1/*
   2 *  linux/mm/page_alloc.c
   3 *
   4 *  Manages the free list, the system allocates free pages here.
   5 *  Note that kmalloc() lives in slab.c
   6 *
   7 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   8 *  Swap reorganised 29.12.95, Stephen Tweedie
   9 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  10 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  11 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  12 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  13 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  14 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  15 */
  16
  17#include <linux/stddef.h>
  18#include <linux/mm.h>
  19#include <linux/swap.h>
  20#include <linux/interrupt.h>
  21#include <linux/pagemap.h>
  22#include <linux/jiffies.h>
  23#include <linux/bootmem.h>
  24#include <linux/memblock.h>
  25#include <linux/compiler.h>
  26#include <linux/kernel.h>
  27#include <linux/kmemcheck.h>
  28#include <linux/kasan.h>
 
  29#include <linux/module.h>
  30#include <linux/suspend.h>
  31#include <linux/pagevec.h>
  32#include <linux/blkdev.h>
  33#include <linux/slab.h>
  34#include <linux/ratelimit.h>
  35#include <linux/oom.h>
  36#include <linux/notifier.h>
  37#include <linux/topology.h>
  38#include <linux/sysctl.h>
  39#include <linux/cpu.h>
  40#include <linux/cpuset.h>
 
  41#include <linux/memory_hotplug.h>
  42#include <linux/nodemask.h>
  43#include <linux/vmalloc.h>
  44#include <linux/vmstat.h>
  45#include <linux/mempolicy.h>
  46#include <linux/memremap.h>
  47#include <linux/stop_machine.h>
  48#include <linux/sort.h>
  49#include <linux/pfn.h>
  50#include <linux/backing-dev.h>
  51#include <linux/fault-inject.h>
  52#include <linux/page-isolation.h>
  53#include <linux/page_ext.h>
  54#include <linux/debugobjects.h>
  55#include <linux/kmemleak.h>
  56#include <linux/compaction.h>
  57#include <trace/events/kmem.h>
 
  58#include <linux/prefetch.h>
  59#include <linux/mm_inline.h>
 
  60#include <linux/migrate.h>
  61#include <linux/page_ext.h>
  62#include <linux/hugetlb.h>
  63#include <linux/sched/rt.h>
  64#include <linux/page_owner.h>
  65#include <linux/kthread.h>
  66
  67#include <asm/sections.h>
  68#include <asm/tlbflush.h>
 
 
 
 
  69#include <asm/div64.h>
  70#include "internal.h"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  71
  72/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
  73static DEFINE_MUTEX(pcp_batch_high_lock);
  74#define MIN_PERCPU_PAGELIST_FRACTION	(8)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  75
  76#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  77DEFINE_PER_CPU(int, numa_node);
  78EXPORT_PER_CPU_SYMBOL(numa_node);
  79#endif
  80
 
 
  81#ifdef CONFIG_HAVE_MEMORYLESS_NODES
  82/*
  83 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  84 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  85 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  86 * defined in <linux/topology.h>.
  87 */
  88DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
  89EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  90int _node_numa_mem_[MAX_NUMNODES];
 
 
 
 
 
 
  91#endif
  92
  93/*
  94 * Array of node states.
  95 */
  96nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  97	[N_POSSIBLE] = NODE_MASK_ALL,
  98	[N_ONLINE] = { { [0] = 1UL } },
  99#ifndef CONFIG_NUMA
 100	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
 101#ifdef CONFIG_HIGHMEM
 102	[N_HIGH_MEMORY] = { { [0] = 1UL } },
 103#endif
 104#ifdef CONFIG_MOVABLE_NODE
 105	[N_MEMORY] = { { [0] = 1UL } },
 106#endif
 107	[N_CPU] = { { [0] = 1UL } },
 108#endif	/* NUMA */
 109};
 110EXPORT_SYMBOL(node_states);
 111
 112/* Protect totalram_pages and zone->managed_pages */
 113static DEFINE_SPINLOCK(managed_page_count_lock);
 114
 115unsigned long totalram_pages __read_mostly;
 116unsigned long totalreserve_pages __read_mostly;
 117unsigned long totalcma_pages __read_mostly;
 118
 119int percpu_pagelist_fraction;
 120gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 121
 122/*
 123 * A cached value of the page's pageblock's migratetype, used when the page is
 124 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 125 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 126 * Also the migratetype set in the page does not necessarily match the pcplist
 127 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 128 * other index - this ensures that it will be put on the correct CMA freelist.
 129 */
 130static inline int get_pcppage_migratetype(struct page *page)
 131{
 132	return page->index;
 133}
 134
 135static inline void set_pcppage_migratetype(struct page *page, int migratetype)
 136{
 137	page->index = migratetype;
 138}
 139
 140#ifdef CONFIG_PM_SLEEP
 141/*
 142 * The following functions are used by the suspend/hibernate code to temporarily
 143 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 144 * while devices are suspended.  To avoid races with the suspend/hibernate code,
 145 * they should always be called with pm_mutex held (gfp_allowed_mask also should
 146 * only be modified with pm_mutex held, unless the suspend/hibernate code is
 147 * guaranteed not to run in parallel with that modification).
 148 */
 149
 150static gfp_t saved_gfp_mask;
 151
 152void pm_restore_gfp_mask(void)
 153{
 154	WARN_ON(!mutex_is_locked(&pm_mutex));
 155	if (saved_gfp_mask) {
 156		gfp_allowed_mask = saved_gfp_mask;
 157		saved_gfp_mask = 0;
 158	}
 159}
 160
 161void pm_restrict_gfp_mask(void)
 162{
 163	WARN_ON(!mutex_is_locked(&pm_mutex));
 164	WARN_ON(saved_gfp_mask);
 165	saved_gfp_mask = gfp_allowed_mask;
 166	gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
 167}
 168
 169bool pm_suspended_storage(void)
 170{
 171	if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
 172		return false;
 173	return true;
 174}
 175#endif /* CONFIG_PM_SLEEP */
 176
 177#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 178unsigned int pageblock_order __read_mostly;
 179#endif
 180
 181static void __free_pages_ok(struct page *page, unsigned int order);
 
 182
 183/*
 184 * results with 256, 32 in the lowmem_reserve sysctl:
 185 *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 186 *	1G machine -> (16M dma, 784M normal, 224M high)
 187 *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 188 *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 189 *	HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
 190 *
 191 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 192 * don't need any ZONE_NORMAL reservation
 193 */
 194int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
 195#ifdef CONFIG_ZONE_DMA
 196	 256,
 197#endif
 198#ifdef CONFIG_ZONE_DMA32
 199	 256,
 200#endif
 
 201#ifdef CONFIG_HIGHMEM
 202	 32,
 203#endif
 204	 32,
 205};
 206
 207EXPORT_SYMBOL(totalram_pages);
 208
 209static char * const zone_names[MAX_NR_ZONES] = {
 210#ifdef CONFIG_ZONE_DMA
 211	 "DMA",
 212#endif
 213#ifdef CONFIG_ZONE_DMA32
 214	 "DMA32",
 215#endif
 216	 "Normal",
 217#ifdef CONFIG_HIGHMEM
 218	 "HighMem",
 219#endif
 220	 "Movable",
 221#ifdef CONFIG_ZONE_DEVICE
 222	 "Device",
 223#endif
 224};
 225
 226char * const migratetype_names[MIGRATE_TYPES] = {
 227	"Unmovable",
 228	"Movable",
 229	"Reclaimable",
 230	"HighAtomic",
 231#ifdef CONFIG_CMA
 232	"CMA",
 233#endif
 234#ifdef CONFIG_MEMORY_ISOLATION
 235	"Isolate",
 236#endif
 237};
 238
 239compound_page_dtor * const compound_page_dtors[] = {
 240	NULL,
 241	free_compound_page,
 242#ifdef CONFIG_HUGETLB_PAGE
 243	free_huge_page,
 244#endif
 245#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 246	free_transhuge_page,
 247#endif
 248};
 249
 250int min_free_kbytes = 1024;
 251int user_min_free_kbytes = -1;
 252int watermark_scale_factor = 10;
 253
 254static unsigned long __meminitdata nr_kernel_pages;
 255static unsigned long __meminitdata nr_all_pages;
 256static unsigned long __meminitdata dma_reserve;
 257
 258#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
 259static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
 260static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
 261static unsigned long __initdata required_kernelcore;
 262static unsigned long __initdata required_movablecore;
 263static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
 264static bool mirrored_kernelcore;
 265
 266/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
 267int movable_zone;
 268EXPORT_SYMBOL(movable_zone);
 269#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
 270
 271#if MAX_NUMNODES > 1
 272int nr_node_ids __read_mostly = MAX_NUMNODES;
 273int nr_online_nodes __read_mostly = 1;
 274EXPORT_SYMBOL(nr_node_ids);
 275EXPORT_SYMBOL(nr_online_nodes);
 276#endif
 277
 
 
 
 
 
 
 278int page_group_by_mobility_disabled __read_mostly;
 279
 280#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 281static inline void reset_deferred_meminit(pg_data_t *pgdat)
 
 
 
 
 
 
 
 282{
 283	pgdat->first_deferred_pfn = ULONG_MAX;
 284}
 285
 286/* Returns true if the struct page for the pfn is uninitialised */
 287static inline bool __meminit early_page_uninitialised(unsigned long pfn)
 
 
 
 
 
 
 
 
 
 
 
 288{
 289	if (pfn >= NODE_DATA(early_pfn_to_nid(pfn))->first_deferred_pfn)
 290		return true;
 291
 292	return false;
 293}
 
 294
 295static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid)
 
 
 296{
 297	if (pfn >= NODE_DATA(nid)->first_deferred_pfn)
 298		return true;
 
 
 
 
 299
 300	return false;
 
 
 
 
 
 
 
 301}
 302
 303/*
 304 * Returns false when the remaining initialisation should be deferred until
 305 * later in the boot cycle when it can be parallelised.
 
 
 
 
 306 */
 307static inline bool update_defer_init(pg_data_t *pgdat,
 308				unsigned long pfn, unsigned long zone_end,
 309				unsigned long *nr_initialised)
 310{
 311	unsigned long max_initialise;
 
 
 312
 313	/* Always populate low zones for address-contrained allocations */
 314	if (zone_end < pgdat_end_pfn(pgdat))
 315		return true;
 
 316	/*
 317	 * Initialise at least 2G of a node but also take into account that
 318	 * two large system hashes that can take up 1GB for 0.25TB/node.
 
 319	 */
 320	max_initialise = max(2UL << (30 - PAGE_SHIFT),
 321		(pgdat->node_spanned_pages >> 8));
 322
 323	(*nr_initialised)++;
 324	if ((*nr_initialised > max_initialise) &&
 325	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {
 326		pgdat->first_deferred_pfn = pfn;
 327		return false;
 328	}
 329
 330	return true;
 331}
 332#else
 333static inline void reset_deferred_meminit(pg_data_t *pgdat)
 334{
 335}
 336
 337static inline bool early_page_uninitialised(unsigned long pfn)
 
 338{
 339	return false;
 340}
 341
 342static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid)
 
 
 
 
 
 
 
 
 
 343{
 344	return false;
 345}
 
 346
 347static inline bool update_defer_init(pg_data_t *pgdat,
 348				unsigned long pfn, unsigned long zone_end,
 349				unsigned long *nr_initialised)
 350{
 351	return true;
 352}
 353#endif
 354
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 355
 356void set_pageblock_migratetype(struct page *page, int migratetype)
 357{
 358	if (unlikely(page_group_by_mobility_disabled &&
 359		     migratetype < MIGRATE_PCPTYPES))
 360		migratetype = MIGRATE_UNMOVABLE;
 361
 362	set_pageblock_flags_group(page, (unsigned long)migratetype,
 363					PB_migrate, PB_migrate_end);
 364}
 365
 366#ifdef CONFIG_DEBUG_VM
 367static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 368{
 369	int ret = 0;
 370	unsigned seq;
 371	unsigned long pfn = page_to_pfn(page);
 372	unsigned long sp, start_pfn;
 373
 374	do {
 375		seq = zone_span_seqbegin(zone);
 376		start_pfn = zone->zone_start_pfn;
 377		sp = zone->spanned_pages;
 378		if (!zone_spans_pfn(zone, pfn))
 379			ret = 1;
 380	} while (zone_span_seqretry(zone, seq));
 381
 382	if (ret)
 383		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
 384			pfn, zone_to_nid(zone), zone->name,
 385			start_pfn, start_pfn + sp);
 386
 387	return ret;
 388}
 389
 390static int page_is_consistent(struct zone *zone, struct page *page)
 391{
 392	if (!pfn_valid_within(page_to_pfn(page)))
 393		return 0;
 394	if (zone != page_zone(page))
 395		return 0;
 396
 397	return 1;
 398}
 399/*
 400 * Temporary debugging check for pages not lying within a given zone.
 401 */
 402static int bad_range(struct zone *zone, struct page *page)
 403{
 404	if (page_outside_zone_boundaries(zone, page))
 405		return 1;
 406	if (!page_is_consistent(zone, page))
 407		return 1;
 408
 409	return 0;
 410}
 411#else
 412static inline int bad_range(struct zone *zone, struct page *page)
 413{
 414	return 0;
 415}
 416#endif
 417
 418static void bad_page(struct page *page, const char *reason,
 419		unsigned long bad_flags)
 420{
 421	static unsigned long resume;
 422	static unsigned long nr_shown;
 423	static unsigned long nr_unshown;
 424
 425	/* Don't complain about poisoned pages */
 426	if (PageHWPoison(page)) {
 427		page_mapcount_reset(page); /* remove PageBuddy */
 428		return;
 429	}
 430
 431	/*
 432	 * Allow a burst of 60 reports, then keep quiet for that minute;
 433	 * or allow a steady drip of one report per second.
 434	 */
 435	if (nr_shown == 60) {
 436		if (time_before(jiffies, resume)) {
 437			nr_unshown++;
 438			goto out;
 439		}
 440		if (nr_unshown) {
 441			pr_alert(
 442			      "BUG: Bad page state: %lu messages suppressed\n",
 443				nr_unshown);
 444			nr_unshown = 0;
 445		}
 446		nr_shown = 0;
 447	}
 448	if (nr_shown++ == 0)
 449		resume = jiffies + 60 * HZ;
 450
 451	pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
 452		current->comm, page_to_pfn(page));
 453	__dump_page(page, reason);
 454	bad_flags &= page->flags;
 455	if (bad_flags)
 456		pr_alert("bad because of flags: %#lx(%pGp)\n",
 457						bad_flags, &bad_flags);
 458	dump_page_owner(page);
 459
 460	print_modules();
 461	dump_stack();
 462out:
 463	/* Leave bad fields for debug, except PageBuddy could make trouble */
 464	page_mapcount_reset(page); /* remove PageBuddy */
 465	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 466}
 467
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 468/*
 469 * Higher-order pages are called "compound pages".  They are structured thusly:
 470 *
 471 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
 472 *
 473 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 474 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
 475 *
 476 * The first tail page's ->compound_dtor holds the offset in array of compound
 477 * page destructors. See compound_page_dtors.
 478 *
 479 * The first tail page's ->compound_order holds the order of allocation.
 480 * This usage means that zero-order pages may not be compound.
 481 */
 482
 483void free_compound_page(struct page *page)
 484{
 485	__free_pages_ok(page, compound_order(page));
 486}
 487
 488void prep_compound_page(struct page *page, unsigned int order)
 489{
 490	int i;
 491	int nr_pages = 1 << order;
 492
 493	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
 494	set_compound_order(page, order);
 495	__SetPageHead(page);
 496	for (i = 1; i < nr_pages; i++) {
 497		struct page *p = page + i;
 498		set_page_count(p, 0);
 499		p->mapping = TAIL_MAPPING;
 500		set_compound_head(p, page);
 501	}
 502	atomic_set(compound_mapcount_ptr(page), -1);
 503}
 504
 505#ifdef CONFIG_DEBUG_PAGEALLOC
 506unsigned int _debug_guardpage_minorder;
 507bool _debug_pagealloc_enabled __read_mostly
 508			= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
 509EXPORT_SYMBOL(_debug_pagealloc_enabled);
 510bool _debug_guardpage_enabled __read_mostly;
 511
 512static int __init early_debug_pagealloc(char *buf)
 513{
 514	if (!buf)
 515		return -EINVAL;
 516
 517	if (strcmp(buf, "on") == 0)
 518		_debug_pagealloc_enabled = true;
 519
 520	if (strcmp(buf, "off") == 0)
 521		_debug_pagealloc_enabled = false;
 522
 523	return 0;
 
 524}
 525early_param("debug_pagealloc", early_debug_pagealloc);
 526
 527static bool need_debug_guardpage(void)
 528{
 529	/* If we don't use debug_pagealloc, we don't need guard page */
 530	if (!debug_pagealloc_enabled())
 531		return false;
 532
 533	return true;
 534}
 535
 536static void init_debug_guardpage(void)
 
 537{
 538	if (!debug_pagealloc_enabled())
 539		return;
 540
 541	_debug_guardpage_enabled = true;
 
 
 
 542}
 543
 544struct page_ext_operations debug_guardpage_ops = {
 545	.need = need_debug_guardpage,
 546	.init = init_debug_guardpage,
 547};
 548
 549static int __init debug_guardpage_minorder_setup(char *buf)
 550{
 551	unsigned long res;
 
 552
 553	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
 554		pr_err("Bad debug_guardpage_minorder value\n");
 555		return 0;
 556	}
 557	_debug_guardpage_minorder = res;
 558	pr_info("Setting debug_guardpage_minorder to %lu\n", res);
 559	return 0;
 560}
 561__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
 562
 563static inline void set_page_guard(struct zone *zone, struct page *page,
 564				unsigned int order, int migratetype)
 565{
 566	struct page_ext *page_ext;
 
 
 
 
 567
 568	if (!debug_guardpage_enabled())
 569		return;
 
 570
 571	page_ext = lookup_page_ext(page);
 572	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
 
 
 
 573
 574	INIT_LIST_HEAD(&page->lru);
 575	set_page_private(page, order);
 576	/* Guard pages are not available for any usage */
 577	__mod_zone_freepage_state(zone, -(1 << order), migratetype);
 
 578}
 
 579
 580static inline void clear_page_guard(struct zone *zone, struct page *page,
 581				unsigned int order, int migratetype)
 
 582{
 583	struct page_ext *page_ext;
 584
 585	if (!debug_guardpage_enabled())
 586		return;
 
 587
 588	page_ext = lookup_page_ext(page);
 589	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
 
 
 
 590
 591	set_page_private(page, 0);
 592	if (!is_migrate_isolate(migratetype))
 593		__mod_zone_freepage_state(zone, (1 << order), migratetype);
 594}
 595#else
 596struct page_ext_operations debug_guardpage_ops = { NULL, };
 597static inline void set_page_guard(struct zone *zone, struct page *page,
 598				unsigned int order, int migratetype) {}
 599static inline void clear_page_guard(struct zone *zone, struct page *page,
 600				unsigned int order, int migratetype) {}
 601#endif
 602
 603static inline void set_page_order(struct page *page, unsigned int order)
 
 
 
 
 
 
 604{
 605	set_page_private(page, order);
 606	__SetPageBuddy(page);
 
 607}
 608
 609static inline void rmv_page_order(struct page *page)
 
 610{
 
 
 
 
 
 611	__ClearPageBuddy(page);
 612	set_page_private(page, 0);
 
 
 
 
 
 
 
 
 613}
 614
 615/*
 616 * This function checks whether a page is free && is the buddy
 617 * we can do coalesce a page and its buddy if
 618 * (a) the buddy is not in a hole &&
 619 * (b) the buddy is in the buddy system &&
 620 * (c) a page and its buddy have the same order &&
 621 * (d) a page and its buddy are in the same zone.
 622 *
 623 * For recording whether a page is in the buddy system, we set ->_mapcount
 624 * PAGE_BUDDY_MAPCOUNT_VALUE.
 625 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
 626 * serialized by zone->lock.
 627 *
 628 * For recording page's order, we use page_private(page).
 629 */
 630static inline int page_is_buddy(struct page *page, struct page *buddy,
 631							unsigned int order)
 
 632{
 633	if (!pfn_valid_within(page_to_pfn(buddy)))
 634		return 0;
 635
 636	if (page_is_guard(buddy) && page_order(buddy) == order) {
 637		if (page_zone_id(page) != page_zone_id(buddy))
 638			return 0;
 639
 640		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
 641
 642		return 1;
 643	}
 644
 645	if (PageBuddy(buddy) && page_order(buddy) == order) {
 646		/*
 647		 * zone check is done late to avoid uselessly
 648		 * calculating zone/node ids for pages that could
 649		 * never merge.
 650		 */
 651		if (page_zone_id(page) != page_zone_id(buddy))
 652			return 0;
 653
 654		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
 
 655
 656		return 1;
 657	}
 658	return 0;
 659}
 660
 661/*
 662 * Freeing function for a buddy system allocator.
 663 *
 664 * The concept of a buddy system is to maintain direct-mapped table
 665 * (containing bit values) for memory blocks of various "orders".
 666 * The bottom level table contains the map for the smallest allocatable
 667 * units of memory (here, pages), and each level above it describes
 668 * pairs of units from the levels below, hence, "buddies".
 669 * At a high level, all that happens here is marking the table entry
 670 * at the bottom level available, and propagating the changes upward
 671 * as necessary, plus some accounting needed to play nicely with other
 672 * parts of the VM system.
 673 * At each level, we keep a list of pages, which are heads of continuous
 674 * free pages of length of (1 << order) and marked with _mapcount
 675 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
 676 * field.
 677 * So when we are allocating or freeing one, we can derive the state of the
 678 * other.  That is, if we allocate a small block, and both were
 679 * free, the remainder of the region must be split into blocks.
 680 * If a block is freed, and its buddy is also free, then this
 681 * triggers coalescing into a block of larger size.
 682 *
 683 * -- nyc
 684 */
 685
 686static inline void __free_one_page(struct page *page,
 687		unsigned long pfn,
 688		struct zone *zone, unsigned int order,
 689		int migratetype)
 690{
 691	unsigned long page_idx;
 692	unsigned long combined_idx;
 693	unsigned long uninitialized_var(buddy_idx);
 694	struct page *buddy;
 695	unsigned int max_order;
 696
 697	max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
 698
 699	VM_BUG_ON(!zone_is_initialized(zone));
 700	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
 701
 702	VM_BUG_ON(migratetype == -1);
 703	if (likely(!is_migrate_isolate(migratetype)))
 704		__mod_zone_freepage_state(zone, 1 << order, migratetype);
 705
 706	page_idx = pfn & ((1 << MAX_ORDER) - 1);
 707
 708	VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
 709	VM_BUG_ON_PAGE(bad_range(zone, page), page);
 710
 711continue_merging:
 712	while (order < max_order - 1) {
 713		buddy_idx = __find_buddy_index(page_idx, order);
 714		buddy = page + (buddy_idx - page_idx);
 715		if (!page_is_buddy(page, buddy, order))
 
 
 
 
 716			goto done_merging;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 717		/*
 718		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
 719		 * merge with it and move up one order.
 720		 */
 721		if (page_is_guard(buddy)) {
 722			clear_page_guard(zone, buddy, order, migratetype);
 723		} else {
 724			list_del(&buddy->lru);
 725			zone->free_area[order].nr_free--;
 726			rmv_page_order(buddy);
 727		}
 728		combined_idx = buddy_idx & page_idx;
 729		page = page + (combined_idx - page_idx);
 730		page_idx = combined_idx;
 731		order++;
 732	}
 733	if (max_order < MAX_ORDER) {
 734		/* If we are here, it means order is >= pageblock_order.
 735		 * We want to prevent merge between freepages on isolate
 736		 * pageblock and normal pageblock. Without this, pageblock
 737		 * isolation could cause incorrect freepage or CMA accounting.
 738		 *
 739		 * We don't want to hit this code for the more frequent
 740		 * low-order merging.
 741		 */
 742		if (unlikely(has_isolate_pageblock(zone))) {
 743			int buddy_mt;
 744
 745			buddy_idx = __find_buddy_index(page_idx, order);
 746			buddy = page + (buddy_idx - page_idx);
 747			buddy_mt = get_pageblock_migratetype(buddy);
 748
 749			if (migratetype != buddy_mt
 750					&& (is_migrate_isolate(migratetype) ||
 751						is_migrate_isolate(buddy_mt)))
 752				goto done_merging;
 753		}
 754		max_order++;
 755		goto continue_merging;
 756	}
 757
 758done_merging:
 759	set_page_order(page, order);
 
 
 760
 761	/*
 762	 * If this is not the largest possible page, check if the buddy
 763	 * of the next-highest order is free. If it is, it's possible
 764	 * that pages are being freed that will coalesce soon. In case,
 765	 * that is happening, add the free page to the tail of the list
 766	 * so it's less likely to be used soon and more likely to be merged
 767	 * as a higher order page
 768	 */
 769	if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
 770		struct page *higher_page, *higher_buddy;
 771		combined_idx = buddy_idx & page_idx;
 772		higher_page = page + (combined_idx - page_idx);
 773		buddy_idx = __find_buddy_index(combined_idx, order + 1);
 774		higher_buddy = higher_page + (buddy_idx - combined_idx);
 775		if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
 776			list_add_tail(&page->lru,
 777				&zone->free_area[order].free_list[migratetype]);
 778			goto out;
 779		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 780	}
 781
 782	list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 783out:
 784	zone->free_area[order].nr_free++;
 
 785}
 
 
 
 
 
 
 
 
 
 
 786
 787static inline int free_pages_check(struct page *page)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 788{
 789	const char *bad_reason = NULL;
 790	unsigned long bad_flags = 0;
 791
 792	if (unlikely(atomic_read(&page->_mapcount) != -1))
 793		bad_reason = "nonzero mapcount";
 794	if (unlikely(page->mapping != NULL))
 795		bad_reason = "non-NULL mapping";
 796	if (unlikely(page_ref_count(page) != 0))
 797		bad_reason = "nonzero _count";
 798	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
 799		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
 800		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
 
 
 801	}
 802#ifdef CONFIG_MEMCG
 803	if (unlikely(page->mem_cgroup))
 804		bad_reason = "page still charged to cgroup";
 805#endif
 806	if (unlikely(bad_reason)) {
 807		bad_page(page, bad_reason, bad_flags);
 808		return 1;
 809	}
 810	page_cpupid_reset_last(page);
 811	if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
 812		page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
 813	return 0;
 814}
 815
 816/*
 817 * Frees a number of pages from the PCP lists
 818 * Assumes all pages on list are in same zone, and of same order.
 819 * count is the number of pages to free.
 820 *
 821 * If the zone was previously in an "all pages pinned" state then look to
 822 * see if this freeing clears that state.
 823 *
 824 * And clear the zone's pages_scanned counter, to hold off the "all pages are
 825 * pinned" detection logic.
 826 */
 827static void free_pcppages_bulk(struct zone *zone, int count,
 828					struct per_cpu_pages *pcp)
 829{
 830	int migratetype = 0;
 831	int batch_free = 0;
 832	int to_free = count;
 833	unsigned long nr_scanned;
 834
 835	spin_lock(&zone->lock);
 836	nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
 837	if (nr_scanned)
 838		__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
 839
 840	while (to_free) {
 841		struct page *page;
 842		struct list_head *list;
 843
 844		/*
 845		 * Remove pages from lists in a round-robin fashion. A
 846		 * batch_free count is maintained that is incremented when an
 847		 * empty list is encountered.  This is so more pages are freed
 848		 * off fuller lists instead of spinning excessively around empty
 849		 * lists
 850		 */
 851		do {
 852			batch_free++;
 853			if (++migratetype == MIGRATE_PCPTYPES)
 854				migratetype = 0;
 855			list = &pcp->lists[migratetype];
 856		} while (list_empty(list));
 857
 858		/* This is the only non-empty list. Free them all. */
 859		if (batch_free == MIGRATE_PCPTYPES)
 860			batch_free = to_free;
 861
 862		do {
 863			int mt;	/* migratetype of the to-be-freed page */
 864
 865			page = list_last_entry(list, struct page, lru);
 866			/* must delete as __free_one_page list manipulates */
 867			list_del(&page->lru);
 868
 869			mt = get_pcppage_migratetype(page);
 870			/* MIGRATE_ISOLATE page should not go to pcplists */
 871			VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
 872			/* Pageblock could have been isolated meanwhile */
 873			if (unlikely(has_isolate_pageblock(zone)))
 874				mt = get_pageblock_migratetype(page);
 875
 876			__free_one_page(page, page_to_pfn(page), zone, 0, mt);
 877			trace_mm_page_pcpu_drain(page, 0, mt);
 878		} while (--to_free && --batch_free && !list_empty(list));
 879	}
 880	spin_unlock(&zone->lock);
 881}
 882
 883static void free_one_page(struct zone *zone,
 884				struct page *page, unsigned long pfn,
 885				unsigned int order,
 886				int migratetype)
 887{
 888	unsigned long nr_scanned;
 889	spin_lock(&zone->lock);
 890	nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
 891	if (nr_scanned)
 892		__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
 893
 894	if (unlikely(has_isolate_pageblock(zone) ||
 895		is_migrate_isolate(migratetype))) {
 896		migratetype = get_pfnblock_migratetype(page, pfn);
 897	}
 898	__free_one_page(page, pfn, zone, order, migratetype);
 899	spin_unlock(&zone->lock);
 900}
 901
 902static int free_tail_pages_check(struct page *head_page, struct page *page)
 903{
 
 904	int ret = 1;
 905
 906	/*
 907	 * We rely page->lru.next never has bit 0 set, unless the page
 908	 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
 909	 */
 910	BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
 911
 912	if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
 913		ret = 0;
 914		goto out;
 915	}
 916	switch (page - head_page) {
 917	case 1:
 918		/* the first tail page: ->mapping is compound_mapcount() */
 919		if (unlikely(compound_mapcount(page))) {
 920			bad_page(page, "nonzero compound_mapcount", 0);
 
 
 
 
 
 
 
 
 921			goto out;
 922		}
 923		break;
 924	case 2:
 925		/*
 926		 * the second tail page: ->mapping is
 927		 * page_deferred_list().next -- ignore value.
 928		 */
 929		break;
 930	default:
 931		if (page->mapping != TAIL_MAPPING) {
 932			bad_page(page, "corrupted mapping in tail page", 0);
 933			goto out;
 934		}
 935		break;
 936	}
 937	if (unlikely(!PageTail(page))) {
 938		bad_page(page, "PageTail not set", 0);
 939		goto out;
 940	}
 941	if (unlikely(compound_head(page) != head_page)) {
 942		bad_page(page, "compound_head not consistent", 0);
 943		goto out;
 944	}
 945	ret = 0;
 946out:
 947	page->mapping = NULL;
 948	clear_compound_head(page);
 949	return ret;
 950}
 951
 952static void __meminit __init_single_page(struct page *page, unsigned long pfn,
 953				unsigned long zone, int nid)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 954{
 955	set_page_links(page, zone, nid, pfn);
 956	init_page_count(page);
 957	page_mapcount_reset(page);
 958	page_cpupid_reset_last(page);
 959
 960	INIT_LIST_HEAD(&page->lru);
 961#ifdef WANT_PAGE_VIRTUAL
 962	/* The shift won't overflow because ZONE_NORMAL is below 4G. */
 963	if (!is_highmem_idx(zone))
 964		set_page_address(page, __va(pfn << PAGE_SHIFT));
 965#endif
 966}
 967
 968static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone,
 969					int nid)
 970{
 971	return __init_single_page(pfn_to_page(pfn), pfn, zone, nid);
 972}
 973
 974#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 975static void init_reserved_page(unsigned long pfn)
 976{
 977	pg_data_t *pgdat;
 978	int nid, zid;
 979
 980	if (!early_page_uninitialised(pfn))
 981		return;
 982
 983	nid = early_pfn_to_nid(pfn);
 984	pgdat = NODE_DATA(nid);
 985
 986	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
 987		struct zone *zone = &pgdat->node_zones[zid];
 988
 989		if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
 990			break;
 991	}
 992	__init_single_pfn(pfn, zid, nid);
 993}
 994#else
 995static inline void init_reserved_page(unsigned long pfn)
 996{
 997}
 998#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
 999
1000/*
1001 * Initialised pages do not have PageReserved set. This function is
1002 * called for each range allocated by the bootmem allocator and
1003 * marks the pages PageReserved. The remaining valid pages are later
1004 * sent to the buddy page allocator.
1005 */
1006void __meminit reserve_bootmem_region(unsigned long start, unsigned long end)
1007{
1008	unsigned long start_pfn = PFN_DOWN(start);
1009	unsigned long end_pfn = PFN_UP(end);
1010
1011	for (; start_pfn < end_pfn; start_pfn++) {
1012		if (pfn_valid(start_pfn)) {
1013			struct page *page = pfn_to_page(start_pfn);
1014
1015			init_reserved_page(start_pfn);
 
1016
1017			/* Avoid false-positive PageTail() */
1018			INIT_LIST_HEAD(&page->lru);
1019
1020			SetPageReserved(page);
1021		}
 
 
 
1022	}
1023}
1024
1025static bool free_pages_prepare(struct page *page, unsigned int order)
1026{
1027	bool compound = PageCompound(page);
1028	int i, bad = 0;
1029
1030	VM_BUG_ON_PAGE(PageTail(page), page);
1031	VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1032
1033	trace_mm_page_free(page, order);
1034	kmemcheck_free_shadow(page, order);
1035	kasan_free_pages(page, order);
 
 
 
1036
1037	if (PageAnon(page))
1038		page->mapping = NULL;
1039	bad += free_pages_check(page);
1040	for (i = 1; i < (1 << order); i++) {
1041		if (compound)
1042			bad += free_tail_pages_check(page, page + i);
1043		bad += free_pages_check(page + i);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1044	}
1045	if (bad)
1046		return false;
1047
 
 
1048	reset_page_owner(page, order);
 
1049
1050	if (!PageHighMem(page)) {
1051		debug_check_no_locks_freed(page_address(page),
1052					   PAGE_SIZE << order);
1053		debug_check_no_obj_freed(page_address(page),
1054					   PAGE_SIZE << order);
1055	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1056	arch_free_page(page, order);
1057	kernel_poison_pages(page, 1 << order, 0);
1058	kernel_map_pages(page, 1 << order, 0);
1059
1060	return true;
1061}
1062
1063static void __free_pages_ok(struct page *page, unsigned int order)
 
 
 
 
 
 
 
1064{
1065	unsigned long flags;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1066	int migratetype;
1067	unsigned long pfn = page_to_pfn(page);
 
1068
1069	if (!free_pages_prepare(page, order))
1070		return;
1071
 
 
 
 
 
1072	migratetype = get_pfnblock_migratetype(page, pfn);
1073	local_irq_save(flags);
 
 
1074	__count_vm_events(PGFREE, 1 << order);
1075	free_one_page(page_zone(page), page, pfn, order, migratetype);
1076	local_irq_restore(flags);
1077}
1078
1079static void __init __free_pages_boot_core(struct page *page,
1080					unsigned long pfn, unsigned int order)
1081{
1082	unsigned int nr_pages = 1 << order;
1083	struct page *p = page;
1084	unsigned int loop;
1085
 
 
 
 
 
1086	prefetchw(p);
1087	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1088		prefetchw(p + 1);
1089		__ClearPageReserved(p);
1090		set_page_count(p, 0);
1091	}
1092	__ClearPageReserved(p);
1093	set_page_count(p, 0);
1094
1095	page_zone(page)->managed_pages += nr_pages;
1096	set_page_refcounted(page);
1097	__free_pages(page, order);
1098}
1099
1100#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
1101	defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1102
1103static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1104
1105int __meminit early_pfn_to_nid(unsigned long pfn)
1106{
1107	static DEFINE_SPINLOCK(early_pfn_lock);
1108	int nid;
1109
1110	spin_lock(&early_pfn_lock);
1111	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1112	if (nid < 0)
1113		nid = 0;
1114	spin_unlock(&early_pfn_lock);
1115
1116	return nid;
1117}
1118#endif
1119
1120#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1121static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
1122					struct mminit_pfnnid_cache *state)
1123{
1124	int nid;
1125
1126	nid = __early_pfn_to_nid(pfn, state);
1127	if (nid >= 0 && nid != node)
1128		return false;
1129	return true;
1130}
1131
1132/* Only safe to use early in boot when initialisation is single-threaded */
1133static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1134{
1135	return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
1136}
1137
1138#else
1139
1140static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1141{
1142	return true;
1143}
1144static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
1145					struct mminit_pfnnid_cache *state)
1146{
1147	return true;
1148}
1149#endif
1150
 
 
1151
1152void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
1153							unsigned int order)
1154{
1155	if (early_page_uninitialised(pfn))
1156		return;
1157	return __free_pages_boot_core(page, pfn, order);
1158}
1159
1160/*
1161 * Check that the whole (or subset of) a pageblock given by the interval of
1162 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1163 * with the migration of free compaction scanner. The scanners then need to
1164 * use only pfn_valid_within() check for arches that allow holes within
1165 * pageblocks.
1166 *
1167 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1168 *
1169 * It's possible on some configurations to have a setup like node0 node1 node0
1170 * i.e. it's possible that all pages within a zones range of pages do not
1171 * belong to a single zone. We assume that a border between node0 and node1
1172 * can occur within a single pageblock, but not a node0 node1 node0
1173 * interleaving within a single pageblock. It is therefore sufficient to check
1174 * the first and last page of a pageblock and avoid checking each individual
1175 * page in a pageblock.
 
 
 
 
 
 
 
 
 
1176 */
1177struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1178				     unsigned long end_pfn, struct zone *zone)
1179{
1180	struct page *start_page;
1181	struct page *end_page;
1182
1183	/* end_pfn is one past the range we are checking */
1184	end_pfn--;
1185
1186	if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1187		return NULL;
1188
1189	start_page = pfn_to_page(start_pfn);
 
 
1190
1191	if (page_zone(start_page) != zone)
1192		return NULL;
1193
1194	end_page = pfn_to_page(end_pfn);
1195
1196	/* This gives a shorter code than deriving page_zone(end_page) */
1197	if (page_zone_id(start_page) != page_zone_id(end_page))
1198		return NULL;
1199
1200	return start_page;
1201}
1202
1203void set_zone_contiguous(struct zone *zone)
1204{
1205	unsigned long block_start_pfn = zone->zone_start_pfn;
1206	unsigned long block_end_pfn;
1207
1208	block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1209	for (; block_start_pfn < zone_end_pfn(zone);
1210			block_start_pfn = block_end_pfn,
1211			 block_end_pfn += pageblock_nr_pages) {
1212
1213		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1214
1215		if (!__pageblock_pfn_to_page(block_start_pfn,
1216					     block_end_pfn, zone))
1217			return;
1218	}
1219
1220	/* We confirm that there is no hole */
1221	zone->contiguous = true;
1222}
1223
1224void clear_zone_contiguous(struct zone *zone)
1225{
1226	zone->contiguous = false;
1227}
1228
1229#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1230static void __init deferred_free_range(struct page *page,
1231					unsigned long pfn, int nr_pages)
1232{
1233	int i;
1234
1235	if (!page)
1236		return;
1237
1238	/* Free a large naturally-aligned chunk if possible */
1239	if (nr_pages == MAX_ORDER_NR_PAGES &&
1240	    (pfn & (MAX_ORDER_NR_PAGES-1)) == 0) {
1241		set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1242		__free_pages_boot_core(page, pfn, MAX_ORDER-1);
1243		return;
1244	}
1245
1246	for (i = 0; i < nr_pages; i++, page++, pfn++)
1247		__free_pages_boot_core(page, pfn, 0);
1248}
1249
1250/* Completion tracking for deferred_init_memmap() threads */
1251static atomic_t pgdat_init_n_undone __initdata;
1252static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1253
1254static inline void __init pgdat_init_report_one_done(void)
1255{
1256	if (atomic_dec_and_test(&pgdat_init_n_undone))
1257		complete(&pgdat_init_all_done_comp);
1258}
1259
1260/* Initialise remaining memory on a node */
1261static int __init deferred_init_memmap(void *data)
1262{
1263	pg_data_t *pgdat = data;
1264	int nid = pgdat->node_id;
1265	struct mminit_pfnnid_cache nid_init_state = { };
1266	unsigned long start = jiffies;
1267	unsigned long nr_pages = 0;
1268	unsigned long walk_start, walk_end;
1269	int i, zid;
1270	struct zone *zone;
1271	unsigned long first_init_pfn = pgdat->first_deferred_pfn;
1272	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1273
1274	if (first_init_pfn == ULONG_MAX) {
1275		pgdat_init_report_one_done();
1276		return 0;
1277	}
1278
1279	/* Bind memory initialisation thread to a local node if possible */
1280	if (!cpumask_empty(cpumask))
1281		set_cpus_allowed_ptr(current, cpumask);
1282
1283	/* Sanity check boundaries */
1284	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1285	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1286	pgdat->first_deferred_pfn = ULONG_MAX;
1287
1288	/* Only the highest zone is deferred so find it */
1289	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1290		zone = pgdat->node_zones + zid;
1291		if (first_init_pfn < zone_end_pfn(zone))
1292			break;
1293	}
1294
1295	for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) {
1296		unsigned long pfn, end_pfn;
1297		struct page *page = NULL;
1298		struct page *free_base_page = NULL;
1299		unsigned long free_base_pfn = 0;
1300		int nr_to_free = 0;
1301
1302		end_pfn = min(walk_end, zone_end_pfn(zone));
1303		pfn = first_init_pfn;
1304		if (pfn < walk_start)
1305			pfn = walk_start;
1306		if (pfn < zone->zone_start_pfn)
1307			pfn = zone->zone_start_pfn;
1308
1309		for (; pfn < end_pfn; pfn++) {
1310			if (!pfn_valid_within(pfn))
1311				goto free_range;
1312
1313			/*
1314			 * Ensure pfn_valid is checked every
1315			 * MAX_ORDER_NR_PAGES for memory holes
1316			 */
1317			if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) {
1318				if (!pfn_valid(pfn)) {
1319					page = NULL;
1320					goto free_range;
1321				}
1322			}
1323
1324			if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) {
1325				page = NULL;
1326				goto free_range;
1327			}
1328
1329			/* Minimise pfn page lookups and scheduler checks */
1330			if (page && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) {
1331				page++;
1332			} else {
1333				nr_pages += nr_to_free;
1334				deferred_free_range(free_base_page,
1335						free_base_pfn, nr_to_free);
1336				free_base_page = NULL;
1337				free_base_pfn = nr_to_free = 0;
1338
1339				page = pfn_to_page(pfn);
1340				cond_resched();
1341			}
1342
1343			if (page->flags) {
1344				VM_BUG_ON(page_zone(page) != zone);
1345				goto free_range;
1346			}
1347
1348			__init_single_page(page, pfn, zid, nid);
1349			if (!free_base_page) {
1350				free_base_page = page;
1351				free_base_pfn = pfn;
1352				nr_to_free = 0;
1353			}
1354			nr_to_free++;
1355
1356			/* Where possible, batch up pages for a single free */
1357			continue;
1358free_range:
1359			/* Free the current block of pages to allocator */
1360			nr_pages += nr_to_free;
1361			deferred_free_range(free_base_page, free_base_pfn,
1362								nr_to_free);
1363			free_base_page = NULL;
1364			free_base_pfn = nr_to_free = 0;
1365		}
1366
1367		first_init_pfn = max(end_pfn, first_init_pfn);
1368	}
1369
1370	/* Sanity check that the next zone really is unpopulated */
1371	WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1372
1373	pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
1374					jiffies_to_msecs(jiffies - start));
1375
1376	pgdat_init_report_one_done();
1377	return 0;
1378}
1379#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1380
1381void __init page_alloc_init_late(void)
1382{
1383	struct zone *zone;
1384
1385#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1386	int nid;
1387
1388	/* There will be num_node_state(N_MEMORY) threads */
1389	atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1390	for_each_node_state(nid, N_MEMORY) {
1391		kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1392	}
1393
1394	/* Block until all are initialised */
1395	wait_for_completion(&pgdat_init_all_done_comp);
1396
1397	/* Reinit limits that are based on free pages after the kernel is up */
1398	files_maxfiles_init();
1399#endif
1400
1401	for_each_populated_zone(zone)
1402		set_zone_contiguous(zone);
1403}
1404
1405#ifdef CONFIG_CMA
1406/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
1407void __init init_cma_reserved_pageblock(struct page *page)
1408{
1409	unsigned i = pageblock_nr_pages;
1410	struct page *p = page;
1411
1412	do {
1413		__ClearPageReserved(p);
1414		set_page_count(p, 0);
1415	} while (++p, --i);
1416
1417	set_pageblock_migratetype(page, MIGRATE_CMA);
1418
1419	if (pageblock_order >= MAX_ORDER) {
1420		i = pageblock_nr_pages;
1421		p = page;
1422		do {
1423			set_page_refcounted(p);
1424			__free_pages(p, MAX_ORDER - 1);
1425			p += MAX_ORDER_NR_PAGES;
1426		} while (i -= MAX_ORDER_NR_PAGES);
1427	} else {
1428		set_page_refcounted(page);
1429		__free_pages(page, pageblock_order);
1430	}
1431
1432	adjust_managed_page_count(page, pageblock_nr_pages);
1433}
1434#endif
1435
1436/*
1437 * The order of subdivision here is critical for the IO subsystem.
1438 * Please do not alter this order without good reasons and regression
1439 * testing. Specifically, as large blocks of memory are subdivided,
1440 * the order in which smaller blocks are delivered depends on the order
1441 * they're subdivided in this function. This is the primary factor
1442 * influencing the order in which pages are delivered to the IO
1443 * subsystem according to empirical testing, and this is also justified
1444 * by considering the behavior of a buddy system containing a single
1445 * large block of memory acted on by a series of small allocations.
1446 * This behavior is a critical factor in sglist merging's success.
1447 *
1448 * -- nyc
1449 */
1450static inline void expand(struct zone *zone, struct page *page,
1451	int low, int high, struct free_area *area,
1452	int migratetype)
1453{
1454	unsigned long size = 1 << high;
1455
1456	while (high > low) {
1457		area--;
1458		high--;
1459		size >>= 1;
1460		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1461
1462		if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
1463			debug_guardpage_enabled() &&
1464			high < debug_guardpage_minorder()) {
1465			/*
1466			 * Mark as guard pages (or page), that will allow to
1467			 * merge back to allocator when buddy will be freed.
1468			 * Corresponding page table entries will not be touched,
1469			 * pages will stay not present in virtual address space
1470			 */
1471			set_page_guard(zone, &page[size], high, migratetype);
1472			continue;
1473		}
1474		list_add(&page[size].lru, &area->free_list[migratetype]);
1475		area->nr_free++;
1476		set_page_order(&page[size], high);
 
 
 
 
 
 
 
 
1477	}
 
 
 
1478}
1479
1480/*
1481 * This page is about to be returned from the page allocator
1482 */
1483static inline int check_new_page(struct page *page)
1484{
1485	const char *bad_reason = NULL;
1486	unsigned long bad_flags = 0;
 
1487
1488	if (unlikely(atomic_read(&page->_mapcount) != -1))
1489		bad_reason = "nonzero mapcount";
1490	if (unlikely(page->mapping != NULL))
1491		bad_reason = "non-NULL mapping";
1492	if (unlikely(page_ref_count(page) != 0))
1493		bad_reason = "nonzero _count";
1494	if (unlikely(page->flags & __PG_HWPOISON)) {
1495		bad_reason = "HWPoisoned (hardware-corrupted)";
1496		bad_flags = __PG_HWPOISON;
1497	}
1498	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
1499		bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
1500		bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
1501	}
1502#ifdef CONFIG_MEMCG
1503	if (unlikely(page->mem_cgroup))
1504		bad_reason = "page still charged to cgroup";
1505#endif
1506	if (unlikely(bad_reason)) {
1507		bad_page(page, bad_reason, bad_flags);
1508		return 1;
1509	}
1510	return 0;
 
1511}
1512
1513static inline bool free_pages_prezeroed(bool poisoned)
1514{
1515	return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
1516		page_poisoning_enabled() && poisoned;
 
 
 
 
 
 
 
 
 
 
 
 
1517}
1518
1519static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
1520								int alloc_flags)
1521{
1522	int i;
1523	bool poisoned = true;
 
1524
1525	for (i = 0; i < (1 << order); i++) {
1526		struct page *p = page + i;
1527		if (unlikely(check_new_page(p)))
1528			return 1;
1529		if (poisoned)
1530			poisoned &= page_is_poisoned(p);
1531	}
 
 
 
 
1532
1533	set_page_private(page, 0);
1534	set_page_refcounted(page);
1535
1536	arch_alloc_page(page, order);
1537	kernel_map_pages(page, 1 << order, 1);
1538	kernel_poison_pages(page, 1 << order, 1);
1539	kasan_alloc_pages(page, order);
1540
1541	if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO))
1542		for (i = 0; i < (1 << order); i++)
1543			clear_highpage(page + i);
1544
1545	if (order && (gfp_flags & __GFP_COMP))
1546		prep_compound_page(page, order);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1547
1548	set_page_owner(page, order, gfp_flags);
 
 
 
 
 
 
 
 
 
 
1549
1550	/*
1551	 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
1552	 * allocate the page. The expectation is that the caller is taking
1553	 * steps that will free more memory. The caller should avoid the page
1554	 * being used for !PFMEMALLOC purposes.
1555	 */
1556	if (alloc_flags & ALLOC_NO_WATERMARKS)
1557		set_page_pfmemalloc(page);
1558	else
1559		clear_page_pfmemalloc(page);
1560
1561	return 0;
1562}
1563
1564/*
1565 * Go through the free lists for the given migratetype and remove
1566 * the smallest available page from the freelists
1567 */
1568static inline
1569struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
1570						int migratetype)
1571{
1572	unsigned int current_order;
1573	struct free_area *area;
1574	struct page *page;
1575
1576	/* Find a page of the appropriate size in the preferred list */
1577	for (current_order = order; current_order < MAX_ORDER; ++current_order) {
1578		area = &(zone->free_area[current_order]);
1579		page = list_first_entry_or_null(&area->free_list[migratetype],
1580							struct page, lru);
1581		if (!page)
1582			continue;
1583		list_del(&page->lru);
1584		rmv_page_order(page);
1585		area->nr_free--;
1586		expand(zone, page, order, current_order, area, migratetype);
1587		set_pcppage_migratetype(page, migratetype);
 
 
 
1588		return page;
1589	}
1590
1591	return NULL;
1592}
1593
1594
1595/*
1596 * This array describes the order lists are fallen back to when
1597 * the free lists for the desirable migrate type are depleted
 
 
1598 */
1599static int fallbacks[MIGRATE_TYPES][4] = {
1600	[MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
1601	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
1602	[MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
1603#ifdef CONFIG_CMA
1604	[MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
1605#endif
1606#ifdef CONFIG_MEMORY_ISOLATION
1607	[MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
1608#endif
1609};
1610
1611#ifdef CONFIG_CMA
1612static struct page *__rmqueue_cma_fallback(struct zone *zone,
1613					unsigned int order)
1614{
1615	return __rmqueue_smallest(zone, order, MIGRATE_CMA);
1616}
1617#else
1618static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
1619					unsigned int order) { return NULL; }
1620#endif
1621
1622/*
1623 * Move the free pages in a range to the free lists of the requested type.
1624 * Note that start_page and end_pages are not aligned on a pageblock
1625 * boundary. If alignment is required, use move_freepages_block()
1626 */
1627int move_freepages(struct zone *zone,
1628			  struct page *start_page, struct page *end_page,
1629			  int migratetype)
1630{
1631	struct page *page;
 
1632	unsigned int order;
1633	int pages_moved = 0;
1634
1635#ifndef CONFIG_HOLES_IN_ZONE
1636	/*
1637	 * page_zone is not safe to call in this context when
1638	 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
1639	 * anyway as we check zone boundaries in move_freepages_block().
1640	 * Remove at a later date when no bug reports exist related to
1641	 * grouping pages by mobility
1642	 */
1643	VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
1644#endif
1645
1646	for (page = start_page; page <= end_page;) {
1647		/* Make sure we are not inadvertently changing nodes */
1648		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
1649
1650		if (!pfn_valid_within(page_to_pfn(page))) {
1651			page++;
1652			continue;
1653		}
1654
1655		if (!PageBuddy(page)) {
1656			page++;
 
 
 
 
 
 
 
 
1657			continue;
1658		}
1659
1660		order = page_order(page);
1661		list_move(&page->lru,
1662			  &zone->free_area[order].free_list[migratetype]);
1663		page += 1 << order;
 
 
 
1664		pages_moved += 1 << order;
1665	}
1666
1667	return pages_moved;
1668}
1669
1670int move_freepages_block(struct zone *zone, struct page *page,
1671				int migratetype)
1672{
1673	unsigned long start_pfn, end_pfn;
1674	struct page *start_page, *end_page;
1675
1676	start_pfn = page_to_pfn(page);
1677	start_pfn = start_pfn & ~(pageblock_nr_pages-1);
1678	start_page = pfn_to_page(start_pfn);
1679	end_page = start_page + pageblock_nr_pages - 1;
1680	end_pfn = start_pfn + pageblock_nr_pages - 1;
 
1681
1682	/* Do not cross zone boundaries */
1683	if (!zone_spans_pfn(zone, start_pfn))
1684		start_page = page;
1685	if (!zone_spans_pfn(zone, end_pfn))
1686		return 0;
1687
1688	return move_freepages(zone, start_page, end_page, migratetype);
 
1689}
1690
1691static void change_pageblock_range(struct page *pageblock_page,
1692					int start_order, int migratetype)
1693{
1694	int nr_pageblocks = 1 << (start_order - pageblock_order);
1695
1696	while (nr_pageblocks--) {
1697		set_pageblock_migratetype(pageblock_page, migratetype);
1698		pageblock_page += pageblock_nr_pages;
1699	}
1700}
1701
1702/*
1703 * When we are falling back to another migratetype during allocation, try to
1704 * steal extra free pages from the same pageblocks to satisfy further
1705 * allocations, instead of polluting multiple pageblocks.
1706 *
1707 * If we are stealing a relatively large buddy page, it is likely there will
1708 * be more free pages in the pageblock, so try to steal them all. For
1709 * reclaimable and unmovable allocations, we steal regardless of page size,
1710 * as fragmentation caused by those allocations polluting movable pageblocks
1711 * is worse than movable allocations stealing from unmovable and reclaimable
1712 * pageblocks.
1713 */
1714static bool can_steal_fallback(unsigned int order, int start_mt)
1715{
1716	/*
1717	 * Leaving this order check is intended, although there is
1718	 * relaxed order check in next check. The reason is that
1719	 * we can actually steal whole pageblock if this condition met,
1720	 * but, below check doesn't guarantee it and that is just heuristic
1721	 * so could be changed anytime.
1722	 */
1723	if (order >= pageblock_order)
1724		return true;
1725
1726	if (order >= pageblock_order / 2 ||
1727		start_mt == MIGRATE_RECLAIMABLE ||
1728		start_mt == MIGRATE_UNMOVABLE ||
1729		page_group_by_mobility_disabled)
1730		return true;
1731
1732	return false;
1733}
1734
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1735/*
1736 * This function implements actual steal behaviour. If order is large enough,
1737 * we can steal whole pageblock. If not, we first move freepages in this
1738 * pageblock and check whether half of pages are moved or not. If half of
1739 * pages are moved, we can change migratetype of pageblock and permanently
1740 * use it's pages as requested migratetype in the future.
 
1741 */
1742static void steal_suitable_fallback(struct zone *zone, struct page *page,
1743							  int start_type)
1744{
1745	unsigned int current_order = page_order(page);
1746	int pages;
 
 
 
 
 
 
 
 
 
 
1747
1748	/* Take ownership for orders >= pageblock_order */
1749	if (current_order >= pageblock_order) {
1750		change_pageblock_range(page, current_order, start_type);
1751		return;
1752	}
1753
1754	pages = move_freepages_block(zone, page, start_type);
 
 
 
 
 
 
1755
1756	/* Claim the whole block if over half of it is free */
1757	if (pages >= (1 << (pageblock_order-1)) ||
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1758			page_group_by_mobility_disabled)
1759		set_pageblock_migratetype(page, start_type);
 
 
 
 
 
1760}
1761
1762/*
1763 * Check whether there is a suitable fallback freepage with requested order.
1764 * If only_stealable is true, this function returns fallback_mt only if
1765 * we can steal other freepages all together. This would help to reduce
1766 * fragmentation due to mixed migratetype pages in one pageblock.
1767 */
1768int find_suitable_fallback(struct free_area *area, unsigned int order,
1769			int migratetype, bool only_stealable, bool *can_steal)
1770{
1771	int i;
1772	int fallback_mt;
1773
1774	if (area->nr_free == 0)
1775		return -1;
1776
1777	*can_steal = false;
1778	for (i = 0;; i++) {
1779		fallback_mt = fallbacks[migratetype][i];
1780		if (fallback_mt == MIGRATE_TYPES)
1781			break;
1782
1783		if (list_empty(&area->free_list[fallback_mt]))
1784			continue;
1785
1786		if (can_steal_fallback(order, migratetype))
1787			*can_steal = true;
1788
1789		if (!only_stealable)
1790			return fallback_mt;
1791
1792		if (*can_steal)
1793			return fallback_mt;
1794	}
1795
1796	return -1;
1797}
1798
1799/*
1800 * Reserve a pageblock for exclusive use of high-order atomic allocations if
1801 * there are no empty page blocks that contain a page with a suitable order
1802 */
1803static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
1804				unsigned int alloc_order)
1805{
1806	int mt;
1807	unsigned long max_managed, flags;
1808
1809	/*
1810	 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
 
 
1811	 * Check is race-prone but harmless.
1812	 */
1813	max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
 
 
1814	if (zone->nr_reserved_highatomic >= max_managed)
1815		return;
1816
1817	spin_lock_irqsave(&zone->lock, flags);
1818
1819	/* Recheck the nr_reserved_highatomic limit under the lock */
1820	if (zone->nr_reserved_highatomic >= max_managed)
1821		goto out_unlock;
1822
1823	/* Yoink! */
1824	mt = get_pageblock_migratetype(page);
1825	if (mt != MIGRATE_HIGHATOMIC &&
1826			!is_migrate_isolate(mt) && !is_migrate_cma(mt)) {
1827		zone->nr_reserved_highatomic += pageblock_nr_pages;
1828		set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
1829		move_freepages_block(zone, page, MIGRATE_HIGHATOMIC);
1830	}
1831
1832out_unlock:
1833	spin_unlock_irqrestore(&zone->lock, flags);
1834}
1835
1836/*
1837 * Used when an allocation is about to fail under memory pressure. This
1838 * potentially hurts the reliability of high-order allocations when under
1839 * intense memory pressure but failed atomic allocations should be easier
1840 * to recover from than an OOM.
 
 
 
1841 */
1842static void unreserve_highatomic_pageblock(const struct alloc_context *ac)
 
1843{
1844	struct zonelist *zonelist = ac->zonelist;
1845	unsigned long flags;
1846	struct zoneref *z;
1847	struct zone *zone;
1848	struct page *page;
1849	int order;
 
1850
1851	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
1852								ac->nodemask) {
1853		/* Preserve at least one pageblock */
1854		if (zone->nr_reserved_highatomic <= pageblock_nr_pages)
 
 
 
 
1855			continue;
1856
1857		spin_lock_irqsave(&zone->lock, flags);
1858		for (order = 0; order < MAX_ORDER; order++) {
1859			struct free_area *area = &(zone->free_area[order]);
1860
1861			page = list_first_entry_or_null(
1862					&area->free_list[MIGRATE_HIGHATOMIC],
1863					struct page, lru);
1864			if (!page)
1865				continue;
1866
1867			/*
1868			 * It should never happen but changes to locking could
1869			 * inadvertently allow a per-cpu drain to add pages
1870			 * to MIGRATE_HIGHATOMIC while unreserving so be safe
1871			 * and watch for underflows.
 
1872			 */
1873			zone->nr_reserved_highatomic -= min(pageblock_nr_pages,
1874				zone->nr_reserved_highatomic);
 
 
 
 
 
 
 
 
 
 
1875
1876			/*
1877			 * Convert to ac->migratetype and avoid the normal
1878			 * pageblock stealing heuristics. Minimally, the caller
1879			 * is doing the work and needs the pages. More
1880			 * importantly, if the block was always converted to
1881			 * MIGRATE_UNMOVABLE or another type then the number
1882			 * of pageblocks that cannot be completely freed
1883			 * may increase.
1884			 */
1885			set_pageblock_migratetype(page, ac->migratetype);
1886			move_freepages_block(zone, page, ac->migratetype);
1887			spin_unlock_irqrestore(&zone->lock, flags);
1888			return;
 
 
 
1889		}
1890		spin_unlock_irqrestore(&zone->lock, flags);
1891	}
 
 
1892}
1893
1894/* Remove an element from the buddy allocator from the fallback list */
1895static inline struct page *
1896__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
 
 
 
 
 
 
 
 
 
 
1897{
1898	struct free_area *area;
1899	unsigned int current_order;
 
1900	struct page *page;
1901	int fallback_mt;
1902	bool can_steal;
1903
1904	/* Find the largest possible block of pages in the other list */
1905	for (current_order = MAX_ORDER-1;
1906				current_order >= order && current_order <= MAX_ORDER-1;
 
 
 
 
 
 
 
 
 
 
 
1907				--current_order) {
1908		area = &(zone->free_area[current_order]);
1909		fallback_mt = find_suitable_fallback(area, current_order,
1910				start_migratetype, false, &can_steal);
1911		if (fallback_mt == -1)
1912			continue;
1913
1914		page = list_first_entry(&area->free_list[fallback_mt],
1915						struct page, lru);
1916		if (can_steal)
1917			steal_suitable_fallback(zone, page, start_migratetype);
1918
1919		/* Remove the page from the freelists */
1920		area->nr_free--;
1921		list_del(&page->lru);
1922		rmv_page_order(page);
1923
1924		expand(zone, page, order, current_order, area,
1925					start_migratetype);
1926		/*
1927		 * The pcppage_migratetype may differ from pageblock's
1928		 * migratetype depending on the decisions in
1929		 * find_suitable_fallback(). This is OK as long as it does not
1930		 * differ for MIGRATE_CMA pageblocks. Those can be used as
1931		 * fallback only via special __rmqueue_cma_fallback() function
1932		 */
1933		set_pcppage_migratetype(page, start_migratetype);
1934
1935		trace_mm_page_alloc_extfrag(page, order, current_order,
1936			start_migratetype, fallback_mt);
1937
1938		return page;
 
 
 
 
 
 
 
 
1939	}
1940
1941	return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1942}
1943
1944/*
1945 * Do the hard work of removing an element from the buddy allocator.
1946 * Call me with the zone->lock already held.
1947 */
1948static struct page *__rmqueue(struct zone *zone, unsigned int order,
1949				int migratetype)
 
1950{
1951	struct page *page;
1952
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1953	page = __rmqueue_smallest(zone, order, migratetype);
1954	if (unlikely(!page)) {
1955		if (migratetype == MIGRATE_MOVABLE)
1956			page = __rmqueue_cma_fallback(zone, order);
1957
1958		if (!page)
1959			page = __rmqueue_fallback(zone, order, migratetype);
 
1960	}
1961
1962	trace_mm_page_alloc_zone_locked(page, order, migratetype);
1963	return page;
1964}
1965
1966/*
1967 * Obtain a specified number of elements from the buddy allocator, all under
1968 * a single hold of the lock, for efficiency.  Add them to the supplied list.
1969 * Returns the number of new pages which were placed at *list.
1970 */
1971static int rmqueue_bulk(struct zone *zone, unsigned int order,
1972			unsigned long count, struct list_head *list,
1973			int migratetype, bool cold)
1974{
 
1975	int i;
1976
1977	spin_lock(&zone->lock);
1978	for (i = 0; i < count; ++i) {
1979		struct page *page = __rmqueue(zone, order, migratetype);
 
1980		if (unlikely(page == NULL))
1981			break;
1982
1983		/*
1984		 * Split buddy pages returned by expand() are received here
1985		 * in physical page order. The page is added to the callers and
1986		 * list and the list head then moves forward. From the callers
1987		 * perspective, the linked list is ordered by page number in
1988		 * some conditions. This is useful for IO devices that can
1989		 * merge IO requests if the physical pages are ordered
1990		 * properly.
 
1991		 */
1992		if (likely(!cold))
1993			list_add(&page->lru, list);
1994		else
1995			list_add_tail(&page->lru, list);
1996		list = &page->lru;
1997		if (is_migrate_cma(get_pcppage_migratetype(page)))
1998			__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
1999					      -(1 << order));
2000	}
 
2001	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2002	spin_unlock(&zone->lock);
 
2003	return i;
2004}
2005
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2006#ifdef CONFIG_NUMA
2007/*
2008 * Called from the vmstat counter updater to drain pagesets of this
2009 * currently executing processor on remote nodes after they have
2010 * expired.
2011 *
2012 * Note that this function must be called with the thread pinned to
2013 * a single processor.
2014 */
2015void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2016{
2017	unsigned long flags;
2018	int to_drain, batch;
2019
2020	local_irq_save(flags);
2021	batch = READ_ONCE(pcp->batch);
2022	to_drain = min(pcp->count, batch);
2023	if (to_drain > 0) {
2024		free_pcppages_bulk(zone, to_drain, pcp);
2025		pcp->count -= to_drain;
 
2026	}
2027	local_irq_restore(flags);
2028}
2029#endif
2030
2031/*
2032 * Drain pcplists of the indicated processor and zone.
2033 *
2034 * The processor must either be the current processor and the
2035 * thread pinned to the current processor or a processor that
2036 * is not online.
2037 */
2038static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2039{
2040	unsigned long flags;
2041	struct per_cpu_pageset *pset;
2042	struct per_cpu_pages *pcp;
2043
2044	local_irq_save(flags);
2045	pset = per_cpu_ptr(zone->pageset, cpu);
2046
2047	pcp = &pset->pcp;
2048	if (pcp->count) {
2049		free_pcppages_bulk(zone, pcp->count, pcp);
2050		pcp->count = 0;
 
2051	}
2052	local_irq_restore(flags);
2053}
2054
2055/*
2056 * Drain pcplists of all zones on the indicated processor.
2057 *
2058 * The processor must either be the current processor and the
2059 * thread pinned to the current processor or a processor that
2060 * is not online.
2061 */
2062static void drain_pages(unsigned int cpu)
2063{
2064	struct zone *zone;
2065
2066	for_each_populated_zone(zone) {
2067		drain_pages_zone(cpu, zone);
2068	}
2069}
2070
2071/*
2072 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2073 *
2074 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
2075 * the single zone's pages.
2076 */
2077void drain_local_pages(struct zone *zone)
2078{
2079	int cpu = smp_processor_id();
2080
2081	if (zone)
2082		drain_pages_zone(cpu, zone);
2083	else
2084		drain_pages(cpu);
2085}
2086
2087/*
2088 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
2089 *
2090 * When zone parameter is non-NULL, spill just the single zone's pages.
2091 *
2092 * Note that this code is protected against sending an IPI to an offline
2093 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
2094 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
2095 * nothing keeps CPUs from showing up after we populated the cpumask and
2096 * before the call to on_each_cpu_mask().
2097 */
2098void drain_all_pages(struct zone *zone)
2099{
2100	int cpu;
2101
2102	/*
2103	 * Allocate in the BSS so we wont require allocation in
2104	 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
2105	 */
2106	static cpumask_t cpus_with_pcps;
2107
2108	/*
 
 
 
 
 
 
 
 
 
 
 
2109	 * We don't care about racing with CPU hotplug event
2110	 * as offline notification will cause the notified
2111	 * cpu to drain that CPU pcps and on_each_cpu_mask
2112	 * disables preemption as part of its processing
2113	 */
2114	for_each_online_cpu(cpu) {
2115		struct per_cpu_pageset *pcp;
2116		struct zone *z;
2117		bool has_pcps = false;
2118
2119		if (zone) {
2120			pcp = per_cpu_ptr(zone->pageset, cpu);
2121			if (pcp->pcp.count)
 
 
 
 
 
 
2122				has_pcps = true;
2123		} else {
2124			for_each_populated_zone(z) {
2125				pcp = per_cpu_ptr(z->pageset, cpu);
2126				if (pcp->pcp.count) {
2127					has_pcps = true;
2128					break;
2129				}
2130			}
2131		}
2132
2133		if (has_pcps)
2134			cpumask_set_cpu(cpu, &cpus_with_pcps);
2135		else
2136			cpumask_clear_cpu(cpu, &cpus_with_pcps);
2137	}
2138	on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
2139								zone, 1);
 
 
 
 
 
 
 
2140}
2141
2142#ifdef CONFIG_HIBERNATION
 
 
 
 
 
 
 
 
2143
2144void mark_free_pages(struct zone *zone)
 
2145{
2146	unsigned long pfn, max_zone_pfn;
2147	unsigned long flags;
2148	unsigned int order, t;
2149	struct page *page;
2150
2151	if (zone_is_empty(zone))
2152		return;
2153
2154	spin_lock_irqsave(&zone->lock, flags);
 
 
 
2155
2156	max_zone_pfn = zone_end_pfn(zone);
2157	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2158		if (pfn_valid(pfn)) {
2159			page = pfn_to_page(pfn);
2160			if (!swsusp_page_is_forbidden(page))
2161				swsusp_unset_page_free(page);
2162		}
2163
2164	for_each_migratetype_order(order, t) {
2165		list_for_each_entry(page,
2166				&zone->free_area[order].free_list[t], lru) {
2167			unsigned long i;
2168
2169			pfn = page_to_pfn(page);
2170			for (i = 0; i < (1UL << order); i++)
2171				swsusp_set_page_free(pfn_to_page(pfn + i));
2172		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2173	}
2174	spin_unlock_irqrestore(&zone->lock, flags);
2175}
2176#endif /* CONFIG_PM */
2177
2178/*
2179 * Free a 0-order page
2180 * cold == true ? free a cold page : free a hot page
2181 */
2182void free_hot_cold_page(struct page *page, bool cold)
2183{
2184	struct zone *zone = page_zone(page);
2185	struct per_cpu_pages *pcp;
2186	unsigned long flags;
2187	unsigned long pfn = page_to_pfn(page);
2188	int migratetype;
2189
2190	if (!free_pages_prepare(page, 0))
2191		return;
2192
2193	migratetype = get_pfnblock_migratetype(page, pfn);
2194	set_pcppage_migratetype(page, migratetype);
2195	local_irq_save(flags);
2196	__count_vm_event(PGFREE);
2197
2198	/*
2199	 * We only track unmovable, reclaimable and movable on pcp lists.
2200	 * Free ISOLATE pages back to the allocator because they are being
2201	 * offlined but treat RESERVE as movable pages so we can get those
2202	 * areas back if necessary. Otherwise, we may have to free
2203	 * excessively into the page allocator
2204	 */
2205	if (migratetype >= MIGRATE_PCPTYPES) {
 
2206		if (unlikely(is_migrate_isolate(migratetype))) {
2207			free_one_page(zone, page, pfn, 0, migratetype);
2208			goto out;
2209		}
2210		migratetype = MIGRATE_MOVABLE;
2211	}
2212
2213	pcp = &this_cpu_ptr(zone->pageset)->pcp;
2214	if (!cold)
2215		list_add(&page->lru, &pcp->lists[migratetype]);
2216	else
2217		list_add_tail(&page->lru, &pcp->lists[migratetype]);
2218	pcp->count++;
2219	if (pcp->count >= pcp->high) {
2220		unsigned long batch = READ_ONCE(pcp->batch);
2221		free_pcppages_bulk(zone, batch, pcp);
2222		pcp->count -= batch;
2223	}
2224
2225out:
2226	local_irq_restore(flags);
2227}
2228
2229/*
2230 * Free a list of 0-order pages
2231 */
2232void free_hot_cold_page_list(struct list_head *list, bool cold)
2233{
2234	struct page *page, *next;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2235
2236	list_for_each_entry_safe(page, next, list, lru) {
2237		trace_mm_page_free_batched(page, cold);
2238		free_hot_cold_page(page, cold);
 
 
 
 
 
 
 
2239	}
 
 
 
 
 
 
2240}
2241
2242/*
2243 * split_page takes a non-compound higher-order page, and splits it into
2244 * n (1<<order) sub-pages: page[0..n]
2245 * Each sub-page must be freed individually.
2246 *
2247 * Note: this is probably too low level an operation for use in drivers.
2248 * Please consult with lkml before using this in your driver.
2249 */
2250void split_page(struct page *page, unsigned int order)
2251{
2252	int i;
2253	gfp_t gfp_mask;
2254
2255	VM_BUG_ON_PAGE(PageCompound(page), page);
2256	VM_BUG_ON_PAGE(!page_count(page), page);
2257
2258#ifdef CONFIG_KMEMCHECK
2259	/*
2260	 * Split shadow pages too, because free(page[0]) would
2261	 * otherwise free the whole shadow.
2262	 */
2263	if (kmemcheck_page_is_tracked(page))
2264		split_page(virt_to_page(page[0].shadow), order);
2265#endif
2266
2267	gfp_mask = get_page_owner_gfp(page);
2268	set_page_owner(page, 0, gfp_mask);
2269	for (i = 1; i < (1 << order); i++) {
2270		set_page_refcounted(page + i);
2271		set_page_owner(page + i, 0, gfp_mask);
2272	}
2273}
2274EXPORT_SYMBOL_GPL(split_page);
2275
2276int __isolate_free_page(struct page *page, unsigned int order)
2277{
2278	unsigned long watermark;
2279	struct zone *zone;
2280	int mt;
2281
2282	BUG_ON(!PageBuddy(page));
2283
2284	zone = page_zone(page);
2285	mt = get_pageblock_migratetype(page);
2286
2287	if (!is_migrate_isolate(mt)) {
2288		/* Obey watermarks as if the page was being allocated */
2289		watermark = low_wmark_pages(zone) + (1 << order);
2290		if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
 
 
 
 
 
 
2291			return 0;
2292
2293		__mod_zone_freepage_state(zone, -(1UL << order), mt);
2294	}
2295
2296	/* Remove page from free list */
2297	list_del(&page->lru);
2298	zone->free_area[order].nr_free--;
2299	rmv_page_order(page);
2300
2301	set_page_owner(page, order, __GFP_MOVABLE);
2302
2303	/* Set the pageblock if the isolated page is at least a pageblock */
 
2304	if (order >= pageblock_order - 1) {
2305		struct page *endpage = page + (1 << order) - 1;
2306		for (; page < endpage; page += pageblock_nr_pages) {
2307			int mt = get_pageblock_migratetype(page);
2308			if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
 
 
 
 
2309				set_pageblock_migratetype(page,
2310							  MIGRATE_MOVABLE);
2311		}
2312	}
2313
2314
2315	return 1UL << order;
2316}
2317
2318/*
2319 * Similar to split_page except the page is already free. As this is only
2320 * being used for migration, the migratetype of the block also changes.
2321 * As this is called with interrupts disabled, the caller is responsible
2322 * for calling arch_alloc_page() and kernel_map_page() after interrupts
2323 * are enabled.
2324 *
2325 * Note: this is probably too low level an operation for use in drivers.
2326 * Please consult with lkml before using this in your driver.
2327 */
2328int split_free_page(struct page *page)
2329{
2330	unsigned int order;
2331	int nr_pages;
2332
2333	order = page_order(page);
2334
2335	nr_pages = __isolate_free_page(page, order);
2336	if (!nr_pages)
2337		return 0;
2338
2339	/* Split into individual pages */
2340	set_page_refcounted(page);
2341	split_page(page, order);
2342	return nr_pages;
2343}
2344
2345/*
2346 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
2347 */
2348static inline
2349struct page *buffered_rmqueue(struct zone *preferred_zone,
2350			struct zone *zone, unsigned int order,
2351			gfp_t gfp_flags, int alloc_flags, int migratetype)
2352{
2353	unsigned long flags;
2354	struct page *page;
2355	bool cold = ((gfp_flags & __GFP_COLD) != 0);
2356
2357	if (likely(order == 0)) {
2358		struct per_cpu_pages *pcp;
2359		struct list_head *list;
2360
2361		local_irq_save(flags);
2362		pcp = &this_cpu_ptr(zone->pageset)->pcp;
2363		list = &pcp->lists[migratetype];
2364		if (list_empty(list)) {
2365			pcp->count += rmqueue_bulk(zone, 0,
2366					pcp->batch, list,
2367					migratetype, cold);
2368			if (unlikely(list_empty(list)))
2369				goto failed;
2370		}
2371
2372		if (cold)
2373			page = list_last_entry(list, struct page, lru);
2374		else
2375			page = list_first_entry(list, struct page, lru);
 
 
 
 
 
2376
2377		list_del(&page->lru);
2378		pcp->count--;
2379	} else {
2380		/*
2381		 * We most definitely don't want callers attempting to
2382		 * allocate greater than order-1 page units with __GFP_NOFAIL.
2383		 */
2384		WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
2385		spin_lock_irqsave(&zone->lock, flags);
2386
 
2387		page = NULL;
2388		if (alloc_flags & ALLOC_HARDER) {
 
2389			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
2390			if (page)
2391				trace_mm_page_alloc_zone_locked(page, order, migratetype);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2392		}
2393		if (!page)
2394			page = __rmqueue(zone, order, migratetype);
2395		spin_unlock(&zone->lock);
2396		if (!page)
2397			goto failed;
2398		__mod_zone_freepage_state(zone, -(1 << order),
2399					  get_pcppage_migratetype(page));
2400	}
 
2401
2402	__mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
2403	if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 &&
2404	    !test_bit(ZONE_FAIR_DEPLETED, &zone->flags))
2405		set_bit(ZONE_FAIR_DEPLETED, &zone->flags);
2406
2407	__count_zone_vm_events(PGALLOC, zone, 1 << order);
2408	zone_statistics(preferred_zone, zone, gfp_flags);
2409	local_irq_restore(flags);
2410
2411	VM_BUG_ON_PAGE(bad_range(zone, page), page);
2412	return page;
2413
2414failed:
2415	local_irq_restore(flags);
2416	return NULL;
2417}
2418
2419#ifdef CONFIG_FAIL_PAGE_ALLOC
 
 
 
2420
2421static struct {
2422	struct fault_attr attr;
 
 
2423
2424	bool ignore_gfp_highmem;
2425	bool ignore_gfp_reclaim;
2426	u32 min_order;
2427} fail_page_alloc = {
2428	.attr = FAULT_ATTR_INITIALIZER,
2429	.ignore_gfp_reclaim = true,
2430	.ignore_gfp_highmem = true,
2431	.min_order = 1,
2432};
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2433
2434static int __init setup_fail_page_alloc(char *str)
 
 
 
 
 
 
2435{
2436	return setup_fault_attr(&fail_page_alloc.attr, str);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2437}
2438__setup("fail_page_alloc=", setup_fail_page_alloc);
2439
2440static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
 
 
 
2441{
2442	if (order < fail_page_alloc.min_order)
2443		return false;
2444	if (gfp_mask & __GFP_NOFAIL)
2445		return false;
2446	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
2447		return false;
2448	if (fail_page_alloc.ignore_gfp_reclaim &&
2449			(gfp_mask & __GFP_DIRECT_RECLAIM))
2450		return false;
2451
2452	return should_fail(&fail_page_alloc.attr, 1 << order);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2453}
2454
2455#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
 
 
 
2456
2457static int __init fail_page_alloc_debugfs(void)
 
 
 
 
 
 
 
 
 
 
 
2458{
2459	umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
2460	struct dentry *dir;
2461
2462	dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
2463					&fail_page_alloc.attr);
2464	if (IS_ERR(dir))
2465		return PTR_ERR(dir);
2466
2467	if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
2468				&fail_page_alloc.ignore_gfp_reclaim))
2469		goto fail;
2470	if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
2471				&fail_page_alloc.ignore_gfp_highmem))
2472		goto fail;
2473	if (!debugfs_create_u32("min-order", mode, dir,
2474				&fail_page_alloc.min_order))
2475		goto fail;
2476
2477	return 0;
2478fail:
2479	debugfs_remove_recursive(dir);
 
 
2480
2481	return -ENOMEM;
2482}
 
 
 
 
2483
2484late_initcall(fail_page_alloc_debugfs);
 
2485
2486#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
 
 
 
 
 
 
2487
2488#else /* CONFIG_FAIL_PAGE_ALLOC */
 
 
2489
2490static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
2491{
2492	return false;
2493}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2494
2495#endif /* CONFIG_FAIL_PAGE_ALLOC */
 
2496
2497/*
2498 * Return true if free base pages are above 'mark'. For high-order checks it
2499 * will return true of the order-0 watermark is reached and there is at least
2500 * one free page of a suitable size. Checking now avoids taking the zone lock
2501 * to check in the allocation paths if no pages are free.
2502 */
2503static bool __zone_watermark_ok(struct zone *z, unsigned int order,
2504			unsigned long mark, int classzone_idx, int alloc_flags,
2505			long free_pages)
2506{
2507	long min = mark;
2508	int o;
2509	const int alloc_harder = (alloc_flags & ALLOC_HARDER);
2510
2511	/* free_pages may go negative - that's OK */
2512	free_pages -= (1 << order) - 1;
2513
2514	if (alloc_flags & ALLOC_HIGH)
2515		min -= min / 2;
 
 
 
 
 
2516
2517	/*
2518	 * If the caller does not have rights to ALLOC_HARDER then subtract
2519	 * the high-atomic reserves. This will over-estimate the size of the
2520	 * atomic reserve but it avoids a search.
2521	 */
2522	if (likely(!alloc_harder))
2523		free_pages -= z->nr_reserved_highatomic;
2524	else
2525		min -= min / 4;
 
2526
2527#ifdef CONFIG_CMA
2528	/* If allocation can't use CMA areas don't use free CMA pages */
2529	if (!(alloc_flags & ALLOC_CMA))
2530		free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
2531#endif
 
 
 
 
2532
2533	/*
2534	 * Check watermarks for an order-0 allocation request. If these
2535	 * are not met, then a high-order request also cannot go ahead
2536	 * even if a suitable page happened to be free.
2537	 */
2538	if (free_pages <= min + z->lowmem_reserve[classzone_idx])
2539		return false;
2540
2541	/* If this is an order-0 request then the watermark is fine */
2542	if (!order)
2543		return true;
2544
2545	/* For a high-order request, check at least one suitable page is free */
2546	for (o = order; o < MAX_ORDER; o++) {
2547		struct free_area *area = &z->free_area[o];
2548		int mt;
2549
2550		if (!area->nr_free)
2551			continue;
2552
2553		if (alloc_harder)
2554			return true;
2555
2556		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
2557			if (!list_empty(&area->free_list[mt]))
2558				return true;
2559		}
2560
2561#ifdef CONFIG_CMA
2562		if ((alloc_flags & ALLOC_CMA) &&
2563		    !list_empty(&area->free_list[MIGRATE_CMA])) {
2564			return true;
2565		}
2566#endif
 
 
 
 
2567	}
2568	return false;
2569}
2570
2571bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
2572		      int classzone_idx, int alloc_flags)
2573{
2574	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
2575					zone_page_state(z, NR_FREE_PAGES));
2576}
2577
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2578bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
2579			unsigned long mark, int classzone_idx)
2580{
2581	long free_pages = zone_page_state(z, NR_FREE_PAGES);
2582
2583	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
2584		free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
2585
2586	return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
2587								free_pages);
2588}
2589
2590#ifdef CONFIG_NUMA
2591static bool zone_local(struct zone *local_zone, struct zone *zone)
2592{
2593	return local_zone->node == zone->node;
2594}
2595
2596static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
2597{
2598	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <
2599				RECLAIM_DISTANCE;
2600}
2601#else	/* CONFIG_NUMA */
2602static bool zone_local(struct zone *local_zone, struct zone *zone)
2603{
2604	return true;
2605}
2606
2607static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
2608{
2609	return true;
2610}
2611#endif	/* CONFIG_NUMA */
2612
2613static void reset_alloc_batches(struct zone *preferred_zone)
 
 
 
 
 
 
 
 
 
2614{
2615	struct zone *zone = preferred_zone->zone_pgdat->node_zones;
2616
2617	do {
2618		mod_zone_page_state(zone, NR_ALLOC_BATCH,
2619			high_wmark_pages(zone) - low_wmark_pages(zone) -
2620			atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
2621		clear_bit(ZONE_FAIR_DEPLETED, &zone->flags);
2622	} while (zone++ != preferred_zone);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2623}
2624
2625/*
2626 * get_page_from_freelist goes through the zonelist trying to allocate
2627 * a page.
2628 */
2629static struct page *
2630get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
2631						const struct alloc_context *ac)
2632{
2633	struct zonelist *zonelist = ac->zonelist;
2634	struct zoneref *z;
2635	struct page *page = NULL;
2636	struct zone *zone;
2637	int nr_fair_skipped = 0;
2638	bool zonelist_rescan;
2639
2640zonelist_scan:
2641	zonelist_rescan = false;
2642
 
2643	/*
2644	 * Scan zonelist, looking for a zone with enough free.
2645	 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
2646	 */
2647	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2648								ac->nodemask) {
 
 
 
2649		unsigned long mark;
2650
2651		if (cpusets_enabled() &&
2652			(alloc_flags & ALLOC_CPUSET) &&
2653			!cpuset_zone_allowed(zone, gfp_mask))
2654				continue;
2655		/*
2656		 * Distribute pages in proportion to the individual
2657		 * zone size to ensure fair page aging.  The zone a
2658		 * page was allocated in should have no effect on the
2659		 * time the page has in memory before being reclaimed.
2660		 */
2661		if (alloc_flags & ALLOC_FAIR) {
2662			if (!zone_local(ac->preferred_zone, zone))
2663				break;
2664			if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) {
2665				nr_fair_skipped++;
2666				continue;
2667			}
2668		}
2669		/*
2670		 * When allocating a page cache page for writing, we
2671		 * want to get it from a zone that is within its dirty
2672		 * limit, such that no single zone holds more than its
2673		 * proportional share of globally allowed dirty pages.
2674		 * The dirty limits take into account the zone's
2675		 * lowmem reserves and high watermark so that kswapd
2676		 * should be able to balance it without having to
2677		 * write pages from its LRU list.
2678		 *
2679		 * This may look like it could increase pressure on
2680		 * lower zones by failing allocations in higher zones
2681		 * before they are full.  But the pages that do spill
2682		 * over are limited as the lower zones are protected
2683		 * by this very same mechanism.  It should not become
2684		 * a practical burden to them.
2685		 *
2686		 * XXX: For now, allow allocations to potentially
2687		 * exceed the per-zone dirty limit in the slowpath
2688		 * (spread_dirty_pages unset) before going into reclaim,
2689		 * which is important when on a NUMA setup the allowed
2690		 * zones are together not big enough to reach the
2691		 * global limit.  The proper fix for these situations
2692		 * will require awareness of zones in the
2693		 * dirty-throttling and the flusher threads.
2694		 */
2695		if (ac->spread_dirty_pages && !zone_dirty_ok(zone))
2696			continue;
 
 
 
2697
2698		mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
2699		if (!zone_watermark_ok(zone, order, mark,
2700				       ac->classzone_idx, alloc_flags)) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2701			int ret;
2702
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2703			/* Checked here to keep the fast path fast */
2704			BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
2705			if (alloc_flags & ALLOC_NO_WATERMARKS)
2706				goto try_this_zone;
2707
2708			if (zone_reclaim_mode == 0 ||
2709			    !zone_allows_reclaim(ac->preferred_zone, zone))
2710				continue;
2711
2712			ret = zone_reclaim(zone, gfp_mask, order);
2713			switch (ret) {
2714			case ZONE_RECLAIM_NOSCAN:
2715				/* did not scan */
2716				continue;
2717			case ZONE_RECLAIM_FULL:
2718				/* scanned but unreclaimable */
2719				continue;
2720			default:
2721				/* did we reclaim enough */
2722				if (zone_watermark_ok(zone, order, mark,
2723						ac->classzone_idx, alloc_flags))
2724					goto try_this_zone;
2725
2726				continue;
2727			}
2728		}
2729
2730try_this_zone:
2731		page = buffered_rmqueue(ac->preferred_zone, zone, order,
2732				gfp_mask, alloc_flags, ac->migratetype);
2733		if (page) {
2734			if (prep_new_page(page, order, gfp_mask, alloc_flags))
2735				goto try_this_zone;
2736
2737			/*
2738			 * If this is a high-order atomic allocation then check
2739			 * if the pageblock should be reserved for the future
2740			 */
2741			if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
2742				reserve_highatomic_pageblock(page, zone, order);
2743
2744			return page;
 
 
 
 
 
 
 
 
 
 
 
 
 
2745		}
2746	}
2747
2748	/*
2749	 * The first pass makes sure allocations are spread fairly within the
2750	 * local node.  However, the local node might have free pages left
2751	 * after the fairness batches are exhausted, and remote zones haven't
2752	 * even been considered yet.  Try once more without fairness, and
2753	 * include remote zones now, before entering the slowpath and waking
2754	 * kswapd: prefer spilling to a remote zone over swapping locally.
2755	 */
2756	if (alloc_flags & ALLOC_FAIR) {
2757		alloc_flags &= ~ALLOC_FAIR;
2758		if (nr_fair_skipped) {
2759			zonelist_rescan = true;
2760			reset_alloc_batches(ac->preferred_zone);
2761		}
2762		if (nr_online_nodes > 1)
2763			zonelist_rescan = true;
2764	}
2765
2766	if (zonelist_rescan)
2767		goto zonelist_scan;
2768
2769	return NULL;
2770}
2771
2772/*
2773 * Large machines with many possible nodes should not always dump per-node
2774 * meminfo in irq context.
2775 */
2776static inline bool should_suppress_show_mem(void)
2777{
2778	bool ret = false;
2779
2780#if NODES_SHIFT > 8
2781	ret = in_interrupt();
2782#endif
2783	return ret;
2784}
2785
2786static DEFINE_RATELIMIT_STATE(nopage_rs,
2787		DEFAULT_RATELIMIT_INTERVAL,
2788		DEFAULT_RATELIMIT_BURST);
2789
2790void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, const char *fmt, ...)
2791{
2792	unsigned int filter = SHOW_MEM_FILTER_NODES;
2793
2794	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
2795	    debug_guardpage_minorder() > 0)
2796		return;
2797
2798	/*
2799	 * This documents exceptions given to allocations in certain
2800	 * contexts that are allowed to allocate outside current's set
2801	 * of allowed nodes.
2802	 */
2803	if (!(gfp_mask & __GFP_NOMEMALLOC))
2804		if (test_thread_flag(TIF_MEMDIE) ||
2805		    (current->flags & (PF_MEMALLOC | PF_EXITING)))
2806			filter &= ~SHOW_MEM_FILTER_NODES;
2807	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
2808		filter &= ~SHOW_MEM_FILTER_NODES;
2809
2810	if (fmt) {
2811		struct va_format vaf;
2812		va_list args;
2813
2814		va_start(args, fmt);
2815
2816		vaf.fmt = fmt;
2817		vaf.va = &args;
 
 
 
2818
2819		pr_warn("%pV", &vaf);
 
 
 
2820
2821		va_end(args);
2822	}
 
 
 
 
 
2823
2824	pr_warn("%s: page allocation failure: order:%u, mode:%#x(%pGg)\n",
2825		current->comm, order, gfp_mask, &gfp_mask);
2826	dump_stack();
2827	if (!should_suppress_show_mem())
2828		show_mem(filter);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2829}
2830
2831static inline struct page *
2832__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
2833	const struct alloc_context *ac, unsigned long *did_some_progress)
2834{
2835	struct oom_control oc = {
2836		.zonelist = ac->zonelist,
2837		.nodemask = ac->nodemask,
 
2838		.gfp_mask = gfp_mask,
2839		.order = order,
2840	};
2841	struct page *page;
2842
2843	*did_some_progress = 0;
2844
2845	/*
2846	 * Acquire the oom lock.  If that fails, somebody else is
2847	 * making progress for us.
2848	 */
2849	if (!mutex_trylock(&oom_lock)) {
2850		*did_some_progress = 1;
2851		schedule_timeout_uninterruptible(1);
2852		return NULL;
2853	}
2854
2855	/*
2856	 * Go through the zonelist yet one more time, keep very high watermark
2857	 * here, this is only to catch a parallel oom killing, we must fail if
2858	 * we're still under heavy pressure.
2859	 */
2860	page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order,
2861					ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
 
 
 
2862	if (page)
2863		goto out;
2864
2865	if (!(gfp_mask & __GFP_NOFAIL)) {
2866		/* Coredumps can quickly deplete all memory reserves */
2867		if (current->flags & PF_DUMPCORE)
2868			goto out;
2869		/* The OOM killer will not help higher order allocs */
2870		if (order > PAGE_ALLOC_COSTLY_ORDER)
2871			goto out;
2872		/* The OOM killer does not needlessly kill tasks for lowmem */
2873		if (ac->high_zoneidx < ZONE_NORMAL)
2874			goto out;
2875		/* The OOM killer does not compensate for IO-less reclaim */
2876		if (!(gfp_mask & __GFP_FS)) {
2877			/*
2878			 * XXX: Page reclaim didn't yield anything,
2879			 * and the OOM killer can't be invoked, but
2880			 * keep looping as per tradition.
2881			 *
2882			 * But do not keep looping if oom_killer_disable()
2883			 * was already called, for the system is trying to
2884			 * enter a quiescent state during suspend.
2885			 */
2886			*did_some_progress = !oom_killer_disabled;
2887			goto out;
2888		}
2889		if (pm_suspended_storage())
2890			goto out;
2891		/* The OOM killer may not free memory on a specific node */
2892		if (gfp_mask & __GFP_THISNODE)
2893			goto out;
2894	}
2895	/* Exhausted what can be done so it's blamo time */
2896	if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
 
 
2897		*did_some_progress = 1;
2898
2899		if (gfp_mask & __GFP_NOFAIL) {
2900			page = get_page_from_freelist(gfp_mask, order,
2901					ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac);
2902			/*
2903			 * fallback to ignore cpuset restriction if our nodes
2904			 * are depleted
2905			 */
2906			if (!page)
2907				page = get_page_from_freelist(gfp_mask, order,
2908					ALLOC_NO_WATERMARKS, ac);
2909		}
2910	}
2911out:
2912	mutex_unlock(&oom_lock);
2913	return page;
2914}
2915
 
 
 
 
 
 
2916#ifdef CONFIG_COMPACTION
2917/* Try memory compaction for high-order allocations before reclaim */
2918static struct page *
2919__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2920		int alloc_flags, const struct alloc_context *ac,
2921		enum migrate_mode mode, int *contended_compaction,
2922		bool *deferred_compaction)
2923{
2924	unsigned long compact_result;
2925	struct page *page;
 
2926
2927	if (!order)
2928		return NULL;
2929
2930	current->flags |= PF_MEMALLOC;
2931	compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
2932						mode, contended_compaction);
2933	current->flags &= ~PF_MEMALLOC;
2934
2935	switch (compact_result) {
2936	case COMPACT_DEFERRED:
2937		*deferred_compaction = true;
2938		/* fall-through */
2939	case COMPACT_SKIPPED:
2940		return NULL;
2941	default:
2942		break;
2943	}
2944
 
 
2945	/*
2946	 * At least in one zone compaction wasn't deferred or skipped, so let's
2947	 * count a compaction stall
2948	 */
2949	count_vm_event(COMPACTSTALL);
2950
2951	page = get_page_from_freelist(gfp_mask, order,
2952					alloc_flags & ~ALLOC_NO_WATERMARKS, ac);
 
 
 
 
 
2953
2954	if (page) {
2955		struct zone *zone = page_zone(page);
2956
2957		zone->compact_blockskip_flush = false;
2958		compaction_defer_reset(zone, order, true);
2959		count_vm_event(COMPACTSUCCESS);
2960		return page;
2961	}
2962
2963	/*
2964	 * It's bad if compaction run occurs and fails. The most likely reason
2965	 * is that pages exist, but not enough to satisfy watermarks.
2966	 */
2967	count_vm_event(COMPACTFAIL);
2968
2969	cond_resched();
2970
2971	return NULL;
2972}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2973#else
2974static inline struct page *
2975__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2976		int alloc_flags, const struct alloc_context *ac,
2977		enum migrate_mode mode, int *contended_compaction,
2978		bool *deferred_compaction)
2979{
 
2980	return NULL;
2981}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2982#endif /* CONFIG_COMPACTION */
2983
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2984/* Perform direct synchronous page reclaim */
2985static int
2986__perform_reclaim(gfp_t gfp_mask, unsigned int order,
2987					const struct alloc_context *ac)
2988{
2989	struct reclaim_state reclaim_state;
2990	int progress;
2991
2992	cond_resched();
2993
2994	/* We now go into synchronous reclaim */
2995	cpuset_memory_pressure_bump();
2996	current->flags |= PF_MEMALLOC;
2997	lockdep_set_current_reclaim_state(gfp_mask);
2998	reclaim_state.reclaimed_slab = 0;
2999	current->reclaim_state = &reclaim_state;
3000
3001	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
3002								ac->nodemask);
3003
3004	current->reclaim_state = NULL;
3005	lockdep_clear_current_reclaim_state();
3006	current->flags &= ~PF_MEMALLOC;
3007
3008	cond_resched();
3009
3010	return progress;
3011}
3012
3013/* The really slow allocator path where we enter direct reclaim */
3014static inline struct page *
3015__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
3016		int alloc_flags, const struct alloc_context *ac,
3017		unsigned long *did_some_progress)
3018{
3019	struct page *page = NULL;
 
3020	bool drained = false;
3021
 
3022	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
3023	if (unlikely(!(*did_some_progress)))
3024		return NULL;
3025
3026retry:
3027	page = get_page_from_freelist(gfp_mask, order,
3028					alloc_flags & ~ALLOC_NO_WATERMARKS, ac);
3029
3030	/*
3031	 * If an allocation failed after direct reclaim, it could be because
3032	 * pages are pinned on the per-cpu lists or in high alloc reserves.
3033	 * Shrink them them and try again
3034	 */
3035	if (!page && !drained) {
3036		unreserve_highatomic_pageblock(ac);
3037		drain_all_pages(NULL);
3038		drained = true;
3039		goto retry;
3040	}
 
 
3041
3042	return page;
3043}
3044
3045static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
 
3046{
3047	struct zoneref *z;
3048	struct zone *zone;
 
 
3049
3050	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
3051						ac->high_zoneidx, ac->nodemask)
3052		wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone));
 
 
 
 
 
 
3053}
3054
3055static inline int
3056gfp_to_alloc_flags(gfp_t gfp_mask)
3057{
3058	int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
3059
3060	/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
3061	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
 
 
 
 
 
3062
3063	/*
3064	 * The caller may dip into page reserves a bit more if the caller
3065	 * cannot run direct reclaim, or if the caller has realtime scheduling
3066	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
3067	 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
3068	 */
3069	alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
 
3070
3071	if (gfp_mask & __GFP_ATOMIC) {
3072		/*
3073		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
3074		 * if it can't schedule.
3075		 */
3076		if (!(gfp_mask & __GFP_NOMEMALLOC))
3077			alloc_flags |= ALLOC_HARDER;
 
 
 
 
 
3078		/*
3079		 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
3080		 * comment for __cpuset_node_allowed().
 
3081		 */
3082		alloc_flags &= ~ALLOC_CPUSET;
3083	} else if (unlikely(rt_task(current)) && !in_interrupt())
3084		alloc_flags |= ALLOC_HARDER;
3085
3086	if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
3087		if (gfp_mask & __GFP_MEMALLOC)
3088			alloc_flags |= ALLOC_NO_WATERMARKS;
3089		else if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
3090			alloc_flags |= ALLOC_NO_WATERMARKS;
3091		else if (!in_interrupt() &&
3092				((current->flags & PF_MEMALLOC) ||
3093				 unlikely(test_thread_flag(TIF_MEMDIE))))
3094			alloc_flags |= ALLOC_NO_WATERMARKS;
3095	}
3096#ifdef CONFIG_CMA
3097	if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
3098		alloc_flags |= ALLOC_CMA;
3099#endif
3100	return alloc_flags;
3101}
3102
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3103bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
3104{
3105	return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS);
3106}
3107
3108static inline bool is_thp_gfp_mask(gfp_t gfp_mask)
 
 
 
 
 
 
 
 
 
 
 
 
 
3109{
3110	return (gfp_mask & (GFP_TRANSHUGE | __GFP_KSWAPD_RECLAIM)) == GFP_TRANSHUGE;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3111}
3112
3113static inline struct page *
3114__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
3115						struct alloc_context *ac)
3116{
3117	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
 
 
3118	struct page *page = NULL;
3119	int alloc_flags;
3120	unsigned long pages_reclaimed = 0;
3121	unsigned long did_some_progress;
3122	enum migrate_mode migration_mode = MIGRATE_ASYNC;
3123	bool deferred_compaction = false;
3124	int contended_compaction = COMPACT_CONTENDED_NONE;
 
 
 
 
3125
3126	/*
3127	 * In the slowpath, we sanity check order to avoid ever trying to
3128	 * reclaim >= MAX_ORDER areas which will never succeed. Callers may
3129	 * be using allocators in order of preference for an area that is
3130	 * too large.
3131	 */
3132	if (order >= MAX_ORDER) {
3133		WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
3134		return NULL;
3135	}
3136
3137	/*
3138	 * We also sanity check to catch abuse of atomic reserves being used by
3139	 * callers that are not in atomic context.
 
3140	 */
3141	if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
3142				(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
3143		gfp_mask &= ~__GFP_ATOMIC;
3144
3145retry:
3146	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3147		wake_all_kswapds(order, ac);
3148
3149	/*
3150	 * OK, we're below the kswapd watermark and have kicked background
3151	 * reclaim. Now things get more complex, so set up alloc_flags according
3152	 * to how we want to proceed.
 
3153	 */
3154	alloc_flags = gfp_to_alloc_flags(gfp_mask);
 
 
 
3155
3156	/*
3157	 * Find the true preferred zone if the allocation is unconstrained by
3158	 * cpusets.
3159	 */
3160	if (!(alloc_flags & ALLOC_CPUSET) && !ac->nodemask) {
3161		struct zoneref *preferred_zoneref;
3162		preferred_zoneref = first_zones_zonelist(ac->zonelist,
3163				ac->high_zoneidx, NULL, &ac->preferred_zone);
3164		ac->classzone_idx = zonelist_zone_idx(preferred_zoneref);
 
 
3165	}
3166
3167	/* This is the last chance, in general, before the goto nopage. */
3168	page = get_page_from_freelist(gfp_mask, order,
3169				alloc_flags & ~ALLOC_NO_WATERMARKS, ac);
 
 
 
 
 
3170	if (page)
3171		goto got_pg;
3172
3173	/* Allocate without watermarks if the context allows */
3174	if (alloc_flags & ALLOC_NO_WATERMARKS) {
3175		/*
3176		 * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
3177		 * the allocation is high priority and these type of
3178		 * allocations are system rather than user orientated
3179		 */
3180		ac->zonelist = node_zonelist(numa_node_id(), gfp_mask);
3181		page = get_page_from_freelist(gfp_mask, order,
3182						ALLOC_NO_WATERMARKS, ac);
 
 
 
 
 
 
 
3183		if (page)
3184			goto got_pg;
3185	}
3186
3187	/* Caller is not willing to reclaim, we can't balance anything */
3188	if (!can_direct_reclaim) {
3189		/*
3190		 * All existing users of the __GFP_NOFAIL are blockable, so warn
3191		 * of any new users that actually allow this type of allocation
3192		 * to fail.
3193		 */
3194		WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL);
3195		goto nopage;
3196	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3197
3198	/* Avoid recursion of direct reclaim */
3199	if (current->flags & PF_MEMALLOC) {
3200		/*
3201		 * __GFP_NOFAIL request from this context is rather bizarre
3202		 * because we cannot reclaim anything and only can loop waiting
3203		 * for somebody to do a work for us.
3204		 */
3205		if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3206			cond_resched();
3207			goto retry;
3208		}
3209		goto nopage;
3210	}
3211
3212	/* Avoid allocations with no watermarks from looping endlessly */
3213	if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
3214		goto nopage;
 
 
 
 
 
 
3215
3216	/*
3217	 * Try direct compaction. The first pass is asynchronous. Subsequent
3218	 * attempts after direct reclaim are synchronous
 
3219	 */
3220	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
3221					migration_mode,
3222					&contended_compaction,
3223					&deferred_compaction);
 
 
 
 
3224	if (page)
3225		goto got_pg;
3226
3227	/* Checks for THP-specific high-order allocations */
3228	if (is_thp_gfp_mask(gfp_mask)) {
3229		/*
3230		 * If compaction is deferred for high-order allocations, it is
3231		 * because sync compaction recently failed. If this is the case
3232		 * and the caller requested a THP allocation, we do not want
3233		 * to heavily disrupt the system, so we fail the allocation
3234		 * instead of entering direct reclaim.
3235		 */
3236		if (deferred_compaction)
3237			goto nopage;
3238
3239		/*
3240		 * In all zones where compaction was attempted (and not
3241		 * deferred or skipped), lock contention has been detected.
3242		 * For THP allocation we do not want to disrupt the others
3243		 * so we fallback to base pages instead.
3244		 */
3245		if (contended_compaction == COMPACT_CONTENDED_LOCK)
3246			goto nopage;
3247
3248		/*
3249		 * If compaction was aborted due to need_resched(), we do not
3250		 * want to further increase allocation latency, unless it is
3251		 * khugepaged trying to collapse.
3252		 */
3253		if (contended_compaction == COMPACT_CONTENDED_SCHED
3254			&& !(current->flags & PF_KTHREAD))
3255			goto nopage;
3256	}
3257
3258	/*
3259	 * It can become very expensive to allocate transparent hugepages at
3260	 * fault, so use asynchronous memory compaction for THP unless it is
3261	 * khugepaged trying to collapse.
3262	 */
3263	if (!is_thp_gfp_mask(gfp_mask) || (current->flags & PF_KTHREAD))
3264		migration_mode = MIGRATE_SYNC_LIGHT;
3265
3266	/* Try direct reclaim and then allocating */
3267	page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
3268							&did_some_progress);
3269	if (page)
3270		goto got_pg;
3271
 
 
 
 
 
 
3272	/* Do not loop if specifically requested */
3273	if (gfp_mask & __GFP_NORETRY)
3274		goto noretry;
 
 
 
 
 
 
 
 
3275
3276	/* Keep reclaiming pages as long as there is reasonable progress */
3277	pages_reclaimed += did_some_progress;
3278	if ((did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) ||
3279	    ((gfp_mask & __GFP_REPEAT) && pages_reclaimed < (1 << order))) {
3280		/* Wait for some write requests to complete then retry */
3281		wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50);
3282		goto retry;
3283	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3284
3285	/* Reclaim has failed us, start killing things */
3286	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
3287	if (page)
3288		goto got_pg;
3289
 
 
 
 
 
 
3290	/* Retry as long as the OOM killer is making progress */
3291	if (did_some_progress)
 
3292		goto retry;
 
3293
3294noretry:
3295	/*
3296	 * High-order allocations do not necessarily loop after
3297	 * direct reclaim and reclaim/compaction depends on compaction
3298	 * being called after reclaim so call directly if necessary
3299	 */
3300	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags,
3301					    ac, migration_mode,
3302					    &contended_compaction,
3303					    &deferred_compaction);
3304	if (page)
3305		goto got_pg;
3306nopage:
3307	warn_alloc_failed(gfp_mask, order, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3308got_pg:
3309	return page;
3310}
3311
3312/*
3313 * This is the 'heart' of the zoned buddy allocator.
3314 */
3315struct page *
3316__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
3317			struct zonelist *zonelist, nodemask_t *nodemask)
3318{
3319	struct zoneref *preferred_zoneref;
3320	struct page *page = NULL;
3321	unsigned int cpuset_mems_cookie;
3322	int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
3323	gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
3324	struct alloc_context ac = {
3325		.high_zoneidx = gfp_zone(gfp_mask),
3326		.nodemask = nodemask,
3327		.migratetype = gfpflags_to_migratetype(gfp_mask),
3328	};
3329
3330	gfp_mask &= gfp_allowed_mask;
3331
3332	lockdep_trace_alloc(gfp_mask);
 
 
 
 
 
 
 
 
 
 
3333
3334	might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
3335
3336	if (should_fail_alloc_page(gfp_mask, order))
3337		return NULL;
 
 
 
 
 
3338
3339	/*
3340	 * Check the zones suitable for the gfp_mask contain at least one
3341	 * valid zone. It's possible to have an empty zonelist as a result
3342	 * of __GFP_THISNODE and a memoryless node
3343	 */
3344	if (unlikely(!zonelist->_zonerefs->zone))
3345		return NULL;
3346
3347	if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE)
3348		alloc_flags |= ALLOC_CMA;
3349
3350retry_cpuset:
3351	cpuset_mems_cookie = read_mems_allowed_begin();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3352
3353	/* We set it here, as __alloc_pages_slowpath might have changed it */
3354	ac.zonelist = zonelist;
 
 
 
 
3355
3356	/* Dirty zone balancing only done in the fast path */
3357	ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE);
 
3358
3359	/* The preferred zone is used for statistics later */
3360	preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx,
3361				ac.nodemask ? : &cpuset_current_mems_allowed,
3362				&ac.preferred_zone);
3363	if (!ac.preferred_zone)
3364		goto out;
3365	ac.classzone_idx = zonelist_zone_idx(preferred_zoneref);
3366
3367	/* First allocation attempt */
3368	alloc_mask = gfp_mask|__GFP_HARDWALL;
3369	page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
3370	if (unlikely(!page)) {
3371		/*
3372		 * Runtime PM, block IO and its error handling path
3373		 * can deadlock because I/O on the device might not
3374		 * complete.
3375		 */
3376		alloc_mask = memalloc_noio_flags(gfp_mask);
3377		ac.spread_dirty_pages = false;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3378
3379		page = __alloc_pages_slowpath(alloc_mask, order, &ac);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3380	}
3381
3382	if (kmemcheck_enabled && page)
3383		kmemcheck_pagealloc_alloc(page, order, gfp_mask);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3384
3385	trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
 
3386
3387out:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3388	/*
3389	 * When updating a task's mems_allowed, it is possible to race with
3390	 * parallel threads in such a way that an allocation can fail while
3391	 * the mask is being updated. If a page allocation is about to fail,
3392	 * check if the cpuset changed during allocation and if so, retry.
3393	 */
3394	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
3395		goto retry_cpuset;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3396
3397	return page;
3398}
3399EXPORT_SYMBOL(__alloc_pages_nodemask);
 
 
 
 
 
 
 
 
 
3400
3401/*
3402 * Common helper functions.
 
 
3403 */
3404unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
3405{
3406	struct page *page;
3407
3408	/*
3409	 * __get_free_pages() returns a 32-bit address, which cannot represent
3410	 * a highmem page
3411	 */
3412	VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
3413
3414	page = alloc_pages(gfp_mask, order);
3415	if (!page)
3416		return 0;
3417	return (unsigned long) page_address(page);
3418}
3419EXPORT_SYMBOL(__get_free_pages);
3420
3421unsigned long get_zeroed_page(gfp_t gfp_mask)
3422{
3423	return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
3424}
3425EXPORT_SYMBOL(get_zeroed_page);
3426
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3427void __free_pages(struct page *page, unsigned int order)
3428{
3429	if (put_page_testzero(page)) {
3430		if (order == 0)
3431			free_hot_cold_page(page, false);
3432		else
3433			__free_pages_ok(page, order);
3434	}
3435}
3436
 
 
 
 
 
 
3437EXPORT_SYMBOL(__free_pages);
3438
3439void free_pages(unsigned long addr, unsigned int order)
3440{
3441	if (addr != 0) {
3442		VM_BUG_ON(!virt_addr_valid((void *)addr));
3443		__free_pages(virt_to_page((void *)addr), order);
3444	}
3445}
3446
3447EXPORT_SYMBOL(free_pages);
3448
3449/*
3450 * Page Fragment:
3451 *  An arbitrary-length arbitrary-offset area of memory which resides
3452 *  within a 0 or higher order page.  Multiple fragments within that page
3453 *  are individually refcounted, in the page's reference counter.
3454 *
3455 * The page_frag functions below provide a simple allocation framework for
3456 * page fragments.  This is used by the network stack and network device
3457 * drivers to provide a backing region of memory for use as either an
3458 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
3459 */
3460static struct page *__page_frag_refill(struct page_frag_cache *nc,
3461				       gfp_t gfp_mask)
3462{
3463	struct page *page = NULL;
3464	gfp_t gfp = gfp_mask;
3465
3466#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
3467	gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
3468		    __GFP_NOMEMALLOC;
3469	page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
3470				PAGE_FRAG_CACHE_MAX_ORDER);
3471	nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
3472#endif
3473	if (unlikely(!page))
3474		page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
3475
3476	nc->va = page ? page_address(page) : NULL;
3477
3478	return page;
3479}
3480
3481void *__alloc_page_frag(struct page_frag_cache *nc,
3482			unsigned int fragsz, gfp_t gfp_mask)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3483{
3484	unsigned int size = PAGE_SIZE;
3485	struct page *page;
3486	int offset;
3487
3488	if (unlikely(!nc->va)) {
3489refill:
3490		page = __page_frag_refill(nc, gfp_mask);
3491		if (!page)
3492			return NULL;
3493
3494#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
3495		/* if size can vary use size else just use PAGE_SIZE */
3496		size = nc->size;
3497#endif
3498		/* Even if we own the page, we do not use atomic_set().
3499		 * This would break get_page_unless_zero() users.
3500		 */
3501		page_ref_add(page, size - 1);
3502
3503		/* reset page count bias and offset to start of new frag */
3504		nc->pfmemalloc = page_is_pfmemalloc(page);
3505		nc->pagecnt_bias = size;
3506		nc->offset = size;
3507	}
3508
3509	offset = nc->offset - fragsz;
3510	if (unlikely(offset < 0)) {
3511		page = virt_to_page(nc->va);
3512
3513		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
3514			goto refill;
3515
 
 
 
 
 
3516#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
3517		/* if size can vary use size else just use PAGE_SIZE */
3518		size = nc->size;
3519#endif
3520		/* OK, page count is 0, we can safely set it */
3521		set_page_count(page, size);
3522
3523		/* reset page count bias and offset to start of new frag */
3524		nc->pagecnt_bias = size;
3525		offset = size - fragsz;
 
 
 
 
 
 
 
 
 
 
 
 
3526	}
3527
3528	nc->pagecnt_bias--;
 
3529	nc->offset = offset;
3530
3531	return nc->va + offset;
3532}
3533EXPORT_SYMBOL(__alloc_page_frag);
3534
3535/*
3536 * Frees a page fragment allocated out of either a compound or order 0 page.
3537 */
3538void __free_page_frag(void *addr)
3539{
3540	struct page *page = virt_to_head_page(addr);
3541
3542	if (unlikely(put_page_testzero(page)))
3543		__free_pages_ok(page, compound_order(page));
3544}
3545EXPORT_SYMBOL(__free_page_frag);
3546
3547/*
3548 * alloc_kmem_pages charges newly allocated pages to the kmem resource counter
3549 * of the current memory cgroup if __GFP_ACCOUNT is set, other than that it is
3550 * equivalent to alloc_pages.
3551 *
3552 * It should be used when the caller would like to use kmalloc, but since the
3553 * allocation is large, it has to fall back to the page allocator.
3554 */
3555struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order)
3556{
3557	struct page *page;
3558
3559	page = alloc_pages(gfp_mask, order);
3560	if (page && memcg_kmem_charge(page, gfp_mask, order) != 0) {
3561		__free_pages(page, order);
3562		page = NULL;
3563	}
3564	return page;
3565}
3566
3567struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
3568{
3569	struct page *page;
3570
3571	page = alloc_pages_node(nid, gfp_mask, order);
3572	if (page && memcg_kmem_charge(page, gfp_mask, order) != 0) {
3573		__free_pages(page, order);
3574		page = NULL;
3575	}
3576	return page;
3577}
3578
3579/*
3580 * __free_kmem_pages and free_kmem_pages will free pages allocated with
3581 * alloc_kmem_pages.
3582 */
3583void __free_kmem_pages(struct page *page, unsigned int order)
3584{
3585	memcg_kmem_uncharge(page, order);
3586	__free_pages(page, order);
3587}
3588
3589void free_kmem_pages(unsigned long addr, unsigned int order)
3590{
3591	if (addr != 0) {
3592		VM_BUG_ON(!virt_addr_valid((void *)addr));
3593		__free_kmem_pages(virt_to_page((void *)addr), order);
3594	}
3595}
 
3596
3597static void *make_alloc_exact(unsigned long addr, unsigned int order,
3598		size_t size)
3599{
3600	if (addr) {
3601		unsigned long alloc_end = addr + (PAGE_SIZE << order);
3602		unsigned long used = addr + PAGE_ALIGN(size);
3603
3604		split_page(virt_to_page((void *)addr), order);
3605		while (used < alloc_end) {
3606			free_page(used);
3607			used += PAGE_SIZE;
3608		}
 
 
 
 
3609	}
3610	return (void *)addr;
3611}
3612
3613/**
3614 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
3615 * @size: the number of bytes to allocate
3616 * @gfp_mask: GFP flags for the allocation
3617 *
3618 * This function is similar to alloc_pages(), except that it allocates the
3619 * minimum number of pages to satisfy the request.  alloc_pages() can only
3620 * allocate memory in power-of-two pages.
3621 *
3622 * This function is also limited by MAX_ORDER.
3623 *
3624 * Memory allocated by this function must be released by free_pages_exact().
 
 
3625 */
3626void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
3627{
3628	unsigned int order = get_order(size);
3629	unsigned long addr;
3630
 
 
 
3631	addr = __get_free_pages(gfp_mask, order);
3632	return make_alloc_exact(addr, order, size);
3633}
3634EXPORT_SYMBOL(alloc_pages_exact);
3635
3636/**
3637 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
3638 *			   pages on a node.
3639 * @nid: the preferred node ID where memory should be allocated
3640 * @size: the number of bytes to allocate
3641 * @gfp_mask: GFP flags for the allocation
3642 *
3643 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
3644 * back.
 
 
3645 */
3646void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
3647{
3648	unsigned int order = get_order(size);
3649	struct page *p = alloc_pages_node(nid, gfp_mask, order);
 
 
 
 
 
3650	if (!p)
3651		return NULL;
3652	return make_alloc_exact((unsigned long)page_address(p), order, size);
3653}
3654
3655/**
3656 * free_pages_exact - release memory allocated via alloc_pages_exact()
3657 * @virt: the value returned by alloc_pages_exact.
3658 * @size: size of allocation, same value as passed to alloc_pages_exact().
3659 *
3660 * Release the memory allocated by a previous call to alloc_pages_exact.
3661 */
3662void free_pages_exact(void *virt, size_t size)
3663{
3664	unsigned long addr = (unsigned long)virt;
3665	unsigned long end = addr + PAGE_ALIGN(size);
3666
3667	while (addr < end) {
3668		free_page(addr);
3669		addr += PAGE_SIZE;
3670	}
3671}
3672EXPORT_SYMBOL(free_pages_exact);
3673
3674/**
3675 * nr_free_zone_pages - count number of pages beyond high watermark
3676 * @offset: The zone index of the highest zone
3677 *
3678 * nr_free_zone_pages() counts the number of counts pages which are beyond the
3679 * high watermark within all zones at or below a given zone index.  For each
3680 * zone, the number of pages is calculated as:
3681 *     managed_pages - high_pages
 
 
 
3682 */
3683static unsigned long nr_free_zone_pages(int offset)
3684{
3685	struct zoneref *z;
3686	struct zone *zone;
3687
3688	/* Just pick one node, since fallback list is circular */
3689	unsigned long sum = 0;
3690
3691	struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
3692
3693	for_each_zone_zonelist(zone, z, zonelist, offset) {
3694		unsigned long size = zone->managed_pages;
3695		unsigned long high = high_wmark_pages(zone);
3696		if (size > high)
3697			sum += size - high;
3698	}
3699
3700	return sum;
3701}
3702
3703/**
3704 * nr_free_buffer_pages - count number of pages beyond high watermark
3705 *
3706 * nr_free_buffer_pages() counts the number of pages which are beyond the high
3707 * watermark within ZONE_DMA and ZONE_NORMAL.
 
 
 
3708 */
3709unsigned long nr_free_buffer_pages(void)
3710{
3711	return nr_free_zone_pages(gfp_zone(GFP_USER));
3712}
3713EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
3714
3715/**
3716 * nr_free_pagecache_pages - count number of pages beyond high watermark
3717 *
3718 * nr_free_pagecache_pages() counts the number of pages which are beyond the
3719 * high watermark within all zones.
3720 */
3721unsigned long nr_free_pagecache_pages(void)
3722{
3723	return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
3724}
3725
3726static inline void show_node(struct zone *zone)
3727{
3728	if (IS_ENABLED(CONFIG_NUMA))
3729		printk("Node %d ", zone_to_nid(zone));
3730}
3731
3732long si_mem_available(void)
3733{
3734	long available;
3735	unsigned long pagecache;
3736	unsigned long wmark_low = 0;
3737	unsigned long pages[NR_LRU_LISTS];
3738	struct zone *zone;
3739	int lru;
3740
3741	for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
3742		pages[lru] = global_page_state(NR_LRU_BASE + lru);
3743
3744	for_each_zone(zone)
3745		wmark_low += zone->watermark[WMARK_LOW];
3746
3747	/*
3748	 * Estimate the amount of memory available for userspace allocations,
3749	 * without causing swapping.
3750	 */
3751	available = global_page_state(NR_FREE_PAGES) - totalreserve_pages;
3752
3753	/*
3754	 * Not all the page cache can be freed, otherwise the system will
3755	 * start swapping. Assume at least half of the page cache, or the
3756	 * low watermark worth of cache, needs to stay.
3757	 */
3758	pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
3759	pagecache -= min(pagecache / 2, wmark_low);
3760	available += pagecache;
3761
3762	/*
3763	 * Part of the reclaimable slab consists of items that are in use,
3764	 * and cannot be freed. Cap this estimate at the low watermark.
3765	 */
3766	available += global_page_state(NR_SLAB_RECLAIMABLE) -
3767		     min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
3768
3769	if (available < 0)
3770		available = 0;
3771	return available;
3772}
3773EXPORT_SYMBOL_GPL(si_mem_available);
3774
3775void si_meminfo(struct sysinfo *val)
3776{
3777	val->totalram = totalram_pages;
3778	val->sharedram = global_page_state(NR_SHMEM);
3779	val->freeram = global_page_state(NR_FREE_PAGES);
3780	val->bufferram = nr_blockdev_pages();
3781	val->totalhigh = totalhigh_pages;
3782	val->freehigh = nr_free_highpages();
3783	val->mem_unit = PAGE_SIZE;
3784}
3785
3786EXPORT_SYMBOL(si_meminfo);
3787
3788#ifdef CONFIG_NUMA
3789void si_meminfo_node(struct sysinfo *val, int nid)
3790{
3791	int zone_type;		/* needs to be signed */
3792	unsigned long managed_pages = 0;
3793	pg_data_t *pgdat = NODE_DATA(nid);
3794
3795	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
3796		managed_pages += pgdat->node_zones[zone_type].managed_pages;
3797	val->totalram = managed_pages;
3798	val->sharedram = node_page_state(nid, NR_SHMEM);
3799	val->freeram = node_page_state(nid, NR_FREE_PAGES);
3800#ifdef CONFIG_HIGHMEM
3801	val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages;
3802	val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
3803			NR_FREE_PAGES);
3804#else
3805	val->totalhigh = 0;
3806	val->freehigh = 0;
3807#endif
3808	val->mem_unit = PAGE_SIZE;
3809}
3810#endif
3811
3812/*
3813 * Determine whether the node should be displayed or not, depending on whether
3814 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
3815 */
3816bool skip_free_areas_node(unsigned int flags, int nid)
3817{
3818	bool ret = false;
3819	unsigned int cpuset_mems_cookie;
3820
3821	if (!(flags & SHOW_MEM_FILTER_NODES))
3822		goto out;
3823
3824	do {
3825		cpuset_mems_cookie = read_mems_allowed_begin();
3826		ret = !node_isset(nid, cpuset_current_mems_allowed);
3827	} while (read_mems_allowed_retry(cpuset_mems_cookie));
3828out:
3829	return ret;
3830}
3831
3832#define K(x) ((x) << (PAGE_SHIFT-10))
3833
3834static void show_migration_types(unsigned char type)
3835{
3836	static const char types[MIGRATE_TYPES] = {
3837		[MIGRATE_UNMOVABLE]	= 'U',
3838		[MIGRATE_MOVABLE]	= 'M',
3839		[MIGRATE_RECLAIMABLE]	= 'E',
3840		[MIGRATE_HIGHATOMIC]	= 'H',
3841#ifdef CONFIG_CMA
3842		[MIGRATE_CMA]		= 'C',
3843#endif
3844#ifdef CONFIG_MEMORY_ISOLATION
3845		[MIGRATE_ISOLATE]	= 'I',
3846#endif
3847	};
3848	char tmp[MIGRATE_TYPES + 1];
3849	char *p = tmp;
3850	int i;
3851
3852	for (i = 0; i < MIGRATE_TYPES; i++) {
3853		if (type & (1 << i))
3854			*p++ = types[i];
3855	}
3856
3857	*p = '\0';
3858	printk("(%s) ", tmp);
3859}
3860
3861/*
3862 * Show free area list (used inside shift_scroll-lock stuff)
3863 * We also calculate the percentage fragmentation. We do this by counting the
3864 * memory on each free list with the exception of the first item on the list.
3865 *
3866 * Bits in @filter:
3867 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
3868 *   cpuset.
3869 */
3870void show_free_areas(unsigned int filter)
3871{
3872	unsigned long free_pcp = 0;
3873	int cpu;
3874	struct zone *zone;
3875
3876	for_each_populated_zone(zone) {
3877		if (skip_free_areas_node(filter, zone_to_nid(zone)))
3878			continue;
3879
3880		for_each_online_cpu(cpu)
3881			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
3882	}
3883
3884	printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
3885		" active_file:%lu inactive_file:%lu isolated_file:%lu\n"
3886		" unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
3887		" slab_reclaimable:%lu slab_unreclaimable:%lu\n"
3888		" mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
3889		" free:%lu free_pcp:%lu free_cma:%lu\n",
3890		global_page_state(NR_ACTIVE_ANON),
3891		global_page_state(NR_INACTIVE_ANON),
3892		global_page_state(NR_ISOLATED_ANON),
3893		global_page_state(NR_ACTIVE_FILE),
3894		global_page_state(NR_INACTIVE_FILE),
3895		global_page_state(NR_ISOLATED_FILE),
3896		global_page_state(NR_UNEVICTABLE),
3897		global_page_state(NR_FILE_DIRTY),
3898		global_page_state(NR_WRITEBACK),
3899		global_page_state(NR_UNSTABLE_NFS),
3900		global_page_state(NR_SLAB_RECLAIMABLE),
3901		global_page_state(NR_SLAB_UNRECLAIMABLE),
3902		global_page_state(NR_FILE_MAPPED),
3903		global_page_state(NR_SHMEM),
3904		global_page_state(NR_PAGETABLE),
3905		global_page_state(NR_BOUNCE),
3906		global_page_state(NR_FREE_PAGES),
3907		free_pcp,
3908		global_page_state(NR_FREE_CMA_PAGES));
3909
3910	for_each_populated_zone(zone) {
3911		int i;
3912
3913		if (skip_free_areas_node(filter, zone_to_nid(zone)))
3914			continue;
3915
3916		free_pcp = 0;
3917		for_each_online_cpu(cpu)
3918			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
3919
3920		show_node(zone);
3921		printk("%s"
3922			" free:%lukB"
3923			" min:%lukB"
3924			" low:%lukB"
3925			" high:%lukB"
3926			" active_anon:%lukB"
3927			" inactive_anon:%lukB"
3928			" active_file:%lukB"
3929			" inactive_file:%lukB"
3930			" unevictable:%lukB"
3931			" isolated(anon):%lukB"
3932			" isolated(file):%lukB"
3933			" present:%lukB"
3934			" managed:%lukB"
3935			" mlocked:%lukB"
3936			" dirty:%lukB"
3937			" writeback:%lukB"
3938			" mapped:%lukB"
3939			" shmem:%lukB"
3940			" slab_reclaimable:%lukB"
3941			" slab_unreclaimable:%lukB"
3942			" kernel_stack:%lukB"
3943			" pagetables:%lukB"
3944			" unstable:%lukB"
3945			" bounce:%lukB"
3946			" free_pcp:%lukB"
3947			" local_pcp:%ukB"
3948			" free_cma:%lukB"
3949			" writeback_tmp:%lukB"
3950			" pages_scanned:%lu"
3951			" all_unreclaimable? %s"
3952			"\n",
3953			zone->name,
3954			K(zone_page_state(zone, NR_FREE_PAGES)),
3955			K(min_wmark_pages(zone)),
3956			K(low_wmark_pages(zone)),
3957			K(high_wmark_pages(zone)),
3958			K(zone_page_state(zone, NR_ACTIVE_ANON)),
3959			K(zone_page_state(zone, NR_INACTIVE_ANON)),
3960			K(zone_page_state(zone, NR_ACTIVE_FILE)),
3961			K(zone_page_state(zone, NR_INACTIVE_FILE)),
3962			K(zone_page_state(zone, NR_UNEVICTABLE)),
3963			K(zone_page_state(zone, NR_ISOLATED_ANON)),
3964			K(zone_page_state(zone, NR_ISOLATED_FILE)),
3965			K(zone->present_pages),
3966			K(zone->managed_pages),
3967			K(zone_page_state(zone, NR_MLOCK)),
3968			K(zone_page_state(zone, NR_FILE_DIRTY)),
3969			K(zone_page_state(zone, NR_WRITEBACK)),
3970			K(zone_page_state(zone, NR_FILE_MAPPED)),
3971			K(zone_page_state(zone, NR_SHMEM)),
3972			K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
3973			K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
3974			zone_page_state(zone, NR_KERNEL_STACK) *
3975				THREAD_SIZE / 1024,
3976			K(zone_page_state(zone, NR_PAGETABLE)),
3977			K(zone_page_state(zone, NR_UNSTABLE_NFS)),
3978			K(zone_page_state(zone, NR_BOUNCE)),
3979			K(free_pcp),
3980			K(this_cpu_read(zone->pageset->pcp.count)),
3981			K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
3982			K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
3983			K(zone_page_state(zone, NR_PAGES_SCANNED)),
3984			(!zone_reclaimable(zone) ? "yes" : "no")
3985			);
3986		printk("lowmem_reserve[]:");
3987		for (i = 0; i < MAX_NR_ZONES; i++)
3988			printk(" %ld", zone->lowmem_reserve[i]);
3989		printk("\n");
3990	}
3991
3992	for_each_populated_zone(zone) {
3993		unsigned int order;
3994		unsigned long nr[MAX_ORDER], flags, total = 0;
3995		unsigned char types[MAX_ORDER];
3996
3997		if (skip_free_areas_node(filter, zone_to_nid(zone)))
3998			continue;
3999		show_node(zone);
4000		printk("%s: ", zone->name);
4001
4002		spin_lock_irqsave(&zone->lock, flags);
4003		for (order = 0; order < MAX_ORDER; order++) {
4004			struct free_area *area = &zone->free_area[order];
4005			int type;
4006
4007			nr[order] = area->nr_free;
4008			total += nr[order] << order;
4009
4010			types[order] = 0;
4011			for (type = 0; type < MIGRATE_TYPES; type++) {
4012				if (!list_empty(&area->free_list[type]))
4013					types[order] |= 1 << type;
4014			}
4015		}
4016		spin_unlock_irqrestore(&zone->lock, flags);
4017		for (order = 0; order < MAX_ORDER; order++) {
4018			printk("%lu*%lukB ", nr[order], K(1UL) << order);
4019			if (nr[order])
4020				show_migration_types(types[order]);
4021		}
4022		printk("= %lukB\n", K(total));
4023	}
4024
4025	hugetlb_show_meminfo();
4026
4027	printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
4028
4029	show_swap_cache_info();
4030}
4031
4032static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
4033{
4034	zoneref->zone = zone;
4035	zoneref->zone_idx = zone_idx(zone);
4036}
4037
4038/*
4039 * Builds allocation fallback zone lists.
4040 *
4041 * Add all populated zones of a node to the zonelist.
4042 */
4043static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
4044				int nr_zones)
4045{
4046	struct zone *zone;
4047	enum zone_type zone_type = MAX_NR_ZONES;
 
4048
4049	do {
4050		zone_type--;
4051		zone = pgdat->node_zones + zone_type;
4052		if (populated_zone(zone)) {
4053			zoneref_set_zone(zone,
4054				&zonelist->_zonerefs[nr_zones++]);
4055			check_highest_zone(zone_type);
4056		}
4057	} while (zone_type);
4058
4059	return nr_zones;
4060}
4061
4062
4063/*
4064 *  zonelist_order:
4065 *  0 = automatic detection of better ordering.
4066 *  1 = order by ([node] distance, -zonetype)
4067 *  2 = order by (-zonetype, [node] distance)
4068 *
4069 *  If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
4070 *  the same zonelist. So only NUMA can configure this param.
4071 */
4072#define ZONELIST_ORDER_DEFAULT  0
4073#define ZONELIST_ORDER_NODE     1
4074#define ZONELIST_ORDER_ZONE     2
4075
4076/* zonelist order in the kernel.
4077 * set_zonelist_order() will set this to NODE or ZONE.
4078 */
4079static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
4080static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
4081
4082
4083#ifdef CONFIG_NUMA
4084/* The value user specified ....changed by config */
4085static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
4086/* string for sysctl */
4087#define NUMA_ZONELIST_ORDER_LEN	16
4088char numa_zonelist_order[16] = "default";
4089
4090/*
4091 * interface for configure zonelist ordering.
4092 * command line option "numa_zonelist_order"
4093 *	= "[dD]efault	- default, automatic configuration.
4094 *	= "[nN]ode 	- order by node locality, then by zone within node
4095 *	= "[zZ]one      - order by zone, then by locality within zone
4096 */
4097
4098static int __parse_numa_zonelist_order(char *s)
4099{
4100	if (*s == 'd' || *s == 'D') {
4101		user_zonelist_order = ZONELIST_ORDER_DEFAULT;
4102	} else if (*s == 'n' || *s == 'N') {
4103		user_zonelist_order = ZONELIST_ORDER_NODE;
4104	} else if (*s == 'z' || *s == 'Z') {
4105		user_zonelist_order = ZONELIST_ORDER_ZONE;
4106	} else {
4107		pr_warn("Ignoring invalid numa_zonelist_order value:  %s\n", s);
4108		return -EINVAL;
4109	}
4110	return 0;
4111}
4112
4113static __init int setup_numa_zonelist_order(char *s)
4114{
4115	int ret;
4116
4117	if (!s)
4118		return 0;
4119
4120	ret = __parse_numa_zonelist_order(s);
4121	if (ret == 0)
4122		strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
4123
4124	return ret;
4125}
4126early_param("numa_zonelist_order", setup_numa_zonelist_order);
4127
4128/*
4129 * sysctl handler for numa_zonelist_order
4130 */
4131int numa_zonelist_order_handler(struct ctl_table *table, int write,
4132		void __user *buffer, size_t *length,
4133		loff_t *ppos)
4134{
4135	char saved_string[NUMA_ZONELIST_ORDER_LEN];
4136	int ret;
4137	static DEFINE_MUTEX(zl_order_mutex);
4138
4139	mutex_lock(&zl_order_mutex);
4140	if (write) {
4141		if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) {
4142			ret = -EINVAL;
4143			goto out;
4144		}
4145		strcpy(saved_string, (char *)table->data);
4146	}
4147	ret = proc_dostring(table, write, buffer, length, ppos);
4148	if (ret)
4149		goto out;
4150	if (write) {
4151		int oldval = user_zonelist_order;
4152
4153		ret = __parse_numa_zonelist_order((char *)table->data);
4154		if (ret) {
4155			/*
4156			 * bogus value.  restore saved string
4157			 */
4158			strncpy((char *)table->data, saved_string,
4159				NUMA_ZONELIST_ORDER_LEN);
4160			user_zonelist_order = oldval;
4161		} else if (oldval != user_zonelist_order) {
4162			mutex_lock(&zonelists_mutex);
4163			build_all_zonelists(NULL, NULL);
4164			mutex_unlock(&zonelists_mutex);
4165		}
4166	}
4167out:
4168	mutex_unlock(&zl_order_mutex);
4169	return ret;
4170}
4171
4172
4173#define MAX_NODE_LOAD (nr_online_nodes)
4174static int node_load[MAX_NUMNODES];
4175
4176/**
4177 * find_next_best_node - find the next node that should appear in a given node's fallback list
4178 * @node: node whose fallback list we're appending
4179 * @used_node_mask: nodemask_t of already used nodes
4180 *
4181 * We use a number of factors to determine which is the next node that should
4182 * appear on a given node's fallback list.  The node should not have appeared
4183 * already in @node's fallback list, and it should be the next closest node
4184 * according to the distance array (which contains arbitrary distance values
4185 * from each node to each node in the system), and should also prefer nodes
4186 * with no CPUs, since presumably they'll have very little allocation pressure
4187 * on them otherwise.
4188 * It returns -1 if no node is found.
 
4189 */
4190static int find_next_best_node(int node, nodemask_t *used_node_mask)
4191{
4192	int n, val;
4193	int min_val = INT_MAX;
4194	int best_node = NUMA_NO_NODE;
4195	const struct cpumask *tmp = cpumask_of_node(0);
4196
4197	/* Use the local node if we haven't already */
4198	if (!node_isset(node, *used_node_mask)) {
 
 
 
4199		node_set(node, *used_node_mask);
4200		return node;
4201	}
4202
4203	for_each_node_state(n, N_MEMORY) {
4204
4205		/* Don't want a node to appear more than once */
4206		if (node_isset(n, *used_node_mask))
4207			continue;
4208
4209		/* Use the distance array to find the distance */
4210		val = node_distance(node, n);
4211
4212		/* Penalize nodes under us ("prefer the next node") */
4213		val += (n < node);
4214
4215		/* Give preference to headless and unused nodes */
4216		tmp = cpumask_of_node(n);
4217		if (!cpumask_empty(tmp))
4218			val += PENALTY_FOR_NODE_WITH_CPUS;
4219
4220		/* Slight preference for less loaded node */
4221		val *= (MAX_NODE_LOAD*MAX_NUMNODES);
4222		val += node_load[n];
4223
4224		if (val < min_val) {
4225			min_val = val;
4226			best_node = n;
4227		}
4228	}
4229
4230	if (best_node >= 0)
4231		node_set(best_node, *used_node_mask);
4232
4233	return best_node;
4234}
4235
4236
4237/*
4238 * Build zonelists ordered by node and zones within node.
4239 * This results in maximum locality--normal zone overflows into local
4240 * DMA zone, if any--but risks exhausting DMA zone.
4241 */
4242static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
 
4243{
4244	int j;
4245	struct zonelist *zonelist;
4246
4247	zonelist = &pgdat->node_zonelists[0];
4248	for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
4249		;
4250	j = build_zonelists_node(NODE_DATA(node), zonelist, j);
4251	zonelist->_zonerefs[j].zone = NULL;
4252	zonelist->_zonerefs[j].zone_idx = 0;
 
 
 
 
 
 
4253}
4254
4255/*
4256 * Build gfp_thisnode zonelists
4257 */
4258static void build_thisnode_zonelists(pg_data_t *pgdat)
4259{
4260	int j;
4261	struct zonelist *zonelist;
4262
4263	zonelist = &pgdat->node_zonelists[1];
4264	j = build_zonelists_node(pgdat, zonelist, 0);
4265	zonelist->_zonerefs[j].zone = NULL;
4266	zonelist->_zonerefs[j].zone_idx = 0;
 
4267}
4268
4269/*
4270 * Build zonelists ordered by zone and nodes within zones.
4271 * This results in conserving DMA zone[s] until all Normal memory is
4272 * exhausted, but results in overflowing to remote node while memory
4273 * may still exist in local DMA zone.
4274 */
4275static int node_order[MAX_NUMNODES];
4276
4277static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
4278{
4279	int pos, j, node;
4280	int zone_type;		/* needs to be signed */
4281	struct zone *z;
4282	struct zonelist *zonelist;
4283
4284	zonelist = &pgdat->node_zonelists[0];
4285	pos = 0;
4286	for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
4287		for (j = 0; j < nr_nodes; j++) {
4288			node = node_order[j];
4289			z = &NODE_DATA(node)->node_zones[zone_type];
4290			if (populated_zone(z)) {
4291				zoneref_set_zone(z,
4292					&zonelist->_zonerefs[pos++]);
4293				check_highest_zone(zone_type);
4294			}
4295		}
4296	}
4297	zonelist->_zonerefs[pos].zone = NULL;
4298	zonelist->_zonerefs[pos].zone_idx = 0;
4299}
4300
4301#if defined(CONFIG_64BIT)
4302/*
4303 * Devices that require DMA32/DMA are relatively rare and do not justify a
4304 * penalty to every machine in case the specialised case applies. Default
4305 * to Node-ordering on 64-bit NUMA machines
4306 */
4307static int default_zonelist_order(void)
4308{
4309	return ZONELIST_ORDER_NODE;
4310}
4311#else
4312/*
4313 * On 32-bit, the Normal zone needs to be preserved for allocations accessible
4314 * by the kernel. If processes running on node 0 deplete the low memory zone
4315 * then reclaim will occur more frequency increasing stalls and potentially
4316 * be easier to OOM if a large percentage of the zone is under writeback or
4317 * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set.
4318 * Hence, default to zone ordering on 32-bit.
4319 */
4320static int default_zonelist_order(void)
4321{
4322	return ZONELIST_ORDER_ZONE;
4323}
4324#endif /* CONFIG_64BIT */
4325
4326static void set_zonelist_order(void)
4327{
4328	if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
4329		current_zonelist_order = default_zonelist_order();
4330	else
4331		current_zonelist_order = user_zonelist_order;
4332}
4333
4334static void build_zonelists(pg_data_t *pgdat)
4335{
4336	int i, node, load;
4337	nodemask_t used_mask;
 
4338	int local_node, prev_node;
4339	struct zonelist *zonelist;
4340	unsigned int order = current_zonelist_order;
4341
4342	/* initialize zonelists */
4343	for (i = 0; i < MAX_ZONELISTS; i++) {
4344		zonelist = pgdat->node_zonelists + i;
4345		zonelist->_zonerefs[0].zone = NULL;
4346		zonelist->_zonerefs[0].zone_idx = 0;
4347	}
4348
4349	/* NUMA-aware ordering of nodes */
4350	local_node = pgdat->node_id;
4351	load = nr_online_nodes;
4352	prev_node = local_node;
4353	nodes_clear(used_mask);
4354
4355	memset(node_order, 0, sizeof(node_order));
4356	i = 0;
4357
4358	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
4359		/*
4360		 * We don't want to pressure a particular node.
4361		 * So adding penalty to the first node in same
4362		 * distance group to make it round-robin.
4363		 */
4364		if (node_distance(local_node, node) !=
4365		    node_distance(local_node, prev_node))
4366			node_load[node] = load;
4367
 
4368		prev_node = node;
4369		load--;
4370		if (order == ZONELIST_ORDER_NODE)
4371			build_zonelists_in_node_order(pgdat, node);
4372		else
4373			node_order[i++] = node;	/* remember order */
4374	}
4375
4376	if (order == ZONELIST_ORDER_ZONE) {
4377		/* calculate node order -- i.e., DMA last! */
4378		build_zonelists_in_zone_order(pgdat, i);
4379	}
4380
 
4381	build_thisnode_zonelists(pgdat);
 
 
 
 
4382}
4383
4384#ifdef CONFIG_HAVE_MEMORYLESS_NODES
4385/*
4386 * Return node id of node used for "local" allocations.
4387 * I.e., first node id of first zone in arg node's generic zonelist.
4388 * Used for initializing percpu 'numa_mem', which is used primarily
4389 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
4390 */
4391int local_memory_node(int node)
4392{
4393	struct zone *zone;
4394
4395	(void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
4396				   gfp_zone(GFP_KERNEL),
4397				   NULL,
4398				   &zone);
4399	return zone->node;
4400}
4401#endif
4402
 
 
4403#else	/* CONFIG_NUMA */
4404
4405static void set_zonelist_order(void)
4406{
4407	current_zonelist_order = ZONELIST_ORDER_ZONE;
4408}
4409
4410static void build_zonelists(pg_data_t *pgdat)
4411{
4412	int node, local_node;
4413	enum zone_type j;
4414	struct zonelist *zonelist;
4415
4416	local_node = pgdat->node_id;
4417
4418	zonelist = &pgdat->node_zonelists[0];
4419	j = build_zonelists_node(pgdat, zonelist, 0);
 
4420
4421	/*
4422	 * Now we build the zonelist so that it contains the zones
4423	 * of all the other nodes.
4424	 * We don't want to pressure a particular node, so when
4425	 * building the zones for node N, we make sure that the
4426	 * zones coming right after the local ones are those from
4427	 * node N+1 (modulo N)
4428	 */
4429	for (node = local_node + 1; node < MAX_NUMNODES; node++) {
4430		if (!node_online(node))
4431			continue;
4432		j = build_zonelists_node(NODE_DATA(node), zonelist, j);
 
4433	}
4434	for (node = 0; node < local_node; node++) {
4435		if (!node_online(node))
4436			continue;
4437		j = build_zonelists_node(NODE_DATA(node), zonelist, j);
 
4438	}
4439
4440	zonelist->_zonerefs[j].zone = NULL;
4441	zonelist->_zonerefs[j].zone_idx = 0;
4442}
4443
4444#endif	/* CONFIG_NUMA */
4445
4446/*
4447 * Boot pageset table. One per cpu which is going to be used for all
4448 * zones and all nodes. The parameters will be set in such a way
4449 * that an item put on a list will immediately be handed over to
4450 * the buddy list. This is safe since pageset manipulation is done
4451 * with interrupts disabled.
4452 *
4453 * The boot_pagesets must be kept even after bootup is complete for
4454 * unused processors and/or zones. They do play a role for bootstrapping
4455 * hotplugged processors.
4456 *
4457 * zoneinfo_show() and maybe other functions do
4458 * not check if the processor is online before following the pageset pointer.
4459 * Other parts of the kernel may not check if the zone is available.
4460 */
4461static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
4462static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
4463static void setup_zone_pageset(struct zone *zone);
 
 
 
4464
4465/*
4466 * Global mutex to protect against size modification of zonelists
4467 * as well as to serialize pageset setup for the new populated zone.
4468 */
4469DEFINE_MUTEX(zonelists_mutex);
4470
4471/* return values int ....just for stop_machine() */
4472static int __build_all_zonelists(void *data)
4473{
4474	int nid;
4475	int cpu;
4476	pg_data_t *self = data;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4477
4478#ifdef CONFIG_NUMA
4479	memset(node_load, 0, sizeof(node_load));
4480#endif
4481
 
 
 
 
4482	if (self && !node_online(self->node_id)) {
4483		build_zonelists(self);
4484	}
4485
4486	for_each_online_node(nid) {
4487		pg_data_t *pgdat = NODE_DATA(nid);
4488
4489		build_zonelists(pgdat);
4490	}
4491
4492	/*
4493	 * Initialize the boot_pagesets that are going to be used
4494	 * for bootstrapping processors. The real pagesets for
4495	 * each zone will be allocated later when the per cpu
4496	 * allocator is available.
4497	 *
4498	 * boot_pagesets are used also for bootstrapping offline
4499	 * cpus if the system is already booted because the pagesets
4500	 * are needed to initialize allocators on a specific cpu too.
4501	 * F.e. the percpu allocator needs the page allocator which
4502	 * needs the percpu allocator in order to allocate its pagesets
4503	 * (a chicken-egg dilemma).
4504	 */
4505	for_each_possible_cpu(cpu) {
4506		setup_pageset(&per_cpu(boot_pageset, cpu), 0);
4507
4508#ifdef CONFIG_HAVE_MEMORYLESS_NODES
4509		/*
4510		 * We now know the "local memory node" for each node--
4511		 * i.e., the node of the first zone in the generic zonelist.
4512		 * Set up numa_mem percpu variable for on-line cpus.  During
4513		 * boot, only the boot cpu should be on-line;  we'll init the
4514		 * secondary cpus' numa_mem as they come on-line.  During
4515		 * node/memory hotplug, we'll fixup all on-line cpus.
4516		 */
4517		if (cpu_online(cpu))
4518			set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
4519#endif
4520	}
4521
4522	return 0;
 
4523}
4524
4525static noinline void __init
4526build_all_zonelists_init(void)
4527{
 
 
4528	__build_all_zonelists(NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4529	mminit_verify_zonelist();
4530	cpuset_init_current_mems_allowed();
4531}
4532
4533/*
4534 * Called with zonelists_mutex held always
4535 * unless system_state == SYSTEM_BOOTING.
4536 *
4537 * __ref due to (1) call of __meminit annotated setup_zone_pageset
4538 * [we're only called with non-NULL zone through __meminit paths] and
4539 * (2) call of __init annotated helper build_all_zonelists_init
4540 * [protected by SYSTEM_BOOTING].
4541 */
4542void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
4543{
4544	set_zonelist_order();
4545
4546	if (system_state == SYSTEM_BOOTING) {
4547		build_all_zonelists_init();
4548	} else {
4549#ifdef CONFIG_MEMORY_HOTPLUG
4550		if (zone)
4551			setup_zone_pageset(zone);
4552#endif
4553		/* we have to stop all cpus to guarantee there is no user
4554		   of zonelist */
4555		stop_machine(__build_all_zonelists, pgdat, NULL);
4556		/* cpuset refresh routine should be here */
4557	}
4558	vm_total_pages = nr_free_pagecache_pages();
 
4559	/*
4560	 * Disable grouping by mobility if the number of pages in the
4561	 * system is too low to allow the mechanism to work. It would be
4562	 * more accurate, but expensive to check per-zone. This check is
4563	 * made on memory-hotadd so a system can start with mobility
4564	 * disabled and enable it later
4565	 */
4566	if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
4567		page_group_by_mobility_disabled = 1;
4568	else
4569		page_group_by_mobility_disabled = 0;
4570
4571	pr_info("Built %i zonelists in %s order, mobility grouping %s.  Total pages: %ld\n",
4572		nr_online_nodes,
4573		zonelist_order_name[current_zonelist_order],
4574		page_group_by_mobility_disabled ? "off" : "on",
4575		vm_total_pages);
4576#ifdef CONFIG_NUMA
4577	pr_info("Policy zone: %s\n", zone_names[policy_zone]);
4578#endif
4579}
4580
4581/*
4582 * Helper functions to size the waitqueue hash table.
4583 * Essentially these want to choose hash table sizes sufficiently
4584 * large so that collisions trying to wait on pages are rare.
4585 * But in fact, the number of active page waitqueues on typical
4586 * systems is ridiculously low, less than 200. So this is even
4587 * conservative, even though it seems large.
4588 *
4589 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
4590 * waitqueues, i.e. the size of the waitq table given the number of pages.
4591 */
4592#define PAGES_PER_WAITQUEUE	256
4593
4594#ifndef CONFIG_MEMORY_HOTPLUG
4595static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
4596{
4597	unsigned long size = 1;
4598
4599	pages /= PAGES_PER_WAITQUEUE;
4600
4601	while (size < pages)
4602		size <<= 1;
4603
4604	/*
4605	 * Once we have dozens or even hundreds of threads sleeping
4606	 * on IO we've got bigger problems than wait queue collision.
4607	 * Limit the size of the wait table to a reasonable size.
4608	 */
4609	size = min(size, 4096UL);
4610
4611	return max(size, 4UL);
4612}
4613#else
4614/*
4615 * A zone's size might be changed by hot-add, so it is not possible to determine
4616 * a suitable size for its wait_table.  So we use the maximum size now.
4617 *
4618 * The max wait table size = 4096 x sizeof(wait_queue_head_t).   ie:
4619 *
4620 *    i386 (preemption config)    : 4096 x 16 = 64Kbyte.
4621 *    ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
4622 *    ia64, x86-64 (preemption)   : 4096 x 24 = 96Kbyte.
4623 *
4624 * The maximum entries are prepared when a zone's memory is (512K + 256) pages
4625 * or more by the traditional way. (See above).  It equals:
4626 *
4627 *    i386, x86-64, powerpc(4K page size) : =  ( 2G + 1M)byte.
4628 *    ia64(16K page size)                 : =  ( 8G + 4M)byte.
4629 *    powerpc (64K page size)             : =  (32G +16M)byte.
4630 */
4631static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
4632{
4633	return 4096UL;
4634}
4635#endif
4636
4637/*
4638 * This is an integer logarithm so that shifts can be used later
4639 * to extract the more random high bits from the multiplicative
4640 * hash function before the remainder is taken.
4641 */
4642static inline unsigned long wait_table_bits(unsigned long size)
4643{
4644	return ffz(~size);
4645}
4646
4647/*
4648 * Initially all pages are reserved - free ones are freed
4649 * up by free_all_bootmem() once the early boot process is
4650 * done. Non-atomic initialization, single-pass.
4651 */
4652void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
4653		unsigned long start_pfn, enum memmap_context context)
4654{
4655	struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn));
4656	unsigned long end_pfn = start_pfn + size;
4657	pg_data_t *pgdat = NODE_DATA(nid);
4658	unsigned long pfn;
4659	unsigned long nr_initialised = 0;
4660#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4661	struct memblock_region *r = NULL, *tmp;
4662#endif
4663
4664	if (highest_memmap_pfn < end_pfn - 1)
4665		highest_memmap_pfn = end_pfn - 1;
4666
4667	/*
4668	 * Honor reservation requested by the driver for this ZONE_DEVICE
4669	 * memory
4670	 */
4671	if (altmap && start_pfn == altmap->base_pfn)
4672		start_pfn += altmap->reserve;
4673
4674	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
4675		/*
4676		 * There can be holes in boot-time mem_map[]s handed to this
4677		 * function.  They do not exist on hotplugged memory.
4678		 */
4679		if (context != MEMMAP_EARLY)
4680			goto not_early;
4681
4682		if (!early_pfn_valid(pfn))
4683			continue;
4684		if (!early_pfn_in_nid(pfn, nid))
4685			continue;
4686		if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised))
4687			break;
4688
4689#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4690		/*
4691		 * If not mirrored_kernelcore and ZONE_MOVABLE exists, range
4692		 * from zone_movable_pfn[nid] to end of each node should be
4693		 * ZONE_MOVABLE not ZONE_NORMAL. skip it.
4694		 */
4695		if (!mirrored_kernelcore && zone_movable_pfn[nid])
4696			if (zone == ZONE_NORMAL && pfn >= zone_movable_pfn[nid])
4697				continue;
4698
4699		/*
4700		 * Check given memblock attribute by firmware which can affect
4701		 * kernel memory layout.  If zone==ZONE_MOVABLE but memory is
4702		 * mirrored, it's an overlapped memmap init. skip it.
4703		 */
4704		if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
4705			if (!r || pfn >= memblock_region_memory_end_pfn(r)) {
4706				for_each_memblock(memory, tmp)
4707					if (pfn < memblock_region_memory_end_pfn(tmp))
4708						break;
4709				r = tmp;
4710			}
4711			if (pfn >= memblock_region_memory_base_pfn(r) &&
4712			    memblock_is_mirror(r)) {
4713				/* already initialized as NORMAL */
4714				pfn = memblock_region_memory_end_pfn(r);
4715				continue;
4716			}
4717		}
4718#endif
4719
4720not_early:
4721		/*
4722		 * Mark the block movable so that blocks are reserved for
4723		 * movable at startup. This will force kernel allocations
4724		 * to reserve their blocks rather than leaking throughout
4725		 * the address space during boot when many long-lived
4726		 * kernel allocations are made.
4727		 *
4728		 * bitmap is created for zone's valid pfn range. but memmap
4729		 * can be created for invalid pages (for alignment)
4730		 * check here not to call set_pageblock_migratetype() against
4731		 * pfn out of zone.
4732		 */
4733		if (!(pfn & (pageblock_nr_pages - 1))) {
4734			struct page *page = pfn_to_page(pfn);
4735
4736			__init_single_page(page, pfn, zone, nid);
4737			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
4738		} else {
4739			__init_single_pfn(pfn, zone, nid);
4740		}
4741	}
4742}
4743
4744static void __meminit zone_init_free_lists(struct zone *zone)
4745{
4746	unsigned int order, t;
4747	for_each_migratetype_order(order, t) {
4748		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
4749		zone->free_area[order].nr_free = 0;
4750	}
4751}
4752
4753#ifndef __HAVE_ARCH_MEMMAP_INIT
4754#define memmap_init(size, nid, zone, start_pfn) \
4755	memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
4756#endif
4757
4758static int zone_batchsize(struct zone *zone)
4759{
4760#ifdef CONFIG_MMU
4761	int batch;
4762
4763	/*
4764	 * The per-cpu-pages pools are set to around 1000th of the
4765	 * size of the zone.  But no more than 1/2 of a meg.
4766	 *
4767	 * OK, so we don't know how big the cache is.  So guess.
4768	 */
4769	batch = zone->managed_pages / 1024;
4770	if (batch * PAGE_SIZE > 512 * 1024)
4771		batch = (512 * 1024) / PAGE_SIZE;
4772	batch /= 4;		/* We effectively *= 4 below */
4773	if (batch < 1)
4774		batch = 1;
4775
4776	/*
4777	 * Clamp the batch to a 2^n - 1 value. Having a power
4778	 * of 2 value was found to be more likely to have
4779	 * suboptimal cache aliasing properties in some cases.
4780	 *
4781	 * For example if 2 tasks are alternately allocating
4782	 * batches of pages, one task can end up with a lot
4783	 * of pages of one half of the possible page colors
4784	 * and the other with pages of the other colors.
4785	 */
4786	batch = rounddown_pow_of_two(batch + batch/2) - 1;
4787
4788	return batch;
4789
4790#else
4791	/* The deferral and batching of frees should be suppressed under NOMMU
4792	 * conditions.
4793	 *
4794	 * The problem is that NOMMU needs to be able to allocate large chunks
4795	 * of contiguous memory as there's no hardware page translation to
4796	 * assemble apparent contiguous memory from discontiguous pages.
4797	 *
4798	 * Queueing large contiguous runs of pages for batching, however,
4799	 * causes the pages to actually be freed in smaller chunks.  As there
4800	 * can be a significant delay between the individual batches being
4801	 * recycled, this leads to the once large chunks of space being
4802	 * fragmented and becoming unavailable for high-order allocations.
4803	 */
4804	return 0;
4805#endif
4806}
4807
4808/*
4809 * pcp->high and pcp->batch values are related and dependent on one another:
4810 * ->batch must never be higher then ->high.
4811 * The following function updates them in a safe manner without read side
4812 * locking.
4813 *
4814 * Any new users of pcp->batch and pcp->high should ensure they can cope with
4815 * those fields changing asynchronously (acording the the above rule).
4816 *
4817 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
4818 * outside of boot time (or some other assurance that no concurrent updaters
4819 * exist).
4820 */
4821static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
4822		unsigned long batch)
4823{
4824       /* start with a fail safe value for batch */
4825	pcp->batch = 1;
4826	smp_wmb();
4827
4828       /* Update high, then batch, in order */
4829	pcp->high = high;
4830	smp_wmb();
4831
4832	pcp->batch = batch;
4833}
4834
4835/* a companion to pageset_set_high() */
4836static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
4837{
4838	pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
4839}
4840
4841static void pageset_init(struct per_cpu_pageset *p)
4842{
4843	struct per_cpu_pages *pcp;
4844	int migratetype;
4845
4846	memset(p, 0, sizeof(*p));
4847
4848	pcp = &p->pcp;
4849	pcp->count = 0;
4850	for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
4851		INIT_LIST_HEAD(&pcp->lists[migratetype]);
4852}
4853
4854static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
4855{
4856	pageset_init(p);
4857	pageset_set_batch(p, batch);
4858}
4859
4860/*
4861 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
4862 * to the value high for the pageset p.
4863 */
4864static void pageset_set_high(struct per_cpu_pageset *p,
4865				unsigned long high)
4866{
4867	unsigned long batch = max(1UL, high / 4);
4868	if ((high / 4) > (PAGE_SHIFT * 8))
4869		batch = PAGE_SHIFT * 8;
4870
4871	pageset_update(&p->pcp, high, batch);
4872}
4873
4874static void pageset_set_high_and_batch(struct zone *zone,
4875				       struct per_cpu_pageset *pcp)
4876{
4877	if (percpu_pagelist_fraction)
4878		pageset_set_high(pcp,
4879			(zone->managed_pages /
4880				percpu_pagelist_fraction));
4881	else
4882		pageset_set_batch(pcp, zone_batchsize(zone));
4883}
4884
4885static void __meminit zone_pageset_init(struct zone *zone, int cpu)
4886{
4887	struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
4888
4889	pageset_init(pcp);
4890	pageset_set_high_and_batch(zone, pcp);
4891}
4892
4893static void __meminit setup_zone_pageset(struct zone *zone)
4894{
4895	int cpu;
4896	zone->pageset = alloc_percpu(struct per_cpu_pageset);
4897	for_each_possible_cpu(cpu)
4898		zone_pageset_init(zone, cpu);
4899}
4900
4901/*
4902 * Allocate per cpu pagesets and initialize them.
4903 * Before this call only boot pagesets were available.
4904 */
4905void __init setup_per_cpu_pageset(void)
4906{
4907	struct zone *zone;
4908
4909	for_each_populated_zone(zone)
4910		setup_zone_pageset(zone);
4911}
4912
4913static noinline __init_refok
4914int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
4915{
4916	int i;
4917	size_t alloc_size;
4918
4919	/*
4920	 * The per-page waitqueue mechanism uses hashed waitqueues
4921	 * per zone.
4922	 */
4923	zone->wait_table_hash_nr_entries =
4924		 wait_table_hash_nr_entries(zone_size_pages);
4925	zone->wait_table_bits =
4926		wait_table_bits(zone->wait_table_hash_nr_entries);
4927	alloc_size = zone->wait_table_hash_nr_entries
4928					* sizeof(wait_queue_head_t);
4929
4930	if (!slab_is_available()) {
4931		zone->wait_table = (wait_queue_head_t *)
4932			memblock_virt_alloc_node_nopanic(
4933				alloc_size, zone->zone_pgdat->node_id);
4934	} else {
4935		/*
4936		 * This case means that a zone whose size was 0 gets new memory
4937		 * via memory hot-add.
4938		 * But it may be the case that a new node was hot-added.  In
4939		 * this case vmalloc() will not be able to use this new node's
4940		 * memory - this wait_table must be initialized to use this new
4941		 * node itself as well.
4942		 * To use this new node's memory, further consideration will be
4943		 * necessary.
4944		 */
4945		zone->wait_table = vmalloc(alloc_size);
4946	}
4947	if (!zone->wait_table)
4948		return -ENOMEM;
4949
4950	for (i = 0; i < zone->wait_table_hash_nr_entries; ++i)
4951		init_waitqueue_head(zone->wait_table + i);
4952
4953	return 0;
4954}
4955
4956static __meminit void zone_pcp_init(struct zone *zone)
4957{
4958	/*
4959	 * per cpu subsystem is not up at this point. The following code
4960	 * relies on the ability of the linker to provide the
4961	 * offset of a (static) per cpu variable into the per cpu area.
 
 
 
4962	 */
4963	zone->pageset = &boot_pageset;
4964
4965	if (populated_zone(zone))
4966		printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
4967			zone->name, zone->present_pages,
4968					 zone_batchsize(zone));
4969}
4970
4971int __meminit init_currently_empty_zone(struct zone *zone,
4972					unsigned long zone_start_pfn,
4973					unsigned long size)
4974{
4975	struct pglist_data *pgdat = zone->zone_pgdat;
4976	int ret;
4977	ret = zone_wait_table_init(zone, size);
4978	if (ret)
4979		return ret;
4980	pgdat->nr_zones = zone_idx(zone) + 1;
4981
4982	zone->zone_start_pfn = zone_start_pfn;
4983
4984	mminit_dprintk(MMINIT_TRACE, "memmap_init",
4985			"Initialising map node %d zone %lu pfns %lu -> %lu\n",
4986			pgdat->node_id,
4987			(unsigned long)zone_idx(zone),
4988			zone_start_pfn, (zone_start_pfn + size));
4989
4990	zone_init_free_lists(zone);
4991
 
 
4992	return 0;
 
4993}
4994
4995#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4996#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
4997
4998/*
4999 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
5000 */
5001int __meminit __early_pfn_to_nid(unsigned long pfn,
5002					struct mminit_pfnnid_cache *state)
5003{
5004	unsigned long start_pfn, end_pfn;
5005	int nid;
5006
5007	if (state->last_start <= pfn && pfn < state->last_end)
5008		return state->last_nid;
5009
5010	nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
5011	if (nid != -1) {
5012		state->last_start = start_pfn;
5013		state->last_end = end_pfn;
5014		state->last_nid = nid;
5015	}
5016
5017	return nid;
5018}
5019#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
5020
5021/**
5022 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
5023 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
5024 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
5025 *
5026 * If an architecture guarantees that all ranges registered contain no holes
5027 * and may be freed, this this function may be used instead of calling
5028 * memblock_free_early_nid() manually.
5029 */
5030void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
5031{
5032	unsigned long start_pfn, end_pfn;
5033	int i, this_nid;
5034
5035	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
5036		start_pfn = min(start_pfn, max_low_pfn);
5037		end_pfn = min(end_pfn, max_low_pfn);
5038
5039		if (start_pfn < end_pfn)
5040			memblock_free_early_nid(PFN_PHYS(start_pfn),
5041					(end_pfn - start_pfn) << PAGE_SHIFT,
5042					this_nid);
5043	}
5044}
5045
5046/**
5047 * sparse_memory_present_with_active_regions - Call memory_present for each active range
5048 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
5049 *
5050 * If an architecture guarantees that all ranges registered contain no holes and may
5051 * be freed, this function may be used instead of calling memory_present() manually.
 
5052 */
5053void __init sparse_memory_present_with_active_regions(int nid)
 
5054{
5055	unsigned long start_pfn, end_pfn;
5056	int i, this_nid;
5057
5058	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
5059		memory_present(this_nid, start_pfn, end_pfn);
5060}
5061
5062/**
5063 * get_pfn_range_for_nid - Return the start and end page frames for a node
5064 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
5065 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
5066 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
5067 *
5068 * It returns the start and end page frame of a node based on information
5069 * provided by memblock_set_node(). If called for a node
5070 * with no available memory, a warning is printed and the start and end
5071 * PFNs will be 0.
5072 */
5073void __meminit get_pfn_range_for_nid(unsigned int nid,
5074			unsigned long *start_pfn, unsigned long *end_pfn)
5075{
5076	unsigned long this_start_pfn, this_end_pfn;
5077	int i;
5078
5079	*start_pfn = -1UL;
5080	*end_pfn = 0;
5081
5082	for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
5083		*start_pfn = min(*start_pfn, this_start_pfn);
5084		*end_pfn = max(*end_pfn, this_end_pfn);
5085	}
5086
5087	if (*start_pfn == -1UL)
5088		*start_pfn = 0;
 
 
 
 
 
 
 
 
5089}
5090
5091/*
5092 * This finds a zone that can be used for ZONE_MOVABLE pages. The
5093 * assumption is made that zones within a node are ordered in monotonic
5094 * increasing memory addresses so that the "highest" populated zone is used
5095 */
5096static void __init find_usable_zone_for_movable(void)
5097{
5098	int zone_index;
5099	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
5100		if (zone_index == ZONE_MOVABLE)
5101			continue;
5102
5103		if (arch_zone_highest_possible_pfn[zone_index] >
5104				arch_zone_lowest_possible_pfn[zone_index])
5105			break;
5106	}
5107
5108	VM_BUG_ON(zone_index == -1);
5109	movable_zone = zone_index;
5110}
5111
5112/*
5113 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
5114 * because it is sized independent of architecture. Unlike the other zones,
5115 * the starting point for ZONE_MOVABLE is not fixed. It may be different
5116 * in each node depending on the size of each node and how evenly kernelcore
5117 * is distributed. This helper function adjusts the zone ranges
5118 * provided by the architecture for a given node by using the end of the
5119 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
5120 * zones within a node are in order of monotonic increases memory addresses
5121 */
5122static void __meminit adjust_zone_range_for_zone_movable(int nid,
5123					unsigned long zone_type,
5124					unsigned long node_start_pfn,
5125					unsigned long node_end_pfn,
5126					unsigned long *zone_start_pfn,
5127					unsigned long *zone_end_pfn)
5128{
5129	/* Only adjust if ZONE_MOVABLE is on this node */
5130	if (zone_movable_pfn[nid]) {
5131		/* Size ZONE_MOVABLE */
5132		if (zone_type == ZONE_MOVABLE) {
5133			*zone_start_pfn = zone_movable_pfn[nid];
5134			*zone_end_pfn = min(node_end_pfn,
5135				arch_zone_highest_possible_pfn[movable_zone]);
5136
5137		/* Check if this whole range is within ZONE_MOVABLE */
5138		} else if (*zone_start_pfn >= zone_movable_pfn[nid])
5139			*zone_start_pfn = *zone_end_pfn;
5140	}
5141}
5142
5143/*
5144 * Return the number of pages a zone spans in a node, including holes
5145 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
5146 */
5147static unsigned long __meminit zone_spanned_pages_in_node(int nid,
5148					unsigned long zone_type,
5149					unsigned long node_start_pfn,
5150					unsigned long node_end_pfn,
5151					unsigned long *zone_start_pfn,
5152					unsigned long *zone_end_pfn,
5153					unsigned long *ignored)
5154{
5155	/* When hotadd a new node from cpu_up(), the node should be empty */
5156	if (!node_start_pfn && !node_end_pfn)
5157		return 0;
5158
5159	/* Get the start and end of the zone */
5160	*zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
5161	*zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
5162	adjust_zone_range_for_zone_movable(nid, zone_type,
5163				node_start_pfn, node_end_pfn,
5164				zone_start_pfn, zone_end_pfn);
5165
5166	/* Check that this node has pages within the zone's required range */
5167	if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
5168		return 0;
5169
5170	/* Move the zone boundaries inside the node if necessary */
5171	*zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
5172	*zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
5173
5174	/* Return the spanned pages */
5175	return *zone_end_pfn - *zone_start_pfn;
5176}
5177
5178/*
5179 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
5180 * then all holes in the requested range will be accounted for.
5181 */
5182unsigned long __meminit __absent_pages_in_range(int nid,
5183				unsigned long range_start_pfn,
5184				unsigned long range_end_pfn)
5185{
5186	unsigned long nr_absent = range_end_pfn - range_start_pfn;
5187	unsigned long start_pfn, end_pfn;
5188	int i;
5189
5190	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
5191		start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
5192		end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
5193		nr_absent -= end_pfn - start_pfn;
5194	}
5195	return nr_absent;
5196}
5197
5198/**
5199 * absent_pages_in_range - Return number of page frames in holes within a range
5200 * @start_pfn: The start PFN to start searching for holes
5201 * @end_pfn: The end PFN to stop searching for holes
5202 *
5203 * It returns the number of pages frames in memory holes within a range.
5204 */
5205unsigned long __init absent_pages_in_range(unsigned long start_pfn,
5206							unsigned long end_pfn)
5207{
5208	return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
5209}
5210
5211/* Return the number of page frames in holes in a zone on a node */
5212static unsigned long __meminit zone_absent_pages_in_node(int nid,
5213					unsigned long zone_type,
5214					unsigned long node_start_pfn,
5215					unsigned long node_end_pfn,
5216					unsigned long *ignored)
5217{
5218	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
5219	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
5220	unsigned long zone_start_pfn, zone_end_pfn;
5221	unsigned long nr_absent;
5222
5223	/* When hotadd a new node from cpu_up(), the node should be empty */
5224	if (!node_start_pfn && !node_end_pfn)
5225		return 0;
5226
5227	zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
5228	zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
5229
5230	adjust_zone_range_for_zone_movable(nid, zone_type,
5231			node_start_pfn, node_end_pfn,
5232			&zone_start_pfn, &zone_end_pfn);
5233	nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
5234
5235	/*
5236	 * ZONE_MOVABLE handling.
5237	 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
5238	 * and vice versa.
5239	 */
5240	if (zone_movable_pfn[nid]) {
5241		if (mirrored_kernelcore) {
5242			unsigned long start_pfn, end_pfn;
5243			struct memblock_region *r;
5244
5245			for_each_memblock(memory, r) {
5246				start_pfn = clamp(memblock_region_memory_base_pfn(r),
5247						  zone_start_pfn, zone_end_pfn);
5248				end_pfn = clamp(memblock_region_memory_end_pfn(r),
5249						zone_start_pfn, zone_end_pfn);
5250
5251				if (zone_type == ZONE_MOVABLE &&
5252				    memblock_is_mirror(r))
5253					nr_absent += end_pfn - start_pfn;
5254
5255				if (zone_type == ZONE_NORMAL &&
5256				    !memblock_is_mirror(r))
5257					nr_absent += end_pfn - start_pfn;
5258			}
5259		} else {
5260			if (zone_type == ZONE_NORMAL)
5261				nr_absent += node_end_pfn - zone_movable_pfn[nid];
5262		}
5263	}
5264
5265	return nr_absent;
5266}
5267
5268#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
5269static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
5270					unsigned long zone_type,
5271					unsigned long node_start_pfn,
5272					unsigned long node_end_pfn,
5273					unsigned long *zone_start_pfn,
5274					unsigned long *zone_end_pfn,
5275					unsigned long *zones_size)
5276{
5277	unsigned int zone;
5278
5279	*zone_start_pfn = node_start_pfn;
5280	for (zone = 0; zone < zone_type; zone++)
5281		*zone_start_pfn += zones_size[zone];
5282
5283	*zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
 
 
5284
5285	return zones_size[zone_type];
 
5286}
5287
5288static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
5289						unsigned long zone_type,
5290						unsigned long node_start_pfn,
5291						unsigned long node_end_pfn,
5292						unsigned long *zholes_size)
5293{
5294	if (!zholes_size)
5295		return 0;
5296
5297	return zholes_size[zone_type];
5298}
5299
5300#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
5301
5302static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
5303						unsigned long node_start_pfn,
5304						unsigned long node_end_pfn,
5305						unsigned long *zones_size,
5306						unsigned long *zholes_size)
5307{
5308	unsigned long realtotalpages = 0, totalpages = 0;
5309	enum zone_type i;
5310
5311	for (i = 0; i < MAX_NR_ZONES; i++) {
5312		struct zone *zone = pgdat->node_zones + i;
5313		unsigned long zone_start_pfn, zone_end_pfn;
5314		unsigned long size, real_size;
5315
5316		size = zone_spanned_pages_in_node(pgdat->node_id, i,
5317						  node_start_pfn,
5318						  node_end_pfn,
5319						  &zone_start_pfn,
5320						  &zone_end_pfn,
5321						  zones_size);
5322		real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
5323						  node_start_pfn, node_end_pfn,
5324						  zholes_size);
5325		if (size)
5326			zone->zone_start_pfn = zone_start_pfn;
5327		else
5328			zone->zone_start_pfn = 0;
5329		zone->spanned_pages = size;
5330		zone->present_pages = real_size;
5331
5332		totalpages += size;
5333		realtotalpages += real_size;
 
5334	}
5335
5336	pgdat->node_spanned_pages = totalpages;
5337	pgdat->node_present_pages = realtotalpages;
5338	printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
5339							realtotalpages);
5340}
5341
5342#ifndef CONFIG_SPARSEMEM
5343/*
5344 * Calculate the size of the zone->blockflags rounded to an unsigned long
5345 * Start by making sure zonesize is a multiple of pageblock_order by rounding
5346 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
5347 * round what is now in bits to nearest long in bits, then return it in
5348 * bytes.
5349 */
5350static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
5351{
5352	unsigned long usemapsize;
5353
5354	zonesize += zone_start_pfn & (pageblock_nr_pages-1);
5355	usemapsize = roundup(zonesize, pageblock_nr_pages);
5356	usemapsize = usemapsize >> pageblock_order;
5357	usemapsize *= NR_PAGEBLOCK_BITS;
5358	usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
5359
5360	return usemapsize / 8;
5361}
5362
5363static void __init setup_usemap(struct pglist_data *pgdat,
5364				struct zone *zone,
5365				unsigned long zone_start_pfn,
5366				unsigned long zonesize)
5367{
5368	unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
5369	zone->pageblock_flags = NULL;
5370	if (usemapsize)
5371		zone->pageblock_flags =
5372			memblock_virt_alloc_node_nopanic(usemapsize,
5373							 pgdat->node_id);
5374}
5375#else
5376static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
5377				unsigned long zone_start_pfn, unsigned long zonesize) {}
5378#endif /* CONFIG_SPARSEMEM */
5379
5380#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
5381
5382/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
5383void __paginginit set_pageblock_order(void)
5384{
5385	unsigned int order;
5386
5387	/* Check that pageblock_nr_pages has not already been setup */
5388	if (pageblock_order)
5389		return;
5390
5391	if (HPAGE_SHIFT > PAGE_SHIFT)
5392		order = HUGETLB_PAGE_ORDER;
5393	else
5394		order = MAX_ORDER - 1;
5395
 
 
5396	/*
5397	 * Assume the largest contiguous order of interest is a huge page.
5398	 * This value may be variable depending on boot parameters on IA64 and
5399	 * powerpc.
5400	 */
5401	pageblock_order = order;
5402}
5403#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
5404
5405/*
5406 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
5407 * is unused as pageblock_order is set at compile-time. See
5408 * include/linux/pageblock-flags.h for the values of pageblock_order based on
5409 * the kernel config
5410 */
5411void __paginginit set_pageblock_order(void)
5412{
5413}
5414
5415#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
5416
5417static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
5418						   unsigned long present_pages)
5419{
5420	unsigned long pages = spanned_pages;
5421
5422	/*
5423	 * Provide a more accurate estimation if there are holes within
5424	 * the zone and SPARSEMEM is in use. If there are holes within the
5425	 * zone, each populated memory region may cost us one or two extra
5426	 * memmap pages due to alignment because memmap pages for each
5427	 * populated regions may not naturally algined on page boundary.
5428	 * So the (present_pages >> 4) heuristic is a tradeoff for that.
5429	 */
5430	if (spanned_pages > present_pages + (present_pages >> 4) &&
5431	    IS_ENABLED(CONFIG_SPARSEMEM))
5432		pages = present_pages;
5433
5434	return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
 
5435}
5436
5437/*
5438 * Set up the zone data structures:
5439 *   - mark all pages reserved
5440 *   - mark all memory queues empty
5441 *   - clear the memory bitmaps
5442 *
5443 * NOTE: pgdat should get zeroed by caller.
5444 */
5445static void __paginginit free_area_init_core(struct pglist_data *pgdat)
5446{
5447	enum zone_type j;
5448	int nid = pgdat->node_id;
5449	int ret;
5450
5451	pgdat_resize_init(pgdat);
5452#ifdef CONFIG_NUMA_BALANCING
5453	spin_lock_init(&pgdat->numabalancing_migrate_lock);
5454	pgdat->numabalancing_migrate_nr_pages = 0;
5455	pgdat->numabalancing_migrate_next_window = jiffies;
5456#endif
5457#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5458	spin_lock_init(&pgdat->split_queue_lock);
5459	INIT_LIST_HEAD(&pgdat->split_queue);
5460	pgdat->split_queue_len = 0;
5461#endif
5462	init_waitqueue_head(&pgdat->kswapd_wait);
5463	init_waitqueue_head(&pgdat->pfmemalloc_wait);
5464#ifdef CONFIG_COMPACTION
5465	init_waitqueue_head(&pgdat->kcompactd_wait);
5466#endif
5467	pgdat_page_ext_init(pgdat);
5468
5469	for (j = 0; j < MAX_NR_ZONES; j++) {
5470		struct zone *zone = pgdat->node_zones + j;
5471		unsigned long size, realsize, freesize, memmap_pages;
5472		unsigned long zone_start_pfn = zone->zone_start_pfn;
5473
5474		size = zone->spanned_pages;
5475		realsize = freesize = zone->present_pages;
5476
5477		/*
5478		 * Adjust freesize so that it accounts for how much memory
5479		 * is used by this zone for memmap. This affects the watermark
5480		 * and per-cpu initialisations
5481		 */
5482		memmap_pages = calc_memmap_size(size, realsize);
5483		if (!is_highmem_idx(j)) {
5484			if (freesize >= memmap_pages) {
5485				freesize -= memmap_pages;
5486				if (memmap_pages)
5487					printk(KERN_DEBUG
5488					       "  %s zone: %lu pages used for memmap\n",
5489					       zone_names[j], memmap_pages);
5490			} else
5491				pr_warn("  %s zone: %lu pages exceeds freesize %lu\n",
5492					zone_names[j], memmap_pages, freesize);
5493		}
5494
5495		/* Account for reserved pages */
5496		if (j == 0 && freesize > dma_reserve) {
5497			freesize -= dma_reserve;
5498			printk(KERN_DEBUG "  %s zone: %lu pages reserved\n",
5499					zone_names[0], dma_reserve);
5500		}
5501
5502		if (!is_highmem_idx(j))
5503			nr_kernel_pages += freesize;
5504		/* Charge for highmem memmap if there are enough kernel pages */
5505		else if (nr_kernel_pages > memmap_pages * 2)
5506			nr_kernel_pages -= memmap_pages;
5507		nr_all_pages += freesize;
5508
5509		/*
5510		 * Set an approximate value for lowmem here, it will be adjusted
5511		 * when the bootmem allocator frees pages into the buddy system.
5512		 * And all highmem pages will be managed by the buddy system.
5513		 */
5514		zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
5515#ifdef CONFIG_NUMA
5516		zone->node = nid;
5517		zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio)
5518						/ 100;
5519		zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100;
5520#endif
5521		zone->name = zone_names[j];
5522		spin_lock_init(&zone->lock);
5523		spin_lock_init(&zone->lru_lock);
5524		zone_seqlock_init(zone);
5525		zone->zone_pgdat = pgdat;
5526		zone_pcp_init(zone);
5527
5528		/* For bootup, initialized properly in watermark setup */
5529		mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages);
5530
5531		lruvec_init(&zone->lruvec);
5532		if (!size)
5533			continue;
5534
5535		set_pageblock_order();
5536		setup_usemap(pgdat, zone, zone_start_pfn, size);
5537		ret = init_currently_empty_zone(zone, zone_start_pfn, size);
5538		BUG_ON(ret);
5539		memmap_init(size, nid, j, zone_start_pfn);
5540	}
5541}
5542
5543static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
5544{
5545	unsigned long __maybe_unused start = 0;
5546	unsigned long __maybe_unused offset = 0;
5547
5548	/* Skip empty nodes */
5549	if (!pgdat->node_spanned_pages)
5550		return;
5551
5552#ifdef CONFIG_FLAT_NODE_MEM_MAP
5553	start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
5554	offset = pgdat->node_start_pfn - start;
5555	/* ia64 gets its own node_mem_map, before this, without bootmem */
5556	if (!pgdat->node_mem_map) {
5557		unsigned long size, end;
5558		struct page *map;
5559
5560		/*
5561		 * The zone's endpoints aren't required to be MAX_ORDER
5562		 * aligned but the node_mem_map endpoints must be in order
5563		 * for the buddy allocator to function correctly.
5564		 */
5565		end = pgdat_end_pfn(pgdat);
5566		end = ALIGN(end, MAX_ORDER_NR_PAGES);
5567		size =  (end - start) * sizeof(struct page);
5568		map = alloc_remap(pgdat->node_id, size);
5569		if (!map)
5570			map = memblock_virt_alloc_node_nopanic(size,
5571							       pgdat->node_id);
5572		pgdat->node_mem_map = map + offset;
5573	}
5574#ifndef CONFIG_NEED_MULTIPLE_NODES
5575	/*
5576	 * With no DISCONTIG, the global mem_map is just set as node 0's
5577	 */
5578	if (pgdat == NODE_DATA(0)) {
5579		mem_map = NODE_DATA(0)->node_mem_map;
5580#if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
5581		if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
5582			mem_map -= offset;
5583#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
5584	}
5585#endif
5586#endif /* CONFIG_FLAT_NODE_MEM_MAP */
5587}
5588
5589void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
5590		unsigned long node_start_pfn, unsigned long *zholes_size)
5591{
5592	pg_data_t *pgdat = NODE_DATA(nid);
5593	unsigned long start_pfn = 0;
5594	unsigned long end_pfn = 0;
5595
5596	/* pg_data_t should be reset to zero when it's allocated */
5597	WARN_ON(pgdat->nr_zones || pgdat->classzone_idx);
5598
5599	reset_deferred_meminit(pgdat);
5600	pgdat->node_id = nid;
5601	pgdat->node_start_pfn = node_start_pfn;
5602#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5603	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
5604	pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
5605		(u64)start_pfn << PAGE_SHIFT,
5606		end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
5607#else
5608	start_pfn = node_start_pfn;
5609#endif
5610	calculate_node_totalpages(pgdat, start_pfn, end_pfn,
5611				  zones_size, zholes_size);
5612
5613	alloc_node_mem_map(pgdat);
5614#ifdef CONFIG_FLAT_NODE_MEM_MAP
5615	printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
5616		nid, (unsigned long)pgdat,
5617		(unsigned long)pgdat->node_mem_map);
5618#endif
5619
5620	free_area_init_core(pgdat);
5621}
5622
5623#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5624
5625#if MAX_NUMNODES > 1
5626/*
5627 * Figure out the number of possible node ids.
5628 */
5629void __init setup_nr_node_ids(void)
5630{
5631	unsigned int highest;
5632
5633	highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
5634	nr_node_ids = highest + 1;
5635}
5636#endif
5637
5638/**
5639 * node_map_pfn_alignment - determine the maximum internode alignment
5640 *
5641 * This function should be called after node map is populated and sorted.
5642 * It calculates the maximum power of two alignment which can distinguish
5643 * all the nodes.
5644 *
5645 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
5646 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the
5647 * nodes are shifted by 256MiB, 256MiB.  Note that if only the last node is
5648 * shifted, 1GiB is enough and this function will indicate so.
5649 *
5650 * This is used to test whether pfn -> nid mapping of the chosen memory
5651 * model has fine enough granularity to avoid incorrect mapping for the
5652 * populated node map.
5653 *
5654 * Returns the determined alignment in pfn's.  0 if there is no alignment
5655 * requirement (single node).
5656 */
5657unsigned long __init node_map_pfn_alignment(void)
5658{
5659	unsigned long accl_mask = 0, last_end = 0;
5660	unsigned long start, end, mask;
5661	int last_nid = -1;
5662	int i, nid;
5663
5664	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
5665		if (!start || last_nid < 0 || last_nid == nid) {
5666			last_nid = nid;
5667			last_end = end;
5668			continue;
5669		}
5670
5671		/*
5672		 * Start with a mask granular enough to pin-point to the
5673		 * start pfn and tick off bits one-by-one until it becomes
5674		 * too coarse to separate the current node from the last.
5675		 */
5676		mask = ~((1 << __ffs(start)) - 1);
5677		while (mask && last_end <= (start & (mask << 1)))
5678			mask <<= 1;
5679
5680		/* accumulate all internode masks */
5681		accl_mask |= mask;
5682	}
5683
5684	/* convert mask to number of pages */
5685	return ~accl_mask + 1;
5686}
5687
5688/* Find the lowest pfn for a node */
5689static unsigned long __init find_min_pfn_for_node(int nid)
5690{
5691	unsigned long min_pfn = ULONG_MAX;
5692	unsigned long start_pfn;
5693	int i;
5694
5695	for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
5696		min_pfn = min(min_pfn, start_pfn);
5697
5698	if (min_pfn == ULONG_MAX) {
5699		pr_warn("Could not find start_pfn for node %d\n", nid);
5700		return 0;
5701	}
5702
5703	return min_pfn;
5704}
5705
5706/**
5707 * find_min_pfn_with_active_regions - Find the minimum PFN registered
5708 *
5709 * It returns the minimum PFN based on information provided via
5710 * memblock_set_node().
5711 */
5712unsigned long __init find_min_pfn_with_active_regions(void)
5713{
5714	return find_min_pfn_for_node(MAX_NUMNODES);
5715}
5716
5717/*
5718 * early_calculate_totalpages()
5719 * Sum pages in active regions for movable zone.
5720 * Populate N_MEMORY for calculating usable_nodes.
5721 */
5722static unsigned long __init early_calculate_totalpages(void)
5723{
5724	unsigned long totalpages = 0;
5725	unsigned long start_pfn, end_pfn;
5726	int i, nid;
5727
5728	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
5729		unsigned long pages = end_pfn - start_pfn;
5730
5731		totalpages += pages;
5732		if (pages)
5733			node_set_state(nid, N_MEMORY);
5734	}
5735	return totalpages;
5736}
5737
5738/*
5739 * Find the PFN the Movable zone begins in each node. Kernel memory
5740 * is spread evenly between nodes as long as the nodes have enough
5741 * memory. When they don't, some nodes will have more kernelcore than
5742 * others
5743 */
5744static void __init find_zone_movable_pfns_for_nodes(void)
5745{
5746	int i, nid;
5747	unsigned long usable_startpfn;
5748	unsigned long kernelcore_node, kernelcore_remaining;
5749	/* save the state before borrow the nodemask */
5750	nodemask_t saved_node_state = node_states[N_MEMORY];
5751	unsigned long totalpages = early_calculate_totalpages();
5752	int usable_nodes = nodes_weight(node_states[N_MEMORY]);
5753	struct memblock_region *r;
5754
5755	/* Need to find movable_zone earlier when movable_node is specified. */
5756	find_usable_zone_for_movable();
5757
5758	/*
5759	 * If movable_node is specified, ignore kernelcore and movablecore
5760	 * options.
5761	 */
5762	if (movable_node_is_enabled()) {
5763		for_each_memblock(memory, r) {
5764			if (!memblock_is_hotpluggable(r))
5765				continue;
5766
5767			nid = r->nid;
5768
5769			usable_startpfn = PFN_DOWN(r->base);
5770			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
5771				min(usable_startpfn, zone_movable_pfn[nid]) :
5772				usable_startpfn;
5773		}
5774
5775		goto out2;
5776	}
5777
5778	/*
5779	 * If kernelcore=mirror is specified, ignore movablecore option
5780	 */
5781	if (mirrored_kernelcore) {
5782		bool mem_below_4gb_not_mirrored = false;
5783
5784		for_each_memblock(memory, r) {
5785			if (memblock_is_mirror(r))
5786				continue;
5787
5788			nid = r->nid;
5789
5790			usable_startpfn = memblock_region_memory_base_pfn(r);
5791
5792			if (usable_startpfn < 0x100000) {
5793				mem_below_4gb_not_mirrored = true;
5794				continue;
5795			}
5796
5797			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
5798				min(usable_startpfn, zone_movable_pfn[nid]) :
5799				usable_startpfn;
5800		}
5801
5802		if (mem_below_4gb_not_mirrored)
5803			pr_warn("This configuration results in unmirrored kernel memory.");
5804
5805		goto out2;
5806	}
5807
5808	/*
5809	 * If movablecore=nn[KMG] was specified, calculate what size of
5810	 * kernelcore that corresponds so that memory usable for
5811	 * any allocation type is evenly spread. If both kernelcore
5812	 * and movablecore are specified, then the value of kernelcore
5813	 * will be used for required_kernelcore if it's greater than
5814	 * what movablecore would have allowed.
5815	 */
5816	if (required_movablecore) {
5817		unsigned long corepages;
5818
5819		/*
5820		 * Round-up so that ZONE_MOVABLE is at least as large as what
5821		 * was requested by the user
5822		 */
5823		required_movablecore =
5824			roundup(required_movablecore, MAX_ORDER_NR_PAGES);
5825		required_movablecore = min(totalpages, required_movablecore);
5826		corepages = totalpages - required_movablecore;
5827
5828		required_kernelcore = max(required_kernelcore, corepages);
5829	}
5830
5831	/*
5832	 * If kernelcore was not specified or kernelcore size is larger
5833	 * than totalpages, there is no ZONE_MOVABLE.
5834	 */
5835	if (!required_kernelcore || required_kernelcore >= totalpages)
5836		goto out;
5837
5838	/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
5839	usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
5840
5841restart:
5842	/* Spread kernelcore memory as evenly as possible throughout nodes */
5843	kernelcore_node = required_kernelcore / usable_nodes;
5844	for_each_node_state(nid, N_MEMORY) {
5845		unsigned long start_pfn, end_pfn;
5846
5847		/*
5848		 * Recalculate kernelcore_node if the division per node
5849		 * now exceeds what is necessary to satisfy the requested
5850		 * amount of memory for the kernel
5851		 */
5852		if (required_kernelcore < kernelcore_node)
5853			kernelcore_node = required_kernelcore / usable_nodes;
5854
5855		/*
5856		 * As the map is walked, we track how much memory is usable
5857		 * by the kernel using kernelcore_remaining. When it is
5858		 * 0, the rest of the node is usable by ZONE_MOVABLE
5859		 */
5860		kernelcore_remaining = kernelcore_node;
5861
5862		/* Go through each range of PFNs within this node */
5863		for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
5864			unsigned long size_pages;
5865
5866			start_pfn = max(start_pfn, zone_movable_pfn[nid]);
5867			if (start_pfn >= end_pfn)
5868				continue;
5869
5870			/* Account for what is only usable for kernelcore */
5871			if (start_pfn < usable_startpfn) {
5872				unsigned long kernel_pages;
5873				kernel_pages = min(end_pfn, usable_startpfn)
5874								- start_pfn;
5875
5876				kernelcore_remaining -= min(kernel_pages,
5877							kernelcore_remaining);
5878				required_kernelcore -= min(kernel_pages,
5879							required_kernelcore);
5880
5881				/* Continue if range is now fully accounted */
5882				if (end_pfn <= usable_startpfn) {
5883
5884					/*
5885					 * Push zone_movable_pfn to the end so
5886					 * that if we have to rebalance
5887					 * kernelcore across nodes, we will
5888					 * not double account here
5889					 */
5890					zone_movable_pfn[nid] = end_pfn;
5891					continue;
5892				}
5893				start_pfn = usable_startpfn;
5894			}
5895
5896			/*
5897			 * The usable PFN range for ZONE_MOVABLE is from
5898			 * start_pfn->end_pfn. Calculate size_pages as the
5899			 * number of pages used as kernelcore
5900			 */
5901			size_pages = end_pfn - start_pfn;
5902			if (size_pages > kernelcore_remaining)
5903				size_pages = kernelcore_remaining;
5904			zone_movable_pfn[nid] = start_pfn + size_pages;
5905
5906			/*
5907			 * Some kernelcore has been met, update counts and
5908			 * break if the kernelcore for this node has been
5909			 * satisfied
5910			 */
5911			required_kernelcore -= min(required_kernelcore,
5912								size_pages);
5913			kernelcore_remaining -= size_pages;
5914			if (!kernelcore_remaining)
5915				break;
5916		}
5917	}
5918
5919	/*
5920	 * If there is still required_kernelcore, we do another pass with one
5921	 * less node in the count. This will push zone_movable_pfn[nid] further
5922	 * along on the nodes that still have memory until kernelcore is
5923	 * satisfied
5924	 */
5925	usable_nodes--;
5926	if (usable_nodes && required_kernelcore > usable_nodes)
5927		goto restart;
5928
5929out2:
5930	/* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
5931	for (nid = 0; nid < MAX_NUMNODES; nid++)
5932		zone_movable_pfn[nid] =
5933			roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
5934
5935out:
5936	/* restore the node_state */
5937	node_states[N_MEMORY] = saved_node_state;
5938}
5939
5940/* Any regular or high memory on that node ? */
5941static void check_for_memory(pg_data_t *pgdat, int nid)
5942{
5943	enum zone_type zone_type;
5944
5945	if (N_MEMORY == N_NORMAL_MEMORY)
5946		return;
5947
5948	for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
5949		struct zone *zone = &pgdat->node_zones[zone_type];
5950		if (populated_zone(zone)) {
5951			node_set_state(nid, N_HIGH_MEMORY);
5952			if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
5953			    zone_type <= ZONE_NORMAL)
5954				node_set_state(nid, N_NORMAL_MEMORY);
5955			break;
5956		}
5957	}
5958}
5959
5960/**
5961 * free_area_init_nodes - Initialise all pg_data_t and zone data
5962 * @max_zone_pfn: an array of max PFNs for each zone
5963 *
5964 * This will call free_area_init_node() for each active node in the system.
5965 * Using the page ranges provided by memblock_set_node(), the size of each
5966 * zone in each node and their holes is calculated. If the maximum PFN
5967 * between two adjacent zones match, it is assumed that the zone is empty.
5968 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
5969 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
5970 * starts where the previous one ended. For example, ZONE_DMA32 starts
5971 * at arch_max_dma_pfn.
5972 */
5973void __init free_area_init_nodes(unsigned long *max_zone_pfn)
5974{
5975	unsigned long start_pfn, end_pfn;
5976	int i, nid;
5977
5978	/* Record where the zone boundaries are */
5979	memset(arch_zone_lowest_possible_pfn, 0,
5980				sizeof(arch_zone_lowest_possible_pfn));
5981	memset(arch_zone_highest_possible_pfn, 0,
5982				sizeof(arch_zone_highest_possible_pfn));
5983	arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
5984	arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
5985	for (i = 1; i < MAX_NR_ZONES; i++) {
5986		if (i == ZONE_MOVABLE)
5987			continue;
5988		arch_zone_lowest_possible_pfn[i] =
5989			arch_zone_highest_possible_pfn[i-1];
5990		arch_zone_highest_possible_pfn[i] =
5991			max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
5992	}
5993	arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
5994	arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
5995
5996	/* Find the PFNs that ZONE_MOVABLE begins at in each node */
5997	memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
5998	find_zone_movable_pfns_for_nodes();
5999
6000	/* Print out the zone ranges */
6001	pr_info("Zone ranges:\n");
6002	for (i = 0; i < MAX_NR_ZONES; i++) {
6003		if (i == ZONE_MOVABLE)
6004			continue;
6005		pr_info("  %-8s ", zone_names[i]);
6006		if (arch_zone_lowest_possible_pfn[i] ==
6007				arch_zone_highest_possible_pfn[i])
6008			pr_cont("empty\n");
6009		else
6010			pr_cont("[mem %#018Lx-%#018Lx]\n",
6011				(u64)arch_zone_lowest_possible_pfn[i]
6012					<< PAGE_SHIFT,
6013				((u64)arch_zone_highest_possible_pfn[i]
6014					<< PAGE_SHIFT) - 1);
6015	}
6016
6017	/* Print out the PFNs ZONE_MOVABLE begins at in each node */
6018	pr_info("Movable zone start for each node\n");
6019	for (i = 0; i < MAX_NUMNODES; i++) {
6020		if (zone_movable_pfn[i])
6021			pr_info("  Node %d: %#018Lx\n", i,
6022			       (u64)zone_movable_pfn[i] << PAGE_SHIFT);
6023	}
6024
6025	/* Print out the early node map */
6026	pr_info("Early memory node ranges\n");
6027	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
6028		pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,
6029			(u64)start_pfn << PAGE_SHIFT,
6030			((u64)end_pfn << PAGE_SHIFT) - 1);
6031
6032	/* Initialise every node */
6033	mminit_verify_pageflags_layout();
6034	setup_nr_node_ids();
6035	for_each_online_node(nid) {
6036		pg_data_t *pgdat = NODE_DATA(nid);
6037		free_area_init_node(nid, NULL,
6038				find_min_pfn_for_node(nid), NULL);
6039
6040		/* Any memory on that node */
6041		if (pgdat->node_present_pages)
6042			node_set_state(nid, N_MEMORY);
6043		check_for_memory(pgdat, nid);
6044	}
6045}
6046
6047static int __init cmdline_parse_core(char *p, unsigned long *core)
6048{
6049	unsigned long long coremem;
6050	if (!p)
6051		return -EINVAL;
6052
6053	coremem = memparse(p, &p);
6054	*core = coremem >> PAGE_SHIFT;
6055
6056	/* Paranoid check that UL is enough for the coremem value */
6057	WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
6058
6059	return 0;
6060}
6061
6062/*
6063 * kernelcore=size sets the amount of memory for use for allocations that
6064 * cannot be reclaimed or migrated.
6065 */
6066static int __init cmdline_parse_kernelcore(char *p)
6067{
6068	/* parse kernelcore=mirror */
6069	if (parse_option_str(p, "mirror")) {
6070		mirrored_kernelcore = true;
6071		return 0;
6072	}
6073
6074	return cmdline_parse_core(p, &required_kernelcore);
6075}
6076
6077/*
6078 * movablecore=size sets the amount of memory for use for allocations that
6079 * can be reclaimed or migrated.
6080 */
6081static int __init cmdline_parse_movablecore(char *p)
6082{
6083	return cmdline_parse_core(p, &required_movablecore);
6084}
6085
6086early_param("kernelcore", cmdline_parse_kernelcore);
6087early_param("movablecore", cmdline_parse_movablecore);
6088
6089#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6090
6091void adjust_managed_page_count(struct page *page, long count)
6092{
6093	spin_lock(&managed_page_count_lock);
6094	page_zone(page)->managed_pages += count;
6095	totalram_pages += count;
6096#ifdef CONFIG_HIGHMEM
6097	if (PageHighMem(page))
6098		totalhigh_pages += count;
6099#endif
6100	spin_unlock(&managed_page_count_lock);
6101}
6102EXPORT_SYMBOL(adjust_managed_page_count);
6103
6104unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
6105{
6106	void *pos;
6107	unsigned long pages = 0;
6108
6109	start = (void *)PAGE_ALIGN((unsigned long)start);
6110	end = (void *)((unsigned long)end & PAGE_MASK);
6111	for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6112		if ((unsigned int)poison <= 0xFF)
6113			memset(pos, poison, PAGE_SIZE);
6114		free_reserved_page(virt_to_page(pos));
 
6115	}
6116
6117	if (pages && s)
6118		pr_info("Freeing %s memory: %ldK (%p - %p)\n",
6119			s, pages << (PAGE_SHIFT - 10), start, end);
6120
6121	return pages;
6122}
6123EXPORT_SYMBOL(free_reserved_area);
6124
6125#ifdef	CONFIG_HIGHMEM
6126void free_highmem_page(struct page *page)
6127{
6128	__free_reserved_page(page);
6129	totalram_pages++;
6130	page_zone(page)->managed_pages++;
6131	totalhigh_pages++;
6132}
6133#endif
6134
6135
6136void __init mem_init_print_info(const char *str)
6137{
6138	unsigned long physpages, codesize, datasize, rosize, bss_size;
6139	unsigned long init_code_size, init_data_size;
6140
6141	physpages = get_num_physpages();
6142	codesize = _etext - _stext;
6143	datasize = _edata - _sdata;
6144	rosize = __end_rodata - __start_rodata;
6145	bss_size = __bss_stop - __bss_start;
6146	init_data_size = __init_end - __init_begin;
6147	init_code_size = _einittext - _sinittext;
6148
6149	/*
6150	 * Detect special cases and adjust section sizes accordingly:
6151	 * 1) .init.* may be embedded into .data sections
6152	 * 2) .init.text.* may be out of [__init_begin, __init_end],
6153	 *    please refer to arch/tile/kernel/vmlinux.lds.S.
6154	 * 3) .rodata.* may be embedded into .text or .data sections.
6155	 */
6156#define adj_init_size(start, end, size, pos, adj) \
6157	do { \
6158		if (start <= pos && pos < end && size > adj) \
6159			size -= adj; \
6160	} while (0)
6161
6162	adj_init_size(__init_begin, __init_end, init_data_size,
6163		     _sinittext, init_code_size);
6164	adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
6165	adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
6166	adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
6167	adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
 
 
6168
6169#undef	adj_init_size
 
6170
6171	pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
6172#ifdef	CONFIG_HIGHMEM
6173		", %luK highmem"
6174#endif
6175		"%s%s)\n",
6176		nr_free_pages() << (PAGE_SHIFT - 10),
6177		physpages << (PAGE_SHIFT - 10),
6178		codesize >> 10, datasize >> 10, rosize >> 10,
6179		(init_data_size + init_code_size) >> 10, bss_size >> 10,
6180		(physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
6181		totalcma_pages << (PAGE_SHIFT - 10),
6182#ifdef	CONFIG_HIGHMEM
6183		totalhigh_pages << (PAGE_SHIFT - 10),
6184#endif
6185		str ? ", " : "", str ? str : "");
6186}
6187
6188/**
6189 * set_dma_reserve - set the specified number of pages reserved in the first zone
6190 * @new_dma_reserve: The number of pages to mark reserved
6191 *
6192 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
6193 * In the DMA zone, a significant percentage may be consumed by kernel image
6194 * and other unfreeable allocations which can skew the watermarks badly. This
6195 * function may optionally be used to account for unfreeable pages in the
6196 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
6197 * smaller per-cpu batchsize.
6198 */
6199void __init set_dma_reserve(unsigned long new_dma_reserve)
6200{
6201	dma_reserve = new_dma_reserve;
6202}
6203
6204void __init free_area_init(unsigned long *zones_size)
6205{
6206	free_area_init_node(0, zones_size,
6207			__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
6208}
6209
6210static int page_alloc_cpu_notify(struct notifier_block *self,
6211				 unsigned long action, void *hcpu)
6212{
6213	int cpu = (unsigned long)hcpu;
6214
6215	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
6216		lru_add_drain_cpu(cpu);
6217		drain_pages(cpu);
6218
6219		/*
6220		 * Spill the event counters of the dead processor
6221		 * into the current processors event counters.
6222		 * This artificially elevates the count of the current
6223		 * processor.
6224		 */
6225		vm_events_fold_cpu(cpu);
6226
6227		/*
6228		 * Zero the differential counters of the dead processor
6229		 * so that the vm statistics are consistent.
6230		 *
6231		 * This is only okay since the processor is dead and cannot
6232		 * race with what we are doing.
6233		 */
6234		cpu_vm_stats_fold(cpu);
6235	}
6236	return NOTIFY_OK;
6237}
6238
6239void __init page_alloc_init(void)
6240{
6241	hotcpu_notifier(page_alloc_cpu_notify, 0);
 
 
6242}
6243
6244/*
6245 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
6246 *	or min_free_kbytes changes.
6247 */
6248static void calculate_totalreserve_pages(void)
6249{
6250	struct pglist_data *pgdat;
6251	unsigned long reserve_pages = 0;
6252	enum zone_type i, j;
6253
6254	for_each_online_pgdat(pgdat) {
 
 
 
6255		for (i = 0; i < MAX_NR_ZONES; i++) {
6256			struct zone *zone = pgdat->node_zones + i;
6257			long max = 0;
 
6258
6259			/* Find valid and maximum lowmem_reserve in the zone */
6260			for (j = i; j < MAX_NR_ZONES; j++) {
6261				if (zone->lowmem_reserve[j] > max)
6262					max = zone->lowmem_reserve[j];
6263			}
6264
6265			/* we treat the high watermark as reserved pages. */
6266			max += high_wmark_pages(zone);
6267
6268			if (max > zone->managed_pages)
6269				max = zone->managed_pages;
6270
6271			zone->totalreserve_pages = max;
6272
6273			reserve_pages += max;
6274		}
6275	}
6276	totalreserve_pages = reserve_pages;
6277}
6278
6279/*
6280 * setup_per_zone_lowmem_reserve - called whenever
6281 *	sysctl_lowmem_reserve_ratio changes.  Ensures that each zone
6282 *	has a correct pages reserved value, so an adequate number of
6283 *	pages are left in the zone after a successful __alloc_pages().
6284 */
6285static void setup_per_zone_lowmem_reserve(void)
6286{
6287	struct pglist_data *pgdat;
6288	enum zone_type j, idx;
6289
6290	for_each_online_pgdat(pgdat) {
6291		for (j = 0; j < MAX_NR_ZONES; j++) {
6292			struct zone *zone = pgdat->node_zones + j;
6293			unsigned long managed_pages = zone->managed_pages;
6294
6295			zone->lowmem_reserve[j] = 0;
6296
6297			idx = j;
6298			while (idx) {
6299				struct zone *lower_zone;
6300
6301				idx--;
6302
6303				if (sysctl_lowmem_reserve_ratio[idx] < 1)
6304					sysctl_lowmem_reserve_ratio[idx] = 1;
6305
6306				lower_zone = pgdat->node_zones + idx;
6307				lower_zone->lowmem_reserve[j] = managed_pages /
6308					sysctl_lowmem_reserve_ratio[idx];
6309				managed_pages += lower_zone->managed_pages;
6310			}
6311		}
6312	}
6313
6314	/* update totalreserve_pages */
6315	calculate_totalreserve_pages();
6316}
6317
6318static void __setup_per_zone_wmarks(void)
6319{
6320	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
6321	unsigned long lowmem_pages = 0;
6322	struct zone *zone;
6323	unsigned long flags;
6324
6325	/* Calculate total number of !ZONE_HIGHMEM pages */
6326	for_each_zone(zone) {
6327		if (!is_highmem(zone))
6328			lowmem_pages += zone->managed_pages;
6329	}
6330
6331	for_each_zone(zone) {
6332		u64 tmp;
6333
6334		spin_lock_irqsave(&zone->lock, flags);
6335		tmp = (u64)pages_min * zone->managed_pages;
6336		do_div(tmp, lowmem_pages);
6337		if (is_highmem(zone)) {
6338			/*
6339			 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
6340			 * need highmem pages, so cap pages_min to a small
6341			 * value here.
6342			 *
6343			 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
6344			 * deltas control asynch page reclaim, and so should
6345			 * not be capped for highmem.
6346			 */
6347			unsigned long min_pages;
6348
6349			min_pages = zone->managed_pages / 1024;
6350			min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
6351			zone->watermark[WMARK_MIN] = min_pages;
6352		} else {
6353			/*
6354			 * If it's a lowmem zone, reserve a number of pages
6355			 * proportionate to the zone's size.
6356			 */
6357			zone->watermark[WMARK_MIN] = tmp;
6358		}
6359
6360		/*
6361		 * Set the kswapd watermarks distance according to the
6362		 * scale factor in proportion to available memory, but
6363		 * ensure a minimum size on small systems.
6364		 */
6365		tmp = max_t(u64, tmp >> 2,
6366			    mult_frac(zone->managed_pages,
6367				      watermark_scale_factor, 10000));
6368
6369		zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;
6370		zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
6371
6372		__mod_zone_page_state(zone, NR_ALLOC_BATCH,
6373			high_wmark_pages(zone) - low_wmark_pages(zone) -
6374			atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
6375
6376		spin_unlock_irqrestore(&zone->lock, flags);
6377	}
6378
6379	/* update totalreserve_pages */
6380	calculate_totalreserve_pages();
6381}
6382
6383/**
6384 * setup_per_zone_wmarks - called when min_free_kbytes changes
6385 * or when memory is hot-{added|removed}
6386 *
6387 * Ensures that the watermark[min,low,high] values for each zone are set
6388 * correctly with respect to min_free_kbytes.
6389 */
6390void setup_per_zone_wmarks(void)
6391{
6392	mutex_lock(&zonelists_mutex);
6393	__setup_per_zone_wmarks();
6394	mutex_unlock(&zonelists_mutex);
6395}
6396
6397/*
6398 * The inactive anon list should be small enough that the VM never has to
6399 * do too much work, but large enough that each inactive page has a chance
6400 * to be referenced again before it is swapped out.
6401 *
6402 * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
6403 * INACTIVE_ANON pages on this zone's LRU, maintained by the
6404 * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
6405 * the anonymous pages are kept on the inactive list.
6406 *
6407 * total     target    max
6408 * memory    ratio     inactive anon
6409 * -------------------------------------
6410 *   10MB       1         5MB
6411 *  100MB       1        50MB
6412 *    1GB       3       250MB
6413 *   10GB      10       0.9GB
6414 *  100GB      31         3GB
6415 *    1TB     101        10GB
6416 *   10TB     320        32GB
6417 */
6418static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
6419{
6420	unsigned int gb, ratio;
6421
6422	/* Zone size in gigabytes */
6423	gb = zone->managed_pages >> (30 - PAGE_SHIFT);
6424	if (gb)
6425		ratio = int_sqrt(10 * gb);
6426	else
6427		ratio = 1;
6428
6429	zone->inactive_ratio = ratio;
6430}
6431
6432static void __meminit setup_per_zone_inactive_ratio(void)
6433{
6434	struct zone *zone;
 
 
 
 
 
6435
 
 
 
 
6436	for_each_zone(zone)
6437		calculate_zone_inactive_ratio(zone);
6438}
6439
6440/*
6441 * Initialise min_free_kbytes.
6442 *
6443 * For small machines we want it small (128k min).  For large machines
6444 * we want it large (64MB max).  But it is not linear, because network
6445 * bandwidth does not increase linearly with machine size.  We use
6446 *
6447 *	min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
6448 *	min_free_kbytes = sqrt(lowmem_kbytes * 16)
6449 *
6450 * which yields
6451 *
6452 * 16MB:	512k
6453 * 32MB:	724k
6454 * 64MB:	1024k
6455 * 128MB:	1448k
6456 * 256MB:	2048k
6457 * 512MB:	2896k
6458 * 1024MB:	4096k
6459 * 2048MB:	5792k
6460 * 4096MB:	8192k
6461 * 8192MB:	11584k
6462 * 16384MB:	16384k
6463 */
6464int __meminit init_per_zone_wmark_min(void)
6465{
6466	unsigned long lowmem_kbytes;
6467	int new_min_free_kbytes;
6468
6469	lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
6470	new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
6471
6472	if (new_min_free_kbytes > user_min_free_kbytes) {
6473		min_free_kbytes = new_min_free_kbytes;
6474		if (min_free_kbytes < 128)
6475			min_free_kbytes = 128;
6476		if (min_free_kbytes > 65536)
6477			min_free_kbytes = 65536;
6478	} else {
6479		pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
6480				new_min_free_kbytes, user_min_free_kbytes);
6481	}
 
 
 
 
 
6482	setup_per_zone_wmarks();
6483	refresh_zone_stat_thresholds();
6484	setup_per_zone_lowmem_reserve();
6485	setup_per_zone_inactive_ratio();
 
 
 
 
 
 
 
6486	return 0;
6487}
6488core_initcall(init_per_zone_wmark_min)
6489
6490/*
6491 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
6492 *	that we can call two helper functions whenever min_free_kbytes
6493 *	changes.
6494 */
6495int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
6496	void __user *buffer, size_t *length, loff_t *ppos)
6497{
6498	int rc;
6499
6500	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6501	if (rc)
6502		return rc;
6503
6504	if (write) {
6505		user_min_free_kbytes = min_free_kbytes;
6506		setup_per_zone_wmarks();
6507	}
6508	return 0;
6509}
6510
6511int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
6512	void __user *buffer, size_t *length, loff_t *ppos)
6513{
6514	int rc;
6515
6516	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6517	if (rc)
6518		return rc;
6519
6520	if (write)
6521		setup_per_zone_wmarks();
6522
6523	return 0;
6524}
6525
6526#ifdef CONFIG_NUMA
6527int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
6528	void __user *buffer, size_t *length, loff_t *ppos)
6529{
 
6530	struct zone *zone;
 
 
 
 
 
 
 
 
 
 
 
 
 
6531	int rc;
6532
6533	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6534	if (rc)
6535		return rc;
6536
6537	for_each_zone(zone)
6538		zone->min_unmapped_pages = (zone->managed_pages *
6539				sysctl_min_unmapped_ratio) / 100;
6540	return 0;
6541}
6542
6543int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
6544	void __user *buffer, size_t *length, loff_t *ppos)
6545{
 
6546	struct zone *zone;
 
 
 
 
 
 
 
 
 
 
 
 
6547	int rc;
6548
6549	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6550	if (rc)
6551		return rc;
6552
6553	for_each_zone(zone)
6554		zone->min_slab_pages = (zone->managed_pages *
6555				sysctl_min_slab_ratio) / 100;
6556	return 0;
6557}
6558#endif
6559
6560/*
6561 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
6562 *	proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
6563 *	whenever sysctl_lowmem_reserve_ratio changes.
6564 *
6565 * The reserve ratio obviously has absolutely no relation with the
6566 * minimum watermarks. The lowmem reserve ratio can only make sense
6567 * if in function of the boot time zone sizes.
6568 */
6569int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
6570	void __user *buffer, size_t *length, loff_t *ppos)
6571{
 
 
6572	proc_dointvec_minmax(table, write, buffer, length, ppos);
 
 
 
 
 
 
6573	setup_per_zone_lowmem_reserve();
6574	return 0;
6575}
6576
6577/*
6578 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
6579 * cpu.  It is the fraction of total pages in each zone that a hot per cpu
6580 * pagelist can have before it gets flushed back to buddy allocator.
6581 */
6582int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
6583	void __user *buffer, size_t *length, loff_t *ppos)
6584{
6585	struct zone *zone;
6586	int old_percpu_pagelist_fraction;
6587	int ret;
6588
6589	mutex_lock(&pcp_batch_high_lock);
6590	old_percpu_pagelist_fraction = percpu_pagelist_fraction;
6591
6592	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
6593	if (!write || ret < 0)
6594		goto out;
6595
6596	/* Sanity checking to avoid pcp imbalance */
6597	if (percpu_pagelist_fraction &&
6598	    percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
6599		percpu_pagelist_fraction = old_percpu_pagelist_fraction;
6600		ret = -EINVAL;
6601		goto out;
6602	}
6603
6604	/* No change? */
6605	if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
6606		goto out;
6607
6608	for_each_populated_zone(zone) {
6609		unsigned int cpu;
6610
6611		for_each_possible_cpu(cpu)
6612			pageset_set_high_and_batch(zone,
6613					per_cpu_ptr(zone->pageset, cpu));
6614	}
6615out:
6616	mutex_unlock(&pcp_batch_high_lock);
6617	return ret;
6618}
6619
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6620#ifdef CONFIG_NUMA
6621int hashdist = HASHDIST_DEFAULT;
6622
6623static int __init set_hashdist(char *str)
6624{
6625	if (!str)
6626		return 0;
6627	hashdist = simple_strtoul(str, &str, 0);
6628	return 1;
6629}
6630__setup("hashdist=", set_hashdist);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6631#endif
 
 
6632
6633/*
6634 * allocate a large system hash table from bootmem
6635 * - it is assumed that the hash table must contain an exact power-of-2
6636 *   quantity of entries
6637 * - limit is the number of hash buckets, not the total allocation size
6638 */
6639void *__init alloc_large_system_hash(const char *tablename,
6640				     unsigned long bucketsize,
6641				     unsigned long numentries,
6642				     int scale,
6643				     int flags,
6644				     unsigned int *_hash_shift,
6645				     unsigned int *_hash_mask,
6646				     unsigned long low_limit,
6647				     unsigned long high_limit)
6648{
6649	unsigned long long max = high_limit;
6650	unsigned long log2qty, size;
6651	void *table = NULL;
6652
6653	/* allow the kernel cmdline to have a say */
6654	if (!numentries) {
6655		/* round applicable memory size up to nearest megabyte */
6656		numentries = nr_kernel_pages;
6657
6658		/* It isn't necessary when PAGE_SIZE >= 1MB */
6659		if (PAGE_SHIFT < 20)
6660			numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
6661
6662		/* limit to 1 bucket per 2^scale bytes of low memory */
6663		if (scale > PAGE_SHIFT)
6664			numentries >>= (scale - PAGE_SHIFT);
6665		else
6666			numentries <<= (PAGE_SHIFT - scale);
6667
6668		/* Make sure we've got at least a 0-order allocation.. */
6669		if (unlikely(flags & HASH_SMALL)) {
6670			/* Makes no sense without HASH_EARLY */
6671			WARN_ON(!(flags & HASH_EARLY));
6672			if (!(numentries >> *_hash_shift)) {
6673				numentries = 1UL << *_hash_shift;
6674				BUG_ON(!numentries);
6675			}
6676		} else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
6677			numentries = PAGE_SIZE / bucketsize;
6678	}
6679	numentries = roundup_pow_of_two(numentries);
6680
6681	/* limit allocation size to 1/16 total memory by default */
6682	if (max == 0) {
6683		max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
6684		do_div(max, bucketsize);
6685	}
6686	max = min(max, 0x80000000ULL);
6687
6688	if (numentries < low_limit)
6689		numentries = low_limit;
6690	if (numentries > max)
6691		numentries = max;
6692
6693	log2qty = ilog2(numentries);
6694
6695	do {
6696		size = bucketsize << log2qty;
6697		if (flags & HASH_EARLY)
6698			table = memblock_virt_alloc_nopanic(size, 0);
6699		else if (hashdist)
6700			table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
6701		else {
6702			/*
6703			 * If bucketsize is not a power-of-two, we may free
6704			 * some pages at the end of hash table which
6705			 * alloc_pages_exact() automatically does
6706			 */
6707			if (get_order(size) < MAX_ORDER) {
6708				table = alloc_pages_exact(size, GFP_ATOMIC);
6709				kmemleak_alloc(table, size, 1, GFP_ATOMIC);
6710			}
6711		}
6712	} while (!table && size > PAGE_SIZE && --log2qty);
6713
6714	if (!table)
6715		panic("Failed to allocate %s hash table\n", tablename);
6716
6717	pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
6718		tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
6719
6720	if (_hash_shift)
6721		*_hash_shift = log2qty;
6722	if (_hash_mask)
6723		*_hash_mask = (1 << log2qty) - 1;
6724
6725	return table;
6726}
6727
6728/* Return a pointer to the bitmap storing bits affecting a block of pages */
6729static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
6730							unsigned long pfn)
6731{
6732#ifdef CONFIG_SPARSEMEM
6733	return __pfn_to_section(pfn)->pageblock_flags;
6734#else
6735	return zone->pageblock_flags;
6736#endif /* CONFIG_SPARSEMEM */
6737}
6738
6739static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
6740{
6741#ifdef CONFIG_SPARSEMEM
6742	pfn &= (PAGES_PER_SECTION-1);
6743	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
6744#else
6745	pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages);
6746	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
6747#endif /* CONFIG_SPARSEMEM */
6748}
6749
6750/**
6751 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
6752 * @page: The page within the block of interest
6753 * @pfn: The target page frame number
6754 * @end_bitidx: The last bit of interest to retrieve
6755 * @mask: mask of bits that the caller is interested in
6756 *
6757 * Return: pageblock_bits flags
6758 */
6759unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
6760					unsigned long end_bitidx,
6761					unsigned long mask)
6762{
6763	struct zone *zone;
6764	unsigned long *bitmap;
6765	unsigned long bitidx, word_bitidx;
6766	unsigned long word;
6767
6768	zone = page_zone(page);
6769	bitmap = get_pageblock_bitmap(zone, pfn);
6770	bitidx = pfn_to_bitidx(zone, pfn);
6771	word_bitidx = bitidx / BITS_PER_LONG;
6772	bitidx &= (BITS_PER_LONG-1);
6773
6774	word = bitmap[word_bitidx];
6775	bitidx += end_bitidx;
6776	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
6777}
6778
6779/**
6780 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
6781 * @page: The page within the block of interest
6782 * @flags: The flags to set
6783 * @pfn: The target page frame number
6784 * @end_bitidx: The last bit of interest
6785 * @mask: mask of bits that the caller is interested in
6786 */
6787void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
6788					unsigned long pfn,
6789					unsigned long end_bitidx,
6790					unsigned long mask)
6791{
6792	struct zone *zone;
6793	unsigned long *bitmap;
6794	unsigned long bitidx, word_bitidx;
6795	unsigned long old_word, word;
6796
6797	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
6798
6799	zone = page_zone(page);
6800	bitmap = get_pageblock_bitmap(zone, pfn);
6801	bitidx = pfn_to_bitidx(zone, pfn);
6802	word_bitidx = bitidx / BITS_PER_LONG;
6803	bitidx &= (BITS_PER_LONG-1);
6804
6805	VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page);
6806
6807	bitidx += end_bitidx;
6808	mask <<= (BITS_PER_LONG - bitidx - 1);
6809	flags <<= (BITS_PER_LONG - bitidx - 1);
6810
6811	word = READ_ONCE(bitmap[word_bitidx]);
6812	for (;;) {
6813		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
6814		if (word == old_word)
6815			break;
6816		word = old_word;
6817	}
6818}
6819
6820/*
6821 * This function checks whether pageblock includes unmovable pages or not.
6822 * If @count is not zero, it is okay to include less @count unmovable pages
6823 *
6824 * PageLRU check without isolation or lru_lock could race so that
6825 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
6826 * expect this function should be exact.
6827 */
6828bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
6829			 bool skip_hwpoisoned_pages)
6830{
6831	unsigned long pfn, iter, found;
6832	int mt;
6833
6834	/*
6835	 * For avoiding noise data, lru_add_drain_all() should be called
6836	 * If ZONE_MOVABLE, the zone never contains unmovable pages
6837	 */
6838	if (zone_idx(zone) == ZONE_MOVABLE)
6839		return false;
6840	mt = get_pageblock_migratetype(page);
6841	if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
6842		return false;
6843
6844	pfn = page_to_pfn(page);
6845	for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
6846		unsigned long check = pfn + iter;
6847
6848		if (!pfn_valid_within(check))
6849			continue;
6850
6851		page = pfn_to_page(check);
6852
6853		/*
6854		 * Hugepages are not in LRU lists, but they're movable.
6855		 * We need not scan over tail pages bacause we don't
6856		 * handle each tail page individually in migration.
6857		 */
6858		if (PageHuge(page)) {
6859			iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
6860			continue;
6861		}
6862
6863		/*
6864		 * We can't use page_count without pin a page
6865		 * because another CPU can free compound page.
6866		 * This check already skips compound tails of THP
6867		 * because their page->_count is zero at all time.
6868		 */
6869		if (!page_ref_count(page)) {
6870			if (PageBuddy(page))
6871				iter += (1 << page_order(page)) - 1;
6872			continue;
6873		}
6874
6875		/*
6876		 * The HWPoisoned page may be not in buddy system, and
6877		 * page_count() is not 0.
6878		 */
6879		if (skip_hwpoisoned_pages && PageHWPoison(page))
6880			continue;
6881
6882		if (!PageLRU(page))
6883			found++;
6884		/*
6885		 * If there are RECLAIMABLE pages, we need to check
6886		 * it.  But now, memory offline itself doesn't call
6887		 * shrink_node_slabs() and it still to be fixed.
6888		 */
6889		/*
6890		 * If the page is not RAM, page_count()should be 0.
6891		 * we don't need more check. This is an _used_ not-movable page.
6892		 *
6893		 * The problematic thing here is PG_reserved pages. PG_reserved
6894		 * is set to both of a memory hole page and a _used_ kernel
6895		 * page at boot.
6896		 */
6897		if (found > count)
6898			return true;
6899	}
6900	return false;
6901}
6902
6903bool is_pageblock_removable_nolock(struct page *page)
6904{
6905	struct zone *zone;
6906	unsigned long pfn;
6907
6908	/*
6909	 * We have to be careful here because we are iterating over memory
6910	 * sections which are not zone aware so we might end up outside of
6911	 * the zone but still within the section.
6912	 * We have to take care about the node as well. If the node is offline
6913	 * its NODE_DATA will be NULL - see page_zone.
6914	 */
6915	if (!node_online(page_to_nid(page)))
6916		return false;
6917
6918	zone = page_zone(page);
6919	pfn = page_to_pfn(page);
6920	if (!zone_spans_pfn(zone, pfn))
6921		return false;
6922
6923	return !has_unmovable_pages(zone, page, 0, true);
6924}
6925
6926#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
6927
6928static unsigned long pfn_max_align_down(unsigned long pfn)
6929{
6930	return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
6931			     pageblock_nr_pages) - 1);
6932}
6933
6934static unsigned long pfn_max_align_up(unsigned long pfn)
6935{
6936	return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
6937				pageblock_nr_pages));
6938}
6939
6940/* [start, end) must belong to a single zone. */
6941static int __alloc_contig_migrate_range(struct compact_control *cc,
6942					unsigned long start, unsigned long end)
6943{
6944	/* This function is based on compact_zone() from compaction.c. */
6945	unsigned long nr_reclaimed;
6946	unsigned long pfn = start;
6947	unsigned int tries = 0;
6948	int ret = 0;
 
 
 
 
 
 
 
 
6949
6950	migrate_prep();
6951
6952	while (pfn < end || !list_empty(&cc->migratepages)) {
6953		if (fatal_signal_pending(current)) {
6954			ret = -EINTR;
6955			break;
6956		}
6957
6958		if (list_empty(&cc->migratepages)) {
6959			cc->nr_migratepages = 0;
6960			pfn = isolate_migratepages_range(cc, pfn, end);
6961			if (!pfn) {
6962				ret = -EINTR;
6963				break;
6964			}
6965			tries = 0;
6966		} else if (++tries == 5) {
6967			ret = ret < 0 ? ret : -EBUSY;
6968			break;
6969		}
6970
6971		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
6972							&cc->migratepages);
6973		cc->nr_migratepages -= nr_reclaimed;
6974
6975		ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
6976				    NULL, 0, cc->mode, MR_CMA);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6977	}
 
 
6978	if (ret < 0) {
 
 
6979		putback_movable_pages(&cc->migratepages);
6980		return ret;
6981	}
6982	return 0;
 
 
 
 
 
6983}
6984
6985/**
6986 * alloc_contig_range() -- tries to allocate given range of pages
6987 * @start:	start PFN to allocate
6988 * @end:	one-past-the-last PFN to allocate
6989 * @migratetype:	migratetype of the underlaying pageblocks (either
6990 *			#MIGRATE_MOVABLE or #MIGRATE_CMA).  All pageblocks
6991 *			in range must have the same migratetype and it must
6992 *			be either of the two.
 
6993 *
6994 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
6995 * aligned, however it's the caller's responsibility to guarantee that
6996 * we are the only thread that changes migrate type of pageblocks the
6997 * pages fall in.
6998 *
6999 * The PFN range must belong to a single zone.
 
 
7000 *
7001 * Returns zero on success or negative error code.  On success all
7002 * pages which PFN is in [start, end) are allocated for the caller and
7003 * need to be freed with free_contig_range().
7004 */
7005int alloc_contig_range(unsigned long start, unsigned long end,
7006		       unsigned migratetype)
7007{
7008	unsigned long outer_start, outer_end;
7009	unsigned int order;
7010	int ret = 0;
7011
7012	struct compact_control cc = {
7013		.nr_migratepages = 0,
7014		.order = -1,
7015		.zone = page_zone(pfn_to_page(start)),
7016		.mode = MIGRATE_SYNC,
7017		.ignore_skip_hint = true,
 
 
 
7018	};
7019	INIT_LIST_HEAD(&cc.migratepages);
7020
7021	/*
7022	 * What we do here is we mark all pageblocks in range as
7023	 * MIGRATE_ISOLATE.  Because pageblock and max order pages may
7024	 * have different sizes, and due to the way page allocator
7025	 * work, we align the range to biggest of the two pages so
7026	 * that page allocator won't try to merge buddies from
7027	 * different pageblocks and change MIGRATE_ISOLATE to some
7028	 * other migration type.
7029	 *
7030	 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
7031	 * migrate the pages from an unaligned range (ie. pages that
7032	 * we are interested in).  This will put all the pages in
7033	 * range back to page allocator as MIGRATE_ISOLATE.
7034	 *
7035	 * When this is done, we take the pages in range from page
7036	 * allocator removing them from the buddy system.  This way
7037	 * page allocator will never consider using them.
7038	 *
7039	 * This lets us mark the pageblocks back as
7040	 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
7041	 * aligned range but not in the unaligned, original range are
7042	 * put back to page allocator so that buddy can use them.
7043	 */
7044
7045	ret = start_isolate_page_range(pfn_max_align_down(start),
7046				       pfn_max_align_up(end), migratetype,
7047				       false);
7048	if (ret)
7049		return ret;
 
 
7050
7051	/*
7052	 * In case of -EBUSY, we'd like to know which page causes problem.
7053	 * So, just fall through. We will check it in test_pages_isolated().
 
 
 
 
 
 
7054	 */
7055	ret = __alloc_contig_migrate_range(&cc, start, end);
7056	if (ret && ret != -EBUSY)
7057		goto done;
 
7058
7059	/*
7060	 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
7061	 * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
7062	 * more, all pages in [start, end) are free in page allocator.
7063	 * What we are going to do is to allocate all pages from
7064	 * [start, end) (that is remove them from page allocator).
7065	 *
7066	 * The only problem is that pages at the beginning and at the
7067	 * end of interesting range may be not aligned with pages that
7068	 * page allocator holds, ie. they can be part of higher order
7069	 * pages.  Because of this, we reserve the bigger range and
7070	 * once this is done free the pages we are not interested in.
7071	 *
7072	 * We don't have to hold zone->lock here because the pages are
7073	 * isolated thus they won't get removed from buddy.
7074	 */
7075
7076	lru_add_drain_all();
7077	drain_all_pages(cc.zone);
7078
7079	order = 0;
7080	outer_start = start;
7081	while (!PageBuddy(pfn_to_page(outer_start))) {
7082		if (++order >= MAX_ORDER) {
7083			outer_start = start;
7084			break;
7085		}
7086		outer_start &= ~0UL << order;
7087	}
7088
7089	if (outer_start != start) {
7090		order = page_order(pfn_to_page(outer_start));
7091
7092		/*
7093		 * outer_start page could be small order buddy page and
7094		 * it doesn't include start page. Adjust outer_start
7095		 * in this case to report failed page properly
7096		 * on tracepoint in test_pages_isolated()
7097		 */
7098		if (outer_start + (1UL << order) <= start)
7099			outer_start = start;
7100	}
7101
7102	/* Make sure the range is really isolated. */
7103	if (test_pages_isolated(outer_start, end, false)) {
7104		pr_info("%s: [%lx, %lx) PFNs busy\n",
7105			__func__, outer_start, end);
7106		ret = -EBUSY;
7107		goto done;
7108	}
7109
7110	/* Grab isolated pages from freelists. */
7111	outer_end = isolate_freepages_range(&cc, outer_start, end);
7112	if (!outer_end) {
7113		ret = -EBUSY;
7114		goto done;
7115	}
7116
7117	/* Free head and tail (if any) */
7118	if (start != outer_start)
7119		free_contig_range(outer_start, start - outer_start);
7120	if (end != outer_end)
7121		free_contig_range(end, outer_end - end);
7122
7123done:
7124	undo_isolate_page_range(pfn_max_align_down(start),
7125				pfn_max_align_up(end), migratetype);
7126	return ret;
7127}
 
 
 
 
 
 
 
 
 
 
7128
7129void free_contig_range(unsigned long pfn, unsigned nr_pages)
 
7130{
7131	unsigned int count = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7132
7133	for (; nr_pages--; pfn++) {
7134		struct page *page = pfn_to_page(pfn);
7135
7136		count += page_count(page) != 1;
7137		__free_page(page);
7138	}
7139	WARN(count != 0, "%d pages are still in use!\n", count);
7140}
7141#endif
7142
7143#ifdef CONFIG_MEMORY_HOTPLUG
7144/*
7145 * The zone indicated has a new number of managed_pages; batch sizes and percpu
7146 * page high values need to be recalulated.
 
 
 
 
7147 */
7148void __meminit zone_pcp_update(struct zone *zone)
7149{
7150	unsigned cpu;
7151	mutex_lock(&pcp_batch_high_lock);
7152	for_each_possible_cpu(cpu)
7153		pageset_set_high_and_batch(zone,
7154				per_cpu_ptr(zone->pageset, cpu));
 
 
 
 
 
7155	mutex_unlock(&pcp_batch_high_lock);
7156}
7157#endif
7158
7159void zone_pcp_reset(struct zone *zone)
7160{
7161	unsigned long flags;
7162	int cpu;
7163	struct per_cpu_pageset *pset;
7164
7165	/* avoid races with drain_pages()  */
7166	local_irq_save(flags);
7167	if (zone->pageset != &boot_pageset) {
7168		for_each_online_cpu(cpu) {
7169			pset = per_cpu_ptr(zone->pageset, cpu);
7170			drain_zonestat(zone, pset);
 
 
 
 
 
 
7171		}
7172		free_percpu(zone->pageset);
7173		zone->pageset = &boot_pageset;
7174	}
7175	local_irq_restore(flags);
7176}
7177
7178#ifdef CONFIG_MEMORY_HOTREMOVE
7179/*
7180 * All pages in the range must be isolated before calling this.
 
7181 */
7182void
7183__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
7184{
 
7185	struct page *page;
7186	struct zone *zone;
7187	unsigned int order, i;
7188	unsigned long pfn;
7189	unsigned long flags;
7190	/* find the first valid pfn */
7191	for (pfn = start_pfn; pfn < end_pfn; pfn++)
7192		if (pfn_valid(pfn))
7193			break;
7194	if (pfn == end_pfn)
7195		return;
7196	zone = page_zone(pfn_to_page(pfn));
7197	spin_lock_irqsave(&zone->lock, flags);
7198	pfn = start_pfn;
7199	while (pfn < end_pfn) {
7200		if (!pfn_valid(pfn)) {
7201			pfn++;
7202			continue;
7203		}
7204		page = pfn_to_page(pfn);
7205		/*
7206		 * The HWPoisoned page may be not in buddy system, and
7207		 * page_count() is not 0.
7208		 */
7209		if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
7210			pfn++;
7211			SetPageReserved(page);
 
 
 
 
 
 
 
 
 
7212			continue;
7213		}
7214
7215		BUG_ON(page_count(page));
7216		BUG_ON(!PageBuddy(page));
7217		order = page_order(page);
7218#ifdef CONFIG_DEBUG_VM
7219		pr_info("remove from free list %lx %d %lx\n",
7220			pfn, 1 << order, end_pfn);
7221#endif
7222		list_del(&page->lru);
7223		rmv_page_order(page);
7224		zone->free_area[order].nr_free--;
7225		for (i = 0; i < (1 << order); i++)
7226			SetPageReserved((page+i));
7227		pfn += (1 << order);
7228	}
7229	spin_unlock_irqrestore(&zone->lock, flags);
7230}
7231#endif
7232
 
 
 
7233bool is_free_buddy_page(struct page *page)
7234{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7235	struct zone *zone = page_zone(page);
7236	unsigned long pfn = page_to_pfn(page);
7237	unsigned long flags;
7238	unsigned int order;
 
7239
7240	spin_lock_irqsave(&zone->lock, flags);
7241	for (order = 0; order < MAX_ORDER; order++) {
7242		struct page *page_head = page - (pfn & ((1 << order) - 1));
 
7243
7244		if (PageBuddy(page_head) && page_order(page_head) >= order)
 
 
 
 
 
 
 
 
 
 
 
7245			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7246	}
 
 
 
 
 
 
7247	spin_unlock_irqrestore(&zone->lock, flags);
7248
7249	return order < MAX_ORDER;
 
 
 
 
 
 
 
7250}