hugetlb.c - mm/hugetlb.c - Linux diff v6.8 - Bootlin Elixir Cross Referencer

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Generic hugetlb support.
   4 * (C) Nadia Yvette Chambers, April 2004
   5 */
   6#include <linux/list.h>
   7#include <linux/init.h>
   8#include <linux/mm.h>
   9#include <linux/seq_file.h>
  10#include <linux/sysctl.h>
  11#include <linux/highmem.h>
  12#include <linux/mmu_notifier.h>
  13#include <linux/nodemask.h>
  14#include <linux/pagemap.h>
  15#include <linux/mempolicy.h>
  16#include <linux/compiler.h>
  17#include <linux/cpuset.h>
  18#include <linux/mutex.h>
  19#include <linux/memblock.h>
  20#include <linux/sysfs.h>
  21#include <linux/slab.h>
  22#include <linux/sched/mm.h>
  23#include <linux/mmdebug.h>
  24#include <linux/sched/signal.h>
  25#include <linux/rmap.h>
  26#include <linux/string_helpers.h>
  27#include <linux/swap.h>
  28#include <linux/swapops.h>
  29#include <linux/jhash.h>
  30#include <linux/numa.h>
  31#include <linux/llist.h>
  32#include <linux/cma.h>
  33#include <linux/migrate.h>
  34#include <linux/nospec.h>
  35#include <linux/delayacct.h>
  36#include <linux/memory.h>
  37#include <linux/mm_inline.h>
  38
  39#include <asm/page.h>
  40#include <asm/pgalloc.h>
  41#include <asm/tlb.h>
  42
  43#include <linux/io.h>
  44#include <linux/hugetlb.h>
  45#include <linux/hugetlb_cgroup.h>
  46#include <linux/node.h>
  47#include <linux/page_owner.h>
  48#include "internal.h"
  49#include "hugetlb_vmemmap.h"
  50
  51int hugetlb_max_hstate __read_mostly;
  52unsigned int default_hstate_idx;
  53struct hstate hstates[HUGE_MAX_HSTATE];
  54
  55#ifdef CONFIG_CMA
  56static struct cma *hugetlb_cma[MAX_NUMNODES];
  57static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
  58static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
  59{
  60	return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
  61				1 << order);
  62}
  63#else
  64static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
  65{
  66	return false;
  67}
  68#endif
  69static unsigned long hugetlb_cma_size __initdata;
  70
 
 
 
 
 
 
  71__initdata LIST_HEAD(huge_boot_pages);
  72
  73/* for command line parsing */
  74static struct hstate * __initdata parsed_hstate;
  75static unsigned long __initdata default_hstate_max_huge_pages;
  76static bool __initdata parsed_valid_hugepagesz = true;
  77static bool __initdata parsed_default_hugepagesz;
  78static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
  79
  80/*
  81 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  82 * free_huge_pages, and surplus_huge_pages.
  83 */
  84DEFINE_SPINLOCK(hugetlb_lock);
  85
  86/*
  87 * Serializes faults on the same logical page.  This is used to
  88 * prevent spurious OOMs when the hugepage pool is fully utilized.
  89 */
  90static int num_fault_mutexes;
  91struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
  92
  93/* Forward declaration */
  94static int hugetlb_acct_memory(struct hstate *h, long delta);
  95static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
  96static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
  97static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
  98static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
  99		unsigned long start, unsigned long end);
 100static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
 101
 102static inline bool subpool_is_free(struct hugepage_subpool *spool)
 103{
 104	if (spool->count)
 105		return false;
 106	if (spool->max_hpages != -1)
 107		return spool->used_hpages == 0;
 108	if (spool->min_hpages != -1)
 109		return spool->rsv_hpages == spool->min_hpages;
 110
 111	return true;
 112}
 113
 114static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
 115						unsigned long irq_flags)
 116{
 117	spin_unlock_irqrestore(&spool->lock, irq_flags);
 118
 119	/* If no pages are used, and no other handles to the subpool
 120	 * remain, give up any reservations based on minimum size and
 121	 * free the subpool */
 122	if (subpool_is_free(spool)) {
 123		if (spool->min_hpages != -1)
 124			hugetlb_acct_memory(spool->hstate,
 125						-spool->min_hpages);
 126		kfree(spool);
 127	}
 128}
 129
 130struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 131						long min_hpages)
 132{
 133	struct hugepage_subpool *spool;
 134
 135	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 136	if (!spool)
 137		return NULL;
 138
 139	spin_lock_init(&spool->lock);
 140	spool->count = 1;
 141	spool->max_hpages = max_hpages;
 142	spool->hstate = h;
 143	spool->min_hpages = min_hpages;
 144
 145	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
 146		kfree(spool);
 147		return NULL;
 148	}
 149	spool->rsv_hpages = min_hpages;
 150
 151	return spool;
 152}
 153
 154void hugepage_put_subpool(struct hugepage_subpool *spool)
 155{
 156	unsigned long flags;
 157
 158	spin_lock_irqsave(&spool->lock, flags);
 159	BUG_ON(!spool->count);
 160	spool->count--;
 161	unlock_or_release_subpool(spool, flags);
 162}
 163
 164/*
 165 * Subpool accounting for allocating and reserving pages.
 166 * Return -ENOMEM if there are not enough resources to satisfy the
 167 * request.  Otherwise, return the number of pages by which the
 168 * global pools must be adjusted (upward).  The returned value may
 169 * only be different than the passed value (delta) in the case where
 170 * a subpool minimum size must be maintained.
 171 */
 172static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
 173				      long delta)
 174{
 175	long ret = delta;
 176
 177	if (!spool)
 178		return ret;
 179
 180	spin_lock_irq(&spool->lock);
 181
 182	if (spool->max_hpages != -1) {		/* maximum size accounting */
 183		if ((spool->used_hpages + delta) <= spool->max_hpages)
 184			spool->used_hpages += delta;
 185		else {
 186			ret = -ENOMEM;
 187			goto unlock_ret;
 188		}
 189	}
 190
 191	/* minimum size accounting */
 192	if (spool->min_hpages != -1 && spool->rsv_hpages) {
 193		if (delta > spool->rsv_hpages) {
 194			/*
 195			 * Asking for more reserves than those already taken on
 196			 * behalf of subpool.  Return difference.
 197			 */
 198			ret = delta - spool->rsv_hpages;
 199			spool->rsv_hpages = 0;
 200		} else {
 201			ret = 0;	/* reserves already accounted for */
 202			spool->rsv_hpages -= delta;
 203		}
 204	}
 205
 206unlock_ret:
 207	spin_unlock_irq(&spool->lock);
 208	return ret;
 209}
 210
 211/*
 212 * Subpool accounting for freeing and unreserving pages.
 213 * Return the number of global page reservations that must be dropped.
 214 * The return value may only be different than the passed value (delta)
 215 * in the case where a subpool minimum size must be maintained.
 216 */
 217static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 218				       long delta)
 219{
 220	long ret = delta;
 221	unsigned long flags;
 222
 223	if (!spool)
 224		return delta;
 225
 226	spin_lock_irqsave(&spool->lock, flags);
 227
 228	if (spool->max_hpages != -1)		/* maximum size accounting */
 229		spool->used_hpages -= delta;
 230
 231	 /* minimum size accounting */
 232	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
 233		if (spool->rsv_hpages + delta <= spool->min_hpages)
 234			ret = 0;
 235		else
 236			ret = spool->rsv_hpages + delta - spool->min_hpages;
 237
 238		spool->rsv_hpages += delta;
 239		if (spool->rsv_hpages > spool->min_hpages)
 240			spool->rsv_hpages = spool->min_hpages;
 241	}
 242
 243	/*
 244	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
 245	 * quota reference, free it now.
 246	 */
 247	unlock_or_release_subpool(spool, flags);
 248
 249	return ret;
 250}
 251
 252static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 253{
 254	return HUGETLBFS_SB(inode->i_sb)->spool;
 255}
 256
 257static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 258{
 259	return subpool_inode(file_inode(vma->vm_file));
 260}
 261
 262/*
 263 * hugetlb vma_lock helper routines
 264 */
 265void hugetlb_vma_lock_read(struct vm_area_struct *vma)
 266{
 267	if (__vma_shareable_lock(vma)) {
 268		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 269
 270		down_read(&vma_lock->rw_sema);
 271	} else if (__vma_private_lock(vma)) {
 272		struct resv_map *resv_map = vma_resv_map(vma);
 273
 274		down_read(&resv_map->rw_sema);
 275	}
 276}
 277
 278void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
 279{
 280	if (__vma_shareable_lock(vma)) {
 281		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 282
 283		up_read(&vma_lock->rw_sema);
 284	} else if (__vma_private_lock(vma)) {
 285		struct resv_map *resv_map = vma_resv_map(vma);
 286
 287		up_read(&resv_map->rw_sema);
 288	}
 289}
 290
 291void hugetlb_vma_lock_write(struct vm_area_struct *vma)
 292{
 293	if (__vma_shareable_lock(vma)) {
 294		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 295
 296		down_write(&vma_lock->rw_sema);
 297	} else if (__vma_private_lock(vma)) {
 298		struct resv_map *resv_map = vma_resv_map(vma);
 299
 300		down_write(&resv_map->rw_sema);
 301	}
 302}
 303
 304void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
 305{
 306	if (__vma_shareable_lock(vma)) {
 307		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 308
 309		up_write(&vma_lock->rw_sema);
 310	} else if (__vma_private_lock(vma)) {
 311		struct resv_map *resv_map = vma_resv_map(vma);
 312
 313		up_write(&resv_map->rw_sema);
 314	}
 315}
 316
 317int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
 318{
 319
 320	if (__vma_shareable_lock(vma)) {
 321		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 322
 323		return down_write_trylock(&vma_lock->rw_sema);
 324	} else if (__vma_private_lock(vma)) {
 325		struct resv_map *resv_map = vma_resv_map(vma);
 326
 327		return down_write_trylock(&resv_map->rw_sema);
 328	}
 329
 330	return 1;
 331}
 332
 333void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
 334{
 335	if (__vma_shareable_lock(vma)) {
 336		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 337
 338		lockdep_assert_held(&vma_lock->rw_sema);
 339	} else if (__vma_private_lock(vma)) {
 340		struct resv_map *resv_map = vma_resv_map(vma);
 341
 342		lockdep_assert_held(&resv_map->rw_sema);
 343	}
 344}
 345
 346void hugetlb_vma_lock_release(struct kref *kref)
 347{
 348	struct hugetlb_vma_lock *vma_lock = container_of(kref,
 349			struct hugetlb_vma_lock, refs);
 350
 351	kfree(vma_lock);
 352}
 353
 354static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
 355{
 356	struct vm_area_struct *vma = vma_lock->vma;
 357
 358	/*
 359	 * vma_lock structure may or not be released as a result of put,
 360	 * it certainly will no longer be attached to vma so clear pointer.
 361	 * Semaphore synchronizes access to vma_lock->vma field.
 362	 */
 363	vma_lock->vma = NULL;
 364	vma->vm_private_data = NULL;
 365	up_write(&vma_lock->rw_sema);
 366	kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
 367}
 368
 369static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
 370{
 371	if (__vma_shareable_lock(vma)) {
 372		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 373
 374		__hugetlb_vma_unlock_write_put(vma_lock);
 375	} else if (__vma_private_lock(vma)) {
 376		struct resv_map *resv_map = vma_resv_map(vma);
 377
 378		/* no free for anon vmas, but still need to unlock */
 379		up_write(&resv_map->rw_sema);
 380	}
 381}
 382
 383static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
 384{
 385	/*
 386	 * Only present in sharable vmas.
 387	 */
 388	if (!vma || !__vma_shareable_lock(vma))
 389		return;
 390
 391	if (vma->vm_private_data) {
 392		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 393
 394		down_write(&vma_lock->rw_sema);
 395		__hugetlb_vma_unlock_write_put(vma_lock);
 396	}
 397}
 398
 399static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
 400{
 401	struct hugetlb_vma_lock *vma_lock;
 402
 403	/* Only establish in (flags) sharable vmas */
 404	if (!vma || !(vma->vm_flags & VM_MAYSHARE))
 405		return;
 406
 407	/* Should never get here with non-NULL vm_private_data */
 408	if (vma->vm_private_data)
 409		return;
 410
 411	vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL);
 412	if (!vma_lock) {
 413		/*
 414		 * If we can not allocate structure, then vma can not
 415		 * participate in pmd sharing.  This is only a possible
 416		 * performance enhancement and memory saving issue.
 417		 * However, the lock is also used to synchronize page
 418		 * faults with truncation.  If the lock is not present,
 419		 * unlikely races could leave pages in a file past i_size
 420		 * until the file is removed.  Warn in the unlikely case of
 421		 * allocation failure.
 422		 */
 423		pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
 424		return;
 425	}
 426
 427	kref_init(&vma_lock->refs);
 428	init_rwsem(&vma_lock->rw_sema);
 429	vma_lock->vma = vma;
 430	vma->vm_private_data = vma_lock;
 431}
 432
 433/* Helper that removes a struct file_region from the resv_map cache and returns
 434 * it for use.
 435 */
 436static struct file_region *
 437get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
 438{
 439	struct file_region *nrg;
 440
 441	VM_BUG_ON(resv->region_cache_count <= 0);
 442
 443	resv->region_cache_count--;
 444	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
 445	list_del(&nrg->link);
 446
 447	nrg->from = from;
 448	nrg->to = to;
 449
 450	return nrg;
 451}
 452
 453static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
 454					      struct file_region *rg)
 455{
 456#ifdef CONFIG_CGROUP_HUGETLB
 457	nrg->reservation_counter = rg->reservation_counter;
 458	nrg->css = rg->css;
 459	if (rg->css)
 460		css_get(rg->css);
 461#endif
 462}
 463
 464/* Helper that records hugetlb_cgroup uncharge info. */
 465static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
 466						struct hstate *h,
 467						struct resv_map *resv,
 468						struct file_region *nrg)
 469{
 470#ifdef CONFIG_CGROUP_HUGETLB
 471	if (h_cg) {
 472		nrg->reservation_counter =
 473			&h_cg->rsvd_hugepage[hstate_index(h)];
 474		nrg->css = &h_cg->css;
 475		/*
 476		 * The caller will hold exactly one h_cg->css reference for the
 477		 * whole contiguous reservation region. But this area might be
 478		 * scattered when there are already some file_regions reside in
 479		 * it. As a result, many file_regions may share only one css
 480		 * reference. In order to ensure that one file_region must hold
 481		 * exactly one h_cg->css reference, we should do css_get for
 482		 * each file_region and leave the reference held by caller
 483		 * untouched.
 484		 */
 485		css_get(&h_cg->css);
 486		if (!resv->pages_per_hpage)
 487			resv->pages_per_hpage = pages_per_huge_page(h);
 488		/* pages_per_hpage should be the same for all entries in
 489		 * a resv_map.
 490		 */
 491		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
 492	} else {
 493		nrg->reservation_counter = NULL;
 494		nrg->css = NULL;
 495	}
 496#endif
 497}
 498
 499static void put_uncharge_info(struct file_region *rg)
 500{
 501#ifdef CONFIG_CGROUP_HUGETLB
 502	if (rg->css)
 503		css_put(rg->css);
 504#endif
 505}
 506
 507static bool has_same_uncharge_info(struct file_region *rg,
 508				   struct file_region *org)
 509{
 510#ifdef CONFIG_CGROUP_HUGETLB
 511	return rg->reservation_counter == org->reservation_counter &&
 
 512	       rg->css == org->css;
 513
 514#else
 515	return true;
 516#endif
 517}
 518
 519static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
 520{
 521	struct file_region *nrg, *prg;
 522
 523	prg = list_prev_entry(rg, link);
 524	if (&prg->link != &resv->regions && prg->to == rg->from &&
 525	    has_same_uncharge_info(prg, rg)) {
 526		prg->to = rg->to;
 527
 528		list_del(&rg->link);
 529		put_uncharge_info(rg);
 530		kfree(rg);
 531
 532		rg = prg;
 533	}
 534
 535	nrg = list_next_entry(rg, link);
 536	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
 537	    has_same_uncharge_info(nrg, rg)) {
 538		nrg->from = rg->from;
 539
 540		list_del(&rg->link);
 541		put_uncharge_info(rg);
 542		kfree(rg);
 543	}
 544}
 545
 546static inline long
 547hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
 548		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
 549		     long *regions_needed)
 550{
 551	struct file_region *nrg;
 552
 553	if (!regions_needed) {
 554		nrg = get_file_region_entry_from_cache(map, from, to);
 555		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
 556		list_add(&nrg->link, rg);
 557		coalesce_file_region(map, nrg);
 558	} else
 559		*regions_needed += 1;
 560
 561	return to - from;
 562}
 563
 564/*
 565 * Must be called with resv->lock held.
 566 *
 567 * Calling this with regions_needed != NULL will count the number of pages
 568 * to be added but will not modify the linked list. And regions_needed will
 569 * indicate the number of file_regions needed in the cache to carry out to add
 570 * the regions for this range.
 571 */
 572static long add_reservation_in_range(struct resv_map *resv, long f, long t,
 573				     struct hugetlb_cgroup *h_cg,
 574				     struct hstate *h, long *regions_needed)
 575{
 576	long add = 0;
 577	struct list_head *head = &resv->regions;
 578	long last_accounted_offset = f;
 579	struct file_region *iter, *trg = NULL;
 580	struct list_head *rg = NULL;
 581
 582	if (regions_needed)
 583		*regions_needed = 0;
 584
 585	/* In this loop, we essentially handle an entry for the range
 586	 * [last_accounted_offset, iter->from), at every iteration, with some
 587	 * bounds checking.
 588	 */
 589	list_for_each_entry_safe(iter, trg, head, link) {
 590		/* Skip irrelevant regions that start before our range. */
 591		if (iter->from < f) {
 592			/* If this region ends after the last accounted offset,
 593			 * then we need to update last_accounted_offset.
 594			 */
 595			if (iter->to > last_accounted_offset)
 596				last_accounted_offset = iter->to;
 597			continue;
 598		}
 599
 600		/* When we find a region that starts beyond our range, we've
 601		 * finished.
 602		 */
 603		if (iter->from >= t) {
 604			rg = iter->link.prev;
 605			break;
 606		}
 607
 608		/* Add an entry for last_accounted_offset -> iter->from, and
 609		 * update last_accounted_offset.
 610		 */
 611		if (iter->from > last_accounted_offset)
 612			add += hugetlb_resv_map_add(resv, iter->link.prev,
 613						    last_accounted_offset,
 614						    iter->from, h, h_cg,
 615						    regions_needed);
 616
 617		last_accounted_offset = iter->to;
 618	}
 619
 620	/* Handle the case where our range extends beyond
 621	 * last_accounted_offset.
 622	 */
 623	if (!rg)
 624		rg = head->prev;
 625	if (last_accounted_offset < t)
 626		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
 627					    t, h, h_cg, regions_needed);
 628
 
 629	return add;
 630}
 631
 632/* Must be called with resv->lock acquired. Will drop lock to allocate entries.
 633 */
 634static int allocate_file_region_entries(struct resv_map *resv,
 635					int regions_needed)
 636	__must_hold(&resv->lock)
 637{
 638	LIST_HEAD(allocated_regions);
 639	int to_allocate = 0, i = 0;
 640	struct file_region *trg = NULL, *rg = NULL;
 641
 642	VM_BUG_ON(regions_needed < 0);
 643
 
 
 644	/*
 645	 * Check for sufficient descriptors in the cache to accommodate
 646	 * the number of in progress add operations plus regions_needed.
 647	 *
 648	 * This is a while loop because when we drop the lock, some other call
 649	 * to region_add or region_del may have consumed some region_entries,
 650	 * so we keep looping here until we finally have enough entries for
 651	 * (adds_in_progress + regions_needed).
 652	 */
 653	while (resv->region_cache_count <
 654	       (resv->adds_in_progress + regions_needed)) {
 655		to_allocate = resv->adds_in_progress + regions_needed -
 656			      resv->region_cache_count;
 657
 658		/* At this point, we should have enough entries in the cache
 659		 * for all the existing adds_in_progress. We should only be
 660		 * needing to allocate for regions_needed.
 661		 */
 662		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
 663
 664		spin_unlock(&resv->lock);
 665		for (i = 0; i < to_allocate; i++) {
 666			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
 667			if (!trg)
 668				goto out_of_memory;
 669			list_add(&trg->link, &allocated_regions);
 670		}
 671
 672		spin_lock(&resv->lock);
 673
 674		list_splice(&allocated_regions, &resv->region_cache);
 675		resv->region_cache_count += to_allocate;
 676	}
 677
 678	return 0;
 679
 680out_of_memory:
 681	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
 682		list_del(&rg->link);
 683		kfree(rg);
 684	}
 685	return -ENOMEM;
 686}
 687
 688/*
 689 * Add the huge page range represented by [f, t) to the reserve
 690 * map.  Regions will be taken from the cache to fill in this range.
 691 * Sufficient regions should exist in the cache due to the previous
 692 * call to region_chg with the same range, but in some cases the cache will not
 693 * have sufficient entries due to races with other code doing region_add or
 694 * region_del.  The extra needed entries will be allocated.
 695 *
 696 * regions_needed is the out value provided by a previous call to region_chg.
 697 *
 698 * Return the number of new huge pages added to the map.  This number is greater
 699 * than or equal to zero.  If file_region entries needed to be allocated for
 700 * this operation and we were not able to allocate, it returns -ENOMEM.
 701 * region_add of regions of length 1 never allocate file_regions and cannot
 702 * fail; region_chg will always allocate at least 1 entry and a region_add for
 703 * 1 page will only require at most 1 entry.
 704 */
 705static long region_add(struct resv_map *resv, long f, long t,
 706		       long in_regions_needed, struct hstate *h,
 707		       struct hugetlb_cgroup *h_cg)
 708{
 709	long add = 0, actual_regions_needed = 0;
 710
 711	spin_lock(&resv->lock);
 712retry:
 713
 714	/* Count how many regions are actually needed to execute this add. */
 715	add_reservation_in_range(resv, f, t, NULL, NULL,
 716				 &actual_regions_needed);
 717
 718	/*
 719	 * Check for sufficient descriptors in the cache to accommodate
 720	 * this add operation. Note that actual_regions_needed may be greater
 721	 * than in_regions_needed, as the resv_map may have been modified since
 722	 * the region_chg call. In this case, we need to make sure that we
 723	 * allocate extra entries, such that we have enough for all the
 724	 * existing adds_in_progress, plus the excess needed for this
 725	 * operation.
 726	 */
 727	if (actual_regions_needed > in_regions_needed &&
 728	    resv->region_cache_count <
 729		    resv->adds_in_progress +
 730			    (actual_regions_needed - in_regions_needed)) {
 731		/* region_add operation of range 1 should never need to
 732		 * allocate file_region entries.
 733		 */
 734		VM_BUG_ON(t - f <= 1);
 735
 736		if (allocate_file_region_entries(
 737			    resv, actual_regions_needed - in_regions_needed)) {
 738			return -ENOMEM;
 739		}
 740
 741		goto retry;
 742	}
 743
 744	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
 745
 746	resv->adds_in_progress -= in_regions_needed;
 747
 748	spin_unlock(&resv->lock);
 749	return add;
 750}
 751
 752/*
 753 * Examine the existing reserve map and determine how many
 754 * huge pages in the specified range [f, t) are NOT currently
 755 * represented.  This routine is called before a subsequent
 756 * call to region_add that will actually modify the reserve
 757 * map to add the specified range [f, t).  region_chg does
 758 * not change the number of huge pages represented by the
 759 * map.  A number of new file_region structures is added to the cache as a
 760 * placeholder, for the subsequent region_add call to use. At least 1
 761 * file_region structure is added.
 762 *
 763 * out_regions_needed is the number of regions added to the
 764 * resv->adds_in_progress.  This value needs to be provided to a follow up call
 765 * to region_add or region_abort for proper accounting.
 766 *
 767 * Returns the number of huge pages that need to be added to the existing
 768 * reservation map for the range [f, t).  This number is greater or equal to
 769 * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 770 * is needed and can not be allocated.
 771 */
 772static long region_chg(struct resv_map *resv, long f, long t,
 773		       long *out_regions_needed)
 774{
 775	long chg = 0;
 776
 777	spin_lock(&resv->lock);
 778
 779	/* Count how many hugepages in this range are NOT represented. */
 780	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
 781				       out_regions_needed);
 782
 783	if (*out_regions_needed == 0)
 784		*out_regions_needed = 1;
 785
 786	if (allocate_file_region_entries(resv, *out_regions_needed))
 787		return -ENOMEM;
 788
 789	resv->adds_in_progress += *out_regions_needed;
 790
 791	spin_unlock(&resv->lock);
 792	return chg;
 793}
 794
 795/*
 796 * Abort the in progress add operation.  The adds_in_progress field
 797 * of the resv_map keeps track of the operations in progress between
 798 * calls to region_chg and region_add.  Operations are sometimes
 799 * aborted after the call to region_chg.  In such cases, region_abort
 800 * is called to decrement the adds_in_progress counter. regions_needed
 801 * is the value returned by the region_chg call, it is used to decrement
 802 * the adds_in_progress counter.
 803 *
 804 * NOTE: The range arguments [f, t) are not needed or used in this
 805 * routine.  They are kept to make reading the calling code easier as
 806 * arguments will match the associated region_chg call.
 807 */
 808static void region_abort(struct resv_map *resv, long f, long t,
 809			 long regions_needed)
 810{
 811	spin_lock(&resv->lock);
 812	VM_BUG_ON(!resv->region_cache_count);
 813	resv->adds_in_progress -= regions_needed;
 814	spin_unlock(&resv->lock);
 815}
 816
 817/*
 818 * Delete the specified range [f, t) from the reserve map.  If the
 819 * t parameter is LONG_MAX, this indicates that ALL regions after f
 820 * should be deleted.  Locate the regions which intersect [f, t)
 821 * and either trim, delete or split the existing regions.
 822 *
 823 * Returns the number of huge pages deleted from the reserve map.
 824 * In the normal case, the return value is zero or more.  In the
 825 * case where a region must be split, a new region descriptor must
 826 * be allocated.  If the allocation fails, -ENOMEM will be returned.
 827 * NOTE: If the parameter t == LONG_MAX, then we will never split
 828 * a region and possibly return -ENOMEM.  Callers specifying
 829 * t == LONG_MAX do not need to check for -ENOMEM error.
 830 */
 831static long region_del(struct resv_map *resv, long f, long t)
 832{
 833	struct list_head *head = &resv->regions;
 834	struct file_region *rg, *trg;
 835	struct file_region *nrg = NULL;
 836	long del = 0;
 837
 838retry:
 839	spin_lock(&resv->lock);
 840	list_for_each_entry_safe(rg, trg, head, link) {
 841		/*
 842		 * Skip regions before the range to be deleted.  file_region
 843		 * ranges are normally of the form [from, to).  However, there
 844		 * may be a "placeholder" entry in the map which is of the form
 845		 * (from, to) with from == to.  Check for placeholder entries
 846		 * at the beginning of the range to be deleted.
 847		 */
 848		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
 849			continue;
 850
 851		if (rg->from >= t)
 852			break;
 853
 854		if (f > rg->from && t < rg->to) { /* Must split region */
 855			/*
 856			 * Check for an entry in the cache before dropping
 857			 * lock and attempting allocation.
 858			 */
 859			if (!nrg &&
 860			    resv->region_cache_count > resv->adds_in_progress) {
 861				nrg = list_first_entry(&resv->region_cache,
 862							struct file_region,
 863							link);
 864				list_del(&nrg->link);
 865				resv->region_cache_count--;
 866			}
 867
 868			if (!nrg) {
 869				spin_unlock(&resv->lock);
 870				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 871				if (!nrg)
 872					return -ENOMEM;
 873				goto retry;
 874			}
 875
 876			del += t - f;
 877			hugetlb_cgroup_uncharge_file_region(
 878				resv, rg, t - f, false);
 879
 880			/* New entry for end of split region */
 881			nrg->from = t;
 882			nrg->to = rg->to;
 883
 884			copy_hugetlb_cgroup_uncharge_info(nrg, rg);
 885
 886			INIT_LIST_HEAD(&nrg->link);
 887
 888			/* Original entry is trimmed */
 889			rg->to = f;
 890
 891			list_add(&nrg->link, &rg->link);
 892			nrg = NULL;
 893			break;
 894		}
 895
 896		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
 897			del += rg->to - rg->from;
 898			hugetlb_cgroup_uncharge_file_region(resv, rg,
 899							    rg->to - rg->from, true);
 900			list_del(&rg->link);
 901			kfree(rg);
 902			continue;
 903		}
 904
 905		if (f <= rg->from) {	/* Trim beginning of region */
 906			hugetlb_cgroup_uncharge_file_region(resv, rg,
 907							    t - rg->from, false);
 908
 909			del += t - rg->from;
 910			rg->from = t;
 911		} else {		/* Trim end of region */
 912			hugetlb_cgroup_uncharge_file_region(resv, rg,
 913							    rg->to - f, false);
 914
 915			del += rg->to - f;
 916			rg->to = f;
 917		}
 918	}
 919
 920	spin_unlock(&resv->lock);
 921	kfree(nrg);
 922	return del;
 923}
 924
 925/*
 926 * A rare out of memory error was encountered which prevented removal of
 927 * the reserve map region for a page.  The huge page itself was free'ed
 928 * and removed from the page cache.  This routine will adjust the subpool
 929 * usage count, and the global reserve count if needed.  By incrementing
 930 * these counts, the reserve map entry which could not be deleted will
 931 * appear as a "reserved" entry instead of simply dangling with incorrect
 932 * counts.
 933 */
 934void hugetlb_fix_reserve_counts(struct inode *inode)
 935{
 936	struct hugepage_subpool *spool = subpool_inode(inode);
 937	long rsv_adjust;
 938	bool reserved = false;
 939
 940	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
 941	if (rsv_adjust > 0) {
 942		struct hstate *h = hstate_inode(inode);
 943
 944		if (!hugetlb_acct_memory(h, 1))
 945			reserved = true;
 946	} else if (!rsv_adjust) {
 947		reserved = true;
 948	}
 949
 950	if (!reserved)
 951		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
 952}
 953
 954/*
 955 * Count and return the number of huge pages in the reserve map
 956 * that intersect with the range [f, t).
 957 */
 958static long region_count(struct resv_map *resv, long f, long t)
 959{
 960	struct list_head *head = &resv->regions;
 961	struct file_region *rg;
 962	long chg = 0;
 963
 964	spin_lock(&resv->lock);
 965	/* Locate each segment we overlap with, and count that overlap. */
 966	list_for_each_entry(rg, head, link) {
 967		long seg_from;
 968		long seg_to;
 969
 970		if (rg->to <= f)
 971			continue;
 972		if (rg->from >= t)
 973			break;
 974
 975		seg_from = max(rg->from, f);
 976		seg_to = min(rg->to, t);
 977
 978		chg += seg_to - seg_from;
 979	}
 980	spin_unlock(&resv->lock);
 981
 982	return chg;
 983}
 984
 985/*
 986 * Convert the address within this vma to the page offset within
 987 * the mapping, huge page units here.
 988 */
 989static pgoff_t vma_hugecache_offset(struct hstate *h,
 990			struct vm_area_struct *vma, unsigned long address)
 991{
 992	return ((address - vma->vm_start) >> huge_page_shift(h)) +
 993			(vma->vm_pgoff >> huge_page_order(h));
 994}
 995
 996/**
 997 * vma_kernel_pagesize - Page size granularity for this VMA.
 998 * @vma: The user mapping.
 999 *
1000 * Folios in this VMA will be aligned to, and at least the size of the
1001 * number of bytes returned by this function.
1002 *
1003 * Return: The default size of the folios allocated when backing a VMA.
 
 
1004 */
1005unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
1006{
1007	if (vma->vm_ops && vma->vm_ops->pagesize)
1008		return vma->vm_ops->pagesize(vma);
1009	return PAGE_SIZE;
1010}
1011EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
1012
1013/*
1014 * Return the page size being used by the MMU to back a VMA. In the majority
1015 * of cases, the page size used by the kernel matches the MMU size. On
1016 * architectures where it differs, an architecture-specific 'strong'
1017 * version of this symbol is required.
1018 */
1019__weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
1020{
1021	return vma_kernel_pagesize(vma);
1022}
1023
1024/*
1025 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
1026 * bits of the reservation map pointer, which are always clear due to
1027 * alignment.
1028 */
1029#define HPAGE_RESV_OWNER    (1UL << 0)
1030#define HPAGE_RESV_UNMAPPED (1UL << 1)
1031#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
1032
1033/*
1034 * These helpers are used to track how many pages are reserved for
1035 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
1036 * is guaranteed to have their future faults succeed.
1037 *
1038 * With the exception of hugetlb_dup_vma_private() which is called at fork(),
1039 * the reserve counters are updated with the hugetlb_lock held. It is safe
1040 * to reset the VMA at fork() time as it is not in use yet and there is no
1041 * chance of the global counters getting corrupted as a result of the values.
1042 *
1043 * The private mapping reservation is represented in a subtly different
1044 * manner to a shared mapping.  A shared mapping has a region map associated
1045 * with the underlying file, this region map represents the backing file
1046 * pages which have ever had a reservation assigned which this persists even
1047 * after the page is instantiated.  A private mapping has a region map
1048 * associated with the original mmap which is attached to all VMAs which
1049 * reference it, this region map represents those offsets which have consumed
1050 * reservation ie. where pages have been instantiated.
1051 */
1052static unsigned long get_vma_private_data(struct vm_area_struct *vma)
1053{
1054	return (unsigned long)vma->vm_private_data;
1055}
1056
1057static void set_vma_private_data(struct vm_area_struct *vma,
1058							unsigned long value)
1059{
1060	vma->vm_private_data = (void *)value;
1061}
1062
1063static void
1064resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
1065					  struct hugetlb_cgroup *h_cg,
1066					  struct hstate *h)
1067{
1068#ifdef CONFIG_CGROUP_HUGETLB
1069	if (!h_cg || !h) {
1070		resv_map->reservation_counter = NULL;
1071		resv_map->pages_per_hpage = 0;
1072		resv_map->css = NULL;
1073	} else {
1074		resv_map->reservation_counter =
1075			&h_cg->rsvd_hugepage[hstate_index(h)];
1076		resv_map->pages_per_hpage = pages_per_huge_page(h);
1077		resv_map->css = &h_cg->css;
1078	}
1079#endif
1080}
1081
1082struct resv_map *resv_map_alloc(void)
1083{
1084	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
1085	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
1086
1087	if (!resv_map || !rg) {
1088		kfree(resv_map);
1089		kfree(rg);
1090		return NULL;
1091	}
1092
1093	kref_init(&resv_map->refs);
1094	spin_lock_init(&resv_map->lock);
1095	INIT_LIST_HEAD(&resv_map->regions);
1096	init_rwsem(&resv_map->rw_sema);
1097
1098	resv_map->adds_in_progress = 0;
1099	/*
1100	 * Initialize these to 0. On shared mappings, 0's here indicate these
1101	 * fields don't do cgroup accounting. On private mappings, these will be
1102	 * re-initialized to the proper values, to indicate that hugetlb cgroup
1103	 * reservations are to be un-charged from here.
1104	 */
1105	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
1106
1107	INIT_LIST_HEAD(&resv_map->region_cache);
1108	list_add(&rg->link, &resv_map->region_cache);
1109	resv_map->region_cache_count = 1;
1110
1111	return resv_map;
1112}
1113
1114void resv_map_release(struct kref *ref)
1115{
1116	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
1117	struct list_head *head = &resv_map->region_cache;
1118	struct file_region *rg, *trg;
1119
1120	/* Clear out any active regions before we release the map. */
1121	region_del(resv_map, 0, LONG_MAX);
1122
1123	/* ... and any entries left in the cache */
1124	list_for_each_entry_safe(rg, trg, head, link) {
1125		list_del(&rg->link);
1126		kfree(rg);
1127	}
1128
1129	VM_BUG_ON(resv_map->adds_in_progress);
1130
1131	kfree(resv_map);
1132}
1133
1134static inline struct resv_map *inode_resv_map(struct inode *inode)
1135{
1136	/*
1137	 * At inode evict time, i_mapping may not point to the original
1138	 * address space within the inode.  This original address space
1139	 * contains the pointer to the resv_map.  So, always use the
1140	 * address space embedded within the inode.
1141	 * The VERY common case is inode->mapping == &inode->i_data but,
1142	 * this may not be true for device special inodes.
1143	 */
1144	return (struct resv_map *)(&inode->i_data)->i_private_data;
1145}
1146
1147static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
1148{
1149	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1150	if (vma->vm_flags & VM_MAYSHARE) {
1151		struct address_space *mapping = vma->vm_file->f_mapping;
1152		struct inode *inode = mapping->host;
1153
1154		return inode_resv_map(inode);
1155
1156	} else {
1157		return (struct resv_map *)(get_vma_private_data(vma) &
1158							~HPAGE_RESV_MASK);
1159	}
1160}
1161
1162static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
1163{
1164	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1165	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1166
1167	set_vma_private_data(vma, (unsigned long)map);
 
1168}
1169
1170static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
1171{
1172	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1173	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1174
1175	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
1176}
1177
1178static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
1179{
1180	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1181
1182	return (get_vma_private_data(vma) & flag) != 0;
1183}
1184
1185bool __vma_private_lock(struct vm_area_struct *vma)
1186{
1187	return !(vma->vm_flags & VM_MAYSHARE) &&
1188		get_vma_private_data(vma) & ~HPAGE_RESV_MASK &&
1189		is_vma_resv_set(vma, HPAGE_RESV_OWNER);
1190}
1191
1192void hugetlb_dup_vma_private(struct vm_area_struct *vma)
1193{
1194	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1195	/*
1196	 * Clear vm_private_data
1197	 * - For shared mappings this is a per-vma semaphore that may be
1198	 *   allocated in a subsequent call to hugetlb_vm_op_open.
1199	 *   Before clearing, make sure pointer is not associated with vma
1200	 *   as this will leak the structure.  This is the case when called
1201	 *   via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already
1202	 *   been called to allocate a new structure.
1203	 * - For MAP_PRIVATE mappings, this is the reserve map which does
1204	 *   not apply to children.  Faults generated by the children are
1205	 *   not guaranteed to succeed, even if read-only.
1206	 */
1207	if (vma->vm_flags & VM_MAYSHARE) {
1208		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
1209
1210		if (vma_lock && vma_lock->vma != vma)
1211			vma->vm_private_data = NULL;
1212	} else
1213		vma->vm_private_data = NULL;
1214}
1215
1216/*
1217 * Reset and decrement one ref on hugepage private reservation.
1218 * Called with mm->mmap_lock writer semaphore held.
1219 * This function should be only used by move_vma() and operate on
1220 * same sized vma. It should never come here with last ref on the
1221 * reservation.
1222 */
1223void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
1224{
1225	/*
1226	 * Clear the old hugetlb private page reservation.
1227	 * It has already been transferred to new_vma.
1228	 *
1229	 * During a mremap() operation of a hugetlb vma we call move_vma()
1230	 * which copies vma into new_vma and unmaps vma. After the copy
1231	 * operation both new_vma and vma share a reference to the resv_map
1232	 * struct, and at that point vma is about to be unmapped. We don't
1233	 * want to return the reservation to the pool at unmap of vma because
1234	 * the reservation still lives on in new_vma, so simply decrement the
1235	 * ref here and remove the resv_map reference from this vma.
1236	 */
1237	struct resv_map *reservations = vma_resv_map(vma);
1238
1239	if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1240		resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
1241		kref_put(&reservations->refs, resv_map_release);
1242	}
1243
1244	hugetlb_dup_vma_private(vma);
1245}
1246
1247/* Returns true if the VMA has associated reserve pages */
1248static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1249{
1250	if (vma->vm_flags & VM_NORESERVE) {
1251		/*
1252		 * This address is already reserved by other process(chg == 0),
1253		 * so, we should decrement reserved count. Without decrementing,
1254		 * reserve count remains after releasing inode, because this
1255		 * allocated page will go into page cache and is regarded as
1256		 * coming from reserved pool in releasing step.  Currently, we
1257		 * don't have any other solution to deal with this situation
1258		 * properly, so add work-around here.
1259		 */
1260		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1261			return true;
1262		else
1263			return false;
1264	}
1265
1266	/* Shared mappings always use reserves */
1267	if (vma->vm_flags & VM_MAYSHARE) {
1268		/*
1269		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1270		 * be a region map for all pages.  The only situation where
1271		 * there is no region map is if a hole was punched via
1272		 * fallocate.  In this case, there really are no reserves to
1273		 * use.  This situation is indicated if chg != 0.
1274		 */
1275		if (chg)
1276			return false;
1277		else
1278			return true;
1279	}
1280
1281	/*
1282	 * Only the process that called mmap() has reserves for
1283	 * private mappings.
1284	 */
1285	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1286		/*
1287		 * Like the shared case above, a hole punch or truncate
1288		 * could have been performed on the private mapping.
1289		 * Examine the value of chg to determine if reserves
1290		 * actually exist or were previously consumed.
1291		 * Very Subtle - The value of chg comes from a previous
1292		 * call to vma_needs_reserves().  The reserve map for
1293		 * private mappings has different (opposite) semantics
1294		 * than that of shared mappings.  vma_needs_reserves()
1295		 * has already taken this difference in semantics into
1296		 * account.  Therefore, the meaning of chg is the same
1297		 * as in the shared case above.  Code could easily be
1298		 * combined, but keeping it separate draws attention to
1299		 * subtle differences.
1300		 */
1301		if (chg)
1302			return false;
1303		else
1304			return true;
1305	}
1306
1307	return false;
1308}
1309
1310static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
1311{
1312	int nid = folio_nid(folio);
1313
1314	lockdep_assert_held(&hugetlb_lock);
1315	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1316
1317	list_move(&folio->lru, &h->hugepage_freelists[nid]);
1318	h->free_huge_pages++;
1319	h->free_huge_pages_node[nid]++;
1320	folio_set_hugetlb_freed(folio);
1321}
1322
1323static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h,
1324								int nid)
1325{
1326	struct folio *folio;
1327	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1328
1329	lockdep_assert_held(&hugetlb_lock);
1330	list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) {
1331		if (pin && !folio_is_longterm_pinnable(folio))
1332			continue;
1333
1334		if (folio_test_hwpoison(folio))
1335			continue;
1336
1337		list_move(&folio->lru, &h->hugepage_activelist);
1338		folio_ref_unfreeze(folio, 1);
1339		folio_clear_hugetlb_freed(folio);
1340		h->free_huge_pages--;
1341		h->free_huge_pages_node[nid]--;
1342		return folio;
1343	}
1344
1345	return NULL;
1346}
1347
1348static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_mask,
1349							int nid, nodemask_t *nmask)
1350{
1351	unsigned int cpuset_mems_cookie;
1352	struct zonelist *zonelist;
1353	struct zone *zone;
1354	struct zoneref *z;
1355	int node = NUMA_NO_NODE;
1356
1357	zonelist = node_zonelist(nid, gfp_mask);
1358
1359retry_cpuset:
1360	cpuset_mems_cookie = read_mems_allowed_begin();
1361	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1362		struct folio *folio;
1363
1364		if (!cpuset_zone_allowed(zone, gfp_mask))
1365			continue;
1366		/*
1367		 * no need to ask again on the same node. Pool is node rather than
1368		 * zone aware
1369		 */
1370		if (zone_to_nid(zone) == node)
1371			continue;
1372		node = zone_to_nid(zone);
1373
1374		folio = dequeue_hugetlb_folio_node_exact(h, node);
1375		if (folio)
1376			return folio;
1377	}
1378	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1379		goto retry_cpuset;
1380
1381	return NULL;
1382}
1383
1384static unsigned long available_huge_pages(struct hstate *h)
1385{
1386	return h->free_huge_pages - h->resv_huge_pages;
1387}
1388
1389static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h,
1390				struct vm_area_struct *vma,
1391				unsigned long address, int avoid_reserve,
1392				long chg)
1393{
1394	struct folio *folio = NULL;
1395	struct mempolicy *mpol;
1396	gfp_t gfp_mask;
1397	nodemask_t *nodemask;
1398	int nid;
1399
1400	/*
1401	 * A child process with MAP_PRIVATE mappings created by their parent
1402	 * have no page reserves. This check ensures that reservations are
1403	 * not "stolen". The child may still get SIGKILLed
1404	 */
1405	if (!vma_has_reserves(vma, chg) && !available_huge_pages(h))
 
1406		goto err;
1407
1408	/* If reserves cannot be used, ensure enough pages are in the pool */
1409	if (avoid_reserve && !available_huge_pages(h))
1410		goto err;
1411
1412	gfp_mask = htlb_alloc_mask(h);
1413	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1414
1415	if (mpol_is_preferred_many(mpol)) {
1416		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
1417							nid, nodemask);
1418
1419		/* Fallback to all nodes if page==NULL */
1420		nodemask = NULL;
1421	}
1422
1423	if (!folio)
1424		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
1425							nid, nodemask);
1426
1427	if (folio && !avoid_reserve && vma_has_reserves(vma, chg)) {
1428		folio_set_hugetlb_restore_reserve(folio);
1429		h->resv_huge_pages--;
1430	}
1431
1432	mpol_cond_put(mpol);
1433	return folio;
1434
1435err:
1436	return NULL;
1437}
1438
1439/*
1440 * common helper functions for hstate_next_node_to_{alloc|free}.
1441 * We may have allocated or freed a huge page based on a different
1442 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1443 * be outside of *nodes_allowed.  Ensure that we use an allowed
1444 * node for alloc or free.
1445 */
1446static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1447{
1448	nid = next_node_in(nid, *nodes_allowed);
1449	VM_BUG_ON(nid >= MAX_NUMNODES);
1450
1451	return nid;
1452}
1453
1454static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1455{
1456	if (!node_isset(nid, *nodes_allowed))
1457		nid = next_node_allowed(nid, nodes_allowed);
1458	return nid;
1459}
1460
1461/*
1462 * returns the previously saved node ["this node"] from which to
1463 * allocate a persistent huge page for the pool and advance the
1464 * next node from which to allocate, handling wrap at end of node
1465 * mask.
1466 */
1467static int hstate_next_node_to_alloc(struct hstate *h,
1468					nodemask_t *nodes_allowed)
1469{
1470	int nid;
1471
1472	VM_BUG_ON(!nodes_allowed);
1473
1474	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1475	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1476
1477	return nid;
1478}
1479
1480/*
1481 * helper for remove_pool_hugetlb_folio() - return the previously saved
1482 * node ["this node"] from which to free a huge page.  Advance the
1483 * next node id whether or not we find a free huge page to free so
1484 * that the next attempt to free addresses the next node.
1485 */
1486static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1487{
1488	int nid;
1489
1490	VM_BUG_ON(!nodes_allowed);
1491
1492	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1493	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1494
1495	return nid;
1496}
1497
1498#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
1499	for (nr_nodes = nodes_weight(*mask);				\
1500		nr_nodes > 0 &&						\
1501		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
1502		nr_nodes--)
1503
1504#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1505	for (nr_nodes = nodes_weight(*mask);				\
1506		nr_nodes > 0 &&						\
1507		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1508		nr_nodes--)
1509
1510/* used to demote non-gigantic_huge pages as well */
1511static void __destroy_compound_gigantic_folio(struct folio *folio,
1512					unsigned int order, bool demote)
1513{
1514	int i;
1515	int nr_pages = 1 << order;
1516	struct page *p;
1517
1518	atomic_set(&folio->_entire_mapcount, 0);
1519	atomic_set(&folio->_nr_pages_mapped, 0);
1520	atomic_set(&folio->_pincount, 0);
1521
1522	for (i = 1; i < nr_pages; i++) {
1523		p = folio_page(folio, i);
1524		p->flags &= ~PAGE_FLAGS_CHECK_AT_FREE;
1525		p->mapping = NULL;
1526		clear_compound_head(p);
1527		if (!demote)
1528			set_page_refcounted(p);
1529	}
1530
1531	__folio_clear_head(folio);
1532}
1533
1534static void destroy_compound_hugetlb_folio_for_demote(struct folio *folio,
1535					unsigned int order)
1536{
1537	__destroy_compound_gigantic_folio(folio, order, true);
1538}
1539
1540#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1541static void destroy_compound_gigantic_folio(struct folio *folio,
1542					unsigned int order)
1543{
1544	__destroy_compound_gigantic_folio(folio, order, false);
1545}
1546
1547static void free_gigantic_folio(struct folio *folio, unsigned int order)
1548{
1549	/*
1550	 * If the page isn't allocated using the cma allocator,
1551	 * cma_release() returns false.
1552	 */
1553#ifdef CONFIG_CMA
1554	int nid = folio_nid(folio);
1555
1556	if (cma_release(hugetlb_cma[nid], &folio->page, 1 << order))
1557		return;
1558#endif
1559
1560	free_contig_range(folio_pfn(folio), 1 << order);
1561}
1562
1563#ifdef CONFIG_CONTIG_ALLOC
1564static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1565		int nid, nodemask_t *nodemask)
1566{
1567	struct page *page;
1568	unsigned long nr_pages = pages_per_huge_page(h);
1569	if (nid == NUMA_NO_NODE)
1570		nid = numa_mem_id();
1571
1572#ifdef CONFIG_CMA
1573	{
 
1574		int node;
1575
1576		if (hugetlb_cma[nid]) {
1577			page = cma_alloc(hugetlb_cma[nid], nr_pages,
1578					huge_page_order(h), true);
1579			if (page)
1580				return page_folio(page);
1581		}
1582
1583		if (!(gfp_mask & __GFP_THISNODE)) {
1584			for_each_node_mask(node, *nodemask) {
1585				if (node == nid || !hugetlb_cma[node])
1586					continue;
1587
1588				page = cma_alloc(hugetlb_cma[node], nr_pages,
1589						huge_page_order(h), true);
1590				if (page)
1591					return page_folio(page);
1592			}
1593		}
1594	}
1595#endif
1596
1597	page = alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
1598	return page ? page_folio(page) : NULL;
1599}
1600
1601#else /* !CONFIG_CONTIG_ALLOC */
1602static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1603					int nid, nodemask_t *nodemask)
1604{
1605	return NULL;
1606}
1607#endif /* CONFIG_CONTIG_ALLOC */
1608
1609#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1610static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1611					int nid, nodemask_t *nodemask)
1612{
1613	return NULL;
1614}
1615static inline void free_gigantic_folio(struct folio *folio,
1616						unsigned int order) { }
1617static inline void destroy_compound_gigantic_folio(struct folio *folio,
1618						unsigned int order) { }
1619#endif
1620
1621static inline void __clear_hugetlb_destructor(struct hstate *h,
1622						struct folio *folio)
1623{
1624	lockdep_assert_held(&hugetlb_lock);
1625
1626	folio_clear_hugetlb(folio);
1627}
1628
1629/*
1630 * Remove hugetlb folio from lists.
1631 * If vmemmap exists for the folio, update dtor so that the folio appears
1632 * as just a compound page.  Otherwise, wait until after allocating vmemmap
1633 * to update dtor.
1634 *
1635 * A reference is held on the folio, except in the case of demote.
1636 *
1637 * Must be called with hugetlb lock held.
1638 */
1639static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1640							bool adjust_surplus,
1641							bool demote)
1642{
1643	int nid = folio_nid(folio);
1644
1645	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
1646	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
1647
1648	lockdep_assert_held(&hugetlb_lock);
1649	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1650		return;
1651
1652	list_del(&folio->lru);
1653
1654	if (folio_test_hugetlb_freed(folio)) {
1655		h->free_huge_pages--;
1656		h->free_huge_pages_node[nid]--;
1657	}
1658	if (adjust_surplus) {
1659		h->surplus_huge_pages--;
1660		h->surplus_huge_pages_node[nid]--;
1661	}
1662
1663	/*
1664	 * We can only clear the hugetlb destructor after allocating vmemmap
1665	 * pages.  Otherwise, someone (memory error handling) may try to write
1666	 * to tail struct pages.
1667	 */
1668	if (!folio_test_hugetlb_vmemmap_optimized(folio))
1669		__clear_hugetlb_destructor(h, folio);
1670
1671	 /*
1672	  * In the case of demote we do not ref count the page as it will soon
1673	  * be turned into a page of smaller size.
1674	 */
1675	if (!demote)
1676		folio_ref_unfreeze(folio, 1);
1677
1678	h->nr_huge_pages--;
1679	h->nr_huge_pages_node[nid]--;
1680}
1681
1682static void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1683							bool adjust_surplus)
1684{
1685	__remove_hugetlb_folio(h, folio, adjust_surplus, false);
1686}
1687
1688static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *folio,
1689							bool adjust_surplus)
1690{
1691	__remove_hugetlb_folio(h, folio, adjust_surplus, true);
1692}
1693
1694static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
1695			     bool adjust_surplus)
1696{
1697	int zeroed;
1698	int nid = folio_nid(folio);
1699
1700	VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
1701
1702	lockdep_assert_held(&hugetlb_lock);
1703
1704	INIT_LIST_HEAD(&folio->lru);
1705	h->nr_huge_pages++;
1706	h->nr_huge_pages_node[nid]++;
1707
1708	if (adjust_surplus) {
1709		h->surplus_huge_pages++;
1710		h->surplus_huge_pages_node[nid]++;
1711	}
1712
1713	folio_set_hugetlb(folio);
1714	folio_change_private(folio, NULL);
1715	/*
1716	 * We have to set hugetlb_vmemmap_optimized again as above
1717	 * folio_change_private(folio, NULL) cleared it.
1718	 */
1719	folio_set_hugetlb_vmemmap_optimized(folio);
1720
1721	/*
1722	 * This folio is about to be managed by the hugetlb allocator and
1723	 * should have no users.  Drop our reference, and check for others
1724	 * just in case.
1725	 */
1726	zeroed = folio_put_testzero(folio);
1727	if (unlikely(!zeroed))
1728		/*
1729		 * It is VERY unlikely soneone else has taken a ref
1730		 * on the folio.  In this case, we simply return as
1731		 * free_huge_folio() will be called when this other ref
1732		 * is dropped.
1733		 */
1734		return;
1735
1736	arch_clear_hugepage_flags(&folio->page);
1737	enqueue_hugetlb_folio(h, folio);
1738}
1739
1740static void __update_and_free_hugetlb_folio(struct hstate *h,
1741						struct folio *folio)
1742{
1743	bool clear_dtor = folio_test_hugetlb_vmemmap_optimized(folio);
 
1744
1745	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1746		return;
1747
1748	/*
1749	 * If we don't know which subpages are hwpoisoned, we can't free
1750	 * the hugepage, so it's leaked intentionally.
1751	 */
1752	if (folio_test_hugetlb_raw_hwp_unreliable(folio))
1753		return;
1754
1755	/*
1756	 * If folio is not vmemmap optimized (!clear_dtor), then the folio
1757	 * is no longer identified as a hugetlb page.  hugetlb_vmemmap_restore_folio
1758	 * can only be passed hugetlb pages and will BUG otherwise.
1759	 */
1760	if (clear_dtor && hugetlb_vmemmap_restore_folio(h, folio)) {
1761		spin_lock_irq(&hugetlb_lock);
1762		/*
1763		 * If we cannot allocate vmemmap pages, just refuse to free the
1764		 * page and put the page back on the hugetlb free list and treat
1765		 * as a surplus page.
1766		 */
1767		add_hugetlb_folio(h, folio, true);
1768		spin_unlock_irq(&hugetlb_lock);
1769		return;
1770	}
1771
1772	/*
1773	 * Move PageHWPoison flag from head page to the raw error pages,
1774	 * which makes any healthy subpages reusable.
1775	 */
1776	if (unlikely(folio_test_hwpoison(folio)))
1777		folio_clear_hugetlb_hwpoison(folio);
1778
1779	/*
1780	 * If vmemmap pages were allocated above, then we need to clear the
1781	 * hugetlb destructor under the hugetlb lock.
1782	 */
1783	if (clear_dtor) {
1784		spin_lock_irq(&hugetlb_lock);
1785		__clear_hugetlb_destructor(h, folio);
1786		spin_unlock_irq(&hugetlb_lock);
1787	}
1788
1789	/*
1790	 * Non-gigantic pages demoted from CMA allocated gigantic pages
1791	 * need to be given back to CMA in free_gigantic_folio.
1792	 */
1793	if (hstate_is_gigantic(h) ||
1794	    hugetlb_cma_folio(folio, huge_page_order(h))) {
1795		destroy_compound_gigantic_folio(folio, huge_page_order(h));
1796		free_gigantic_folio(folio, huge_page_order(h));
1797	} else {
1798		__free_pages(&folio->page, huge_page_order(h));
1799	}
1800}
1801
1802/*
1803 * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
1804 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
1805 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
1806 * the vmemmap pages.
1807 *
1808 * free_hpage_workfn() locklessly retrieves the linked list of pages to be
1809 * freed and frees them one-by-one. As the page->mapping pointer is going
1810 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
1811 * structure of a lockless linked list of huge pages to be freed.
1812 */
1813static LLIST_HEAD(hpage_freelist);
1814
1815static void free_hpage_workfn(struct work_struct *work)
1816{
1817	struct llist_node *node;
1818
1819	node = llist_del_all(&hpage_freelist);
1820
1821	while (node) {
1822		struct folio *folio;
1823		struct hstate *h;
1824
1825		folio = container_of((struct address_space **)node,
1826				     struct folio, mapping);
1827		node = node->next;
1828		folio->mapping = NULL;
1829		/*
1830		 * The VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio) in
1831		 * folio_hstate() is going to trigger because a previous call to
1832		 * remove_hugetlb_folio() will clear the hugetlb bit, so do
1833		 * not use folio_hstate() directly.
1834		 */
1835		h = size_to_hstate(folio_size(folio));
1836
1837		__update_and_free_hugetlb_folio(h, folio);
1838
1839		cond_resched();
1840	}
1841}
1842static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1843
1844static inline void flush_free_hpage_work(struct hstate *h)
1845{
1846	if (hugetlb_vmemmap_optimizable(h))
1847		flush_work(&free_hpage_work);
1848}
1849
1850static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
1851				 bool atomic)
1852{
1853	if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
1854		__update_and_free_hugetlb_folio(h, folio);
1855		return;
1856	}
1857
1858	/*
1859	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
1860	 *
1861	 * Only call schedule_work() if hpage_freelist is previously
1862	 * empty. Otherwise, schedule_work() had been called but the workfn
1863	 * hasn't retrieved the list yet.
1864	 */
1865	if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
1866		schedule_work(&free_hpage_work);
1867}
1868
1869static void bulk_vmemmap_restore_error(struct hstate *h,
1870					struct list_head *folio_list,
1871					struct list_head *non_hvo_folios)
1872{
1873	struct folio *folio, *t_folio;
1874
1875	if (!list_empty(non_hvo_folios)) {
1876		/*
1877		 * Free any restored hugetlb pages so that restore of the
1878		 * entire list can be retried.
1879		 * The idea is that in the common case of ENOMEM errors freeing
1880		 * hugetlb pages with vmemmap we will free up memory so that we
1881		 * can allocate vmemmap for more hugetlb pages.
1882		 */
1883		list_for_each_entry_safe(folio, t_folio, non_hvo_folios, lru) {
1884			list_del(&folio->lru);
1885			spin_lock_irq(&hugetlb_lock);
1886			__clear_hugetlb_destructor(h, folio);
1887			spin_unlock_irq(&hugetlb_lock);
1888			update_and_free_hugetlb_folio(h, folio, false);
1889			cond_resched();
1890		}
1891	} else {
1892		/*
1893		 * In the case where there are no folios which can be
1894		 * immediately freed, we loop through the list trying to restore
1895		 * vmemmap individually in the hope that someone elsewhere may
1896		 * have done something to cause success (such as freeing some
1897		 * memory).  If unable to restore a hugetlb page, the hugetlb
1898		 * page is made a surplus page and removed from the list.
1899		 * If are able to restore vmemmap and free one hugetlb page, we
1900		 * quit processing the list to retry the bulk operation.
1901		 */
1902		list_for_each_entry_safe(folio, t_folio, folio_list, lru)
1903			if (hugetlb_vmemmap_restore_folio(h, folio)) {
1904				list_del(&folio->lru);
1905				spin_lock_irq(&hugetlb_lock);
1906				add_hugetlb_folio(h, folio, true);
1907				spin_unlock_irq(&hugetlb_lock);
1908			} else {
1909				list_del(&folio->lru);
1910				spin_lock_irq(&hugetlb_lock);
1911				__clear_hugetlb_destructor(h, folio);
1912				spin_unlock_irq(&hugetlb_lock);
1913				update_and_free_hugetlb_folio(h, folio, false);
1914				cond_resched();
1915				break;
1916			}
1917	}
1918}
1919
1920static void update_and_free_pages_bulk(struct hstate *h,
1921						struct list_head *folio_list)
1922{
1923	long ret;
1924	struct folio *folio, *t_folio;
1925	LIST_HEAD(non_hvo_folios);
1926
1927	/*
1928	 * First allocate required vmemmmap (if necessary) for all folios.
1929	 * Carefully handle errors and free up any available hugetlb pages
1930	 * in an effort to make forward progress.
1931	 */
1932retry:
1933	ret = hugetlb_vmemmap_restore_folios(h, folio_list, &non_hvo_folios);
1934	if (ret < 0) {
1935		bulk_vmemmap_restore_error(h, folio_list, &non_hvo_folios);
1936		goto retry;
1937	}
1938
1939	/*
1940	 * At this point, list should be empty, ret should be >= 0 and there
1941	 * should only be pages on the non_hvo_folios list.
1942	 * Do note that the non_hvo_folios list could be empty.
1943	 * Without HVO enabled, ret will be 0 and there is no need to call
1944	 * __clear_hugetlb_destructor as this was done previously.
1945	 */
1946	VM_WARN_ON(!list_empty(folio_list));
1947	VM_WARN_ON(ret < 0);
1948	if (!list_empty(&non_hvo_folios) && ret) {
1949		spin_lock_irq(&hugetlb_lock);
1950		list_for_each_entry(folio, &non_hvo_folios, lru)
1951			__clear_hugetlb_destructor(h, folio);
1952		spin_unlock_irq(&hugetlb_lock);
1953	}
1954
1955	list_for_each_entry_safe(folio, t_folio, &non_hvo_folios, lru) {
1956		update_and_free_hugetlb_folio(h, folio, false);
1957		cond_resched();
1958	}
1959}
1960
1961struct hstate *size_to_hstate(unsigned long size)
1962{
1963	struct hstate *h;
1964
1965	for_each_hstate(h) {
1966		if (huge_page_size(h) == size)
1967			return h;
1968	}
1969	return NULL;
1970}
1971
1972void free_huge_folio(struct folio *folio)
1973{
1974	/*
1975	 * Can't pass hstate in here because it is called from the
1976	 * compound page destructor.
1977	 */
1978	struct hstate *h = folio_hstate(folio);
1979	int nid = folio_nid(folio);
1980	struct hugepage_subpool *spool = hugetlb_folio_subpool(folio);
1981	bool restore_reserve;
1982	unsigned long flags;
1983
1984	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1985	VM_BUG_ON_FOLIO(folio_mapcount(folio), folio);
1986
1987	hugetlb_set_folio_subpool(folio, NULL);
1988	if (folio_test_anon(folio))
1989		__ClearPageAnonExclusive(&folio->page);
1990	folio->mapping = NULL;
1991	restore_reserve = folio_test_hugetlb_restore_reserve(folio);
1992	folio_clear_hugetlb_restore_reserve(folio);
1993
1994	/*
1995	 * If HPageRestoreReserve was set on page, page allocation consumed a
1996	 * reservation.  If the page was associated with a subpool, there
1997	 * would have been a page reserved in the subpool before allocation
1998	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1999	 * reservation, do not call hugepage_subpool_put_pages() as this will
2000	 * remove the reserved page from the subpool.
2001	 */
2002	if (!restore_reserve) {
2003		/*
2004		 * A return code of zero implies that the subpool will be
2005		 * under its minimum size if the reservation is not restored
2006		 * after page is free.  Therefore, force restore_reserve
2007		 * operation.
2008		 */
2009		if (hugepage_subpool_put_pages(spool, 1) == 0)
2010			restore_reserve = true;
2011	}
2012
2013	spin_lock_irqsave(&hugetlb_lock, flags);
2014	folio_clear_hugetlb_migratable(folio);
2015	hugetlb_cgroup_uncharge_folio(hstate_index(h),
2016				     pages_per_huge_page(h), folio);
2017	hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
2018					  pages_per_huge_page(h), folio);
2019	mem_cgroup_uncharge(folio);
2020	if (restore_reserve)
2021		h->resv_huge_pages++;
2022
2023	if (folio_test_hugetlb_temporary(folio)) {
2024		remove_hugetlb_folio(h, folio, false);
2025		spin_unlock_irqrestore(&hugetlb_lock, flags);
2026		update_and_free_hugetlb_folio(h, folio, true);
2027	} else if (h->surplus_huge_pages_node[nid]) {
2028		/* remove the page from active list */
2029		remove_hugetlb_folio(h, folio, true);
2030		spin_unlock_irqrestore(&hugetlb_lock, flags);
2031		update_and_free_hugetlb_folio(h, folio, true);
2032	} else {
2033		arch_clear_hugepage_flags(&folio->page);
2034		enqueue_hugetlb_folio(h, folio);
2035		spin_unlock_irqrestore(&hugetlb_lock, flags);
2036	}
2037}
2038
2039/*
2040 * Must be called with the hugetlb lock held
2041 */
2042static void __prep_account_new_huge_page(struct hstate *h, int nid)
2043{
2044	lockdep_assert_held(&hugetlb_lock);
2045	h->nr_huge_pages++;
2046	h->nr_huge_pages_node[nid]++;
2047}
2048
2049static void init_new_hugetlb_folio(struct hstate *h, struct folio *folio)
2050{
2051	folio_set_hugetlb(folio);
2052	INIT_LIST_HEAD(&folio->lru);
2053	hugetlb_set_folio_subpool(folio, NULL);
2054	set_hugetlb_cgroup(folio, NULL);
2055	set_hugetlb_cgroup_rsvd(folio, NULL);
 
2056}
2057
2058static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
2059{
2060	init_new_hugetlb_folio(h, folio);
2061	hugetlb_vmemmap_optimize_folio(h, folio);
2062}
2063
2064static void prep_new_hugetlb_folio(struct hstate *h, struct folio *folio, int nid)
2065{
2066	__prep_new_hugetlb_folio(h, folio);
2067	spin_lock_irq(&hugetlb_lock);
2068	__prep_account_new_huge_page(h, nid);
2069	spin_unlock_irq(&hugetlb_lock);
2070}
2071
2072static bool __prep_compound_gigantic_folio(struct folio *folio,
2073					unsigned int order, bool demote)
2074{
2075	int i, j;
2076	int nr_pages = 1 << order;
2077	struct page *p;
2078
2079	__folio_clear_reserved(folio);
2080	for (i = 0; i < nr_pages; i++) {
2081		p = folio_page(folio, i);
2082
 
 
 
 
 
2083		/*
2084		 * For gigantic hugepages allocated through bootmem at
2085		 * boot, it's safer to be consistent with the not-gigantic
2086		 * hugepages and clear the PG_reserved bit from all tail pages
2087		 * too.  Otherwise drivers using get_user_pages() to access tail
2088		 * pages may get the reference counting wrong if they see
2089		 * PG_reserved set on a tail page (despite the head page not
2090		 * having PG_reserved set).  Enforcing this consistency between
2091		 * head and tail pages allows drivers to optimize away a check
2092		 * on the head page when they need know if put_page() is needed
2093		 * after get_user_pages().
2094		 */
2095		if (i != 0)	/* head page cleared above */
2096			__ClearPageReserved(p);
2097		/*
2098		 * Subtle and very unlikely
2099		 *
2100		 * Gigantic 'page allocators' such as memblock or cma will
2101		 * return a set of pages with each page ref counted.  We need
2102		 * to turn this set of pages into a compound page with tail
2103		 * page ref counts set to zero.  Code such as speculative page
2104		 * cache adding could take a ref on a 'to be' tail page.
2105		 * We need to respect any increased ref count, and only set
2106		 * the ref count to zero if count is currently 1.  If count
2107		 * is not 1, we return an error.  An error return indicates
2108		 * the set of pages can not be converted to a gigantic page.
2109		 * The caller who allocated the pages should then discard the
2110		 * pages using the appropriate free interface.
2111		 *
2112		 * In the case of demote, the ref count will be zero.
2113		 */
2114		if (!demote) {
2115			if (!page_ref_freeze(p, 1)) {
2116				pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
2117				goto out_error;
2118			}
2119		} else {
2120			VM_BUG_ON_PAGE(page_count(p), p);
2121		}
2122		if (i != 0)
2123			set_compound_head(p, &folio->page);
2124	}
2125	__folio_set_head(folio);
2126	/* we rely on prep_new_hugetlb_folio to set the destructor */
2127	folio_set_order(folio, order);
2128	atomic_set(&folio->_entire_mapcount, -1);
2129	atomic_set(&folio->_nr_pages_mapped, 0);
2130	atomic_set(&folio->_pincount, 0);
2131	return true;
2132
2133out_error:
2134	/* undo page modifications made above */
2135	for (j = 0; j < i; j++) {
2136		p = folio_page(folio, j);
2137		if (j != 0)
2138			clear_compound_head(p);
2139		set_page_refcounted(p);
2140	}
2141	/* need to clear PG_reserved on remaining tail pages  */
2142	for (; j < nr_pages; j++) {
2143		p = folio_page(folio, j);
2144		__ClearPageReserved(p);
2145	}
 
 
2146	return false;
2147}
2148
2149static bool prep_compound_gigantic_folio(struct folio *folio,
2150							unsigned int order)
2151{
2152	return __prep_compound_gigantic_folio(folio, order, false);
2153}
2154
2155static bool prep_compound_gigantic_folio_for_demote(struct folio *folio,
2156							unsigned int order)
2157{
2158	return __prep_compound_gigantic_folio(folio, order, true);
2159}
2160
2161/*
2162 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
2163 * transparent huge pages.  See the PageTransHuge() documentation for more
2164 * details.
2165 */
2166int PageHuge(struct page *page)
2167{
2168	struct folio *folio;
2169
2170	if (!PageCompound(page))
2171		return 0;
2172	folio = page_folio(page);
2173	return folio_test_hugetlb(folio);
 
2174}
2175EXPORT_SYMBOL_GPL(PageHuge);
2176
2177/*
 
 
 
 
 
 
 
 
 
 
 
 
2178 * Find and lock address space (mapping) in write mode.
2179 *
2180 * Upon entry, the page is locked which means that page_mapping() is
2181 * stable.  Due to locking order, we can only trylock_write.  If we can
2182 * not get the lock, simply return NULL to caller.
2183 */
2184struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
2185{
2186	struct address_space *mapping = page_mapping(hpage);
2187
2188	if (!mapping)
2189		return mapping;
2190
2191	if (i_mmap_trylock_write(mapping))
2192		return mapping;
2193
2194	return NULL;
2195}
2196
2197static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2198		gfp_t gfp_mask, int nid, nodemask_t *nmask,
2199		nodemask_t *node_alloc_noretry)
2200{
2201	int order = huge_page_order(h);
2202	struct page *page;
2203	bool alloc_try_hard = true;
2204	bool retry = true;
2205
2206	/*
2207	 * By default we always try hard to allocate the page with
2208	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
2209	 * a loop (to adjust global huge page counts) and previous allocation
2210	 * failed, do not continue to try hard on the same node.  Use the
2211	 * node_alloc_noretry bitmap to manage this state information.
2212	 */
2213	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
2214		alloc_try_hard = false;
2215	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
2216	if (alloc_try_hard)
2217		gfp_mask |= __GFP_RETRY_MAYFAIL;
2218	if (nid == NUMA_NO_NODE)
2219		nid = numa_mem_id();
2220retry:
2221	page = __alloc_pages(gfp_mask, order, nid, nmask);
2222
2223	/* Freeze head page */
2224	if (page && !page_ref_freeze(page, 1)) {
2225		__free_pages(page, order);
2226		if (retry) {	/* retry once */
2227			retry = false;
2228			goto retry;
2229		}
2230		/* WOW!  twice in a row. */
2231		pr_warn("HugeTLB head page unexpected inflated ref count\n");
2232		page = NULL;
2233	}
2234
2235	/*
2236	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
2237	 * indicates an overall state change.  Clear bit so that we resume
2238	 * normal 'try hard' allocations.
2239	 */
2240	if (node_alloc_noretry && page && !alloc_try_hard)
2241		node_clear(nid, *node_alloc_noretry);
2242
2243	/*
2244	 * If we tried hard to get a page but failed, set bit so that
2245	 * subsequent attempts will not try as hard until there is an
2246	 * overall state change.
2247	 */
2248	if (node_alloc_noretry && !page && alloc_try_hard)
2249		node_set(nid, *node_alloc_noretry);
2250
2251	if (!page) {
2252		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
2253		return NULL;
2254	}
2255
2256	__count_vm_event(HTLB_BUDDY_PGALLOC);
2257	return page_folio(page);
2258}
2259
2260static struct folio *__alloc_fresh_hugetlb_folio(struct hstate *h,
2261				gfp_t gfp_mask, int nid, nodemask_t *nmask,
2262				nodemask_t *node_alloc_noretry)
 
 
 
 
2263{
2264	struct folio *folio;
2265	bool retry = false;
2266
2267retry:
2268	if (hstate_is_gigantic(h))
2269		folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask);
2270	else
2271		folio = alloc_buddy_hugetlb_folio(h, gfp_mask,
2272				nid, nmask, node_alloc_noretry);
2273	if (!folio)
2274		return NULL;
2275
2276	if (hstate_is_gigantic(h)) {
2277		if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) {
2278			/*
2279			 * Rare failure to convert pages to compound page.
2280			 * Free pages and try again - ONCE!
2281			 */
2282			free_gigantic_folio(folio, huge_page_order(h));
2283			if (!retry) {
2284				retry = true;
2285				goto retry;
2286			}
 
2287			return NULL;
2288		}
2289	}
 
2290
2291	return folio;
2292}
2293
2294static struct folio *only_alloc_fresh_hugetlb_folio(struct hstate *h,
2295		gfp_t gfp_mask, int nid, nodemask_t *nmask,
2296		nodemask_t *node_alloc_noretry)
2297{
2298	struct folio *folio;
2299
2300	folio = __alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask,
2301						node_alloc_noretry);
2302	if (folio)
2303		init_new_hugetlb_folio(h, folio);
2304	return folio;
2305}
2306
2307/*
2308 * Common helper to allocate a fresh hugetlb page. All specific allocators
2309 * should use this function to get new hugetlb pages
2310 *
2311 * Note that returned page is 'frozen':  ref count of head page and all tail
2312 * pages is zero.
2313 */
2314static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
2315		gfp_t gfp_mask, int nid, nodemask_t *nmask,
2316		nodemask_t *node_alloc_noretry)
2317{
2318	struct folio *folio;
 
 
2319
2320	folio = __alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask,
 
2321						node_alloc_noretry);
2322	if (!folio)
2323		return NULL;
2324
2325	prep_new_hugetlb_folio(h, folio, folio_nid(folio));
2326	return folio;
2327}
2328
2329static void prep_and_add_allocated_folios(struct hstate *h,
2330					struct list_head *folio_list)
2331{
2332	unsigned long flags;
2333	struct folio *folio, *tmp_f;
2334
2335	/* Send list for bulk vmemmap optimization processing */
2336	hugetlb_vmemmap_optimize_folios(h, folio_list);
2337
2338	/* Add all new pool pages to free lists in one lock cycle */
2339	spin_lock_irqsave(&hugetlb_lock, flags);
2340	list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
2341		__prep_account_new_huge_page(h, folio_nid(folio));
2342		enqueue_hugetlb_folio(h, folio);
2343	}
2344	spin_unlock_irqrestore(&hugetlb_lock, flags);
2345}
2346
2347/*
2348 * Allocates a fresh hugetlb page in a node interleaved manner.  The page
2349 * will later be added to the appropriate hugetlb pool.
2350 */
2351static struct folio *alloc_pool_huge_folio(struct hstate *h,
2352					nodemask_t *nodes_allowed,
2353					nodemask_t *node_alloc_noretry)
2354{
2355	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2356	int nr_nodes, node;
2357
2358	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2359		struct folio *folio;
2360
2361		folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, node,
2362					nodes_allowed, node_alloc_noretry);
2363		if (folio)
2364			return folio;
2365	}
2366
2367	return NULL;
2368}
2369
2370/*
2371 * Remove huge page from pool from next node to free.  Attempt to keep
2372 * persistent huge pages more or less balanced over allowed nodes.
2373 * This routine only 'removes' the hugetlb page.  The caller must make
2374 * an additional call to free the page to low level allocators.
2375 * Called with hugetlb_lock locked.
2376 */
2377static struct folio *remove_pool_hugetlb_folio(struct hstate *h,
2378		nodemask_t *nodes_allowed, bool acct_surplus)
 
2379{
2380	int nr_nodes, node;
2381	struct folio *folio = NULL;
2382
2383	lockdep_assert_held(&hugetlb_lock);
2384	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2385		/*
2386		 * If we're returning unused surplus pages, only examine
2387		 * nodes with surplus pages.
2388		 */
2389		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
2390		    !list_empty(&h->hugepage_freelists[node])) {
2391			folio = list_entry(h->hugepage_freelists[node].next,
2392					  struct folio, lru);
2393			remove_hugetlb_folio(h, folio, acct_surplus);
2394			break;
2395		}
2396	}
2397
2398	return folio;
2399}
2400
2401/*
2402 * Dissolve a given free hugepage into free buddy pages. This function does
2403 * nothing for in-use hugepages and non-hugepages.
2404 * This function returns values like below:
2405 *
2406 *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
2407 *           when the system is under memory pressure and the feature of
2408 *           freeing unused vmemmap pages associated with each hugetlb page
2409 *           is enabled.
2410 *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
2411 *           (allocated or reserved.)
2412 *       0:  successfully dissolved free hugepages or the page is not a
2413 *           hugepage (considered as already dissolved)
2414 */
2415int dissolve_free_huge_page(struct page *page)
2416{
2417	int rc = -EBUSY;
2418	struct folio *folio = page_folio(page);
2419
2420retry:
2421	/* Not to disrupt normal path by vainly holding hugetlb_lock */
2422	if (!folio_test_hugetlb(folio))
2423		return 0;
2424
2425	spin_lock_irq(&hugetlb_lock);
2426	if (!folio_test_hugetlb(folio)) {
2427		rc = 0;
2428		goto out;
2429	}
2430
2431	if (!folio_ref_count(folio)) {
2432		struct hstate *h = folio_hstate(folio);
2433		if (!available_huge_pages(h))
 
2434			goto out;
2435
2436		/*
2437		 * We should make sure that the page is already on the free list
2438		 * when it is dissolved.
2439		 */
2440		if (unlikely(!folio_test_hugetlb_freed(folio))) {
2441			spin_unlock_irq(&hugetlb_lock);
2442			cond_resched();
2443
2444			/*
2445			 * Theoretically, we should return -EBUSY when we
2446			 * encounter this race. In fact, we have a chance
2447			 * to successfully dissolve the page if we do a
2448			 * retry. Because the race window is quite small.
2449			 * If we seize this opportunity, it is an optimization
2450			 * for increasing the success rate of dissolving page.
2451			 */
2452			goto retry;
2453		}
2454
2455		remove_hugetlb_folio(h, folio, false);
2456		h->max_huge_pages--;
2457		spin_unlock_irq(&hugetlb_lock);
2458
2459		/*
2460		 * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
2461		 * before freeing the page.  update_and_free_hugtlb_folio will fail to
2462		 * free the page if it can not allocate required vmemmap.  We
2463		 * need to adjust max_huge_pages if the page is not freed.
2464		 * Attempt to allocate vmemmmap here so that we can take
2465		 * appropriate action on failure.
2466		 *
2467		 * The folio_test_hugetlb check here is because
2468		 * remove_hugetlb_folio will clear hugetlb folio flag for
2469		 * non-vmemmap optimized hugetlb folios.
2470		 */
2471		if (folio_test_hugetlb(folio)) {
2472			rc = hugetlb_vmemmap_restore_folio(h, folio);
2473			if (rc) {
2474				spin_lock_irq(&hugetlb_lock);
2475				add_hugetlb_folio(h, folio, false);
2476				h->max_huge_pages++;
2477				goto out;
2478			}
2479		} else
2480			rc = 0;
 
 
 
 
 
2481
2482		update_and_free_hugetlb_folio(h, folio, false);
2483		return rc;
2484	}
2485out:
2486	spin_unlock_irq(&hugetlb_lock);
2487	return rc;
2488}
2489
2490/*
2491 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
2492 * make specified memory blocks removable from the system.
2493 * Note that this will dissolve a free gigantic hugepage completely, if any
2494 * part of it lies within the given range.
2495 * Also note that if dissolve_free_huge_page() returns with an error, all
2496 * free hugepages that were dissolved before that error are lost.
2497 */
2498int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
2499{
2500	unsigned long pfn;
2501	struct page *page;
2502	int rc = 0;
2503	unsigned int order;
2504	struct hstate *h;
2505
2506	if (!hugepages_supported())
2507		return rc;
2508
2509	order = huge_page_order(&default_hstate);
2510	for_each_hstate(h)
2511		order = min(order, huge_page_order(h));
2512
2513	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) {
2514		page = pfn_to_page(pfn);
2515		rc = dissolve_free_huge_page(page);
2516		if (rc)
2517			break;
2518	}
2519
2520	return rc;
2521}
2522
2523/*
2524 * Allocates a fresh surplus page from the page allocator.
2525 */
2526static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h,
2527				gfp_t gfp_mask,	int nid, nodemask_t *nmask)
2528{
2529	struct folio *folio = NULL;
2530
2531	if (hstate_is_gigantic(h))
2532		return NULL;
2533
2534	spin_lock_irq(&hugetlb_lock);
2535	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
2536		goto out_unlock;
2537	spin_unlock_irq(&hugetlb_lock);
2538
2539	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
2540	if (!folio)
2541		return NULL;
2542
2543	spin_lock_irq(&hugetlb_lock);
2544	/*
2545	 * We could have raced with the pool size change.
2546	 * Double check that and simply deallocate the new page
2547	 * if we would end up overcommiting the surpluses. Abuse
2548	 * temporary page to workaround the nasty free_huge_folio
2549	 * codeflow
2550	 */
2551	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2552		folio_set_hugetlb_temporary(folio);
2553		spin_unlock_irq(&hugetlb_lock);
2554		free_huge_folio(folio);
2555		return NULL;
 
 
 
2556	}
2557
2558	h->surplus_huge_pages++;
2559	h->surplus_huge_pages_node[folio_nid(folio)]++;
2560
2561out_unlock:
2562	spin_unlock_irq(&hugetlb_lock);
2563
2564	return folio;
2565}
2566
2567static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
2568				     int nid, nodemask_t *nmask)
2569{
2570	struct folio *folio;
2571
2572	if (hstate_is_gigantic(h))
2573		return NULL;
2574
2575	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
2576	if (!folio)
2577		return NULL;
2578
2579	/* fresh huge pages are frozen */
2580	folio_ref_unfreeze(folio, 1);
2581	/*
2582	 * We do not account these pages as surplus because they are only
2583	 * temporary and will be released properly on the last reference
2584	 */
2585	folio_set_hugetlb_temporary(folio);
2586
2587	return folio;
2588}
2589
2590/*
2591 * Use the VMA's mpolicy to allocate a huge page from the buddy.
2592 */
2593static
2594struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h,
2595		struct vm_area_struct *vma, unsigned long addr)
2596{
2597	struct folio *folio = NULL;
2598	struct mempolicy *mpol;
2599	gfp_t gfp_mask = htlb_alloc_mask(h);
2600	int nid;
2601	nodemask_t *nodemask;
2602
2603	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2604	if (mpol_is_preferred_many(mpol)) {
2605		gfp_t gfp = gfp_mask | __GFP_NOWARN;
2606
2607		gfp &=  ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2608		folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask);
2609
2610		/* Fallback to all nodes if page==NULL */
2611		nodemask = NULL;
2612	}
2613
2614	if (!folio)
2615		folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask);
2616	mpol_cond_put(mpol);
2617	return folio;
 
2618}
2619
2620/* folio migration callback function */
2621struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
2622		nodemask_t *nmask, gfp_t gfp_mask)
2623{
2624	spin_lock_irq(&hugetlb_lock);
2625	if (available_huge_pages(h)) {
2626		struct folio *folio;
2627
2628		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
2629						preferred_nid, nmask);
2630		if (folio) {
2631			spin_unlock_irq(&hugetlb_lock);
2632			return folio;
2633		}
2634	}
2635	spin_unlock_irq(&hugetlb_lock);
2636
2637	return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2638}
2639
2640/*
2641 * Increase the hugetlb pool such that it can accommodate a reservation
2642 * of size 'delta'.
2643 */
2644static int gather_surplus_pages(struct hstate *h, long delta)
2645	__must_hold(&hugetlb_lock)
2646{
2647	LIST_HEAD(surplus_list);
2648	struct folio *folio, *tmp;
2649	int ret;
2650	long i;
2651	long needed, allocated;
2652	bool alloc_ok = true;
2653
2654	lockdep_assert_held(&hugetlb_lock);
2655	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2656	if (needed <= 0) {
2657		h->resv_huge_pages += delta;
2658		return 0;
2659	}
2660
2661	allocated = 0;
 
2662
2663	ret = -ENOMEM;
2664retry:
2665	spin_unlock_irq(&hugetlb_lock);
2666	for (i = 0; i < needed; i++) {
2667		folio = alloc_surplus_hugetlb_folio(h, htlb_alloc_mask(h),
2668				NUMA_NO_NODE, NULL);
2669		if (!folio) {
2670			alloc_ok = false;
2671			break;
2672		}
2673		list_add(&folio->lru, &surplus_list);
2674		cond_resched();
2675	}
2676	allocated += i;
2677
2678	/*
2679	 * After retaking hugetlb_lock, we need to recalculate 'needed'
2680	 * because either resv_huge_pages or free_huge_pages may have changed.
2681	 */
2682	spin_lock_irq(&hugetlb_lock);
2683	needed = (h->resv_huge_pages + delta) -
2684			(h->free_huge_pages + allocated);
2685	if (needed > 0) {
2686		if (alloc_ok)
2687			goto retry;
2688		/*
2689		 * We were not able to allocate enough pages to
2690		 * satisfy the entire reservation so we free what
2691		 * we've allocated so far.
2692		 */
2693		goto free;
2694	}
2695	/*
2696	 * The surplus_list now contains _at_least_ the number of extra pages
2697	 * needed to accommodate the reservation.  Add the appropriate number
2698	 * of pages to the hugetlb pool and free the extras back to the buddy
2699	 * allocator.  Commit the entire reservation here to prevent another
2700	 * process from stealing the pages as they are added to the pool but
2701	 * before they are reserved.
2702	 */
2703	needed += allocated;
2704	h->resv_huge_pages += delta;
2705	ret = 0;
2706
2707	/* Free the needed pages to the hugetlb pool */
2708	list_for_each_entry_safe(folio, tmp, &surplus_list, lru) {
 
 
2709		if ((--needed) < 0)
2710			break;
2711		/* Add the page to the hugetlb allocator */
2712		enqueue_hugetlb_folio(h, folio);
 
 
 
 
 
2713	}
2714free:
2715	spin_unlock_irq(&hugetlb_lock);
2716
2717	/*
2718	 * Free unnecessary surplus pages to the buddy allocator.
2719	 * Pages have no ref count, call free_huge_folio directly.
2720	 */
2721	list_for_each_entry_safe(folio, tmp, &surplus_list, lru)
2722		free_huge_folio(folio);
2723	spin_lock_irq(&hugetlb_lock);
2724
2725	return ret;
2726}
2727
2728/*
2729 * This routine has two main purposes:
2730 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2731 *    in unused_resv_pages.  This corresponds to the prior adjustments made
2732 *    to the associated reservation map.
2733 * 2) Free any unused surplus pages that may have been allocated to satisfy
2734 *    the reservation.  As many as unused_resv_pages may be freed.
2735 */
2736static void return_unused_surplus_pages(struct hstate *h,
2737					unsigned long unused_resv_pages)
2738{
2739	unsigned long nr_pages;
 
2740	LIST_HEAD(page_list);
2741
2742	lockdep_assert_held(&hugetlb_lock);
2743	/* Uncommit the reservation */
2744	h->resv_huge_pages -= unused_resv_pages;
2745
2746	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
 
2747		goto out;
2748
2749	/*
2750	 * Part (or even all) of the reservation could have been backed
2751	 * by pre-allocated pages. Only free surplus pages.
2752	 */
2753	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2754
2755	/*
2756	 * We want to release as many surplus pages as possible, spread
2757	 * evenly across all nodes with memory. Iterate across these nodes
2758	 * until we can no longer free unreserved surplus pages. This occurs
2759	 * when the nodes with surplus pages have no free pages.
2760	 * remove_pool_hugetlb_folio() will balance the freed pages across the
2761	 * on-line nodes with memory and will handle the hstate accounting.
2762	 */
2763	while (nr_pages--) {
2764		struct folio *folio;
2765
2766		folio = remove_pool_hugetlb_folio(h, &node_states[N_MEMORY], 1);
2767		if (!folio)
2768			goto out;
2769
2770		list_add(&folio->lru, &page_list);
2771	}
2772
2773out:
2774	spin_unlock_irq(&hugetlb_lock);
2775	update_and_free_pages_bulk(h, &page_list);
2776	spin_lock_irq(&hugetlb_lock);
2777}
2778
2779
2780/*
2781 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2782 * are used by the huge page allocation routines to manage reservations.
2783 *
2784 * vma_needs_reservation is called to determine if the huge page at addr
2785 * within the vma has an associated reservation.  If a reservation is
2786 * needed, the value 1 is returned.  The caller is then responsible for
2787 * managing the global reservation and subpool usage counts.  After
2788 * the huge page has been allocated, vma_commit_reservation is called
2789 * to add the page to the reservation map.  If the page allocation fails,
2790 * the reservation must be ended instead of committed.  vma_end_reservation
2791 * is called in such cases.
2792 *
2793 * In the normal case, vma_commit_reservation returns the same value
2794 * as the preceding vma_needs_reservation call.  The only time this
2795 * is not the case is if a reserve map was changed between calls.  It
2796 * is the responsibility of the caller to notice the difference and
2797 * take appropriate action.
2798 *
2799 * vma_add_reservation is used in error paths where a reservation must
2800 * be restored when a newly allocated huge page must be freed.  It is
2801 * to be called after calling vma_needs_reservation to determine if a
2802 * reservation exists.
2803 *
2804 * vma_del_reservation is used in error paths where an entry in the reserve
2805 * map was created during huge page allocation and must be removed.  It is to
2806 * be called after calling vma_needs_reservation to determine if a reservation
2807 * exists.
2808 */
2809enum vma_resv_mode {
2810	VMA_NEEDS_RESV,
2811	VMA_COMMIT_RESV,
2812	VMA_END_RESV,
2813	VMA_ADD_RESV,
2814	VMA_DEL_RESV,
2815};
2816static long __vma_reservation_common(struct hstate *h,
2817				struct vm_area_struct *vma, unsigned long addr,
2818				enum vma_resv_mode mode)
2819{
2820	struct resv_map *resv;
2821	pgoff_t idx;
2822	long ret;
2823	long dummy_out_regions_needed;
2824
2825	resv = vma_resv_map(vma);
2826	if (!resv)
2827		return 1;
2828
2829	idx = vma_hugecache_offset(h, vma, addr);
2830	switch (mode) {
2831	case VMA_NEEDS_RESV:
2832		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2833		/* We assume that vma_reservation_* routines always operate on
2834		 * 1 page, and that adding to resv map a 1 page entry can only
2835		 * ever require 1 region.
2836		 */
2837		VM_BUG_ON(dummy_out_regions_needed != 1);
2838		break;
2839	case VMA_COMMIT_RESV:
2840		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2841		/* region_add calls of range 1 should never fail. */
2842		VM_BUG_ON(ret < 0);
2843		break;
2844	case VMA_END_RESV:
2845		region_abort(resv, idx, idx + 1, 1);
2846		ret = 0;
2847		break;
2848	case VMA_ADD_RESV:
2849		if (vma->vm_flags & VM_MAYSHARE) {
2850			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2851			/* region_add calls of range 1 should never fail. */
2852			VM_BUG_ON(ret < 0);
2853		} else {
2854			region_abort(resv, idx, idx + 1, 1);
2855			ret = region_del(resv, idx, idx + 1);
2856		}
2857		break;
2858	case VMA_DEL_RESV:
2859		if (vma->vm_flags & VM_MAYSHARE) {
2860			region_abort(resv, idx, idx + 1, 1);
2861			ret = region_del(resv, idx, idx + 1);
2862		} else {
2863			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2864			/* region_add calls of range 1 should never fail. */
2865			VM_BUG_ON(ret < 0);
2866		}
2867		break;
2868	default:
2869		BUG();
2870	}
2871
2872	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2873		return ret;
2874	/*
2875	 * We know private mapping must have HPAGE_RESV_OWNER set.
2876	 *
2877	 * In most cases, reserves always exist for private mappings.
2878	 * However, a file associated with mapping could have been
2879	 * hole punched or truncated after reserves were consumed.
2880	 * As subsequent fault on such a range will not use reserves.
2881	 * Subtle - The reserve map for private mappings has the
2882	 * opposite meaning than that of shared mappings.  If NO
2883	 * entry is in the reserve map, it means a reservation exists.
2884	 * If an entry exists in the reserve map, it means the
2885	 * reservation has already been consumed.  As a result, the
2886	 * return value of this routine is the opposite of the
2887	 * value returned from reserve map manipulation routines above.
2888	 */
2889	if (ret > 0)
2890		return 0;
2891	if (ret == 0)
2892		return 1;
2893	return ret;
2894}
2895
2896static long vma_needs_reservation(struct hstate *h,
2897			struct vm_area_struct *vma, unsigned long addr)
2898{
2899	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2900}
2901
2902static long vma_commit_reservation(struct hstate *h,
2903			struct vm_area_struct *vma, unsigned long addr)
2904{
2905	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2906}
2907
2908static void vma_end_reservation(struct hstate *h,
2909			struct vm_area_struct *vma, unsigned long addr)
2910{
2911	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2912}
2913
2914static long vma_add_reservation(struct hstate *h,
2915			struct vm_area_struct *vma, unsigned long addr)
2916{
2917	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2918}
2919
2920static long vma_del_reservation(struct hstate *h,
2921			struct vm_area_struct *vma, unsigned long addr)
2922{
2923	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
2924}
2925
2926/*
2927 * This routine is called to restore reservation information on error paths.
2928 * It should ONLY be called for folios allocated via alloc_hugetlb_folio(),
2929 * and the hugetlb mutex should remain held when calling this routine.
2930 *
2931 * It handles two specific cases:
2932 * 1) A reservation was in place and the folio consumed the reservation.
2933 *    hugetlb_restore_reserve is set in the folio.
2934 * 2) No reservation was in place for the page, so hugetlb_restore_reserve is
2935 *    not set.  However, alloc_hugetlb_folio always updates the reserve map.
2936 *
2937 * In case 1, free_huge_folio later in the error path will increment the
2938 * global reserve count.  But, free_huge_folio does not have enough context
2939 * to adjust the reservation map.  This case deals primarily with private
2940 * mappings.  Adjust the reserve map here to be consistent with global
2941 * reserve count adjustments to be made by free_huge_folio.  Make sure the
2942 * reserve map indicates there is a reservation present.
2943 *
2944 * In case 2, simply undo reserve map modifications done by alloc_hugetlb_folio.
2945 */
2946void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2947			unsigned long address, struct folio *folio)
2948{
2949	long rc = vma_needs_reservation(h, vma, address);
2950
2951	if (folio_test_hugetlb_restore_reserve(folio)) {
2952		if (unlikely(rc < 0))
2953			/*
2954			 * Rare out of memory condition in reserve map
2955			 * manipulation.  Clear hugetlb_restore_reserve so
2956			 * that global reserve count will not be incremented
2957			 * by free_huge_folio.  This will make it appear
2958			 * as though the reservation for this folio was
2959			 * consumed.  This may prevent the task from
2960			 * faulting in the folio at a later time.  This
2961			 * is better than inconsistent global huge page
2962			 * accounting of reserve counts.
2963			 */
2964			folio_clear_hugetlb_restore_reserve(folio);
2965		else if (rc)
2966			(void)vma_add_reservation(h, vma, address);
2967		else
2968			vma_end_reservation(h, vma, address);
2969	} else {
2970		if (!rc) {
2971			/*
2972			 * This indicates there is an entry in the reserve map
2973			 * not added by alloc_hugetlb_folio.  We know it was added
2974			 * before the alloc_hugetlb_folio call, otherwise
2975			 * hugetlb_restore_reserve would be set on the folio.
2976			 * Remove the entry so that a subsequent allocation
2977			 * does not consume a reservation.
2978			 */
2979			rc = vma_del_reservation(h, vma, address);
2980			if (rc < 0)
2981				/*
2982				 * VERY rare out of memory condition.  Since
2983				 * we can not delete the entry, set
2984				 * hugetlb_restore_reserve so that the reserve
2985				 * count will be incremented when the folio
2986				 * is freed.  This reserve will be consumed
2987				 * on a subsequent allocation.
2988				 */
2989				folio_set_hugetlb_restore_reserve(folio);
2990		} else if (rc < 0) {
2991			/*
2992			 * Rare out of memory condition from
2993			 * vma_needs_reservation call.  Memory allocation is
2994			 * only attempted if a new entry is needed.  Therefore,
2995			 * this implies there is not an entry in the
2996			 * reserve map.
2997			 *
2998			 * For shared mappings, no entry in the map indicates
2999			 * no reservation.  We are done.
3000			 */
3001			if (!(vma->vm_flags & VM_MAYSHARE))
3002				/*
3003				 * For private mappings, no entry indicates
3004				 * a reservation is present.  Since we can
3005				 * not add an entry, set hugetlb_restore_reserve
3006				 * on the folio so reserve count will be
3007				 * incremented when freed.  This reserve will
3008				 * be consumed on a subsequent allocation.
3009				 */
3010				folio_set_hugetlb_restore_reserve(folio);
3011		} else
3012			/*
3013			 * No reservation present, do nothing
3014			 */
3015			 vma_end_reservation(h, vma, address);
3016	}
3017}
3018
3019/*
3020 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
3021 * the old one
3022 * @h: struct hstate old page belongs to
3023 * @old_folio: Old folio to dissolve
3024 * @list: List to isolate the page in case we need to
3025 * Returns 0 on success, otherwise negated error.
3026 */
3027static int alloc_and_dissolve_hugetlb_folio(struct hstate *h,
3028			struct folio *old_folio, struct list_head *list)
3029{
3030	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
3031	int nid = folio_nid(old_folio);
3032	struct folio *new_folio;
3033	int ret = 0;
3034
3035	/*
3036	 * Before dissolving the folio, we need to allocate a new one for the
3037	 * pool to remain stable.  Here, we allocate the folio and 'prep' it
3038	 * by doing everything but actually updating counters and adding to
3039	 * the pool.  This simplifies and let us do most of the processing
3040	 * under the lock.
3041	 */
3042	new_folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid, NULL, NULL);
3043	if (!new_folio)
3044		return -ENOMEM;
3045	__prep_new_hugetlb_folio(h, new_folio);
3046
3047retry:
3048	spin_lock_irq(&hugetlb_lock);
3049	if (!folio_test_hugetlb(old_folio)) {
3050		/*
3051		 * Freed from under us. Drop new_folio too.
3052		 */
3053		goto free_new;
3054	} else if (folio_ref_count(old_folio)) {
3055		bool isolated;
3056
3057		/*
3058		 * Someone has grabbed the folio, try to isolate it here.
3059		 * Fail with -EBUSY if not possible.
3060		 */
3061		spin_unlock_irq(&hugetlb_lock);
3062		isolated = isolate_hugetlb(old_folio, list);
3063		ret = isolated ? 0 : -EBUSY;
3064		spin_lock_irq(&hugetlb_lock);
3065		goto free_new;
3066	} else if (!folio_test_hugetlb_freed(old_folio)) {
3067		/*
3068		 * Folio's refcount is 0 but it has not been enqueued in the
3069		 * freelist yet. Race window is small, so we can succeed here if
3070		 * we retry.
3071		 */
3072		spin_unlock_irq(&hugetlb_lock);
3073		cond_resched();
3074		goto retry;
3075	} else {
3076		/*
3077		 * Ok, old_folio is still a genuine free hugepage. Remove it from
3078		 * the freelist and decrease the counters. These will be
3079		 * incremented again when calling __prep_account_new_huge_page()
3080		 * and enqueue_hugetlb_folio() for new_folio. The counters will
3081		 * remain stable since this happens under the lock.
3082		 */
3083		remove_hugetlb_folio(h, old_folio, false);
3084
3085		/*
3086		 * Ref count on new_folio is already zero as it was dropped
3087		 * earlier.  It can be directly added to the pool free list.
3088		 */
3089		__prep_account_new_huge_page(h, nid);
3090		enqueue_hugetlb_folio(h, new_folio);
 
3091
3092		/*
3093		 * Folio has been replaced, we can safely free the old one.
3094		 */
3095		spin_unlock_irq(&hugetlb_lock);
3096		update_and_free_hugetlb_folio(h, old_folio, false);
3097	}
3098
3099	return ret;
3100
3101free_new:
3102	spin_unlock_irq(&hugetlb_lock);
3103	/* Folio has a zero ref count, but needs a ref to be freed */
3104	folio_ref_unfreeze(new_folio, 1);
3105	update_and_free_hugetlb_folio(h, new_folio, false);
3106
3107	return ret;
3108}
3109
3110int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
3111{
3112	struct hstate *h;
3113	struct folio *folio = page_folio(page);
3114	int ret = -EBUSY;
3115
3116	/*
3117	 * The page might have been dissolved from under our feet, so make sure
3118	 * to carefully check the state under the lock.
3119	 * Return success when racing as if we dissolved the page ourselves.
3120	 */
3121	spin_lock_irq(&hugetlb_lock);
3122	if (folio_test_hugetlb(folio)) {
3123		h = folio_hstate(folio);
 
3124	} else {
3125		spin_unlock_irq(&hugetlb_lock);
3126		return 0;
3127	}
3128	spin_unlock_irq(&hugetlb_lock);
3129
3130	/*
3131	 * Fence off gigantic pages as there is a cyclic dependency between
3132	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
3133	 * of bailing out right away without further retrying.
3134	 */
3135	if (hstate_is_gigantic(h))
3136		return -ENOMEM;
3137
3138	if (folio_ref_count(folio) && isolate_hugetlb(folio, list))
3139		ret = 0;
3140	else if (!folio_ref_count(folio))
3141		ret = alloc_and_dissolve_hugetlb_folio(h, folio, list);
3142
3143	return ret;
3144}
3145
3146struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
3147				    unsigned long addr, int avoid_reserve)
3148{
3149	struct hugepage_subpool *spool = subpool_vma(vma);
3150	struct hstate *h = hstate_vma(vma);
3151	struct folio *folio;
3152	long map_chg, map_commit, nr_pages = pages_per_huge_page(h);
3153	long gbl_chg;
3154	int memcg_charge_ret, ret, idx;
3155	struct hugetlb_cgroup *h_cg = NULL;
3156	struct mem_cgroup *memcg;
3157	bool deferred_reserve;
3158	gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL;
3159
3160	memcg = get_mem_cgroup_from_current();
3161	memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages);
3162	if (memcg_charge_ret == -ENOMEM) {
3163		mem_cgroup_put(memcg);
3164		return ERR_PTR(-ENOMEM);
3165	}
3166
3167	idx = hstate_index(h);
3168	/*
3169	 * Examine the region/reserve map to determine if the process
3170	 * has a reservation for the page to be allocated.  A return
3171	 * code of zero indicates a reservation exists (no change).
3172	 */
3173	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
3174	if (map_chg < 0) {
3175		if (!memcg_charge_ret)
3176			mem_cgroup_cancel_charge(memcg, nr_pages);
3177		mem_cgroup_put(memcg);
3178		return ERR_PTR(-ENOMEM);
3179	}
3180
3181	/*
3182	 * Processes that did not create the mapping will have no
3183	 * reserves as indicated by the region/reserve map. Check
3184	 * that the allocation will not exceed the subpool limit.
3185	 * Allocations for MAP_NORESERVE mappings also need to be
3186	 * checked against any subpool limit.
3187	 */
3188	if (map_chg || avoid_reserve) {
3189		gbl_chg = hugepage_subpool_get_pages(spool, 1);
3190		if (gbl_chg < 0)
3191			goto out_end_reservation;
 
 
3192
3193		/*
3194		 * Even though there was no reservation in the region/reserve
3195		 * map, there could be reservations associated with the
3196		 * subpool that can be used.  This would be indicated if the
3197		 * return value of hugepage_subpool_get_pages() is zero.
3198		 * However, if avoid_reserve is specified we still avoid even
3199		 * the subpool reservations.
3200		 */
3201		if (avoid_reserve)
3202			gbl_chg = 1;
3203	}
3204
3205	/* If this allocation is not consuming a reservation, charge it now.
3206	 */
3207	deferred_reserve = map_chg || avoid_reserve;
3208	if (deferred_reserve) {
3209		ret = hugetlb_cgroup_charge_cgroup_rsvd(
3210			idx, pages_per_huge_page(h), &h_cg);
3211		if (ret)
3212			goto out_subpool_put;
3213	}
3214
3215	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
3216	if (ret)
3217		goto out_uncharge_cgroup_reservation;
3218
3219	spin_lock_irq(&hugetlb_lock);
3220	/*
3221	 * glb_chg is passed to indicate whether or not a page must be taken
3222	 * from the global free pool (global change).  gbl_chg == 0 indicates
3223	 * a reservation exists for the allocation.
3224	 */
3225	folio = dequeue_hugetlb_folio_vma(h, vma, addr, avoid_reserve, gbl_chg);
3226	if (!folio) {
3227		spin_unlock_irq(&hugetlb_lock);
3228		folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr);
3229		if (!folio)
3230			goto out_uncharge_cgroup;
3231		spin_lock_irq(&hugetlb_lock);
3232		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
3233			folio_set_hugetlb_restore_reserve(folio);
3234			h->resv_huge_pages--;
3235		}
3236		list_add(&folio->lru, &h->hugepage_activelist);
3237		folio_ref_unfreeze(folio, 1);
3238		/* Fall through */
3239	}
3240
3241	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio);
3242	/* If allocation is not consuming a reservation, also store the
3243	 * hugetlb_cgroup pointer on the page.
3244	 */
3245	if (deferred_reserve) {
3246		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
3247						  h_cg, folio);
3248	}
3249
3250	spin_unlock_irq(&hugetlb_lock);
3251
3252	hugetlb_set_folio_subpool(folio, spool);
3253
3254	map_commit = vma_commit_reservation(h, vma, addr);
3255	if (unlikely(map_chg > map_commit)) {
3256		/*
3257		 * The page was added to the reservation map between
3258		 * vma_needs_reservation and vma_commit_reservation.
3259		 * This indicates a race with hugetlb_reserve_pages.
3260		 * Adjust for the subpool count incremented above AND
3261		 * in hugetlb_reserve_pages for the same page.  Also,
3262		 * the reservation count added in hugetlb_reserve_pages
3263		 * no longer applies.
3264		 */
3265		long rsv_adjust;
3266
3267		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
3268		hugetlb_acct_memory(h, -rsv_adjust);
3269		if (deferred_reserve)
3270			hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
3271					pages_per_huge_page(h), folio);
3272	}
3273
3274	if (!memcg_charge_ret)
3275		mem_cgroup_commit_charge(folio, memcg);
3276	mem_cgroup_put(memcg);
3277
3278	return folio;
3279
3280out_uncharge_cgroup:
3281	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
3282out_uncharge_cgroup_reservation:
3283	if (deferred_reserve)
3284		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
3285						    h_cg);
3286out_subpool_put:
3287	if (map_chg || avoid_reserve)
3288		hugepage_subpool_put_pages(spool, 1);
3289out_end_reservation:
3290	vma_end_reservation(h, vma, addr);
3291	if (!memcg_charge_ret)
3292		mem_cgroup_cancel_charge(memcg, nr_pages);
3293	mem_cgroup_put(memcg);
3294	return ERR_PTR(-ENOSPC);
3295}
3296
3297int alloc_bootmem_huge_page(struct hstate *h, int nid)
3298	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
3299int __alloc_bootmem_huge_page(struct hstate *h, int nid)
3300{
3301	struct huge_bootmem_page *m = NULL; /* initialize for clang */
3302	int nr_nodes, node;
3303
3304	/* do node specific alloc */
3305	if (nid != NUMA_NO_NODE) {
3306		m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
3307				0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
3308		if (!m)
3309			return 0;
3310		goto found;
3311	}
3312	/* allocate from next node when distributing huge pages */
3313	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
3314		m = memblock_alloc_try_nid_raw(
 
 
3315				huge_page_size(h), huge_page_size(h),
3316				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3317		/*
3318		 * Use the beginning of the huge page to store the
3319		 * huge_bootmem_page struct (until gather_bootmem
3320		 * puts them into the mem_map).
3321		 */
3322		if (!m)
3323			return 0;
3324		goto found;
 
3325	}
 
3326
3327found:
3328
3329	/*
3330	 * Only initialize the head struct page in memmap_init_reserved_pages,
3331	 * rest of the struct pages will be initialized by the HugeTLB
3332	 * subsystem itself.
3333	 * The head struct page is used to get folio information by the HugeTLB
3334	 * subsystem like zone id and node id.
3335	 */
3336	memblock_reserved_mark_noinit(virt_to_phys((void *)m + PAGE_SIZE),
3337		huge_page_size(h) - PAGE_SIZE);
3338	/* Put them into a private list first because mem_map is not up yet */
3339	INIT_LIST_HEAD(&m->list);
3340	list_add(&m->list, &huge_boot_pages);
3341	m->hstate = h;
3342	return 1;
3343}
3344
3345/* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
3346static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
3347					unsigned long start_page_number,
3348					unsigned long end_page_number)
3349{
3350	enum zone_type zone = zone_idx(folio_zone(folio));
3351	int nid = folio_nid(folio);
3352	unsigned long head_pfn = folio_pfn(folio);
3353	unsigned long pfn, end_pfn = head_pfn + end_page_number;
3354	int ret;
3355
3356	for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {
3357		struct page *page = pfn_to_page(pfn);
3358
3359		__init_single_page(page, pfn, zone, nid);
3360		prep_compound_tail((struct page *)folio, pfn - head_pfn);
3361		ret = page_ref_freeze(page, 1);
3362		VM_BUG_ON(!ret);
3363	}
3364}
3365
3366static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
3367					      struct hstate *h,
3368					      unsigned long nr_pages)
3369{
3370	int ret;
3371
3372	/* Prepare folio head */
3373	__folio_clear_reserved(folio);
3374	__folio_set_head(folio);
3375	ret = folio_ref_freeze(folio, 1);
3376	VM_BUG_ON(!ret);
3377	/* Initialize the necessary tail struct pages */
3378	hugetlb_folio_init_tail_vmemmap(folio, 1, nr_pages);
3379	prep_compound_head((struct page *)folio, huge_page_order(h));
3380}
3381
3382static void __init prep_and_add_bootmem_folios(struct hstate *h,
3383					struct list_head *folio_list)
3384{
3385	unsigned long flags;
3386	struct folio *folio, *tmp_f;
3387
3388	/* Send list for bulk vmemmap optimization processing */
3389	hugetlb_vmemmap_optimize_folios(h, folio_list);
3390
3391	/* Add all new pool pages to free lists in one lock cycle */
3392	spin_lock_irqsave(&hugetlb_lock, flags);
3393	list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
3394		if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
3395			/*
3396			 * If HVO fails, initialize all tail struct pages
3397			 * We do not worry about potential long lock hold
3398			 * time as this is early in boot and there should
3399			 * be no contention.
3400			 */
3401			hugetlb_folio_init_tail_vmemmap(folio,
3402					HUGETLB_VMEMMAP_RESERVE_PAGES,
3403					pages_per_huge_page(h));
3404		}
3405		__prep_account_new_huge_page(h, folio_nid(folio));
3406		enqueue_hugetlb_folio(h, folio);
3407	}
3408	spin_unlock_irqrestore(&hugetlb_lock, flags);
3409}
3410
3411/*
3412 * Put bootmem huge pages into the standard lists after mem_map is up.
3413 * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
3414 */
3415static void __init gather_bootmem_prealloc(void)
3416{
3417	LIST_HEAD(folio_list);
3418	struct huge_bootmem_page *m;
3419	struct hstate *h = NULL, *prev_h = NULL;
3420
3421	list_for_each_entry(m, &huge_boot_pages, list) {
3422		struct page *page = virt_to_page(m);
3423		struct folio *folio = (void *)page;
3424
3425		h = m->hstate;
3426		/*
3427		 * It is possible to have multiple huge page sizes (hstates)
3428		 * in this list.  If so, process each size separately.
3429		 */
3430		if (h != prev_h && prev_h != NULL)
3431			prep_and_add_bootmem_folios(prev_h, &folio_list);
3432		prev_h = h;
3433
3434		VM_BUG_ON(!hstate_is_gigantic(h));
3435		WARN_ON(folio_ref_count(folio) != 1);
3436
3437		hugetlb_folio_init_vmemmap(folio, h,
3438					   HUGETLB_VMEMMAP_RESERVE_PAGES);
3439		init_new_hugetlb_folio(h, folio);
3440		list_add(&folio->lru, &folio_list);
 
 
 
3441
3442		/*
3443		 * We need to restore the 'stolen' pages to totalram_pages
3444		 * in order to fix confusing memory reports from free(1) and
3445		 * other side-effects, like CommitLimit going negative.
3446		 */
3447		adjust_managed_page_count(page, pages_per_huge_page(h));
3448		cond_resched();
3449	}
3450
3451	prep_and_add_bootmem_folios(h, &folio_list);
3452}
3453
3454static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
3455{
3456	unsigned long i;
3457	char buf[32];
3458
3459	for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
3460		if (hstate_is_gigantic(h)) {
3461			if (!alloc_bootmem_huge_page(h, nid))
3462				break;
3463		} else {
3464			struct folio *folio;
3465			gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
3466
3467			folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
3468					&node_states[N_MEMORY], NULL);
3469			if (!folio)
3470				break;
3471			free_huge_folio(folio); /* free it into the hugepage allocator */
3472		}
3473		cond_resched();
3474	}
3475	if (i == h->max_huge_pages_node[nid])
3476		return;
3477
3478	string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3479	pr_warn("HugeTLB: allocating %u of page size %s failed node%d.  Only allocated %lu hugepages.\n",
3480		h->max_huge_pages_node[nid], buf, nid, i);
3481	h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
3482	h->max_huge_pages_node[nid] = i;
3483}
3484
3485/*
3486 * NOTE: this routine is called in different contexts for gigantic and
3487 * non-gigantic pages.
3488 * - For gigantic pages, this is called early in the boot process and
3489 *   pages are allocated from memblock allocated or something similar.
3490 *   Gigantic pages are actually added to pools later with the routine
3491 *   gather_bootmem_prealloc.
3492 * - For non-gigantic pages, this is called later in the boot process after
3493 *   all of mm is up and functional.  Pages are allocated from buddy and
3494 *   then added to hugetlb pools.
3495 */
3496static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
3497{
3498	unsigned long i;
3499	struct folio *folio;
3500	LIST_HEAD(folio_list);
3501	nodemask_t *node_alloc_noretry;
3502	bool node_specific_alloc = false;
3503
3504	/* skip gigantic hugepages allocation if hugetlb_cma enabled */
3505	if (hstate_is_gigantic(h) && hugetlb_cma_size) {
3506		pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
3507		return;
3508	}
3509
3510	/* do node specific alloc */
3511	for_each_online_node(i) {
3512		if (h->max_huge_pages_node[i] > 0) {
3513			hugetlb_hstate_alloc_pages_onenode(h, i);
3514			node_specific_alloc = true;
3515		}
3516	}
3517
3518	if (node_specific_alloc)
3519		return;
3520
3521	/* below will do all node balanced alloc */
3522	if (!hstate_is_gigantic(h)) {
3523		/*
3524		 * Bit mask controlling how hard we retry per-node allocations.
3525		 * Ignore errors as lower level routines can deal with
3526		 * node_alloc_noretry == NULL.  If this kmalloc fails at boot
3527		 * time, we are likely in bigger trouble.
3528		 */
3529		node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
3530						GFP_KERNEL);
3531	} else {
3532		/* allocations done at boot time */
3533		node_alloc_noretry = NULL;
3534	}
3535
3536	/* bit mask controlling how hard we retry per-node allocations */
3537	if (node_alloc_noretry)
3538		nodes_clear(*node_alloc_noretry);
3539
3540	for (i = 0; i < h->max_huge_pages; ++i) {
3541		if (hstate_is_gigantic(h)) {
3542			/*
3543			 * gigantic pages not added to list as they are not
3544			 * added to pools now.
3545			 */
3546			if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
3547				break;
3548		} else {
3549			folio = alloc_pool_huge_folio(h, &node_states[N_MEMORY],
3550							node_alloc_noretry);
3551			if (!folio)
3552				break;
3553			list_add(&folio->lru, &folio_list);
3554		}
 
 
3555		cond_resched();
3556	}
3557
3558	/* list will be empty if hstate_is_gigantic */
3559	prep_and_add_allocated_folios(h, &folio_list);
3560
3561	if (i < h->max_huge_pages) {
3562		char buf[32];
3563
3564		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3565		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
3566			h->max_huge_pages, buf, i);
3567		h->max_huge_pages = i;
3568	}
 
3569	kfree(node_alloc_noretry);
3570}
3571
3572static void __init hugetlb_init_hstates(void)
3573{
3574	struct hstate *h, *h2;
3575
3576	for_each_hstate(h) {
 
 
 
3577		/* oversize hugepages were init'ed in early boot */
3578		if (!hstate_is_gigantic(h))
3579			hugetlb_hstate_alloc_pages(h);
3580
3581		/*
3582		 * Set demote order for each hstate.  Note that
3583		 * h->demote_order is initially 0.
3584		 * - We can not demote gigantic pages if runtime freeing
3585		 *   is not supported, so skip this.
3586		 * - If CMA allocation is possible, we can not demote
3587		 *   HUGETLB_PAGE_ORDER or smaller size pages.
3588		 */
3589		if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3590			continue;
3591		if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER)
3592			continue;
3593		for_each_hstate(h2) {
3594			if (h2 == h)
3595				continue;
3596			if (h2->order < h->order &&
3597			    h2->order > h->demote_order)
3598				h->demote_order = h2->order;
3599		}
3600	}
 
3601}
3602
3603static void __init report_hugepages(void)
3604{
3605	struct hstate *h;
3606
3607	for_each_hstate(h) {
3608		char buf[32];
3609
3610		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3611		pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
3612			buf, h->free_huge_pages);
3613		pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
3614			hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
3615	}
3616}
3617
3618#ifdef CONFIG_HIGHMEM
3619static void try_to_free_low(struct hstate *h, unsigned long count,
3620						nodemask_t *nodes_allowed)
3621{
3622	int i;
3623	LIST_HEAD(page_list);
3624
3625	lockdep_assert_held(&hugetlb_lock);
3626	if (hstate_is_gigantic(h))
3627		return;
3628
3629	/*
3630	 * Collect pages to be freed on a list, and free after dropping lock
3631	 */
3632	for_each_node_mask(i, *nodes_allowed) {
3633		struct folio *folio, *next;
3634		struct list_head *freel = &h->hugepage_freelists[i];
3635		list_for_each_entry_safe(folio, next, freel, lru) {
3636			if (count >= h->nr_huge_pages)
3637				goto out;
3638			if (folio_test_highmem(folio))
3639				continue;
3640			remove_hugetlb_folio(h, folio, false);
3641			list_add(&folio->lru, &page_list);
3642		}
3643	}
3644
3645out:
3646	spin_unlock_irq(&hugetlb_lock);
3647	update_and_free_pages_bulk(h, &page_list);
3648	spin_lock_irq(&hugetlb_lock);
3649}
3650#else
3651static inline void try_to_free_low(struct hstate *h, unsigned long count,
3652						nodemask_t *nodes_allowed)
3653{
3654}
3655#endif
3656
3657/*
3658 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
3659 * balanced by operating on them in a round-robin fashion.
3660 * Returns 1 if an adjustment was made.
3661 */
3662static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
3663				int delta)
3664{
3665	int nr_nodes, node;
3666
3667	lockdep_assert_held(&hugetlb_lock);
3668	VM_BUG_ON(delta != -1 && delta != 1);
3669
3670	if (delta < 0) {
3671		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
3672			if (h->surplus_huge_pages_node[node])
3673				goto found;
3674		}
3675	} else {
3676		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3677			if (h->surplus_huge_pages_node[node] <
3678					h->nr_huge_pages_node[node])
3679				goto found;
3680		}
3681	}
3682	return 0;
3683
3684found:
3685	h->surplus_huge_pages += delta;
3686	h->surplus_huge_pages_node[node] += delta;
3687	return 1;
3688}
3689
3690#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
3691static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
3692			      nodemask_t *nodes_allowed)
3693{
3694	unsigned long min_count;
3695	unsigned long allocated;
3696	struct folio *folio;
3697	LIST_HEAD(page_list);
3698	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
3699
3700	/*
3701	 * Bit mask controlling how hard we retry per-node allocations.
3702	 * If we can not allocate the bit mask, do not attempt to allocate
3703	 * the requested huge pages.
3704	 */
3705	if (node_alloc_noretry)
3706		nodes_clear(*node_alloc_noretry);
3707	else
3708		return -ENOMEM;
3709
3710	/*
3711	 * resize_lock mutex prevents concurrent adjustments to number of
3712	 * pages in hstate via the proc/sysfs interfaces.
3713	 */
3714	mutex_lock(&h->resize_lock);
3715	flush_free_hpage_work(h);
3716	spin_lock_irq(&hugetlb_lock);
3717
3718	/*
3719	 * Check for a node specific request.
3720	 * Changing node specific huge page count may require a corresponding
3721	 * change to the global count.  In any case, the passed node mask
3722	 * (nodes_allowed) will restrict alloc/free to the specified node.
3723	 */
3724	if (nid != NUMA_NO_NODE) {
3725		unsigned long old_count = count;
3726
3727		count += persistent_huge_pages(h) -
3728			 (h->nr_huge_pages_node[nid] -
3729			  h->surplus_huge_pages_node[nid]);
3730		/*
3731		 * User may have specified a large count value which caused the
3732		 * above calculation to overflow.  In this case, they wanted
3733		 * to allocate as many huge pages as possible.  Set count to
3734		 * largest possible value to align with their intention.
3735		 */
3736		if (count < old_count)
3737			count = ULONG_MAX;
3738	}
3739
3740	/*
3741	 * Gigantic pages runtime allocation depend on the capability for large
3742	 * page range allocation.
3743	 * If the system does not provide this feature, return an error when
3744	 * the user tries to allocate gigantic pages but let the user free the
3745	 * boottime allocated gigantic pages.
3746	 */
3747	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
3748		if (count > persistent_huge_pages(h)) {
3749			spin_unlock_irq(&hugetlb_lock);
3750			mutex_unlock(&h->resize_lock);
3751			NODEMASK_FREE(node_alloc_noretry);
3752			return -EINVAL;
3753		}
3754		/* Fall through to decrease pool */
3755	}
3756
3757	/*
3758	 * Increase the pool size
3759	 * First take pages out of surplus state.  Then make up the
3760	 * remaining difference by allocating fresh huge pages.
3761	 *
3762	 * We might race with alloc_surplus_hugetlb_folio() here and be unable
3763	 * to convert a surplus huge page to a normal huge page. That is
3764	 * not critical, though, it just means the overall size of the
3765	 * pool might be one hugepage larger than it needs to be, but
3766	 * within all the constraints specified by the sysctls.
3767	 */
3768	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3769		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3770			break;
3771	}
3772
3773	allocated = 0;
3774	while (count > (persistent_huge_pages(h) + allocated)) {
3775		/*
3776		 * If this allocation races such that we no longer need the
3777		 * page, free_huge_folio will handle it by freeing the page
3778		 * and reducing the surplus.
3779		 */
3780		spin_unlock_irq(&hugetlb_lock);
3781
3782		/* yield cpu to avoid soft lockup */
3783		cond_resched();
3784
3785		folio = alloc_pool_huge_folio(h, nodes_allowed,
3786						node_alloc_noretry);
3787		if (!folio) {
3788			prep_and_add_allocated_folios(h, &page_list);
3789			spin_lock_irq(&hugetlb_lock);
3790			goto out;
3791		}
3792
3793		list_add(&folio->lru, &page_list);
3794		allocated++;
3795
3796		/* Bail for signals. Probably ctrl-c from user */
3797		if (signal_pending(current)) {
3798			prep_and_add_allocated_folios(h, &page_list);
3799			spin_lock_irq(&hugetlb_lock);
3800			goto out;
3801		}
3802
3803		spin_lock_irq(&hugetlb_lock);
3804	}
3805
3806	/* Add allocated pages to the pool */
3807	if (!list_empty(&page_list)) {
3808		spin_unlock_irq(&hugetlb_lock);
3809		prep_and_add_allocated_folios(h, &page_list);
3810		spin_lock_irq(&hugetlb_lock);
3811	}
3812
3813	/*
3814	 * Decrease the pool size
3815	 * First return free pages to the buddy allocator (being careful
3816	 * to keep enough around to satisfy reservations).  Then place
3817	 * pages into surplus state as needed so the pool will shrink
3818	 * to the desired size as pages become free.
3819	 *
3820	 * By placing pages into the surplus state independent of the
3821	 * overcommit value, we are allowing the surplus pool size to
3822	 * exceed overcommit. There are few sane options here. Since
3823	 * alloc_surplus_hugetlb_folio() is checking the global counter,
3824	 * though, we'll note that we're not allowed to exceed surplus
3825	 * and won't grow the pool anywhere else. Not until one of the
3826	 * sysctls are changed, or the surplus pages go out of use.
3827	 */
3828	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3829	min_count = max(count, min_count);
3830	try_to_free_low(h, min_count, nodes_allowed);
3831
3832	/*
3833	 * Collect pages to be removed on list without dropping lock
3834	 */
3835	while (min_count < persistent_huge_pages(h)) {
3836		folio = remove_pool_hugetlb_folio(h, nodes_allowed, 0);
3837		if (!folio)
3838			break;
3839
3840		list_add(&folio->lru, &page_list);
3841	}
3842	/* free the pages after dropping lock */
3843	spin_unlock_irq(&hugetlb_lock);
3844	update_and_free_pages_bulk(h, &page_list);
3845	flush_free_hpage_work(h);
3846	spin_lock_irq(&hugetlb_lock);
3847
3848	while (count < persistent_huge_pages(h)) {
3849		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3850			break;
3851	}
3852out:
3853	h->max_huge_pages = persistent_huge_pages(h);
3854	spin_unlock_irq(&hugetlb_lock);
3855	mutex_unlock(&h->resize_lock);
3856
3857	NODEMASK_FREE(node_alloc_noretry);
3858
3859	return 0;
3860}
3861
3862static int demote_free_hugetlb_folio(struct hstate *h, struct folio *folio)
3863{
3864	int i, nid = folio_nid(folio);
3865	struct hstate *target_hstate;
3866	struct page *subpage;
3867	struct folio *inner_folio;
3868	int rc = 0;
3869
3870	target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);
3871
3872	remove_hugetlb_folio_for_demote(h, folio, false);
3873	spin_unlock_irq(&hugetlb_lock);
3874
3875	/*
3876	 * If vmemmap already existed for folio, the remove routine above would
3877	 * have cleared the hugetlb folio flag.  Hence the folio is technically
3878	 * no longer a hugetlb folio.  hugetlb_vmemmap_restore_folio can only be
3879	 * passed hugetlb folios and will BUG otherwise.
3880	 */
3881	if (folio_test_hugetlb(folio)) {
3882		rc = hugetlb_vmemmap_restore_folio(h, folio);
3883		if (rc) {
3884			/* Allocation of vmemmmap failed, we can not demote folio */
3885			spin_lock_irq(&hugetlb_lock);
3886			folio_ref_unfreeze(folio, 1);
3887			add_hugetlb_folio(h, folio, false);
3888			return rc;
3889		}
3890	}
3891
3892	/*
3893	 * Use destroy_compound_hugetlb_folio_for_demote for all huge page
3894	 * sizes as it will not ref count folios.
3895	 */
3896	destroy_compound_hugetlb_folio_for_demote(folio, huge_page_order(h));
3897
3898	/*
3899	 * Taking target hstate mutex synchronizes with set_max_huge_pages.
3900	 * Without the mutex, pages added to target hstate could be marked
3901	 * as surplus.
3902	 *
3903	 * Note that we already hold h->resize_lock.  To prevent deadlock,
3904	 * use the convention of always taking larger size hstate mutex first.
3905	 */
3906	mutex_lock(&target_hstate->resize_lock);
3907	for (i = 0; i < pages_per_huge_page(h);
3908				i += pages_per_huge_page(target_hstate)) {
3909		subpage = folio_page(folio, i);
3910		inner_folio = page_folio(subpage);
3911		if (hstate_is_gigantic(target_hstate))
3912			prep_compound_gigantic_folio_for_demote(inner_folio,
3913							target_hstate->order);
3914		else
3915			prep_compound_page(subpage, target_hstate->order);
3916		folio_change_private(inner_folio, NULL);
3917		prep_new_hugetlb_folio(target_hstate, inner_folio, nid);
3918		free_huge_folio(inner_folio);
3919	}
3920	mutex_unlock(&target_hstate->resize_lock);
3921
3922	spin_lock_irq(&hugetlb_lock);
3923
3924	/*
3925	 * Not absolutely necessary, but for consistency update max_huge_pages
3926	 * based on pool changes for the demoted page.
3927	 */
3928	h->max_huge_pages--;
3929	target_hstate->max_huge_pages +=
3930		pages_per_huge_page(h) / pages_per_huge_page(target_hstate);
3931
3932	return rc;
3933}
3934
3935static int demote_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
3936	__must_hold(&hugetlb_lock)
3937{
3938	int nr_nodes, node;
3939	struct folio *folio;
3940
3941	lockdep_assert_held(&hugetlb_lock);
3942
3943	/* We should never get here if no demote order */
3944	if (!h->demote_order) {
3945		pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
3946		return -EINVAL;		/* internal error */
3947	}
3948
3949	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3950		list_for_each_entry(folio, &h->hugepage_freelists[node], lru) {
3951			if (folio_test_hwpoison(folio))
3952				continue;
3953			return demote_free_hugetlb_folio(h, folio);
3954		}
3955	}
3956
3957	/*
3958	 * Only way to get here is if all pages on free lists are poisoned.
3959	 * Return -EBUSY so that caller will not retry.
3960	 */
3961	return -EBUSY;
3962}
3963
3964#define HSTATE_ATTR_RO(_name) \
3965	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
3966
3967#define HSTATE_ATTR_WO(_name) \
3968	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)
3969
3970#define HSTATE_ATTR(_name) \
3971	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
 
3972
3973static struct kobject *hugepages_kobj;
3974static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
3975
3976static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
3977
3978static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
3979{
3980	int i;
3981
3982	for (i = 0; i < HUGE_MAX_HSTATE; i++)
3983		if (hstate_kobjs[i] == kobj) {
3984			if (nidp)
3985				*nidp = NUMA_NO_NODE;
3986			return &hstates[i];
3987		}
3988
3989	return kobj_to_node_hstate(kobj, nidp);
3990}
3991
3992static ssize_t nr_hugepages_show_common(struct kobject *kobj,
3993					struct kobj_attribute *attr, char *buf)
3994{
3995	struct hstate *h;
3996	unsigned long nr_huge_pages;
3997	int nid;
3998
3999	h = kobj_to_hstate(kobj, &nid);
4000	if (nid == NUMA_NO_NODE)
4001		nr_huge_pages = h->nr_huge_pages;
4002	else
4003		nr_huge_pages = h->nr_huge_pages_node[nid];
4004
4005	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
4006}
4007
4008static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
4009					   struct hstate *h, int nid,
4010					   unsigned long count, size_t len)
4011{
4012	int err;
4013	nodemask_t nodes_allowed, *n_mask;
4014
4015	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
4016		return -EINVAL;
4017
4018	if (nid == NUMA_NO_NODE) {
4019		/*
4020		 * global hstate attribute
4021		 */
4022		if (!(obey_mempolicy &&
4023				init_nodemask_of_mempolicy(&nodes_allowed)))
4024			n_mask = &node_states[N_MEMORY];
4025		else
4026			n_mask = &nodes_allowed;
4027	} else {
4028		/*
4029		 * Node specific request.  count adjustment happens in
4030		 * set_max_huge_pages() after acquiring hugetlb_lock.
4031		 */
4032		init_nodemask_of_node(&nodes_allowed, nid);
4033		n_mask = &nodes_allowed;
4034	}
4035
4036	err = set_max_huge_pages(h, count, nid, n_mask);
4037
4038	return err ? err : len;
4039}
4040
4041static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
4042					 struct kobject *kobj, const char *buf,
4043					 size_t len)
4044{
4045	struct hstate *h;
4046	unsigned long count;
4047	int nid;
4048	int err;
4049
4050	err = kstrtoul(buf, 10, &count);
4051	if (err)
4052		return err;
4053
4054	h = kobj_to_hstate(kobj, &nid);
4055	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
4056}
4057
4058static ssize_t nr_hugepages_show(struct kobject *kobj,
4059				       struct kobj_attribute *attr, char *buf)
4060{
4061	return nr_hugepages_show_common(kobj, attr, buf);
4062}
4063
4064static ssize_t nr_hugepages_store(struct kobject *kobj,
4065	       struct kobj_attribute *attr, const char *buf, size_t len)
4066{
4067	return nr_hugepages_store_common(false, kobj, buf, len);
4068}
4069HSTATE_ATTR(nr_hugepages);
4070
4071#ifdef CONFIG_NUMA
4072
4073/*
4074 * hstate attribute for optionally mempolicy-based constraint on persistent
4075 * huge page alloc/free.
4076 */
4077static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
4078					   struct kobj_attribute *attr,
4079					   char *buf)
4080{
4081	return nr_hugepages_show_common(kobj, attr, buf);
4082}
4083
4084static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
4085	       struct kobj_attribute *attr, const char *buf, size_t len)
4086{
4087	return nr_hugepages_store_common(true, kobj, buf, len);
4088}
4089HSTATE_ATTR(nr_hugepages_mempolicy);
4090#endif
4091
4092
4093static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
4094					struct kobj_attribute *attr, char *buf)
4095{
4096	struct hstate *h = kobj_to_hstate(kobj, NULL);
4097	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
4098}
4099
4100static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
4101		struct kobj_attribute *attr, const char *buf, size_t count)
4102{
4103	int err;
4104	unsigned long input;
4105	struct hstate *h = kobj_to_hstate(kobj, NULL);
4106
4107	if (hstate_is_gigantic(h))
4108		return -EINVAL;
4109
4110	err = kstrtoul(buf, 10, &input);
4111	if (err)
4112		return err;
4113
4114	spin_lock_irq(&hugetlb_lock);
4115	h->nr_overcommit_huge_pages = input;
4116	spin_unlock_irq(&hugetlb_lock);
4117
4118	return count;
4119}
4120HSTATE_ATTR(nr_overcommit_hugepages);
4121
4122static ssize_t free_hugepages_show(struct kobject *kobj,
4123					struct kobj_attribute *attr, char *buf)
4124{
4125	struct hstate *h;
4126	unsigned long free_huge_pages;
4127	int nid;
4128
4129	h = kobj_to_hstate(kobj, &nid);
4130	if (nid == NUMA_NO_NODE)
4131		free_huge_pages = h->free_huge_pages;
4132	else
4133		free_huge_pages = h->free_huge_pages_node[nid];
4134
4135	return sysfs_emit(buf, "%lu\n", free_huge_pages);
4136}
4137HSTATE_ATTR_RO(free_hugepages);
4138
4139static ssize_t resv_hugepages_show(struct kobject *kobj,
4140					struct kobj_attribute *attr, char *buf)
4141{
4142	struct hstate *h = kobj_to_hstate(kobj, NULL);
4143	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
4144}
4145HSTATE_ATTR_RO(resv_hugepages);
4146
4147static ssize_t surplus_hugepages_show(struct kobject *kobj,
4148					struct kobj_attribute *attr, char *buf)
4149{
4150	struct hstate *h;
4151	unsigned long surplus_huge_pages;
4152	int nid;
4153
4154	h = kobj_to_hstate(kobj, &nid);
4155	if (nid == NUMA_NO_NODE)
4156		surplus_huge_pages = h->surplus_huge_pages;
4157	else
4158		surplus_huge_pages = h->surplus_huge_pages_node[nid];
4159
4160	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
4161}
4162HSTATE_ATTR_RO(surplus_hugepages);
4163
4164static ssize_t demote_store(struct kobject *kobj,
4165	       struct kobj_attribute *attr, const char *buf, size_t len)
4166{
4167	unsigned long nr_demote;
4168	unsigned long nr_available;
4169	nodemask_t nodes_allowed, *n_mask;
4170	struct hstate *h;
4171	int err;
4172	int nid;
4173
4174	err = kstrtoul(buf, 10, &nr_demote);
4175	if (err)
4176		return err;
4177	h = kobj_to_hstate(kobj, &nid);
4178
4179	if (nid != NUMA_NO_NODE) {
4180		init_nodemask_of_node(&nodes_allowed, nid);
4181		n_mask = &nodes_allowed;
4182	} else {
4183		n_mask = &node_states[N_MEMORY];
4184	}
4185
4186	/* Synchronize with other sysfs operations modifying huge pages */
4187	mutex_lock(&h->resize_lock);
4188	spin_lock_irq(&hugetlb_lock);
4189
4190	while (nr_demote) {
4191		/*
4192		 * Check for available pages to demote each time thorough the
4193		 * loop as demote_pool_huge_page will drop hugetlb_lock.
4194		 */
4195		if (nid != NUMA_NO_NODE)
4196			nr_available = h->free_huge_pages_node[nid];
4197		else
4198			nr_available = h->free_huge_pages;
4199		nr_available -= h->resv_huge_pages;
4200		if (!nr_available)
4201			break;
4202
4203		err = demote_pool_huge_page(h, n_mask);
4204		if (err)
4205			break;
4206
4207		nr_demote--;
4208	}
4209
4210	spin_unlock_irq(&hugetlb_lock);
4211	mutex_unlock(&h->resize_lock);
4212
4213	if (err)
4214		return err;
4215	return len;
4216}
4217HSTATE_ATTR_WO(demote);
4218
4219static ssize_t demote_size_show(struct kobject *kobj,
4220					struct kobj_attribute *attr, char *buf)
4221{
4222	struct hstate *h = kobj_to_hstate(kobj, NULL);
4223	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
4224
4225	return sysfs_emit(buf, "%lukB\n", demote_size);
4226}
4227
4228static ssize_t demote_size_store(struct kobject *kobj,
4229					struct kobj_attribute *attr,
4230					const char *buf, size_t count)
4231{
4232	struct hstate *h, *demote_hstate;
4233	unsigned long demote_size;
4234	unsigned int demote_order;
4235
4236	demote_size = (unsigned long)memparse(buf, NULL);
4237
4238	demote_hstate = size_to_hstate(demote_size);
4239	if (!demote_hstate)
4240		return -EINVAL;
4241	demote_order = demote_hstate->order;
4242	if (demote_order < HUGETLB_PAGE_ORDER)
4243		return -EINVAL;
4244
4245	/* demote order must be smaller than hstate order */
4246	h = kobj_to_hstate(kobj, NULL);
4247	if (demote_order >= h->order)
4248		return -EINVAL;
4249
4250	/* resize_lock synchronizes access to demote size and writes */
4251	mutex_lock(&h->resize_lock);
4252	h->demote_order = demote_order;
4253	mutex_unlock(&h->resize_lock);
4254
4255	return count;
4256}
4257HSTATE_ATTR(demote_size);
4258
4259static struct attribute *hstate_attrs[] = {
4260	&nr_hugepages_attr.attr,
4261	&nr_overcommit_hugepages_attr.attr,
4262	&free_hugepages_attr.attr,
4263	&resv_hugepages_attr.attr,
4264	&surplus_hugepages_attr.attr,
4265#ifdef CONFIG_NUMA
4266	&nr_hugepages_mempolicy_attr.attr,
4267#endif
4268	NULL,
4269};
4270
4271static const struct attribute_group hstate_attr_group = {
4272	.attrs = hstate_attrs,
4273};
4274
4275static struct attribute *hstate_demote_attrs[] = {
4276	&demote_size_attr.attr,
4277	&demote_attr.attr,
4278	NULL,
4279};
4280
4281static const struct attribute_group hstate_demote_attr_group = {
4282	.attrs = hstate_demote_attrs,
4283};
4284
4285static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
4286				    struct kobject **hstate_kobjs,
4287				    const struct attribute_group *hstate_attr_group)
4288{
4289	int retval;
4290	int hi = hstate_index(h);
4291
4292	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
4293	if (!hstate_kobjs[hi])
4294		return -ENOMEM;
4295
4296	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
4297	if (retval) {
4298		kobject_put(hstate_kobjs[hi]);
4299		hstate_kobjs[hi] = NULL;
4300		return retval;
4301	}
4302
4303	if (h->demote_order) {
4304		retval = sysfs_create_group(hstate_kobjs[hi],
4305					    &hstate_demote_attr_group);
4306		if (retval) {
4307			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
4308			sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
4309			kobject_put(hstate_kobjs[hi]);
4310			hstate_kobjs[hi] = NULL;
4311			return retval;
4312		}
4313	}
4314
4315	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4316}
4317
4318#ifdef CONFIG_NUMA
4319static bool hugetlb_sysfs_initialized __ro_after_init;
4320
4321/*
4322 * node_hstate/s - associate per node hstate attributes, via their kobjects,
4323 * with node devices in node_devices[] using a parallel array.  The array
4324 * index of a node device or _hstate == node id.
4325 * This is here to avoid any static dependency of the node device driver, in
4326 * the base kernel, on the hugetlb module.
4327 */
4328struct node_hstate {
4329	struct kobject		*hugepages_kobj;
4330	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
4331};
4332static struct node_hstate node_hstates[MAX_NUMNODES];
4333
4334/*
4335 * A subset of global hstate attributes for node devices
4336 */
4337static struct attribute *per_node_hstate_attrs[] = {
4338	&nr_hugepages_attr.attr,
4339	&free_hugepages_attr.attr,
4340	&surplus_hugepages_attr.attr,
4341	NULL,
4342};
4343
4344static const struct attribute_group per_node_hstate_attr_group = {
4345	.attrs = per_node_hstate_attrs,
4346};
4347
4348/*
4349 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
4350 * Returns node id via non-NULL nidp.
4351 */
4352static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4353{
4354	int nid;
4355
4356	for (nid = 0; nid < nr_node_ids; nid++) {
4357		struct node_hstate *nhs = &node_hstates[nid];
4358		int i;
4359		for (i = 0; i < HUGE_MAX_HSTATE; i++)
4360			if (nhs->hstate_kobjs[i] == kobj) {
4361				if (nidp)
4362					*nidp = nid;
4363				return &hstates[i];
4364			}
4365	}
4366
4367	BUG();
4368	return NULL;
4369}
4370
4371/*
4372 * Unregister hstate attributes from a single node device.
4373 * No-op if no hstate attributes attached.
4374 */
4375void hugetlb_unregister_node(struct node *node)
4376{
4377	struct hstate *h;
4378	struct node_hstate *nhs = &node_hstates[node->dev.id];
4379
4380	if (!nhs->hugepages_kobj)
4381		return;		/* no hstate attributes */
4382
4383	for_each_hstate(h) {
4384		int idx = hstate_index(h);
4385		struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];
4386
4387		if (!hstate_kobj)
4388			continue;
4389		if (h->demote_order)
4390			sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
4391		sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
4392		kobject_put(hstate_kobj);
4393		nhs->hstate_kobjs[idx] = NULL;
4394	}
4395
4396	kobject_put(nhs->hugepages_kobj);
4397	nhs->hugepages_kobj = NULL;
4398}
4399
4400
4401/*
4402 * Register hstate attributes for a single node device.
4403 * No-op if attributes already registered.
4404 */
4405void hugetlb_register_node(struct node *node)
4406{
4407	struct hstate *h;
4408	struct node_hstate *nhs = &node_hstates[node->dev.id];
4409	int err;
4410
4411	if (!hugetlb_sysfs_initialized)
4412		return;
4413
4414	if (nhs->hugepages_kobj)
4415		return;		/* already allocated */
4416
4417	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
4418							&node->dev.kobj);
4419	if (!nhs->hugepages_kobj)
4420		return;
4421
4422	for_each_hstate(h) {
4423		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
4424						nhs->hstate_kobjs,
4425						&per_node_hstate_attr_group);
4426		if (err) {
4427			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
4428				h->name, node->dev.id);
4429			hugetlb_unregister_node(node);
4430			break;
4431		}
4432	}
4433}
4434
4435/*
4436 * hugetlb init time:  register hstate attributes for all registered node
4437 * devices of nodes that have memory.  All on-line nodes should have
4438 * registered their associated device by this time.
4439 */
4440static void __init hugetlb_register_all_nodes(void)
4441{
4442	int nid;
4443
4444	for_each_online_node(nid)
4445		hugetlb_register_node(node_devices[nid]);
 
 
 
 
 
 
 
 
 
 
4446}
4447#else	/* !CONFIG_NUMA */
4448
4449static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4450{
4451	BUG();
4452	if (nidp)
4453		*nidp = -1;
4454	return NULL;
4455}
4456
4457static void hugetlb_register_all_nodes(void) { }
4458
4459#endif
4460
4461#ifdef CONFIG_CMA
4462static void __init hugetlb_cma_check(void);
4463#else
4464static inline __init void hugetlb_cma_check(void)
4465{
4466}
4467#endif
4468
4469static void __init hugetlb_sysfs_init(void)
4470{
4471	struct hstate *h;
4472	int err;
4473
4474	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
4475	if (!hugepages_kobj)
4476		return;
4477
4478	for_each_hstate(h) {
4479		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
4480					 hstate_kobjs, &hstate_attr_group);
4481		if (err)
4482			pr_err("HugeTLB: Unable to add hstate %s", h->name);
4483	}
4484
4485#ifdef CONFIG_NUMA
4486	hugetlb_sysfs_initialized = true;
4487#endif
4488	hugetlb_register_all_nodes();
4489}
4490
4491#ifdef CONFIG_SYSCTL
4492static void hugetlb_sysctl_init(void);
4493#else
4494static inline void hugetlb_sysctl_init(void) { }
4495#endif
4496
4497static int __init hugetlb_init(void)
4498{
4499	int i;
4500
4501	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
4502			__NR_HPAGEFLAGS);
4503
4504	if (!hugepages_supported()) {
4505		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
4506			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
4507		return 0;
4508	}
4509
4510	/*
4511	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
4512	 * architectures depend on setup being done here.
4513	 */
4514	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
4515	if (!parsed_default_hugepagesz) {
4516		/*
4517		 * If we did not parse a default huge page size, set
4518		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
4519		 * number of huge pages for this default size was implicitly
4520		 * specified, set that here as well.
4521		 * Note that the implicit setting will overwrite an explicit
4522		 * setting.  A warning will be printed in this case.
4523		 */
4524		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
4525		if (default_hstate_max_huge_pages) {
4526			if (default_hstate.max_huge_pages) {
4527				char buf[32];
4528
4529				string_get_size(huge_page_size(&default_hstate),
4530					1, STRING_UNITS_2, buf, 32);
4531				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
4532					default_hstate.max_huge_pages, buf);
4533				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
4534					default_hstate_max_huge_pages);
4535			}
4536			default_hstate.max_huge_pages =
4537				default_hstate_max_huge_pages;
4538
4539			for_each_online_node(i)
4540				default_hstate.max_huge_pages_node[i] =
4541					default_hugepages_in_node[i];
4542		}
4543	}
4544
4545	hugetlb_cma_check();
4546	hugetlb_init_hstates();
4547	gather_bootmem_prealloc();
4548	report_hugepages();
4549
4550	hugetlb_sysfs_init();
 
4551	hugetlb_cgroup_file_init();
4552	hugetlb_sysctl_init();
4553
4554#ifdef CONFIG_SMP
4555	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
4556#else
4557	num_fault_mutexes = 1;
4558#endif
4559	hugetlb_fault_mutex_table =
4560		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
4561			      GFP_KERNEL);
4562	BUG_ON(!hugetlb_fault_mutex_table);
4563
4564	for (i = 0; i < num_fault_mutexes; i++)
4565		mutex_init(&hugetlb_fault_mutex_table[i]);
4566	return 0;
4567}
4568subsys_initcall(hugetlb_init);
4569
4570/* Overwritten by architectures with more huge page sizes */
4571bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
4572{
4573	return size == HPAGE_SIZE;
4574}
4575
4576void __init hugetlb_add_hstate(unsigned int order)
4577{
4578	struct hstate *h;
4579	unsigned long i;
4580
4581	if (size_to_hstate(PAGE_SIZE << order)) {
4582		return;
4583	}
4584	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
4585	BUG_ON(order < order_base_2(__NR_USED_SUBPAGE));
4586	h = &hstates[hugetlb_max_hstate++];
4587	mutex_init(&h->resize_lock);
4588	h->order = order;
4589	h->mask = ~(huge_page_size(h) - 1);
4590	for (i = 0; i < MAX_NUMNODES; ++i)
4591		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
4592	INIT_LIST_HEAD(&h->hugepage_activelist);
4593	h->next_nid_to_alloc = first_memory_node;
4594	h->next_nid_to_free = first_memory_node;
4595	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
4596					huge_page_size(h)/SZ_1K);
 
4597
4598	parsed_hstate = h;
4599}
4600
4601bool __init __weak hugetlb_node_alloc_supported(void)
4602{
4603	return true;
4604}
4605
4606static void __init hugepages_clear_pages_in_node(void)
4607{
4608	if (!hugetlb_max_hstate) {
4609		default_hstate_max_huge_pages = 0;
4610		memset(default_hugepages_in_node, 0,
4611			sizeof(default_hugepages_in_node));
4612	} else {
4613		parsed_hstate->max_huge_pages = 0;
4614		memset(parsed_hstate->max_huge_pages_node, 0,
4615			sizeof(parsed_hstate->max_huge_pages_node));
4616	}
4617}
4618
4619/*
4620 * hugepages command line processing
4621 * hugepages normally follows a valid hugepagsz or default_hugepagsz
4622 * specification.  If not, ignore the hugepages value.  hugepages can also
4623 * be the first huge page command line  option in which case it implicitly
4624 * specifies the number of huge pages for the default size.
4625 */
4626static int __init hugepages_setup(char *s)
4627{
4628	unsigned long *mhp;
4629	static unsigned long *last_mhp;
4630	int node = NUMA_NO_NODE;
4631	int count;
4632	unsigned long tmp;
4633	char *p = s;
4634
4635	if (!parsed_valid_hugepagesz) {
4636		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
4637		parsed_valid_hugepagesz = true;
4638		return 1;
4639	}
4640
4641	/*
4642	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
4643	 * yet, so this hugepages= parameter goes to the "default hstate".
4644	 * Otherwise, it goes with the previously parsed hugepagesz or
4645	 * default_hugepagesz.
4646	 */
4647	else if (!hugetlb_max_hstate)
4648		mhp = &default_hstate_max_huge_pages;
4649	else
4650		mhp = &parsed_hstate->max_huge_pages;
4651
4652	if (mhp == last_mhp) {
4653		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
4654		return 1;
4655	}
4656
4657	while (*p) {
4658		count = 0;
4659		if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4660			goto invalid;
4661		/* Parameter is node format */
4662		if (p[count] == ':') {
4663			if (!hugetlb_node_alloc_supported()) {
4664				pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
4665				return 1;
4666			}
4667			if (tmp >= MAX_NUMNODES || !node_online(tmp))
4668				goto invalid;
4669			node = array_index_nospec(tmp, MAX_NUMNODES);
4670			p += count + 1;
4671			/* Parse hugepages */
4672			if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4673				goto invalid;
4674			if (!hugetlb_max_hstate)
4675				default_hugepages_in_node[node] = tmp;
4676			else
4677				parsed_hstate->max_huge_pages_node[node] = tmp;
4678			*mhp += tmp;
4679			/* Go to parse next node*/
4680			if (p[count] == ',')
4681				p += count + 1;
4682			else
4683				break;
4684		} else {
4685			if (p != s)
4686				goto invalid;
4687			*mhp = tmp;
4688			break;
4689		}
4690	}
4691
4692	/*
4693	 * Global state is always initialized later in hugetlb_init.
4694	 * But we need to allocate gigantic hstates here early to still
4695	 * use the bootmem allocator.
4696	 */
4697	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
4698		hugetlb_hstate_alloc_pages(parsed_hstate);
4699
4700	last_mhp = mhp;
4701
4702	return 1;
4703
4704invalid:
4705	pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
4706	hugepages_clear_pages_in_node();
4707	return 1;
4708}
4709__setup("hugepages=", hugepages_setup);
4710
4711/*
4712 * hugepagesz command line processing
4713 * A specific huge page size can only be specified once with hugepagesz.
4714 * hugepagesz is followed by hugepages on the command line.  The global
4715 * variable 'parsed_valid_hugepagesz' is used to determine if prior
4716 * hugepagesz argument was valid.
4717 */
4718static int __init hugepagesz_setup(char *s)
4719{
4720	unsigned long size;
4721	struct hstate *h;
4722
4723	parsed_valid_hugepagesz = false;
4724	size = (unsigned long)memparse(s, NULL);
4725
4726	if (!arch_hugetlb_valid_size(size)) {
4727		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
4728		return 1;
4729	}
4730
4731	h = size_to_hstate(size);
4732	if (h) {
4733		/*
4734		 * hstate for this size already exists.  This is normally
4735		 * an error, but is allowed if the existing hstate is the
4736		 * default hstate.  More specifically, it is only allowed if
4737		 * the number of huge pages for the default hstate was not
4738		 * previously specified.
4739		 */
4740		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
4741		    default_hstate.max_huge_pages) {
4742			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
4743			return 1;
4744		}
4745
4746		/*
4747		 * No need to call hugetlb_add_hstate() as hstate already
4748		 * exists.  But, do set parsed_hstate so that a following
4749		 * hugepages= parameter will be applied to this hstate.
4750		 */
4751		parsed_hstate = h;
4752		parsed_valid_hugepagesz = true;
4753		return 1;
4754	}
4755
4756	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4757	parsed_valid_hugepagesz = true;
4758	return 1;
4759}
4760__setup("hugepagesz=", hugepagesz_setup);
4761
4762/*
4763 * default_hugepagesz command line input
4764 * Only one instance of default_hugepagesz allowed on command line.
4765 */
4766static int __init default_hugepagesz_setup(char *s)
4767{
4768	unsigned long size;
4769	int i;
4770
4771	parsed_valid_hugepagesz = false;
4772	if (parsed_default_hugepagesz) {
4773		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
4774		return 1;
4775	}
4776
4777	size = (unsigned long)memparse(s, NULL);
4778
4779	if (!arch_hugetlb_valid_size(size)) {
4780		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
4781		return 1;
4782	}
4783
4784	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4785	parsed_valid_hugepagesz = true;
4786	parsed_default_hugepagesz = true;
4787	default_hstate_idx = hstate_index(size_to_hstate(size));
4788
4789	/*
4790	 * The number of default huge pages (for this size) could have been
4791	 * specified as the first hugetlb parameter: hugepages=X.  If so,
4792	 * then default_hstate_max_huge_pages is set.  If the default huge
4793	 * page size is gigantic (> MAX_PAGE_ORDER), then the pages must be
4794	 * allocated here from bootmem allocator.
4795	 */
4796	if (default_hstate_max_huge_pages) {
4797		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
4798		for_each_online_node(i)
4799			default_hstate.max_huge_pages_node[i] =
4800				default_hugepages_in_node[i];
4801		if (hstate_is_gigantic(&default_hstate))
4802			hugetlb_hstate_alloc_pages(&default_hstate);
4803		default_hstate_max_huge_pages = 0;
4804	}
4805
4806	return 1;
4807}
4808__setup("default_hugepagesz=", default_hugepagesz_setup);
4809
4810static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
4811{
4812#ifdef CONFIG_NUMA
4813	struct mempolicy *mpol = get_task_policy(current);
4814
4815	/*
4816	 * Only enforce MPOL_BIND policy which overlaps with cpuset policy
4817	 * (from policy_nodemask) specifically for hugetlb case
4818	 */
4819	if (mpol->mode == MPOL_BIND &&
4820		(apply_policy_zone(mpol, gfp_zone(gfp)) &&
4821		 cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
4822		return &mpol->nodes;
4823#endif
4824	return NULL;
4825}
4826
4827static unsigned int allowed_mems_nr(struct hstate *h)
4828{
4829	int node;
4830	unsigned int nr = 0;
4831	nodemask_t *mbind_nodemask;
4832	unsigned int *array = h->free_huge_pages_node;
4833	gfp_t gfp_mask = htlb_alloc_mask(h);
4834
4835	mbind_nodemask = policy_mbind_nodemask(gfp_mask);
 
4836	for_each_node_mask(node, cpuset_current_mems_allowed) {
4837		if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
4838			nr += array[node];
4839	}
4840
4841	return nr;
4842}
4843
4844#ifdef CONFIG_SYSCTL
4845static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
4846					  void *buffer, size_t *length,
4847					  loff_t *ppos, unsigned long *out)
4848{
4849	struct ctl_table dup_table;
4850
4851	/*
4852	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
4853	 * can duplicate the @table and alter the duplicate of it.
4854	 */
4855	dup_table = *table;
4856	dup_table.data = out;
4857
4858	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
4859}
4860
4861static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
4862			 struct ctl_table *table, int write,
4863			 void *buffer, size_t *length, loff_t *ppos)
4864{
4865	struct hstate *h = &default_hstate;
4866	unsigned long tmp = h->max_huge_pages;
4867	int ret;
4868
4869	if (!hugepages_supported())
4870		return -EOPNOTSUPP;
4871
4872	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4873					     &tmp);
4874	if (ret)
4875		goto out;
4876
4877	if (write)
4878		ret = __nr_hugepages_store_common(obey_mempolicy, h,
4879						  NUMA_NO_NODE, tmp, *length);
4880out:
4881	return ret;
4882}
4883
4884static int hugetlb_sysctl_handler(struct ctl_table *table, int write,
4885			  void *buffer, size_t *length, loff_t *ppos)
4886{
4887
4888	return hugetlb_sysctl_handler_common(false, table, write,
4889							buffer, length, ppos);
4890}
4891
4892#ifdef CONFIG_NUMA
4893static int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
4894			  void *buffer, size_t *length, loff_t *ppos)
4895{
4896	return hugetlb_sysctl_handler_common(true, table, write,
4897							buffer, length, ppos);
4898}
4899#endif /* CONFIG_NUMA */
4900
4901static int hugetlb_overcommit_handler(struct ctl_table *table, int write,
4902		void *buffer, size_t *length, loff_t *ppos)
4903{
4904	struct hstate *h = &default_hstate;
4905	unsigned long tmp;
4906	int ret;
4907
4908	if (!hugepages_supported())
4909		return -EOPNOTSUPP;
4910
4911	tmp = h->nr_overcommit_huge_pages;
4912
4913	if (write && hstate_is_gigantic(h))
4914		return -EINVAL;
4915
4916	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4917					     &tmp);
4918	if (ret)
4919		goto out;
4920
4921	if (write) {
4922		spin_lock_irq(&hugetlb_lock);
4923		h->nr_overcommit_huge_pages = tmp;
4924		spin_unlock_irq(&hugetlb_lock);
4925	}
4926out:
4927	return ret;
4928}
4929
4930static struct ctl_table hugetlb_table[] = {
4931	{
4932		.procname	= "nr_hugepages",
4933		.data		= NULL,
4934		.maxlen		= sizeof(unsigned long),
4935		.mode		= 0644,
4936		.proc_handler	= hugetlb_sysctl_handler,
4937	},
4938#ifdef CONFIG_NUMA
4939	{
4940		.procname       = "nr_hugepages_mempolicy",
4941		.data           = NULL,
4942		.maxlen         = sizeof(unsigned long),
4943		.mode           = 0644,
4944		.proc_handler   = &hugetlb_mempolicy_sysctl_handler,
4945	},
4946#endif
4947	{
4948		.procname	= "hugetlb_shm_group",
4949		.data		= &sysctl_hugetlb_shm_group,
4950		.maxlen		= sizeof(gid_t),
4951		.mode		= 0644,
4952		.proc_handler	= proc_dointvec,
4953	},
4954	{
4955		.procname	= "nr_overcommit_hugepages",
4956		.data		= NULL,
4957		.maxlen		= sizeof(unsigned long),
4958		.mode		= 0644,
4959		.proc_handler	= hugetlb_overcommit_handler,
4960	},
4961	{ }
4962};
4963
4964static void hugetlb_sysctl_init(void)
4965{
4966	register_sysctl_init("vm", hugetlb_table);
4967}
4968#endif /* CONFIG_SYSCTL */
4969
4970void hugetlb_report_meminfo(struct seq_file *m)
4971{
4972	struct hstate *h;
4973	unsigned long total = 0;
4974
4975	if (!hugepages_supported())
4976		return;
4977
4978	for_each_hstate(h) {
4979		unsigned long count = h->nr_huge_pages;
4980
4981		total += huge_page_size(h) * count;
4982
4983		if (h == &default_hstate)
4984			seq_printf(m,
4985				   "HugePages_Total:   %5lu\n"
4986				   "HugePages_Free:    %5lu\n"
4987				   "HugePages_Rsvd:    %5lu\n"
4988				   "HugePages_Surp:    %5lu\n"
4989				   "Hugepagesize:   %8lu kB\n",
4990				   count,
4991				   h->free_huge_pages,
4992				   h->resv_huge_pages,
4993				   h->surplus_huge_pages,
4994				   huge_page_size(h) / SZ_1K);
4995	}
4996
4997	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
4998}
4999
5000int hugetlb_report_node_meminfo(char *buf, int len, int nid)
5001{
5002	struct hstate *h = &default_hstate;
5003
5004	if (!hugepages_supported())
5005		return 0;
5006
5007	return sysfs_emit_at(buf, len,
5008			     "Node %d HugePages_Total: %5u\n"
5009			     "Node %d HugePages_Free:  %5u\n"
5010			     "Node %d HugePages_Surp:  %5u\n",
5011			     nid, h->nr_huge_pages_node[nid],
5012			     nid, h->free_huge_pages_node[nid],
5013			     nid, h->surplus_huge_pages_node[nid]);
5014}
5015
5016void hugetlb_show_meminfo_node(int nid)
5017{
5018	struct hstate *h;
 
5019
5020	if (!hugepages_supported())
5021		return;
5022
5023	for_each_hstate(h)
5024		printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
5025			nid,
5026			h->nr_huge_pages_node[nid],
5027			h->free_huge_pages_node[nid],
5028			h->surplus_huge_pages_node[nid],
5029			huge_page_size(h) / SZ_1K);
 
5030}
5031
5032void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
5033{
5034	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
5035		   K(atomic_long_read(&mm->hugetlb_usage)));
5036}
5037
5038/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
5039unsigned long hugetlb_total_pages(void)
5040{
5041	struct hstate *h;
5042	unsigned long nr_total_pages = 0;
5043
5044	for_each_hstate(h)
5045		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
5046	return nr_total_pages;
5047}
5048
5049static int hugetlb_acct_memory(struct hstate *h, long delta)
5050{
5051	int ret = -ENOMEM;
5052
5053	if (!delta)
5054		return 0;
5055
5056	spin_lock_irq(&hugetlb_lock);
5057	/*
5058	 * When cpuset is configured, it breaks the strict hugetlb page
5059	 * reservation as the accounting is done on a global variable. Such
5060	 * reservation is completely rubbish in the presence of cpuset because
5061	 * the reservation is not checked against page availability for the
5062	 * current cpuset. Application can still potentially OOM'ed by kernel
5063	 * with lack of free htlb page in cpuset that the task is in.
5064	 * Attempt to enforce strict accounting with cpuset is almost
5065	 * impossible (or too ugly) because cpuset is too fluid that
5066	 * task or memory node can be dynamically moved between cpusets.
5067	 *
5068	 * The change of semantics for shared hugetlb mapping with cpuset is
5069	 * undesirable. However, in order to preserve some of the semantics,
5070	 * we fall back to check against current free page availability as
5071	 * a best attempt and hopefully to minimize the impact of changing
5072	 * semantics that cpuset has.
5073	 *
5074	 * Apart from cpuset, we also have memory policy mechanism that
5075	 * also determines from which node the kernel will allocate memory
5076	 * in a NUMA system. So similar to cpuset, we also should consider
5077	 * the memory policy of the current task. Similar to the description
5078	 * above.
5079	 */
5080	if (delta > 0) {
5081		if (gather_surplus_pages(h, delta) < 0)
5082			goto out;
5083
5084		if (delta > allowed_mems_nr(h)) {
5085			return_unused_surplus_pages(h, delta);
5086			goto out;
5087		}
5088	}
5089
5090	ret = 0;
5091	if (delta < 0)
5092		return_unused_surplus_pages(h, (unsigned long) -delta);
5093
5094out:
5095	spin_unlock_irq(&hugetlb_lock);
5096	return ret;
5097}
5098
5099static void hugetlb_vm_op_open(struct vm_area_struct *vma)
5100{
5101	struct resv_map *resv = vma_resv_map(vma);
5102
5103	/*
5104	 * HPAGE_RESV_OWNER indicates a private mapping.
5105	 * This new VMA should share its siblings reservation map if present.
5106	 * The VMA will only ever have a valid reservation map pointer where
5107	 * it is being copied for another still existing VMA.  As that VMA
5108	 * has a reference to the reservation map it cannot disappear until
5109	 * after this open call completes.  It is therefore safe to take a
5110	 * new reference here without additional locking.
5111	 */
5112	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
5113		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
5114		kref_get(&resv->refs);
5115	}
5116
5117	/*
5118	 * vma_lock structure for sharable mappings is vma specific.
5119	 * Clear old pointer (if copied via vm_area_dup) and allocate
5120	 * new structure.  Before clearing, make sure vma_lock is not
5121	 * for this vma.
5122	 */
5123	if (vma->vm_flags & VM_MAYSHARE) {
5124		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
5125
5126		if (vma_lock) {
5127			if (vma_lock->vma != vma) {
5128				vma->vm_private_data = NULL;
5129				hugetlb_vma_lock_alloc(vma);
5130			} else
5131				pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__);
5132		} else
5133			hugetlb_vma_lock_alloc(vma);
5134	}
5135}
5136
5137static void hugetlb_vm_op_close(struct vm_area_struct *vma)
5138{
5139	struct hstate *h = hstate_vma(vma);
5140	struct resv_map *resv;
5141	struct hugepage_subpool *spool = subpool_vma(vma);
5142	unsigned long reserve, start, end;
5143	long gbl_reserve;
5144
5145	hugetlb_vma_lock_free(vma);
5146
5147	resv = vma_resv_map(vma);
5148	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
5149		return;
5150
5151	start = vma_hugecache_offset(h, vma, vma->vm_start);
5152	end = vma_hugecache_offset(h, vma, vma->vm_end);
5153
5154	reserve = (end - start) - region_count(resv, start, end);
5155	hugetlb_cgroup_uncharge_counter(resv, start, end);
5156	if (reserve) {
5157		/*
5158		 * Decrement reserve counts.  The global reserve count may be
5159		 * adjusted if the subpool has a minimum size.
5160		 */
5161		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
5162		hugetlb_acct_memory(h, -gbl_reserve);
5163	}
5164
5165	kref_put(&resv->refs, resv_map_release);
5166}
5167
5168static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
5169{
5170	if (addr & ~(huge_page_mask(hstate_vma(vma))))
5171		return -EINVAL;
5172
5173	/*
5174	 * PMD sharing is only possible for PUD_SIZE-aligned address ranges
5175	 * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this
5176	 * split, unshare PMDs in the PUD_SIZE interval surrounding addr now.
5177	 */
5178	if (addr & ~PUD_MASK) {
5179		/*
5180		 * hugetlb_vm_op_split is called right before we attempt to
5181		 * split the VMA. We will need to unshare PMDs in the old and
5182		 * new VMAs, so let's unshare before we split.
5183		 */
5184		unsigned long floor = addr & PUD_MASK;
5185		unsigned long ceil = floor + PUD_SIZE;
5186
5187		if (floor >= vma->vm_start && ceil <= vma->vm_end)
5188			hugetlb_unshare_pmds(vma, floor, ceil);
5189	}
5190
5191	return 0;
5192}
5193
5194static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
5195{
5196	return huge_page_size(hstate_vma(vma));
5197}
5198
5199/*
5200 * We cannot handle pagefaults against hugetlb pages at all.  They cause
5201 * handle_mm_fault() to try to instantiate regular-sized pages in the
5202 * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
5203 * this far.
5204 */
5205static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
5206{
5207	BUG();
5208	return 0;
5209}
5210
5211/*
5212 * When a new function is introduced to vm_operations_struct and added
5213 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
5214 * This is because under System V memory model, mappings created via
5215 * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
5216 * their original vm_ops are overwritten with shm_vm_ops.
5217 */
5218const struct vm_operations_struct hugetlb_vm_ops = {
5219	.fault = hugetlb_vm_op_fault,
5220	.open = hugetlb_vm_op_open,
5221	.close = hugetlb_vm_op_close,
5222	.may_split = hugetlb_vm_op_split,
5223	.pagesize = hugetlb_vm_op_pagesize,
5224};
5225
5226static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
5227				int writable)
5228{
5229	pte_t entry;
5230	unsigned int shift = huge_page_shift(hstate_vma(vma));
5231
5232	if (writable) {
5233		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
5234					 vma->vm_page_prot)));
5235	} else {
5236		entry = huge_pte_wrprotect(mk_huge_pte(page,
5237					   vma->vm_page_prot));
5238	}
5239	entry = pte_mkyoung(entry);
 
5240	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
5241
5242	return entry;
5243}
5244
5245static void set_huge_ptep_writable(struct vm_area_struct *vma,
5246				   unsigned long address, pte_t *ptep)
5247{
5248	pte_t entry;
5249
5250	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
5251	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
5252		update_mmu_cache(vma, address, ptep);
5253}
5254
5255bool is_hugetlb_entry_migration(pte_t pte)
5256{
5257	swp_entry_t swp;
5258
5259	if (huge_pte_none(pte) || pte_present(pte))
5260		return false;
5261	swp = pte_to_swp_entry(pte);
5262	if (is_migration_entry(swp))
5263		return true;
5264	else
5265		return false;
5266}
5267
5268bool is_hugetlb_entry_hwpoisoned(pte_t pte)
5269{
5270	swp_entry_t swp;
5271
5272	if (huge_pte_none(pte) || pte_present(pte))
5273		return false;
5274	swp = pte_to_swp_entry(pte);
5275	if (is_hwpoison_entry(swp))
5276		return true;
5277	else
5278		return false;
5279}
5280
5281static void
5282hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
5283		      struct folio *new_folio, pte_t old, unsigned long sz)
5284{
5285	pte_t newpte = make_huge_pte(vma, &new_folio->page, 1);
5286
5287	__folio_mark_uptodate(new_folio);
5288	hugetlb_add_new_anon_rmap(new_folio, vma, addr);
5289	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old))
5290		newpte = huge_pte_mkuffd_wp(newpte);
5291	set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
5292	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
5293	folio_set_hugetlb_migratable(new_folio);
 
5294}
5295
5296int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
5297			    struct vm_area_struct *dst_vma,
5298			    struct vm_area_struct *src_vma)
5299{
5300	pte_t *src_pte, *dst_pte, entry;
5301	struct folio *pte_folio;
5302	unsigned long addr;
5303	bool cow = is_cow_mapping(src_vma->vm_flags);
5304	struct hstate *h = hstate_vma(src_vma);
5305	unsigned long sz = huge_page_size(h);
5306	unsigned long npages = pages_per_huge_page(h);
 
5307	struct mmu_notifier_range range;
5308	unsigned long last_addr_mask;
5309	int ret = 0;
5310
5311	if (cow) {
5312		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src,
5313					src_vma->vm_start,
5314					src_vma->vm_end);
5315		mmu_notifier_invalidate_range_start(&range);
5316		vma_assert_write_locked(src_vma);
5317		raw_write_seqcount_begin(&src->write_protect_seq);
5318	} else {
5319		/*
5320		 * For shared mappings the vma lock must be held before
5321		 * calling hugetlb_walk() in the src vma. Otherwise, the
5322		 * returned ptep could go away if part of a shared pmd and
5323		 * another thread calls huge_pmd_unshare.
5324		 */
5325		hugetlb_vma_lock_read(src_vma);
5326	}
5327
5328	last_addr_mask = hugetlb_mask_last_page(h);
5329	for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
5330		spinlock_t *src_ptl, *dst_ptl;
5331		src_pte = hugetlb_walk(src_vma, addr, sz);
5332		if (!src_pte) {
5333			addr |= last_addr_mask;
5334			continue;
5335		}
5336		dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
5337		if (!dst_pte) {
5338			ret = -ENOMEM;
5339			break;
5340		}
5341
5342		/*
5343		 * If the pagetables are shared don't copy or take references.
5344		 *
5345		 * dst_pte == src_pte is the common case of src/dest sharing.
 
5346		 * However, src could have 'unshared' and dst shares with
5347		 * another vma. So page_count of ptep page is checked instead
5348		 * to reliably determine whether pte is shared.
 
5349		 */
5350		if (page_count(virt_to_page(dst_pte)) > 1) {
5351			addr |= last_addr_mask;
5352			continue;
5353		}
5354
5355		dst_ptl = huge_pte_lock(h, dst, dst_pte);
5356		src_ptl = huge_pte_lockptr(h, src, src_pte);
5357		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5358		entry = huge_ptep_get(src_pte);
 
5359again:
5360		if (huge_pte_none(entry)) {
5361			/*
5362			 * Skip if src entry none.
 
 
5363			 */
5364			;
5365		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
5366			if (!userfaultfd_wp(dst_vma))
5367				entry = huge_pte_clear_uffd_wp(entry);
5368			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5369		} else if (unlikely(is_hugetlb_entry_migration(entry))) {
5370			swp_entry_t swp_entry = pte_to_swp_entry(entry);
5371			bool uffd_wp = pte_swp_uffd_wp(entry);
5372
5373			if (!is_readable_migration_entry(swp_entry) && cow) {
5374				/*
5375				 * COW mappings require pages in both
5376				 * parent and child to be set to read.
5377				 */
5378				swp_entry = make_readable_migration_entry(
5379							swp_offset(swp_entry));
5380				entry = swp_entry_to_pte(swp_entry);
5381				if (userfaultfd_wp(src_vma) && uffd_wp)
5382					entry = pte_swp_mkuffd_wp(entry);
5383				set_huge_pte_at(src, addr, src_pte, entry, sz);
5384			}
5385			if (!userfaultfd_wp(dst_vma))
5386				entry = huge_pte_clear_uffd_wp(entry);
5387			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5388		} else if (unlikely(is_pte_marker(entry))) {
5389			pte_marker marker = copy_pte_marker(
5390				pte_to_swp_entry(entry), dst_vma);
5391
5392			if (marker)
5393				set_huge_pte_at(dst, addr, dst_pte,
5394						make_pte_marker(marker), sz);
5395		} else {
5396			entry = huge_ptep_get(src_pte);
5397			pte_folio = page_folio(pte_page(entry));
5398			folio_get(pte_folio);
5399
5400			/*
5401			 * Failing to duplicate the anon rmap is a rare case
5402			 * where we see pinned hugetlb pages while they're
5403			 * prone to COW. We need to do the COW earlier during
5404			 * fork.
5405			 *
5406			 * When pre-allocating the page or copying data, we
5407			 * need to be without the pgtable locks since we could
5408			 * sleep during the process.
5409			 */
5410			if (!folio_test_anon(pte_folio)) {
5411				hugetlb_add_file_rmap(pte_folio);
5412			} else if (hugetlb_try_dup_anon_rmap(pte_folio, src_vma)) {
5413				pte_t src_pte_old = entry;
5414				struct folio *new_folio;
5415
5416				spin_unlock(src_ptl);
5417				spin_unlock(dst_ptl);
5418				/* Do not use reserve as it's private owned */
5419				new_folio = alloc_hugetlb_folio(dst_vma, addr, 1);
5420				if (IS_ERR(new_folio)) {
5421					folio_put(pte_folio);
5422					ret = PTR_ERR(new_folio);
5423					break;
5424				}
5425				ret = copy_user_large_folio(new_folio,
5426							    pte_folio,
5427							    addr, dst_vma);
5428				folio_put(pte_folio);
5429				if (ret) {
5430					folio_put(new_folio);
5431					break;
5432				}
 
 
 
5433
5434				/* Install the new hugetlb folio if src pte stable */
5435				dst_ptl = huge_pte_lock(h, dst, dst_pte);
5436				src_ptl = huge_pte_lockptr(h, src, src_pte);
5437				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5438				entry = huge_ptep_get(src_pte);
5439				if (!pte_same(src_pte_old, entry)) {
5440					restore_reserve_on_error(h, dst_vma, addr,
5441								new_folio);
5442					folio_put(new_folio);
5443					/* huge_ptep of dst_pte won't change as in child */
5444					goto again;
5445				}
5446				hugetlb_install_folio(dst_vma, dst_pte, addr,
5447						      new_folio, src_pte_old, sz);
5448				spin_unlock(src_ptl);
5449				spin_unlock(dst_ptl);
5450				continue;
5451			}
5452
5453			if (cow) {
5454				/*
5455				 * No need to notify as we are downgrading page
5456				 * table protection not changing it to point
5457				 * to a new page.
5458				 *
5459				 * See Documentation/mm/mmu_notifier.rst
5460				 */
5461				huge_ptep_set_wrprotect(src, addr, src_pte);
5462				entry = huge_pte_wrprotect(entry);
5463			}
5464
5465			if (!userfaultfd_wp(dst_vma))
5466				entry = huge_pte_clear_uffd_wp(entry);
5467
5468			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5469			hugetlb_count_add(npages, dst);
5470		}
5471		spin_unlock(src_ptl);
5472		spin_unlock(dst_ptl);
5473	}
5474
5475	if (cow) {
5476		raw_write_seqcount_end(&src->write_protect_seq);
5477		mmu_notifier_invalidate_range_end(&range);
5478	} else {
5479		hugetlb_vma_unlock_read(src_vma);
5480	}
5481
5482	return ret;
5483}
5484
5485static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
5486			  unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte,
5487			  unsigned long sz)
5488{
5489	struct hstate *h = hstate_vma(vma);
5490	struct mm_struct *mm = vma->vm_mm;
5491	spinlock_t *src_ptl, *dst_ptl;
5492	pte_t pte;
5493
5494	dst_ptl = huge_pte_lock(h, mm, dst_pte);
5495	src_ptl = huge_pte_lockptr(h, mm, src_pte);
5496
5497	/*
5498	 * We don't have to worry about the ordering of src and dst ptlocks
5499	 * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
5500	 */
5501	if (src_ptl != dst_ptl)
5502		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5503
5504	pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
5505	set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
5506
5507	if (src_ptl != dst_ptl)
5508		spin_unlock(src_ptl);
5509	spin_unlock(dst_ptl);
5510}
5511
5512int move_hugetlb_page_tables(struct vm_area_struct *vma,
5513			     struct vm_area_struct *new_vma,
5514			     unsigned long old_addr, unsigned long new_addr,
5515			     unsigned long len)
5516{
5517	struct hstate *h = hstate_vma(vma);
5518	struct address_space *mapping = vma->vm_file->f_mapping;
5519	unsigned long sz = huge_page_size(h);
5520	struct mm_struct *mm = vma->vm_mm;
5521	unsigned long old_end = old_addr + len;
5522	unsigned long last_addr_mask;
5523	pte_t *src_pte, *dst_pte;
5524	struct mmu_notifier_range range;
5525	bool shared_pmd = false;
5526
5527	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr,
5528				old_end);
5529	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5530	/*
5531	 * In case of shared PMDs, we should cover the maximum possible
5532	 * range.
5533	 */
5534	flush_cache_range(vma, range.start, range.end);
5535
5536	mmu_notifier_invalidate_range_start(&range);
5537	last_addr_mask = hugetlb_mask_last_page(h);
5538	/* Prevent race with file truncation */
5539	hugetlb_vma_lock_write(vma);
5540	i_mmap_lock_write(mapping);
5541	for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
5542		src_pte = hugetlb_walk(vma, old_addr, sz);
5543		if (!src_pte) {
5544			old_addr |= last_addr_mask;
5545			new_addr |= last_addr_mask;
5546			continue;
5547		}
5548		if (huge_pte_none(huge_ptep_get(src_pte)))
5549			continue;
5550
5551		if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) {
5552			shared_pmd = true;
5553			old_addr |= last_addr_mask;
5554			new_addr |= last_addr_mask;
5555			continue;
5556		}
5557
5558		dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
5559		if (!dst_pte)
5560			break;
5561
5562		move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
5563	}
5564
5565	if (shared_pmd)
5566		flush_hugetlb_tlb_range(vma, range.start, range.end);
5567	else
5568		flush_hugetlb_tlb_range(vma, old_end - len, old_end);
5569	mmu_notifier_invalidate_range_end(&range);
5570	i_mmap_unlock_write(mapping);
5571	hugetlb_vma_unlock_write(vma);
5572
5573	return len + old_addr - old_end;
5574}
5575
5576void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
5577			    unsigned long start, unsigned long end,
5578			    struct page *ref_page, zap_flags_t zap_flags)
5579{
5580	struct mm_struct *mm = vma->vm_mm;
5581	unsigned long address;
5582	pte_t *ptep;
5583	pte_t pte;
5584	spinlock_t *ptl;
5585	struct page *page;
5586	struct hstate *h = hstate_vma(vma);
5587	unsigned long sz = huge_page_size(h);
5588	unsigned long last_addr_mask;
5589	bool force_flush = false;
5590
5591	WARN_ON(!is_vm_hugetlb_page(vma));
5592	BUG_ON(start & ~huge_page_mask(h));
5593	BUG_ON(end & ~huge_page_mask(h));
5594
5595	/*
5596	 * This is a hugetlb vma, all the pte entries should point
5597	 * to huge page.
5598	 */
5599	tlb_change_page_size(tlb, sz);
5600	tlb_start_vma(tlb, vma);
5601
5602	last_addr_mask = hugetlb_mask_last_page(h);
 
 
 
 
 
 
5603	address = start;
5604	for (; address < end; address += sz) {
5605		ptep = hugetlb_walk(vma, address, sz);
5606		if (!ptep) {
5607			address |= last_addr_mask;
5608			continue;
5609		}
5610
5611		ptl = huge_pte_lock(h, mm, ptep);
5612		if (huge_pmd_unshare(mm, vma, address, ptep)) {
5613			spin_unlock(ptl);
5614			tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
5615			force_flush = true;
5616			address |= last_addr_mask;
 
5617			continue;
5618		}
5619
5620		pte = huge_ptep_get(ptep);
5621		if (huge_pte_none(pte)) {
5622			spin_unlock(ptl);
5623			continue;
5624		}
5625
5626		/*
5627		 * Migrating hugepage or HWPoisoned hugepage is already
5628		 * unmapped and its refcount is dropped, so just clear pte here.
5629		 */
5630		if (unlikely(!pte_present(pte))) {
5631			/*
5632			 * If the pte was wr-protected by uffd-wp in any of the
5633			 * swap forms, meanwhile the caller does not want to
5634			 * drop the uffd-wp bit in this zap, then replace the
5635			 * pte with a marker.
5636			 */
5637			if (pte_swp_uffd_wp_any(pte) &&
5638			    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5639				set_huge_pte_at(mm, address, ptep,
5640						make_pte_marker(PTE_MARKER_UFFD_WP),
5641						sz);
5642			else
5643				huge_pte_clear(mm, address, ptep, sz);
5644			spin_unlock(ptl);
5645			continue;
5646		}
5647
5648		page = pte_page(pte);
5649		/*
5650		 * If a reference page is supplied, it is because a specific
5651		 * page is being unmapped, not a range. Ensure the page we
5652		 * are about to unmap is the actual page of interest.
5653		 */
5654		if (ref_page) {
5655			if (page != ref_page) {
5656				spin_unlock(ptl);
5657				continue;
5658			}
5659			/*
5660			 * Mark the VMA as having unmapped its page so that
5661			 * future faults in this VMA will fail rather than
5662			 * looking like data was lost
5663			 */
5664			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
5665		}
5666
5667		pte = huge_ptep_get_and_clear(mm, address, ptep);
5668		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5669		if (huge_pte_dirty(pte))
5670			set_page_dirty(page);
5671		/* Leave a uffd-wp pte marker if needed */
5672		if (huge_pte_uffd_wp(pte) &&
5673		    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5674			set_huge_pte_at(mm, address, ptep,
5675					make_pte_marker(PTE_MARKER_UFFD_WP),
5676					sz);
5677		hugetlb_count_sub(pages_per_huge_page(h), mm);
5678		hugetlb_remove_rmap(page_folio(page));
5679
5680		spin_unlock(ptl);
5681		tlb_remove_page_size(tlb, page, huge_page_size(h));
5682		/*
5683		 * Bail out after unmapping reference page if supplied
5684		 */
5685		if (ref_page)
5686			break;
5687	}
 
5688	tlb_end_vma(tlb, vma);
5689
5690	/*
5691	 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
5692	 * could defer the flush until now, since by holding i_mmap_rwsem we
5693	 * guaranteed that the last refernece would not be dropped. But we must
5694	 * do the flushing before we return, as otherwise i_mmap_rwsem will be
5695	 * dropped and the last reference to the shared PMDs page might be
5696	 * dropped as well.
5697	 *
5698	 * In theory we could defer the freeing of the PMD pages as well, but
5699	 * huge_pmd_unshare() relies on the exact page_count for the PMD page to
5700	 * detect sharing, so we cannot defer the release of the page either.
5701	 * Instead, do flush now.
5702	 */
5703	if (force_flush)
5704		tlb_flush_mmu_tlbonly(tlb);
5705}
5706
5707void __hugetlb_zap_begin(struct vm_area_struct *vma,
5708			 unsigned long *start, unsigned long *end)
5709{
5710	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
5711		return;
5712
5713	adjust_range_if_pmd_sharing_possible(vma, start, end);
5714	hugetlb_vma_lock_write(vma);
5715	if (vma->vm_file)
5716		i_mmap_lock_write(vma->vm_file->f_mapping);
5717}
5718
5719void __hugetlb_zap_end(struct vm_area_struct *vma,
5720		       struct zap_details *details)
5721{
5722	zap_flags_t zap_flags = details ? details->zap_flags : 0;
5723
5724	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
5725		return;
5726
5727	if (zap_flags & ZAP_FLAG_UNMAP) {	/* final unmap */
5728		/*
5729		 * Unlock and free the vma lock before releasing i_mmap_rwsem.
5730		 * When the vma_lock is freed, this makes the vma ineligible
5731		 * for pmd sharing.  And, i_mmap_rwsem is required to set up
5732		 * pmd sharing.  This is important as page tables for this
5733		 * unmapped range will be asynchrously deleted.  If the page
5734		 * tables are shared, there will be issues when accessed by
5735		 * someone else.
5736		 */
5737		__hugetlb_vma_unlock_write_free(vma);
5738	} else {
5739		hugetlb_vma_unlock_write(vma);
5740	}
5741
5742	if (vma->vm_file)
5743		i_mmap_unlock_write(vma->vm_file->f_mapping);
5744}
5745
5746void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
5747			  unsigned long end, struct page *ref_page,
5748			  zap_flags_t zap_flags)
5749{
5750	struct mmu_notifier_range range;
5751	struct mmu_gather tlb;
5752
5753	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
5754				start, end);
5755	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5756	mmu_notifier_invalidate_range_start(&range);
5757	tlb_gather_mmu(&tlb, vma->vm_mm);
5758
5759	__unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags);
5760
5761	mmu_notifier_invalidate_range_end(&range);
5762	tlb_finish_mmu(&tlb);
5763}
5764
5765/*
5766 * This is called when the original mapper is failing to COW a MAP_PRIVATE
5767 * mapping it owns the reserve page for. The intention is to unmap the page
5768 * from other VMAs and let the children be SIGKILLed if they are faulting the
5769 * same region.
5770 */
5771static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
5772			      struct page *page, unsigned long address)
5773{
5774	struct hstate *h = hstate_vma(vma);
5775	struct vm_area_struct *iter_vma;
5776	struct address_space *mapping;
5777	pgoff_t pgoff;
5778
5779	/*
5780	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
5781	 * from page cache lookup which is in HPAGE_SIZE units.
5782	 */
5783	address = address & huge_page_mask(h);
5784	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
5785			vma->vm_pgoff;
5786	mapping = vma->vm_file->f_mapping;
5787
5788	/*
5789	 * Take the mapping lock for the duration of the table walk. As
5790	 * this mapping should be shared between all the VMAs,
5791	 * __unmap_hugepage_range() is called as the lock is already held
5792	 */
5793	i_mmap_lock_write(mapping);
5794	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
5795		/* Do not unmap the current VMA */
5796		if (iter_vma == vma)
5797			continue;
5798
5799		/*
5800		 * Shared VMAs have their own reserves and do not affect
5801		 * MAP_PRIVATE accounting but it is possible that a shared
5802		 * VMA is using the same page so check and skip such VMAs.
5803		 */
5804		if (iter_vma->vm_flags & VM_MAYSHARE)
5805			continue;
5806
5807		/*
5808		 * Unmap the page from other VMAs without their own reserves.
5809		 * They get marked to be SIGKILLed if they fault in these
5810		 * areas. This is because a future no-page fault on this VMA
5811		 * could insert a zeroed page instead of the data existing
5812		 * from the time of fork. This would look like data corruption
5813		 */
5814		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
5815			unmap_hugepage_range(iter_vma, address,
5816					     address + huge_page_size(h), page, 0);
5817	}
5818	i_mmap_unlock_write(mapping);
5819}
5820
5821/*
5822 * hugetlb_wp() should be called with page lock of the original hugepage held.
5823 * Called with hugetlb_fault_mutex_table held and pte_page locked so we
5824 * cannot race with other handlers or page migration.
5825 * Keep the pte_same checks anyway to make transition from the mutex easier.
5826 */
5827static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
5828		       unsigned long address, pte_t *ptep, unsigned int flags,
5829		       struct folio *pagecache_folio, spinlock_t *ptl)
5830{
5831	const bool unshare = flags & FAULT_FLAG_UNSHARE;
5832	pte_t pte = huge_ptep_get(ptep);
5833	struct hstate *h = hstate_vma(vma);
5834	struct folio *old_folio;
5835	struct folio *new_folio;
5836	int outside_reserve = 0;
5837	vm_fault_t ret = 0;
5838	unsigned long haddr = address & huge_page_mask(h);
5839	struct mmu_notifier_range range;
5840
5841	/*
5842	 * Never handle CoW for uffd-wp protected pages.  It should be only
5843	 * handled when the uffd-wp protection is removed.
5844	 *
5845	 * Note that only the CoW optimization path (in hugetlb_no_page())
5846	 * can trigger this, because hugetlb_fault() will always resolve
5847	 * uffd-wp bit first.
5848	 */
5849	if (!unshare && huge_pte_uffd_wp(pte))
5850		return 0;
5851
5852	/*
5853	 * hugetlb does not support FOLL_FORCE-style write faults that keep the
5854	 * PTE mapped R/O such as maybe_mkwrite() would do.
5855	 */
5856	if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE)))
5857		return VM_FAULT_SIGSEGV;
5858
5859	/* Let's take out MAP_SHARED mappings first. */
5860	if (vma->vm_flags & VM_MAYSHARE) {
5861		set_huge_ptep_writable(vma, haddr, ptep);
5862		return 0;
5863	}
5864
5865	old_folio = page_folio(pte_page(pte));
5866
5867	delayacct_wpcopy_start();
5868
5869retry_avoidcopy:
5870	/*
5871	 * If no-one else is actually using this page, we're the exclusive
5872	 * owner and can reuse this page.
5873	 */
5874	if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) {
5875		if (!PageAnonExclusive(&old_folio->page)) {
5876			folio_move_anon_rmap(old_folio, vma);
5877			SetPageAnonExclusive(&old_folio->page);
5878		}
5879		if (likely(!unshare))
5880			set_huge_ptep_writable(vma, haddr, ptep);
5881
5882		delayacct_wpcopy_end();
5883		return 0;
5884	}
5885	VM_BUG_ON_PAGE(folio_test_anon(old_folio) &&
5886		       PageAnonExclusive(&old_folio->page), &old_folio->page);
5887
5888	/*
5889	 * If the process that created a MAP_PRIVATE mapping is about to
5890	 * perform a COW due to a shared page count, attempt to satisfy
5891	 * the allocation without using the existing reserves. The pagecache
5892	 * page is used to determine if the reserve at this address was
5893	 * consumed or not. If reserves were used, a partial faulted mapping
5894	 * at the time of fork() could consume its reserves on COW instead
5895	 * of the full address range.
5896	 */
5897	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
5898			old_folio != pagecache_folio)
5899		outside_reserve = 1;
5900
5901	folio_get(old_folio);
5902
5903	/*
5904	 * Drop page table lock as buddy allocator may be called. It will
5905	 * be acquired again before returning to the caller, as expected.
5906	 */
5907	spin_unlock(ptl);
5908	new_folio = alloc_hugetlb_folio(vma, haddr, outside_reserve);
5909
5910	if (IS_ERR(new_folio)) {
5911		/*
5912		 * If a process owning a MAP_PRIVATE mapping fails to COW,
5913		 * it is due to references held by a child and an insufficient
5914		 * huge page pool. To guarantee the original mappers
5915		 * reliability, unmap the page from child processes. The child
5916		 * may get SIGKILLed if it later faults.
5917		 */
5918		if (outside_reserve) {
5919			struct address_space *mapping = vma->vm_file->f_mapping;
5920			pgoff_t idx;
5921			u32 hash;
5922
5923			folio_put(old_folio);
 
5924			/*
5925			 * Drop hugetlb_fault_mutex and vma_lock before
5926			 * unmapping.  unmapping needs to hold vma_lock
5927			 * in write mode.  Dropping vma_lock in read mode
5928			 * here is OK as COW mappings do not interact with
5929			 * PMD sharing.
5930			 *
5931			 * Reacquire both after unmap operation.
5932			 */
5933			idx = vma_hugecache_offset(h, vma, haddr);
5934			hash = hugetlb_fault_mutex_hash(mapping, idx);
5935			hugetlb_vma_unlock_read(vma);
5936			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 
5937
5938			unmap_ref_private(mm, vma, &old_folio->page, haddr);
5939
 
5940			mutex_lock(&hugetlb_fault_mutex_table[hash]);
5941			hugetlb_vma_lock_read(vma);
5942			spin_lock(ptl);
5943			ptep = hugetlb_walk(vma, haddr, huge_page_size(h));
5944			if (likely(ptep &&
5945				   pte_same(huge_ptep_get(ptep), pte)))
5946				goto retry_avoidcopy;
5947			/*
5948			 * race occurs while re-acquiring page table
5949			 * lock, and our job is done.
5950			 */
5951			delayacct_wpcopy_end();
5952			return 0;
5953		}
5954
5955		ret = vmf_error(PTR_ERR(new_folio));
5956		goto out_release_old;
5957	}
5958
5959	/*
5960	 * When the original hugepage is shared one, it does not have
5961	 * anon_vma prepared.
5962	 */
5963	if (unlikely(anon_vma_prepare(vma))) {
5964		ret = VM_FAULT_OOM;
5965		goto out_release_all;
5966	}
5967
5968	if (copy_user_large_folio(new_folio, old_folio, address, vma)) {
5969		ret = VM_FAULT_HWPOISON_LARGE;
5970		goto out_release_all;
5971	}
5972	__folio_mark_uptodate(new_folio);
5973
5974	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr,
5975				haddr + huge_page_size(h));
5976	mmu_notifier_invalidate_range_start(&range);
5977
5978	/*
5979	 * Retake the page table lock to check for racing updates
5980	 * before the page tables are altered
5981	 */
5982	spin_lock(ptl);
5983	ptep = hugetlb_walk(vma, haddr, huge_page_size(h));
5984	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
5985		pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare);
5986
5987		/* Break COW or unshare */
5988		huge_ptep_clear_flush(vma, haddr, ptep);
5989		hugetlb_remove_rmap(old_folio);
5990		hugetlb_add_new_anon_rmap(new_folio, vma, haddr);
5991		if (huge_pte_uffd_wp(pte))
5992			newpte = huge_pte_mkuffd_wp(newpte);
5993		set_huge_pte_at(mm, haddr, ptep, newpte, huge_page_size(h));
5994		folio_set_hugetlb_migratable(new_folio);
5995		/* Make the old page be freed below */
5996		new_folio = old_folio;
5997	}
5998	spin_unlock(ptl);
5999	mmu_notifier_invalidate_range_end(&range);
6000out_release_all:
6001	/*
6002	 * No restore in case of successful pagetable update (Break COW or
6003	 * unshare)
6004	 */
6005	if (new_folio != old_folio)
6006		restore_reserve_on_error(h, vma, haddr, new_folio);
6007	folio_put(new_folio);
6008out_release_old:
6009	folio_put(old_folio);
6010
6011	spin_lock(ptl); /* Caller expects lock to be held */
6012
6013	delayacct_wpcopy_end();
6014	return ret;
6015}
6016
 
 
 
 
 
 
 
 
 
 
 
 
 
6017/*
6018 * Return whether there is a pagecache page to back given address within VMA.
 
6019 */
6020static bool hugetlbfs_pagecache_present(struct hstate *h,
6021			struct vm_area_struct *vma, unsigned long address)
6022{
6023	struct address_space *mapping = vma->vm_file->f_mapping;
6024	pgoff_t idx = linear_page_index(vma, address);
6025	struct folio *folio;
6026
6027	folio = filemap_get_folio(mapping, idx);
6028	if (IS_ERR(folio))
6029		return false;
6030	folio_put(folio);
6031	return true;
 
 
6032}
6033
6034int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
6035			   pgoff_t idx)
6036{
6037	struct inode *inode = mapping->host;
6038	struct hstate *h = hstate_inode(inode);
6039	int err;
6040
6041	idx <<= huge_page_order(h);
6042	__folio_set_locked(folio);
6043	err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL);
6044
6045	if (unlikely(err)) {
6046		__folio_clear_locked(folio);
6047		return err;
6048	}
6049	folio_clear_hugetlb_restore_reserve(folio);
6050
6051	/*
6052	 * mark folio dirty so that it will not be removed from cache/file
6053	 * by non-hugetlbfs specific code paths.
6054	 */
6055	folio_mark_dirty(folio);
6056
6057	spin_lock(&inode->i_lock);
6058	inode->i_blocks += blocks_per_huge_page(h);
6059	spin_unlock(&inode->i_lock);
6060	return 0;
6061}
6062
6063static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
6064						  struct address_space *mapping,
6065						  pgoff_t idx,
6066						  unsigned int flags,
6067						  unsigned long haddr,
6068						  unsigned long addr,
6069						  unsigned long reason)
6070{
 
6071	u32 hash;
6072	struct vm_fault vmf = {
6073		.vma = vma,
6074		.address = haddr,
6075		.real_address = addr,
6076		.flags = flags,
6077
6078		/*
6079		 * Hard to debug if it ends up being
6080		 * used by a callee that assumes
6081		 * something about the other
6082		 * uninitialized fields... same as in
6083		 * memory.c
6084		 */
6085	};
6086
6087	/*
6088	 * vma_lock and hugetlb_fault_mutex must be dropped before handling
6089	 * userfault. Also mmap_lock could be dropped due to handling
6090	 * userfault, any vma operation should be careful from here.
6091	 */
6092	hugetlb_vma_unlock_read(vma);
6093	hash = hugetlb_fault_mutex_hash(mapping, idx);
6094	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6095	return handle_userfault(&vmf, reason);
6096}
6097
6098/*
6099 * Recheck pte with pgtable lock.  Returns true if pte didn't change, or
6100 * false if pte changed or is changing.
6101 */
6102static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm,
6103			       pte_t *ptep, pte_t old_pte)
6104{
6105	spinlock_t *ptl;
6106	bool same;
6107
6108	ptl = huge_pte_lock(h, mm, ptep);
6109	same = pte_same(huge_ptep_get(ptep), old_pte);
6110	spin_unlock(ptl);
6111
6112	return same;
6113}
6114
6115static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
6116			struct vm_area_struct *vma,
6117			struct address_space *mapping, pgoff_t idx,
6118			unsigned long address, pte_t *ptep,
6119			pte_t old_pte, unsigned int flags)
6120{
6121	struct hstate *h = hstate_vma(vma);
6122	vm_fault_t ret = VM_FAULT_SIGBUS;
6123	int anon_rmap = 0;
6124	unsigned long size;
6125	struct folio *folio;
6126	pte_t new_pte;
6127	spinlock_t *ptl;
6128	unsigned long haddr = address & huge_page_mask(h);
6129	bool new_folio, new_pagecache_folio = false;
6130	u32 hash = hugetlb_fault_mutex_hash(mapping, idx);
6131
6132	/*
6133	 * Currently, we are forced to kill the process in the event the
6134	 * original mapper has unmapped pages from the child due to a failed
6135	 * COW/unsharing. Warn that such a situation has occurred as it may not
6136	 * be obvious.
6137	 */
6138	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
6139		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
6140			   current->pid);
6141		goto out;
6142	}
6143
6144	/*
6145	 * Use page lock to guard against racing truncation
6146	 * before we get page_table_lock.
 
6147	 */
6148	new_folio = false;
6149	folio = filemap_lock_hugetlb_folio(h, mapping, idx);
6150	if (IS_ERR(folio)) {
6151		size = i_size_read(mapping->host) >> huge_page_shift(h);
6152		if (idx >= size)
6153			goto out;
 
 
6154		/* Check for page in userfault range */
6155		if (userfaultfd_missing(vma)) {
6156			/*
6157			 * Since hugetlb_no_page() was examining pte
6158			 * without pgtable lock, we need to re-test under
6159			 * lock because the pte may not be stable and could
6160			 * have changed from under us.  Try to detect
6161			 * either changed or during-changing ptes and retry
6162			 * properly when needed.
6163			 *
6164			 * Note that userfaultfd is actually fine with
6165			 * false positives (e.g. caused by pte changed),
6166			 * but not wrong logical events (e.g. caused by
6167			 * reading a pte during changing).  The latter can
6168			 * confuse the userspace, so the strictness is very
6169			 * much preferred.  E.g., MISSING event should
6170			 * never happen on the page after UFFDIO_COPY has
6171			 * correctly installed the page and returned.
6172			 */
6173			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
6174				ret = 0;
6175				goto out;
6176			}
6177
6178			return hugetlb_handle_userfault(vma, mapping, idx, flags,
6179							haddr, address,
6180							VM_UFFD_MISSING);
6181		}
6182
6183		folio = alloc_hugetlb_folio(vma, haddr, 0);
6184		if (IS_ERR(folio)) {
6185			/*
6186			 * Returning error will result in faulting task being
6187			 * sent SIGBUS.  The hugetlb fault mutex prevents two
6188			 * tasks from racing to fault in the same page which
6189			 * could result in false unable to allocate errors.
6190			 * Page migration does not take the fault mutex, but
6191			 * does a clear then write of pte's under page table
6192			 * lock.  Page fault code could race with migration,
6193			 * notice the clear pte and try to allocate a page
6194			 * here.  Before returning error, get ptl and make
6195			 * sure there really is no pte entry.
6196			 */
6197			if (hugetlb_pte_stable(h, mm, ptep, old_pte))
6198				ret = vmf_error(PTR_ERR(folio));
6199			else
6200				ret = 0;
 
6201			goto out;
6202		}
6203		clear_huge_page(&folio->page, address, pages_per_huge_page(h));
6204		__folio_mark_uptodate(folio);
6205		new_folio = true;
6206
6207		if (vma->vm_flags & VM_MAYSHARE) {
6208			int err = hugetlb_add_to_page_cache(folio, mapping, idx);
6209			if (err) {
6210				/*
6211				 * err can't be -EEXIST which implies someone
6212				 * else consumed the reservation since hugetlb
6213				 * fault mutex is held when add a hugetlb page
6214				 * to the page cache. So it's safe to call
6215				 * restore_reserve_on_error() here.
6216				 */
6217				restore_reserve_on_error(h, vma, haddr, folio);
6218				folio_put(folio);
6219				goto out;
6220			}
6221			new_pagecache_folio = true;
6222		} else {
6223			folio_lock(folio);
6224			if (unlikely(anon_vma_prepare(vma))) {
6225				ret = VM_FAULT_OOM;
6226				goto backout_unlocked;
6227			}
6228			anon_rmap = 1;
6229		}
6230	} else {
6231		/*
6232		 * If memory error occurs between mmap() and fault, some process
6233		 * don't have hwpoisoned swap entry for errored virtual address.
6234		 * So we need to block hugepage fault by PG_hwpoison bit check.
6235		 */
6236		if (unlikely(folio_test_hwpoison(folio))) {
6237			ret = VM_FAULT_HWPOISON_LARGE |
6238				VM_FAULT_SET_HINDEX(hstate_index(h));
6239			goto backout_unlocked;
6240		}
6241
6242		/* Check for page in userfault range. */
6243		if (userfaultfd_minor(vma)) {
6244			folio_unlock(folio);
6245			folio_put(folio);
6246			/* See comment in userfaultfd_missing() block above */
6247			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
6248				ret = 0;
6249				goto out;
6250			}
6251			return hugetlb_handle_userfault(vma, mapping, idx, flags,
6252							haddr, address,
6253							VM_UFFD_MINOR);
6254		}
6255	}
6256
6257	/*
6258	 * If we are going to COW a private mapping later, we examine the
6259	 * pending reservations for this page now. This will ensure that
6260	 * any allocations necessary to record that reservation occur outside
6261	 * the spinlock.
6262	 */
6263	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
6264		if (vma_needs_reservation(h, vma, haddr) < 0) {
6265			ret = VM_FAULT_OOM;
6266			goto backout_unlocked;
6267		}
6268		/* Just decrements count, does not deallocate */
6269		vma_end_reservation(h, vma, haddr);
6270	}
6271
6272	ptl = huge_pte_lock(h, mm, ptep);
6273	ret = 0;
6274	/* If pte changed from under us, retry */
6275	if (!pte_same(huge_ptep_get(ptep), old_pte))
6276		goto backout;
6277
6278	if (anon_rmap)
6279		hugetlb_add_new_anon_rmap(folio, vma, haddr);
6280	else
6281		hugetlb_add_file_rmap(folio);
6282	new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE)
 
6283				&& (vma->vm_flags & VM_SHARED)));
6284	/*
6285	 * If this pte was previously wr-protected, keep it wr-protected even
6286	 * if populated.
6287	 */
6288	if (unlikely(pte_marker_uffd_wp(old_pte)))
6289		new_pte = huge_pte_mkuffd_wp(new_pte);
6290	set_huge_pte_at(mm, haddr, ptep, new_pte, huge_page_size(h));
6291
6292	hugetlb_count_add(pages_per_huge_page(h), mm);
6293	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
6294		/* Optimization, do the COW without a second fault */
6295		ret = hugetlb_wp(mm, vma, address, ptep, flags, folio, ptl);
6296	}
6297
6298	spin_unlock(ptl);
6299
6300	/*
6301	 * Only set hugetlb_migratable in newly allocated pages.  Existing pages
6302	 * found in the pagecache may not have hugetlb_migratable if they have
6303	 * been isolated for migration.
6304	 */
6305	if (new_folio)
6306		folio_set_hugetlb_migratable(folio);
6307
6308	folio_unlock(folio);
6309out:
6310	hugetlb_vma_unlock_read(vma);
6311	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6312	return ret;
6313
6314backout:
6315	spin_unlock(ptl);
6316backout_unlocked:
6317	if (new_folio && !new_pagecache_folio)
6318		restore_reserve_on_error(h, vma, haddr, folio);
6319
6320	folio_unlock(folio);
6321	folio_put(folio);
6322	goto out;
6323}
6324
6325#ifdef CONFIG_SMP
6326u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
6327{
6328	unsigned long key[2];
6329	u32 hash;
6330
6331	key[0] = (unsigned long) mapping;
6332	key[1] = idx;
6333
6334	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
6335
6336	return hash & (num_fault_mutexes - 1);
6337}
6338#else
6339/*
6340 * For uniprocessor systems we always use a single mutex, so just
6341 * return 0 and avoid the hashing overhead.
6342 */
6343u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
6344{
6345	return 0;
6346}
6347#endif
6348
6349vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
6350			unsigned long address, unsigned int flags)
6351{
6352	pte_t *ptep, entry;
6353	spinlock_t *ptl;
6354	vm_fault_t ret;
6355	u32 hash;
6356	pgoff_t idx;
6357	struct folio *folio = NULL;
6358	struct folio *pagecache_folio = NULL;
6359	struct hstate *h = hstate_vma(vma);
6360	struct address_space *mapping;
6361	int need_wait_lock = 0;
6362	unsigned long haddr = address & huge_page_mask(h);
6363
6364	/* TODO: Handle faults under the VMA lock */
6365	if (flags & FAULT_FLAG_VMA_LOCK) {
6366		vma_end_read(vma);
6367		return VM_FAULT_RETRY;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6368	}
6369
6370	/*
6371	 * Serialize hugepage allocation and instantiation, so that we don't
6372	 * get spurious allocation failures if two CPUs race to instantiate
6373	 * the same page in the page cache.
6374	 */
6375	mapping = vma->vm_file->f_mapping;
6376	idx = vma_hugecache_offset(h, vma, haddr);
6377	hash = hugetlb_fault_mutex_hash(mapping, idx);
6378	mutex_lock(&hugetlb_fault_mutex_table[hash]);
6379
6380	/*
6381	 * Acquire vma lock before calling huge_pte_alloc and hold
6382	 * until finished with ptep.  This prevents huge_pmd_unshare from
6383	 * being called elsewhere and making the ptep no longer valid.
6384	 */
6385	hugetlb_vma_lock_read(vma);
6386	ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
6387	if (!ptep) {
6388		hugetlb_vma_unlock_read(vma);
6389		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6390		return VM_FAULT_OOM;
6391	}
6392
6393	entry = huge_ptep_get(ptep);
6394	if (huge_pte_none_mostly(entry)) {
6395		if (is_pte_marker(entry)) {
6396			pte_marker marker =
6397				pte_marker_get(pte_to_swp_entry(entry));
6398
6399			if (marker & PTE_MARKER_POISONED) {
6400				ret = VM_FAULT_HWPOISON_LARGE;
6401				goto out_mutex;
6402			}
6403		}
6404
6405		/*
6406		 * Other PTE markers should be handled the same way as none PTE.
6407		 *
6408		 * hugetlb_no_page will drop vma lock and hugetlb fault
6409		 * mutex internally, which make us return immediately.
6410		 */
6411		return hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
6412				      entry, flags);
6413	}
6414
6415	ret = 0;
6416
6417	/*
6418	 * entry could be a migration/hwpoison entry at this point, so this
6419	 * check prevents the kernel from going below assuming that we have
6420	 * an active hugepage in pagecache. This goto expects the 2nd page
6421	 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
6422	 * properly handle it.
6423	 */
6424	if (!pte_present(entry)) {
6425		if (unlikely(is_hugetlb_entry_migration(entry))) {
6426			/*
6427			 * Release the hugetlb fault lock now, but retain
6428			 * the vma lock, because it is needed to guard the
6429			 * huge_pte_lockptr() later in
6430			 * migration_entry_wait_huge(). The vma lock will
6431			 * be released there.
6432			 */
6433			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6434			migration_entry_wait_huge(vma, ptep);
6435			return 0;
6436		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
6437			ret = VM_FAULT_HWPOISON_LARGE |
6438			    VM_FAULT_SET_HINDEX(hstate_index(h));
6439		goto out_mutex;
6440	}
6441
6442	/*
6443	 * If we are going to COW/unshare the mapping later, we examine the
6444	 * pending reservations for this page now. This will ensure that any
6445	 * allocations necessary to record that reservation occur outside the
6446	 * spinlock. Also lookup the pagecache page now as it is used to
6447	 * determine if a reservation has been consumed.
 
6448	 */
6449	if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
6450	    !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) {
6451		if (vma_needs_reservation(h, vma, haddr) < 0) {
6452			ret = VM_FAULT_OOM;
6453			goto out_mutex;
6454		}
6455		/* Just decrements count, does not deallocate */
6456		vma_end_reservation(h, vma, haddr);
6457
6458		pagecache_folio = filemap_lock_hugetlb_folio(h, mapping, idx);
6459		if (IS_ERR(pagecache_folio))
6460			pagecache_folio = NULL;
6461	}
6462
6463	ptl = huge_pte_lock(h, mm, ptep);
6464
6465	/* Check for a racing update before calling hugetlb_wp() */
6466	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
6467		goto out_ptl;
6468
6469	/* Handle userfault-wp first, before trying to lock more pages */
6470	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) &&
6471	    (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
6472		if (!userfaultfd_wp_async(vma)) {
6473			struct vm_fault vmf = {
6474				.vma = vma,
6475				.address = haddr,
6476				.real_address = address,
6477				.flags = flags,
6478			};
6479
6480			spin_unlock(ptl);
6481			if (pagecache_folio) {
6482				folio_unlock(pagecache_folio);
6483				folio_put(pagecache_folio);
6484			}
6485			hugetlb_vma_unlock_read(vma);
6486			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6487			return handle_userfault(&vmf, VM_UFFD_WP);
6488		}
6489
6490		entry = huge_pte_clear_uffd_wp(entry);
6491		set_huge_pte_at(mm, haddr, ptep, entry,
6492				huge_page_size(hstate_vma(vma)));
6493		/* Fallthrough to CoW */
6494	}
6495
6496	/*
6497	 * hugetlb_wp() requires page locks of pte_page(entry) and
6498	 * pagecache_folio, so here we need take the former one
6499	 * when folio != pagecache_folio or !pagecache_folio.
6500	 */
6501	folio = page_folio(pte_page(entry));
6502	if (folio != pagecache_folio)
6503		if (!folio_trylock(folio)) {
6504			need_wait_lock = 1;
6505			goto out_ptl;
6506		}
6507
6508	folio_get(folio);
6509
6510	if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
6511		if (!huge_pte_write(entry)) {
6512			ret = hugetlb_wp(mm, vma, address, ptep, flags,
6513					 pagecache_folio, ptl);
6514			goto out_put_page;
6515		} else if (likely(flags & FAULT_FLAG_WRITE)) {
6516			entry = huge_pte_mkdirty(entry);
6517		}
 
6518	}
6519	entry = pte_mkyoung(entry);
6520	if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
6521						flags & FAULT_FLAG_WRITE))
6522		update_mmu_cache(vma, haddr, ptep);
6523out_put_page:
6524	if (folio != pagecache_folio)
6525		folio_unlock(folio);
6526	folio_put(folio);
6527out_ptl:
6528	spin_unlock(ptl);
6529
6530	if (pagecache_folio) {
6531		folio_unlock(pagecache_folio);
6532		folio_put(pagecache_folio);
6533	}
6534out_mutex:
6535	hugetlb_vma_unlock_read(vma);
6536	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 
6537	/*
6538	 * Generally it's safe to hold refcount during waiting page lock. But
6539	 * here we just wait to defer the next page fault to avoid busy loop and
6540	 * the page is not used after unlocked before returning from the current
6541	 * page fault. So we are safe from accessing freed page, even if we wait
6542	 * here without taking refcount.
6543	 */
6544	if (need_wait_lock)
6545		folio_wait_locked(folio);
6546	return ret;
6547}
6548
6549#ifdef CONFIG_USERFAULTFD
6550/*
6551 * Can probably be eliminated, but still used by hugetlb_mfill_atomic_pte().
6552 */
6553static struct folio *alloc_hugetlb_folio_vma(struct hstate *h,
6554		struct vm_area_struct *vma, unsigned long address)
6555{
6556	struct mempolicy *mpol;
6557	nodemask_t *nodemask;
6558	struct folio *folio;
6559	gfp_t gfp_mask;
6560	int node;
6561
6562	gfp_mask = htlb_alloc_mask(h);
6563	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
6564	folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask);
6565	mpol_cond_put(mpol);
6566
6567	return folio;
6568}
6569
6570/*
6571 * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte
6572 * with modifications for hugetlb pages.
6573 */
6574int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
6575			     struct vm_area_struct *dst_vma,
6576			     unsigned long dst_addr,
6577			     unsigned long src_addr,
6578			     uffd_flags_t flags,
6579			     struct folio **foliop)
 
6580{
6581	struct mm_struct *dst_mm = dst_vma->vm_mm;
6582	bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE);
6583	bool wp_enabled = (flags & MFILL_ATOMIC_WP);
6584	struct hstate *h = hstate_vma(dst_vma);
6585	struct address_space *mapping = dst_vma->vm_file->f_mapping;
6586	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
6587	unsigned long size;
6588	int vm_shared = dst_vma->vm_flags & VM_SHARED;
6589	pte_t _dst_pte;
6590	spinlock_t *ptl;
6591	int ret = -ENOMEM;
6592	struct folio *folio;
6593	int writable;
6594	bool folio_in_pagecache = false;
6595
6596	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) {
6597		ptl = huge_pte_lock(h, dst_mm, dst_pte);
6598
6599		/* Don't overwrite any existing PTEs (even markers) */
6600		if (!huge_pte_none(huge_ptep_get(dst_pte))) {
6601			spin_unlock(ptl);
6602			return -EEXIST;
6603		}
6604
6605		_dst_pte = make_pte_marker(PTE_MARKER_POISONED);
6606		set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte,
6607				huge_page_size(h));
6608
6609		/* No need to invalidate - it was non-present before */
6610		update_mmu_cache(dst_vma, dst_addr, dst_pte);
6611
6612		spin_unlock(ptl);
6613		return 0;
6614	}
6615
6616	if (is_continue) {
6617		ret = -EFAULT;
6618		folio = filemap_lock_hugetlb_folio(h, mapping, idx);
6619		if (IS_ERR(folio))
6620			goto out;
6621		folio_in_pagecache = true;
6622	} else if (!*foliop) {
6623		/* If a folio already exists, then it's UFFDIO_COPY for
6624		 * a non-missing case. Return -EEXIST.
6625		 */
6626		if (vm_shared &&
6627		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6628			ret = -EEXIST;
6629			goto out;
6630		}
6631
6632		folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0);
6633		if (IS_ERR(folio)) {
6634			ret = -ENOMEM;
6635			goto out;
6636		}
6637
6638		ret = copy_folio_from_user(folio, (const void __user *) src_addr,
6639					   false);
 
6640
6641		/* fallback to copy_from_user outside mmap_lock */
6642		if (unlikely(ret)) {
6643			ret = -ENOENT;
6644			/* Free the allocated folio which may have
6645			 * consumed a reservation.
6646			 */
6647			restore_reserve_on_error(h, dst_vma, dst_addr, folio);
6648			folio_put(folio);
6649
6650			/* Allocate a temporary folio to hold the copied
6651			 * contents.
6652			 */
6653			folio = alloc_hugetlb_folio_vma(h, dst_vma, dst_addr);
6654			if (!folio) {
6655				ret = -ENOMEM;
6656				goto out;
6657			}
6658			*foliop = folio;
6659			/* Set the outparam foliop and return to the caller to
6660			 * copy the contents outside the lock. Don't free the
6661			 * folio.
6662			 */
6663			goto out;
6664		}
6665	} else {
6666		if (vm_shared &&
6667		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6668			folio_put(*foliop);
6669			ret = -EEXIST;
6670			*foliop = NULL;
6671			goto out;
6672		}
6673
6674		folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0);
6675		if (IS_ERR(folio)) {
6676			folio_put(*foliop);
6677			ret = -ENOMEM;
6678			*foliop = NULL;
6679			goto out;
6680		}
6681		ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma);
6682		folio_put(*foliop);
6683		*foliop = NULL;
6684		if (ret) {
6685			folio_put(folio);
6686			goto out;
6687		}
 
 
 
6688	}
6689
6690	/*
6691	 * The memory barrier inside __folio_mark_uptodate makes sure that
6692	 * preceding stores to the page contents become visible before
6693	 * the set_pte_at() write.
6694	 */
6695	__folio_mark_uptodate(folio);
6696
6697	/* Add shared, newly allocated pages to the page cache. */
6698	if (vm_shared && !is_continue) {
6699		size = i_size_read(mapping->host) >> huge_page_shift(h);
6700		ret = -EFAULT;
6701		if (idx >= size)
6702			goto out_release_nounlock;
6703
6704		/*
6705		 * Serialization between remove_inode_hugepages() and
6706		 * hugetlb_add_to_page_cache() below happens through the
6707		 * hugetlb_fault_mutex_table that here must be hold by
6708		 * the caller.
6709		 */
6710		ret = hugetlb_add_to_page_cache(folio, mapping, idx);
6711		if (ret)
6712			goto out_release_nounlock;
6713		folio_in_pagecache = true;
6714	}
6715
6716	ptl = huge_pte_lock(h, dst_mm, dst_pte);
 
6717
6718	ret = -EIO;
6719	if (folio_test_hwpoison(folio))
 
 
 
 
 
 
 
 
 
 
6720		goto out_release_unlock;
6721
6722	/*
6723	 * We allow to overwrite a pte marker: consider when both MISSING|WP
6724	 * registered, we firstly wr-protect a none pte which has no page cache
6725	 * page backing it, then access the page.
6726	 */
6727	ret = -EEXIST;
6728	if (!huge_pte_none_mostly(huge_ptep_get(dst_pte)))
6729		goto out_release_unlock;
6730
6731	if (folio_in_pagecache)
6732		hugetlb_add_file_rmap(folio);
6733	else
6734		hugetlb_add_new_anon_rmap(folio, dst_vma, dst_addr);
 
 
6735
6736	/*
6737	 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
6738	 * with wp flag set, don't set pte write bit.
6739	 */
6740	if (wp_enabled || (is_continue && !vm_shared))
6741		writable = 0;
6742	else
6743		writable = dst_vma->vm_flags & VM_WRITE;
6744
6745	_dst_pte = make_huge_pte(dst_vma, &folio->page, writable);
6746	/*
6747	 * Always mark UFFDIO_COPY page dirty; note that this may not be
6748	 * extremely important for hugetlbfs for now since swapping is not
6749	 * supported, but we should still be clear in that this page cannot be
6750	 * thrown away at will, even if write bit not set.
6751	 */
6752	_dst_pte = huge_pte_mkdirty(_dst_pte);
6753	_dst_pte = pte_mkyoung(_dst_pte);
6754
6755	if (wp_enabled)
6756		_dst_pte = huge_pte_mkuffd_wp(_dst_pte);
6757
6758	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, huge_page_size(h));
6759
 
 
6760	hugetlb_count_add(pages_per_huge_page(h), dst_mm);
6761
6762	/* No need to invalidate - it was non-present before */
6763	update_mmu_cache(dst_vma, dst_addr, dst_pte);
6764
6765	spin_unlock(ptl);
6766	if (!is_continue)
6767		folio_set_hugetlb_migratable(folio);
6768	if (vm_shared || is_continue)
6769		folio_unlock(folio);
6770	ret = 0;
6771out:
6772	return ret;
6773out_release_unlock:
6774	spin_unlock(ptl);
6775	if (vm_shared || is_continue)
6776		folio_unlock(folio);
6777out_release_nounlock:
6778	if (!folio_in_pagecache)
6779		restore_reserve_on_error(h, dst_vma, dst_addr, folio);
6780	folio_put(folio);
6781	goto out;
6782}
6783#endif /* CONFIG_USERFAULTFD */
6784
6785struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma,
6786				      unsigned long address, unsigned int flags,
6787				      unsigned int *page_mask)
6788{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6789	struct hstate *h = hstate_vma(vma);
6790	struct mm_struct *mm = vma->vm_mm;
6791	unsigned long haddr = address & huge_page_mask(h);
6792	struct page *page = NULL;
6793	spinlock_t *ptl;
6794	pte_t *pte, entry;
6795	int ret;
6796
6797	hugetlb_vma_lock_read(vma);
6798	pte = hugetlb_walk(vma, haddr, huge_page_size(h));
6799	if (!pte)
6800		goto out_unlock;
 
6801
6802	ptl = huge_pte_lock(h, mm, pte);
6803	entry = huge_ptep_get(pte);
6804	if (pte_present(entry)) {
6805		page = pte_page(entry);
 
 
 
 
6806
6807		if (!huge_pte_write(entry)) {
6808			if (flags & FOLL_WRITE) {
6809				page = NULL;
6810				goto out;
6811			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6812
6813			if (gup_must_unshare(vma, flags, page)) {
6814				/* Tell the caller to do unsharing */
6815				page = ERR_PTR(-EMLINK);
6816				goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6817			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6818		}
6819
6820		page = nth_page(page, ((address & ~huge_page_mask(h)) >> PAGE_SHIFT));
 
6821
6822		/*
6823		 * Note that page may be a sub-page, and with vmemmap
6824		 * optimizations the page struct may be read only.
6825		 * try_grab_page() will increase the ref count on the
6826		 * head page, so this will be OK.
6827		 *
6828		 * try_grab_page() should always be able to get the page here,
6829		 * because we hold the ptl lock and have verified pte_present().
6830		 */
6831		ret = try_grab_page(page, flags);
 
 
 
 
 
 
 
 
6832
6833		if (WARN_ON_ONCE(ret)) {
6834			page = ERR_PTR(ret);
6835			goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6836		}
6837
6838		*page_mask = (1U << huge_page_order(h)) - 1;
6839	}
6840out:
6841	spin_unlock(ptl);
6842out_unlock:
6843	hugetlb_vma_unlock_read(vma);
6844
 
 
 
6845	/*
6846	 * Fixup retval for dump requests: if pagecache doesn't exist,
6847	 * don't try to allocate a new page but just skip it.
 
6848	 */
6849	if (!page && (flags & FOLL_DUMP) &&
6850	    !hugetlbfs_pagecache_present(h, vma, address))
6851		page = ERR_PTR(-EFAULT);
6852
6853	return page;
6854}
6855
6856long hugetlb_change_protection(struct vm_area_struct *vma,
6857		unsigned long address, unsigned long end,
6858		pgprot_t newprot, unsigned long cp_flags)
6859{
6860	struct mm_struct *mm = vma->vm_mm;
6861	unsigned long start = address;
6862	pte_t *ptep;
6863	pte_t pte;
6864	struct hstate *h = hstate_vma(vma);
6865	long pages = 0, psize = huge_page_size(h);
6866	bool shared_pmd = false;
6867	struct mmu_notifier_range range;
6868	unsigned long last_addr_mask;
6869	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
6870	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
6871
6872	/*
6873	 * In the case of shared PMDs, the area to flush could be beyond
6874	 * start/end.  Set range.start/range.end to cover the maximum possible
6875	 * range if PMD sharing is possible.
6876	 */
6877	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
6878				0, mm, start, end);
6879	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
6880
6881	BUG_ON(address >= end);
6882	flush_cache_range(vma, range.start, range.end);
6883
6884	mmu_notifier_invalidate_range_start(&range);
6885	hugetlb_vma_lock_write(vma);
6886	i_mmap_lock_write(vma->vm_file->f_mapping);
6887	last_addr_mask = hugetlb_mask_last_page(h);
6888	for (; address < end; address += psize) {
6889		spinlock_t *ptl;
6890		ptep = hugetlb_walk(vma, address, psize);
6891		if (!ptep) {
6892			if (!uffd_wp) {
6893				address |= last_addr_mask;
6894				continue;
6895			}
6896			/*
6897			 * Userfaultfd wr-protect requires pgtable
6898			 * pre-allocations to install pte markers.
6899			 */
6900			ptep = huge_pte_alloc(mm, vma, address, psize);
6901			if (!ptep) {
6902				pages = -ENOMEM;
6903				break;
6904			}
6905		}
6906		ptl = huge_pte_lock(h, mm, ptep);
6907		if (huge_pmd_unshare(mm, vma, address, ptep)) {
6908			/*
6909			 * When uffd-wp is enabled on the vma, unshare
6910			 * shouldn't happen at all.  Warn about it if it
6911			 * happened due to some reason.
6912			 */
6913			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
6914			pages++;
6915			spin_unlock(ptl);
6916			shared_pmd = true;
6917			address |= last_addr_mask;
6918			continue;
6919		}
6920		pte = huge_ptep_get(ptep);
6921		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
6922			/* Nothing to do. */
6923		} else if (unlikely(is_hugetlb_entry_migration(pte))) {
 
 
6924			swp_entry_t entry = pte_to_swp_entry(pte);
6925			struct page *page = pfn_swap_entry_to_page(entry);
6926			pte_t newpte = pte;
6927
6928			if (is_writable_migration_entry(entry)) {
6929				if (PageAnon(page))
6930					entry = make_readable_exclusive_migration_entry(
6931								swp_offset(entry));
6932				else
6933					entry = make_readable_migration_entry(
6934								swp_offset(entry));
6935				newpte = swp_entry_to_pte(entry);
 
 
6936				pages++;
6937			}
6938
6939			if (uffd_wp)
6940				newpte = pte_swp_mkuffd_wp(newpte);
6941			else if (uffd_wp_resolve)
6942				newpte = pte_swp_clear_uffd_wp(newpte);
6943			if (!pte_same(pte, newpte))
6944				set_huge_pte_at(mm, address, ptep, newpte, psize);
6945		} else if (unlikely(is_pte_marker(pte))) {
6946			/* No other markers apply for now. */
6947			WARN_ON_ONCE(!pte_marker_uffd_wp(pte));
6948			if (uffd_wp_resolve)
6949				/* Safe to modify directly (non-present->none). */
6950				huge_pte_clear(mm, address, ptep, psize);
6951		} else if (!huge_pte_none(pte)) {
6952			pte_t old_pte;
6953			unsigned int shift = huge_page_shift(hstate_vma(vma));
6954
6955			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
6956			pte = huge_pte_modify(old_pte, newprot);
6957			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
6958			if (uffd_wp)
6959				pte = huge_pte_mkuffd_wp(pte);
6960			else if (uffd_wp_resolve)
6961				pte = huge_pte_clear_uffd_wp(pte);
6962			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
6963			pages++;
6964		} else {
6965			/* None pte */
6966			if (unlikely(uffd_wp))
6967				/* Safe to modify directly (none->non-present). */
6968				set_huge_pte_at(mm, address, ptep,
6969						make_pte_marker(PTE_MARKER_UFFD_WP),
6970						psize);
6971		}
6972		spin_unlock(ptl);
6973	}
6974	/*
6975	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
6976	 * may have cleared our pud entry and done put_page on the page table:
6977	 * once we release i_mmap_rwsem, another task can do the final put_page
6978	 * and that page table be reused and filled with junk.  If we actually
6979	 * did unshare a page of pmds, flush the range corresponding to the pud.
6980	 */
6981	if (shared_pmd)
6982		flush_hugetlb_tlb_range(vma, range.start, range.end);
6983	else
6984		flush_hugetlb_tlb_range(vma, start, end);
6985	/*
6986	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are
6987	 * downgrading page table protection not changing it to point to a new
6988	 * page.
6989	 *
6990	 * See Documentation/mm/mmu_notifier.rst
6991	 */
6992	i_mmap_unlock_write(vma->vm_file->f_mapping);
6993	hugetlb_vma_unlock_write(vma);
6994	mmu_notifier_invalidate_range_end(&range);
6995
6996	return pages > 0 ? (pages << h->order) : pages;
6997}
6998
6999/* Return true if reservation was successful, false otherwise.  */
7000bool hugetlb_reserve_pages(struct inode *inode,
7001					long from, long to,
7002					struct vm_area_struct *vma,
7003					vm_flags_t vm_flags)
7004{
7005	long chg = -1, add = -1;
7006	struct hstate *h = hstate_inode(inode);
7007	struct hugepage_subpool *spool = subpool_inode(inode);
7008	struct resv_map *resv_map;
7009	struct hugetlb_cgroup *h_cg = NULL;
7010	long gbl_reserve, regions_needed = 0;
7011
7012	/* This should never happen */
7013	if (from > to) {
7014		VM_WARN(1, "%s called with a negative range\n", __func__);
7015		return false;
7016	}
7017
7018	/*
7019	 * vma specific semaphore used for pmd sharing and fault/truncation
7020	 * synchronization
7021	 */
7022	hugetlb_vma_lock_alloc(vma);
7023
7024	/*
7025	 * Only apply hugepage reservation if asked. At fault time, an
7026	 * attempt will be made for VM_NORESERVE to allocate a page
7027	 * without using reserves
7028	 */
7029	if (vm_flags & VM_NORESERVE)
7030		return true;
7031
7032	/*
7033	 * Shared mappings base their reservation on the number of pages that
7034	 * are already allocated on behalf of the file. Private mappings need
7035	 * to reserve the full area even if read-only as mprotect() may be
7036	 * called to make the mapping read-write. Assume !vma is a shm mapping
7037	 */
7038	if (!vma || vma->vm_flags & VM_MAYSHARE) {
7039		/*
7040		 * resv_map can not be NULL as hugetlb_reserve_pages is only
7041		 * called for inodes for which resv_maps were created (see
7042		 * hugetlbfs_get_inode).
7043		 */
7044		resv_map = inode_resv_map(inode);
7045
7046		chg = region_chg(resv_map, from, to, &regions_needed);
 
7047	} else {
7048		/* Private mapping. */
7049		resv_map = resv_map_alloc();
7050		if (!resv_map)
7051			goto out_err;
7052
7053		chg = to - from;
7054
7055		set_vma_resv_map(vma, resv_map);
7056		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
7057	}
7058
7059	if (chg < 0)
7060		goto out_err;
7061
7062	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
7063				chg * pages_per_huge_page(h), &h_cg) < 0)
7064		goto out_err;
7065
7066	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
7067		/* For private mappings, the hugetlb_cgroup uncharge info hangs
7068		 * of the resv_map.
7069		 */
7070		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
7071	}
7072
7073	/*
7074	 * There must be enough pages in the subpool for the mapping. If
7075	 * the subpool has a minimum size, there may be some global
7076	 * reservations already in place (gbl_reserve).
7077	 */
7078	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
7079	if (gbl_reserve < 0)
7080		goto out_uncharge_cgroup;
7081
7082	/*
7083	 * Check enough hugepages are available for the reservation.
7084	 * Hand the pages back to the subpool if there are not
7085	 */
7086	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
7087		goto out_put_pages;
7088
7089	/*
7090	 * Account for the reservations made. Shared mappings record regions
7091	 * that have reservations as they are shared by multiple VMAs.
7092	 * When the last VMA disappears, the region map says how much
7093	 * the reservation was and the page cache tells how much of
7094	 * the reservation was consumed. Private mappings are per-VMA and
7095	 * only the consumed reservations are tracked. When the VMA
7096	 * disappears, the original reservation is the VMA size and the
7097	 * consumed reservations are stored in the map. Hence, nothing
7098	 * else has to be done for private mappings here
7099	 */
7100	if (!vma || vma->vm_flags & VM_MAYSHARE) {
7101		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
7102
7103		if (unlikely(add < 0)) {
7104			hugetlb_acct_memory(h, -gbl_reserve);
7105			goto out_put_pages;
7106		} else if (unlikely(chg > add)) {
7107			/*
7108			 * pages in this range were added to the reserve
7109			 * map between region_chg and region_add.  This
7110			 * indicates a race with alloc_hugetlb_folio.  Adjust
7111			 * the subpool and reserve counts modified above
7112			 * based on the difference.
7113			 */
7114			long rsv_adjust;
7115
7116			/*
7117			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
7118			 * reference to h_cg->css. See comment below for detail.
7119			 */
7120			hugetlb_cgroup_uncharge_cgroup_rsvd(
7121				hstate_index(h),
7122				(chg - add) * pages_per_huge_page(h), h_cg);
7123
7124			rsv_adjust = hugepage_subpool_put_pages(spool,
7125								chg - add);
7126			hugetlb_acct_memory(h, -rsv_adjust);
7127		} else if (h_cg) {
7128			/*
7129			 * The file_regions will hold their own reference to
7130			 * h_cg->css. So we should release the reference held
7131			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
7132			 * done.
7133			 */
7134			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
7135		}
7136	}
7137	return true;
7138
7139out_put_pages:
7140	/* put back original number of pages, chg */
7141	(void)hugepage_subpool_put_pages(spool, chg);
7142out_uncharge_cgroup:
7143	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
7144					    chg * pages_per_huge_page(h), h_cg);
7145out_err:
7146	hugetlb_vma_lock_free(vma);
7147	if (!vma || vma->vm_flags & VM_MAYSHARE)
7148		/* Only call region_abort if the region_chg succeeded but the
7149		 * region_add failed or didn't run.
7150		 */
7151		if (chg >= 0 && add < 0)
7152			region_abort(resv_map, from, to, regions_needed);
7153	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
7154		kref_put(&resv_map->refs, resv_map_release);
7155		set_vma_resv_map(vma, NULL);
7156	}
7157	return false;
7158}
7159
7160long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
7161								long freed)
7162{
7163	struct hstate *h = hstate_inode(inode);
7164	struct resv_map *resv_map = inode_resv_map(inode);
7165	long chg = 0;
7166	struct hugepage_subpool *spool = subpool_inode(inode);
7167	long gbl_reserve;
7168
7169	/*
7170	 * Since this routine can be called in the evict inode path for all
7171	 * hugetlbfs inodes, resv_map could be NULL.
7172	 */
7173	if (resv_map) {
7174		chg = region_del(resv_map, start, end);
7175		/*
7176		 * region_del() can fail in the rare case where a region
7177		 * must be split and another region descriptor can not be
7178		 * allocated.  If end == LONG_MAX, it will not fail.
7179		 */
7180		if (chg < 0)
7181			return chg;
7182	}
7183
7184	spin_lock(&inode->i_lock);
7185	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
7186	spin_unlock(&inode->i_lock);
7187
7188	/*
7189	 * If the subpool has a minimum size, the number of global
7190	 * reservations to be released may be adjusted.
7191	 *
7192	 * Note that !resv_map implies freed == 0. So (chg - freed)
7193	 * won't go negative.
7194	 */
7195	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
7196	hugetlb_acct_memory(h, -gbl_reserve);
7197
7198	return 0;
7199}
7200
7201#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
7202static unsigned long page_table_shareable(struct vm_area_struct *svma,
7203				struct vm_area_struct *vma,
7204				unsigned long addr, pgoff_t idx)
7205{
7206	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
7207				svma->vm_start;
7208	unsigned long sbase = saddr & PUD_MASK;
7209	unsigned long s_end = sbase + PUD_SIZE;
7210
7211	/* Allow segments to share if only one is marked locked */
7212	unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED_MASK;
7213	unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED_MASK;
7214
7215	/*
7216	 * match the virtual addresses, permission and the alignment of the
7217	 * page table page.
7218	 *
7219	 * Also, vma_lock (vm_private_data) is required for sharing.
7220	 */
7221	if (pmd_index(addr) != pmd_index(saddr) ||
7222	    vm_flags != svm_flags ||
7223	    !range_in_vma(svma, sbase, s_end) ||
7224	    !svma->vm_private_data)
7225		return 0;
7226
7227	return saddr;
7228}
7229
7230bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
7231{
7232	unsigned long start = addr & PUD_MASK;
7233	unsigned long end = start + PUD_SIZE;
7234
7235#ifdef CONFIG_USERFAULTFD
7236	if (uffd_disable_huge_pmd_share(vma))
7237		return false;
7238#endif
7239	/*
7240	 * check on proper vm_flags and page table alignment
7241	 */
7242	if (!(vma->vm_flags & VM_MAYSHARE))
7243		return false;
7244	if (!vma->vm_private_data)	/* vma lock required for sharing */
7245		return false;
7246	if (!range_in_vma(vma, start, end))
 
 
 
 
7247		return false;
7248	return true;
 
7249}
7250
7251/*
7252 * Determine if start,end range within vma could be mapped by shared pmd.
7253 * If yes, adjust start and end to cover range associated with possible
7254 * shared pmd mappings.
7255 */
7256void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
7257				unsigned long *start, unsigned long *end)
7258{
7259	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
7260		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
7261
7262	/*
7263	 * vma needs to span at least one aligned PUD size, and the range
7264	 * must be at least partially within in.
7265	 */
7266	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
7267		(*end <= v_start) || (*start >= v_end))
7268		return;
7269
7270	/* Extend the range to be PUD aligned for a worst case scenario */
7271	if (*start > v_start)
7272		*start = ALIGN_DOWN(*start, PUD_SIZE);
7273
7274	if (*end < v_end)
7275		*end = ALIGN(*end, PUD_SIZE);
7276}
7277
7278/*
7279 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
7280 * and returns the corresponding pte. While this is not necessary for the
7281 * !shared pmd case because we can allocate the pmd later as well, it makes the
7282 * code much cleaner. pmd allocation is essential for the shared case because
7283 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
7284 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
7285 * bad pmd for sharing.
 
 
 
 
 
 
 
 
7286 */
7287pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
7288		      unsigned long addr, pud_t *pud)
7289{
7290	struct address_space *mapping = vma->vm_file->f_mapping;
7291	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
7292			vma->vm_pgoff;
7293	struct vm_area_struct *svma;
7294	unsigned long saddr;
7295	pte_t *spte = NULL;
7296	pte_t *pte;
 
7297
7298	i_mmap_lock_read(mapping);
7299	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
7300		if (svma == vma)
7301			continue;
7302
7303		saddr = page_table_shareable(svma, vma, addr, idx);
7304		if (saddr) {
7305			spte = hugetlb_walk(svma, saddr,
7306					    vma_mmu_pagesize(svma));
7307			if (spte) {
7308				get_page(virt_to_page(spte));
7309				break;
7310			}
7311		}
7312	}
7313
7314	if (!spte)
7315		goto out;
7316
7317	spin_lock(&mm->page_table_lock);
7318	if (pud_none(*pud)) {
7319		pud_populate(mm, pud,
7320				(pmd_t *)((unsigned long)spte & PAGE_MASK));
7321		mm_inc_nr_pmds(mm);
7322	} else {
7323		put_page(virt_to_page(spte));
7324	}
7325	spin_unlock(&mm->page_table_lock);
7326out:
7327	pte = (pte_t *)pmd_alloc(mm, pud, addr);
7328	i_mmap_unlock_read(mapping);
7329	return pte;
7330}
7331
7332/*
7333 * unmap huge page backed by shared pte.
7334 *
7335 * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
7336 * indicated by page_count > 1, unmap is achieved by clearing pud and
7337 * decrementing the ref count. If count == 1, the pte page is not shared.
7338 *
7339 * Called with page table lock held.
7340 *
7341 * returns: 1 successfully unmapped a shared pte page
7342 *	    0 the underlying pte page is not shared, or it is the last user
7343 */
7344int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7345					unsigned long addr, pte_t *ptep)
7346{
7347	pgd_t *pgd = pgd_offset(mm, addr);
7348	p4d_t *p4d = p4d_offset(pgd, addr);
7349	pud_t *pud = pud_offset(p4d, addr);
7350
7351	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
7352	hugetlb_vma_assert_locked(vma);
7353	BUG_ON(page_count(virt_to_page(ptep)) == 0);
7354	if (page_count(virt_to_page(ptep)) == 1)
7355		return 0;
7356
7357	pud_clear(pud);
7358	put_page(virt_to_page(ptep));
7359	mm_dec_nr_pmds(mm);
 
7360	return 1;
7361}
7362
7363#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
7364
7365pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
7366		      unsigned long addr, pud_t *pud)
7367{
7368	return NULL;
7369}
7370
7371int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7372				unsigned long addr, pte_t *ptep)
7373{
7374	return 0;
7375}
7376
7377void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
7378				unsigned long *start, unsigned long *end)
7379{
7380}
7381
7382bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
7383{
7384	return false;
7385}
7386#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
7387
7388#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
7389pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
7390			unsigned long addr, unsigned long sz)
7391{
7392	pgd_t *pgd;
7393	p4d_t *p4d;
7394	pud_t *pud;
7395	pte_t *pte = NULL;
7396
7397	pgd = pgd_offset(mm, addr);
7398	p4d = p4d_alloc(mm, pgd, addr);
7399	if (!p4d)
7400		return NULL;
7401	pud = pud_alloc(mm, p4d, addr);
7402	if (pud) {
7403		if (sz == PUD_SIZE) {
7404			pte = (pte_t *)pud;
7405		} else {
7406			BUG_ON(sz != PMD_SIZE);
7407			if (want_pmd_share(vma, addr) && pud_none(*pud))
7408				pte = huge_pmd_share(mm, vma, addr, pud);
7409			else
7410				pte = (pte_t *)pmd_alloc(mm, pud, addr);
7411		}
7412	}
7413
7414	if (pte) {
7415		pte_t pteval = ptep_get_lockless(pte);
7416
7417		BUG_ON(pte_present(pteval) && !pte_huge(pteval));
7418	}
7419
7420	return pte;
7421}
7422
7423/*
7424 * huge_pte_offset() - Walk the page table to resolve the hugepage
7425 * entry at address @addr
7426 *
7427 * Return: Pointer to page table entry (PUD or PMD) for
7428 * address @addr, or NULL if a !p*d_present() entry is encountered and the
7429 * size @sz doesn't match the hugepage size at this level of the page
7430 * table.
7431 */
7432pte_t *huge_pte_offset(struct mm_struct *mm,
7433		       unsigned long addr, unsigned long sz)
7434{
7435	pgd_t *pgd;
7436	p4d_t *p4d;
7437	pud_t *pud;
7438	pmd_t *pmd;
7439
7440	pgd = pgd_offset(mm, addr);
7441	if (!pgd_present(*pgd))
7442		return NULL;
7443	p4d = p4d_offset(pgd, addr);
7444	if (!p4d_present(*p4d))
7445		return NULL;
7446
7447	pud = pud_offset(p4d, addr);
7448	if (sz == PUD_SIZE)
7449		/* must be pud huge, non-present or none */
7450		return (pte_t *)pud;
7451	if (!pud_present(*pud))
7452		return NULL;
7453	/* must have a valid entry and size to go further */
7454
7455	pmd = pmd_offset(pud, addr);
7456	/* must be pmd huge, non-present or none */
7457	return (pte_t *)pmd;
7458}
7459
 
 
7460/*
7461 * Return a mask that can be used to update an address to the last huge
7462 * page in a page table page mapping size.  Used to skip non-present
7463 * page table entries when linearly scanning address ranges.  Architectures
7464 * with unique huge page to page table relationships can define their own
7465 * version of this routine.
7466 */
7467unsigned long hugetlb_mask_last_page(struct hstate *h)
7468{
7469	unsigned long hp_size = huge_page_size(h);
7470
7471	if (hp_size == PUD_SIZE)
7472		return P4D_SIZE - PUD_SIZE;
7473	else if (hp_size == PMD_SIZE)
7474		return PUD_SIZE - PMD_SIZE;
7475	else
7476		return 0UL;
7477}
7478
7479#else
 
 
 
 
 
 
7480
7481/* See description above.  Architectures can provide their own version. */
7482__weak unsigned long hugetlb_mask_last_page(struct hstate *h)
 
7483{
7484#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
7485	if (huge_page_size(h) == PMD_SIZE)
7486		return PUD_SIZE - PMD_SIZE;
7487#endif
7488	return 0UL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7489}
7490
7491#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
 
 
 
 
 
7492
7493/*
7494 * These functions are overwritable if your architecture needs its own
7495 * behavior.
7496 */
7497bool isolate_hugetlb(struct folio *folio, struct list_head *list)
 
 
 
 
 
 
 
 
7498{
7499	bool ret = true;
7500
7501	spin_lock_irq(&hugetlb_lock);
7502	if (!folio_test_hugetlb(folio) ||
7503	    !folio_test_hugetlb_migratable(folio) ||
7504	    !folio_try_get(folio)) {
7505		ret = false;
7506		goto unlock;
7507	}
7508	folio_clear_hugetlb_migratable(folio);
7509	list_move_tail(&folio->lru, list);
7510unlock:
7511	spin_unlock_irq(&hugetlb_lock);
7512	return ret;
7513}
7514
7515int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
7516{
7517	int ret = 0;
7518
7519	*hugetlb = false;
7520	spin_lock_irq(&hugetlb_lock);
7521	if (folio_test_hugetlb(folio)) {
7522		*hugetlb = true;
7523		if (folio_test_hugetlb_freed(folio))
7524			ret = 0;
7525		else if (folio_test_hugetlb_migratable(folio) || unpoison)
7526			ret = folio_try_get(folio);
7527		else
7528			ret = -EBUSY;
7529	}
7530	spin_unlock_irq(&hugetlb_lock);
7531	return ret;
7532}
7533
7534int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
7535				bool *migratable_cleared)
7536{
7537	int ret;
7538
7539	spin_lock_irq(&hugetlb_lock);
7540	ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared);
7541	spin_unlock_irq(&hugetlb_lock);
7542	return ret;
7543}
7544
7545void folio_putback_active_hugetlb(struct folio *folio)
7546{
7547	spin_lock_irq(&hugetlb_lock);
7548	folio_set_hugetlb_migratable(folio);
7549	list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist);
7550	spin_unlock_irq(&hugetlb_lock);
7551	folio_put(folio);
7552}
7553
7554void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason)
7555{
7556	struct hstate *h = folio_hstate(old_folio);
7557
7558	hugetlb_cgroup_migrate(old_folio, new_folio);
7559	set_page_owner_migrate_reason(&new_folio->page, reason);
7560
7561	/*
7562	 * transfer temporary state of the new hugetlb folio. This is
7563	 * reverse to other transitions because the newpage is going to
7564	 * be final while the old one will be freed so it takes over
7565	 * the temporary status.
7566	 *
7567	 * Also note that we have to transfer the per-node surplus state
7568	 * here as well otherwise the global surplus count will not match
7569	 * the per-node's.
7570	 */
7571	if (folio_test_hugetlb_temporary(new_folio)) {
7572		int old_nid = folio_nid(old_folio);
7573		int new_nid = folio_nid(new_folio);
7574
7575		folio_set_hugetlb_temporary(old_folio);
7576		folio_clear_hugetlb_temporary(new_folio);
7577
 
 
7578
7579		/*
7580		 * There is no need to transfer the per-node surplus state
7581		 * when we do not cross the node.
7582		 */
7583		if (new_nid == old_nid)
7584			return;
7585		spin_lock_irq(&hugetlb_lock);
7586		if (h->surplus_huge_pages_node[old_nid]) {
7587			h->surplus_huge_pages_node[old_nid]--;
7588			h->surplus_huge_pages_node[new_nid]++;
7589		}
7590		spin_unlock_irq(&hugetlb_lock);
7591	}
7592}
7593
7594static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
7595				   unsigned long start,
7596				   unsigned long end)
 
 
7597{
7598	struct hstate *h = hstate_vma(vma);
7599	unsigned long sz = huge_page_size(h);
7600	struct mm_struct *mm = vma->vm_mm;
7601	struct mmu_notifier_range range;
7602	unsigned long address;
7603	spinlock_t *ptl;
7604	pte_t *ptep;
7605
7606	if (!(vma->vm_flags & VM_MAYSHARE))
7607		return;
7608
 
 
 
7609	if (start >= end)
7610		return;
7611
7612	flush_cache_range(vma, start, end);
7613	/*
7614	 * No need to call adjust_range_if_pmd_sharing_possible(), because
7615	 * we have already done the PUD_SIZE alignment.
7616	 */
7617	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
7618				start, end);
7619	mmu_notifier_invalidate_range_start(&range);
7620	hugetlb_vma_lock_write(vma);
7621	i_mmap_lock_write(vma->vm_file->f_mapping);
7622	for (address = start; address < end; address += PUD_SIZE) {
7623		ptep = hugetlb_walk(vma, address, sz);
 
 
7624		if (!ptep)
7625			continue;
7626		ptl = huge_pte_lock(h, mm, ptep);
7627		huge_pmd_unshare(mm, vma, address, ptep);
 
7628		spin_unlock(ptl);
7629	}
7630	flush_hugetlb_tlb_range(vma, start, end);
7631	i_mmap_unlock_write(vma->vm_file->f_mapping);
7632	hugetlb_vma_unlock_write(vma);
7633	/*
7634	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see
7635	 * Documentation/mm/mmu_notifier.rst.
7636	 */
7637	mmu_notifier_invalidate_range_end(&range);
7638}
7639
7640/*
7641 * This function will unconditionally remove all the shared pmd pgtable entries
7642 * within the specific vma for a hugetlbfs memory range.
7643 */
7644void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
7645{
7646	hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE),
7647			ALIGN_DOWN(vma->vm_end, PUD_SIZE));
7648}
7649
7650#ifdef CONFIG_CMA
7651static bool cma_reserve_called __initdata;
7652
7653static int __init cmdline_parse_hugetlb_cma(char *p)
7654{
7655	int nid, count = 0;
7656	unsigned long tmp;
7657	char *s = p;
7658
7659	while (*s) {
7660		if (sscanf(s, "%lu%n", &tmp, &count) != 1)
7661			break;
7662
7663		if (s[count] == ':') {
7664			if (tmp >= MAX_NUMNODES)
7665				break;
7666			nid = array_index_nospec(tmp, MAX_NUMNODES);
7667
7668			s += count + 1;
7669			tmp = memparse(s, &s);
7670			hugetlb_cma_size_in_node[nid] = tmp;
7671			hugetlb_cma_size += tmp;
7672
7673			/*
7674			 * Skip the separator if have one, otherwise
7675			 * break the parsing.
7676			 */
7677			if (*s == ',')
7678				s++;
7679			else
7680				break;
7681		} else {
7682			hugetlb_cma_size = memparse(p, &p);
7683			break;
7684		}
7685	}
7686
7687	return 0;
7688}
7689
7690early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
7691
7692void __init hugetlb_cma_reserve(int order)
7693{
7694	unsigned long size, reserved, per_node;
7695	bool node_specific_cma_alloc = false;
7696	int nid;
7697
7698	cma_reserve_called = true;
7699
7700	if (!hugetlb_cma_size)
7701		return;
7702
7703	for (nid = 0; nid < MAX_NUMNODES; nid++) {
7704		if (hugetlb_cma_size_in_node[nid] == 0)
7705			continue;
7706
7707		if (!node_online(nid)) {
7708			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
7709			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7710			hugetlb_cma_size_in_node[nid] = 0;
7711			continue;
7712		}
7713
7714		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
7715			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
7716				nid, (PAGE_SIZE << order) / SZ_1M);
7717			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7718			hugetlb_cma_size_in_node[nid] = 0;
7719		} else {
7720			node_specific_cma_alloc = true;
7721		}
7722	}
7723
7724	/* Validate the CMA size again in case some invalid nodes specified. */
7725	if (!hugetlb_cma_size)
7726		return;
7727
7728	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
7729		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
7730			(PAGE_SIZE << order) / SZ_1M);
7731		hugetlb_cma_size = 0;
7732		return;
7733	}
7734
7735	if (!node_specific_cma_alloc) {
7736		/*
7737		 * If 3 GB area is requested on a machine with 4 numa nodes,
7738		 * let's allocate 1 GB on first three nodes and ignore the last one.
7739		 */
7740		per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
7741		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
7742			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
7743	}
7744
7745	reserved = 0;
7746	for_each_online_node(nid) {
7747		int res;
7748		char name[CMA_MAX_NAME];
7749
7750		if (node_specific_cma_alloc) {
7751			if (hugetlb_cma_size_in_node[nid] == 0)
7752				continue;
7753
7754			size = hugetlb_cma_size_in_node[nid];
7755		} else {
7756			size = min(per_node, hugetlb_cma_size - reserved);
7757		}
7758
7759		size = round_up(size, PAGE_SIZE << order);
7760
7761		snprintf(name, sizeof(name), "hugetlb%d", nid);
7762		/*
7763		 * Note that 'order per bit' is based on smallest size that
7764		 * may be returned to CMA allocator in the case of
7765		 * huge page demotion.
7766		 */
7767		res = cma_declare_contiguous_nid(0, size, 0,
7768						PAGE_SIZE << HUGETLB_PAGE_ORDER,
7769						 0, false, name,
7770						 &hugetlb_cma[nid], nid);
7771		if (res) {
7772			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
7773				res, nid);
7774			continue;
7775		}
7776
7777		reserved += size;
7778		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
7779			size / SZ_1M, nid);
7780
7781		if (reserved >= hugetlb_cma_size)
7782			break;
7783	}
7784
7785	if (!reserved)
7786		/*
7787		 * hugetlb_cma_size is used to determine if allocations from
7788		 * cma are possible.  Set to zero if no cma regions are set up.
7789		 */
7790		hugetlb_cma_size = 0;
7791}
7792
7793static void __init hugetlb_cma_check(void)
7794{
7795	if (!hugetlb_cma_size || cma_reserve_called)
7796		return;
7797
7798	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
7799}
7800
7801#endif /* CONFIG_CMA */

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Generic hugetlb support.
   4 * (C) Nadia Yvette Chambers, April 2004
   5 */
   6#include <linux/list.h>
   7#include <linux/init.h>
   8#include <linux/mm.h>
   9#include <linux/seq_file.h>
  10#include <linux/sysctl.h>
  11#include <linux/highmem.h>
  12#include <linux/mmu_notifier.h>
  13#include <linux/nodemask.h>
  14#include <linux/pagemap.h>
  15#include <linux/mempolicy.h>
  16#include <linux/compiler.h>
  17#include <linux/cpuset.h>
  18#include <linux/mutex.h>
  19#include <linux/memblock.h>
  20#include <linux/sysfs.h>
  21#include <linux/slab.h>
  22#include <linux/sched/mm.h>
  23#include <linux/mmdebug.h>
  24#include <linux/sched/signal.h>
  25#include <linux/rmap.h>
  26#include <linux/string_helpers.h>
  27#include <linux/swap.h>
  28#include <linux/swapops.h>
  29#include <linux/jhash.h>
  30#include <linux/numa.h>
  31#include <linux/llist.h>
  32#include <linux/cma.h>
  33#include <linux/migrate.h>
 
 
 
 
  34
  35#include <asm/page.h>
  36#include <asm/pgalloc.h>
  37#include <asm/tlb.h>
  38
  39#include <linux/io.h>
  40#include <linux/hugetlb.h>
  41#include <linux/hugetlb_cgroup.h>
  42#include <linux/node.h>
  43#include <linux/page_owner.h>
  44#include "internal.h"
  45#include "hugetlb_vmemmap.h"
  46
  47int hugetlb_max_hstate __read_mostly;
  48unsigned int default_hstate_idx;
  49struct hstate hstates[HUGE_MAX_HSTATE];
  50
  51#ifdef CONFIG_CMA
  52static struct cma *hugetlb_cma[MAX_NUMNODES];
 
 
 
 
 
 
 
 
 
 
 
  53#endif
  54static unsigned long hugetlb_cma_size __initdata;
  55
  56/*
  57 * Minimum page order among possible hugepage sizes, set to a proper value
  58 * at boot time.
  59 */
  60static unsigned int minimum_order __read_mostly = UINT_MAX;
  61
  62__initdata LIST_HEAD(huge_boot_pages);
  63
  64/* for command line parsing */
  65static struct hstate * __initdata parsed_hstate;
  66static unsigned long __initdata default_hstate_max_huge_pages;
  67static bool __initdata parsed_valid_hugepagesz = true;
  68static bool __initdata parsed_default_hugepagesz;
 
  69
  70/*
  71 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  72 * free_huge_pages, and surplus_huge_pages.
  73 */
  74DEFINE_SPINLOCK(hugetlb_lock);
  75
  76/*
  77 * Serializes faults on the same logical page.  This is used to
  78 * prevent spurious OOMs when the hugepage pool is fully utilized.
  79 */
  80static int num_fault_mutexes;
  81struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
  82
  83/* Forward declaration */
  84static int hugetlb_acct_memory(struct hstate *h, long delta);
 
 
 
 
 
 
  85
  86static inline bool subpool_is_free(struct hugepage_subpool *spool)
  87{
  88	if (spool->count)
  89		return false;
  90	if (spool->max_hpages != -1)
  91		return spool->used_hpages == 0;
  92	if (spool->min_hpages != -1)
  93		return spool->rsv_hpages == spool->min_hpages;
  94
  95	return true;
  96}
  97
  98static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
  99						unsigned long irq_flags)
 100{
 101	spin_unlock_irqrestore(&spool->lock, irq_flags);
 102
 103	/* If no pages are used, and no other handles to the subpool
 104	 * remain, give up any reservations based on minimum size and
 105	 * free the subpool */
 106	if (subpool_is_free(spool)) {
 107		if (spool->min_hpages != -1)
 108			hugetlb_acct_memory(spool->hstate,
 109						-spool->min_hpages);
 110		kfree(spool);
 111	}
 112}
 113
 114struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 115						long min_hpages)
 116{
 117	struct hugepage_subpool *spool;
 118
 119	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 120	if (!spool)
 121		return NULL;
 122
 123	spin_lock_init(&spool->lock);
 124	spool->count = 1;
 125	spool->max_hpages = max_hpages;
 126	spool->hstate = h;
 127	spool->min_hpages = min_hpages;
 128
 129	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
 130		kfree(spool);
 131		return NULL;
 132	}
 133	spool->rsv_hpages = min_hpages;
 134
 135	return spool;
 136}
 137
 138void hugepage_put_subpool(struct hugepage_subpool *spool)
 139{
 140	unsigned long flags;
 141
 142	spin_lock_irqsave(&spool->lock, flags);
 143	BUG_ON(!spool->count);
 144	spool->count--;
 145	unlock_or_release_subpool(spool, flags);
 146}
 147
 148/*
 149 * Subpool accounting for allocating and reserving pages.
 150 * Return -ENOMEM if there are not enough resources to satisfy the
 151 * request.  Otherwise, return the number of pages by which the
 152 * global pools must be adjusted (upward).  The returned value may
 153 * only be different than the passed value (delta) in the case where
 154 * a subpool minimum size must be maintained.
 155 */
 156static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
 157				      long delta)
 158{
 159	long ret = delta;
 160
 161	if (!spool)
 162		return ret;
 163
 164	spin_lock_irq(&spool->lock);
 165
 166	if (spool->max_hpages != -1) {		/* maximum size accounting */
 167		if ((spool->used_hpages + delta) <= spool->max_hpages)
 168			spool->used_hpages += delta;
 169		else {
 170			ret = -ENOMEM;
 171			goto unlock_ret;
 172		}
 173	}
 174
 175	/* minimum size accounting */
 176	if (spool->min_hpages != -1 && spool->rsv_hpages) {
 177		if (delta > spool->rsv_hpages) {
 178			/*
 179			 * Asking for more reserves than those already taken on
 180			 * behalf of subpool.  Return difference.
 181			 */
 182			ret = delta - spool->rsv_hpages;
 183			spool->rsv_hpages = 0;
 184		} else {
 185			ret = 0;	/* reserves already accounted for */
 186			spool->rsv_hpages -= delta;
 187		}
 188	}
 189
 190unlock_ret:
 191	spin_unlock_irq(&spool->lock);
 192	return ret;
 193}
 194
 195/*
 196 * Subpool accounting for freeing and unreserving pages.
 197 * Return the number of global page reservations that must be dropped.
 198 * The return value may only be different than the passed value (delta)
 199 * in the case where a subpool minimum size must be maintained.
 200 */
 201static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 202				       long delta)
 203{
 204	long ret = delta;
 205	unsigned long flags;
 206
 207	if (!spool)
 208		return delta;
 209
 210	spin_lock_irqsave(&spool->lock, flags);
 211
 212	if (spool->max_hpages != -1)		/* maximum size accounting */
 213		spool->used_hpages -= delta;
 214
 215	 /* minimum size accounting */
 216	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
 217		if (spool->rsv_hpages + delta <= spool->min_hpages)
 218			ret = 0;
 219		else
 220			ret = spool->rsv_hpages + delta - spool->min_hpages;
 221
 222		spool->rsv_hpages += delta;
 223		if (spool->rsv_hpages > spool->min_hpages)
 224			spool->rsv_hpages = spool->min_hpages;
 225	}
 226
 227	/*
 228	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
 229	 * quota reference, free it now.
 230	 */
 231	unlock_or_release_subpool(spool, flags);
 232
 233	return ret;
 234}
 235
 236static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 237{
 238	return HUGETLBFS_SB(inode->i_sb)->spool;
 239}
 240
 241static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 242{
 243	return subpool_inode(file_inode(vma->vm_file));
 244}
 245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 246/* Helper that removes a struct file_region from the resv_map cache and returns
 247 * it for use.
 248 */
 249static struct file_region *
 250get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
 251{
 252	struct file_region *nrg = NULL;
 253
 254	VM_BUG_ON(resv->region_cache_count <= 0);
 255
 256	resv->region_cache_count--;
 257	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
 258	list_del(&nrg->link);
 259
 260	nrg->from = from;
 261	nrg->to = to;
 262
 263	return nrg;
 264}
 265
 266static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
 267					      struct file_region *rg)
 268{
 269#ifdef CONFIG_CGROUP_HUGETLB
 270	nrg->reservation_counter = rg->reservation_counter;
 271	nrg->css = rg->css;
 272	if (rg->css)
 273		css_get(rg->css);
 274#endif
 275}
 276
 277/* Helper that records hugetlb_cgroup uncharge info. */
 278static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
 279						struct hstate *h,
 280						struct resv_map *resv,
 281						struct file_region *nrg)
 282{
 283#ifdef CONFIG_CGROUP_HUGETLB
 284	if (h_cg) {
 285		nrg->reservation_counter =
 286			&h_cg->rsvd_hugepage[hstate_index(h)];
 287		nrg->css = &h_cg->css;
 288		/*
 289		 * The caller will hold exactly one h_cg->css reference for the
 290		 * whole contiguous reservation region. But this area might be
 291		 * scattered when there are already some file_regions reside in
 292		 * it. As a result, many file_regions may share only one css
 293		 * reference. In order to ensure that one file_region must hold
 294		 * exactly one h_cg->css reference, we should do css_get for
 295		 * each file_region and leave the reference held by caller
 296		 * untouched.
 297		 */
 298		css_get(&h_cg->css);
 299		if (!resv->pages_per_hpage)
 300			resv->pages_per_hpage = pages_per_huge_page(h);
 301		/* pages_per_hpage should be the same for all entries in
 302		 * a resv_map.
 303		 */
 304		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
 305	} else {
 306		nrg->reservation_counter = NULL;
 307		nrg->css = NULL;
 308	}
 309#endif
 310}
 311
 312static void put_uncharge_info(struct file_region *rg)
 313{
 314#ifdef CONFIG_CGROUP_HUGETLB
 315	if (rg->css)
 316		css_put(rg->css);
 317#endif
 318}
 319
 320static bool has_same_uncharge_info(struct file_region *rg,
 321				   struct file_region *org)
 322{
 323#ifdef CONFIG_CGROUP_HUGETLB
 324	return rg && org &&
 325	       rg->reservation_counter == org->reservation_counter &&
 326	       rg->css == org->css;
 327
 328#else
 329	return true;
 330#endif
 331}
 332
 333static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
 334{
 335	struct file_region *nrg = NULL, *prg = NULL;
 336
 337	prg = list_prev_entry(rg, link);
 338	if (&prg->link != &resv->regions && prg->to == rg->from &&
 339	    has_same_uncharge_info(prg, rg)) {
 340		prg->to = rg->to;
 341
 342		list_del(&rg->link);
 343		put_uncharge_info(rg);
 344		kfree(rg);
 345
 346		rg = prg;
 347	}
 348
 349	nrg = list_next_entry(rg, link);
 350	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
 351	    has_same_uncharge_info(nrg, rg)) {
 352		nrg->from = rg->from;
 353
 354		list_del(&rg->link);
 355		put_uncharge_info(rg);
 356		kfree(rg);
 357	}
 358}
 359
 360static inline long
 361hugetlb_resv_map_add(struct resv_map *map, struct file_region *rg, long from,
 362		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
 363		     long *regions_needed)
 364{
 365	struct file_region *nrg;
 366
 367	if (!regions_needed) {
 368		nrg = get_file_region_entry_from_cache(map, from, to);
 369		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
 370		list_add(&nrg->link, rg->link.prev);
 371		coalesce_file_region(map, nrg);
 372	} else
 373		*regions_needed += 1;
 374
 375	return to - from;
 376}
 377
 378/*
 379 * Must be called with resv->lock held.
 380 *
 381 * Calling this with regions_needed != NULL will count the number of pages
 382 * to be added but will not modify the linked list. And regions_needed will
 383 * indicate the number of file_regions needed in the cache to carry out to add
 384 * the regions for this range.
 385 */
 386static long add_reservation_in_range(struct resv_map *resv, long f, long t,
 387				     struct hugetlb_cgroup *h_cg,
 388				     struct hstate *h, long *regions_needed)
 389{
 390	long add = 0;
 391	struct list_head *head = &resv->regions;
 392	long last_accounted_offset = f;
 393	struct file_region *rg = NULL, *trg = NULL;
 
 394
 395	if (regions_needed)
 396		*regions_needed = 0;
 397
 398	/* In this loop, we essentially handle an entry for the range
 399	 * [last_accounted_offset, rg->from), at every iteration, with some
 400	 * bounds checking.
 401	 */
 402	list_for_each_entry_safe(rg, trg, head, link) {
 403		/* Skip irrelevant regions that start before our range. */
 404		if (rg->from < f) {
 405			/* If this region ends after the last accounted offset,
 406			 * then we need to update last_accounted_offset.
 407			 */
 408			if (rg->to > last_accounted_offset)
 409				last_accounted_offset = rg->to;
 410			continue;
 411		}
 412
 413		/* When we find a region that starts beyond our range, we've
 414		 * finished.
 415		 */
 416		if (rg->from >= t)
 
 417			break;
 
 418
 419		/* Add an entry for last_accounted_offset -> rg->from, and
 420		 * update last_accounted_offset.
 421		 */
 422		if (rg->from > last_accounted_offset)
 423			add += hugetlb_resv_map_add(resv, rg,
 424						    last_accounted_offset,
 425						    rg->from, h, h_cg,
 426						    regions_needed);
 427
 428		last_accounted_offset = rg->to;
 429	}
 430
 431	/* Handle the case where our range extends beyond
 432	 * last_accounted_offset.
 433	 */
 
 
 434	if (last_accounted_offset < t)
 435		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
 436					    t, h, h_cg, regions_needed);
 437
 438	VM_BUG_ON(add < 0);
 439	return add;
 440}
 441
 442/* Must be called with resv->lock acquired. Will drop lock to allocate entries.
 443 */
 444static int allocate_file_region_entries(struct resv_map *resv,
 445					int regions_needed)
 446	__must_hold(&resv->lock)
 447{
 448	struct list_head allocated_regions;
 449	int to_allocate = 0, i = 0;
 450	struct file_region *trg = NULL, *rg = NULL;
 451
 452	VM_BUG_ON(regions_needed < 0);
 453
 454	INIT_LIST_HEAD(&allocated_regions);
 455
 456	/*
 457	 * Check for sufficient descriptors in the cache to accommodate
 458	 * the number of in progress add operations plus regions_needed.
 459	 *
 460	 * This is a while loop because when we drop the lock, some other call
 461	 * to region_add or region_del may have consumed some region_entries,
 462	 * so we keep looping here until we finally have enough entries for
 463	 * (adds_in_progress + regions_needed).
 464	 */
 465	while (resv->region_cache_count <
 466	       (resv->adds_in_progress + regions_needed)) {
 467		to_allocate = resv->adds_in_progress + regions_needed -
 468			      resv->region_cache_count;
 469
 470		/* At this point, we should have enough entries in the cache
 471		 * for all the existing adds_in_progress. We should only be
 472		 * needing to allocate for regions_needed.
 473		 */
 474		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
 475
 476		spin_unlock(&resv->lock);
 477		for (i = 0; i < to_allocate; i++) {
 478			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
 479			if (!trg)
 480				goto out_of_memory;
 481			list_add(&trg->link, &allocated_regions);
 482		}
 483
 484		spin_lock(&resv->lock);
 485
 486		list_splice(&allocated_regions, &resv->region_cache);
 487		resv->region_cache_count += to_allocate;
 488	}
 489
 490	return 0;
 491
 492out_of_memory:
 493	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
 494		list_del(&rg->link);
 495		kfree(rg);
 496	}
 497	return -ENOMEM;
 498}
 499
 500/*
 501 * Add the huge page range represented by [f, t) to the reserve
 502 * map.  Regions will be taken from the cache to fill in this range.
 503 * Sufficient regions should exist in the cache due to the previous
 504 * call to region_chg with the same range, but in some cases the cache will not
 505 * have sufficient entries due to races with other code doing region_add or
 506 * region_del.  The extra needed entries will be allocated.
 507 *
 508 * regions_needed is the out value provided by a previous call to region_chg.
 509 *
 510 * Return the number of new huge pages added to the map.  This number is greater
 511 * than or equal to zero.  If file_region entries needed to be allocated for
 512 * this operation and we were not able to allocate, it returns -ENOMEM.
 513 * region_add of regions of length 1 never allocate file_regions and cannot
 514 * fail; region_chg will always allocate at least 1 entry and a region_add for
 515 * 1 page will only require at most 1 entry.
 516 */
 517static long region_add(struct resv_map *resv, long f, long t,
 518		       long in_regions_needed, struct hstate *h,
 519		       struct hugetlb_cgroup *h_cg)
 520{
 521	long add = 0, actual_regions_needed = 0;
 522
 523	spin_lock(&resv->lock);
 524retry:
 525
 526	/* Count how many regions are actually needed to execute this add. */
 527	add_reservation_in_range(resv, f, t, NULL, NULL,
 528				 &actual_regions_needed);
 529
 530	/*
 531	 * Check for sufficient descriptors in the cache to accommodate
 532	 * this add operation. Note that actual_regions_needed may be greater
 533	 * than in_regions_needed, as the resv_map may have been modified since
 534	 * the region_chg call. In this case, we need to make sure that we
 535	 * allocate extra entries, such that we have enough for all the
 536	 * existing adds_in_progress, plus the excess needed for this
 537	 * operation.
 538	 */
 539	if (actual_regions_needed > in_regions_needed &&
 540	    resv->region_cache_count <
 541		    resv->adds_in_progress +
 542			    (actual_regions_needed - in_regions_needed)) {
 543		/* region_add operation of range 1 should never need to
 544		 * allocate file_region entries.
 545		 */
 546		VM_BUG_ON(t - f <= 1);
 547
 548		if (allocate_file_region_entries(
 549			    resv, actual_regions_needed - in_regions_needed)) {
 550			return -ENOMEM;
 551		}
 552
 553		goto retry;
 554	}
 555
 556	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
 557
 558	resv->adds_in_progress -= in_regions_needed;
 559
 560	spin_unlock(&resv->lock);
 561	return add;
 562}
 563
 564/*
 565 * Examine the existing reserve map and determine how many
 566 * huge pages in the specified range [f, t) are NOT currently
 567 * represented.  This routine is called before a subsequent
 568 * call to region_add that will actually modify the reserve
 569 * map to add the specified range [f, t).  region_chg does
 570 * not change the number of huge pages represented by the
 571 * map.  A number of new file_region structures is added to the cache as a
 572 * placeholder, for the subsequent region_add call to use. At least 1
 573 * file_region structure is added.
 574 *
 575 * out_regions_needed is the number of regions added to the
 576 * resv->adds_in_progress.  This value needs to be provided to a follow up call
 577 * to region_add or region_abort for proper accounting.
 578 *
 579 * Returns the number of huge pages that need to be added to the existing
 580 * reservation map for the range [f, t).  This number is greater or equal to
 581 * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 582 * is needed and can not be allocated.
 583 */
 584static long region_chg(struct resv_map *resv, long f, long t,
 585		       long *out_regions_needed)
 586{
 587	long chg = 0;
 588
 589	spin_lock(&resv->lock);
 590
 591	/* Count how many hugepages in this range are NOT represented. */
 592	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
 593				       out_regions_needed);
 594
 595	if (*out_regions_needed == 0)
 596		*out_regions_needed = 1;
 597
 598	if (allocate_file_region_entries(resv, *out_regions_needed))
 599		return -ENOMEM;
 600
 601	resv->adds_in_progress += *out_regions_needed;
 602
 603	spin_unlock(&resv->lock);
 604	return chg;
 605}
 606
 607/*
 608 * Abort the in progress add operation.  The adds_in_progress field
 609 * of the resv_map keeps track of the operations in progress between
 610 * calls to region_chg and region_add.  Operations are sometimes
 611 * aborted after the call to region_chg.  In such cases, region_abort
 612 * is called to decrement the adds_in_progress counter. regions_needed
 613 * is the value returned by the region_chg call, it is used to decrement
 614 * the adds_in_progress counter.
 615 *
 616 * NOTE: The range arguments [f, t) are not needed or used in this
 617 * routine.  They are kept to make reading the calling code easier as
 618 * arguments will match the associated region_chg call.
 619 */
 620static void region_abort(struct resv_map *resv, long f, long t,
 621			 long regions_needed)
 622{
 623	spin_lock(&resv->lock);
 624	VM_BUG_ON(!resv->region_cache_count);
 625	resv->adds_in_progress -= regions_needed;
 626	spin_unlock(&resv->lock);
 627}
 628
 629/*
 630 * Delete the specified range [f, t) from the reserve map.  If the
 631 * t parameter is LONG_MAX, this indicates that ALL regions after f
 632 * should be deleted.  Locate the regions which intersect [f, t)
 633 * and either trim, delete or split the existing regions.
 634 *
 635 * Returns the number of huge pages deleted from the reserve map.
 636 * In the normal case, the return value is zero or more.  In the
 637 * case where a region must be split, a new region descriptor must
 638 * be allocated.  If the allocation fails, -ENOMEM will be returned.
 639 * NOTE: If the parameter t == LONG_MAX, then we will never split
 640 * a region and possibly return -ENOMEM.  Callers specifying
 641 * t == LONG_MAX do not need to check for -ENOMEM error.
 642 */
 643static long region_del(struct resv_map *resv, long f, long t)
 644{
 645	struct list_head *head = &resv->regions;
 646	struct file_region *rg, *trg;
 647	struct file_region *nrg = NULL;
 648	long del = 0;
 649
 650retry:
 651	spin_lock(&resv->lock);
 652	list_for_each_entry_safe(rg, trg, head, link) {
 653		/*
 654		 * Skip regions before the range to be deleted.  file_region
 655		 * ranges are normally of the form [from, to).  However, there
 656		 * may be a "placeholder" entry in the map which is of the form
 657		 * (from, to) with from == to.  Check for placeholder entries
 658		 * at the beginning of the range to be deleted.
 659		 */
 660		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
 661			continue;
 662
 663		if (rg->from >= t)
 664			break;
 665
 666		if (f > rg->from && t < rg->to) { /* Must split region */
 667			/*
 668			 * Check for an entry in the cache before dropping
 669			 * lock and attempting allocation.
 670			 */
 671			if (!nrg &&
 672			    resv->region_cache_count > resv->adds_in_progress) {
 673				nrg = list_first_entry(&resv->region_cache,
 674							struct file_region,
 675							link);
 676				list_del(&nrg->link);
 677				resv->region_cache_count--;
 678			}
 679
 680			if (!nrg) {
 681				spin_unlock(&resv->lock);
 682				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 683				if (!nrg)
 684					return -ENOMEM;
 685				goto retry;
 686			}
 687
 688			del += t - f;
 689			hugetlb_cgroup_uncharge_file_region(
 690				resv, rg, t - f, false);
 691
 692			/* New entry for end of split region */
 693			nrg->from = t;
 694			nrg->to = rg->to;
 695
 696			copy_hugetlb_cgroup_uncharge_info(nrg, rg);
 697
 698			INIT_LIST_HEAD(&nrg->link);
 699
 700			/* Original entry is trimmed */
 701			rg->to = f;
 702
 703			list_add(&nrg->link, &rg->link);
 704			nrg = NULL;
 705			break;
 706		}
 707
 708		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
 709			del += rg->to - rg->from;
 710			hugetlb_cgroup_uncharge_file_region(resv, rg,
 711							    rg->to - rg->from, true);
 712			list_del(&rg->link);
 713			kfree(rg);
 714			continue;
 715		}
 716
 717		if (f <= rg->from) {	/* Trim beginning of region */
 718			hugetlb_cgroup_uncharge_file_region(resv, rg,
 719							    t - rg->from, false);
 720
 721			del += t - rg->from;
 722			rg->from = t;
 723		} else {		/* Trim end of region */
 724			hugetlb_cgroup_uncharge_file_region(resv, rg,
 725							    rg->to - f, false);
 726
 727			del += rg->to - f;
 728			rg->to = f;
 729		}
 730	}
 731
 732	spin_unlock(&resv->lock);
 733	kfree(nrg);
 734	return del;
 735}
 736
 737/*
 738 * A rare out of memory error was encountered which prevented removal of
 739 * the reserve map region for a page.  The huge page itself was free'ed
 740 * and removed from the page cache.  This routine will adjust the subpool
 741 * usage count, and the global reserve count if needed.  By incrementing
 742 * these counts, the reserve map entry which could not be deleted will
 743 * appear as a "reserved" entry instead of simply dangling with incorrect
 744 * counts.
 745 */
 746void hugetlb_fix_reserve_counts(struct inode *inode)
 747{
 748	struct hugepage_subpool *spool = subpool_inode(inode);
 749	long rsv_adjust;
 750	bool reserved = false;
 751
 752	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
 753	if (rsv_adjust > 0) {
 754		struct hstate *h = hstate_inode(inode);
 755
 756		if (!hugetlb_acct_memory(h, 1))
 757			reserved = true;
 758	} else if (!rsv_adjust) {
 759		reserved = true;
 760	}
 761
 762	if (!reserved)
 763		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
 764}
 765
 766/*
 767 * Count and return the number of huge pages in the reserve map
 768 * that intersect with the range [f, t).
 769 */
 770static long region_count(struct resv_map *resv, long f, long t)
 771{
 772	struct list_head *head = &resv->regions;
 773	struct file_region *rg;
 774	long chg = 0;
 775
 776	spin_lock(&resv->lock);
 777	/* Locate each segment we overlap with, and count that overlap. */
 778	list_for_each_entry(rg, head, link) {
 779		long seg_from;
 780		long seg_to;
 781
 782		if (rg->to <= f)
 783			continue;
 784		if (rg->from >= t)
 785			break;
 786
 787		seg_from = max(rg->from, f);
 788		seg_to = min(rg->to, t);
 789
 790		chg += seg_to - seg_from;
 791	}
 792	spin_unlock(&resv->lock);
 793
 794	return chg;
 795}
 796
 797/*
 798 * Convert the address within this vma to the page offset within
 799 * the mapping, in pagecache page units; huge pages here.
 800 */
 801static pgoff_t vma_hugecache_offset(struct hstate *h,
 802			struct vm_area_struct *vma, unsigned long address)
 803{
 804	return ((address - vma->vm_start) >> huge_page_shift(h)) +
 805			(vma->vm_pgoff >> huge_page_order(h));
 806}
 807
 808pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
 809				     unsigned long address)
 810{
 811	return vma_hugecache_offset(hstate_vma(vma), vma, address);
 812}
 813EXPORT_SYMBOL_GPL(linear_hugepage_index);
 814
 815/*
 816 * Return the size of the pages allocated when backing a VMA. In the majority
 817 * cases this will be same size as used by the page table entries.
 818 */
 819unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
 820{
 821	if (vma->vm_ops && vma->vm_ops->pagesize)
 822		return vma->vm_ops->pagesize(vma);
 823	return PAGE_SIZE;
 824}
 825EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
 826
 827/*
 828 * Return the page size being used by the MMU to back a VMA. In the majority
 829 * of cases, the page size used by the kernel matches the MMU size. On
 830 * architectures where it differs, an architecture-specific 'strong'
 831 * version of this symbol is required.
 832 */
 833__weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 834{
 835	return vma_kernel_pagesize(vma);
 836}
 837
 838/*
 839 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
 840 * bits of the reservation map pointer, which are always clear due to
 841 * alignment.
 842 */
 843#define HPAGE_RESV_OWNER    (1UL << 0)
 844#define HPAGE_RESV_UNMAPPED (1UL << 1)
 845#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
 846
 847/*
 848 * These helpers are used to track how many pages are reserved for
 849 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
 850 * is guaranteed to have their future faults succeed.
 851 *
 852 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
 853 * the reserve counters are updated with the hugetlb_lock held. It is safe
 854 * to reset the VMA at fork() time as it is not in use yet and there is no
 855 * chance of the global counters getting corrupted as a result of the values.
 856 *
 857 * The private mapping reservation is represented in a subtly different
 858 * manner to a shared mapping.  A shared mapping has a region map associated
 859 * with the underlying file, this region map represents the backing file
 860 * pages which have ever had a reservation assigned which this persists even
 861 * after the page is instantiated.  A private mapping has a region map
 862 * associated with the original mmap which is attached to all VMAs which
 863 * reference it, this region map represents those offsets which have consumed
 864 * reservation ie. where pages have been instantiated.
 865 */
 866static unsigned long get_vma_private_data(struct vm_area_struct *vma)
 867{
 868	return (unsigned long)vma->vm_private_data;
 869}
 870
 871static void set_vma_private_data(struct vm_area_struct *vma,
 872							unsigned long value)
 873{
 874	vma->vm_private_data = (void *)value;
 875}
 876
 877static void
 878resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
 879					  struct hugetlb_cgroup *h_cg,
 880					  struct hstate *h)
 881{
 882#ifdef CONFIG_CGROUP_HUGETLB
 883	if (!h_cg || !h) {
 884		resv_map->reservation_counter = NULL;
 885		resv_map->pages_per_hpage = 0;
 886		resv_map->css = NULL;
 887	} else {
 888		resv_map->reservation_counter =
 889			&h_cg->rsvd_hugepage[hstate_index(h)];
 890		resv_map->pages_per_hpage = pages_per_huge_page(h);
 891		resv_map->css = &h_cg->css;
 892	}
 893#endif
 894}
 895
 896struct resv_map *resv_map_alloc(void)
 897{
 898	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
 899	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
 900
 901	if (!resv_map || !rg) {
 902		kfree(resv_map);
 903		kfree(rg);
 904		return NULL;
 905	}
 906
 907	kref_init(&resv_map->refs);
 908	spin_lock_init(&resv_map->lock);
 909	INIT_LIST_HEAD(&resv_map->regions);
 
 910
 911	resv_map->adds_in_progress = 0;
 912	/*
 913	 * Initialize these to 0. On shared mappings, 0's here indicate these
 914	 * fields don't do cgroup accounting. On private mappings, these will be
 915	 * re-initialized to the proper values, to indicate that hugetlb cgroup
 916	 * reservations are to be un-charged from here.
 917	 */
 918	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
 919
 920	INIT_LIST_HEAD(&resv_map->region_cache);
 921	list_add(&rg->link, &resv_map->region_cache);
 922	resv_map->region_cache_count = 1;
 923
 924	return resv_map;
 925}
 926
 927void resv_map_release(struct kref *ref)
 928{
 929	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
 930	struct list_head *head = &resv_map->region_cache;
 931	struct file_region *rg, *trg;
 932
 933	/* Clear out any active regions before we release the map. */
 934	region_del(resv_map, 0, LONG_MAX);
 935
 936	/* ... and any entries left in the cache */
 937	list_for_each_entry_safe(rg, trg, head, link) {
 938		list_del(&rg->link);
 939		kfree(rg);
 940	}
 941
 942	VM_BUG_ON(resv_map->adds_in_progress);
 943
 944	kfree(resv_map);
 945}
 946
 947static inline struct resv_map *inode_resv_map(struct inode *inode)
 948{
 949	/*
 950	 * At inode evict time, i_mapping may not point to the original
 951	 * address space within the inode.  This original address space
 952	 * contains the pointer to the resv_map.  So, always use the
 953	 * address space embedded within the inode.
 954	 * The VERY common case is inode->mapping == &inode->i_data but,
 955	 * this may not be true for device special inodes.
 956	 */
 957	return (struct resv_map *)(&inode->i_data)->private_data;
 958}
 959
 960static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 961{
 962	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 963	if (vma->vm_flags & VM_MAYSHARE) {
 964		struct address_space *mapping = vma->vm_file->f_mapping;
 965		struct inode *inode = mapping->host;
 966
 967		return inode_resv_map(inode);
 968
 969	} else {
 970		return (struct resv_map *)(get_vma_private_data(vma) &
 971							~HPAGE_RESV_MASK);
 972	}
 973}
 974
 975static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 976{
 977	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 978	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
 979
 980	set_vma_private_data(vma, (get_vma_private_data(vma) &
 981				HPAGE_RESV_MASK) | (unsigned long)map);
 982}
 983
 984static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 985{
 986	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 987	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
 988
 989	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
 990}
 991
 992static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 993{
 994	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
 995
 996	return (get_vma_private_data(vma) & flag) != 0;
 997}
 998
 999/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
1000void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
 
 
 
 
 
 
1001{
1002	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1003	if (!(vma->vm_flags & VM_MAYSHARE))
1004		vma->vm_private_data = (void *)0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1005}
1006
1007/* Returns true if the VMA has associated reserve pages */
1008static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1009{
1010	if (vma->vm_flags & VM_NORESERVE) {
1011		/*
1012		 * This address is already reserved by other process(chg == 0),
1013		 * so, we should decrement reserved count. Without decrementing,
1014		 * reserve count remains after releasing inode, because this
1015		 * allocated page will go into page cache and is regarded as
1016		 * coming from reserved pool in releasing step.  Currently, we
1017		 * don't have any other solution to deal with this situation
1018		 * properly, so add work-around here.
1019		 */
1020		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1021			return true;
1022		else
1023			return false;
1024	}
1025
1026	/* Shared mappings always use reserves */
1027	if (vma->vm_flags & VM_MAYSHARE) {
1028		/*
1029		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1030		 * be a region map for all pages.  The only situation where
1031		 * there is no region map is if a hole was punched via
1032		 * fallocate.  In this case, there really are no reserves to
1033		 * use.  This situation is indicated if chg != 0.
1034		 */
1035		if (chg)
1036			return false;
1037		else
1038			return true;
1039	}
1040
1041	/*
1042	 * Only the process that called mmap() has reserves for
1043	 * private mappings.
1044	 */
1045	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1046		/*
1047		 * Like the shared case above, a hole punch or truncate
1048		 * could have been performed on the private mapping.
1049		 * Examine the value of chg to determine if reserves
1050		 * actually exist or were previously consumed.
1051		 * Very Subtle - The value of chg comes from a previous
1052		 * call to vma_needs_reserves().  The reserve map for
1053		 * private mappings has different (opposite) semantics
1054		 * than that of shared mappings.  vma_needs_reserves()
1055		 * has already taken this difference in semantics into
1056		 * account.  Therefore, the meaning of chg is the same
1057		 * as in the shared case above.  Code could easily be
1058		 * combined, but keeping it separate draws attention to
1059		 * subtle differences.
1060		 */
1061		if (chg)
1062			return false;
1063		else
1064			return true;
1065	}
1066
1067	return false;
1068}
1069
1070static void enqueue_huge_page(struct hstate *h, struct page *page)
1071{
1072	int nid = page_to_nid(page);
1073
1074	lockdep_assert_held(&hugetlb_lock);
1075	list_move(&page->lru, &h->hugepage_freelists[nid]);
 
 
1076	h->free_huge_pages++;
1077	h->free_huge_pages_node[nid]++;
1078	SetHPageFreed(page);
1079}
1080
1081static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
 
1082{
1083	struct page *page;
1084	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1085
1086	lockdep_assert_held(&hugetlb_lock);
1087	list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
1088		if (pin && !is_pinnable_page(page))
1089			continue;
1090
1091		if (PageHWPoison(page))
1092			continue;
1093
1094		list_move(&page->lru, &h->hugepage_activelist);
1095		set_page_refcounted(page);
1096		ClearHPageFreed(page);
1097		h->free_huge_pages--;
1098		h->free_huge_pages_node[nid]--;
1099		return page;
1100	}
1101
1102	return NULL;
1103}
1104
1105static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
1106		nodemask_t *nmask)
1107{
1108	unsigned int cpuset_mems_cookie;
1109	struct zonelist *zonelist;
1110	struct zone *zone;
1111	struct zoneref *z;
1112	int node = NUMA_NO_NODE;
1113
1114	zonelist = node_zonelist(nid, gfp_mask);
1115
1116retry_cpuset:
1117	cpuset_mems_cookie = read_mems_allowed_begin();
1118	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1119		struct page *page;
1120
1121		if (!cpuset_zone_allowed(zone, gfp_mask))
1122			continue;
1123		/*
1124		 * no need to ask again on the same node. Pool is node rather than
1125		 * zone aware
1126		 */
1127		if (zone_to_nid(zone) == node)
1128			continue;
1129		node = zone_to_nid(zone);
1130
1131		page = dequeue_huge_page_node_exact(h, node);
1132		if (page)
1133			return page;
1134	}
1135	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1136		goto retry_cpuset;
1137
1138	return NULL;
1139}
1140
1141static struct page *dequeue_huge_page_vma(struct hstate *h,
 
 
 
 
 
1142				struct vm_area_struct *vma,
1143				unsigned long address, int avoid_reserve,
1144				long chg)
1145{
1146	struct page *page;
1147	struct mempolicy *mpol;
1148	gfp_t gfp_mask;
1149	nodemask_t *nodemask;
1150	int nid;
1151
1152	/*
1153	 * A child process with MAP_PRIVATE mappings created by their parent
1154	 * have no page reserves. This check ensures that reservations are
1155	 * not "stolen". The child may still get SIGKILLed
1156	 */
1157	if (!vma_has_reserves(vma, chg) &&
1158			h->free_huge_pages - h->resv_huge_pages == 0)
1159		goto err;
1160
1161	/* If reserves cannot be used, ensure enough pages are in the pool */
1162	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
1163		goto err;
1164
1165	gfp_mask = htlb_alloc_mask(h);
1166	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1167	page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1168	if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
1169		SetHPageRestoreReserve(page);
 
 
 
 
 
 
 
 
 
 
 
 
1170		h->resv_huge_pages--;
1171	}
1172
1173	mpol_cond_put(mpol);
1174	return page;
1175
1176err:
1177	return NULL;
1178}
1179
1180/*
1181 * common helper functions for hstate_next_node_to_{alloc|free}.
1182 * We may have allocated or freed a huge page based on a different
1183 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1184 * be outside of *nodes_allowed.  Ensure that we use an allowed
1185 * node for alloc or free.
1186 */
1187static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1188{
1189	nid = next_node_in(nid, *nodes_allowed);
1190	VM_BUG_ON(nid >= MAX_NUMNODES);
1191
1192	return nid;
1193}
1194
1195static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1196{
1197	if (!node_isset(nid, *nodes_allowed))
1198		nid = next_node_allowed(nid, nodes_allowed);
1199	return nid;
1200}
1201
1202/*
1203 * returns the previously saved node ["this node"] from which to
1204 * allocate a persistent huge page for the pool and advance the
1205 * next node from which to allocate, handling wrap at end of node
1206 * mask.
1207 */
1208static int hstate_next_node_to_alloc(struct hstate *h,
1209					nodemask_t *nodes_allowed)
1210{
1211	int nid;
1212
1213	VM_BUG_ON(!nodes_allowed);
1214
1215	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1216	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1217
1218	return nid;
1219}
1220
1221/*
1222 * helper for remove_pool_huge_page() - return the previously saved
1223 * node ["this node"] from which to free a huge page.  Advance the
1224 * next node id whether or not we find a free huge page to free so
1225 * that the next attempt to free addresses the next node.
1226 */
1227static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1228{
1229	int nid;
1230
1231	VM_BUG_ON(!nodes_allowed);
1232
1233	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1234	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1235
1236	return nid;
1237}
1238
1239#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
1240	for (nr_nodes = nodes_weight(*mask);				\
1241		nr_nodes > 0 &&						\
1242		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
1243		nr_nodes--)
1244
1245#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1246	for (nr_nodes = nodes_weight(*mask);				\
1247		nr_nodes > 0 &&						\
1248		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1249		nr_nodes--)
1250
1251#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1252static void destroy_compound_gigantic_page(struct page *page,
1253					unsigned int order)
1254{
1255	int i;
1256	int nr_pages = 1 << order;
1257	struct page *p = page + 1;
1258
1259	atomic_set(compound_mapcount_ptr(page), 0);
1260	atomic_set(compound_pincount_ptr(page), 0);
1261
1262	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
 
 
 
 
1263		clear_compound_head(p);
1264		set_page_refcounted(p);
 
1265	}
1266
1267	set_compound_order(page, 0);
1268	page[1].compound_nr = 0;
1269	__ClearPageHead(page);
 
 
 
 
 
 
 
 
 
 
 
1270}
1271
1272static void free_gigantic_page(struct page *page, unsigned int order)
1273{
1274	/*
1275	 * If the page isn't allocated using the cma allocator,
1276	 * cma_release() returns false.
1277	 */
1278#ifdef CONFIG_CMA
1279	if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order))
 
 
1280		return;
1281#endif
1282
1283	free_contig_range(page_to_pfn(page), 1 << order);
1284}
1285
1286#ifdef CONFIG_CONTIG_ALLOC
1287static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1288		int nid, nodemask_t *nodemask)
1289{
 
1290	unsigned long nr_pages = pages_per_huge_page(h);
1291	if (nid == NUMA_NO_NODE)
1292		nid = numa_mem_id();
1293
1294#ifdef CONFIG_CMA
1295	{
1296		struct page *page;
1297		int node;
1298
1299		if (hugetlb_cma[nid]) {
1300			page = cma_alloc(hugetlb_cma[nid], nr_pages,
1301					huge_page_order(h), true);
1302			if (page)
1303				return page;
1304		}
1305
1306		if (!(gfp_mask & __GFP_THISNODE)) {
1307			for_each_node_mask(node, *nodemask) {
1308				if (node == nid || !hugetlb_cma[node])
1309					continue;
1310
1311				page = cma_alloc(hugetlb_cma[node], nr_pages,
1312						huge_page_order(h), true);
1313				if (page)
1314					return page;
1315			}
1316		}
1317	}
1318#endif
1319
1320	return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
 
1321}
1322
1323#else /* !CONFIG_CONTIG_ALLOC */
1324static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1325					int nid, nodemask_t *nodemask)
1326{
1327	return NULL;
1328}
1329#endif /* CONFIG_CONTIG_ALLOC */
1330
1331#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1332static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1333					int nid, nodemask_t *nodemask)
1334{
1335	return NULL;
1336}
1337static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1338static inline void destroy_compound_gigantic_page(struct page *page,
 
1339						unsigned int order) { }
1340#endif
1341
 
 
 
 
 
 
 
 
1342/*
1343 * Remove hugetlb page from lists, and update dtor so that page appears
1344 * as just a compound page.  A reference is held on the page.
 
 
 
 
1345 *
1346 * Must be called with hugetlb lock held.
1347 */
1348static void remove_hugetlb_page(struct hstate *h, struct page *page,
1349							bool adjust_surplus)
 
1350{
1351	int nid = page_to_nid(page);
1352
1353	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1354	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page);
1355
1356	lockdep_assert_held(&hugetlb_lock);
1357	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1358		return;
1359
1360	list_del(&page->lru);
1361
1362	if (HPageFreed(page)) {
1363		h->free_huge_pages--;
1364		h->free_huge_pages_node[nid]--;
1365	}
1366	if (adjust_surplus) {
1367		h->surplus_huge_pages--;
1368		h->surplus_huge_pages_node[nid]--;
1369	}
1370
1371	set_page_refcounted(page);
1372	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
 
 
 
 
 
 
 
 
 
 
 
 
1373
1374	h->nr_huge_pages--;
1375	h->nr_huge_pages_node[nid]--;
1376}
1377
1378static void add_hugetlb_page(struct hstate *h, struct page *page,
 
 
 
 
 
 
 
 
 
 
 
 
1379			     bool adjust_surplus)
1380{
1381	int zeroed;
1382	int nid = page_to_nid(page);
1383
1384	VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
1385
1386	lockdep_assert_held(&hugetlb_lock);
1387
1388	INIT_LIST_HEAD(&page->lru);
1389	h->nr_huge_pages++;
1390	h->nr_huge_pages_node[nid]++;
1391
1392	if (adjust_surplus) {
1393		h->surplus_huge_pages++;
1394		h->surplus_huge_pages_node[nid]++;
1395	}
1396
1397	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1398	set_page_private(page, 0);
1399	SetHPageVmemmapOptimized(page);
 
 
 
 
1400
1401	/*
1402	 * This page is now managed by the hugetlb allocator and has
1403	 * no users -- drop the last reference.
 
1404	 */
1405	zeroed = put_page_testzero(page);
1406	VM_BUG_ON_PAGE(!zeroed, page);
1407	arch_clear_hugepage_flags(page);
1408	enqueue_huge_page(h, page);
 
 
 
 
 
 
 
 
1409}
1410
1411static void __update_and_free_page(struct hstate *h, struct page *page)
 
1412{
1413	int i;
1414	struct page *subpage = page;
1415
1416	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1417		return;
1418
1419	if (alloc_huge_page_vmemmap(h, page)) {
 
 
 
 
 
 
 
 
 
 
 
 
1420		spin_lock_irq(&hugetlb_lock);
1421		/*
1422		 * If we cannot allocate vmemmap pages, just refuse to free the
1423		 * page and put the page back on the hugetlb free list and treat
1424		 * as a surplus page.
1425		 */
1426		add_hugetlb_page(h, page, true);
1427		spin_unlock_irq(&hugetlb_lock);
1428		return;
1429	}
1430
1431	for (i = 0; i < pages_per_huge_page(h);
1432	     i++, subpage = mem_map_next(subpage, page, i)) {
1433		subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1434				1 << PG_referenced | 1 << PG_dirty |
1435				1 << PG_active | 1 << PG_private |
1436				1 << PG_writeback);
 
 
 
 
 
 
 
 
 
1437	}
1438	if (hstate_is_gigantic(h)) {
1439		destroy_compound_gigantic_page(page, huge_page_order(h));
1440		free_gigantic_page(page, huge_page_order(h));
 
 
 
 
 
 
1441	} else {
1442		__free_pages(page, huge_page_order(h));
1443	}
1444}
1445
1446/*
1447 * As update_and_free_page() can be called under any context, so we cannot
1448 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
1449 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
1450 * the vmemmap pages.
1451 *
1452 * free_hpage_workfn() locklessly retrieves the linked list of pages to be
1453 * freed and frees them one-by-one. As the page->mapping pointer is going
1454 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
1455 * structure of a lockless linked list of huge pages to be freed.
1456 */
1457static LLIST_HEAD(hpage_freelist);
1458
1459static void free_hpage_workfn(struct work_struct *work)
1460{
1461	struct llist_node *node;
1462
1463	node = llist_del_all(&hpage_freelist);
1464
1465	while (node) {
1466		struct page *page;
1467		struct hstate *h;
1468
1469		page = container_of((struct address_space **)node,
1470				     struct page, mapping);
1471		node = node->next;
1472		page->mapping = NULL;
1473		/*
1474		 * The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate()
1475		 * is going to trigger because a previous call to
1476		 * remove_hugetlb_page() will set_compound_page_dtor(page,
1477		 * NULL_COMPOUND_DTOR), so do not use page_hstate() directly.
1478		 */
1479		h = size_to_hstate(page_size(page));
1480
1481		__update_and_free_page(h, page);
1482
1483		cond_resched();
1484	}
1485}
1486static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1487
1488static inline void flush_free_hpage_work(struct hstate *h)
1489{
1490	if (free_vmemmap_pages_per_hpage(h))
1491		flush_work(&free_hpage_work);
1492}
1493
1494static void update_and_free_page(struct hstate *h, struct page *page,
1495				 bool atomic)
1496{
1497	if (!HPageVmemmapOptimized(page) || !atomic) {
1498		__update_and_free_page(h, page);
1499		return;
1500	}
1501
1502	/*
1503	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
1504	 *
1505	 * Only call schedule_work() if hpage_freelist is previously
1506	 * empty. Otherwise, schedule_work() had been called but the workfn
1507	 * hasn't retrieved the list yet.
1508	 */
1509	if (llist_add((struct llist_node *)&page->mapping, &hpage_freelist))
1510		schedule_work(&free_hpage_work);
1511}
1512
1513static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1514{
1515	struct page *page, *t_page;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1516
1517	list_for_each_entry_safe(page, t_page, list, lru) {
1518		update_and_free_page(h, page, false);
1519		cond_resched();
1520	}
1521}
1522
1523struct hstate *size_to_hstate(unsigned long size)
1524{
1525	struct hstate *h;
1526
1527	for_each_hstate(h) {
1528		if (huge_page_size(h) == size)
1529			return h;
1530	}
1531	return NULL;
1532}
1533
1534void free_huge_page(struct page *page)
1535{
1536	/*
1537	 * Can't pass hstate in here because it is called from the
1538	 * compound page destructor.
1539	 */
1540	struct hstate *h = page_hstate(page);
1541	int nid = page_to_nid(page);
1542	struct hugepage_subpool *spool = hugetlb_page_subpool(page);
1543	bool restore_reserve;
1544	unsigned long flags;
1545
1546	VM_BUG_ON_PAGE(page_count(page), page);
1547	VM_BUG_ON_PAGE(page_mapcount(page), page);
1548
1549	hugetlb_set_page_subpool(page, NULL);
1550	page->mapping = NULL;
1551	restore_reserve = HPageRestoreReserve(page);
1552	ClearHPageRestoreReserve(page);
 
 
1553
1554	/*
1555	 * If HPageRestoreReserve was set on page, page allocation consumed a
1556	 * reservation.  If the page was associated with a subpool, there
1557	 * would have been a page reserved in the subpool before allocation
1558	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1559	 * reservation, do not call hugepage_subpool_put_pages() as this will
1560	 * remove the reserved page from the subpool.
1561	 */
1562	if (!restore_reserve) {
1563		/*
1564		 * A return code of zero implies that the subpool will be
1565		 * under its minimum size if the reservation is not restored
1566		 * after page is free.  Therefore, force restore_reserve
1567		 * operation.
1568		 */
1569		if (hugepage_subpool_put_pages(spool, 1) == 0)
1570			restore_reserve = true;
1571	}
1572
1573	spin_lock_irqsave(&hugetlb_lock, flags);
1574	ClearHPageMigratable(page);
1575	hugetlb_cgroup_uncharge_page(hstate_index(h),
1576				     pages_per_huge_page(h), page);
1577	hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
1578					  pages_per_huge_page(h), page);
 
1579	if (restore_reserve)
1580		h->resv_huge_pages++;
1581
1582	if (HPageTemporary(page)) {
1583		remove_hugetlb_page(h, page, false);
1584		spin_unlock_irqrestore(&hugetlb_lock, flags);
1585		update_and_free_page(h, page, true);
1586	} else if (h->surplus_huge_pages_node[nid]) {
1587		/* remove the page from active list */
1588		remove_hugetlb_page(h, page, true);
1589		spin_unlock_irqrestore(&hugetlb_lock, flags);
1590		update_and_free_page(h, page, true);
1591	} else {
1592		arch_clear_hugepage_flags(page);
1593		enqueue_huge_page(h, page);
1594		spin_unlock_irqrestore(&hugetlb_lock, flags);
1595	}
1596}
1597
1598/*
1599 * Must be called with the hugetlb lock held
1600 */
1601static void __prep_account_new_huge_page(struct hstate *h, int nid)
1602{
1603	lockdep_assert_held(&hugetlb_lock);
1604	h->nr_huge_pages++;
1605	h->nr_huge_pages_node[nid]++;
1606}
1607
1608static void __prep_new_huge_page(struct hstate *h, struct page *page)
1609{
1610	free_huge_page_vmemmap(h, page);
1611	INIT_LIST_HEAD(&page->lru);
1612	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1613	hugetlb_set_page_subpool(page, NULL);
1614	set_hugetlb_cgroup(page, NULL);
1615	set_hugetlb_cgroup_rsvd(page, NULL);
1616}
1617
1618static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1619{
1620	__prep_new_huge_page(h, page);
 
 
 
 
 
 
1621	spin_lock_irq(&hugetlb_lock);
1622	__prep_account_new_huge_page(h, nid);
1623	spin_unlock_irq(&hugetlb_lock);
1624}
1625
1626static bool prep_compound_gigantic_page(struct page *page, unsigned int order)
 
1627{
1628	int i, j;
1629	int nr_pages = 1 << order;
1630	struct page *p = page + 1;
 
 
 
 
1631
1632	/* we rely on prep_new_huge_page to set the destructor */
1633	set_compound_order(page, order);
1634	__ClearPageReserved(page);
1635	__SetPageHead(page);
1636	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1637		/*
1638		 * For gigantic hugepages allocated through bootmem at
1639		 * boot, it's safer to be consistent with the not-gigantic
1640		 * hugepages and clear the PG_reserved bit from all tail pages
1641		 * too.  Otherwise drivers using get_user_pages() to access tail
1642		 * pages may get the reference counting wrong if they see
1643		 * PG_reserved set on a tail page (despite the head page not
1644		 * having PG_reserved set).  Enforcing this consistency between
1645		 * head and tail pages allows drivers to optimize away a check
1646		 * on the head page when they need know if put_page() is needed
1647		 * after get_user_pages().
1648		 */
1649		__ClearPageReserved(p);
 
1650		/*
1651		 * Subtle and very unlikely
1652		 *
1653		 * Gigantic 'page allocators' such as memblock or cma will
1654		 * return a set of pages with each page ref counted.  We need
1655		 * to turn this set of pages into a compound page with tail
1656		 * page ref counts set to zero.  Code such as speculative page
1657		 * cache adding could take a ref on a 'to be' tail page.
1658		 * We need to respect any increased ref count, and only set
1659		 * the ref count to zero if count is currently 1.  If count
1660		 * is not 1, we call synchronize_rcu in the hope that a rcu
1661		 * grace period will cause ref count to drop and then retry.
1662		 * If count is still inflated on retry we return an error and
1663		 * must discard the pages.
1664		 */
1665		if (!page_ref_freeze(p, 1)) {
1666			pr_info("HugeTLB unexpected inflated ref count on freshly allocated page\n");
1667			synchronize_rcu();
1668			if (!page_ref_freeze(p, 1))
 
1669				goto out_error;
 
 
 
1670		}
1671		set_page_count(p, 0);
1672		set_compound_head(p, page);
1673	}
1674	atomic_set(compound_mapcount_ptr(page), -1);
1675	atomic_set(compound_pincount_ptr(page), 0);
 
 
 
 
1676	return true;
1677
1678out_error:
1679	/* undo tail page modifications made above */
1680	p = page + 1;
1681	for (j = 1; j < i; j++, p = mem_map_next(p, page, j)) {
1682		clear_compound_head(p);
 
1683		set_page_refcounted(p);
1684	}
1685	/* need to clear PG_reserved on remaining tail pages  */
1686	for (; j < nr_pages; j++, p = mem_map_next(p, page, j))
 
1687		__ClearPageReserved(p);
1688	set_compound_order(page, 0);
1689	page[1].compound_nr = 0;
1690	__ClearPageHead(page);
1691	return false;
1692}
1693
 
 
 
 
 
 
 
 
 
 
 
 
1694/*
1695 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1696 * transparent huge pages.  See the PageTransHuge() documentation for more
1697 * details.
1698 */
1699int PageHuge(struct page *page)
1700{
 
 
1701	if (!PageCompound(page))
1702		return 0;
1703
1704	page = compound_head(page);
1705	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1706}
1707EXPORT_SYMBOL_GPL(PageHuge);
1708
1709/*
1710 * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1711 * normal or transparent huge pages.
1712 */
1713int PageHeadHuge(struct page *page_head)
1714{
1715	if (!PageHead(page_head))
1716		return 0;
1717
1718	return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR;
1719}
1720
1721/*
1722 * Find and lock address space (mapping) in write mode.
1723 *
1724 * Upon entry, the page is locked which means that page_mapping() is
1725 * stable.  Due to locking order, we can only trylock_write.  If we can
1726 * not get the lock, simply return NULL to caller.
1727 */
1728struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
1729{
1730	struct address_space *mapping = page_mapping(hpage);
1731
1732	if (!mapping)
1733		return mapping;
1734
1735	if (i_mmap_trylock_write(mapping))
1736		return mapping;
1737
1738	return NULL;
1739}
1740
1741pgoff_t hugetlb_basepage_index(struct page *page)
1742{
1743	struct page *page_head = compound_head(page);
1744	pgoff_t index = page_index(page_head);
1745	unsigned long compound_idx;
1746
1747	if (compound_order(page_head) >= MAX_ORDER)
1748		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1749	else
1750		compound_idx = page - page_head;
1751
1752	return (index << compound_order(page_head)) + compound_idx;
1753}
1754
1755static struct page *alloc_buddy_huge_page(struct hstate *h,
1756		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1757		nodemask_t *node_alloc_noretry)
1758{
1759	int order = huge_page_order(h);
1760	struct page *page;
1761	bool alloc_try_hard = true;
 
1762
1763	/*
1764	 * By default we always try hard to allocate the page with
1765	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
1766	 * a loop (to adjust global huge page counts) and previous allocation
1767	 * failed, do not continue to try hard on the same node.  Use the
1768	 * node_alloc_noretry bitmap to manage this state information.
1769	 */
1770	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
1771		alloc_try_hard = false;
1772	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
1773	if (alloc_try_hard)
1774		gfp_mask |= __GFP_RETRY_MAYFAIL;
1775	if (nid == NUMA_NO_NODE)
1776		nid = numa_mem_id();
 
1777	page = __alloc_pages(gfp_mask, order, nid, nmask);
1778	if (page)
1779		__count_vm_event(HTLB_BUDDY_PGALLOC);
1780	else
1781		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
 
 
 
 
 
 
 
 
1782
1783	/*
1784	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
1785	 * indicates an overall state change.  Clear bit so that we resume
1786	 * normal 'try hard' allocations.
1787	 */
1788	if (node_alloc_noretry && page && !alloc_try_hard)
1789		node_clear(nid, *node_alloc_noretry);
1790
1791	/*
1792	 * If we tried hard to get a page but failed, set bit so that
1793	 * subsequent attempts will not try as hard until there is an
1794	 * overall state change.
1795	 */
1796	if (node_alloc_noretry && !page && alloc_try_hard)
1797		node_set(nid, *node_alloc_noretry);
1798
1799	return page;
 
 
 
 
 
 
1800}
1801
1802/*
1803 * Common helper to allocate a fresh hugetlb page. All specific allocators
1804 * should use this function to get new hugetlb pages
1805 */
1806static struct page *alloc_fresh_huge_page(struct hstate *h,
1807		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1808		nodemask_t *node_alloc_noretry)
1809{
1810	struct page *page;
1811	bool retry = false;
1812
1813retry:
1814	if (hstate_is_gigantic(h))
1815		page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
1816	else
1817		page = alloc_buddy_huge_page(h, gfp_mask,
1818				nid, nmask, node_alloc_noretry);
1819	if (!page)
1820		return NULL;
1821
1822	if (hstate_is_gigantic(h)) {
1823		if (!prep_compound_gigantic_page(page, huge_page_order(h))) {
1824			/*
1825			 * Rare failure to convert pages to compound page.
1826			 * Free pages and try again - ONCE!
1827			 */
1828			free_gigantic_page(page, huge_page_order(h));
1829			if (!retry) {
1830				retry = true;
1831				goto retry;
1832			}
1833			pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
1834			return NULL;
1835		}
1836	}
1837	prep_new_huge_page(h, page, page_to_nid(page));
1838
1839	return page;
 
 
 
 
 
 
 
 
 
 
 
 
 
1840}
1841
1842/*
1843 * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
1844 * manner.
 
 
 
1845 */
1846static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1847				nodemask_t *node_alloc_noretry)
 
1848{
1849	struct page *page;
1850	int nr_nodes, node;
1851	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
1852
1853	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1854		page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
1855						node_alloc_noretry);
1856		if (page)
1857			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1858	}
 
 
 
 
 
 
 
 
 
 
 
 
 
1859
1860	if (!page)
1861		return 0;
1862
1863	put_page(page); /* free it into the hugepage allocator */
 
 
 
 
1864
1865	return 1;
1866}
1867
1868/*
1869 * Remove huge page from pool from next node to free.  Attempt to keep
1870 * persistent huge pages more or less balanced over allowed nodes.
1871 * This routine only 'removes' the hugetlb page.  The caller must make
1872 * an additional call to free the page to low level allocators.
1873 * Called with hugetlb_lock locked.
1874 */
1875static struct page *remove_pool_huge_page(struct hstate *h,
1876						nodemask_t *nodes_allowed,
1877						 bool acct_surplus)
1878{
1879	int nr_nodes, node;
1880	struct page *page = NULL;
1881
1882	lockdep_assert_held(&hugetlb_lock);
1883	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1884		/*
1885		 * If we're returning unused surplus pages, only examine
1886		 * nodes with surplus pages.
1887		 */
1888		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1889		    !list_empty(&h->hugepage_freelists[node])) {
1890			page = list_entry(h->hugepage_freelists[node].next,
1891					  struct page, lru);
1892			remove_hugetlb_page(h, page, acct_surplus);
1893			break;
1894		}
1895	}
1896
1897	return page;
1898}
1899
1900/*
1901 * Dissolve a given free hugepage into free buddy pages. This function does
1902 * nothing for in-use hugepages and non-hugepages.
1903 * This function returns values like below:
1904 *
1905 *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
1906 *           when the system is under memory pressure and the feature of
1907 *           freeing unused vmemmap pages associated with each hugetlb page
1908 *           is enabled.
1909 *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
1910 *           (allocated or reserved.)
1911 *       0:  successfully dissolved free hugepages or the page is not a
1912 *           hugepage (considered as already dissolved)
1913 */
1914int dissolve_free_huge_page(struct page *page)
1915{
1916	int rc = -EBUSY;
 
1917
1918retry:
1919	/* Not to disrupt normal path by vainly holding hugetlb_lock */
1920	if (!PageHuge(page))
1921		return 0;
1922
1923	spin_lock_irq(&hugetlb_lock);
1924	if (!PageHuge(page)) {
1925		rc = 0;
1926		goto out;
1927	}
1928
1929	if (!page_count(page)) {
1930		struct page *head = compound_head(page);
1931		struct hstate *h = page_hstate(head);
1932		if (h->free_huge_pages - h->resv_huge_pages == 0)
1933			goto out;
1934
1935		/*
1936		 * We should make sure that the page is already on the free list
1937		 * when it is dissolved.
1938		 */
1939		if (unlikely(!HPageFreed(head))) {
1940			spin_unlock_irq(&hugetlb_lock);
1941			cond_resched();
1942
1943			/*
1944			 * Theoretically, we should return -EBUSY when we
1945			 * encounter this race. In fact, we have a chance
1946			 * to successfully dissolve the page if we do a
1947			 * retry. Because the race window is quite small.
1948			 * If we seize this opportunity, it is an optimization
1949			 * for increasing the success rate of dissolving page.
1950			 */
1951			goto retry;
1952		}
1953
1954		remove_hugetlb_page(h, head, false);
1955		h->max_huge_pages--;
1956		spin_unlock_irq(&hugetlb_lock);
1957
1958		/*
1959		 * Normally update_and_free_page will allocate required vmemmmap
1960		 * before freeing the page.  update_and_free_page will fail to
1961		 * free the page if it can not allocate required vmemmap.  We
1962		 * need to adjust max_huge_pages if the page is not freed.
1963		 * Attempt to allocate vmemmmap here so that we can take
1964		 * appropriate action on failure.
1965		 */
1966		rc = alloc_huge_page_vmemmap(h, head);
1967		if (!rc) {
1968			/*
1969			 * Move PageHWPoison flag from head page to the raw
1970			 * error page, which makes any subpages rather than
1971			 * the error page reusable.
1972			 */
1973			if (PageHWPoison(head) && page != head) {
1974				SetPageHWPoison(page);
1975				ClearPageHWPoison(head);
 
1976			}
1977			update_and_free_page(h, head, false);
1978		} else {
1979			spin_lock_irq(&hugetlb_lock);
1980			add_hugetlb_page(h, head, false);
1981			h->max_huge_pages++;
1982			spin_unlock_irq(&hugetlb_lock);
1983		}
1984
 
1985		return rc;
1986	}
1987out:
1988	spin_unlock_irq(&hugetlb_lock);
1989	return rc;
1990}
1991
1992/*
1993 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1994 * make specified memory blocks removable from the system.
1995 * Note that this will dissolve a free gigantic hugepage completely, if any
1996 * part of it lies within the given range.
1997 * Also note that if dissolve_free_huge_page() returns with an error, all
1998 * free hugepages that were dissolved before that error are lost.
1999 */
2000int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
2001{
2002	unsigned long pfn;
2003	struct page *page;
2004	int rc = 0;
 
 
2005
2006	if (!hugepages_supported())
2007		return rc;
2008
2009	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
 
 
 
 
2010		page = pfn_to_page(pfn);
2011		rc = dissolve_free_huge_page(page);
2012		if (rc)
2013			break;
2014	}
2015
2016	return rc;
2017}
2018
2019/*
2020 * Allocates a fresh surplus page from the page allocator.
2021 */
2022static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
2023		int nid, nodemask_t *nmask)
2024{
2025	struct page *page = NULL;
2026
2027	if (hstate_is_gigantic(h))
2028		return NULL;
2029
2030	spin_lock_irq(&hugetlb_lock);
2031	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
2032		goto out_unlock;
2033	spin_unlock_irq(&hugetlb_lock);
2034
2035	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
2036	if (!page)
2037		return NULL;
2038
2039	spin_lock_irq(&hugetlb_lock);
2040	/*
2041	 * We could have raced with the pool size change.
2042	 * Double check that and simply deallocate the new page
2043	 * if we would end up overcommiting the surpluses. Abuse
2044	 * temporary page to workaround the nasty free_huge_page
2045	 * codeflow
2046	 */
2047	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2048		SetHPageTemporary(page);
2049		spin_unlock_irq(&hugetlb_lock);
2050		put_page(page);
2051		return NULL;
2052	} else {
2053		h->surplus_huge_pages++;
2054		h->surplus_huge_pages_node[page_to_nid(page)]++;
2055	}
2056
 
 
 
2057out_unlock:
2058	spin_unlock_irq(&hugetlb_lock);
2059
2060	return page;
2061}
2062
2063static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
2064				     int nid, nodemask_t *nmask)
2065{
2066	struct page *page;
2067
2068	if (hstate_is_gigantic(h))
2069		return NULL;
2070
2071	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
2072	if (!page)
2073		return NULL;
2074
 
 
2075	/*
2076	 * We do not account these pages as surplus because they are only
2077	 * temporary and will be released properly on the last reference
2078	 */
2079	SetHPageTemporary(page);
2080
2081	return page;
2082}
2083
2084/*
2085 * Use the VMA's mpolicy to allocate a huge page from the buddy.
2086 */
2087static
2088struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
2089		struct vm_area_struct *vma, unsigned long addr)
2090{
2091	struct page *page;
2092	struct mempolicy *mpol;
2093	gfp_t gfp_mask = htlb_alloc_mask(h);
2094	int nid;
2095	nodemask_t *nodemask;
2096
2097	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2098	page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);
 
 
 
 
 
 
 
 
 
 
 
2099	mpol_cond_put(mpol);
2100
2101	return page;
2102}
2103
2104/* page migration callback function */
2105struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
2106		nodemask_t *nmask, gfp_t gfp_mask)
2107{
2108	spin_lock_irq(&hugetlb_lock);
2109	if (h->free_huge_pages - h->resv_huge_pages > 0) {
2110		struct page *page;
2111
2112		page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
2113		if (page) {
 
2114			spin_unlock_irq(&hugetlb_lock);
2115			return page;
2116		}
2117	}
2118	spin_unlock_irq(&hugetlb_lock);
2119
2120	return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
2121}
2122
2123/* mempolicy aware migration callback */
2124struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
2125		unsigned long address)
2126{
2127	struct mempolicy *mpol;
2128	nodemask_t *nodemask;
2129	struct page *page;
2130	gfp_t gfp_mask;
2131	int node;
2132
2133	gfp_mask = htlb_alloc_mask(h);
2134	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
2135	page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask);
2136	mpol_cond_put(mpol);
2137
2138	return page;
2139}
2140
2141/*
2142 * Increase the hugetlb pool such that it can accommodate a reservation
2143 * of size 'delta'.
2144 */
2145static int gather_surplus_pages(struct hstate *h, long delta)
2146	__must_hold(&hugetlb_lock)
2147{
2148	struct list_head surplus_list;
2149	struct page *page, *tmp;
2150	int ret;
2151	long i;
2152	long needed, allocated;
2153	bool alloc_ok = true;
2154
2155	lockdep_assert_held(&hugetlb_lock);
2156	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2157	if (needed <= 0) {
2158		h->resv_huge_pages += delta;
2159		return 0;
2160	}
2161
2162	allocated = 0;
2163	INIT_LIST_HEAD(&surplus_list);
2164
2165	ret = -ENOMEM;
2166retry:
2167	spin_unlock_irq(&hugetlb_lock);
2168	for (i = 0; i < needed; i++) {
2169		page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
2170				NUMA_NO_NODE, NULL);
2171		if (!page) {
2172			alloc_ok = false;
2173			break;
2174		}
2175		list_add(&page->lru, &surplus_list);
2176		cond_resched();
2177	}
2178	allocated += i;
2179
2180	/*
2181	 * After retaking hugetlb_lock, we need to recalculate 'needed'
2182	 * because either resv_huge_pages or free_huge_pages may have changed.
2183	 */
2184	spin_lock_irq(&hugetlb_lock);
2185	needed = (h->resv_huge_pages + delta) -
2186			(h->free_huge_pages + allocated);
2187	if (needed > 0) {
2188		if (alloc_ok)
2189			goto retry;
2190		/*
2191		 * We were not able to allocate enough pages to
2192		 * satisfy the entire reservation so we free what
2193		 * we've allocated so far.
2194		 */
2195		goto free;
2196	}
2197	/*
2198	 * The surplus_list now contains _at_least_ the number of extra pages
2199	 * needed to accommodate the reservation.  Add the appropriate number
2200	 * of pages to the hugetlb pool and free the extras back to the buddy
2201	 * allocator.  Commit the entire reservation here to prevent another
2202	 * process from stealing the pages as they are added to the pool but
2203	 * before they are reserved.
2204	 */
2205	needed += allocated;
2206	h->resv_huge_pages += delta;
2207	ret = 0;
2208
2209	/* Free the needed pages to the hugetlb pool */
2210	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
2211		int zeroed;
2212
2213		if ((--needed) < 0)
2214			break;
2215		/*
2216		 * This page is now managed by the hugetlb allocator and has
2217		 * no users -- drop the buddy allocator's reference.
2218		 */
2219		zeroed = put_page_testzero(page);
2220		VM_BUG_ON_PAGE(!zeroed, page);
2221		enqueue_huge_page(h, page);
2222	}
2223free:
2224	spin_unlock_irq(&hugetlb_lock);
2225
2226	/* Free unnecessary surplus pages to the buddy allocator */
2227	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
2228		put_page(page);
 
 
 
2229	spin_lock_irq(&hugetlb_lock);
2230
2231	return ret;
2232}
2233
2234/*
2235 * This routine has two main purposes:
2236 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2237 *    in unused_resv_pages.  This corresponds to the prior adjustments made
2238 *    to the associated reservation map.
2239 * 2) Free any unused surplus pages that may have been allocated to satisfy
2240 *    the reservation.  As many as unused_resv_pages may be freed.
2241 */
2242static void return_unused_surplus_pages(struct hstate *h,
2243					unsigned long unused_resv_pages)
2244{
2245	unsigned long nr_pages;
2246	struct page *page;
2247	LIST_HEAD(page_list);
2248
2249	lockdep_assert_held(&hugetlb_lock);
2250	/* Uncommit the reservation */
2251	h->resv_huge_pages -= unused_resv_pages;
2252
2253	/* Cannot return gigantic pages currently */
2254	if (hstate_is_gigantic(h))
2255		goto out;
2256
2257	/*
2258	 * Part (or even all) of the reservation could have been backed
2259	 * by pre-allocated pages. Only free surplus pages.
2260	 */
2261	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2262
2263	/*
2264	 * We want to release as many surplus pages as possible, spread
2265	 * evenly across all nodes with memory. Iterate across these nodes
2266	 * until we can no longer free unreserved surplus pages. This occurs
2267	 * when the nodes with surplus pages have no free pages.
2268	 * remove_pool_huge_page() will balance the freed pages across the
2269	 * on-line nodes with memory and will handle the hstate accounting.
2270	 */
2271	while (nr_pages--) {
2272		page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1);
2273		if (!page)
 
 
2274			goto out;
2275
2276		list_add(&page->lru, &page_list);
2277	}
2278
2279out:
2280	spin_unlock_irq(&hugetlb_lock);
2281	update_and_free_pages_bulk(h, &page_list);
2282	spin_lock_irq(&hugetlb_lock);
2283}
2284
2285
2286/*
2287 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2288 * are used by the huge page allocation routines to manage reservations.
2289 *
2290 * vma_needs_reservation is called to determine if the huge page at addr
2291 * within the vma has an associated reservation.  If a reservation is
2292 * needed, the value 1 is returned.  The caller is then responsible for
2293 * managing the global reservation and subpool usage counts.  After
2294 * the huge page has been allocated, vma_commit_reservation is called
2295 * to add the page to the reservation map.  If the page allocation fails,
2296 * the reservation must be ended instead of committed.  vma_end_reservation
2297 * is called in such cases.
2298 *
2299 * In the normal case, vma_commit_reservation returns the same value
2300 * as the preceding vma_needs_reservation call.  The only time this
2301 * is not the case is if a reserve map was changed between calls.  It
2302 * is the responsibility of the caller to notice the difference and
2303 * take appropriate action.
2304 *
2305 * vma_add_reservation is used in error paths where a reservation must
2306 * be restored when a newly allocated huge page must be freed.  It is
2307 * to be called after calling vma_needs_reservation to determine if a
2308 * reservation exists.
2309 *
2310 * vma_del_reservation is used in error paths where an entry in the reserve
2311 * map was created during huge page allocation and must be removed.  It is to
2312 * be called after calling vma_needs_reservation to determine if a reservation
2313 * exists.
2314 */
2315enum vma_resv_mode {
2316	VMA_NEEDS_RESV,
2317	VMA_COMMIT_RESV,
2318	VMA_END_RESV,
2319	VMA_ADD_RESV,
2320	VMA_DEL_RESV,
2321};
2322static long __vma_reservation_common(struct hstate *h,
2323				struct vm_area_struct *vma, unsigned long addr,
2324				enum vma_resv_mode mode)
2325{
2326	struct resv_map *resv;
2327	pgoff_t idx;
2328	long ret;
2329	long dummy_out_regions_needed;
2330
2331	resv = vma_resv_map(vma);
2332	if (!resv)
2333		return 1;
2334
2335	idx = vma_hugecache_offset(h, vma, addr);
2336	switch (mode) {
2337	case VMA_NEEDS_RESV:
2338		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2339		/* We assume that vma_reservation_* routines always operate on
2340		 * 1 page, and that adding to resv map a 1 page entry can only
2341		 * ever require 1 region.
2342		 */
2343		VM_BUG_ON(dummy_out_regions_needed != 1);
2344		break;
2345	case VMA_COMMIT_RESV:
2346		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2347		/* region_add calls of range 1 should never fail. */
2348		VM_BUG_ON(ret < 0);
2349		break;
2350	case VMA_END_RESV:
2351		region_abort(resv, idx, idx + 1, 1);
2352		ret = 0;
2353		break;
2354	case VMA_ADD_RESV:
2355		if (vma->vm_flags & VM_MAYSHARE) {
2356			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2357			/* region_add calls of range 1 should never fail. */
2358			VM_BUG_ON(ret < 0);
2359		} else {
2360			region_abort(resv, idx, idx + 1, 1);
2361			ret = region_del(resv, idx, idx + 1);
2362		}
2363		break;
2364	case VMA_DEL_RESV:
2365		if (vma->vm_flags & VM_MAYSHARE) {
2366			region_abort(resv, idx, idx + 1, 1);
2367			ret = region_del(resv, idx, idx + 1);
2368		} else {
2369			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2370			/* region_add calls of range 1 should never fail. */
2371			VM_BUG_ON(ret < 0);
2372		}
2373		break;
2374	default:
2375		BUG();
2376	}
2377
2378	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2379		return ret;
2380	/*
2381	 * We know private mapping must have HPAGE_RESV_OWNER set.
2382	 *
2383	 * In most cases, reserves always exist for private mappings.
2384	 * However, a file associated with mapping could have been
2385	 * hole punched or truncated after reserves were consumed.
2386	 * As subsequent fault on such a range will not use reserves.
2387	 * Subtle - The reserve map for private mappings has the
2388	 * opposite meaning than that of shared mappings.  If NO
2389	 * entry is in the reserve map, it means a reservation exists.
2390	 * If an entry exists in the reserve map, it means the
2391	 * reservation has already been consumed.  As a result, the
2392	 * return value of this routine is the opposite of the
2393	 * value returned from reserve map manipulation routines above.
2394	 */
2395	if (ret > 0)
2396		return 0;
2397	if (ret == 0)
2398		return 1;
2399	return ret;
2400}
2401
2402static long vma_needs_reservation(struct hstate *h,
2403			struct vm_area_struct *vma, unsigned long addr)
2404{
2405	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2406}
2407
2408static long vma_commit_reservation(struct hstate *h,
2409			struct vm_area_struct *vma, unsigned long addr)
2410{
2411	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2412}
2413
2414static void vma_end_reservation(struct hstate *h,
2415			struct vm_area_struct *vma, unsigned long addr)
2416{
2417	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2418}
2419
2420static long vma_add_reservation(struct hstate *h,
2421			struct vm_area_struct *vma, unsigned long addr)
2422{
2423	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2424}
2425
2426static long vma_del_reservation(struct hstate *h,
2427			struct vm_area_struct *vma, unsigned long addr)
2428{
2429	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
2430}
2431
2432/*
2433 * This routine is called to restore reservation information on error paths.
2434 * It should ONLY be called for pages allocated via alloc_huge_page(), and
2435 * the hugetlb mutex should remain held when calling this routine.
2436 *
2437 * It handles two specific cases:
2438 * 1) A reservation was in place and the page consumed the reservation.
2439 *    HPageRestoreReserve is set in the page.
2440 * 2) No reservation was in place for the page, so HPageRestoreReserve is
2441 *    not set.  However, alloc_huge_page always updates the reserve map.
2442 *
2443 * In case 1, free_huge_page later in the error path will increment the
2444 * global reserve count.  But, free_huge_page does not have enough context
2445 * to adjust the reservation map.  This case deals primarily with private
2446 * mappings.  Adjust the reserve map here to be consistent with global
2447 * reserve count adjustments to be made by free_huge_page.  Make sure the
2448 * reserve map indicates there is a reservation present.
2449 *
2450 * In case 2, simply undo reserve map modifications done by alloc_huge_page.
2451 */
2452void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2453			unsigned long address, struct page *page)
2454{
2455	long rc = vma_needs_reservation(h, vma, address);
2456
2457	if (HPageRestoreReserve(page)) {
2458		if (unlikely(rc < 0))
2459			/*
2460			 * Rare out of memory condition in reserve map
2461			 * manipulation.  Clear HPageRestoreReserve so that
2462			 * global reserve count will not be incremented
2463			 * by free_huge_page.  This will make it appear
2464			 * as though the reservation for this page was
2465			 * consumed.  This may prevent the task from
2466			 * faulting in the page at a later time.  This
2467			 * is better than inconsistent global huge page
2468			 * accounting of reserve counts.
2469			 */
2470			ClearHPageRestoreReserve(page);
2471		else if (rc)
2472			(void)vma_add_reservation(h, vma, address);
2473		else
2474			vma_end_reservation(h, vma, address);
2475	} else {
2476		if (!rc) {
2477			/*
2478			 * This indicates there is an entry in the reserve map
2479			 * not added by alloc_huge_page.  We know it was added
2480			 * before the alloc_huge_page call, otherwise
2481			 * HPageRestoreReserve would be set on the page.
2482			 * Remove the entry so that a subsequent allocation
2483			 * does not consume a reservation.
2484			 */
2485			rc = vma_del_reservation(h, vma, address);
2486			if (rc < 0)
2487				/*
2488				 * VERY rare out of memory condition.  Since
2489				 * we can not delete the entry, set
2490				 * HPageRestoreReserve so that the reserve
2491				 * count will be incremented when the page
2492				 * is freed.  This reserve will be consumed
2493				 * on a subsequent allocation.
2494				 */
2495				SetHPageRestoreReserve(page);
2496		} else if (rc < 0) {
2497			/*
2498			 * Rare out of memory condition from
2499			 * vma_needs_reservation call.  Memory allocation is
2500			 * only attempted if a new entry is needed.  Therefore,
2501			 * this implies there is not an entry in the
2502			 * reserve map.
2503			 *
2504			 * For shared mappings, no entry in the map indicates
2505			 * no reservation.  We are done.
2506			 */
2507			if (!(vma->vm_flags & VM_MAYSHARE))
2508				/*
2509				 * For private mappings, no entry indicates
2510				 * a reservation is present.  Since we can
2511				 * not add an entry, set SetHPageRestoreReserve
2512				 * on the page so reserve count will be
2513				 * incremented when freed.  This reserve will
2514				 * be consumed on a subsequent allocation.
2515				 */
2516				SetHPageRestoreReserve(page);
2517		} else
2518			/*
2519			 * No reservation present, do nothing
2520			 */
2521			 vma_end_reservation(h, vma, address);
2522	}
2523}
2524
2525/*
2526 * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
 
2527 * @h: struct hstate old page belongs to
2528 * @old_page: Old page to dissolve
2529 * @list: List to isolate the page in case we need to
2530 * Returns 0 on success, otherwise negated error.
2531 */
2532static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
2533					struct list_head *list)
2534{
2535	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2536	int nid = page_to_nid(old_page);
2537	struct page *new_page;
2538	int ret = 0;
2539
2540	/*
2541	 * Before dissolving the page, we need to allocate a new one for the
2542	 * pool to remain stable.  Here, we allocate the page and 'prep' it
2543	 * by doing everything but actually updating counters and adding to
2544	 * the pool.  This simplifies and let us do most of the processing
2545	 * under the lock.
2546	 */
2547	new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
2548	if (!new_page)
2549		return -ENOMEM;
2550	__prep_new_huge_page(h, new_page);
2551
2552retry:
2553	spin_lock_irq(&hugetlb_lock);
2554	if (!PageHuge(old_page)) {
2555		/*
2556		 * Freed from under us. Drop new_page too.
2557		 */
2558		goto free_new;
2559	} else if (page_count(old_page)) {
 
 
2560		/*
2561		 * Someone has grabbed the page, try to isolate it here.
2562		 * Fail with -EBUSY if not possible.
2563		 */
2564		spin_unlock_irq(&hugetlb_lock);
2565		if (!isolate_huge_page(old_page, list))
2566			ret = -EBUSY;
2567		spin_lock_irq(&hugetlb_lock);
2568		goto free_new;
2569	} else if (!HPageFreed(old_page)) {
2570		/*
2571		 * Page's refcount is 0 but it has not been enqueued in the
2572		 * freelist yet. Race window is small, so we can succeed here if
2573		 * we retry.
2574		 */
2575		spin_unlock_irq(&hugetlb_lock);
2576		cond_resched();
2577		goto retry;
2578	} else {
2579		/*
2580		 * Ok, old_page is still a genuine free hugepage. Remove it from
2581		 * the freelist and decrease the counters. These will be
2582		 * incremented again when calling __prep_account_new_huge_page()
2583		 * and enqueue_huge_page() for new_page. The counters will remain
2584		 * stable since this happens under the lock.
2585		 */
2586		remove_hugetlb_page(h, old_page, false);
2587
2588		/*
2589		 * Reference count trick is needed because allocator gives us
2590		 * referenced page but the pool requires pages with 0 refcount.
2591		 */
2592		__prep_account_new_huge_page(h, nid);
2593		page_ref_dec(new_page);
2594		enqueue_huge_page(h, new_page);
2595
2596		/*
2597		 * Pages have been replaced, we can safely free the old one.
2598		 */
2599		spin_unlock_irq(&hugetlb_lock);
2600		update_and_free_page(h, old_page, false);
2601	}
2602
2603	return ret;
2604
2605free_new:
2606	spin_unlock_irq(&hugetlb_lock);
2607	update_and_free_page(h, new_page, false);
 
 
2608
2609	return ret;
2610}
2611
2612int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
2613{
2614	struct hstate *h;
2615	struct page *head;
2616	int ret = -EBUSY;
2617
2618	/*
2619	 * The page might have been dissolved from under our feet, so make sure
2620	 * to carefully check the state under the lock.
2621	 * Return success when racing as if we dissolved the page ourselves.
2622	 */
2623	spin_lock_irq(&hugetlb_lock);
2624	if (PageHuge(page)) {
2625		head = compound_head(page);
2626		h = page_hstate(head);
2627	} else {
2628		spin_unlock_irq(&hugetlb_lock);
2629		return 0;
2630	}
2631	spin_unlock_irq(&hugetlb_lock);
2632
2633	/*
2634	 * Fence off gigantic pages as there is a cyclic dependency between
2635	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
2636	 * of bailing out right away without further retrying.
2637	 */
2638	if (hstate_is_gigantic(h))
2639		return -ENOMEM;
2640
2641	if (page_count(head) && isolate_huge_page(head, list))
2642		ret = 0;
2643	else if (!page_count(head))
2644		ret = alloc_and_dissolve_huge_page(h, head, list);
2645
2646	return ret;
2647}
2648
2649struct page *alloc_huge_page(struct vm_area_struct *vma,
2650				    unsigned long addr, int avoid_reserve)
2651{
2652	struct hugepage_subpool *spool = subpool_vma(vma);
2653	struct hstate *h = hstate_vma(vma);
2654	struct page *page;
2655	long map_chg, map_commit;
2656	long gbl_chg;
2657	int ret, idx;
2658	struct hugetlb_cgroup *h_cg;
 
2659	bool deferred_reserve;
 
 
 
 
 
 
 
 
2660
2661	idx = hstate_index(h);
2662	/*
2663	 * Examine the region/reserve map to determine if the process
2664	 * has a reservation for the page to be allocated.  A return
2665	 * code of zero indicates a reservation exists (no change).
2666	 */
2667	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2668	if (map_chg < 0)
 
 
 
2669		return ERR_PTR(-ENOMEM);
 
2670
2671	/*
2672	 * Processes that did not create the mapping will have no
2673	 * reserves as indicated by the region/reserve map. Check
2674	 * that the allocation will not exceed the subpool limit.
2675	 * Allocations for MAP_NORESERVE mappings also need to be
2676	 * checked against any subpool limit.
2677	 */
2678	if (map_chg || avoid_reserve) {
2679		gbl_chg = hugepage_subpool_get_pages(spool, 1);
2680		if (gbl_chg < 0) {
2681			vma_end_reservation(h, vma, addr);
2682			return ERR_PTR(-ENOSPC);
2683		}
2684
2685		/*
2686		 * Even though there was no reservation in the region/reserve
2687		 * map, there could be reservations associated with the
2688		 * subpool that can be used.  This would be indicated if the
2689		 * return value of hugepage_subpool_get_pages() is zero.
2690		 * However, if avoid_reserve is specified we still avoid even
2691		 * the subpool reservations.
2692		 */
2693		if (avoid_reserve)
2694			gbl_chg = 1;
2695	}
2696
2697	/* If this allocation is not consuming a reservation, charge it now.
2698	 */
2699	deferred_reserve = map_chg || avoid_reserve;
2700	if (deferred_reserve) {
2701		ret = hugetlb_cgroup_charge_cgroup_rsvd(
2702			idx, pages_per_huge_page(h), &h_cg);
2703		if (ret)
2704			goto out_subpool_put;
2705	}
2706
2707	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2708	if (ret)
2709		goto out_uncharge_cgroup_reservation;
2710
2711	spin_lock_irq(&hugetlb_lock);
2712	/*
2713	 * glb_chg is passed to indicate whether or not a page must be taken
2714	 * from the global free pool (global change).  gbl_chg == 0 indicates
2715	 * a reservation exists for the allocation.
2716	 */
2717	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2718	if (!page) {
2719		spin_unlock_irq(&hugetlb_lock);
2720		page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
2721		if (!page)
2722			goto out_uncharge_cgroup;
 
2723		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2724			SetHPageRestoreReserve(page);
2725			h->resv_huge_pages--;
2726		}
2727		spin_lock_irq(&hugetlb_lock);
2728		list_add(&page->lru, &h->hugepage_activelist);
2729		/* Fall through */
2730	}
2731	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
 
2732	/* If allocation is not consuming a reservation, also store the
2733	 * hugetlb_cgroup pointer on the page.
2734	 */
2735	if (deferred_reserve) {
2736		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
2737						  h_cg, page);
2738	}
2739
2740	spin_unlock_irq(&hugetlb_lock);
2741
2742	hugetlb_set_page_subpool(page, spool);
2743
2744	map_commit = vma_commit_reservation(h, vma, addr);
2745	if (unlikely(map_chg > map_commit)) {
2746		/*
2747		 * The page was added to the reservation map between
2748		 * vma_needs_reservation and vma_commit_reservation.
2749		 * This indicates a race with hugetlb_reserve_pages.
2750		 * Adjust for the subpool count incremented above AND
2751		 * in hugetlb_reserve_pages for the same page.  Also,
2752		 * the reservation count added in hugetlb_reserve_pages
2753		 * no longer applies.
2754		 */
2755		long rsv_adjust;
2756
2757		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2758		hugetlb_acct_memory(h, -rsv_adjust);
2759		if (deferred_reserve)
2760			hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
2761					pages_per_huge_page(h), page);
2762	}
2763	return page;
 
 
 
 
 
2764
2765out_uncharge_cgroup:
2766	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2767out_uncharge_cgroup_reservation:
2768	if (deferred_reserve)
2769		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
2770						    h_cg);
2771out_subpool_put:
2772	if (map_chg || avoid_reserve)
2773		hugepage_subpool_put_pages(spool, 1);
 
2774	vma_end_reservation(h, vma, addr);
 
 
 
2775	return ERR_PTR(-ENOSPC);
2776}
2777
2778int alloc_bootmem_huge_page(struct hstate *h)
2779	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
2780int __alloc_bootmem_huge_page(struct hstate *h)
2781{
2782	struct huge_bootmem_page *m;
2783	int nr_nodes, node;
2784
 
 
 
 
 
 
 
 
 
2785	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2786		void *addr;
2787
2788		addr = memblock_alloc_try_nid_raw(
2789				huge_page_size(h), huge_page_size(h),
2790				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
2791		if (addr) {
2792			/*
2793			 * Use the beginning of the huge page to store the
2794			 * huge_bootmem_page struct (until gather_bootmem
2795			 * puts them into the mem_map).
2796			 */
2797			m = addr;
2798			goto found;
2799		}
2800	}
2801	return 0;
2802
2803found:
2804	BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
 
 
 
 
 
 
 
 
 
2805	/* Put them into a private list first because mem_map is not up yet */
2806	INIT_LIST_HEAD(&m->list);
2807	list_add(&m->list, &huge_boot_pages);
2808	m->hstate = h;
2809	return 1;
2810}
2811
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2812/*
2813 * Put bootmem huge pages into the standard lists after mem_map is up.
2814 * Note: This only applies to gigantic (order > MAX_ORDER) pages.
2815 */
2816static void __init gather_bootmem_prealloc(void)
2817{
 
2818	struct huge_bootmem_page *m;
 
2819
2820	list_for_each_entry(m, &huge_boot_pages, list) {
2821		struct page *page = virt_to_page(m);
2822		struct hstate *h = m->hstate;
 
 
 
 
 
 
 
 
 
2823
2824		VM_BUG_ON(!hstate_is_gigantic(h));
2825		WARN_ON(page_count(page) != 1);
2826		if (prep_compound_gigantic_page(page, huge_page_order(h))) {
2827			WARN_ON(PageReserved(page));
2828			prep_new_huge_page(h, page, page_to_nid(page));
2829			put_page(page); /* add to the hugepage allocator */
2830		} else {
2831			free_gigantic_page(page, huge_page_order(h));
2832			pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
2833		}
2834
2835		/*
2836		 * We need to restore the 'stolen' pages to totalram_pages
2837		 * in order to fix confusing memory reports from free(1) and
2838		 * other side-effects, like CommitLimit going negative.
2839		 */
2840		adjust_managed_page_count(page, pages_per_huge_page(h));
2841		cond_resched();
2842	}
 
 
2843}
2844
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2845static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2846{
2847	unsigned long i;
 
 
2848	nodemask_t *node_alloc_noretry;
 
2849
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2850	if (!hstate_is_gigantic(h)) {
2851		/*
2852		 * Bit mask controlling how hard we retry per-node allocations.
2853		 * Ignore errors as lower level routines can deal with
2854		 * node_alloc_noretry == NULL.  If this kmalloc fails at boot
2855		 * time, we are likely in bigger trouble.
2856		 */
2857		node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
2858						GFP_KERNEL);
2859	} else {
2860		/* allocations done at boot time */
2861		node_alloc_noretry = NULL;
2862	}
2863
2864	/* bit mask controlling how hard we retry per-node allocations */
2865	if (node_alloc_noretry)
2866		nodes_clear(*node_alloc_noretry);
2867
2868	for (i = 0; i < h->max_huge_pages; ++i) {
2869		if (hstate_is_gigantic(h)) {
2870			if (hugetlb_cma_size) {
2871				pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
2872				goto free;
2873			}
2874			if (!alloc_bootmem_huge_page(h))
 
 
 
 
 
2875				break;
2876		} else if (!alloc_pool_huge_page(h,
2877					 &node_states[N_MEMORY],
2878					 node_alloc_noretry))
2879			break;
2880		cond_resched();
2881	}
 
 
 
 
2882	if (i < h->max_huge_pages) {
2883		char buf[32];
2884
2885		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2886		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
2887			h->max_huge_pages, buf, i);
2888		h->max_huge_pages = i;
2889	}
2890free:
2891	kfree(node_alloc_noretry);
2892}
2893
2894static void __init hugetlb_init_hstates(void)
2895{
2896	struct hstate *h;
2897
2898	for_each_hstate(h) {
2899		if (minimum_order > huge_page_order(h))
2900			minimum_order = huge_page_order(h);
2901
2902		/* oversize hugepages were init'ed in early boot */
2903		if (!hstate_is_gigantic(h))
2904			hugetlb_hstate_alloc_pages(h);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2905	}
2906	VM_BUG_ON(minimum_order == UINT_MAX);
2907}
2908
2909static void __init report_hugepages(void)
2910{
2911	struct hstate *h;
2912
2913	for_each_hstate(h) {
2914		char buf[32];
2915
2916		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2917		pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2918			buf, h->free_huge_pages);
 
 
2919	}
2920}
2921
2922#ifdef CONFIG_HIGHMEM
2923static void try_to_free_low(struct hstate *h, unsigned long count,
2924						nodemask_t *nodes_allowed)
2925{
2926	int i;
2927	LIST_HEAD(page_list);
2928
2929	lockdep_assert_held(&hugetlb_lock);
2930	if (hstate_is_gigantic(h))
2931		return;
2932
2933	/*
2934	 * Collect pages to be freed on a list, and free after dropping lock
2935	 */
2936	for_each_node_mask(i, *nodes_allowed) {
2937		struct page *page, *next;
2938		struct list_head *freel = &h->hugepage_freelists[i];
2939		list_for_each_entry_safe(page, next, freel, lru) {
2940			if (count >= h->nr_huge_pages)
2941				goto out;
2942			if (PageHighMem(page))
2943				continue;
2944			remove_hugetlb_page(h, page, false);
2945			list_add(&page->lru, &page_list);
2946		}
2947	}
2948
2949out:
2950	spin_unlock_irq(&hugetlb_lock);
2951	update_and_free_pages_bulk(h, &page_list);
2952	spin_lock_irq(&hugetlb_lock);
2953}
2954#else
2955static inline void try_to_free_low(struct hstate *h, unsigned long count,
2956						nodemask_t *nodes_allowed)
2957{
2958}
2959#endif
2960
2961/*
2962 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2963 * balanced by operating on them in a round-robin fashion.
2964 * Returns 1 if an adjustment was made.
2965 */
2966static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2967				int delta)
2968{
2969	int nr_nodes, node;
2970
2971	lockdep_assert_held(&hugetlb_lock);
2972	VM_BUG_ON(delta != -1 && delta != 1);
2973
2974	if (delta < 0) {
2975		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2976			if (h->surplus_huge_pages_node[node])
2977				goto found;
2978		}
2979	} else {
2980		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2981			if (h->surplus_huge_pages_node[node] <
2982					h->nr_huge_pages_node[node])
2983				goto found;
2984		}
2985	}
2986	return 0;
2987
2988found:
2989	h->surplus_huge_pages += delta;
2990	h->surplus_huge_pages_node[node] += delta;
2991	return 1;
2992}
2993
2994#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2995static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
2996			      nodemask_t *nodes_allowed)
2997{
2998	unsigned long min_count, ret;
2999	struct page *page;
 
3000	LIST_HEAD(page_list);
3001	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
3002
3003	/*
3004	 * Bit mask controlling how hard we retry per-node allocations.
3005	 * If we can not allocate the bit mask, do not attempt to allocate
3006	 * the requested huge pages.
3007	 */
3008	if (node_alloc_noretry)
3009		nodes_clear(*node_alloc_noretry);
3010	else
3011		return -ENOMEM;
3012
3013	/*
3014	 * resize_lock mutex prevents concurrent adjustments to number of
3015	 * pages in hstate via the proc/sysfs interfaces.
3016	 */
3017	mutex_lock(&h->resize_lock);
3018	flush_free_hpage_work(h);
3019	spin_lock_irq(&hugetlb_lock);
3020
3021	/*
3022	 * Check for a node specific request.
3023	 * Changing node specific huge page count may require a corresponding
3024	 * change to the global count.  In any case, the passed node mask
3025	 * (nodes_allowed) will restrict alloc/free to the specified node.
3026	 */
3027	if (nid != NUMA_NO_NODE) {
3028		unsigned long old_count = count;
3029
3030		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
 
 
3031		/*
3032		 * User may have specified a large count value which caused the
3033		 * above calculation to overflow.  In this case, they wanted
3034		 * to allocate as many huge pages as possible.  Set count to
3035		 * largest possible value to align with their intention.
3036		 */
3037		if (count < old_count)
3038			count = ULONG_MAX;
3039	}
3040
3041	/*
3042	 * Gigantic pages runtime allocation depend on the capability for large
3043	 * page range allocation.
3044	 * If the system does not provide this feature, return an error when
3045	 * the user tries to allocate gigantic pages but let the user free the
3046	 * boottime allocated gigantic pages.
3047	 */
3048	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
3049		if (count > persistent_huge_pages(h)) {
3050			spin_unlock_irq(&hugetlb_lock);
3051			mutex_unlock(&h->resize_lock);
3052			NODEMASK_FREE(node_alloc_noretry);
3053			return -EINVAL;
3054		}
3055		/* Fall through to decrease pool */
3056	}
3057
3058	/*
3059	 * Increase the pool size
3060	 * First take pages out of surplus state.  Then make up the
3061	 * remaining difference by allocating fresh huge pages.
3062	 *
3063	 * We might race with alloc_surplus_huge_page() here and be unable
3064	 * to convert a surplus huge page to a normal huge page. That is
3065	 * not critical, though, it just means the overall size of the
3066	 * pool might be one hugepage larger than it needs to be, but
3067	 * within all the constraints specified by the sysctls.
3068	 */
3069	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3070		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3071			break;
3072	}
3073
3074	while (count > persistent_huge_pages(h)) {
 
3075		/*
3076		 * If this allocation races such that we no longer need the
3077		 * page, free_huge_page will handle it by freeing the page
3078		 * and reducing the surplus.
3079		 */
3080		spin_unlock_irq(&hugetlb_lock);
3081
3082		/* yield cpu to avoid soft lockup */
3083		cond_resched();
3084
3085		ret = alloc_pool_huge_page(h, nodes_allowed,
3086						node_alloc_noretry);
3087		spin_lock_irq(&hugetlb_lock);
3088		if (!ret)
 
3089			goto out;
 
 
 
 
3090
3091		/* Bail for signals. Probably ctrl-c from user */
3092		if (signal_pending(current))
 
 
3093			goto out;
 
 
 
 
 
 
 
 
 
 
3094	}
3095
3096	/*
3097	 * Decrease the pool size
3098	 * First return free pages to the buddy allocator (being careful
3099	 * to keep enough around to satisfy reservations).  Then place
3100	 * pages into surplus state as needed so the pool will shrink
3101	 * to the desired size as pages become free.
3102	 *
3103	 * By placing pages into the surplus state independent of the
3104	 * overcommit value, we are allowing the surplus pool size to
3105	 * exceed overcommit. There are few sane options here. Since
3106	 * alloc_surplus_huge_page() is checking the global counter,
3107	 * though, we'll note that we're not allowed to exceed surplus
3108	 * and won't grow the pool anywhere else. Not until one of the
3109	 * sysctls are changed, or the surplus pages go out of use.
3110	 */
3111	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3112	min_count = max(count, min_count);
3113	try_to_free_low(h, min_count, nodes_allowed);
3114
3115	/*
3116	 * Collect pages to be removed on list without dropping lock
3117	 */
3118	while (min_count < persistent_huge_pages(h)) {
3119		page = remove_pool_huge_page(h, nodes_allowed, 0);
3120		if (!page)
3121			break;
3122
3123		list_add(&page->lru, &page_list);
3124	}
3125	/* free the pages after dropping lock */
3126	spin_unlock_irq(&hugetlb_lock);
3127	update_and_free_pages_bulk(h, &page_list);
3128	flush_free_hpage_work(h);
3129	spin_lock_irq(&hugetlb_lock);
3130
3131	while (count < persistent_huge_pages(h)) {
3132		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3133			break;
3134	}
3135out:
3136	h->max_huge_pages = persistent_huge_pages(h);
3137	spin_unlock_irq(&hugetlb_lock);
3138	mutex_unlock(&h->resize_lock);
3139
3140	NODEMASK_FREE(node_alloc_noretry);
3141
3142	return 0;
3143}
3144
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3145#define HSTATE_ATTR_RO(_name) \
3146	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
3147
 
 
 
3148#define HSTATE_ATTR(_name) \
3149	static struct kobj_attribute _name##_attr = \
3150		__ATTR(_name, 0644, _name##_show, _name##_store)
3151
3152static struct kobject *hugepages_kobj;
3153static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
3154
3155static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
3156
3157static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
3158{
3159	int i;
3160
3161	for (i = 0; i < HUGE_MAX_HSTATE; i++)
3162		if (hstate_kobjs[i] == kobj) {
3163			if (nidp)
3164				*nidp = NUMA_NO_NODE;
3165			return &hstates[i];
3166		}
3167
3168	return kobj_to_node_hstate(kobj, nidp);
3169}
3170
3171static ssize_t nr_hugepages_show_common(struct kobject *kobj,
3172					struct kobj_attribute *attr, char *buf)
3173{
3174	struct hstate *h;
3175	unsigned long nr_huge_pages;
3176	int nid;
3177
3178	h = kobj_to_hstate(kobj, &nid);
3179	if (nid == NUMA_NO_NODE)
3180		nr_huge_pages = h->nr_huge_pages;
3181	else
3182		nr_huge_pages = h->nr_huge_pages_node[nid];
3183
3184	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
3185}
3186
3187static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
3188					   struct hstate *h, int nid,
3189					   unsigned long count, size_t len)
3190{
3191	int err;
3192	nodemask_t nodes_allowed, *n_mask;
3193
3194	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3195		return -EINVAL;
3196
3197	if (nid == NUMA_NO_NODE) {
3198		/*
3199		 * global hstate attribute
3200		 */
3201		if (!(obey_mempolicy &&
3202				init_nodemask_of_mempolicy(&nodes_allowed)))
3203			n_mask = &node_states[N_MEMORY];
3204		else
3205			n_mask = &nodes_allowed;
3206	} else {
3207		/*
3208		 * Node specific request.  count adjustment happens in
3209		 * set_max_huge_pages() after acquiring hugetlb_lock.
3210		 */
3211		init_nodemask_of_node(&nodes_allowed, nid);
3212		n_mask = &nodes_allowed;
3213	}
3214
3215	err = set_max_huge_pages(h, count, nid, n_mask);
3216
3217	return err ? err : len;
3218}
3219
3220static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
3221					 struct kobject *kobj, const char *buf,
3222					 size_t len)
3223{
3224	struct hstate *h;
3225	unsigned long count;
3226	int nid;
3227	int err;
3228
3229	err = kstrtoul(buf, 10, &count);
3230	if (err)
3231		return err;
3232
3233	h = kobj_to_hstate(kobj, &nid);
3234	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
3235}
3236
3237static ssize_t nr_hugepages_show(struct kobject *kobj,
3238				       struct kobj_attribute *attr, char *buf)
3239{
3240	return nr_hugepages_show_common(kobj, attr, buf);
3241}
3242
3243static ssize_t nr_hugepages_store(struct kobject *kobj,
3244	       struct kobj_attribute *attr, const char *buf, size_t len)
3245{
3246	return nr_hugepages_store_common(false, kobj, buf, len);
3247}
3248HSTATE_ATTR(nr_hugepages);
3249
3250#ifdef CONFIG_NUMA
3251
3252/*
3253 * hstate attribute for optionally mempolicy-based constraint on persistent
3254 * huge page alloc/free.
3255 */
3256static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
3257					   struct kobj_attribute *attr,
3258					   char *buf)
3259{
3260	return nr_hugepages_show_common(kobj, attr, buf);
3261}
3262
3263static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
3264	       struct kobj_attribute *attr, const char *buf, size_t len)
3265{
3266	return nr_hugepages_store_common(true, kobj, buf, len);
3267}
3268HSTATE_ATTR(nr_hugepages_mempolicy);
3269#endif
3270
3271
3272static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
3273					struct kobj_attribute *attr, char *buf)
3274{
3275	struct hstate *h = kobj_to_hstate(kobj, NULL);
3276	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
3277}
3278
3279static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
3280		struct kobj_attribute *attr, const char *buf, size_t count)
3281{
3282	int err;
3283	unsigned long input;
3284	struct hstate *h = kobj_to_hstate(kobj, NULL);
3285
3286	if (hstate_is_gigantic(h))
3287		return -EINVAL;
3288
3289	err = kstrtoul(buf, 10, &input);
3290	if (err)
3291		return err;
3292
3293	spin_lock_irq(&hugetlb_lock);
3294	h->nr_overcommit_huge_pages = input;
3295	spin_unlock_irq(&hugetlb_lock);
3296
3297	return count;
3298}
3299HSTATE_ATTR(nr_overcommit_hugepages);
3300
3301static ssize_t free_hugepages_show(struct kobject *kobj,
3302					struct kobj_attribute *attr, char *buf)
3303{
3304	struct hstate *h;
3305	unsigned long free_huge_pages;
3306	int nid;
3307
3308	h = kobj_to_hstate(kobj, &nid);
3309	if (nid == NUMA_NO_NODE)
3310		free_huge_pages = h->free_huge_pages;
3311	else
3312		free_huge_pages = h->free_huge_pages_node[nid];
3313
3314	return sysfs_emit(buf, "%lu\n", free_huge_pages);
3315}
3316HSTATE_ATTR_RO(free_hugepages);
3317
3318static ssize_t resv_hugepages_show(struct kobject *kobj,
3319					struct kobj_attribute *attr, char *buf)
3320{
3321	struct hstate *h = kobj_to_hstate(kobj, NULL);
3322	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
3323}
3324HSTATE_ATTR_RO(resv_hugepages);
3325
3326static ssize_t surplus_hugepages_show(struct kobject *kobj,
3327					struct kobj_attribute *attr, char *buf)
3328{
3329	struct hstate *h;
3330	unsigned long surplus_huge_pages;
3331	int nid;
3332
3333	h = kobj_to_hstate(kobj, &nid);
3334	if (nid == NUMA_NO_NODE)
3335		surplus_huge_pages = h->surplus_huge_pages;
3336	else
3337		surplus_huge_pages = h->surplus_huge_pages_node[nid];
3338
3339	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
3340}
3341HSTATE_ATTR_RO(surplus_hugepages);
3342
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3343static struct attribute *hstate_attrs[] = {
3344	&nr_hugepages_attr.attr,
3345	&nr_overcommit_hugepages_attr.attr,
3346	&free_hugepages_attr.attr,
3347	&resv_hugepages_attr.attr,
3348	&surplus_hugepages_attr.attr,
3349#ifdef CONFIG_NUMA
3350	&nr_hugepages_mempolicy_attr.attr,
3351#endif
3352	NULL,
3353};
3354
3355static const struct attribute_group hstate_attr_group = {
3356	.attrs = hstate_attrs,
3357};
3358
 
 
 
 
 
 
 
 
 
 
3359static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
3360				    struct kobject **hstate_kobjs,
3361				    const struct attribute_group *hstate_attr_group)
3362{
3363	int retval;
3364	int hi = hstate_index(h);
3365
3366	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
3367	if (!hstate_kobjs[hi])
3368		return -ENOMEM;
3369
3370	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
3371	if (retval) {
3372		kobject_put(hstate_kobjs[hi]);
3373		hstate_kobjs[hi] = NULL;
 
3374	}
3375
3376	return retval;
3377}
 
 
 
 
 
 
 
 
 
3378
3379static void __init hugetlb_sysfs_init(void)
3380{
3381	struct hstate *h;
3382	int err;
3383
3384	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
3385	if (!hugepages_kobj)
3386		return;
3387
3388	for_each_hstate(h) {
3389		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
3390					 hstate_kobjs, &hstate_attr_group);
3391		if (err)
3392			pr_err("HugeTLB: Unable to add hstate %s", h->name);
3393	}
3394}
3395
3396#ifdef CONFIG_NUMA
 
3397
3398/*
3399 * node_hstate/s - associate per node hstate attributes, via their kobjects,
3400 * with node devices in node_devices[] using a parallel array.  The array
3401 * index of a node device or _hstate == node id.
3402 * This is here to avoid any static dependency of the node device driver, in
3403 * the base kernel, on the hugetlb module.
3404 */
3405struct node_hstate {
3406	struct kobject		*hugepages_kobj;
3407	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
3408};
3409static struct node_hstate node_hstates[MAX_NUMNODES];
3410
3411/*
3412 * A subset of global hstate attributes for node devices
3413 */
3414static struct attribute *per_node_hstate_attrs[] = {
3415	&nr_hugepages_attr.attr,
3416	&free_hugepages_attr.attr,
3417	&surplus_hugepages_attr.attr,
3418	NULL,
3419};
3420
3421static const struct attribute_group per_node_hstate_attr_group = {
3422	.attrs = per_node_hstate_attrs,
3423};
3424
3425/*
3426 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
3427 * Returns node id via non-NULL nidp.
3428 */
3429static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3430{
3431	int nid;
3432
3433	for (nid = 0; nid < nr_node_ids; nid++) {
3434		struct node_hstate *nhs = &node_hstates[nid];
3435		int i;
3436		for (i = 0; i < HUGE_MAX_HSTATE; i++)
3437			if (nhs->hstate_kobjs[i] == kobj) {
3438				if (nidp)
3439					*nidp = nid;
3440				return &hstates[i];
3441			}
3442	}
3443
3444	BUG();
3445	return NULL;
3446}
3447
3448/*
3449 * Unregister hstate attributes from a single node device.
3450 * No-op if no hstate attributes attached.
3451 */
3452static void hugetlb_unregister_node(struct node *node)
3453{
3454	struct hstate *h;
3455	struct node_hstate *nhs = &node_hstates[node->dev.id];
3456
3457	if (!nhs->hugepages_kobj)
3458		return;		/* no hstate attributes */
3459
3460	for_each_hstate(h) {
3461		int idx = hstate_index(h);
3462		if (nhs->hstate_kobjs[idx]) {
3463			kobject_put(nhs->hstate_kobjs[idx]);
3464			nhs->hstate_kobjs[idx] = NULL;
3465		}
 
 
 
 
 
3466	}
3467
3468	kobject_put(nhs->hugepages_kobj);
3469	nhs->hugepages_kobj = NULL;
3470}
3471
3472
3473/*
3474 * Register hstate attributes for a single node device.
3475 * No-op if attributes already registered.
3476 */
3477static void hugetlb_register_node(struct node *node)
3478{
3479	struct hstate *h;
3480	struct node_hstate *nhs = &node_hstates[node->dev.id];
3481	int err;
3482
 
 
 
3483	if (nhs->hugepages_kobj)
3484		return;		/* already allocated */
3485
3486	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
3487							&node->dev.kobj);
3488	if (!nhs->hugepages_kobj)
3489		return;
3490
3491	for_each_hstate(h) {
3492		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
3493						nhs->hstate_kobjs,
3494						&per_node_hstate_attr_group);
3495		if (err) {
3496			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
3497				h->name, node->dev.id);
3498			hugetlb_unregister_node(node);
3499			break;
3500		}
3501	}
3502}
3503
3504/*
3505 * hugetlb init time:  register hstate attributes for all registered node
3506 * devices of nodes that have memory.  All on-line nodes should have
3507 * registered their associated device by this time.
3508 */
3509static void __init hugetlb_register_all_nodes(void)
3510{
3511	int nid;
3512
3513	for_each_node_state(nid, N_MEMORY) {
3514		struct node *node = node_devices[nid];
3515		if (node->dev.id == nid)
3516			hugetlb_register_node(node);
3517	}
3518
3519	/*
3520	 * Let the node device driver know we're here so it can
3521	 * [un]register hstate attributes on node hotplug.
3522	 */
3523	register_hugetlbfs_with_node(hugetlb_register_node,
3524				     hugetlb_unregister_node);
3525}
3526#else	/* !CONFIG_NUMA */
3527
3528static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3529{
3530	BUG();
3531	if (nidp)
3532		*nidp = -1;
3533	return NULL;
3534}
3535
3536static void hugetlb_register_all_nodes(void) { }
3537
3538#endif
3539
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3540static int __init hugetlb_init(void)
3541{
3542	int i;
3543
3544	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
3545			__NR_HPAGEFLAGS);
3546
3547	if (!hugepages_supported()) {
3548		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
3549			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
3550		return 0;
3551	}
3552
3553	/*
3554	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
3555	 * architectures depend on setup being done here.
3556	 */
3557	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
3558	if (!parsed_default_hugepagesz) {
3559		/*
3560		 * If we did not parse a default huge page size, set
3561		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
3562		 * number of huge pages for this default size was implicitly
3563		 * specified, set that here as well.
3564		 * Note that the implicit setting will overwrite an explicit
3565		 * setting.  A warning will be printed in this case.
3566		 */
3567		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
3568		if (default_hstate_max_huge_pages) {
3569			if (default_hstate.max_huge_pages) {
3570				char buf[32];
3571
3572				string_get_size(huge_page_size(&default_hstate),
3573					1, STRING_UNITS_2, buf, 32);
3574				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
3575					default_hstate.max_huge_pages, buf);
3576				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
3577					default_hstate_max_huge_pages);
3578			}
3579			default_hstate.max_huge_pages =
3580				default_hstate_max_huge_pages;
 
 
 
 
3581		}
3582	}
3583
3584	hugetlb_cma_check();
3585	hugetlb_init_hstates();
3586	gather_bootmem_prealloc();
3587	report_hugepages();
3588
3589	hugetlb_sysfs_init();
3590	hugetlb_register_all_nodes();
3591	hugetlb_cgroup_file_init();
 
3592
3593#ifdef CONFIG_SMP
3594	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
3595#else
3596	num_fault_mutexes = 1;
3597#endif
3598	hugetlb_fault_mutex_table =
3599		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
3600			      GFP_KERNEL);
3601	BUG_ON(!hugetlb_fault_mutex_table);
3602
3603	for (i = 0; i < num_fault_mutexes; i++)
3604		mutex_init(&hugetlb_fault_mutex_table[i]);
3605	return 0;
3606}
3607subsys_initcall(hugetlb_init);
3608
3609/* Overwritten by architectures with more huge page sizes */
3610bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
3611{
3612	return size == HPAGE_SIZE;
3613}
3614
3615void __init hugetlb_add_hstate(unsigned int order)
3616{
3617	struct hstate *h;
3618	unsigned long i;
3619
3620	if (size_to_hstate(PAGE_SIZE << order)) {
3621		return;
3622	}
3623	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
3624	BUG_ON(order == 0);
3625	h = &hstates[hugetlb_max_hstate++];
3626	mutex_init(&h->resize_lock);
3627	h->order = order;
3628	h->mask = ~(huge_page_size(h) - 1);
3629	for (i = 0; i < MAX_NUMNODES; ++i)
3630		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
3631	INIT_LIST_HEAD(&h->hugepage_activelist);
3632	h->next_nid_to_alloc = first_memory_node;
3633	h->next_nid_to_free = first_memory_node;
3634	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
3635					huge_page_size(h)/1024);
3636	hugetlb_vmemmap_init(h);
3637
3638	parsed_hstate = h;
3639}
3640
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3641/*
3642 * hugepages command line processing
3643 * hugepages normally follows a valid hugepagsz or default_hugepagsz
3644 * specification.  If not, ignore the hugepages value.  hugepages can also
3645 * be the first huge page command line  option in which case it implicitly
3646 * specifies the number of huge pages for the default size.
3647 */
3648static int __init hugepages_setup(char *s)
3649{
3650	unsigned long *mhp;
3651	static unsigned long *last_mhp;
 
 
 
 
3652
3653	if (!parsed_valid_hugepagesz) {
3654		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
3655		parsed_valid_hugepagesz = true;
3656		return 0;
3657	}
3658
3659	/*
3660	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
3661	 * yet, so this hugepages= parameter goes to the "default hstate".
3662	 * Otherwise, it goes with the previously parsed hugepagesz or
3663	 * default_hugepagesz.
3664	 */
3665	else if (!hugetlb_max_hstate)
3666		mhp = &default_hstate_max_huge_pages;
3667	else
3668		mhp = &parsed_hstate->max_huge_pages;
3669
3670	if (mhp == last_mhp) {
3671		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
3672		return 0;
3673	}
3674
3675	if (sscanf(s, "%lu", mhp) <= 0)
3676		*mhp = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3677
3678	/*
3679	 * Global state is always initialized later in hugetlb_init.
3680	 * But we need to allocate gigantic hstates here early to still
3681	 * use the bootmem allocator.
3682	 */
3683	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
3684		hugetlb_hstate_alloc_pages(parsed_hstate);
3685
3686	last_mhp = mhp;
3687
3688	return 1;
 
 
 
 
 
3689}
3690__setup("hugepages=", hugepages_setup);
3691
3692/*
3693 * hugepagesz command line processing
3694 * A specific huge page size can only be specified once with hugepagesz.
3695 * hugepagesz is followed by hugepages on the command line.  The global
3696 * variable 'parsed_valid_hugepagesz' is used to determine if prior
3697 * hugepagesz argument was valid.
3698 */
3699static int __init hugepagesz_setup(char *s)
3700{
3701	unsigned long size;
3702	struct hstate *h;
3703
3704	parsed_valid_hugepagesz = false;
3705	size = (unsigned long)memparse(s, NULL);
3706
3707	if (!arch_hugetlb_valid_size(size)) {
3708		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
3709		return 0;
3710	}
3711
3712	h = size_to_hstate(size);
3713	if (h) {
3714		/*
3715		 * hstate for this size already exists.  This is normally
3716		 * an error, but is allowed if the existing hstate is the
3717		 * default hstate.  More specifically, it is only allowed if
3718		 * the number of huge pages for the default hstate was not
3719		 * previously specified.
3720		 */
3721		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
3722		    default_hstate.max_huge_pages) {
3723			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
3724			return 0;
3725		}
3726
3727		/*
3728		 * No need to call hugetlb_add_hstate() as hstate already
3729		 * exists.  But, do set parsed_hstate so that a following
3730		 * hugepages= parameter will be applied to this hstate.
3731		 */
3732		parsed_hstate = h;
3733		parsed_valid_hugepagesz = true;
3734		return 1;
3735	}
3736
3737	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
3738	parsed_valid_hugepagesz = true;
3739	return 1;
3740}
3741__setup("hugepagesz=", hugepagesz_setup);
3742
3743/*
3744 * default_hugepagesz command line input
3745 * Only one instance of default_hugepagesz allowed on command line.
3746 */
3747static int __init default_hugepagesz_setup(char *s)
3748{
3749	unsigned long size;
 
3750
3751	parsed_valid_hugepagesz = false;
3752	if (parsed_default_hugepagesz) {
3753		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
3754		return 0;
3755	}
3756
3757	size = (unsigned long)memparse(s, NULL);
3758
3759	if (!arch_hugetlb_valid_size(size)) {
3760		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
3761		return 0;
3762	}
3763
3764	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
3765	parsed_valid_hugepagesz = true;
3766	parsed_default_hugepagesz = true;
3767	default_hstate_idx = hstate_index(size_to_hstate(size));
3768
3769	/*
3770	 * The number of default huge pages (for this size) could have been
3771	 * specified as the first hugetlb parameter: hugepages=X.  If so,
3772	 * then default_hstate_max_huge_pages is set.  If the default huge
3773	 * page size is gigantic (>= MAX_ORDER), then the pages must be
3774	 * allocated here from bootmem allocator.
3775	 */
3776	if (default_hstate_max_huge_pages) {
3777		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
 
 
 
3778		if (hstate_is_gigantic(&default_hstate))
3779			hugetlb_hstate_alloc_pages(&default_hstate);
3780		default_hstate_max_huge_pages = 0;
3781	}
3782
3783	return 1;
3784}
3785__setup("default_hugepagesz=", default_hugepagesz_setup);
3786
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3787static unsigned int allowed_mems_nr(struct hstate *h)
3788{
3789	int node;
3790	unsigned int nr = 0;
3791	nodemask_t *mpol_allowed;
3792	unsigned int *array = h->free_huge_pages_node;
3793	gfp_t gfp_mask = htlb_alloc_mask(h);
3794
3795	mpol_allowed = policy_nodemask_current(gfp_mask);
3796
3797	for_each_node_mask(node, cpuset_current_mems_allowed) {
3798		if (!mpol_allowed || node_isset(node, *mpol_allowed))
3799			nr += array[node];
3800	}
3801
3802	return nr;
3803}
3804
3805#ifdef CONFIG_SYSCTL
3806static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
3807					  void *buffer, size_t *length,
3808					  loff_t *ppos, unsigned long *out)
3809{
3810	struct ctl_table dup_table;
3811
3812	/*
3813	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
3814	 * can duplicate the @table and alter the duplicate of it.
3815	 */
3816	dup_table = *table;
3817	dup_table.data = out;
3818
3819	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
3820}
3821
3822static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
3823			 struct ctl_table *table, int write,
3824			 void *buffer, size_t *length, loff_t *ppos)
3825{
3826	struct hstate *h = &default_hstate;
3827	unsigned long tmp = h->max_huge_pages;
3828	int ret;
3829
3830	if (!hugepages_supported())
3831		return -EOPNOTSUPP;
3832
3833	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3834					     &tmp);
3835	if (ret)
3836		goto out;
3837
3838	if (write)
3839		ret = __nr_hugepages_store_common(obey_mempolicy, h,
3840						  NUMA_NO_NODE, tmp, *length);
3841out:
3842	return ret;
3843}
3844
3845int hugetlb_sysctl_handler(struct ctl_table *table, int write,
3846			  void *buffer, size_t *length, loff_t *ppos)
3847{
3848
3849	return hugetlb_sysctl_handler_common(false, table, write,
3850							buffer, length, ppos);
3851}
3852
3853#ifdef CONFIG_NUMA
3854int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
3855			  void *buffer, size_t *length, loff_t *ppos)
3856{
3857	return hugetlb_sysctl_handler_common(true, table, write,
3858							buffer, length, ppos);
3859}
3860#endif /* CONFIG_NUMA */
3861
3862int hugetlb_overcommit_handler(struct ctl_table *table, int write,
3863		void *buffer, size_t *length, loff_t *ppos)
3864{
3865	struct hstate *h = &default_hstate;
3866	unsigned long tmp;
3867	int ret;
3868
3869	if (!hugepages_supported())
3870		return -EOPNOTSUPP;
3871
3872	tmp = h->nr_overcommit_huge_pages;
3873
3874	if (write && hstate_is_gigantic(h))
3875		return -EINVAL;
3876
3877	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3878					     &tmp);
3879	if (ret)
3880		goto out;
3881
3882	if (write) {
3883		spin_lock_irq(&hugetlb_lock);
3884		h->nr_overcommit_huge_pages = tmp;
3885		spin_unlock_irq(&hugetlb_lock);
3886	}
3887out:
3888	return ret;
3889}
3890
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3891#endif /* CONFIG_SYSCTL */
3892
3893void hugetlb_report_meminfo(struct seq_file *m)
3894{
3895	struct hstate *h;
3896	unsigned long total = 0;
3897
3898	if (!hugepages_supported())
3899		return;
3900
3901	for_each_hstate(h) {
3902		unsigned long count = h->nr_huge_pages;
3903
3904		total += huge_page_size(h) * count;
3905
3906		if (h == &default_hstate)
3907			seq_printf(m,
3908				   "HugePages_Total:   %5lu\n"
3909				   "HugePages_Free:    %5lu\n"
3910				   "HugePages_Rsvd:    %5lu\n"
3911				   "HugePages_Surp:    %5lu\n"
3912				   "Hugepagesize:   %8lu kB\n",
3913				   count,
3914				   h->free_huge_pages,
3915				   h->resv_huge_pages,
3916				   h->surplus_huge_pages,
3917				   huge_page_size(h) / SZ_1K);
3918	}
3919
3920	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
3921}
3922
3923int hugetlb_report_node_meminfo(char *buf, int len, int nid)
3924{
3925	struct hstate *h = &default_hstate;
3926
3927	if (!hugepages_supported())
3928		return 0;
3929
3930	return sysfs_emit_at(buf, len,
3931			     "Node %d HugePages_Total: %5u\n"
3932			     "Node %d HugePages_Free:  %5u\n"
3933			     "Node %d HugePages_Surp:  %5u\n",
3934			     nid, h->nr_huge_pages_node[nid],
3935			     nid, h->free_huge_pages_node[nid],
3936			     nid, h->surplus_huge_pages_node[nid]);
3937}
3938
3939void hugetlb_show_meminfo(void)
3940{
3941	struct hstate *h;
3942	int nid;
3943
3944	if (!hugepages_supported())
3945		return;
3946
3947	for_each_node_state(nid, N_MEMORY)
3948		for_each_hstate(h)
3949			pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3950				nid,
3951				h->nr_huge_pages_node[nid],
3952				h->free_huge_pages_node[nid],
3953				h->surplus_huge_pages_node[nid],
3954				huge_page_size(h) / SZ_1K);
3955}
3956
3957void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3958{
3959	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3960		   atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3961}
3962
3963/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3964unsigned long hugetlb_total_pages(void)
3965{
3966	struct hstate *h;
3967	unsigned long nr_total_pages = 0;
3968
3969	for_each_hstate(h)
3970		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3971	return nr_total_pages;
3972}
3973
3974static int hugetlb_acct_memory(struct hstate *h, long delta)
3975{
3976	int ret = -ENOMEM;
3977
3978	if (!delta)
3979		return 0;
3980
3981	spin_lock_irq(&hugetlb_lock);
3982	/*
3983	 * When cpuset is configured, it breaks the strict hugetlb page
3984	 * reservation as the accounting is done on a global variable. Such
3985	 * reservation is completely rubbish in the presence of cpuset because
3986	 * the reservation is not checked against page availability for the
3987	 * current cpuset. Application can still potentially OOM'ed by kernel
3988	 * with lack of free htlb page in cpuset that the task is in.
3989	 * Attempt to enforce strict accounting with cpuset is almost
3990	 * impossible (or too ugly) because cpuset is too fluid that
3991	 * task or memory node can be dynamically moved between cpusets.
3992	 *
3993	 * The change of semantics for shared hugetlb mapping with cpuset is
3994	 * undesirable. However, in order to preserve some of the semantics,
3995	 * we fall back to check against current free page availability as
3996	 * a best attempt and hopefully to minimize the impact of changing
3997	 * semantics that cpuset has.
3998	 *
3999	 * Apart from cpuset, we also have memory policy mechanism that
4000	 * also determines from which node the kernel will allocate memory
4001	 * in a NUMA system. So similar to cpuset, we also should consider
4002	 * the memory policy of the current task. Similar to the description
4003	 * above.
4004	 */
4005	if (delta > 0) {
4006		if (gather_surplus_pages(h, delta) < 0)
4007			goto out;
4008
4009		if (delta > allowed_mems_nr(h)) {
4010			return_unused_surplus_pages(h, delta);
4011			goto out;
4012		}
4013	}
4014
4015	ret = 0;
4016	if (delta < 0)
4017		return_unused_surplus_pages(h, (unsigned long) -delta);
4018
4019out:
4020	spin_unlock_irq(&hugetlb_lock);
4021	return ret;
4022}
4023
4024static void hugetlb_vm_op_open(struct vm_area_struct *vma)
4025{
4026	struct resv_map *resv = vma_resv_map(vma);
4027
4028	/*
 
4029	 * This new VMA should share its siblings reservation map if present.
4030	 * The VMA will only ever have a valid reservation map pointer where
4031	 * it is being copied for another still existing VMA.  As that VMA
4032	 * has a reference to the reservation map it cannot disappear until
4033	 * after this open call completes.  It is therefore safe to take a
4034	 * new reference here without additional locking.
4035	 */
4036	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
4037		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
4038		kref_get(&resv->refs);
4039	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4040}
4041
4042static void hugetlb_vm_op_close(struct vm_area_struct *vma)
4043{
4044	struct hstate *h = hstate_vma(vma);
4045	struct resv_map *resv = vma_resv_map(vma);
4046	struct hugepage_subpool *spool = subpool_vma(vma);
4047	unsigned long reserve, start, end;
4048	long gbl_reserve;
4049
 
 
 
4050	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4051		return;
4052
4053	start = vma_hugecache_offset(h, vma, vma->vm_start);
4054	end = vma_hugecache_offset(h, vma, vma->vm_end);
4055
4056	reserve = (end - start) - region_count(resv, start, end);
4057	hugetlb_cgroup_uncharge_counter(resv, start, end);
4058	if (reserve) {
4059		/*
4060		 * Decrement reserve counts.  The global reserve count may be
4061		 * adjusted if the subpool has a minimum size.
4062		 */
4063		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
4064		hugetlb_acct_memory(h, -gbl_reserve);
4065	}
4066
4067	kref_put(&resv->refs, resv_map_release);
4068}
4069
4070static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
4071{
4072	if (addr & ~(huge_page_mask(hstate_vma(vma))))
4073		return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4074	return 0;
4075}
4076
4077static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
4078{
4079	return huge_page_size(hstate_vma(vma));
4080}
4081
4082/*
4083 * We cannot handle pagefaults against hugetlb pages at all.  They cause
4084 * handle_mm_fault() to try to instantiate regular-sized pages in the
4085 * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
4086 * this far.
4087 */
4088static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
4089{
4090	BUG();
4091	return 0;
4092}
4093
4094/*
4095 * When a new function is introduced to vm_operations_struct and added
4096 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
4097 * This is because under System V memory model, mappings created via
4098 * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
4099 * their original vm_ops are overwritten with shm_vm_ops.
4100 */
4101const struct vm_operations_struct hugetlb_vm_ops = {
4102	.fault = hugetlb_vm_op_fault,
4103	.open = hugetlb_vm_op_open,
4104	.close = hugetlb_vm_op_close,
4105	.may_split = hugetlb_vm_op_split,
4106	.pagesize = hugetlb_vm_op_pagesize,
4107};
4108
4109static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
4110				int writable)
4111{
4112	pte_t entry;
4113	unsigned int shift = huge_page_shift(hstate_vma(vma));
4114
4115	if (writable) {
4116		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
4117					 vma->vm_page_prot)));
4118	} else {
4119		entry = huge_pte_wrprotect(mk_huge_pte(page,
4120					   vma->vm_page_prot));
4121	}
4122	entry = pte_mkyoung(entry);
4123	entry = pte_mkhuge(entry);
4124	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
4125
4126	return entry;
4127}
4128
4129static void set_huge_ptep_writable(struct vm_area_struct *vma,
4130				   unsigned long address, pte_t *ptep)
4131{
4132	pte_t entry;
4133
4134	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
4135	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
4136		update_mmu_cache(vma, address, ptep);
4137}
4138
4139bool is_hugetlb_entry_migration(pte_t pte)
4140{
4141	swp_entry_t swp;
4142
4143	if (huge_pte_none(pte) || pte_present(pte))
4144		return false;
4145	swp = pte_to_swp_entry(pte);
4146	if (is_migration_entry(swp))
4147		return true;
4148	else
4149		return false;
4150}
4151
4152static bool is_hugetlb_entry_hwpoisoned(pte_t pte)
4153{
4154	swp_entry_t swp;
4155
4156	if (huge_pte_none(pte) || pte_present(pte))
4157		return false;
4158	swp = pte_to_swp_entry(pte);
4159	if (is_hwpoison_entry(swp))
4160		return true;
4161	else
4162		return false;
4163}
4164
4165static void
4166hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
4167		     struct page *new_page)
4168{
4169	__SetPageUptodate(new_page);
4170	set_huge_pte_at(vma->vm_mm, addr, ptep, make_huge_pte(vma, new_page, 1));
4171	hugepage_add_new_anon_rmap(new_page, vma, addr);
 
 
 
 
4172	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
4173	ClearHPageRestoreReserve(new_page);
4174	SetHPageMigratable(new_page);
4175}
4176
4177int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
4178			    struct vm_area_struct *vma)
 
4179{
4180	pte_t *src_pte, *dst_pte, entry, dst_entry;
4181	struct page *ptepage;
4182	unsigned long addr;
4183	bool cow = is_cow_mapping(vma->vm_flags);
4184	struct hstate *h = hstate_vma(vma);
4185	unsigned long sz = huge_page_size(h);
4186	unsigned long npages = pages_per_huge_page(h);
4187	struct address_space *mapping = vma->vm_file->f_mapping;
4188	struct mmu_notifier_range range;
 
4189	int ret = 0;
4190
4191	if (cow) {
4192		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src,
4193					vma->vm_start,
4194					vma->vm_end);
4195		mmu_notifier_invalidate_range_start(&range);
 
 
4196	} else {
4197		/*
4198		 * For shared mappings i_mmap_rwsem must be held to call
4199		 * huge_pte_alloc, otherwise the returned ptep could go
4200		 * away if part of a shared pmd and another thread calls
4201		 * huge_pmd_unshare.
4202		 */
4203		i_mmap_lock_read(mapping);
4204	}
4205
4206	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
 
4207		spinlock_t *src_ptl, *dst_ptl;
4208		src_pte = huge_pte_offset(src, addr, sz);
4209		if (!src_pte)
 
4210			continue;
4211		dst_pte = huge_pte_alloc(dst, vma, addr, sz);
 
4212		if (!dst_pte) {
4213			ret = -ENOMEM;
4214			break;
4215		}
4216
4217		/*
4218		 * If the pagetables are shared don't copy or take references.
 
4219		 * dst_pte == src_pte is the common case of src/dest sharing.
4220		 *
4221		 * However, src could have 'unshared' and dst shares with
4222		 * another vma.  If dst_pte !none, this implies sharing.
4223		 * Check here before taking page table lock, and once again
4224		 * after taking the lock below.
4225		 */
4226		dst_entry = huge_ptep_get(dst_pte);
4227		if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
4228			continue;
 
4229
4230		dst_ptl = huge_pte_lock(h, dst, dst_pte);
4231		src_ptl = huge_pte_lockptr(h, src, src_pte);
4232		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4233		entry = huge_ptep_get(src_pte);
4234		dst_entry = huge_ptep_get(dst_pte);
4235again:
4236		if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
4237			/*
4238			 * Skip if src entry none.  Also, skip in the
4239			 * unlikely case dst entry !none as this implies
4240			 * sharing with another vma.
4241			 */
4242			;
4243		} else if (unlikely(is_hugetlb_entry_migration(entry) ||
4244				    is_hugetlb_entry_hwpoisoned(entry))) {
 
 
 
4245			swp_entry_t swp_entry = pte_to_swp_entry(entry);
 
4246
4247			if (is_writable_migration_entry(swp_entry) && cow) {
4248				/*
4249				 * COW mappings require pages in both
4250				 * parent and child to be set to read.
4251				 */
4252				swp_entry = make_readable_migration_entry(
4253							swp_offset(swp_entry));
4254				entry = swp_entry_to_pte(swp_entry);
4255				set_huge_swap_pte_at(src, addr, src_pte,
4256						     entry, sz);
 
4257			}
4258			set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
 
 
 
 
 
 
 
 
 
4259		} else {
4260			entry = huge_ptep_get(src_pte);
4261			ptepage = pte_page(entry);
4262			get_page(ptepage);
4263
4264			/*
4265			 * This is a rare case where we see pinned hugetlb
4266			 * pages while they're prone to COW.  We need to do the
4267			 * COW earlier during fork.
 
4268			 *
4269			 * When pre-allocating the page or copying data, we
4270			 * need to be without the pgtable locks since we could
4271			 * sleep during the process.
4272			 */
4273			if (unlikely(page_needs_cow_for_dma(vma, ptepage))) {
 
 
4274				pte_t src_pte_old = entry;
4275				struct page *new;
4276
4277				spin_unlock(src_ptl);
4278				spin_unlock(dst_ptl);
4279				/* Do not use reserve as it's private owned */
4280				new = alloc_huge_page(vma, addr, 1);
4281				if (IS_ERR(new)) {
4282					put_page(ptepage);
4283					ret = PTR_ERR(new);
 
 
 
 
 
 
 
 
4284					break;
4285				}
4286				copy_user_huge_page(new, ptepage, addr, vma,
4287						    npages);
4288				put_page(ptepage);
4289
4290				/* Install the new huge page if src pte stable */
4291				dst_ptl = huge_pte_lock(h, dst, dst_pte);
4292				src_ptl = huge_pte_lockptr(h, src, src_pte);
4293				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4294				entry = huge_ptep_get(src_pte);
4295				if (!pte_same(src_pte_old, entry)) {
4296					restore_reserve_on_error(h, vma, addr,
4297								new);
4298					put_page(new);
4299					/* dst_entry won't change as in child */
4300					goto again;
4301				}
4302				hugetlb_install_page(vma, dst_pte, addr, new);
 
4303				spin_unlock(src_ptl);
4304				spin_unlock(dst_ptl);
4305				continue;
4306			}
4307
4308			if (cow) {
4309				/*
4310				 * No need to notify as we are downgrading page
4311				 * table protection not changing it to point
4312				 * to a new page.
4313				 *
4314				 * See Documentation/vm/mmu_notifier.rst
4315				 */
4316				huge_ptep_set_wrprotect(src, addr, src_pte);
4317				entry = huge_pte_wrprotect(entry);
4318			}
4319
4320			page_dup_rmap(ptepage, true);
4321			set_huge_pte_at(dst, addr, dst_pte, entry);
 
 
4322			hugetlb_count_add(npages, dst);
4323		}
4324		spin_unlock(src_ptl);
4325		spin_unlock(dst_ptl);
4326	}
4327
4328	if (cow)
 
4329		mmu_notifier_invalidate_range_end(&range);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4330	else
4331		i_mmap_unlock_read(mapping);
 
 
 
4332
4333	return ret;
4334}
4335
4336void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
4337			    unsigned long start, unsigned long end,
4338			    struct page *ref_page)
4339{
4340	struct mm_struct *mm = vma->vm_mm;
4341	unsigned long address;
4342	pte_t *ptep;
4343	pte_t pte;
4344	spinlock_t *ptl;
4345	struct page *page;
4346	struct hstate *h = hstate_vma(vma);
4347	unsigned long sz = huge_page_size(h);
4348	struct mmu_notifier_range range;
 
4349
4350	WARN_ON(!is_vm_hugetlb_page(vma));
4351	BUG_ON(start & ~huge_page_mask(h));
4352	BUG_ON(end & ~huge_page_mask(h));
4353
4354	/*
4355	 * This is a hugetlb vma, all the pte entries should point
4356	 * to huge page.
4357	 */
4358	tlb_change_page_size(tlb, sz);
4359	tlb_start_vma(tlb, vma);
4360
4361	/*
4362	 * If sharing possible, alert mmu notifiers of worst case.
4363	 */
4364	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start,
4365				end);
4366	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
4367	mmu_notifier_invalidate_range_start(&range);
4368	address = start;
4369	for (; address < end; address += sz) {
4370		ptep = huge_pte_offset(mm, address, sz);
4371		if (!ptep)
 
4372			continue;
 
4373
4374		ptl = huge_pte_lock(h, mm, ptep);
4375		if (huge_pmd_unshare(mm, vma, &address, ptep)) {
4376			spin_unlock(ptl);
4377			/*
4378			 * We just unmapped a page of PMDs by clearing a PUD.
4379			 * The caller's TLB flush range should cover this area.
4380			 */
4381			continue;
4382		}
4383
4384		pte = huge_ptep_get(ptep);
4385		if (huge_pte_none(pte)) {
4386			spin_unlock(ptl);
4387			continue;
4388		}
4389
4390		/*
4391		 * Migrating hugepage or HWPoisoned hugepage is already
4392		 * unmapped and its refcount is dropped, so just clear pte here.
4393		 */
4394		if (unlikely(!pte_present(pte))) {
4395			huge_pte_clear(mm, address, ptep, sz);
 
 
 
 
 
 
 
 
 
 
 
 
4396			spin_unlock(ptl);
4397			continue;
4398		}
4399
4400		page = pte_page(pte);
4401		/*
4402		 * If a reference page is supplied, it is because a specific
4403		 * page is being unmapped, not a range. Ensure the page we
4404		 * are about to unmap is the actual page of interest.
4405		 */
4406		if (ref_page) {
4407			if (page != ref_page) {
4408				spin_unlock(ptl);
4409				continue;
4410			}
4411			/*
4412			 * Mark the VMA as having unmapped its page so that
4413			 * future faults in this VMA will fail rather than
4414			 * looking like data was lost
4415			 */
4416			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
4417		}
4418
4419		pte = huge_ptep_get_and_clear(mm, address, ptep);
4420		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
4421		if (huge_pte_dirty(pte))
4422			set_page_dirty(page);
4423
 
 
 
 
 
4424		hugetlb_count_sub(pages_per_huge_page(h), mm);
4425		page_remove_rmap(page, true);
4426
4427		spin_unlock(ptl);
4428		tlb_remove_page_size(tlb, page, huge_page_size(h));
4429		/*
4430		 * Bail out after unmapping reference page if supplied
4431		 */
4432		if (ref_page)
4433			break;
4434	}
4435	mmu_notifier_invalidate_range_end(&range);
4436	tlb_end_vma(tlb, vma);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4437}
4438
4439void __unmap_hugepage_range_final(struct mmu_gather *tlb,
4440			  struct vm_area_struct *vma, unsigned long start,
4441			  unsigned long end, struct page *ref_page)
4442{
4443	__unmap_hugepage_range(tlb, vma, start, end, ref_page);
4444
4445	/*
4446	 * Clear this flag so that x86's huge_pmd_share page_table_shareable
4447	 * test will fail on a vma being torn down, and not grab a page table
4448	 * on its way out.  We're lucky that the flag has such an appropriate
4449	 * name, and can in fact be safely cleared here. We could clear it
4450	 * before the __unmap_hugepage_range above, but all that's necessary
4451	 * is to clear it before releasing the i_mmap_rwsem. This works
4452	 * because in the context this is called, the VMA is about to be
4453	 * destroyed and the i_mmap_rwsem is held.
4454	 */
4455	vma->vm_flags &= ~VM_MAYSHARE;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4456}
4457
4458void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
4459			  unsigned long end, struct page *ref_page)
 
4460{
 
4461	struct mmu_gather tlb;
4462
 
 
 
 
4463	tlb_gather_mmu(&tlb, vma->vm_mm);
4464	__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
 
 
 
4465	tlb_finish_mmu(&tlb);
4466}
4467
4468/*
4469 * This is called when the original mapper is failing to COW a MAP_PRIVATE
4470 * mapping it owns the reserve page for. The intention is to unmap the page
4471 * from other VMAs and let the children be SIGKILLed if they are faulting the
4472 * same region.
4473 */
4474static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
4475			      struct page *page, unsigned long address)
4476{
4477	struct hstate *h = hstate_vma(vma);
4478	struct vm_area_struct *iter_vma;
4479	struct address_space *mapping;
4480	pgoff_t pgoff;
4481
4482	/*
4483	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
4484	 * from page cache lookup which is in HPAGE_SIZE units.
4485	 */
4486	address = address & huge_page_mask(h);
4487	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
4488			vma->vm_pgoff;
4489	mapping = vma->vm_file->f_mapping;
4490
4491	/*
4492	 * Take the mapping lock for the duration of the table walk. As
4493	 * this mapping should be shared between all the VMAs,
4494	 * __unmap_hugepage_range() is called as the lock is already held
4495	 */
4496	i_mmap_lock_write(mapping);
4497	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
4498		/* Do not unmap the current VMA */
4499		if (iter_vma == vma)
4500			continue;
4501
4502		/*
4503		 * Shared VMAs have their own reserves and do not affect
4504		 * MAP_PRIVATE accounting but it is possible that a shared
4505		 * VMA is using the same page so check and skip such VMAs.
4506		 */
4507		if (iter_vma->vm_flags & VM_MAYSHARE)
4508			continue;
4509
4510		/*
4511		 * Unmap the page from other VMAs without their own reserves.
4512		 * They get marked to be SIGKILLed if they fault in these
4513		 * areas. This is because a future no-page fault on this VMA
4514		 * could insert a zeroed page instead of the data existing
4515		 * from the time of fork. This would look like data corruption
4516		 */
4517		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
4518			unmap_hugepage_range(iter_vma, address,
4519					     address + huge_page_size(h), page);
4520	}
4521	i_mmap_unlock_write(mapping);
4522}
4523
4524/*
4525 * Hugetlb_cow() should be called with page lock of the original hugepage held.
4526 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
4527 * cannot race with other handlers or page migration.
4528 * Keep the pte_same checks anyway to make transition from the mutex easier.
4529 */
4530static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
4531		       unsigned long address, pte_t *ptep,
4532		       struct page *pagecache_page, spinlock_t *ptl)
4533{
4534	pte_t pte;
 
4535	struct hstate *h = hstate_vma(vma);
4536	struct page *old_page, *new_page;
 
4537	int outside_reserve = 0;
4538	vm_fault_t ret = 0;
4539	unsigned long haddr = address & huge_page_mask(h);
4540	struct mmu_notifier_range range;
4541
4542	pte = huge_ptep_get(ptep);
4543	old_page = pte_page(pte);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4544
4545retry_avoidcopy:
4546	/* If no-one else is actually using this page, avoid the copy
4547	 * and just make the page writable */
4548	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
4549		page_move_anon_rmap(old_page, vma);
4550		set_huge_ptep_writable(vma, haddr, ptep);
 
 
 
 
 
 
 
 
4551		return 0;
4552	}
 
 
4553
4554	/*
4555	 * If the process that created a MAP_PRIVATE mapping is about to
4556	 * perform a COW due to a shared page count, attempt to satisfy
4557	 * the allocation without using the existing reserves. The pagecache
4558	 * page is used to determine if the reserve at this address was
4559	 * consumed or not. If reserves were used, a partial faulted mapping
4560	 * at the time of fork() could consume its reserves on COW instead
4561	 * of the full address range.
4562	 */
4563	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
4564			old_page != pagecache_page)
4565		outside_reserve = 1;
4566
4567	get_page(old_page);
4568
4569	/*
4570	 * Drop page table lock as buddy allocator may be called. It will
4571	 * be acquired again before returning to the caller, as expected.
4572	 */
4573	spin_unlock(ptl);
4574	new_page = alloc_huge_page(vma, haddr, outside_reserve);
4575
4576	if (IS_ERR(new_page)) {
4577		/*
4578		 * If a process owning a MAP_PRIVATE mapping fails to COW,
4579		 * it is due to references held by a child and an insufficient
4580		 * huge page pool. To guarantee the original mappers
4581		 * reliability, unmap the page from child processes. The child
4582		 * may get SIGKILLed if it later faults.
4583		 */
4584		if (outside_reserve) {
4585			struct address_space *mapping = vma->vm_file->f_mapping;
4586			pgoff_t idx;
4587			u32 hash;
4588
4589			put_page(old_page);
4590			BUG_ON(huge_pte_none(pte));
4591			/*
4592			 * Drop hugetlb_fault_mutex and i_mmap_rwsem before
4593			 * unmapping.  unmapping needs to hold i_mmap_rwsem
4594			 * in write mode.  Dropping i_mmap_rwsem in read mode
4595			 * here is OK as COW mappings do not interact with
4596			 * PMD sharing.
4597			 *
4598			 * Reacquire both after unmap operation.
4599			 */
4600			idx = vma_hugecache_offset(h, vma, haddr);
4601			hash = hugetlb_fault_mutex_hash(mapping, idx);
 
4602			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4603			i_mmap_unlock_read(mapping);
4604
4605			unmap_ref_private(mm, vma, old_page, haddr);
4606
4607			i_mmap_lock_read(mapping);
4608			mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
4609			spin_lock(ptl);
4610			ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4611			if (likely(ptep &&
4612				   pte_same(huge_ptep_get(ptep), pte)))
4613				goto retry_avoidcopy;
4614			/*
4615			 * race occurs while re-acquiring page table
4616			 * lock, and our job is done.
4617			 */
 
4618			return 0;
4619		}
4620
4621		ret = vmf_error(PTR_ERR(new_page));
4622		goto out_release_old;
4623	}
4624
4625	/*
4626	 * When the original hugepage is shared one, it does not have
4627	 * anon_vma prepared.
4628	 */
4629	if (unlikely(anon_vma_prepare(vma))) {
4630		ret = VM_FAULT_OOM;
4631		goto out_release_all;
4632	}
4633
4634	copy_user_huge_page(new_page, old_page, address, vma,
4635			    pages_per_huge_page(h));
4636	__SetPageUptodate(new_page);
 
 
4637
4638	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,
4639				haddr + huge_page_size(h));
4640	mmu_notifier_invalidate_range_start(&range);
4641
4642	/*
4643	 * Retake the page table lock to check for racing updates
4644	 * before the page tables are altered
4645	 */
4646	spin_lock(ptl);
4647	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4648	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
4649		ClearHPageRestoreReserve(new_page);
4650
4651		/* Break COW */
4652		huge_ptep_clear_flush(vma, haddr, ptep);
4653		mmu_notifier_invalidate_range(mm, range.start, range.end);
4654		set_huge_pte_at(mm, haddr, ptep,
4655				make_huge_pte(vma, new_page, 1));
4656		page_remove_rmap(old_page, true);
4657		hugepage_add_new_anon_rmap(new_page, vma, haddr);
4658		SetHPageMigratable(new_page);
4659		/* Make the old page be freed below */
4660		new_page = old_page;
4661	}
4662	spin_unlock(ptl);
4663	mmu_notifier_invalidate_range_end(&range);
4664out_release_all:
4665	/* No restore in case of successful pagetable update (Break COW) */
4666	if (new_page != old_page)
4667		restore_reserve_on_error(h, vma, haddr, new_page);
4668	put_page(new_page);
 
 
 
4669out_release_old:
4670	put_page(old_page);
4671
4672	spin_lock(ptl); /* Caller expects lock to be held */
 
 
4673	return ret;
4674}
4675
4676/* Return the pagecache page at a given address within a VMA */
4677static struct page *hugetlbfs_pagecache_page(struct hstate *h,
4678			struct vm_area_struct *vma, unsigned long address)
4679{
4680	struct address_space *mapping;
4681	pgoff_t idx;
4682
4683	mapping = vma->vm_file->f_mapping;
4684	idx = vma_hugecache_offset(h, vma, address);
4685
4686	return find_lock_page(mapping, idx);
4687}
4688
4689/*
4690 * Return whether there is a pagecache page to back given address within VMA.
4691 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
4692 */
4693static bool hugetlbfs_pagecache_present(struct hstate *h,
4694			struct vm_area_struct *vma, unsigned long address)
4695{
4696	struct address_space *mapping;
4697	pgoff_t idx;
4698	struct page *page;
4699
4700	mapping = vma->vm_file->f_mapping;
4701	idx = vma_hugecache_offset(h, vma, address);
4702
4703	page = find_get_page(mapping, idx);
4704	if (page)
4705		put_page(page);
4706	return page != NULL;
4707}
4708
4709int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
4710			   pgoff_t idx)
4711{
4712	struct inode *inode = mapping->host;
4713	struct hstate *h = hstate_inode(inode);
4714	int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
 
 
 
 
4715
4716	if (err)
 
4717		return err;
4718	ClearHPageRestoreReserve(page);
 
4719
4720	/*
4721	 * set page dirty so that it will not be removed from cache/file
4722	 * by non-hugetlbfs specific code paths.
4723	 */
4724	set_page_dirty(page);
4725
4726	spin_lock(&inode->i_lock);
4727	inode->i_blocks += blocks_per_huge_page(h);
4728	spin_unlock(&inode->i_lock);
4729	return 0;
4730}
4731
4732static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
4733						  struct address_space *mapping,
4734						  pgoff_t idx,
4735						  unsigned int flags,
4736						  unsigned long haddr,
 
4737						  unsigned long reason)
4738{
4739	vm_fault_t ret;
4740	u32 hash;
4741	struct vm_fault vmf = {
4742		.vma = vma,
4743		.address = haddr,
 
4744		.flags = flags,
4745
4746		/*
4747		 * Hard to debug if it ends up being
4748		 * used by a callee that assumes
4749		 * something about the other
4750		 * uninitialized fields... same as in
4751		 * memory.c
4752		 */
4753	};
4754
4755	/*
4756	 * hugetlb_fault_mutex and i_mmap_rwsem must be
4757	 * dropped before handling userfault.  Reacquire
4758	 * after handling fault to make calling code simpler.
4759	 */
 
4760	hash = hugetlb_fault_mutex_hash(mapping, idx);
4761	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4762	i_mmap_unlock_read(mapping);
4763	ret = handle_userfault(&vmf, reason);
4764	i_mmap_lock_read(mapping);
4765	mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
 
 
 
 
 
 
 
 
 
 
 
4766
4767	return ret;
4768}
4769
4770static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
4771			struct vm_area_struct *vma,
4772			struct address_space *mapping, pgoff_t idx,
4773			unsigned long address, pte_t *ptep, unsigned int flags)
 
4774{
4775	struct hstate *h = hstate_vma(vma);
4776	vm_fault_t ret = VM_FAULT_SIGBUS;
4777	int anon_rmap = 0;
4778	unsigned long size;
4779	struct page *page;
4780	pte_t new_pte;
4781	spinlock_t *ptl;
4782	unsigned long haddr = address & huge_page_mask(h);
4783	bool new_page, new_pagecache_page = false;
 
4784
4785	/*
4786	 * Currently, we are forced to kill the process in the event the
4787	 * original mapper has unmapped pages from the child due to a failed
4788	 * COW. Warn that such a situation has occurred as it may not be obvious
 
4789	 */
4790	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
4791		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
4792			   current->pid);
4793		return ret;
4794	}
4795
4796	/*
4797	 * We can not race with truncation due to holding i_mmap_rwsem.
4798	 * i_size is modified when holding i_mmap_rwsem, so check here
4799	 * once for faults beyond end of file.
4800	 */
4801	size = i_size_read(mapping->host) >> huge_page_shift(h);
4802	if (idx >= size)
4803		goto out;
4804
4805retry:
4806	new_page = false;
4807	page = find_lock_page(mapping, idx);
4808	if (!page) {
4809		/* Check for page in userfault range */
4810		if (userfaultfd_missing(vma)) {
4811			ret = hugetlb_handle_userfault(vma, mapping, idx,
4812						       flags, haddr,
4813						       VM_UFFD_MISSING);
4814			goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4815		}
4816
4817		page = alloc_huge_page(vma, haddr, 0);
4818		if (IS_ERR(page)) {
4819			/*
4820			 * Returning error will result in faulting task being
4821			 * sent SIGBUS.  The hugetlb fault mutex prevents two
4822			 * tasks from racing to fault in the same page which
4823			 * could result in false unable to allocate errors.
4824			 * Page migration does not take the fault mutex, but
4825			 * does a clear then write of pte's under page table
4826			 * lock.  Page fault code could race with migration,
4827			 * notice the clear pte and try to allocate a page
4828			 * here.  Before returning error, get ptl and make
4829			 * sure there really is no pte entry.
4830			 */
4831			ptl = huge_pte_lock(h, mm, ptep);
4832			ret = 0;
4833			if (huge_pte_none(huge_ptep_get(ptep)))
4834				ret = vmf_error(PTR_ERR(page));
4835			spin_unlock(ptl);
4836			goto out;
4837		}
4838		clear_huge_page(page, address, pages_per_huge_page(h));
4839		__SetPageUptodate(page);
4840		new_page = true;
4841
4842		if (vma->vm_flags & VM_MAYSHARE) {
4843			int err = huge_add_to_page_cache(page, mapping, idx);
4844			if (err) {
4845				put_page(page);
4846				if (err == -EEXIST)
4847					goto retry;
 
 
 
 
 
 
4848				goto out;
4849			}
4850			new_pagecache_page = true;
4851		} else {
4852			lock_page(page);
4853			if (unlikely(anon_vma_prepare(vma))) {
4854				ret = VM_FAULT_OOM;
4855				goto backout_unlocked;
4856			}
4857			anon_rmap = 1;
4858		}
4859	} else {
4860		/*
4861		 * If memory error occurs between mmap() and fault, some process
4862		 * don't have hwpoisoned swap entry for errored virtual address.
4863		 * So we need to block hugepage fault by PG_hwpoison bit check.
4864		 */
4865		if (unlikely(PageHWPoison(page))) {
4866			ret = VM_FAULT_HWPOISON_LARGE |
4867				VM_FAULT_SET_HINDEX(hstate_index(h));
4868			goto backout_unlocked;
4869		}
4870
4871		/* Check for page in userfault range. */
4872		if (userfaultfd_minor(vma)) {
4873			unlock_page(page);
4874			put_page(page);
4875			ret = hugetlb_handle_userfault(vma, mapping, idx,
4876						       flags, haddr,
4877						       VM_UFFD_MINOR);
4878			goto out;
 
 
 
 
4879		}
4880	}
4881
4882	/*
4883	 * If we are going to COW a private mapping later, we examine the
4884	 * pending reservations for this page now. This will ensure that
4885	 * any allocations necessary to record that reservation occur outside
4886	 * the spinlock.
4887	 */
4888	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4889		if (vma_needs_reservation(h, vma, haddr) < 0) {
4890			ret = VM_FAULT_OOM;
4891			goto backout_unlocked;
4892		}
4893		/* Just decrements count, does not deallocate */
4894		vma_end_reservation(h, vma, haddr);
4895	}
4896
4897	ptl = huge_pte_lock(h, mm, ptep);
4898	ret = 0;
4899	if (!huge_pte_none(huge_ptep_get(ptep)))
 
4900		goto backout;
4901
4902	if (anon_rmap) {
4903		ClearHPageRestoreReserve(page);
4904		hugepage_add_new_anon_rmap(page, vma, haddr);
4905	} else
4906		page_dup_rmap(page, true);
4907	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
4908				&& (vma->vm_flags & VM_SHARED)));
4909	set_huge_pte_at(mm, haddr, ptep, new_pte);
 
 
 
 
 
 
4910
4911	hugetlb_count_add(pages_per_huge_page(h), mm);
4912	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4913		/* Optimization, do the COW without a second fault */
4914		ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
4915	}
4916
4917	spin_unlock(ptl);
4918
4919	/*
4920	 * Only set HPageMigratable in newly allocated pages.  Existing pages
4921	 * found in the pagecache may not have HPageMigratableset if they have
4922	 * been isolated for migration.
4923	 */
4924	if (new_page)
4925		SetHPageMigratable(page);
4926
4927	unlock_page(page);
4928out:
 
 
4929	return ret;
4930
4931backout:
4932	spin_unlock(ptl);
4933backout_unlocked:
4934	unlock_page(page);
4935	/* restore reserve for newly allocated pages not in page cache */
4936	if (new_page && !new_pagecache_page)
4937		restore_reserve_on_error(h, vma, haddr, page);
4938	put_page(page);
4939	goto out;
4940}
4941
4942#ifdef CONFIG_SMP
4943u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
4944{
4945	unsigned long key[2];
4946	u32 hash;
4947
4948	key[0] = (unsigned long) mapping;
4949	key[1] = idx;
4950
4951	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
4952
4953	return hash & (num_fault_mutexes - 1);
4954}
4955#else
4956/*
4957 * For uniprocessor systems we always use a single mutex, so just
4958 * return 0 and avoid the hashing overhead.
4959 */
4960u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
4961{
4962	return 0;
4963}
4964#endif
4965
4966vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
4967			unsigned long address, unsigned int flags)
4968{
4969	pte_t *ptep, entry;
4970	spinlock_t *ptl;
4971	vm_fault_t ret;
4972	u32 hash;
4973	pgoff_t idx;
4974	struct page *page = NULL;
4975	struct page *pagecache_page = NULL;
4976	struct hstate *h = hstate_vma(vma);
4977	struct address_space *mapping;
4978	int need_wait_lock = 0;
4979	unsigned long haddr = address & huge_page_mask(h);
4980
4981	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4982	if (ptep) {
4983		/*
4984		 * Since we hold no locks, ptep could be stale.  That is
4985		 * OK as we are only making decisions based on content and
4986		 * not actually modifying content here.
4987		 */
4988		entry = huge_ptep_get(ptep);
4989		if (unlikely(is_hugetlb_entry_migration(entry))) {
4990			migration_entry_wait_huge(vma, mm, ptep);
4991			return 0;
4992		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
4993			return VM_FAULT_HWPOISON_LARGE |
4994				VM_FAULT_SET_HINDEX(hstate_index(h));
4995	}
4996
4997	/*
4998	 * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold
4999	 * until finished with ptep.  This serves two purposes:
5000	 * 1) It prevents huge_pmd_unshare from being called elsewhere
5001	 *    and making the ptep no longer valid.
5002	 * 2) It synchronizes us with i_size modifications during truncation.
5003	 *
5004	 * ptep could have already be assigned via huge_pte_offset.  That
5005	 * is OK, as huge_pte_alloc will return the same value unless
5006	 * something has changed.
5007	 */
5008	mapping = vma->vm_file->f_mapping;
5009	i_mmap_lock_read(mapping);
5010	ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
5011	if (!ptep) {
5012		i_mmap_unlock_read(mapping);
5013		return VM_FAULT_OOM;
5014	}
5015
5016	/*
5017	 * Serialize hugepage allocation and instantiation, so that we don't
5018	 * get spurious allocation failures if two CPUs race to instantiate
5019	 * the same page in the page cache.
5020	 */
 
5021	idx = vma_hugecache_offset(h, vma, haddr);
5022	hash = hugetlb_fault_mutex_hash(mapping, idx);
5023	mutex_lock(&hugetlb_fault_mutex_table[hash]);
5024
 
 
 
 
 
 
 
 
 
 
 
 
 
5025	entry = huge_ptep_get(ptep);
5026	if (huge_pte_none(entry)) {
5027		ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
5028		goto out_mutex;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5029	}
5030
5031	ret = 0;
5032
5033	/*
5034	 * entry could be a migration/hwpoison entry at this point, so this
5035	 * check prevents the kernel from going below assuming that we have
5036	 * an active hugepage in pagecache. This goto expects the 2nd page
5037	 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
5038	 * properly handle it.
5039	 */
5040	if (!pte_present(entry))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5041		goto out_mutex;
 
5042
5043	/*
5044	 * If we are going to COW the mapping later, we examine the pending
5045	 * reservations for this page now. This will ensure that any
5046	 * allocations necessary to record that reservation occur outside the
5047	 * spinlock. For private mappings, we also lookup the pagecache
5048	 * page now as it is used to determine if a reservation has been
5049	 * consumed.
5050	 */
5051	if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
 
5052		if (vma_needs_reservation(h, vma, haddr) < 0) {
5053			ret = VM_FAULT_OOM;
5054			goto out_mutex;
5055		}
5056		/* Just decrements count, does not deallocate */
5057		vma_end_reservation(h, vma, haddr);
5058
5059		if (!(vma->vm_flags & VM_MAYSHARE))
5060			pagecache_page = hugetlbfs_pagecache_page(h,
5061								vma, haddr);
5062	}
5063
5064	ptl = huge_pte_lock(h, mm, ptep);
5065
5066	/* Check for a racing update before calling hugetlb_cow */
5067	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
5068		goto out_ptl;
5069
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5070	/*
5071	 * hugetlb_cow() requires page locks of pte_page(entry) and
5072	 * pagecache_page, so here we need take the former one
5073	 * when page != pagecache_page or !pagecache_page.
5074	 */
5075	page = pte_page(entry);
5076	if (page != pagecache_page)
5077		if (!trylock_page(page)) {
5078			need_wait_lock = 1;
5079			goto out_ptl;
5080		}
5081
5082	get_page(page);
5083
5084	if (flags & FAULT_FLAG_WRITE) {
5085		if (!huge_pte_write(entry)) {
5086			ret = hugetlb_cow(mm, vma, address, ptep,
5087					  pagecache_page, ptl);
5088			goto out_put_page;
 
 
5089		}
5090		entry = huge_pte_mkdirty(entry);
5091	}
5092	entry = pte_mkyoung(entry);
5093	if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
5094						flags & FAULT_FLAG_WRITE))
5095		update_mmu_cache(vma, haddr, ptep);
5096out_put_page:
5097	if (page != pagecache_page)
5098		unlock_page(page);
5099	put_page(page);
5100out_ptl:
5101	spin_unlock(ptl);
5102
5103	if (pagecache_page) {
5104		unlock_page(pagecache_page);
5105		put_page(pagecache_page);
5106	}
5107out_mutex:
 
5108	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5109	i_mmap_unlock_read(mapping);
5110	/*
5111	 * Generally it's safe to hold refcount during waiting page lock. But
5112	 * here we just wait to defer the next page fault to avoid busy loop and
5113	 * the page is not used after unlocked before returning from the current
5114	 * page fault. So we are safe from accessing freed page, even if we wait
5115	 * here without taking refcount.
5116	 */
5117	if (need_wait_lock)
5118		wait_on_page_locked(page);
5119	return ret;
5120}
5121
5122#ifdef CONFIG_USERFAULTFD
5123/*
5124 * Used by userfaultfd UFFDIO_COPY.  Based on mcopy_atomic_pte with
5125 * modifications for huge pages.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5126 */
5127int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
5128			    pte_t *dst_pte,
5129			    struct vm_area_struct *dst_vma,
5130			    unsigned long dst_addr,
5131			    unsigned long src_addr,
5132			    enum mcopy_atomic_mode mode,
5133			    struct page **pagep)
5134{
5135	bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE);
 
 
5136	struct hstate *h = hstate_vma(dst_vma);
5137	struct address_space *mapping = dst_vma->vm_file->f_mapping;
5138	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
5139	unsigned long size;
5140	int vm_shared = dst_vma->vm_flags & VM_SHARED;
5141	pte_t _dst_pte;
5142	spinlock_t *ptl;
5143	int ret = -ENOMEM;
5144	struct page *page;
5145	int writable;
5146	bool new_pagecache_page = false;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5147
5148	if (is_continue) {
5149		ret = -EFAULT;
5150		page = find_lock_page(mapping, idx);
5151		if (!page)
5152			goto out;
5153	} else if (!*pagep) {
5154		/* If a page already exists, then it's UFFDIO_COPY for
 
5155		 * a non-missing case. Return -EEXIST.
5156		 */
5157		if (vm_shared &&
5158		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
5159			ret = -EEXIST;
5160			goto out;
5161		}
5162
5163		page = alloc_huge_page(dst_vma, dst_addr, 0);
5164		if (IS_ERR(page)) {
5165			ret = -ENOMEM;
5166			goto out;
5167		}
5168
5169		ret = copy_huge_page_from_user(page,
5170						(const void __user *) src_addr,
5171						pages_per_huge_page(h), false);
5172
5173		/* fallback to copy_from_user outside mmap_lock */
5174		if (unlikely(ret)) {
5175			ret = -ENOENT;
5176			/* Free the allocated page which may have
5177			 * consumed a reservation.
5178			 */
5179			restore_reserve_on_error(h, dst_vma, dst_addr, page);
5180			put_page(page);
5181
5182			/* Allocate a temporary page to hold the copied
5183			 * contents.
5184			 */
5185			page = alloc_huge_page_vma(h, dst_vma, dst_addr);
5186			if (!page) {
5187				ret = -ENOMEM;
5188				goto out;
5189			}
5190			*pagep = page;
5191			/* Set the outparam pagep and return to the caller to
5192			 * copy the contents outside the lock. Don't free the
5193			 * page.
5194			 */
5195			goto out;
5196		}
5197	} else {
5198		if (vm_shared &&
5199		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
5200			put_page(*pagep);
5201			ret = -EEXIST;
5202			*pagep = NULL;
5203			goto out;
5204		}
5205
5206		page = alloc_huge_page(dst_vma, dst_addr, 0);
5207		if (IS_ERR(page)) {
 
5208			ret = -ENOMEM;
5209			*pagep = NULL;
 
 
 
 
 
 
 
5210			goto out;
5211		}
5212		copy_huge_page(page, *pagep);
5213		put_page(*pagep);
5214		*pagep = NULL;
5215	}
5216
5217	/*
5218	 * The memory barrier inside __SetPageUptodate makes sure that
5219	 * preceding stores to the page contents become visible before
5220	 * the set_pte_at() write.
5221	 */
5222	__SetPageUptodate(page);
5223
5224	/* Add shared, newly allocated pages to the page cache. */
5225	if (vm_shared && !is_continue) {
5226		size = i_size_read(mapping->host) >> huge_page_shift(h);
5227		ret = -EFAULT;
5228		if (idx >= size)
5229			goto out_release_nounlock;
5230
5231		/*
5232		 * Serialization between remove_inode_hugepages() and
5233		 * huge_add_to_page_cache() below happens through the
5234		 * hugetlb_fault_mutex_table that here must be hold by
5235		 * the caller.
5236		 */
5237		ret = huge_add_to_page_cache(page, mapping, idx);
5238		if (ret)
5239			goto out_release_nounlock;
5240		new_pagecache_page = true;
5241	}
5242
5243	ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
5244	spin_lock(ptl);
5245
5246	/*
5247	 * Recheck the i_size after holding PT lock to make sure not
5248	 * to leave any page mapped (as page_mapped()) beyond the end
5249	 * of the i_size (remove_inode_hugepages() is strict about
5250	 * enforcing that). If we bail out here, we'll also leave a
5251	 * page in the radix tree in the vm_shared case beyond the end
5252	 * of the i_size, but remove_inode_hugepages() will take care
5253	 * of it as soon as we drop the hugetlb_fault_mutex_table.
5254	 */
5255	size = i_size_read(mapping->host) >> huge_page_shift(h);
5256	ret = -EFAULT;
5257	if (idx >= size)
5258		goto out_release_unlock;
5259
 
 
 
 
 
5260	ret = -EEXIST;
5261	if (!huge_pte_none(huge_ptep_get(dst_pte)))
5262		goto out_release_unlock;
5263
5264	if (vm_shared) {
5265		page_dup_rmap(page, true);
5266	} else {
5267		ClearHPageRestoreReserve(page);
5268		hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
5269	}
5270
5271	/* For CONTINUE on a non-shared VMA, don't set VM_WRITE for CoW. */
5272	if (is_continue && !vm_shared)
 
 
 
5273		writable = 0;
5274	else
5275		writable = dst_vma->vm_flags & VM_WRITE;
5276
5277	_dst_pte = make_huge_pte(dst_vma, page, writable);
5278	if (writable)
5279		_dst_pte = huge_pte_mkdirty(_dst_pte);
 
 
 
 
 
5280	_dst_pte = pte_mkyoung(_dst_pte);
5281
5282	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
 
 
 
5283
5284	(void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte,
5285					dst_vma->vm_flags & VM_WRITE);
5286	hugetlb_count_add(pages_per_huge_page(h), dst_mm);
5287
5288	/* No need to invalidate - it was non-present before */
5289	update_mmu_cache(dst_vma, dst_addr, dst_pte);
5290
5291	spin_unlock(ptl);
5292	if (!is_continue)
5293		SetHPageMigratable(page);
5294	if (vm_shared || is_continue)
5295		unlock_page(page);
5296	ret = 0;
5297out:
5298	return ret;
5299out_release_unlock:
5300	spin_unlock(ptl);
5301	if (vm_shared || is_continue)
5302		unlock_page(page);
5303out_release_nounlock:
5304	if (!new_pagecache_page)
5305		restore_reserve_on_error(h, dst_vma, dst_addr, page);
5306	put_page(page);
5307	goto out;
5308}
5309#endif /* CONFIG_USERFAULTFD */
5310
5311static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
5312				 int refs, struct page **pages,
5313				 struct vm_area_struct **vmas)
5314{
5315	int nr;
5316
5317	for (nr = 0; nr < refs; nr++) {
5318		if (likely(pages))
5319			pages[nr] = mem_map_offset(page, nr);
5320		if (vmas)
5321			vmas[nr] = vma;
5322	}
5323}
5324
5325long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
5326			 struct page **pages, struct vm_area_struct **vmas,
5327			 unsigned long *position, unsigned long *nr_pages,
5328			 long i, unsigned int flags, int *locked)
5329{
5330	unsigned long pfn_offset;
5331	unsigned long vaddr = *position;
5332	unsigned long remainder = *nr_pages;
5333	struct hstate *h = hstate_vma(vma);
5334	int err = -EFAULT, refs;
 
 
 
 
 
5335
5336	while (vaddr < vma->vm_end && remainder) {
5337		pte_t *pte;
5338		spinlock_t *ptl = NULL;
5339		int absent;
5340		struct page *page;
5341
5342		/*
5343		 * If we have a pending SIGKILL, don't keep faulting pages and
5344		 * potentially allocating memory.
5345		 */
5346		if (fatal_signal_pending(current)) {
5347			remainder = 0;
5348			break;
5349		}
5350
5351		/*
5352		 * Some archs (sparc64, sh*) have multiple pte_ts to
5353		 * each hugepage.  We have to make sure we get the
5354		 * first, for the page indexing below to work.
5355		 *
5356		 * Note that page table lock is not held when pte is null.
5357		 */
5358		pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
5359				      huge_page_size(h));
5360		if (pte)
5361			ptl = huge_pte_lock(h, mm, pte);
5362		absent = !pte || huge_pte_none(huge_ptep_get(pte));
5363
5364		/*
5365		 * When coredumping, it suits get_dump_page if we just return
5366		 * an error where there's an empty slot with no huge pagecache
5367		 * to back it.  This way, we avoid allocating a hugepage, and
5368		 * the sparse dumpfile avoids allocating disk blocks, but its
5369		 * huge holes still show up with zeroes where they need to be.
5370		 */
5371		if (absent && (flags & FOLL_DUMP) &&
5372		    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
5373			if (pte)
5374				spin_unlock(ptl);
5375			remainder = 0;
5376			break;
5377		}
5378
5379		/*
5380		 * We need call hugetlb_fault for both hugepages under migration
5381		 * (in which case hugetlb_fault waits for the migration,) and
5382		 * hwpoisoned hugepages (in which case we need to prevent the
5383		 * caller from accessing to them.) In order to do this, we use
5384		 * here is_swap_pte instead of is_hugetlb_entry_migration and
5385		 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
5386		 * both cases, and because we can't follow correct pages
5387		 * directly from any kind of swap entries.
5388		 */
5389		if (absent || is_swap_pte(huge_ptep_get(pte)) ||
5390		    ((flags & FOLL_WRITE) &&
5391		      !huge_pte_write(huge_ptep_get(pte)))) {
5392			vm_fault_t ret;
5393			unsigned int fault_flags = 0;
5394
5395			if (pte)
5396				spin_unlock(ptl);
5397			if (flags & FOLL_WRITE)
5398				fault_flags |= FAULT_FLAG_WRITE;
5399			if (locked)
5400				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
5401					FAULT_FLAG_KILLABLE;
5402			if (flags & FOLL_NOWAIT)
5403				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
5404					FAULT_FLAG_RETRY_NOWAIT;
5405			if (flags & FOLL_TRIED) {
5406				/*
5407				 * Note: FAULT_FLAG_ALLOW_RETRY and
5408				 * FAULT_FLAG_TRIED can co-exist
5409				 */
5410				fault_flags |= FAULT_FLAG_TRIED;
5411			}
5412			ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
5413			if (ret & VM_FAULT_ERROR) {
5414				err = vm_fault_to_errno(ret, flags);
5415				remainder = 0;
5416				break;
5417			}
5418			if (ret & VM_FAULT_RETRY) {
5419				if (locked &&
5420				    !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
5421					*locked = 0;
5422				*nr_pages = 0;
5423				/*
5424				 * VM_FAULT_RETRY must not return an
5425				 * error, it will return zero
5426				 * instead.
5427				 *
5428				 * No need to update "position" as the
5429				 * caller will not check it after
5430				 * *nr_pages is set to 0.
5431				 */
5432				return i;
5433			}
5434			continue;
5435		}
5436
5437		pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
5438		page = pte_page(huge_ptep_get(pte));
5439
5440		/*
5441		 * If subpage information not requested, update counters
5442		 * and skip the same_page loop below.
 
 
 
 
 
5443		 */
5444		if (!pages && !vmas && !pfn_offset &&
5445		    (vaddr + huge_page_size(h) < vma->vm_end) &&
5446		    (remainder >= pages_per_huge_page(h))) {
5447			vaddr += huge_page_size(h);
5448			remainder -= pages_per_huge_page(h);
5449			i += pages_per_huge_page(h);
5450			spin_unlock(ptl);
5451			continue;
5452		}
5453
5454		/* vaddr may not be aligned to PAGE_SIZE */
5455		refs = min3(pages_per_huge_page(h) - pfn_offset, remainder,
5456		    (vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT);
5457
5458		if (pages || vmas)
5459			record_subpages_vmas(mem_map_offset(page, pfn_offset),
5460					     vma, refs,
5461					     likely(pages) ? pages + i : NULL,
5462					     vmas ? vmas + i : NULL);
5463
5464		if (pages) {
5465			/*
5466			 * try_grab_compound_head() should always succeed here,
5467			 * because: a) we hold the ptl lock, and b) we've just
5468			 * checked that the huge page is present in the page
5469			 * tables. If the huge page is present, then the tail
5470			 * pages must also be present. The ptl prevents the
5471			 * head page and tail pages from being rearranged in
5472			 * any way. So this page must be available at this
5473			 * point, unless the page refcount overflowed:
5474			 */
5475			if (WARN_ON_ONCE(!try_grab_compound_head(pages[i],
5476								 refs,
5477								 flags))) {
5478				spin_unlock(ptl);
5479				remainder = 0;
5480				err = -ENOMEM;
5481				break;
5482			}
5483		}
5484
5485		vaddr += (refs << PAGE_SHIFT);
5486		remainder -= refs;
5487		i += refs;
 
 
 
5488
5489		spin_unlock(ptl);
5490	}
5491	*nr_pages = remainder;
5492	/*
5493	 * setting position is actually required only if remainder is
5494	 * not zero but it's faster not to add a "if (remainder)"
5495	 * branch.
5496	 */
5497	*position = vaddr;
 
 
5498
5499	return i ? i : err;
5500}
5501
5502unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
5503		unsigned long address, unsigned long end, pgprot_t newprot)
 
5504{
5505	struct mm_struct *mm = vma->vm_mm;
5506	unsigned long start = address;
5507	pte_t *ptep;
5508	pte_t pte;
5509	struct hstate *h = hstate_vma(vma);
5510	unsigned long pages = 0;
5511	bool shared_pmd = false;
5512	struct mmu_notifier_range range;
 
 
 
5513
5514	/*
5515	 * In the case of shared PMDs, the area to flush could be beyond
5516	 * start/end.  Set range.start/range.end to cover the maximum possible
5517	 * range if PMD sharing is possible.
5518	 */
5519	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
5520				0, vma, mm, start, end);
5521	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5522
5523	BUG_ON(address >= end);
5524	flush_cache_range(vma, range.start, range.end);
5525
5526	mmu_notifier_invalidate_range_start(&range);
 
5527	i_mmap_lock_write(vma->vm_file->f_mapping);
5528	for (; address < end; address += huge_page_size(h)) {
 
5529		spinlock_t *ptl;
5530		ptep = huge_pte_offset(mm, address, huge_page_size(h));
5531		if (!ptep)
5532			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
5533		ptl = huge_pte_lock(h, mm, ptep);
5534		if (huge_pmd_unshare(mm, vma, &address, ptep)) {
 
 
 
 
 
 
5535			pages++;
5536			spin_unlock(ptl);
5537			shared_pmd = true;
 
5538			continue;
5539		}
5540		pte = huge_ptep_get(ptep);
5541		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
5542			spin_unlock(ptl);
5543			continue;
5544		}
5545		if (unlikely(is_hugetlb_entry_migration(pte))) {
5546			swp_entry_t entry = pte_to_swp_entry(pte);
 
 
5547
5548			if (is_writable_migration_entry(entry)) {
5549				pte_t newpte;
5550
5551				entry = make_readable_migration_entry(
5552							swp_offset(entry));
 
 
5553				newpte = swp_entry_to_pte(entry);
5554				set_huge_swap_pte_at(mm, address, ptep,
5555						     newpte, huge_page_size(h));
5556				pages++;
5557			}
5558			spin_unlock(ptl);
5559			continue;
5560		}
5561		if (!huge_pte_none(pte)) {
 
 
 
 
 
 
 
 
 
 
5562			pte_t old_pte;
5563			unsigned int shift = huge_page_shift(hstate_vma(vma));
5564
5565			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
5566			pte = pte_mkhuge(huge_pte_modify(old_pte, newprot));
5567			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
 
 
 
 
5568			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
5569			pages++;
 
 
 
 
 
 
 
5570		}
5571		spin_unlock(ptl);
5572	}
5573	/*
5574	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
5575	 * may have cleared our pud entry and done put_page on the page table:
5576	 * once we release i_mmap_rwsem, another task can do the final put_page
5577	 * and that page table be reused and filled with junk.  If we actually
5578	 * did unshare a page of pmds, flush the range corresponding to the pud.
5579	 */
5580	if (shared_pmd)
5581		flush_hugetlb_tlb_range(vma, range.start, range.end);
5582	else
5583		flush_hugetlb_tlb_range(vma, start, end);
5584	/*
5585	 * No need to call mmu_notifier_invalidate_range() we are downgrading
5586	 * page table protection not changing it to point to a new page.
 
5587	 *
5588	 * See Documentation/vm/mmu_notifier.rst
5589	 */
5590	i_mmap_unlock_write(vma->vm_file->f_mapping);
 
5591	mmu_notifier_invalidate_range_end(&range);
5592
5593	return pages << h->order;
5594}
5595
5596/* Return true if reservation was successful, false otherwise.  */
5597bool hugetlb_reserve_pages(struct inode *inode,
5598					long from, long to,
5599					struct vm_area_struct *vma,
5600					vm_flags_t vm_flags)
5601{
5602	long chg, add = -1;
5603	struct hstate *h = hstate_inode(inode);
5604	struct hugepage_subpool *spool = subpool_inode(inode);
5605	struct resv_map *resv_map;
5606	struct hugetlb_cgroup *h_cg = NULL;
5607	long gbl_reserve, regions_needed = 0;
5608
5609	/* This should never happen */
5610	if (from > to) {
5611		VM_WARN(1, "%s called with a negative range\n", __func__);
5612		return false;
5613	}
5614
5615	/*
 
 
 
 
 
 
5616	 * Only apply hugepage reservation if asked. At fault time, an
5617	 * attempt will be made for VM_NORESERVE to allocate a page
5618	 * without using reserves
5619	 */
5620	if (vm_flags & VM_NORESERVE)
5621		return true;
5622
5623	/*
5624	 * Shared mappings base their reservation on the number of pages that
5625	 * are already allocated on behalf of the file. Private mappings need
5626	 * to reserve the full area even if read-only as mprotect() may be
5627	 * called to make the mapping read-write. Assume !vma is a shm mapping
5628	 */
5629	if (!vma || vma->vm_flags & VM_MAYSHARE) {
5630		/*
5631		 * resv_map can not be NULL as hugetlb_reserve_pages is only
5632		 * called for inodes for which resv_maps were created (see
5633		 * hugetlbfs_get_inode).
5634		 */
5635		resv_map = inode_resv_map(inode);
5636
5637		chg = region_chg(resv_map, from, to, &regions_needed);
5638
5639	} else {
5640		/* Private mapping. */
5641		resv_map = resv_map_alloc();
5642		if (!resv_map)
5643			return false;
5644
5645		chg = to - from;
5646
5647		set_vma_resv_map(vma, resv_map);
5648		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
5649	}
5650
5651	if (chg < 0)
5652		goto out_err;
5653
5654	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
5655				chg * pages_per_huge_page(h), &h_cg) < 0)
5656		goto out_err;
5657
5658	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
5659		/* For private mappings, the hugetlb_cgroup uncharge info hangs
5660		 * of the resv_map.
5661		 */
5662		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
5663	}
5664
5665	/*
5666	 * There must be enough pages in the subpool for the mapping. If
5667	 * the subpool has a minimum size, there may be some global
5668	 * reservations already in place (gbl_reserve).
5669	 */
5670	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
5671	if (gbl_reserve < 0)
5672		goto out_uncharge_cgroup;
5673
5674	/*
5675	 * Check enough hugepages are available for the reservation.
5676	 * Hand the pages back to the subpool if there are not
5677	 */
5678	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
5679		goto out_put_pages;
5680
5681	/*
5682	 * Account for the reservations made. Shared mappings record regions
5683	 * that have reservations as they are shared by multiple VMAs.
5684	 * When the last VMA disappears, the region map says how much
5685	 * the reservation was and the page cache tells how much of
5686	 * the reservation was consumed. Private mappings are per-VMA and
5687	 * only the consumed reservations are tracked. When the VMA
5688	 * disappears, the original reservation is the VMA size and the
5689	 * consumed reservations are stored in the map. Hence, nothing
5690	 * else has to be done for private mappings here
5691	 */
5692	if (!vma || vma->vm_flags & VM_MAYSHARE) {
5693		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
5694
5695		if (unlikely(add < 0)) {
5696			hugetlb_acct_memory(h, -gbl_reserve);
5697			goto out_put_pages;
5698		} else if (unlikely(chg > add)) {
5699			/*
5700			 * pages in this range were added to the reserve
5701			 * map between region_chg and region_add.  This
5702			 * indicates a race with alloc_huge_page.  Adjust
5703			 * the subpool and reserve counts modified above
5704			 * based on the difference.
5705			 */
5706			long rsv_adjust;
5707
5708			/*
5709			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
5710			 * reference to h_cg->css. See comment below for detail.
5711			 */
5712			hugetlb_cgroup_uncharge_cgroup_rsvd(
5713				hstate_index(h),
5714				(chg - add) * pages_per_huge_page(h), h_cg);
5715
5716			rsv_adjust = hugepage_subpool_put_pages(spool,
5717								chg - add);
5718			hugetlb_acct_memory(h, -rsv_adjust);
5719		} else if (h_cg) {
5720			/*
5721			 * The file_regions will hold their own reference to
5722			 * h_cg->css. So we should release the reference held
5723			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
5724			 * done.
5725			 */
5726			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
5727		}
5728	}
5729	return true;
5730
5731out_put_pages:
5732	/* put back original number of pages, chg */
5733	(void)hugepage_subpool_put_pages(spool, chg);
5734out_uncharge_cgroup:
5735	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
5736					    chg * pages_per_huge_page(h), h_cg);
5737out_err:
 
5738	if (!vma || vma->vm_flags & VM_MAYSHARE)
5739		/* Only call region_abort if the region_chg succeeded but the
5740		 * region_add failed or didn't run.
5741		 */
5742		if (chg >= 0 && add < 0)
5743			region_abort(resv_map, from, to, regions_needed);
5744	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
5745		kref_put(&resv_map->refs, resv_map_release);
 
 
5746	return false;
5747}
5748
5749long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
5750								long freed)
5751{
5752	struct hstate *h = hstate_inode(inode);
5753	struct resv_map *resv_map = inode_resv_map(inode);
5754	long chg = 0;
5755	struct hugepage_subpool *spool = subpool_inode(inode);
5756	long gbl_reserve;
5757
5758	/*
5759	 * Since this routine can be called in the evict inode path for all
5760	 * hugetlbfs inodes, resv_map could be NULL.
5761	 */
5762	if (resv_map) {
5763		chg = region_del(resv_map, start, end);
5764		/*
5765		 * region_del() can fail in the rare case where a region
5766		 * must be split and another region descriptor can not be
5767		 * allocated.  If end == LONG_MAX, it will not fail.
5768		 */
5769		if (chg < 0)
5770			return chg;
5771	}
5772
5773	spin_lock(&inode->i_lock);
5774	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
5775	spin_unlock(&inode->i_lock);
5776
5777	/*
5778	 * If the subpool has a minimum size, the number of global
5779	 * reservations to be released may be adjusted.
5780	 *
5781	 * Note that !resv_map implies freed == 0. So (chg - freed)
5782	 * won't go negative.
5783	 */
5784	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
5785	hugetlb_acct_memory(h, -gbl_reserve);
5786
5787	return 0;
5788}
5789
5790#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
5791static unsigned long page_table_shareable(struct vm_area_struct *svma,
5792				struct vm_area_struct *vma,
5793				unsigned long addr, pgoff_t idx)
5794{
5795	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
5796				svma->vm_start;
5797	unsigned long sbase = saddr & PUD_MASK;
5798	unsigned long s_end = sbase + PUD_SIZE;
5799
5800	/* Allow segments to share if only one is marked locked */
5801	unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
5802	unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
5803
5804	/*
5805	 * match the virtual addresses, permission and the alignment of the
5806	 * page table page.
 
 
5807	 */
5808	if (pmd_index(addr) != pmd_index(saddr) ||
5809	    vm_flags != svm_flags ||
5810	    !range_in_vma(svma, sbase, s_end))
 
5811		return 0;
5812
5813	return saddr;
5814}
5815
5816static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
5817{
5818	unsigned long base = addr & PUD_MASK;
5819	unsigned long end = base + PUD_SIZE;
5820
 
 
 
 
5821	/*
5822	 * check on proper vm_flags and page table alignment
5823	 */
5824	if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
5825		return true;
5826	return false;
5827}
5828
5829bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
5830{
5831#ifdef CONFIG_USERFAULTFD
5832	if (uffd_disable_huge_pmd_share(vma))
5833		return false;
5834#endif
5835	return vma_shareable(vma, addr);
5836}
5837
5838/*
5839 * Determine if start,end range within vma could be mapped by shared pmd.
5840 * If yes, adjust start and end to cover range associated with possible
5841 * shared pmd mappings.
5842 */
5843void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
5844				unsigned long *start, unsigned long *end)
5845{
5846	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
5847		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
5848
5849	/*
5850	 * vma needs to span at least one aligned PUD size, and the range
5851	 * must be at least partially within in.
5852	 */
5853	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
5854		(*end <= v_start) || (*start >= v_end))
5855		return;
5856
5857	/* Extend the range to be PUD aligned for a worst case scenario */
5858	if (*start > v_start)
5859		*start = ALIGN_DOWN(*start, PUD_SIZE);
5860
5861	if (*end < v_end)
5862		*end = ALIGN(*end, PUD_SIZE);
5863}
5864
5865/*
5866 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
5867 * and returns the corresponding pte. While this is not necessary for the
5868 * !shared pmd case because we can allocate the pmd later as well, it makes the
5869 * code much cleaner.
5870 *
5871 * This routine must be called with i_mmap_rwsem held in at least read mode if
5872 * sharing is possible.  For hugetlbfs, this prevents removal of any page
5873 * table entries associated with the address space.  This is important as we
5874 * are setting up sharing based on existing page table entries (mappings).
5875 *
5876 * NOTE: This routine is only called from huge_pte_alloc.  Some callers of
5877 * huge_pte_alloc know that sharing is not possible and do not take
5878 * i_mmap_rwsem as a performance optimization.  This is handled by the
5879 * if !vma_shareable check at the beginning of the routine. i_mmap_rwsem is
5880 * only required for subsequent processing.
5881 */
5882pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
5883		      unsigned long addr, pud_t *pud)
5884{
5885	struct address_space *mapping = vma->vm_file->f_mapping;
5886	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
5887			vma->vm_pgoff;
5888	struct vm_area_struct *svma;
5889	unsigned long saddr;
5890	pte_t *spte = NULL;
5891	pte_t *pte;
5892	spinlock_t *ptl;
5893
5894	i_mmap_assert_locked(mapping);
5895	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
5896		if (svma == vma)
5897			continue;
5898
5899		saddr = page_table_shareable(svma, vma, addr, idx);
5900		if (saddr) {
5901			spte = huge_pte_offset(svma->vm_mm, saddr,
5902					       vma_mmu_pagesize(svma));
5903			if (spte) {
5904				get_page(virt_to_page(spte));
5905				break;
5906			}
5907		}
5908	}
5909
5910	if (!spte)
5911		goto out;
5912
5913	ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
5914	if (pud_none(*pud)) {
5915		pud_populate(mm, pud,
5916				(pmd_t *)((unsigned long)spte & PAGE_MASK));
5917		mm_inc_nr_pmds(mm);
5918	} else {
5919		put_page(virt_to_page(spte));
5920	}
5921	spin_unlock(ptl);
5922out:
5923	pte = (pte_t *)pmd_alloc(mm, pud, addr);
 
5924	return pte;
5925}
5926
5927/*
5928 * unmap huge page backed by shared pte.
5929 *
5930 * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
5931 * indicated by page_count > 1, unmap is achieved by clearing pud and
5932 * decrementing the ref count. If count == 1, the pte page is not shared.
5933 *
5934 * Called with page table lock held and i_mmap_rwsem held in write mode.
5935 *
5936 * returns: 1 successfully unmapped a shared pte page
5937 *	    0 the underlying pte page is not shared, or it is the last user
5938 */
5939int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
5940					unsigned long *addr, pte_t *ptep)
5941{
5942	pgd_t *pgd = pgd_offset(mm, *addr);
5943	p4d_t *p4d = p4d_offset(pgd, *addr);
5944	pud_t *pud = pud_offset(p4d, *addr);
5945
5946	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
 
5947	BUG_ON(page_count(virt_to_page(ptep)) == 0);
5948	if (page_count(virt_to_page(ptep)) == 1)
5949		return 0;
5950
5951	pud_clear(pud);
5952	put_page(virt_to_page(ptep));
5953	mm_dec_nr_pmds(mm);
5954	*addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
5955	return 1;
5956}
5957
5958#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
 
5959pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
5960		      unsigned long addr, pud_t *pud)
5961{
5962	return NULL;
5963}
5964
5965int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
5966				unsigned long *addr, pte_t *ptep)
5967{
5968	return 0;
5969}
5970
5971void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
5972				unsigned long *start, unsigned long *end)
5973{
5974}
5975
5976bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
5977{
5978	return false;
5979}
5980#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
5981
5982#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
5983pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
5984			unsigned long addr, unsigned long sz)
5985{
5986	pgd_t *pgd;
5987	p4d_t *p4d;
5988	pud_t *pud;
5989	pte_t *pte = NULL;
5990
5991	pgd = pgd_offset(mm, addr);
5992	p4d = p4d_alloc(mm, pgd, addr);
5993	if (!p4d)
5994		return NULL;
5995	pud = pud_alloc(mm, p4d, addr);
5996	if (pud) {
5997		if (sz == PUD_SIZE) {
5998			pte = (pte_t *)pud;
5999		} else {
6000			BUG_ON(sz != PMD_SIZE);
6001			if (want_pmd_share(vma, addr) && pud_none(*pud))
6002				pte = huge_pmd_share(mm, vma, addr, pud);
6003			else
6004				pte = (pte_t *)pmd_alloc(mm, pud, addr);
6005		}
6006	}
6007	BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
 
 
 
 
 
6008
6009	return pte;
6010}
6011
6012/*
6013 * huge_pte_offset() - Walk the page table to resolve the hugepage
6014 * entry at address @addr
6015 *
6016 * Return: Pointer to page table entry (PUD or PMD) for
6017 * address @addr, or NULL if a !p*d_present() entry is encountered and the
6018 * size @sz doesn't match the hugepage size at this level of the page
6019 * table.
6020 */
6021pte_t *huge_pte_offset(struct mm_struct *mm,
6022		       unsigned long addr, unsigned long sz)
6023{
6024	pgd_t *pgd;
6025	p4d_t *p4d;
6026	pud_t *pud;
6027	pmd_t *pmd;
6028
6029	pgd = pgd_offset(mm, addr);
6030	if (!pgd_present(*pgd))
6031		return NULL;
6032	p4d = p4d_offset(pgd, addr);
6033	if (!p4d_present(*p4d))
6034		return NULL;
6035
6036	pud = pud_offset(p4d, addr);
6037	if (sz == PUD_SIZE)
6038		/* must be pud huge, non-present or none */
6039		return (pte_t *)pud;
6040	if (!pud_present(*pud))
6041		return NULL;
6042	/* must have a valid entry and size to go further */
6043
6044	pmd = pmd_offset(pud, addr);
6045	/* must be pmd huge, non-present or none */
6046	return (pte_t *)pmd;
6047}
6048
6049#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
6050
6051/*
6052 * These functions are overwritable if your architecture needs its own
6053 * behavior.
6054 */
6055struct page * __weak
6056follow_huge_addr(struct mm_struct *mm, unsigned long address,
6057			      int write)
6058{
6059	return ERR_PTR(-EINVAL);
 
 
 
 
 
 
 
 
6060}
6061
6062struct page * __weak
6063follow_huge_pd(struct vm_area_struct *vma,
6064	       unsigned long address, hugepd_t hpd, int flags, int pdshift)
6065{
6066	WARN(1, "hugepd follow called with no support for hugepage directory format\n");
6067	return NULL;
6068}
6069
6070struct page * __weak
6071follow_huge_pmd(struct mm_struct *mm, unsigned long address,
6072		pmd_t *pmd, int flags)
6073{
6074	struct page *page = NULL;
6075	spinlock_t *ptl;
6076	pte_t pte;
6077
6078	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
6079	if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
6080			 (FOLL_PIN | FOLL_GET)))
6081		return NULL;
6082
6083retry:
6084	ptl = pmd_lockptr(mm, pmd);
6085	spin_lock(ptl);
6086	/*
6087	 * make sure that the address range covered by this pmd is not
6088	 * unmapped from other threads.
6089	 */
6090	if (!pmd_huge(*pmd))
6091		goto out;
6092	pte = huge_ptep_get((pte_t *)pmd);
6093	if (pte_present(pte)) {
6094		page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
6095		/*
6096		 * try_grab_page() should always succeed here, because: a) we
6097		 * hold the pmd (ptl) lock, and b) we've just checked that the
6098		 * huge pmd (head) page is present in the page tables. The ptl
6099		 * prevents the head page and tail pages from being rearranged
6100		 * in any way. So this page must be available at this point,
6101		 * unless the page refcount overflowed:
6102		 */
6103		if (WARN_ON_ONCE(!try_grab_page(page, flags))) {
6104			page = NULL;
6105			goto out;
6106		}
6107	} else {
6108		if (is_hugetlb_entry_migration(pte)) {
6109			spin_unlock(ptl);
6110			__migration_entry_wait(mm, (pte_t *)pmd, ptl);
6111			goto retry;
6112		}
6113		/*
6114		 * hwpoisoned entry is treated as no_page_table in
6115		 * follow_page_mask().
6116		 */
6117	}
6118out:
6119	spin_unlock(ptl);
6120	return page;
6121}
6122
6123struct page * __weak
6124follow_huge_pud(struct mm_struct *mm, unsigned long address,
6125		pud_t *pud, int flags)
6126{
6127	if (flags & (FOLL_GET | FOLL_PIN))
6128		return NULL;
6129
6130	return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
6131}
6132
6133struct page * __weak
6134follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags)
6135{
6136	if (flags & (FOLL_GET | FOLL_PIN))
6137		return NULL;
6138
6139	return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT);
6140}
6141
6142bool isolate_huge_page(struct page *page, struct list_head *list)
6143{
6144	bool ret = true;
6145
6146	spin_lock_irq(&hugetlb_lock);
6147	if (!PageHeadHuge(page) ||
6148	    !HPageMigratable(page) ||
6149	    !get_page_unless_zero(page)) {
6150		ret = false;
6151		goto unlock;
6152	}
6153	ClearHPageMigratable(page);
6154	list_move_tail(&page->lru, list);
6155unlock:
6156	spin_unlock_irq(&hugetlb_lock);
6157	return ret;
6158}
6159
6160int get_hwpoison_huge_page(struct page *page, bool *hugetlb)
6161{
6162	int ret = 0;
6163
6164	*hugetlb = false;
6165	spin_lock_irq(&hugetlb_lock);
6166	if (PageHeadHuge(page)) {
6167		*hugetlb = true;
6168		if (HPageFreed(page) || HPageMigratable(page))
6169			ret = get_page_unless_zero(page);
 
 
6170		else
6171			ret = -EBUSY;
6172	}
6173	spin_unlock_irq(&hugetlb_lock);
6174	return ret;
6175}
6176
6177void putback_active_hugepage(struct page *page)
 
 
 
 
 
 
 
 
 
 
 
6178{
6179	spin_lock_irq(&hugetlb_lock);
6180	SetHPageMigratable(page);
6181	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
6182	spin_unlock_irq(&hugetlb_lock);
6183	put_page(page);
6184}
6185
6186void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
6187{
6188	struct hstate *h = page_hstate(oldpage);
6189
6190	hugetlb_cgroup_migrate(oldpage, newpage);
6191	set_page_owner_migrate_reason(newpage, reason);
6192
6193	/*
6194	 * transfer temporary state of the new huge page. This is
6195	 * reverse to other transitions because the newpage is going to
6196	 * be final while the old one will be freed so it takes over
6197	 * the temporary status.
6198	 *
6199	 * Also note that we have to transfer the per-node surplus state
6200	 * here as well otherwise the global surplus count will not match
6201	 * the per-node's.
6202	 */
6203	if (HPageTemporary(newpage)) {
6204		int old_nid = page_to_nid(oldpage);
6205		int new_nid = page_to_nid(newpage);
 
 
 
6206
6207		SetHPageTemporary(oldpage);
6208		ClearHPageTemporary(newpage);
6209
6210		/*
6211		 * There is no need to transfer the per-node surplus state
6212		 * when we do not cross the node.
6213		 */
6214		if (new_nid == old_nid)
6215			return;
6216		spin_lock_irq(&hugetlb_lock);
6217		if (h->surplus_huge_pages_node[old_nid]) {
6218			h->surplus_huge_pages_node[old_nid]--;
6219			h->surplus_huge_pages_node[new_nid]++;
6220		}
6221		spin_unlock_irq(&hugetlb_lock);
6222	}
6223}
6224
6225/*
6226 * This function will unconditionally remove all the shared pmd pgtable entries
6227 * within the specific vma for a hugetlbfs memory range.
6228 */
6229void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
6230{
6231	struct hstate *h = hstate_vma(vma);
6232	unsigned long sz = huge_page_size(h);
6233	struct mm_struct *mm = vma->vm_mm;
6234	struct mmu_notifier_range range;
6235	unsigned long address, start, end;
6236	spinlock_t *ptl;
6237	pte_t *ptep;
6238
6239	if (!(vma->vm_flags & VM_MAYSHARE))
6240		return;
6241
6242	start = ALIGN(vma->vm_start, PUD_SIZE);
6243	end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
6244
6245	if (start >= end)
6246		return;
6247
 
6248	/*
6249	 * No need to call adjust_range_if_pmd_sharing_possible(), because
6250	 * we have already done the PUD_SIZE alignment.
6251	 */
6252	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
6253				start, end);
6254	mmu_notifier_invalidate_range_start(&range);
 
6255	i_mmap_lock_write(vma->vm_file->f_mapping);
6256	for (address = start; address < end; address += PUD_SIZE) {
6257		unsigned long tmp = address;
6258
6259		ptep = huge_pte_offset(mm, address, sz);
6260		if (!ptep)
6261			continue;
6262		ptl = huge_pte_lock(h, mm, ptep);
6263		/* We don't want 'address' to be changed */
6264		huge_pmd_unshare(mm, vma, &tmp, ptep);
6265		spin_unlock(ptl);
6266	}
6267	flush_hugetlb_tlb_range(vma, start, end);
6268	i_mmap_unlock_write(vma->vm_file->f_mapping);
 
6269	/*
6270	 * No need to call mmu_notifier_invalidate_range(), see
6271	 * Documentation/vm/mmu_notifier.rst.
6272	 */
6273	mmu_notifier_invalidate_range_end(&range);
6274}
6275
 
 
 
 
 
 
 
 
 
 
6276#ifdef CONFIG_CMA
6277static bool cma_reserve_called __initdata;
6278
6279static int __init cmdline_parse_hugetlb_cma(char *p)
6280{
6281	hugetlb_cma_size = memparse(p, &p);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6282	return 0;
6283}
6284
6285early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
6286
6287void __init hugetlb_cma_reserve(int order)
6288{
6289	unsigned long size, reserved, per_node;
 
6290	int nid;
6291
6292	cma_reserve_called = true;
6293
6294	if (!hugetlb_cma_size)
6295		return;
6296
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6297	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
6298		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
6299			(PAGE_SIZE << order) / SZ_1M);
 
6300		return;
6301	}
6302
6303	/*
6304	 * If 3 GB area is requested on a machine with 4 numa nodes,
6305	 * let's allocate 1 GB on first three nodes and ignore the last one.
6306	 */
6307	per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
6308	pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
6309		hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
 
 
6310
6311	reserved = 0;
6312	for_each_node_state(nid, N_ONLINE) {
6313		int res;
6314		char name[CMA_MAX_NAME];
6315
6316		size = min(per_node, hugetlb_cma_size - reserved);
 
 
 
 
 
 
 
 
6317		size = round_up(size, PAGE_SIZE << order);
6318
6319		snprintf(name, sizeof(name), "hugetlb%d", nid);
6320		res = cma_declare_contiguous_nid(0, size, 0, PAGE_SIZE << order,
 
 
 
 
 
 
6321						 0, false, name,
6322						 &hugetlb_cma[nid], nid);
6323		if (res) {
6324			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
6325				res, nid);
6326			continue;
6327		}
6328
6329		reserved += size;
6330		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
6331			size / SZ_1M, nid);
6332
6333		if (reserved >= hugetlb_cma_size)
6334			break;
6335	}
 
 
 
 
 
 
 
6336}
6337
6338void __init hugetlb_cma_check(void)
6339{
6340	if (!hugetlb_cma_size || cma_reserve_called)
6341		return;
6342
6343	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
6344}
6345
6346#endif /* CONFIG_CMA */