1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Generic hugetlb support.
   4 * (C) Nadia Yvette Chambers, April 2004
   5 */
   6#include <linux/list.h>
   7#include <linux/init.h>
   8#include <linux/mm.h>
   9#include <linux/seq_file.h>
  10#include <linux/sysctl.h>
  11#include <linux/highmem.h>
  12#include <linux/mmu_notifier.h>
  13#include <linux/nodemask.h>
  14#include <linux/pagemap.h>
  15#include <linux/mempolicy.h>
  16#include <linux/compiler.h>
  17#include <linux/cpuset.h>
  18#include <linux/mutex.h>
  19#include <linux/memblock.h>
  20#include <linux/sysfs.h>
  21#include <linux/slab.h>
  22#include <linux/sched/mm.h>
  23#include <linux/mmdebug.h>
  24#include <linux/sched/signal.h>
  25#include <linux/rmap.h>
  26#include <linux/string_helpers.h>
  27#include <linux/swap.h>
  28#include <linux/swapops.h>
  29#include <linux/jhash.h>
  30#include <linux/numa.h>
  31#include <linux/llist.h>
  32#include <linux/cma.h>
  33#include <linux/migrate.h>
  34#include <linux/nospec.h>
  35#include <linux/delayacct.h>
  36#include <linux/memory.h>
  37#include <linux/mm_inline.h>
  38#include <linux/padata.h>
  39
  40#include <asm/page.h>
  41#include <asm/pgalloc.h>
  42#include <asm/tlb.h>
  43
  44#include <linux/io.h>
  45#include <linux/hugetlb.h>
  46#include <linux/hugetlb_cgroup.h>
  47#include <linux/node.h>
  48#include <linux/page_owner.h>
  49#include "internal.h"
  50#include "hugetlb_vmemmap.h"
  51
  52int hugetlb_max_hstate __read_mostly;
  53unsigned int default_hstate_idx;
  54struct hstate hstates[HUGE_MAX_HSTATE];
  55
  56#ifdef CONFIG_CMA
  57static struct cma *hugetlb_cma[MAX_NUMNODES];
  58static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
 
 
 
 
 
 
 
 
 
 
  59#endif
  60static unsigned long hugetlb_cma_size __initdata;
  61
  62__initdata struct list_head huge_boot_pages[MAX_NUMNODES];
  63
  64/* for command line parsing */
  65static struct hstate * __initdata parsed_hstate;
  66static unsigned long __initdata default_hstate_max_huge_pages;
  67static bool __initdata parsed_valid_hugepagesz = true;
  68static bool __initdata parsed_default_hugepagesz;
  69static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
  70
  71/*
  72 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  73 * free_huge_pages, and surplus_huge_pages.
  74 */
  75__cacheline_aligned_in_smp DEFINE_SPINLOCK(hugetlb_lock);
  76
  77/*
  78 * Serializes faults on the same logical page.  This is used to
  79 * prevent spurious OOMs when the hugepage pool is fully utilized.
  80 */
  81static int num_fault_mutexes __ro_after_init;
  82struct mutex *hugetlb_fault_mutex_table __ro_after_init;
  83
  84/* Forward declaration */
  85static int hugetlb_acct_memory(struct hstate *h, long delta);
  86static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
  87static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
  88static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
  89static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
  90		unsigned long start, unsigned long end);
  91static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
  92
  93static void hugetlb_free_folio(struct folio *folio)
  94{
  95#ifdef CONFIG_CMA
  96	int nid = folio_nid(folio);
  97
  98	if (cma_free_folio(hugetlb_cma[nid], folio))
  99		return;
 100#endif
 101	folio_put(folio);
 102}
 103
 104static inline bool subpool_is_free(struct hugepage_subpool *spool)
 105{
 106	if (spool->count)
 107		return false;
 108	if (spool->max_hpages != -1)
 109		return spool->used_hpages == 0;
 110	if (spool->min_hpages != -1)
 111		return spool->rsv_hpages == spool->min_hpages;
 112
 113	return true;
 114}
 115
 116static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
 117						unsigned long irq_flags)
 118{
 119	spin_unlock_irqrestore(&spool->lock, irq_flags);
 120
 121	/* If no pages are used, and no other handles to the subpool
 122	 * remain, give up any reservations based on minimum size and
 123	 * free the subpool */
 124	if (subpool_is_free(spool)) {
 125		if (spool->min_hpages != -1)
 126			hugetlb_acct_memory(spool->hstate,
 127						-spool->min_hpages);
 128		kfree(spool);
 129	}
 130}
 131
 132struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 133						long min_hpages)
 134{
 135	struct hugepage_subpool *spool;
 136
 137	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 138	if (!spool)
 139		return NULL;
 140
 141	spin_lock_init(&spool->lock);
 142	spool->count = 1;
 143	spool->max_hpages = max_hpages;
 144	spool->hstate = h;
 145	spool->min_hpages = min_hpages;
 146
 147	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
 148		kfree(spool);
 149		return NULL;
 150	}
 151	spool->rsv_hpages = min_hpages;
 152
 153	return spool;
 154}
 155
 156void hugepage_put_subpool(struct hugepage_subpool *spool)
 157{
 158	unsigned long flags;
 159
 160	spin_lock_irqsave(&spool->lock, flags);
 161	BUG_ON(!spool->count);
 162	spool->count--;
 163	unlock_or_release_subpool(spool, flags);
 164}
 165
 166/*
 167 * Subpool accounting for allocating and reserving pages.
 168 * Return -ENOMEM if there are not enough resources to satisfy the
 169 * request.  Otherwise, return the number of pages by which the
 170 * global pools must be adjusted (upward).  The returned value may
 171 * only be different than the passed value (delta) in the case where
 172 * a subpool minimum size must be maintained.
 173 */
 174static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
 175				      long delta)
 176{
 177	long ret = delta;
 178
 179	if (!spool)
 180		return ret;
 181
 182	spin_lock_irq(&spool->lock);
 183
 184	if (spool->max_hpages != -1) {		/* maximum size accounting */
 185		if ((spool->used_hpages + delta) <= spool->max_hpages)
 186			spool->used_hpages += delta;
 187		else {
 188			ret = -ENOMEM;
 189			goto unlock_ret;
 190		}
 191	}
 192
 193	/* minimum size accounting */
 194	if (spool->min_hpages != -1 && spool->rsv_hpages) {
 195		if (delta > spool->rsv_hpages) {
 196			/*
 197			 * Asking for more reserves than those already taken on
 198			 * behalf of subpool.  Return difference.
 199			 */
 200			ret = delta - spool->rsv_hpages;
 201			spool->rsv_hpages = 0;
 202		} else {
 203			ret = 0;	/* reserves already accounted for */
 204			spool->rsv_hpages -= delta;
 205		}
 206	}
 207
 208unlock_ret:
 209	spin_unlock_irq(&spool->lock);
 210	return ret;
 211}
 212
 213/*
 214 * Subpool accounting for freeing and unreserving pages.
 215 * Return the number of global page reservations that must be dropped.
 216 * The return value may only be different than the passed value (delta)
 217 * in the case where a subpool minimum size must be maintained.
 218 */
 219static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 220				       long delta)
 221{
 222	long ret = delta;
 223	unsigned long flags;
 224
 225	if (!spool)
 226		return delta;
 227
 228	spin_lock_irqsave(&spool->lock, flags);
 229
 230	if (spool->max_hpages != -1)		/* maximum size accounting */
 231		spool->used_hpages -= delta;
 232
 233	 /* minimum size accounting */
 234	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
 235		if (spool->rsv_hpages + delta <= spool->min_hpages)
 236			ret = 0;
 237		else
 238			ret = spool->rsv_hpages + delta - spool->min_hpages;
 239
 240		spool->rsv_hpages += delta;
 241		if (spool->rsv_hpages > spool->min_hpages)
 242			spool->rsv_hpages = spool->min_hpages;
 243	}
 244
 245	/*
 246	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
 247	 * quota reference, free it now.
 248	 */
 249	unlock_or_release_subpool(spool, flags);
 250
 251	return ret;
 252}
 253
 254static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 255{
 256	return HUGETLBFS_SB(inode->i_sb)->spool;
 257}
 258
 259static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 260{
 261	return subpool_inode(file_inode(vma->vm_file));
 262}
 263
 264/*
 265 * hugetlb vma_lock helper routines
 266 */
 267void hugetlb_vma_lock_read(struct vm_area_struct *vma)
 268{
 269	if (__vma_shareable_lock(vma)) {
 270		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 271
 272		down_read(&vma_lock->rw_sema);
 273	} else if (__vma_private_lock(vma)) {
 274		struct resv_map *resv_map = vma_resv_map(vma);
 275
 276		down_read(&resv_map->rw_sema);
 277	}
 278}
 279
 280void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
 281{
 282	if (__vma_shareable_lock(vma)) {
 283		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 284
 285		up_read(&vma_lock->rw_sema);
 286	} else if (__vma_private_lock(vma)) {
 287		struct resv_map *resv_map = vma_resv_map(vma);
 288
 289		up_read(&resv_map->rw_sema);
 290	}
 291}
 292
 293void hugetlb_vma_lock_write(struct vm_area_struct *vma)
 294{
 295	if (__vma_shareable_lock(vma)) {
 296		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 297
 298		down_write(&vma_lock->rw_sema);
 299	} else if (__vma_private_lock(vma)) {
 300		struct resv_map *resv_map = vma_resv_map(vma);
 301
 302		down_write(&resv_map->rw_sema);
 303	}
 304}
 305
 306void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
 307{
 308	if (__vma_shareable_lock(vma)) {
 309		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 310
 311		up_write(&vma_lock->rw_sema);
 312	} else if (__vma_private_lock(vma)) {
 313		struct resv_map *resv_map = vma_resv_map(vma);
 314
 315		up_write(&resv_map->rw_sema);
 316	}
 317}
 318
 319int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
 320{
 321
 322	if (__vma_shareable_lock(vma)) {
 323		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 324
 325		return down_write_trylock(&vma_lock->rw_sema);
 326	} else if (__vma_private_lock(vma)) {
 327		struct resv_map *resv_map = vma_resv_map(vma);
 328
 329		return down_write_trylock(&resv_map->rw_sema);
 330	}
 331
 332	return 1;
 333}
 334
 335void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
 336{
 337	if (__vma_shareable_lock(vma)) {
 338		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 339
 340		lockdep_assert_held(&vma_lock->rw_sema);
 341	} else if (__vma_private_lock(vma)) {
 342		struct resv_map *resv_map = vma_resv_map(vma);
 343
 344		lockdep_assert_held(&resv_map->rw_sema);
 345	}
 346}
 347
 348void hugetlb_vma_lock_release(struct kref *kref)
 349{
 350	struct hugetlb_vma_lock *vma_lock = container_of(kref,
 351			struct hugetlb_vma_lock, refs);
 352
 353	kfree(vma_lock);
 354}
 355
 356static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
 357{
 358	struct vm_area_struct *vma = vma_lock->vma;
 359
 360	/*
 361	 * vma_lock structure may or not be released as a result of put,
 362	 * it certainly will no longer be attached to vma so clear pointer.
 363	 * Semaphore synchronizes access to vma_lock->vma field.
 364	 */
 365	vma_lock->vma = NULL;
 366	vma->vm_private_data = NULL;
 367	up_write(&vma_lock->rw_sema);
 368	kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
 369}
 370
 371static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
 372{
 373	if (__vma_shareable_lock(vma)) {
 374		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 375
 376		__hugetlb_vma_unlock_write_put(vma_lock);
 377	} else if (__vma_private_lock(vma)) {
 378		struct resv_map *resv_map = vma_resv_map(vma);
 379
 380		/* no free for anon vmas, but still need to unlock */
 381		up_write(&resv_map->rw_sema);
 382	}
 383}
 384
 385static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
 386{
 387	/*
 388	 * Only present in sharable vmas.
 389	 */
 390	if (!vma || !__vma_shareable_lock(vma))
 391		return;
 392
 393	if (vma->vm_private_data) {
 394		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 395
 396		down_write(&vma_lock->rw_sema);
 397		__hugetlb_vma_unlock_write_put(vma_lock);
 398	}
 399}
 400
 401static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
 402{
 403	struct hugetlb_vma_lock *vma_lock;
 404
 405	/* Only establish in (flags) sharable vmas */
 406	if (!vma || !(vma->vm_flags & VM_MAYSHARE))
 407		return;
 408
 409	/* Should never get here with non-NULL vm_private_data */
 410	if (vma->vm_private_data)
 411		return;
 412
 413	vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL);
 414	if (!vma_lock) {
 415		/*
 416		 * If we can not allocate structure, then vma can not
 417		 * participate in pmd sharing.  This is only a possible
 418		 * performance enhancement and memory saving issue.
 419		 * However, the lock is also used to synchronize page
 420		 * faults with truncation.  If the lock is not present,
 421		 * unlikely races could leave pages in a file past i_size
 422		 * until the file is removed.  Warn in the unlikely case of
 423		 * allocation failure.
 424		 */
 425		pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
 426		return;
 427	}
 428
 429	kref_init(&vma_lock->refs);
 430	init_rwsem(&vma_lock->rw_sema);
 431	vma_lock->vma = vma;
 432	vma->vm_private_data = vma_lock;
 433}
 434
 435/* Helper that removes a struct file_region from the resv_map cache and returns
 436 * it for use.
 437 */
 438static struct file_region *
 439get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
 440{
 441	struct file_region *nrg;
 442
 443	VM_BUG_ON(resv->region_cache_count <= 0);
 444
 445	resv->region_cache_count--;
 446	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
 447	list_del(&nrg->link);
 448
 449	nrg->from = from;
 450	nrg->to = to;
 451
 452	return nrg;
 453}
 454
 455static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
 456					      struct file_region *rg)
 457{
 458#ifdef CONFIG_CGROUP_HUGETLB
 459	nrg->reservation_counter = rg->reservation_counter;
 460	nrg->css = rg->css;
 461	if (rg->css)
 462		css_get(rg->css);
 463#endif
 464}
 465
 466/* Helper that records hugetlb_cgroup uncharge info. */
 467static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
 468						struct hstate *h,
 469						struct resv_map *resv,
 470						struct file_region *nrg)
 471{
 472#ifdef CONFIG_CGROUP_HUGETLB
 473	if (h_cg) {
 474		nrg->reservation_counter =
 475			&h_cg->rsvd_hugepage[hstate_index(h)];
 476		nrg->css = &h_cg->css;
 477		/*
 478		 * The caller will hold exactly one h_cg->css reference for the
 479		 * whole contiguous reservation region. But this area might be
 480		 * scattered when there are already some file_regions reside in
 481		 * it. As a result, many file_regions may share only one css
 482		 * reference. In order to ensure that one file_region must hold
 483		 * exactly one h_cg->css reference, we should do css_get for
 484		 * each file_region and leave the reference held by caller
 485		 * untouched.
 486		 */
 487		css_get(&h_cg->css);
 488		if (!resv->pages_per_hpage)
 489			resv->pages_per_hpage = pages_per_huge_page(h);
 490		/* pages_per_hpage should be the same for all entries in
 491		 * a resv_map.
 492		 */
 493		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
 494	} else {
 495		nrg->reservation_counter = NULL;
 496		nrg->css = NULL;
 497	}
 498#endif
 499}
 500
 501static void put_uncharge_info(struct file_region *rg)
 502{
 503#ifdef CONFIG_CGROUP_HUGETLB
 504	if (rg->css)
 505		css_put(rg->css);
 506#endif
 507}
 508
 509static bool has_same_uncharge_info(struct file_region *rg,
 510				   struct file_region *org)
 511{
 512#ifdef CONFIG_CGROUP_HUGETLB
 513	return rg->reservation_counter == org->reservation_counter &&
 514	       rg->css == org->css;
 515
 516#else
 517	return true;
 518#endif
 519}
 520
 521static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
 522{
 523	struct file_region *nrg, *prg;
 524
 525	prg = list_prev_entry(rg, link);
 526	if (&prg->link != &resv->regions && prg->to == rg->from &&
 527	    has_same_uncharge_info(prg, rg)) {
 528		prg->to = rg->to;
 529
 530		list_del(&rg->link);
 531		put_uncharge_info(rg);
 532		kfree(rg);
 533
 534		rg = prg;
 535	}
 536
 537	nrg = list_next_entry(rg, link);
 538	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
 539	    has_same_uncharge_info(nrg, rg)) {
 540		nrg->from = rg->from;
 541
 542		list_del(&rg->link);
 543		put_uncharge_info(rg);
 544		kfree(rg);
 545	}
 546}
 547
 548static inline long
 549hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
 550		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
 551		     long *regions_needed)
 552{
 553	struct file_region *nrg;
 554
 555	if (!regions_needed) {
 556		nrg = get_file_region_entry_from_cache(map, from, to);
 557		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
 558		list_add(&nrg->link, rg);
 559		coalesce_file_region(map, nrg);
 560	} else
 561		*regions_needed += 1;
 562
 563	return to - from;
 564}
 565
 566/*
 567 * Must be called with resv->lock held.
 568 *
 569 * Calling this with regions_needed != NULL will count the number of pages
 570 * to be added but will not modify the linked list. And regions_needed will
 571 * indicate the number of file_regions needed in the cache to carry out to add
 572 * the regions for this range.
 573 */
 574static long add_reservation_in_range(struct resv_map *resv, long f, long t,
 575				     struct hugetlb_cgroup *h_cg,
 576				     struct hstate *h, long *regions_needed)
 577{
 578	long add = 0;
 579	struct list_head *head = &resv->regions;
 580	long last_accounted_offset = f;
 581	struct file_region *iter, *trg = NULL;
 582	struct list_head *rg = NULL;
 583
 584	if (regions_needed)
 585		*regions_needed = 0;
 586
 587	/* In this loop, we essentially handle an entry for the range
 588	 * [last_accounted_offset, iter->from), at every iteration, with some
 589	 * bounds checking.
 590	 */
 591	list_for_each_entry_safe(iter, trg, head, link) {
 592		/* Skip irrelevant regions that start before our range. */
 593		if (iter->from < f) {
 594			/* If this region ends after the last accounted offset,
 595			 * then we need to update last_accounted_offset.
 596			 */
 597			if (iter->to > last_accounted_offset)
 598				last_accounted_offset = iter->to;
 599			continue;
 600		}
 601
 602		/* When we find a region that starts beyond our range, we've
 603		 * finished.
 604		 */
 605		if (iter->from >= t) {
 606			rg = iter->link.prev;
 607			break;
 608		}
 609
 610		/* Add an entry for last_accounted_offset -> iter->from, and
 611		 * update last_accounted_offset.
 612		 */
 613		if (iter->from > last_accounted_offset)
 614			add += hugetlb_resv_map_add(resv, iter->link.prev,
 615						    last_accounted_offset,
 616						    iter->from, h, h_cg,
 617						    regions_needed);
 618
 619		last_accounted_offset = iter->to;
 620	}
 621
 622	/* Handle the case where our range extends beyond
 623	 * last_accounted_offset.
 624	 */
 625	if (!rg)
 626		rg = head->prev;
 627	if (last_accounted_offset < t)
 628		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
 629					    t, h, h_cg, regions_needed);
 630
 631	return add;
 632}
 633
 634/* Must be called with resv->lock acquired. Will drop lock to allocate entries.
 635 */
 636static int allocate_file_region_entries(struct resv_map *resv,
 637					int regions_needed)
 638	__must_hold(&resv->lock)
 639{
 640	LIST_HEAD(allocated_regions);
 641	int to_allocate = 0, i = 0;
 642	struct file_region *trg = NULL, *rg = NULL;
 643
 644	VM_BUG_ON(regions_needed < 0);
 645
 646	/*
 647	 * Check for sufficient descriptors in the cache to accommodate
 648	 * the number of in progress add operations plus regions_needed.
 649	 *
 650	 * This is a while loop because when we drop the lock, some other call
 651	 * to region_add or region_del may have consumed some region_entries,
 652	 * so we keep looping here until we finally have enough entries for
 653	 * (adds_in_progress + regions_needed).
 654	 */
 655	while (resv->region_cache_count <
 656	       (resv->adds_in_progress + regions_needed)) {
 657		to_allocate = resv->adds_in_progress + regions_needed -
 658			      resv->region_cache_count;
 659
 660		/* At this point, we should have enough entries in the cache
 661		 * for all the existing adds_in_progress. We should only be
 662		 * needing to allocate for regions_needed.
 663		 */
 664		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
 665
 666		spin_unlock(&resv->lock);
 667		for (i = 0; i < to_allocate; i++) {
 668			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
 669			if (!trg)
 670				goto out_of_memory;
 671			list_add(&trg->link, &allocated_regions);
 672		}
 673
 674		spin_lock(&resv->lock);
 675
 676		list_splice(&allocated_regions, &resv->region_cache);
 677		resv->region_cache_count += to_allocate;
 678	}
 679
 680	return 0;
 681
 682out_of_memory:
 683	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
 684		list_del(&rg->link);
 685		kfree(rg);
 686	}
 687	return -ENOMEM;
 688}
 689
 690/*
 691 * Add the huge page range represented by [f, t) to the reserve
 692 * map.  Regions will be taken from the cache to fill in this range.
 693 * Sufficient regions should exist in the cache due to the previous
 694 * call to region_chg with the same range, but in some cases the cache will not
 695 * have sufficient entries due to races with other code doing region_add or
 696 * region_del.  The extra needed entries will be allocated.
 697 *
 698 * regions_needed is the out value provided by a previous call to region_chg.
 699 *
 700 * Return the number of new huge pages added to the map.  This number is greater
 701 * than or equal to zero.  If file_region entries needed to be allocated for
 702 * this operation and we were not able to allocate, it returns -ENOMEM.
 703 * region_add of regions of length 1 never allocate file_regions and cannot
 704 * fail; region_chg will always allocate at least 1 entry and a region_add for
 705 * 1 page will only require at most 1 entry.
 706 */
 707static long region_add(struct resv_map *resv, long f, long t,
 708		       long in_regions_needed, struct hstate *h,
 709		       struct hugetlb_cgroup *h_cg)
 710{
 711	long add = 0, actual_regions_needed = 0;
 712
 713	spin_lock(&resv->lock);
 714retry:
 715
 716	/* Count how many regions are actually needed to execute this add. */
 717	add_reservation_in_range(resv, f, t, NULL, NULL,
 718				 &actual_regions_needed);
 719
 720	/*
 721	 * Check for sufficient descriptors in the cache to accommodate
 722	 * this add operation. Note that actual_regions_needed may be greater
 723	 * than in_regions_needed, as the resv_map may have been modified since
 724	 * the region_chg call. In this case, we need to make sure that we
 725	 * allocate extra entries, such that we have enough for all the
 726	 * existing adds_in_progress, plus the excess needed for this
 727	 * operation.
 728	 */
 729	if (actual_regions_needed > in_regions_needed &&
 730	    resv->region_cache_count <
 731		    resv->adds_in_progress +
 732			    (actual_regions_needed - in_regions_needed)) {
 733		/* region_add operation of range 1 should never need to
 734		 * allocate file_region entries.
 735		 */
 736		VM_BUG_ON(t - f <= 1);
 737
 738		if (allocate_file_region_entries(
 739			    resv, actual_regions_needed - in_regions_needed)) {
 740			return -ENOMEM;
 741		}
 742
 743		goto retry;
 744	}
 745
 746	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
 747
 748	resv->adds_in_progress -= in_regions_needed;
 749
 750	spin_unlock(&resv->lock);
 751	return add;
 752}
 753
 754/*
 755 * Examine the existing reserve map and determine how many
 756 * huge pages in the specified range [f, t) are NOT currently
 757 * represented.  This routine is called before a subsequent
 758 * call to region_add that will actually modify the reserve
 759 * map to add the specified range [f, t).  region_chg does
 760 * not change the number of huge pages represented by the
 761 * map.  A number of new file_region structures is added to the cache as a
 762 * placeholder, for the subsequent region_add call to use. At least 1
 763 * file_region structure is added.
 764 *
 765 * out_regions_needed is the number of regions added to the
 766 * resv->adds_in_progress.  This value needs to be provided to a follow up call
 767 * to region_add or region_abort for proper accounting.
 768 *
 769 * Returns the number of huge pages that need to be added to the existing
 770 * reservation map for the range [f, t).  This number is greater or equal to
 771 * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 772 * is needed and can not be allocated.
 773 */
 774static long region_chg(struct resv_map *resv, long f, long t,
 775		       long *out_regions_needed)
 776{
 777	long chg = 0;
 778
 779	spin_lock(&resv->lock);
 780
 781	/* Count how many hugepages in this range are NOT represented. */
 782	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
 783				       out_regions_needed);
 784
 785	if (*out_regions_needed == 0)
 786		*out_regions_needed = 1;
 787
 788	if (allocate_file_region_entries(resv, *out_regions_needed))
 789		return -ENOMEM;
 790
 791	resv->adds_in_progress += *out_regions_needed;
 792
 793	spin_unlock(&resv->lock);
 794	return chg;
 795}
 796
 797/*
 798 * Abort the in progress add operation.  The adds_in_progress field
 799 * of the resv_map keeps track of the operations in progress between
 800 * calls to region_chg and region_add.  Operations are sometimes
 801 * aborted after the call to region_chg.  In such cases, region_abort
 802 * is called to decrement the adds_in_progress counter. regions_needed
 803 * is the value returned by the region_chg call, it is used to decrement
 804 * the adds_in_progress counter.
 805 *
 806 * NOTE: The range arguments [f, t) are not needed or used in this
 807 * routine.  They are kept to make reading the calling code easier as
 808 * arguments will match the associated region_chg call.
 809 */
 810static void region_abort(struct resv_map *resv, long f, long t,
 811			 long regions_needed)
 812{
 813	spin_lock(&resv->lock);
 814	VM_BUG_ON(!resv->region_cache_count);
 815	resv->adds_in_progress -= regions_needed;
 816	spin_unlock(&resv->lock);
 817}
 818
 819/*
 820 * Delete the specified range [f, t) from the reserve map.  If the
 821 * t parameter is LONG_MAX, this indicates that ALL regions after f
 822 * should be deleted.  Locate the regions which intersect [f, t)
 823 * and either trim, delete or split the existing regions.
 824 *
 825 * Returns the number of huge pages deleted from the reserve map.
 826 * In the normal case, the return value is zero or more.  In the
 827 * case where a region must be split, a new region descriptor must
 828 * be allocated.  If the allocation fails, -ENOMEM will be returned.
 829 * NOTE: If the parameter t == LONG_MAX, then we will never split
 830 * a region and possibly return -ENOMEM.  Callers specifying
 831 * t == LONG_MAX do not need to check for -ENOMEM error.
 832 */
 833static long region_del(struct resv_map *resv, long f, long t)
 834{
 835	struct list_head *head = &resv->regions;
 836	struct file_region *rg, *trg;
 837	struct file_region *nrg = NULL;
 838	long del = 0;
 839
 840retry:
 841	spin_lock(&resv->lock);
 842	list_for_each_entry_safe(rg, trg, head, link) {
 843		/*
 844		 * Skip regions before the range to be deleted.  file_region
 845		 * ranges are normally of the form [from, to).  However, there
 846		 * may be a "placeholder" entry in the map which is of the form
 847		 * (from, to) with from == to.  Check for placeholder entries
 848		 * at the beginning of the range to be deleted.
 849		 */
 850		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
 851			continue;
 852
 853		if (rg->from >= t)
 854			break;
 855
 856		if (f > rg->from && t < rg->to) { /* Must split region */
 857			/*
 858			 * Check for an entry in the cache before dropping
 859			 * lock and attempting allocation.
 860			 */
 861			if (!nrg &&
 862			    resv->region_cache_count > resv->adds_in_progress) {
 863				nrg = list_first_entry(&resv->region_cache,
 864							struct file_region,
 865							link);
 866				list_del(&nrg->link);
 867				resv->region_cache_count--;
 868			}
 869
 870			if (!nrg) {
 871				spin_unlock(&resv->lock);
 872				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 873				if (!nrg)
 874					return -ENOMEM;
 875				goto retry;
 876			}
 877
 878			del += t - f;
 879			hugetlb_cgroup_uncharge_file_region(
 880				resv, rg, t - f, false);
 881
 882			/* New entry for end of split region */
 883			nrg->from = t;
 884			nrg->to = rg->to;
 885
 886			copy_hugetlb_cgroup_uncharge_info(nrg, rg);
 887
 888			INIT_LIST_HEAD(&nrg->link);
 889
 890			/* Original entry is trimmed */
 891			rg->to = f;
 892
 893			list_add(&nrg->link, &rg->link);
 894			nrg = NULL;
 895			break;
 896		}
 897
 898		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
 899			del += rg->to - rg->from;
 900			hugetlb_cgroup_uncharge_file_region(resv, rg,
 901							    rg->to - rg->from, true);
 902			list_del(&rg->link);
 903			kfree(rg);
 904			continue;
 905		}
 906
 907		if (f <= rg->from) {	/* Trim beginning of region */
 908			hugetlb_cgroup_uncharge_file_region(resv, rg,
 909							    t - rg->from, false);
 910
 911			del += t - rg->from;
 912			rg->from = t;
 913		} else {		/* Trim end of region */
 914			hugetlb_cgroup_uncharge_file_region(resv, rg,
 915							    rg->to - f, false);
 916
 917			del += rg->to - f;
 918			rg->to = f;
 919		}
 920	}
 921
 922	spin_unlock(&resv->lock);
 923	kfree(nrg);
 924	return del;
 925}
 926
 927/*
 928 * A rare out of memory error was encountered which prevented removal of
 929 * the reserve map region for a page.  The huge page itself was free'ed
 930 * and removed from the page cache.  This routine will adjust the subpool
 931 * usage count, and the global reserve count if needed.  By incrementing
 932 * these counts, the reserve map entry which could not be deleted will
 933 * appear as a "reserved" entry instead of simply dangling with incorrect
 934 * counts.
 935 */
 936void hugetlb_fix_reserve_counts(struct inode *inode)
 937{
 938	struct hugepage_subpool *spool = subpool_inode(inode);
 939	long rsv_adjust;
 940	bool reserved = false;
 941
 942	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
 943	if (rsv_adjust > 0) {
 944		struct hstate *h = hstate_inode(inode);
 945
 946		if (!hugetlb_acct_memory(h, 1))
 947			reserved = true;
 948	} else if (!rsv_adjust) {
 949		reserved = true;
 950	}
 951
 952	if (!reserved)
 953		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
 954}
 955
 956/*
 957 * Count and return the number of huge pages in the reserve map
 958 * that intersect with the range [f, t).
 959 */
 960static long region_count(struct resv_map *resv, long f, long t)
 961{
 962	struct list_head *head = &resv->regions;
 963	struct file_region *rg;
 964	long chg = 0;
 965
 966	spin_lock(&resv->lock);
 967	/* Locate each segment we overlap with, and count that overlap. */
 968	list_for_each_entry(rg, head, link) {
 969		long seg_from;
 970		long seg_to;
 971
 972		if (rg->to <= f)
 973			continue;
 974		if (rg->from >= t)
 975			break;
 976
 977		seg_from = max(rg->from, f);
 978		seg_to = min(rg->to, t);
 979
 980		chg += seg_to - seg_from;
 981	}
 982	spin_unlock(&resv->lock);
 983
 984	return chg;
 985}
 986
 987/*
 988 * Convert the address within this vma to the page offset within
 989 * the mapping, huge page units here.
 990 */
 991static pgoff_t vma_hugecache_offset(struct hstate *h,
 992			struct vm_area_struct *vma, unsigned long address)
 993{
 994	return ((address - vma->vm_start) >> huge_page_shift(h)) +
 995			(vma->vm_pgoff >> huge_page_order(h));
 996}
 997
 998/**
 999 * vma_kernel_pagesize - Page size granularity for this VMA.
1000 * @vma: The user mapping.
1001 *
1002 * Folios in this VMA will be aligned to, and at least the size of the
1003 * number of bytes returned by this function.
1004 *
1005 * Return: The default size of the folios allocated when backing a VMA.
1006 */
1007unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
1008{
1009	if (vma->vm_ops && vma->vm_ops->pagesize)
1010		return vma->vm_ops->pagesize(vma);
1011	return PAGE_SIZE;
1012}
1013EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
1014
1015/*
1016 * Return the page size being used by the MMU to back a VMA. In the majority
1017 * of cases, the page size used by the kernel matches the MMU size. On
1018 * architectures where it differs, an architecture-specific 'strong'
1019 * version of this symbol is required.
1020 */
1021__weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
1022{
1023	return vma_kernel_pagesize(vma);
1024}
1025
1026/*
1027 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
1028 * bits of the reservation map pointer, which are always clear due to
1029 * alignment.
1030 */
1031#define HPAGE_RESV_OWNER    (1UL << 0)
1032#define HPAGE_RESV_UNMAPPED (1UL << 1)
1033#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
1034
1035/*
1036 * These helpers are used to track how many pages are reserved for
1037 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
1038 * is guaranteed to have their future faults succeed.
1039 *
1040 * With the exception of hugetlb_dup_vma_private() which is called at fork(),
1041 * the reserve counters are updated with the hugetlb_lock held. It is safe
1042 * to reset the VMA at fork() time as it is not in use yet and there is no
1043 * chance of the global counters getting corrupted as a result of the values.
1044 *
1045 * The private mapping reservation is represented in a subtly different
1046 * manner to a shared mapping.  A shared mapping has a region map associated
1047 * with the underlying file, this region map represents the backing file
1048 * pages which have ever had a reservation assigned which this persists even
1049 * after the page is instantiated.  A private mapping has a region map
1050 * associated with the original mmap which is attached to all VMAs which
1051 * reference it, this region map represents those offsets which have consumed
1052 * reservation ie. where pages have been instantiated.
1053 */
1054static unsigned long get_vma_private_data(struct vm_area_struct *vma)
1055{
1056	return (unsigned long)vma->vm_private_data;
1057}
1058
1059static void set_vma_private_data(struct vm_area_struct *vma,
1060							unsigned long value)
1061{
1062	vma->vm_private_data = (void *)value;
1063}
1064
1065static void
1066resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
1067					  struct hugetlb_cgroup *h_cg,
1068					  struct hstate *h)
1069{
1070#ifdef CONFIG_CGROUP_HUGETLB
1071	if (!h_cg || !h) {
1072		resv_map->reservation_counter = NULL;
1073		resv_map->pages_per_hpage = 0;
1074		resv_map->css = NULL;
1075	} else {
1076		resv_map->reservation_counter =
1077			&h_cg->rsvd_hugepage[hstate_index(h)];
1078		resv_map->pages_per_hpage = pages_per_huge_page(h);
1079		resv_map->css = &h_cg->css;
1080	}
1081#endif
1082}
1083
1084struct resv_map *resv_map_alloc(void)
1085{
1086	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
1087	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
1088
1089	if (!resv_map || !rg) {
1090		kfree(resv_map);
1091		kfree(rg);
1092		return NULL;
1093	}
1094
1095	kref_init(&resv_map->refs);
1096	spin_lock_init(&resv_map->lock);
1097	INIT_LIST_HEAD(&resv_map->regions);
1098	init_rwsem(&resv_map->rw_sema);
1099
1100	resv_map->adds_in_progress = 0;
1101	/*
1102	 * Initialize these to 0. On shared mappings, 0's here indicate these
1103	 * fields don't do cgroup accounting. On private mappings, these will be
1104	 * re-initialized to the proper values, to indicate that hugetlb cgroup
1105	 * reservations are to be un-charged from here.
1106	 */
1107	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
1108
1109	INIT_LIST_HEAD(&resv_map->region_cache);
1110	list_add(&rg->link, &resv_map->region_cache);
1111	resv_map->region_cache_count = 1;
1112
1113	return resv_map;
1114}
1115
1116void resv_map_release(struct kref *ref)
1117{
1118	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
1119	struct list_head *head = &resv_map->region_cache;
1120	struct file_region *rg, *trg;
1121
1122	/* Clear out any active regions before we release the map. */
1123	region_del(resv_map, 0, LONG_MAX);
1124
1125	/* ... and any entries left in the cache */
1126	list_for_each_entry_safe(rg, trg, head, link) {
1127		list_del(&rg->link);
1128		kfree(rg);
1129	}
1130
1131	VM_BUG_ON(resv_map->adds_in_progress);
1132
1133	kfree(resv_map);
1134}
1135
1136static inline struct resv_map *inode_resv_map(struct inode *inode)
1137{
1138	/*
1139	 * At inode evict time, i_mapping may not point to the original
1140	 * address space within the inode.  This original address space
1141	 * contains the pointer to the resv_map.  So, always use the
1142	 * address space embedded within the inode.
1143	 * The VERY common case is inode->mapping == &inode->i_data but,
1144	 * this may not be true for device special inodes.
1145	 */
1146	return (struct resv_map *)(&inode->i_data)->i_private_data;
1147}
1148
1149static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
1150{
1151	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1152	if (vma->vm_flags & VM_MAYSHARE) {
1153		struct address_space *mapping = vma->vm_file->f_mapping;
1154		struct inode *inode = mapping->host;
1155
1156		return inode_resv_map(inode);
1157
1158	} else {
1159		return (struct resv_map *)(get_vma_private_data(vma) &
1160							~HPAGE_RESV_MASK);
1161	}
1162}
1163
1164static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
1165{
1166	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1167	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1168
1169	set_vma_private_data(vma, (unsigned long)map);
1170}
1171
1172static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
1173{
1174	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1175	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1176
1177	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
1178}
1179
1180static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
1181{
1182	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1183
1184	return (get_vma_private_data(vma) & flag) != 0;
1185}
1186
1187bool __vma_private_lock(struct vm_area_struct *vma)
1188{
1189	return !(vma->vm_flags & VM_MAYSHARE) &&
1190		get_vma_private_data(vma) & ~HPAGE_RESV_MASK &&
1191		is_vma_resv_set(vma, HPAGE_RESV_OWNER);
1192}
1193
1194void hugetlb_dup_vma_private(struct vm_area_struct *vma)
1195{
1196	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1197	/*
1198	 * Clear vm_private_data
1199	 * - For shared mappings this is a per-vma semaphore that may be
1200	 *   allocated in a subsequent call to hugetlb_vm_op_open.
1201	 *   Before clearing, make sure pointer is not associated with vma
1202	 *   as this will leak the structure.  This is the case when called
1203	 *   via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already
1204	 *   been called to allocate a new structure.
1205	 * - For MAP_PRIVATE mappings, this is the reserve map which does
1206	 *   not apply to children.  Faults generated by the children are
1207	 *   not guaranteed to succeed, even if read-only.
1208	 */
1209	if (vma->vm_flags & VM_MAYSHARE) {
1210		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
1211
1212		if (vma_lock && vma_lock->vma != vma)
1213			vma->vm_private_data = NULL;
1214	} else
1215		vma->vm_private_data = NULL;
1216}
1217
1218/*
1219 * Reset and decrement one ref on hugepage private reservation.
1220 * Called with mm->mmap_lock writer semaphore held.
1221 * This function should be only used by move_vma() and operate on
1222 * same sized vma. It should never come here with last ref on the
1223 * reservation.
1224 */
1225void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
1226{
1227	/*
1228	 * Clear the old hugetlb private page reservation.
1229	 * It has already been transferred to new_vma.
1230	 *
1231	 * During a mremap() operation of a hugetlb vma we call move_vma()
1232	 * which copies vma into new_vma and unmaps vma. After the copy
1233	 * operation both new_vma and vma share a reference to the resv_map
1234	 * struct, and at that point vma is about to be unmapped. We don't
1235	 * want to return the reservation to the pool at unmap of vma because
1236	 * the reservation still lives on in new_vma, so simply decrement the
1237	 * ref here and remove the resv_map reference from this vma.
1238	 */
1239	struct resv_map *reservations = vma_resv_map(vma);
1240
1241	if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1242		resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
1243		kref_put(&reservations->refs, resv_map_release);
1244	}
1245
1246	hugetlb_dup_vma_private(vma);
1247}
1248
1249/* Returns true if the VMA has associated reserve pages */
1250static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1251{
1252	if (vma->vm_flags & VM_NORESERVE) {
1253		/*
1254		 * This address is already reserved by other process(chg == 0),
1255		 * so, we should decrement reserved count. Without decrementing,
1256		 * reserve count remains after releasing inode, because this
1257		 * allocated page will go into page cache and is regarded as
1258		 * coming from reserved pool in releasing step.  Currently, we
1259		 * don't have any other solution to deal with this situation
1260		 * properly, so add work-around here.
1261		 */
1262		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1263			return true;
1264		else
1265			return false;
1266	}
1267
1268	/* Shared mappings always use reserves */
1269	if (vma->vm_flags & VM_MAYSHARE) {
1270		/*
1271		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1272		 * be a region map for all pages.  The only situation where
1273		 * there is no region map is if a hole was punched via
1274		 * fallocate.  In this case, there really are no reserves to
1275		 * use.  This situation is indicated if chg != 0.
1276		 */
1277		if (chg)
1278			return false;
1279		else
1280			return true;
1281	}
1282
1283	/*
1284	 * Only the process that called mmap() has reserves for
1285	 * private mappings.
1286	 */
1287	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1288		/*
1289		 * Like the shared case above, a hole punch or truncate
1290		 * could have been performed on the private mapping.
1291		 * Examine the value of chg to determine if reserves
1292		 * actually exist or were previously consumed.
1293		 * Very Subtle - The value of chg comes from a previous
1294		 * call to vma_needs_reserves().  The reserve map for
1295		 * private mappings has different (opposite) semantics
1296		 * than that of shared mappings.  vma_needs_reserves()
1297		 * has already taken this difference in semantics into
1298		 * account.  Therefore, the meaning of chg is the same
1299		 * as in the shared case above.  Code could easily be
1300		 * combined, but keeping it separate draws attention to
1301		 * subtle differences.
1302		 */
1303		if (chg)
1304			return false;
1305		else
1306			return true;
1307	}
1308
1309	return false;
1310}
1311
1312static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
1313{
1314	int nid = folio_nid(folio);
1315
1316	lockdep_assert_held(&hugetlb_lock);
1317	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1318
1319	list_move(&folio->lru, &h->hugepage_freelists[nid]);
1320	h->free_huge_pages++;
1321	h->free_huge_pages_node[nid]++;
1322	folio_set_hugetlb_freed(folio);
1323}
1324
1325static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h,
1326								int nid)
1327{
1328	struct folio *folio;
1329	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1330
1331	lockdep_assert_held(&hugetlb_lock);
1332	list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) {
1333		if (pin && !folio_is_longterm_pinnable(folio))
1334			continue;
1335
1336		if (folio_test_hwpoison(folio))
1337			continue;
1338
1339		list_move(&folio->lru, &h->hugepage_activelist);
1340		folio_ref_unfreeze(folio, 1);
1341		folio_clear_hugetlb_freed(folio);
1342		h->free_huge_pages--;
1343		h->free_huge_pages_node[nid]--;
1344		return folio;
1345	}
1346
1347	return NULL;
1348}
1349
1350static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_mask,
1351							int nid, nodemask_t *nmask)
1352{
1353	unsigned int cpuset_mems_cookie;
1354	struct zonelist *zonelist;
1355	struct zone *zone;
1356	struct zoneref *z;
1357	int node = NUMA_NO_NODE;
1358
1359	/* 'nid' should not be NUMA_NO_NODE. Try to catch any misuse of it and rectifiy. */
1360	if (nid == NUMA_NO_NODE)
1361		nid = numa_node_id();
1362
1363	zonelist = node_zonelist(nid, gfp_mask);
1364
1365retry_cpuset:
1366	cpuset_mems_cookie = read_mems_allowed_begin();
1367	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1368		struct folio *folio;
1369
1370		if (!cpuset_zone_allowed(zone, gfp_mask))
1371			continue;
1372		/*
1373		 * no need to ask again on the same node. Pool is node rather than
1374		 * zone aware
1375		 */
1376		if (zone_to_nid(zone) == node)
1377			continue;
1378		node = zone_to_nid(zone);
1379
1380		folio = dequeue_hugetlb_folio_node_exact(h, node);
1381		if (folio)
1382			return folio;
1383	}
1384	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1385		goto retry_cpuset;
1386
1387	return NULL;
1388}
1389
1390static unsigned long available_huge_pages(struct hstate *h)
1391{
1392	return h->free_huge_pages - h->resv_huge_pages;
1393}
1394
1395static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h,
1396				struct vm_area_struct *vma,
1397				unsigned long address, long chg)
 
1398{
1399	struct folio *folio = NULL;
1400	struct mempolicy *mpol;
1401	gfp_t gfp_mask;
1402	nodemask_t *nodemask;
1403	int nid;
1404
1405	/*
1406	 * A child process with MAP_PRIVATE mappings created by their parent
1407	 * have no page reserves. This check ensures that reservations are
1408	 * not "stolen". The child may still get SIGKILLed
1409	 */
1410	if (!vma_has_reserves(vma, chg) && !available_huge_pages(h))
1411		goto err;
1412
 
 
 
 
1413	gfp_mask = htlb_alloc_mask(h);
1414	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1415
1416	if (mpol_is_preferred_many(mpol)) {
1417		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
1418							nid, nodemask);
1419
1420		/* Fallback to all nodes if page==NULL */
1421		nodemask = NULL;
1422	}
1423
1424	if (!folio)
1425		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
1426							nid, nodemask);
1427
1428	if (folio && vma_has_reserves(vma, chg)) {
1429		folio_set_hugetlb_restore_reserve(folio);
1430		h->resv_huge_pages--;
1431	}
1432
1433	mpol_cond_put(mpol);
1434	return folio;
1435
1436err:
1437	return NULL;
1438}
1439
1440/*
1441 * common helper functions for hstate_next_node_to_{alloc|free}.
1442 * We may have allocated or freed a huge page based on a different
1443 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1444 * be outside of *nodes_allowed.  Ensure that we use an allowed
1445 * node for alloc or free.
1446 */
1447static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1448{
1449	nid = next_node_in(nid, *nodes_allowed);
1450	VM_BUG_ON(nid >= MAX_NUMNODES);
1451
1452	return nid;
1453}
1454
1455static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1456{
1457	if (!node_isset(nid, *nodes_allowed))
1458		nid = next_node_allowed(nid, nodes_allowed);
1459	return nid;
1460}
1461
1462/*
1463 * returns the previously saved node ["this node"] from which to
1464 * allocate a persistent huge page for the pool and advance the
1465 * next node from which to allocate, handling wrap at end of node
1466 * mask.
1467 */
1468static int hstate_next_node_to_alloc(int *next_node,
1469					nodemask_t *nodes_allowed)
1470{
1471	int nid;
1472
1473	VM_BUG_ON(!nodes_allowed);
1474
1475	nid = get_valid_node_allowed(*next_node, nodes_allowed);
1476	*next_node = next_node_allowed(nid, nodes_allowed);
1477
1478	return nid;
1479}
1480
1481/*
1482 * helper for remove_pool_hugetlb_folio() - return the previously saved
1483 * node ["this node"] from which to free a huge page.  Advance the
1484 * next node id whether or not we find a free huge page to free so
1485 * that the next attempt to free addresses the next node.
1486 */
1487static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1488{
1489	int nid;
1490
1491	VM_BUG_ON(!nodes_allowed);
1492
1493	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1494	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1495
1496	return nid;
1497}
1498
1499#define for_each_node_mask_to_alloc(next_node, nr_nodes, node, mask)		\
1500	for (nr_nodes = nodes_weight(*mask);				\
1501		nr_nodes > 0 &&						\
1502		((node = hstate_next_node_to_alloc(next_node, mask)) || 1);	\
1503		nr_nodes--)
1504
1505#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1506	for (nr_nodes = nodes_weight(*mask);				\
1507		nr_nodes > 0 &&						\
1508		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1509		nr_nodes--)
1510
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1511#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1512#ifdef CONFIG_CONTIG_ALLOC
1513static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1514		int nid, nodemask_t *nodemask)
1515{
1516	struct folio *folio;
1517	int order = huge_page_order(h);
1518	bool retried = false;
1519
1520	if (nid == NUMA_NO_NODE)
1521		nid = numa_mem_id();
1522retry:
1523	folio = NULL;
1524#ifdef CONFIG_CMA
1525	{
1526		int node;
1527
1528		if (hugetlb_cma[nid])
1529			folio = cma_alloc_folio(hugetlb_cma[nid], order, gfp_mask);
 
 
 
 
1530
1531		if (!folio && !(gfp_mask & __GFP_THISNODE)) {
1532			for_each_node_mask(node, *nodemask) {
1533				if (node == nid || !hugetlb_cma[node])
1534					continue;
1535
1536				folio = cma_alloc_folio(hugetlb_cma[node], order, gfp_mask);
1537				if (folio)
1538					break;
 
1539			}
1540		}
1541	}
1542#endif
1543	if (!folio) {
1544		folio = folio_alloc_gigantic(order, gfp_mask, nid, nodemask);
1545		if (!folio)
1546			return NULL;
1547	}
1548
1549	if (folio_ref_freeze(folio, 1))
1550		return folio;
1551
1552	pr_warn("HugeTLB: unexpected refcount on PFN %lu\n", folio_pfn(folio));
1553	hugetlb_free_folio(folio);
1554	if (!retried) {
1555		retried = true;
1556		goto retry;
1557	}
1558	return NULL;
1559}
1560
1561#else /* !CONFIG_CONTIG_ALLOC */
1562static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1563					int nid, nodemask_t *nodemask)
1564{
1565	return NULL;
1566}
1567#endif /* CONFIG_CONTIG_ALLOC */
1568
1569#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1570static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1571					int nid, nodemask_t *nodemask)
1572{
1573	return NULL;
1574}
 
 
 
 
1575#endif
1576
 
 
 
 
 
 
 
 
1577/*
1578 * Remove hugetlb folio from lists.
1579 * If vmemmap exists for the folio, clear the hugetlb flag so that the
1580 * folio appears as just a compound page.  Otherwise, wait until after
1581 * allocating vmemmap to clear the flag.
 
 
1582 *
1583 * Must be called with hugetlb lock held.
1584 */
1585static void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1586							bool adjust_surplus)
 
1587{
1588	int nid = folio_nid(folio);
1589
1590	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
1591	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
1592
1593	lockdep_assert_held(&hugetlb_lock);
1594	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1595		return;
1596
1597	list_del(&folio->lru);
1598
1599	if (folio_test_hugetlb_freed(folio)) {
1600		folio_clear_hugetlb_freed(folio);
1601		h->free_huge_pages--;
1602		h->free_huge_pages_node[nid]--;
1603	}
1604	if (adjust_surplus) {
1605		h->surplus_huge_pages--;
1606		h->surplus_huge_pages_node[nid]--;
1607	}
1608
1609	/*
1610	 * We can only clear the hugetlb flag after allocating vmemmap
1611	 * pages.  Otherwise, someone (memory error handling) may try to write
1612	 * to tail struct pages.
1613	 */
1614	if (!folio_test_hugetlb_vmemmap_optimized(folio))
1615		__folio_clear_hugetlb(folio);
 
 
 
 
 
 
 
1616
1617	h->nr_huge_pages--;
1618	h->nr_huge_pages_node[nid]--;
1619}
1620
 
 
 
 
 
 
 
 
 
 
 
 
1621static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
1622			     bool adjust_surplus)
1623{
 
1624	int nid = folio_nid(folio);
1625
1626	VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
1627
1628	lockdep_assert_held(&hugetlb_lock);
1629
1630	INIT_LIST_HEAD(&folio->lru);
1631	h->nr_huge_pages++;
1632	h->nr_huge_pages_node[nid]++;
1633
1634	if (adjust_surplus) {
1635		h->surplus_huge_pages++;
1636		h->surplus_huge_pages_node[nid]++;
1637	}
1638
1639	__folio_set_hugetlb(folio);
1640	folio_change_private(folio, NULL);
1641	/*
1642	 * We have to set hugetlb_vmemmap_optimized again as above
1643	 * folio_change_private(folio, NULL) cleared it.
1644	 */
1645	folio_set_hugetlb_vmemmap_optimized(folio);
1646
1647	arch_clear_hugetlb_flags(folio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1648	enqueue_hugetlb_folio(h, folio);
1649}
1650
1651static void __update_and_free_hugetlb_folio(struct hstate *h,
1652						struct folio *folio)
1653{
1654	bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio);
1655
1656	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1657		return;
1658
1659	/*
1660	 * If we don't know which subpages are hwpoisoned, we can't free
1661	 * the hugepage, so it's leaked intentionally.
1662	 */
1663	if (folio_test_hugetlb_raw_hwp_unreliable(folio))
1664		return;
1665
1666	/*
1667	 * If folio is not vmemmap optimized (!clear_flag), then the folio
1668	 * is no longer identified as a hugetlb page.  hugetlb_vmemmap_restore_folio
1669	 * can only be passed hugetlb pages and will BUG otherwise.
1670	 */
1671	if (clear_flag && hugetlb_vmemmap_restore_folio(h, folio)) {
1672		spin_lock_irq(&hugetlb_lock);
1673		/*
1674		 * If we cannot allocate vmemmap pages, just refuse to free the
1675		 * page and put the page back on the hugetlb free list and treat
1676		 * as a surplus page.
1677		 */
1678		add_hugetlb_folio(h, folio, true);
1679		spin_unlock_irq(&hugetlb_lock);
1680		return;
1681	}
1682
1683	/*
 
 
 
 
 
 
 
1684	 * If vmemmap pages were allocated above, then we need to clear the
1685	 * hugetlb flag under the hugetlb lock.
1686	 */
1687	if (folio_test_hugetlb(folio)) {
1688		spin_lock_irq(&hugetlb_lock);
1689		__folio_clear_hugetlb(folio);
1690		spin_unlock_irq(&hugetlb_lock);
1691	}
1692
1693	/*
1694	 * Move PageHWPoison flag from head page to the raw error pages,
1695	 * which makes any healthy subpages reusable.
1696	 */
1697	if (unlikely(folio_test_hwpoison(folio)))
1698		folio_clear_hugetlb_hwpoison(folio);
1699
1700	folio_ref_unfreeze(folio, 1);
1701
1702	INIT_LIST_HEAD(&folio->_deferred_list);
1703	hugetlb_free_folio(folio);
1704}
1705
1706/*
1707 * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
1708 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
1709 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
1710 * the vmemmap pages.
1711 *
1712 * free_hpage_workfn() locklessly retrieves the linked list of pages to be
1713 * freed and frees them one-by-one. As the page->mapping pointer is going
1714 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
1715 * structure of a lockless linked list of huge pages to be freed.
1716 */
1717static LLIST_HEAD(hpage_freelist);
1718
1719static void free_hpage_workfn(struct work_struct *work)
1720{
1721	struct llist_node *node;
1722
1723	node = llist_del_all(&hpage_freelist);
1724
1725	while (node) {
1726		struct folio *folio;
1727		struct hstate *h;
1728
1729		folio = container_of((struct address_space **)node,
1730				     struct folio, mapping);
1731		node = node->next;
1732		folio->mapping = NULL;
1733		/*
1734		 * The VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio) in
1735		 * folio_hstate() is going to trigger because a previous call to
1736		 * remove_hugetlb_folio() will clear the hugetlb bit, so do
1737		 * not use folio_hstate() directly.
1738		 */
1739		h = size_to_hstate(folio_size(folio));
1740
1741		__update_and_free_hugetlb_folio(h, folio);
1742
1743		cond_resched();
1744	}
1745}
1746static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1747
1748static inline void flush_free_hpage_work(struct hstate *h)
1749{
1750	if (hugetlb_vmemmap_optimizable(h))
1751		flush_work(&free_hpage_work);
1752}
1753
1754static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
1755				 bool atomic)
1756{
1757	if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
1758		__update_and_free_hugetlb_folio(h, folio);
1759		return;
1760	}
1761
1762	/*
1763	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
1764	 *
1765	 * Only call schedule_work() if hpage_freelist is previously
1766	 * empty. Otherwise, schedule_work() had been called but the workfn
1767	 * hasn't retrieved the list yet.
1768	 */
1769	if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
1770		schedule_work(&free_hpage_work);
1771}
1772
1773static void bulk_vmemmap_restore_error(struct hstate *h,
1774					struct list_head *folio_list,
1775					struct list_head *non_hvo_folios)
1776{
1777	struct folio *folio, *t_folio;
1778
1779	if (!list_empty(non_hvo_folios)) {
1780		/*
1781		 * Free any restored hugetlb pages so that restore of the
1782		 * entire list can be retried.
1783		 * The idea is that in the common case of ENOMEM errors freeing
1784		 * hugetlb pages with vmemmap we will free up memory so that we
1785		 * can allocate vmemmap for more hugetlb pages.
1786		 */
1787		list_for_each_entry_safe(folio, t_folio, non_hvo_folios, lru) {
1788			list_del(&folio->lru);
1789			spin_lock_irq(&hugetlb_lock);
1790			__folio_clear_hugetlb(folio);
1791			spin_unlock_irq(&hugetlb_lock);
1792			update_and_free_hugetlb_folio(h, folio, false);
1793			cond_resched();
1794		}
1795	} else {
1796		/*
1797		 * In the case where there are no folios which can be
1798		 * immediately freed, we loop through the list trying to restore
1799		 * vmemmap individually in the hope that someone elsewhere may
1800		 * have done something to cause success (such as freeing some
1801		 * memory).  If unable to restore a hugetlb page, the hugetlb
1802		 * page is made a surplus page and removed from the list.
1803		 * If are able to restore vmemmap and free one hugetlb page, we
1804		 * quit processing the list to retry the bulk operation.
1805		 */
1806		list_for_each_entry_safe(folio, t_folio, folio_list, lru)
1807			if (hugetlb_vmemmap_restore_folio(h, folio)) {
1808				list_del(&folio->lru);
1809				spin_lock_irq(&hugetlb_lock);
1810				add_hugetlb_folio(h, folio, true);
1811				spin_unlock_irq(&hugetlb_lock);
1812			} else {
1813				list_del(&folio->lru);
1814				spin_lock_irq(&hugetlb_lock);
1815				__folio_clear_hugetlb(folio);
1816				spin_unlock_irq(&hugetlb_lock);
1817				update_and_free_hugetlb_folio(h, folio, false);
1818				cond_resched();
1819				break;
1820			}
1821	}
1822}
1823
1824static void update_and_free_pages_bulk(struct hstate *h,
1825						struct list_head *folio_list)
1826{
1827	long ret;
1828	struct folio *folio, *t_folio;
1829	LIST_HEAD(non_hvo_folios);
1830
1831	/*
1832	 * First allocate required vmemmmap (if necessary) for all folios.
1833	 * Carefully handle errors and free up any available hugetlb pages
1834	 * in an effort to make forward progress.
1835	 */
1836retry:
1837	ret = hugetlb_vmemmap_restore_folios(h, folio_list, &non_hvo_folios);
1838	if (ret < 0) {
1839		bulk_vmemmap_restore_error(h, folio_list, &non_hvo_folios);
1840		goto retry;
1841	}
1842
1843	/*
1844	 * At this point, list should be empty, ret should be >= 0 and there
1845	 * should only be pages on the non_hvo_folios list.
1846	 * Do note that the non_hvo_folios list could be empty.
1847	 * Without HVO enabled, ret will be 0 and there is no need to call
1848	 * __folio_clear_hugetlb as this was done previously.
1849	 */
1850	VM_WARN_ON(!list_empty(folio_list));
1851	VM_WARN_ON(ret < 0);
1852	if (!list_empty(&non_hvo_folios) && ret) {
1853		spin_lock_irq(&hugetlb_lock);
1854		list_for_each_entry(folio, &non_hvo_folios, lru)
1855			__folio_clear_hugetlb(folio);
1856		spin_unlock_irq(&hugetlb_lock);
1857	}
1858
1859	list_for_each_entry_safe(folio, t_folio, &non_hvo_folios, lru) {
1860		update_and_free_hugetlb_folio(h, folio, false);
1861		cond_resched();
1862	}
1863}
1864
1865struct hstate *size_to_hstate(unsigned long size)
1866{
1867	struct hstate *h;
1868
1869	for_each_hstate(h) {
1870		if (huge_page_size(h) == size)
1871			return h;
1872	}
1873	return NULL;
1874}
1875
1876void free_huge_folio(struct folio *folio)
1877{
1878	/*
1879	 * Can't pass hstate in here because it is called from the
1880	 * generic mm code.
1881	 */
1882	struct hstate *h = folio_hstate(folio);
1883	int nid = folio_nid(folio);
1884	struct hugepage_subpool *spool = hugetlb_folio_subpool(folio);
1885	bool restore_reserve;
1886	unsigned long flags;
1887
1888	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1889	VM_BUG_ON_FOLIO(folio_mapcount(folio), folio);
1890
1891	hugetlb_set_folio_subpool(folio, NULL);
1892	if (folio_test_anon(folio))
1893		__ClearPageAnonExclusive(&folio->page);
1894	folio->mapping = NULL;
1895	restore_reserve = folio_test_hugetlb_restore_reserve(folio);
1896	folio_clear_hugetlb_restore_reserve(folio);
1897
1898	/*
1899	 * If HPageRestoreReserve was set on page, page allocation consumed a
1900	 * reservation.  If the page was associated with a subpool, there
1901	 * would have been a page reserved in the subpool before allocation
1902	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1903	 * reservation, do not call hugepage_subpool_put_pages() as this will
1904	 * remove the reserved page from the subpool.
1905	 */
1906	if (!restore_reserve) {
1907		/*
1908		 * A return code of zero implies that the subpool will be
1909		 * under its minimum size if the reservation is not restored
1910		 * after page is free.  Therefore, force restore_reserve
1911		 * operation.
1912		 */
1913		if (hugepage_subpool_put_pages(spool, 1) == 0)
1914			restore_reserve = true;
1915	}
1916
1917	spin_lock_irqsave(&hugetlb_lock, flags);
1918	folio_clear_hugetlb_migratable(folio);
1919	hugetlb_cgroup_uncharge_folio(hstate_index(h),
1920				     pages_per_huge_page(h), folio);
1921	hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
1922					  pages_per_huge_page(h), folio);
1923	lruvec_stat_mod_folio(folio, NR_HUGETLB, -pages_per_huge_page(h));
1924	mem_cgroup_uncharge(folio);
1925	if (restore_reserve)
1926		h->resv_huge_pages++;
1927
1928	if (folio_test_hugetlb_temporary(folio)) {
1929		remove_hugetlb_folio(h, folio, false);
1930		spin_unlock_irqrestore(&hugetlb_lock, flags);
1931		update_and_free_hugetlb_folio(h, folio, true);
1932	} else if (h->surplus_huge_pages_node[nid]) {
1933		/* remove the page from active list */
1934		remove_hugetlb_folio(h, folio, true);
1935		spin_unlock_irqrestore(&hugetlb_lock, flags);
1936		update_and_free_hugetlb_folio(h, folio, true);
1937	} else {
1938		arch_clear_hugetlb_flags(folio);
1939		enqueue_hugetlb_folio(h, folio);
1940		spin_unlock_irqrestore(&hugetlb_lock, flags);
1941	}
1942}
1943
1944/*
1945 * Must be called with the hugetlb lock held
1946 */
1947static void __prep_account_new_huge_page(struct hstate *h, int nid)
1948{
1949	lockdep_assert_held(&hugetlb_lock);
1950	h->nr_huge_pages++;
1951	h->nr_huge_pages_node[nid]++;
1952}
1953
1954static void init_new_hugetlb_folio(struct hstate *h, struct folio *folio)
1955{
1956	__folio_set_hugetlb(folio);
1957	INIT_LIST_HEAD(&folio->lru);
1958	hugetlb_set_folio_subpool(folio, NULL);
1959	set_hugetlb_cgroup(folio, NULL);
1960	set_hugetlb_cgroup_rsvd(folio, NULL);
1961}
1962
1963static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
1964{
1965	init_new_hugetlb_folio(h, folio);
1966	hugetlb_vmemmap_optimize_folio(h, folio);
1967}
1968
1969static void prep_new_hugetlb_folio(struct hstate *h, struct folio *folio, int nid)
1970{
1971	__prep_new_hugetlb_folio(h, folio);
1972	spin_lock_irq(&hugetlb_lock);
1973	__prep_account_new_huge_page(h, nid);
1974	spin_unlock_irq(&hugetlb_lock);
1975}
1976
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1977/*
1978 * Find and lock address space (mapping) in write mode.
1979 *
1980 * Upon entry, the folio is locked which means that folio_mapping() is
1981 * stable.  Due to locking order, we can only trylock_write.  If we can
1982 * not get the lock, simply return NULL to caller.
1983 */
1984struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio)
1985{
1986	struct address_space *mapping = folio_mapping(folio);
1987
1988	if (!mapping)
1989		return mapping;
1990
1991	if (i_mmap_trylock_write(mapping))
1992		return mapping;
1993
1994	return NULL;
1995}
1996
1997static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
1998		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1999		nodemask_t *node_alloc_noretry)
2000{
2001	int order = huge_page_order(h);
2002	struct folio *folio;
2003	bool alloc_try_hard = true;
2004	bool retry = true;
2005
2006	/*
2007	 * By default we always try hard to allocate the folio with
2008	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating folios in
2009	 * a loop (to adjust global huge page counts) and previous allocation
2010	 * failed, do not continue to try hard on the same node.  Use the
2011	 * node_alloc_noretry bitmap to manage this state information.
2012	 */
2013	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
2014		alloc_try_hard = false;
 
2015	if (alloc_try_hard)
2016		gfp_mask |= __GFP_RETRY_MAYFAIL;
2017	if (nid == NUMA_NO_NODE)
2018		nid = numa_mem_id();
2019retry:
2020	folio = __folio_alloc(gfp_mask, order, nid, nmask);
2021	/* Ensure hugetlb folio won't have large_rmappable flag set. */
2022	if (folio)
2023		folio_clear_large_rmappable(folio);
2024
2025	if (folio && !folio_ref_freeze(folio, 1)) {
2026		folio_put(folio);
 
2027		if (retry) {	/* retry once */
2028			retry = false;
2029			goto retry;
2030		}
2031		/* WOW!  twice in a row. */
2032		pr_warn("HugeTLB unexpected inflated folio ref count\n");
2033		folio = NULL;
2034	}
2035
2036	/*
2037	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
2038	 * folio this indicates an overall state change.  Clear bit so
2039	 * that we resume normal 'try hard' allocations.
2040	 */
2041	if (node_alloc_noretry && folio && !alloc_try_hard)
2042		node_clear(nid, *node_alloc_noretry);
2043
2044	/*
2045	 * If we tried hard to get a folio but failed, set bit so that
2046	 * subsequent attempts will not try as hard until there is an
2047	 * overall state change.
2048	 */
2049	if (node_alloc_noretry && !folio && alloc_try_hard)
2050		node_set(nid, *node_alloc_noretry);
2051
2052	if (!folio) {
2053		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
2054		return NULL;
2055	}
2056
2057	__count_vm_event(HTLB_BUDDY_PGALLOC);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2058	return folio;
2059}
2060
2061static struct folio *only_alloc_fresh_hugetlb_folio(struct hstate *h,
2062		gfp_t gfp_mask, int nid, nodemask_t *nmask,
2063		nodemask_t *node_alloc_noretry)
2064{
2065	struct folio *folio;
2066
2067	if (hstate_is_gigantic(h))
2068		folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask);
2069	else
2070		folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid, nmask, node_alloc_noretry);
2071	if (folio)
2072		init_new_hugetlb_folio(h, folio);
2073	return folio;
2074}
2075
2076/*
2077 * Common helper to allocate a fresh hugetlb page. All specific allocators
2078 * should use this function to get new hugetlb pages
2079 *
2080 * Note that returned page is 'frozen':  ref count of head page and all tail
2081 * pages is zero.
2082 */
2083static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
2084		gfp_t gfp_mask, int nid, nodemask_t *nmask)
 
2085{
2086	struct folio *folio;
2087
2088	if (hstate_is_gigantic(h))
2089		folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask);
2090	else
2091		folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
2092	if (!folio)
2093		return NULL;
2094
2095	prep_new_hugetlb_folio(h, folio, folio_nid(folio));
2096	return folio;
2097}
2098
2099static void prep_and_add_allocated_folios(struct hstate *h,
2100					struct list_head *folio_list)
2101{
2102	unsigned long flags;
2103	struct folio *folio, *tmp_f;
2104
2105	/* Send list for bulk vmemmap optimization processing */
2106	hugetlb_vmemmap_optimize_folios(h, folio_list);
2107
2108	/* Add all new pool pages to free lists in one lock cycle */
2109	spin_lock_irqsave(&hugetlb_lock, flags);
2110	list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
2111		__prep_account_new_huge_page(h, folio_nid(folio));
2112		enqueue_hugetlb_folio(h, folio);
2113	}
2114	spin_unlock_irqrestore(&hugetlb_lock, flags);
2115}
2116
2117/*
2118 * Allocates a fresh hugetlb page in a node interleaved manner.  The page
2119 * will later be added to the appropriate hugetlb pool.
2120 */
2121static struct folio *alloc_pool_huge_folio(struct hstate *h,
2122					nodemask_t *nodes_allowed,
2123					nodemask_t *node_alloc_noretry,
2124					int *next_node)
2125{
2126	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2127	int nr_nodes, node;
2128
2129	for_each_node_mask_to_alloc(next_node, nr_nodes, node, nodes_allowed) {
2130		struct folio *folio;
2131
2132		folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, node,
2133					nodes_allowed, node_alloc_noretry);
2134		if (folio)
2135			return folio;
2136	}
2137
2138	return NULL;
2139}
2140
2141/*
2142 * Remove huge page from pool from next node to free.  Attempt to keep
2143 * persistent huge pages more or less balanced over allowed nodes.
2144 * This routine only 'removes' the hugetlb page.  The caller must make
2145 * an additional call to free the page to low level allocators.
2146 * Called with hugetlb_lock locked.
2147 */
2148static struct folio *remove_pool_hugetlb_folio(struct hstate *h,
2149		nodemask_t *nodes_allowed, bool acct_surplus)
2150{
2151	int nr_nodes, node;
2152	struct folio *folio = NULL;
2153
2154	lockdep_assert_held(&hugetlb_lock);
2155	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2156		/*
2157		 * If we're returning unused surplus pages, only examine
2158		 * nodes with surplus pages.
2159		 */
2160		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
2161		    !list_empty(&h->hugepage_freelists[node])) {
2162			folio = list_entry(h->hugepage_freelists[node].next,
2163					  struct folio, lru);
2164			remove_hugetlb_folio(h, folio, acct_surplus);
2165			break;
2166		}
2167	}
2168
2169	return folio;
2170}
2171
2172/*
2173 * Dissolve a given free hugetlb folio into free buddy pages. This function
2174 * does nothing for in-use hugetlb folios and non-hugetlb folios.
2175 * This function returns values like below:
2176 *
2177 *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
2178 *           when the system is under memory pressure and the feature of
2179 *           freeing unused vmemmap pages associated with each hugetlb page
2180 *           is enabled.
2181 *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
2182 *           (allocated or reserved.)
2183 *       0:  successfully dissolved free hugepages or the page is not a
2184 *           hugepage (considered as already dissolved)
2185 */
2186int dissolve_free_hugetlb_folio(struct folio *folio)
2187{
2188	int rc = -EBUSY;
 
2189
2190retry:
2191	/* Not to disrupt normal path by vainly holding hugetlb_lock */
2192	if (!folio_test_hugetlb(folio))
2193		return 0;
2194
2195	spin_lock_irq(&hugetlb_lock);
2196	if (!folio_test_hugetlb(folio)) {
2197		rc = 0;
2198		goto out;
2199	}
2200
2201	if (!folio_ref_count(folio)) {
2202		struct hstate *h = folio_hstate(folio);
2203		if (!available_huge_pages(h))
2204			goto out;
2205
2206		/*
2207		 * We should make sure that the page is already on the free list
2208		 * when it is dissolved.
2209		 */
2210		if (unlikely(!folio_test_hugetlb_freed(folio))) {
2211			spin_unlock_irq(&hugetlb_lock);
2212			cond_resched();
2213
2214			/*
2215			 * Theoretically, we should return -EBUSY when we
2216			 * encounter this race. In fact, we have a chance
2217			 * to successfully dissolve the page if we do a
2218			 * retry. Because the race window is quite small.
2219			 * If we seize this opportunity, it is an optimization
2220			 * for increasing the success rate of dissolving page.
2221			 */
2222			goto retry;
2223		}
2224
2225		remove_hugetlb_folio(h, folio, false);
2226		h->max_huge_pages--;
2227		spin_unlock_irq(&hugetlb_lock);
2228
2229		/*
2230		 * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
2231		 * before freeing the page.  update_and_free_hugtlb_folio will fail to
2232		 * free the page if it can not allocate required vmemmap.  We
2233		 * need to adjust max_huge_pages if the page is not freed.
2234		 * Attempt to allocate vmemmmap here so that we can take
2235		 * appropriate action on failure.
2236		 *
2237		 * The folio_test_hugetlb check here is because
2238		 * remove_hugetlb_folio will clear hugetlb folio flag for
2239		 * non-vmemmap optimized hugetlb folios.
2240		 */
2241		if (folio_test_hugetlb(folio)) {
2242			rc = hugetlb_vmemmap_restore_folio(h, folio);
2243			if (rc) {
2244				spin_lock_irq(&hugetlb_lock);
2245				add_hugetlb_folio(h, folio, false);
2246				h->max_huge_pages++;
2247				goto out;
2248			}
2249		} else
2250			rc = 0;
2251
2252		update_and_free_hugetlb_folio(h, folio, false);
2253		return rc;
2254	}
2255out:
2256	spin_unlock_irq(&hugetlb_lock);
2257	return rc;
2258}
2259
2260/*
2261 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
2262 * make specified memory blocks removable from the system.
2263 * Note that this will dissolve a free gigantic hugepage completely, if any
2264 * part of it lies within the given range.
2265 * Also note that if dissolve_free_hugetlb_folio() returns with an error, all
2266 * free hugetlb folios that were dissolved before that error are lost.
2267 */
2268int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn)
2269{
2270	unsigned long pfn;
2271	struct folio *folio;
2272	int rc = 0;
2273	unsigned int order;
2274	struct hstate *h;
2275
2276	if (!hugepages_supported())
2277		return rc;
2278
2279	order = huge_page_order(&default_hstate);
2280	for_each_hstate(h)
2281		order = min(order, huge_page_order(h));
2282
2283	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) {
2284		folio = pfn_folio(pfn);
2285		rc = dissolve_free_hugetlb_folio(folio);
2286		if (rc)
2287			break;
2288	}
2289
2290	return rc;
2291}
2292
2293/*
2294 * Allocates a fresh surplus page from the page allocator.
2295 */
2296static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h,
2297				gfp_t gfp_mask,	int nid, nodemask_t *nmask)
2298{
2299	struct folio *folio = NULL;
2300
2301	if (hstate_is_gigantic(h))
2302		return NULL;
2303
2304	spin_lock_irq(&hugetlb_lock);
2305	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
2306		goto out_unlock;
2307	spin_unlock_irq(&hugetlb_lock);
2308
2309	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask);
2310	if (!folio)
2311		return NULL;
2312
2313	spin_lock_irq(&hugetlb_lock);
2314	/*
2315	 * We could have raced with the pool size change.
2316	 * Double check that and simply deallocate the new page
2317	 * if we would end up overcommiting the surpluses. Abuse
2318	 * temporary page to workaround the nasty free_huge_folio
2319	 * codeflow
2320	 */
2321	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2322		folio_set_hugetlb_temporary(folio);
2323		spin_unlock_irq(&hugetlb_lock);
2324		free_huge_folio(folio);
2325		return NULL;
2326	}
2327
2328	h->surplus_huge_pages++;
2329	h->surplus_huge_pages_node[folio_nid(folio)]++;
2330
2331out_unlock:
2332	spin_unlock_irq(&hugetlb_lock);
2333
2334	return folio;
2335}
2336
2337static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
2338				     int nid, nodemask_t *nmask)
2339{
2340	struct folio *folio;
2341
2342	if (hstate_is_gigantic(h))
2343		return NULL;
2344
2345	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask);
2346	if (!folio)
2347		return NULL;
2348
2349	/* fresh huge pages are frozen */
2350	folio_ref_unfreeze(folio, 1);
2351	/*
2352	 * We do not account these pages as surplus because they are only
2353	 * temporary and will be released properly on the last reference
2354	 */
2355	folio_set_hugetlb_temporary(folio);
2356
2357	return folio;
2358}
2359
2360/*
2361 * Use the VMA's mpolicy to allocate a huge page from the buddy.
2362 */
2363static
2364struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h,
2365		struct vm_area_struct *vma, unsigned long addr)
2366{
2367	struct folio *folio = NULL;
2368	struct mempolicy *mpol;
2369	gfp_t gfp_mask = htlb_alloc_mask(h);
2370	int nid;
2371	nodemask_t *nodemask;
2372
2373	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2374	if (mpol_is_preferred_many(mpol)) {
2375		gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2376
 
2377		folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask);
2378
2379		/* Fallback to all nodes if page==NULL */
2380		nodemask = NULL;
2381	}
2382
2383	if (!folio)
2384		folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask);
2385	mpol_cond_put(mpol);
2386	return folio;
2387}
2388
2389struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
2390		nodemask_t *nmask, gfp_t gfp_mask)
2391{
2392	struct folio *folio;
2393
2394	spin_lock_irq(&hugetlb_lock);
2395	folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid,
2396					       nmask);
2397	if (folio) {
2398		VM_BUG_ON(!h->resv_huge_pages);
2399		h->resv_huge_pages--;
2400	}
2401
2402	spin_unlock_irq(&hugetlb_lock);
2403	return folio;
2404}
2405
2406/* folio migration callback function */
2407struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
2408		nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback)
2409{
2410	spin_lock_irq(&hugetlb_lock);
2411	if (available_huge_pages(h)) {
2412		struct folio *folio;
2413
2414		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
2415						preferred_nid, nmask);
2416		if (folio) {
2417			spin_unlock_irq(&hugetlb_lock);
2418			return folio;
2419		}
2420	}
2421	spin_unlock_irq(&hugetlb_lock);
2422
2423	/* We cannot fallback to other nodes, as we could break the per-node pool. */
2424	if (!allow_alloc_fallback)
2425		gfp_mask |= __GFP_THISNODE;
2426
2427	return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask);
2428}
2429
2430static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
2431{
2432#ifdef CONFIG_NUMA
2433	struct mempolicy *mpol = get_task_policy(current);
2434
2435	/*
2436	 * Only enforce MPOL_BIND policy which overlaps with cpuset policy
2437	 * (from policy_nodemask) specifically for hugetlb case
2438	 */
2439	if (mpol->mode == MPOL_BIND &&
2440		(apply_policy_zone(mpol, gfp_zone(gfp)) &&
2441		 cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
2442		return &mpol->nodes;
2443#endif
2444	return NULL;
2445}
2446
2447/*
2448 * Increase the hugetlb pool such that it can accommodate a reservation
2449 * of size 'delta'.
2450 */
2451static int gather_surplus_pages(struct hstate *h, long delta)
2452	__must_hold(&hugetlb_lock)
2453{
2454	LIST_HEAD(surplus_list);
2455	struct folio *folio, *tmp;
2456	int ret;
2457	long i;
2458	long needed, allocated;
2459	bool alloc_ok = true;
2460	int node;
2461	nodemask_t *mbind_nodemask = policy_mbind_nodemask(htlb_alloc_mask(h));
2462
2463	lockdep_assert_held(&hugetlb_lock);
2464	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2465	if (needed <= 0) {
2466		h->resv_huge_pages += delta;
2467		return 0;
2468	}
2469
2470	allocated = 0;
2471
2472	ret = -ENOMEM;
2473retry:
2474	spin_unlock_irq(&hugetlb_lock);
2475	for (i = 0; i < needed; i++) {
2476		folio = NULL;
2477		for_each_node_mask(node, cpuset_current_mems_allowed) {
2478			if (!mbind_nodemask || node_isset(node, *mbind_nodemask)) {
2479				folio = alloc_surplus_hugetlb_folio(h, htlb_alloc_mask(h),
2480						node, NULL);
2481				if (folio)
2482					break;
2483			}
2484		}
2485		if (!folio) {
2486			alloc_ok = false;
2487			break;
2488		}
2489		list_add(&folio->lru, &surplus_list);
2490		cond_resched();
2491	}
2492	allocated += i;
2493
2494	/*
2495	 * After retaking hugetlb_lock, we need to recalculate 'needed'
2496	 * because either resv_huge_pages or free_huge_pages may have changed.
2497	 */
2498	spin_lock_irq(&hugetlb_lock);
2499	needed = (h->resv_huge_pages + delta) -
2500			(h->free_huge_pages + allocated);
2501	if (needed > 0) {
2502		if (alloc_ok)
2503			goto retry;
2504		/*
2505		 * We were not able to allocate enough pages to
2506		 * satisfy the entire reservation so we free what
2507		 * we've allocated so far.
2508		 */
2509		goto free;
2510	}
2511	/*
2512	 * The surplus_list now contains _at_least_ the number of extra pages
2513	 * needed to accommodate the reservation.  Add the appropriate number
2514	 * of pages to the hugetlb pool and free the extras back to the buddy
2515	 * allocator.  Commit the entire reservation here to prevent another
2516	 * process from stealing the pages as they are added to the pool but
2517	 * before they are reserved.
2518	 */
2519	needed += allocated;
2520	h->resv_huge_pages += delta;
2521	ret = 0;
2522
2523	/* Free the needed pages to the hugetlb pool */
2524	list_for_each_entry_safe(folio, tmp, &surplus_list, lru) {
2525		if ((--needed) < 0)
2526			break;
2527		/* Add the page to the hugetlb allocator */
2528		enqueue_hugetlb_folio(h, folio);
2529	}
2530free:
2531	spin_unlock_irq(&hugetlb_lock);
2532
2533	/*
2534	 * Free unnecessary surplus pages to the buddy allocator.
2535	 * Pages have no ref count, call free_huge_folio directly.
2536	 */
2537	list_for_each_entry_safe(folio, tmp, &surplus_list, lru)
2538		free_huge_folio(folio);
2539	spin_lock_irq(&hugetlb_lock);
2540
2541	return ret;
2542}
2543
2544/*
2545 * This routine has two main purposes:
2546 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2547 *    in unused_resv_pages.  This corresponds to the prior adjustments made
2548 *    to the associated reservation map.
2549 * 2) Free any unused surplus pages that may have been allocated to satisfy
2550 *    the reservation.  As many as unused_resv_pages may be freed.
2551 */
2552static void return_unused_surplus_pages(struct hstate *h,
2553					unsigned long unused_resv_pages)
2554{
2555	unsigned long nr_pages;
2556	LIST_HEAD(page_list);
2557
2558	lockdep_assert_held(&hugetlb_lock);
2559	/* Uncommit the reservation */
2560	h->resv_huge_pages -= unused_resv_pages;
2561
2562	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
2563		goto out;
2564
2565	/*
2566	 * Part (or even all) of the reservation could have been backed
2567	 * by pre-allocated pages. Only free surplus pages.
2568	 */
2569	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2570
2571	/*
2572	 * We want to release as many surplus pages as possible, spread
2573	 * evenly across all nodes with memory. Iterate across these nodes
2574	 * until we can no longer free unreserved surplus pages. This occurs
2575	 * when the nodes with surplus pages have no free pages.
2576	 * remove_pool_hugetlb_folio() will balance the freed pages across the
2577	 * on-line nodes with memory and will handle the hstate accounting.
2578	 */
2579	while (nr_pages--) {
2580		struct folio *folio;
2581
2582		folio = remove_pool_hugetlb_folio(h, &node_states[N_MEMORY], 1);
2583		if (!folio)
2584			goto out;
2585
2586		list_add(&folio->lru, &page_list);
2587	}
2588
2589out:
2590	spin_unlock_irq(&hugetlb_lock);
2591	update_and_free_pages_bulk(h, &page_list);
2592	spin_lock_irq(&hugetlb_lock);
2593}
2594
2595
2596/*
2597 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2598 * are used by the huge page allocation routines to manage reservations.
2599 *
2600 * vma_needs_reservation is called to determine if the huge page at addr
2601 * within the vma has an associated reservation.  If a reservation is
2602 * needed, the value 1 is returned.  The caller is then responsible for
2603 * managing the global reservation and subpool usage counts.  After
2604 * the huge page has been allocated, vma_commit_reservation is called
2605 * to add the page to the reservation map.  If the page allocation fails,
2606 * the reservation must be ended instead of committed.  vma_end_reservation
2607 * is called in such cases.
2608 *
2609 * In the normal case, vma_commit_reservation returns the same value
2610 * as the preceding vma_needs_reservation call.  The only time this
2611 * is not the case is if a reserve map was changed between calls.  It
2612 * is the responsibility of the caller to notice the difference and
2613 * take appropriate action.
2614 *
2615 * vma_add_reservation is used in error paths where a reservation must
2616 * be restored when a newly allocated huge page must be freed.  It is
2617 * to be called after calling vma_needs_reservation to determine if a
2618 * reservation exists.
2619 *
2620 * vma_del_reservation is used in error paths where an entry in the reserve
2621 * map was created during huge page allocation and must be removed.  It is to
2622 * be called after calling vma_needs_reservation to determine if a reservation
2623 * exists.
2624 */
2625enum vma_resv_mode {
2626	VMA_NEEDS_RESV,
2627	VMA_COMMIT_RESV,
2628	VMA_END_RESV,
2629	VMA_ADD_RESV,
2630	VMA_DEL_RESV,
2631};
2632static long __vma_reservation_common(struct hstate *h,
2633				struct vm_area_struct *vma, unsigned long addr,
2634				enum vma_resv_mode mode)
2635{
2636	struct resv_map *resv;
2637	pgoff_t idx;
2638	long ret;
2639	long dummy_out_regions_needed;
2640
2641	resv = vma_resv_map(vma);
2642	if (!resv)
2643		return 1;
2644
2645	idx = vma_hugecache_offset(h, vma, addr);
2646	switch (mode) {
2647	case VMA_NEEDS_RESV:
2648		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2649		/* We assume that vma_reservation_* routines always operate on
2650		 * 1 page, and that adding to resv map a 1 page entry can only
2651		 * ever require 1 region.
2652		 */
2653		VM_BUG_ON(dummy_out_regions_needed != 1);
2654		break;
2655	case VMA_COMMIT_RESV:
2656		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2657		/* region_add calls of range 1 should never fail. */
2658		VM_BUG_ON(ret < 0);
2659		break;
2660	case VMA_END_RESV:
2661		region_abort(resv, idx, idx + 1, 1);
2662		ret = 0;
2663		break;
2664	case VMA_ADD_RESV:
2665		if (vma->vm_flags & VM_MAYSHARE) {
2666			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2667			/* region_add calls of range 1 should never fail. */
2668			VM_BUG_ON(ret < 0);
2669		} else {
2670			region_abort(resv, idx, idx + 1, 1);
2671			ret = region_del(resv, idx, idx + 1);
2672		}
2673		break;
2674	case VMA_DEL_RESV:
2675		if (vma->vm_flags & VM_MAYSHARE) {
2676			region_abort(resv, idx, idx + 1, 1);
2677			ret = region_del(resv, idx, idx + 1);
2678		} else {
2679			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2680			/* region_add calls of range 1 should never fail. */
2681			VM_BUG_ON(ret < 0);
2682		}
2683		break;
2684	default:
2685		BUG();
2686	}
2687
2688	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2689		return ret;
2690	/*
2691	 * We know private mapping must have HPAGE_RESV_OWNER set.
2692	 *
2693	 * In most cases, reserves always exist for private mappings.
2694	 * However, a file associated with mapping could have been
2695	 * hole punched or truncated after reserves were consumed.
2696	 * As subsequent fault on such a range will not use reserves.
2697	 * Subtle - The reserve map for private mappings has the
2698	 * opposite meaning than that of shared mappings.  If NO
2699	 * entry is in the reserve map, it means a reservation exists.
2700	 * If an entry exists in the reserve map, it means the
2701	 * reservation has already been consumed.  As a result, the
2702	 * return value of this routine is the opposite of the
2703	 * value returned from reserve map manipulation routines above.
2704	 */
2705	if (ret > 0)
2706		return 0;
2707	if (ret == 0)
2708		return 1;
2709	return ret;
2710}
2711
2712static long vma_needs_reservation(struct hstate *h,
2713			struct vm_area_struct *vma, unsigned long addr)
2714{
2715	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2716}
2717
2718static long vma_commit_reservation(struct hstate *h,
2719			struct vm_area_struct *vma, unsigned long addr)
2720{
2721	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2722}
2723
2724static void vma_end_reservation(struct hstate *h,
2725			struct vm_area_struct *vma, unsigned long addr)
2726{
2727	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2728}
2729
2730static long vma_add_reservation(struct hstate *h,
2731			struct vm_area_struct *vma, unsigned long addr)
2732{
2733	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2734}
2735
2736static long vma_del_reservation(struct hstate *h,
2737			struct vm_area_struct *vma, unsigned long addr)
2738{
2739	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
2740}
2741
2742/*
2743 * This routine is called to restore reservation information on error paths.
2744 * It should ONLY be called for folios allocated via alloc_hugetlb_folio(),
2745 * and the hugetlb mutex should remain held when calling this routine.
2746 *
2747 * It handles two specific cases:
2748 * 1) A reservation was in place and the folio consumed the reservation.
2749 *    hugetlb_restore_reserve is set in the folio.
2750 * 2) No reservation was in place for the page, so hugetlb_restore_reserve is
2751 *    not set.  However, alloc_hugetlb_folio always updates the reserve map.
2752 *
2753 * In case 1, free_huge_folio later in the error path will increment the
2754 * global reserve count.  But, free_huge_folio does not have enough context
2755 * to adjust the reservation map.  This case deals primarily with private
2756 * mappings.  Adjust the reserve map here to be consistent with global
2757 * reserve count adjustments to be made by free_huge_folio.  Make sure the
2758 * reserve map indicates there is a reservation present.
2759 *
2760 * In case 2, simply undo reserve map modifications done by alloc_hugetlb_folio.
2761 */
2762void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2763			unsigned long address, struct folio *folio)
2764{
2765	long rc = vma_needs_reservation(h, vma, address);
2766
2767	if (folio_test_hugetlb_restore_reserve(folio)) {
2768		if (unlikely(rc < 0))
2769			/*
2770			 * Rare out of memory condition in reserve map
2771			 * manipulation.  Clear hugetlb_restore_reserve so
2772			 * that global reserve count will not be incremented
2773			 * by free_huge_folio.  This will make it appear
2774			 * as though the reservation for this folio was
2775			 * consumed.  This may prevent the task from
2776			 * faulting in the folio at a later time.  This
2777			 * is better than inconsistent global huge page
2778			 * accounting of reserve counts.
2779			 */
2780			folio_clear_hugetlb_restore_reserve(folio);
2781		else if (rc)
2782			(void)vma_add_reservation(h, vma, address);
2783		else
2784			vma_end_reservation(h, vma, address);
2785	} else {
2786		if (!rc) {
2787			/*
2788			 * This indicates there is an entry in the reserve map
2789			 * not added by alloc_hugetlb_folio.  We know it was added
2790			 * before the alloc_hugetlb_folio call, otherwise
2791			 * hugetlb_restore_reserve would be set on the folio.
2792			 * Remove the entry so that a subsequent allocation
2793			 * does not consume a reservation.
2794			 */
2795			rc = vma_del_reservation(h, vma, address);
2796			if (rc < 0)
2797				/*
2798				 * VERY rare out of memory condition.  Since
2799				 * we can not delete the entry, set
2800				 * hugetlb_restore_reserve so that the reserve
2801				 * count will be incremented when the folio
2802				 * is freed.  This reserve will be consumed
2803				 * on a subsequent allocation.
2804				 */
2805				folio_set_hugetlb_restore_reserve(folio);
2806		} else if (rc < 0) {
2807			/*
2808			 * Rare out of memory condition from
2809			 * vma_needs_reservation call.  Memory allocation is
2810			 * only attempted if a new entry is needed.  Therefore,
2811			 * this implies there is not an entry in the
2812			 * reserve map.
2813			 *
2814			 * For shared mappings, no entry in the map indicates
2815			 * no reservation.  We are done.
2816			 */
2817			if (!(vma->vm_flags & VM_MAYSHARE))
2818				/*
2819				 * For private mappings, no entry indicates
2820				 * a reservation is present.  Since we can
2821				 * not add an entry, set hugetlb_restore_reserve
2822				 * on the folio so reserve count will be
2823				 * incremented when freed.  This reserve will
2824				 * be consumed on a subsequent allocation.
2825				 */
2826				folio_set_hugetlb_restore_reserve(folio);
2827		} else
2828			/*
2829			 * No reservation present, do nothing
2830			 */
2831			 vma_end_reservation(h, vma, address);
2832	}
2833}
2834
2835/*
2836 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
2837 * the old one
2838 * @h: struct hstate old page belongs to
2839 * @old_folio: Old folio to dissolve
2840 * @list: List to isolate the page in case we need to
2841 * Returns 0 on success, otherwise negated error.
2842 */
2843static int alloc_and_dissolve_hugetlb_folio(struct hstate *h,
2844			struct folio *old_folio, struct list_head *list)
2845{
2846	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2847	int nid = folio_nid(old_folio);
2848	struct folio *new_folio = NULL;
2849	int ret = 0;
2850
 
 
 
 
 
 
 
 
 
 
 
 
2851retry:
2852	spin_lock_irq(&hugetlb_lock);
2853	if (!folio_test_hugetlb(old_folio)) {
2854		/*
2855		 * Freed from under us. Drop new_folio too.
2856		 */
2857		goto free_new;
2858	} else if (folio_ref_count(old_folio)) {
2859		bool isolated;
2860
2861		/*
2862		 * Someone has grabbed the folio, try to isolate it here.
2863		 * Fail with -EBUSY if not possible.
2864		 */
2865		spin_unlock_irq(&hugetlb_lock);
2866		isolated = isolate_hugetlb(old_folio, list);
2867		ret = isolated ? 0 : -EBUSY;
2868		spin_lock_irq(&hugetlb_lock);
2869		goto free_new;
2870	} else if (!folio_test_hugetlb_freed(old_folio)) {
2871		/*
2872		 * Folio's refcount is 0 but it has not been enqueued in the
2873		 * freelist yet. Race window is small, so we can succeed here if
2874		 * we retry.
2875		 */
2876		spin_unlock_irq(&hugetlb_lock);
2877		cond_resched();
2878		goto retry;
2879	} else {
2880		if (!new_folio) {
2881			spin_unlock_irq(&hugetlb_lock);
2882			new_folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid,
2883							      NULL, NULL);
2884			if (!new_folio)
2885				return -ENOMEM;
2886			__prep_new_hugetlb_folio(h, new_folio);
2887			goto retry;
2888		}
2889
2890		/*
2891		 * Ok, old_folio is still a genuine free hugepage. Remove it from
2892		 * the freelist and decrease the counters. These will be
2893		 * incremented again when calling __prep_account_new_huge_page()
2894		 * and enqueue_hugetlb_folio() for new_folio. The counters will
2895		 * remain stable since this happens under the lock.
2896		 */
2897		remove_hugetlb_folio(h, old_folio, false);
2898
2899		/*
2900		 * Ref count on new_folio is already zero as it was dropped
2901		 * earlier.  It can be directly added to the pool free list.
2902		 */
2903		__prep_account_new_huge_page(h, nid);
2904		enqueue_hugetlb_folio(h, new_folio);
2905
2906		/*
2907		 * Folio has been replaced, we can safely free the old one.
2908		 */
2909		spin_unlock_irq(&hugetlb_lock);
2910		update_and_free_hugetlb_folio(h, old_folio, false);
2911	}
2912
2913	return ret;
2914
2915free_new:
2916	spin_unlock_irq(&hugetlb_lock);
2917	if (new_folio)
2918		update_and_free_hugetlb_folio(h, new_folio, false);
 
2919
2920	return ret;
2921}
2922
2923int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
2924{
2925	struct hstate *h;
2926	struct folio *folio = page_folio(page);
2927	int ret = -EBUSY;
2928
2929	/*
2930	 * The page might have been dissolved from under our feet, so make sure
2931	 * to carefully check the state under the lock.
2932	 * Return success when racing as if we dissolved the page ourselves.
2933	 */
2934	spin_lock_irq(&hugetlb_lock);
2935	if (folio_test_hugetlb(folio)) {
2936		h = folio_hstate(folio);
2937	} else {
2938		spin_unlock_irq(&hugetlb_lock);
2939		return 0;
2940	}
2941	spin_unlock_irq(&hugetlb_lock);
2942
2943	/*
2944	 * Fence off gigantic pages as there is a cyclic dependency between
2945	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
2946	 * of bailing out right away without further retrying.
2947	 */
2948	if (hstate_is_gigantic(h))
2949		return -ENOMEM;
2950
2951	if (folio_ref_count(folio) && isolate_hugetlb(folio, list))
2952		ret = 0;
2953	else if (!folio_ref_count(folio))
2954		ret = alloc_and_dissolve_hugetlb_folio(h, folio, list);
2955
2956	return ret;
2957}
2958
2959struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
2960				    unsigned long addr, int avoid_reserve)
2961{
2962	struct hugepage_subpool *spool = subpool_vma(vma);
2963	struct hstate *h = hstate_vma(vma);
2964	struct folio *folio;
2965	long map_chg, map_commit, nr_pages = pages_per_huge_page(h);
2966	long gbl_chg;
2967	int memcg_charge_ret, ret, idx;
2968	struct hugetlb_cgroup *h_cg = NULL;
2969	struct mem_cgroup *memcg;
2970	bool deferred_reserve;
2971	gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL;
2972
2973	memcg = get_mem_cgroup_from_current();
2974	memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages);
2975	if (memcg_charge_ret == -ENOMEM) {
2976		mem_cgroup_put(memcg);
2977		return ERR_PTR(-ENOMEM);
2978	}
2979
2980	idx = hstate_index(h);
2981	/*
2982	 * Examine the region/reserve map to determine if the process
2983	 * has a reservation for the page to be allocated.  A return
2984	 * code of zero indicates a reservation exists (no change).
2985	 */
2986	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2987	if (map_chg < 0) {
2988		if (!memcg_charge_ret)
2989			mem_cgroup_cancel_charge(memcg, nr_pages);
2990		mem_cgroup_put(memcg);
2991		return ERR_PTR(-ENOMEM);
2992	}
2993
2994	/*
2995	 * Processes that did not create the mapping will have no
2996	 * reserves as indicated by the region/reserve map. Check
2997	 * that the allocation will not exceed the subpool limit.
2998	 * Allocations for MAP_NORESERVE mappings also need to be
2999	 * checked against any subpool limit.
3000	 */
3001	if (map_chg || avoid_reserve) {
3002		gbl_chg = hugepage_subpool_get_pages(spool, 1);
3003		if (gbl_chg < 0)
3004			goto out_end_reservation;
 
 
 
 
 
 
 
 
 
 
 
3005	}
3006
3007	/* If this allocation is not consuming a reservation, charge it now.
3008	 */
3009	deferred_reserve = map_chg || avoid_reserve;
3010	if (deferred_reserve) {
3011		ret = hugetlb_cgroup_charge_cgroup_rsvd(
3012			idx, pages_per_huge_page(h), &h_cg);
3013		if (ret)
3014			goto out_subpool_put;
3015	}
3016
3017	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
3018	if (ret)
3019		goto out_uncharge_cgroup_reservation;
3020
3021	spin_lock_irq(&hugetlb_lock);
3022	/*
3023	 * glb_chg is passed to indicate whether or not a page must be taken
3024	 * from the global free pool (global change).  gbl_chg == 0 indicates
3025	 * a reservation exists for the allocation.
3026	 */
3027	folio = dequeue_hugetlb_folio_vma(h, vma, addr, gbl_chg);
3028	if (!folio) {
3029		spin_unlock_irq(&hugetlb_lock);
3030		folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr);
3031		if (!folio)
3032			goto out_uncharge_cgroup;
3033		spin_lock_irq(&hugetlb_lock);
3034		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
3035			folio_set_hugetlb_restore_reserve(folio);
3036			h->resv_huge_pages--;
3037		}
3038		list_add(&folio->lru, &h->hugepage_activelist);
3039		folio_ref_unfreeze(folio, 1);
3040		/* Fall through */
3041	}
3042
3043	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio);
3044	/* If allocation is not consuming a reservation, also store the
3045	 * hugetlb_cgroup pointer on the page.
3046	 */
3047	if (deferred_reserve) {
3048		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
3049						  h_cg, folio);
3050	}
3051
3052	spin_unlock_irq(&hugetlb_lock);
3053
3054	hugetlb_set_folio_subpool(folio, spool);
3055
3056	map_commit = vma_commit_reservation(h, vma, addr);
3057	if (unlikely(map_chg > map_commit)) {
3058		/*
3059		 * The page was added to the reservation map between
3060		 * vma_needs_reservation and vma_commit_reservation.
3061		 * This indicates a race with hugetlb_reserve_pages.
3062		 * Adjust for the subpool count incremented above AND
3063		 * in hugetlb_reserve_pages for the same page.  Also,
3064		 * the reservation count added in hugetlb_reserve_pages
3065		 * no longer applies.
3066		 */
3067		long rsv_adjust;
3068
3069		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
3070		hugetlb_acct_memory(h, -rsv_adjust);
3071		if (deferred_reserve) {
3072			spin_lock_irq(&hugetlb_lock);
3073			hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
3074					pages_per_huge_page(h), folio);
3075			spin_unlock_irq(&hugetlb_lock);
3076		}
3077	}
3078
3079	if (!memcg_charge_ret)
3080		mem_cgroup_commit_charge(folio, memcg);
3081	lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h));
3082	mem_cgroup_put(memcg);
3083
3084	return folio;
3085
3086out_uncharge_cgroup:
3087	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
3088out_uncharge_cgroup_reservation:
3089	if (deferred_reserve)
3090		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
3091						    h_cg);
3092out_subpool_put:
3093	if (map_chg || avoid_reserve)
3094		hugepage_subpool_put_pages(spool, 1);
3095out_end_reservation:
3096	vma_end_reservation(h, vma, addr);
3097	if (!memcg_charge_ret)
3098		mem_cgroup_cancel_charge(memcg, nr_pages);
3099	mem_cgroup_put(memcg);
3100	return ERR_PTR(-ENOSPC);
3101}
3102
3103int alloc_bootmem_huge_page(struct hstate *h, int nid)
3104	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
3105int __alloc_bootmem_huge_page(struct hstate *h, int nid)
3106{
3107	struct huge_bootmem_page *m = NULL; /* initialize for clang */
3108	int nr_nodes, node = nid;
3109
3110	/* do node specific alloc */
3111	if (nid != NUMA_NO_NODE) {
3112		m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
3113				0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
3114		if (!m)
3115			return 0;
3116		goto found;
3117	}
3118	/* allocate from next node when distributing huge pages */
3119	for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, &node_states[N_MEMORY]) {
3120		m = memblock_alloc_try_nid_raw(
3121				huge_page_size(h), huge_page_size(h),
3122				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3123		/*
3124		 * Use the beginning of the huge page to store the
3125		 * huge_bootmem_page struct (until gather_bootmem
3126		 * puts them into the mem_map).
3127		 */
3128		if (!m)
3129			return 0;
3130		goto found;
3131	}
3132
3133found:
3134
3135	/*
3136	 * Only initialize the head struct page in memmap_init_reserved_pages,
3137	 * rest of the struct pages will be initialized by the HugeTLB
3138	 * subsystem itself.
3139	 * The head struct page is used to get folio information by the HugeTLB
3140	 * subsystem like zone id and node id.
3141	 */
3142	memblock_reserved_mark_noinit(virt_to_phys((void *)m + PAGE_SIZE),
3143		huge_page_size(h) - PAGE_SIZE);
3144	/* Put them into a private list first because mem_map is not up yet */
3145	INIT_LIST_HEAD(&m->list);
3146	list_add(&m->list, &huge_boot_pages[node]);
3147	m->hstate = h;
3148	return 1;
3149}
3150
3151/* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
3152static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
3153					unsigned long start_page_number,
3154					unsigned long end_page_number)
3155{
3156	enum zone_type zone = zone_idx(folio_zone(folio));
3157	int nid = folio_nid(folio);
3158	unsigned long head_pfn = folio_pfn(folio);
3159	unsigned long pfn, end_pfn = head_pfn + end_page_number;
3160	int ret;
3161
3162	for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {
3163		struct page *page = pfn_to_page(pfn);
3164
3165		__ClearPageReserved(folio_page(folio, pfn - head_pfn));
3166		__init_single_page(page, pfn, zone, nid);
3167		prep_compound_tail((struct page *)folio, pfn - head_pfn);
3168		ret = page_ref_freeze(page, 1);
3169		VM_BUG_ON(!ret);
3170	}
3171}
3172
3173static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
3174					      struct hstate *h,
3175					      unsigned long nr_pages)
3176{
3177	int ret;
3178
3179	/* Prepare folio head */
3180	__folio_clear_reserved(folio);
3181	__folio_set_head(folio);
3182	ret = folio_ref_freeze(folio, 1);
3183	VM_BUG_ON(!ret);
3184	/* Initialize the necessary tail struct pages */
3185	hugetlb_folio_init_tail_vmemmap(folio, 1, nr_pages);
3186	prep_compound_head((struct page *)folio, huge_page_order(h));
3187}
3188
3189static void __init prep_and_add_bootmem_folios(struct hstate *h,
3190					struct list_head *folio_list)
3191{
3192	unsigned long flags;
3193	struct folio *folio, *tmp_f;
3194
3195	/* Send list for bulk vmemmap optimization processing */
3196	hugetlb_vmemmap_optimize_folios(h, folio_list);
3197
 
 
3198	list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
3199		if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
3200			/*
3201			 * If HVO fails, initialize all tail struct pages
3202			 * We do not worry about potential long lock hold
3203			 * time as this is early in boot and there should
3204			 * be no contention.
3205			 */
3206			hugetlb_folio_init_tail_vmemmap(folio,
3207					HUGETLB_VMEMMAP_RESERVE_PAGES,
3208					pages_per_huge_page(h));
3209		}
3210		/* Subdivide locks to achieve better parallel performance */
3211		spin_lock_irqsave(&hugetlb_lock, flags);
3212		__prep_account_new_huge_page(h, folio_nid(folio));
3213		enqueue_hugetlb_folio(h, folio);
3214		spin_unlock_irqrestore(&hugetlb_lock, flags);
3215	}
 
3216}
3217
3218/*
3219 * Put bootmem huge pages into the standard lists after mem_map is up.
3220 * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
3221 */
3222static void __init gather_bootmem_prealloc_node(unsigned long nid)
3223{
3224	LIST_HEAD(folio_list);
3225	struct huge_bootmem_page *m;
3226	struct hstate *h = NULL, *prev_h = NULL;
3227
3228	list_for_each_entry(m, &huge_boot_pages[nid], list) {
3229		struct page *page = virt_to_page(m);
3230		struct folio *folio = (void *)page;
3231
3232		h = m->hstate;
3233		/*
3234		 * It is possible to have multiple huge page sizes (hstates)
3235		 * in this list.  If so, process each size separately.
3236		 */
3237		if (h != prev_h && prev_h != NULL)
3238			prep_and_add_bootmem_folios(prev_h, &folio_list);
3239		prev_h = h;
3240
3241		VM_BUG_ON(!hstate_is_gigantic(h));
3242		WARN_ON(folio_ref_count(folio) != 1);
3243
3244		hugetlb_folio_init_vmemmap(folio, h,
3245					   HUGETLB_VMEMMAP_RESERVE_PAGES);
3246		init_new_hugetlb_folio(h, folio);
3247		list_add(&folio->lru, &folio_list);
3248
3249		/*
3250		 * We need to restore the 'stolen' pages to totalram_pages
3251		 * in order to fix confusing memory reports from free(1) and
3252		 * other side-effects, like CommitLimit going negative.
3253		 */
3254		adjust_managed_page_count(page, pages_per_huge_page(h));
3255		cond_resched();
3256	}
3257
3258	prep_and_add_bootmem_folios(h, &folio_list);
3259}
3260
3261static void __init gather_bootmem_prealloc_parallel(unsigned long start,
3262						    unsigned long end, void *arg)
3263{
3264	int nid;
3265
3266	for (nid = start; nid < end; nid++)
3267		gather_bootmem_prealloc_node(nid);
3268}
3269
3270static void __init gather_bootmem_prealloc(void)
3271{
3272	struct padata_mt_job job = {
3273		.thread_fn	= gather_bootmem_prealloc_parallel,
3274		.fn_arg		= NULL,
3275		.start		= 0,
3276		.size		= nr_node_ids,
3277		.align		= 1,
3278		.min_chunk	= 1,
3279		.max_threads	= num_node_state(N_MEMORY),
3280		.numa_aware	= true,
3281	};
3282
3283	padata_do_multithreaded(&job);
3284}
3285
3286static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
3287{
3288	unsigned long i;
3289	char buf[32];
3290	LIST_HEAD(folio_list);
3291
3292	for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
3293		if (hstate_is_gigantic(h)) {
3294			if (!alloc_bootmem_huge_page(h, nid))
3295				break;
3296		} else {
3297			struct folio *folio;
3298			gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
3299
3300			folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
3301					&node_states[N_MEMORY], NULL);
3302			if (!folio)
3303				break;
3304			list_add(&folio->lru, &folio_list);
3305		}
3306		cond_resched();
3307	}
3308
3309	if (!list_empty(&folio_list))
3310		prep_and_add_allocated_folios(h, &folio_list);
3311
3312	if (i == h->max_huge_pages_node[nid])
3313		return;
3314
3315	string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3316	pr_warn("HugeTLB: allocating %u of page size %s failed node%d.  Only allocated %lu hugepages.\n",
3317		h->max_huge_pages_node[nid], buf, nid, i);
3318	h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
3319	h->max_huge_pages_node[nid] = i;
3320}
3321
3322static bool __init hugetlb_hstate_alloc_pages_specific_nodes(struct hstate *h)
3323{
3324	int i;
3325	bool node_specific_alloc = false;
3326
3327	for_each_online_node(i) {
3328		if (h->max_huge_pages_node[i] > 0) {
3329			hugetlb_hstate_alloc_pages_onenode(h, i);
3330			node_specific_alloc = true;
3331		}
3332	}
3333
3334	return node_specific_alloc;
3335}
3336
3337static void __init hugetlb_hstate_alloc_pages_errcheck(unsigned long allocated, struct hstate *h)
3338{
3339	if (allocated < h->max_huge_pages) {
3340		char buf[32];
3341
3342		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3343		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
3344			h->max_huge_pages, buf, allocated);
3345		h->max_huge_pages = allocated;
3346	}
3347}
3348
3349static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned long end, void *arg)
3350{
3351	struct hstate *h = (struct hstate *)arg;
3352	int i, num = end - start;
3353	nodemask_t node_alloc_noretry;
3354	LIST_HEAD(folio_list);
3355	int next_node = first_online_node;
3356
3357	/* Bit mask controlling how hard we retry per-node allocations.*/
3358	nodes_clear(node_alloc_noretry);
3359
3360	for (i = 0; i < num; ++i) {
3361		struct folio *folio = alloc_pool_huge_folio(h, &node_states[N_MEMORY],
3362						&node_alloc_noretry, &next_node);
3363		if (!folio)
3364			break;
3365
3366		list_move(&folio->lru, &folio_list);
3367		cond_resched();
3368	}
3369
3370	prep_and_add_allocated_folios(h, &folio_list);
3371}
3372
3373static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
3374{
3375	unsigned long i;
3376
3377	for (i = 0; i < h->max_huge_pages; ++i) {
3378		if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
3379			break;
3380		cond_resched();
3381	}
3382
3383	return i;
3384}
3385
3386static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
3387{
3388	struct padata_mt_job job = {
3389		.fn_arg		= h,
3390		.align		= 1,
3391		.numa_aware	= true
3392	};
3393
3394	job.thread_fn	= hugetlb_pages_alloc_boot_node;
3395	job.start	= 0;
3396	job.size	= h->max_huge_pages;
3397
3398	/*
3399	 * job.max_threads is twice the num_node_state(N_MEMORY),
3400	 *
3401	 * Tests below indicate that a multiplier of 2 significantly improves
3402	 * performance, and although larger values also provide improvements,
3403	 * the gains are marginal.
3404	 *
3405	 * Therefore, choosing 2 as the multiplier strikes a good balance between
3406	 * enhancing parallel processing capabilities and maintaining efficient
3407	 * resource management.
3408	 *
3409	 * +------------+-------+-------+-------+-------+-------+
3410	 * | multiplier |   1   |   2   |   3   |   4   |   5   |
3411	 * +------------+-------+-------+-------+-------+-------+
3412	 * | 256G 2node | 358ms | 215ms | 157ms | 134ms | 126ms |
3413	 * | 2T   4node | 979ms | 679ms | 543ms | 489ms | 481ms |
3414	 * | 50G  2node | 71ms  | 44ms  | 37ms  | 30ms  | 31ms  |
3415	 * +------------+-------+-------+-------+-------+-------+
3416	 */
3417	job.max_threads	= num_node_state(N_MEMORY) * 2;
3418	job.min_chunk	= h->max_huge_pages / num_node_state(N_MEMORY) / 2;
3419	padata_do_multithreaded(&job);
3420
3421	return h->nr_huge_pages;
3422}
3423
3424/*
3425 * NOTE: this routine is called in different contexts for gigantic and
3426 * non-gigantic pages.
3427 * - For gigantic pages, this is called early in the boot process and
3428 *   pages are allocated from memblock allocated or something similar.
3429 *   Gigantic pages are actually added to pools later with the routine
3430 *   gather_bootmem_prealloc.
3431 * - For non-gigantic pages, this is called later in the boot process after
3432 *   all of mm is up and functional.  Pages are allocated from buddy and
3433 *   then added to hugetlb pools.
3434 */
3435static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
3436{
3437	unsigned long allocated;
3438	static bool initialized __initdata;
 
 
 
3439
3440	/* skip gigantic hugepages allocation if hugetlb_cma enabled */
3441	if (hstate_is_gigantic(h) && hugetlb_cma_size) {
3442		pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
3443		return;
3444	}
3445
3446	/* hugetlb_hstate_alloc_pages will be called many times, initialize huge_boot_pages once */
3447	if (!initialized) {
3448		int i = 0;
3449
3450		for (i = 0; i < MAX_NUMNODES; i++)
3451			INIT_LIST_HEAD(&huge_boot_pages[i]);
3452		initialized = true;
3453	}
3454
3455	/* do node specific alloc */
3456	if (hugetlb_hstate_alloc_pages_specific_nodes(h))
3457		return;
3458
3459	/* below will do all node balanced alloc */
3460	if (hstate_is_gigantic(h))
3461		allocated = hugetlb_gigantic_pages_alloc_boot(h);
3462	else
3463		allocated = hugetlb_pages_alloc_boot(h);
 
 
 
 
 
 
 
 
 
 
 
 
 
3464
3465	hugetlb_hstate_alloc_pages_errcheck(allocated, h);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3466}
3467
3468static void __init hugetlb_init_hstates(void)
3469{
3470	struct hstate *h, *h2;
3471
3472	for_each_hstate(h) {
3473		/* oversize hugepages were init'ed in early boot */
3474		if (!hstate_is_gigantic(h))
3475			hugetlb_hstate_alloc_pages(h);
3476
3477		/*
3478		 * Set demote order for each hstate.  Note that
3479		 * h->demote_order is initially 0.
3480		 * - We can not demote gigantic pages if runtime freeing
3481		 *   is not supported, so skip this.
3482		 * - If CMA allocation is possible, we can not demote
3483		 *   HUGETLB_PAGE_ORDER or smaller size pages.
3484		 */
3485		if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3486			continue;
3487		if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER)
3488			continue;
3489		for_each_hstate(h2) {
3490			if (h2 == h)
3491				continue;
3492			if (h2->order < h->order &&
3493			    h2->order > h->demote_order)
3494				h->demote_order = h2->order;
3495		}
3496	}
3497}
3498
3499static void __init report_hugepages(void)
3500{
3501	struct hstate *h;
3502
3503	for_each_hstate(h) {
3504		char buf[32];
3505
3506		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3507		pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
3508			buf, h->free_huge_pages);
3509		pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
3510			hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
3511	}
3512}
3513
3514#ifdef CONFIG_HIGHMEM
3515static void try_to_free_low(struct hstate *h, unsigned long count,
3516						nodemask_t *nodes_allowed)
3517{
3518	int i;
3519	LIST_HEAD(page_list);
3520
3521	lockdep_assert_held(&hugetlb_lock);
3522	if (hstate_is_gigantic(h))
3523		return;
3524
3525	/*
3526	 * Collect pages to be freed on a list, and free after dropping lock
3527	 */
3528	for_each_node_mask(i, *nodes_allowed) {
3529		struct folio *folio, *next;
3530		struct list_head *freel = &h->hugepage_freelists[i];
3531		list_for_each_entry_safe(folio, next, freel, lru) {
3532			if (count >= h->nr_huge_pages)
3533				goto out;
3534			if (folio_test_highmem(folio))
3535				continue;
3536			remove_hugetlb_folio(h, folio, false);
3537			list_add(&folio->lru, &page_list);
3538		}
3539	}
3540
3541out:
3542	spin_unlock_irq(&hugetlb_lock);
3543	update_and_free_pages_bulk(h, &page_list);
3544	spin_lock_irq(&hugetlb_lock);
3545}
3546#else
3547static inline void try_to_free_low(struct hstate *h, unsigned long count,
3548						nodemask_t *nodes_allowed)
3549{
3550}
3551#endif
3552
3553/*
3554 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
3555 * balanced by operating on them in a round-robin fashion.
3556 * Returns 1 if an adjustment was made.
3557 */
3558static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
3559				int delta)
3560{
3561	int nr_nodes, node;
3562
3563	lockdep_assert_held(&hugetlb_lock);
3564	VM_BUG_ON(delta != -1 && delta != 1);
3565
3566	if (delta < 0) {
3567		for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, nodes_allowed) {
3568			if (h->surplus_huge_pages_node[node])
3569				goto found;
3570		}
3571	} else {
3572		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3573			if (h->surplus_huge_pages_node[node] <
3574					h->nr_huge_pages_node[node])
3575				goto found;
3576		}
3577	}
3578	return 0;
3579
3580found:
3581	h->surplus_huge_pages += delta;
3582	h->surplus_huge_pages_node[node] += delta;
3583	return 1;
3584}
3585
3586#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
3587static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
3588			      nodemask_t *nodes_allowed)
3589{
3590	unsigned long min_count;
3591	unsigned long allocated;
3592	struct folio *folio;
3593	LIST_HEAD(page_list);
3594	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
3595
3596	/*
3597	 * Bit mask controlling how hard we retry per-node allocations.
3598	 * If we can not allocate the bit mask, do not attempt to allocate
3599	 * the requested huge pages.
3600	 */
3601	if (node_alloc_noretry)
3602		nodes_clear(*node_alloc_noretry);
3603	else
3604		return -ENOMEM;
3605
3606	/*
3607	 * resize_lock mutex prevents concurrent adjustments to number of
3608	 * pages in hstate via the proc/sysfs interfaces.
3609	 */
3610	mutex_lock(&h->resize_lock);
3611	flush_free_hpage_work(h);
3612	spin_lock_irq(&hugetlb_lock);
3613
3614	/*
3615	 * Check for a node specific request.
3616	 * Changing node specific huge page count may require a corresponding
3617	 * change to the global count.  In any case, the passed node mask
3618	 * (nodes_allowed) will restrict alloc/free to the specified node.
3619	 */
3620	if (nid != NUMA_NO_NODE) {
3621		unsigned long old_count = count;
3622
3623		count += persistent_huge_pages(h) -
3624			 (h->nr_huge_pages_node[nid] -
3625			  h->surplus_huge_pages_node[nid]);
3626		/*
3627		 * User may have specified a large count value which caused the
3628		 * above calculation to overflow.  In this case, they wanted
3629		 * to allocate as many huge pages as possible.  Set count to
3630		 * largest possible value to align with their intention.
3631		 */
3632		if (count < old_count)
3633			count = ULONG_MAX;
3634	}
3635
3636	/*
3637	 * Gigantic pages runtime allocation depend on the capability for large
3638	 * page range allocation.
3639	 * If the system does not provide this feature, return an error when
3640	 * the user tries to allocate gigantic pages but let the user free the
3641	 * boottime allocated gigantic pages.
3642	 */
3643	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
3644		if (count > persistent_huge_pages(h)) {
3645			spin_unlock_irq(&hugetlb_lock);
3646			mutex_unlock(&h->resize_lock);
3647			NODEMASK_FREE(node_alloc_noretry);
3648			return -EINVAL;
3649		}
3650		/* Fall through to decrease pool */
3651	}
3652
3653	/*
3654	 * Increase the pool size
3655	 * First take pages out of surplus state.  Then make up the
3656	 * remaining difference by allocating fresh huge pages.
3657	 *
3658	 * We might race with alloc_surplus_hugetlb_folio() here and be unable
3659	 * to convert a surplus huge page to a normal huge page. That is
3660	 * not critical, though, it just means the overall size of the
3661	 * pool might be one hugepage larger than it needs to be, but
3662	 * within all the constraints specified by the sysctls.
3663	 */
3664	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3665		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3666			break;
3667	}
3668
3669	allocated = 0;
3670	while (count > (persistent_huge_pages(h) + allocated)) {
3671		/*
3672		 * If this allocation races such that we no longer need the
3673		 * page, free_huge_folio will handle it by freeing the page
3674		 * and reducing the surplus.
3675		 */
3676		spin_unlock_irq(&hugetlb_lock);
3677
3678		/* yield cpu to avoid soft lockup */
3679		cond_resched();
3680
3681		folio = alloc_pool_huge_folio(h, nodes_allowed,
3682						node_alloc_noretry,
3683						&h->next_nid_to_alloc);
3684		if (!folio) {
3685			prep_and_add_allocated_folios(h, &page_list);
3686			spin_lock_irq(&hugetlb_lock);
3687			goto out;
3688		}
3689
3690		list_add(&folio->lru, &page_list);
3691		allocated++;
3692
3693		/* Bail for signals. Probably ctrl-c from user */
3694		if (signal_pending(current)) {
3695			prep_and_add_allocated_folios(h, &page_list);
3696			spin_lock_irq(&hugetlb_lock);
3697			goto out;
3698		}
3699
3700		spin_lock_irq(&hugetlb_lock);
3701	}
3702
3703	/* Add allocated pages to the pool */
3704	if (!list_empty(&page_list)) {
3705		spin_unlock_irq(&hugetlb_lock);
3706		prep_and_add_allocated_folios(h, &page_list);
3707		spin_lock_irq(&hugetlb_lock);
3708	}
3709
3710	/*
3711	 * Decrease the pool size
3712	 * First return free pages to the buddy allocator (being careful
3713	 * to keep enough around to satisfy reservations).  Then place
3714	 * pages into surplus state as needed so the pool will shrink
3715	 * to the desired size as pages become free.
3716	 *
3717	 * By placing pages into the surplus state independent of the
3718	 * overcommit value, we are allowing the surplus pool size to
3719	 * exceed overcommit. There are few sane options here. Since
3720	 * alloc_surplus_hugetlb_folio() is checking the global counter,
3721	 * though, we'll note that we're not allowed to exceed surplus
3722	 * and won't grow the pool anywhere else. Not until one of the
3723	 * sysctls are changed, or the surplus pages go out of use.
3724	 */
3725	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3726	min_count = max(count, min_count);
3727	try_to_free_low(h, min_count, nodes_allowed);
3728
3729	/*
3730	 * Collect pages to be removed on list without dropping lock
3731	 */
3732	while (min_count < persistent_huge_pages(h)) {
3733		folio = remove_pool_hugetlb_folio(h, nodes_allowed, 0);
3734		if (!folio)
3735			break;
3736
3737		list_add(&folio->lru, &page_list);
3738	}
3739	/* free the pages after dropping lock */
3740	spin_unlock_irq(&hugetlb_lock);
3741	update_and_free_pages_bulk(h, &page_list);
3742	flush_free_hpage_work(h);
3743	spin_lock_irq(&hugetlb_lock);
3744
3745	while (count < persistent_huge_pages(h)) {
3746		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3747			break;
3748	}
3749out:
3750	h->max_huge_pages = persistent_huge_pages(h);
3751	spin_unlock_irq(&hugetlb_lock);
3752	mutex_unlock(&h->resize_lock);
3753
3754	NODEMASK_FREE(node_alloc_noretry);
3755
3756	return 0;
3757}
3758
3759static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,
3760				       struct list_head *src_list)
3761{
3762	long rc;
3763	struct folio *folio, *next;
3764	LIST_HEAD(dst_list);
3765	LIST_HEAD(ret_list);
 
3766
3767	rc = hugetlb_vmemmap_restore_folios(src, src_list, &ret_list);
3768	list_splice_init(&ret_list, src_list);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3769
3770	/*
3771	 * Taking target hstate mutex synchronizes with set_max_huge_pages.
3772	 * Without the mutex, pages added to target hstate could be marked
3773	 * as surplus.
3774	 *
3775	 * Note that we already hold src->resize_lock.  To prevent deadlock,
3776	 * use the convention of always taking larger size hstate mutex first.
3777	 */
3778	mutex_lock(&dst->resize_lock);
3779
3780	list_for_each_entry_safe(folio, next, src_list, lru) {
3781		int i;
3782
3783		if (folio_test_hugetlb_vmemmap_optimized(folio))
3784			continue;
3785
3786		list_del(&folio->lru);
3787
3788		split_page_owner(&folio->page, huge_page_order(src), huge_page_order(dst));
3789		pgalloc_tag_split(folio, huge_page_order(src), huge_page_order(dst));
3790
3791		for (i = 0; i < pages_per_huge_page(src); i += pages_per_huge_page(dst)) {
3792			struct page *page = folio_page(folio, i);
3793
3794			page->mapping = NULL;
3795			clear_compound_head(page);
3796			prep_compound_page(page, dst->order);
3797
3798			init_new_hugetlb_folio(dst, page_folio(page));
3799			list_add(&page->lru, &dst_list);
3800		}
3801	}
 
3802
3803	prep_and_add_allocated_folios(dst, &dst_list);
3804
3805	mutex_unlock(&dst->resize_lock);
 
 
 
 
 
 
3806
3807	return rc;
3808}
3809
3810static long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
3811				  unsigned long nr_to_demote)
3812	__must_hold(&hugetlb_lock)
3813{
3814	int nr_nodes, node;
3815	struct hstate *dst;
3816	long rc = 0;
3817	long nr_demoted = 0;
3818
3819	lockdep_assert_held(&hugetlb_lock);
3820
3821	/* We should never get here if no demote order */
3822	if (!src->demote_order) {
3823		pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
3824		return -EINVAL;		/* internal error */
3825	}
3826	dst = size_to_hstate(PAGE_SIZE << src->demote_order);
3827
3828	for_each_node_mask_to_free(src, nr_nodes, node, nodes_allowed) {
3829		LIST_HEAD(list);
3830		struct folio *folio, *next;
3831
3832		list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) {
3833			if (folio_test_hwpoison(folio))
3834				continue;
3835
3836			remove_hugetlb_folio(src, folio, false);
3837			list_add(&folio->lru, &list);
3838
3839			if (++nr_demoted == nr_to_demote)
3840				break;
3841		}
3842
3843		spin_unlock_irq(&hugetlb_lock);
3844
3845		rc = demote_free_hugetlb_folios(src, dst, &list);
3846
3847		spin_lock_irq(&hugetlb_lock);
3848
3849		list_for_each_entry_safe(folio, next, &list, lru) {
3850			list_del(&folio->lru);
3851			add_hugetlb_folio(src, folio, false);
3852
3853			nr_demoted--;
3854		}
3855
3856		if (rc < 0 || nr_demoted == nr_to_demote)
3857			break;
3858	}
3859
3860	/*
3861	 * Not absolutely necessary, but for consistency update max_huge_pages
3862	 * based on pool changes for the demoted page.
3863	 */
3864	src->max_huge_pages -= nr_demoted;
3865	dst->max_huge_pages += nr_demoted << (huge_page_order(src) - huge_page_order(dst));
3866
3867	if (rc < 0)
3868		return rc;
3869
3870	if (nr_demoted)
3871		return nr_demoted;
3872	/*
3873	 * Only way to get here is if all pages on free lists are poisoned.
3874	 * Return -EBUSY so that caller will not retry.
3875	 */
3876	return -EBUSY;
3877}
3878
3879#define HSTATE_ATTR_RO(_name) \
3880	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
3881
3882#define HSTATE_ATTR_WO(_name) \
3883	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)
3884
3885#define HSTATE_ATTR(_name) \
3886	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
3887
3888static struct kobject *hugepages_kobj;
3889static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
3890
3891static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
3892
3893static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
3894{
3895	int i;
3896
3897	for (i = 0; i < HUGE_MAX_HSTATE; i++)
3898		if (hstate_kobjs[i] == kobj) {
3899			if (nidp)
3900				*nidp = NUMA_NO_NODE;
3901			return &hstates[i];
3902		}
3903
3904	return kobj_to_node_hstate(kobj, nidp);
3905}
3906
3907static ssize_t nr_hugepages_show_common(struct kobject *kobj,
3908					struct kobj_attribute *attr, char *buf)
3909{
3910	struct hstate *h;
3911	unsigned long nr_huge_pages;
3912	int nid;
3913
3914	h = kobj_to_hstate(kobj, &nid);
3915	if (nid == NUMA_NO_NODE)
3916		nr_huge_pages = h->nr_huge_pages;
3917	else
3918		nr_huge_pages = h->nr_huge_pages_node[nid];
3919
3920	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
3921}
3922
3923static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
3924					   struct hstate *h, int nid,
3925					   unsigned long count, size_t len)
3926{
3927	int err;
3928	nodemask_t nodes_allowed, *n_mask;
3929
3930	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3931		return -EINVAL;
3932
3933	if (nid == NUMA_NO_NODE) {
3934		/*
3935		 * global hstate attribute
3936		 */
3937		if (!(obey_mempolicy &&
3938				init_nodemask_of_mempolicy(&nodes_allowed)))
3939			n_mask = &node_states[N_MEMORY];
3940		else
3941			n_mask = &nodes_allowed;
3942	} else {
3943		/*
3944		 * Node specific request.  count adjustment happens in
3945		 * set_max_huge_pages() after acquiring hugetlb_lock.
3946		 */
3947		init_nodemask_of_node(&nodes_allowed, nid);
3948		n_mask = &nodes_allowed;
3949	}
3950
3951	err = set_max_huge_pages(h, count, nid, n_mask);
3952
3953	return err ? err : len;
3954}
3955
3956static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
3957					 struct kobject *kobj, const char *buf,
3958					 size_t len)
3959{
3960	struct hstate *h;
3961	unsigned long count;
3962	int nid;
3963	int err;
3964
3965	err = kstrtoul(buf, 10, &count);
3966	if (err)
3967		return err;
3968
3969	h = kobj_to_hstate(kobj, &nid);
3970	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
3971}
3972
3973static ssize_t nr_hugepages_show(struct kobject *kobj,
3974				       struct kobj_attribute *attr, char *buf)
3975{
3976	return nr_hugepages_show_common(kobj, attr, buf);
3977}
3978
3979static ssize_t nr_hugepages_store(struct kobject *kobj,
3980	       struct kobj_attribute *attr, const char *buf, size_t len)
3981{
3982	return nr_hugepages_store_common(false, kobj, buf, len);
3983}
3984HSTATE_ATTR(nr_hugepages);
3985
3986#ifdef CONFIG_NUMA
3987
3988/*
3989 * hstate attribute for optionally mempolicy-based constraint on persistent
3990 * huge page alloc/free.
3991 */
3992static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
3993					   struct kobj_attribute *attr,
3994					   char *buf)
3995{
3996	return nr_hugepages_show_common(kobj, attr, buf);
3997}
3998
3999static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
4000	       struct kobj_attribute *attr, const char *buf, size_t len)
4001{
4002	return nr_hugepages_store_common(true, kobj, buf, len);
4003}
4004HSTATE_ATTR(nr_hugepages_mempolicy);
4005#endif
4006
4007
4008static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
4009					struct kobj_attribute *attr, char *buf)
4010{
4011	struct hstate *h = kobj_to_hstate(kobj, NULL);
4012	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
4013}
4014
4015static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
4016		struct kobj_attribute *attr, const char *buf, size_t count)
4017{
4018	int err;
4019	unsigned long input;
4020	struct hstate *h = kobj_to_hstate(kobj, NULL);
4021
4022	if (hstate_is_gigantic(h))
4023		return -EINVAL;
4024
4025	err = kstrtoul(buf, 10, &input);
4026	if (err)
4027		return err;
4028
4029	spin_lock_irq(&hugetlb_lock);
4030	h->nr_overcommit_huge_pages = input;
4031	spin_unlock_irq(&hugetlb_lock);
4032
4033	return count;
4034}
4035HSTATE_ATTR(nr_overcommit_hugepages);
4036
4037static ssize_t free_hugepages_show(struct kobject *kobj,
4038					struct kobj_attribute *attr, char *buf)
4039{
4040	struct hstate *h;
4041	unsigned long free_huge_pages;
4042	int nid;
4043
4044	h = kobj_to_hstate(kobj, &nid);
4045	if (nid == NUMA_NO_NODE)
4046		free_huge_pages = h->free_huge_pages;
4047	else
4048		free_huge_pages = h->free_huge_pages_node[nid];
4049
4050	return sysfs_emit(buf, "%lu\n", free_huge_pages);
4051}
4052HSTATE_ATTR_RO(free_hugepages);
4053
4054static ssize_t resv_hugepages_show(struct kobject *kobj,
4055					struct kobj_attribute *attr, char *buf)
4056{
4057	struct hstate *h = kobj_to_hstate(kobj, NULL);
4058	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
4059}
4060HSTATE_ATTR_RO(resv_hugepages);
4061
4062static ssize_t surplus_hugepages_show(struct kobject *kobj,
4063					struct kobj_attribute *attr, char *buf)
4064{
4065	struct hstate *h;
4066	unsigned long surplus_huge_pages;
4067	int nid;
4068
4069	h = kobj_to_hstate(kobj, &nid);
4070	if (nid == NUMA_NO_NODE)
4071		surplus_huge_pages = h->surplus_huge_pages;
4072	else
4073		surplus_huge_pages = h->surplus_huge_pages_node[nid];
4074
4075	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
4076}
4077HSTATE_ATTR_RO(surplus_hugepages);
4078
4079static ssize_t demote_store(struct kobject *kobj,
4080	       struct kobj_attribute *attr, const char *buf, size_t len)
4081{
4082	unsigned long nr_demote;
4083	unsigned long nr_available;
4084	nodemask_t nodes_allowed, *n_mask;
4085	struct hstate *h;
4086	int err;
4087	int nid;
4088
4089	err = kstrtoul(buf, 10, &nr_demote);
4090	if (err)
4091		return err;
4092	h = kobj_to_hstate(kobj, &nid);
4093
4094	if (nid != NUMA_NO_NODE) {
4095		init_nodemask_of_node(&nodes_allowed, nid);
4096		n_mask = &nodes_allowed;
4097	} else {
4098		n_mask = &node_states[N_MEMORY];
4099	}
4100
4101	/* Synchronize with other sysfs operations modifying huge pages */
4102	mutex_lock(&h->resize_lock);
4103	spin_lock_irq(&hugetlb_lock);
4104
4105	while (nr_demote) {
4106		long rc;
4107
4108		/*
4109		 * Check for available pages to demote each time thorough the
4110		 * loop as demote_pool_huge_page will drop hugetlb_lock.
4111		 */
4112		if (nid != NUMA_NO_NODE)
4113			nr_available = h->free_huge_pages_node[nid];
4114		else
4115			nr_available = h->free_huge_pages;
4116		nr_available -= h->resv_huge_pages;
4117		if (!nr_available)
4118			break;
4119
4120		rc = demote_pool_huge_page(h, n_mask, nr_demote);
4121		if (rc < 0) {
4122			err = rc;
4123			break;
4124		}
4125
4126		nr_demote -= rc;
4127	}
4128
4129	spin_unlock_irq(&hugetlb_lock);
4130	mutex_unlock(&h->resize_lock);
4131
4132	if (err)
4133		return err;
4134	return len;
4135}
4136HSTATE_ATTR_WO(demote);
4137
4138static ssize_t demote_size_show(struct kobject *kobj,
4139					struct kobj_attribute *attr, char *buf)
4140{
4141	struct hstate *h = kobj_to_hstate(kobj, NULL);
4142	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
4143
4144	return sysfs_emit(buf, "%lukB\n", demote_size);
4145}
4146
4147static ssize_t demote_size_store(struct kobject *kobj,
4148					struct kobj_attribute *attr,
4149					const char *buf, size_t count)
4150{
4151	struct hstate *h, *demote_hstate;
4152	unsigned long demote_size;
4153	unsigned int demote_order;
4154
4155	demote_size = (unsigned long)memparse(buf, NULL);
4156
4157	demote_hstate = size_to_hstate(demote_size);
4158	if (!demote_hstate)
4159		return -EINVAL;
4160	demote_order = demote_hstate->order;
4161	if (demote_order < HUGETLB_PAGE_ORDER)
4162		return -EINVAL;
4163
4164	/* demote order must be smaller than hstate order */
4165	h = kobj_to_hstate(kobj, NULL);
4166	if (demote_order >= h->order)
4167		return -EINVAL;
4168
4169	/* resize_lock synchronizes access to demote size and writes */
4170	mutex_lock(&h->resize_lock);
4171	h->demote_order = demote_order;
4172	mutex_unlock(&h->resize_lock);
4173
4174	return count;
4175}
4176HSTATE_ATTR(demote_size);
4177
4178static struct attribute *hstate_attrs[] = {
4179	&nr_hugepages_attr.attr,
4180	&nr_overcommit_hugepages_attr.attr,
4181	&free_hugepages_attr.attr,
4182	&resv_hugepages_attr.attr,
4183	&surplus_hugepages_attr.attr,
4184#ifdef CONFIG_NUMA
4185	&nr_hugepages_mempolicy_attr.attr,
4186#endif
4187	NULL,
4188};
4189
4190static const struct attribute_group hstate_attr_group = {
4191	.attrs = hstate_attrs,
4192};
4193
4194static struct attribute *hstate_demote_attrs[] = {
4195	&demote_size_attr.attr,
4196	&demote_attr.attr,
4197	NULL,
4198};
4199
4200static const struct attribute_group hstate_demote_attr_group = {
4201	.attrs = hstate_demote_attrs,
4202};
4203
4204static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
4205				    struct kobject **hstate_kobjs,
4206				    const struct attribute_group *hstate_attr_group)
4207{
4208	int retval;
4209	int hi = hstate_index(h);
4210
4211	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
4212	if (!hstate_kobjs[hi])
4213		return -ENOMEM;
4214
4215	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
4216	if (retval) {
4217		kobject_put(hstate_kobjs[hi]);
4218		hstate_kobjs[hi] = NULL;
4219		return retval;
4220	}
4221
4222	if (h->demote_order) {
4223		retval = sysfs_create_group(hstate_kobjs[hi],
4224					    &hstate_demote_attr_group);
4225		if (retval) {
4226			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
4227			sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
4228			kobject_put(hstate_kobjs[hi]);
4229			hstate_kobjs[hi] = NULL;
4230			return retval;
4231		}
4232	}
4233
4234	return 0;
4235}
4236
4237#ifdef CONFIG_NUMA
4238static bool hugetlb_sysfs_initialized __ro_after_init;
4239
4240/*
4241 * node_hstate/s - associate per node hstate attributes, via their kobjects,
4242 * with node devices in node_devices[] using a parallel array.  The array
4243 * index of a node device or _hstate == node id.
4244 * This is here to avoid any static dependency of the node device driver, in
4245 * the base kernel, on the hugetlb module.
4246 */
4247struct node_hstate {
4248	struct kobject		*hugepages_kobj;
4249	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
4250};
4251static struct node_hstate node_hstates[MAX_NUMNODES];
4252
4253/*
4254 * A subset of global hstate attributes for node devices
4255 */
4256static struct attribute *per_node_hstate_attrs[] = {
4257	&nr_hugepages_attr.attr,
4258	&free_hugepages_attr.attr,
4259	&surplus_hugepages_attr.attr,
4260	NULL,
4261};
4262
4263static const struct attribute_group per_node_hstate_attr_group = {
4264	.attrs = per_node_hstate_attrs,
4265};
4266
4267/*
4268 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
4269 * Returns node id via non-NULL nidp.
4270 */
4271static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4272{
4273	int nid;
4274
4275	for (nid = 0; nid < nr_node_ids; nid++) {
4276		struct node_hstate *nhs = &node_hstates[nid];
4277		int i;
4278		for (i = 0; i < HUGE_MAX_HSTATE; i++)
4279			if (nhs->hstate_kobjs[i] == kobj) {
4280				if (nidp)
4281					*nidp = nid;
4282				return &hstates[i];
4283			}
4284	}
4285
4286	BUG();
4287	return NULL;
4288}
4289
4290/*
4291 * Unregister hstate attributes from a single node device.
4292 * No-op if no hstate attributes attached.
4293 */
4294void hugetlb_unregister_node(struct node *node)
4295{
4296	struct hstate *h;
4297	struct node_hstate *nhs = &node_hstates[node->dev.id];
4298
4299	if (!nhs->hugepages_kobj)
4300		return;		/* no hstate attributes */
4301
4302	for_each_hstate(h) {
4303		int idx = hstate_index(h);
4304		struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];
4305
4306		if (!hstate_kobj)
4307			continue;
4308		if (h->demote_order)
4309			sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
4310		sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
4311		kobject_put(hstate_kobj);
4312		nhs->hstate_kobjs[idx] = NULL;
4313	}
4314
4315	kobject_put(nhs->hugepages_kobj);
4316	nhs->hugepages_kobj = NULL;
4317}
4318
4319
4320/*
4321 * Register hstate attributes for a single node device.
4322 * No-op if attributes already registered.
4323 */
4324void hugetlb_register_node(struct node *node)
4325{
4326	struct hstate *h;
4327	struct node_hstate *nhs = &node_hstates[node->dev.id];
4328	int err;
4329
4330	if (!hugetlb_sysfs_initialized)
4331		return;
4332
4333	if (nhs->hugepages_kobj)
4334		return;		/* already allocated */
4335
4336	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
4337							&node->dev.kobj);
4338	if (!nhs->hugepages_kobj)
4339		return;
4340
4341	for_each_hstate(h) {
4342		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
4343						nhs->hstate_kobjs,
4344						&per_node_hstate_attr_group);
4345		if (err) {
4346			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
4347				h->name, node->dev.id);
4348			hugetlb_unregister_node(node);
4349			break;
4350		}
4351	}
4352}
4353
4354/*
4355 * hugetlb init time:  register hstate attributes for all registered node
4356 * devices of nodes that have memory.  All on-line nodes should have
4357 * registered their associated device by this time.
4358 */
4359static void __init hugetlb_register_all_nodes(void)
4360{
4361	int nid;
4362
4363	for_each_online_node(nid)
4364		hugetlb_register_node(node_devices[nid]);
4365}
4366#else	/* !CONFIG_NUMA */
4367
4368static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4369{
4370	BUG();
4371	if (nidp)
4372		*nidp = -1;
4373	return NULL;
4374}
4375
4376static void hugetlb_register_all_nodes(void) { }
4377
4378#endif
4379
4380#ifdef CONFIG_CMA
4381static void __init hugetlb_cma_check(void);
4382#else
4383static inline __init void hugetlb_cma_check(void)
4384{
4385}
4386#endif
4387
4388static void __init hugetlb_sysfs_init(void)
4389{
4390	struct hstate *h;
4391	int err;
4392
4393	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
4394	if (!hugepages_kobj)
4395		return;
4396
4397	for_each_hstate(h) {
4398		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
4399					 hstate_kobjs, &hstate_attr_group);
4400		if (err)
4401			pr_err("HugeTLB: Unable to add hstate %s", h->name);
4402	}
4403
4404#ifdef CONFIG_NUMA
4405	hugetlb_sysfs_initialized = true;
4406#endif
4407	hugetlb_register_all_nodes();
4408}
4409
4410#ifdef CONFIG_SYSCTL
4411static void hugetlb_sysctl_init(void);
4412#else
4413static inline void hugetlb_sysctl_init(void) { }
4414#endif
4415
4416static int __init hugetlb_init(void)
4417{
4418	int i;
4419
4420	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
4421			__NR_HPAGEFLAGS);
4422
4423	if (!hugepages_supported()) {
4424		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
4425			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
4426		return 0;
4427	}
4428
4429	/*
4430	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
4431	 * architectures depend on setup being done here.
4432	 */
4433	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
4434	if (!parsed_default_hugepagesz) {
4435		/*
4436		 * If we did not parse a default huge page size, set
4437		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
4438		 * number of huge pages for this default size was implicitly
4439		 * specified, set that here as well.
4440		 * Note that the implicit setting will overwrite an explicit
4441		 * setting.  A warning will be printed in this case.
4442		 */
4443		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
4444		if (default_hstate_max_huge_pages) {
4445			if (default_hstate.max_huge_pages) {
4446				char buf[32];
4447
4448				string_get_size(huge_page_size(&default_hstate),
4449					1, STRING_UNITS_2, buf, 32);
4450				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
4451					default_hstate.max_huge_pages, buf);
4452				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
4453					default_hstate_max_huge_pages);
4454			}
4455			default_hstate.max_huge_pages =
4456				default_hstate_max_huge_pages;
4457
4458			for_each_online_node(i)
4459				default_hstate.max_huge_pages_node[i] =
4460					default_hugepages_in_node[i];
4461		}
4462	}
4463
4464	hugetlb_cma_check();
4465	hugetlb_init_hstates();
4466	gather_bootmem_prealloc();
4467	report_hugepages();
4468
4469	hugetlb_sysfs_init();
4470	hugetlb_cgroup_file_init();
4471	hugetlb_sysctl_init();
4472
4473#ifdef CONFIG_SMP
4474	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
4475#else
4476	num_fault_mutexes = 1;
4477#endif
4478	hugetlb_fault_mutex_table =
4479		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
4480			      GFP_KERNEL);
4481	BUG_ON(!hugetlb_fault_mutex_table);
4482
4483	for (i = 0; i < num_fault_mutexes; i++)
4484		mutex_init(&hugetlb_fault_mutex_table[i]);
4485	return 0;
4486}
4487subsys_initcall(hugetlb_init);
4488
4489/* Overwritten by architectures with more huge page sizes */
4490bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
4491{
4492	return size == HPAGE_SIZE;
4493}
4494
4495void __init hugetlb_add_hstate(unsigned int order)
4496{
4497	struct hstate *h;
4498	unsigned long i;
4499
4500	if (size_to_hstate(PAGE_SIZE << order)) {
4501		return;
4502	}
4503	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
4504	BUG_ON(order < order_base_2(__NR_USED_SUBPAGE));
4505	h = &hstates[hugetlb_max_hstate++];
4506	__mutex_init(&h->resize_lock, "resize mutex", &h->resize_key);
4507	h->order = order;
4508	h->mask = ~(huge_page_size(h) - 1);
4509	for (i = 0; i < MAX_NUMNODES; ++i)
4510		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
4511	INIT_LIST_HEAD(&h->hugepage_activelist);
4512	h->next_nid_to_alloc = first_memory_node;
4513	h->next_nid_to_free = first_memory_node;
4514	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
4515					huge_page_size(h)/SZ_1K);
4516
4517	parsed_hstate = h;
4518}
4519
4520bool __init __weak hugetlb_node_alloc_supported(void)
4521{
4522	return true;
4523}
4524
4525static void __init hugepages_clear_pages_in_node(void)
4526{
4527	if (!hugetlb_max_hstate) {
4528		default_hstate_max_huge_pages = 0;
4529		memset(default_hugepages_in_node, 0,
4530			sizeof(default_hugepages_in_node));
4531	} else {
4532		parsed_hstate->max_huge_pages = 0;
4533		memset(parsed_hstate->max_huge_pages_node, 0,
4534			sizeof(parsed_hstate->max_huge_pages_node));
4535	}
4536}
4537
4538/*
4539 * hugepages command line processing
4540 * hugepages normally follows a valid hugepagsz or default_hugepagsz
4541 * specification.  If not, ignore the hugepages value.  hugepages can also
4542 * be the first huge page command line  option in which case it implicitly
4543 * specifies the number of huge pages for the default size.
4544 */
4545static int __init hugepages_setup(char *s)
4546{
4547	unsigned long *mhp;
4548	static unsigned long *last_mhp;
4549	int node = NUMA_NO_NODE;
4550	int count;
4551	unsigned long tmp;
4552	char *p = s;
4553
4554	if (!parsed_valid_hugepagesz) {
4555		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
4556		parsed_valid_hugepagesz = true;
4557		return 1;
4558	}
4559
4560	/*
4561	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
4562	 * yet, so this hugepages= parameter goes to the "default hstate".
4563	 * Otherwise, it goes with the previously parsed hugepagesz or
4564	 * default_hugepagesz.
4565	 */
4566	else if (!hugetlb_max_hstate)
4567		mhp = &default_hstate_max_huge_pages;
4568	else
4569		mhp = &parsed_hstate->max_huge_pages;
4570
4571	if (mhp == last_mhp) {
4572		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
4573		return 1;
4574	}
4575
4576	while (*p) {
4577		count = 0;
4578		if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4579			goto invalid;
4580		/* Parameter is node format */
4581		if (p[count] == ':') {
4582			if (!hugetlb_node_alloc_supported()) {
4583				pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
4584				return 1;
4585			}
4586			if (tmp >= MAX_NUMNODES || !node_online(tmp))
4587				goto invalid;
4588			node = array_index_nospec(tmp, MAX_NUMNODES);
4589			p += count + 1;
4590			/* Parse hugepages */
4591			if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4592				goto invalid;
4593			if (!hugetlb_max_hstate)
4594				default_hugepages_in_node[node] = tmp;
4595			else
4596				parsed_hstate->max_huge_pages_node[node] = tmp;
4597			*mhp += tmp;
4598			/* Go to parse next node*/
4599			if (p[count] == ',')
4600				p += count + 1;
4601			else
4602				break;
4603		} else {
4604			if (p != s)
4605				goto invalid;
4606			*mhp = tmp;
4607			break;
4608		}
4609	}
4610
4611	/*
4612	 * Global state is always initialized later in hugetlb_init.
4613	 * But we need to allocate gigantic hstates here early to still
4614	 * use the bootmem allocator.
4615	 */
4616	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
4617		hugetlb_hstate_alloc_pages(parsed_hstate);
4618
4619	last_mhp = mhp;
4620
4621	return 1;
4622
4623invalid:
4624	pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
4625	hugepages_clear_pages_in_node();
4626	return 1;
4627}
4628__setup("hugepages=", hugepages_setup);
4629
4630/*
4631 * hugepagesz command line processing
4632 * A specific huge page size can only be specified once with hugepagesz.
4633 * hugepagesz is followed by hugepages on the command line.  The global
4634 * variable 'parsed_valid_hugepagesz' is used to determine if prior
4635 * hugepagesz argument was valid.
4636 */
4637static int __init hugepagesz_setup(char *s)
4638{
4639	unsigned long size;
4640	struct hstate *h;
4641
4642	parsed_valid_hugepagesz = false;
4643	size = (unsigned long)memparse(s, NULL);
4644
4645	if (!arch_hugetlb_valid_size(size)) {
4646		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
4647		return 1;
4648	}
4649
4650	h = size_to_hstate(size);
4651	if (h) {
4652		/*
4653		 * hstate for this size already exists.  This is normally
4654		 * an error, but is allowed if the existing hstate is the
4655		 * default hstate.  More specifically, it is only allowed if
4656		 * the number of huge pages for the default hstate was not
4657		 * previously specified.
4658		 */
4659		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
4660		    default_hstate.max_huge_pages) {
4661			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
4662			return 1;
4663		}
4664
4665		/*
4666		 * No need to call hugetlb_add_hstate() as hstate already
4667		 * exists.  But, do set parsed_hstate so that a following
4668		 * hugepages= parameter will be applied to this hstate.
4669		 */
4670		parsed_hstate = h;
4671		parsed_valid_hugepagesz = true;
4672		return 1;
4673	}
4674
4675	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4676	parsed_valid_hugepagesz = true;
4677	return 1;
4678}
4679__setup("hugepagesz=", hugepagesz_setup);
4680
4681/*
4682 * default_hugepagesz command line input
4683 * Only one instance of default_hugepagesz allowed on command line.
4684 */
4685static int __init default_hugepagesz_setup(char *s)
4686{
4687	unsigned long size;
4688	int i;
4689
4690	parsed_valid_hugepagesz = false;
4691	if (parsed_default_hugepagesz) {
4692		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
4693		return 1;
4694	}
4695
4696	size = (unsigned long)memparse(s, NULL);
4697
4698	if (!arch_hugetlb_valid_size(size)) {
4699		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
4700		return 1;
4701	}
4702
4703	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4704	parsed_valid_hugepagesz = true;
4705	parsed_default_hugepagesz = true;
4706	default_hstate_idx = hstate_index(size_to_hstate(size));
4707
4708	/*
4709	 * The number of default huge pages (for this size) could have been
4710	 * specified as the first hugetlb parameter: hugepages=X.  If so,
4711	 * then default_hstate_max_huge_pages is set.  If the default huge
4712	 * page size is gigantic (> MAX_PAGE_ORDER), then the pages must be
4713	 * allocated here from bootmem allocator.
4714	 */
4715	if (default_hstate_max_huge_pages) {
4716		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
4717		for_each_online_node(i)
4718			default_hstate.max_huge_pages_node[i] =
4719				default_hugepages_in_node[i];
4720		if (hstate_is_gigantic(&default_hstate))
4721			hugetlb_hstate_alloc_pages(&default_hstate);
4722		default_hstate_max_huge_pages = 0;
4723	}
4724
4725	return 1;
4726}
4727__setup("default_hugepagesz=", default_hugepagesz_setup);
4728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4729static unsigned int allowed_mems_nr(struct hstate *h)
4730{
4731	int node;
4732	unsigned int nr = 0;
4733	nodemask_t *mbind_nodemask;
4734	unsigned int *array = h->free_huge_pages_node;
4735	gfp_t gfp_mask = htlb_alloc_mask(h);
4736
4737	mbind_nodemask = policy_mbind_nodemask(gfp_mask);
4738	for_each_node_mask(node, cpuset_current_mems_allowed) {
4739		if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
4740			nr += array[node];
4741	}
4742
4743	return nr;
4744}
4745
4746#ifdef CONFIG_SYSCTL
4747static int proc_hugetlb_doulongvec_minmax(const struct ctl_table *table, int write,
4748					  void *buffer, size_t *length,
4749					  loff_t *ppos, unsigned long *out)
4750{
4751	struct ctl_table dup_table;
4752
4753	/*
4754	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
4755	 * can duplicate the @table and alter the duplicate of it.
4756	 */
4757	dup_table = *table;
4758	dup_table.data = out;
4759
4760	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
4761}
4762
4763static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
4764			 const struct ctl_table *table, int write,
4765			 void *buffer, size_t *length, loff_t *ppos)
4766{
4767	struct hstate *h = &default_hstate;
4768	unsigned long tmp = h->max_huge_pages;
4769	int ret;
4770
4771	if (!hugepages_supported())
4772		return -EOPNOTSUPP;
4773
4774	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4775					     &tmp);
4776	if (ret)
4777		goto out;
4778
4779	if (write)
4780		ret = __nr_hugepages_store_common(obey_mempolicy, h,
4781						  NUMA_NO_NODE, tmp, *length);
4782out:
4783	return ret;
4784}
4785
4786static int hugetlb_sysctl_handler(const struct ctl_table *table, int write,
4787			  void *buffer, size_t *length, loff_t *ppos)
4788{
4789
4790	return hugetlb_sysctl_handler_common(false, table, write,
4791							buffer, length, ppos);
4792}
4793
4794#ifdef CONFIG_NUMA
4795static int hugetlb_mempolicy_sysctl_handler(const struct ctl_table *table, int write,
4796			  void *buffer, size_t *length, loff_t *ppos)
4797{
4798	return hugetlb_sysctl_handler_common(true, table, write,
4799							buffer, length, ppos);
4800}
4801#endif /* CONFIG_NUMA */
4802
4803static int hugetlb_overcommit_handler(const struct ctl_table *table, int write,
4804		void *buffer, size_t *length, loff_t *ppos)
4805{
4806	struct hstate *h = &default_hstate;
4807	unsigned long tmp;
4808	int ret;
4809
4810	if (!hugepages_supported())
4811		return -EOPNOTSUPP;
4812
4813	tmp = h->nr_overcommit_huge_pages;
4814
4815	if (write && hstate_is_gigantic(h))
4816		return -EINVAL;
4817
4818	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4819					     &tmp);
4820	if (ret)
4821		goto out;
4822
4823	if (write) {
4824		spin_lock_irq(&hugetlb_lock);
4825		h->nr_overcommit_huge_pages = tmp;
4826		spin_unlock_irq(&hugetlb_lock);
4827	}
4828out:
4829	return ret;
4830}
4831
4832static struct ctl_table hugetlb_table[] = {
4833	{
4834		.procname	= "nr_hugepages",
4835		.data		= NULL,
4836		.maxlen		= sizeof(unsigned long),
4837		.mode		= 0644,
4838		.proc_handler	= hugetlb_sysctl_handler,
4839	},
4840#ifdef CONFIG_NUMA
4841	{
4842		.procname       = "nr_hugepages_mempolicy",
4843		.data           = NULL,
4844		.maxlen         = sizeof(unsigned long),
4845		.mode           = 0644,
4846		.proc_handler   = &hugetlb_mempolicy_sysctl_handler,
4847	},
4848#endif
4849	{
4850		.procname	= "hugetlb_shm_group",
4851		.data		= &sysctl_hugetlb_shm_group,
4852		.maxlen		= sizeof(gid_t),
4853		.mode		= 0644,
4854		.proc_handler	= proc_dointvec,
4855	},
4856	{
4857		.procname	= "nr_overcommit_hugepages",
4858		.data		= NULL,
4859		.maxlen		= sizeof(unsigned long),
4860		.mode		= 0644,
4861		.proc_handler	= hugetlb_overcommit_handler,
4862	},
 
4863};
4864
4865static void hugetlb_sysctl_init(void)
4866{
4867	register_sysctl_init("vm", hugetlb_table);
4868}
4869#endif /* CONFIG_SYSCTL */
4870
4871void hugetlb_report_meminfo(struct seq_file *m)
4872{
4873	struct hstate *h;
4874	unsigned long total = 0;
4875
4876	if (!hugepages_supported())
4877		return;
4878
4879	for_each_hstate(h) {
4880		unsigned long count = h->nr_huge_pages;
4881
4882		total += huge_page_size(h) * count;
4883
4884		if (h == &default_hstate)
4885			seq_printf(m,
4886				   "HugePages_Total:   %5lu\n"
4887				   "HugePages_Free:    %5lu\n"
4888				   "HugePages_Rsvd:    %5lu\n"
4889				   "HugePages_Surp:    %5lu\n"
4890				   "Hugepagesize:   %8lu kB\n",
4891				   count,
4892				   h->free_huge_pages,
4893				   h->resv_huge_pages,
4894				   h->surplus_huge_pages,
4895				   huge_page_size(h) / SZ_1K);
4896	}
4897
4898	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
4899}
4900
4901int hugetlb_report_node_meminfo(char *buf, int len, int nid)
4902{
4903	struct hstate *h = &default_hstate;
4904
4905	if (!hugepages_supported())
4906		return 0;
4907
4908	return sysfs_emit_at(buf, len,
4909			     "Node %d HugePages_Total: %5u\n"
4910			     "Node %d HugePages_Free:  %5u\n"
4911			     "Node %d HugePages_Surp:  %5u\n",
4912			     nid, h->nr_huge_pages_node[nid],
4913			     nid, h->free_huge_pages_node[nid],
4914			     nid, h->surplus_huge_pages_node[nid]);
4915}
4916
4917void hugetlb_show_meminfo_node(int nid)
4918{
4919	struct hstate *h;
4920
4921	if (!hugepages_supported())
4922		return;
4923
4924	for_each_hstate(h)
4925		printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
4926			nid,
4927			h->nr_huge_pages_node[nid],
4928			h->free_huge_pages_node[nid],
4929			h->surplus_huge_pages_node[nid],
4930			huge_page_size(h) / SZ_1K);
4931}
4932
4933void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
4934{
4935	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
4936		   K(atomic_long_read(&mm->hugetlb_usage)));
4937}
4938
4939/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
4940unsigned long hugetlb_total_pages(void)
4941{
4942	struct hstate *h;
4943	unsigned long nr_total_pages = 0;
4944
4945	for_each_hstate(h)
4946		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
4947	return nr_total_pages;
4948}
4949
4950static int hugetlb_acct_memory(struct hstate *h, long delta)
4951{
4952	int ret = -ENOMEM;
4953
4954	if (!delta)
4955		return 0;
4956
4957	spin_lock_irq(&hugetlb_lock);
4958	/*
4959	 * When cpuset is configured, it breaks the strict hugetlb page
4960	 * reservation as the accounting is done on a global variable. Such
4961	 * reservation is completely rubbish in the presence of cpuset because
4962	 * the reservation is not checked against page availability for the
4963	 * current cpuset. Application can still potentially OOM'ed by kernel
4964	 * with lack of free htlb page in cpuset that the task is in.
4965	 * Attempt to enforce strict accounting with cpuset is almost
4966	 * impossible (or too ugly) because cpuset is too fluid that
4967	 * task or memory node can be dynamically moved between cpusets.
4968	 *
4969	 * The change of semantics for shared hugetlb mapping with cpuset is
4970	 * undesirable. However, in order to preserve some of the semantics,
4971	 * we fall back to check against current free page availability as
4972	 * a best attempt and hopefully to minimize the impact of changing
4973	 * semantics that cpuset has.
4974	 *
4975	 * Apart from cpuset, we also have memory policy mechanism that
4976	 * also determines from which node the kernel will allocate memory
4977	 * in a NUMA system. So similar to cpuset, we also should consider
4978	 * the memory policy of the current task. Similar to the description
4979	 * above.
4980	 */
4981	if (delta > 0) {
4982		if (gather_surplus_pages(h, delta) < 0)
4983			goto out;
4984
4985		if (delta > allowed_mems_nr(h)) {
4986			return_unused_surplus_pages(h, delta);
4987			goto out;
4988		}
4989	}
4990
4991	ret = 0;
4992	if (delta < 0)
4993		return_unused_surplus_pages(h, (unsigned long) -delta);
4994
4995out:
4996	spin_unlock_irq(&hugetlb_lock);
4997	return ret;
4998}
4999
5000static void hugetlb_vm_op_open(struct vm_area_struct *vma)
5001{
5002	struct resv_map *resv = vma_resv_map(vma);
5003
5004	/*
5005	 * HPAGE_RESV_OWNER indicates a private mapping.
5006	 * This new VMA should share its siblings reservation map if present.
5007	 * The VMA will only ever have a valid reservation map pointer where
5008	 * it is being copied for another still existing VMA.  As that VMA
5009	 * has a reference to the reservation map it cannot disappear until
5010	 * after this open call completes.  It is therefore safe to take a
5011	 * new reference here without additional locking.
5012	 */
5013	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
5014		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
5015		kref_get(&resv->refs);
5016	}
5017
5018	/*
5019	 * vma_lock structure for sharable mappings is vma specific.
5020	 * Clear old pointer (if copied via vm_area_dup) and allocate
5021	 * new structure.  Before clearing, make sure vma_lock is not
5022	 * for this vma.
5023	 */
5024	if (vma->vm_flags & VM_MAYSHARE) {
5025		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
5026
5027		if (vma_lock) {
5028			if (vma_lock->vma != vma) {
5029				vma->vm_private_data = NULL;
5030				hugetlb_vma_lock_alloc(vma);
5031			} else
5032				pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__);
5033		} else
5034			hugetlb_vma_lock_alloc(vma);
5035	}
5036}
5037
5038static void hugetlb_vm_op_close(struct vm_area_struct *vma)
5039{
5040	struct hstate *h = hstate_vma(vma);
5041	struct resv_map *resv;
5042	struct hugepage_subpool *spool = subpool_vma(vma);
5043	unsigned long reserve, start, end;
5044	long gbl_reserve;
5045
5046	hugetlb_vma_lock_free(vma);
5047
5048	resv = vma_resv_map(vma);
5049	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
5050		return;
5051
5052	start = vma_hugecache_offset(h, vma, vma->vm_start);
5053	end = vma_hugecache_offset(h, vma, vma->vm_end);
5054
5055	reserve = (end - start) - region_count(resv, start, end);
5056	hugetlb_cgroup_uncharge_counter(resv, start, end);
5057	if (reserve) {
5058		/*
5059		 * Decrement reserve counts.  The global reserve count may be
5060		 * adjusted if the subpool has a minimum size.
5061		 */
5062		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
5063		hugetlb_acct_memory(h, -gbl_reserve);
5064	}
5065
5066	kref_put(&resv->refs, resv_map_release);
5067}
5068
5069static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
5070{
5071	if (addr & ~(huge_page_mask(hstate_vma(vma))))
5072		return -EINVAL;
5073
5074	/*
5075	 * PMD sharing is only possible for PUD_SIZE-aligned address ranges
5076	 * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this
5077	 * split, unshare PMDs in the PUD_SIZE interval surrounding addr now.
5078	 */
5079	if (addr & ~PUD_MASK) {
5080		/*
5081		 * hugetlb_vm_op_split is called right before we attempt to
5082		 * split the VMA. We will need to unshare PMDs in the old and
5083		 * new VMAs, so let's unshare before we split.
5084		 */
5085		unsigned long floor = addr & PUD_MASK;
5086		unsigned long ceil = floor + PUD_SIZE;
5087
5088		if (floor >= vma->vm_start && ceil <= vma->vm_end)
5089			hugetlb_unshare_pmds(vma, floor, ceil);
5090	}
5091
5092	return 0;
5093}
5094
5095static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
5096{
5097	return huge_page_size(hstate_vma(vma));
5098}
5099
5100/*
5101 * We cannot handle pagefaults against hugetlb pages at all.  They cause
5102 * handle_mm_fault() to try to instantiate regular-sized pages in the
5103 * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
5104 * this far.
5105 */
5106static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
5107{
5108	BUG();
5109	return 0;
5110}
5111
5112/*
5113 * When a new function is introduced to vm_operations_struct and added
5114 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
5115 * This is because under System V memory model, mappings created via
5116 * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
5117 * their original vm_ops are overwritten with shm_vm_ops.
5118 */
5119const struct vm_operations_struct hugetlb_vm_ops = {
5120	.fault = hugetlb_vm_op_fault,
5121	.open = hugetlb_vm_op_open,
5122	.close = hugetlb_vm_op_close,
5123	.may_split = hugetlb_vm_op_split,
5124	.pagesize = hugetlb_vm_op_pagesize,
5125};
5126
5127static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
5128				int writable)
5129{
5130	pte_t entry;
5131	unsigned int shift = huge_page_shift(hstate_vma(vma));
5132
5133	if (writable) {
5134		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
5135					 vma->vm_page_prot)));
5136	} else {
5137		entry = huge_pte_wrprotect(mk_huge_pte(page,
5138					   vma->vm_page_prot));
5139	}
5140	entry = pte_mkyoung(entry);
5141	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
5142
5143	return entry;
5144}
5145
5146static void set_huge_ptep_writable(struct vm_area_struct *vma,
5147				   unsigned long address, pte_t *ptep)
5148{
5149	pte_t entry;
5150
5151	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(vma->vm_mm, address, ptep)));
5152	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
5153		update_mmu_cache(vma, address, ptep);
5154}
5155
5156bool is_hugetlb_entry_migration(pte_t pte)
5157{
5158	swp_entry_t swp;
5159
5160	if (huge_pte_none(pte) || pte_present(pte))
5161		return false;
5162	swp = pte_to_swp_entry(pte);
5163	if (is_migration_entry(swp))
5164		return true;
5165	else
5166		return false;
5167}
5168
5169bool is_hugetlb_entry_hwpoisoned(pte_t pte)
5170{
5171	swp_entry_t swp;
5172
5173	if (huge_pte_none(pte) || pte_present(pte))
5174		return false;
5175	swp = pte_to_swp_entry(pte);
5176	if (is_hwpoison_entry(swp))
5177		return true;
5178	else
5179		return false;
5180}
5181
5182static void
5183hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
5184		      struct folio *new_folio, pte_t old, unsigned long sz)
5185{
5186	pte_t newpte = make_huge_pte(vma, &new_folio->page, 1);
5187
5188	__folio_mark_uptodate(new_folio);
5189	hugetlb_add_new_anon_rmap(new_folio, vma, addr);
5190	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old))
5191		newpte = huge_pte_mkuffd_wp(newpte);
5192	set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
5193	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
5194	folio_set_hugetlb_migratable(new_folio);
5195}
5196
5197int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
5198			    struct vm_area_struct *dst_vma,
5199			    struct vm_area_struct *src_vma)
5200{
5201	pte_t *src_pte, *dst_pte, entry;
5202	struct folio *pte_folio;
5203	unsigned long addr;
5204	bool cow = is_cow_mapping(src_vma->vm_flags);
5205	struct hstate *h = hstate_vma(src_vma);
5206	unsigned long sz = huge_page_size(h);
5207	unsigned long npages = pages_per_huge_page(h);
5208	struct mmu_notifier_range range;
5209	unsigned long last_addr_mask;
5210	int ret = 0;
5211
5212	if (cow) {
5213		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src,
5214					src_vma->vm_start,
5215					src_vma->vm_end);
5216		mmu_notifier_invalidate_range_start(&range);
5217		vma_assert_write_locked(src_vma);
5218		raw_write_seqcount_begin(&src->write_protect_seq);
5219	} else {
5220		/*
5221		 * For shared mappings the vma lock must be held before
5222		 * calling hugetlb_walk() in the src vma. Otherwise, the
5223		 * returned ptep could go away if part of a shared pmd and
5224		 * another thread calls huge_pmd_unshare.
5225		 */
5226		hugetlb_vma_lock_read(src_vma);
5227	}
5228
5229	last_addr_mask = hugetlb_mask_last_page(h);
5230	for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
5231		spinlock_t *src_ptl, *dst_ptl;
5232		src_pte = hugetlb_walk(src_vma, addr, sz);
5233		if (!src_pte) {
5234			addr |= last_addr_mask;
5235			continue;
5236		}
5237		dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
5238		if (!dst_pte) {
5239			ret = -ENOMEM;
5240			break;
5241		}
5242
5243		/*
5244		 * If the pagetables are shared don't copy or take references.
5245		 *
5246		 * dst_pte == src_pte is the common case of src/dest sharing.
5247		 * However, src could have 'unshared' and dst shares with
5248		 * another vma. So page_count of ptep page is checked instead
5249		 * to reliably determine whether pte is shared.
5250		 */
5251		if (page_count(virt_to_page(dst_pte)) > 1) {
5252			addr |= last_addr_mask;
5253			continue;
5254		}
5255
5256		dst_ptl = huge_pte_lock(h, dst, dst_pte);
5257		src_ptl = huge_pte_lockptr(h, src, src_pte);
5258		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5259		entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
5260again:
5261		if (huge_pte_none(entry)) {
5262			/*
5263			 * Skip if src entry none.
5264			 */
5265			;
5266		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
5267			if (!userfaultfd_wp(dst_vma))
5268				entry = huge_pte_clear_uffd_wp(entry);
5269			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5270		} else if (unlikely(is_hugetlb_entry_migration(entry))) {
5271			swp_entry_t swp_entry = pte_to_swp_entry(entry);
5272			bool uffd_wp = pte_swp_uffd_wp(entry);
5273
5274			if (!is_readable_migration_entry(swp_entry) && cow) {
5275				/*
5276				 * COW mappings require pages in both
5277				 * parent and child to be set to read.
5278				 */
5279				swp_entry = make_readable_migration_entry(
5280							swp_offset(swp_entry));
5281				entry = swp_entry_to_pte(swp_entry);
5282				if (userfaultfd_wp(src_vma) && uffd_wp)
5283					entry = pte_swp_mkuffd_wp(entry);
5284				set_huge_pte_at(src, addr, src_pte, entry, sz);
5285			}
5286			if (!userfaultfd_wp(dst_vma))
5287				entry = huge_pte_clear_uffd_wp(entry);
5288			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5289		} else if (unlikely(is_pte_marker(entry))) {
5290			pte_marker marker = copy_pte_marker(
5291				pte_to_swp_entry(entry), dst_vma);
5292
5293			if (marker)
5294				set_huge_pte_at(dst, addr, dst_pte,
5295						make_pte_marker(marker), sz);
5296		} else {
5297			entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
5298			pte_folio = page_folio(pte_page(entry));
5299			folio_get(pte_folio);
5300
5301			/*
5302			 * Failing to duplicate the anon rmap is a rare case
5303			 * where we see pinned hugetlb pages while they're
5304			 * prone to COW. We need to do the COW earlier during
5305			 * fork.
5306			 *
5307			 * When pre-allocating the page or copying data, we
5308			 * need to be without the pgtable locks since we could
5309			 * sleep during the process.
5310			 */
5311			if (!folio_test_anon(pte_folio)) {
5312				hugetlb_add_file_rmap(pte_folio);
5313			} else if (hugetlb_try_dup_anon_rmap(pte_folio, src_vma)) {
5314				pte_t src_pte_old = entry;
5315				struct folio *new_folio;
5316
5317				spin_unlock(src_ptl);
5318				spin_unlock(dst_ptl);
5319				/* Do not use reserve as it's private owned */
5320				new_folio = alloc_hugetlb_folio(dst_vma, addr, 1);
5321				if (IS_ERR(new_folio)) {
5322					folio_put(pte_folio);
5323					ret = PTR_ERR(new_folio);
5324					break;
5325				}
5326				ret = copy_user_large_folio(new_folio, pte_folio,
 
5327							    addr, dst_vma);
5328				folio_put(pte_folio);
5329				if (ret) {
5330					folio_put(new_folio);
5331					break;
5332				}
5333
5334				/* Install the new hugetlb folio if src pte stable */
5335				dst_ptl = huge_pte_lock(h, dst, dst_pte);
5336				src_ptl = huge_pte_lockptr(h, src, src_pte);
5337				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5338				entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
5339				if (!pte_same(src_pte_old, entry)) {
5340					restore_reserve_on_error(h, dst_vma, addr,
5341								new_folio);
5342					folio_put(new_folio);
5343					/* huge_ptep of dst_pte won't change as in child */
5344					goto again;
5345				}
5346				hugetlb_install_folio(dst_vma, dst_pte, addr,
5347						      new_folio, src_pte_old, sz);
5348				spin_unlock(src_ptl);
5349				spin_unlock(dst_ptl);
5350				continue;
5351			}
5352
5353			if (cow) {
5354				/*
5355				 * No need to notify as we are downgrading page
5356				 * table protection not changing it to point
5357				 * to a new page.
5358				 *
5359				 * See Documentation/mm/mmu_notifier.rst
5360				 */
5361				huge_ptep_set_wrprotect(src, addr, src_pte);
5362				entry = huge_pte_wrprotect(entry);
5363			}
5364
5365			if (!userfaultfd_wp(dst_vma))
5366				entry = huge_pte_clear_uffd_wp(entry);
5367
5368			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5369			hugetlb_count_add(npages, dst);
5370		}
5371		spin_unlock(src_ptl);
5372		spin_unlock(dst_ptl);
5373	}
5374
5375	if (cow) {
5376		raw_write_seqcount_end(&src->write_protect_seq);
5377		mmu_notifier_invalidate_range_end(&range);
5378	} else {
5379		hugetlb_vma_unlock_read(src_vma);
5380	}
5381
5382	return ret;
5383}
5384
5385static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
5386			  unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte,
5387			  unsigned long sz)
5388{
5389	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
5390	struct hstate *h = hstate_vma(vma);
5391	struct mm_struct *mm = vma->vm_mm;
5392	spinlock_t *src_ptl, *dst_ptl;
5393	pte_t pte;
5394
5395	dst_ptl = huge_pte_lock(h, mm, dst_pte);
5396	src_ptl = huge_pte_lockptr(h, mm, src_pte);
5397
5398	/*
5399	 * We don't have to worry about the ordering of src and dst ptlocks
5400	 * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
5401	 */
5402	if (src_ptl != dst_ptl)
5403		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5404
5405	pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);
5406
5407	if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
5408		huge_pte_clear(mm, new_addr, dst_pte, sz);
5409	else {
5410		if (need_clear_uffd_wp) {
5411			if (pte_present(pte))
5412				pte = huge_pte_clear_uffd_wp(pte);
5413			else if (is_swap_pte(pte))
5414				pte = pte_swp_clear_uffd_wp(pte);
5415		}
5416		set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
5417	}
5418
5419	if (src_ptl != dst_ptl)
5420		spin_unlock(src_ptl);
5421	spin_unlock(dst_ptl);
5422}
5423
5424int move_hugetlb_page_tables(struct vm_area_struct *vma,
5425			     struct vm_area_struct *new_vma,
5426			     unsigned long old_addr, unsigned long new_addr,
5427			     unsigned long len)
5428{
5429	struct hstate *h = hstate_vma(vma);
5430	struct address_space *mapping = vma->vm_file->f_mapping;
5431	unsigned long sz = huge_page_size(h);
5432	struct mm_struct *mm = vma->vm_mm;
5433	unsigned long old_end = old_addr + len;
5434	unsigned long last_addr_mask;
5435	pte_t *src_pte, *dst_pte;
5436	struct mmu_notifier_range range;
5437	bool shared_pmd = false;
5438
5439	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr,
5440				old_end);
5441	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5442	/*
5443	 * In case of shared PMDs, we should cover the maximum possible
5444	 * range.
5445	 */
5446	flush_cache_range(vma, range.start, range.end);
5447
5448	mmu_notifier_invalidate_range_start(&range);
5449	last_addr_mask = hugetlb_mask_last_page(h);
5450	/* Prevent race with file truncation */
5451	hugetlb_vma_lock_write(vma);
5452	i_mmap_lock_write(mapping);
5453	for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
5454		src_pte = hugetlb_walk(vma, old_addr, sz);
5455		if (!src_pte) {
5456			old_addr |= last_addr_mask;
5457			new_addr |= last_addr_mask;
5458			continue;
5459		}
5460		if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte)))
5461			continue;
5462
5463		if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) {
5464			shared_pmd = true;
5465			old_addr |= last_addr_mask;
5466			new_addr |= last_addr_mask;
5467			continue;
5468		}
5469
5470		dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
5471		if (!dst_pte)
5472			break;
5473
5474		move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
5475	}
5476
5477	if (shared_pmd)
5478		flush_hugetlb_tlb_range(vma, range.start, range.end);
5479	else
5480		flush_hugetlb_tlb_range(vma, old_end - len, old_end);
5481	mmu_notifier_invalidate_range_end(&range);
5482	i_mmap_unlock_write(mapping);
5483	hugetlb_vma_unlock_write(vma);
5484
5485	return len + old_addr - old_end;
5486}
5487
5488void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
5489			    unsigned long start, unsigned long end,
5490			    struct page *ref_page, zap_flags_t zap_flags)
5491{
5492	struct mm_struct *mm = vma->vm_mm;
5493	unsigned long address;
5494	pte_t *ptep;
5495	pte_t pte;
5496	spinlock_t *ptl;
5497	struct page *page;
5498	struct hstate *h = hstate_vma(vma);
5499	unsigned long sz = huge_page_size(h);
5500	bool adjust_reservation = false;
5501	unsigned long last_addr_mask;
5502	bool force_flush = false;
5503
5504	WARN_ON(!is_vm_hugetlb_page(vma));
5505	BUG_ON(start & ~huge_page_mask(h));
5506	BUG_ON(end & ~huge_page_mask(h));
5507
5508	/*
5509	 * This is a hugetlb vma, all the pte entries should point
5510	 * to huge page.
5511	 */
5512	tlb_change_page_size(tlb, sz);
5513	tlb_start_vma(tlb, vma);
5514
5515	last_addr_mask = hugetlb_mask_last_page(h);
5516	address = start;
5517	for (; address < end; address += sz) {
5518		ptep = hugetlb_walk(vma, address, sz);
5519		if (!ptep) {
5520			address |= last_addr_mask;
5521			continue;
5522		}
5523
5524		ptl = huge_pte_lock(h, mm, ptep);
5525		if (huge_pmd_unshare(mm, vma, address, ptep)) {
5526			spin_unlock(ptl);
5527			tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
5528			force_flush = true;
5529			address |= last_addr_mask;
5530			continue;
5531		}
5532
5533		pte = huge_ptep_get(mm, address, ptep);
5534		if (huge_pte_none(pte)) {
5535			spin_unlock(ptl);
5536			continue;
5537		}
5538
5539		/*
5540		 * Migrating hugepage or HWPoisoned hugepage is already
5541		 * unmapped and its refcount is dropped, so just clear pte here.
5542		 */
5543		if (unlikely(!pte_present(pte))) {
5544			/*
5545			 * If the pte was wr-protected by uffd-wp in any of the
5546			 * swap forms, meanwhile the caller does not want to
5547			 * drop the uffd-wp bit in this zap, then replace the
5548			 * pte with a marker.
5549			 */
5550			if (pte_swp_uffd_wp_any(pte) &&
5551			    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5552				set_huge_pte_at(mm, address, ptep,
5553						make_pte_marker(PTE_MARKER_UFFD_WP),
5554						sz);
5555			else
5556				huge_pte_clear(mm, address, ptep, sz);
5557			spin_unlock(ptl);
5558			continue;
5559		}
5560
5561		page = pte_page(pte);
5562		/*
5563		 * If a reference page is supplied, it is because a specific
5564		 * page is being unmapped, not a range. Ensure the page we
5565		 * are about to unmap is the actual page of interest.
5566		 */
5567		if (ref_page) {
5568			if (page != ref_page) {
5569				spin_unlock(ptl);
5570				continue;
5571			}
5572			/*
5573			 * Mark the VMA as having unmapped its page so that
5574			 * future faults in this VMA will fail rather than
5575			 * looking like data was lost
5576			 */
5577			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
5578		}
5579
5580		pte = huge_ptep_get_and_clear(mm, address, ptep, sz);
5581		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5582		if (huge_pte_dirty(pte))
5583			set_page_dirty(page);
5584		/* Leave a uffd-wp pte marker if needed */
5585		if (huge_pte_uffd_wp(pte) &&
5586		    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5587			set_huge_pte_at(mm, address, ptep,
5588					make_pte_marker(PTE_MARKER_UFFD_WP),
5589					sz);
5590		hugetlb_count_sub(pages_per_huge_page(h), mm);
5591		hugetlb_remove_rmap(page_folio(page));
5592
5593		/*
5594		 * Restore the reservation for anonymous page, otherwise the
5595		 * backing page could be stolen by someone.
5596		 * If there we are freeing a surplus, do not set the restore
5597		 * reservation bit.
5598		 */
5599		if (!h->surplus_huge_pages && __vma_private_lock(vma) &&
5600		    folio_test_anon(page_folio(page))) {
5601			folio_set_hugetlb_restore_reserve(page_folio(page));
5602			/* Reservation to be adjusted after the spin lock */
5603			adjust_reservation = true;
5604		}
5605
5606		spin_unlock(ptl);
5607
5608		/*
5609		 * Adjust the reservation for the region that will have the
5610		 * reserve restored. Keep in mind that vma_needs_reservation() changes
5611		 * resv->adds_in_progress if it succeeds. If this is not done,
5612		 * do_exit() will not see it, and will keep the reservation
5613		 * forever.
5614		 */
5615		if (adjust_reservation) {
5616			int rc = vma_needs_reservation(h, vma, address);
5617
5618			if (rc < 0)
5619				/* Pressumably allocate_file_region_entries failed
5620				 * to allocate a file_region struct. Clear
5621				 * hugetlb_restore_reserve so that global reserve
5622				 * count will not be incremented by free_huge_folio.
5623				 * Act as if we consumed the reservation.
5624				 */
5625				folio_clear_hugetlb_restore_reserve(page_folio(page));
5626			else if (rc)
5627				vma_add_reservation(h, vma, address);
5628		}
5629
5630		tlb_remove_page_size(tlb, page, huge_page_size(h));
5631		/*
5632		 * Bail out after unmapping reference page if supplied
5633		 */
5634		if (ref_page)
5635			break;
5636	}
5637	tlb_end_vma(tlb, vma);
5638
5639	/*
5640	 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
5641	 * could defer the flush until now, since by holding i_mmap_rwsem we
5642	 * guaranteed that the last refernece would not be dropped. But we must
5643	 * do the flushing before we return, as otherwise i_mmap_rwsem will be
5644	 * dropped and the last reference to the shared PMDs page might be
5645	 * dropped as well.
5646	 *
5647	 * In theory we could defer the freeing of the PMD pages as well, but
5648	 * huge_pmd_unshare() relies on the exact page_count for the PMD page to
5649	 * detect sharing, so we cannot defer the release of the page either.
5650	 * Instead, do flush now.
5651	 */
5652	if (force_flush)
5653		tlb_flush_mmu_tlbonly(tlb);
5654}
5655
5656void __hugetlb_zap_begin(struct vm_area_struct *vma,
5657			 unsigned long *start, unsigned long *end)
5658{
5659	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
5660		return;
5661
5662	adjust_range_if_pmd_sharing_possible(vma, start, end);
5663	hugetlb_vma_lock_write(vma);
5664	if (vma->vm_file)
5665		i_mmap_lock_write(vma->vm_file->f_mapping);
5666}
5667
5668void __hugetlb_zap_end(struct vm_area_struct *vma,
5669		       struct zap_details *details)
5670{
5671	zap_flags_t zap_flags = details ? details->zap_flags : 0;
5672
5673	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
5674		return;
5675
5676	if (zap_flags & ZAP_FLAG_UNMAP) {	/* final unmap */
5677		/*
5678		 * Unlock and free the vma lock before releasing i_mmap_rwsem.
5679		 * When the vma_lock is freed, this makes the vma ineligible
5680		 * for pmd sharing.  And, i_mmap_rwsem is required to set up
5681		 * pmd sharing.  This is important as page tables for this
5682		 * unmapped range will be asynchrously deleted.  If the page
5683		 * tables are shared, there will be issues when accessed by
5684		 * someone else.
5685		 */
5686		__hugetlb_vma_unlock_write_free(vma);
5687	} else {
5688		hugetlb_vma_unlock_write(vma);
5689	}
5690
5691	if (vma->vm_file)
5692		i_mmap_unlock_write(vma->vm_file->f_mapping);
5693}
5694
5695void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
5696			  unsigned long end, struct page *ref_page,
5697			  zap_flags_t zap_flags)
5698{
5699	struct mmu_notifier_range range;
5700	struct mmu_gather tlb;
5701
5702	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
5703				start, end);
5704	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5705	mmu_notifier_invalidate_range_start(&range);
5706	tlb_gather_mmu(&tlb, vma->vm_mm);
5707
5708	__unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags);
5709
5710	mmu_notifier_invalidate_range_end(&range);
5711	tlb_finish_mmu(&tlb);
5712}
5713
5714/*
5715 * This is called when the original mapper is failing to COW a MAP_PRIVATE
5716 * mapping it owns the reserve page for. The intention is to unmap the page
5717 * from other VMAs and let the children be SIGKILLed if they are faulting the
5718 * same region.
5719 */
5720static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
5721			      struct page *page, unsigned long address)
5722{
5723	struct hstate *h = hstate_vma(vma);
5724	struct vm_area_struct *iter_vma;
5725	struct address_space *mapping;
5726	pgoff_t pgoff;
5727
5728	/*
5729	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
5730	 * from page cache lookup which is in HPAGE_SIZE units.
5731	 */
5732	address = address & huge_page_mask(h);
5733	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
5734			vma->vm_pgoff;
5735	mapping = vma->vm_file->f_mapping;
5736
5737	/*
5738	 * Take the mapping lock for the duration of the table walk. As
5739	 * this mapping should be shared between all the VMAs,
5740	 * __unmap_hugepage_range() is called as the lock is already held
5741	 */
5742	i_mmap_lock_write(mapping);
5743	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
5744		/* Do not unmap the current VMA */
5745		if (iter_vma == vma)
5746			continue;
5747
5748		/*
5749		 * Shared VMAs have their own reserves and do not affect
5750		 * MAP_PRIVATE accounting but it is possible that a shared
5751		 * VMA is using the same page so check and skip such VMAs.
5752		 */
5753		if (iter_vma->vm_flags & VM_MAYSHARE)
5754			continue;
5755
5756		/*
5757		 * Unmap the page from other VMAs without their own reserves.
5758		 * They get marked to be SIGKILLed if they fault in these
5759		 * areas. This is because a future no-page fault on this VMA
5760		 * could insert a zeroed page instead of the data existing
5761		 * from the time of fork. This would look like data corruption
5762		 */
5763		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
5764			unmap_hugepage_range(iter_vma, address,
5765					     address + huge_page_size(h), page, 0);
5766	}
5767	i_mmap_unlock_write(mapping);
5768}
5769
5770/*
5771 * hugetlb_wp() should be called with page lock of the original hugepage held.
5772 * Called with hugetlb_fault_mutex_table held and pte_page locked so we
5773 * cannot race with other handlers or page migration.
5774 * Keep the pte_same checks anyway to make transition from the mutex easier.
5775 */
5776static vm_fault_t hugetlb_wp(struct folio *pagecache_folio,
5777		       struct vm_fault *vmf)
 
5778{
5779	struct vm_area_struct *vma = vmf->vma;
5780	struct mm_struct *mm = vma->vm_mm;
5781	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
5782	pte_t pte = huge_ptep_get(mm, vmf->address, vmf->pte);
5783	struct hstate *h = hstate_vma(vma);
5784	struct folio *old_folio;
5785	struct folio *new_folio;
5786	int outside_reserve = 0;
5787	vm_fault_t ret = 0;
 
5788	struct mmu_notifier_range range;
5789
5790	/*
5791	 * Never handle CoW for uffd-wp protected pages.  It should be only
5792	 * handled when the uffd-wp protection is removed.
5793	 *
5794	 * Note that only the CoW optimization path (in hugetlb_no_page())
5795	 * can trigger this, because hugetlb_fault() will always resolve
5796	 * uffd-wp bit first.
5797	 */
5798	if (!unshare && huge_pte_uffd_wp(pte))
5799		return 0;
5800
5801	/*
5802	 * hugetlb does not support FOLL_FORCE-style write faults that keep the
5803	 * PTE mapped R/O such as maybe_mkwrite() would do.
5804	 */
5805	if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE)))
5806		return VM_FAULT_SIGSEGV;
5807
5808	/* Let's take out MAP_SHARED mappings first. */
5809	if (vma->vm_flags & VM_MAYSHARE) {
5810		set_huge_ptep_writable(vma, vmf->address, vmf->pte);
5811		return 0;
5812	}
5813
5814	old_folio = page_folio(pte_page(pte));
5815
5816	delayacct_wpcopy_start();
5817
5818retry_avoidcopy:
5819	/*
5820	 * If no-one else is actually using this page, we're the exclusive
5821	 * owner and can reuse this page.
5822	 *
5823	 * Note that we don't rely on the (safer) folio refcount here, because
5824	 * copying the hugetlb folio when there are unexpected (temporary)
5825	 * folio references could harm simple fork()+exit() users when
5826	 * we run out of free hugetlb folios: we would have to kill processes
5827	 * in scenarios that used to work. As a side effect, there can still
5828	 * be leaks between processes, for example, with FOLL_GET users.
5829	 */
5830	if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) {
5831		if (!PageAnonExclusive(&old_folio->page)) {
5832			folio_move_anon_rmap(old_folio, vma);
5833			SetPageAnonExclusive(&old_folio->page);
5834		}
5835		if (likely(!unshare))
5836			set_huge_ptep_writable(vma, vmf->address, vmf->pte);
5837
5838		delayacct_wpcopy_end();
5839		return 0;
5840	}
5841	VM_BUG_ON_PAGE(folio_test_anon(old_folio) &&
5842		       PageAnonExclusive(&old_folio->page), &old_folio->page);
5843
5844	/*
5845	 * If the process that created a MAP_PRIVATE mapping is about to
5846	 * perform a COW due to a shared page count, attempt to satisfy
5847	 * the allocation without using the existing reserves. The pagecache
5848	 * page is used to determine if the reserve at this address was
5849	 * consumed or not. If reserves were used, a partial faulted mapping
5850	 * at the time of fork() could consume its reserves on COW instead
5851	 * of the full address range.
5852	 */
5853	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
5854			old_folio != pagecache_folio)
5855		outside_reserve = 1;
5856
5857	folio_get(old_folio);
5858
5859	/*
5860	 * Drop page table lock as buddy allocator may be called. It will
5861	 * be acquired again before returning to the caller, as expected.
5862	 */
5863	spin_unlock(vmf->ptl);
5864	new_folio = alloc_hugetlb_folio(vma, vmf->address, outside_reserve);
5865
5866	if (IS_ERR(new_folio)) {
5867		/*
5868		 * If a process owning a MAP_PRIVATE mapping fails to COW,
5869		 * it is due to references held by a child and an insufficient
5870		 * huge page pool. To guarantee the original mappers
5871		 * reliability, unmap the page from child processes. The child
5872		 * may get SIGKILLed if it later faults.
5873		 */
5874		if (outside_reserve) {
5875			struct address_space *mapping = vma->vm_file->f_mapping;
5876			pgoff_t idx;
5877			u32 hash;
5878
5879			folio_put(old_folio);
5880			/*
5881			 * Drop hugetlb_fault_mutex and vma_lock before
5882			 * unmapping.  unmapping needs to hold vma_lock
5883			 * in write mode.  Dropping vma_lock in read mode
5884			 * here is OK as COW mappings do not interact with
5885			 * PMD sharing.
5886			 *
5887			 * Reacquire both after unmap operation.
5888			 */
5889			idx = vma_hugecache_offset(h, vma, vmf->address);
5890			hash = hugetlb_fault_mutex_hash(mapping, idx);
5891			hugetlb_vma_unlock_read(vma);
5892			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5893
5894			unmap_ref_private(mm, vma, &old_folio->page,
5895					vmf->address);
5896
5897			mutex_lock(&hugetlb_fault_mutex_table[hash]);
5898			hugetlb_vma_lock_read(vma);
5899			spin_lock(vmf->ptl);
5900			vmf->pte = hugetlb_walk(vma, vmf->address,
5901					huge_page_size(h));
5902			if (likely(vmf->pte &&
5903				   pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), pte)))
5904				goto retry_avoidcopy;
5905			/*
5906			 * race occurs while re-acquiring page table
5907			 * lock, and our job is done.
5908			 */
5909			delayacct_wpcopy_end();
5910			return 0;
5911		}
5912
5913		ret = vmf_error(PTR_ERR(new_folio));
5914		goto out_release_old;
5915	}
5916
5917	/*
5918	 * When the original hugepage is shared one, it does not have
5919	 * anon_vma prepared.
5920	 */
5921	ret = __vmf_anon_prepare(vmf);
5922	if (unlikely(ret))
5923		goto out_release_all;
 
5924
5925	if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) {
5926		ret = VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(hstate_index(h));
5927		goto out_release_all;
5928	}
5929	__folio_mark_uptodate(new_folio);
5930
5931	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address,
5932				vmf->address + huge_page_size(h));
5933	mmu_notifier_invalidate_range_start(&range);
5934
5935	/*
5936	 * Retake the page table lock to check for racing updates
5937	 * before the page tables are altered
5938	 */
5939	spin_lock(vmf->ptl);
5940	vmf->pte = hugetlb_walk(vma, vmf->address, huge_page_size(h));
5941	if (likely(vmf->pte && pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), pte))) {
5942		pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare);
5943
5944		/* Break COW or unshare */
5945		huge_ptep_clear_flush(vma, vmf->address, vmf->pte);
5946		hugetlb_remove_rmap(old_folio);
5947		hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address);
5948		if (huge_pte_uffd_wp(pte))
5949			newpte = huge_pte_mkuffd_wp(newpte);
5950		set_huge_pte_at(mm, vmf->address, vmf->pte, newpte,
5951				huge_page_size(h));
5952		folio_set_hugetlb_migratable(new_folio);
5953		/* Make the old page be freed below */
5954		new_folio = old_folio;
5955	}
5956	spin_unlock(vmf->ptl);
5957	mmu_notifier_invalidate_range_end(&range);
5958out_release_all:
5959	/*
5960	 * No restore in case of successful pagetable update (Break COW or
5961	 * unshare)
5962	 */
5963	if (new_folio != old_folio)
5964		restore_reserve_on_error(h, vma, vmf->address, new_folio);
5965	folio_put(new_folio);
5966out_release_old:
5967	folio_put(old_folio);
5968
5969	spin_lock(vmf->ptl); /* Caller expects lock to be held */
5970
5971	delayacct_wpcopy_end();
5972	return ret;
5973}
5974
5975/*
5976 * Return whether there is a pagecache page to back given address within VMA.
5977 */
5978bool hugetlbfs_pagecache_present(struct hstate *h,
5979				 struct vm_area_struct *vma, unsigned long address)
5980{
5981	struct address_space *mapping = vma->vm_file->f_mapping;
5982	pgoff_t idx = linear_page_index(vma, address);
5983	struct folio *folio;
5984
5985	folio = filemap_get_folio(mapping, idx);
5986	if (IS_ERR(folio))
5987		return false;
5988	folio_put(folio);
5989	return true;
5990}
5991
5992int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
5993			   pgoff_t idx)
5994{
5995	struct inode *inode = mapping->host;
5996	struct hstate *h = hstate_inode(inode);
5997	int err;
5998
5999	idx <<= huge_page_order(h);
6000	__folio_set_locked(folio);
6001	err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL);
6002
6003	if (unlikely(err)) {
6004		__folio_clear_locked(folio);
6005		return err;
6006	}
6007	folio_clear_hugetlb_restore_reserve(folio);
6008
6009	/*
6010	 * mark folio dirty so that it will not be removed from cache/file
6011	 * by non-hugetlbfs specific code paths.
6012	 */
6013	folio_mark_dirty(folio);
6014
6015	spin_lock(&inode->i_lock);
6016	inode->i_blocks += blocks_per_huge_page(h);
6017	spin_unlock(&inode->i_lock);
6018	return 0;
6019}
6020
6021static inline vm_fault_t hugetlb_handle_userfault(struct vm_fault *vmf,
6022						  struct address_space *mapping,
 
 
 
 
6023						  unsigned long reason)
6024{
6025	u32 hash;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6026
6027	/*
6028	 * vma_lock and hugetlb_fault_mutex must be dropped before handling
6029	 * userfault. Also mmap_lock could be dropped due to handling
6030	 * userfault, any vma operation should be careful from here.
6031	 */
6032	hugetlb_vma_unlock_read(vmf->vma);
6033	hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
6034	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6035	return handle_userfault(vmf, reason);
6036}
6037
6038/*
6039 * Recheck pte with pgtable lock.  Returns true if pte didn't change, or
6040 * false if pte changed or is changing.
6041 */
6042static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, unsigned long addr,
6043			       pte_t *ptep, pte_t old_pte)
6044{
6045	spinlock_t *ptl;
6046	bool same;
6047
6048	ptl = huge_pte_lock(h, mm, ptep);
6049	same = pte_same(huge_ptep_get(mm, addr, ptep), old_pte);
6050	spin_unlock(ptl);
6051
6052	return same;
6053}
6054
6055static vm_fault_t hugetlb_no_page(struct address_space *mapping,
6056			struct vm_fault *vmf)
 
 
 
6057{
6058	struct vm_area_struct *vma = vmf->vma;
6059	struct mm_struct *mm = vma->vm_mm;
6060	struct hstate *h = hstate_vma(vma);
6061	vm_fault_t ret = VM_FAULT_SIGBUS;
6062	int anon_rmap = 0;
6063	unsigned long size;
6064	struct folio *folio;
6065	pte_t new_pte;
 
 
6066	bool new_folio, new_pagecache_folio = false;
6067	u32 hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
6068
6069	/*
6070	 * Currently, we are forced to kill the process in the event the
6071	 * original mapper has unmapped pages from the child due to a failed
6072	 * COW/unsharing. Warn that such a situation has occurred as it may not
6073	 * be obvious.
6074	 */
6075	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
6076		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
6077			   current->pid);
6078		goto out;
6079	}
6080
6081	/*
6082	 * Use page lock to guard against racing truncation
6083	 * before we get page_table_lock.
6084	 */
6085	new_folio = false;
6086	folio = filemap_lock_hugetlb_folio(h, mapping, vmf->pgoff);
6087	if (IS_ERR(folio)) {
6088		size = i_size_read(mapping->host) >> huge_page_shift(h);
6089		if (vmf->pgoff >= size)
6090			goto out;
6091		/* Check for page in userfault range */
6092		if (userfaultfd_missing(vma)) {
6093			/*
6094			 * Since hugetlb_no_page() was examining pte
6095			 * without pgtable lock, we need to re-test under
6096			 * lock because the pte may not be stable and could
6097			 * have changed from under us.  Try to detect
6098			 * either changed or during-changing ptes and retry
6099			 * properly when needed.
6100			 *
6101			 * Note that userfaultfd is actually fine with
6102			 * false positives (e.g. caused by pte changed),
6103			 * but not wrong logical events (e.g. caused by
6104			 * reading a pte during changing).  The latter can
6105			 * confuse the userspace, so the strictness is very
6106			 * much preferred.  E.g., MISSING event should
6107			 * never happen on the page after UFFDIO_COPY has
6108			 * correctly installed the page and returned.
6109			 */
6110			if (!hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte)) {
6111				ret = 0;
6112				goto out;
6113			}
6114
6115			return hugetlb_handle_userfault(vmf, mapping,
 
6116							VM_UFFD_MISSING);
6117		}
6118
6119		if (!(vma->vm_flags & VM_MAYSHARE)) {
6120			ret = __vmf_anon_prepare(vmf);
6121			if (unlikely(ret))
6122				goto out;
6123		}
6124
6125		folio = alloc_hugetlb_folio(vma, vmf->address, 0);
6126		if (IS_ERR(folio)) {
6127			/*
6128			 * Returning error will result in faulting task being
6129			 * sent SIGBUS.  The hugetlb fault mutex prevents two
6130			 * tasks from racing to fault in the same page which
6131			 * could result in false unable to allocate errors.
6132			 * Page migration does not take the fault mutex, but
6133			 * does a clear then write of pte's under page table
6134			 * lock.  Page fault code could race with migration,
6135			 * notice the clear pte and try to allocate a page
6136			 * here.  Before returning error, get ptl and make
6137			 * sure there really is no pte entry.
6138			 */
6139			if (hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte))
6140				ret = vmf_error(PTR_ERR(folio));
6141			else
6142				ret = 0;
6143			goto out;
6144		}
6145		folio_zero_user(folio, vmf->real_address);
6146		__folio_mark_uptodate(folio);
6147		new_folio = true;
6148
6149		if (vma->vm_flags & VM_MAYSHARE) {
6150			int err = hugetlb_add_to_page_cache(folio, mapping,
6151							vmf->pgoff);
6152			if (err) {
6153				/*
6154				 * err can't be -EEXIST which implies someone
6155				 * else consumed the reservation since hugetlb
6156				 * fault mutex is held when add a hugetlb page
6157				 * to the page cache. So it's safe to call
6158				 * restore_reserve_on_error() here.
6159				 */
6160				restore_reserve_on_error(h, vma, vmf->address,
6161							folio);
6162				folio_put(folio);
6163				ret = VM_FAULT_SIGBUS;
6164				goto out;
6165			}
6166			new_pagecache_folio = true;
6167		} else {
6168			folio_lock(folio);
 
 
 
 
6169			anon_rmap = 1;
6170		}
6171	} else {
6172		/*
6173		 * If memory error occurs between mmap() and fault, some process
6174		 * don't have hwpoisoned swap entry for errored virtual address.
6175		 * So we need to block hugepage fault by PG_hwpoison bit check.
6176		 */
6177		if (unlikely(folio_test_hwpoison(folio))) {
6178			ret = VM_FAULT_HWPOISON_LARGE |
6179				VM_FAULT_SET_HINDEX(hstate_index(h));
6180			goto backout_unlocked;
6181		}
6182
6183		/* Check for page in userfault range. */
6184		if (userfaultfd_minor(vma)) {
6185			folio_unlock(folio);
6186			folio_put(folio);
6187			/* See comment in userfaultfd_missing() block above */
6188			if (!hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte)) {
6189				ret = 0;
6190				goto out;
6191			}
6192			return hugetlb_handle_userfault(vmf, mapping,
 
6193							VM_UFFD_MINOR);
6194		}
6195	}
6196
6197	/*
6198	 * If we are going to COW a private mapping later, we examine the
6199	 * pending reservations for this page now. This will ensure that
6200	 * any allocations necessary to record that reservation occur outside
6201	 * the spinlock.
6202	 */
6203	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
6204		if (vma_needs_reservation(h, vma, vmf->address) < 0) {
6205			ret = VM_FAULT_OOM;
6206			goto backout_unlocked;
6207		}
6208		/* Just decrements count, does not deallocate */
6209		vma_end_reservation(h, vma, vmf->address);
6210	}
6211
6212	vmf->ptl = huge_pte_lock(h, mm, vmf->pte);
6213	ret = 0;
6214	/* If pte changed from under us, retry */
6215	if (!pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), vmf->orig_pte))
6216		goto backout;
6217
6218	if (anon_rmap)
6219		hugetlb_add_new_anon_rmap(folio, vma, vmf->address);
6220	else
6221		hugetlb_add_file_rmap(folio);
6222	new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE)
6223				&& (vma->vm_flags & VM_SHARED)));
6224	/*
6225	 * If this pte was previously wr-protected, keep it wr-protected even
6226	 * if populated.
6227	 */
6228	if (unlikely(pte_marker_uffd_wp(vmf->orig_pte)))
6229		new_pte = huge_pte_mkuffd_wp(new_pte);
6230	set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h));
6231
6232	hugetlb_count_add(pages_per_huge_page(h), mm);
6233	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
6234		/* Optimization, do the COW without a second fault */
6235		ret = hugetlb_wp(folio, vmf);
6236	}
6237
6238	spin_unlock(vmf->ptl);
6239
6240	/*
6241	 * Only set hugetlb_migratable in newly allocated pages.  Existing pages
6242	 * found in the pagecache may not have hugetlb_migratable if they have
6243	 * been isolated for migration.
6244	 */
6245	if (new_folio)
6246		folio_set_hugetlb_migratable(folio);
6247
6248	folio_unlock(folio);
6249out:
6250	hugetlb_vma_unlock_read(vma);
6251
6252	/*
6253	 * We must check to release the per-VMA lock. __vmf_anon_prepare() is
6254	 * the only way ret can be set to VM_FAULT_RETRY.
6255	 */
6256	if (unlikely(ret & VM_FAULT_RETRY))
6257		vma_end_read(vma);
6258
6259	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6260	return ret;
6261
6262backout:
6263	spin_unlock(vmf->ptl);
6264backout_unlocked:
6265	if (new_folio && !new_pagecache_folio)
6266		restore_reserve_on_error(h, vma, vmf->address, folio);
6267
6268	folio_unlock(folio);
6269	folio_put(folio);
6270	goto out;
6271}
6272
6273#ifdef CONFIG_SMP
6274u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
6275{
6276	unsigned long key[2];
6277	u32 hash;
6278
6279	key[0] = (unsigned long) mapping;
6280	key[1] = idx;
6281
6282	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
6283
6284	return hash & (num_fault_mutexes - 1);
6285}
6286#else
6287/*
6288 * For uniprocessor systems we always use a single mutex, so just
6289 * return 0 and avoid the hashing overhead.
6290 */
6291u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
6292{
6293	return 0;
6294}
6295#endif
6296
6297vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
6298			unsigned long address, unsigned int flags)
6299{
 
 
6300	vm_fault_t ret;
6301	u32 hash;
 
6302	struct folio *folio = NULL;
6303	struct folio *pagecache_folio = NULL;
6304	struct hstate *h = hstate_vma(vma);
6305	struct address_space *mapping;
6306	int need_wait_lock = 0;
6307	struct vm_fault vmf = {
6308		.vma = vma,
6309		.address = address & huge_page_mask(h),
6310		.real_address = address,
6311		.flags = flags,
6312		.pgoff = vma_hugecache_offset(h, vma,
6313				address & huge_page_mask(h)),
6314		/* TODO: Track hugetlb faults using vm_fault */
6315
6316		/*
6317		 * Some fields may not be initialized, be careful as it may
6318		 * be hard to debug if called functions make assumptions
6319		 */
6320	};
6321
6322	/*
6323	 * Serialize hugepage allocation and instantiation, so that we don't
6324	 * get spurious allocation failures if two CPUs race to instantiate
6325	 * the same page in the page cache.
6326	 */
6327	mapping = vma->vm_file->f_mapping;
6328	hash = hugetlb_fault_mutex_hash(mapping, vmf.pgoff);
 
6329	mutex_lock(&hugetlb_fault_mutex_table[hash]);
6330
6331	/*
6332	 * Acquire vma lock before calling huge_pte_alloc and hold
6333	 * until finished with vmf.pte.  This prevents huge_pmd_unshare from
6334	 * being called elsewhere and making the vmf.pte no longer valid.
6335	 */
6336	hugetlb_vma_lock_read(vma);
6337	vmf.pte = huge_pte_alloc(mm, vma, vmf.address, huge_page_size(h));
6338	if (!vmf.pte) {
6339		hugetlb_vma_unlock_read(vma);
6340		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6341		return VM_FAULT_OOM;
6342	}
6343
6344	vmf.orig_pte = huge_ptep_get(mm, vmf.address, vmf.pte);
6345	if (huge_pte_none_mostly(vmf.orig_pte)) {
6346		if (is_pte_marker(vmf.orig_pte)) {
6347			pte_marker marker =
6348				pte_marker_get(pte_to_swp_entry(vmf.orig_pte));
6349
6350			if (marker & PTE_MARKER_POISONED) {
6351				ret = VM_FAULT_HWPOISON_LARGE |
6352				      VM_FAULT_SET_HINDEX(hstate_index(h));
6353				goto out_mutex;
6354			} else if (WARN_ON_ONCE(marker & PTE_MARKER_GUARD)) {
6355				/* This isn't supported in hugetlb. */
6356				ret = VM_FAULT_SIGSEGV;
6357				goto out_mutex;
6358			}
6359		}
6360
6361		/*
6362		 * Other PTE markers should be handled the same way as none PTE.
6363		 *
6364		 * hugetlb_no_page will drop vma lock and hugetlb fault
6365		 * mutex internally, which make us return immediately.
6366		 */
6367		return hugetlb_no_page(mapping, &vmf);
 
6368	}
6369
6370	ret = 0;
6371
6372	/*
6373	 * vmf.orig_pte could be a migration/hwpoison vmf.orig_pte at this
6374	 * point, so this check prevents the kernel from going below assuming
6375	 * that we have an active hugepage in pagecache. This goto expects
6376	 * the 2nd page fault, and is_hugetlb_entry_(migration|hwpoisoned)
6377	 * check will properly handle it.
6378	 */
6379	if (!pte_present(vmf.orig_pte)) {
6380		if (unlikely(is_hugetlb_entry_migration(vmf.orig_pte))) {
6381			/*
6382			 * Release the hugetlb fault lock now, but retain
6383			 * the vma lock, because it is needed to guard the
6384			 * huge_pte_lockptr() later in
6385			 * migration_entry_wait_huge(). The vma lock will
6386			 * be released there.
6387			 */
6388			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6389			migration_entry_wait_huge(vma, vmf.address, vmf.pte);
6390			return 0;
6391		} else if (unlikely(is_hugetlb_entry_hwpoisoned(vmf.orig_pte)))
6392			ret = VM_FAULT_HWPOISON_LARGE |
6393			    VM_FAULT_SET_HINDEX(hstate_index(h));
6394		goto out_mutex;
6395	}
6396
6397	/*
6398	 * If we are going to COW/unshare the mapping later, we examine the
6399	 * pending reservations for this page now. This will ensure that any
6400	 * allocations necessary to record that reservation occur outside the
6401	 * spinlock. Also lookup the pagecache page now as it is used to
6402	 * determine if a reservation has been consumed.
6403	 */
6404	if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
6405	    !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf.orig_pte)) {
6406		if (vma_needs_reservation(h, vma, vmf.address) < 0) {
6407			ret = VM_FAULT_OOM;
6408			goto out_mutex;
6409		}
6410		/* Just decrements count, does not deallocate */
6411		vma_end_reservation(h, vma, vmf.address);
6412
6413		pagecache_folio = filemap_lock_hugetlb_folio(h, mapping,
6414							     vmf.pgoff);
6415		if (IS_ERR(pagecache_folio))
6416			pagecache_folio = NULL;
6417	}
6418
6419	vmf.ptl = huge_pte_lock(h, mm, vmf.pte);
6420
6421	/* Check for a racing update before calling hugetlb_wp() */
6422	if (unlikely(!pte_same(vmf.orig_pte, huge_ptep_get(mm, vmf.address, vmf.pte))))
6423		goto out_ptl;
6424
6425	/* Handle userfault-wp first, before trying to lock more pages */
6426	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(mm, vmf.address, vmf.pte)) &&
6427	    (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) {
6428		if (!userfaultfd_wp_async(vma)) {
6429			spin_unlock(vmf.ptl);
 
 
 
 
 
 
 
6430			if (pagecache_folio) {
6431				folio_unlock(pagecache_folio);
6432				folio_put(pagecache_folio);
6433			}
6434			hugetlb_vma_unlock_read(vma);
6435			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6436			return handle_userfault(&vmf, VM_UFFD_WP);
6437		}
6438
6439		vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte);
6440		set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte,
6441				huge_page_size(hstate_vma(vma)));
6442		/* Fallthrough to CoW */
6443	}
6444
6445	/*
6446	 * hugetlb_wp() requires page locks of pte_page(vmf.orig_pte) and
6447	 * pagecache_folio, so here we need take the former one
6448	 * when folio != pagecache_folio or !pagecache_folio.
6449	 */
6450	folio = page_folio(pte_page(vmf.orig_pte));
6451	if (folio != pagecache_folio)
6452		if (!folio_trylock(folio)) {
6453			need_wait_lock = 1;
6454			goto out_ptl;
6455		}
6456
6457	folio_get(folio);
6458
6459	if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
6460		if (!huge_pte_write(vmf.orig_pte)) {
6461			ret = hugetlb_wp(pagecache_folio, &vmf);
 
6462			goto out_put_page;
6463		} else if (likely(flags & FAULT_FLAG_WRITE)) {
6464			vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
6465		}
6466	}
6467	vmf.orig_pte = pte_mkyoung(vmf.orig_pte);
6468	if (huge_ptep_set_access_flags(vma, vmf.address, vmf.pte, vmf.orig_pte,
6469						flags & FAULT_FLAG_WRITE))
6470		update_mmu_cache(vma, vmf.address, vmf.pte);
6471out_put_page:
6472	if (folio != pagecache_folio)
6473		folio_unlock(folio);
6474	folio_put(folio);
6475out_ptl:
6476	spin_unlock(vmf.ptl);
6477
6478	if (pagecache_folio) {
6479		folio_unlock(pagecache_folio);
6480		folio_put(pagecache_folio);
6481	}
6482out_mutex:
6483	hugetlb_vma_unlock_read(vma);
6484
6485	/*
6486	 * We must check to release the per-VMA lock. __vmf_anon_prepare() in
6487	 * hugetlb_wp() is the only way ret can be set to VM_FAULT_RETRY.
6488	 */
6489	if (unlikely(ret & VM_FAULT_RETRY))
6490		vma_end_read(vma);
6491
6492	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6493	/*
6494	 * Generally it's safe to hold refcount during waiting page lock. But
6495	 * here we just wait to defer the next page fault to avoid busy loop and
6496	 * the page is not used after unlocked before returning from the current
6497	 * page fault. So we are safe from accessing freed page, even if we wait
6498	 * here without taking refcount.
6499	 */
6500	if (need_wait_lock)
6501		folio_wait_locked(folio);
6502	return ret;
6503}
6504
6505#ifdef CONFIG_USERFAULTFD
6506/*
6507 * Can probably be eliminated, but still used by hugetlb_mfill_atomic_pte().
6508 */
6509static struct folio *alloc_hugetlb_folio_vma(struct hstate *h,
6510		struct vm_area_struct *vma, unsigned long address)
6511{
6512	struct mempolicy *mpol;
6513	nodemask_t *nodemask;
6514	struct folio *folio;
6515	gfp_t gfp_mask;
6516	int node;
6517
6518	gfp_mask = htlb_alloc_mask(h);
6519	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
6520	/*
6521	 * This is used to allocate a temporary hugetlb to hold the copied
6522	 * content, which will then be copied again to the final hugetlb
6523	 * consuming a reservation. Set the alloc_fallback to false to indicate
6524	 * that breaking the per-node hugetlb pool is not allowed in this case.
6525	 */
6526	folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask, false);
6527	mpol_cond_put(mpol);
6528
6529	return folio;
6530}
6531
6532/*
6533 * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte
6534 * with modifications for hugetlb pages.
6535 */
6536int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
6537			     struct vm_area_struct *dst_vma,
6538			     unsigned long dst_addr,
6539			     unsigned long src_addr,
6540			     uffd_flags_t flags,
6541			     struct folio **foliop)
6542{
6543	struct mm_struct *dst_mm = dst_vma->vm_mm;
6544	bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE);
6545	bool wp_enabled = (flags & MFILL_ATOMIC_WP);
6546	struct hstate *h = hstate_vma(dst_vma);
6547	struct address_space *mapping = dst_vma->vm_file->f_mapping;
6548	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
6549	unsigned long size = huge_page_size(h);
6550	int vm_shared = dst_vma->vm_flags & VM_SHARED;
6551	pte_t _dst_pte;
6552	spinlock_t *ptl;
6553	int ret = -ENOMEM;
6554	struct folio *folio;
6555	int writable;
6556	bool folio_in_pagecache = false;
6557
6558	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) {
6559		ptl = huge_pte_lock(h, dst_mm, dst_pte);
6560
6561		/* Don't overwrite any existing PTEs (even markers) */
6562		if (!huge_pte_none(huge_ptep_get(dst_mm, dst_addr, dst_pte))) {
6563			spin_unlock(ptl);
6564			return -EEXIST;
6565		}
6566
6567		_dst_pte = make_pte_marker(PTE_MARKER_POISONED);
6568		set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);
 
6569
6570		/* No need to invalidate - it was non-present before */
6571		update_mmu_cache(dst_vma, dst_addr, dst_pte);
6572
6573		spin_unlock(ptl);
6574		return 0;
6575	}
6576
6577	if (is_continue) {
6578		ret = -EFAULT;
6579		folio = filemap_lock_hugetlb_folio(h, mapping, idx);
6580		if (IS_ERR(folio))
6581			goto out;
6582		folio_in_pagecache = true;
6583	} else if (!*foliop) {
6584		/* If a folio already exists, then it's UFFDIO_COPY for
6585		 * a non-missing case. Return -EEXIST.
6586		 */
6587		if (vm_shared &&
6588		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6589			ret = -EEXIST;
6590			goto out;
6591		}
6592
6593		folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0);
6594		if (IS_ERR(folio)) {
6595			ret = -ENOMEM;
6596			goto out;
6597		}
6598
6599		ret = copy_folio_from_user(folio, (const void __user *) src_addr,
6600					   false);
6601
6602		/* fallback to copy_from_user outside mmap_lock */
6603		if (unlikely(ret)) {
6604			ret = -ENOENT;
6605			/* Free the allocated folio which may have
6606			 * consumed a reservation.
6607			 */
6608			restore_reserve_on_error(h, dst_vma, dst_addr, folio);
6609			folio_put(folio);
6610
6611			/* Allocate a temporary folio to hold the copied
6612			 * contents.
6613			 */
6614			folio = alloc_hugetlb_folio_vma(h, dst_vma, dst_addr);
6615			if (!folio) {
6616				ret = -ENOMEM;
6617				goto out;
6618			}
6619			*foliop = folio;
6620			/* Set the outparam foliop and return to the caller to
6621			 * copy the contents outside the lock. Don't free the
6622			 * folio.
6623			 */
6624			goto out;
6625		}
6626	} else {
6627		if (vm_shared &&
6628		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6629			folio_put(*foliop);
6630			ret = -EEXIST;
6631			*foliop = NULL;
6632			goto out;
6633		}
6634
6635		folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0);
6636		if (IS_ERR(folio)) {
6637			folio_put(*foliop);
6638			ret = -ENOMEM;
6639			*foliop = NULL;
6640			goto out;
6641		}
6642		ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma);
6643		folio_put(*foliop);
6644		*foliop = NULL;
6645		if (ret) {
6646			folio_put(folio);
6647			goto out;
6648		}
6649	}
6650
6651	/*
6652	 * If we just allocated a new page, we need a memory barrier to ensure
6653	 * that preceding stores to the page become visible before the
6654	 * set_pte_at() write. The memory barrier inside __folio_mark_uptodate
6655	 * is what we need.
6656	 *
6657	 * In the case where we have not allocated a new page (is_continue),
6658	 * the page must already be uptodate. UFFDIO_CONTINUE already includes
6659	 * an earlier smp_wmb() to ensure that prior stores will be visible
6660	 * before the set_pte_at() write.
6661	 */
6662	if (!is_continue)
6663		__folio_mark_uptodate(folio);
6664	else
6665		WARN_ON_ONCE(!folio_test_uptodate(folio));
6666
6667	/* Add shared, newly allocated pages to the page cache. */
6668	if (vm_shared && !is_continue) {
 
6669		ret = -EFAULT;
6670		if (idx >= (i_size_read(mapping->host) >> huge_page_shift(h)))
6671			goto out_release_nounlock;
6672
6673		/*
6674		 * Serialization between remove_inode_hugepages() and
6675		 * hugetlb_add_to_page_cache() below happens through the
6676		 * hugetlb_fault_mutex_table that here must be hold by
6677		 * the caller.
6678		 */
6679		ret = hugetlb_add_to_page_cache(folio, mapping, idx);
6680		if (ret)
6681			goto out_release_nounlock;
6682		folio_in_pagecache = true;
6683	}
6684
6685	ptl = huge_pte_lock(h, dst_mm, dst_pte);
6686
6687	ret = -EIO;
6688	if (folio_test_hwpoison(folio))
6689		goto out_release_unlock;
6690
6691	/*
6692	 * We allow to overwrite a pte marker: consider when both MISSING|WP
6693	 * registered, we firstly wr-protect a none pte which has no page cache
6694	 * page backing it, then access the page.
6695	 */
6696	ret = -EEXIST;
6697	if (!huge_pte_none_mostly(huge_ptep_get(dst_mm, dst_addr, dst_pte)))
6698		goto out_release_unlock;
6699
6700	if (folio_in_pagecache)
6701		hugetlb_add_file_rmap(folio);
6702	else
6703		hugetlb_add_new_anon_rmap(folio, dst_vma, dst_addr);
6704
6705	/*
6706	 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
6707	 * with wp flag set, don't set pte write bit.
6708	 */
6709	if (wp_enabled || (is_continue && !vm_shared))
6710		writable = 0;
6711	else
6712		writable = dst_vma->vm_flags & VM_WRITE;
6713
6714	_dst_pte = make_huge_pte(dst_vma, &folio->page, writable);
6715	/*
6716	 * Always mark UFFDIO_COPY page dirty; note that this may not be
6717	 * extremely important for hugetlbfs for now since swapping is not
6718	 * supported, but we should still be clear in that this page cannot be
6719	 * thrown away at will, even if write bit not set.
6720	 */
6721	_dst_pte = huge_pte_mkdirty(_dst_pte);
6722	_dst_pte = pte_mkyoung(_dst_pte);
6723
6724	if (wp_enabled)
6725		_dst_pte = huge_pte_mkuffd_wp(_dst_pte);
6726
6727	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);
6728
6729	hugetlb_count_add(pages_per_huge_page(h), dst_mm);
6730
6731	/* No need to invalidate - it was non-present before */
6732	update_mmu_cache(dst_vma, dst_addr, dst_pte);
6733
6734	spin_unlock(ptl);
6735	if (!is_continue)
6736		folio_set_hugetlb_migratable(folio);
6737	if (vm_shared || is_continue)
6738		folio_unlock(folio);
6739	ret = 0;
6740out:
6741	return ret;
6742out_release_unlock:
6743	spin_unlock(ptl);
6744	if (vm_shared || is_continue)
6745		folio_unlock(folio);
6746out_release_nounlock:
6747	if (!folio_in_pagecache)
6748		restore_reserve_on_error(h, dst_vma, dst_addr, folio);
6749	folio_put(folio);
6750	goto out;
6751}
6752#endif /* CONFIG_USERFAULTFD */
6753
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6754long hugetlb_change_protection(struct vm_area_struct *vma,
6755		unsigned long address, unsigned long end,
6756		pgprot_t newprot, unsigned long cp_flags)
6757{
6758	struct mm_struct *mm = vma->vm_mm;
6759	unsigned long start = address;
6760	pte_t *ptep;
6761	pte_t pte;
6762	struct hstate *h = hstate_vma(vma);
6763	long pages = 0, psize = huge_page_size(h);
6764	bool shared_pmd = false;
6765	struct mmu_notifier_range range;
6766	unsigned long last_addr_mask;
6767	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
6768	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
6769
6770	/*
6771	 * In the case of shared PMDs, the area to flush could be beyond
6772	 * start/end.  Set range.start/range.end to cover the maximum possible
6773	 * range if PMD sharing is possible.
6774	 */
6775	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
6776				0, mm, start, end);
6777	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
6778
6779	BUG_ON(address >= end);
6780	flush_cache_range(vma, range.start, range.end);
6781
6782	mmu_notifier_invalidate_range_start(&range);
6783	hugetlb_vma_lock_write(vma);
6784	i_mmap_lock_write(vma->vm_file->f_mapping);
6785	last_addr_mask = hugetlb_mask_last_page(h);
6786	for (; address < end; address += psize) {
6787		spinlock_t *ptl;
6788		ptep = hugetlb_walk(vma, address, psize);
6789		if (!ptep) {
6790			if (!uffd_wp) {
6791				address |= last_addr_mask;
6792				continue;
6793			}
6794			/*
6795			 * Userfaultfd wr-protect requires pgtable
6796			 * pre-allocations to install pte markers.
6797			 */
6798			ptep = huge_pte_alloc(mm, vma, address, psize);
6799			if (!ptep) {
6800				pages = -ENOMEM;
6801				break;
6802			}
6803		}
6804		ptl = huge_pte_lock(h, mm, ptep);
6805		if (huge_pmd_unshare(mm, vma, address, ptep)) {
6806			/*
6807			 * When uffd-wp is enabled on the vma, unshare
6808			 * shouldn't happen at all.  Warn about it if it
6809			 * happened due to some reason.
6810			 */
6811			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
6812			pages++;
6813			spin_unlock(ptl);
6814			shared_pmd = true;
6815			address |= last_addr_mask;
6816			continue;
6817		}
6818		pte = huge_ptep_get(mm, address, ptep);
6819		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
6820			/* Nothing to do. */
6821		} else if (unlikely(is_hugetlb_entry_migration(pte))) {
6822			swp_entry_t entry = pte_to_swp_entry(pte);
6823			struct page *page = pfn_swap_entry_to_page(entry);
6824			pte_t newpte = pte;
6825
6826			if (is_writable_migration_entry(entry)) {
6827				if (PageAnon(page))
6828					entry = make_readable_exclusive_migration_entry(
6829								swp_offset(entry));
6830				else
6831					entry = make_readable_migration_entry(
6832								swp_offset(entry));
6833				newpte = swp_entry_to_pte(entry);
6834				pages++;
6835			}
6836
6837			if (uffd_wp)
6838				newpte = pte_swp_mkuffd_wp(newpte);
6839			else if (uffd_wp_resolve)
6840				newpte = pte_swp_clear_uffd_wp(newpte);
6841			if (!pte_same(pte, newpte))
6842				set_huge_pte_at(mm, address, ptep, newpte, psize);
6843		} else if (unlikely(is_pte_marker(pte))) {
6844			/*
6845			 * Do nothing on a poison marker; page is
6846			 * corrupted, permissons do not apply.  Here
6847			 * pte_marker_uffd_wp()==true implies !poison
6848			 * because they're mutual exclusive.
6849			 */
6850			if (pte_marker_uffd_wp(pte) && uffd_wp_resolve)
6851				/* Safe to modify directly (non-present->none). */
6852				huge_pte_clear(mm, address, ptep, psize);
6853		} else if (!huge_pte_none(pte)) {
6854			pte_t old_pte;
6855			unsigned int shift = huge_page_shift(hstate_vma(vma));
6856
6857			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
6858			pte = huge_pte_modify(old_pte, newprot);
6859			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
6860			if (uffd_wp)
6861				pte = huge_pte_mkuffd_wp(pte);
6862			else if (uffd_wp_resolve)
6863				pte = huge_pte_clear_uffd_wp(pte);
6864			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
6865			pages++;
6866		} else {
6867			/* None pte */
6868			if (unlikely(uffd_wp))
6869				/* Safe to modify directly (none->non-present). */
6870				set_huge_pte_at(mm, address, ptep,
6871						make_pte_marker(PTE_MARKER_UFFD_WP),
6872						psize);
6873		}
6874		spin_unlock(ptl);
6875	}
6876	/*
6877	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
6878	 * may have cleared our pud entry and done put_page on the page table:
6879	 * once we release i_mmap_rwsem, another task can do the final put_page
6880	 * and that page table be reused and filled with junk.  If we actually
6881	 * did unshare a page of pmds, flush the range corresponding to the pud.
6882	 */
6883	if (shared_pmd)
6884		flush_hugetlb_tlb_range(vma, range.start, range.end);
6885	else
6886		flush_hugetlb_tlb_range(vma, start, end);
6887	/*
6888	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are
6889	 * downgrading page table protection not changing it to point to a new
6890	 * page.
6891	 *
6892	 * See Documentation/mm/mmu_notifier.rst
6893	 */
6894	i_mmap_unlock_write(vma->vm_file->f_mapping);
6895	hugetlb_vma_unlock_write(vma);
6896	mmu_notifier_invalidate_range_end(&range);
6897
6898	return pages > 0 ? (pages << h->order) : pages;
6899}
6900
6901/* Return true if reservation was successful, false otherwise.  */
6902bool hugetlb_reserve_pages(struct inode *inode,
6903					long from, long to,
6904					struct vm_area_struct *vma,
6905					vm_flags_t vm_flags)
6906{
6907	long chg = -1, add = -1;
6908	struct hstate *h = hstate_inode(inode);
6909	struct hugepage_subpool *spool = subpool_inode(inode);
6910	struct resv_map *resv_map;
6911	struct hugetlb_cgroup *h_cg = NULL;
6912	long gbl_reserve, regions_needed = 0;
6913
6914	/* This should never happen */
6915	if (from > to) {
6916		VM_WARN(1, "%s called with a negative range\n", __func__);
6917		return false;
6918	}
6919
6920	/*
6921	 * vma specific semaphore used for pmd sharing and fault/truncation
6922	 * synchronization
6923	 */
6924	hugetlb_vma_lock_alloc(vma);
6925
6926	/*
6927	 * Only apply hugepage reservation if asked. At fault time, an
6928	 * attempt will be made for VM_NORESERVE to allocate a page
6929	 * without using reserves
6930	 */
6931	if (vm_flags & VM_NORESERVE)
6932		return true;
6933
6934	/*
6935	 * Shared mappings base their reservation on the number of pages that
6936	 * are already allocated on behalf of the file. Private mappings need
6937	 * to reserve the full area even if read-only as mprotect() may be
6938	 * called to make the mapping read-write. Assume !vma is a shm mapping
6939	 */
6940	if (!vma || vma->vm_flags & VM_MAYSHARE) {
6941		/*
6942		 * resv_map can not be NULL as hugetlb_reserve_pages is only
6943		 * called for inodes for which resv_maps were created (see
6944		 * hugetlbfs_get_inode).
6945		 */
6946		resv_map = inode_resv_map(inode);
6947
6948		chg = region_chg(resv_map, from, to, &regions_needed);
6949	} else {
6950		/* Private mapping. */
6951		resv_map = resv_map_alloc();
6952		if (!resv_map)
6953			goto out_err;
6954
6955		chg = to - from;
6956
6957		set_vma_resv_map(vma, resv_map);
6958		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
6959	}
6960
6961	if (chg < 0)
6962		goto out_err;
6963
6964	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
6965				chg * pages_per_huge_page(h), &h_cg) < 0)
6966		goto out_err;
6967
6968	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
6969		/* For private mappings, the hugetlb_cgroup uncharge info hangs
6970		 * of the resv_map.
6971		 */
6972		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
6973	}
6974
6975	/*
6976	 * There must be enough pages in the subpool for the mapping. If
6977	 * the subpool has a minimum size, there may be some global
6978	 * reservations already in place (gbl_reserve).
6979	 */
6980	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
6981	if (gbl_reserve < 0)
6982		goto out_uncharge_cgroup;
6983
6984	/*
6985	 * Check enough hugepages are available for the reservation.
6986	 * Hand the pages back to the subpool if there are not
6987	 */
6988	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
6989		goto out_put_pages;
6990
6991	/*
6992	 * Account for the reservations made. Shared mappings record regions
6993	 * that have reservations as they are shared by multiple VMAs.
6994	 * When the last VMA disappears, the region map says how much
6995	 * the reservation was and the page cache tells how much of
6996	 * the reservation was consumed. Private mappings are per-VMA and
6997	 * only the consumed reservations are tracked. When the VMA
6998	 * disappears, the original reservation is the VMA size and the
6999	 * consumed reservations are stored in the map. Hence, nothing
7000	 * else has to be done for private mappings here
7001	 */
7002	if (!vma || vma->vm_flags & VM_MAYSHARE) {
7003		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
7004
7005		if (unlikely(add < 0)) {
7006			hugetlb_acct_memory(h, -gbl_reserve);
7007			goto out_put_pages;
7008		} else if (unlikely(chg > add)) {
7009			/*
7010			 * pages in this range were added to the reserve
7011			 * map between region_chg and region_add.  This
7012			 * indicates a race with alloc_hugetlb_folio.  Adjust
7013			 * the subpool and reserve counts modified above
7014			 * based on the difference.
7015			 */
7016			long rsv_adjust;
7017
7018			/*
7019			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
7020			 * reference to h_cg->css. See comment below for detail.
7021			 */
7022			hugetlb_cgroup_uncharge_cgroup_rsvd(
7023				hstate_index(h),
7024				(chg - add) * pages_per_huge_page(h), h_cg);
7025
7026			rsv_adjust = hugepage_subpool_put_pages(spool,
7027								chg - add);
7028			hugetlb_acct_memory(h, -rsv_adjust);
7029		} else if (h_cg) {
7030			/*
7031			 * The file_regions will hold their own reference to
7032			 * h_cg->css. So we should release the reference held
7033			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
7034			 * done.
7035			 */
7036			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
7037		}
7038	}
7039	return true;
7040
7041out_put_pages:
7042	/* put back original number of pages, chg */
7043	(void)hugepage_subpool_put_pages(spool, chg);
7044out_uncharge_cgroup:
7045	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
7046					    chg * pages_per_huge_page(h), h_cg);
7047out_err:
7048	hugetlb_vma_lock_free(vma);
7049	if (!vma || vma->vm_flags & VM_MAYSHARE)
7050		/* Only call region_abort if the region_chg succeeded but the
7051		 * region_add failed or didn't run.
7052		 */
7053		if (chg >= 0 && add < 0)
7054			region_abort(resv_map, from, to, regions_needed);
7055	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
7056		kref_put(&resv_map->refs, resv_map_release);
7057		set_vma_resv_map(vma, NULL);
7058	}
7059	return false;
7060}
7061
7062long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
7063								long freed)
7064{
7065	struct hstate *h = hstate_inode(inode);
7066	struct resv_map *resv_map = inode_resv_map(inode);
7067	long chg = 0;
7068	struct hugepage_subpool *spool = subpool_inode(inode);
7069	long gbl_reserve;
7070
7071	/*
7072	 * Since this routine can be called in the evict inode path for all
7073	 * hugetlbfs inodes, resv_map could be NULL.
7074	 */
7075	if (resv_map) {
7076		chg = region_del(resv_map, start, end);
7077		/*
7078		 * region_del() can fail in the rare case where a region
7079		 * must be split and another region descriptor can not be
7080		 * allocated.  If end == LONG_MAX, it will not fail.
7081		 */
7082		if (chg < 0)
7083			return chg;
7084	}
7085
7086	spin_lock(&inode->i_lock);
7087	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
7088	spin_unlock(&inode->i_lock);
7089
7090	/*
7091	 * If the subpool has a minimum size, the number of global
7092	 * reservations to be released may be adjusted.
7093	 *
7094	 * Note that !resv_map implies freed == 0. So (chg - freed)
7095	 * won't go negative.
7096	 */
7097	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
7098	hugetlb_acct_memory(h, -gbl_reserve);
7099
7100	return 0;
7101}
7102
7103#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
7104static unsigned long page_table_shareable(struct vm_area_struct *svma,
7105				struct vm_area_struct *vma,
7106				unsigned long addr, pgoff_t idx)
7107{
7108	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
7109				svma->vm_start;
7110	unsigned long sbase = saddr & PUD_MASK;
7111	unsigned long s_end = sbase + PUD_SIZE;
7112
7113	/* Allow segments to share if only one is marked locked */
7114	unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED_MASK;
7115	unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED_MASK;
7116
7117	/*
7118	 * match the virtual addresses, permission and the alignment of the
7119	 * page table page.
7120	 *
7121	 * Also, vma_lock (vm_private_data) is required for sharing.
7122	 */
7123	if (pmd_index(addr) != pmd_index(saddr) ||
7124	    vm_flags != svm_flags ||
7125	    !range_in_vma(svma, sbase, s_end) ||
7126	    !svma->vm_private_data)
7127		return 0;
7128
7129	return saddr;
7130}
7131
7132bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
7133{
7134	unsigned long start = addr & PUD_MASK;
7135	unsigned long end = start + PUD_SIZE;
7136
7137#ifdef CONFIG_USERFAULTFD
7138	if (uffd_disable_huge_pmd_share(vma))
7139		return false;
7140#endif
7141	/*
7142	 * check on proper vm_flags and page table alignment
7143	 */
7144	if (!(vma->vm_flags & VM_MAYSHARE))
7145		return false;
7146	if (!vma->vm_private_data)	/* vma lock required for sharing */
7147		return false;
7148	if (!range_in_vma(vma, start, end))
7149		return false;
7150	return true;
7151}
7152
7153/*
7154 * Determine if start,end range within vma could be mapped by shared pmd.
7155 * If yes, adjust start and end to cover range associated with possible
7156 * shared pmd mappings.
7157 */
7158void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
7159				unsigned long *start, unsigned long *end)
7160{
7161	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
7162		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
7163
7164	/*
7165	 * vma needs to span at least one aligned PUD size, and the range
7166	 * must be at least partially within in.
7167	 */
7168	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
7169		(*end <= v_start) || (*start >= v_end))
7170		return;
7171
7172	/* Extend the range to be PUD aligned for a worst case scenario */
7173	if (*start > v_start)
7174		*start = ALIGN_DOWN(*start, PUD_SIZE);
7175
7176	if (*end < v_end)
7177		*end = ALIGN(*end, PUD_SIZE);
7178}
7179
7180/*
7181 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
7182 * and returns the corresponding pte. While this is not necessary for the
7183 * !shared pmd case because we can allocate the pmd later as well, it makes the
7184 * code much cleaner. pmd allocation is essential for the shared case because
7185 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
7186 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
7187 * bad pmd for sharing.
7188 */
7189pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
7190		      unsigned long addr, pud_t *pud)
7191{
7192	struct address_space *mapping = vma->vm_file->f_mapping;
7193	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
7194			vma->vm_pgoff;
7195	struct vm_area_struct *svma;
7196	unsigned long saddr;
7197	pte_t *spte = NULL;
7198	pte_t *pte;
7199
7200	i_mmap_lock_read(mapping);
7201	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
7202		if (svma == vma)
7203			continue;
7204
7205		saddr = page_table_shareable(svma, vma, addr, idx);
7206		if (saddr) {
7207			spte = hugetlb_walk(svma, saddr,
7208					    vma_mmu_pagesize(svma));
7209			if (spte) {
7210				ptdesc_pmd_pts_inc(virt_to_ptdesc(spte));
7211				break;
7212			}
7213		}
7214	}
7215
7216	if (!spte)
7217		goto out;
7218
7219	spin_lock(&mm->page_table_lock);
7220	if (pud_none(*pud)) {
7221		pud_populate(mm, pud,
7222				(pmd_t *)((unsigned long)spte & PAGE_MASK));
7223		mm_inc_nr_pmds(mm);
7224	} else {
7225		ptdesc_pmd_pts_dec(virt_to_ptdesc(spte));
7226	}
7227	spin_unlock(&mm->page_table_lock);
7228out:
7229	pte = (pte_t *)pmd_alloc(mm, pud, addr);
7230	i_mmap_unlock_read(mapping);
7231	return pte;
7232}
7233
7234/*
7235 * unmap huge page backed by shared pte.
7236 *
 
 
 
 
7237 * Called with page table lock held.
7238 *
7239 * returns: 1 successfully unmapped a shared pte page
7240 *	    0 the underlying pte page is not shared, or it is the last user
7241 */
7242int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7243					unsigned long addr, pte_t *ptep)
7244{
7245	unsigned long sz = huge_page_size(hstate_vma(vma));
7246	pgd_t *pgd = pgd_offset(mm, addr);
7247	p4d_t *p4d = p4d_offset(pgd, addr);
7248	pud_t *pud = pud_offset(p4d, addr);
7249
7250	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
7251	hugetlb_vma_assert_locked(vma);
7252	if (sz != PMD_SIZE)
7253		return 0;
7254	if (!ptdesc_pmd_pts_count(virt_to_ptdesc(ptep)))
7255		return 0;
7256
7257	pud_clear(pud);
7258	ptdesc_pmd_pts_dec(virt_to_ptdesc(ptep));
7259	mm_dec_nr_pmds(mm);
7260	return 1;
7261}
7262
7263#else /* !CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */
7264
7265pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
7266		      unsigned long addr, pud_t *pud)
7267{
7268	return NULL;
7269}
7270
7271int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7272				unsigned long addr, pte_t *ptep)
7273{
7274	return 0;
7275}
7276
7277void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
7278				unsigned long *start, unsigned long *end)
7279{
7280}
7281
7282bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
7283{
7284	return false;
7285}
7286#endif /* CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */
7287
7288#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
7289pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
7290			unsigned long addr, unsigned long sz)
7291{
7292	pgd_t *pgd;
7293	p4d_t *p4d;
7294	pud_t *pud;
7295	pte_t *pte = NULL;
7296
7297	pgd = pgd_offset(mm, addr);
7298	p4d = p4d_alloc(mm, pgd, addr);
7299	if (!p4d)
7300		return NULL;
7301	pud = pud_alloc(mm, p4d, addr);
7302	if (pud) {
7303		if (sz == PUD_SIZE) {
7304			pte = (pte_t *)pud;
7305		} else {
7306			BUG_ON(sz != PMD_SIZE);
7307			if (want_pmd_share(vma, addr) && pud_none(*pud))
7308				pte = huge_pmd_share(mm, vma, addr, pud);
7309			else
7310				pte = (pte_t *)pmd_alloc(mm, pud, addr);
7311		}
7312	}
7313
7314	if (pte) {
7315		pte_t pteval = ptep_get_lockless(pte);
7316
7317		BUG_ON(pte_present(pteval) && !pte_huge(pteval));
7318	}
7319
7320	return pte;
7321}
7322
7323/*
7324 * huge_pte_offset() - Walk the page table to resolve the hugepage
7325 * entry at address @addr
7326 *
7327 * Return: Pointer to page table entry (PUD or PMD) for
7328 * address @addr, or NULL if a !p*d_present() entry is encountered and the
7329 * size @sz doesn't match the hugepage size at this level of the page
7330 * table.
7331 */
7332pte_t *huge_pte_offset(struct mm_struct *mm,
7333		       unsigned long addr, unsigned long sz)
7334{
7335	pgd_t *pgd;
7336	p4d_t *p4d;
7337	pud_t *pud;
7338	pmd_t *pmd;
7339
7340	pgd = pgd_offset(mm, addr);
7341	if (!pgd_present(*pgd))
7342		return NULL;
7343	p4d = p4d_offset(pgd, addr);
7344	if (!p4d_present(*p4d))
7345		return NULL;
7346
7347	pud = pud_offset(p4d, addr);
7348	if (sz == PUD_SIZE)
7349		/* must be pud huge, non-present or none */
7350		return (pte_t *)pud;
7351	if (!pud_present(*pud))
7352		return NULL;
7353	/* must have a valid entry and size to go further */
7354
7355	pmd = pmd_offset(pud, addr);
7356	/* must be pmd huge, non-present or none */
7357	return (pte_t *)pmd;
7358}
7359
7360/*
7361 * Return a mask that can be used to update an address to the last huge
7362 * page in a page table page mapping size.  Used to skip non-present
7363 * page table entries when linearly scanning address ranges.  Architectures
7364 * with unique huge page to page table relationships can define their own
7365 * version of this routine.
7366 */
7367unsigned long hugetlb_mask_last_page(struct hstate *h)
7368{
7369	unsigned long hp_size = huge_page_size(h);
7370
7371	if (hp_size == PUD_SIZE)
7372		return P4D_SIZE - PUD_SIZE;
7373	else if (hp_size == PMD_SIZE)
7374		return PUD_SIZE - PMD_SIZE;
7375	else
7376		return 0UL;
7377}
7378
7379#else
7380
7381/* See description above.  Architectures can provide their own version. */
7382__weak unsigned long hugetlb_mask_last_page(struct hstate *h)
7383{
7384#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
7385	if (huge_page_size(h) == PMD_SIZE)
7386		return PUD_SIZE - PMD_SIZE;
7387#endif
7388	return 0UL;
7389}
7390
7391#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
7392
 
 
 
 
7393bool isolate_hugetlb(struct folio *folio, struct list_head *list)
7394{
7395	bool ret = true;
7396
7397	spin_lock_irq(&hugetlb_lock);
7398	if (!folio_test_hugetlb(folio) ||
7399	    !folio_test_hugetlb_migratable(folio) ||
7400	    !folio_try_get(folio)) {
7401		ret = false;
7402		goto unlock;
7403	}
7404	folio_clear_hugetlb_migratable(folio);
7405	list_move_tail(&folio->lru, list);
7406unlock:
7407	spin_unlock_irq(&hugetlb_lock);
7408	return ret;
7409}
7410
7411int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
7412{
7413	int ret = 0;
7414
7415	*hugetlb = false;
7416	spin_lock_irq(&hugetlb_lock);
7417	if (folio_test_hugetlb(folio)) {
7418		*hugetlb = true;
7419		if (folio_test_hugetlb_freed(folio))
7420			ret = 0;
7421		else if (folio_test_hugetlb_migratable(folio) || unpoison)
7422			ret = folio_try_get(folio);
7423		else
7424			ret = -EBUSY;
7425	}
7426	spin_unlock_irq(&hugetlb_lock);
7427	return ret;
7428}
7429
7430int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
7431				bool *migratable_cleared)
7432{
7433	int ret;
7434
7435	spin_lock_irq(&hugetlb_lock);
7436	ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared);
7437	spin_unlock_irq(&hugetlb_lock);
7438	return ret;
7439}
7440
7441void folio_putback_active_hugetlb(struct folio *folio)
7442{
7443	spin_lock_irq(&hugetlb_lock);
7444	folio_set_hugetlb_migratable(folio);
7445	list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist);
7446	spin_unlock_irq(&hugetlb_lock);
7447	folio_put(folio);
7448}
7449
7450void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason)
7451{
7452	struct hstate *h = folio_hstate(old_folio);
7453
7454	hugetlb_cgroup_migrate(old_folio, new_folio);
7455	set_page_owner_migrate_reason(&new_folio->page, reason);
7456
7457	/*
7458	 * transfer temporary state of the new hugetlb folio. This is
7459	 * reverse to other transitions because the newpage is going to
7460	 * be final while the old one will be freed so it takes over
7461	 * the temporary status.
7462	 *
7463	 * Also note that we have to transfer the per-node surplus state
7464	 * here as well otherwise the global surplus count will not match
7465	 * the per-node's.
7466	 */
7467	if (folio_test_hugetlb_temporary(new_folio)) {
7468		int old_nid = folio_nid(old_folio);
7469		int new_nid = folio_nid(new_folio);
7470
7471		folio_set_hugetlb_temporary(old_folio);
7472		folio_clear_hugetlb_temporary(new_folio);
7473
7474
7475		/*
7476		 * There is no need to transfer the per-node surplus state
7477		 * when we do not cross the node.
7478		 */
7479		if (new_nid == old_nid)
7480			return;
7481		spin_lock_irq(&hugetlb_lock);
7482		if (h->surplus_huge_pages_node[old_nid]) {
7483			h->surplus_huge_pages_node[old_nid]--;
7484			h->surplus_huge_pages_node[new_nid]++;
7485		}
7486		spin_unlock_irq(&hugetlb_lock);
7487	}
7488}
7489
7490static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
7491				   unsigned long start,
7492				   unsigned long end)
7493{
7494	struct hstate *h = hstate_vma(vma);
7495	unsigned long sz = huge_page_size(h);
7496	struct mm_struct *mm = vma->vm_mm;
7497	struct mmu_notifier_range range;
7498	unsigned long address;
7499	spinlock_t *ptl;
7500	pte_t *ptep;
7501
7502	if (!(vma->vm_flags & VM_MAYSHARE))
7503		return;
7504
7505	if (start >= end)
7506		return;
7507
7508	flush_cache_range(vma, start, end);
7509	/*
7510	 * No need to call adjust_range_if_pmd_sharing_possible(), because
7511	 * we have already done the PUD_SIZE alignment.
7512	 */
7513	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
7514				start, end);
7515	mmu_notifier_invalidate_range_start(&range);
7516	hugetlb_vma_lock_write(vma);
7517	i_mmap_lock_write(vma->vm_file->f_mapping);
7518	for (address = start; address < end; address += PUD_SIZE) {
7519		ptep = hugetlb_walk(vma, address, sz);
7520		if (!ptep)
7521			continue;
7522		ptl = huge_pte_lock(h, mm, ptep);
7523		huge_pmd_unshare(mm, vma, address, ptep);
7524		spin_unlock(ptl);
7525	}
7526	flush_hugetlb_tlb_range(vma, start, end);
7527	i_mmap_unlock_write(vma->vm_file->f_mapping);
7528	hugetlb_vma_unlock_write(vma);
7529	/*
7530	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see
7531	 * Documentation/mm/mmu_notifier.rst.
7532	 */
7533	mmu_notifier_invalidate_range_end(&range);
7534}
7535
7536/*
7537 * This function will unconditionally remove all the shared pmd pgtable entries
7538 * within the specific vma for a hugetlbfs memory range.
7539 */
7540void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
7541{
7542	hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE),
7543			ALIGN_DOWN(vma->vm_end, PUD_SIZE));
7544}
7545
7546#ifdef CONFIG_CMA
7547static bool cma_reserve_called __initdata;
7548
7549static int __init cmdline_parse_hugetlb_cma(char *p)
7550{
7551	int nid, count = 0;
7552	unsigned long tmp;
7553	char *s = p;
7554
7555	while (*s) {
7556		if (sscanf(s, "%lu%n", &tmp, &count) != 1)
7557			break;
7558
7559		if (s[count] == ':') {
7560			if (tmp >= MAX_NUMNODES)
7561				break;
7562			nid = array_index_nospec(tmp, MAX_NUMNODES);
7563
7564			s += count + 1;
7565			tmp = memparse(s, &s);
7566			hugetlb_cma_size_in_node[nid] = tmp;
7567			hugetlb_cma_size += tmp;
7568
7569			/*
7570			 * Skip the separator if have one, otherwise
7571			 * break the parsing.
7572			 */
7573			if (*s == ',')
7574				s++;
7575			else
7576				break;
7577		} else {
7578			hugetlb_cma_size = memparse(p, &p);
7579			break;
7580		}
7581	}
7582
7583	return 0;
7584}
7585
7586early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
7587
7588void __init hugetlb_cma_reserve(int order)
7589{
7590	unsigned long size, reserved, per_node;
7591	bool node_specific_cma_alloc = false;
7592	int nid;
7593
7594	/*
7595	 * HugeTLB CMA reservation is required for gigantic
7596	 * huge pages which could not be allocated via the
7597	 * page allocator. Just warn if there is any change
7598	 * breaking this assumption.
7599	 */
7600	VM_WARN_ON(order <= MAX_PAGE_ORDER);
7601	cma_reserve_called = true;
7602
7603	if (!hugetlb_cma_size)
7604		return;
7605
7606	for (nid = 0; nid < MAX_NUMNODES; nid++) {
7607		if (hugetlb_cma_size_in_node[nid] == 0)
7608			continue;
7609
7610		if (!node_online(nid)) {
7611			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
7612			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7613			hugetlb_cma_size_in_node[nid] = 0;
7614			continue;
7615		}
7616
7617		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
7618			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
7619				nid, (PAGE_SIZE << order) / SZ_1M);
7620			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7621			hugetlb_cma_size_in_node[nid] = 0;
7622		} else {
7623			node_specific_cma_alloc = true;
7624		}
7625	}
7626
7627	/* Validate the CMA size again in case some invalid nodes specified. */
7628	if (!hugetlb_cma_size)
7629		return;
7630
7631	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
7632		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
7633			(PAGE_SIZE << order) / SZ_1M);
7634		hugetlb_cma_size = 0;
7635		return;
7636	}
7637
7638	if (!node_specific_cma_alloc) {
7639		/*
7640		 * If 3 GB area is requested on a machine with 4 numa nodes,
7641		 * let's allocate 1 GB on first three nodes and ignore the last one.
7642		 */
7643		per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
7644		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
7645			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
7646	}
7647
7648	reserved = 0;
7649	for_each_online_node(nid) {
7650		int res;
7651		char name[CMA_MAX_NAME];
7652
7653		if (node_specific_cma_alloc) {
7654			if (hugetlb_cma_size_in_node[nid] == 0)
7655				continue;
7656
7657			size = hugetlb_cma_size_in_node[nid];
7658		} else {
7659			size = min(per_node, hugetlb_cma_size - reserved);
7660		}
7661
7662		size = round_up(size, PAGE_SIZE << order);
7663
7664		snprintf(name, sizeof(name), "hugetlb%d", nid);
7665		/*
7666		 * Note that 'order per bit' is based on smallest size that
7667		 * may be returned to CMA allocator in the case of
7668		 * huge page demotion.
7669		 */
7670		res = cma_declare_contiguous_nid(0, size, 0,
7671					PAGE_SIZE << order,
7672					HUGETLB_PAGE_ORDER, false, name,
7673					&hugetlb_cma[nid], nid);
7674		if (res) {
7675			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
7676				res, nid);
7677			continue;
7678		}
7679
7680		reserved += size;
7681		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
7682			size / SZ_1M, nid);
7683
7684		if (reserved >= hugetlb_cma_size)
7685			break;
7686	}
7687
7688	if (!reserved)
7689		/*
7690		 * hugetlb_cma_size is used to determine if allocations from
7691		 * cma are possible.  Set to zero if no cma regions are set up.
7692		 */
7693		hugetlb_cma_size = 0;
7694}
7695
7696static void __init hugetlb_cma_check(void)
7697{
7698	if (!hugetlb_cma_size || cma_reserve_called)
7699		return;
7700
7701	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
7702}
7703
7704#endif /* CONFIG_CMA */