1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Generic hugetlb support.
   4 * (C) Nadia Yvette Chambers, April 2004
   5 */
   6#include <linux/list.h>
   7#include <linux/init.h>
   8#include <linux/mm.h>
   9#include <linux/seq_file.h>
  10#include <linux/sysctl.h>
  11#include <linux/highmem.h>
  12#include <linux/mmu_notifier.h>
  13#include <linux/nodemask.h>
  14#include <linux/pagemap.h>
  15#include <linux/mempolicy.h>
  16#include <linux/compiler.h>
  17#include <linux/cpuset.h>
  18#include <linux/mutex.h>
  19#include <linux/memblock.h>
  20#include <linux/sysfs.h>
  21#include <linux/slab.h>
  22#include <linux/sched/mm.h>
  23#include <linux/mmdebug.h>
  24#include <linux/sched/signal.h>
  25#include <linux/rmap.h>
  26#include <linux/string_helpers.h>
  27#include <linux/swap.h>
  28#include <linux/swapops.h>
  29#include <linux/jhash.h>
  30#include <linux/numa.h>
  31#include <linux/llist.h>
  32#include <linux/cma.h>
  33#include <linux/migrate.h>
  34#include <linux/nospec.h>
  35#include <linux/delayacct.h>
  36#include <linux/memory.h>
  37
  38#include <asm/page.h>
  39#include <asm/pgalloc.h>
  40#include <asm/tlb.h>
  41
  42#include <linux/io.h>
  43#include <linux/hugetlb.h>
  44#include <linux/hugetlb_cgroup.h>
  45#include <linux/node.h>
 
  46#include <linux/page_owner.h>
  47#include "internal.h"
  48#include "hugetlb_vmemmap.h"
  49
  50int hugetlb_max_hstate __read_mostly;
  51unsigned int default_hstate_idx;
  52struct hstate hstates[HUGE_MAX_HSTATE];
  53
  54#ifdef CONFIG_CMA
  55static struct cma *hugetlb_cma[MAX_NUMNODES];
  56static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
  57static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
  58{
  59	return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
  60				1 << order);
  61}
  62#else
  63static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
  64{
  65	return false;
  66}
  67#endif
  68static unsigned long hugetlb_cma_size __initdata;
  69
 
 
 
 
 
 
  70__initdata LIST_HEAD(huge_boot_pages);
  71
  72/* for command line parsing */
  73static struct hstate * __initdata parsed_hstate;
  74static unsigned long __initdata default_hstate_max_huge_pages;
  75static bool __initdata parsed_valid_hugepagesz = true;
  76static bool __initdata parsed_default_hugepagesz;
  77static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
  78
  79/*
  80 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
  81 * free_huge_pages, and surplus_huge_pages.
  82 */
  83DEFINE_SPINLOCK(hugetlb_lock);
  84
  85/*
  86 * Serializes faults on the same logical page.  This is used to
  87 * prevent spurious OOMs when the hugepage pool is fully utilized.
  88 */
  89static int num_fault_mutexes;
  90struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
  91
  92/* Forward declaration */
  93static int hugetlb_acct_memory(struct hstate *h, long delta);
  94static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
  95static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
  96static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
  97static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
  98		unsigned long start, unsigned long end);
  99
 100static inline bool subpool_is_free(struct hugepage_subpool *spool)
 101{
 102	if (spool->count)
 103		return false;
 104	if (spool->max_hpages != -1)
 105		return spool->used_hpages == 0;
 106	if (spool->min_hpages != -1)
 107		return spool->rsv_hpages == spool->min_hpages;
 108
 109	return true;
 110}
 111
 112static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
 113						unsigned long irq_flags)
 114{
 115	spin_unlock_irqrestore(&spool->lock, irq_flags);
 116
 117	/* If no pages are used, and no other handles to the subpool
 118	 * remain, give up any reservations based on minimum size and
 119	 * free the subpool */
 120	if (subpool_is_free(spool)) {
 121		if (spool->min_hpages != -1)
 122			hugetlb_acct_memory(spool->hstate,
 123						-spool->min_hpages);
 124		kfree(spool);
 125	}
 126}
 127
 128struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 129						long min_hpages)
 130{
 131	struct hugepage_subpool *spool;
 132
 133	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 134	if (!spool)
 135		return NULL;
 136
 137	spin_lock_init(&spool->lock);
 138	spool->count = 1;
 139	spool->max_hpages = max_hpages;
 140	spool->hstate = h;
 141	spool->min_hpages = min_hpages;
 142
 143	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
 144		kfree(spool);
 145		return NULL;
 146	}
 147	spool->rsv_hpages = min_hpages;
 148
 149	return spool;
 150}
 151
 152void hugepage_put_subpool(struct hugepage_subpool *spool)
 153{
 154	unsigned long flags;
 155
 156	spin_lock_irqsave(&spool->lock, flags);
 157	BUG_ON(!spool->count);
 158	spool->count--;
 159	unlock_or_release_subpool(spool, flags);
 160}
 161
 162/*
 163 * Subpool accounting for allocating and reserving pages.
 164 * Return -ENOMEM if there are not enough resources to satisfy the
 165 * request.  Otherwise, return the number of pages by which the
 166 * global pools must be adjusted (upward).  The returned value may
 167 * only be different than the passed value (delta) in the case where
 168 * a subpool minimum size must be maintained.
 169 */
 170static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
 171				      long delta)
 172{
 173	long ret = delta;
 174
 175	if (!spool)
 176		return ret;
 177
 178	spin_lock_irq(&spool->lock);
 179
 180	if (spool->max_hpages != -1) {		/* maximum size accounting */
 181		if ((spool->used_hpages + delta) <= spool->max_hpages)
 182			spool->used_hpages += delta;
 183		else {
 184			ret = -ENOMEM;
 185			goto unlock_ret;
 186		}
 187	}
 188
 189	/* minimum size accounting */
 190	if (spool->min_hpages != -1 && spool->rsv_hpages) {
 191		if (delta > spool->rsv_hpages) {
 192			/*
 193			 * Asking for more reserves than those already taken on
 194			 * behalf of subpool.  Return difference.
 195			 */
 196			ret = delta - spool->rsv_hpages;
 197			spool->rsv_hpages = 0;
 198		} else {
 199			ret = 0;	/* reserves already accounted for */
 200			spool->rsv_hpages -= delta;
 201		}
 202	}
 203
 204unlock_ret:
 205	spin_unlock_irq(&spool->lock);
 206	return ret;
 207}
 208
 209/*
 210 * Subpool accounting for freeing and unreserving pages.
 211 * Return the number of global page reservations that must be dropped.
 212 * The return value may only be different than the passed value (delta)
 213 * in the case where a subpool minimum size must be maintained.
 214 */
 215static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 216				       long delta)
 217{
 218	long ret = delta;
 219	unsigned long flags;
 220
 221	if (!spool)
 222		return delta;
 223
 224	spin_lock_irqsave(&spool->lock, flags);
 225
 226	if (spool->max_hpages != -1)		/* maximum size accounting */
 227		spool->used_hpages -= delta;
 228
 229	 /* minimum size accounting */
 230	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
 231		if (spool->rsv_hpages + delta <= spool->min_hpages)
 232			ret = 0;
 233		else
 234			ret = spool->rsv_hpages + delta - spool->min_hpages;
 235
 236		spool->rsv_hpages += delta;
 237		if (spool->rsv_hpages > spool->min_hpages)
 238			spool->rsv_hpages = spool->min_hpages;
 239	}
 240
 241	/*
 242	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
 243	 * quota reference, free it now.
 244	 */
 245	unlock_or_release_subpool(spool, flags);
 246
 247	return ret;
 248}
 249
 250static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 251{
 252	return HUGETLBFS_SB(inode->i_sb)->spool;
 253}
 254
 255static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 256{
 257	return subpool_inode(file_inode(vma->vm_file));
 258}
 259
 260/*
 261 * hugetlb vma_lock helper routines
 262 */
 263static bool __vma_shareable_lock(struct vm_area_struct *vma)
 264{
 265	return vma->vm_flags & (VM_MAYSHARE | VM_SHARED) &&
 266		vma->vm_private_data;
 267}
 268
 269void hugetlb_vma_lock_read(struct vm_area_struct *vma)
 270{
 271	if (__vma_shareable_lock(vma)) {
 272		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 273
 274		down_read(&vma_lock->rw_sema);
 275	}
 276}
 277
 278void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
 279{
 280	if (__vma_shareable_lock(vma)) {
 281		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 282
 283		up_read(&vma_lock->rw_sema);
 284	}
 285}
 286
 287void hugetlb_vma_lock_write(struct vm_area_struct *vma)
 288{
 289	if (__vma_shareable_lock(vma)) {
 290		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 291
 292		down_write(&vma_lock->rw_sema);
 293	}
 294}
 295
 296void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
 297{
 298	if (__vma_shareable_lock(vma)) {
 299		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 300
 301		up_write(&vma_lock->rw_sema);
 302	}
 303}
 304
 305int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
 306{
 307	struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 308
 309	if (!__vma_shareable_lock(vma))
 310		return 1;
 311
 312	return down_write_trylock(&vma_lock->rw_sema);
 313}
 314
 315void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
 316{
 317	if (__vma_shareable_lock(vma)) {
 318		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 319
 320		lockdep_assert_held(&vma_lock->rw_sema);
 321	}
 322}
 323
 324void hugetlb_vma_lock_release(struct kref *kref)
 325{
 326	struct hugetlb_vma_lock *vma_lock = container_of(kref,
 327			struct hugetlb_vma_lock, refs);
 328
 329	kfree(vma_lock);
 330}
 331
 332static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
 333{
 334	struct vm_area_struct *vma = vma_lock->vma;
 335
 336	/*
 337	 * vma_lock structure may or not be released as a result of put,
 338	 * it certainly will no longer be attached to vma so clear pointer.
 339	 * Semaphore synchronizes access to vma_lock->vma field.
 340	 */
 341	vma_lock->vma = NULL;
 342	vma->vm_private_data = NULL;
 343	up_write(&vma_lock->rw_sema);
 344	kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
 345}
 346
 347static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
 348{
 349	if (__vma_shareable_lock(vma)) {
 350		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 351
 352		__hugetlb_vma_unlock_write_put(vma_lock);
 353	}
 354}
 355
 356static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
 357{
 358	/*
 359	 * Only present in sharable vmas.
 360	 */
 361	if (!vma || !__vma_shareable_lock(vma))
 362		return;
 363
 364	if (vma->vm_private_data) {
 365		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
 366
 367		down_write(&vma_lock->rw_sema);
 368		__hugetlb_vma_unlock_write_put(vma_lock);
 369	}
 370}
 371
 372static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
 373{
 374	struct hugetlb_vma_lock *vma_lock;
 375
 376	/* Only establish in (flags) sharable vmas */
 377	if (!vma || !(vma->vm_flags & VM_MAYSHARE))
 378		return;
 379
 380	/* Should never get here with non-NULL vm_private_data */
 381	if (vma->vm_private_data)
 382		return;
 383
 384	vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL);
 385	if (!vma_lock) {
 386		/*
 387		 * If we can not allocate structure, then vma can not
 388		 * participate in pmd sharing.  This is only a possible
 389		 * performance enhancement and memory saving issue.
 390		 * However, the lock is also used to synchronize page
 391		 * faults with truncation.  If the lock is not present,
 392		 * unlikely races could leave pages in a file past i_size
 393		 * until the file is removed.  Warn in the unlikely case of
 394		 * allocation failure.
 395		 */
 396		pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
 397		return;
 398	}
 399
 400	kref_init(&vma_lock->refs);
 401	init_rwsem(&vma_lock->rw_sema);
 402	vma_lock->vma = vma;
 403	vma->vm_private_data = vma_lock;
 404}
 405
 406/* Helper that removes a struct file_region from the resv_map cache and returns
 407 * it for use.
 408 */
 409static struct file_region *
 410get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
 411{
 412	struct file_region *nrg;
 413
 414	VM_BUG_ON(resv->region_cache_count <= 0);
 415
 416	resv->region_cache_count--;
 417	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
 
 418	list_del(&nrg->link);
 419
 420	nrg->from = from;
 421	nrg->to = to;
 422
 423	return nrg;
 424}
 425
 426static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
 427					      struct file_region *rg)
 428{
 429#ifdef CONFIG_CGROUP_HUGETLB
 430	nrg->reservation_counter = rg->reservation_counter;
 431	nrg->css = rg->css;
 432	if (rg->css)
 433		css_get(rg->css);
 434#endif
 435}
 436
 437/* Helper that records hugetlb_cgroup uncharge info. */
 438static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
 439						struct hstate *h,
 440						struct resv_map *resv,
 441						struct file_region *nrg)
 442{
 443#ifdef CONFIG_CGROUP_HUGETLB
 444	if (h_cg) {
 445		nrg->reservation_counter =
 446			&h_cg->rsvd_hugepage[hstate_index(h)];
 447		nrg->css = &h_cg->css;
 448		/*
 449		 * The caller will hold exactly one h_cg->css reference for the
 450		 * whole contiguous reservation region. But this area might be
 451		 * scattered when there are already some file_regions reside in
 452		 * it. As a result, many file_regions may share only one css
 453		 * reference. In order to ensure that one file_region must hold
 454		 * exactly one h_cg->css reference, we should do css_get for
 455		 * each file_region and leave the reference held by caller
 456		 * untouched.
 457		 */
 458		css_get(&h_cg->css);
 459		if (!resv->pages_per_hpage)
 460			resv->pages_per_hpage = pages_per_huge_page(h);
 461		/* pages_per_hpage should be the same for all entries in
 462		 * a resv_map.
 463		 */
 464		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
 465	} else {
 466		nrg->reservation_counter = NULL;
 467		nrg->css = NULL;
 468	}
 469#endif
 470}
 471
 472static void put_uncharge_info(struct file_region *rg)
 473{
 474#ifdef CONFIG_CGROUP_HUGETLB
 475	if (rg->css)
 476		css_put(rg->css);
 477#endif
 478}
 479
 480static bool has_same_uncharge_info(struct file_region *rg,
 481				   struct file_region *org)
 482{
 483#ifdef CONFIG_CGROUP_HUGETLB
 484	return rg->reservation_counter == org->reservation_counter &&
 
 485	       rg->css == org->css;
 486
 487#else
 488	return true;
 489#endif
 490}
 491
 492static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
 493{
 494	struct file_region *nrg, *prg;
 495
 496	prg = list_prev_entry(rg, link);
 497	if (&prg->link != &resv->regions && prg->to == rg->from &&
 498	    has_same_uncharge_info(prg, rg)) {
 499		prg->to = rg->to;
 500
 501		list_del(&rg->link);
 502		put_uncharge_info(rg);
 503		kfree(rg);
 504
 505		rg = prg;
 
 506	}
 507
 508	nrg = list_next_entry(rg, link);
 509	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
 510	    has_same_uncharge_info(nrg, rg)) {
 511		nrg->from = rg->from;
 512
 513		list_del(&rg->link);
 514		put_uncharge_info(rg);
 515		kfree(rg);
 516	}
 517}
 518
 519static inline long
 520hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
 521		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
 522		     long *regions_needed)
 523{
 524	struct file_region *nrg;
 525
 526	if (!regions_needed) {
 527		nrg = get_file_region_entry_from_cache(map, from, to);
 528		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
 529		list_add(&nrg->link, rg);
 530		coalesce_file_region(map, nrg);
 531	} else
 532		*regions_needed += 1;
 533
 534	return to - from;
 
 
 535}
 536
 537/*
 538 * Must be called with resv->lock held.
 539 *
 540 * Calling this with regions_needed != NULL will count the number of pages
 541 * to be added but will not modify the linked list. And regions_needed will
 542 * indicate the number of file_regions needed in the cache to carry out to add
 543 * the regions for this range.
 544 */
 545static long add_reservation_in_range(struct resv_map *resv, long f, long t,
 546				     struct hugetlb_cgroup *h_cg,
 547				     struct hstate *h, long *regions_needed)
 
 548{
 549	long add = 0;
 550	struct list_head *head = &resv->regions;
 551	long last_accounted_offset = f;
 552	struct file_region *iter, *trg = NULL;
 553	struct list_head *rg = NULL;
 554
 555	if (regions_needed)
 556		*regions_needed = 0;
 557
 558	/* In this loop, we essentially handle an entry for the range
 559	 * [last_accounted_offset, iter->from), at every iteration, with some
 560	 * bounds checking.
 561	 */
 562	list_for_each_entry_safe(iter, trg, head, link) {
 563		/* Skip irrelevant regions that start before our range. */
 564		if (iter->from < f) {
 565			/* If this region ends after the last accounted offset,
 566			 * then we need to update last_accounted_offset.
 567			 */
 568			if (iter->to > last_accounted_offset)
 569				last_accounted_offset = iter->to;
 570			continue;
 571		}
 572
 573		/* When we find a region that starts beyond our range, we've
 574		 * finished.
 575		 */
 576		if (iter->from >= t) {
 577			rg = iter->link.prev;
 578			break;
 579		}
 580
 581		/* Add an entry for last_accounted_offset -> iter->from, and
 582		 * update last_accounted_offset.
 583		 */
 584		if (iter->from > last_accounted_offset)
 585			add += hugetlb_resv_map_add(resv, iter->link.prev,
 586						    last_accounted_offset,
 587						    iter->from, h, h_cg,
 588						    regions_needed);
 
 
 
 
 
 
 
 589
 590		last_accounted_offset = iter->to;
 591	}
 592
 593	/* Handle the case where our range extends beyond
 594	 * last_accounted_offset.
 595	 */
 596	if (!rg)
 597		rg = head->prev;
 598	if (last_accounted_offset < t)
 599		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
 600					    t, h, h_cg, regions_needed);
 
 
 
 
 
 
 601
 
 602	return add;
 603}
 604
 605/* Must be called with resv->lock acquired. Will drop lock to allocate entries.
 606 */
 607static int allocate_file_region_entries(struct resv_map *resv,
 608					int regions_needed)
 609	__must_hold(&resv->lock)
 610{
 611	LIST_HEAD(allocated_regions);
 612	int to_allocate = 0, i = 0;
 613	struct file_region *trg = NULL, *rg = NULL;
 614
 615	VM_BUG_ON(regions_needed < 0);
 616
 
 
 617	/*
 618	 * Check for sufficient descriptors in the cache to accommodate
 619	 * the number of in progress add operations plus regions_needed.
 620	 *
 621	 * This is a while loop because when we drop the lock, some other call
 622	 * to region_add or region_del may have consumed some region_entries,
 623	 * so we keep looping here until we finally have enough entries for
 624	 * (adds_in_progress + regions_needed).
 625	 */
 626	while (resv->region_cache_count <
 627	       (resv->adds_in_progress + regions_needed)) {
 628		to_allocate = resv->adds_in_progress + regions_needed -
 629			      resv->region_cache_count;
 630
 631		/* At this point, we should have enough entries in the cache
 632		 * for all the existing adds_in_progress. We should only be
 633		 * needing to allocate for regions_needed.
 634		 */
 635		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
 636
 637		spin_unlock(&resv->lock);
 638		for (i = 0; i < to_allocate; i++) {
 639			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
 640			if (!trg)
 641				goto out_of_memory;
 642			list_add(&trg->link, &allocated_regions);
 643		}
 644
 645		spin_lock(&resv->lock);
 646
 647		list_splice(&allocated_regions, &resv->region_cache);
 648		resv->region_cache_count += to_allocate;
 
 
 
 649	}
 650
 651	return 0;
 652
 653out_of_memory:
 654	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
 655		list_del(&rg->link);
 656		kfree(rg);
 657	}
 658	return -ENOMEM;
 659}
 660
 661/*
 662 * Add the huge page range represented by [f, t) to the reserve
 663 * map.  Regions will be taken from the cache to fill in this range.
 664 * Sufficient regions should exist in the cache due to the previous
 665 * call to region_chg with the same range, but in some cases the cache will not
 666 * have sufficient entries due to races with other code doing region_add or
 667 * region_del.  The extra needed entries will be allocated.
 668 *
 669 * regions_needed is the out value provided by a previous call to region_chg.
 670 *
 671 * Return the number of new huge pages added to the map.  This number is greater
 672 * than or equal to zero.  If file_region entries needed to be allocated for
 673 * this operation and we were not able to allocate, it returns -ENOMEM.
 674 * region_add of regions of length 1 never allocate file_regions and cannot
 675 * fail; region_chg will always allocate at least 1 entry and a region_add for
 676 * 1 page will only require at most 1 entry.
 677 */
 678static long region_add(struct resv_map *resv, long f, long t,
 679		       long in_regions_needed, struct hstate *h,
 680		       struct hugetlb_cgroup *h_cg)
 681{
 682	long add = 0, actual_regions_needed = 0;
 683
 684	spin_lock(&resv->lock);
 685retry:
 686
 687	/* Count how many regions are actually needed to execute this add. */
 688	add_reservation_in_range(resv, f, t, NULL, NULL,
 689				 &actual_regions_needed);
 690
 691	/*
 692	 * Check for sufficient descriptors in the cache to accommodate
 693	 * this add operation. Note that actual_regions_needed may be greater
 694	 * than in_regions_needed, as the resv_map may have been modified since
 695	 * the region_chg call. In this case, we need to make sure that we
 696	 * allocate extra entries, such that we have enough for all the
 697	 * existing adds_in_progress, plus the excess needed for this
 698	 * operation.
 699	 */
 700	if (actual_regions_needed > in_regions_needed &&
 701	    resv->region_cache_count <
 702		    resv->adds_in_progress +
 703			    (actual_regions_needed - in_regions_needed)) {
 704		/* region_add operation of range 1 should never need to
 705		 * allocate file_region entries.
 706		 */
 707		VM_BUG_ON(t - f <= 1);
 708
 709		if (allocate_file_region_entries(
 710			    resv, actual_regions_needed - in_regions_needed)) {
 711			return -ENOMEM;
 712		}
 713
 714		goto retry;
 715	}
 716
 717	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
 718
 719	resv->adds_in_progress -= in_regions_needed;
 720
 721	spin_unlock(&resv->lock);
 
 722	return add;
 723}
 724
 725/*
 726 * Examine the existing reserve map and determine how many
 727 * huge pages in the specified range [f, t) are NOT currently
 728 * represented.  This routine is called before a subsequent
 729 * call to region_add that will actually modify the reserve
 730 * map to add the specified range [f, t).  region_chg does
 731 * not change the number of huge pages represented by the
 732 * map.  A number of new file_region structures is added to the cache as a
 733 * placeholder, for the subsequent region_add call to use. At least 1
 734 * file_region structure is added.
 735 *
 736 * out_regions_needed is the number of regions added to the
 737 * resv->adds_in_progress.  This value needs to be provided to a follow up call
 738 * to region_add or region_abort for proper accounting.
 739 *
 740 * Returns the number of huge pages that need to be added to the existing
 741 * reservation map for the range [f, t).  This number is greater or equal to
 742 * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 743 * is needed and can not be allocated.
 744 */
 745static long region_chg(struct resv_map *resv, long f, long t,
 746		       long *out_regions_needed)
 747{
 748	long chg = 0;
 749
 750	spin_lock(&resv->lock);
 751
 752	/* Count how many hugepages in this range are NOT represented. */
 753	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
 754				       out_regions_needed);
 755
 756	if (*out_regions_needed == 0)
 757		*out_regions_needed = 1;
 758
 759	if (allocate_file_region_entries(resv, *out_regions_needed))
 760		return -ENOMEM;
 761
 762	resv->adds_in_progress += *out_regions_needed;
 763
 764	spin_unlock(&resv->lock);
 765	return chg;
 766}
 767
 768/*
 769 * Abort the in progress add operation.  The adds_in_progress field
 770 * of the resv_map keeps track of the operations in progress between
 771 * calls to region_chg and region_add.  Operations are sometimes
 772 * aborted after the call to region_chg.  In such cases, region_abort
 773 * is called to decrement the adds_in_progress counter. regions_needed
 774 * is the value returned by the region_chg call, it is used to decrement
 775 * the adds_in_progress counter.
 776 *
 777 * NOTE: The range arguments [f, t) are not needed or used in this
 778 * routine.  They are kept to make reading the calling code easier as
 779 * arguments will match the associated region_chg call.
 780 */
 781static void region_abort(struct resv_map *resv, long f, long t,
 782			 long regions_needed)
 783{
 784	spin_lock(&resv->lock);
 785	VM_BUG_ON(!resv->region_cache_count);
 786	resv->adds_in_progress -= regions_needed;
 787	spin_unlock(&resv->lock);
 788}
 789
 790/*
 791 * Delete the specified range [f, t) from the reserve map.  If the
 792 * t parameter is LONG_MAX, this indicates that ALL regions after f
 793 * should be deleted.  Locate the regions which intersect [f, t)
 794 * and either trim, delete or split the existing regions.
 795 *
 796 * Returns the number of huge pages deleted from the reserve map.
 797 * In the normal case, the return value is zero or more.  In the
 798 * case where a region must be split, a new region descriptor must
 799 * be allocated.  If the allocation fails, -ENOMEM will be returned.
 800 * NOTE: If the parameter t == LONG_MAX, then we will never split
 801 * a region and possibly return -ENOMEM.  Callers specifying
 802 * t == LONG_MAX do not need to check for -ENOMEM error.
 803 */
 804static long region_del(struct resv_map *resv, long f, long t)
 805{
 806	struct list_head *head = &resv->regions;
 807	struct file_region *rg, *trg;
 808	struct file_region *nrg = NULL;
 809	long del = 0;
 810
 811retry:
 812	spin_lock(&resv->lock);
 813	list_for_each_entry_safe(rg, trg, head, link) {
 814		/*
 815		 * Skip regions before the range to be deleted.  file_region
 816		 * ranges are normally of the form [from, to).  However, there
 817		 * may be a "placeholder" entry in the map which is of the form
 818		 * (from, to) with from == to.  Check for placeholder entries
 819		 * at the beginning of the range to be deleted.
 820		 */
 821		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
 822			continue;
 823
 824		if (rg->from >= t)
 825			break;
 826
 827		if (f > rg->from && t < rg->to) { /* Must split region */
 828			/*
 829			 * Check for an entry in the cache before dropping
 830			 * lock and attempting allocation.
 831			 */
 832			if (!nrg &&
 833			    resv->region_cache_count > resv->adds_in_progress) {
 834				nrg = list_first_entry(&resv->region_cache,
 835							struct file_region,
 836							link);
 837				list_del(&nrg->link);
 838				resv->region_cache_count--;
 839			}
 840
 841			if (!nrg) {
 842				spin_unlock(&resv->lock);
 843				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 844				if (!nrg)
 845					return -ENOMEM;
 846				goto retry;
 847			}
 848
 849			del += t - f;
 850			hugetlb_cgroup_uncharge_file_region(
 851				resv, rg, t - f, false);
 852
 853			/* New entry for end of split region */
 854			nrg->from = t;
 855			nrg->to = rg->to;
 856
 857			copy_hugetlb_cgroup_uncharge_info(nrg, rg);
 858
 859			INIT_LIST_HEAD(&nrg->link);
 860
 861			/* Original entry is trimmed */
 862			rg->to = f;
 863
 
 
 
 864			list_add(&nrg->link, &rg->link);
 865			nrg = NULL;
 866			break;
 867		}
 868
 869		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
 870			del += rg->to - rg->from;
 871			hugetlb_cgroup_uncharge_file_region(resv, rg,
 872							    rg->to - rg->from, true);
 873			list_del(&rg->link);
 874			kfree(rg);
 875			continue;
 876		}
 877
 878		if (f <= rg->from) {	/* Trim beginning of region */
 879			hugetlb_cgroup_uncharge_file_region(resv, rg,
 880							    t - rg->from, false);
 881
 882			del += t - rg->from;
 883			rg->from = t;
 884		} else {		/* Trim end of region */
 885			hugetlb_cgroup_uncharge_file_region(resv, rg,
 886							    rg->to - f, false);
 887
 
 
 
 888			del += rg->to - f;
 889			rg->to = f;
 
 
 
 890		}
 891	}
 892
 893	spin_unlock(&resv->lock);
 894	kfree(nrg);
 895	return del;
 896}
 897
 898/*
 899 * A rare out of memory error was encountered which prevented removal of
 900 * the reserve map region for a page.  The huge page itself was free'ed
 901 * and removed from the page cache.  This routine will adjust the subpool
 902 * usage count, and the global reserve count if needed.  By incrementing
 903 * these counts, the reserve map entry which could not be deleted will
 904 * appear as a "reserved" entry instead of simply dangling with incorrect
 905 * counts.
 906 */
 907void hugetlb_fix_reserve_counts(struct inode *inode)
 908{
 909	struct hugepage_subpool *spool = subpool_inode(inode);
 910	long rsv_adjust;
 911	bool reserved = false;
 912
 913	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
 914	if (rsv_adjust > 0) {
 915		struct hstate *h = hstate_inode(inode);
 916
 917		if (!hugetlb_acct_memory(h, 1))
 918			reserved = true;
 919	} else if (!rsv_adjust) {
 920		reserved = true;
 921	}
 922
 923	if (!reserved)
 924		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
 925}
 926
 927/*
 928 * Count and return the number of huge pages in the reserve map
 929 * that intersect with the range [f, t).
 930 */
 931static long region_count(struct resv_map *resv, long f, long t)
 932{
 933	struct list_head *head = &resv->regions;
 934	struct file_region *rg;
 935	long chg = 0;
 936
 937	spin_lock(&resv->lock);
 938	/* Locate each segment we overlap with, and count that overlap. */
 939	list_for_each_entry(rg, head, link) {
 940		long seg_from;
 941		long seg_to;
 942
 943		if (rg->to <= f)
 944			continue;
 945		if (rg->from >= t)
 946			break;
 947
 948		seg_from = max(rg->from, f);
 949		seg_to = min(rg->to, t);
 950
 951		chg += seg_to - seg_from;
 952	}
 953	spin_unlock(&resv->lock);
 954
 955	return chg;
 956}
 957
 958/*
 959 * Convert the address within this vma to the page offset within
 960 * the mapping, in pagecache page units; huge pages here.
 961 */
 962static pgoff_t vma_hugecache_offset(struct hstate *h,
 963			struct vm_area_struct *vma, unsigned long address)
 964{
 965	return ((address - vma->vm_start) >> huge_page_shift(h)) +
 966			(vma->vm_pgoff >> huge_page_order(h));
 967}
 968
 969pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
 970				     unsigned long address)
 971{
 972	return vma_hugecache_offset(hstate_vma(vma), vma, address);
 973}
 974EXPORT_SYMBOL_GPL(linear_hugepage_index);
 975
 976/*
 977 * Return the size of the pages allocated when backing a VMA. In the majority
 978 * cases this will be same size as used by the page table entries.
 979 */
 980unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
 981{
 982	if (vma->vm_ops && vma->vm_ops->pagesize)
 983		return vma->vm_ops->pagesize(vma);
 984	return PAGE_SIZE;
 985}
 986EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
 987
 988/*
 989 * Return the page size being used by the MMU to back a VMA. In the majority
 990 * of cases, the page size used by the kernel matches the MMU size. On
 991 * architectures where it differs, an architecture-specific 'strong'
 992 * version of this symbol is required.
 993 */
 994__weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 995{
 996	return vma_kernel_pagesize(vma);
 997}
 998
 999/*
1000 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
1001 * bits of the reservation map pointer, which are always clear due to
1002 * alignment.
1003 */
1004#define HPAGE_RESV_OWNER    (1UL << 0)
1005#define HPAGE_RESV_UNMAPPED (1UL << 1)
1006#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
1007
1008/*
1009 * These helpers are used to track how many pages are reserved for
1010 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
1011 * is guaranteed to have their future faults succeed.
1012 *
1013 * With the exception of hugetlb_dup_vma_private() which is called at fork(),
1014 * the reserve counters are updated with the hugetlb_lock held. It is safe
1015 * to reset the VMA at fork() time as it is not in use yet and there is no
1016 * chance of the global counters getting corrupted as a result of the values.
1017 *
1018 * The private mapping reservation is represented in a subtly different
1019 * manner to a shared mapping.  A shared mapping has a region map associated
1020 * with the underlying file, this region map represents the backing file
1021 * pages which have ever had a reservation assigned which this persists even
1022 * after the page is instantiated.  A private mapping has a region map
1023 * associated with the original mmap which is attached to all VMAs which
1024 * reference it, this region map represents those offsets which have consumed
1025 * reservation ie. where pages have been instantiated.
1026 */
1027static unsigned long get_vma_private_data(struct vm_area_struct *vma)
1028{
1029	return (unsigned long)vma->vm_private_data;
1030}
1031
1032static void set_vma_private_data(struct vm_area_struct *vma,
1033							unsigned long value)
1034{
1035	vma->vm_private_data = (void *)value;
1036}
1037
1038static void
1039resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
1040					  struct hugetlb_cgroup *h_cg,
1041					  struct hstate *h)
1042{
1043#ifdef CONFIG_CGROUP_HUGETLB
1044	if (!h_cg || !h) {
1045		resv_map->reservation_counter = NULL;
1046		resv_map->pages_per_hpage = 0;
1047		resv_map->css = NULL;
1048	} else {
1049		resv_map->reservation_counter =
1050			&h_cg->rsvd_hugepage[hstate_index(h)];
1051		resv_map->pages_per_hpage = pages_per_huge_page(h);
1052		resv_map->css = &h_cg->css;
1053	}
1054#endif
1055}
1056
1057struct resv_map *resv_map_alloc(void)
1058{
1059	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
1060	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
1061
1062	if (!resv_map || !rg) {
1063		kfree(resv_map);
1064		kfree(rg);
1065		return NULL;
1066	}
1067
1068	kref_init(&resv_map->refs);
1069	spin_lock_init(&resv_map->lock);
1070	INIT_LIST_HEAD(&resv_map->regions);
1071
1072	resv_map->adds_in_progress = 0;
1073	/*
1074	 * Initialize these to 0. On shared mappings, 0's here indicate these
1075	 * fields don't do cgroup accounting. On private mappings, these will be
1076	 * re-initialized to the proper values, to indicate that hugetlb cgroup
1077	 * reservations are to be un-charged from here.
1078	 */
1079	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
1080
1081	INIT_LIST_HEAD(&resv_map->region_cache);
1082	list_add(&rg->link, &resv_map->region_cache);
1083	resv_map->region_cache_count = 1;
1084
1085	return resv_map;
1086}
1087
1088void resv_map_release(struct kref *ref)
1089{
1090	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
1091	struct list_head *head = &resv_map->region_cache;
1092	struct file_region *rg, *trg;
1093
1094	/* Clear out any active regions before we release the map. */
1095	region_del(resv_map, 0, LONG_MAX);
1096
1097	/* ... and any entries left in the cache */
1098	list_for_each_entry_safe(rg, trg, head, link) {
1099		list_del(&rg->link);
1100		kfree(rg);
1101	}
1102
1103	VM_BUG_ON(resv_map->adds_in_progress);
1104
1105	kfree(resv_map);
1106}
1107
1108static inline struct resv_map *inode_resv_map(struct inode *inode)
1109{
1110	/*
1111	 * At inode evict time, i_mapping may not point to the original
1112	 * address space within the inode.  This original address space
1113	 * contains the pointer to the resv_map.  So, always use the
1114	 * address space embedded within the inode.
1115	 * The VERY common case is inode->mapping == &inode->i_data but,
1116	 * this may not be true for device special inodes.
1117	 */
1118	return (struct resv_map *)(&inode->i_data)->private_data;
1119}
1120
1121static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
1122{
1123	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1124	if (vma->vm_flags & VM_MAYSHARE) {
1125		struct address_space *mapping = vma->vm_file->f_mapping;
1126		struct inode *inode = mapping->host;
1127
1128		return inode_resv_map(inode);
1129
1130	} else {
1131		return (struct resv_map *)(get_vma_private_data(vma) &
1132							~HPAGE_RESV_MASK);
1133	}
1134}
1135
1136static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
1137{
1138	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1139	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1140
1141	set_vma_private_data(vma, (get_vma_private_data(vma) &
1142				HPAGE_RESV_MASK) | (unsigned long)map);
1143}
1144
1145static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
1146{
1147	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1148	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1149
1150	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
1151}
1152
1153static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
1154{
1155	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1156
1157	return (get_vma_private_data(vma) & flag) != 0;
1158}
1159
1160void hugetlb_dup_vma_private(struct vm_area_struct *vma)
 
1161{
1162	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1163	/*
1164	 * Clear vm_private_data
1165	 * - For shared mappings this is a per-vma semaphore that may be
1166	 *   allocated in a subsequent call to hugetlb_vm_op_open.
1167	 *   Before clearing, make sure pointer is not associated with vma
1168	 *   as this will leak the structure.  This is the case when called
1169	 *   via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already
1170	 *   been called to allocate a new structure.
1171	 * - For MAP_PRIVATE mappings, this is the reserve map which does
1172	 *   not apply to children.  Faults generated by the children are
1173	 *   not guaranteed to succeed, even if read-only.
1174	 */
1175	if (vma->vm_flags & VM_MAYSHARE) {
1176		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
1177
1178		if (vma_lock && vma_lock->vma != vma)
1179			vma->vm_private_data = NULL;
1180	} else
1181		vma->vm_private_data = NULL;
1182}
1183
1184/*
1185 * Reset and decrement one ref on hugepage private reservation.
1186 * Called with mm->mmap_lock writer semaphore held.
1187 * This function should be only used by move_vma() and operate on
1188 * same sized vma. It should never come here with last ref on the
1189 * reservation.
1190 */
1191void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
1192{
1193	/*
1194	 * Clear the old hugetlb private page reservation.
1195	 * It has already been transferred to new_vma.
1196	 *
1197	 * During a mremap() operation of a hugetlb vma we call move_vma()
1198	 * which copies vma into new_vma and unmaps vma. After the copy
1199	 * operation both new_vma and vma share a reference to the resv_map
1200	 * struct, and at that point vma is about to be unmapped. We don't
1201	 * want to return the reservation to the pool at unmap of vma because
1202	 * the reservation still lives on in new_vma, so simply decrement the
1203	 * ref here and remove the resv_map reference from this vma.
1204	 */
1205	struct resv_map *reservations = vma_resv_map(vma);
1206
1207	if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1208		resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
1209		kref_put(&reservations->refs, resv_map_release);
1210	}
1211
1212	hugetlb_dup_vma_private(vma);
1213}
1214
1215/* Returns true if the VMA has associated reserve pages */
1216static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1217{
1218	if (vma->vm_flags & VM_NORESERVE) {
1219		/*
1220		 * This address is already reserved by other process(chg == 0),
1221		 * so, we should decrement reserved count. Without decrementing,
1222		 * reserve count remains after releasing inode, because this
1223		 * allocated page will go into page cache and is regarded as
1224		 * coming from reserved pool in releasing step.  Currently, we
1225		 * don't have any other solution to deal with this situation
1226		 * properly, so add work-around here.
1227		 */
1228		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1229			return true;
1230		else
1231			return false;
1232	}
1233
1234	/* Shared mappings always use reserves */
1235	if (vma->vm_flags & VM_MAYSHARE) {
1236		/*
1237		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1238		 * be a region map for all pages.  The only situation where
1239		 * there is no region map is if a hole was punched via
1240		 * fallocate.  In this case, there really are no reserves to
1241		 * use.  This situation is indicated if chg != 0.
1242		 */
1243		if (chg)
1244			return false;
1245		else
1246			return true;
1247	}
1248
1249	/*
1250	 * Only the process that called mmap() has reserves for
1251	 * private mappings.
1252	 */
1253	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1254		/*
1255		 * Like the shared case above, a hole punch or truncate
1256		 * could have been performed on the private mapping.
1257		 * Examine the value of chg to determine if reserves
1258		 * actually exist or were previously consumed.
1259		 * Very Subtle - The value of chg comes from a previous
1260		 * call to vma_needs_reserves().  The reserve map for
1261		 * private mappings has different (opposite) semantics
1262		 * than that of shared mappings.  vma_needs_reserves()
1263		 * has already taken this difference in semantics into
1264		 * account.  Therefore, the meaning of chg is the same
1265		 * as in the shared case above.  Code could easily be
1266		 * combined, but keeping it separate draws attention to
1267		 * subtle differences.
1268		 */
1269		if (chg)
1270			return false;
1271		else
1272			return true;
1273	}
1274
1275	return false;
1276}
1277
1278static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
1279{
1280	int nid = folio_nid(folio);
1281
1282	lockdep_assert_held(&hugetlb_lock);
1283	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1284
1285	list_move(&folio->lru, &h->hugepage_freelists[nid]);
1286	h->free_huge_pages++;
1287	h->free_huge_pages_node[nid]++;
1288	folio_set_hugetlb_freed(folio);
1289}
1290
1291static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
1292{
1293	struct page *page;
1294	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1295
1296	lockdep_assert_held(&hugetlb_lock);
1297	list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
1298		if (pin && !is_longterm_pinnable_page(page))
1299			continue;
1300
1301		if (PageHWPoison(page))
1302			continue;
1303
1304		list_move(&page->lru, &h->hugepage_activelist);
1305		set_page_refcounted(page);
1306		ClearHPageFreed(page);
1307		h->free_huge_pages--;
1308		h->free_huge_pages_node[nid]--;
1309		return page;
1310	}
1311
1312	return NULL;
 
 
 
 
 
 
 
 
 
 
1313}
1314
1315static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
1316		nodemask_t *nmask)
1317{
1318	unsigned int cpuset_mems_cookie;
1319	struct zonelist *zonelist;
1320	struct zone *zone;
1321	struct zoneref *z;
1322	int node = NUMA_NO_NODE;
1323
1324	zonelist = node_zonelist(nid, gfp_mask);
1325
1326retry_cpuset:
1327	cpuset_mems_cookie = read_mems_allowed_begin();
1328	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1329		struct page *page;
1330
1331		if (!cpuset_zone_allowed(zone, gfp_mask))
1332			continue;
1333		/*
1334		 * no need to ask again on the same node. Pool is node rather than
1335		 * zone aware
1336		 */
1337		if (zone_to_nid(zone) == node)
1338			continue;
1339		node = zone_to_nid(zone);
1340
1341		page = dequeue_huge_page_node_exact(h, node);
1342		if (page)
1343			return page;
1344	}
1345	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1346		goto retry_cpuset;
1347
1348	return NULL;
1349}
1350
1351static unsigned long available_huge_pages(struct hstate *h)
1352{
1353	return h->free_huge_pages - h->resv_huge_pages;
1354}
1355
1356static struct page *dequeue_huge_page_vma(struct hstate *h,
1357				struct vm_area_struct *vma,
1358				unsigned long address, int avoid_reserve,
1359				long chg)
1360{
1361	struct page *page = NULL;
1362	struct mempolicy *mpol;
1363	gfp_t gfp_mask;
1364	nodemask_t *nodemask;
1365	int nid;
1366
1367	/*
1368	 * A child process with MAP_PRIVATE mappings created by their parent
1369	 * have no page reserves. This check ensures that reservations are
1370	 * not "stolen". The child may still get SIGKILLed
1371	 */
1372	if (!vma_has_reserves(vma, chg) && !available_huge_pages(h))
 
1373		goto err;
1374
1375	/* If reserves cannot be used, ensure enough pages are in the pool */
1376	if (avoid_reserve && !available_huge_pages(h))
1377		goto err;
1378
1379	gfp_mask = htlb_alloc_mask(h);
1380	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1381
1382	if (mpol_is_preferred_many(mpol)) {
1383		page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1384
1385		/* Fallback to all nodes if page==NULL */
1386		nodemask = NULL;
1387	}
1388
1389	if (!page)
1390		page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1391
1392	if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
1393		SetHPageRestoreReserve(page);
1394		h->resv_huge_pages--;
1395	}
1396
1397	mpol_cond_put(mpol);
1398	return page;
1399
1400err:
1401	return NULL;
1402}
1403
1404/*
1405 * common helper functions for hstate_next_node_to_{alloc|free}.
1406 * We may have allocated or freed a huge page based on a different
1407 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1408 * be outside of *nodes_allowed.  Ensure that we use an allowed
1409 * node for alloc or free.
1410 */
1411static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1412{
1413	nid = next_node_in(nid, *nodes_allowed);
1414	VM_BUG_ON(nid >= MAX_NUMNODES);
1415
1416	return nid;
1417}
1418
1419static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1420{
1421	if (!node_isset(nid, *nodes_allowed))
1422		nid = next_node_allowed(nid, nodes_allowed);
1423	return nid;
1424}
1425
1426/*
1427 * returns the previously saved node ["this node"] from which to
1428 * allocate a persistent huge page for the pool and advance the
1429 * next node from which to allocate, handling wrap at end of node
1430 * mask.
1431 */
1432static int hstate_next_node_to_alloc(struct hstate *h,
1433					nodemask_t *nodes_allowed)
1434{
1435	int nid;
1436
1437	VM_BUG_ON(!nodes_allowed);
1438
1439	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1440	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1441
1442	return nid;
1443}
1444
1445/*
1446 * helper for remove_pool_huge_page() - return the previously saved
1447 * node ["this node"] from which to free a huge page.  Advance the
1448 * next node id whether or not we find a free huge page to free so
1449 * that the next attempt to free addresses the next node.
1450 */
1451static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1452{
1453	int nid;
1454
1455	VM_BUG_ON(!nodes_allowed);
1456
1457	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1458	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1459
1460	return nid;
1461}
1462
1463#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
1464	for (nr_nodes = nodes_weight(*mask);				\
1465		nr_nodes > 0 &&						\
1466		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
1467		nr_nodes--)
1468
1469#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1470	for (nr_nodes = nodes_weight(*mask);				\
1471		nr_nodes > 0 &&						\
1472		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1473		nr_nodes--)
1474
1475/* used to demote non-gigantic_huge pages as well */
1476static void __destroy_compound_gigantic_folio(struct folio *folio,
1477					unsigned int order, bool demote)
1478{
1479	int i;
1480	int nr_pages = 1 << order;
1481	struct page *p;
 
 
 
 
1482
1483	atomic_set(folio_mapcount_ptr(folio), 0);
1484	atomic_set(folio_subpages_mapcount_ptr(folio), 0);
1485	atomic_set(folio_pincount_ptr(folio), 0);
1486
1487	for (i = 1; i < nr_pages; i++) {
1488		p = folio_page(folio, i);
1489		p->mapping = NULL;
1490		clear_compound_head(p);
1491		if (!demote)
1492			set_page_refcounted(p);
1493	}
1494
1495	folio_set_compound_order(folio, 0);
1496	__folio_clear_head(folio);
1497}
1498
1499static void destroy_compound_hugetlb_folio_for_demote(struct folio *folio,
1500					unsigned int order)
1501{
1502	__destroy_compound_gigantic_folio(folio, order, true);
1503}
1504
1505#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1506static void destroy_compound_gigantic_folio(struct folio *folio,
1507					unsigned int order)
1508{
1509	__destroy_compound_gigantic_folio(folio, order, false);
1510}
1511
1512static void free_gigantic_folio(struct folio *folio, unsigned int order)
1513{
1514	/*
1515	 * If the page isn't allocated using the cma allocator,
1516	 * cma_release() returns false.
1517	 */
1518#ifdef CONFIG_CMA
1519	int nid = folio_nid(folio);
1520
1521	if (cma_release(hugetlb_cma[nid], &folio->page, 1 << order))
1522		return;
1523#endif
1524
1525	free_contig_range(folio_pfn(folio), 1 << order);
1526}
1527
1528#ifdef CONFIG_CONTIG_ALLOC
1529static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1530		int nid, nodemask_t *nodemask)
1531{
1532	struct page *page;
1533	unsigned long nr_pages = pages_per_huge_page(h);
1534	if (nid == NUMA_NO_NODE)
1535		nid = numa_mem_id();
1536
1537#ifdef CONFIG_CMA
1538	{
 
1539		int node;
1540
1541		if (hugetlb_cma[nid]) {
1542			page = cma_alloc(hugetlb_cma[nid], nr_pages,
1543					huge_page_order(h), true);
1544			if (page)
1545				return page_folio(page);
1546		}
1547
1548		if (!(gfp_mask & __GFP_THISNODE)) {
1549			for_each_node_mask(node, *nodemask) {
1550				if (node == nid || !hugetlb_cma[node])
1551					continue;
1552
1553				page = cma_alloc(hugetlb_cma[node], nr_pages,
1554						huge_page_order(h), true);
1555				if (page)
1556					return page_folio(page);
1557			}
1558		}
1559	}
1560#endif
1561
1562	page = alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
1563	return page ? page_folio(page) : NULL;
1564}
1565
 
 
1566#else /* !CONFIG_CONTIG_ALLOC */
1567static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1568					int nid, nodemask_t *nodemask)
1569{
1570	return NULL;
1571}
1572#endif /* CONFIG_CONTIG_ALLOC */
1573
1574#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1575static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1576					int nid, nodemask_t *nodemask)
1577{
1578	return NULL;
1579}
1580static inline void free_gigantic_folio(struct folio *folio,
1581						unsigned int order) { }
1582static inline void destroy_compound_gigantic_folio(struct folio *folio,
1583						unsigned int order) { }
1584#endif
1585
1586/*
1587 * Remove hugetlb folio from lists, and update dtor so that the folio appears
1588 * as just a compound page.
1589 *
1590 * A reference is held on the folio, except in the case of demote.
1591 *
1592 * Must be called with hugetlb lock held.
1593 */
1594static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1595							bool adjust_surplus,
1596							bool demote)
1597{
1598	int nid = folio_nid(folio);
1599
1600	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
1601	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
1602
1603	lockdep_assert_held(&hugetlb_lock);
1604	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1605		return;
1606
1607	list_del(&folio->lru);
1608
1609	if (folio_test_hugetlb_freed(folio)) {
1610		h->free_huge_pages--;
1611		h->free_huge_pages_node[nid]--;
1612	}
1613	if (adjust_surplus) {
1614		h->surplus_huge_pages--;
1615		h->surplus_huge_pages_node[nid]--;
1616	}
1617
1618	/*
1619	 * Very subtle
1620	 *
1621	 * For non-gigantic pages set the destructor to the normal compound
1622	 * page dtor.  This is needed in case someone takes an additional
1623	 * temporary ref to the page, and freeing is delayed until they drop
1624	 * their reference.
1625	 *
1626	 * For gigantic pages set the destructor to the null dtor.  This
1627	 * destructor will never be called.  Before freeing the gigantic
1628	 * page destroy_compound_gigantic_folio will turn the folio into a
1629	 * simple group of pages.  After this the destructor does not
1630	 * apply.
1631	 *
1632	 * This handles the case where more than one ref is held when and
1633	 * after update_and_free_hugetlb_folio is called.
1634	 *
1635	 * In the case of demote we do not ref count the page as it will soon
1636	 * be turned into a page of smaller size.
1637	 */
1638	if (!demote)
1639		folio_ref_unfreeze(folio, 1);
1640	if (hstate_is_gigantic(h))
1641		folio_set_compound_dtor(folio, NULL_COMPOUND_DTOR);
1642	else
1643		folio_set_compound_dtor(folio, COMPOUND_PAGE_DTOR);
1644
1645	h->nr_huge_pages--;
1646	h->nr_huge_pages_node[nid]--;
1647}
1648
1649static void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1650							bool adjust_surplus)
1651{
1652	__remove_hugetlb_folio(h, folio, adjust_surplus, false);
1653}
1654
1655static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *folio,
1656							bool adjust_surplus)
1657{
1658	__remove_hugetlb_folio(h, folio, adjust_surplus, true);
1659}
1660
1661static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
1662			     bool adjust_surplus)
1663{
1664	int zeroed;
1665	int nid = folio_nid(folio);
1666
1667	VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
1668
1669	lockdep_assert_held(&hugetlb_lock);
1670
1671	INIT_LIST_HEAD(&folio->lru);
1672	h->nr_huge_pages++;
1673	h->nr_huge_pages_node[nid]++;
1674
1675	if (adjust_surplus) {
1676		h->surplus_huge_pages++;
1677		h->surplus_huge_pages_node[nid]++;
1678	}
1679
1680	folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
1681	folio_change_private(folio, NULL);
1682	/*
1683	 * We have to set hugetlb_vmemmap_optimized again as above
1684	 * folio_change_private(folio, NULL) cleared it.
1685	 */
1686	folio_set_hugetlb_vmemmap_optimized(folio);
1687
1688	/*
1689	 * This folio is about to be managed by the hugetlb allocator and
1690	 * should have no users.  Drop our reference, and check for others
1691	 * just in case.
1692	 */
1693	zeroed = folio_put_testzero(folio);
1694	if (unlikely(!zeroed))
1695		/*
1696		 * It is VERY unlikely soneone else has taken a ref on
1697		 * the page.  In this case, we simply return as the
1698		 * hugetlb destructor (free_huge_page) will be called
1699		 * when this other ref is dropped.
1700		 */
1701		return;
1702
1703	arch_clear_hugepage_flags(&folio->page);
1704	enqueue_hugetlb_folio(h, folio);
1705}
1706
1707static void __update_and_free_page(struct hstate *h, struct page *page)
1708{
1709	int i;
1710	struct folio *folio = page_folio(page);
1711	struct page *subpage;
1712
1713	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1714		return;
1715
1716	/*
1717	 * If we don't know which subpages are hwpoisoned, we can't free
1718	 * the hugepage, so it's leaked intentionally.
1719	 */
1720	if (folio_test_hugetlb_raw_hwp_unreliable(folio))
1721		return;
1722
1723	if (hugetlb_vmemmap_restore(h, page)) {
1724		spin_lock_irq(&hugetlb_lock);
1725		/*
1726		 * If we cannot allocate vmemmap pages, just refuse to free the
1727		 * page and put the page back on the hugetlb free list and treat
1728		 * as a surplus page.
1729		 */
1730		add_hugetlb_folio(h, folio, true);
1731		spin_unlock_irq(&hugetlb_lock);
1732		return;
1733	}
1734
1735	/*
1736	 * Move PageHWPoison flag from head page to the raw error pages,
1737	 * which makes any healthy subpages reusable.
1738	 */
1739	if (unlikely(folio_test_hwpoison(folio)))
1740		hugetlb_clear_page_hwpoison(&folio->page);
1741
1742	for (i = 0; i < pages_per_huge_page(h); i++) {
1743		subpage = folio_page(folio, i);
1744		subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1745				1 << PG_referenced | 1 << PG_dirty |
1746				1 << PG_active | 1 << PG_private |
1747				1 << PG_writeback);
1748	}
1749
1750	/*
1751	 * Non-gigantic pages demoted from CMA allocated gigantic pages
1752	 * need to be given back to CMA in free_gigantic_folio.
1753	 */
1754	if (hstate_is_gigantic(h) ||
1755	    hugetlb_cma_folio(folio, huge_page_order(h))) {
1756		destroy_compound_gigantic_folio(folio, huge_page_order(h));
1757		free_gigantic_folio(folio, huge_page_order(h));
 
 
 
 
1758	} else {
1759		__free_pages(page, huge_page_order(h));
1760	}
1761}
1762
1763/*
1764 * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
1765 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
1766 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
1767 * the vmemmap pages.
1768 *
1769 * free_hpage_workfn() locklessly retrieves the linked list of pages to be
1770 * freed and frees them one-by-one. As the page->mapping pointer is going
1771 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
1772 * structure of a lockless linked list of huge pages to be freed.
1773 */
1774static LLIST_HEAD(hpage_freelist);
1775
1776static void free_hpage_workfn(struct work_struct *work)
1777{
1778	struct llist_node *node;
1779
1780	node = llist_del_all(&hpage_freelist);
1781
1782	while (node) {
1783		struct page *page;
1784		struct hstate *h;
1785
1786		page = container_of((struct address_space **)node,
1787				     struct page, mapping);
1788		node = node->next;
1789		page->mapping = NULL;
1790		/*
1791		 * The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate()
1792		 * is going to trigger because a previous call to
1793		 * remove_hugetlb_folio() will call folio_set_compound_dtor
1794		 * (folio, NULL_COMPOUND_DTOR), so do not use page_hstate()
1795		 * directly.
1796		 */
1797		h = size_to_hstate(page_size(page));
1798
1799		__update_and_free_page(h, page);
1800
1801		cond_resched();
 
 
1802	}
 
1803}
1804static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1805
1806static inline void flush_free_hpage_work(struct hstate *h)
 
 
 
 
 
 
1807{
1808	if (hugetlb_vmemmap_optimizable(h))
1809		flush_work(&free_hpage_work);
1810}
1811
1812static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
1813				 bool atomic)
1814{
1815	if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
1816		__update_and_free_page(h, &folio->page);
1817		return;
1818	}
1819
1820	/*
1821	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
1822	 *
1823	 * Only call schedule_work() if hpage_freelist is previously
1824	 * empty. Otherwise, schedule_work() had been called but the workfn
1825	 * hasn't retrieved the list yet.
1826	 */
1827	if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
1828		schedule_work(&free_hpage_work);
1829}
1830
1831static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list)
 
 
 
 
1832{
1833	struct page *page, *t_page;
1834	struct folio *folio;
1835
1836	list_for_each_entry_safe(page, t_page, list, lru) {
1837		folio = page_folio(page);
1838		update_and_free_hugetlb_folio(h, folio, false);
1839		cond_resched();
1840	}
1841}
1842
1843struct hstate *size_to_hstate(unsigned long size)
1844{
1845	struct hstate *h;
 
1846
1847	for_each_hstate(h) {
1848		if (huge_page_size(h) == size)
1849			return h;
1850	}
1851	return NULL;
1852}
1853
1854void free_huge_page(struct page *page)
1855{
1856	/*
1857	 * Can't pass hstate in here because it is called from the
1858	 * compound page destructor.
1859	 */
1860	struct folio *folio = page_folio(page);
1861	struct hstate *h = folio_hstate(folio);
1862	int nid = folio_nid(folio);
1863	struct hugepage_subpool *spool = hugetlb_folio_subpool(folio);
1864	bool restore_reserve;
1865	unsigned long flags;
1866
1867	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1868	VM_BUG_ON_FOLIO(folio_mapcount(folio), folio);
1869
1870	hugetlb_set_folio_subpool(folio, NULL);
1871	if (folio_test_anon(folio))
1872		__ClearPageAnonExclusive(&folio->page);
1873	folio->mapping = NULL;
1874	restore_reserve = folio_test_hugetlb_restore_reserve(folio);
1875	folio_clear_hugetlb_restore_reserve(folio);
1876
1877	/*
1878	 * If HPageRestoreReserve was set on page, page allocation consumed a
1879	 * reservation.  If the page was associated with a subpool, there
1880	 * would have been a page reserved in the subpool before allocation
1881	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1882	 * reservation, do not call hugepage_subpool_put_pages() as this will
1883	 * remove the reserved page from the subpool.
1884	 */
1885	if (!restore_reserve) {
1886		/*
1887		 * A return code of zero implies that the subpool will be
1888		 * under its minimum size if the reservation is not restored
1889		 * after page is free.  Therefore, force restore_reserve
1890		 * operation.
1891		 */
1892		if (hugepage_subpool_put_pages(spool, 1) == 0)
1893			restore_reserve = true;
1894	}
1895
1896	spin_lock_irqsave(&hugetlb_lock, flags);
1897	folio_clear_hugetlb_migratable(folio);
1898	hugetlb_cgroup_uncharge_folio(hstate_index(h),
1899				     pages_per_huge_page(h), folio);
1900	hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
1901					  pages_per_huge_page(h), folio);
1902	if (restore_reserve)
1903		h->resv_huge_pages++;
1904
1905	if (folio_test_hugetlb_temporary(folio)) {
1906		remove_hugetlb_folio(h, folio, false);
1907		spin_unlock_irqrestore(&hugetlb_lock, flags);
1908		update_and_free_hugetlb_folio(h, folio, true);
1909	} else if (h->surplus_huge_pages_node[nid]) {
1910		/* remove the page from active list */
1911		remove_hugetlb_folio(h, folio, true);
1912		spin_unlock_irqrestore(&hugetlb_lock, flags);
1913		update_and_free_hugetlb_folio(h, folio, true);
 
1914	} else {
1915		arch_clear_hugepage_flags(page);
1916		enqueue_hugetlb_folio(h, folio);
1917		spin_unlock_irqrestore(&hugetlb_lock, flags);
1918	}
 
1919}
1920
1921/*
1922 * Must be called with the hugetlb lock held
 
 
 
 
 
 
 
1923 */
1924static void __prep_account_new_huge_page(struct hstate *h, int nid)
 
 
1925{
1926	lockdep_assert_held(&hugetlb_lock);
1927	h->nr_huge_pages++;
1928	h->nr_huge_pages_node[nid]++;
 
 
 
 
 
 
 
 
1929}
 
1930
1931static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
1932{
1933	hugetlb_vmemmap_optimize(h, &folio->page);
1934	INIT_LIST_HEAD(&folio->lru);
1935	folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
1936	hugetlb_set_folio_subpool(folio, NULL);
1937	set_hugetlb_cgroup(folio, NULL);
1938	set_hugetlb_cgroup_rsvd(folio, NULL);
 
 
 
 
 
 
 
 
 
 
1939}
1940
1941static void prep_new_hugetlb_folio(struct hstate *h, struct folio *folio, int nid)
1942{
1943	__prep_new_hugetlb_folio(h, folio);
1944	spin_lock_irq(&hugetlb_lock);
1945	__prep_account_new_huge_page(h, nid);
1946	spin_unlock_irq(&hugetlb_lock);
 
 
 
 
1947}
1948
1949static bool __prep_compound_gigantic_folio(struct folio *folio,
1950					unsigned int order, bool demote)
1951{
1952	int i, j;
1953	int nr_pages = 1 << order;
1954	struct page *p;
1955
1956	__folio_clear_reserved(folio);
1957	__folio_set_head(folio);
1958	/* we rely on prep_new_hugetlb_folio to set the destructor */
1959	folio_set_compound_order(folio, order);
1960	for (i = 0; i < nr_pages; i++) {
1961		p = folio_page(folio, i);
1962
 
 
 
 
 
1963		/*
1964		 * For gigantic hugepages allocated through bootmem at
1965		 * boot, it's safer to be consistent with the not-gigantic
1966		 * hugepages and clear the PG_reserved bit from all tail pages
1967		 * too.  Otherwise drivers using get_user_pages() to access tail
1968		 * pages may get the reference counting wrong if they see
1969		 * PG_reserved set on a tail page (despite the head page not
1970		 * having PG_reserved set).  Enforcing this consistency between
1971		 * head and tail pages allows drivers to optimize away a check
1972		 * on the head page when they need know if put_page() is needed
1973		 * after get_user_pages().
1974		 */
1975		if (i != 0)	/* head page cleared above */
1976			__ClearPageReserved(p);
1977		/*
1978		 * Subtle and very unlikely
1979		 *
1980		 * Gigantic 'page allocators' such as memblock or cma will
1981		 * return a set of pages with each page ref counted.  We need
1982		 * to turn this set of pages into a compound page with tail
1983		 * page ref counts set to zero.  Code such as speculative page
1984		 * cache adding could take a ref on a 'to be' tail page.
1985		 * We need to respect any increased ref count, and only set
1986		 * the ref count to zero if count is currently 1.  If count
1987		 * is not 1, we return an error.  An error return indicates
1988		 * the set of pages can not be converted to a gigantic page.
1989		 * The caller who allocated the pages should then discard the
1990		 * pages using the appropriate free interface.
1991		 *
1992		 * In the case of demote, the ref count will be zero.
1993		 */
1994		if (!demote) {
1995			if (!page_ref_freeze(p, 1)) {
1996				pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
1997				goto out_error;
1998			}
1999		} else {
2000			VM_BUG_ON_PAGE(page_count(p), p);
2001		}
2002		if (i != 0)
2003			set_compound_head(p, &folio->page);
2004	}
2005	atomic_set(folio_mapcount_ptr(folio), -1);
2006	atomic_set(folio_subpages_mapcount_ptr(folio), 0);
2007	atomic_set(folio_pincount_ptr(folio), 0);
2008	return true;
2009
2010out_error:
2011	/* undo page modifications made above */
2012	for (j = 0; j < i; j++) {
2013		p = folio_page(folio, j);
2014		if (j != 0)
2015			clear_compound_head(p);
2016		set_page_refcounted(p);
2017	}
2018	/* need to clear PG_reserved on remaining tail pages  */
2019	for (; j < nr_pages; j++) {
2020		p = folio_page(folio, j);
2021		__ClearPageReserved(p);
 
 
2022	}
2023	folio_set_compound_order(folio, 0);
2024	__folio_clear_head(folio);
2025	return false;
2026}
2027
2028static bool prep_compound_gigantic_folio(struct folio *folio,
2029							unsigned int order)
2030{
2031	return __prep_compound_gigantic_folio(folio, order, false);
2032}
2033
2034static bool prep_compound_gigantic_folio_for_demote(struct folio *folio,
2035							unsigned int order)
2036{
2037	return __prep_compound_gigantic_folio(folio, order, true);
2038}
2039
2040/*
2041 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
2042 * transparent huge pages.  See the PageTransHuge() documentation for more
2043 * details.
2044 */
2045int PageHuge(struct page *page)
2046{
2047	if (!PageCompound(page))
2048		return 0;
2049
2050	page = compound_head(page);
2051	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
2052}
2053EXPORT_SYMBOL_GPL(PageHuge);
2054
2055/*
2056 * PageHeadHuge() only returns true for hugetlbfs head page, but not for
2057 * normal or transparent huge pages.
2058 */
2059int PageHeadHuge(struct page *page_head)
2060{
2061	if (!PageHead(page_head))
2062		return 0;
2063
2064	return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR;
2065}
2066EXPORT_SYMBOL_GPL(PageHeadHuge);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2067
2068/*
2069 * Find and lock address space (mapping) in write mode.
2070 *
2071 * Upon entry, the page is locked which means that page_mapping() is
2072 * stable.  Due to locking order, we can only trylock_write.  If we can
2073 * not get the lock, simply return NULL to caller.
 
 
 
 
2074 */
2075struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
2076{
2077	struct address_space *mapping = page_mapping(hpage);
2078
 
 
2079	if (!mapping)
2080		return mapping;
2081
 
 
 
2082	if (i_mmap_trylock_write(mapping))
2083		return mapping;
2084
2085	return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2086}
2087
2088pgoff_t hugetlb_basepage_index(struct page *page)
2089{
2090	struct page *page_head = compound_head(page);
2091	pgoff_t index = page_index(page_head);
2092	unsigned long compound_idx;
2093
 
 
 
2094	if (compound_order(page_head) >= MAX_ORDER)
2095		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
2096	else
2097		compound_idx = page - page_head;
2098
2099	return (index << compound_order(page_head)) + compound_idx;
2100}
2101
2102static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
2103		gfp_t gfp_mask, int nid, nodemask_t *nmask,
2104		nodemask_t *node_alloc_noretry)
2105{
2106	int order = huge_page_order(h);
2107	struct page *page;
2108	bool alloc_try_hard = true;
2109	bool retry = true;
2110
2111	/*
2112	 * By default we always try hard to allocate the page with
2113	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
2114	 * a loop (to adjust global huge page counts) and previous allocation
2115	 * failed, do not continue to try hard on the same node.  Use the
2116	 * node_alloc_noretry bitmap to manage this state information.
2117	 */
2118	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
2119		alloc_try_hard = false;
2120	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
2121	if (alloc_try_hard)
2122		gfp_mask |= __GFP_RETRY_MAYFAIL;
2123	if (nid == NUMA_NO_NODE)
2124		nid = numa_mem_id();
2125retry:
2126	page = __alloc_pages(gfp_mask, order, nid, nmask);
2127
2128	/* Freeze head page */
2129	if (page && !page_ref_freeze(page, 1)) {
2130		__free_pages(page, order);
2131		if (retry) {	/* retry once */
2132			retry = false;
2133			goto retry;
2134		}
2135		/* WOW!  twice in a row. */
2136		pr_warn("HugeTLB head page unexpected inflated ref count\n");
2137		page = NULL;
2138	}
2139
2140	/*
2141	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
2142	 * indicates an overall state change.  Clear bit so that we resume
2143	 * normal 'try hard' allocations.
2144	 */
2145	if (node_alloc_noretry && page && !alloc_try_hard)
2146		node_clear(nid, *node_alloc_noretry);
2147
2148	/*
2149	 * If we tried hard to get a page but failed, set bit so that
2150	 * subsequent attempts will not try as hard until there is an
2151	 * overall state change.
2152	 */
2153	if (node_alloc_noretry && !page && alloc_try_hard)
2154		node_set(nid, *node_alloc_noretry);
2155
2156	if (!page) {
2157		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
2158		return NULL;
2159	}
2160
2161	__count_vm_event(HTLB_BUDDY_PGALLOC);
2162	return page_folio(page);
2163}
2164
2165/*
2166 * Common helper to allocate a fresh hugetlb page. All specific allocators
2167 * should use this function to get new hugetlb pages
2168 *
2169 * Note that returned page is 'frozen':  ref count of head page and all tail
2170 * pages is zero.
2171 */
2172static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
2173		gfp_t gfp_mask, int nid, nodemask_t *nmask,
2174		nodemask_t *node_alloc_noretry)
2175{
2176	struct folio *folio;
2177	bool retry = false;
2178
2179retry:
2180	if (hstate_is_gigantic(h))
2181		folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask);
2182	else
2183		folio = alloc_buddy_hugetlb_folio(h, gfp_mask,
2184				nid, nmask, node_alloc_noretry);
2185	if (!folio)
2186		return NULL;
2187	if (hstate_is_gigantic(h)) {
2188		if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) {
2189			/*
2190			 * Rare failure to convert pages to compound page.
2191			 * Free pages and try again - ONCE!
2192			 */
2193			free_gigantic_folio(folio, huge_page_order(h));
2194			if (!retry) {
2195				retry = true;
2196				goto retry;
2197			}
2198			return NULL;
2199		}
2200	}
2201	prep_new_hugetlb_folio(h, folio, folio_nid(folio));
2202
2203	return folio;
 
 
 
 
2204}
2205
2206/*
2207 * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
2208 * manner.
2209 */
2210static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
2211				nodemask_t *node_alloc_noretry)
2212{
2213	struct folio *folio;
2214	int nr_nodes, node;
2215	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2216
2217	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2218		folio = alloc_fresh_hugetlb_folio(h, gfp_mask, node,
2219					nodes_allowed, node_alloc_noretry);
2220		if (folio) {
2221			free_huge_page(&folio->page); /* free it into the hugepage allocator */
2222			return 1;
2223		}
2224	}
2225
2226	return 0;
 
 
 
 
 
2227}
2228
2229/*
2230 * Remove huge page from pool from next node to free.  Attempt to keep
2231 * persistent huge pages more or less balanced over allowed nodes.
2232 * This routine only 'removes' the hugetlb page.  The caller must make
2233 * an additional call to free the page to low level allocators.
2234 * Called with hugetlb_lock locked.
2235 */
2236static struct page *remove_pool_huge_page(struct hstate *h,
2237						nodemask_t *nodes_allowed,
2238						 bool acct_surplus)
2239{
2240	int nr_nodes, node;
2241	struct page *page = NULL;
2242	struct folio *folio;
2243
2244	lockdep_assert_held(&hugetlb_lock);
2245	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2246		/*
2247		 * If we're returning unused surplus pages, only examine
2248		 * nodes with surplus pages.
2249		 */
2250		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
2251		    !list_empty(&h->hugepage_freelists[node])) {
2252			page = list_entry(h->hugepage_freelists[node].next,
 
2253					  struct page, lru);
2254			folio = page_folio(page);
2255			remove_hugetlb_folio(h, folio, acct_surplus);
 
 
 
 
 
 
 
2256			break;
2257		}
2258	}
2259
2260	return page;
2261}
2262
2263/*
2264 * Dissolve a given free hugepage into free buddy pages. This function does
2265 * nothing for in-use hugepages and non-hugepages.
2266 * This function returns values like below:
2267 *
2268 *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
2269 *           when the system is under memory pressure and the feature of
2270 *           freeing unused vmemmap pages associated with each hugetlb page
2271 *           is enabled.
2272 *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
2273 *           (allocated or reserved.)
2274 *       0:  successfully dissolved free hugepages or the page is not a
2275 *           hugepage (considered as already dissolved)
2276 */
2277int dissolve_free_huge_page(struct page *page)
2278{
2279	int rc = -EBUSY;
2280	struct folio *folio = page_folio(page);
2281
2282retry:
2283	/* Not to disrupt normal path by vainly holding hugetlb_lock */
2284	if (!folio_test_hugetlb(folio))
2285		return 0;
2286
2287	spin_lock_irq(&hugetlb_lock);
2288	if (!folio_test_hugetlb(folio)) {
2289		rc = 0;
2290		goto out;
2291	}
2292
2293	if (!folio_ref_count(folio)) {
2294		struct hstate *h = folio_hstate(folio);
2295		if (!available_huge_pages(h))
 
 
2296			goto out;
2297
2298		/*
2299		 * We should make sure that the page is already on the free list
2300		 * when it is dissolved.
2301		 */
2302		if (unlikely(!folio_test_hugetlb_freed(folio))) {
2303			spin_unlock_irq(&hugetlb_lock);
2304			cond_resched();
2305
2306			/*
2307			 * Theoretically, we should return -EBUSY when we
2308			 * encounter this race. In fact, we have a chance
2309			 * to successfully dissolve the page if we do a
2310			 * retry. Because the race window is quite small.
2311			 * If we seize this opportunity, it is an optimization
2312			 * for increasing the success rate of dissolving page.
2313			 */
2314			goto retry;
2315		}
2316
2317		remove_hugetlb_folio(h, folio, false);
 
2318		h->max_huge_pages--;
2319		spin_unlock_irq(&hugetlb_lock);
2320
2321		/*
2322		 * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
2323		 * before freeing the page.  update_and_free_hugtlb_folio will fail to
2324		 * free the page if it can not allocate required vmemmap.  We
2325		 * need to adjust max_huge_pages if the page is not freed.
2326		 * Attempt to allocate vmemmmap here so that we can take
2327		 * appropriate action on failure.
2328		 */
2329		rc = hugetlb_vmemmap_restore(h, &folio->page);
2330		if (!rc) {
2331			update_and_free_hugetlb_folio(h, folio, false);
2332		} else {
2333			spin_lock_irq(&hugetlb_lock);
2334			add_hugetlb_folio(h, folio, false);
2335			h->max_huge_pages++;
2336			spin_unlock_irq(&hugetlb_lock);
2337		}
2338
2339		return rc;
2340	}
2341out:
2342	spin_unlock_irq(&hugetlb_lock);
2343	return rc;
2344}
2345
2346/*
2347 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
2348 * make specified memory blocks removable from the system.
2349 * Note that this will dissolve a free gigantic hugepage completely, if any
2350 * part of it lies within the given range.
2351 * Also note that if dissolve_free_huge_page() returns with an error, all
2352 * free hugepages that were dissolved before that error are lost.
2353 */
2354int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
2355{
2356	unsigned long pfn;
2357	struct page *page;
2358	int rc = 0;
2359	unsigned int order;
2360	struct hstate *h;
2361
2362	if (!hugepages_supported())
2363		return rc;
2364
2365	order = huge_page_order(&default_hstate);
2366	for_each_hstate(h)
2367		order = min(order, huge_page_order(h));
2368
2369	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) {
2370		page = pfn_to_page(pfn);
2371		rc = dissolve_free_huge_page(page);
2372		if (rc)
2373			break;
2374	}
2375
2376	return rc;
2377}
2378
2379/*
2380 * Allocates a fresh surplus page from the page allocator.
2381 */
2382static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
2383						int nid, nodemask_t *nmask)
2384{
2385	struct folio *folio = NULL;
2386
2387	if (hstate_is_gigantic(h))
2388		return NULL;
2389
2390	spin_lock_irq(&hugetlb_lock);
2391	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
2392		goto out_unlock;
2393	spin_unlock_irq(&hugetlb_lock);
2394
2395	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
2396	if (!folio)
2397		return NULL;
2398
2399	spin_lock_irq(&hugetlb_lock);
2400	/*
2401	 * We could have raced with the pool size change.
2402	 * Double check that and simply deallocate the new page
2403	 * if we would end up overcommiting the surpluses. Abuse
2404	 * temporary page to workaround the nasty free_huge_page
2405	 * codeflow
2406	 */
2407	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2408		folio_set_hugetlb_temporary(folio);
2409		spin_unlock_irq(&hugetlb_lock);
2410		free_huge_page(&folio->page);
2411		return NULL;
 
 
 
2412	}
2413
2414	h->surplus_huge_pages++;
2415	h->surplus_huge_pages_node[folio_nid(folio)]++;
2416
2417out_unlock:
2418	spin_unlock_irq(&hugetlb_lock);
2419
2420	return &folio->page;
2421}
2422
2423static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
2424				     int nid, nodemask_t *nmask)
2425{
2426	struct folio *folio;
2427
2428	if (hstate_is_gigantic(h))
2429		return NULL;
2430
2431	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
2432	if (!folio)
2433		return NULL;
2434
2435	/* fresh huge pages are frozen */
2436	folio_ref_unfreeze(folio, 1);
2437	/*
2438	 * We do not account these pages as surplus because they are only
2439	 * temporary and will be released properly on the last reference
2440	 */
2441	folio_set_hugetlb_temporary(folio);
2442
2443	return &folio->page;
2444}
2445
2446/*
2447 * Use the VMA's mpolicy to allocate a huge page from the buddy.
2448 */
2449static
2450struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
2451		struct vm_area_struct *vma, unsigned long addr)
2452{
2453	struct page *page = NULL;
2454	struct mempolicy *mpol;
2455	gfp_t gfp_mask = htlb_alloc_mask(h);
2456	int nid;
2457	nodemask_t *nodemask;
2458
2459	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2460	if (mpol_is_preferred_many(mpol)) {
2461		gfp_t gfp = gfp_mask | __GFP_NOWARN;
2462
2463		gfp &=  ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2464		page = alloc_surplus_huge_page(h, gfp, nid, nodemask);
2465
2466		/* Fallback to all nodes if page==NULL */
2467		nodemask = NULL;
2468	}
2469
2470	if (!page)
2471		page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);
2472	mpol_cond_put(mpol);
 
2473	return page;
2474}
2475
2476/* page migration callback function */
2477struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
2478		nodemask_t *nmask, gfp_t gfp_mask)
2479{
2480	spin_lock_irq(&hugetlb_lock);
2481	if (available_huge_pages(h)) {
2482		struct page *page;
2483
2484		page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
2485		if (page) {
2486			spin_unlock_irq(&hugetlb_lock);
2487			return page;
2488		}
2489	}
2490	spin_unlock_irq(&hugetlb_lock);
2491
2492	return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
2493}
2494
2495/* mempolicy aware migration callback */
2496struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
2497		unsigned long address)
2498{
2499	struct mempolicy *mpol;
2500	nodemask_t *nodemask;
2501	struct page *page;
2502	gfp_t gfp_mask;
2503	int node;
2504
2505	gfp_mask = htlb_alloc_mask(h);
2506	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
2507	page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask);
2508	mpol_cond_put(mpol);
2509
2510	return page;
2511}
2512
2513/*
2514 * Increase the hugetlb pool such that it can accommodate a reservation
2515 * of size 'delta'.
2516 */
2517static int gather_surplus_pages(struct hstate *h, long delta)
2518	__must_hold(&hugetlb_lock)
2519{
2520	LIST_HEAD(surplus_list);
2521	struct page *page, *tmp;
2522	int ret;
2523	long i;
2524	long needed, allocated;
2525	bool alloc_ok = true;
2526
2527	lockdep_assert_held(&hugetlb_lock);
2528	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2529	if (needed <= 0) {
2530		h->resv_huge_pages += delta;
2531		return 0;
2532	}
2533
2534	allocated = 0;
 
2535
2536	ret = -ENOMEM;
2537retry:
2538	spin_unlock_irq(&hugetlb_lock);
2539	for (i = 0; i < needed; i++) {
2540		page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
2541				NUMA_NO_NODE, NULL);
2542		if (!page) {
2543			alloc_ok = false;
2544			break;
2545		}
2546		list_add(&page->lru, &surplus_list);
2547		cond_resched();
2548	}
2549	allocated += i;
2550
2551	/*
2552	 * After retaking hugetlb_lock, we need to recalculate 'needed'
2553	 * because either resv_huge_pages or free_huge_pages may have changed.
2554	 */
2555	spin_lock_irq(&hugetlb_lock);
2556	needed = (h->resv_huge_pages + delta) -
2557			(h->free_huge_pages + allocated);
2558	if (needed > 0) {
2559		if (alloc_ok)
2560			goto retry;
2561		/*
2562		 * We were not able to allocate enough pages to
2563		 * satisfy the entire reservation so we free what
2564		 * we've allocated so far.
2565		 */
2566		goto free;
2567	}
2568	/*
2569	 * The surplus_list now contains _at_least_ the number of extra pages
2570	 * needed to accommodate the reservation.  Add the appropriate number
2571	 * of pages to the hugetlb pool and free the extras back to the buddy
2572	 * allocator.  Commit the entire reservation here to prevent another
2573	 * process from stealing the pages as they are added to the pool but
2574	 * before they are reserved.
2575	 */
2576	needed += allocated;
2577	h->resv_huge_pages += delta;
2578	ret = 0;
2579
2580	/* Free the needed pages to the hugetlb pool */
2581	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
2582		if ((--needed) < 0)
2583			break;
2584		/* Add the page to the hugetlb allocator */
2585		enqueue_hugetlb_folio(h, page_folio(page));
 
 
 
 
 
2586	}
2587free:
2588	spin_unlock_irq(&hugetlb_lock);
2589
2590	/*
2591	 * Free unnecessary surplus pages to the buddy allocator.
2592	 * Pages have no ref count, call free_huge_page directly.
2593	 */
2594	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
2595		free_huge_page(page);
2596	spin_lock_irq(&hugetlb_lock);
2597
2598	return ret;
2599}
2600
2601/*
2602 * This routine has two main purposes:
2603 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2604 *    in unused_resv_pages.  This corresponds to the prior adjustments made
2605 *    to the associated reservation map.
2606 * 2) Free any unused surplus pages that may have been allocated to satisfy
2607 *    the reservation.  As many as unused_resv_pages may be freed.
 
 
 
 
 
 
2608 */
2609static void return_unused_surplus_pages(struct hstate *h,
2610					unsigned long unused_resv_pages)
2611{
2612	unsigned long nr_pages;
2613	struct page *page;
2614	LIST_HEAD(page_list);
2615
2616	lockdep_assert_held(&hugetlb_lock);
2617	/* Uncommit the reservation */
2618	h->resv_huge_pages -= unused_resv_pages;
2619
2620	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
2621		goto out;
2622
2623	/*
2624	 * Part (or even all) of the reservation could have been backed
2625	 * by pre-allocated pages. Only free surplus pages.
2626	 */
2627	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2628
2629	/*
2630	 * We want to release as many surplus pages as possible, spread
2631	 * evenly across all nodes with memory. Iterate across these nodes
2632	 * until we can no longer free unreserved surplus pages. This occurs
2633	 * when the nodes with surplus pages have no free pages.
2634	 * remove_pool_huge_page() will balance the freed pages across the
2635	 * on-line nodes with memory and will handle the hstate accounting.
 
 
 
 
2636	 */
2637	while (nr_pages--) {
2638		page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1);
2639		if (!page)
 
2640			goto out;
2641
2642		list_add(&page->lru, &page_list);
2643	}
2644
2645out:
2646	spin_unlock_irq(&hugetlb_lock);
2647	update_and_free_pages_bulk(h, &page_list);
2648	spin_lock_irq(&hugetlb_lock);
2649}
2650
2651
2652/*
2653 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2654 * are used by the huge page allocation routines to manage reservations.
2655 *
2656 * vma_needs_reservation is called to determine if the huge page at addr
2657 * within the vma has an associated reservation.  If a reservation is
2658 * needed, the value 1 is returned.  The caller is then responsible for
2659 * managing the global reservation and subpool usage counts.  After
2660 * the huge page has been allocated, vma_commit_reservation is called
2661 * to add the page to the reservation map.  If the page allocation fails,
2662 * the reservation must be ended instead of committed.  vma_end_reservation
2663 * is called in such cases.
2664 *
2665 * In the normal case, vma_commit_reservation returns the same value
2666 * as the preceding vma_needs_reservation call.  The only time this
2667 * is not the case is if a reserve map was changed between calls.  It
2668 * is the responsibility of the caller to notice the difference and
2669 * take appropriate action.
2670 *
2671 * vma_add_reservation is used in error paths where a reservation must
2672 * be restored when a newly allocated huge page must be freed.  It is
2673 * to be called after calling vma_needs_reservation to determine if a
2674 * reservation exists.
2675 *
2676 * vma_del_reservation is used in error paths where an entry in the reserve
2677 * map was created during huge page allocation and must be removed.  It is to
2678 * be called after calling vma_needs_reservation to determine if a reservation
2679 * exists.
2680 */
2681enum vma_resv_mode {
2682	VMA_NEEDS_RESV,
2683	VMA_COMMIT_RESV,
2684	VMA_END_RESV,
2685	VMA_ADD_RESV,
2686	VMA_DEL_RESV,
2687};
2688static long __vma_reservation_common(struct hstate *h,
2689				struct vm_area_struct *vma, unsigned long addr,
2690				enum vma_resv_mode mode)
2691{
2692	struct resv_map *resv;
2693	pgoff_t idx;
2694	long ret;
2695	long dummy_out_regions_needed;
2696
2697	resv = vma_resv_map(vma);
2698	if (!resv)
2699		return 1;
2700
2701	idx = vma_hugecache_offset(h, vma, addr);
2702	switch (mode) {
2703	case VMA_NEEDS_RESV:
2704		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2705		/* We assume that vma_reservation_* routines always operate on
2706		 * 1 page, and that adding to resv map a 1 page entry can only
2707		 * ever require 1 region.
2708		 */
2709		VM_BUG_ON(dummy_out_regions_needed != 1);
2710		break;
2711	case VMA_COMMIT_RESV:
2712		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2713		/* region_add calls of range 1 should never fail. */
2714		VM_BUG_ON(ret < 0);
2715		break;
2716	case VMA_END_RESV:
2717		region_abort(resv, idx, idx + 1, 1);
2718		ret = 0;
2719		break;
2720	case VMA_ADD_RESV:
2721		if (vma->vm_flags & VM_MAYSHARE) {
2722			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2723			/* region_add calls of range 1 should never fail. */
2724			VM_BUG_ON(ret < 0);
2725		} else {
2726			region_abort(resv, idx, idx + 1, 1);
2727			ret = region_del(resv, idx, idx + 1);
2728		}
2729		break;
2730	case VMA_DEL_RESV:
2731		if (vma->vm_flags & VM_MAYSHARE) {
2732			region_abort(resv, idx, idx + 1, 1);
2733			ret = region_del(resv, idx, idx + 1);
2734		} else {
2735			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2736			/* region_add calls of range 1 should never fail. */
2737			VM_BUG_ON(ret < 0);
2738		}
2739		break;
2740	default:
2741		BUG();
2742	}
2743
2744	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2745		return ret;
2746	/*
2747	 * We know private mapping must have HPAGE_RESV_OWNER set.
2748	 *
2749	 * In most cases, reserves always exist for private mappings.
2750	 * However, a file associated with mapping could have been
2751	 * hole punched or truncated after reserves were consumed.
2752	 * As subsequent fault on such a range will not use reserves.
2753	 * Subtle - The reserve map for private mappings has the
2754	 * opposite meaning than that of shared mappings.  If NO
2755	 * entry is in the reserve map, it means a reservation exists.
2756	 * If an entry exists in the reserve map, it means the
2757	 * reservation has already been consumed.  As a result, the
2758	 * return value of this routine is the opposite of the
2759	 * value returned from reserve map manipulation routines above.
2760	 */
2761	if (ret > 0)
2762		return 0;
2763	if (ret == 0)
2764		return 1;
2765	return ret;
 
2766}
2767
2768static long vma_needs_reservation(struct hstate *h,
2769			struct vm_area_struct *vma, unsigned long addr)
2770{
2771	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2772}
2773
2774static long vma_commit_reservation(struct hstate *h,
2775			struct vm_area_struct *vma, unsigned long addr)
2776{
2777	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2778}
2779
2780static void vma_end_reservation(struct hstate *h,
2781			struct vm_area_struct *vma, unsigned long addr)
2782{
2783	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2784}
2785
2786static long vma_add_reservation(struct hstate *h,
2787			struct vm_area_struct *vma, unsigned long addr)
2788{
2789	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2790}
2791
2792static long vma_del_reservation(struct hstate *h,
2793			struct vm_area_struct *vma, unsigned long addr)
2794{
2795	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
2796}
2797
2798/*
2799 * This routine is called to restore reservation information on error paths.
2800 * It should ONLY be called for pages allocated via alloc_huge_page(), and
2801 * the hugetlb mutex should remain held when calling this routine.
2802 *
2803 * It handles two specific cases:
2804 * 1) A reservation was in place and the page consumed the reservation.
2805 *    HPageRestoreReserve is set in the page.
2806 * 2) No reservation was in place for the page, so HPageRestoreReserve is
2807 *    not set.  However, alloc_huge_page always updates the reserve map.
2808 *
2809 * In case 1, free_huge_page later in the error path will increment the
2810 * global reserve count.  But, free_huge_page does not have enough context
2811 * to adjust the reservation map.  This case deals primarily with private
2812 * mappings.  Adjust the reserve map here to be consistent with global
2813 * reserve count adjustments to be made by free_huge_page.  Make sure the
2814 * reserve map indicates there is a reservation present.
2815 *
2816 * In case 2, simply undo reserve map modifications done by alloc_huge_page.
2817 */
2818void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2819			unsigned long address, struct page *page)
2820{
2821	long rc = vma_needs_reservation(h, vma, address);
 
2822
2823	if (HPageRestoreReserve(page)) {
2824		if (unlikely(rc < 0))
2825			/*
2826			 * Rare out of memory condition in reserve map
2827			 * manipulation.  Clear HPageRestoreReserve so that
2828			 * global reserve count will not be incremented
2829			 * by free_huge_page.  This will make it appear
2830			 * as though the reservation for this page was
2831			 * consumed.  This may prevent the task from
2832			 * faulting in the page at a later time.  This
2833			 * is better than inconsistent global huge page
2834			 * accounting of reserve counts.
2835			 */
2836			ClearHPageRestoreReserve(page);
2837		else if (rc)
2838			(void)vma_add_reservation(h, vma, address);
2839		else
2840			vma_end_reservation(h, vma, address);
2841	} else {
2842		if (!rc) {
2843			/*
2844			 * This indicates there is an entry in the reserve map
2845			 * not added by alloc_huge_page.  We know it was added
2846			 * before the alloc_huge_page call, otherwise
2847			 * HPageRestoreReserve would be set on the page.
2848			 * Remove the entry so that a subsequent allocation
2849			 * does not consume a reservation.
2850			 */
2851			rc = vma_del_reservation(h, vma, address);
2852			if (rc < 0)
2853				/*
2854				 * VERY rare out of memory condition.  Since
2855				 * we can not delete the entry, set
2856				 * HPageRestoreReserve so that the reserve
2857				 * count will be incremented when the page
2858				 * is freed.  This reserve will be consumed
2859				 * on a subsequent allocation.
2860				 */
2861				SetHPageRestoreReserve(page);
2862		} else if (rc < 0) {
2863			/*
2864			 * Rare out of memory condition from
2865			 * vma_needs_reservation call.  Memory allocation is
2866			 * only attempted if a new entry is needed.  Therefore,
2867			 * this implies there is not an entry in the
2868			 * reserve map.
2869			 *
2870			 * For shared mappings, no entry in the map indicates
2871			 * no reservation.  We are done.
2872			 */
2873			if (!(vma->vm_flags & VM_MAYSHARE))
2874				/*
2875				 * For private mappings, no entry indicates
2876				 * a reservation is present.  Since we can
2877				 * not add an entry, set SetHPageRestoreReserve
2878				 * on the page so reserve count will be
2879				 * incremented when freed.  This reserve will
2880				 * be consumed on a subsequent allocation.
2881				 */
2882				SetHPageRestoreReserve(page);
2883		} else
2884			/*
2885			 * No reservation present, do nothing
2886			 */
2887			 vma_end_reservation(h, vma, address);
2888	}
2889}
2890
2891/*
2892 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
2893 * the old one
2894 * @h: struct hstate old page belongs to
2895 * @old_folio: Old folio to dissolve
2896 * @list: List to isolate the page in case we need to
2897 * Returns 0 on success, otherwise negated error.
2898 */
2899static int alloc_and_dissolve_hugetlb_folio(struct hstate *h,
2900			struct folio *old_folio, struct list_head *list)
2901{
2902	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2903	int nid = folio_nid(old_folio);
2904	struct folio *new_folio;
2905	int ret = 0;
2906
2907	/*
2908	 * Before dissolving the folio, we need to allocate a new one for the
2909	 * pool to remain stable.  Here, we allocate the folio and 'prep' it
2910	 * by doing everything but actually updating counters and adding to
2911	 * the pool.  This simplifies and let us do most of the processing
2912	 * under the lock.
2913	 */
2914	new_folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid, NULL, NULL);
2915	if (!new_folio)
2916		return -ENOMEM;
2917	__prep_new_hugetlb_folio(h, new_folio);
2918
2919retry:
2920	spin_lock_irq(&hugetlb_lock);
2921	if (!folio_test_hugetlb(old_folio)) {
2922		/*
2923		 * Freed from under us. Drop new_folio too.
2924		 */
2925		goto free_new;
2926	} else if (folio_ref_count(old_folio)) {
2927		/*
2928		 * Someone has grabbed the folio, try to isolate it here.
2929		 * Fail with -EBUSY if not possible.
2930		 */
2931		spin_unlock_irq(&hugetlb_lock);
2932		ret = isolate_hugetlb(&old_folio->page, list);
2933		spin_lock_irq(&hugetlb_lock);
2934		goto free_new;
2935	} else if (!folio_test_hugetlb_freed(old_folio)) {
2936		/*
2937		 * Folio's refcount is 0 but it has not been enqueued in the
2938		 * freelist yet. Race window is small, so we can succeed here if
2939		 * we retry.
2940		 */
2941		spin_unlock_irq(&hugetlb_lock);
2942		cond_resched();
2943		goto retry;
2944	} else {
2945		/*
2946		 * Ok, old_folio is still a genuine free hugepage. Remove it from
2947		 * the freelist and decrease the counters. These will be
2948		 * incremented again when calling __prep_account_new_huge_page()
2949		 * and enqueue_hugetlb_folio() for new_folio. The counters will
2950		 * remain stable since this happens under the lock.
2951		 */
2952		remove_hugetlb_folio(h, old_folio, false);
2953
2954		/*
2955		 * Ref count on new_folio is already zero as it was dropped
2956		 * earlier.  It can be directly added to the pool free list.
2957		 */
2958		__prep_account_new_huge_page(h, nid);
2959		enqueue_hugetlb_folio(h, new_folio);
2960
2961		/*
2962		 * Folio has been replaced, we can safely free the old one.
2963		 */
2964		spin_unlock_irq(&hugetlb_lock);
2965		update_and_free_hugetlb_folio(h, old_folio, false);
2966	}
2967
2968	return ret;
2969
2970free_new:
2971	spin_unlock_irq(&hugetlb_lock);
2972	/* Folio has a zero ref count, but needs a ref to be freed */
2973	folio_ref_unfreeze(new_folio, 1);
2974	update_and_free_hugetlb_folio(h, new_folio, false);
2975
2976	return ret;
2977}
2978
2979int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
2980{
2981	struct hstate *h;
2982	struct folio *folio = page_folio(page);
2983	int ret = -EBUSY;
2984
2985	/*
2986	 * The page might have been dissolved from under our feet, so make sure
2987	 * to carefully check the state under the lock.
2988	 * Return success when racing as if we dissolved the page ourselves.
2989	 */
2990	spin_lock_irq(&hugetlb_lock);
2991	if (folio_test_hugetlb(folio)) {
2992		h = folio_hstate(folio);
2993	} else {
2994		spin_unlock_irq(&hugetlb_lock);
2995		return 0;
2996	}
2997	spin_unlock_irq(&hugetlb_lock);
2998
2999	/*
3000	 * Fence off gigantic pages as there is a cyclic dependency between
3001	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
3002	 * of bailing out right away without further retrying.
3003	 */
3004	if (hstate_is_gigantic(h))
3005		return -ENOMEM;
3006
3007	if (folio_ref_count(folio) && !isolate_hugetlb(&folio->page, list))
3008		ret = 0;
3009	else if (!folio_ref_count(folio))
3010		ret = alloc_and_dissolve_hugetlb_folio(h, folio, list);
3011
3012	return ret;
3013}
3014
3015struct page *alloc_huge_page(struct vm_area_struct *vma,
3016				    unsigned long addr, int avoid_reserve)
3017{
3018	struct hugepage_subpool *spool = subpool_vma(vma);
3019	struct hstate *h = hstate_vma(vma);
3020	struct page *page;
3021	struct folio *folio;
3022	long map_chg, map_commit;
3023	long gbl_chg;
3024	int ret, idx;
3025	struct hugetlb_cgroup *h_cg;
3026	bool deferred_reserve;
3027
3028	idx = hstate_index(h);
3029	/*
3030	 * Examine the region/reserve map to determine if the process
3031	 * has a reservation for the page to be allocated.  A return
3032	 * code of zero indicates a reservation exists (no change).
3033	 */
3034	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
3035	if (map_chg < 0)
3036		return ERR_PTR(-ENOMEM);
3037
3038	/*
3039	 * Processes that did not create the mapping will have no
3040	 * reserves as indicated by the region/reserve map. Check
3041	 * that the allocation will not exceed the subpool limit.
3042	 * Allocations for MAP_NORESERVE mappings also need to be
3043	 * checked against any subpool limit.
3044	 */
3045	if (map_chg || avoid_reserve) {
3046		gbl_chg = hugepage_subpool_get_pages(spool, 1);
3047		if (gbl_chg < 0) {
3048			vma_end_reservation(h, vma, addr);
3049			return ERR_PTR(-ENOSPC);
3050		}
3051
3052		/*
3053		 * Even though there was no reservation in the region/reserve
3054		 * map, there could be reservations associated with the
3055		 * subpool that can be used.  This would be indicated if the
3056		 * return value of hugepage_subpool_get_pages() is zero.
3057		 * However, if avoid_reserve is specified we still avoid even
3058		 * the subpool reservations.
3059		 */
3060		if (avoid_reserve)
3061			gbl_chg = 1;
3062	}
3063
3064	/* If this allocation is not consuming a reservation, charge it now.
3065	 */
3066	deferred_reserve = map_chg || avoid_reserve;
3067	if (deferred_reserve) {
3068		ret = hugetlb_cgroup_charge_cgroup_rsvd(
3069			idx, pages_per_huge_page(h), &h_cg);
3070		if (ret)
3071			goto out_subpool_put;
3072	}
3073
3074	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
3075	if (ret)
3076		goto out_uncharge_cgroup_reservation;
3077
3078	spin_lock_irq(&hugetlb_lock);
3079	/*
3080	 * glb_chg is passed to indicate whether or not a page must be taken
3081	 * from the global free pool (global change).  gbl_chg == 0 indicates
3082	 * a reservation exists for the allocation.
3083	 */
3084	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
3085	if (!page) {
3086		spin_unlock_irq(&hugetlb_lock);
3087		page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
3088		if (!page)
3089			goto out_uncharge_cgroup;
3090		spin_lock_irq(&hugetlb_lock);
3091		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
3092			SetHPageRestoreReserve(page);
3093			h->resv_huge_pages--;
3094		}
3095		list_add(&page->lru, &h->hugepage_activelist);
3096		set_page_refcounted(page);
3097		/* Fall through */
3098	}
3099	folio = page_folio(page);
3100	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
3101	/* If allocation is not consuming a reservation, also store the
3102	 * hugetlb_cgroup pointer on the page.
3103	 */
3104	if (deferred_reserve) {
3105		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
3106						  h_cg, page);
3107	}
3108
3109	spin_unlock_irq(&hugetlb_lock);
3110
3111	hugetlb_set_page_subpool(page, spool);
3112
3113	map_commit = vma_commit_reservation(h, vma, addr);
3114	if (unlikely(map_chg > map_commit)) {
3115		/*
3116		 * The page was added to the reservation map between
3117		 * vma_needs_reservation and vma_commit_reservation.
3118		 * This indicates a race with hugetlb_reserve_pages.
3119		 * Adjust for the subpool count incremented above AND
3120		 * in hugetlb_reserve_pages for the same page.  Also,
3121		 * the reservation count added in hugetlb_reserve_pages
3122		 * no longer applies.
3123		 */
3124		long rsv_adjust;
3125
3126		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
3127		hugetlb_acct_memory(h, -rsv_adjust);
3128		if (deferred_reserve)
3129			hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
3130					pages_per_huge_page(h), folio);
3131	}
3132	return page;
3133
3134out_uncharge_cgroup:
3135	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
3136out_uncharge_cgroup_reservation:
3137	if (deferred_reserve)
3138		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
3139						    h_cg);
3140out_subpool_put:
3141	if (map_chg || avoid_reserve)
3142		hugepage_subpool_put_pages(spool, 1);
3143	vma_end_reservation(h, vma, addr);
3144	return ERR_PTR(-ENOSPC);
3145}
3146
3147int alloc_bootmem_huge_page(struct hstate *h, int nid)
3148	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
3149int __alloc_bootmem_huge_page(struct hstate *h, int nid)
3150{
3151	struct huge_bootmem_page *m = NULL; /* initialize for clang */
3152	int nr_nodes, node;
3153
3154	/* do node specific alloc */
3155	if (nid != NUMA_NO_NODE) {
3156		m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
3157				0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
3158		if (!m)
3159			return 0;
3160		goto found;
3161	}
3162	/* allocate from next node when distributing huge pages */
3163	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
3164		m = memblock_alloc_try_nid_raw(
 
 
3165				huge_page_size(h), huge_page_size(h),
3166				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3167		/*
3168		 * Use the beginning of the huge page to store the
3169		 * huge_bootmem_page struct (until gather_bootmem
3170		 * puts them into the mem_map).
3171		 */
3172		if (!m)
3173			return 0;
3174		goto found;
 
3175	}
 
3176
3177found:
 
3178	/* Put them into a private list first because mem_map is not up yet */
3179	INIT_LIST_HEAD(&m->list);
3180	list_add(&m->list, &huge_boot_pages);
3181	m->hstate = h;
3182	return 1;
3183}
3184
3185/*
3186 * Put bootmem huge pages into the standard lists after mem_map is up.
3187 * Note: This only applies to gigantic (order > MAX_ORDER) pages.
3188 */
 
 
 
 
 
 
3189static void __init gather_bootmem_prealloc(void)
3190{
3191	struct huge_bootmem_page *m;
3192
3193	list_for_each_entry(m, &huge_boot_pages, list) {
3194		struct page *page = virt_to_page(m);
3195		struct folio *folio = page_folio(page);
3196		struct hstate *h = m->hstate;
3197
3198		VM_BUG_ON(!hstate_is_gigantic(h));
3199		WARN_ON(folio_ref_count(folio) != 1);
3200		if (prep_compound_gigantic_folio(folio, huge_page_order(h))) {
3201			WARN_ON(folio_test_reserved(folio));
3202			prep_new_hugetlb_folio(h, folio, folio_nid(folio));
3203			free_huge_page(page); /* add to the hugepage allocator */
3204		} else {
3205			/* VERY unlikely inflated ref count on a tail page */
3206			free_gigantic_folio(folio, huge_page_order(h));
3207		}
3208
3209		/*
3210		 * We need to restore the 'stolen' pages to totalram_pages
3211		 * in order to fix confusing memory reports from free(1) and
3212		 * other side-effects, like CommitLimit going negative.
 
3213		 */
3214		adjust_managed_page_count(page, pages_per_huge_page(h));
 
3215		cond_resched();
3216	}
3217}
3218static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
3219{
3220	unsigned long i;
3221	char buf[32];
3222
3223	for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
3224		if (hstate_is_gigantic(h)) {
3225			if (!alloc_bootmem_huge_page(h, nid))
3226				break;
3227		} else {
3228			struct folio *folio;
3229			gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
3230
3231			folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
3232					&node_states[N_MEMORY], NULL);
3233			if (!folio)
3234				break;
3235			free_huge_page(&folio->page); /* free it into the hugepage allocator */
3236		}
3237		cond_resched();
3238	}
3239	if (i == h->max_huge_pages_node[nid])
3240		return;
3241
3242	string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3243	pr_warn("HugeTLB: allocating %u of page size %s failed node%d.  Only allocated %lu hugepages.\n",
3244		h->max_huge_pages_node[nid], buf, nid, i);
3245	h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
3246	h->max_huge_pages_node[nid] = i;
3247}
3248
3249static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
3250{
3251	unsigned long i;
3252	nodemask_t *node_alloc_noretry;
3253	bool node_specific_alloc = false;
3254
3255	/* skip gigantic hugepages allocation if hugetlb_cma enabled */
3256	if (hstate_is_gigantic(h) && hugetlb_cma_size) {
3257		pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
3258		return;
3259	}
3260
3261	/* do node specific alloc */
3262	for_each_online_node(i) {
3263		if (h->max_huge_pages_node[i] > 0) {
3264			hugetlb_hstate_alloc_pages_onenode(h, i);
3265			node_specific_alloc = true;
3266		}
3267	}
3268
3269	if (node_specific_alloc)
3270		return;
3271
3272	/* below will do all node balanced alloc */
3273	if (!hstate_is_gigantic(h)) {
3274		/*
3275		 * Bit mask controlling how hard we retry per-node allocations.
3276		 * Ignore errors as lower level routines can deal with
3277		 * node_alloc_noretry == NULL.  If this kmalloc fails at boot
3278		 * time, we are likely in bigger trouble.
3279		 */
3280		node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
3281						GFP_KERNEL);
3282	} else {
3283		/* allocations done at boot time */
3284		node_alloc_noretry = NULL;
3285	}
3286
3287	/* bit mask controlling how hard we retry per-node allocations */
3288	if (node_alloc_noretry)
3289		nodes_clear(*node_alloc_noretry);
3290
3291	for (i = 0; i < h->max_huge_pages; ++i) {
3292		if (hstate_is_gigantic(h)) {
3293			if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
 
 
 
 
3294				break;
3295		} else if (!alloc_pool_huge_page(h,
3296					 &node_states[N_MEMORY],
3297					 node_alloc_noretry))
3298			break;
3299		cond_resched();
3300	}
3301	if (i < h->max_huge_pages) {
3302		char buf[32];
3303
3304		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3305		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
3306			h->max_huge_pages, buf, i);
3307		h->max_huge_pages = i;
3308	}
 
3309	kfree(node_alloc_noretry);
3310}
3311
3312static void __init hugetlb_init_hstates(void)
3313{
3314	struct hstate *h, *h2;
3315
3316	for_each_hstate(h) {
 
 
 
3317		/* oversize hugepages were init'ed in early boot */
3318		if (!hstate_is_gigantic(h))
3319			hugetlb_hstate_alloc_pages(h);
3320
3321		/*
3322		 * Set demote order for each hstate.  Note that
3323		 * h->demote_order is initially 0.
3324		 * - We can not demote gigantic pages if runtime freeing
3325		 *   is not supported, so skip this.
3326		 * - If CMA allocation is possible, we can not demote
3327		 *   HUGETLB_PAGE_ORDER or smaller size pages.
3328		 */
3329		if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3330			continue;
3331		if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER)
3332			continue;
3333		for_each_hstate(h2) {
3334			if (h2 == h)
3335				continue;
3336			if (h2->order < h->order &&
3337			    h2->order > h->demote_order)
3338				h->demote_order = h2->order;
3339		}
3340	}
 
3341}
3342
3343static void __init report_hugepages(void)
3344{
3345	struct hstate *h;
3346
3347	for_each_hstate(h) {
3348		char buf[32];
3349
3350		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3351		pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
3352			buf, h->free_huge_pages);
3353		pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
3354			hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
3355	}
3356}
3357
3358#ifdef CONFIG_HIGHMEM
3359static void try_to_free_low(struct hstate *h, unsigned long count,
3360						nodemask_t *nodes_allowed)
3361{
3362	int i;
3363	LIST_HEAD(page_list);
3364
3365	lockdep_assert_held(&hugetlb_lock);
3366	if (hstate_is_gigantic(h))
3367		return;
3368
3369	/*
3370	 * Collect pages to be freed on a list, and free after dropping lock
3371	 */
3372	for_each_node_mask(i, *nodes_allowed) {
3373		struct page *page, *next;
3374		struct list_head *freel = &h->hugepage_freelists[i];
3375		list_for_each_entry_safe(page, next, freel, lru) {
3376			if (count >= h->nr_huge_pages)
3377				goto out;
3378			if (PageHighMem(page))
3379				continue;
3380			remove_hugetlb_folio(h, page_folio(page), false);
3381			list_add(&page->lru, &page_list);
 
 
3382		}
3383	}
3384
3385out:
3386	spin_unlock_irq(&hugetlb_lock);
3387	update_and_free_pages_bulk(h, &page_list);
3388	spin_lock_irq(&hugetlb_lock);
3389}
3390#else
3391static inline void try_to_free_low(struct hstate *h, unsigned long count,
3392						nodemask_t *nodes_allowed)
3393{
3394}
3395#endif
3396
3397/*
3398 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
3399 * balanced by operating on them in a round-robin fashion.
3400 * Returns 1 if an adjustment was made.
3401 */
3402static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
3403				int delta)
3404{
3405	int nr_nodes, node;
3406
3407	lockdep_assert_held(&hugetlb_lock);
3408	VM_BUG_ON(delta != -1 && delta != 1);
3409
3410	if (delta < 0) {
3411		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
3412			if (h->surplus_huge_pages_node[node])
3413				goto found;
3414		}
3415	} else {
3416		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3417			if (h->surplus_huge_pages_node[node] <
3418					h->nr_huge_pages_node[node])
3419				goto found;
3420		}
3421	}
3422	return 0;
3423
3424found:
3425	h->surplus_huge_pages += delta;
3426	h->surplus_huge_pages_node[node] += delta;
3427	return 1;
3428}
3429
3430#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
3431static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
3432			      nodemask_t *nodes_allowed)
3433{
3434	unsigned long min_count, ret;
3435	struct page *page;
3436	LIST_HEAD(page_list);
3437	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
3438
3439	/*
3440	 * Bit mask controlling how hard we retry per-node allocations.
3441	 * If we can not allocate the bit mask, do not attempt to allocate
3442	 * the requested huge pages.
3443	 */
3444	if (node_alloc_noretry)
3445		nodes_clear(*node_alloc_noretry);
3446	else
3447		return -ENOMEM;
3448
3449	/*
3450	 * resize_lock mutex prevents concurrent adjustments to number of
3451	 * pages in hstate via the proc/sysfs interfaces.
3452	 */
3453	mutex_lock(&h->resize_lock);
3454	flush_free_hpage_work(h);
3455	spin_lock_irq(&hugetlb_lock);
3456
3457	/*
3458	 * Check for a node specific request.
3459	 * Changing node specific huge page count may require a corresponding
3460	 * change to the global count.  In any case, the passed node mask
3461	 * (nodes_allowed) will restrict alloc/free to the specified node.
3462	 */
3463	if (nid != NUMA_NO_NODE) {
3464		unsigned long old_count = count;
3465
3466		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
3467		/*
3468		 * User may have specified a large count value which caused the
3469		 * above calculation to overflow.  In this case, they wanted
3470		 * to allocate as many huge pages as possible.  Set count to
3471		 * largest possible value to align with their intention.
3472		 */
3473		if (count < old_count)
3474			count = ULONG_MAX;
3475	}
3476
3477	/*
3478	 * Gigantic pages runtime allocation depend on the capability for large
3479	 * page range allocation.
3480	 * If the system does not provide this feature, return an error when
3481	 * the user tries to allocate gigantic pages but let the user free the
3482	 * boottime allocated gigantic pages.
3483	 */
3484	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
3485		if (count > persistent_huge_pages(h)) {
3486			spin_unlock_irq(&hugetlb_lock);
3487			mutex_unlock(&h->resize_lock);
3488			NODEMASK_FREE(node_alloc_noretry);
3489			return -EINVAL;
3490		}
3491		/* Fall through to decrease pool */
3492	}
3493
3494	/*
3495	 * Increase the pool size
3496	 * First take pages out of surplus state.  Then make up the
3497	 * remaining difference by allocating fresh huge pages.
3498	 *
3499	 * We might race with alloc_surplus_huge_page() here and be unable
3500	 * to convert a surplus huge page to a normal huge page. That is
3501	 * not critical, though, it just means the overall size of the
3502	 * pool might be one hugepage larger than it needs to be, but
3503	 * within all the constraints specified by the sysctls.
3504	 */
3505	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3506		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3507			break;
3508	}
3509
3510	while (count > persistent_huge_pages(h)) {
3511		/*
3512		 * If this allocation races such that we no longer need the
3513		 * page, free_huge_page will handle it by freeing the page
3514		 * and reducing the surplus.
3515		 */
3516		spin_unlock_irq(&hugetlb_lock);
3517
3518		/* yield cpu to avoid soft lockup */
3519		cond_resched();
3520
3521		ret = alloc_pool_huge_page(h, nodes_allowed,
3522						node_alloc_noretry);
3523		spin_lock_irq(&hugetlb_lock);
3524		if (!ret)
3525			goto out;
3526
3527		/* Bail for signals. Probably ctrl-c from user */
3528		if (signal_pending(current))
3529			goto out;
3530	}
3531
3532	/*
3533	 * Decrease the pool size
3534	 * First return free pages to the buddy allocator (being careful
3535	 * to keep enough around to satisfy reservations).  Then place
3536	 * pages into surplus state as needed so the pool will shrink
3537	 * to the desired size as pages become free.
3538	 *
3539	 * By placing pages into the surplus state independent of the
3540	 * overcommit value, we are allowing the surplus pool size to
3541	 * exceed overcommit. There are few sane options here. Since
3542	 * alloc_surplus_huge_page() is checking the global counter,
3543	 * though, we'll note that we're not allowed to exceed surplus
3544	 * and won't grow the pool anywhere else. Not until one of the
3545	 * sysctls are changed, or the surplus pages go out of use.
3546	 */
3547	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3548	min_count = max(count, min_count);
3549	try_to_free_low(h, min_count, nodes_allowed);
3550
3551	/*
3552	 * Collect pages to be removed on list without dropping lock
3553	 */
3554	while (min_count < persistent_huge_pages(h)) {
3555		page = remove_pool_huge_page(h, nodes_allowed, 0);
3556		if (!page)
3557			break;
3558
3559		list_add(&page->lru, &page_list);
3560	}
3561	/* free the pages after dropping lock */
3562	spin_unlock_irq(&hugetlb_lock);
3563	update_and_free_pages_bulk(h, &page_list);
3564	flush_free_hpage_work(h);
3565	spin_lock_irq(&hugetlb_lock);
3566
3567	while (count < persistent_huge_pages(h)) {
3568		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3569			break;
3570	}
3571out:
3572	h->max_huge_pages = persistent_huge_pages(h);
3573	spin_unlock_irq(&hugetlb_lock);
3574	mutex_unlock(&h->resize_lock);
3575
3576	NODEMASK_FREE(node_alloc_noretry);
3577
3578	return 0;
3579}
3580
3581static int demote_free_huge_page(struct hstate *h, struct page *page)
3582{
3583	int i, nid = page_to_nid(page);
3584	struct hstate *target_hstate;
3585	struct folio *folio = page_folio(page);
3586	struct page *subpage;
3587	int rc = 0;
3588
3589	target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);
3590
3591	remove_hugetlb_folio_for_demote(h, folio, false);
3592	spin_unlock_irq(&hugetlb_lock);
3593
3594	rc = hugetlb_vmemmap_restore(h, page);
3595	if (rc) {
3596		/* Allocation of vmemmmap failed, we can not demote page */
3597		spin_lock_irq(&hugetlb_lock);
3598		set_page_refcounted(page);
3599		add_hugetlb_folio(h, page_folio(page), false);
3600		return rc;
3601	}
3602
3603	/*
3604	 * Use destroy_compound_hugetlb_folio_for_demote for all huge page
3605	 * sizes as it will not ref count pages.
3606	 */
3607	destroy_compound_hugetlb_folio_for_demote(folio, huge_page_order(h));
3608
3609	/*
3610	 * Taking target hstate mutex synchronizes with set_max_huge_pages.
3611	 * Without the mutex, pages added to target hstate could be marked
3612	 * as surplus.
3613	 *
3614	 * Note that we already hold h->resize_lock.  To prevent deadlock,
3615	 * use the convention of always taking larger size hstate mutex first.
3616	 */
3617	mutex_lock(&target_hstate->resize_lock);
3618	for (i = 0; i < pages_per_huge_page(h);
3619				i += pages_per_huge_page(target_hstate)) {
3620		subpage = nth_page(page, i);
3621		folio = page_folio(subpage);
3622		if (hstate_is_gigantic(target_hstate))
3623			prep_compound_gigantic_folio_for_demote(folio,
3624							target_hstate->order);
3625		else
3626			prep_compound_page(subpage, target_hstate->order);
3627		set_page_private(subpage, 0);
3628		prep_new_hugetlb_folio(target_hstate, folio, nid);
3629		free_huge_page(subpage);
3630	}
3631	mutex_unlock(&target_hstate->resize_lock);
3632
3633	spin_lock_irq(&hugetlb_lock);
3634
3635	/*
3636	 * Not absolutely necessary, but for consistency update max_huge_pages
3637	 * based on pool changes for the demoted page.
3638	 */
3639	h->max_huge_pages--;
3640	target_hstate->max_huge_pages +=
3641		pages_per_huge_page(h) / pages_per_huge_page(target_hstate);
3642
3643	return rc;
3644}
3645
3646static int demote_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
3647	__must_hold(&hugetlb_lock)
3648{
3649	int nr_nodes, node;
3650	struct page *page;
3651
3652	lockdep_assert_held(&hugetlb_lock);
3653
3654	/* We should never get here if no demote order */
3655	if (!h->demote_order) {
3656		pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
3657		return -EINVAL;		/* internal error */
3658	}
3659
3660	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3661		list_for_each_entry(page, &h->hugepage_freelists[node], lru) {
3662			if (PageHWPoison(page))
3663				continue;
3664
3665			return demote_free_huge_page(h, page);
3666		}
3667	}
3668
3669	/*
3670	 * Only way to get here is if all pages on free lists are poisoned.
3671	 * Return -EBUSY so that caller will not retry.
3672	 */
3673	return -EBUSY;
3674}
3675
3676#define HSTATE_ATTR_RO(_name) \
3677	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
3678
3679#define HSTATE_ATTR_WO(_name) \
3680	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)
3681
3682#define HSTATE_ATTR(_name) \
3683	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
 
3684
3685static struct kobject *hugepages_kobj;
3686static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
3687
3688static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
3689
3690static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
3691{
3692	int i;
3693
3694	for (i = 0; i < HUGE_MAX_HSTATE; i++)
3695		if (hstate_kobjs[i] == kobj) {
3696			if (nidp)
3697				*nidp = NUMA_NO_NODE;
3698			return &hstates[i];
3699		}
3700
3701	return kobj_to_node_hstate(kobj, nidp);
3702}
3703
3704static ssize_t nr_hugepages_show_common(struct kobject *kobj,
3705					struct kobj_attribute *attr, char *buf)
3706{
3707	struct hstate *h;
3708	unsigned long nr_huge_pages;
3709	int nid;
3710
3711	h = kobj_to_hstate(kobj, &nid);
3712	if (nid == NUMA_NO_NODE)
3713		nr_huge_pages = h->nr_huge_pages;
3714	else
3715		nr_huge_pages = h->nr_huge_pages_node[nid];
3716
3717	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
3718}
3719
3720static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
3721					   struct hstate *h, int nid,
3722					   unsigned long count, size_t len)
3723{
3724	int err;
3725	nodemask_t nodes_allowed, *n_mask;
3726
3727	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3728		return -EINVAL;
3729
3730	if (nid == NUMA_NO_NODE) {
3731		/*
3732		 * global hstate attribute
3733		 */
3734		if (!(obey_mempolicy &&
3735				init_nodemask_of_mempolicy(&nodes_allowed)))
3736			n_mask = &node_states[N_MEMORY];
3737		else
3738			n_mask = &nodes_allowed;
3739	} else {
3740		/*
3741		 * Node specific request.  count adjustment happens in
3742		 * set_max_huge_pages() after acquiring hugetlb_lock.
3743		 */
3744		init_nodemask_of_node(&nodes_allowed, nid);
3745		n_mask = &nodes_allowed;
3746	}
3747
3748	err = set_max_huge_pages(h, count, nid, n_mask);
3749
3750	return err ? err : len;
3751}
3752
3753static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
3754					 struct kobject *kobj, const char *buf,
3755					 size_t len)
3756{
3757	struct hstate *h;
3758	unsigned long count;
3759	int nid;
3760	int err;
3761
3762	err = kstrtoul(buf, 10, &count);
3763	if (err)
3764		return err;
3765
3766	h = kobj_to_hstate(kobj, &nid);
3767	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
3768}
3769
3770static ssize_t nr_hugepages_show(struct kobject *kobj,
3771				       struct kobj_attribute *attr, char *buf)
3772{
3773	return nr_hugepages_show_common(kobj, attr, buf);
3774}
3775
3776static ssize_t nr_hugepages_store(struct kobject *kobj,
3777	       struct kobj_attribute *attr, const char *buf, size_t len)
3778{
3779	return nr_hugepages_store_common(false, kobj, buf, len);
3780}
3781HSTATE_ATTR(nr_hugepages);
3782
3783#ifdef CONFIG_NUMA
3784
3785/*
3786 * hstate attribute for optionally mempolicy-based constraint on persistent
3787 * huge page alloc/free.
3788 */
3789static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
3790					   struct kobj_attribute *attr,
3791					   char *buf)
3792{
3793	return nr_hugepages_show_common(kobj, attr, buf);
3794}
3795
3796static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
3797	       struct kobj_attribute *attr, const char *buf, size_t len)
3798{
3799	return nr_hugepages_store_common(true, kobj, buf, len);
3800}
3801HSTATE_ATTR(nr_hugepages_mempolicy);
3802#endif
3803
3804
3805static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
3806					struct kobj_attribute *attr, char *buf)
3807{
3808	struct hstate *h = kobj_to_hstate(kobj, NULL);
3809	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
3810}
3811
3812static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
3813		struct kobj_attribute *attr, const char *buf, size_t count)
3814{
3815	int err;
3816	unsigned long input;
3817	struct hstate *h = kobj_to_hstate(kobj, NULL);
3818
3819	if (hstate_is_gigantic(h))
3820		return -EINVAL;
3821
3822	err = kstrtoul(buf, 10, &input);
3823	if (err)
3824		return err;
3825
3826	spin_lock_irq(&hugetlb_lock);
3827	h->nr_overcommit_huge_pages = input;
3828	spin_unlock_irq(&hugetlb_lock);
3829
3830	return count;
3831}
3832HSTATE_ATTR(nr_overcommit_hugepages);
3833
3834static ssize_t free_hugepages_show(struct kobject *kobj,
3835					struct kobj_attribute *attr, char *buf)
3836{
3837	struct hstate *h;
3838	unsigned long free_huge_pages;
3839	int nid;
3840
3841	h = kobj_to_hstate(kobj, &nid);
3842	if (nid == NUMA_NO_NODE)
3843		free_huge_pages = h->free_huge_pages;
3844	else
3845		free_huge_pages = h->free_huge_pages_node[nid];
3846
3847	return sysfs_emit(buf, "%lu\n", free_huge_pages);
3848}
3849HSTATE_ATTR_RO(free_hugepages);
3850
3851static ssize_t resv_hugepages_show(struct kobject *kobj,
3852					struct kobj_attribute *attr, char *buf)
3853{
3854	struct hstate *h = kobj_to_hstate(kobj, NULL);
3855	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
3856}
3857HSTATE_ATTR_RO(resv_hugepages);
3858
3859static ssize_t surplus_hugepages_show(struct kobject *kobj,
3860					struct kobj_attribute *attr, char *buf)
3861{
3862	struct hstate *h;
3863	unsigned long surplus_huge_pages;
3864	int nid;
3865
3866	h = kobj_to_hstate(kobj, &nid);
3867	if (nid == NUMA_NO_NODE)
3868		surplus_huge_pages = h->surplus_huge_pages;
3869	else
3870		surplus_huge_pages = h->surplus_huge_pages_node[nid];
3871
3872	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
3873}
3874HSTATE_ATTR_RO(surplus_hugepages);
3875
3876static ssize_t demote_store(struct kobject *kobj,
3877	       struct kobj_attribute *attr, const char *buf, size_t len)
3878{
3879	unsigned long nr_demote;
3880	unsigned long nr_available;
3881	nodemask_t nodes_allowed, *n_mask;
3882	struct hstate *h;
3883	int err;
3884	int nid;
3885
3886	err = kstrtoul(buf, 10, &nr_demote);
3887	if (err)
3888		return err;
3889	h = kobj_to_hstate(kobj, &nid);
3890
3891	if (nid != NUMA_NO_NODE) {
3892		init_nodemask_of_node(&nodes_allowed, nid);
3893		n_mask = &nodes_allowed;
3894	} else {
3895		n_mask = &node_states[N_MEMORY];
3896	}
3897
3898	/* Synchronize with other sysfs operations modifying huge pages */
3899	mutex_lock(&h->resize_lock);
3900	spin_lock_irq(&hugetlb_lock);
3901
3902	while (nr_demote) {
3903		/*
3904		 * Check for available pages to demote each time thorough the
3905		 * loop as demote_pool_huge_page will drop hugetlb_lock.
3906		 */
3907		if (nid != NUMA_NO_NODE)
3908			nr_available = h->free_huge_pages_node[nid];
3909		else
3910			nr_available = h->free_huge_pages;
3911		nr_available -= h->resv_huge_pages;
3912		if (!nr_available)
3913			break;
3914
3915		err = demote_pool_huge_page(h, n_mask);
3916		if (err)
3917			break;
3918
3919		nr_demote--;
3920	}
3921
3922	spin_unlock_irq(&hugetlb_lock);
3923	mutex_unlock(&h->resize_lock);
3924
3925	if (err)
3926		return err;
3927	return len;
3928}
3929HSTATE_ATTR_WO(demote);
3930
3931static ssize_t demote_size_show(struct kobject *kobj,
3932					struct kobj_attribute *attr, char *buf)
3933{
3934	struct hstate *h = kobj_to_hstate(kobj, NULL);
3935	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
3936
3937	return sysfs_emit(buf, "%lukB\n", demote_size);
3938}
3939
3940static ssize_t demote_size_store(struct kobject *kobj,
3941					struct kobj_attribute *attr,
3942					const char *buf, size_t count)
3943{
3944	struct hstate *h, *demote_hstate;
3945	unsigned long demote_size;
3946	unsigned int demote_order;
3947
3948	demote_size = (unsigned long)memparse(buf, NULL);
3949
3950	demote_hstate = size_to_hstate(demote_size);
3951	if (!demote_hstate)
3952		return -EINVAL;
3953	demote_order = demote_hstate->order;
3954	if (demote_order < HUGETLB_PAGE_ORDER)
3955		return -EINVAL;
3956
3957	/* demote order must be smaller than hstate order */
3958	h = kobj_to_hstate(kobj, NULL);
3959	if (demote_order >= h->order)
3960		return -EINVAL;
3961
3962	/* resize_lock synchronizes access to demote size and writes */
3963	mutex_lock(&h->resize_lock);
3964	h->demote_order = demote_order;
3965	mutex_unlock(&h->resize_lock);
3966
3967	return count;
3968}
3969HSTATE_ATTR(demote_size);
3970
3971static struct attribute *hstate_attrs[] = {
3972	&nr_hugepages_attr.attr,
3973	&nr_overcommit_hugepages_attr.attr,
3974	&free_hugepages_attr.attr,
3975	&resv_hugepages_attr.attr,
3976	&surplus_hugepages_attr.attr,
3977#ifdef CONFIG_NUMA
3978	&nr_hugepages_mempolicy_attr.attr,
3979#endif
3980	NULL,
3981};
3982
3983static const struct attribute_group hstate_attr_group = {
3984	.attrs = hstate_attrs,
3985};
3986
3987static struct attribute *hstate_demote_attrs[] = {
3988	&demote_size_attr.attr,
3989	&demote_attr.attr,
3990	NULL,
3991};
3992
3993static const struct attribute_group hstate_demote_attr_group = {
3994	.attrs = hstate_demote_attrs,
3995};
3996
3997static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
3998				    struct kobject **hstate_kobjs,
3999				    const struct attribute_group *hstate_attr_group)
4000{
4001	int retval;
4002	int hi = hstate_index(h);
4003
4004	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
4005	if (!hstate_kobjs[hi])
4006		return -ENOMEM;
4007
4008	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
4009	if (retval) {
4010		kobject_put(hstate_kobjs[hi]);
4011		hstate_kobjs[hi] = NULL;
4012		return retval;
4013	}
4014
4015	if (h->demote_order) {
4016		retval = sysfs_create_group(hstate_kobjs[hi],
4017					    &hstate_demote_attr_group);
4018		if (retval) {
4019			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
4020			sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
4021			kobject_put(hstate_kobjs[hi]);
4022			hstate_kobjs[hi] = NULL;
4023			return retval;
4024		}
4025	}
4026
4027	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4028}
4029
4030#ifdef CONFIG_NUMA
4031static bool hugetlb_sysfs_initialized __ro_after_init;
4032
4033/*
4034 * node_hstate/s - associate per node hstate attributes, via their kobjects,
4035 * with node devices in node_devices[] using a parallel array.  The array
4036 * index of a node device or _hstate == node id.
4037 * This is here to avoid any static dependency of the node device driver, in
4038 * the base kernel, on the hugetlb module.
4039 */
4040struct node_hstate {
4041	struct kobject		*hugepages_kobj;
4042	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
4043};
4044static struct node_hstate node_hstates[MAX_NUMNODES];
4045
4046/*
4047 * A subset of global hstate attributes for node devices
4048 */
4049static struct attribute *per_node_hstate_attrs[] = {
4050	&nr_hugepages_attr.attr,
4051	&free_hugepages_attr.attr,
4052	&surplus_hugepages_attr.attr,
4053	NULL,
4054};
4055
4056static const struct attribute_group per_node_hstate_attr_group = {
4057	.attrs = per_node_hstate_attrs,
4058};
4059
4060/*
4061 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
4062 * Returns node id via non-NULL nidp.
4063 */
4064static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4065{
4066	int nid;
4067
4068	for (nid = 0; nid < nr_node_ids; nid++) {
4069		struct node_hstate *nhs = &node_hstates[nid];
4070		int i;
4071		for (i = 0; i < HUGE_MAX_HSTATE; i++)
4072			if (nhs->hstate_kobjs[i] == kobj) {
4073				if (nidp)
4074					*nidp = nid;
4075				return &hstates[i];
4076			}
4077	}
4078
4079	BUG();
4080	return NULL;
4081}
4082
4083/*
4084 * Unregister hstate attributes from a single node device.
4085 * No-op if no hstate attributes attached.
4086 */
4087void hugetlb_unregister_node(struct node *node)
4088{
4089	struct hstate *h;
4090	struct node_hstate *nhs = &node_hstates[node->dev.id];
4091
4092	if (!nhs->hugepages_kobj)
4093		return;		/* no hstate attributes */
4094
4095	for_each_hstate(h) {
4096		int idx = hstate_index(h);
4097		struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];
4098
4099		if (!hstate_kobj)
4100			continue;
4101		if (h->demote_order)
4102			sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
4103		sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
4104		kobject_put(hstate_kobj);
4105		nhs->hstate_kobjs[idx] = NULL;
4106	}
4107
4108	kobject_put(nhs->hugepages_kobj);
4109	nhs->hugepages_kobj = NULL;
4110}
4111
4112
4113/*
4114 * Register hstate attributes for a single node device.
4115 * No-op if attributes already registered.
4116 */
4117void hugetlb_register_node(struct node *node)
4118{
4119	struct hstate *h;
4120	struct node_hstate *nhs = &node_hstates[node->dev.id];
4121	int err;
4122
4123	if (!hugetlb_sysfs_initialized)
4124		return;
4125
4126	if (nhs->hugepages_kobj)
4127		return;		/* already allocated */
4128
4129	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
4130							&node->dev.kobj);
4131	if (!nhs->hugepages_kobj)
4132		return;
4133
4134	for_each_hstate(h) {
4135		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
4136						nhs->hstate_kobjs,
4137						&per_node_hstate_attr_group);
4138		if (err) {
4139			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
4140				h->name, node->dev.id);
4141			hugetlb_unregister_node(node);
4142			break;
4143		}
4144	}
4145}
4146
4147/*
4148 * hugetlb init time:  register hstate attributes for all registered node
4149 * devices of nodes that have memory.  All on-line nodes should have
4150 * registered their associated device by this time.
4151 */
4152static void __init hugetlb_register_all_nodes(void)
4153{
4154	int nid;
4155
4156	for_each_online_node(nid)
4157		hugetlb_register_node(node_devices[nid]);
 
 
 
 
 
 
 
 
 
 
4158}
4159#else	/* !CONFIG_NUMA */
4160
4161static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4162{
4163	BUG();
4164	if (nidp)
4165		*nidp = -1;
4166	return NULL;
4167}
4168
4169static void hugetlb_register_all_nodes(void) { }
4170
4171#endif
4172
4173#ifdef CONFIG_CMA
4174static void __init hugetlb_cma_check(void);
4175#else
4176static inline __init void hugetlb_cma_check(void)
4177{
4178}
4179#endif
4180
4181static void __init hugetlb_sysfs_init(void)
4182{
4183	struct hstate *h;
4184	int err;
4185
4186	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
4187	if (!hugepages_kobj)
4188		return;
4189
4190	for_each_hstate(h) {
4191		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
4192					 hstate_kobjs, &hstate_attr_group);
4193		if (err)
4194			pr_err("HugeTLB: Unable to add hstate %s", h->name);
4195	}
4196
4197#ifdef CONFIG_NUMA
4198	hugetlb_sysfs_initialized = true;
4199#endif
4200	hugetlb_register_all_nodes();
4201}
4202
4203static int __init hugetlb_init(void)
4204{
4205	int i;
4206
4207	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
4208			__NR_HPAGEFLAGS);
4209
4210	if (!hugepages_supported()) {
4211		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
4212			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
4213		return 0;
4214	}
4215
4216	/*
4217	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
4218	 * architectures depend on setup being done here.
4219	 */
4220	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
4221	if (!parsed_default_hugepagesz) {
4222		/*
4223		 * If we did not parse a default huge page size, set
4224		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
4225		 * number of huge pages for this default size was implicitly
4226		 * specified, set that here as well.
4227		 * Note that the implicit setting will overwrite an explicit
4228		 * setting.  A warning will be printed in this case.
4229		 */
4230		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
4231		if (default_hstate_max_huge_pages) {
4232			if (default_hstate.max_huge_pages) {
4233				char buf[32];
4234
4235				string_get_size(huge_page_size(&default_hstate),
4236					1, STRING_UNITS_2, buf, 32);
4237				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
4238					default_hstate.max_huge_pages, buf);
4239				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
4240					default_hstate_max_huge_pages);
4241			}
4242			default_hstate.max_huge_pages =
4243				default_hstate_max_huge_pages;
4244
4245			for_each_online_node(i)
4246				default_hstate.max_huge_pages_node[i] =
4247					default_hugepages_in_node[i];
4248		}
4249	}
4250
4251	hugetlb_cma_check();
4252	hugetlb_init_hstates();
4253	gather_bootmem_prealloc();
4254	report_hugepages();
4255
4256	hugetlb_sysfs_init();
 
4257	hugetlb_cgroup_file_init();
4258
4259#ifdef CONFIG_SMP
4260	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
4261#else
4262	num_fault_mutexes = 1;
4263#endif
4264	hugetlb_fault_mutex_table =
4265		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
4266			      GFP_KERNEL);
4267	BUG_ON(!hugetlb_fault_mutex_table);
4268
4269	for (i = 0; i < num_fault_mutexes; i++)
4270		mutex_init(&hugetlb_fault_mutex_table[i]);
4271	return 0;
4272}
4273subsys_initcall(hugetlb_init);
4274
4275/* Overwritten by architectures with more huge page sizes */
4276bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
4277{
4278	return size == HPAGE_SIZE;
4279}
4280
4281void __init hugetlb_add_hstate(unsigned int order)
4282{
4283	struct hstate *h;
4284	unsigned long i;
4285
4286	if (size_to_hstate(PAGE_SIZE << order)) {
4287		return;
4288	}
4289	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
4290	BUG_ON(order == 0);
4291	h = &hstates[hugetlb_max_hstate++];
4292	mutex_init(&h->resize_lock);
4293	h->order = order;
4294	h->mask = ~(huge_page_size(h) - 1);
 
 
4295	for (i = 0; i < MAX_NUMNODES; ++i)
4296		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
4297	INIT_LIST_HEAD(&h->hugepage_activelist);
4298	h->next_nid_to_alloc = first_memory_node;
4299	h->next_nid_to_free = first_memory_node;
4300	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
4301					huge_page_size(h)/SZ_1K);
4302
4303	parsed_hstate = h;
4304}
4305
4306bool __init __weak hugetlb_node_alloc_supported(void)
4307{
4308	return true;
4309}
4310
4311static void __init hugepages_clear_pages_in_node(void)
4312{
4313	if (!hugetlb_max_hstate) {
4314		default_hstate_max_huge_pages = 0;
4315		memset(default_hugepages_in_node, 0,
4316			sizeof(default_hugepages_in_node));
4317	} else {
4318		parsed_hstate->max_huge_pages = 0;
4319		memset(parsed_hstate->max_huge_pages_node, 0,
4320			sizeof(parsed_hstate->max_huge_pages_node));
4321	}
4322}
4323
4324/*
4325 * hugepages command line processing
4326 * hugepages normally follows a valid hugepagsz or default_hugepagsz
4327 * specification.  If not, ignore the hugepages value.  hugepages can also
4328 * be the first huge page command line  option in which case it implicitly
4329 * specifies the number of huge pages for the default size.
4330 */
4331static int __init hugepages_setup(char *s)
4332{
4333	unsigned long *mhp;
4334	static unsigned long *last_mhp;
4335	int node = NUMA_NO_NODE;
4336	int count;
4337	unsigned long tmp;
4338	char *p = s;
4339
4340	if (!parsed_valid_hugepagesz) {
4341		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
4342		parsed_valid_hugepagesz = true;
4343		return 1;
4344	}
4345
4346	/*
4347	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
4348	 * yet, so this hugepages= parameter goes to the "default hstate".
4349	 * Otherwise, it goes with the previously parsed hugepagesz or
4350	 * default_hugepagesz.
4351	 */
4352	else if (!hugetlb_max_hstate)
4353		mhp = &default_hstate_max_huge_pages;
4354	else
4355		mhp = &parsed_hstate->max_huge_pages;
4356
4357	if (mhp == last_mhp) {
4358		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
4359		return 1;
4360	}
4361
4362	while (*p) {
4363		count = 0;
4364		if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4365			goto invalid;
4366		/* Parameter is node format */
4367		if (p[count] == ':') {
4368			if (!hugetlb_node_alloc_supported()) {
4369				pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
4370				return 1;
4371			}
4372			if (tmp >= MAX_NUMNODES || !node_online(tmp))
4373				goto invalid;
4374			node = array_index_nospec(tmp, MAX_NUMNODES);
4375			p += count + 1;
4376			/* Parse hugepages */
4377			if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4378				goto invalid;
4379			if (!hugetlb_max_hstate)
4380				default_hugepages_in_node[node] = tmp;
4381			else
4382				parsed_hstate->max_huge_pages_node[node] = tmp;
4383			*mhp += tmp;
4384			/* Go to parse next node*/
4385			if (p[count] == ',')
4386				p += count + 1;
4387			else
4388				break;
4389		} else {
4390			if (p != s)
4391				goto invalid;
4392			*mhp = tmp;
4393			break;
4394		}
4395	}
4396
4397	/*
4398	 * Global state is always initialized later in hugetlb_init.
4399	 * But we need to allocate gigantic hstates here early to still
4400	 * use the bootmem allocator.
4401	 */
4402	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
4403		hugetlb_hstate_alloc_pages(parsed_hstate);
4404
4405	last_mhp = mhp;
4406
4407	return 1;
4408
4409invalid:
4410	pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
4411	hugepages_clear_pages_in_node();
4412	return 1;
4413}
4414__setup("hugepages=", hugepages_setup);
4415
4416/*
4417 * hugepagesz command line processing
4418 * A specific huge page size can only be specified once with hugepagesz.
4419 * hugepagesz is followed by hugepages on the command line.  The global
4420 * variable 'parsed_valid_hugepagesz' is used to determine if prior
4421 * hugepagesz argument was valid.
4422 */
4423static int __init hugepagesz_setup(char *s)
4424{
4425	unsigned long size;
4426	struct hstate *h;
4427
4428	parsed_valid_hugepagesz = false;
4429	size = (unsigned long)memparse(s, NULL);
4430
4431	if (!arch_hugetlb_valid_size(size)) {
4432		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
4433		return 1;
4434	}
4435
4436	h = size_to_hstate(size);
4437	if (h) {
4438		/*
4439		 * hstate for this size already exists.  This is normally
4440		 * an error, but is allowed if the existing hstate is the
4441		 * default hstate.  More specifically, it is only allowed if
4442		 * the number of huge pages for the default hstate was not
4443		 * previously specified.
4444		 */
4445		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
4446		    default_hstate.max_huge_pages) {
4447			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
4448			return 1;
4449		}
4450
4451		/*
4452		 * No need to call hugetlb_add_hstate() as hstate already
4453		 * exists.  But, do set parsed_hstate so that a following
4454		 * hugepages= parameter will be applied to this hstate.
4455		 */
4456		parsed_hstate = h;
4457		parsed_valid_hugepagesz = true;
4458		return 1;
4459	}
4460
4461	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4462	parsed_valid_hugepagesz = true;
4463	return 1;
4464}
4465__setup("hugepagesz=", hugepagesz_setup);
4466
4467/*
4468 * default_hugepagesz command line input
4469 * Only one instance of default_hugepagesz allowed on command line.
4470 */
4471static int __init default_hugepagesz_setup(char *s)
4472{
4473	unsigned long size;
4474	int i;
4475
4476	parsed_valid_hugepagesz = false;
4477	if (parsed_default_hugepagesz) {
4478		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
4479		return 1;
4480	}
4481
4482	size = (unsigned long)memparse(s, NULL);
4483
4484	if (!arch_hugetlb_valid_size(size)) {
4485		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
4486		return 1;
4487	}
4488
4489	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4490	parsed_valid_hugepagesz = true;
4491	parsed_default_hugepagesz = true;
4492	default_hstate_idx = hstate_index(size_to_hstate(size));
4493
4494	/*
4495	 * The number of default huge pages (for this size) could have been
4496	 * specified as the first hugetlb parameter: hugepages=X.  If so,
4497	 * then default_hstate_max_huge_pages is set.  If the default huge
4498	 * page size is gigantic (>= MAX_ORDER), then the pages must be
4499	 * allocated here from bootmem allocator.
4500	 */
4501	if (default_hstate_max_huge_pages) {
4502		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
4503		for_each_online_node(i)
4504			default_hstate.max_huge_pages_node[i] =
4505				default_hugepages_in_node[i];
4506		if (hstate_is_gigantic(&default_hstate))
4507			hugetlb_hstate_alloc_pages(&default_hstate);
4508		default_hstate_max_huge_pages = 0;
4509	}
4510
4511	return 1;
4512}
4513__setup("default_hugepagesz=", default_hugepagesz_setup);
4514
4515static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
4516{
4517#ifdef CONFIG_NUMA
4518	struct mempolicy *mpol = get_task_policy(current);
4519
4520	/*
4521	 * Only enforce MPOL_BIND policy which overlaps with cpuset policy
4522	 * (from policy_nodemask) specifically for hugetlb case
4523	 */
4524	if (mpol->mode == MPOL_BIND &&
4525		(apply_policy_zone(mpol, gfp_zone(gfp)) &&
4526		 cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
4527		return &mpol->nodes;
4528#endif
4529	return NULL;
4530}
4531
4532static unsigned int allowed_mems_nr(struct hstate *h)
4533{
4534	int node;
4535	unsigned int nr = 0;
4536	nodemask_t *mbind_nodemask;
4537	unsigned int *array = h->free_huge_pages_node;
4538	gfp_t gfp_mask = htlb_alloc_mask(h);
4539
4540	mbind_nodemask = policy_mbind_nodemask(gfp_mask);
 
4541	for_each_node_mask(node, cpuset_current_mems_allowed) {
4542		if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
 
4543			nr += array[node];
4544	}
4545
4546	return nr;
4547}
4548
4549#ifdef CONFIG_SYSCTL
4550static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
4551					  void *buffer, size_t *length,
4552					  loff_t *ppos, unsigned long *out)
4553{
4554	struct ctl_table dup_table;
4555
4556	/*
4557	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
4558	 * can duplicate the @table and alter the duplicate of it.
4559	 */
4560	dup_table = *table;
4561	dup_table.data = out;
4562
4563	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
4564}
4565
4566static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
4567			 struct ctl_table *table, int write,
4568			 void *buffer, size_t *length, loff_t *ppos)
4569{
4570	struct hstate *h = &default_hstate;
4571	unsigned long tmp = h->max_huge_pages;
4572	int ret;
4573
4574	if (!hugepages_supported())
4575		return -EOPNOTSUPP;
4576
4577	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4578					     &tmp);
4579	if (ret)
4580		goto out;
4581
4582	if (write)
4583		ret = __nr_hugepages_store_common(obey_mempolicy, h,
4584						  NUMA_NO_NODE, tmp, *length);
4585out:
4586	return ret;
4587}
4588
4589int hugetlb_sysctl_handler(struct ctl_table *table, int write,
4590			  void *buffer, size_t *length, loff_t *ppos)
4591{
4592
4593	return hugetlb_sysctl_handler_common(false, table, write,
4594							buffer, length, ppos);
4595}
4596
4597#ifdef CONFIG_NUMA
4598int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
4599			  void *buffer, size_t *length, loff_t *ppos)
4600{
4601	return hugetlb_sysctl_handler_common(true, table, write,
4602							buffer, length, ppos);
4603}
4604#endif /* CONFIG_NUMA */
4605
4606int hugetlb_overcommit_handler(struct ctl_table *table, int write,
4607		void *buffer, size_t *length, loff_t *ppos)
4608{
4609	struct hstate *h = &default_hstate;
4610	unsigned long tmp;
4611	int ret;
4612
4613	if (!hugepages_supported())
4614		return -EOPNOTSUPP;
4615
4616	tmp = h->nr_overcommit_huge_pages;
4617
4618	if (write && hstate_is_gigantic(h))
4619		return -EINVAL;
4620
4621	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4622					     &tmp);
4623	if (ret)
4624		goto out;
4625
4626	if (write) {
4627		spin_lock_irq(&hugetlb_lock);
4628		h->nr_overcommit_huge_pages = tmp;
4629		spin_unlock_irq(&hugetlb_lock);
4630	}
4631out:
4632	return ret;
4633}
4634
4635#endif /* CONFIG_SYSCTL */
4636
4637void hugetlb_report_meminfo(struct seq_file *m)
4638{
4639	struct hstate *h;
4640	unsigned long total = 0;
4641
4642	if (!hugepages_supported())
4643		return;
4644
4645	for_each_hstate(h) {
4646		unsigned long count = h->nr_huge_pages;
4647
4648		total += huge_page_size(h) * count;
4649
4650		if (h == &default_hstate)
4651			seq_printf(m,
4652				   "HugePages_Total:   %5lu\n"
4653				   "HugePages_Free:    %5lu\n"
4654				   "HugePages_Rsvd:    %5lu\n"
4655				   "HugePages_Surp:    %5lu\n"
4656				   "Hugepagesize:   %8lu kB\n",
4657				   count,
4658				   h->free_huge_pages,
4659				   h->resv_huge_pages,
4660				   h->surplus_huge_pages,
4661				   huge_page_size(h) / SZ_1K);
4662	}
4663
4664	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
4665}
4666
4667int hugetlb_report_node_meminfo(char *buf, int len, int nid)
4668{
4669	struct hstate *h = &default_hstate;
4670
4671	if (!hugepages_supported())
4672		return 0;
4673
4674	return sysfs_emit_at(buf, len,
4675			     "Node %d HugePages_Total: %5u\n"
4676			     "Node %d HugePages_Free:  %5u\n"
4677			     "Node %d HugePages_Surp:  %5u\n",
4678			     nid, h->nr_huge_pages_node[nid],
4679			     nid, h->free_huge_pages_node[nid],
4680			     nid, h->surplus_huge_pages_node[nid]);
4681}
4682
4683void hugetlb_show_meminfo_node(int nid)
4684{
4685	struct hstate *h;
 
4686
4687	if (!hugepages_supported())
4688		return;
4689
4690	for_each_hstate(h)
4691		printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
4692			nid,
4693			h->nr_huge_pages_node[nid],
4694			h->free_huge_pages_node[nid],
4695			h->surplus_huge_pages_node[nid],
4696			huge_page_size(h) / SZ_1K);
 
4697}
4698
4699void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
4700{
4701	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
4702		   atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
4703}
4704
4705/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
4706unsigned long hugetlb_total_pages(void)
4707{
4708	struct hstate *h;
4709	unsigned long nr_total_pages = 0;
4710
4711	for_each_hstate(h)
4712		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
4713	return nr_total_pages;
4714}
4715
4716static int hugetlb_acct_memory(struct hstate *h, long delta)
4717{
4718	int ret = -ENOMEM;
4719
4720	if (!delta)
4721		return 0;
4722
4723	spin_lock_irq(&hugetlb_lock);
4724	/*
4725	 * When cpuset is configured, it breaks the strict hugetlb page
4726	 * reservation as the accounting is done on a global variable. Such
4727	 * reservation is completely rubbish in the presence of cpuset because
4728	 * the reservation is not checked against page availability for the
4729	 * current cpuset. Application can still potentially OOM'ed by kernel
4730	 * with lack of free htlb page in cpuset that the task is in.
4731	 * Attempt to enforce strict accounting with cpuset is almost
4732	 * impossible (or too ugly) because cpuset is too fluid that
4733	 * task or memory node can be dynamically moved between cpusets.
4734	 *
4735	 * The change of semantics for shared hugetlb mapping with cpuset is
4736	 * undesirable. However, in order to preserve some of the semantics,
4737	 * we fall back to check against current free page availability as
4738	 * a best attempt and hopefully to minimize the impact of changing
4739	 * semantics that cpuset has.
4740	 *
4741	 * Apart from cpuset, we also have memory policy mechanism that
4742	 * also determines from which node the kernel will allocate memory
4743	 * in a NUMA system. So similar to cpuset, we also should consider
4744	 * the memory policy of the current task. Similar to the description
4745	 * above.
4746	 */
4747	if (delta > 0) {
4748		if (gather_surplus_pages(h, delta) < 0)
4749			goto out;
4750
4751		if (delta > allowed_mems_nr(h)) {
4752			return_unused_surplus_pages(h, delta);
4753			goto out;
4754		}
4755	}
4756
4757	ret = 0;
4758	if (delta < 0)
4759		return_unused_surplus_pages(h, (unsigned long) -delta);
4760
4761out:
4762	spin_unlock_irq(&hugetlb_lock);
4763	return ret;
4764}
4765
4766static void hugetlb_vm_op_open(struct vm_area_struct *vma)
4767{
4768	struct resv_map *resv = vma_resv_map(vma);
4769
4770	/*
4771	 * HPAGE_RESV_OWNER indicates a private mapping.
4772	 * This new VMA should share its siblings reservation map if present.
4773	 * The VMA will only ever have a valid reservation map pointer where
4774	 * it is being copied for another still existing VMA.  As that VMA
4775	 * has a reference to the reservation map it cannot disappear until
4776	 * after this open call completes.  It is therefore safe to take a
4777	 * new reference here without additional locking.
4778	 */
4779	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
4780		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
4781		kref_get(&resv->refs);
4782	}
4783
4784	/*
4785	 * vma_lock structure for sharable mappings is vma specific.
4786	 * Clear old pointer (if copied via vm_area_dup) and allocate
4787	 * new structure.  Before clearing, make sure vma_lock is not
4788	 * for this vma.
4789	 */
4790	if (vma->vm_flags & VM_MAYSHARE) {
4791		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
4792
4793		if (vma_lock) {
4794			if (vma_lock->vma != vma) {
4795				vma->vm_private_data = NULL;
4796				hugetlb_vma_lock_alloc(vma);
4797			} else
4798				pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__);
4799		} else
4800			hugetlb_vma_lock_alloc(vma);
4801	}
4802}
4803
4804static void hugetlb_vm_op_close(struct vm_area_struct *vma)
4805{
4806	struct hstate *h = hstate_vma(vma);
4807	struct resv_map *resv;
4808	struct hugepage_subpool *spool = subpool_vma(vma);
4809	unsigned long reserve, start, end;
4810	long gbl_reserve;
4811
4812	hugetlb_vma_lock_free(vma);
4813
4814	resv = vma_resv_map(vma);
4815	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4816		return;
4817
4818	start = vma_hugecache_offset(h, vma, vma->vm_start);
4819	end = vma_hugecache_offset(h, vma, vma->vm_end);
4820
4821	reserve = (end - start) - region_count(resv, start, end);
4822	hugetlb_cgroup_uncharge_counter(resv, start, end);
4823	if (reserve) {
4824		/*
4825		 * Decrement reserve counts.  The global reserve count may be
4826		 * adjusted if the subpool has a minimum size.
4827		 */
4828		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
4829		hugetlb_acct_memory(h, -gbl_reserve);
4830	}
4831
4832	kref_put(&resv->refs, resv_map_release);
4833}
4834
4835static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
4836{
4837	if (addr & ~(huge_page_mask(hstate_vma(vma))))
4838		return -EINVAL;
4839
4840	/*
4841	 * PMD sharing is only possible for PUD_SIZE-aligned address ranges
4842	 * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this
4843	 * split, unshare PMDs in the PUD_SIZE interval surrounding addr now.
4844	 */
4845	if (addr & ~PUD_MASK) {
4846		/*
4847		 * hugetlb_vm_op_split is called right before we attempt to
4848		 * split the VMA. We will need to unshare PMDs in the old and
4849		 * new VMAs, so let's unshare before we split.
4850		 */
4851		unsigned long floor = addr & PUD_MASK;
4852		unsigned long ceil = floor + PUD_SIZE;
4853
4854		if (floor >= vma->vm_start && ceil <= vma->vm_end)
4855			hugetlb_unshare_pmds(vma, floor, ceil);
4856	}
4857
4858	return 0;
4859}
4860
4861static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
4862{
4863	return huge_page_size(hstate_vma(vma));
 
 
4864}
4865
4866/*
4867 * We cannot handle pagefaults against hugetlb pages at all.  They cause
4868 * handle_mm_fault() to try to instantiate regular-sized pages in the
4869 * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
4870 * this far.
4871 */
4872static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
4873{
4874	BUG();
4875	return 0;
4876}
4877
4878/*
4879 * When a new function is introduced to vm_operations_struct and added
4880 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
4881 * This is because under System V memory model, mappings created via
4882 * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
4883 * their original vm_ops are overwritten with shm_vm_ops.
4884 */
4885const struct vm_operations_struct hugetlb_vm_ops = {
4886	.fault = hugetlb_vm_op_fault,
4887	.open = hugetlb_vm_op_open,
4888	.close = hugetlb_vm_op_close,
4889	.may_split = hugetlb_vm_op_split,
4890	.pagesize = hugetlb_vm_op_pagesize,
4891};
4892
4893static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
4894				int writable)
4895{
4896	pte_t entry;
4897	unsigned int shift = huge_page_shift(hstate_vma(vma));
4898
4899	if (writable) {
4900		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
4901					 vma->vm_page_prot)));
4902	} else {
4903		entry = huge_pte_wrprotect(mk_huge_pte(page,
4904					   vma->vm_page_prot));
4905	}
4906	entry = pte_mkyoung(entry);
4907	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
 
4908
4909	return entry;
4910}
4911
4912static void set_huge_ptep_writable(struct vm_area_struct *vma,
4913				   unsigned long address, pte_t *ptep)
4914{
4915	pte_t entry;
4916
4917	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
4918	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
4919		update_mmu_cache(vma, address, ptep);
4920}
4921
4922bool is_hugetlb_entry_migration(pte_t pte)
4923{
4924	swp_entry_t swp;
4925
4926	if (huge_pte_none(pte) || pte_present(pte))
4927		return false;
4928	swp = pte_to_swp_entry(pte);
4929	if (is_migration_entry(swp))
4930		return true;
4931	else
4932		return false;
4933}
4934
4935static bool is_hugetlb_entry_hwpoisoned(pte_t pte)
4936{
4937	swp_entry_t swp;
4938
4939	if (huge_pte_none(pte) || pte_present(pte))
4940		return false;
4941	swp = pte_to_swp_entry(pte);
4942	if (is_hwpoison_entry(swp))
4943		return true;
4944	else
4945		return false;
4946}
4947
4948static void
4949hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
4950		     struct page *new_page)
4951{
4952	__SetPageUptodate(new_page);
4953	hugepage_add_new_anon_rmap(new_page, vma, addr);
4954	set_huge_pte_at(vma->vm_mm, addr, ptep, make_huge_pte(vma, new_page, 1));
4955	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
4956	SetHPageMigratable(new_page);
4957}
4958
4959int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
4960			    struct vm_area_struct *dst_vma,
4961			    struct vm_area_struct *src_vma)
4962{
4963	pte_t *src_pte, *dst_pte, entry;
4964	struct page *ptepage;
4965	unsigned long addr;
4966	bool cow = is_cow_mapping(src_vma->vm_flags);
4967	struct hstate *h = hstate_vma(src_vma);
4968	unsigned long sz = huge_page_size(h);
4969	unsigned long npages = pages_per_huge_page(h);
4970	struct mmu_notifier_range range;
4971	unsigned long last_addr_mask;
4972	int ret = 0;
4973
 
 
4974	if (cow) {
4975		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src_vma, src,
4976					src_vma->vm_start,
4977					src_vma->vm_end);
4978		mmu_notifier_invalidate_range_start(&range);
4979		mmap_assert_write_locked(src);
4980		raw_write_seqcount_begin(&src->write_protect_seq);
4981	} else {
4982		/*
4983		 * For shared mappings the vma lock must be held before
4984		 * calling huge_pte_offset in the src vma. Otherwise, the
4985		 * returned ptep could go away if part of a shared pmd and
4986		 * another thread calls huge_pmd_unshare.
4987		 */
4988		hugetlb_vma_lock_read(src_vma);
4989	}
4990
4991	last_addr_mask = hugetlb_mask_last_page(h);
4992	for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
4993		spinlock_t *src_ptl, *dst_ptl;
4994		src_pte = huge_pte_offset(src, addr, sz);
4995		if (!src_pte) {
4996			addr |= last_addr_mask;
4997			continue;
4998		}
4999		dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
5000		if (!dst_pte) {
5001			ret = -ENOMEM;
5002			break;
5003		}
5004
5005		/*
5006		 * If the pagetables are shared don't copy or take references.
5007		 *
5008		 * dst_pte == src_pte is the common case of src/dest sharing.
 
5009		 * However, src could have 'unshared' and dst shares with
5010		 * another vma. So page_count of ptep page is checked instead
5011		 * to reliably determine whether pte is shared.
 
5012		 */
5013		if (page_count(virt_to_page(dst_pte)) > 1) {
5014			addr |= last_addr_mask;
5015			continue;
5016		}
5017
5018		dst_ptl = huge_pte_lock(h, dst, dst_pte);
5019		src_ptl = huge_pte_lockptr(h, src, src_pte);
5020		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5021		entry = huge_ptep_get(src_pte);
5022again:
5023		if (huge_pte_none(entry)) {
5024			/*
5025			 * Skip if src entry none.
 
 
5026			 */
5027			;
5028		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
5029			bool uffd_wp = huge_pte_uffd_wp(entry);
5030
5031			if (!userfaultfd_wp(dst_vma) && uffd_wp)
5032				entry = huge_pte_clear_uffd_wp(entry);
5033			set_huge_pte_at(dst, addr, dst_pte, entry);
5034		} else if (unlikely(is_hugetlb_entry_migration(entry))) {
5035			swp_entry_t swp_entry = pte_to_swp_entry(entry);
5036			bool uffd_wp = huge_pte_uffd_wp(entry);
5037
5038			if (!is_readable_migration_entry(swp_entry) && cow) {
5039				/*
5040				 * COW mappings require pages in both
5041				 * parent and child to be set to read.
5042				 */
5043				swp_entry = make_readable_migration_entry(
5044							swp_offset(swp_entry));
5045				entry = swp_entry_to_pte(swp_entry);
5046				if (userfaultfd_wp(src_vma) && uffd_wp)
5047					entry = huge_pte_mkuffd_wp(entry);
5048				set_huge_pte_at(src, addr, src_pte, entry);
5049			}
5050			if (!userfaultfd_wp(dst_vma) && uffd_wp)
5051				entry = huge_pte_clear_uffd_wp(entry);
5052			set_huge_pte_at(dst, addr, dst_pte, entry);
5053		} else if (unlikely(is_pte_marker(entry))) {
5054			/* No swap on hugetlb */
5055			WARN_ON_ONCE(
5056			    is_swapin_error_entry(pte_to_swp_entry(entry)));
5057			/*
5058			 * We copy the pte marker only if the dst vma has
5059			 * uffd-wp enabled.
5060			 */
5061			if (userfaultfd_wp(dst_vma))
5062				set_huge_pte_at(dst, addr, dst_pte, entry);
5063		} else {
5064			entry = huge_ptep_get(src_pte);
5065			ptepage = pte_page(entry);
5066			get_page(ptepage);
5067
5068			/*
5069			 * Failing to duplicate the anon rmap is a rare case
5070			 * where we see pinned hugetlb pages while they're
5071			 * prone to COW. We need to do the COW earlier during
5072			 * fork.
5073			 *
5074			 * When pre-allocating the page or copying data, we
5075			 * need to be without the pgtable locks since we could
5076			 * sleep during the process.
5077			 */
5078			if (!PageAnon(ptepage)) {
5079				page_dup_file_rmap(ptepage, true);
5080			} else if (page_try_dup_anon_rmap(ptepage, true,
5081							  src_vma)) {
5082				pte_t src_pte_old = entry;
5083				struct page *new;
5084
5085				spin_unlock(src_ptl);
5086				spin_unlock(dst_ptl);
5087				/* Do not use reserve as it's private owned */
5088				new = alloc_huge_page(dst_vma, addr, 1);
5089				if (IS_ERR(new)) {
5090					put_page(ptepage);
5091					ret = PTR_ERR(new);
5092					break;
5093				}
5094				copy_user_huge_page(new, ptepage, addr, dst_vma,
5095						    npages);
5096				put_page(ptepage);
5097
5098				/* Install the new huge page if src pte stable */
5099				dst_ptl = huge_pte_lock(h, dst, dst_pte);
5100				src_ptl = huge_pte_lockptr(h, src, src_pte);
5101				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5102				entry = huge_ptep_get(src_pte);
5103				if (!pte_same(src_pte_old, entry)) {
5104					restore_reserve_on_error(h, dst_vma, addr,
5105								new);
5106					put_page(new);
5107					/* huge_ptep of dst_pte won't change as in child */
5108					goto again;
5109				}
5110				hugetlb_install_page(dst_vma, dst_pte, addr, new);
5111				spin_unlock(src_ptl);
5112				spin_unlock(dst_ptl);
5113				continue;
5114			}
5115
5116			if (cow) {
5117				/*
5118				 * No need to notify as we are downgrading page
5119				 * table protection not changing it to point
5120				 * to a new page.
5121				 *
5122				 * See Documentation/mm/mmu_notifier.rst
5123				 */
5124				huge_ptep_set_wrprotect(src, addr, src_pte);
5125				entry = huge_pte_wrprotect(entry);
5126			}
5127
 
 
 
5128			set_huge_pte_at(dst, addr, dst_pte, entry);
5129			hugetlb_count_add(npages, dst);
5130		}
5131		spin_unlock(src_ptl);
5132		spin_unlock(dst_ptl);
5133	}
5134
5135	if (cow) {
5136		raw_write_seqcount_end(&src->write_protect_seq);
5137		mmu_notifier_invalidate_range_end(&range);
5138	} else {
5139		hugetlb_vma_unlock_read(src_vma);
5140	}
5141
5142	return ret;
5143}
5144
5145static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
5146			  unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte)
5147{
5148	struct hstate *h = hstate_vma(vma);
5149	struct mm_struct *mm = vma->vm_mm;
5150	spinlock_t *src_ptl, *dst_ptl;
5151	pte_t pte;
5152
5153	dst_ptl = huge_pte_lock(h, mm, dst_pte);
5154	src_ptl = huge_pte_lockptr(h, mm, src_pte);
5155
5156	/*
5157	 * We don't have to worry about the ordering of src and dst ptlocks
5158	 * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
5159	 */
5160	if (src_ptl != dst_ptl)
5161		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5162
5163	pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
5164	set_huge_pte_at(mm, new_addr, dst_pte, pte);
5165
5166	if (src_ptl != dst_ptl)
5167		spin_unlock(src_ptl);
5168	spin_unlock(dst_ptl);
5169}
5170
5171int move_hugetlb_page_tables(struct vm_area_struct *vma,
5172			     struct vm_area_struct *new_vma,
5173			     unsigned long old_addr, unsigned long new_addr,
5174			     unsigned long len)
5175{
5176	struct hstate *h = hstate_vma(vma);
5177	struct address_space *mapping = vma->vm_file->f_mapping;
5178	unsigned long sz = huge_page_size(h);
5179	struct mm_struct *mm = vma->vm_mm;
5180	unsigned long old_end = old_addr + len;
5181	unsigned long last_addr_mask;
5182	pte_t *src_pte, *dst_pte;
5183	struct mmu_notifier_range range;
5184	bool shared_pmd = false;
5185
5186	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, old_addr,
5187				old_end);
5188	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5189	/*
5190	 * In case of shared PMDs, we should cover the maximum possible
5191	 * range.
5192	 */
5193	flush_cache_range(vma, range.start, range.end);
5194
5195	mmu_notifier_invalidate_range_start(&range);
5196	last_addr_mask = hugetlb_mask_last_page(h);
5197	/* Prevent race with file truncation */
5198	hugetlb_vma_lock_write(vma);
5199	i_mmap_lock_write(mapping);
5200	for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
5201		src_pte = huge_pte_offset(mm, old_addr, sz);
5202		if (!src_pte) {
5203			old_addr |= last_addr_mask;
5204			new_addr |= last_addr_mask;
5205			continue;
5206		}
5207		if (huge_pte_none(huge_ptep_get(src_pte)))
5208			continue;
5209
5210		if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) {
5211			shared_pmd = true;
5212			old_addr |= last_addr_mask;
5213			new_addr |= last_addr_mask;
5214			continue;
5215		}
5216
5217		dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
5218		if (!dst_pte)
5219			break;
5220
5221		move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte);
5222	}
5223
5224	if (shared_pmd)
5225		flush_tlb_range(vma, range.start, range.end);
5226	else
5227		flush_tlb_range(vma, old_end - len, old_end);
5228	mmu_notifier_invalidate_range_end(&range);
5229	i_mmap_unlock_write(mapping);
5230	hugetlb_vma_unlock_write(vma);
5231
5232	return len + old_addr - old_end;
5233}
5234
5235static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
5236				   unsigned long start, unsigned long end,
5237				   struct page *ref_page, zap_flags_t zap_flags)
5238{
5239	struct mm_struct *mm = vma->vm_mm;
5240	unsigned long address;
5241	pte_t *ptep;
5242	pte_t pte;
5243	spinlock_t *ptl;
5244	struct page *page;
5245	struct hstate *h = hstate_vma(vma);
5246	unsigned long sz = huge_page_size(h);
5247	unsigned long last_addr_mask;
5248	bool force_flush = false;
5249
5250	WARN_ON(!is_vm_hugetlb_page(vma));
5251	BUG_ON(start & ~huge_page_mask(h));
5252	BUG_ON(end & ~huge_page_mask(h));
5253
5254	/*
5255	 * This is a hugetlb vma, all the pte entries should point
5256	 * to huge page.
5257	 */
5258	tlb_change_page_size(tlb, sz);
5259	tlb_start_vma(tlb, vma);
5260
5261	last_addr_mask = hugetlb_mask_last_page(h);
 
 
 
 
 
 
5262	address = start;
5263	for (; address < end; address += sz) {
5264		ptep = huge_pte_offset(mm, address, sz);
5265		if (!ptep) {
5266			address |= last_addr_mask;
5267			continue;
5268		}
5269
5270		ptl = huge_pte_lock(h, mm, ptep);
5271		if (huge_pmd_unshare(mm, vma, address, ptep)) {
5272			spin_unlock(ptl);
5273			tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
5274			force_flush = true;
5275			address |= last_addr_mask;
 
5276			continue;
5277		}
5278
5279		pte = huge_ptep_get(ptep);
5280		if (huge_pte_none(pte)) {
5281			spin_unlock(ptl);
5282			continue;
5283		}
5284
5285		/*
5286		 * Migrating hugepage or HWPoisoned hugepage is already
5287		 * unmapped and its refcount is dropped, so just clear pte here.
5288		 */
5289		if (unlikely(!pte_present(pte))) {
5290			/*
5291			 * If the pte was wr-protected by uffd-wp in any of the
5292			 * swap forms, meanwhile the caller does not want to
5293			 * drop the uffd-wp bit in this zap, then replace the
5294			 * pte with a marker.
5295			 */
5296			if (pte_swp_uffd_wp_any(pte) &&
5297			    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5298				set_huge_pte_at(mm, address, ptep,
5299						make_pte_marker(PTE_MARKER_UFFD_WP));
5300			else
5301				huge_pte_clear(mm, address, ptep, sz);
5302			spin_unlock(ptl);
5303			continue;
5304		}
5305
5306		page = pte_page(pte);
5307		/*
5308		 * If a reference page is supplied, it is because a specific
5309		 * page is being unmapped, not a range. Ensure the page we
5310		 * are about to unmap is the actual page of interest.
5311		 */
5312		if (ref_page) {
5313			if (page != ref_page) {
5314				spin_unlock(ptl);
5315				continue;
5316			}
5317			/*
5318			 * Mark the VMA as having unmapped its page so that
5319			 * future faults in this VMA will fail rather than
5320			 * looking like data was lost
5321			 */
5322			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
5323		}
5324
5325		pte = huge_ptep_get_and_clear(mm, address, ptep);
5326		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5327		if (huge_pte_dirty(pte))
5328			set_page_dirty(page);
5329		/* Leave a uffd-wp pte marker if needed */
5330		if (huge_pte_uffd_wp(pte) &&
5331		    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5332			set_huge_pte_at(mm, address, ptep,
5333					make_pte_marker(PTE_MARKER_UFFD_WP));
5334		hugetlb_count_sub(pages_per_huge_page(h), mm);
5335		page_remove_rmap(page, vma, true);
5336
5337		spin_unlock(ptl);
5338		tlb_remove_page_size(tlb, page, huge_page_size(h));
5339		/*
5340		 * Bail out after unmapping reference page if supplied
5341		 */
5342		if (ref_page)
5343			break;
5344	}
 
5345	tlb_end_vma(tlb, vma);
5346
5347	/*
5348	 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
5349	 * could defer the flush until now, since by holding i_mmap_rwsem we
5350	 * guaranteed that the last refernece would not be dropped. But we must
5351	 * do the flushing before we return, as otherwise i_mmap_rwsem will be
5352	 * dropped and the last reference to the shared PMDs page might be
5353	 * dropped as well.
5354	 *
5355	 * In theory we could defer the freeing of the PMD pages as well, but
5356	 * huge_pmd_unshare() relies on the exact page_count for the PMD page to
5357	 * detect sharing, so we cannot defer the release of the page either.
5358	 * Instead, do flush now.
5359	 */
5360	if (force_flush)
5361		tlb_flush_mmu_tlbonly(tlb);
5362}
5363
5364void __unmap_hugepage_range_final(struct mmu_gather *tlb,
5365			  struct vm_area_struct *vma, unsigned long start,
5366			  unsigned long end, struct page *ref_page,
5367			  zap_flags_t zap_flags)
5368{
5369	hugetlb_vma_lock_write(vma);
5370	i_mmap_lock_write(vma->vm_file->f_mapping);
5371
5372	/* mmu notification performed in caller */
5373	__unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags);
5374
5375	if (zap_flags & ZAP_FLAG_UNMAP) {	/* final unmap */
5376		/*
5377		 * Unlock and free the vma lock before releasing i_mmap_rwsem.
5378		 * When the vma_lock is freed, this makes the vma ineligible
5379		 * for pmd sharing.  And, i_mmap_rwsem is required to set up
5380		 * pmd sharing.  This is important as page tables for this
5381		 * unmapped range will be asynchrously deleted.  If the page
5382		 * tables are shared, there will be issues when accessed by
5383		 * someone else.
5384		 */
5385		__hugetlb_vma_unlock_write_free(vma);
5386		i_mmap_unlock_write(vma->vm_file->f_mapping);
5387	} else {
5388		i_mmap_unlock_write(vma->vm_file->f_mapping);
5389		hugetlb_vma_unlock_write(vma);
5390	}
5391}
5392
5393void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
5394			  unsigned long end, struct page *ref_page,
5395			  zap_flags_t zap_flags)
5396{
5397	struct mmu_notifier_range range;
5398	struct mmu_gather tlb;
 
 
5399
5400	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
5401				start, end);
5402	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5403	mmu_notifier_invalidate_range_start(&range);
5404	tlb_gather_mmu(&tlb, vma->vm_mm);
 
 
 
5405
5406	__unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags);
5407
5408	mmu_notifier_invalidate_range_end(&range);
5409	tlb_finish_mmu(&tlb);
 
5410}
5411
5412/*
5413 * This is called when the original mapper is failing to COW a MAP_PRIVATE
5414 * mapping it owns the reserve page for. The intention is to unmap the page
5415 * from other VMAs and let the children be SIGKILLed if they are faulting the
5416 * same region.
5417 */
5418static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
5419			      struct page *page, unsigned long address)
5420{
5421	struct hstate *h = hstate_vma(vma);
5422	struct vm_area_struct *iter_vma;
5423	struct address_space *mapping;
5424	pgoff_t pgoff;
5425
5426	/*
5427	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
5428	 * from page cache lookup which is in HPAGE_SIZE units.
5429	 */
5430	address = address & huge_page_mask(h);
5431	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
5432			vma->vm_pgoff;
5433	mapping = vma->vm_file->f_mapping;
5434
5435	/*
5436	 * Take the mapping lock for the duration of the table walk. As
5437	 * this mapping should be shared between all the VMAs,
5438	 * __unmap_hugepage_range() is called as the lock is already held
5439	 */
5440	i_mmap_lock_write(mapping);
5441	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
5442		/* Do not unmap the current VMA */
5443		if (iter_vma == vma)
5444			continue;
5445
5446		/*
5447		 * Shared VMAs have their own reserves and do not affect
5448		 * MAP_PRIVATE accounting but it is possible that a shared
5449		 * VMA is using the same page so check and skip such VMAs.
5450		 */
5451		if (iter_vma->vm_flags & VM_MAYSHARE)
5452			continue;
5453
5454		/*
5455		 * Unmap the page from other VMAs without their own reserves.
5456		 * They get marked to be SIGKILLed if they fault in these
5457		 * areas. This is because a future no-page fault on this VMA
5458		 * could insert a zeroed page instead of the data existing
5459		 * from the time of fork. This would look like data corruption
5460		 */
5461		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
5462			unmap_hugepage_range(iter_vma, address,
5463					     address + huge_page_size(h), page, 0);
5464	}
5465	i_mmap_unlock_write(mapping);
5466}
5467
5468/*
5469 * hugetlb_wp() should be called with page lock of the original hugepage held.
5470 * Called with hugetlb_fault_mutex_table held and pte_page locked so we
5471 * cannot race with other handlers or page migration.
5472 * Keep the pte_same checks anyway to make transition from the mutex easier.
5473 */
5474static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
5475		       unsigned long address, pte_t *ptep, unsigned int flags,
5476		       struct page *pagecache_page, spinlock_t *ptl)
5477{
5478	const bool unshare = flags & FAULT_FLAG_UNSHARE;
5479	pte_t pte;
5480	struct hstate *h = hstate_vma(vma);
5481	struct page *old_page, *new_page;
5482	int outside_reserve = 0;
5483	vm_fault_t ret = 0;
5484	unsigned long haddr = address & huge_page_mask(h);
5485	struct mmu_notifier_range range;
5486
5487	/*
5488	 * hugetlb does not support FOLL_FORCE-style write faults that keep the
5489	 * PTE mapped R/O such as maybe_mkwrite() would do.
5490	 */
5491	if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE)))
5492		return VM_FAULT_SIGSEGV;
5493
5494	/* Let's take out MAP_SHARED mappings first. */
5495	if (vma->vm_flags & VM_MAYSHARE) {
5496		set_huge_ptep_writable(vma, haddr, ptep);
5497		return 0;
5498	}
5499
5500	pte = huge_ptep_get(ptep);
5501	old_page = pte_page(pte);
5502
5503	delayacct_wpcopy_start();
5504
5505retry_avoidcopy:
5506	/*
5507	 * If no-one else is actually using this page, we're the exclusive
5508	 * owner and can reuse this page.
5509	 */
5510	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
5511		if (!PageAnonExclusive(old_page))
5512			page_move_anon_rmap(old_page, vma);
5513		if (likely(!unshare))
5514			set_huge_ptep_writable(vma, haddr, ptep);
5515
5516		delayacct_wpcopy_end();
5517		return 0;
5518	}
5519	VM_BUG_ON_PAGE(PageAnon(old_page) && PageAnonExclusive(old_page),
5520		       old_page);
5521
5522	/*
5523	 * If the process that created a MAP_PRIVATE mapping is about to
5524	 * perform a COW due to a shared page count, attempt to satisfy
5525	 * the allocation without using the existing reserves. The pagecache
5526	 * page is used to determine if the reserve at this address was
5527	 * consumed or not. If reserves were used, a partial faulted mapping
5528	 * at the time of fork() could consume its reserves on COW instead
5529	 * of the full address range.
5530	 */
5531	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
5532			old_page != pagecache_page)
5533		outside_reserve = 1;
5534
5535	get_page(old_page);
5536
5537	/*
5538	 * Drop page table lock as buddy allocator may be called. It will
5539	 * be acquired again before returning to the caller, as expected.
5540	 */
5541	spin_unlock(ptl);
5542	new_page = alloc_huge_page(vma, haddr, outside_reserve);
5543
5544	if (IS_ERR(new_page)) {
5545		/*
5546		 * If a process owning a MAP_PRIVATE mapping fails to COW,
5547		 * it is due to references held by a child and an insufficient
5548		 * huge page pool. To guarantee the original mappers
5549		 * reliability, unmap the page from child processes. The child
5550		 * may get SIGKILLed if it later faults.
5551		 */
5552		if (outside_reserve) {
5553			struct address_space *mapping = vma->vm_file->f_mapping;
5554			pgoff_t idx;
5555			u32 hash;
5556
5557			put_page(old_page);
5558			/*
5559			 * Drop hugetlb_fault_mutex and vma_lock before
5560			 * unmapping.  unmapping needs to hold vma_lock
5561			 * in write mode.  Dropping vma_lock in read mode
5562			 * here is OK as COW mappings do not interact with
5563			 * PMD sharing.
5564			 *
5565			 * Reacquire both after unmap operation.
5566			 */
5567			idx = vma_hugecache_offset(h, vma, haddr);
5568			hash = hugetlb_fault_mutex_hash(mapping, idx);
5569			hugetlb_vma_unlock_read(vma);
5570			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5571
5572			unmap_ref_private(mm, vma, old_page, haddr);
5573
5574			mutex_lock(&hugetlb_fault_mutex_table[hash]);
5575			hugetlb_vma_lock_read(vma);
5576			spin_lock(ptl);
5577			ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
5578			if (likely(ptep &&
5579				   pte_same(huge_ptep_get(ptep), pte)))
5580				goto retry_avoidcopy;
5581			/*
5582			 * race occurs while re-acquiring page table
5583			 * lock, and our job is done.
5584			 */
5585			delayacct_wpcopy_end();
5586			return 0;
5587		}
5588
5589		ret = vmf_error(PTR_ERR(new_page));
5590		goto out_release_old;
5591	}
5592
5593	/*
5594	 * When the original hugepage is shared one, it does not have
5595	 * anon_vma prepared.
5596	 */
5597	if (unlikely(anon_vma_prepare(vma))) {
5598		ret = VM_FAULT_OOM;
5599		goto out_release_all;
5600	}
5601
5602	copy_user_huge_page(new_page, old_page, address, vma,
5603			    pages_per_huge_page(h));
5604	__SetPageUptodate(new_page);
5605
5606	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,
5607				haddr + huge_page_size(h));
5608	mmu_notifier_invalidate_range_start(&range);
5609
5610	/*
5611	 * Retake the page table lock to check for racing updates
5612	 * before the page tables are altered
5613	 */
5614	spin_lock(ptl);
5615	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
5616	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
5617		/* Break COW or unshare */
 
 
5618		huge_ptep_clear_flush(vma, haddr, ptep);
5619		mmu_notifier_invalidate_range(mm, range.start, range.end);
5620		page_remove_rmap(old_page, vma, true);
5621		hugepage_add_new_anon_rmap(new_page, vma, haddr);
5622		set_huge_pte_at(mm, haddr, ptep,
5623				make_huge_pte(vma, new_page, !unshare));
5624		SetHPageMigratable(new_page);
 
 
5625		/* Make the old page be freed below */
5626		new_page = old_page;
5627	}
5628	spin_unlock(ptl);
5629	mmu_notifier_invalidate_range_end(&range);
5630out_release_all:
5631	/*
5632	 * No restore in case of successful pagetable update (Break COW or
5633	 * unshare)
5634	 */
5635	if (new_page != old_page)
5636		restore_reserve_on_error(h, vma, haddr, new_page);
5637	put_page(new_page);
5638out_release_old:
5639	put_page(old_page);
5640
5641	spin_lock(ptl); /* Caller expects lock to be held */
5642
5643	delayacct_wpcopy_end();
5644	return ret;
5645}
5646
 
 
 
 
 
 
 
 
 
 
 
 
 
5647/*
5648 * Return whether there is a pagecache page to back given address within VMA.
5649 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
5650 */
5651static bool hugetlbfs_pagecache_present(struct hstate *h,
5652			struct vm_area_struct *vma, unsigned long address)
5653{
5654	struct address_space *mapping;
5655	pgoff_t idx;
5656	struct page *page;
5657
5658	mapping = vma->vm_file->f_mapping;
5659	idx = vma_hugecache_offset(h, vma, address);
5660
5661	page = find_get_page(mapping, idx);
5662	if (page)
5663		put_page(page);
5664	return page != NULL;
5665}
5666
5667int hugetlb_add_to_page_cache(struct page *page, struct address_space *mapping,
5668			   pgoff_t idx)
5669{
5670	struct folio *folio = page_folio(page);
5671	struct inode *inode = mapping->host;
5672	struct hstate *h = hstate_inode(inode);
5673	int err;
5674
5675	__folio_set_locked(folio);
5676	err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL);
5677
5678	if (unlikely(err)) {
5679		__folio_clear_locked(folio);
5680		return err;
5681	}
5682	ClearHPageRestoreReserve(page);
5683
5684	/*
5685	 * mark folio dirty so that it will not be removed from cache/file
5686	 * by non-hugetlbfs specific code paths.
5687	 */
5688	folio_mark_dirty(folio);
5689
5690	spin_lock(&inode->i_lock);
5691	inode->i_blocks += blocks_per_huge_page(h);
5692	spin_unlock(&inode->i_lock);
5693	return 0;
5694}
5695
5696static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
5697						  struct address_space *mapping,
5698						  pgoff_t idx,
5699						  unsigned int flags,
5700						  unsigned long haddr,
5701						  unsigned long addr,
5702						  unsigned long reason)
5703{
5704	u32 hash;
5705	struct vm_fault vmf = {
5706		.vma = vma,
5707		.address = haddr,
5708		.real_address = addr,
5709		.flags = flags,
5710
5711		/*
5712		 * Hard to debug if it ends up being
5713		 * used by a callee that assumes
5714		 * something about the other
5715		 * uninitialized fields... same as in
5716		 * memory.c
5717		 */
5718	};
5719
5720	/*
5721	 * vma_lock and hugetlb_fault_mutex must be dropped before handling
5722	 * userfault. Also mmap_lock could be dropped due to handling
5723	 * userfault, any vma operation should be careful from here.
5724	 */
5725	hugetlb_vma_unlock_read(vma);
5726	hash = hugetlb_fault_mutex_hash(mapping, idx);
5727	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5728	return handle_userfault(&vmf, reason);
5729}
5730
5731/*
5732 * Recheck pte with pgtable lock.  Returns true if pte didn't change, or
5733 * false if pte changed or is changing.
5734 */
5735static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm,
5736			       pte_t *ptep, pte_t old_pte)
5737{
5738	spinlock_t *ptl;
5739	bool same;
5740
5741	ptl = huge_pte_lock(h, mm, ptep);
5742	same = pte_same(huge_ptep_get(ptep), old_pte);
5743	spin_unlock(ptl);
5744
5745	return same;
5746}
5747
5748static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
5749			struct vm_area_struct *vma,
5750			struct address_space *mapping, pgoff_t idx,
5751			unsigned long address, pte_t *ptep,
5752			pte_t old_pte, unsigned int flags)
5753{
5754	struct hstate *h = hstate_vma(vma);
5755	vm_fault_t ret = VM_FAULT_SIGBUS;
5756	int anon_rmap = 0;
5757	unsigned long size;
5758	struct page *page;
5759	pte_t new_pte;
5760	spinlock_t *ptl;
5761	unsigned long haddr = address & huge_page_mask(h);
5762	bool new_page, new_pagecache_page = false;
5763	u32 hash = hugetlb_fault_mutex_hash(mapping, idx);
5764
5765	/*
5766	 * Currently, we are forced to kill the process in the event the
5767	 * original mapper has unmapped pages from the child due to a failed
5768	 * COW/unsharing. Warn that such a situation has occurred as it may not
5769	 * be obvious.
5770	 */
5771	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
5772		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
5773			   current->pid);
5774		goto out;
5775	}
5776
5777	/*
5778	 * Use page lock to guard against racing truncation
5779	 * before we get page_table_lock.
 
5780	 */
5781	new_page = false;
 
 
 
 
5782	page = find_lock_page(mapping, idx);
5783	if (!page) {
5784		size = i_size_read(mapping->host) >> huge_page_shift(h);
5785		if (idx >= size)
5786			goto out;
5787		/* Check for page in userfault range */
5788		if (userfaultfd_missing(vma)) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5789			/*
5790			 * Since hugetlb_no_page() was examining pte
5791			 * without pgtable lock, we need to re-test under
5792			 * lock because the pte may not be stable and could
5793			 * have changed from under us.  Try to detect
5794			 * either changed or during-changing ptes and retry
5795			 * properly when needed.
5796			 *
5797			 * Note that userfaultfd is actually fine with
5798			 * false positives (e.g. caused by pte changed),
5799			 * but not wrong logical events (e.g. caused by
5800			 * reading a pte during changing).  The latter can
5801			 * confuse the userspace, so the strictness is very
5802			 * much preferred.  E.g., MISSING event should
5803			 * never happen on the page after UFFDIO_COPY has
5804			 * correctly installed the page and returned.
5805			 */
5806			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
5807				ret = 0;
5808				goto out;
5809			}
5810
5811			return hugetlb_handle_userfault(vma, mapping, idx, flags,
5812							haddr, address,
5813							VM_UFFD_MISSING);
5814		}
5815
5816		page = alloc_huge_page(vma, haddr, 0);
5817		if (IS_ERR(page)) {
5818			/*
5819			 * Returning error will result in faulting task being
5820			 * sent SIGBUS.  The hugetlb fault mutex prevents two
5821			 * tasks from racing to fault in the same page which
5822			 * could result in false unable to allocate errors.
5823			 * Page migration does not take the fault mutex, but
5824			 * does a clear then write of pte's under page table
5825			 * lock.  Page fault code could race with migration,
5826			 * notice the clear pte and try to allocate a page
5827			 * here.  Before returning error, get ptl and make
5828			 * sure there really is no pte entry.
5829			 */
5830			if (hugetlb_pte_stable(h, mm, ptep, old_pte))
5831				ret = vmf_error(PTR_ERR(page));
5832			else
5833				ret = 0;
 
 
 
 
 
5834			goto out;
5835		}
5836		clear_huge_page(page, address, pages_per_huge_page(h));
5837		__SetPageUptodate(page);
5838		new_page = true;
5839
5840		if (vma->vm_flags & VM_MAYSHARE) {
5841			int err = hugetlb_add_to_page_cache(page, mapping, idx);
5842			if (err) {
5843				/*
5844				 * err can't be -EEXIST which implies someone
5845				 * else consumed the reservation since hugetlb
5846				 * fault mutex is held when add a hugetlb page
5847				 * to the page cache. So it's safe to call
5848				 * restore_reserve_on_error() here.
5849				 */
5850				restore_reserve_on_error(h, vma, haddr, page);
5851				put_page(page);
 
 
5852				goto out;
5853			}
5854			new_pagecache_page = true;
5855		} else {
5856			lock_page(page);
5857			if (unlikely(anon_vma_prepare(vma))) {
5858				ret = VM_FAULT_OOM;
5859				goto backout_unlocked;
5860			}
5861			anon_rmap = 1;
5862		}
5863	} else {
5864		/*
5865		 * If memory error occurs between mmap() and fault, some process
5866		 * don't have hwpoisoned swap entry for errored virtual address.
5867		 * So we need to block hugepage fault by PG_hwpoison bit check.
5868		 */
5869		if (unlikely(PageHWPoison(page))) {
5870			ret = VM_FAULT_HWPOISON_LARGE |
5871				VM_FAULT_SET_HINDEX(hstate_index(h));
5872			goto backout_unlocked;
5873		}
5874
5875		/* Check for page in userfault range. */
5876		if (userfaultfd_minor(vma)) {
5877			unlock_page(page);
5878			put_page(page);
5879			/* See comment in userfaultfd_missing() block above */
5880			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
5881				ret = 0;
5882				goto out;
5883			}
5884			return hugetlb_handle_userfault(vma, mapping, idx, flags,
5885							haddr, address,
5886							VM_UFFD_MINOR);
5887		}
5888	}
5889
5890	/*
5891	 * If we are going to COW a private mapping later, we examine the
5892	 * pending reservations for this page now. This will ensure that
5893	 * any allocations necessary to record that reservation occur outside
5894	 * the spinlock.
5895	 */
5896	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
5897		if (vma_needs_reservation(h, vma, haddr) < 0) {
5898			ret = VM_FAULT_OOM;
5899			goto backout_unlocked;
5900		}
5901		/* Just decrements count, does not deallocate */
5902		vma_end_reservation(h, vma, haddr);
5903	}
5904
5905	ptl = huge_pte_lock(h, mm, ptep);
5906	ret = 0;
5907	/* If pte changed from under us, retry */
5908	if (!pte_same(huge_ptep_get(ptep), old_pte))
5909		goto backout;
5910
5911	if (anon_rmap)
 
5912		hugepage_add_new_anon_rmap(page, vma, haddr);
5913	else
5914		page_dup_file_rmap(page, true);
5915	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
5916				&& (vma->vm_flags & VM_SHARED)));
5917	/*
5918	 * If this pte was previously wr-protected, keep it wr-protected even
5919	 * if populated.
5920	 */
5921	if (unlikely(pte_marker_uffd_wp(old_pte)))
5922		new_pte = huge_pte_wrprotect(huge_pte_mkuffd_wp(new_pte));
5923	set_huge_pte_at(mm, haddr, ptep, new_pte);
5924
5925	hugetlb_count_add(pages_per_huge_page(h), mm);
5926	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
5927		/* Optimization, do the COW without a second fault */
5928		ret = hugetlb_wp(mm, vma, address, ptep, flags, page, ptl);
5929	}
5930
5931	spin_unlock(ptl);
5932
5933	/*
5934	 * Only set HPageMigratable in newly allocated pages.  Existing pages
5935	 * found in the pagecache may not have HPageMigratableset if they have
5936	 * been isolated for migration.
5937	 */
5938	if (new_page)
5939		SetHPageMigratable(page);
5940
5941	unlock_page(page);
5942out:
5943	hugetlb_vma_unlock_read(vma);
5944	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5945	return ret;
5946
5947backout:
5948	spin_unlock(ptl);
5949backout_unlocked:
5950	if (new_page && !new_pagecache_page)
5951		restore_reserve_on_error(h, vma, haddr, page);
5952
5953	unlock_page(page);
 
5954	put_page(page);
5955	goto out;
5956}
5957
5958#ifdef CONFIG_SMP
5959u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
5960{
5961	unsigned long key[2];
5962	u32 hash;
5963
5964	key[0] = (unsigned long) mapping;
5965	key[1] = idx;
5966
5967	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
5968
5969	return hash & (num_fault_mutexes - 1);
5970}
5971#else
5972/*
5973 * For uniprocessor systems we always use a single mutex, so just
5974 * return 0 and avoid the hashing overhead.
5975 */
5976u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
5977{
5978	return 0;
5979}
5980#endif
5981
5982vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
5983			unsigned long address, unsigned int flags)
5984{
5985	pte_t *ptep, entry;
5986	spinlock_t *ptl;
5987	vm_fault_t ret;
5988	u32 hash;
5989	pgoff_t idx;
5990	struct page *page = NULL;
5991	struct page *pagecache_page = NULL;
5992	struct hstate *h = hstate_vma(vma);
5993	struct address_space *mapping;
5994	int need_wait_lock = 0;
5995	unsigned long haddr = address & huge_page_mask(h);
5996
5997	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
5998	if (ptep) {
5999		/*
6000		 * Since we hold no locks, ptep could be stale.  That is
6001		 * OK as we are only making decisions based on content and
6002		 * not actually modifying content here.
6003		 */
6004		entry = huge_ptep_get(ptep);
6005		if (unlikely(is_hugetlb_entry_migration(entry))) {
6006			migration_entry_wait_huge(vma, ptep);
6007			return 0;
6008		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
6009			return VM_FAULT_HWPOISON_LARGE |
6010				VM_FAULT_SET_HINDEX(hstate_index(h));
6011	}
6012
6013	/*
6014	 * Serialize hugepage allocation and instantiation, so that we don't
6015	 * get spurious allocation failures if two CPUs race to instantiate
6016	 * the same page in the page cache.
6017	 */
6018	mapping = vma->vm_file->f_mapping;
6019	idx = vma_hugecache_offset(h, vma, haddr);
6020	hash = hugetlb_fault_mutex_hash(mapping, idx);
6021	mutex_lock(&hugetlb_fault_mutex_table[hash]);
6022
6023	/*
6024	 * Acquire vma lock before calling huge_pte_alloc and hold
6025	 * until finished with ptep.  This prevents huge_pmd_unshare from
6026	 * being called elsewhere and making the ptep no longer valid.
6027	 *
6028	 * ptep could have already be assigned via huge_pte_offset.  That
6029	 * is OK, as huge_pte_alloc will return the same value unless
6030	 * something has changed.
6031	 */
6032	hugetlb_vma_lock_read(vma);
6033	ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
 
6034	if (!ptep) {
6035		hugetlb_vma_unlock_read(vma);
6036		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6037		return VM_FAULT_OOM;
6038	}
6039
 
 
 
 
 
 
 
 
 
6040	entry = huge_ptep_get(ptep);
6041	/* PTE markers should be handled the same way as none pte */
6042	if (huge_pte_none_mostly(entry))
6043		/*
6044		 * hugetlb_no_page will drop vma lock and hugetlb fault
6045		 * mutex internally, which make us return immediately.
6046		 */
6047		return hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
6048				      entry, flags);
6049
6050	ret = 0;
6051
6052	/*
6053	 * entry could be a migration/hwpoison entry at this point, so this
6054	 * check prevents the kernel from going below assuming that we have
6055	 * an active hugepage in pagecache. This goto expects the 2nd page
6056	 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
6057	 * properly handle it.
6058	 */
6059	if (!pte_present(entry))
6060		goto out_mutex;
6061
6062	/*
6063	 * If we are going to COW/unshare the mapping later, we examine the
6064	 * pending reservations for this page now. This will ensure that any
6065	 * allocations necessary to record that reservation occur outside the
6066	 * spinlock. Also lookup the pagecache page now as it is used to
6067	 * determine if a reservation has been consumed.
 
6068	 */
6069	if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
6070	    !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) {
6071		if (vma_needs_reservation(h, vma, haddr) < 0) {
6072			ret = VM_FAULT_OOM;
6073			goto out_mutex;
6074		}
6075		/* Just decrements count, does not deallocate */
6076		vma_end_reservation(h, vma, haddr);
6077
6078		pagecache_page = find_lock_page(mapping, idx);
 
 
6079	}
6080
6081	ptl = huge_pte_lock(h, mm, ptep);
6082
6083	/* Check for a racing update before calling hugetlb_wp() */
6084	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
6085		goto out_ptl;
6086
6087	/* Handle userfault-wp first, before trying to lock more pages */
6088	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) &&
6089	    (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
6090		struct vm_fault vmf = {
6091			.vma = vma,
6092			.address = haddr,
6093			.real_address = address,
6094			.flags = flags,
6095		};
6096
6097		spin_unlock(ptl);
6098		if (pagecache_page) {
6099			unlock_page(pagecache_page);
6100			put_page(pagecache_page);
6101		}
6102		hugetlb_vma_unlock_read(vma);
6103		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6104		return handle_userfault(&vmf, VM_UFFD_WP);
6105	}
6106
6107	/*
6108	 * hugetlb_wp() requires page locks of pte_page(entry) and
6109	 * pagecache_page, so here we need take the former one
6110	 * when page != pagecache_page or !pagecache_page.
6111	 */
6112	page = pte_page(entry);
6113	if (page != pagecache_page)
6114		if (!trylock_page(page)) {
6115			need_wait_lock = 1;
6116			goto out_ptl;
6117		}
6118
6119	get_page(page);
6120
6121	if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
6122		if (!huge_pte_write(entry)) {
6123			ret = hugetlb_wp(mm, vma, address, ptep, flags,
6124					 pagecache_page, ptl);
6125			goto out_put_page;
6126		} else if (likely(flags & FAULT_FLAG_WRITE)) {
6127			entry = huge_pte_mkdirty(entry);
6128		}
 
6129	}
6130	entry = pte_mkyoung(entry);
6131	if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
6132						flags & FAULT_FLAG_WRITE))
6133		update_mmu_cache(vma, haddr, ptep);
6134out_put_page:
6135	if (page != pagecache_page)
6136		unlock_page(page);
6137	put_page(page);
6138out_ptl:
6139	spin_unlock(ptl);
6140
6141	if (pagecache_page) {
6142		unlock_page(pagecache_page);
6143		put_page(pagecache_page);
6144	}
6145out_mutex:
6146	hugetlb_vma_unlock_read(vma);
6147	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 
6148	/*
6149	 * Generally it's safe to hold refcount during waiting page lock. But
6150	 * here we just wait to defer the next page fault to avoid busy loop and
6151	 * the page is not used after unlocked before returning from the current
6152	 * page fault. So we are safe from accessing freed page, even if we wait
6153	 * here without taking refcount.
6154	 */
6155	if (need_wait_lock)
6156		wait_on_page_locked(page);
6157	return ret;
6158}
6159
6160#ifdef CONFIG_USERFAULTFD
6161/*
6162 * Used by userfaultfd UFFDIO_COPY.  Based on mcopy_atomic_pte with
6163 * modifications for huge pages.
6164 */
6165int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
6166			    pte_t *dst_pte,
6167			    struct vm_area_struct *dst_vma,
6168			    unsigned long dst_addr,
6169			    unsigned long src_addr,
6170			    enum mcopy_atomic_mode mode,
6171			    struct page **pagep,
6172			    bool wp_copy)
6173{
6174	bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE);
6175	struct hstate *h = hstate_vma(dst_vma);
6176	struct address_space *mapping = dst_vma->vm_file->f_mapping;
6177	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
6178	unsigned long size;
6179	int vm_shared = dst_vma->vm_flags & VM_SHARED;
 
6180	pte_t _dst_pte;
6181	spinlock_t *ptl;
6182	int ret = -ENOMEM;
6183	struct page *page;
6184	int writable;
6185	bool page_in_pagecache = false;
6186
6187	if (is_continue) {
6188		ret = -EFAULT;
6189		page = find_lock_page(mapping, idx);
6190		if (!page)
6191			goto out;
6192		page_in_pagecache = true;
6193	} else if (!*pagep) {
6194		/* If a page already exists, then it's UFFDIO_COPY for
6195		 * a non-missing case. Return -EEXIST.
6196		 */
6197		if (vm_shared &&
6198		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6199			ret = -EEXIST;
6200			goto out;
6201		}
6202
 
 
6203		page = alloc_huge_page(dst_vma, dst_addr, 0);
6204		if (IS_ERR(page)) {
6205			ret = -ENOMEM;
6206			goto out;
6207		}
6208
6209		ret = copy_huge_page_from_user(page,
6210						(const void __user *) src_addr,
6211						pages_per_huge_page(h), false);
6212
6213		/* fallback to copy_from_user outside mmap_lock */
6214		if (unlikely(ret)) {
6215			ret = -ENOENT;
6216			/* Free the allocated page which may have
6217			 * consumed a reservation.
6218			 */
6219			restore_reserve_on_error(h, dst_vma, dst_addr, page);
6220			put_page(page);
6221
6222			/* Allocate a temporary page to hold the copied
6223			 * contents.
6224			 */
6225			page = alloc_huge_page_vma(h, dst_vma, dst_addr);
6226			if (!page) {
6227				ret = -ENOMEM;
6228				goto out;
6229			}
6230			*pagep = page;
6231			/* Set the outparam pagep and return to the caller to
6232			 * copy the contents outside the lock. Don't free the
6233			 * page.
6234			 */
6235			goto out;
6236		}
6237	} else {
6238		if (vm_shared &&
6239		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6240			put_page(*pagep);
6241			ret = -EEXIST;
6242			*pagep = NULL;
6243			goto out;
6244		}
6245
6246		page = alloc_huge_page(dst_vma, dst_addr, 0);
6247		if (IS_ERR(page)) {
6248			put_page(*pagep);
6249			ret = -ENOMEM;
6250			*pagep = NULL;
6251			goto out;
6252		}
6253		copy_user_huge_page(page, *pagep, dst_addr, dst_vma,
6254				    pages_per_huge_page(h));
6255		put_page(*pagep);
6256		*pagep = NULL;
6257	}
6258
6259	/*
6260	 * The memory barrier inside __SetPageUptodate makes sure that
6261	 * preceding stores to the page contents become visible before
6262	 * the set_pte_at() write.
6263	 */
6264	__SetPageUptodate(page);
6265
6266	/* Add shared, newly allocated pages to the page cache. */
6267	if (vm_shared && !is_continue) {
 
 
 
 
 
6268		size = i_size_read(mapping->host) >> huge_page_shift(h);
6269		ret = -EFAULT;
6270		if (idx >= size)
6271			goto out_release_nounlock;
6272
6273		/*
6274		 * Serialization between remove_inode_hugepages() and
6275		 * hugetlb_add_to_page_cache() below happens through the
6276		 * hugetlb_fault_mutex_table that here must be hold by
6277		 * the caller.
6278		 */
6279		ret = hugetlb_add_to_page_cache(page, mapping, idx);
6280		if (ret)
6281			goto out_release_nounlock;
6282		page_in_pagecache = true;
6283	}
6284
6285	ptl = huge_pte_lock(h, dst_mm, dst_pte);
 
6286
6287	ret = -EIO;
6288	if (PageHWPoison(page))
 
 
 
 
 
 
 
 
 
 
6289		goto out_release_unlock;
6290
6291	/*
6292	 * We allow to overwrite a pte marker: consider when both MISSING|WP
6293	 * registered, we firstly wr-protect a none pte which has no page cache
6294	 * page backing it, then access the page.
6295	 */
6296	ret = -EEXIST;
6297	if (!huge_pte_none_mostly(huge_ptep_get(dst_pte)))
6298		goto out_release_unlock;
6299
6300	if (page_in_pagecache)
6301		page_dup_file_rmap(page, true);
6302	else
 
6303		hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
 
6304
6305	/*
6306	 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
6307	 * with wp flag set, don't set pte write bit.
6308	 */
6309	if (wp_copy || (is_continue && !vm_shared))
6310		writable = 0;
6311	else
6312		writable = dst_vma->vm_flags & VM_WRITE;
6313
6314	_dst_pte = make_huge_pte(dst_vma, page, writable);
6315	/*
6316	 * Always mark UFFDIO_COPY page dirty; note that this may not be
6317	 * extremely important for hugetlbfs for now since swapping is not
6318	 * supported, but we should still be clear in that this page cannot be
6319	 * thrown away at will, even if write bit not set.
6320	 */
6321	_dst_pte = huge_pte_mkdirty(_dst_pte);
6322	_dst_pte = pte_mkyoung(_dst_pte);
6323
6324	if (wp_copy)
6325		_dst_pte = huge_pte_mkuffd_wp(_dst_pte);
6326
6327	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
6328
 
 
6329	hugetlb_count_add(pages_per_huge_page(h), dst_mm);
6330
6331	/* No need to invalidate - it was non-present before */
6332	update_mmu_cache(dst_vma, dst_addr, dst_pte);
6333
6334	spin_unlock(ptl);
6335	if (!is_continue)
6336		SetHPageMigratable(page);
6337	if (vm_shared || is_continue)
6338		unlock_page(page);
6339	ret = 0;
6340out:
6341	return ret;
6342out_release_unlock:
6343	spin_unlock(ptl);
6344	if (vm_shared || is_continue)
6345		unlock_page(page);
6346out_release_nounlock:
6347	if (!page_in_pagecache)
6348		restore_reserve_on_error(h, dst_vma, dst_addr, page);
6349	put_page(page);
6350	goto out;
6351}
6352#endif /* CONFIG_USERFAULTFD */
6353
6354static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
6355				 int refs, struct page **pages,
6356				 struct vm_area_struct **vmas)
6357{
6358	int nr;
6359
6360	for (nr = 0; nr < refs; nr++) {
6361		if (likely(pages))
6362			pages[nr] = nth_page(page, nr);
6363		if (vmas)
6364			vmas[nr] = vma;
6365	}
6366}
6367
6368static inline bool __follow_hugetlb_must_fault(struct vm_area_struct *vma,
6369					       unsigned int flags, pte_t *pte,
6370					       bool *unshare)
6371{
6372	pte_t pteval = huge_ptep_get(pte);
6373
6374	*unshare = false;
6375	if (is_swap_pte(pteval))
6376		return true;
6377	if (huge_pte_write(pteval))
6378		return false;
6379	if (flags & FOLL_WRITE)
6380		return true;
6381	if (gup_must_unshare(vma, flags, pte_page(pteval))) {
6382		*unshare = true;
6383		return true;
6384	}
6385	return false;
6386}
6387
6388struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma,
6389				unsigned long address, unsigned int flags)
6390{
6391	struct hstate *h = hstate_vma(vma);
6392	struct mm_struct *mm = vma->vm_mm;
6393	unsigned long haddr = address & huge_page_mask(h);
6394	struct page *page = NULL;
6395	spinlock_t *ptl;
6396	pte_t *pte, entry;
6397
6398	/*
6399	 * FOLL_PIN is not supported for follow_page(). Ordinary GUP goes via
6400	 * follow_hugetlb_page().
6401	 */
6402	if (WARN_ON_ONCE(flags & FOLL_PIN))
6403		return NULL;
6404
6405retry:
6406	pte = huge_pte_offset(mm, haddr, huge_page_size(h));
6407	if (!pte)
6408		return NULL;
6409
6410	ptl = huge_pte_lock(h, mm, pte);
6411	entry = huge_ptep_get(pte);
6412	if (pte_present(entry)) {
6413		page = pte_page(entry) +
6414				((address & ~huge_page_mask(h)) >> PAGE_SHIFT);
6415		/*
6416		 * Note that page may be a sub-page, and with vmemmap
6417		 * optimizations the page struct may be read only.
6418		 * try_grab_page() will increase the ref count on the
6419		 * head page, so this will be OK.
6420		 *
6421		 * try_grab_page() should always be able to get the page here,
6422		 * because we hold the ptl lock and have verified pte_present().
6423		 */
6424		if (try_grab_page(page, flags)) {
6425			page = NULL;
6426			goto out;
6427		}
6428	} else {
6429		if (is_hugetlb_entry_migration(entry)) {
6430			spin_unlock(ptl);
6431			__migration_entry_wait_huge(pte, ptl);
6432			goto retry;
6433		}
6434		/*
6435		 * hwpoisoned entry is treated as no_page_table in
6436		 * follow_page_mask().
6437		 */
6438	}
6439out:
6440	spin_unlock(ptl);
6441	return page;
6442}
6443
6444long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
6445			 struct page **pages, struct vm_area_struct **vmas,
6446			 unsigned long *position, unsigned long *nr_pages,
6447			 long i, unsigned int flags, int *locked)
6448{
6449	unsigned long pfn_offset;
6450	unsigned long vaddr = *position;
6451	unsigned long remainder = *nr_pages;
6452	struct hstate *h = hstate_vma(vma);
6453	int err = -EFAULT, refs;
6454
6455	while (vaddr < vma->vm_end && remainder) {
6456		pte_t *pte;
6457		spinlock_t *ptl = NULL;
6458		bool unshare = false;
6459		int absent;
6460		struct page *page;
6461
6462		/*
6463		 * If we have a pending SIGKILL, don't keep faulting pages and
6464		 * potentially allocating memory.
6465		 */
6466		if (fatal_signal_pending(current)) {
6467			remainder = 0;
6468			break;
6469		}
6470
6471		/*
6472		 * Some archs (sparc64, sh*) have multiple pte_ts to
6473		 * each hugepage.  We have to make sure we get the
6474		 * first, for the page indexing below to work.
6475		 *
6476		 * Note that page table lock is not held when pte is null.
6477		 */
6478		pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
6479				      huge_page_size(h));
6480		if (pte)
6481			ptl = huge_pte_lock(h, mm, pte);
6482		absent = !pte || huge_pte_none(huge_ptep_get(pte));
6483
6484		/*
6485		 * When coredumping, it suits get_dump_page if we just return
6486		 * an error where there's an empty slot with no huge pagecache
6487		 * to back it.  This way, we avoid allocating a hugepage, and
6488		 * the sparse dumpfile avoids allocating disk blocks, but its
6489		 * huge holes still show up with zeroes where they need to be.
6490		 */
6491		if (absent && (flags & FOLL_DUMP) &&
6492		    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
6493			if (pte)
6494				spin_unlock(ptl);
6495			remainder = 0;
6496			break;
6497		}
6498
6499		/*
6500		 * We need call hugetlb_fault for both hugepages under migration
6501		 * (in which case hugetlb_fault waits for the migration,) and
6502		 * hwpoisoned hugepages (in which case we need to prevent the
6503		 * caller from accessing to them.) In order to do this, we use
6504		 * here is_swap_pte instead of is_hugetlb_entry_migration and
6505		 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
6506		 * both cases, and because we can't follow correct pages
6507		 * directly from any kind of swap entries.
6508		 */
6509		if (absent ||
6510		    __follow_hugetlb_must_fault(vma, flags, pte, &unshare)) {
 
6511			vm_fault_t ret;
6512			unsigned int fault_flags = 0;
6513
6514			if (pte)
6515				spin_unlock(ptl);
6516			if (flags & FOLL_WRITE)
6517				fault_flags |= FAULT_FLAG_WRITE;
6518			else if (unshare)
6519				fault_flags |= FAULT_FLAG_UNSHARE;
6520			if (locked) {
6521				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
6522					FAULT_FLAG_KILLABLE;
6523				if (flags & FOLL_INTERRUPTIBLE)
6524					fault_flags |= FAULT_FLAG_INTERRUPTIBLE;
6525			}
6526			if (flags & FOLL_NOWAIT)
6527				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
6528					FAULT_FLAG_RETRY_NOWAIT;
6529			if (flags & FOLL_TRIED) {
6530				/*
6531				 * Note: FAULT_FLAG_ALLOW_RETRY and
6532				 * FAULT_FLAG_TRIED can co-exist
6533				 */
6534				fault_flags |= FAULT_FLAG_TRIED;
6535			}
6536			ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
6537			if (ret & VM_FAULT_ERROR) {
6538				err = vm_fault_to_errno(ret, flags);
6539				remainder = 0;
6540				break;
6541			}
6542			if (ret & VM_FAULT_RETRY) {
6543				if (locked &&
6544				    !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
6545					*locked = 0;
6546				*nr_pages = 0;
6547				/*
6548				 * VM_FAULT_RETRY must not return an
6549				 * error, it will return zero
6550				 * instead.
6551				 *
6552				 * No need to update "position" as the
6553				 * caller will not check it after
6554				 * *nr_pages is set to 0.
6555				 */
6556				return i;
6557			}
6558			continue;
6559		}
6560
6561		pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
6562		page = pte_page(huge_ptep_get(pte));
6563
6564		VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
6565			       !PageAnonExclusive(page), page);
6566
6567		/*
6568		 * If subpage information not requested, update counters
6569		 * and skip the same_page loop below.
6570		 */
6571		if (!pages && !vmas && !pfn_offset &&
6572		    (vaddr + huge_page_size(h) < vma->vm_end) &&
6573		    (remainder >= pages_per_huge_page(h))) {
6574			vaddr += huge_page_size(h);
6575			remainder -= pages_per_huge_page(h);
6576			i += pages_per_huge_page(h);
6577			spin_unlock(ptl);
6578			continue;
6579		}
6580
6581		/* vaddr may not be aligned to PAGE_SIZE */
6582		refs = min3(pages_per_huge_page(h) - pfn_offset, remainder,
6583		    (vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT);
6584
6585		if (pages || vmas)
6586			record_subpages_vmas(nth_page(page, pfn_offset),
6587					     vma, refs,
6588					     likely(pages) ? pages + i : NULL,
6589					     vmas ? vmas + i : NULL);
6590
6591		if (pages) {
 
6592			/*
6593			 * try_grab_folio() should always succeed here,
6594			 * because: a) we hold the ptl lock, and b) we've just
6595			 * checked that the huge page is present in the page
6596			 * tables. If the huge page is present, then the tail
6597			 * pages must also be present. The ptl prevents the
6598			 * head page and tail pages from being rearranged in
6599			 * any way. As this is hugetlb, the pages will never
6600			 * be p2pdma or not longterm pinable. So this page
6601			 * must be available at this point, unless the page
6602			 * refcount overflowed:
6603			 */
6604			if (WARN_ON_ONCE(!try_grab_folio(pages[i], refs,
6605							 flags))) {
6606				spin_unlock(ptl);
6607				remainder = 0;
6608				err = -ENOMEM;
6609				break;
6610			}
6611		}
6612
6613		vaddr += (refs << PAGE_SHIFT);
6614		remainder -= refs;
6615		i += refs;
6616
 
 
 
 
 
 
 
 
 
 
 
 
6617		spin_unlock(ptl);
6618	}
6619	*nr_pages = remainder;
6620	/*
6621	 * setting position is actually required only if remainder is
6622	 * not zero but it's faster not to add a "if (remainder)"
6623	 * branch.
6624	 */
6625	*position = vaddr;
6626
6627	return i ? i : err;
6628}
6629
 
 
 
 
 
 
 
 
6630unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
6631		unsigned long address, unsigned long end,
6632		pgprot_t newprot, unsigned long cp_flags)
6633{
6634	struct mm_struct *mm = vma->vm_mm;
6635	unsigned long start = address;
6636	pte_t *ptep;
6637	pte_t pte;
6638	struct hstate *h = hstate_vma(vma);
6639	unsigned long pages = 0, psize = huge_page_size(h);
6640	bool shared_pmd = false;
6641	struct mmu_notifier_range range;
6642	unsigned long last_addr_mask;
6643	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
6644	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
6645
6646	/*
6647	 * In the case of shared PMDs, the area to flush could be beyond
6648	 * start/end.  Set range.start/range.end to cover the maximum possible
6649	 * range if PMD sharing is possible.
6650	 */
6651	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
6652				0, vma, mm, start, end);
6653	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
6654
6655	BUG_ON(address >= end);
6656	flush_cache_range(vma, range.start, range.end);
6657
6658	mmu_notifier_invalidate_range_start(&range);
6659	hugetlb_vma_lock_write(vma);
6660	i_mmap_lock_write(vma->vm_file->f_mapping);
6661	last_addr_mask = hugetlb_mask_last_page(h);
6662	for (; address < end; address += psize) {
6663		spinlock_t *ptl;
6664		ptep = huge_pte_offset(mm, address, psize);
6665		if (!ptep) {
6666			if (!uffd_wp) {
6667				address |= last_addr_mask;
6668				continue;
6669			}
6670			/*
6671			 * Userfaultfd wr-protect requires pgtable
6672			 * pre-allocations to install pte markers.
6673			 */
6674			ptep = huge_pte_alloc(mm, vma, address, psize);
6675			if (!ptep)
6676				break;
6677		}
6678		ptl = huge_pte_lock(h, mm, ptep);
6679		if (huge_pmd_unshare(mm, vma, address, ptep)) {
6680			/*
6681			 * When uffd-wp is enabled on the vma, unshare
6682			 * shouldn't happen at all.  Warn about it if it
6683			 * happened due to some reason.
6684			 */
6685			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
6686			pages++;
6687			spin_unlock(ptl);
6688			shared_pmd = true;
6689			address |= last_addr_mask;
6690			continue;
6691		}
6692		pte = huge_ptep_get(ptep);
6693		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
6694			/* Nothing to do. */
6695		} else if (unlikely(is_hugetlb_entry_migration(pte))) {
 
 
6696			swp_entry_t entry = pte_to_swp_entry(pte);
6697			struct page *page = pfn_swap_entry_to_page(entry);
6698			pte_t newpte = pte;
6699
6700			if (is_writable_migration_entry(entry)) {
6701				if (PageAnon(page))
6702					entry = make_readable_exclusive_migration_entry(
6703								swp_offset(entry));
6704				else
6705					entry = make_readable_migration_entry(
6706								swp_offset(entry));
6707				newpte = swp_entry_to_pte(entry);
 
 
6708				pages++;
6709			}
6710
6711			if (uffd_wp)
6712				newpte = pte_swp_mkuffd_wp(newpte);
6713			else if (uffd_wp_resolve)
6714				newpte = pte_swp_clear_uffd_wp(newpte);
6715			if (!pte_same(pte, newpte))
6716				set_huge_pte_at(mm, address, ptep, newpte);
6717		} else if (unlikely(is_pte_marker(pte))) {
6718			/* No other markers apply for now. */
6719			WARN_ON_ONCE(!pte_marker_uffd_wp(pte));
6720			if (uffd_wp_resolve)
6721				/* Safe to modify directly (non-present->none). */
6722				huge_pte_clear(mm, address, ptep, psize);
6723		} else if (!huge_pte_none(pte)) {
6724			pte_t old_pte;
6725			unsigned int shift = huge_page_shift(hstate_vma(vma));
6726
6727			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
6728			pte = huge_pte_modify(old_pte, newprot);
6729			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
6730			if (uffd_wp)
6731				pte = huge_pte_mkuffd_wp(huge_pte_wrprotect(pte));
6732			else if (uffd_wp_resolve)
6733				pte = huge_pte_clear_uffd_wp(pte);
6734			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
6735			pages++;
6736		} else {
6737			/* None pte */
6738			if (unlikely(uffd_wp))
6739				/* Safe to modify directly (none->non-present). */
6740				set_huge_pte_at(mm, address, ptep,
6741						make_pte_marker(PTE_MARKER_UFFD_WP));
6742		}
6743		spin_unlock(ptl);
6744	}
6745	/*
6746	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
6747	 * may have cleared our pud entry and done put_page on the page table:
6748	 * once we release i_mmap_rwsem, another task can do the final put_page
6749	 * and that page table be reused and filled with junk.  If we actually
6750	 * did unshare a page of pmds, flush the range corresponding to the pud.
6751	 */
6752	if (shared_pmd)
6753		flush_hugetlb_tlb_range(vma, range.start, range.end);
6754	else
6755		flush_hugetlb_tlb_range(vma, start, end);
6756	/*
6757	 * No need to call mmu_notifier_invalidate_range() we are downgrading
6758	 * page table protection not changing it to point to a new page.
6759	 *
6760	 * See Documentation/mm/mmu_notifier.rst
6761	 */
6762	i_mmap_unlock_write(vma->vm_file->f_mapping);
6763	hugetlb_vma_unlock_write(vma);
6764	mmu_notifier_invalidate_range_end(&range);
6765
6766	return pages << h->order;
6767}
6768
6769/* Return true if reservation was successful, false otherwise.  */
6770bool hugetlb_reserve_pages(struct inode *inode,
6771					long from, long to,
6772					struct vm_area_struct *vma,
6773					vm_flags_t vm_flags)
6774{
6775	long chg, add = -1;
6776	struct hstate *h = hstate_inode(inode);
6777	struct hugepage_subpool *spool = subpool_inode(inode);
6778	struct resv_map *resv_map;
6779	struct hugetlb_cgroup *h_cg = NULL;
6780	long gbl_reserve, regions_needed = 0;
6781
6782	/* This should never happen */
6783	if (from > to) {
6784		VM_WARN(1, "%s called with a negative range\n", __func__);
6785		return false;
6786	}
6787
6788	/*
6789	 * vma specific semaphore used for pmd sharing and fault/truncation
6790	 * synchronization
6791	 */
6792	hugetlb_vma_lock_alloc(vma);
6793
6794	/*
6795	 * Only apply hugepage reservation if asked. At fault time, an
6796	 * attempt will be made for VM_NORESERVE to allocate a page
6797	 * without using reserves
6798	 */
6799	if (vm_flags & VM_NORESERVE)
6800		return true;
6801
6802	/*
6803	 * Shared mappings base their reservation on the number of pages that
6804	 * are already allocated on behalf of the file. Private mappings need
6805	 * to reserve the full area even if read-only as mprotect() may be
6806	 * called to make the mapping read-write. Assume !vma is a shm mapping
6807	 */
6808	if (!vma || vma->vm_flags & VM_MAYSHARE) {
6809		/*
6810		 * resv_map can not be NULL as hugetlb_reserve_pages is only
6811		 * called for inodes for which resv_maps were created (see
6812		 * hugetlbfs_get_inode).
6813		 */
6814		resv_map = inode_resv_map(inode);
6815
6816		chg = region_chg(resv_map, from, to, &regions_needed);
 
6817	} else {
6818		/* Private mapping. */
6819		resv_map = resv_map_alloc();
6820		if (!resv_map)
6821			goto out_err;
6822
6823		chg = to - from;
6824
6825		set_vma_resv_map(vma, resv_map);
6826		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
6827	}
6828
6829	if (chg < 0)
 
6830		goto out_err;
 
6831
6832	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
6833				chg * pages_per_huge_page(h), &h_cg) < 0)
 
 
 
6834		goto out_err;
 
6835
6836	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
6837		/* For private mappings, the hugetlb_cgroup uncharge info hangs
6838		 * of the resv_map.
6839		 */
6840		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
6841	}
6842
6843	/*
6844	 * There must be enough pages in the subpool for the mapping. If
6845	 * the subpool has a minimum size, there may be some global
6846	 * reservations already in place (gbl_reserve).
6847	 */
6848	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
6849	if (gbl_reserve < 0)
 
6850		goto out_uncharge_cgroup;
 
6851
6852	/*
6853	 * Check enough hugepages are available for the reservation.
6854	 * Hand the pages back to the subpool if there are not
6855	 */
6856	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
 
6857		goto out_put_pages;
 
6858
6859	/*
6860	 * Account for the reservations made. Shared mappings record regions
6861	 * that have reservations as they are shared by multiple VMAs.
6862	 * When the last VMA disappears, the region map says how much
6863	 * the reservation was and the page cache tells how much of
6864	 * the reservation was consumed. Private mappings are per-VMA and
6865	 * only the consumed reservations are tracked. When the VMA
6866	 * disappears, the original reservation is the VMA size and the
6867	 * consumed reservations are stored in the map. Hence, nothing
6868	 * else has to be done for private mappings here
6869	 */
6870	if (!vma || vma->vm_flags & VM_MAYSHARE) {
6871		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
6872
6873		if (unlikely(add < 0)) {
6874			hugetlb_acct_memory(h, -gbl_reserve);
6875			goto out_put_pages;
6876		} else if (unlikely(chg > add)) {
6877			/*
6878			 * pages in this range were added to the reserve
6879			 * map between region_chg and region_add.  This
6880			 * indicates a race with alloc_huge_page.  Adjust
6881			 * the subpool and reserve counts modified above
6882			 * based on the difference.
6883			 */
6884			long rsv_adjust;
6885
6886			/*
6887			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
6888			 * reference to h_cg->css. See comment below for detail.
6889			 */
6890			hugetlb_cgroup_uncharge_cgroup_rsvd(
6891				hstate_index(h),
6892				(chg - add) * pages_per_huge_page(h), h_cg);
6893
6894			rsv_adjust = hugepage_subpool_put_pages(spool,
6895								chg - add);
6896			hugetlb_acct_memory(h, -rsv_adjust);
6897		} else if (h_cg) {
6898			/*
6899			 * The file_regions will hold their own reference to
6900			 * h_cg->css. So we should release the reference held
6901			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
6902			 * done.
6903			 */
6904			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
6905		}
6906	}
6907	return true;
6908
6909out_put_pages:
6910	/* put back original number of pages, chg */
6911	(void)hugepage_subpool_put_pages(spool, chg);
6912out_uncharge_cgroup:
6913	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
6914					    chg * pages_per_huge_page(h), h_cg);
6915out_err:
6916	hugetlb_vma_lock_free(vma);
6917	if (!vma || vma->vm_flags & VM_MAYSHARE)
6918		/* Only call region_abort if the region_chg succeeded but the
6919		 * region_add failed or didn't run.
6920		 */
6921		if (chg >= 0 && add < 0)
6922			region_abort(resv_map, from, to, regions_needed);
6923	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
6924		kref_put(&resv_map->refs, resv_map_release);
6925	return false;
6926}
6927
6928long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
6929								long freed)
6930{
6931	struct hstate *h = hstate_inode(inode);
6932	struct resv_map *resv_map = inode_resv_map(inode);
6933	long chg = 0;
6934	struct hugepage_subpool *spool = subpool_inode(inode);
6935	long gbl_reserve;
6936
6937	/*
6938	 * Since this routine can be called in the evict inode path for all
6939	 * hugetlbfs inodes, resv_map could be NULL.
6940	 */
6941	if (resv_map) {
6942		chg = region_del(resv_map, start, end);
6943		/*
6944		 * region_del() can fail in the rare case where a region
6945		 * must be split and another region descriptor can not be
6946		 * allocated.  If end == LONG_MAX, it will not fail.
6947		 */
6948		if (chg < 0)
6949			return chg;
6950	}
6951
6952	spin_lock(&inode->i_lock);
6953	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
6954	spin_unlock(&inode->i_lock);
6955
6956	/*
6957	 * If the subpool has a minimum size, the number of global
6958	 * reservations to be released may be adjusted.
6959	 *
6960	 * Note that !resv_map implies freed == 0. So (chg - freed)
6961	 * won't go negative.
6962	 */
6963	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
6964	hugetlb_acct_memory(h, -gbl_reserve);
6965
6966	return 0;
6967}
6968
6969#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
6970static unsigned long page_table_shareable(struct vm_area_struct *svma,
6971				struct vm_area_struct *vma,
6972				unsigned long addr, pgoff_t idx)
6973{
6974	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
6975				svma->vm_start;
6976	unsigned long sbase = saddr & PUD_MASK;
6977	unsigned long s_end = sbase + PUD_SIZE;
6978
6979	/* Allow segments to share if only one is marked locked */
6980	unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
6981	unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
6982
6983	/*
6984	 * match the virtual addresses, permission and the alignment of the
6985	 * page table page.
6986	 *
6987	 * Also, vma_lock (vm_private_data) is required for sharing.
6988	 */
6989	if (pmd_index(addr) != pmd_index(saddr) ||
6990	    vm_flags != svm_flags ||
6991	    !range_in_vma(svma, sbase, s_end) ||
6992	    !svma->vm_private_data)
6993		return 0;
6994
6995	return saddr;
6996}
6997
6998bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
6999{
7000	unsigned long start = addr & PUD_MASK;
7001	unsigned long end = start + PUD_SIZE;
7002
7003#ifdef CONFIG_USERFAULTFD
7004	if (uffd_disable_huge_pmd_share(vma))
7005		return false;
7006#endif
7007	/*
7008	 * check on proper vm_flags and page table alignment
7009	 */
7010	if (!(vma->vm_flags & VM_MAYSHARE))
7011		return false;
7012	if (!vma->vm_private_data)	/* vma lock required for sharing */
7013		return false;
7014	if (!range_in_vma(vma, start, end))
7015		return false;
7016	return true;
7017}
7018
7019/*
7020 * Determine if start,end range within vma could be mapped by shared pmd.
7021 * If yes, adjust start and end to cover range associated with possible
7022 * shared pmd mappings.
7023 */
7024void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
7025				unsigned long *start, unsigned long *end)
7026{
7027	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
7028		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
7029
7030	/*
7031	 * vma needs to span at least one aligned PUD size, and the range
7032	 * must be at least partially within in.
7033	 */
7034	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
7035		(*end <= v_start) || (*start >= v_end))
7036		return;
7037
7038	/* Extend the range to be PUD aligned for a worst case scenario */
7039	if (*start > v_start)
7040		*start = ALIGN_DOWN(*start, PUD_SIZE);
7041
7042	if (*end < v_end)
7043		*end = ALIGN(*end, PUD_SIZE);
 
 
 
 
7044}
7045
7046/*
7047 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
7048 * and returns the corresponding pte. While this is not necessary for the
7049 * !shared pmd case because we can allocate the pmd later as well, it makes the
7050 * code much cleaner. pmd allocation is essential for the shared case because
7051 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
7052 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
7053 * bad pmd for sharing.
 
 
7054 */
7055pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
7056		      unsigned long addr, pud_t *pud)
7057{
 
7058	struct address_space *mapping = vma->vm_file->f_mapping;
7059	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
7060			vma->vm_pgoff;
7061	struct vm_area_struct *svma;
7062	unsigned long saddr;
7063	pte_t *spte = NULL;
7064	pte_t *pte;
7065	spinlock_t *ptl;
7066
7067	i_mmap_lock_read(mapping);
 
 
7068	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
7069		if (svma == vma)
7070			continue;
7071
7072		saddr = page_table_shareable(svma, vma, addr, idx);
7073		if (saddr) {
7074			spte = huge_pte_offset(svma->vm_mm, saddr,
7075					       vma_mmu_pagesize(svma));
7076			if (spte) {
7077				get_page(virt_to_page(spte));
7078				break;
7079			}
7080		}
7081	}
7082
7083	if (!spte)
7084		goto out;
7085
7086	ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
7087	if (pud_none(*pud)) {
7088		pud_populate(mm, pud,
7089				(pmd_t *)((unsigned long)spte & PAGE_MASK));
7090		mm_inc_nr_pmds(mm);
7091	} else {
7092		put_page(virt_to_page(spte));
7093	}
7094	spin_unlock(ptl);
7095out:
7096	pte = (pte_t *)pmd_alloc(mm, pud, addr);
7097	i_mmap_unlock_read(mapping);
7098	return pte;
7099}
7100
7101/*
7102 * unmap huge page backed by shared pte.
7103 *
7104 * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
7105 * indicated by page_count > 1, unmap is achieved by clearing pud and
7106 * decrementing the ref count. If count == 1, the pte page is not shared.
7107 *
7108 * Called with page table lock held.
7109 *
7110 * returns: 1 successfully unmapped a shared pte page
7111 *	    0 the underlying pte page is not shared, or it is the last user
7112 */
7113int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7114					unsigned long addr, pte_t *ptep)
7115{
7116	pgd_t *pgd = pgd_offset(mm, addr);
7117	p4d_t *p4d = p4d_offset(pgd, addr);
7118	pud_t *pud = pud_offset(p4d, addr);
7119
7120	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
7121	hugetlb_vma_assert_locked(vma);
7122	BUG_ON(page_count(virt_to_page(ptep)) == 0);
7123	if (page_count(virt_to_page(ptep)) == 1)
7124		return 0;
7125
7126	pud_clear(pud);
7127	put_page(virt_to_page(ptep));
7128	mm_dec_nr_pmds(mm);
 
7129	return 1;
7130}
7131
7132#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
7133
7134pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
7135		      unsigned long addr, pud_t *pud)
7136{
7137	return NULL;
7138}
7139
7140int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7141				unsigned long addr, pte_t *ptep)
7142{
7143	return 0;
7144}
7145
7146void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
7147				unsigned long *start, unsigned long *end)
7148{
7149}
7150
7151bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
7152{
7153	return false;
7154}
7155#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
7156
7157#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
7158pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
7159			unsigned long addr, unsigned long sz)
7160{
7161	pgd_t *pgd;
7162	p4d_t *p4d;
7163	pud_t *pud;
7164	pte_t *pte = NULL;
7165
7166	pgd = pgd_offset(mm, addr);
7167	p4d = p4d_alloc(mm, pgd, addr);
7168	if (!p4d)
7169		return NULL;
7170	pud = pud_alloc(mm, p4d, addr);
7171	if (pud) {
7172		if (sz == PUD_SIZE) {
7173			pte = (pte_t *)pud;
7174		} else {
7175			BUG_ON(sz != PMD_SIZE);
7176			if (want_pmd_share(vma, addr) && pud_none(*pud))
7177				pte = huge_pmd_share(mm, vma, addr, pud);
7178			else
7179				pte = (pte_t *)pmd_alloc(mm, pud, addr);
7180		}
7181	}
7182	BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
7183
7184	return pte;
7185}
7186
7187/*
7188 * huge_pte_offset() - Walk the page table to resolve the hugepage
7189 * entry at address @addr
7190 *
7191 * Return: Pointer to page table entry (PUD or PMD) for
7192 * address @addr, or NULL if a !p*d_present() entry is encountered and the
7193 * size @sz doesn't match the hugepage size at this level of the page
7194 * table.
7195 */
7196pte_t *huge_pte_offset(struct mm_struct *mm,
7197		       unsigned long addr, unsigned long sz)
7198{
7199	pgd_t *pgd;
7200	p4d_t *p4d;
7201	pud_t *pud;
7202	pmd_t *pmd;
7203
7204	pgd = pgd_offset(mm, addr);
7205	if (!pgd_present(*pgd))
7206		return NULL;
7207	p4d = p4d_offset(pgd, addr);
7208	if (!p4d_present(*p4d))
7209		return NULL;
7210
7211	pud = pud_offset(p4d, addr);
7212	if (sz == PUD_SIZE)
7213		/* must be pud huge, non-present or none */
7214		return (pte_t *)pud;
7215	if (!pud_present(*pud))
7216		return NULL;
7217	/* must have a valid entry and size to go further */
7218
7219	pmd = pmd_offset(pud, addr);
7220	/* must be pmd huge, non-present or none */
7221	return (pte_t *)pmd;
7222}
7223
7224/*
7225 * Return a mask that can be used to update an address to the last huge
7226 * page in a page table page mapping size.  Used to skip non-present
7227 * page table entries when linearly scanning address ranges.  Architectures
7228 * with unique huge page to page table relationships can define their own
7229 * version of this routine.
7230 */
7231unsigned long hugetlb_mask_last_page(struct hstate *h)
7232{
7233	unsigned long hp_size = huge_page_size(h);
7234
7235	if (hp_size == PUD_SIZE)
7236		return P4D_SIZE - PUD_SIZE;
7237	else if (hp_size == PMD_SIZE)
7238		return PUD_SIZE - PMD_SIZE;
7239	else
7240		return 0UL;
7241}
7242
7243#else
7244
7245/* See description above.  Architectures can provide their own version. */
7246__weak unsigned long hugetlb_mask_last_page(struct hstate *h)
7247{
7248#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
7249	if (huge_page_size(h) == PMD_SIZE)
7250		return PUD_SIZE - PMD_SIZE;
7251#endif
7252	return 0UL;
7253}
7254
7255#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
7256
7257/*
7258 * These functions are overwritable if your architecture needs its own
7259 * behavior.
7260 */
7261int isolate_hugetlb(struct page *page, struct list_head *list)
 
 
7262{
7263	int ret = 0;
 
7264
7265	spin_lock_irq(&hugetlb_lock);
7266	if (!PageHeadHuge(page) ||
7267	    !HPageMigratable(page) ||
7268	    !get_page_unless_zero(page)) {
7269		ret = -EBUSY;
7270		goto unlock;
7271	}
7272	ClearHPageMigratable(page);
7273	list_move_tail(&page->lru, list);
7274unlock:
7275	spin_unlock_irq(&hugetlb_lock);
7276	return ret;
7277}
7278
7279int get_hwpoison_huge_page(struct page *page, bool *hugetlb, bool unpoison)
 
 
7280{
7281	int ret = 0;
 
 
7282
7283	*hugetlb = false;
7284	spin_lock_irq(&hugetlb_lock);
7285	if (PageHeadHuge(page)) {
7286		*hugetlb = true;
7287		if (HPageFreed(page))
7288			ret = 0;
7289		else if (HPageMigratable(page) || unpoison)
7290			ret = get_page_unless_zero(page);
7291		else
7292			ret = -EBUSY;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7293	}
7294	spin_unlock_irq(&hugetlb_lock);
7295	return ret;
 
7296}
7297
7298int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
7299				bool *migratable_cleared)
 
7300{
7301	int ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7302
7303	spin_lock_irq(&hugetlb_lock);
7304	ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared);
7305	spin_unlock_irq(&hugetlb_lock);
 
 
 
 
 
 
 
7306	return ret;
7307}
7308
7309void putback_active_hugepage(struct page *page)
7310{
7311	spin_lock_irq(&hugetlb_lock);
7312	SetHPageMigratable(page);
 
7313	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
7314	spin_unlock_irq(&hugetlb_lock);
7315	put_page(page);
7316}
7317
7318void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason)
7319{
7320	struct hstate *h = folio_hstate(old_folio);
7321
7322	hugetlb_cgroup_migrate(old_folio, new_folio);
7323	set_page_owner_migrate_reason(&new_folio->page, reason);
7324
7325	/*
7326	 * transfer temporary state of the new hugetlb folio. This is
7327	 * reverse to other transitions because the newpage is going to
7328	 * be final while the old one will be freed so it takes over
7329	 * the temporary status.
7330	 *
7331	 * Also note that we have to transfer the per-node surplus state
7332	 * here as well otherwise the global surplus count will not match
7333	 * the per-node's.
7334	 */
7335	if (folio_test_hugetlb_temporary(new_folio)) {
7336		int old_nid = folio_nid(old_folio);
7337		int new_nid = folio_nid(new_folio);
7338
7339		folio_set_hugetlb_temporary(old_folio);
7340		folio_clear_hugetlb_temporary(new_folio);
7341
 
 
7342
7343		/*
7344		 * There is no need to transfer the per-node surplus state
7345		 * when we do not cross the node.
7346		 */
7347		if (new_nid == old_nid)
7348			return;
7349		spin_lock_irq(&hugetlb_lock);
7350		if (h->surplus_huge_pages_node[old_nid]) {
7351			h->surplus_huge_pages_node[old_nid]--;
7352			h->surplus_huge_pages_node[new_nid]++;
7353		}
7354		spin_unlock_irq(&hugetlb_lock);
7355	}
7356}
7357
7358static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
7359				   unsigned long start,
7360				   unsigned long end)
7361{
7362	struct hstate *h = hstate_vma(vma);
7363	unsigned long sz = huge_page_size(h);
7364	struct mm_struct *mm = vma->vm_mm;
7365	struct mmu_notifier_range range;
7366	unsigned long address;
7367	spinlock_t *ptl;
7368	pte_t *ptep;
7369
7370	if (!(vma->vm_flags & VM_MAYSHARE))
7371		return;
7372
7373	if (start >= end)
7374		return;
7375
7376	flush_cache_range(vma, start, end);
7377	/*
7378	 * No need to call adjust_range_if_pmd_sharing_possible(), because
7379	 * we have already done the PUD_SIZE alignment.
7380	 */
7381	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
7382				start, end);
7383	mmu_notifier_invalidate_range_start(&range);
7384	hugetlb_vma_lock_write(vma);
7385	i_mmap_lock_write(vma->vm_file->f_mapping);
7386	for (address = start; address < end; address += PUD_SIZE) {
7387		ptep = huge_pte_offset(mm, address, sz);
7388		if (!ptep)
7389			continue;
7390		ptl = huge_pte_lock(h, mm, ptep);
7391		huge_pmd_unshare(mm, vma, address, ptep);
7392		spin_unlock(ptl);
7393	}
7394	flush_hugetlb_tlb_range(vma, start, end);
7395	i_mmap_unlock_write(vma->vm_file->f_mapping);
7396	hugetlb_vma_unlock_write(vma);
7397	/*
7398	 * No need to call mmu_notifier_invalidate_range(), see
7399	 * Documentation/mm/mmu_notifier.rst.
7400	 */
7401	mmu_notifier_invalidate_range_end(&range);
7402}
7403
7404/*
7405 * This function will unconditionally remove all the shared pmd pgtable entries
7406 * within the specific vma for a hugetlbfs memory range.
7407 */
7408void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
7409{
7410	hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE),
7411			ALIGN_DOWN(vma->vm_end, PUD_SIZE));
7412}
7413
7414#ifdef CONFIG_CMA
7415static bool cma_reserve_called __initdata;
7416
7417static int __init cmdline_parse_hugetlb_cma(char *p)
7418{
7419	int nid, count = 0;
7420	unsigned long tmp;
7421	char *s = p;
7422
7423	while (*s) {
7424		if (sscanf(s, "%lu%n", &tmp, &count) != 1)
7425			break;
7426
7427		if (s[count] == ':') {
7428			if (tmp >= MAX_NUMNODES)
7429				break;
7430			nid = array_index_nospec(tmp, MAX_NUMNODES);
7431
7432			s += count + 1;
7433			tmp = memparse(s, &s);
7434			hugetlb_cma_size_in_node[nid] = tmp;
7435			hugetlb_cma_size += tmp;
7436
7437			/*
7438			 * Skip the separator if have one, otherwise
7439			 * break the parsing.
7440			 */
7441			if (*s == ',')
7442				s++;
7443			else
7444				break;
7445		} else {
7446			hugetlb_cma_size = memparse(p, &p);
7447			break;
7448		}
7449	}
7450
7451	return 0;
7452}
7453
7454early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
7455
7456void __init hugetlb_cma_reserve(int order)
7457{
7458	unsigned long size, reserved, per_node;
7459	bool node_specific_cma_alloc = false;
7460	int nid;
7461
7462	cma_reserve_called = true;
7463
7464	if (!hugetlb_cma_size)
7465		return;
7466
7467	for (nid = 0; nid < MAX_NUMNODES; nid++) {
7468		if (hugetlb_cma_size_in_node[nid] == 0)
7469			continue;
7470
7471		if (!node_online(nid)) {
7472			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
7473			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7474			hugetlb_cma_size_in_node[nid] = 0;
7475			continue;
7476		}
7477
7478		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
7479			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
7480				nid, (PAGE_SIZE << order) / SZ_1M);
7481			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7482			hugetlb_cma_size_in_node[nid] = 0;
7483		} else {
7484			node_specific_cma_alloc = true;
7485		}
7486	}
7487
7488	/* Validate the CMA size again in case some invalid nodes specified. */
7489	if (!hugetlb_cma_size)
7490		return;
7491
7492	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
7493		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
7494			(PAGE_SIZE << order) / SZ_1M);
7495		hugetlb_cma_size = 0;
7496		return;
7497	}
7498
7499	if (!node_specific_cma_alloc) {
7500		/*
7501		 * If 3 GB area is requested on a machine with 4 numa nodes,
7502		 * let's allocate 1 GB on first three nodes and ignore the last one.
7503		 */
7504		per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
7505		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
7506			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
7507	}
7508
7509	reserved = 0;
7510	for_each_online_node(nid) {
7511		int res;
7512		char name[CMA_MAX_NAME];
7513
7514		if (node_specific_cma_alloc) {
7515			if (hugetlb_cma_size_in_node[nid] == 0)
7516				continue;
7517
7518			size = hugetlb_cma_size_in_node[nid];
7519		} else {
7520			size = min(per_node, hugetlb_cma_size - reserved);
7521		}
7522
 
7523		size = round_up(size, PAGE_SIZE << order);
7524
7525		snprintf(name, sizeof(name), "hugetlb%d", nid);
7526		/*
7527		 * Note that 'order per bit' is based on smallest size that
7528		 * may be returned to CMA allocator in the case of
7529		 * huge page demotion.
7530		 */
7531		res = cma_declare_contiguous_nid(0, size, 0,
7532						PAGE_SIZE << HUGETLB_PAGE_ORDER,
7533						 0, false, name,
7534						 &hugetlb_cma[nid], nid);
7535		if (res) {
7536			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
7537				res, nid);
7538			continue;
7539		}
7540
7541		reserved += size;
7542		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
7543			size / SZ_1M, nid);
7544
7545		if (reserved >= hugetlb_cma_size)
7546			break;
7547	}
7548
7549	if (!reserved)
7550		/*
7551		 * hugetlb_cma_size is used to determine if allocations from
7552		 * cma are possible.  Set to zero if no cma regions are set up.
7553		 */
7554		hugetlb_cma_size = 0;
7555}
7556
7557static void __init hugetlb_cma_check(void)
7558{
7559	if (!hugetlb_cma_size || cma_reserve_called)
7560		return;
7561
7562	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
7563}
7564
7565#endif /* CONFIG_CMA */