1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/mm/memory.c
   4 *
   5 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   6 */
   7
   8/*
   9 * demand-loading started 01.12.91 - seems it is high on the list of
  10 * things wanted, and it should be easy to implement. - Linus
  11 */
  12
  13/*
  14 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
  15 * pages started 02.12.91, seems to work. - Linus.
  16 *
  17 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
  18 * would have taken more than the 6M I have free, but it worked well as
  19 * far as I could see.
  20 *
  21 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
  22 */
  23
  24/*
  25 * Real VM (paging to/from disk) started 18.12.91. Much more work and
  26 * thought has to go into this. Oh, well..
  27 * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
  28 *		Found it. Everything seems to work now.
  29 * 20.12.91  -  Ok, making the swap-device changeable like the root.
  30 */
  31
  32/*
  33 * 05.04.94  -  Multi-page memory management added for v1.1.
  34 *              Idea by Alex Bligh (alex@cconcepts.co.uk)
  35 *
  36 * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
  37 *		(Gerhard.Wichert@pdb.siemens.de)
  38 *
  39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
  40 */
  41
  42#include <linux/kernel_stat.h>
  43#include <linux/mm.h>
  44#include <linux/mm_inline.h>
  45#include <linux/sched/mm.h>
 
  46#include <linux/sched/numa_balancing.h>
  47#include <linux/sched/task.h>
  48#include <linux/hugetlb.h>
  49#include <linux/mman.h>
  50#include <linux/swap.h>
  51#include <linux/highmem.h>
  52#include <linux/pagemap.h>
  53#include <linux/memremap.h>
  54#include <linux/kmsan.h>
  55#include <linux/ksm.h>
  56#include <linux/rmap.h>
  57#include <linux/export.h>
  58#include <linux/delayacct.h>
  59#include <linux/init.h>
  60#include <linux/pfn_t.h>
  61#include <linux/writeback.h>
  62#include <linux/memcontrol.h>
  63#include <linux/mmu_notifier.h>
  64#include <linux/swapops.h>
  65#include <linux/elf.h>
  66#include <linux/gfp.h>
  67#include <linux/migrate.h>
  68#include <linux/string.h>
  69#include <linux/memory-tiers.h>
  70#include <linux/debugfs.h>
  71#include <linux/userfaultfd_k.h>
  72#include <linux/dax.h>
  73#include <linux/oom.h>
  74#include <linux/numa.h>
  75#include <linux/perf_event.h>
  76#include <linux/ptrace.h>
  77#include <linux/vmalloc.h>
  78#include <linux/sched/sysctl.h>
  79
  80#include <trace/events/kmem.h>
  81
  82#include <asm/io.h>
  83#include <asm/mmu_context.h>
  84#include <asm/pgalloc.h>
  85#include <linux/uaccess.h>
  86#include <asm/tlb.h>
  87#include <asm/tlbflush.h>
  88
  89#include "pgalloc-track.h"
  90#include "internal.h"
  91#include "swap.h"
  92
  93#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
  94#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
  95#endif
  96
  97#ifndef CONFIG_NUMA
  98unsigned long max_mapnr;
  99EXPORT_SYMBOL(max_mapnr);
 100
 101struct page *mem_map;
 102EXPORT_SYMBOL(mem_map);
 103#endif
 104
 105static vm_fault_t do_fault(struct vm_fault *vmf);
 106static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
 107static bool vmf_pte_changed(struct vm_fault *vmf);
 108
 109/*
 110 * Return true if the original pte was a uffd-wp pte marker (so the pte was
 111 * wr-protected).
 112 */
 113static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
 114{
 115	if (!userfaultfd_wp(vmf->vma))
 116		return false;
 117	if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
 118		return false;
 119
 120	return pte_marker_uffd_wp(vmf->orig_pte);
 121}
 122
 123/*
 124 * A number of key systems in x86 including ioremap() rely on the assumption
 125 * that high_memory defines the upper bound on direct map memory, then end
 126 * of ZONE_NORMAL.
 
 
 127 */
 128void *high_memory;
 129EXPORT_SYMBOL(high_memory);
 130
 131/*
 132 * Randomize the address space (stacks, mmaps, brk, etc.).
 133 *
 134 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
 135 *   as ancient (libc5 based) binaries can segfault. )
 136 */
 137int randomize_va_space __read_mostly =
 138#ifdef CONFIG_COMPAT_BRK
 139					1;
 140#else
 141					2;
 142#endif
 143
 
 
 
 
 
 
 
 
 
 
 
 
 144#ifndef arch_wants_old_prefaulted_pte
 145static inline bool arch_wants_old_prefaulted_pte(void)
 146{
 147	/*
 148	 * Transitioning a PTE from 'old' to 'young' can be expensive on
 149	 * some architectures, even if it's performed in hardware. By
 150	 * default, "false" means prefaulted entries will be 'young'.
 151	 */
 152	return false;
 153}
 154#endif
 155
 156static int __init disable_randmaps(char *s)
 157{
 158	randomize_va_space = 0;
 159	return 1;
 160}
 161__setup("norandmaps", disable_randmaps);
 162
 163unsigned long zero_pfn __read_mostly;
 164EXPORT_SYMBOL(zero_pfn);
 165
 166unsigned long highest_memmap_pfn __read_mostly;
 167
 168/*
 169 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
 170 */
 171static int __init init_zero_pfn(void)
 172{
 173	zero_pfn = page_to_pfn(ZERO_PAGE(0));
 174	return 0;
 175}
 176early_initcall(init_zero_pfn);
 177
 178void mm_trace_rss_stat(struct mm_struct *mm, int member)
 179{
 180	trace_rss_stat(mm, member);
 181}
 182
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 183/*
 184 * Note: this doesn't free the actual pages themselves. That
 185 * has been handled earlier when unmapping all the memory regions.
 186 */
 187static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
 188			   unsigned long addr)
 189{
 190	pgtable_t token = pmd_pgtable(*pmd);
 191	pmd_clear(pmd);
 192	pte_free_tlb(tlb, token, addr);
 193	mm_dec_nr_ptes(tlb->mm);
 194}
 195
 196static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 197				unsigned long addr, unsigned long end,
 198				unsigned long floor, unsigned long ceiling)
 199{
 200	pmd_t *pmd;
 201	unsigned long next;
 202	unsigned long start;
 203
 204	start = addr;
 205	pmd = pmd_offset(pud, addr);
 206	do {
 207		next = pmd_addr_end(addr, end);
 208		if (pmd_none_or_clear_bad(pmd))
 209			continue;
 210		free_pte_range(tlb, pmd, addr);
 211	} while (pmd++, addr = next, addr != end);
 212
 213	start &= PUD_MASK;
 214	if (start < floor)
 215		return;
 216	if (ceiling) {
 217		ceiling &= PUD_MASK;
 218		if (!ceiling)
 219			return;
 220	}
 221	if (end - 1 > ceiling - 1)
 222		return;
 223
 224	pmd = pmd_offset(pud, start);
 225	pud_clear(pud);
 226	pmd_free_tlb(tlb, pmd, start);
 227	mm_dec_nr_pmds(tlb->mm);
 228}
 229
 230static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
 231				unsigned long addr, unsigned long end,
 232				unsigned long floor, unsigned long ceiling)
 233{
 234	pud_t *pud;
 235	unsigned long next;
 236	unsigned long start;
 237
 238	start = addr;
 239	pud = pud_offset(p4d, addr);
 240	do {
 241		next = pud_addr_end(addr, end);
 242		if (pud_none_or_clear_bad(pud))
 243			continue;
 244		free_pmd_range(tlb, pud, addr, next, floor, ceiling);
 245	} while (pud++, addr = next, addr != end);
 246
 247	start &= P4D_MASK;
 248	if (start < floor)
 249		return;
 250	if (ceiling) {
 251		ceiling &= P4D_MASK;
 252		if (!ceiling)
 253			return;
 254	}
 255	if (end - 1 > ceiling - 1)
 256		return;
 257
 258	pud = pud_offset(p4d, start);
 259	p4d_clear(p4d);
 260	pud_free_tlb(tlb, pud, start);
 261	mm_dec_nr_puds(tlb->mm);
 262}
 263
 264static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
 265				unsigned long addr, unsigned long end,
 266				unsigned long floor, unsigned long ceiling)
 267{
 268	p4d_t *p4d;
 269	unsigned long next;
 270	unsigned long start;
 271
 272	start = addr;
 273	p4d = p4d_offset(pgd, addr);
 274	do {
 275		next = p4d_addr_end(addr, end);
 276		if (p4d_none_or_clear_bad(p4d))
 277			continue;
 278		free_pud_range(tlb, p4d, addr, next, floor, ceiling);
 279	} while (p4d++, addr = next, addr != end);
 280
 281	start &= PGDIR_MASK;
 282	if (start < floor)
 283		return;
 284	if (ceiling) {
 285		ceiling &= PGDIR_MASK;
 286		if (!ceiling)
 287			return;
 288	}
 289	if (end - 1 > ceiling - 1)
 290		return;
 291
 292	p4d = p4d_offset(pgd, start);
 293	pgd_clear(pgd);
 294	p4d_free_tlb(tlb, p4d, start);
 295}
 296
 297/*
 298 * This function frees user-level page tables of a process.
 299 */
 300void free_pgd_range(struct mmu_gather *tlb,
 301			unsigned long addr, unsigned long end,
 302			unsigned long floor, unsigned long ceiling)
 303{
 304	pgd_t *pgd;
 305	unsigned long next;
 306
 307	/*
 308	 * The next few lines have given us lots of grief...
 309	 *
 310	 * Why are we testing PMD* at this top level?  Because often
 311	 * there will be no work to do at all, and we'd prefer not to
 312	 * go all the way down to the bottom just to discover that.
 313	 *
 314	 * Why all these "- 1"s?  Because 0 represents both the bottom
 315	 * of the address space and the top of it (using -1 for the
 316	 * top wouldn't help much: the masks would do the wrong thing).
 317	 * The rule is that addr 0 and floor 0 refer to the bottom of
 318	 * the address space, but end 0 and ceiling 0 refer to the top
 319	 * Comparisons need to use "end - 1" and "ceiling - 1" (though
 320	 * that end 0 case should be mythical).
 321	 *
 322	 * Wherever addr is brought up or ceiling brought down, we must
 323	 * be careful to reject "the opposite 0" before it confuses the
 324	 * subsequent tests.  But what about where end is brought down
 325	 * by PMD_SIZE below? no, end can't go down to 0 there.
 326	 *
 327	 * Whereas we round start (addr) and ceiling down, by different
 328	 * masks at different levels, in order to test whether a table
 329	 * now has no other vmas using it, so can be freed, we don't
 330	 * bother to round floor or end up - the tests don't need that.
 331	 */
 332
 333	addr &= PMD_MASK;
 334	if (addr < floor) {
 335		addr += PMD_SIZE;
 336		if (!addr)
 337			return;
 338	}
 339	if (ceiling) {
 340		ceiling &= PMD_MASK;
 341		if (!ceiling)
 342			return;
 343	}
 344	if (end - 1 > ceiling - 1)
 345		end -= PMD_SIZE;
 346	if (addr > end - 1)
 347		return;
 348	/*
 349	 * We add page table cache pages with PAGE_SIZE,
 350	 * (see pte_free_tlb()), flush the tlb if we need
 351	 */
 352	tlb_change_page_size(tlb, PAGE_SIZE);
 353	pgd = pgd_offset(tlb->mm, addr);
 354	do {
 355		next = pgd_addr_end(addr, end);
 356		if (pgd_none_or_clear_bad(pgd))
 357			continue;
 358		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
 359	} while (pgd++, addr = next, addr != end);
 360}
 361
 362void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
 363		   struct vm_area_struct *vma, unsigned long floor,
 364		   unsigned long ceiling, bool mm_wr_locked)
 365{
 366	struct unlink_vma_file_batch vb;
 367
 368	do {
 369		unsigned long addr = vma->vm_start;
 370		struct vm_area_struct *next;
 371
 372		/*
 373		 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
 374		 * be 0.  This will underflow and is okay.
 375		 */
 376		next = mas_find(mas, ceiling - 1);
 377		if (unlikely(xa_is_zero(next)))
 378			next = NULL;
 379
 380		/*
 381		 * Hide vma from rmap and truncate_pagecache before freeing
 382		 * pgtables
 383		 */
 384		if (mm_wr_locked)
 385			vma_start_write(vma);
 386		unlink_anon_vmas(vma);
 
 387
 388		if (is_vm_hugetlb_page(vma)) {
 389			unlink_file_vma(vma);
 390			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
 391				floor, next ? next->vm_start : ceiling);
 392		} else {
 393			unlink_file_vma_batch_init(&vb);
 394			unlink_file_vma_batch_add(&vb, vma);
 395
 396			/*
 397			 * Optimization: gather nearby vmas into one call down
 398			 */
 399			while (next && next->vm_start <= vma->vm_end + PMD_SIZE
 400			       && !is_vm_hugetlb_page(next)) {
 401				vma = next;
 402				next = mas_find(mas, ceiling - 1);
 403				if (unlikely(xa_is_zero(next)))
 404					next = NULL;
 405				if (mm_wr_locked)
 406					vma_start_write(vma);
 407				unlink_anon_vmas(vma);
 408				unlink_file_vma_batch_add(&vb, vma);
 409			}
 410			unlink_file_vma_batch_final(&vb);
 411			free_pgd_range(tlb, addr, vma->vm_end,
 412				floor, next ? next->vm_start : ceiling);
 413		}
 414		vma = next;
 415	} while (vma);
 416}
 417
 418void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
 419{
 420	spinlock_t *ptl = pmd_lock(mm, pmd);
 421
 422	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
 423		mm_inc_nr_ptes(mm);
 424		/*
 425		 * Ensure all pte setup (eg. pte page lock and page clearing) are
 426		 * visible before the pte is made visible to other CPUs by being
 427		 * put into page tables.
 428		 *
 429		 * The other side of the story is the pointer chasing in the page
 430		 * table walking code (when walking the page table without locking;
 431		 * ie. most of the time). Fortunately, these data accesses consist
 432		 * of a chain of data-dependent loads, meaning most CPUs (alpha
 433		 * being the notable exception) will already guarantee loads are
 434		 * seen in-order. See the alpha page table accessors for the
 435		 * smp_rmb() barriers in page table walking code.
 436		 */
 437		smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
 438		pmd_populate(mm, pmd, *pte);
 439		*pte = NULL;
 440	}
 441	spin_unlock(ptl);
 442}
 443
 444int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
 445{
 
 446	pgtable_t new = pte_alloc_one(mm);
 447	if (!new)
 448		return -ENOMEM;
 449
 450	pmd_install(mm, pmd, &new);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 451	if (new)
 452		pte_free(mm, new);
 453	return 0;
 454}
 455
 456int __pte_alloc_kernel(pmd_t *pmd)
 457{
 458	pte_t *new = pte_alloc_one_kernel(&init_mm);
 459	if (!new)
 460		return -ENOMEM;
 461
 
 
 462	spin_lock(&init_mm.page_table_lock);
 463	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
 464		smp_wmb(); /* See comment in pmd_install() */
 465		pmd_populate_kernel(&init_mm, pmd, new);
 466		new = NULL;
 467	}
 468	spin_unlock(&init_mm.page_table_lock);
 469	if (new)
 470		pte_free_kernel(&init_mm, new);
 471	return 0;
 472}
 473
 474static inline void init_rss_vec(int *rss)
 475{
 476	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
 477}
 478
 479static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
 480{
 481	int i;
 482
 
 
 483	for (i = 0; i < NR_MM_COUNTERS; i++)
 484		if (rss[i])
 485			add_mm_counter(mm, i, rss[i]);
 486}
 487
 488/*
 489 * This function is called to print an error when a bad pte
 490 * is found. For example, we might have a PFN-mapped pte in
 491 * a region that doesn't allow it.
 492 *
 493 * The calling function must still handle the error.
 494 */
 495static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
 496			  pte_t pte, struct page *page)
 497{
 498	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
 499	p4d_t *p4d = p4d_offset(pgd, addr);
 500	pud_t *pud = pud_offset(p4d, addr);
 501	pmd_t *pmd = pmd_offset(pud, addr);
 502	struct address_space *mapping;
 503	pgoff_t index;
 504	static unsigned long resume;
 505	static unsigned long nr_shown;
 506	static unsigned long nr_unshown;
 507
 508	/*
 509	 * Allow a burst of 60 reports, then keep quiet for that minute;
 510	 * or allow a steady drip of one report per second.
 511	 */
 512	if (nr_shown == 60) {
 513		if (time_before(jiffies, resume)) {
 514			nr_unshown++;
 515			return;
 516		}
 517		if (nr_unshown) {
 518			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
 519				 nr_unshown);
 520			nr_unshown = 0;
 521		}
 522		nr_shown = 0;
 523	}
 524	if (nr_shown++ == 0)
 525		resume = jiffies + 60 * HZ;
 526
 527	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
 528	index = linear_page_index(vma, addr);
 529
 530	pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
 531		 current->comm,
 532		 (long long)pte_val(pte), (long long)pmd_val(*pmd));
 533	if (page)
 534		dump_page(page, "bad pte");
 535	pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
 536		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
 537	pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
 538		 vma->vm_file,
 539		 vma->vm_ops ? vma->vm_ops->fault : NULL,
 540		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
 541		 mapping ? mapping->a_ops->read_folio : NULL);
 542	dump_stack();
 543	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 544}
 545
 546/*
 547 * vm_normal_page -- This function gets the "struct page" associated with a pte.
 548 *
 549 * "Special" mappings do not wish to be associated with a "struct page" (either
 550 * it doesn't exist, or it exists but they don't want to touch it). In this
 551 * case, NULL is returned here. "Normal" mappings do have a struct page.
 552 *
 553 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
 554 * pte bit, in which case this function is trivial. Secondly, an architecture
 555 * may not have a spare pte bit, which requires a more complicated scheme,
 556 * described below.
 557 *
 558 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
 559 * special mapping (even if there are underlying and valid "struct pages").
 560 * COWed pages of a VM_PFNMAP are always normal.
 561 *
 562 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
 563 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
 564 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
 565 * mapping will always honor the rule
 566 *
 567 *	pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
 568 *
 569 * And for normal mappings this is false.
 570 *
 571 * This restricts such mappings to be a linear translation from virtual address
 572 * to pfn. To get around this restriction, we allow arbitrary mappings so long
 573 * as the vma is not a COW mapping; in that case, we know that all ptes are
 574 * special (because none can have been COWed).
 575 *
 576 *
 577 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
 578 *
 579 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
 580 * page" backing, however the difference is that _all_ pages with a struct
 581 * page (that is, those where pfn_valid is true) are refcounted and considered
 582 * normal pages by the VM. The only exception are zeropages, which are
 583 * *never* refcounted.
 584 *
 585 * The disadvantage is that pages are refcounted (which can be slower and
 586 * simply not an option for some PFNMAP users). The advantage is that we
 587 * don't have to follow the strict linearity rule of PFNMAP mappings in
 588 * order to support COWable mappings.
 589 *
 590 */
 591struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
 592			    pte_t pte)
 593{
 594	unsigned long pfn = pte_pfn(pte);
 595
 596	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
 597		if (likely(!pte_special(pte)))
 598			goto check_pfn;
 599		if (vma->vm_ops && vma->vm_ops->find_special_page)
 600			return vma->vm_ops->find_special_page(vma, addr);
 601		if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
 602			return NULL;
 603		if (is_zero_pfn(pfn))
 604			return NULL;
 605		if (pte_devmap(pte))
 606		/*
 607		 * NOTE: New users of ZONE_DEVICE will not set pte_devmap()
 608		 * and will have refcounts incremented on their struct pages
 609		 * when they are inserted into PTEs, thus they are safe to
 610		 * return here. Legacy ZONE_DEVICE pages that set pte_devmap()
 611		 * do not have refcounts. Example of legacy ZONE_DEVICE is
 612		 * MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
 613		 */
 614			return NULL;
 615
 616		print_bad_pte(vma, addr, pte, NULL);
 617		return NULL;
 618	}
 619
 620	/* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
 621
 622	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
 623		if (vma->vm_flags & VM_MIXEDMAP) {
 624			if (!pfn_valid(pfn))
 625				return NULL;
 626			if (is_zero_pfn(pfn))
 627				return NULL;
 628			goto out;
 629		} else {
 630			unsigned long off;
 631			off = (addr - vma->vm_start) >> PAGE_SHIFT;
 632			if (pfn == vma->vm_pgoff + off)
 633				return NULL;
 634			if (!is_cow_mapping(vma->vm_flags))
 635				return NULL;
 636		}
 637	}
 638
 639	if (is_zero_pfn(pfn))
 640		return NULL;
 641
 642check_pfn:
 643	if (unlikely(pfn > highest_memmap_pfn)) {
 644		print_bad_pte(vma, addr, pte, NULL);
 645		return NULL;
 646	}
 647
 648	/*
 649	 * NOTE! We still have PageReserved() pages in the page tables.
 650	 * eg. VDSO mappings can cause them to exist.
 651	 */
 652out:
 653	VM_WARN_ON_ONCE(is_zero_pfn(pfn));
 654	return pfn_to_page(pfn);
 655}
 656
 657struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
 658			    pte_t pte)
 659{
 660	struct page *page = vm_normal_page(vma, addr, pte);
 661
 662	if (page)
 663		return page_folio(page);
 664	return NULL;
 665}
 666
 667#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
 668struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 669				pmd_t pmd)
 670{
 671	unsigned long pfn = pmd_pfn(pmd);
 672
 673	/* Currently it's only used for huge pfnmaps */
 674	if (unlikely(pmd_special(pmd)))
 675		return NULL;
 676
 
 677	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
 678		if (vma->vm_flags & VM_MIXEDMAP) {
 679			if (!pfn_valid(pfn))
 680				return NULL;
 681			goto out;
 682		} else {
 683			unsigned long off;
 684			off = (addr - vma->vm_start) >> PAGE_SHIFT;
 685			if (pfn == vma->vm_pgoff + off)
 686				return NULL;
 687			if (!is_cow_mapping(vma->vm_flags))
 688				return NULL;
 689		}
 690	}
 691
 692	if (pmd_devmap(pmd))
 693		return NULL;
 694	if (is_huge_zero_pmd(pmd))
 695		return NULL;
 696	if (unlikely(pfn > highest_memmap_pfn))
 697		return NULL;
 698
 699	/*
 700	 * NOTE! We still have PageReserved() pages in the page tables.
 701	 * eg. VDSO mappings can cause them to exist.
 702	 */
 703out:
 704	return pfn_to_page(pfn);
 705}
 706
 707struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
 708				  unsigned long addr, pmd_t pmd)
 709{
 710	struct page *page = vm_normal_page_pmd(vma, addr, pmd);
 711
 712	if (page)
 713		return page_folio(page);
 714	return NULL;
 715}
 716#endif
 717
 718static void restore_exclusive_pte(struct vm_area_struct *vma,
 719				  struct page *page, unsigned long address,
 720				  pte_t *ptep)
 721{
 722	struct folio *folio = page_folio(page);
 723	pte_t orig_pte;
 724	pte_t pte;
 725	swp_entry_t entry;
 726
 727	orig_pte = ptep_get(ptep);
 728	pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
 729	if (pte_swp_soft_dirty(orig_pte))
 730		pte = pte_mksoft_dirty(pte);
 731
 732	entry = pte_to_swp_entry(orig_pte);
 733	if (pte_swp_uffd_wp(orig_pte))
 734		pte = pte_mkuffd_wp(pte);
 735	else if (is_writable_device_exclusive_entry(entry))
 736		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
 737
 738	VM_BUG_ON_FOLIO(pte_write(pte) && (!folio_test_anon(folio) &&
 739					   PageAnonExclusive(page)), folio);
 740
 741	/*
 742	 * No need to take a page reference as one was already
 743	 * created when the swap entry was made.
 744	 */
 745	if (folio_test_anon(folio))
 746		folio_add_anon_rmap_pte(folio, page, vma, address, RMAP_NONE);
 747	else
 748		/*
 749		 * Currently device exclusive access only supports anonymous
 750		 * memory so the entry shouldn't point to a filebacked page.
 751		 */
 752		WARN_ON_ONCE(1);
 753
 754	set_pte_at(vma->vm_mm, address, ptep, pte);
 
 755
 756	/*
 757	 * No need to invalidate - it was non-present before. However
 758	 * secondary CPUs may have mappings that need invalidating.
 759	 */
 760	update_mmu_cache(vma, address, ptep);
 761}
 762
 763/*
 764 * Tries to restore an exclusive pte if the page lock can be acquired without
 765 * sleeping.
 766 */
 767static int
 768try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
 769			unsigned long addr)
 770{
 771	swp_entry_t entry = pte_to_swp_entry(ptep_get(src_pte));
 772	struct page *page = pfn_swap_entry_to_page(entry);
 773
 774	if (trylock_page(page)) {
 775		restore_exclusive_pte(vma, page, addr, src_pte);
 776		unlock_page(page);
 777		return 0;
 778	}
 779
 780	return -EBUSY;
 781}
 782
 783/*
 784 * copy one vm_area from one task to the other. Assumes the page tables
 785 * already present in the new task to be cleared in the whole range
 786 * covered by this vma.
 787 */
 788
 789static unsigned long
 790copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 791		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
 792		struct vm_area_struct *src_vma, unsigned long addr, int *rss)
 793{
 794	unsigned long vm_flags = dst_vma->vm_flags;
 795	pte_t orig_pte = ptep_get(src_pte);
 796	pte_t pte = orig_pte;
 797	struct folio *folio;
 798	struct page *page;
 799	swp_entry_t entry = pte_to_swp_entry(orig_pte);
 800
 801	if (likely(!non_swap_entry(entry))) {
 802		if (swap_duplicate(entry) < 0)
 803			return -EIO;
 804
 805		/* make sure dst_mm is on swapoff's mmlist. */
 806		if (unlikely(list_empty(&dst_mm->mmlist))) {
 807			spin_lock(&mmlist_lock);
 808			if (list_empty(&dst_mm->mmlist))
 809				list_add(&dst_mm->mmlist,
 810						&src_mm->mmlist);
 811			spin_unlock(&mmlist_lock);
 812		}
 813		/* Mark the swap entry as shared. */
 814		if (pte_swp_exclusive(orig_pte)) {
 815			pte = pte_swp_clear_exclusive(orig_pte);
 816			set_pte_at(src_mm, addr, src_pte, pte);
 817		}
 818		rss[MM_SWAPENTS]++;
 819	} else if (is_migration_entry(entry)) {
 820		folio = pfn_swap_entry_folio(entry);
 821
 822		rss[mm_counter(folio)]++;
 823
 824		if (!is_readable_migration_entry(entry) &&
 825				is_cow_mapping(vm_flags)) {
 826			/*
 827			 * COW mappings require pages in both parent and child
 828			 * to be set to read. A previously exclusive entry is
 829			 * now shared.
 830			 */
 831			entry = make_readable_migration_entry(
 832							swp_offset(entry));
 833			pte = swp_entry_to_pte(entry);
 834			if (pte_swp_soft_dirty(orig_pte))
 835				pte = pte_swp_mksoft_dirty(pte);
 836			if (pte_swp_uffd_wp(orig_pte))
 837				pte = pte_swp_mkuffd_wp(pte);
 838			set_pte_at(src_mm, addr, src_pte, pte);
 839		}
 840	} else if (is_device_private_entry(entry)) {
 841		page = pfn_swap_entry_to_page(entry);
 842		folio = page_folio(page);
 843
 844		/*
 845		 * Update rss count even for unaddressable pages, as
 846		 * they should treated just like normal pages in this
 847		 * respect.
 848		 *
 849		 * We will likely want to have some new rss counters
 850		 * for unaddressable pages, at some point. But for now
 851		 * keep things as they are.
 852		 */
 853		folio_get(folio);
 854		rss[mm_counter(folio)]++;
 855		/* Cannot fail as these pages cannot get pinned. */
 856		folio_try_dup_anon_rmap_pte(folio, page, src_vma);
 857
 858		/*
 859		 * We do not preserve soft-dirty information, because so
 860		 * far, checkpoint/restore is the only feature that
 861		 * requires that. And checkpoint/restore does not work
 862		 * when a device driver is involved (you cannot easily
 863		 * save and restore device driver state).
 864		 */
 865		if (is_writable_device_private_entry(entry) &&
 866		    is_cow_mapping(vm_flags)) {
 867			entry = make_readable_device_private_entry(
 868							swp_offset(entry));
 869			pte = swp_entry_to_pte(entry);
 870			if (pte_swp_uffd_wp(orig_pte))
 871				pte = pte_swp_mkuffd_wp(pte);
 872			set_pte_at(src_mm, addr, src_pte, pte);
 873		}
 874	} else if (is_device_exclusive_entry(entry)) {
 875		/*
 876		 * Make device exclusive entries present by restoring the
 877		 * original entry then copying as for a present pte. Device
 878		 * exclusive entries currently only support private writable
 879		 * (ie. COW) mappings.
 880		 */
 881		VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
 882		if (try_restore_exclusive_pte(src_pte, src_vma, addr))
 883			return -EBUSY;
 884		return -ENOENT;
 885	} else if (is_pte_marker_entry(entry)) {
 886		pte_marker marker = copy_pte_marker(entry, dst_vma);
 887
 888		if (marker)
 889			set_pte_at(dst_mm, addr, dst_pte,
 890				   make_pte_marker(marker));
 891		return 0;
 892	}
 893	if (!userfaultfd_wp(dst_vma))
 894		pte = pte_swp_clear_uffd_wp(pte);
 895	set_pte_at(dst_mm, addr, dst_pte, pte);
 896	return 0;
 897}
 898
 899/*
 900 * Copy a present and normal page.
 901 *
 902 * NOTE! The usual case is that this isn't required;
 903 * instead, the caller can just increase the page refcount
 904 * and re-use the pte the traditional way.
 
 
 
 
 
 
 
 
 905 *
 906 * And if we need a pre-allocated page but don't yet have
 907 * one, return a negative error to let the preallocation
 908 * code know so that it can do so outside the page table
 909 * lock.
 910 */
 911static inline int
 912copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
 913		  pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
 914		  struct folio **prealloc, struct page *page)
 915{
 916	struct folio *new_folio;
 917	pte_t pte;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 918
 919	new_folio = *prealloc;
 920	if (!new_folio)
 921		return -EAGAIN;
 922
 923	/*
 924	 * We have a prealloc page, all good!  Take it
 925	 * over and copy the page & arm it.
 926	 */
 927
 928	if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma))
 929		return -EHWPOISON;
 930
 931	*prealloc = NULL;
 932	__folio_mark_uptodate(new_folio);
 933	folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE);
 934	folio_add_lru_vma(new_folio, dst_vma);
 935	rss[MM_ANONPAGES]++;
 
 936
 937	/* All done, just insert the new page copy in the child */
 938	pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
 939	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
 940	if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
 941		/* Uffd-wp needs to be delivered to dest pte as well */
 942		pte = pte_mkuffd_wp(pte);
 943	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
 944	return 0;
 945}
 946
 947static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
 948		struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte,
 949		pte_t pte, unsigned long addr, int nr)
 950{
 951	struct mm_struct *src_mm = src_vma->vm_mm;
 952
 953	/* If it's a COW mapping, write protect it both processes. */
 954	if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) {
 955		wrprotect_ptes(src_mm, addr, src_pte, nr);
 956		pte = pte_wrprotect(pte);
 957	}
 958
 959	/* If it's a shared mapping, mark it clean in the child. */
 960	if (src_vma->vm_flags & VM_SHARED)
 961		pte = pte_mkclean(pte);
 962	pte = pte_mkold(pte);
 963
 964	if (!userfaultfd_wp(dst_vma))
 965		pte = pte_clear_uffd_wp(pte);
 966
 967	set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
 968}
 969
 970/*
 971 * Copy one present PTE, trying to batch-process subsequent PTEs that map
 972 * consecutive pages of the same folio by copying them as well.
 973 *
 974 * Returns -EAGAIN if one preallocated page is required to copy the next PTE.
 975 * Otherwise, returns the number of copied PTEs (at least 1).
 976 */
 977static inline int
 978copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
 979		 pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr,
 980		 int max_nr, int *rss, struct folio **prealloc)
 981{
 
 
 
 982	struct page *page;
 983	struct folio *folio;
 984	bool any_writable;
 985	fpb_t flags = 0;
 986	int err, nr;
 987
 988	page = vm_normal_page(src_vma, addr, pte);
 989	if (unlikely(!page))
 990		goto copy_pte;
 991
 992	folio = page_folio(page);
 
 
 
 993
 994	/*
 995	 * If we likely have to copy, just don't bother with batching. Make
 996	 * sure that the common "small folio" case is as fast as possible
 997	 * by keeping the batching logic separate.
 998	 */
 999	if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {
1000		if (src_vma->vm_flags & VM_SHARED)
1001			flags |= FPB_IGNORE_DIRTY;
1002		if (!vma_soft_dirty_enabled(src_vma))
1003			flags |= FPB_IGNORE_SOFT_DIRTY;
1004
1005		nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,
1006				     &any_writable, NULL, NULL);
1007		folio_ref_add(folio, nr);
1008		if (folio_test_anon(folio)) {
1009			if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
1010								  nr, src_vma))) {
1011				folio_ref_sub(folio, nr);
1012				return -EAGAIN;
1013			}
1014			rss[MM_ANONPAGES] += nr;
1015			VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
1016		} else {
1017			folio_dup_file_rmap_ptes(folio, page, nr);
1018			rss[mm_counter_file(folio)] += nr;
1019		}
1020		if (any_writable)
1021			pte = pte_mkwrite(pte, src_vma);
1022		__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
1023				    addr, nr);
1024		return nr;
1025	}
1026
1027	folio_get(folio);
1028	if (folio_test_anon(folio)) {
1029		/*
1030		 * If this page may have been pinned by the parent process,
1031		 * copy the page immediately for the child so that we'll always
1032		 * guarantee the pinned page won't be randomly replaced in the
1033		 * future.
1034		 */
1035		if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, src_vma))) {
1036			/* Page may be pinned, we have to copy. */
1037			folio_put(folio);
1038			err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
1039						addr, rss, prealloc, page);
1040			return err ? err : 1;
1041		}
1042		rss[MM_ANONPAGES]++;
1043		VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
1044	} else {
1045		folio_dup_file_rmap_pte(folio, page);
1046		rss[mm_counter_file(folio)]++;
1047	}
1048
1049copy_pte:
1050	__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
1051	return 1;
 
 
 
 
 
 
 
 
 
 
1052}
1053
1054static inline struct folio *folio_prealloc(struct mm_struct *src_mm,
1055		struct vm_area_struct *vma, unsigned long addr, bool need_zero)
 
1056{
1057	struct folio *new_folio;
1058
1059	if (need_zero)
1060		new_folio = vma_alloc_zeroed_movable_folio(vma, addr);
1061	else
1062		new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr);
1063
1064	if (!new_folio)
1065		return NULL;
1066
1067	if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
1068		folio_put(new_folio);
1069		return NULL;
1070	}
1071	folio_throttle_swaprate(new_folio, GFP_KERNEL);
1072
1073	return new_folio;
1074}
1075
1076static int
1077copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1078	       pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1079	       unsigned long end)
1080{
1081	struct mm_struct *dst_mm = dst_vma->vm_mm;
1082	struct mm_struct *src_mm = src_vma->vm_mm;
1083	pte_t *orig_src_pte, *orig_dst_pte;
1084	pte_t *src_pte, *dst_pte;
1085	pmd_t dummy_pmdval;
1086	pte_t ptent;
1087	spinlock_t *src_ptl, *dst_ptl;
1088	int progress, max_nr, ret = 0;
1089	int rss[NR_MM_COUNTERS];
1090	swp_entry_t entry = (swp_entry_t){0};
1091	struct folio *prealloc = NULL;
1092	int nr;
1093
1094again:
1095	progress = 0;
1096	init_rss_vec(rss);
1097
1098	/*
1099	 * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
1100	 * error handling here, assume that exclusive mmap_lock on dst and src
1101	 * protects anon from unexpected THP transitions; with shmem and file
1102	 * protected by mmap_lock-less collapse skipping areas with anon_vma
1103	 * (whereas vma_needs_copy() skips areas without anon_vma).  A rework
1104	 * can remove such assumptions later, but this is good enough for now.
1105	 */
1106	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1107	if (!dst_pte) {
1108		ret = -ENOMEM;
1109		goto out;
1110	}
1111
1112	/*
1113	 * We already hold the exclusive mmap_lock, the copy_pte_range() and
1114	 * retract_page_tables() are using vma->anon_vma to be exclusive, so
1115	 * the PTE page is stable, and there is no need to get pmdval and do
1116	 * pmd_same() check.
1117	 */
1118	src_pte = pte_offset_map_rw_nolock(src_mm, src_pmd, addr, &dummy_pmdval,
1119					   &src_ptl);
1120	if (!src_pte) {
1121		pte_unmap_unlock(dst_pte, dst_ptl);
1122		/* ret == 0 */
1123		goto out;
1124	}
1125	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1126	orig_src_pte = src_pte;
1127	orig_dst_pte = dst_pte;
1128	arch_enter_lazy_mmu_mode();
1129
1130	do {
1131		nr = 1;
1132
1133		/*
1134		 * We are holding two locks at this point - either of them
1135		 * could generate latencies in another task on another CPU.
1136		 */
1137		if (progress >= 32) {
1138			progress = 0;
1139			if (need_resched() ||
1140			    spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
1141				break;
1142		}
1143		ptent = ptep_get(src_pte);
1144		if (pte_none(ptent)) {
1145			progress++;
1146			continue;
1147		}
1148		if (unlikely(!pte_present(ptent))) {
1149			ret = copy_nonpresent_pte(dst_mm, src_mm,
1150						  dst_pte, src_pte,
1151						  dst_vma, src_vma,
1152						  addr, rss);
1153			if (ret == -EIO) {
1154				entry = pte_to_swp_entry(ptep_get(src_pte));
1155				break;
1156			} else if (ret == -EBUSY) {
1157				break;
1158			} else if (!ret) {
1159				progress += 8;
1160				continue;
1161			}
1162			ptent = ptep_get(src_pte);
1163			VM_WARN_ON_ONCE(!pte_present(ptent));
1164
1165			/*
1166			 * Device exclusive entry restored, continue by copying
1167			 * the now present pte.
1168			 */
1169			WARN_ON_ONCE(ret != -ENOENT);
1170		}
1171		/* copy_present_ptes() will clear `*prealloc' if consumed */
1172		max_nr = (end - addr) / PAGE_SIZE;
1173		ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
1174					ptent, addr, max_nr, rss, &prealloc);
1175		/*
1176		 * If we need a pre-allocated page for this pte, drop the
1177		 * locks, allocate, and try again.
1178		 * If copy failed due to hwpoison in source page, break out.
1179		 */
1180		if (unlikely(ret == -EAGAIN || ret == -EHWPOISON))
1181			break;
1182		if (unlikely(prealloc)) {
1183			/*
1184			 * pre-alloc page cannot be reused by next time so as
1185			 * to strictly follow mempolicy (e.g., alloc_page_vma()
1186			 * will allocate page according to address).  This
1187			 * could only happen if one pinned pte changed.
1188			 */
1189			folio_put(prealloc);
1190			prealloc = NULL;
1191		}
1192		nr = ret;
1193		progress += 8 * nr;
1194	} while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr,
1195		 addr != end);
1196
1197	arch_leave_lazy_mmu_mode();
1198	pte_unmap_unlock(orig_src_pte, src_ptl);
 
1199	add_mm_rss_vec(dst_mm, rss);
1200	pte_unmap_unlock(orig_dst_pte, dst_ptl);
1201	cond_resched();
1202
1203	if (ret == -EIO) {
1204		VM_WARN_ON_ONCE(!entry.val);
1205		if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
1206			ret = -ENOMEM;
1207			goto out;
1208		}
1209		entry.val = 0;
1210	} else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) {
1211		goto out;
1212	} else if (ret ==  -EAGAIN) {
1213		prealloc = folio_prealloc(src_mm, src_vma, addr, false);
1214		if (!prealloc)
1215			return -ENOMEM;
1216	} else if (ret < 0) {
1217		VM_WARN_ON_ONCE(1);
1218	}
1219
1220	/* We've captured and resolved the error. Reset, try again. */
1221	ret = 0;
1222
1223	if (addr != end)
1224		goto again;
1225out:
1226	if (unlikely(prealloc))
1227		folio_put(prealloc);
1228	return ret;
1229}
1230
1231static inline int
1232copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1233	       pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1234	       unsigned long end)
1235{
1236	struct mm_struct *dst_mm = dst_vma->vm_mm;
1237	struct mm_struct *src_mm = src_vma->vm_mm;
1238	pmd_t *src_pmd, *dst_pmd;
1239	unsigned long next;
1240
1241	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
1242	if (!dst_pmd)
1243		return -ENOMEM;
1244	src_pmd = pmd_offset(src_pud, addr);
1245	do {
1246		next = pmd_addr_end(addr, end);
1247		if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
1248			|| pmd_devmap(*src_pmd)) {
1249			int err;
1250			VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
1251			err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
1252					    addr, dst_vma, src_vma);
1253			if (err == -ENOMEM)
1254				return -ENOMEM;
1255			if (!err)
1256				continue;
1257			/* fall through */
1258		}
1259		if (pmd_none_or_clear_bad(src_pmd))
1260			continue;
1261		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1262				   addr, next))
1263			return -ENOMEM;
1264	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
1265	return 0;
1266}
1267
1268static inline int
1269copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1270	       p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
1271	       unsigned long end)
1272{
1273	struct mm_struct *dst_mm = dst_vma->vm_mm;
1274	struct mm_struct *src_mm = src_vma->vm_mm;
1275	pud_t *src_pud, *dst_pud;
1276	unsigned long next;
1277
1278	dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1279	if (!dst_pud)
1280		return -ENOMEM;
1281	src_pud = pud_offset(src_p4d, addr);
1282	do {
1283		next = pud_addr_end(addr, end);
1284		if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
1285			int err;
1286
1287			VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
1288			err = copy_huge_pud(dst_mm, src_mm,
1289					    dst_pud, src_pud, addr, src_vma);
1290			if (err == -ENOMEM)
1291				return -ENOMEM;
1292			if (!err)
1293				continue;
1294			/* fall through */
1295		}
1296		if (pud_none_or_clear_bad(src_pud))
1297			continue;
1298		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1299				   addr, next))
1300			return -ENOMEM;
1301	} while (dst_pud++, src_pud++, addr = next, addr != end);
1302	return 0;
1303}
1304
1305static inline int
1306copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1307	       pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
1308	       unsigned long end)
1309{
1310	struct mm_struct *dst_mm = dst_vma->vm_mm;
1311	p4d_t *src_p4d, *dst_p4d;
1312	unsigned long next;
1313
1314	dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1315	if (!dst_p4d)
1316		return -ENOMEM;
1317	src_p4d = p4d_offset(src_pgd, addr);
1318	do {
1319		next = p4d_addr_end(addr, end);
1320		if (p4d_none_or_clear_bad(src_p4d))
1321			continue;
1322		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1323				   addr, next))
1324			return -ENOMEM;
1325	} while (dst_p4d++, src_p4d++, addr = next, addr != end);
1326	return 0;
1327}
1328
1329/*
1330 * Return true if the vma needs to copy the pgtable during this fork().  Return
1331 * false when we can speed up fork() by allowing lazy page faults later until
1332 * when the child accesses the memory range.
1333 */
1334static bool
1335vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1336{
1337	/*
1338	 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
1339	 * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
1340	 * contains uffd-wp protection information, that's something we can't
1341	 * retrieve from page cache, and skip copying will lose those info.
1342	 */
1343	if (userfaultfd_wp(dst_vma))
1344		return true;
1345
1346	if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
1347		return true;
1348
1349	if (src_vma->anon_vma)
1350		return true;
1351
1352	/*
1353	 * Don't copy ptes where a page fault will fill them correctly.  Fork
1354	 * becomes much lighter when there are big shared or private readonly
1355	 * mappings. The tradeoff is that copy_page_range is more efficient
1356	 * than faulting.
1357	 */
1358	return false;
1359}
1360
1361int
1362copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1363{
1364	pgd_t *src_pgd, *dst_pgd;
1365	unsigned long next;
1366	unsigned long addr = src_vma->vm_start;
1367	unsigned long end = src_vma->vm_end;
1368	struct mm_struct *dst_mm = dst_vma->vm_mm;
1369	struct mm_struct *src_mm = src_vma->vm_mm;
1370	struct mmu_notifier_range range;
1371	bool is_cow;
1372	int ret;
1373
1374	if (!vma_needs_copy(dst_vma, src_vma))
 
 
 
 
 
 
 
1375		return 0;
1376
1377	if (is_vm_hugetlb_page(src_vma))
1378		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1379
1380	if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
1381		/*
1382		 * We do not free on error cases below as remove_vma
1383		 * gets called on error from higher level routine
1384		 */
1385		ret = track_pfn_copy(src_vma);
1386		if (ret)
1387			return ret;
1388	}
1389
1390	/*
1391	 * We need to invalidate the secondary MMU mappings only when
1392	 * there could be a permission downgrade on the ptes of the
1393	 * parent mm. And a permission downgrade will only happen if
1394	 * is_cow_mapping() returns true.
1395	 */
1396	is_cow = is_cow_mapping(src_vma->vm_flags);
1397
1398	if (is_cow) {
1399		mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
1400					0, src_mm, addr, end);
1401		mmu_notifier_invalidate_range_start(&range);
1402		/*
1403		 * Disabling preemption is not needed for the write side, as
1404		 * the read side doesn't spin, but goes to the mmap_lock.
1405		 *
1406		 * Use the raw variant of the seqcount_t write API to avoid
1407		 * lockdep complaining about preemptibility.
1408		 */
1409		vma_assert_write_locked(src_vma);
1410		raw_write_seqcount_begin(&src_mm->write_protect_seq);
1411	}
1412
1413	ret = 0;
1414	dst_pgd = pgd_offset(dst_mm, addr);
1415	src_pgd = pgd_offset(src_mm, addr);
1416	do {
1417		next = pgd_addr_end(addr, end);
1418		if (pgd_none_or_clear_bad(src_pgd))
1419			continue;
1420		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1421					    addr, next))) {
1422			untrack_pfn_clear(dst_vma);
1423			ret = -ENOMEM;
1424			break;
1425		}
1426	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
1427
1428	if (is_cow) {
1429		raw_write_seqcount_end(&src_mm->write_protect_seq);
1430		mmu_notifier_invalidate_range_end(&range);
1431	}
1432	return ret;
1433}
1434
1435/* Whether we should zap all COWed (private) pages too */
1436static inline bool should_zap_cows(struct zap_details *details)
1437{
1438	/* By default, zap all pages */
1439	if (!details)
1440		return true;
1441
1442	/* Or, we zap COWed pages only if the caller wants to */
1443	return details->even_cows;
1444}
1445
1446/* Decides whether we should zap this folio with the folio pointer specified */
1447static inline bool should_zap_folio(struct zap_details *details,
1448				    struct folio *folio)
1449{
1450	/* If we can make a decision without *folio.. */
1451	if (should_zap_cows(details))
1452		return true;
1453
1454	/* Otherwise we should only zap non-anon folios */
1455	return !folio_test_anon(folio);
1456}
1457
1458static inline bool zap_drop_markers(struct zap_details *details)
1459{
1460	if (!details)
1461		return false;
1462
1463	return details->zap_flags & ZAP_FLAG_DROP_MARKER;
1464}
1465
1466/*
1467 * This function makes sure that we'll replace the none pte with an uffd-wp
1468 * swap special pte marker when necessary. Must be with the pgtable lock held.
1469 */
1470static inline void
1471zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
1472			      unsigned long addr, pte_t *pte, int nr,
1473			      struct zap_details *details, pte_t pteval)
1474{
1475	/* Zap on anonymous always means dropping everything */
1476	if (vma_is_anonymous(vma))
1477		return;
1478
1479	if (zap_drop_markers(details))
1480		return;
1481
1482	for (;;) {
1483		/* the PFN in the PTE is irrelevant. */
1484		pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
1485		if (--nr == 0)
1486			break;
1487		pte++;
1488		addr += PAGE_SIZE;
1489	}
1490}
1491
1492static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
1493		struct vm_area_struct *vma, struct folio *folio,
1494		struct page *page, pte_t *pte, pte_t ptent, unsigned int nr,
1495		unsigned long addr, struct zap_details *details, int *rss,
1496		bool *force_flush, bool *force_break)
1497{
1498	struct mm_struct *mm = tlb->mm;
1499	bool delay_rmap = false;
1500
1501	if (!folio_test_anon(folio)) {
1502		ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
1503		if (pte_dirty(ptent)) {
1504			folio_mark_dirty(folio);
1505			if (tlb_delay_rmap(tlb)) {
1506				delay_rmap = true;
1507				*force_flush = true;
1508			}
1509		}
1510		if (pte_young(ptent) && likely(vma_has_recency(vma)))
1511			folio_mark_accessed(folio);
1512		rss[mm_counter(folio)] -= nr;
1513	} else {
1514		/* We don't need up-to-date accessed/dirty bits. */
1515		clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
1516		rss[MM_ANONPAGES] -= nr;
1517	}
1518	/* Checking a single PTE in a batch is sufficient. */
1519	arch_check_zapped_pte(vma, ptent);
1520	tlb_remove_tlb_entries(tlb, pte, nr, addr);
1521	if (unlikely(userfaultfd_pte_wp(vma, ptent)))
1522		zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details,
1523					      ptent);
1524
1525	if (!delay_rmap) {
1526		folio_remove_rmap_ptes(folio, page, nr, vma);
1527
1528		if (unlikely(folio_mapcount(folio) < 0))
1529			print_bad_pte(vma, addr, ptent, page);
1530	}
1531	if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
1532		*force_flush = true;
1533		*force_break = true;
1534	}
1535}
1536
1537/*
1538 * Zap or skip at least one present PTE, trying to batch-process subsequent
1539 * PTEs that map consecutive pages of the same folio.
1540 *
1541 * Returns the number of processed (skipped or zapped) PTEs (at least 1).
1542 */
1543static inline int zap_present_ptes(struct mmu_gather *tlb,
1544		struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
1545		unsigned int max_nr, unsigned long addr,
1546		struct zap_details *details, int *rss, bool *force_flush,
1547		bool *force_break)
1548{
1549	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
1550	struct mm_struct *mm = tlb->mm;
1551	struct folio *folio;
1552	struct page *page;
1553	int nr;
1554
1555	page = vm_normal_page(vma, addr, ptent);
1556	if (!page) {
1557		/* We don't need up-to-date accessed/dirty bits. */
1558		ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);
1559		arch_check_zapped_pte(vma, ptent);
1560		tlb_remove_tlb_entry(tlb, pte, addr);
1561		if (userfaultfd_pte_wp(vma, ptent))
1562			zap_install_uffd_wp_if_needed(vma, addr, pte, 1,
1563						      details, ptent);
1564		ksm_might_unmap_zero_page(mm, ptent);
1565		return 1;
1566	}
1567
1568	folio = page_folio(page);
1569	if (unlikely(!should_zap_folio(details, folio)))
1570		return 1;
1571
1572	/*
1573	 * Make sure that the common "small folio" case is as fast as possible
1574	 * by keeping the batching logic separate.
1575	 */
1576	if (unlikely(folio_test_large(folio) && max_nr != 1)) {
1577		nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags,
1578				     NULL, NULL, NULL);
1579
1580		zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,
1581				       addr, details, rss, force_flush,
1582				       force_break);
1583		return nr;
1584	}
1585	zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr,
1586			       details, rss, force_flush, force_break);
1587	return 1;
1588}
1589
1590static unsigned long zap_pte_range(struct mmu_gather *tlb,
1591				struct vm_area_struct *vma, pmd_t *pmd,
1592				unsigned long addr, unsigned long end,
1593				struct zap_details *details)
1594{
1595	bool force_flush = false, force_break = false;
1596	struct mm_struct *mm = tlb->mm;
 
1597	int rss[NR_MM_COUNTERS];
1598	spinlock_t *ptl;
1599	pte_t *start_pte;
1600	pte_t *pte;
1601	swp_entry_t entry;
1602	int nr;
1603
1604	tlb_change_page_size(tlb, PAGE_SIZE);
 
1605	init_rss_vec(rss);
1606	start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1607	if (!pte)
1608		return addr;
1609
1610	flush_tlb_batched_pending(mm);
1611	arch_enter_lazy_mmu_mode();
1612	do {
1613		pte_t ptent = ptep_get(pte);
1614		struct folio *folio;
1615		struct page *page;
1616		int max_nr;
1617
1618		nr = 1;
1619		if (pte_none(ptent))
1620			continue;
1621
1622		if (need_resched())
1623			break;
1624
1625		if (pte_present(ptent)) {
1626			max_nr = (end - addr) / PAGE_SIZE;
1627			nr = zap_present_ptes(tlb, vma, pte, ptent, max_nr,
1628					      addr, details, rss, &force_flush,
1629					      &force_break);
1630			if (unlikely(force_break)) {
1631				addr += nr * PAGE_SIZE;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1632				break;
1633			}
1634			continue;
1635		}
1636
1637		entry = pte_to_swp_entry(ptent);
1638		if (is_device_private_entry(entry) ||
1639		    is_device_exclusive_entry(entry)) {
1640			page = pfn_swap_entry_to_page(entry);
1641			folio = page_folio(page);
1642			if (unlikely(!should_zap_folio(details, folio)))
1643				continue;
1644			/*
1645			 * Both device private/exclusive mappings should only
1646			 * work with anonymous page so far, so we don't need to
1647			 * consider uffd-wp bit when zap. For more information,
1648			 * see zap_install_uffd_wp_if_needed().
1649			 */
1650			WARN_ON_ONCE(!vma_is_anonymous(vma));
1651			rss[mm_counter(folio)]--;
 
 
 
 
1652			if (is_device_private_entry(entry))
1653				folio_remove_rmap_pte(folio, page, vma);
1654			folio_put(folio);
1655		} else if (!non_swap_entry(entry)) {
1656			max_nr = (end - addr) / PAGE_SIZE;
1657			nr = swap_pte_batch(pte, max_nr, ptent);
1658			/* Genuine swap entries, hence a private anon pages */
1659			if (!should_zap_cows(details))
1660				continue;
1661			rss[MM_SWAPENTS] -= nr;
1662			free_swap_and_cache_nr(entry, nr);
1663		} else if (is_migration_entry(entry)) {
1664			folio = pfn_swap_entry_folio(entry);
1665			if (!should_zap_folio(details, folio))
1666				continue;
1667			rss[mm_counter(folio)]--;
1668		} else if (pte_marker_entry_uffd_wp(entry)) {
1669			/*
1670			 * For anon: always drop the marker; for file: only
1671			 * drop the marker if explicitly requested.
1672			 */
1673			if (!vma_is_anonymous(vma) &&
1674			    !zap_drop_markers(details))
1675				continue;
1676		} else if (is_guard_swp_entry(entry)) {
1677			/*
1678			 * Ordinary zapping should not remove guard PTE
1679			 * markers. Only do so if we should remove PTE markers
1680			 * in general.
1681			 */
1682			if (!zap_drop_markers(details))
1683				continue;
1684		} else if (is_hwpoison_entry(entry) ||
1685			   is_poisoned_swp_entry(entry)) {
1686			if (!should_zap_cows(details))
1687				continue;
1688		} else {
1689			/* We should have covered all the swap entry types */
1690			pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
1691			WARN_ON_ONCE(1);
1692		}
1693		clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm);
1694		zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);
1695	} while (pte += nr, addr += PAGE_SIZE * nr, addr != end);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1696
1697	add_mm_rss_vec(mm, rss);
1698	arch_leave_lazy_mmu_mode();
1699
1700	/* Do the actual TLB flush before dropping ptl */
1701	if (force_flush) {
1702		tlb_flush_mmu_tlbonly(tlb);
1703		tlb_flush_rmaps(tlb, vma);
1704	}
1705	pte_unmap_unlock(start_pte, ptl);
1706
1707	/*
1708	 * If we forced a TLB flush (either due to running out of
1709	 * batch buffers or because we needed to flush dirty TLB
1710	 * entries before releasing the ptl), free the batched
1711	 * memory too. Come back again if we didn't do everything.
1712	 */
1713	if (force_flush)
 
1714		tlb_flush_mmu(tlb);
 
 
 
 
 
 
1715
1716	return addr;
1717}
1718
1719static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
1720				struct vm_area_struct *vma, pud_t *pud,
1721				unsigned long addr, unsigned long end,
1722				struct zap_details *details)
1723{
1724	pmd_t *pmd;
1725	unsigned long next;
1726
1727	pmd = pmd_offset(pud, addr);
1728	do {
1729		next = pmd_addr_end(addr, end);
1730		if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1731			if (next - addr != HPAGE_PMD_SIZE)
1732				__split_huge_pmd(vma, pmd, addr, false, NULL);
1733			else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
1734				addr = next;
1735				continue;
1736			}
1737			/* fall through */
1738		} else if (details && details->single_folio &&
1739			   folio_test_pmd_mappable(details->single_folio) &&
1740			   next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
1741			spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
1742			/*
1743			 * Take and drop THP pmd lock so that we cannot return
1744			 * prematurely, while zap_huge_pmd() has cleared *pmd,
1745			 * but not yet decremented compound_mapcount().
1746			 */
1747			spin_unlock(ptl);
1748		}
1749		if (pmd_none(*pmd)) {
1750			addr = next;
1751			continue;
1752		}
1753		addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
1754		if (addr != next)
1755			pmd--;
1756	} while (pmd++, cond_resched(), addr != end);
 
 
 
 
 
 
1757
1758	return addr;
1759}
1760
1761static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1762				struct vm_area_struct *vma, p4d_t *p4d,
1763				unsigned long addr, unsigned long end,
1764				struct zap_details *details)
1765{
1766	pud_t *pud;
1767	unsigned long next;
1768
1769	pud = pud_offset(p4d, addr);
1770	do {
1771		next = pud_addr_end(addr, end);
1772		if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1773			if (next - addr != HPAGE_PUD_SIZE) {
1774				mmap_assert_locked(tlb->mm);
1775				split_huge_pud(vma, pud, addr);
1776			} else if (zap_huge_pud(tlb, vma, pud, addr))
1777				goto next;
1778			/* fall through */
1779		}
1780		if (pud_none_or_clear_bad(pud))
1781			continue;
1782		next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1783next:
1784		cond_resched();
1785	} while (pud++, addr = next, addr != end);
1786
1787	return addr;
1788}
1789
1790static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1791				struct vm_area_struct *vma, pgd_t *pgd,
1792				unsigned long addr, unsigned long end,
1793				struct zap_details *details)
1794{
1795	p4d_t *p4d;
1796	unsigned long next;
1797
1798	p4d = p4d_offset(pgd, addr);
1799	do {
1800		next = p4d_addr_end(addr, end);
1801		if (p4d_none_or_clear_bad(p4d))
1802			continue;
1803		next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1804	} while (p4d++, addr = next, addr != end);
1805
1806	return addr;
1807}
1808
1809void unmap_page_range(struct mmu_gather *tlb,
1810			     struct vm_area_struct *vma,
1811			     unsigned long addr, unsigned long end,
1812			     struct zap_details *details)
1813{
1814	pgd_t *pgd;
1815	unsigned long next;
1816
1817	BUG_ON(addr >= end);
1818	tlb_start_vma(tlb, vma);
1819	pgd = pgd_offset(vma->vm_mm, addr);
1820	do {
1821		next = pgd_addr_end(addr, end);
1822		if (pgd_none_or_clear_bad(pgd))
1823			continue;
1824		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1825	} while (pgd++, addr = next, addr != end);
1826	tlb_end_vma(tlb, vma);
1827}
1828
1829
1830static void unmap_single_vma(struct mmu_gather *tlb,
1831		struct vm_area_struct *vma, unsigned long start_addr,
1832		unsigned long end_addr,
1833		struct zap_details *details, bool mm_wr_locked)
1834{
1835	unsigned long start = max(vma->vm_start, start_addr);
1836	unsigned long end;
1837
1838	if (start >= vma->vm_end)
1839		return;
1840	end = min(vma->vm_end, end_addr);
1841	if (end <= vma->vm_start)
1842		return;
1843
1844	if (vma->vm_file)
1845		uprobe_munmap(vma, start, end);
1846
1847	if (unlikely(vma->vm_flags & VM_PFNMAP))
1848		untrack_pfn(vma, 0, 0, mm_wr_locked);
1849
1850	if (start != end) {
1851		if (unlikely(is_vm_hugetlb_page(vma))) {
1852			/*
1853			 * It is undesirable to test vma->vm_file as it
1854			 * should be non-null for valid hugetlb area.
1855			 * However, vm_file will be NULL in the error
1856			 * cleanup path of mmap_region. When
1857			 * hugetlbfs ->mmap method fails,
1858			 * mmap_region() nullifies vma->vm_file
1859			 * before calling this function to clean up.
1860			 * Since no pte has actually been setup, it is
1861			 * safe to do nothing in this case.
1862			 */
1863			if (vma->vm_file) {
1864				zap_flags_t zap_flags = details ?
1865				    details->zap_flags : 0;
1866				__unmap_hugepage_range(tlb, vma, start, end,
1867							     NULL, zap_flags);
1868			}
1869		} else
1870			unmap_page_range(tlb, vma, start, end, details);
1871	}
1872}
1873
1874/**
1875 * unmap_vmas - unmap a range of memory covered by a list of vma's
1876 * @tlb: address of the caller's struct mmu_gather
1877 * @mas: the maple state
1878 * @vma: the starting vma
1879 * @start_addr: virtual address at which to start unmapping
1880 * @end_addr: virtual address at which to end unmapping
1881 * @tree_end: The maximum index to check
1882 * @mm_wr_locked: lock flag
1883 *
1884 * Unmap all pages in the vma list.
1885 *
1886 * Only addresses between `start' and `end' will be unmapped.
1887 *
1888 * The VMA list must be sorted in ascending virtual address order.
1889 *
1890 * unmap_vmas() assumes that the caller will flush the whole unmapped address
1891 * range after unmap_vmas() returns.  So the only responsibility here is to
1892 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1893 * drops the lock and schedules.
1894 */
1895void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
1896		struct vm_area_struct *vma, unsigned long start_addr,
1897		unsigned long end_addr, unsigned long tree_end,
1898		bool mm_wr_locked)
1899{
1900	struct mmu_notifier_range range;
1901	struct zap_details details = {
1902		.zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
1903		/* Careful - we need to zap private pages too! */
1904		.even_cows = true,
1905	};
1906
1907	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
1908				start_addr, end_addr);
1909	mmu_notifier_invalidate_range_start(&range);
1910	do {
1911		unsigned long start = start_addr;
1912		unsigned long end = end_addr;
1913		hugetlb_zap_begin(vma, &start, &end);
1914		unmap_single_vma(tlb, vma, start, end, &details,
1915				 mm_wr_locked);
1916		hugetlb_zap_end(vma, &details);
1917		vma = mas_find(mas, tree_end - 1);
1918	} while (vma && likely(!xa_is_zero(vma)));
1919	mmu_notifier_invalidate_range_end(&range);
1920}
1921
1922/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1923 * zap_page_range_single - remove user pages in a given range
1924 * @vma: vm_area_struct holding the applicable pages
1925 * @address: starting address of pages to zap
1926 * @size: number of bytes to zap
1927 * @details: details of shared cache invalidation
1928 *
1929 * The range must fit into one VMA.
1930 */
1931void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1932		unsigned long size, struct zap_details *details)
1933{
1934	const unsigned long end = address + size;
1935	struct mmu_notifier_range range;
1936	struct mmu_gather tlb;
1937
1938	lru_add_drain();
1939	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1940				address, end);
1941	hugetlb_zap_begin(vma, &range.start, &range.end);
1942	tlb_gather_mmu(&tlb, vma->vm_mm);
1943	update_hiwater_rss(vma->vm_mm);
1944	mmu_notifier_invalidate_range_start(&range);
1945	/*
1946	 * unmap 'address-end' not 'range.start-range.end' as range
1947	 * could have been expanded for hugetlb pmd sharing.
1948	 */
1949	unmap_single_vma(&tlb, vma, address, end, details, false);
1950	mmu_notifier_invalidate_range_end(&range);
1951	tlb_finish_mmu(&tlb);
1952	hugetlb_zap_end(vma, details);
1953}
1954
1955/**
1956 * zap_vma_ptes - remove ptes mapping the vma
1957 * @vma: vm_area_struct holding ptes to be zapped
1958 * @address: starting address of pages to zap
1959 * @size: number of bytes to zap
1960 *
1961 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1962 *
1963 * The entire address range must be fully contained within the vma.
1964 *
1965 */
1966void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1967		unsigned long size)
1968{
1969	if (!range_in_vma(vma, address, address + size) ||
1970	    		!(vma->vm_flags & VM_PFNMAP))
1971		return;
1972
1973	zap_page_range_single(vma, address, size, NULL);
1974}
1975EXPORT_SYMBOL_GPL(zap_vma_ptes);
1976
1977static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
1978{
1979	pgd_t *pgd;
1980	p4d_t *p4d;
1981	pud_t *pud;
1982	pmd_t *pmd;
1983
1984	pgd = pgd_offset(mm, addr);
1985	p4d = p4d_alloc(mm, pgd, addr);
1986	if (!p4d)
1987		return NULL;
1988	pud = pud_alloc(mm, p4d, addr);
1989	if (!pud)
1990		return NULL;
1991	pmd = pmd_alloc(mm, pud, addr);
1992	if (!pmd)
1993		return NULL;
1994
1995	VM_BUG_ON(pmd_trans_huge(*pmd));
1996	return pmd;
1997}
1998
1999pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
2000			spinlock_t **ptl)
2001{
2002	pmd_t *pmd = walk_to_pmd(mm, addr);
2003
2004	if (!pmd)
2005		return NULL;
2006	return pte_alloc_map_lock(mm, pmd, addr, ptl);
2007}
2008
2009static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma)
2010{
2011	VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP);
2012	/*
2013	 * Whoever wants to forbid the zeropage after some zeropages
2014	 * might already have been mapped has to scan the page tables and
2015	 * bail out on any zeropages. Zeropages in COW mappings can
2016	 * be unshared using FAULT_FLAG_UNSHARE faults.
2017	 */
2018	if (mm_forbids_zeropage(vma->vm_mm))
2019		return false;
2020	/* zeropages in COW mappings are common and unproblematic. */
2021	if (is_cow_mapping(vma->vm_flags))
2022		return true;
2023	/* Mappings that do not allow for writable PTEs are unproblematic. */
2024	if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)))
2025		return true;
2026	/*
2027	 * Why not allow any VMA that has vm_ops->pfn_mkwrite? GUP could
2028	 * find the shared zeropage and longterm-pin it, which would
2029	 * be problematic as soon as the zeropage gets replaced by a different
2030	 * page due to vma->vm_ops->pfn_mkwrite, because what's mapped would
2031	 * now differ to what GUP looked up. FSDAX is incompatible to
2032	 * FOLL_LONGTERM and VM_IO is incompatible to GUP completely (see
2033	 * check_vma_flags).
2034	 */
2035	return vma->vm_ops && vma->vm_ops->pfn_mkwrite &&
2036	       (vma_is_fsdax(vma) || vma->vm_flags & VM_IO);
2037}
2038
2039static int validate_page_before_insert(struct vm_area_struct *vma,
2040				       struct page *page)
2041{
2042	struct folio *folio = page_folio(page);
2043
2044	if (!folio_ref_count(folio))
2045		return -EINVAL;
2046	if (unlikely(is_zero_folio(folio))) {
2047		if (!vm_mixed_zeropage_allowed(vma))
2048			return -EINVAL;
2049		return 0;
2050	}
2051	if (folio_test_anon(folio) || folio_test_slab(folio) ||
2052	    page_has_type(page))
2053		return -EINVAL;
2054	flush_dcache_folio(folio);
2055	return 0;
2056}
2057
2058static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
2059			unsigned long addr, struct page *page, pgprot_t prot)
2060{
2061	struct folio *folio = page_folio(page);
2062	pte_t pteval;
2063
2064	if (!pte_none(ptep_get(pte)))
2065		return -EBUSY;
2066	/* Ok, finally just insert the thing.. */
2067	pteval = mk_pte(page, prot);
2068	if (unlikely(is_zero_folio(folio))) {
2069		pteval = pte_mkspecial(pteval);
2070	} else {
2071		folio_get(folio);
2072		inc_mm_counter(vma->vm_mm, mm_counter_file(folio));
2073		folio_add_file_rmap_pte(folio, page, vma);
2074	}
2075	set_pte_at(vma->vm_mm, addr, pte, pteval);
2076	return 0;
2077}
2078
 
 
 
 
 
 
 
2079static int insert_page(struct vm_area_struct *vma, unsigned long addr,
2080			struct page *page, pgprot_t prot)
2081{
 
2082	int retval;
2083	pte_t *pte;
2084	spinlock_t *ptl;
2085
2086	retval = validate_page_before_insert(vma, page);
2087	if (retval)
2088		goto out;
2089	retval = -ENOMEM;
2090	pte = get_locked_pte(vma->vm_mm, addr, &ptl);
2091	if (!pte)
2092		goto out;
2093	retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
2094	pte_unmap_unlock(pte, ptl);
2095out:
2096	return retval;
2097}
2098
2099static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
 
2100			unsigned long addr, struct page *page, pgprot_t prot)
2101{
2102	int err;
2103
2104	err = validate_page_before_insert(vma, page);
 
 
2105	if (err)
2106		return err;
2107	return insert_page_into_pte_locked(vma, pte, addr, page, prot);
2108}
2109
2110/* insert_pages() amortizes the cost of spinlock operations
2111 * when inserting pages in a loop.
2112 */
2113static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
2114			struct page **pages, unsigned long *num, pgprot_t prot)
2115{
2116	pmd_t *pmd = NULL;
2117	pte_t *start_pte, *pte;
2118	spinlock_t *pte_lock;
2119	struct mm_struct *const mm = vma->vm_mm;
2120	unsigned long curr_page_idx = 0;
2121	unsigned long remaining_pages_total = *num;
2122	unsigned long pages_to_write_in_pmd;
2123	int ret;
2124more:
2125	ret = -EFAULT;
2126	pmd = walk_to_pmd(mm, addr);
2127	if (!pmd)
2128		goto out;
2129
2130	pages_to_write_in_pmd = min_t(unsigned long,
2131		remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
2132
2133	/* Allocate the PTE if necessary; takes PMD lock once only. */
2134	ret = -ENOMEM;
2135	if (pte_alloc(mm, pmd))
2136		goto out;
2137
2138	while (pages_to_write_in_pmd) {
2139		int pte_idx = 0;
2140		const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
2141
2142		start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
2143		if (!start_pte) {
2144			ret = -EFAULT;
2145			goto out;
2146		}
2147		for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
2148			int err = insert_page_in_batch_locked(vma, pte,
2149				addr, pages[curr_page_idx], prot);
2150			if (unlikely(err)) {
2151				pte_unmap_unlock(start_pte, pte_lock);
2152				ret = err;
2153				remaining_pages_total -= pte_idx;
2154				goto out;
2155			}
2156			addr += PAGE_SIZE;
2157			++curr_page_idx;
2158		}
2159		pte_unmap_unlock(start_pte, pte_lock);
2160		pages_to_write_in_pmd -= batch_size;
2161		remaining_pages_total -= batch_size;
2162	}
2163	if (remaining_pages_total)
2164		goto more;
2165	ret = 0;
2166out:
2167	*num = remaining_pages_total;
2168	return ret;
2169}
 
2170
2171/**
2172 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
2173 * @vma: user vma to map to
2174 * @addr: target start user address of these pages
2175 * @pages: source kernel pages
2176 * @num: in: number of pages to map. out: number of pages that were *not*
2177 * mapped. (0 means all pages were successfully mapped).
2178 *
2179 * Preferred over vm_insert_page() when inserting multiple pages.
2180 *
2181 * In case of error, we may have mapped a subset of the provided
2182 * pages. It is the caller's responsibility to account for this case.
2183 *
2184 * The same restrictions apply as in vm_insert_page().
2185 */
2186int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
2187			struct page **pages, unsigned long *num)
2188{
 
2189	const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
2190
2191	if (addr < vma->vm_start || end_addr >= vma->vm_end)
2192		return -EFAULT;
2193	if (!(vma->vm_flags & VM_MIXEDMAP)) {
2194		BUG_ON(mmap_read_trylock(vma->vm_mm));
2195		BUG_ON(vma->vm_flags & VM_PFNMAP);
2196		vm_flags_set(vma, VM_MIXEDMAP);
2197	}
2198	/* Defer page refcount checking till we're about to map that page. */
2199	return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
 
 
 
 
 
 
 
 
 
 
 
 
2200}
2201EXPORT_SYMBOL(vm_insert_pages);
2202
2203/**
2204 * vm_insert_page - insert single page into user vma
2205 * @vma: user vma to map to
2206 * @addr: target user address of this page
2207 * @page: source kernel page
2208 *
2209 * This allows drivers to insert individual pages they've allocated
2210 * into a user vma. The zeropage is supported in some VMAs,
2211 * see vm_mixed_zeropage_allowed().
2212 *
2213 * The page has to be a nice clean _individual_ kernel allocation.
2214 * If you allocate a compound page, you need to have marked it as
2215 * such (__GFP_COMP), or manually just split the page up yourself
2216 * (see split_page()).
2217 *
2218 * NOTE! Traditionally this was done with "remap_pfn_range()" which
2219 * took an arbitrary page protection parameter. This doesn't allow
2220 * that. Your vma protection will have to be set up correctly, which
2221 * means that if you want a shared writable mapping, you'd better
2222 * ask for a shared writable mapping!
2223 *
2224 * The page does not need to be reserved.
2225 *
2226 * Usually this function is called from f_op->mmap() handler
2227 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
2228 * Caller must set VM_MIXEDMAP on vma if it wants to call this
2229 * function from other places, for example from page-fault handler.
2230 *
2231 * Return: %0 on success, negative error code otherwise.
2232 */
2233int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
2234			struct page *page)
2235{
2236	if (addr < vma->vm_start || addr >= vma->vm_end)
2237		return -EFAULT;
 
 
2238	if (!(vma->vm_flags & VM_MIXEDMAP)) {
2239		BUG_ON(mmap_read_trylock(vma->vm_mm));
2240		BUG_ON(vma->vm_flags & VM_PFNMAP);
2241		vm_flags_set(vma, VM_MIXEDMAP);
2242	}
2243	return insert_page(vma, addr, page, vma->vm_page_prot);
2244}
2245EXPORT_SYMBOL(vm_insert_page);
2246
2247/*
2248 * __vm_map_pages - maps range of kernel pages into user vma
2249 * @vma: user vma to map to
2250 * @pages: pointer to array of source kernel pages
2251 * @num: number of pages in page array
2252 * @offset: user's requested vm_pgoff
2253 *
2254 * This allows drivers to map range of kernel pages into a user vma.
2255 * The zeropage is supported in some VMAs, see
2256 * vm_mixed_zeropage_allowed().
2257 *
2258 * Return: 0 on success and error code otherwise.
2259 */
2260static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2261				unsigned long num, unsigned long offset)
2262{
2263	unsigned long count = vma_pages(vma);
2264	unsigned long uaddr = vma->vm_start;
2265	int ret, i;
2266
2267	/* Fail if the user requested offset is beyond the end of the object */
2268	if (offset >= num)
2269		return -ENXIO;
2270
2271	/* Fail if the user requested size exceeds available object size */
2272	if (count > num - offset)
2273		return -ENXIO;
2274
2275	for (i = 0; i < count; i++) {
2276		ret = vm_insert_page(vma, uaddr, pages[offset + i]);
2277		if (ret < 0)
2278			return ret;
2279		uaddr += PAGE_SIZE;
2280	}
2281
2282	return 0;
2283}
2284
2285/**
2286 * vm_map_pages - maps range of kernel pages starts with non zero offset
2287 * @vma: user vma to map to
2288 * @pages: pointer to array of source kernel pages
2289 * @num: number of pages in page array
2290 *
2291 * Maps an object consisting of @num pages, catering for the user's
2292 * requested vm_pgoff
2293 *
2294 * If we fail to insert any page into the vma, the function will return
2295 * immediately leaving any previously inserted pages present.  Callers
2296 * from the mmap handler may immediately return the error as their caller
2297 * will destroy the vma, removing any successfully inserted pages. Other
2298 * callers should make their own arrangements for calling unmap_region().
2299 *
2300 * Context: Process context. Called by mmap handlers.
2301 * Return: 0 on success and error code otherwise.
2302 */
2303int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2304				unsigned long num)
2305{
2306	return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
2307}
2308EXPORT_SYMBOL(vm_map_pages);
2309
2310/**
2311 * vm_map_pages_zero - map range of kernel pages starts with zero offset
2312 * @vma: user vma to map to
2313 * @pages: pointer to array of source kernel pages
2314 * @num: number of pages in page array
2315 *
2316 * Similar to vm_map_pages(), except that it explicitly sets the offset
2317 * to 0. This function is intended for the drivers that did not consider
2318 * vm_pgoff.
2319 *
2320 * Context: Process context. Called by mmap handlers.
2321 * Return: 0 on success and error code otherwise.
2322 */
2323int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2324				unsigned long num)
2325{
2326	return __vm_map_pages(vma, pages, num, 0);
2327}
2328EXPORT_SYMBOL(vm_map_pages_zero);
2329
2330static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2331			pfn_t pfn, pgprot_t prot, bool mkwrite)
2332{
2333	struct mm_struct *mm = vma->vm_mm;
2334	pte_t *pte, entry;
2335	spinlock_t *ptl;
2336
2337	pte = get_locked_pte(mm, addr, &ptl);
2338	if (!pte)
2339		return VM_FAULT_OOM;
2340	entry = ptep_get(pte);
2341	if (!pte_none(entry)) {
2342		if (mkwrite) {
2343			/*
2344			 * For read faults on private mappings the PFN passed
2345			 * in may not match the PFN we have mapped if the
2346			 * mapped PFN is a writeable COW page.  In the mkwrite
2347			 * case we are creating a writable PTE for a shared
2348			 * mapping and we expect the PFNs to match. If they
2349			 * don't match, we are likely racing with block
2350			 * allocation and mapping invalidation so just skip the
2351			 * update.
2352			 */
2353			if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
2354				WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
2355				goto out_unlock;
2356			}
2357			entry = pte_mkyoung(entry);
2358			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2359			if (ptep_set_access_flags(vma, addr, pte, entry, 1))
2360				update_mmu_cache(vma, addr, pte);
2361		}
2362		goto out_unlock;
2363	}
2364
2365	/* Ok, finally just insert the thing.. */
2366	if (pfn_t_devmap(pfn))
2367		entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
2368	else
2369		entry = pte_mkspecial(pfn_t_pte(pfn, prot));
2370
2371	if (mkwrite) {
2372		entry = pte_mkyoung(entry);
2373		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2374	}
2375
2376	set_pte_at(mm, addr, pte, entry);
2377	update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
2378
2379out_unlock:
2380	pte_unmap_unlock(pte, ptl);
2381	return VM_FAULT_NOPAGE;
2382}
2383
2384/**
2385 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
2386 * @vma: user vma to map to
2387 * @addr: target user address of this page
2388 * @pfn: source kernel pfn
2389 * @pgprot: pgprot flags for the inserted page
2390 *
2391 * This is exactly like vmf_insert_pfn(), except that it allows drivers
2392 * to override pgprot on a per-page basis.
2393 *
2394 * This only makes sense for IO mappings, and it makes no sense for
2395 * COW mappings.  In general, using multiple vmas is preferable;
2396 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
2397 * impractical.
2398 *
2399 * pgprot typically only differs from @vma->vm_page_prot when drivers set
2400 * caching- and encryption bits different than those of @vma->vm_page_prot,
2401 * because the caching- or encryption mode may not be known at mmap() time.
2402 *
2403 * This is ok as long as @vma->vm_page_prot is not used by the core vm
2404 * to set caching and encryption bits for those vmas (except for COW pages).
2405 * This is ensured by core vm only modifying these page table entries using
2406 * functions that don't touch caching- or encryption bits, using pte_modify()
2407 * if needed. (See for example mprotect()).
2408 *
2409 * Also when new page-table entries are created, this is only done using the
2410 * fault() callback, and never using the value of vma->vm_page_prot,
2411 * except for page-table entries that point to anonymous pages as the result
2412 * of COW.
2413 *
2414 * Context: Process context.  May allocate using %GFP_KERNEL.
2415 * Return: vm_fault_t value.
2416 */
2417vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2418			unsigned long pfn, pgprot_t pgprot)
2419{
2420	/*
2421	 * Technically, architectures with pte_special can avoid all these
2422	 * restrictions (same for remap_pfn_range).  However we would like
2423	 * consistency in testing and feature parity among all, so we should
2424	 * try to keep these invariants in place for everybody.
2425	 */
2426	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
2427	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
2428						(VM_PFNMAP|VM_MIXEDMAP));
2429	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
2430	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
2431
2432	if (addr < vma->vm_start || addr >= vma->vm_end)
2433		return VM_FAULT_SIGBUS;
2434
2435	if (!pfn_modify_allowed(pfn, pgprot))
2436		return VM_FAULT_SIGBUS;
2437
2438	track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
2439
2440	return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
2441			false);
2442}
2443EXPORT_SYMBOL(vmf_insert_pfn_prot);
2444
2445/**
2446 * vmf_insert_pfn - insert single pfn into user vma
2447 * @vma: user vma to map to
2448 * @addr: target user address of this page
2449 * @pfn: source kernel pfn
2450 *
2451 * Similar to vm_insert_page, this allows drivers to insert individual pages
2452 * they've allocated into a user vma. Same comments apply.
2453 *
2454 * This function should only be called from a vm_ops->fault handler, and
2455 * in that case the handler should return the result of this function.
2456 *
2457 * vma cannot be a COW mapping.
2458 *
2459 * As this is called only for pages that do not currently exist, we
2460 * do not need to flush old virtual caches or the TLB.
2461 *
2462 * Context: Process context.  May allocate using %GFP_KERNEL.
2463 * Return: vm_fault_t value.
2464 */
2465vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2466			unsigned long pfn)
2467{
2468	return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
2469}
2470EXPORT_SYMBOL(vmf_insert_pfn);
2471
2472static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn, bool mkwrite)
2473{
2474	if (unlikely(is_zero_pfn(pfn_t_to_pfn(pfn))) &&
2475	    (mkwrite || !vm_mixed_zeropage_allowed(vma)))
2476		return false;
2477	/* these checks mirror the abort conditions in vm_normal_page */
2478	if (vma->vm_flags & VM_MIXEDMAP)
2479		return true;
2480	if (pfn_t_devmap(pfn))
2481		return true;
2482	if (pfn_t_special(pfn))
2483		return true;
2484	if (is_zero_pfn(pfn_t_to_pfn(pfn)))
2485		return true;
2486	return false;
2487}
2488
2489static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
2490		unsigned long addr, pfn_t pfn, bool mkwrite)
 
2491{
2492	pgprot_t pgprot = vma->vm_page_prot;
2493	int err;
2494
2495	if (!vm_mixed_ok(vma, pfn, mkwrite))
2496		return VM_FAULT_SIGBUS;
2497
2498	if (addr < vma->vm_start || addr >= vma->vm_end)
2499		return VM_FAULT_SIGBUS;
2500
2501	track_pfn_insert(vma, &pgprot, pfn);
2502
2503	if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
2504		return VM_FAULT_SIGBUS;
2505
2506	/*
2507	 * If we don't have pte special, then we have to use the pfn_valid()
2508	 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
2509	 * refcount the page if pfn_valid is true (hence insert_page rather
2510	 * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
2511	 * without pte special, it would there be refcounted as a normal page.
2512	 */
2513	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
2514	    !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
2515		struct page *page;
2516
2517		/*
2518		 * At this point we are committed to insert_page()
2519		 * regardless of whether the caller specified flags that
2520		 * result in pfn_t_has_page() == false.
2521		 */
2522		page = pfn_to_page(pfn_t_to_pfn(pfn));
2523		err = insert_page(vma, addr, page, pgprot);
2524	} else {
2525		return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
2526	}
2527
2528	if (err == -ENOMEM)
2529		return VM_FAULT_OOM;
2530	if (err < 0 && err != -EBUSY)
2531		return VM_FAULT_SIGBUS;
2532
2533	return VM_FAULT_NOPAGE;
2534}
2535
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2536vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2537		pfn_t pfn)
2538{
2539	return __vm_insert_mixed(vma, addr, pfn, false);
2540}
2541EXPORT_SYMBOL(vmf_insert_mixed);
2542
2543/*
2544 *  If the insertion of PTE failed because someone else already added a
2545 *  different entry in the mean time, we treat that as success as we assume
2546 *  the same entry was actually inserted.
2547 */
2548vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
2549		unsigned long addr, pfn_t pfn)
2550{
2551	return __vm_insert_mixed(vma, addr, pfn, true);
2552}
 
2553
2554/*
2555 * maps a range of physical memory into the requested pages. the old
2556 * mappings are removed. any references to nonexistent pages results
2557 * in null mappings (currently treated as "copy-on-access")
2558 */
2559static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
2560			unsigned long addr, unsigned long end,
2561			unsigned long pfn, pgprot_t prot)
2562{
2563	pte_t *pte, *mapped_pte;
2564	spinlock_t *ptl;
2565	int err = 0;
2566
2567	mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
2568	if (!pte)
2569		return -ENOMEM;
2570	arch_enter_lazy_mmu_mode();
2571	do {
2572		BUG_ON(!pte_none(ptep_get(pte)));
2573		if (!pfn_modify_allowed(pfn, prot)) {
2574			err = -EACCES;
2575			break;
2576		}
2577		set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
2578		pfn++;
2579	} while (pte++, addr += PAGE_SIZE, addr != end);
2580	arch_leave_lazy_mmu_mode();
2581	pte_unmap_unlock(mapped_pte, ptl);
2582	return err;
2583}
2584
2585static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
2586			unsigned long addr, unsigned long end,
2587			unsigned long pfn, pgprot_t prot)
2588{
2589	pmd_t *pmd;
2590	unsigned long next;
2591	int err;
2592
2593	pfn -= addr >> PAGE_SHIFT;
2594	pmd = pmd_alloc(mm, pud, addr);
2595	if (!pmd)
2596		return -ENOMEM;
2597	VM_BUG_ON(pmd_trans_huge(*pmd));
2598	do {
2599		next = pmd_addr_end(addr, end);
2600		err = remap_pte_range(mm, pmd, addr, next,
2601				pfn + (addr >> PAGE_SHIFT), prot);
2602		if (err)
2603			return err;
2604	} while (pmd++, addr = next, addr != end);
2605	return 0;
2606}
2607
2608static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
2609			unsigned long addr, unsigned long end,
2610			unsigned long pfn, pgprot_t prot)
2611{
2612	pud_t *pud;
2613	unsigned long next;
2614	int err;
2615
2616	pfn -= addr >> PAGE_SHIFT;
2617	pud = pud_alloc(mm, p4d, addr);
2618	if (!pud)
2619		return -ENOMEM;
2620	do {
2621		next = pud_addr_end(addr, end);
2622		err = remap_pmd_range(mm, pud, addr, next,
2623				pfn + (addr >> PAGE_SHIFT), prot);
2624		if (err)
2625			return err;
2626	} while (pud++, addr = next, addr != end);
2627	return 0;
2628}
2629
2630static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2631			unsigned long addr, unsigned long end,
2632			unsigned long pfn, pgprot_t prot)
2633{
2634	p4d_t *p4d;
2635	unsigned long next;
2636	int err;
2637
2638	pfn -= addr >> PAGE_SHIFT;
2639	p4d = p4d_alloc(mm, pgd, addr);
2640	if (!p4d)
2641		return -ENOMEM;
2642	do {
2643		next = p4d_addr_end(addr, end);
2644		err = remap_pud_range(mm, p4d, addr, next,
2645				pfn + (addr >> PAGE_SHIFT), prot);
2646		if (err)
2647			return err;
2648	} while (p4d++, addr = next, addr != end);
2649	return 0;
2650}
2651
2652static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr,
 
 
 
 
2653		unsigned long pfn, unsigned long size, pgprot_t prot)
2654{
2655	pgd_t *pgd;
2656	unsigned long next;
2657	unsigned long end = addr + PAGE_ALIGN(size);
2658	struct mm_struct *mm = vma->vm_mm;
2659	int err;
2660
2661	if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
2662		return -EINVAL;
2663
2664	/*
2665	 * Physically remapped pages are special. Tell the
2666	 * rest of the world about it:
2667	 *   VM_IO tells people not to look at these pages
2668	 *	(accesses can have side effects).
2669	 *   VM_PFNMAP tells the core MM that the base pages are just
2670	 *	raw PFN mappings, and do not have a "struct page" associated
2671	 *	with them.
2672	 *   VM_DONTEXPAND
2673	 *      Disable vma merging and expanding with mremap().
2674	 *   VM_DONTDUMP
2675	 *      Omit vma from core dump, even when VM_IO turned off.
2676	 *
2677	 * There's a horrible special case to handle copy-on-write
2678	 * behaviour that some programs depend on. We mark the "original"
2679	 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
2680	 * See vm_normal_page() for details.
2681	 */
2682	if (is_cow_mapping(vma->vm_flags)) {
2683		if (addr != vma->vm_start || end != vma->vm_end)
2684			return -EINVAL;
2685		vma->vm_pgoff = pfn;
2686	}
2687
2688	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
2689
2690	BUG_ON(addr >= end);
2691	pfn -= addr >> PAGE_SHIFT;
2692	pgd = pgd_offset(mm, addr);
2693	flush_cache_range(vma, addr, end);
2694	do {
2695		next = pgd_addr_end(addr, end);
2696		err = remap_p4d_range(mm, pgd, addr, next,
2697				pfn + (addr >> PAGE_SHIFT), prot);
2698		if (err)
2699			return err;
2700	} while (pgd++, addr = next, addr != end);
2701
2702	return 0;
2703}
2704
2705/*
2706 * Variant of remap_pfn_range that does not call track_pfn_remap.  The caller
2707 * must have pre-validated the caching bits of the pgprot_t.
2708 */
2709int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
2710		unsigned long pfn, unsigned long size, pgprot_t prot)
2711{
2712	int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);
2713
2714	if (!error)
2715		return 0;
2716
2717	/*
2718	 * A partial pfn range mapping is dangerous: it does not
2719	 * maintain page reference counts, and callers may free
2720	 * pages due to the error. So zap it early.
2721	 */
2722	zap_page_range_single(vma, addr, size, NULL);
2723	return error;
2724}
2725
2726/**
2727 * remap_pfn_range - remap kernel memory to userspace
2728 * @vma: user vma to map to
2729 * @addr: target page aligned user address to start at
2730 * @pfn: page frame number of kernel physical memory address
2731 * @size: size of mapping area
2732 * @prot: page protection flags for this mapping
2733 *
2734 * Note: this is only safe if the mm semaphore is held when called.
2735 *
2736 * Return: %0 on success, negative error code otherwise.
2737 */
2738int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
2739		    unsigned long pfn, unsigned long size, pgprot_t prot)
2740{
2741	int err;
2742
2743	err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
2744	if (err)
2745		return -EINVAL;
2746
2747	err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
2748	if (err)
2749		untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
2750	return err;
2751}
2752EXPORT_SYMBOL(remap_pfn_range);
2753
2754/**
2755 * vm_iomap_memory - remap memory to userspace
2756 * @vma: user vma to map to
2757 * @start: start of the physical memory to be mapped
2758 * @len: size of area
2759 *
2760 * This is a simplified io_remap_pfn_range() for common driver use. The
2761 * driver just needs to give us the physical memory range to be mapped,
2762 * we'll figure out the rest from the vma information.
2763 *
2764 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
2765 * whatever write-combining details or similar.
2766 *
2767 * Return: %0 on success, negative error code otherwise.
2768 */
2769int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
2770{
2771	unsigned long vm_len, pfn, pages;
2772
2773	/* Check that the physical memory area passed in looks valid */
2774	if (start + len < start)
2775		return -EINVAL;
2776	/*
2777	 * You *really* shouldn't map things that aren't page-aligned,
2778	 * but we've historically allowed it because IO memory might
2779	 * just have smaller alignment.
2780	 */
2781	len += start & ~PAGE_MASK;
2782	pfn = start >> PAGE_SHIFT;
2783	pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
2784	if (pfn + pages < pfn)
2785		return -EINVAL;
2786
2787	/* We start the mapping 'vm_pgoff' pages into the area */
2788	if (vma->vm_pgoff > pages)
2789		return -EINVAL;
2790	pfn += vma->vm_pgoff;
2791	pages -= vma->vm_pgoff;
2792
2793	/* Can we fit all of the mapping? */
2794	vm_len = vma->vm_end - vma->vm_start;
2795	if (vm_len >> PAGE_SHIFT > pages)
2796		return -EINVAL;
2797
2798	/* Ok, let it rip */
2799	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
2800}
2801EXPORT_SYMBOL(vm_iomap_memory);
2802
2803static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
2804				     unsigned long addr, unsigned long end,
2805				     pte_fn_t fn, void *data, bool create,
2806				     pgtbl_mod_mask *mask)
2807{
2808	pte_t *pte, *mapped_pte;
2809	int err = 0;
2810	spinlock_t *ptl;
2811
2812	if (create) {
2813		mapped_pte = pte = (mm == &init_mm) ?
2814			pte_alloc_kernel_track(pmd, addr, mask) :
2815			pte_alloc_map_lock(mm, pmd, addr, &ptl);
2816		if (!pte)
2817			return -ENOMEM;
2818	} else {
2819		mapped_pte = pte = (mm == &init_mm) ?
2820			pte_offset_kernel(pmd, addr) :
2821			pte_offset_map_lock(mm, pmd, addr, &ptl);
2822		if (!pte)
2823			return -EINVAL;
2824	}
2825
 
 
2826	arch_enter_lazy_mmu_mode();
2827
2828	if (fn) {
2829		do {
2830			if (create || !pte_none(ptep_get(pte))) {
2831				err = fn(pte++, addr, data);
2832				if (err)
2833					break;
2834			}
2835		} while (addr += PAGE_SIZE, addr != end);
2836	}
2837	*mask |= PGTBL_PTE_MODIFIED;
2838
2839	arch_leave_lazy_mmu_mode();
2840
2841	if (mm != &init_mm)
2842		pte_unmap_unlock(mapped_pte, ptl);
2843	return err;
2844}
2845
2846static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2847				     unsigned long addr, unsigned long end,
2848				     pte_fn_t fn, void *data, bool create,
2849				     pgtbl_mod_mask *mask)
2850{
2851	pmd_t *pmd;
2852	unsigned long next;
2853	int err = 0;
2854
2855	BUG_ON(pud_leaf(*pud));
2856
2857	if (create) {
2858		pmd = pmd_alloc_track(mm, pud, addr, mask);
2859		if (!pmd)
2860			return -ENOMEM;
2861	} else {
2862		pmd = pmd_offset(pud, addr);
2863	}
2864	do {
2865		next = pmd_addr_end(addr, end);
2866		if (pmd_none(*pmd) && !create)
2867			continue;
2868		if (WARN_ON_ONCE(pmd_leaf(*pmd)))
2869			return -EINVAL;
2870		if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
2871			if (!create)
2872				continue;
2873			pmd_clear_bad(pmd);
2874		}
2875		err = apply_to_pte_range(mm, pmd, addr, next,
2876					 fn, data, create, mask);
2877		if (err)
2878			break;
2879	} while (pmd++, addr = next, addr != end);
2880
2881	return err;
2882}
2883
2884static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2885				     unsigned long addr, unsigned long end,
2886				     pte_fn_t fn, void *data, bool create,
2887				     pgtbl_mod_mask *mask)
2888{
2889	pud_t *pud;
2890	unsigned long next;
2891	int err = 0;
2892
2893	if (create) {
2894		pud = pud_alloc_track(mm, p4d, addr, mask);
2895		if (!pud)
2896			return -ENOMEM;
2897	} else {
2898		pud = pud_offset(p4d, addr);
2899	}
2900	do {
2901		next = pud_addr_end(addr, end);
2902		if (pud_none(*pud) && !create)
2903			continue;
2904		if (WARN_ON_ONCE(pud_leaf(*pud)))
2905			return -EINVAL;
2906		if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
2907			if (!create)
2908				continue;
2909			pud_clear_bad(pud);
2910		}
2911		err = apply_to_pmd_range(mm, pud, addr, next,
2912					 fn, data, create, mask);
2913		if (err)
2914			break;
2915	} while (pud++, addr = next, addr != end);
2916
2917	return err;
2918}
2919
2920static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2921				     unsigned long addr, unsigned long end,
2922				     pte_fn_t fn, void *data, bool create,
2923				     pgtbl_mod_mask *mask)
2924{
2925	p4d_t *p4d;
2926	unsigned long next;
2927	int err = 0;
2928
2929	if (create) {
2930		p4d = p4d_alloc_track(mm, pgd, addr, mask);
2931		if (!p4d)
2932			return -ENOMEM;
2933	} else {
2934		p4d = p4d_offset(pgd, addr);
2935	}
2936	do {
2937		next = p4d_addr_end(addr, end);
2938		if (p4d_none(*p4d) && !create)
2939			continue;
2940		if (WARN_ON_ONCE(p4d_leaf(*p4d)))
2941			return -EINVAL;
2942		if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
2943			if (!create)
2944				continue;
2945			p4d_clear_bad(p4d);
2946		}
2947		err = apply_to_pud_range(mm, p4d, addr, next,
2948					 fn, data, create, mask);
2949		if (err)
2950			break;
2951	} while (p4d++, addr = next, addr != end);
2952
2953	return err;
2954}
2955
2956static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2957				 unsigned long size, pte_fn_t fn,
2958				 void *data, bool create)
2959{
2960	pgd_t *pgd;
2961	unsigned long start = addr, next;
2962	unsigned long end = addr + size;
2963	pgtbl_mod_mask mask = 0;
2964	int err = 0;
2965
2966	if (WARN_ON(addr >= end))
2967		return -EINVAL;
2968
2969	pgd = pgd_offset(mm, addr);
2970	do {
2971		next = pgd_addr_end(addr, end);
2972		if (pgd_none(*pgd) && !create)
2973			continue;
2974		if (WARN_ON_ONCE(pgd_leaf(*pgd))) {
2975			err = -EINVAL;
2976			break;
2977		}
2978		if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
2979			if (!create)
2980				continue;
2981			pgd_clear_bad(pgd);
2982		}
2983		err = apply_to_p4d_range(mm, pgd, addr, next,
2984					 fn, data, create, &mask);
2985		if (err)
2986			break;
2987	} while (pgd++, addr = next, addr != end);
2988
2989	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
2990		arch_sync_kernel_mappings(start, start + size);
2991
2992	return err;
2993}
2994
2995/*
2996 * Scan a region of virtual memory, filling in page tables as necessary
2997 * and calling a provided function on each leaf page table.
2998 */
2999int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
3000			unsigned long size, pte_fn_t fn, void *data)
3001{
3002	return __apply_to_page_range(mm, addr, size, fn, data, true);
3003}
3004EXPORT_SYMBOL_GPL(apply_to_page_range);
3005
3006/*
3007 * Scan a region of virtual memory, calling a provided function on
3008 * each leaf page table where it exists.
3009 *
3010 * Unlike apply_to_page_range, this does _not_ fill in page tables
3011 * where they are absent.
3012 */
3013int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
3014				 unsigned long size, pte_fn_t fn, void *data)
3015{
3016	return __apply_to_page_range(mm, addr, size, fn, data, false);
3017}
3018EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
3019
3020/*
3021 * handle_pte_fault chooses page fault handler according to an entry which was
3022 * read non-atomically.  Before making any commitment, on those architectures
3023 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
3024 * parts, do_swap_page must check under lock before unmapping the pte and
3025 * proceeding (but do_wp_page is only called after already making such a check;
3026 * and do_anonymous_page can safely check later on).
3027 */
3028static inline int pte_unmap_same(struct vm_fault *vmf)
 
3029{
3030	int same = 1;
3031#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
3032	if (sizeof(pte_t) > sizeof(unsigned long)) {
3033		spin_lock(vmf->ptl);
3034		same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
3035		spin_unlock(vmf->ptl);
 
3036	}
3037#endif
3038	pte_unmap(vmf->pte);
3039	vmf->pte = NULL;
3040	return same;
3041}
3042
3043/*
3044 * Return:
3045 *	0:		copied succeeded
3046 *	-EHWPOISON:	copy failed due to hwpoison in source page
3047 *	-EAGAIN:	copied failed (some other reason)
3048 */
3049static inline int __wp_page_copy_user(struct page *dst, struct page *src,
3050				      struct vm_fault *vmf)
3051{
3052	int ret;
3053	void *kaddr;
3054	void __user *uaddr;
 
3055	struct vm_area_struct *vma = vmf->vma;
3056	struct mm_struct *mm = vma->vm_mm;
3057	unsigned long addr = vmf->address;
3058
3059	if (likely(src)) {
3060		if (copy_mc_user_highpage(dst, src, addr, vma))
3061			return -EHWPOISON;
3062		return 0;
3063	}
3064
3065	/*
3066	 * If the source page was a PFN mapping, we don't have
3067	 * a "struct page" for it. We do a best-effort copy by
3068	 * just copying from the original user address. If that
3069	 * fails, we just zero-fill it. Live with it.
3070	 */
3071	kaddr = kmap_local_page(dst);
3072	pagefault_disable();
3073	uaddr = (void __user *)(addr & PAGE_MASK);
3074
3075	/*
3076	 * On architectures with software "accessed" bits, we would
3077	 * take a double page fault, so mark it accessed here.
3078	 */
3079	vmf->pte = NULL;
3080	if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
3081		pte_t entry;
3082
3083		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
3084		if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
 
3085			/*
3086			 * Other thread has already handled the fault
3087			 * and update local tlb only
3088			 */
3089			if (vmf->pte)
3090				update_mmu_tlb(vma, addr, vmf->pte);
3091			ret = -EAGAIN;
3092			goto pte_unlock;
3093		}
3094
3095		entry = pte_mkyoung(vmf->orig_pte);
3096		if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
3097			update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
3098	}
3099
3100	/*
3101	 * This really shouldn't fail, because the page is there
3102	 * in the page tables. But it might just be unreadable,
3103	 * in which case we just give up and fill the result with
3104	 * zeroes.
3105	 */
3106	if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
3107		if (vmf->pte)
3108			goto warn;
3109
3110		/* Re-validate under PTL if the page is still mapped */
3111		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
3112		if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
 
3113			/* The PTE changed under us, update local tlb */
3114			if (vmf->pte)
3115				update_mmu_tlb(vma, addr, vmf->pte);
3116			ret = -EAGAIN;
3117			goto pte_unlock;
3118		}
3119
3120		/*
3121		 * The same page can be mapped back since last copy attempt.
3122		 * Try to copy again under PTL.
3123		 */
3124		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
3125			/*
3126			 * Give a warn in case there can be some obscure
3127			 * use-case
3128			 */
3129warn:
3130			WARN_ON_ONCE(1);
3131			clear_page(kaddr);
3132		}
3133	}
3134
3135	ret = 0;
3136
3137pte_unlock:
3138	if (vmf->pte)
3139		pte_unmap_unlock(vmf->pte, vmf->ptl);
3140	pagefault_enable();
3141	kunmap_local(kaddr);
3142	flush_dcache_page(dst);
3143
3144	return ret;
3145}
3146
3147static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
3148{
3149	struct file *vm_file = vma->vm_file;
3150
3151	if (vm_file)
3152		return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
3153
3154	/*
3155	 * Special mappings (e.g. VDSO) do not have any file so fake
3156	 * a default GFP_KERNEL for them.
3157	 */
3158	return GFP_KERNEL;
3159}
3160
3161/*
3162 * Notify the address space that the page is about to become writable so that
3163 * it can prohibit this or wait for the page to get into an appropriate state.
3164 *
3165 * We do this without the lock held, so that it can sleep if it needs to.
3166 */
3167static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
3168{
3169	vm_fault_t ret;
 
3170	unsigned int old_flags = vmf->flags;
3171
3172	vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
3173
3174	if (vmf->vma->vm_file &&
3175	    IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
3176		return VM_FAULT_SIGBUS;
3177
3178	ret = vmf->vma->vm_ops->page_mkwrite(vmf);
3179	/* Restore original flags so that caller is not surprised */
3180	vmf->flags = old_flags;
3181	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
3182		return ret;
3183	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
3184		folio_lock(folio);
3185		if (!folio->mapping) {
3186			folio_unlock(folio);
3187			return 0; /* retry */
3188		}
3189		ret |= VM_FAULT_LOCKED;
3190	} else
3191		VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
3192	return ret;
3193}
3194
3195/*
3196 * Handle dirtying of a page in shared file mapping on a write fault.
3197 *
3198 * The function expects the page to be locked and unlocks it.
3199 */
3200static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
3201{
3202	struct vm_area_struct *vma = vmf->vma;
3203	struct address_space *mapping;
3204	struct folio *folio = page_folio(vmf->page);
3205	bool dirtied;
3206	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
3207
3208	dirtied = folio_mark_dirty(folio);
3209	VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
3210	/*
3211	 * Take a local copy of the address_space - folio.mapping may be zeroed
3212	 * by truncate after folio_unlock().   The address_space itself remains
3213	 * pinned by vma->vm_file's reference.  We rely on folio_unlock()'s
3214	 * release semantics to prevent the compiler from undoing this copying.
3215	 */
3216	mapping = folio_raw_mapping(folio);
3217	folio_unlock(folio);
3218
3219	if (!page_mkwrite)
3220		file_update_time(vma->vm_file);
3221
3222	/*
3223	 * Throttle page dirtying rate down to writeback speed.
3224	 *
3225	 * mapping may be NULL here because some device drivers do not
3226	 * set page.mapping but still dirty their pages
3227	 *
3228	 * Drop the mmap_lock before waiting on IO, if we can. The file
3229	 * is pinning the mapping, as per above.
3230	 */
3231	if ((dirtied || page_mkwrite) && mapping) {
3232		struct file *fpin;
3233
3234		fpin = maybe_unlock_mmap_for_io(vmf, NULL);
3235		balance_dirty_pages_ratelimited(mapping);
3236		if (fpin) {
3237			fput(fpin);
3238			return VM_FAULT_COMPLETED;
3239		}
3240	}
3241
3242	return 0;
3243}
3244
3245/*
3246 * Handle write page faults for pages that can be reused in the current vma
3247 *
3248 * This can happen either due to the mapping being with the VM_SHARED flag,
3249 * or due to us being the last reference standing to the page. In either
3250 * case, all we need to do here is to mark the page as writable and update
3251 * any related book-keeping.
3252 */
3253static inline void wp_page_reuse(struct vm_fault *vmf, struct folio *folio)
3254	__releases(vmf->ptl)
3255{
3256	struct vm_area_struct *vma = vmf->vma;
 
3257	pte_t entry;
3258
3259	VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
3260	VM_WARN_ON(is_zero_pfn(pte_pfn(vmf->orig_pte)));
3261
3262	if (folio) {
3263		VM_BUG_ON(folio_test_anon(folio) &&
3264			  !PageAnonExclusive(vmf->page));
3265		/*
3266		 * Clear the folio's cpupid information as the existing
3267		 * information potentially belongs to a now completely
3268		 * unrelated process.
3269		 */
3270		folio_xchg_last_cpupid(folio, (1 << LAST_CPUPID_SHIFT) - 1);
3271	}
3272
3273	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3274	entry = pte_mkyoung(vmf->orig_pte);
3275	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3276	if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
3277		update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3278	pte_unmap_unlock(vmf->pte, vmf->ptl);
3279	count_vm_event(PGREUSE);
3280}
3281
3282/*
3283 * We could add a bitflag somewhere, but for now, we know that all
3284 * vm_ops that have a ->map_pages have been audited and don't need
3285 * the mmap_lock to be held.
3286 */
3287static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf)
3288{
3289	struct vm_area_struct *vma = vmf->vma;
3290
3291	if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK))
3292		return 0;
3293	vma_end_read(vma);
3294	return VM_FAULT_RETRY;
3295}
3296
3297/**
3298 * __vmf_anon_prepare - Prepare to handle an anonymous fault.
3299 * @vmf: The vm_fault descriptor passed from the fault handler.
3300 *
3301 * When preparing to insert an anonymous page into a VMA from a
3302 * fault handler, call this function rather than anon_vma_prepare().
3303 * If this vma does not already have an associated anon_vma and we are
3304 * only protected by the per-VMA lock, the caller must retry with the
3305 * mmap_lock held.  __anon_vma_prepare() will look at adjacent VMAs to
3306 * determine if this VMA can share its anon_vma, and that's not safe to
3307 * do with only the per-VMA lock held for this VMA.
3308 *
3309 * Return: 0 if fault handling can proceed.  Any other value should be
3310 * returned to the caller.
3311 */
3312vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
3313{
3314	struct vm_area_struct *vma = vmf->vma;
3315	vm_fault_t ret = 0;
3316
3317	if (likely(vma->anon_vma))
3318		return 0;
3319	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3320		if (!mmap_read_trylock(vma->vm_mm))
3321			return VM_FAULT_RETRY;
3322	}
3323	if (__anon_vma_prepare(vma))
3324		ret = VM_FAULT_OOM;
3325	if (vmf->flags & FAULT_FLAG_VMA_LOCK)
3326		mmap_read_unlock(vma->vm_mm);
3327	return ret;
3328}
3329
3330/*
3331 * Handle the case of a page which we actually need to copy to a new page,
3332 * either due to COW or unsharing.
3333 *
3334 * Called with mmap_lock locked and the old page referenced, but
3335 * without the ptl held.
3336 *
3337 * High level logic flow:
3338 *
3339 * - Allocate a page, copy the content of the old page to the new one.
3340 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
3341 * - Take the PTL. If the pte changed, bail out and release the allocated page
3342 * - If the pte is still the way we remember it, update the page table and all
3343 *   relevant references. This includes dropping the reference the page-table
3344 *   held to the old page, as well as updating the rmap.
3345 * - In any case, unlock the PTL and drop the reference we took to the old page.
3346 */
3347static vm_fault_t wp_page_copy(struct vm_fault *vmf)
3348{
3349	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3350	struct vm_area_struct *vma = vmf->vma;
3351	struct mm_struct *mm = vma->vm_mm;
3352	struct folio *old_folio = NULL;
3353	struct folio *new_folio = NULL;
3354	pte_t entry;
3355	int page_copied = 0;
3356	struct mmu_notifier_range range;
3357	vm_fault_t ret;
3358	bool pfn_is_zero;
3359
3360	delayacct_wpcopy_start();
3361
3362	if (vmf->page)
3363		old_folio = page_folio(vmf->page);
3364	ret = vmf_anon_prepare(vmf);
3365	if (unlikely(ret))
3366		goto out;
3367
3368	pfn_is_zero = is_zero_pfn(pte_pfn(vmf->orig_pte));
3369	new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero);
3370	if (!new_folio)
3371		goto oom;
3372
3373	if (!pfn_is_zero) {
3374		int err;
 
 
 
 
 
 
 
 
3375
3376		err = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
3377		if (err) {
3378			/*
3379			 * COW failed, if the fault was solved by other,
3380			 * it's fine. If not, userspace would re-fault on
3381			 * the same address and we will handle the fault
3382			 * from the second attempt.
3383			 * The -EHWPOISON case will not be retried.
3384			 */
3385			folio_put(new_folio);
3386			if (old_folio)
3387				folio_put(old_folio);
3388
3389			delayacct_wpcopy_end();
3390			return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
3391		}
3392		kmsan_copy_page_meta(&new_folio->page, vmf->page);
3393	}
3394
3395	__folio_mark_uptodate(new_folio);
 
 
3396
3397	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
 
 
3398				vmf->address & PAGE_MASK,
3399				(vmf->address & PAGE_MASK) + PAGE_SIZE);
3400	mmu_notifier_invalidate_range_start(&range);
3401
3402	/*
3403	 * Re-check the pte - we dropped the lock
3404	 */
3405	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
3406	if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
3407		if (old_folio) {
3408			if (!folio_test_anon(old_folio)) {
3409				dec_mm_counter(mm, mm_counter_file(old_folio));
3410				inc_mm_counter(mm, MM_ANONPAGES);
 
3411			}
3412		} else {
3413			ksm_might_unmap_zero_page(mm, vmf->orig_pte);
3414			inc_mm_counter(mm, MM_ANONPAGES);
3415		}
3416		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3417		entry = mk_pte(&new_folio->page, vma->vm_page_prot);
3418		entry = pte_sw_mkyoung(entry);
3419		if (unlikely(unshare)) {
3420			if (pte_soft_dirty(vmf->orig_pte))
3421				entry = pte_mksoft_dirty(entry);
3422			if (pte_uffd_wp(vmf->orig_pte))
3423				entry = pte_mkuffd_wp(entry);
3424		} else {
3425			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3426		}
3427
3428		/*
3429		 * Clear the pte entry and flush it first, before updating the
3430		 * pte with the new entry, to keep TLBs on different CPUs in
3431		 * sync. This code used to set the new PTE then flush TLBs, but
3432		 * that left a window where the new PTE could be loaded into
3433		 * some TLBs while the old PTE remains in others.
3434		 */
3435		ptep_clear_flush(vma, vmf->address, vmf->pte);
3436		folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE);
3437		folio_add_lru_vma(new_folio, vma);
3438		BUG_ON(unshare && pte_write(entry));
3439		set_pte_at(mm, vmf->address, vmf->pte, entry);
3440		update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3441		if (old_folio) {
 
 
 
 
3442			/*
3443			 * Only after switching the pte to the new page may
3444			 * we remove the mapcount here. Otherwise another
3445			 * process may come and find the rmap count decremented
3446			 * before the pte is switched to the new page, and
3447			 * "reuse" the old page writing into it while our pte
3448			 * here still points into it and can be read by other
3449			 * threads.
3450			 *
3451			 * The critical issue is to order this
3452			 * folio_remove_rmap_pte() with the ptp_clear_flush
3453			 * above. Those stores are ordered by (if nothing else,)
3454			 * the barrier present in the atomic_add_negative
3455			 * in folio_remove_rmap_pte();
3456			 *
3457			 * Then the TLB flush in ptep_clear_flush ensures that
3458			 * no process can access the old page before the
3459			 * decremented mapcount is visible. And the old page
3460			 * cannot be reused until after the decremented
3461			 * mapcount is visible. So transitively, TLBs to
3462			 * old page will be flushed before it can be reused.
3463			 */
3464			folio_remove_rmap_pte(old_folio, vmf->page, vma);
3465		}
3466
3467		/* Free the old page.. */
3468		new_folio = old_folio;
3469		page_copied = 1;
3470		pte_unmap_unlock(vmf->pte, vmf->ptl);
3471	} else if (vmf->pte) {
3472		update_mmu_tlb(vma, vmf->address, vmf->pte);
3473		pte_unmap_unlock(vmf->pte, vmf->ptl);
3474	}
3475
3476	mmu_notifier_invalidate_range_end(&range);
 
3477
3478	if (new_folio)
3479		folio_put(new_folio);
3480	if (old_folio) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3481		if (page_copied)
3482			free_swap_cache(old_folio);
3483		folio_put(old_folio);
3484	}
3485
3486	delayacct_wpcopy_end();
3487	return 0;
3488oom:
3489	ret = VM_FAULT_OOM;
3490out:
3491	if (old_folio)
3492		folio_put(old_folio);
3493
3494	delayacct_wpcopy_end();
3495	return ret;
3496}
3497
3498/**
3499 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
3500 *			  writeable once the page is prepared
3501 *
3502 * @vmf: structure describing the fault
3503 * @folio: the folio of vmf->page
3504 *
3505 * This function handles all that is needed to finish a write page fault in a
3506 * shared mapping due to PTE being read-only once the mapped page is prepared.
3507 * It handles locking of PTE and modifying it.
3508 *
3509 * The function expects the page to be locked or other protection against
3510 * concurrent faults / writeback (such as DAX radix tree locks).
3511 *
3512 * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
3513 * we acquired PTE lock.
3514 */
3515static vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf, struct folio *folio)
3516{
3517	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
3518	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
3519				       &vmf->ptl);
3520	if (!vmf->pte)
3521		return VM_FAULT_NOPAGE;
3522	/*
3523	 * We might have raced with another page fault while we released the
3524	 * pte_offset_map_lock.
3525	 */
3526	if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
3527		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
3528		pte_unmap_unlock(vmf->pte, vmf->ptl);
3529		return VM_FAULT_NOPAGE;
3530	}
3531	wp_page_reuse(vmf, folio);
3532	return 0;
3533}
3534
3535/*
3536 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
3537 * mapping
3538 */
3539static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
3540{
3541	struct vm_area_struct *vma = vmf->vma;
3542
3543	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
3544		vm_fault_t ret;
3545
3546		pte_unmap_unlock(vmf->pte, vmf->ptl);
3547		ret = vmf_can_call_fault(vmf);
3548		if (ret)
3549			return ret;
3550
3551		vmf->flags |= FAULT_FLAG_MKWRITE;
3552		ret = vma->vm_ops->pfn_mkwrite(vmf);
3553		if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
3554			return ret;
3555		return finish_mkwrite_fault(vmf, NULL);
3556	}
3557	wp_page_reuse(vmf, NULL);
3558	return 0;
3559}
3560
3561static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
3562	__releases(vmf->ptl)
3563{
3564	struct vm_area_struct *vma = vmf->vma;
3565	vm_fault_t ret = 0;
3566
3567	folio_get(folio);
3568
3569	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
3570		vm_fault_t tmp;
3571
3572		pte_unmap_unlock(vmf->pte, vmf->ptl);
3573		tmp = vmf_can_call_fault(vmf);
3574		if (tmp) {
3575			folio_put(folio);
3576			return tmp;
3577		}
3578
3579		tmp = do_page_mkwrite(vmf, folio);
3580		if (unlikely(!tmp || (tmp &
3581				      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
3582			folio_put(folio);
3583			return tmp;
3584		}
3585		tmp = finish_mkwrite_fault(vmf, folio);
3586		if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
3587			folio_unlock(folio);
3588			folio_put(folio);
3589			return tmp;
3590		}
3591	} else {
3592		wp_page_reuse(vmf, folio);
3593		folio_lock(folio);
3594	}
3595	ret |= fault_dirty_shared_page(vmf);
3596	folio_put(folio);
3597
3598	return ret;
3599}
3600
3601static bool wp_can_reuse_anon_folio(struct folio *folio,
3602				    struct vm_area_struct *vma)
3603{
3604	/*
3605	 * We could currently only reuse a subpage of a large folio if no
3606	 * other subpages of the large folios are still mapped. However,
3607	 * let's just consistently not reuse subpages even if we could
3608	 * reuse in that scenario, and give back a large folio a bit
3609	 * sooner.
3610	 */
3611	if (folio_test_large(folio))
3612		return false;
3613
3614	/*
3615	 * We have to verify under folio lock: these early checks are
3616	 * just an optimization to avoid locking the folio and freeing
3617	 * the swapcache if there is little hope that we can reuse.
3618	 *
3619	 * KSM doesn't necessarily raise the folio refcount.
3620	 */
3621	if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
3622		return false;
3623	if (!folio_test_lru(folio))
3624		/*
3625		 * We cannot easily detect+handle references from
3626		 * remote LRU caches or references to LRU folios.
3627		 */
3628		lru_add_drain();
3629	if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
3630		return false;
3631	if (!folio_trylock(folio))
3632		return false;
3633	if (folio_test_swapcache(folio))
3634		folio_free_swap(folio);
3635	if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
3636		folio_unlock(folio);
3637		return false;
3638	}
3639	/*
3640	 * Ok, we've got the only folio reference from our mapping
3641	 * and the folio is locked, it's dark out, and we're wearing
3642	 * sunglasses. Hit it.
3643	 */
3644	folio_move_anon_rmap(folio, vma);
3645	folio_unlock(folio);
3646	return true;
3647}
3648
3649/*
3650 * This routine handles present pages, when
3651 * * users try to write to a shared page (FAULT_FLAG_WRITE)
3652 * * GUP wants to take a R/O pin on a possibly shared anonymous page
3653 *   (FAULT_FLAG_UNSHARE)
3654 *
3655 * It is done by copying the page to a new address and decrementing the
3656 * shared-page counter for the old page.
3657 *
3658 * Note that this routine assumes that the protection checks have been
3659 * done by the caller (the low-level page fault routine in most cases).
3660 * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
3661 * done any necessary COW.
3662 *
3663 * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
3664 * though the page will change only once the write actually happens. This
3665 * avoids a few races, and potentially makes it more efficient.
3666 *
3667 * We enter with non-exclusive mmap_lock (to exclude vma changes,
3668 * but allow concurrent faults), with pte both mapped and locked.
3669 * We return with mmap_lock still held, but pte unmapped and unlocked.
3670 */
3671static vm_fault_t do_wp_page(struct vm_fault *vmf)
3672	__releases(vmf->ptl)
3673{
3674	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3675	struct vm_area_struct *vma = vmf->vma;
3676	struct folio *folio = NULL;
3677	pte_t pte;
3678
3679	if (likely(!unshare)) {
3680		if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
3681			if (!userfaultfd_wp_async(vma)) {
3682				pte_unmap_unlock(vmf->pte, vmf->ptl);
3683				return handle_userfault(vmf, VM_UFFD_WP);
3684			}
3685
3686			/*
3687			 * Nothing needed (cache flush, TLB invalidations,
3688			 * etc.) because we're only removing the uffd-wp bit,
3689			 * which is completely invisible to the user.
3690			 */
3691			pte = pte_clear_uffd_wp(ptep_get(vmf->pte));
3692
3693			set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
3694			/*
3695			 * Update this to be prepared for following up CoW
3696			 * handling
3697			 */
3698			vmf->orig_pte = pte;
3699		}
3700
3701		/*
3702		 * Userfaultfd write-protect can defer flushes. Ensure the TLB
3703		 * is flushed in this case before copying.
3704		 */
3705		if (unlikely(userfaultfd_wp(vmf->vma) &&
3706			     mm_tlb_flush_pending(vmf->vma->vm_mm)))
3707			flush_tlb_page(vmf->vma, vmf->address);
3708	}
3709
3710	vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
3711
3712	if (vmf->page)
3713		folio = page_folio(vmf->page);
3714
3715	/*
3716	 * Shared mapping: we are guaranteed to have VM_WRITE and
3717	 * FAULT_FLAG_WRITE set at this point.
3718	 */
3719	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
 
 
 
 
 
3720		/*
3721		 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
3722		 * VM_PFNMAP VMA.
3723		 *
3724		 * We should not cow pages in a shared writeable mapping.
3725		 * Just mark the pages writable and/or call ops->pfn_mkwrite.
3726		 */
3727		if (!vmf->page)
 
3728			return wp_pfn_shared(vmf);
3729		return wp_page_shared(vmf, folio);
 
 
3730	}
3731
3732	/*
3733	 * Private mapping: create an exclusive anonymous page copy if reuse
3734	 * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
3735	 *
3736	 * If we encounter a page that is marked exclusive, we must reuse
3737	 * the page without further checks.
3738	 */
3739	if (folio && folio_test_anon(folio) &&
3740	    (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) {
3741		if (!PageAnonExclusive(vmf->page))
3742			SetPageAnonExclusive(vmf->page);
3743		if (unlikely(unshare)) {
3744			pte_unmap_unlock(vmf->pte, vmf->ptl);
3745			return 0;
 
 
 
 
3746		}
3747		wp_page_reuse(vmf, folio);
3748		return 0;
 
 
 
 
 
 
 
 
 
3749	}
 
3750	/*
3751	 * Ok, we need to copy. Oh, well..
3752	 */
3753	if (folio)
3754		folio_get(folio);
3755
3756	pte_unmap_unlock(vmf->pte, vmf->ptl);
3757#ifdef CONFIG_KSM
3758	if (folio && folio_test_ksm(folio))
3759		count_vm_event(COW_KSM);
3760#endif
3761	return wp_page_copy(vmf);
3762}
3763
3764static void unmap_mapping_range_vma(struct vm_area_struct *vma,
3765		unsigned long start_addr, unsigned long end_addr,
3766		struct zap_details *details)
3767{
3768	zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
3769}
3770
3771static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
3772					    pgoff_t first_index,
3773					    pgoff_t last_index,
3774					    struct zap_details *details)
3775{
3776	struct vm_area_struct *vma;
3777	pgoff_t vba, vea, zba, zea;
3778
3779	vma_interval_tree_foreach(vma, root, first_index, last_index) {
 
 
3780		vba = vma->vm_pgoff;
3781		vea = vba + vma_pages(vma) - 1;
3782		zba = max(first_index, vba);
3783		zea = min(last_index, vea);
 
 
 
 
3784
3785		unmap_mapping_range_vma(vma,
3786			((zba - vba) << PAGE_SHIFT) + vma->vm_start,
3787			((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
3788				details);
3789	}
3790}
3791
3792/**
3793 * unmap_mapping_folio() - Unmap single folio from processes.
3794 * @folio: The locked folio to be unmapped.
3795 *
3796 * Unmap this folio from any userspace process which still has it mmaped.
3797 * Typically, for efficiency, the range of nearby pages has already been
3798 * unmapped by unmap_mapping_pages() or unmap_mapping_range().  But once
3799 * truncation or invalidation holds the lock on a folio, it may find that
3800 * the page has been remapped again: and then uses unmap_mapping_folio()
3801 * to unmap it finally.
3802 */
3803void unmap_mapping_folio(struct folio *folio)
3804{
3805	struct address_space *mapping = folio->mapping;
3806	struct zap_details details = { };
3807	pgoff_t	first_index;
3808	pgoff_t	last_index;
3809
3810	VM_BUG_ON(!folio_test_locked(folio));
3811
3812	first_index = folio->index;
3813	last_index = folio_next_index(folio) - 1;
3814
3815	details.even_cows = false;
3816	details.single_folio = folio;
3817	details.zap_flags = ZAP_FLAG_DROP_MARKER;
 
3818
3819	i_mmap_lock_read(mapping);
3820	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3821		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3822					 last_index, &details);
3823	i_mmap_unlock_read(mapping);
3824}
3825
3826/**
3827 * unmap_mapping_pages() - Unmap pages from processes.
3828 * @mapping: The address space containing pages to be unmapped.
3829 * @start: Index of first page to be unmapped.
3830 * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
3831 * @even_cows: Whether to unmap even private COWed pages.
3832 *
3833 * Unmap the pages in this address space from any userspace process which
3834 * has them mmaped.  Generally, you want to remove COWed pages as well when
3835 * a file is being truncated, but not when invalidating pages from the page
3836 * cache.
3837 */
3838void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
3839		pgoff_t nr, bool even_cows)
3840{
3841	struct zap_details details = { };
3842	pgoff_t	first_index = start;
3843	pgoff_t	last_index = start + nr - 1;
3844
3845	details.even_cows = even_cows;
3846	if (last_index < first_index)
3847		last_index = ULONG_MAX;
 
 
3848
3849	i_mmap_lock_read(mapping);
3850	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3851		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3852					 last_index, &details);
3853	i_mmap_unlock_read(mapping);
3854}
3855EXPORT_SYMBOL_GPL(unmap_mapping_pages);
3856
3857/**
3858 * unmap_mapping_range - unmap the portion of all mmaps in the specified
3859 * address_space corresponding to the specified byte range in the underlying
3860 * file.
3861 *
3862 * @mapping: the address space containing mmaps to be unmapped.
3863 * @holebegin: byte in first page to unmap, relative to the start of
3864 * the underlying file.  This will be rounded down to a PAGE_SIZE
3865 * boundary.  Note that this is different from truncate_pagecache(), which
3866 * must keep the partial page.  In contrast, we must get rid of
3867 * partial pages.
3868 * @holelen: size of prospective hole in bytes.  This will be rounded
3869 * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
3870 * end of the file.
3871 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
3872 * but 0 when invalidating pagecache, don't throw away private data.
3873 */
3874void unmap_mapping_range(struct address_space *mapping,
3875		loff_t const holebegin, loff_t const holelen, int even_cows)
3876{
3877	pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
3878	pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
3879
3880	/* Check for overflow. */
3881	if (sizeof(holelen) > sizeof(hlen)) {
3882		long long holeend =
3883			(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3884		if (holeend & ~(long long)ULONG_MAX)
3885			hlen = ULONG_MAX - hba + 1;
3886	}
3887
3888	unmap_mapping_pages(mapping, hba, hlen, even_cows);
3889}
3890EXPORT_SYMBOL(unmap_mapping_range);
3891
3892/*
3893 * Restore a potential device exclusive pte to a working pte entry
3894 */
3895static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
3896{
3897	struct folio *folio = page_folio(vmf->page);
3898	struct vm_area_struct *vma = vmf->vma;
3899	struct mmu_notifier_range range;
3900	vm_fault_t ret;
3901
3902	/*
3903	 * We need a reference to lock the folio because we don't hold
3904	 * the PTL so a racing thread can remove the device-exclusive
3905	 * entry and unmap it. If the folio is free the entry must
3906	 * have been removed already. If it happens to have already
3907	 * been re-allocated after being freed all we do is lock and
3908	 * unlock it.
3909	 */
3910	if (!folio_try_get(folio))
3911		return 0;
3912
3913	ret = folio_lock_or_retry(folio, vmf);
3914	if (ret) {
3915		folio_put(folio);
3916		return ret;
3917	}
3918	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
3919				vma->vm_mm, vmf->address & PAGE_MASK,
3920				(vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
3921	mmu_notifier_invalidate_range_start(&range);
3922
3923	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3924				&vmf->ptl);
3925	if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3926		restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
3927
3928	if (vmf->pte)
3929		pte_unmap_unlock(vmf->pte, vmf->ptl);
3930	folio_unlock(folio);
3931	folio_put(folio);
3932
3933	mmu_notifier_invalidate_range_end(&range);
3934	return 0;
3935}
3936
3937static inline bool should_try_to_free_swap(struct folio *folio,
3938					   struct vm_area_struct *vma,
3939					   unsigned int fault_flags)
3940{
3941	if (!folio_test_swapcache(folio))
3942		return false;
3943	if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
3944	    folio_test_mlocked(folio))
3945		return true;
3946	/*
3947	 * If we want to map a page that's in the swapcache writable, we
3948	 * have to detect via the refcount if we're really the exclusive
3949	 * user. Try freeing the swapcache to get rid of the swapcache
3950	 * reference only in case it's likely that we'll be the exlusive user.
3951	 */
3952	return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
3953		folio_ref_count(folio) == (1 + folio_nr_pages(folio));
3954}
3955
3956static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
3957{
3958	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
3959				       vmf->address, &vmf->ptl);
3960	if (!vmf->pte)
3961		return 0;
3962	/*
3963	 * Be careful so that we will only recover a special uffd-wp pte into a
3964	 * none pte.  Otherwise it means the pte could have changed, so retry.
3965	 *
3966	 * This should also cover the case where e.g. the pte changed
3967	 * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
3968	 * So is_pte_marker() check is not enough to safely drop the pte.
3969	 */
3970	if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
3971		pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
3972	pte_unmap_unlock(vmf->pte, vmf->ptl);
3973	return 0;
3974}
3975
3976static vm_fault_t do_pte_missing(struct vm_fault *vmf)
3977{
3978	if (vma_is_anonymous(vmf->vma))
3979		return do_anonymous_page(vmf);
3980	else
3981		return do_fault(vmf);
3982}
3983
3984/*
3985 * This is actually a page-missing access, but with uffd-wp special pte
3986 * installed.  It means this pte was wr-protected before being unmapped.
3987 */
3988static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
3989{
3990	/*
3991	 * Just in case there're leftover special ptes even after the region
3992	 * got unregistered - we can simply clear them.
3993	 */
3994	if (unlikely(!userfaultfd_wp(vmf->vma)))
3995		return pte_marker_clear(vmf);
3996
3997	return do_pte_missing(vmf);
3998}
3999
4000static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
4001{
4002	swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
4003	unsigned long marker = pte_marker_get(entry);
4004
4005	/*
4006	 * PTE markers should never be empty.  If anything weird happened,
4007	 * the best thing to do is to kill the process along with its mm.
4008	 */
4009	if (WARN_ON_ONCE(!marker))
4010		return VM_FAULT_SIGBUS;
4011
4012	/* Higher priority than uffd-wp when data corrupted */
4013	if (marker & PTE_MARKER_POISONED)
4014		return VM_FAULT_HWPOISON;
4015
4016	/* Hitting a guard page is always a fatal condition. */
4017	if (marker & PTE_MARKER_GUARD)
4018		return VM_FAULT_SIGSEGV;
4019
4020	if (pte_marker_entry_uffd_wp(entry))
4021		return pte_marker_handle_uffd_wp(vmf);
4022
4023	/* This is an unknown pte marker */
4024	return VM_FAULT_SIGBUS;
4025}
4026
4027static struct folio *__alloc_swap_folio(struct vm_fault *vmf)
4028{
4029	struct vm_area_struct *vma = vmf->vma;
4030	struct folio *folio;
4031	swp_entry_t entry;
4032
4033	folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, vmf->address);
4034	if (!folio)
4035		return NULL;
4036
4037	entry = pte_to_swp_entry(vmf->orig_pte);
4038	if (mem_cgroup_swapin_charge_folio(folio, vma->vm_mm,
4039					   GFP_KERNEL, entry)) {
4040		folio_put(folio);
4041		return NULL;
4042	}
4043
4044	return folio;
4045}
4046
4047#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4048static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
4049{
4050	struct swap_info_struct *si = swp_swap_info(entry);
4051	pgoff_t offset = swp_offset(entry);
4052	int i;
4053
4054	/*
4055	 * While allocating a large folio and doing swap_read_folio, which is
4056	 * the case the being faulted pte doesn't have swapcache. We need to
4057	 * ensure all PTEs have no cache as well, otherwise, we might go to
4058	 * swap devices while the content is in swapcache.
4059	 */
4060	for (i = 0; i < max_nr; i++) {
4061		if ((si->swap_map[offset + i] & SWAP_HAS_CACHE))
4062			return i;
4063	}
4064
4065	return i;
4066}
4067
4068/*
4069 * Check if the PTEs within a range are contiguous swap entries
4070 * and have consistent swapcache, zeromap.
4071 */
4072static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
4073{
4074	unsigned long addr;
4075	swp_entry_t entry;
4076	int idx;
4077	pte_t pte;
4078
4079	addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
4080	idx = (vmf->address - addr) / PAGE_SIZE;
4081	pte = ptep_get(ptep);
4082
4083	if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
4084		return false;
4085	entry = pte_to_swp_entry(pte);
4086	if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
4087		return false;
4088
4089	/*
4090	 * swap_read_folio() can't handle the case a large folio is hybridly
4091	 * from different backends. And they are likely corner cases. Similar
4092	 * things might be added once zswap support large folios.
4093	 */
4094	if (unlikely(swap_zeromap_batch(entry, nr_pages, NULL) != nr_pages))
4095		return false;
4096	if (unlikely(non_swapcache_batch(entry, nr_pages) != nr_pages))
4097		return false;
4098
4099	return true;
4100}
4101
4102static inline unsigned long thp_swap_suitable_orders(pgoff_t swp_offset,
4103						     unsigned long addr,
4104						     unsigned long orders)
4105{
4106	int order, nr;
4107
4108	order = highest_order(orders);
4109
4110	/*
4111	 * To swap in a THP with nr pages, we require that its first swap_offset
4112	 * is aligned with that number, as it was when the THP was swapped out.
4113	 * This helps filter out most invalid entries.
4114	 */
4115	while (orders) {
4116		nr = 1 << order;
4117		if ((addr >> PAGE_SHIFT) % nr == swp_offset % nr)
4118			break;
4119		order = next_order(&orders, order);
4120	}
4121
4122	return orders;
4123}
4124
4125static struct folio *alloc_swap_folio(struct vm_fault *vmf)
4126{
4127	struct vm_area_struct *vma = vmf->vma;
4128	unsigned long orders;
4129	struct folio *folio;
4130	unsigned long addr;
4131	swp_entry_t entry;
4132	spinlock_t *ptl;
4133	pte_t *pte;
4134	gfp_t gfp;
4135	int order;
4136
4137	/*
4138	 * If uffd is active for the vma we need per-page fault fidelity to
4139	 * maintain the uffd semantics.
4140	 */
4141	if (unlikely(userfaultfd_armed(vma)))
4142		goto fallback;
4143
4144	/*
4145	 * A large swapped out folio could be partially or fully in zswap. We
4146	 * lack handling for such cases, so fallback to swapping in order-0
4147	 * folio.
4148	 */
4149	if (!zswap_never_enabled())
4150		goto fallback;
4151
4152	entry = pte_to_swp_entry(vmf->orig_pte);
4153	/*
4154	 * Get a list of all the (large) orders below PMD_ORDER that are enabled
4155	 * and suitable for swapping THP.
4156	 */
4157	orders = thp_vma_allowable_orders(vma, vma->vm_flags,
4158			TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1);
4159	orders = thp_vma_suitable_orders(vma, vmf->address, orders);
4160	orders = thp_swap_suitable_orders(swp_offset(entry),
4161					  vmf->address, orders);
4162
4163	if (!orders)
4164		goto fallback;
4165
4166	pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
4167				  vmf->address & PMD_MASK, &ptl);
4168	if (unlikely(!pte))
4169		goto fallback;
4170
4171	/*
4172	 * For do_swap_page, find the highest order where the aligned range is
4173	 * completely swap entries with contiguous swap offsets.
4174	 */
4175	order = highest_order(orders);
4176	while (orders) {
4177		addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
4178		if (can_swapin_thp(vmf, pte + pte_index(addr), 1 << order))
4179			break;
4180		order = next_order(&orders, order);
4181	}
4182
4183	pte_unmap_unlock(pte, ptl);
4184
4185	/* Try allocating the highest of the remaining orders. */
4186	gfp = vma_thp_gfp_mask(vma);
4187	while (orders) {
4188		addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
4189		folio = vma_alloc_folio(gfp, order, vma, addr);
4190		if (folio) {
4191			if (!mem_cgroup_swapin_charge_folio(folio, vma->vm_mm,
4192							    gfp, entry))
4193				return folio;
4194			folio_put(folio);
4195		}
4196		order = next_order(&orders, order);
4197	}
4198
4199fallback:
4200	return __alloc_swap_folio(vmf);
4201}
4202#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
4203static struct folio *alloc_swap_folio(struct vm_fault *vmf)
4204{
4205	return __alloc_swap_folio(vmf);
4206}
4207#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4208
4209static DECLARE_WAIT_QUEUE_HEAD(swapcache_wq);
4210
4211/*
4212 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4213 * but allow concurrent faults), and pte mapped but not yet locked.
4214 * We return with pte unmapped and unlocked.
4215 *
4216 * We return with the mmap_lock locked or unlocked in the same cases
4217 * as does filemap_fault().
4218 */
4219vm_fault_t do_swap_page(struct vm_fault *vmf)
4220{
4221	struct vm_area_struct *vma = vmf->vma;
4222	struct folio *swapcache, *folio = NULL;
4223	DECLARE_WAITQUEUE(wait, current);
4224	struct page *page;
4225	struct swap_info_struct *si = NULL;
4226	rmap_t rmap_flags = RMAP_NONE;
4227	bool need_clear_cache = false;
4228	bool exclusive = false;
4229	swp_entry_t entry;
4230	pte_t pte;
 
 
4231	vm_fault_t ret = 0;
4232	void *shadow = NULL;
4233	int nr_pages;
4234	unsigned long page_idx;
4235	unsigned long address;
4236	pte_t *ptep;
4237
4238	if (!pte_unmap_same(vmf))
4239		goto out;
4240
4241	entry = pte_to_swp_entry(vmf->orig_pte);
4242	if (unlikely(non_swap_entry(entry))) {
4243		if (is_migration_entry(entry)) {
4244			migration_entry_wait(vma->vm_mm, vmf->pmd,
4245					     vmf->address);
4246		} else if (is_device_exclusive_entry(entry)) {
4247			vmf->page = pfn_swap_entry_to_page(entry);
4248			ret = remove_device_exclusive_entry(vmf);
4249		} else if (is_device_private_entry(entry)) {
4250			if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4251				/*
4252				 * migrate_to_ram is not yet ready to operate
4253				 * under VMA lock.
4254				 */
4255				vma_end_read(vma);
4256				ret = VM_FAULT_RETRY;
4257				goto out;
4258			}
4259
4260			vmf->page = pfn_swap_entry_to_page(entry);
4261			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4262					vmf->address, &vmf->ptl);
4263			if (unlikely(!vmf->pte ||
4264				     !pte_same(ptep_get(vmf->pte),
4265							vmf->orig_pte)))
4266				goto unlock;
4267
4268			/*
4269			 * Get a page reference while we know the page can't be
4270			 * freed.
4271			 */
4272			get_page(vmf->page);
4273			pte_unmap_unlock(vmf->pte, vmf->ptl);
4274			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
4275			put_page(vmf->page);
4276		} else if (is_hwpoison_entry(entry)) {
4277			ret = VM_FAULT_HWPOISON;
4278		} else if (is_pte_marker_entry(entry)) {
4279			ret = handle_pte_marker(vmf);
4280		} else {
4281			print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
4282			ret = VM_FAULT_SIGBUS;
4283		}
4284		goto out;
4285	}
4286
4287	/* Prevent swapoff from happening to us. */
4288	si = get_swap_device(entry);
4289	if (unlikely(!si))
4290		goto out;
4291
4292	folio = swap_cache_get_folio(entry, vma, vmf->address);
4293	if (folio)
4294		page = folio_file_page(folio, swp_offset(entry));
4295	swapcache = folio;
4296
4297	if (!folio) {
4298		if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
4299		    __swap_count(entry) == 1) {
4300			/* skip swapcache */
4301			folio = alloc_swap_folio(vmf);
4302			if (folio) {
4303				__folio_set_locked(folio);
4304				__folio_set_swapbacked(folio);
4305
4306				nr_pages = folio_nr_pages(folio);
4307				if (folio_test_large(folio))
4308					entry.val = ALIGN_DOWN(entry.val, nr_pages);
4309				/*
4310				 * Prevent parallel swapin from proceeding with
4311				 * the cache flag. Otherwise, another thread
4312				 * may finish swapin first, free the entry, and
4313				 * swapout reusing the same entry. It's
4314				 * undetectable as pte_same() returns true due
4315				 * to entry reuse.
4316				 */
4317				if (swapcache_prepare(entry, nr_pages)) {
4318					/*
4319					 * Relax a bit to prevent rapid
4320					 * repeated page faults.
4321					 */
4322					add_wait_queue(&swapcache_wq, &wait);
4323					schedule_timeout_uninterruptible(1);
4324					remove_wait_queue(&swapcache_wq, &wait);
4325					goto out_page;
4326				}
4327				need_clear_cache = true;
4328
4329				mem_cgroup_swapin_uncharge_swap(entry, nr_pages);
4330
4331				shadow = get_shadow_from_swap_cache(entry);
4332				if (shadow)
4333					workingset_refault(folio, shadow);
4334
4335				folio_add_lru(folio);
4336
4337				/* To provide entry to swap_read_folio() */
4338				folio->swap = entry;
4339				swap_read_folio(folio, NULL);
4340				folio->private = NULL;
4341			}
4342		} else {
4343			folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
4344						vmf);
4345			swapcache = folio;
4346		}
4347
4348		if (!folio) {
4349			/*
4350			 * Back out if somebody else faulted in this pte
4351			 * while we released the pte lock.
4352			 */
4353			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4354					vmf->address, &vmf->ptl);
4355			if (likely(vmf->pte &&
4356				   pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
4357				ret = VM_FAULT_OOM;
 
4358			goto unlock;
4359		}
4360
4361		/* Had to read the page from swap area: Major fault */
4362		ret = VM_FAULT_MAJOR;
4363		count_vm_event(PGMAJFAULT);
4364		count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
4365		page = folio_file_page(folio, swp_offset(entry));
4366	} else if (PageHWPoison(page)) {
4367		/*
4368		 * hwpoisoned dirty swapcache pages are kept for killing
4369		 * owner processes (which may be unknown at hwpoison time)
4370		 */
4371		ret = VM_FAULT_HWPOISON;
 
4372		goto out_release;
4373	}
4374
4375	ret |= folio_lock_or_retry(folio, vmf);
4376	if (ret & VM_FAULT_RETRY)
4377		goto out_release;
4378
4379	if (swapcache) {
4380		/*
4381		 * Make sure folio_free_swap() or swapoff did not release the
4382		 * swapcache from under us.  The page pin, and pte_same test
4383		 * below, are not enough to exclude that.  Even if it is still
4384		 * swapcache, we need to check that the page's swap has not
4385		 * changed.
4386		 */
4387		if (unlikely(!folio_test_swapcache(folio) ||
4388			     page_swap_entry(page).val != entry.val))
4389			goto out_page;
4390
4391		/*
4392		 * KSM sometimes has to copy on read faults, for example, if
4393		 * page->index of !PageKSM() pages would be nonlinear inside the
4394		 * anon VMA -- PageKSM() is lost on actual swapout.
4395		 */
4396		folio = ksm_might_need_to_copy(folio, vma, vmf->address);
4397		if (unlikely(!folio)) {
4398			ret = VM_FAULT_OOM;
4399			folio = swapcache;
4400			goto out_page;
4401		} else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
4402			ret = VM_FAULT_HWPOISON;
4403			folio = swapcache;
4404			goto out_page;
4405		}
4406		if (folio != swapcache)
4407			page = folio_page(folio, 0);
4408
4409		/*
4410		 * If we want to map a page that's in the swapcache writable, we
4411		 * have to detect via the refcount if we're really the exclusive
4412		 * owner. Try removing the extra reference from the local LRU
4413		 * caches if required.
4414		 */
4415		if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
4416		    !folio_test_ksm(folio) && !folio_test_lru(folio))
4417			lru_add_drain();
4418	}
4419
4420	folio_throttle_swaprate(folio, GFP_KERNEL);
4421
4422	/*
4423	 * Back out if somebody else already faulted in this pte.
4424	 */
4425	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4426			&vmf->ptl);
4427	if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
4428		goto out_nomap;
4429
4430	if (unlikely(!folio_test_uptodate(folio))) {
4431		ret = VM_FAULT_SIGBUS;
4432		goto out_nomap;
4433	}
4434
4435	/* allocated large folios for SWP_SYNCHRONOUS_IO */
4436	if (folio_test_large(folio) && !folio_test_swapcache(folio)) {
4437		unsigned long nr = folio_nr_pages(folio);
4438		unsigned long folio_start = ALIGN_DOWN(vmf->address, nr * PAGE_SIZE);
4439		unsigned long idx = (vmf->address - folio_start) / PAGE_SIZE;
4440		pte_t *folio_ptep = vmf->pte - idx;
4441		pte_t folio_pte = ptep_get(folio_ptep);
4442
4443		if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
4444		    swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
4445			goto out_nomap;
4446
4447		page_idx = idx;
4448		address = folio_start;
4449		ptep = folio_ptep;
4450		goto check_folio;
4451	}
4452
4453	nr_pages = 1;
4454	page_idx = 0;
4455	address = vmf->address;
4456	ptep = vmf->pte;
4457	if (folio_test_large(folio) && folio_test_swapcache(folio)) {
4458		int nr = folio_nr_pages(folio);
4459		unsigned long idx = folio_page_idx(folio, page);
4460		unsigned long folio_start = address - idx * PAGE_SIZE;
4461		unsigned long folio_end = folio_start + nr * PAGE_SIZE;
4462		pte_t *folio_ptep;
4463		pte_t folio_pte;
4464
4465		if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start)))
4466			goto check_folio;
4467		if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end)))
4468			goto check_folio;
4469
4470		folio_ptep = vmf->pte - idx;
4471		folio_pte = ptep_get(folio_ptep);
4472		if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
4473		    swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
4474			goto check_folio;
4475
4476		page_idx = idx;
4477		address = folio_start;
4478		ptep = folio_ptep;
4479		nr_pages = nr;
4480		entry = folio->swap;
4481		page = &folio->page;
4482	}
4483
4484check_folio:
4485	/*
4486	 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
4487	 * must never point at an anonymous page in the swapcache that is
4488	 * PG_anon_exclusive. Sanity check that this holds and especially, that
4489	 * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
4490	 * check after taking the PT lock and making sure that nobody
4491	 * concurrently faulted in this page and set PG_anon_exclusive.
4492	 */
4493	BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
4494	BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
4495
4496	/*
4497	 * Check under PT lock (to protect against concurrent fork() sharing
4498	 * the swap entry concurrently) for certainly exclusive pages.
4499	 */
4500	if (!folio_test_ksm(folio)) {
4501		exclusive = pte_swp_exclusive(vmf->orig_pte);
4502		if (folio != swapcache) {
4503			/*
4504			 * We have a fresh page that is not exposed to the
4505			 * swapcache -> certainly exclusive.
4506			 */
4507			exclusive = true;
4508		} else if (exclusive && folio_test_writeback(folio) &&
4509			  data_race(si->flags & SWP_STABLE_WRITES)) {
4510			/*
4511			 * This is tricky: not all swap backends support
4512			 * concurrent page modifications while under writeback.
4513			 *
4514			 * So if we stumble over such a page in the swapcache
4515			 * we must not set the page exclusive, otherwise we can
4516			 * map it writable without further checks and modify it
4517			 * while still under writeback.
4518			 *
4519			 * For these problematic swap backends, simply drop the
4520			 * exclusive marker: this is perfectly fine as we start
4521			 * writeback only if we fully unmapped the page and
4522			 * there are no unexpected references on the page after
4523			 * unmapping succeeded. After fully unmapped, no
4524			 * further GUP references (FOLL_GET and FOLL_PIN) can
4525			 * appear, so dropping the exclusive marker and mapping
4526			 * it only R/O is fine.
4527			 */
4528			exclusive = false;
4529		}
4530	}
4531
4532	/*
4533	 * Some architectures may have to restore extra metadata to the page
4534	 * when reading from swap. This metadata may be indexed by swap entry
4535	 * so this must be called before swap_free().
4536	 */
4537	arch_swap_restore(folio_swap(entry, folio), folio);
4538
4539	/*
4540	 * Remove the swap entry and conditionally try to free up the swapcache.
4541	 * We're already holding a reference on the page but haven't mapped it
4542	 * yet.
 
 
 
 
4543	 */
4544	swap_free_nr(entry, nr_pages);
4545	if (should_try_to_free_swap(folio, vma, vmf->flags))
4546		folio_free_swap(folio);
4547
4548	add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
4549	add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages);
4550	pte = mk_pte(page, vma->vm_page_prot);
 
 
 
 
 
 
 
4551	if (pte_swp_soft_dirty(vmf->orig_pte))
4552		pte = pte_mksoft_dirty(pte);
4553	if (pte_swp_uffd_wp(vmf->orig_pte))
4554		pte = pte_mkuffd_wp(pte);
4555
4556	/*
4557	 * Same logic as in do_wp_page(); however, optimize for pages that are
4558	 * certainly not shared either because we just allocated them without
4559	 * exposing them to the swapcache or because the swap entry indicates
4560	 * exclusivity.
4561	 */
4562	if (!folio_test_ksm(folio) &&
4563	    (exclusive || folio_ref_count(folio) == 1)) {
4564		if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
4565		    !pte_needs_soft_dirty_wp(vma, pte)) {
4566			pte = pte_mkwrite(pte, vma);
4567			if (vmf->flags & FAULT_FLAG_WRITE) {
4568				pte = pte_mkdirty(pte);
4569				vmf->flags &= ~FAULT_FLAG_WRITE;
4570			}
4571		}
4572		rmap_flags |= RMAP_EXCLUSIVE;
4573	}
4574	folio_ref_add(folio, nr_pages - 1);
4575	flush_icache_pages(vma, page, nr_pages);
4576	vmf->orig_pte = pte_advance_pfn(pte, page_idx);
4577
4578	/* ksm created a completely new copy */
4579	if (unlikely(folio != swapcache && swapcache)) {
4580		folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
4581		folio_add_lru_vma(folio, vma);
4582	} else if (!folio_test_anon(folio)) {
4583		/*
4584		 * We currently only expect small !anon folios which are either
4585		 * fully exclusive or fully shared, or new allocated large
4586		 * folios which are fully exclusive. If we ever get large
4587		 * folios within swapcache here, we have to be careful.
4588		 */
4589		VM_WARN_ON_ONCE(folio_test_large(folio) && folio_test_swapcache(folio));
4590		VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
4591		folio_add_new_anon_rmap(folio, vma, address, rmap_flags);
4592	} else {
4593		folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address,
4594					rmap_flags);
4595	}
4596
4597	VM_BUG_ON(!folio_test_anon(folio) ||
4598			(pte_write(pte) && !PageAnonExclusive(page)));
4599	set_ptes(vma->vm_mm, address, ptep, pte, nr_pages);
4600	arch_do_swap_page_nr(vma->vm_mm, vma, address,
4601			pte, pte, nr_pages);
4602
4603	folio_unlock(folio);
4604	if (folio != swapcache && swapcache) {
4605		/*
4606		 * Hold the lock to avoid the swap entry to be reused
4607		 * until we take the PT lock for the pte_same() check
4608		 * (to avoid false positives from pte_same). For
4609		 * further safety release the lock after the swap_free
4610		 * so that the swap count won't change under a
4611		 * parallel locked swapcache.
4612		 */
4613		folio_unlock(swapcache);
4614		folio_put(swapcache);
4615	}
4616
4617	if (vmf->flags & FAULT_FLAG_WRITE) {
4618		ret |= do_wp_page(vmf);
4619		if (ret & VM_FAULT_ERROR)
4620			ret &= VM_FAULT_ERROR;
4621		goto out;
4622	}
4623
4624	/* No need to invalidate - it was non-present before */
4625	update_mmu_cache_range(vmf, vma, address, ptep, nr_pages);
4626unlock:
4627	if (vmf->pte)
4628		pte_unmap_unlock(vmf->pte, vmf->ptl);
4629out:
4630	/* Clear the swap cache pin for direct swapin after PTL unlock */
4631	if (need_clear_cache) {
4632		swapcache_clear(si, entry, nr_pages);
4633		if (waitqueue_active(&swapcache_wq))
4634			wake_up(&swapcache_wq);
4635	}
4636	if (si)
4637		put_swap_device(si);
4638	return ret;
4639out_nomap:
4640	if (vmf->pte)
4641		pte_unmap_unlock(vmf->pte, vmf->ptl);
4642out_page:
4643	folio_unlock(folio);
4644out_release:
4645	folio_put(folio);
4646	if (folio != swapcache && swapcache) {
4647		folio_unlock(swapcache);
4648		folio_put(swapcache);
4649	}
4650	if (need_clear_cache) {
4651		swapcache_clear(si, entry, nr_pages);
4652		if (waitqueue_active(&swapcache_wq))
4653			wake_up(&swapcache_wq);
4654	}
4655	if (si)
4656		put_swap_device(si);
4657	return ret;
4658}
4659
4660static bool pte_range_none(pte_t *pte, int nr_pages)
4661{
4662	int i;
4663
4664	for (i = 0; i < nr_pages; i++) {
4665		if (!pte_none(ptep_get_lockless(pte + i)))
4666			return false;
4667	}
4668
4669	return true;
4670}
4671
4672static struct folio *alloc_anon_folio(struct vm_fault *vmf)
4673{
4674	struct vm_area_struct *vma = vmf->vma;
4675#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4676	unsigned long orders;
4677	struct folio *folio;
4678	unsigned long addr;
4679	pte_t *pte;
4680	gfp_t gfp;
4681	int order;
4682
4683	/*
4684	 * If uffd is active for the vma we need per-page fault fidelity to
4685	 * maintain the uffd semantics.
4686	 */
4687	if (unlikely(userfaultfd_armed(vma)))
4688		goto fallback;
4689
4690	/*
4691	 * Get a list of all the (large) orders below PMD_ORDER that are enabled
4692	 * for this vma. Then filter out the orders that can't be allocated over
4693	 * the faulting address and still be fully contained in the vma.
4694	 */
4695	orders = thp_vma_allowable_orders(vma, vma->vm_flags,
4696			TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1);
4697	orders = thp_vma_suitable_orders(vma, vmf->address, orders);
4698
4699	if (!orders)
4700		goto fallback;
4701
4702	pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK);
4703	if (!pte)
4704		return ERR_PTR(-EAGAIN);
4705
4706	/*
4707	 * Find the highest order where the aligned range is completely
4708	 * pte_none(). Note that all remaining orders will be completely
4709	 * pte_none().
4710	 */
4711	order = highest_order(orders);
4712	while (orders) {
4713		addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
4714		if (pte_range_none(pte + pte_index(addr), 1 << order))
4715			break;
4716		order = next_order(&orders, order);
4717	}
4718
4719	pte_unmap(pte);
4720
4721	if (!orders)
4722		goto fallback;
4723
4724	/* Try allocating the highest of the remaining orders. */
4725	gfp = vma_thp_gfp_mask(vma);
4726	while (orders) {
4727		addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
4728		folio = vma_alloc_folio(gfp, order, vma, addr);
4729		if (folio) {
4730			if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
4731				count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
4732				folio_put(folio);
4733				goto next;
4734			}
4735			folio_throttle_swaprate(folio, gfp);
4736			/*
4737			 * When a folio is not zeroed during allocation
4738			 * (__GFP_ZERO not used) or user folios require special
4739			 * handling, folio_zero_user() is used to make sure
4740			 * that the page corresponding to the faulting address
4741			 * will be hot in the cache after zeroing.
4742			 */
4743			if (user_alloc_needs_zeroing())
4744				folio_zero_user(folio, vmf->address);
4745			return folio;
4746		}
4747next:
4748		count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
4749		order = next_order(&orders, order);
4750	}
4751
4752fallback:
4753#endif
4754	return folio_prealloc(vma->vm_mm, vma, vmf->address, true);
4755}
4756
4757/*
4758 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4759 * but allow concurrent faults), and pte mapped but not yet locked.
4760 * We return with mmap_lock still held, but pte unmapped and unlocked.
4761 */
4762static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4763{
4764	struct vm_area_struct *vma = vmf->vma;
4765	unsigned long addr = vmf->address;
4766	struct folio *folio;
4767	vm_fault_t ret = 0;
4768	int nr_pages = 1;
4769	pte_t entry;
4770
4771	/* File mapping without ->vm_ops ? */
4772	if (vma->vm_flags & VM_SHARED)
4773		return VM_FAULT_SIGBUS;
4774
4775	/*
4776	 * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
4777	 * be distinguished from a transient failure of pte_offset_map().
 
 
 
 
 
 
4778	 */
4779	if (pte_alloc(vma->vm_mm, vmf->pmd))
4780		return VM_FAULT_OOM;
4781
 
 
 
 
4782	/* Use the zero-page for reads */
4783	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4784			!mm_forbids_zeropage(vma->vm_mm)) {
4785		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4786						vma->vm_page_prot));
4787		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4788				vmf->address, &vmf->ptl);
4789		if (!vmf->pte)
4790			goto unlock;
4791		if (vmf_pte_changed(vmf)) {
4792			update_mmu_tlb(vma, vmf->address, vmf->pte);
4793			goto unlock;
4794		}
4795		ret = check_stable_address_space(vma->vm_mm);
4796		if (ret)
4797			goto unlock;
4798		/* Deliver the page fault to userland, check inside PT lock */
4799		if (userfaultfd_missing(vma)) {
4800			pte_unmap_unlock(vmf->pte, vmf->ptl);
4801			return handle_userfault(vmf, VM_UFFD_MISSING);
4802		}
4803		goto setpte;
4804	}
4805
4806	/* Allocate our own private page. */
4807	ret = vmf_anon_prepare(vmf);
4808	if (ret)
4809		return ret;
4810	/* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */
4811	folio = alloc_anon_folio(vmf);
4812	if (IS_ERR(folio))
4813		return 0;
4814	if (!folio)
4815		goto oom;
4816
4817	nr_pages = folio_nr_pages(folio);
4818	addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
 
4819
4820	/*
4821	 * The memory barrier inside __folio_mark_uptodate makes sure that
4822	 * preceding stores to the page contents become visible before
4823	 * the set_pte_at() write.
4824	 */
4825	__folio_mark_uptodate(folio);
4826
4827	entry = mk_pte(&folio->page, vma->vm_page_prot);
4828	entry = pte_sw_mkyoung(entry);
4829	if (vma->vm_flags & VM_WRITE)
4830		entry = pte_mkwrite(pte_mkdirty(entry), vma);
4831
4832	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
4833	if (!vmf->pte)
4834		goto release;
4835	if (nr_pages == 1 && vmf_pte_changed(vmf)) {
4836		update_mmu_tlb(vma, addr, vmf->pte);
4837		goto release;
4838	} else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
4839		update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
4840		goto release;
4841	}
4842
4843	ret = check_stable_address_space(vma->vm_mm);
4844	if (ret)
4845		goto release;
4846
4847	/* Deliver the page fault to userland, check inside PT lock */
4848	if (userfaultfd_missing(vma)) {
4849		pte_unmap_unlock(vmf->pte, vmf->ptl);
4850		folio_put(folio);
4851		return handle_userfault(vmf, VM_UFFD_MISSING);
4852	}
4853
4854	folio_ref_add(folio, nr_pages - 1);
4855	add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
4856	count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_FAULT_ALLOC);
4857	folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
4858	folio_add_lru_vma(folio, vma);
4859setpte:
4860	if (vmf_orig_pte_uffd_wp(vmf))
4861		entry = pte_mkuffd_wp(entry);
4862	set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages);
4863
4864	/* No need to invalidate - it was non-present before */
4865	update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr_pages);
4866unlock:
4867	if (vmf->pte)
4868		pte_unmap_unlock(vmf->pte, vmf->ptl);
4869	return ret;
4870release:
4871	folio_put(folio);
4872	goto unlock;
 
 
4873oom:
4874	return VM_FAULT_OOM;
4875}
4876
4877/*
4878 * The mmap_lock must have been held on entry, and may have been
4879 * released depending on flags and vma->vm_ops->fault() return value.
4880 * See filemap_fault() and __lock_page_retry().
4881 */
4882static vm_fault_t __do_fault(struct vm_fault *vmf)
4883{
4884	struct vm_area_struct *vma = vmf->vma;
4885	struct folio *folio;
4886	vm_fault_t ret;
4887
4888	/*
4889	 * Preallocate pte before we take page_lock because this might lead to
4890	 * deadlocks for memcg reclaim which waits for pages under writeback:
4891	 *				lock_page(A)
4892	 *				SetPageWriteback(A)
4893	 *				unlock_page(A)
4894	 * lock_page(B)
4895	 *				lock_page(B)
4896	 * pte_alloc_one
4897	 *   shrink_folio_list
4898	 *     wait_on_page_writeback(A)
4899	 *				SetPageWriteback(B)
4900	 *				unlock_page(B)
4901	 *				# flush A, B to clear the writeback
4902	 */
4903	if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
4904		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4905		if (!vmf->prealloc_pte)
4906			return VM_FAULT_OOM;
 
4907	}
4908
4909	ret = vma->vm_ops->fault(vmf);
4910	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
4911			    VM_FAULT_DONE_COW)))
4912		return ret;
4913
4914	folio = page_folio(vmf->page);
4915	if (unlikely(PageHWPoison(vmf->page))) {
4916		vm_fault_t poisonret = VM_FAULT_HWPOISON;
4917		if (ret & VM_FAULT_LOCKED) {
4918			if (page_mapped(vmf->page))
4919				unmap_mapping_folio(folio);
4920			/* Retry if a clean folio was removed from the cache. */
4921			if (mapping_evict_folio(folio->mapping, folio))
4922				poisonret = VM_FAULT_NOPAGE;
4923			folio_unlock(folio);
4924		}
4925		folio_put(folio);
4926		vmf->page = NULL;
4927		return poisonret;
4928	}
4929
4930	if (unlikely(!(ret & VM_FAULT_LOCKED)))
4931		folio_lock(folio);
4932	else
4933		VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);
4934
4935	return ret;
4936}
4937
4938#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4939static void deposit_prealloc_pte(struct vm_fault *vmf)
4940{
4941	struct vm_area_struct *vma = vmf->vma;
4942
4943	pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
4944	/*
4945	 * We are going to consume the prealloc table,
4946	 * count that as nr_ptes.
4947	 */
4948	mm_inc_nr_ptes(vma->vm_mm);
4949	vmf->prealloc_pte = NULL;
4950}
4951
4952vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4953{
4954	struct folio *folio = page_folio(page);
4955	struct vm_area_struct *vma = vmf->vma;
4956	bool write = vmf->flags & FAULT_FLAG_WRITE;
4957	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
4958	pmd_t entry;
 
4959	vm_fault_t ret = VM_FAULT_FALLBACK;
4960
4961	/*
4962	 * It is too late to allocate a small folio, we already have a large
4963	 * folio in the pagecache: especially s390 KVM cannot tolerate any
4964	 * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any
4965	 * PMD mappings if THPs are disabled.
4966	 */
4967	if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags))
4968		return ret;
4969
4970	if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
4971		return ret;
4972
4973	if (folio_order(folio) != HPAGE_PMD_ORDER)
4974		return ret;
4975	page = &folio->page;
4976
4977	/*
4978	 * Just backoff if any subpage of a THP is corrupted otherwise
4979	 * the corrupted page may mapped by PMD silently to escape the
4980	 * check.  This kind of THP just can be PTE mapped.  Access to
4981	 * the corrupted subpage should trigger SIGBUS as expected.
4982	 */
4983	if (unlikely(folio_test_has_hwpoisoned(folio)))
4984		return ret;
4985
4986	/*
4987	 * Archs like ppc64 need additional space to store information
4988	 * related to pte entry. Use the preallocated table for that.
4989	 */
4990	if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
4991		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4992		if (!vmf->prealloc_pte)
4993			return VM_FAULT_OOM;
 
4994	}
4995
4996	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
4997	if (unlikely(!pmd_none(*vmf->pmd)))
4998		goto out;
4999
5000	flush_icache_pages(vma, page, HPAGE_PMD_NR);
 
5001
5002	entry = mk_huge_pmd(page, vma->vm_page_prot);
5003	if (write)
5004		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
5005
5006	add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR);
5007	folio_add_file_rmap_pmd(folio, page, vma);
5008
5009	/*
5010	 * deposit and withdraw with pmd lock held
5011	 */
5012	if (arch_needs_pgtable_deposit())
5013		deposit_prealloc_pte(vmf);
5014
5015	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
5016
5017	update_mmu_cache_pmd(vma, haddr, vmf->pmd);
5018
5019	/* fault is handled */
5020	ret = 0;
5021	count_vm_event(THP_FILE_MAPPED);
5022out:
5023	spin_unlock(vmf->ptl);
5024	return ret;
5025}
5026#else
5027vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
5028{
5029	return VM_FAULT_FALLBACK;
5030}
5031#endif
5032
5033/**
5034 * set_pte_range - Set a range of PTEs to point to pages in a folio.
5035 * @vmf: Fault decription.
5036 * @folio: The folio that contains @page.
5037 * @page: The first page to create a PTE for.
5038 * @nr: The number of PTEs to create.
5039 * @addr: The first address to create a PTE for.
5040 */
5041void set_pte_range(struct vm_fault *vmf, struct folio *folio,
5042		struct page *page, unsigned int nr, unsigned long addr)
5043{
5044	struct vm_area_struct *vma = vmf->vma;
5045	bool write = vmf->flags & FAULT_FLAG_WRITE;
5046	bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE);
5047	pte_t entry;
5048
5049	flush_icache_pages(vma, page, nr);
5050	entry = mk_pte(page, vma->vm_page_prot);
5051
5052	if (prefault && arch_wants_old_prefaulted_pte())
5053		entry = pte_mkold(entry);
5054	else
5055		entry = pte_sw_mkyoung(entry);
5056
5057	if (write)
5058		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
5059	if (unlikely(vmf_orig_pte_uffd_wp(vmf)))
5060		entry = pte_mkuffd_wp(entry);
5061	/* copy-on-write page */
5062	if (write && !(vma->vm_flags & VM_SHARED)) {
5063		VM_BUG_ON_FOLIO(nr != 1, folio);
5064		folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
5065		folio_add_lru_vma(folio, vma);
5066	} else {
5067		folio_add_file_rmap_ptes(folio, page, nr, vma);
 
5068	}
5069	set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
5070
5071	/* no need to invalidate: a not-present page won't be cached */
5072	update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
5073}
5074
5075static bool vmf_pte_changed(struct vm_fault *vmf)
5076{
5077	if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
5078		return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
5079
5080	return !pte_none(ptep_get(vmf->pte));
5081}
5082
5083/**
5084 * finish_fault - finish page fault once we have prepared the page to fault
5085 *
5086 * @vmf: structure describing the fault
5087 *
5088 * This function handles all that is needed to finish a page fault once the
5089 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
5090 * given page, adds reverse page mapping, handles memcg charges and LRU
5091 * addition.
5092 *
5093 * The function expects the page to be locked and on success it consumes a
5094 * reference of a page being mapped (for the PTE which maps it).
5095 *
5096 * Return: %0 on success, %VM_FAULT_ code in case of error.
5097 */
5098vm_fault_t finish_fault(struct vm_fault *vmf)
5099{
5100	struct vm_area_struct *vma = vmf->vma;
5101	struct page *page;
5102	struct folio *folio;
5103	vm_fault_t ret;
5104	bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) &&
5105		      !(vma->vm_flags & VM_SHARED);
5106	int type, nr_pages;
5107	unsigned long addr;
5108	bool needs_fallback = false;
5109
5110fallback:
5111	addr = vmf->address;
5112
5113	/* Did we COW the page? */
5114	if (is_cow)
5115		page = vmf->cow_page;
5116	else
5117		page = vmf->page;
5118
5119	/*
5120	 * check even for read faults because we might have lost our CoWed
5121	 * page
5122	 */
5123	if (!(vma->vm_flags & VM_SHARED)) {
5124		ret = check_stable_address_space(vma->vm_mm);
5125		if (ret)
5126			return ret;
5127	}
5128
5129	if (pmd_none(*vmf->pmd)) {
5130		if (PageTransCompound(page)) {
5131			ret = do_set_pmd(vmf, page);
5132			if (ret != VM_FAULT_FALLBACK)
5133				return ret;
5134		}
5135
5136		if (vmf->prealloc_pte)
5137			pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
5138		else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
 
 
 
 
 
 
5139			return VM_FAULT_OOM;
5140	}
5141
5142	folio = page_folio(page);
5143	nr_pages = folio_nr_pages(folio);
5144
5145	/*
5146	 * Using per-page fault to maintain the uffd semantics, and same
5147	 * approach also applies to non-anonymous-shmem faults to avoid
5148	 * inflating the RSS of the process.
5149	 */
5150	if (!vma_is_anon_shmem(vma) || unlikely(userfaultfd_armed(vma)) ||
5151	    unlikely(needs_fallback)) {
5152		nr_pages = 1;
5153	} else if (nr_pages > 1) {
5154		pgoff_t idx = folio_page_idx(folio, page);
5155		/* The page offset of vmf->address within the VMA. */
5156		pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
5157		/* The index of the entry in the pagetable for fault page. */
5158		pgoff_t pte_off = pte_index(vmf->address);
5159
5160		/*
5161		 * Fallback to per-page fault in case the folio size in page
5162		 * cache beyond the VMA limits and PMD pagetable limits.
5163		 */
5164		if (unlikely(vma_off < idx ||
5165			    vma_off + (nr_pages - idx) > vma_pages(vma) ||
5166			    pte_off < idx ||
5167			    pte_off + (nr_pages - idx)  > PTRS_PER_PTE)) {
5168			nr_pages = 1;
5169		} else {
5170			/* Now we can set mappings for the whole large folio. */
5171			addr = vmf->address - idx * PAGE_SIZE;
5172			page = &folio->page;
5173		}
5174	}
5175
5176	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
5177				       addr, &vmf->ptl);
5178	if (!vmf->pte)
5179		return VM_FAULT_NOPAGE;
5180
 
 
 
5181	/* Re-check under ptl */
5182	if (nr_pages == 1 && unlikely(vmf_pte_changed(vmf))) {
5183		update_mmu_tlb(vma, addr, vmf->pte);
 
5184		ret = VM_FAULT_NOPAGE;
5185		goto unlock;
5186	} else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
5187		needs_fallback = true;
5188		pte_unmap_unlock(vmf->pte, vmf->ptl);
5189		goto fallback;
5190	}
5191
5192	folio_ref_add(folio, nr_pages - 1);
5193	set_pte_range(vmf, folio, page, nr_pages, addr);
5194	type = is_cow ? MM_ANONPAGES : mm_counter_file(folio);
5195	add_mm_counter(vma->vm_mm, type, nr_pages);
5196	ret = 0;
5197
5198unlock:
5199	pte_unmap_unlock(vmf->pte, vmf->ptl);
5200	return ret;
5201}
5202
5203static unsigned long fault_around_pages __read_mostly =
5204	65536 >> PAGE_SHIFT;
5205
5206#ifdef CONFIG_DEBUG_FS
5207static int fault_around_bytes_get(void *data, u64 *val)
5208{
5209	*val = fault_around_pages << PAGE_SHIFT;
5210	return 0;
5211}
5212
5213/*
5214 * fault_around_bytes must be rounded down to the nearest page order as it's
5215 * what do_fault_around() expects to see.
5216 */
5217static int fault_around_bytes_set(void *data, u64 val)
5218{
5219	if (val / PAGE_SIZE > PTRS_PER_PTE)
5220		return -EINVAL;
5221
5222	/*
5223	 * The minimum value is 1 page, however this results in no fault-around
5224	 * at all. See should_fault_around().
5225	 */
5226	val = max(val, PAGE_SIZE);
5227	fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT;
5228
5229	return 0;
5230}
5231DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
5232		fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
5233
5234static int __init fault_around_debugfs(void)
5235{
5236	debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
5237				   &fault_around_bytes_fops);
5238	return 0;
5239}
5240late_initcall(fault_around_debugfs);
5241#endif
5242
5243/*
5244 * do_fault_around() tries to map few pages around the fault address. The hope
5245 * is that the pages will be needed soon and this will lower the number of
5246 * faults to handle.
5247 *
5248 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
5249 * not ready to be mapped: not up-to-date, locked, etc.
5250 *
5251 * This function doesn't cross VMA or page table boundaries, in order to call
5252 * map_pages() and acquire a PTE lock only once.
 
5253 *
5254 * fault_around_pages defines how many pages we'll try to map.
 
 
 
5255 * do_fault_around() expects it to be set to a power of two less than or equal
5256 * to PTRS_PER_PTE.
5257 *
5258 * The virtual address of the area that we map is naturally aligned to
5259 * fault_around_pages * PAGE_SIZE rounded down to the machine page size
5260 * (and therefore to page order).  This way it's easier to guarantee
5261 * that we don't cross page table boundaries.
5262 */
5263static vm_fault_t do_fault_around(struct vm_fault *vmf)
5264{
5265	pgoff_t nr_pages = READ_ONCE(fault_around_pages);
5266	pgoff_t pte_off = pte_index(vmf->address);
5267	/* The page offset of vmf->address within the VMA. */
5268	pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
5269	pgoff_t from_pte, to_pte;
5270	vm_fault_t ret;
5271
5272	/* The PTE offset of the start address, clamped to the VMA. */
5273	from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
5274		       pte_off - min(pte_off, vma_off));
5275
5276	/* The PTE offset of the end address, clamped to the VMA and PTE. */
5277	to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
5278		      pte_off + vma_pages(vmf->vma) - vma_off) - 1;
 
 
 
 
 
 
 
5279
5280	if (pmd_none(*vmf->pmd)) {
5281		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
5282		if (!vmf->prealloc_pte)
5283			return VM_FAULT_OOM;
 
5284	}
5285
5286	rcu_read_lock();
5287	ret = vmf->vma->vm_ops->map_pages(vmf,
5288			vmf->pgoff + from_pte - pte_off,
5289			vmf->pgoff + to_pte - pte_off);
5290	rcu_read_unlock();
5291
5292	return ret;
5293}
5294
5295/* Return true if we should do read fault-around, false otherwise */
5296static inline bool should_fault_around(struct vm_fault *vmf)
5297{
5298	/* No ->map_pages?  No way to fault around... */
5299	if (!vmf->vma->vm_ops->map_pages)
5300		return false;
5301
5302	if (uffd_disable_fault_around(vmf->vma))
5303		return false;
5304
5305	/* A single page implies no faulting 'around' at all. */
5306	return fault_around_pages > 1;
5307}
5308
5309static vm_fault_t do_read_fault(struct vm_fault *vmf)
5310{
 
5311	vm_fault_t ret = 0;
5312	struct folio *folio;
5313
5314	/*
5315	 * Let's call ->map_pages() first and use ->fault() as fallback
5316	 * if page by the offset is not ready to be mapped (cold cache or
5317	 * something).
5318	 */
5319	if (should_fault_around(vmf)) {
5320		ret = do_fault_around(vmf);
5321		if (ret)
5322			return ret;
 
 
5323	}
5324
5325	ret = vmf_can_call_fault(vmf);
5326	if (ret)
5327		return ret;
5328
5329	ret = __do_fault(vmf);
5330	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5331		return ret;
5332
5333	ret |= finish_fault(vmf);
5334	folio = page_folio(vmf->page);
5335	folio_unlock(folio);
5336	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5337		folio_put(folio);
5338	return ret;
5339}
5340
5341static vm_fault_t do_cow_fault(struct vm_fault *vmf)
5342{
5343	struct vm_area_struct *vma = vmf->vma;
5344	struct folio *folio;
5345	vm_fault_t ret;
5346
5347	ret = vmf_can_call_fault(vmf);
5348	if (!ret)
5349		ret = vmf_anon_prepare(vmf);
5350	if (ret)
5351		return ret;
5352
5353	folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false);
5354	if (!folio)
5355		return VM_FAULT_OOM;
5356
5357	vmf->cow_page = &folio->page;
 
 
 
 
5358
5359	ret = __do_fault(vmf);
5360	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5361		goto uncharge_out;
5362	if (ret & VM_FAULT_DONE_COW)
5363		return ret;
5364
5365	if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) {
5366		ret = VM_FAULT_HWPOISON;
5367		goto unlock;
5368	}
5369	__folio_mark_uptodate(folio);
5370
5371	ret |= finish_fault(vmf);
5372unlock:
5373	unlock_page(vmf->page);
5374	put_page(vmf->page);
5375	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5376		goto uncharge_out;
5377	return ret;
5378uncharge_out:
5379	folio_put(folio);
5380	return ret;
5381}
5382
5383static vm_fault_t do_shared_fault(struct vm_fault *vmf)
5384{
5385	struct vm_area_struct *vma = vmf->vma;
5386	vm_fault_t ret, tmp;
5387	struct folio *folio;
5388
5389	ret = vmf_can_call_fault(vmf);
5390	if (ret)
5391		return ret;
5392
5393	ret = __do_fault(vmf);
5394	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
5395		return ret;
5396
5397	folio = page_folio(vmf->page);
5398
5399	/*
5400	 * Check if the backing address space wants to know that the page is
5401	 * about to become writable
5402	 */
5403	if (vma->vm_ops->page_mkwrite) {
5404		folio_unlock(folio);
5405		tmp = do_page_mkwrite(vmf, folio);
5406		if (unlikely(!tmp ||
5407				(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
5408			folio_put(folio);
5409			return tmp;
5410		}
5411	}
5412
5413	ret |= finish_fault(vmf);
5414	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
5415					VM_FAULT_RETRY))) {
5416		folio_unlock(folio);
5417		folio_put(folio);
5418		return ret;
5419	}
5420
5421	ret |= fault_dirty_shared_page(vmf);
5422	return ret;
5423}
5424
5425/*
5426 * We enter with non-exclusive mmap_lock (to exclude vma changes,
5427 * but allow concurrent faults).
5428 * The mmap_lock may have been released depending on flags and our
5429 * return value.  See filemap_fault() and __folio_lock_or_retry().
5430 * If mmap_lock is released, vma may become invalid (for example
5431 * by other thread calling munmap()).
5432 */
5433static vm_fault_t do_fault(struct vm_fault *vmf)
5434{
5435	struct vm_area_struct *vma = vmf->vma;
5436	struct mm_struct *vm_mm = vma->vm_mm;
5437	vm_fault_t ret;
5438
5439	/*
5440	 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
5441	 */
5442	if (!vma->vm_ops->fault) {
5443		vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
5444					       vmf->address, &vmf->ptl);
5445		if (unlikely(!vmf->pte))
 
 
5446			ret = VM_FAULT_SIGBUS;
5447		else {
 
 
 
 
5448			/*
5449			 * Make sure this is not a temporary clearing of pte
5450			 * by holding ptl and checking again. A R/M/W update
5451			 * of pte involves: take ptl, clearing the pte so that
5452			 * we don't have concurrent modification by hardware
5453			 * followed by an update.
5454			 */
5455			if (unlikely(pte_none(ptep_get(vmf->pte))))
5456				ret = VM_FAULT_SIGBUS;
5457			else
5458				ret = VM_FAULT_NOPAGE;
5459
5460			pte_unmap_unlock(vmf->pte, vmf->ptl);
5461		}
5462	} else if (!(vmf->flags & FAULT_FLAG_WRITE))
5463		ret = do_read_fault(vmf);
5464	else if (!(vma->vm_flags & VM_SHARED))
5465		ret = do_cow_fault(vmf);
5466	else
5467		ret = do_shared_fault(vmf);
5468
5469	/* preallocated pagetable is unused: free it */
5470	if (vmf->prealloc_pte) {
5471		pte_free(vm_mm, vmf->prealloc_pte);
5472		vmf->prealloc_pte = NULL;
5473	}
5474	return ret;
5475}
5476
5477int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
5478		      unsigned long addr, int *flags,
5479		      bool writable, int *last_cpupid)
5480{
5481	struct vm_area_struct *vma = vmf->vma;
5482
5483	/*
5484	 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
5485	 * much anyway since they can be in shared cache state. This misses
5486	 * the case where a mapping is writable but the process never writes
5487	 * to it but pte_write gets cleared during protection updates and
5488	 * pte_dirty has unpredictable behaviour between PTE scan updates,
5489	 * background writeback, dirty balancing and application behaviour.
5490	 */
5491	if (!writable)
5492		*flags |= TNF_NO_GROUP;
5493
5494	/*
5495	 * Flag if the folio is shared between multiple address spaces. This
5496	 * is later used when determining whether to group tasks together
5497	 */
5498	if (folio_likely_mapped_shared(folio) && (vma->vm_flags & VM_SHARED))
5499		*flags |= TNF_SHARED;
5500	/*
5501	 * For memory tiering mode, cpupid of slow memory page is used
5502	 * to record page access time.  So use default value.
5503	 */
5504	if (folio_use_access_time(folio))
5505		*last_cpupid = (-1 & LAST_CPUPID_MASK);
5506	else
5507		*last_cpupid = folio_last_cpupid(folio);
5508
5509	/* Record the current PID acceesing VMA */
5510	vma_set_access_pid_bit(vma);
5511
5512	count_vm_numa_event(NUMA_HINT_FAULTS);
5513#ifdef CONFIG_NUMA_BALANCING
5514	count_memcg_folio_events(folio, NUMA_HINT_FAULTS, 1);
5515#endif
5516	if (folio_nid(folio) == numa_node_id()) {
5517		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
5518		*flags |= TNF_FAULT_LOCAL;
5519	}
5520
5521	return mpol_misplaced(folio, vmf, addr);
5522}
5523
5524static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
5525					unsigned long fault_addr, pte_t *fault_pte,
5526					bool writable)
5527{
5528	pte_t pte, old_pte;
5529
5530	old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte);
5531	pte = pte_modify(old_pte, vma->vm_page_prot);
5532	pte = pte_mkyoung(pte);
5533	if (writable)
5534		pte = pte_mkwrite(pte, vma);
5535	ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte);
5536	update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1);
5537}
5538
5539static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
5540				       struct folio *folio, pte_t fault_pte,
5541				       bool ignore_writable, bool pte_write_upgrade)
5542{
5543	int nr = pte_pfn(fault_pte) - folio_pfn(folio);
5544	unsigned long start, end, addr = vmf->address;
5545	unsigned long addr_start = addr - (nr << PAGE_SHIFT);
5546	unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE);
5547	pte_t *start_ptep;
5548
5549	/* Stay within the VMA and within the page table. */
5550	start = max3(addr_start, pt_start, vma->vm_start);
5551	end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE,
5552		   vma->vm_end);
5553	start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT);
5554
5555	/* Restore all PTEs' mapping of the large folio */
5556	for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) {
5557		pte_t ptent = ptep_get(start_ptep);
5558		bool writable = false;
5559
5560		if (!pte_present(ptent) || !pte_protnone(ptent))
5561			continue;
5562
5563		if (pfn_folio(pte_pfn(ptent)) != folio)
5564			continue;
5565
5566		if (!ignore_writable) {
5567			ptent = pte_modify(ptent, vma->vm_page_prot);
5568			writable = pte_write(ptent);
5569			if (!writable && pte_write_upgrade &&
5570			    can_change_pte_writable(vma, addr, ptent))
5571				writable = true;
5572		}
5573
5574		numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable);
5575	}
5576}
5577
5578static vm_fault_t do_numa_page(struct vm_fault *vmf)
5579{
5580	struct vm_area_struct *vma = vmf->vma;
5581	struct folio *folio = NULL;
5582	int nid = NUMA_NO_NODE;
5583	bool writable = false, ignore_writable = false;
5584	bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma);
5585	int last_cpupid;
5586	int target_nid;
5587	pte_t pte, old_pte;
5588	int flags = 0, nr_pages;
 
5589
5590	/*
5591	 * The pte cannot be used safely until we verify, while holding the page
5592	 * table lock, that its contents have not changed during fault handling.
 
5593	 */
 
5594	spin_lock(vmf->ptl);
5595	/* Read the live PTE from the page tables: */
5596	old_pte = ptep_get(vmf->pte);
5597
5598	if (unlikely(!pte_same(old_pte, vmf->orig_pte))) {
5599		pte_unmap_unlock(vmf->pte, vmf->ptl);
5600		return 0;
5601	}
5602
 
 
5603	pte = pte_modify(old_pte, vma->vm_page_prot);
5604
5605	/*
5606	 * Detect now whether the PTE could be writable; this information
5607	 * is only valid while holding the PT lock.
5608	 */
5609	writable = pte_write(pte);
5610	if (!writable && pte_write_upgrade &&
5611	    can_change_pte_writable(vma, vmf->address, pte))
5612		writable = true;
5613
5614	folio = vm_normal_folio(vma, vmf->address, pte);
5615	if (!folio || folio_is_zone_device(folio))
5616		goto out_map;
5617
5618	nid = folio_nid(folio);
5619	nr_pages = folio_nr_pages(folio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5620
5621	target_nid = numa_migrate_check(folio, vmf, vmf->address, &flags,
5622					writable, &last_cpupid);
5623	if (target_nid == NUMA_NO_NODE)
5624		goto out_map;
5625	if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) {
5626		flags |= TNF_MIGRATE_FAIL;
5627		goto out_map;
5628	}
5629	/* The folio is isolated and isolation code holds a folio reference. */
5630	pte_unmap_unlock(vmf->pte, vmf->ptl);
5631	writable = false;
5632	ignore_writable = true;
5633
5634	/* Migrate to the requested node */
5635	if (!migrate_misplaced_folio(folio, vma, target_nid)) {
5636		nid = target_nid;
5637		flags |= TNF_MIGRATED;
5638		task_numa_fault(last_cpupid, nid, nr_pages, flags);
5639		return 0;
 
 
 
 
 
 
 
5640	}
5641
5642	flags |= TNF_MIGRATE_FAIL;
5643	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
5644				       vmf->address, &vmf->ptl);
5645	if (unlikely(!vmf->pte))
5646		return 0;
5647	if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
5648		pte_unmap_unlock(vmf->pte, vmf->ptl);
5649		return 0;
5650	}
5651out_map:
5652	/*
5653	 * Make it present again, depending on how arch implements
5654	 * non-accessible ptes, some can allow access by kernel mode.
5655	 */
5656	if (folio && folio_test_large(folio))
5657		numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable,
5658					   pte_write_upgrade);
5659	else
5660		numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte,
5661					    writable);
 
5662	pte_unmap_unlock(vmf->pte, vmf->ptl);
5663
5664	if (nid != NUMA_NO_NODE)
5665		task_numa_fault(last_cpupid, nid, nr_pages, flags);
5666	return 0;
5667}
5668
5669static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
5670{
5671	struct vm_area_struct *vma = vmf->vma;
5672	if (vma_is_anonymous(vma))
5673		return do_huge_pmd_anonymous_page(vmf);
5674	if (vma->vm_ops->huge_fault)
5675		return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
5676	return VM_FAULT_FALLBACK;
5677}
5678
5679/* `inline' is required to avoid gcc 4.1.2 build error */
5680static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
5681{
5682	struct vm_area_struct *vma = vmf->vma;
5683	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
5684	vm_fault_t ret;
5685
5686	if (vma_is_anonymous(vma)) {
5687		if (likely(!unshare) &&
5688		    userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
5689			if (userfaultfd_wp_async(vmf->vma))
5690				goto split;
5691			return handle_userfault(vmf, VM_UFFD_WP);
5692		}
5693		return do_huge_pmd_wp_page(vmf);
5694	}
 
 
5695
5696	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
5697		if (vma->vm_ops->huge_fault) {
5698			ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
5699			if (!(ret & VM_FAULT_FALLBACK))
5700				return ret;
5701		}
5702	}
5703
5704split:
5705	/* COW or write-notify handled on pte level: split pmd. */
5706	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
5707
5708	return VM_FAULT_FALLBACK;
5709}
5710
5711static vm_fault_t create_huge_pud(struct vm_fault *vmf)
5712{
5713#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
5714	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
5715	struct vm_area_struct *vma = vmf->vma;
5716	/* No support for anonymous transparent PUD pages yet */
5717	if (vma_is_anonymous(vma))
5718		return VM_FAULT_FALLBACK;
5719	if (vma->vm_ops->huge_fault)
5720		return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
 
 
 
 
 
 
 
5721#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
5722	return VM_FAULT_FALLBACK;
5723}
5724
5725static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
5726{
5727#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
5728	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
5729	struct vm_area_struct *vma = vmf->vma;
5730	vm_fault_t ret;
5731
5732	/* No support for anonymous transparent PUD pages yet */
5733	if (vma_is_anonymous(vma))
5734		goto split;
5735	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
5736		if (vma->vm_ops->huge_fault) {
5737			ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
5738			if (!(ret & VM_FAULT_FALLBACK))
5739				return ret;
5740		}
5741	}
5742split:
5743	/* COW or write-notify not handled on PUD level: split pud.*/
5744	__split_huge_pud(vma, vmf->pud, vmf->address);
5745#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
5746	return VM_FAULT_FALLBACK;
5747}
5748
5749/*
5750 * These routines also need to handle stuff like marking pages dirty
5751 * and/or accessed for architectures that don't do it in hardware (most
5752 * RISC architectures).  The early dirtying is also good on the i386.
5753 *
5754 * There is also a hook called "update_mmu_cache()" that architectures
5755 * with external mmu caches can use to update those (ie the Sparc or
5756 * PowerPC hashed page tables that act as extended TLBs).
5757 *
5758 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
5759 * concurrent faults).
5760 *
5761 * The mmap_lock may have been released depending on flags and our return value.
5762 * See filemap_fault() and __folio_lock_or_retry().
5763 */
5764static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
5765{
5766	pte_t entry;
5767
5768	if (unlikely(pmd_none(*vmf->pmd))) {
5769		/*
5770		 * Leave __pte_alloc() until later: because vm_ops->fault may
5771		 * want to allocate huge page, and if we expose page table
5772		 * for an instant, it will be difficult to retract from
5773		 * concurrent faults and from rmap lookups.
5774		 */
5775		vmf->pte = NULL;
5776		vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
5777	} else {
5778		pmd_t dummy_pmdval;
5779
5780		/*
5781		 * A regular pmd is established and it can't morph into a huge
5782		 * pmd by anon khugepaged, since that takes mmap_lock in write
5783		 * mode; but shmem or file collapse to THP could still morph
5784		 * it into a huge pmd: just retry later if so.
5785		 *
5786		 * Use the maywrite version to indicate that vmf->pte may be
5787		 * modified, but since we will use pte_same() to detect the
5788		 * change of the !pte_none() entry, there is no need to recheck
5789		 * the pmdval. Here we chooes to pass a dummy variable instead
5790		 * of NULL, which helps new user think about why this place is
5791		 * special.
5792		 */
5793		vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd,
5794						    vmf->address, &dummy_pmdval,
5795						    &vmf->ptl);
5796		if (unlikely(!vmf->pte))
5797			return 0;
5798		vmf->orig_pte = ptep_get_lockless(vmf->pte);
5799		vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
 
 
 
 
 
 
5800
 
 
 
 
 
 
 
 
 
5801		if (pte_none(vmf->orig_pte)) {
5802			pte_unmap(vmf->pte);
5803			vmf->pte = NULL;
5804		}
5805	}
5806
5807	if (!vmf->pte)
5808		return do_pte_missing(vmf);
 
 
 
 
5809
5810	if (!pte_present(vmf->orig_pte))
5811		return do_swap_page(vmf);
5812
5813	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
5814		return do_numa_page(vmf);
5815
 
5816	spin_lock(vmf->ptl);
5817	entry = vmf->orig_pte;
5818	if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
5819		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
5820		goto unlock;
5821	}
5822	if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
5823		if (!pte_write(entry))
5824			return do_wp_page(vmf);
5825		else if (likely(vmf->flags & FAULT_FLAG_WRITE))
5826			entry = pte_mkdirty(entry);
5827	}
5828	entry = pte_mkyoung(entry);
5829	if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
5830				vmf->flags & FAULT_FLAG_WRITE)) {
5831		update_mmu_cache_range(vmf, vmf->vma, vmf->address,
5832				vmf->pte, 1);
5833	} else {
5834		/* Skip spurious TLB flush for retried page fault */
5835		if (vmf->flags & FAULT_FLAG_TRIED)
5836			goto unlock;
5837		/*
5838		 * This is needed only for protection faults but the arch code
5839		 * is not yet telling us if this is a protection fault or not.
5840		 * This still avoids useless tlb flushes for .text page faults
5841		 * with threads.
5842		 */
5843		if (vmf->flags & FAULT_FLAG_WRITE)
5844			flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
5845						     vmf->pte);
5846	}
5847unlock:
5848	pte_unmap_unlock(vmf->pte, vmf->ptl);
5849	return 0;
5850}
5851
5852/*
5853 * On entry, we hold either the VMA lock or the mmap_lock
5854 * (FAULT_FLAG_VMA_LOCK tells you which).  If VM_FAULT_RETRY is set in
5855 * the result, the mmap_lock is not held on exit.  See filemap_fault()
5856 * and __folio_lock_or_retry().
5857 */
5858static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
5859		unsigned long address, unsigned int flags)
5860{
5861	struct vm_fault vmf = {
5862		.vma = vma,
5863		.address = address & PAGE_MASK,
5864		.real_address = address,
5865		.flags = flags,
5866		.pgoff = linear_page_index(vma, address),
5867		.gfp_mask = __get_fault_gfp_mask(vma),
5868	};
 
5869	struct mm_struct *mm = vma->vm_mm;
5870	unsigned long vm_flags = vma->vm_flags;
5871	pgd_t *pgd;
5872	p4d_t *p4d;
5873	vm_fault_t ret;
5874
5875	pgd = pgd_offset(mm, address);
5876	p4d = p4d_alloc(mm, pgd, address);
5877	if (!p4d)
5878		return VM_FAULT_OOM;
5879
5880	vmf.pud = pud_alloc(mm, p4d, address);
5881	if (!vmf.pud)
5882		return VM_FAULT_OOM;
5883retry_pud:
5884	if (pud_none(*vmf.pud) &&
5885	    thp_vma_allowable_order(vma, vm_flags,
5886				TVA_IN_PF | TVA_ENFORCE_SYSFS, PUD_ORDER)) {
5887		ret = create_huge_pud(&vmf);
5888		if (!(ret & VM_FAULT_FALLBACK))
5889			return ret;
5890	} else {
5891		pud_t orig_pud = *vmf.pud;
5892
5893		barrier();
5894		if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
5895
5896			/*
5897			 * TODO once we support anonymous PUDs: NUMA case and
5898			 * FAULT_FLAG_UNSHARE handling.
5899			 */
5900			if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
5901				ret = wp_huge_pud(&vmf, orig_pud);
5902				if (!(ret & VM_FAULT_FALLBACK))
5903					return ret;
5904			} else {
5905				huge_pud_set_accessed(&vmf, orig_pud);
5906				return 0;
5907			}
5908		}
5909	}
5910
5911	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
5912	if (!vmf.pmd)
5913		return VM_FAULT_OOM;
5914
5915	/* Huge pud page fault raced with pmd_alloc? */
5916	if (pud_trans_unstable(vmf.pud))
5917		goto retry_pud;
5918
5919	if (pmd_none(*vmf.pmd) &&
5920	    thp_vma_allowable_order(vma, vm_flags,
5921				TVA_IN_PF | TVA_ENFORCE_SYSFS, PMD_ORDER)) {
5922		ret = create_huge_pmd(&vmf);
5923		if (!(ret & VM_FAULT_FALLBACK))
5924			return ret;
5925	} else {
5926		vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
5927
 
5928		if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
5929			VM_BUG_ON(thp_migration_supported() &&
5930					  !is_pmd_migration_entry(vmf.orig_pmd));
5931			if (is_pmd_migration_entry(vmf.orig_pmd))
5932				pmd_migration_entry_wait(mm, vmf.pmd);
5933			return 0;
5934		}
5935		if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
5936			if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
5937				return do_huge_pmd_numa_page(&vmf);
5938
5939			if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
5940			    !pmd_write(vmf.orig_pmd)) {
5941				ret = wp_huge_pmd(&vmf);
5942				if (!(ret & VM_FAULT_FALLBACK))
5943					return ret;
5944			} else {
5945				huge_pmd_set_accessed(&vmf);
5946				return 0;
5947			}
5948		}
5949	}
5950
5951	return handle_pte_fault(&vmf);
5952}
5953
5954/**
5955 * mm_account_fault - Do page fault accounting
5956 * @mm: mm from which memcg should be extracted. It can be NULL.
5957 * @regs: the pt_regs struct pointer.  When set to NULL, will skip accounting
5958 *        of perf event counters, but we'll still do the per-task accounting to
5959 *        the task who triggered this page fault.
5960 * @address: the faulted address.
5961 * @flags: the fault flags.
5962 * @ret: the fault retcode.
5963 *
5964 * This will take care of most of the page fault accounting.  Meanwhile, it
5965 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
5966 * updates.  However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
5967 * still be in per-arch page fault handlers at the entry of page fault.
5968 */
5969static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
5970				    unsigned long address, unsigned int flags,
5971				    vm_fault_t ret)
5972{
5973	bool major;
5974
5975	/* Incomplete faults will be accounted upon completion. */
5976	if (ret & VM_FAULT_RETRY)
5977		return;
5978
5979	/*
5980	 * To preserve the behavior of older kernels, PGFAULT counters record
5981	 * both successful and failed faults, as opposed to perf counters,
5982	 * which ignore failed cases.
5983	 */
5984	count_vm_event(PGFAULT);
5985	count_memcg_event_mm(mm, PGFAULT);
5986
5987	/*
5988	 * Do not account for unsuccessful faults (e.g. when the address wasn't
5989	 * valid).  That includes arch_vma_access_permitted() failing before
5990	 * reaching here. So this is not a "this many hardware page faults"
5991	 * counter.  We should use the hw profiling for that.
5992	 */
5993	if (ret & VM_FAULT_ERROR)
5994		return;
5995
5996	/*
5997	 * We define the fault as a major fault when the final successful fault
5998	 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
5999	 * handle it immediately previously).
6000	 */
6001	major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
6002
6003	if (major)
6004		current->maj_flt++;
6005	else
6006		current->min_flt++;
6007
6008	/*
6009	 * If the fault is done for GUP, regs will be NULL.  We only do the
6010	 * accounting for the per thread fault counters who triggered the
6011	 * fault, and we skip the perf event updates.
6012	 */
6013	if (!regs)
6014		return;
6015
6016	if (major)
6017		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
6018	else
6019		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
6020}
6021
6022#ifdef CONFIG_LRU_GEN
6023static void lru_gen_enter_fault(struct vm_area_struct *vma)
6024{
6025	/* the LRU algorithm only applies to accesses with recency */
6026	current->in_lru_fault = vma_has_recency(vma);
6027}
6028
6029static void lru_gen_exit_fault(void)
6030{
6031	current->in_lru_fault = false;
6032}
6033#else
6034static void lru_gen_enter_fault(struct vm_area_struct *vma)
6035{
6036}
6037
6038static void lru_gen_exit_fault(void)
6039{
6040}
6041#endif /* CONFIG_LRU_GEN */
6042
6043static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
6044				       unsigned int *flags)
6045{
6046	if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
6047		if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
6048			return VM_FAULT_SIGSEGV;
6049		/*
6050		 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
6051		 * just treat it like an ordinary read-fault otherwise.
6052		 */
6053		if (!is_cow_mapping(vma->vm_flags))
6054			*flags &= ~FAULT_FLAG_UNSHARE;
6055	} else if (*flags & FAULT_FLAG_WRITE) {
6056		/* Write faults on read-only mappings are impossible ... */
6057		if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
6058			return VM_FAULT_SIGSEGV;
6059		/* ... and FOLL_FORCE only applies to COW mappings. */
6060		if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
6061				 !is_cow_mapping(vma->vm_flags)))
6062			return VM_FAULT_SIGSEGV;
6063	}
6064#ifdef CONFIG_PER_VMA_LOCK
6065	/*
6066	 * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
6067	 * the assumption that lock is dropped on VM_FAULT_RETRY.
6068	 */
6069	if (WARN_ON_ONCE((*flags &
6070			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
6071			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
6072		return VM_FAULT_SIGSEGV;
6073#endif
6074
6075	return 0;
6076}
6077
6078/*
6079 * By the time we get here, we already hold the mm semaphore
6080 *
6081 * The mmap_lock may have been released depending on flags and our
6082 * return value.  See filemap_fault() and __folio_lock_or_retry().
6083 */
6084vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
6085			   unsigned int flags, struct pt_regs *regs)
6086{
6087	/* If the fault handler drops the mmap_lock, vma may be freed */
6088	struct mm_struct *mm = vma->vm_mm;
6089	vm_fault_t ret;
6090	bool is_droppable;
6091
6092	__set_current_state(TASK_RUNNING);
6093
6094	ret = sanitize_fault_flags(vma, &flags);
6095	if (ret)
6096		goto out;
 
 
6097
6098	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
6099					    flags & FAULT_FLAG_INSTRUCTION,
6100					    flags & FAULT_FLAG_REMOTE)) {
6101		ret = VM_FAULT_SIGSEGV;
6102		goto out;
6103	}
6104
6105	is_droppable = !!(vma->vm_flags & VM_DROPPABLE);
6106
6107	/*
6108	 * Enable the memcg OOM handling for faults triggered in user
6109	 * space.  Kernel faults are handled more gracefully.
6110	 */
6111	if (flags & FAULT_FLAG_USER)
6112		mem_cgroup_enter_user_fault();
6113
6114	lru_gen_enter_fault(vma);
6115
6116	if (unlikely(is_vm_hugetlb_page(vma)))
6117		ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
6118	else
6119		ret = __handle_mm_fault(vma, address, flags);
6120
6121	/*
6122	 * Warning: It is no longer safe to dereference vma-> after this point,
6123	 * because mmap_lock might have been dropped by __handle_mm_fault(), so
6124	 * vma might be destroyed from underneath us.
6125	 */
6126
6127	lru_gen_exit_fault();
6128
6129	/* If the mapping is droppable, then errors due to OOM aren't fatal. */
6130	if (is_droppable)
6131		ret &= ~VM_FAULT_OOM;
6132
6133	if (flags & FAULT_FLAG_USER) {
6134		mem_cgroup_exit_user_fault();
6135		/*
6136		 * The task may have entered a memcg OOM situation but
6137		 * if the allocation error was handled gracefully (no
6138		 * VM_FAULT_OOM), there is no need to kill anything.
6139		 * Just clean up the OOM state peacefully.
6140		 */
6141		if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
6142			mem_cgroup_oom_synchronize(false);
6143	}
6144out:
6145	mm_account_fault(mm, regs, address, flags, ret);
6146
6147	return ret;
6148}
6149EXPORT_SYMBOL_GPL(handle_mm_fault);
6150
6151#ifdef CONFIG_LOCK_MM_AND_FIND_VMA
6152#include <linux/extable.h>
6153
6154static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
6155{
6156	if (likely(mmap_read_trylock(mm)))
6157		return true;
6158
6159	if (regs && !user_mode(regs)) {
6160		unsigned long ip = exception_ip(regs);
6161		if (!search_exception_tables(ip))
6162			return false;
6163	}
6164
6165	return !mmap_read_lock_killable(mm);
6166}
6167
6168static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
6169{
6170	/*
6171	 * We don't have this operation yet.
6172	 *
6173	 * It should be easy enough to do: it's basically a
6174	 *    atomic_long_try_cmpxchg_acquire()
6175	 * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
6176	 * it also needs the proper lockdep magic etc.
6177	 */
6178	return false;
6179}
6180
6181static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
6182{
6183	mmap_read_unlock(mm);
6184	if (regs && !user_mode(regs)) {
6185		unsigned long ip = exception_ip(regs);
6186		if (!search_exception_tables(ip))
6187			return false;
6188	}
6189	return !mmap_write_lock_killable(mm);
6190}
6191
6192/*
6193 * Helper for page fault handling.
6194 *
6195 * This is kind of equivalend to "mmap_read_lock()" followed
6196 * by "find_extend_vma()", except it's a lot more careful about
6197 * the locking (and will drop the lock on failure).
6198 *
6199 * For example, if we have a kernel bug that causes a page
6200 * fault, we don't want to just use mmap_read_lock() to get
6201 * the mm lock, because that would deadlock if the bug were
6202 * to happen while we're holding the mm lock for writing.
6203 *
6204 * So this checks the exception tables on kernel faults in
6205 * order to only do this all for instructions that are actually
6206 * expected to fault.
6207 *
6208 * We can also actually take the mm lock for writing if we
6209 * need to extend the vma, which helps the VM layer a lot.
6210 */
6211struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
6212			unsigned long addr, struct pt_regs *regs)
6213{
6214	struct vm_area_struct *vma;
6215
6216	if (!get_mmap_lock_carefully(mm, regs))
6217		return NULL;
6218
6219	vma = find_vma(mm, addr);
6220	if (likely(vma && (vma->vm_start <= addr)))
6221		return vma;
6222
6223	/*
6224	 * Well, dang. We might still be successful, but only
6225	 * if we can extend a vma to do so.
6226	 */
6227	if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
6228		mmap_read_unlock(mm);
6229		return NULL;
6230	}
6231
6232	/*
6233	 * We can try to upgrade the mmap lock atomically,
6234	 * in which case we can continue to use the vma
6235	 * we already looked up.
6236	 *
6237	 * Otherwise we'll have to drop the mmap lock and
6238	 * re-take it, and also look up the vma again,
6239	 * re-checking it.
6240	 */
6241	if (!mmap_upgrade_trylock(mm)) {
6242		if (!upgrade_mmap_lock_carefully(mm, regs))
6243			return NULL;
6244
6245		vma = find_vma(mm, addr);
6246		if (!vma)
6247			goto fail;
6248		if (vma->vm_start <= addr)
6249			goto success;
6250		if (!(vma->vm_flags & VM_GROWSDOWN))
6251			goto fail;
6252	}
6253
6254	if (expand_stack_locked(vma, addr))
6255		goto fail;
6256
6257success:
6258	mmap_write_downgrade(mm);
6259	return vma;
6260
6261fail:
6262	mmap_write_unlock(mm);
6263	return NULL;
6264}
6265#endif
6266
6267#ifdef CONFIG_PER_VMA_LOCK
6268/*
6269 * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
6270 * stable and not isolated. If the VMA is not found or is being modified the
6271 * function returns NULL.
6272 */
6273struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
6274					  unsigned long address)
6275{
6276	MA_STATE(mas, &mm->mm_mt, address, address);
6277	struct vm_area_struct *vma;
6278
6279	rcu_read_lock();
6280retry:
6281	vma = mas_walk(&mas);
6282	if (!vma)
6283		goto inval;
6284
6285	if (!vma_start_read(vma))
6286		goto inval;
6287
6288	/* Check if the VMA got isolated after we found it */
6289	if (vma->detached) {
6290		vma_end_read(vma);
6291		count_vm_vma_lock_event(VMA_LOCK_MISS);
6292		/* The area was replaced with another one */
6293		goto retry;
6294	}
6295	/*
6296	 * At this point, we have a stable reference to a VMA: The VMA is
6297	 * locked and we know it hasn't already been isolated.
6298	 * From here on, we can access the VMA without worrying about which
6299	 * fields are accessible for RCU readers.
6300	 */
6301
6302	/* Check since vm_start/vm_end might change before we lock the VMA */
6303	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
6304		goto inval_end_read;
6305
6306	rcu_read_unlock();
6307	return vma;
6308
6309inval_end_read:
6310	vma_end_read(vma);
6311inval:
6312	rcu_read_unlock();
6313	count_vm_vma_lock_event(VMA_LOCK_ABORT);
6314	return NULL;
6315}
6316#endif /* CONFIG_PER_VMA_LOCK */
6317
6318#ifndef __PAGETABLE_P4D_FOLDED
6319/*
6320 * Allocate p4d page table.
6321 * We've already handled the fast-path in-line.
6322 */
6323int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
6324{
6325	p4d_t *new = p4d_alloc_one(mm, address);
6326	if (!new)
6327		return -ENOMEM;
6328
 
 
6329	spin_lock(&mm->page_table_lock);
6330	if (pgd_present(*pgd)) {	/* Another has populated it */
6331		p4d_free(mm, new);
6332	} else {
6333		smp_wmb(); /* See comment in pmd_install() */
6334		pgd_populate(mm, pgd, new);
6335	}
6336	spin_unlock(&mm->page_table_lock);
6337	return 0;
6338}
6339#endif /* __PAGETABLE_P4D_FOLDED */
6340
6341#ifndef __PAGETABLE_PUD_FOLDED
6342/*
6343 * Allocate page upper directory.
6344 * We've already handled the fast-path in-line.
6345 */
6346int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
6347{
6348	pud_t *new = pud_alloc_one(mm, address);
6349	if (!new)
6350		return -ENOMEM;
6351
 
 
6352	spin_lock(&mm->page_table_lock);
6353	if (!p4d_present(*p4d)) {
6354		mm_inc_nr_puds(mm);
6355		smp_wmb(); /* See comment in pmd_install() */
6356		p4d_populate(mm, p4d, new);
6357	} else	/* Another has populated it */
6358		pud_free(mm, new);
6359	spin_unlock(&mm->page_table_lock);
6360	return 0;
6361}
6362#endif /* __PAGETABLE_PUD_FOLDED */
6363
6364#ifndef __PAGETABLE_PMD_FOLDED
6365/*
6366 * Allocate page middle directory.
6367 * We've already handled the fast-path in-line.
6368 */
6369int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
6370{
6371	spinlock_t *ptl;
6372	pmd_t *new = pmd_alloc_one(mm, address);
6373	if (!new)
6374		return -ENOMEM;
6375
 
 
6376	ptl = pud_lock(mm, pud);
6377	if (!pud_present(*pud)) {
6378		mm_inc_nr_pmds(mm);
6379		smp_wmb(); /* See comment in pmd_install() */
6380		pud_populate(mm, pud, new);
6381	} else {	/* Another has populated it */
6382		pmd_free(mm, new);
6383	}
6384	spin_unlock(ptl);
6385	return 0;
6386}
6387#endif /* __PAGETABLE_PMD_FOLDED */
6388
6389static inline void pfnmap_args_setup(struct follow_pfnmap_args *args,
6390				     spinlock_t *lock, pte_t *ptep,
6391				     pgprot_t pgprot, unsigned long pfn_base,
6392				     unsigned long addr_mask, bool writable,
6393				     bool special)
6394{
6395	args->lock = lock;
6396	args->ptep = ptep;
6397	args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT);
6398	args->pgprot = pgprot;
6399	args->writable = writable;
6400	args->special = special;
6401}
6402
6403static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma)
6404{
6405#ifdef CONFIG_LOCKDEP
6406	struct file *file = vma->vm_file;
6407	struct address_space *mapping = file ? file->f_mapping : NULL;
6408
6409	if (mapping)
6410		lockdep_assert(lockdep_is_held(&mapping->i_mmap_rwsem) ||
6411			       lockdep_is_held(&vma->vm_mm->mmap_lock));
6412	else
6413		lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));
6414#endif
6415}
6416
6417/**
6418 * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address
6419 * @args: Pointer to struct @follow_pfnmap_args
6420 *
6421 * The caller needs to setup args->vma and args->address to point to the
6422 * virtual address as the target of such lookup.  On a successful return,
6423 * the results will be put into other output fields.
6424 *
6425 * After the caller finished using the fields, the caller must invoke
6426 * another follow_pfnmap_end() to proper releases the locks and resources
6427 * of such look up request.
6428 *
6429 * During the start() and end() calls, the results in @args will be valid
6430 * as proper locks will be held.  After the end() is called, all the fields
6431 * in @follow_pfnmap_args will be invalid to be further accessed.  Further
6432 * use of such information after end() may require proper synchronizations
6433 * by the caller with page table updates, otherwise it can create a
6434 * security bug.
6435 *
6436 * If the PTE maps a refcounted page, callers are responsible to protect
6437 * against invalidation with MMU notifiers; otherwise access to the PFN at
6438 * a later point in time can trigger use-after-free.
6439 *
6440 * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
6441 * should be taken for read, and the mmap semaphore cannot be released
6442 * before the end() is invoked.
6443 *
6444 * This function must not be used to modify PTE content.
6445 *
6446 * Return: zero on success, negative otherwise.
6447 */
6448int follow_pfnmap_start(struct follow_pfnmap_args *args)
6449{
6450	struct vm_area_struct *vma = args->vma;
6451	unsigned long address = args->address;
6452	struct mm_struct *mm = vma->vm_mm;
6453	spinlock_t *lock;
6454	pgd_t *pgdp;
6455	p4d_t *p4dp, p4d;
6456	pud_t *pudp, pud;
6457	pmd_t *pmdp, pmd;
6458	pte_t *ptep, pte;
6459
6460	pfnmap_lockdep_assert(vma);
6461
6462	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
 
6463		goto out;
6464
6465	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
6466		goto out;
6467retry:
6468	pgdp = pgd_offset(mm, address);
6469	if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
6470		goto out;
6471
6472	p4dp = p4d_offset(pgdp, address);
6473	p4d = READ_ONCE(*p4dp);
6474	if (p4d_none(p4d) || unlikely(p4d_bad(p4d)))
6475		goto out;
6476
6477	pudp = pud_offset(p4dp, address);
6478	pud = READ_ONCE(*pudp);
6479	if (pud_none(pud))
6480		goto out;
6481	if (pud_leaf(pud)) {
6482		lock = pud_lock(mm, pudp);
6483		if (!unlikely(pud_leaf(pud))) {
6484			spin_unlock(lock);
6485			goto retry;
6486		}
6487		pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud),
6488				  pud_pfn(pud), PUD_MASK, pud_write(pud),
6489				  pud_special(pud));
6490		return 0;
6491	}
6492
6493	pmdp = pmd_offset(pudp, address);
6494	pmd = pmdp_get_lockless(pmdp);
6495	if (pmd_leaf(pmd)) {
6496		lock = pmd_lock(mm, pmdp);
6497		if (!unlikely(pmd_leaf(pmd))) {
6498			spin_unlock(lock);
6499			goto retry;
6500		}
6501		pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd),
6502				  pmd_pfn(pmd), PMD_MASK, pmd_write(pmd),
6503				  pmd_special(pmd));
6504		return 0;
 
 
 
 
 
 
6505	}
6506
6507	ptep = pte_offset_map_lock(mm, pmdp, address, &lock);
6508	if (!ptep)
6509		goto out;
6510	pte = ptep_get(ptep);
6511	if (!pte_present(pte))
 
 
 
 
 
 
 
6512		goto unlock;
6513	pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte),
6514			  pte_pfn(pte), PAGE_MASK, pte_write(pte),
6515			  pte_special(pte));
6516	return 0;
6517unlock:
6518	pte_unmap_unlock(ptep, lock);
 
 
6519out:
6520	return -EINVAL;
6521}
6522EXPORT_SYMBOL_GPL(follow_pfnmap_start);
6523
6524/**
6525 * follow_pfnmap_end(): End a follow_pfnmap_start() process
6526 * @args: Pointer to struct @follow_pfnmap_args
 
 
 
 
 
 
 
 
 
6527 *
6528 * Must be used in pair of follow_pfnmap_start().  See the start() function
6529 * above for more information.
 
 
 
 
 
6530 */
6531void follow_pfnmap_end(struct follow_pfnmap_args *args)
 
6532{
6533	if (args->lock)
6534		spin_unlock(args->lock);
6535	if (args->ptep)
6536		pte_unmap(args->ptep);
6537}
6538EXPORT_SYMBOL_GPL(follow_pfnmap_end);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6539
6540#ifdef CONFIG_HAVE_IOREMAP_PROT
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6541/**
6542 * generic_access_phys - generic implementation for iomem mmap access
6543 * @vma: the vma to access
6544 * @addr: userspace address, not relative offset within @vma
6545 * @buf: buffer to read/write
6546 * @len: length of transfer
6547 * @write: set to FOLL_WRITE when writing, otherwise reading
6548 *
6549 * This is a generic implementation for &vm_operations_struct.access for an
6550 * iomem mapping. This callback is used by access_process_vm() when the @vma is
6551 * not page based.
6552 */
6553int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
6554			void *buf, int len, int write)
6555{
6556	resource_size_t phys_addr;
6557	unsigned long prot = 0;
6558	void __iomem *maddr;
 
 
6559	int offset = offset_in_page(addr);
6560	int ret = -EINVAL;
6561	bool writable;
6562	struct follow_pfnmap_args args = { .vma = vma, .address = addr };
 
6563
6564retry:
6565	if (follow_pfnmap_start(&args))
6566		return -EINVAL;
6567	prot = pgprot_val(args.pgprot);
6568	phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT;
6569	writable = args.writable;
6570	follow_pfnmap_end(&args);
 
6571
6572	if ((write & FOLL_WRITE) && !writable)
6573		return -EINVAL;
6574
6575	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
6576	if (!maddr)
6577		return -ENOMEM;
6578
6579	if (follow_pfnmap_start(&args))
6580		goto out_unmap;
6581
6582	if ((prot != pgprot_val(args.pgprot)) ||
6583	    (phys_addr != (args.pfn << PAGE_SHIFT)) ||
6584	    (writable != args.writable)) {
6585		follow_pfnmap_end(&args);
6586		iounmap(maddr);
 
6587		goto retry;
6588	}
6589
6590	if (write)
6591		memcpy_toio(maddr + offset, buf, len);
6592	else
6593		memcpy_fromio(buf, maddr + offset, len);
6594	ret = len;
6595	follow_pfnmap_end(&args);
6596out_unmap:
6597	iounmap(maddr);
6598
6599	return ret;
6600}
6601EXPORT_SYMBOL_GPL(generic_access_phys);
6602#endif
6603
6604/*
6605 * Access another process' address space as given in mm.
6606 */
6607static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
6608			      void *buf, int len, unsigned int gup_flags)
6609{
 
6610	void *old_buf = buf;
6611	int write = gup_flags & FOLL_WRITE;
6612
6613	if (mmap_read_lock_killable(mm))
6614		return 0;
6615
6616	/* Untag the address before looking up the VMA */
6617	addr = untagged_addr_remote(mm, addr);
6618
6619	/* Avoid triggering the temporary warning in __get_user_pages */
6620	if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
6621		return 0;
6622
6623	/* ignore errors, just check how much was successfully transferred */
6624	while (len) {
6625		int bytes, offset;
6626		void *maddr;
6627		struct vm_area_struct *vma = NULL;
6628		struct page *page = get_user_page_vma_remote(mm, addr,
6629							     gup_flags, &vma);
6630
6631		if (IS_ERR(page)) {
6632			/* We might need to expand the stack to access it */
6633			vma = vma_lookup(mm, addr);
6634			if (!vma) {
6635				vma = expand_stack(mm, addr);
6636
6637				/* mmap_lock was dropped on failure */
6638				if (!vma)
6639					return buf - old_buf;
6640
6641				/* Try again if stack expansion worked */
6642				continue;
6643			}
6644
 
 
 
 
 
 
6645			/*
6646			 * Check if this is a VM_IO | VM_PFNMAP VMA, which
6647			 * we can access using slightly different code.
6648			 */
6649			bytes = 0;
6650#ifdef CONFIG_HAVE_IOREMAP_PROT
 
6651			if (vma->vm_ops && vma->vm_ops->access)
6652				bytes = vma->vm_ops->access(vma, addr, buf,
6653							    len, write);
6654#endif
6655			if (bytes <= 0)
6656				break;
 
 
6657		} else {
6658			bytes = len;
6659			offset = addr & (PAGE_SIZE-1);
6660			if (bytes > PAGE_SIZE-offset)
6661				bytes = PAGE_SIZE-offset;
6662
6663			maddr = kmap_local_page(page);
6664			if (write) {
6665				copy_to_user_page(vma, page, addr,
6666						  maddr + offset, buf, bytes);
6667				set_page_dirty_lock(page);
6668			} else {
6669				copy_from_user_page(vma, page, addr,
6670						    buf, maddr + offset, bytes);
6671			}
6672			unmap_and_put_page(page, maddr);
 
6673		}
6674		len -= bytes;
6675		buf += bytes;
6676		addr += bytes;
6677	}
6678	mmap_read_unlock(mm);
6679
6680	return buf - old_buf;
6681}
6682
6683/**
6684 * access_remote_vm - access another process' address space
6685 * @mm:		the mm_struct of the target address space
6686 * @addr:	start address to access
6687 * @buf:	source or destination buffer
6688 * @len:	number of bytes to transfer
6689 * @gup_flags:	flags modifying lookup behaviour
6690 *
6691 * The caller must hold a reference on @mm.
6692 *
6693 * Return: number of bytes copied from source to destination.
6694 */
6695int access_remote_vm(struct mm_struct *mm, unsigned long addr,
6696		void *buf, int len, unsigned int gup_flags)
6697{
6698	return __access_remote_vm(mm, addr, buf, len, gup_flags);
6699}
6700
6701/*
6702 * Access another process' address space.
6703 * Source/target buffer must be kernel space,
6704 * Do not walk the page table directly, use get_user_pages
6705 */
6706int access_process_vm(struct task_struct *tsk, unsigned long addr,
6707		void *buf, int len, unsigned int gup_flags)
6708{
6709	struct mm_struct *mm;
6710	int ret;
6711
6712	mm = get_task_mm(tsk);
6713	if (!mm)
6714		return 0;
6715
6716	ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
6717
6718	mmput(mm);
6719
6720	return ret;
6721}
6722EXPORT_SYMBOL_GPL(access_process_vm);
6723
6724/*
6725 * Print the name of a VMA.
6726 */
6727void print_vma_addr(char *prefix, unsigned long ip)
6728{
6729	struct mm_struct *mm = current->mm;
6730	struct vm_area_struct *vma;
6731
6732	/*
6733	 * we might be running from an atomic context so we cannot sleep
6734	 */
6735	if (!mmap_read_trylock(mm))
6736		return;
6737
6738	vma = vma_lookup(mm, ip);
6739	if (vma && vma->vm_file) {
6740		struct file *f = vma->vm_file;
6741		ip -= vma->vm_start;
6742		ip += vma->vm_pgoff << PAGE_SHIFT;
6743		printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip,
6744				vma->vm_start,
6745				vma->vm_end - vma->vm_start);
 
 
 
 
 
 
 
6746	}
6747	mmap_read_unlock(mm);
6748}
6749
6750#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
6751void __might_fault(const char *file, int line)
6752{
 
 
 
 
 
 
 
 
6753	if (pagefault_disabled())
6754		return;
6755	__might_sleep(file, line);
6756#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
6757	if (current->mm)
6758		might_lock_read(&current->mm->mmap_lock);
6759#endif
6760}
6761EXPORT_SYMBOL(__might_fault);
6762#endif
6763
6764#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
6765/*
6766 * Process all subpages of the specified huge page with the specified
6767 * operation.  The target subpage will be processed last to keep its
6768 * cache lines hot.
6769 */
6770static inline int process_huge_page(
6771	unsigned long addr_hint, unsigned int nr_pages,
6772	int (*process_subpage)(unsigned long addr, int idx, void *arg),
6773	void *arg)
6774{
6775	int i, n, base, l, ret;
6776	unsigned long addr = addr_hint &
6777		~(((unsigned long)nr_pages << PAGE_SHIFT) - 1);
6778
6779	/* Process target subpage last to keep its cache lines hot */
6780	might_sleep();
6781	n = (addr_hint - addr) / PAGE_SIZE;
6782	if (2 * n <= nr_pages) {
6783		/* If target subpage in first half of huge page */
6784		base = 0;
6785		l = n;
6786		/* Process subpages at the end of huge page */
6787		for (i = nr_pages - 1; i >= 2 * n; i--) {
6788			cond_resched();
6789			ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
6790			if (ret)
6791				return ret;
6792		}
6793	} else {
6794		/* If target subpage in second half of huge page */
6795		base = nr_pages - 2 * (nr_pages - n);
6796		l = nr_pages - n;
6797		/* Process subpages at the begin of huge page */
6798		for (i = 0; i < base; i++) {
6799			cond_resched();
6800			ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
6801			if (ret)
6802				return ret;
6803		}
6804	}
6805	/*
6806	 * Process remaining subpages in left-right-left-right pattern
6807	 * towards the target subpage
6808	 */
6809	for (i = 0; i < l; i++) {
6810		int left_idx = base + i;
6811		int right_idx = base + 2 * l - 1 - i;
6812
6813		cond_resched();
6814		ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
6815		if (ret)
6816			return ret;
6817		cond_resched();
6818		ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
6819		if (ret)
6820			return ret;
6821	}
6822	return 0;
6823}
6824
6825static void clear_gigantic_page(struct folio *folio, unsigned long addr_hint,
6826				unsigned int nr_pages)
 
6827{
6828	unsigned long addr = ALIGN_DOWN(addr_hint, folio_size(folio));
6829	int i;
 
6830
6831	might_sleep();
6832	for (i = 0; i < nr_pages; i++) {
 
6833		cond_resched();
6834		clear_user_highpage(folio_page(folio, i), addr + i * PAGE_SIZE);
6835	}
6836}
6837
6838static int clear_subpage(unsigned long addr, int idx, void *arg)
6839{
6840	struct folio *folio = arg;
6841
6842	clear_user_highpage(folio_page(folio, idx), addr);
6843	return 0;
6844}
6845
6846/**
6847 * folio_zero_user - Zero a folio which will be mapped to userspace.
6848 * @folio: The folio to zero.
6849 * @addr_hint: The address will be accessed or the base address if uncelar.
6850 */
6851void folio_zero_user(struct folio *folio, unsigned long addr_hint)
6852{
6853	unsigned int nr_pages = folio_nr_pages(folio);
 
6854
6855	if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
6856		clear_gigantic_page(folio, addr_hint, nr_pages);
6857	else
6858		process_huge_page(addr_hint, nr_pages, clear_subpage, folio);
 
 
6859}
6860
6861static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
6862				   unsigned long addr_hint,
6863				   struct vm_area_struct *vma,
6864				   unsigned int nr_pages)
6865{
6866	unsigned long addr = ALIGN_DOWN(addr_hint, folio_size(dst));
6867	struct page *dst_page;
6868	struct page *src_page;
6869	int i;
 
 
6870
6871	for (i = 0; i < nr_pages; i++) {
6872		dst_page = folio_page(dst, i);
6873		src_page = folio_page(src, i);
6874
6875		cond_resched();
6876		if (copy_mc_user_highpage(dst_page, src_page,
6877					  addr + i*PAGE_SIZE, vma))
6878			return -EHWPOISON;
 
 
6879	}
6880	return 0;
6881}
6882
6883struct copy_subpage_arg {
6884	struct folio *dst;
6885	struct folio *src;
6886	struct vm_area_struct *vma;
6887};
6888
6889static int copy_subpage(unsigned long addr, int idx, void *arg)
6890{
6891	struct copy_subpage_arg *copy_arg = arg;
6892	struct page *dst = folio_page(copy_arg->dst, idx);
6893	struct page *src = folio_page(copy_arg->src, idx);
6894
6895	if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma))
6896		return -EHWPOISON;
6897	return 0;
6898}
6899
6900int copy_user_large_folio(struct folio *dst, struct folio *src,
6901			  unsigned long addr_hint, struct vm_area_struct *vma)
 
6902{
6903	unsigned int nr_pages = folio_nr_pages(dst);
 
6904	struct copy_subpage_arg arg = {
6905		.dst = dst,
6906		.src = src,
6907		.vma = vma,
6908	};
6909
6910	if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
6911		return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages);
 
 
 
6912
6913	return process_huge_page(addr_hint, nr_pages, copy_subpage, &arg);
6914}
6915
6916long copy_folio_from_user(struct folio *dst_folio,
6917			   const void __user *usr_src,
6918			   bool allow_pagefault)
 
6919{
6920	void *kaddr;
 
6921	unsigned long i, rc = 0;
6922	unsigned int nr_pages = folio_nr_pages(dst_folio);
6923	unsigned long ret_val = nr_pages * PAGE_SIZE;
6924	struct page *subpage;
6925
6926	for (i = 0; i < nr_pages; i++) {
6927		subpage = folio_page(dst_folio, i);
6928		kaddr = kmap_local_page(subpage);
6929		if (!allow_pagefault)
6930			pagefault_disable();
6931		rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
6932		if (!allow_pagefault)
6933			pagefault_enable();
6934		kunmap_local(kaddr);
 
 
 
6935
6936		ret_val -= (PAGE_SIZE - rc);
6937		if (rc)
6938			break;
6939
6940		flush_dcache_page(subpage);
6941
6942		cond_resched();
6943	}
6944	return ret_val;
6945}
6946#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
6947
6948#if defined(CONFIG_SPLIT_PTE_PTLOCKS) && ALLOC_SPLIT_PTLOCKS
6949
6950static struct kmem_cache *page_ptl_cachep;
6951
6952void __init ptlock_cache_init(void)
6953{
6954	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
6955			SLAB_PANIC, NULL);
6956}
6957
6958bool ptlock_alloc(struct ptdesc *ptdesc)
6959{
6960	spinlock_t *ptl;
6961
6962	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
6963	if (!ptl)
6964		return false;
6965	ptdesc->ptl = ptl;
6966	return true;
6967}
6968
6969void ptlock_free(struct ptdesc *ptdesc)
6970{
6971	kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
6972}
6973#endif
6974
6975void vma_pgtable_walk_begin(struct vm_area_struct *vma)
6976{
6977	if (is_vm_hugetlb_page(vma))
6978		hugetlb_vma_lock_read(vma);
6979}
6980
6981void vma_pgtable_walk_end(struct vm_area_struct *vma)
6982{
6983	if (is_vm_hugetlb_page(vma))
6984		hugetlb_vma_unlock_read(vma);
6985}