1// SPDX-License-Identifier: GPL-2.0-only
   2#include <linux/kernel.h>
   3#include <linux/errno.h>
   4#include <linux/err.h>
   5#include <linux/spinlock.h>
   6
   7#include <linux/mm.h>
   8#include <linux/memremap.h>
   9#include <linux/pagemap.h>
  10#include <linux/rmap.h>
  11#include <linux/swap.h>
  12#include <linux/swapops.h>
  13#include <linux/secretmem.h>
  14
  15#include <linux/sched/signal.h>
  16#include <linux/rwsem.h>
  17#include <linux/hugetlb.h>
  18#include <linux/migrate.h>
  19#include <linux/mm_inline.h>
  20#include <linux/sched/mm.h>
  21
  22#include <asm/mmu_context.h>
 
  23#include <asm/tlbflush.h>
  24
  25#include "internal.h"
  26
  27struct follow_page_context {
  28	struct dev_pagemap *pgmap;
  29	unsigned int page_mask;
  30};
  31
  32static void hpage_pincount_add(struct page *page, int refs)
  33{
  34	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
  35	VM_BUG_ON_PAGE(page != compound_head(page), page);
  36
  37	atomic_add(refs, compound_pincount_ptr(page));
  38}
  39
  40static void hpage_pincount_sub(struct page *page, int refs)
  41{
  42	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
  43	VM_BUG_ON_PAGE(page != compound_head(page), page);
  44
  45	atomic_sub(refs, compound_pincount_ptr(page));
  46}
  47
  48/* Equivalent to calling put_page() @refs times. */
  49static void put_page_refs(struct page *page, int refs)
  50{
  51#ifdef CONFIG_DEBUG_VM
  52	if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
  53		return;
  54#endif
  55
  56	/*
  57	 * Calling put_page() for each ref is unnecessarily slow. Only the last
  58	 * ref needs a put_page().
  59	 */
  60	if (refs > 1)
  61		page_ref_sub(page, refs - 1);
  62	put_page(page);
  63}
  64
  65/*
  66 * Return the compound head page with ref appropriately incremented,
  67 * or NULL if that failed.
  68 */
  69static inline struct page *try_get_compound_head(struct page *page, int refs)
  70{
  71	struct page *head = compound_head(page);
  72
  73	if (WARN_ON_ONCE(page_ref_count(head) < 0))
  74		return NULL;
  75	if (unlikely(!page_cache_add_speculative(head, refs)))
  76		return NULL;
  77
  78	/*
  79	 * At this point we have a stable reference to the head page; but it
  80	 * could be that between the compound_head() lookup and the refcount
  81	 * increment, the compound page was split, in which case we'd end up
  82	 * holding a reference on a page that has nothing to do with the page
  83	 * we were given anymore.
  84	 * So now that the head page is stable, recheck that the pages still
  85	 * belong together.
  86	 */
  87	if (unlikely(compound_head(page) != head)) {
  88		put_page_refs(head, refs);
  89		return NULL;
  90	}
  91
  92	return head;
  93}
  94
  95/*
  96 * try_grab_compound_head() - attempt to elevate a page's refcount, by a
  97 * flags-dependent amount.
  98 *
  99 * "grab" names in this file mean, "look at flags to decide whether to use
 100 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
 101 *
 102 * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
 103 * same time. (That's true throughout the get_user_pages*() and
 104 * pin_user_pages*() APIs.) Cases:
 105 *
 106 *    FOLL_GET: page's refcount will be incremented by 1.
 107 *    FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
 108 *
 109 * Return: head page (with refcount appropriately incremented) for success, or
 110 * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
 111 * considered failure, and furthermore, a likely bug in the caller, so a warning
 112 * is also emitted.
 113 */
 114__maybe_unused struct page *try_grab_compound_head(struct page *page,
 115						   int refs, unsigned int flags)
 116{
 117	if (flags & FOLL_GET)
 118		return try_get_compound_head(page, refs);
 119	else if (flags & FOLL_PIN) {
 120		int orig_refs = refs;
 121
 122		/*
 123		 * Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a
 124		 * right zone, so fail and let the caller fall back to the slow
 125		 * path.
 126		 */
 127		if (unlikely((flags & FOLL_LONGTERM) &&
 128			     !is_pinnable_page(page)))
 129			return NULL;
 130
 131		/*
 132		 * CAUTION: Don't use compound_head() on the page before this
 133		 * point, the result won't be stable.
 134		 */
 135		page = try_get_compound_head(page, refs);
 136		if (!page)
 137			return NULL;
 138
 139		/*
 140		 * When pinning a compound page of order > 1 (which is what
 141		 * hpage_pincount_available() checks for), use an exact count to
 142		 * track it, via hpage_pincount_add/_sub().
 143		 *
 144		 * However, be sure to *also* increment the normal page refcount
 145		 * field at least once, so that the page really is pinned.
 146		 */
 147		if (hpage_pincount_available(page))
 148			hpage_pincount_add(page, refs);
 149		else
 150			page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
 151
 152		mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
 153				    orig_refs);
 154
 155		return page;
 156	}
 157
 158	WARN_ON_ONCE(1);
 159	return NULL;
 160}
 161
 162static void put_compound_head(struct page *page, int refs, unsigned int flags)
 163{
 164	if (flags & FOLL_PIN) {
 165		mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
 166				    refs);
 167
 168		if (hpage_pincount_available(page))
 169			hpage_pincount_sub(page, refs);
 170		else
 171			refs *= GUP_PIN_COUNTING_BIAS;
 172	}
 173
 174	put_page_refs(page, refs);
 175}
 176
 177/**
 178 * try_grab_page() - elevate a page's refcount by a flag-dependent amount
 179 *
 180 * This might not do anything at all, depending on the flags argument.
 181 *
 182 * "grab" names in this file mean, "look at flags to decide whether to use
 183 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
 184 *
 185 * @page:    pointer to page to be grabbed
 186 * @flags:   gup flags: these are the FOLL_* flag values.
 187 *
 188 * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
 189 * time. Cases:
 190 *
 191 *    FOLL_GET: page's refcount will be incremented by 1.
 192 *    FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
 193 *
 194 * Return: true for success, or if no action was required (if neither FOLL_PIN
 195 * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or
 196 * FOLL_PIN was set, but the page could not be grabbed.
 197 */
 198bool __must_check try_grab_page(struct page *page, unsigned int flags)
 199{
 200	WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
 201
 202	if (flags & FOLL_GET)
 203		return try_get_page(page);
 204	else if (flags & FOLL_PIN) {
 205		int refs = 1;
 206
 207		page = compound_head(page);
 208
 209		if (WARN_ON_ONCE(page_ref_count(page) <= 0))
 210			return false;
 211
 212		if (hpage_pincount_available(page))
 213			hpage_pincount_add(page, 1);
 214		else
 215			refs = GUP_PIN_COUNTING_BIAS;
 216
 217		/*
 218		 * Similar to try_grab_compound_head(): even if using the
 219		 * hpage_pincount_add/_sub() routines, be sure to
 220		 * *also* increment the normal page refcount field at least
 221		 * once, so that the page really is pinned.
 222		 */
 223		page_ref_add(page, refs);
 224
 225		mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
 226	}
 227
 228	return true;
 229}
 230
 231/**
 232 * unpin_user_page() - release a dma-pinned page
 233 * @page:            pointer to page to be released
 234 *
 235 * Pages that were pinned via pin_user_pages*() must be released via either
 236 * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
 237 * that such pages can be separately tracked and uniquely handled. In
 238 * particular, interactions with RDMA and filesystems need special handling.
 239 */
 240void unpin_user_page(struct page *page)
 241{
 242	put_compound_head(compound_head(page), 1, FOLL_PIN);
 243}
 244EXPORT_SYMBOL(unpin_user_page);
 245
 246static inline void compound_range_next(unsigned long i, unsigned long npages,
 247				       struct page **list, struct page **head,
 248				       unsigned int *ntails)
 249{
 250	struct page *next, *page;
 251	unsigned int nr = 1;
 252
 253	if (i >= npages)
 254		return;
 255
 256	next = *list + i;
 257	page = compound_head(next);
 258	if (PageCompound(page) && compound_order(page) >= 1)
 259		nr = min_t(unsigned int,
 260			   page + compound_nr(page) - next, npages - i);
 261
 262	*head = page;
 263	*ntails = nr;
 264}
 265
 266#define for_each_compound_range(__i, __list, __npages, __head, __ntails) \
 267	for (__i = 0, \
 268	     compound_range_next(__i, __npages, __list, &(__head), &(__ntails)); \
 269	     __i < __npages; __i += __ntails, \
 270	     compound_range_next(__i, __npages, __list, &(__head), &(__ntails)))
 271
 272static inline void compound_next(unsigned long i, unsigned long npages,
 273				 struct page **list, struct page **head,
 274				 unsigned int *ntails)
 275{
 276	struct page *page;
 277	unsigned int nr;
 278
 279	if (i >= npages)
 280		return;
 281
 282	page = compound_head(list[i]);
 283	for (nr = i + 1; nr < npages; nr++) {
 284		if (compound_head(list[nr]) != page)
 285			break;
 286	}
 287
 288	*head = page;
 289	*ntails = nr - i;
 290}
 291
 292#define for_each_compound_head(__i, __list, __npages, __head, __ntails) \
 293	for (__i = 0, \
 294	     compound_next(__i, __npages, __list, &(__head), &(__ntails)); \
 295	     __i < __npages; __i += __ntails, \
 296	     compound_next(__i, __npages, __list, &(__head), &(__ntails)))
 297
 298/**
 299 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
 300 * @pages:  array of pages to be maybe marked dirty, and definitely released.
 301 * @npages: number of pages in the @pages array.
 302 * @make_dirty: whether to mark the pages dirty
 303 *
 304 * "gup-pinned page" refers to a page that has had one of the get_user_pages()
 305 * variants called on that page.
 306 *
 307 * For each page in the @pages array, make that page (or its head page, if a
 308 * compound page) dirty, if @make_dirty is true, and if the page was previously
 309 * listed as clean. In any case, releases all pages using unpin_user_page(),
 310 * possibly via unpin_user_pages(), for the non-dirty case.
 311 *
 312 * Please see the unpin_user_page() documentation for details.
 313 *
 314 * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
 315 * required, then the caller should a) verify that this is really correct,
 316 * because _lock() is usually required, and b) hand code it:
 317 * set_page_dirty_lock(), unpin_user_page().
 318 *
 319 */
 320void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
 321				 bool make_dirty)
 322{
 323	unsigned long index;
 324	struct page *head;
 325	unsigned int ntails;
 326
 327	if (!make_dirty) {
 328		unpin_user_pages(pages, npages);
 329		return;
 330	}
 331
 332	for_each_compound_head(index, pages, npages, head, ntails) {
 333		/*
 334		 * Checking PageDirty at this point may race with
 335		 * clear_page_dirty_for_io(), but that's OK. Two key
 336		 * cases:
 337		 *
 338		 * 1) This code sees the page as already dirty, so it
 339		 * skips the call to set_page_dirty(). That could happen
 340		 * because clear_page_dirty_for_io() called
 341		 * page_mkclean(), followed by set_page_dirty().
 342		 * However, now the page is going to get written back,
 343		 * which meets the original intention of setting it
 344		 * dirty, so all is well: clear_page_dirty_for_io() goes
 345		 * on to call TestClearPageDirty(), and write the page
 346		 * back.
 347		 *
 348		 * 2) This code sees the page as clean, so it calls
 349		 * set_page_dirty(). The page stays dirty, despite being
 350		 * written back, so it gets written back again in the
 351		 * next writeback cycle. This is harmless.
 352		 */
 353		if (!PageDirty(head))
 354			set_page_dirty_lock(head);
 355		put_compound_head(head, ntails, FOLL_PIN);
 356	}
 357}
 358EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
 359
 360/**
 361 * unpin_user_page_range_dirty_lock() - release and optionally dirty
 362 * gup-pinned page range
 363 *
 364 * @page:  the starting page of a range maybe marked dirty, and definitely released.
 365 * @npages: number of consecutive pages to release.
 366 * @make_dirty: whether to mark the pages dirty
 367 *
 368 * "gup-pinned page range" refers to a range of pages that has had one of the
 369 * pin_user_pages() variants called on that page.
 370 *
 371 * For the page ranges defined by [page .. page+npages], make that range (or
 372 * its head pages, if a compound page) dirty, if @make_dirty is true, and if the
 373 * page range was previously listed as clean.
 374 *
 375 * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
 376 * required, then the caller should a) verify that this is really correct,
 377 * because _lock() is usually required, and b) hand code it:
 378 * set_page_dirty_lock(), unpin_user_page().
 379 *
 380 */
 381void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
 382				      bool make_dirty)
 383{
 384	unsigned long index;
 385	struct page *head;
 386	unsigned int ntails;
 387
 388	for_each_compound_range(index, &page, npages, head, ntails) {
 389		if (make_dirty && !PageDirty(head))
 390			set_page_dirty_lock(head);
 391		put_compound_head(head, ntails, FOLL_PIN);
 392	}
 393}
 394EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
 395
 396/**
 397 * unpin_user_pages() - release an array of gup-pinned pages.
 398 * @pages:  array of pages to be marked dirty and released.
 399 * @npages: number of pages in the @pages array.
 400 *
 401 * For each page in the @pages array, release the page using unpin_user_page().
 402 *
 403 * Please see the unpin_user_page() documentation for details.
 404 */
 405void unpin_user_pages(struct page **pages, unsigned long npages)
 406{
 407	unsigned long index;
 408	struct page *head;
 409	unsigned int ntails;
 410
 411	/*
 412	 * If this WARN_ON() fires, then the system *might* be leaking pages (by
 413	 * leaving them pinned), but probably not. More likely, gup/pup returned
 414	 * a hard -ERRNO error to the caller, who erroneously passed it here.
 415	 */
 416	if (WARN_ON(IS_ERR_VALUE(npages)))
 417		return;
 418
 419	for_each_compound_head(index, pages, npages, head, ntails)
 420		put_compound_head(head, ntails, FOLL_PIN);
 421}
 422EXPORT_SYMBOL(unpin_user_pages);
 423
 424/*
 425 * Set the MMF_HAS_PINNED if not set yet; after set it'll be there for the mm's
 426 * lifecycle.  Avoid setting the bit unless necessary, or it might cause write
 427 * cache bouncing on large SMP machines for concurrent pinned gups.
 428 */
 429static inline void mm_set_has_pinned_flag(unsigned long *mm_flags)
 430{
 431	if (!test_bit(MMF_HAS_PINNED, mm_flags))
 432		set_bit(MMF_HAS_PINNED, mm_flags);
 433}
 434
 435#ifdef CONFIG_MMU
 436static struct page *no_page_table(struct vm_area_struct *vma,
 437		unsigned int flags)
 438{
 439	/*
 440	 * When core dumping an enormous anonymous area that nobody
 441	 * has touched so far, we don't want to allocate unnecessary pages or
 442	 * page tables.  Return error instead of NULL to skip handle_mm_fault,
 443	 * then get_dump_page() will return NULL to leave a hole in the dump.
 444	 * But we can only make this optimization where a hole would surely
 445	 * be zero-filled if handle_mm_fault() actually did handle it.
 446	 */
 447	if ((flags & FOLL_DUMP) &&
 448			(vma_is_anonymous(vma) || !vma->vm_ops->fault))
 449		return ERR_PTR(-EFAULT);
 450	return NULL;
 451}
 452
 453static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
 454		pte_t *pte, unsigned int flags)
 455{
 456	/* No page to get reference */
 457	if (flags & FOLL_GET)
 458		return -EFAULT;
 459
 460	if (flags & FOLL_TOUCH) {
 461		pte_t entry = *pte;
 462
 463		if (flags & FOLL_WRITE)
 464			entry = pte_mkdirty(entry);
 465		entry = pte_mkyoung(entry);
 466
 467		if (!pte_same(*pte, entry)) {
 468			set_pte_at(vma->vm_mm, address, pte, entry);
 469			update_mmu_cache(vma, address, pte);
 470		}
 471	}
 472
 473	/* Proper page table entry exists, but no corresponding struct page */
 474	return -EEXIST;
 475}
 476
 477/*
 478 * FOLL_FORCE can write to even unwritable pte's, but only
 479 * after we've gone through a COW cycle and they are dirty.
 480 */
 481static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
 482{
 483	return pte_write(pte) ||
 484		((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
 485}
 486
 487static struct page *follow_page_pte(struct vm_area_struct *vma,
 488		unsigned long address, pmd_t *pmd, unsigned int flags,
 489		struct dev_pagemap **pgmap)
 490{
 491	struct mm_struct *mm = vma->vm_mm;
 
 492	struct page *page;
 493	spinlock_t *ptl;
 494	pte_t *ptep, pte;
 495	int ret;
 496
 497	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
 498	if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
 499			 (FOLL_PIN | FOLL_GET)))
 500		return ERR_PTR(-EINVAL);
 501retry:
 502	if (unlikely(pmd_bad(*pmd)))
 503		return no_page_table(vma, flags);
 504
 505	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
 506	pte = *ptep;
 507	if (!pte_present(pte)) {
 508		swp_entry_t entry;
 509		/*
 510		 * KSM's break_ksm() relies upon recognizing a ksm page
 511		 * even while it is being migrated, so for that case we
 512		 * need migration_entry_wait().
 513		 */
 514		if (likely(!(flags & FOLL_MIGRATION)))
 515			goto no_page;
 516		if (pte_none(pte))
 517			goto no_page;
 518		entry = pte_to_swp_entry(pte);
 519		if (!is_migration_entry(entry))
 520			goto no_page;
 521		pte_unmap_unlock(ptep, ptl);
 522		migration_entry_wait(mm, pmd, address);
 523		goto retry;
 524	}
 525	if ((flags & FOLL_NUMA) && pte_protnone(pte))
 526		goto no_page;
 527	if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
 528		pte_unmap_unlock(ptep, ptl);
 529		return NULL;
 530	}
 531
 532	page = vm_normal_page(vma, address, pte);
 533	if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
 534		/*
 535		 * Only return device mapping pages in the FOLL_GET or FOLL_PIN
 536		 * case since they are only valid while holding the pgmap
 537		 * reference.
 538		 */
 539		*pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
 540		if (*pgmap)
 541			page = pte_page(pte);
 542		else
 543			goto no_page;
 544	} else if (unlikely(!page)) {
 545		if (flags & FOLL_DUMP) {
 546			/* Avoid special (like zero) pages in core dumps */
 547			page = ERR_PTR(-EFAULT);
 548			goto out;
 549		}
 550
 551		if (is_zero_pfn(pte_pfn(pte))) {
 552			page = pte_page(pte);
 553		} else {
 
 
 554			ret = follow_pfn_pte(vma, address, ptep, flags);
 555			page = ERR_PTR(ret);
 556			goto out;
 557		}
 558	}
 559
 560	/* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
 561	if (unlikely(!try_grab_page(page, flags))) {
 562		page = ERR_PTR(-ENOMEM);
 563		goto out;
 
 
 
 
 
 
 
 564	}
 565	/*
 566	 * We need to make the page accessible if and only if we are going
 567	 * to access its content (the FOLL_PIN case).  Please see
 568	 * Documentation/core-api/pin_user_pages.rst for details.
 569	 */
 570	if (flags & FOLL_PIN) {
 571		ret = arch_make_page_accessible(page);
 572		if (ret) {
 573			unpin_user_page(page);
 574			page = ERR_PTR(ret);
 575			goto out;
 576		}
 577	}
 578	if (flags & FOLL_TOUCH) {
 579		if ((flags & FOLL_WRITE) &&
 580		    !pte_dirty(pte) && !PageDirty(page))
 581			set_page_dirty(page);
 582		/*
 583		 * pte_mkyoung() would be more correct here, but atomic care
 584		 * is needed to avoid losing the dirty bit: it is easier to use
 585		 * mark_page_accessed().
 586		 */
 587		mark_page_accessed(page);
 588	}
 589	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
 590		/* Do not mlock pte-mapped THP */
 591		if (PageTransCompound(page))
 592			goto out;
 593
 594		/*
 595		 * The preliminary mapping check is mainly to avoid the
 596		 * pointless overhead of lock_page on the ZERO_PAGE
 597		 * which might bounce very badly if there is contention.
 598		 *
 599		 * If the page is already locked, we don't need to
 600		 * handle it now - vmscan will handle it later if and
 601		 * when it attempts to reclaim the page.
 602		 */
 603		if (page->mapping && trylock_page(page)) {
 604			lru_add_drain();  /* push cached pages to LRU */
 605			/*
 606			 * Because we lock page here, and migration is
 607			 * blocked by the pte's page reference, and we
 608			 * know the page is still mapped, we don't even
 609			 * need to check for file-cache page truncation.
 610			 */
 611			mlock_vma_page(page);
 612			unlock_page(page);
 613		}
 614	}
 615out:
 616	pte_unmap_unlock(ptep, ptl);
 617	return page;
 618no_page:
 619	pte_unmap_unlock(ptep, ptl);
 620	if (!pte_none(pte))
 621		return NULL;
 622	return no_page_table(vma, flags);
 623}
 624
 625static struct page *follow_pmd_mask(struct vm_area_struct *vma,
 626				    unsigned long address, pud_t *pudp,
 627				    unsigned int flags,
 628				    struct follow_page_context *ctx)
 629{
 630	pmd_t *pmd, pmdval;
 631	spinlock_t *ptl;
 632	struct page *page;
 633	struct mm_struct *mm = vma->vm_mm;
 634
 635	pmd = pmd_offset(pudp, address);
 636	/*
 637	 * The READ_ONCE() will stabilize the pmdval in a register or
 638	 * on the stack so that it will stop changing under the code.
 639	 */
 640	pmdval = READ_ONCE(*pmd);
 641	if (pmd_none(pmdval))
 642		return no_page_table(vma, flags);
 643	if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
 644		page = follow_huge_pmd(mm, address, pmd, flags);
 645		if (page)
 646			return page;
 647		return no_page_table(vma, flags);
 648	}
 649	if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
 650		page = follow_huge_pd(vma, address,
 651				      __hugepd(pmd_val(pmdval)), flags,
 652				      PMD_SHIFT);
 653		if (page)
 654			return page;
 655		return no_page_table(vma, flags);
 656	}
 657retry:
 658	if (!pmd_present(pmdval)) {
 659		if (likely(!(flags & FOLL_MIGRATION)))
 660			return no_page_table(vma, flags);
 661		VM_BUG_ON(thp_migration_supported() &&
 662				  !is_pmd_migration_entry(pmdval));
 663		if (is_pmd_migration_entry(pmdval))
 664			pmd_migration_entry_wait(mm, pmd);
 665		pmdval = READ_ONCE(*pmd);
 666		/*
 667		 * MADV_DONTNEED may convert the pmd to null because
 668		 * mmap_lock is held in read mode
 669		 */
 670		if (pmd_none(pmdval))
 671			return no_page_table(vma, flags);
 672		goto retry;
 673	}
 674	if (pmd_devmap(pmdval)) {
 675		ptl = pmd_lock(mm, pmd);
 676		page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
 677		spin_unlock(ptl);
 678		if (page)
 679			return page;
 680	}
 681	if (likely(!pmd_trans_huge(pmdval)))
 682		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
 683
 684	if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
 685		return no_page_table(vma, flags);
 686
 687retry_locked:
 688	ptl = pmd_lock(mm, pmd);
 689	if (unlikely(pmd_none(*pmd))) {
 690		spin_unlock(ptl);
 691		return no_page_table(vma, flags);
 692	}
 693	if (unlikely(!pmd_present(*pmd))) {
 694		spin_unlock(ptl);
 695		if (likely(!(flags & FOLL_MIGRATION)))
 696			return no_page_table(vma, flags);
 697		pmd_migration_entry_wait(mm, pmd);
 698		goto retry_locked;
 699	}
 700	if (unlikely(!pmd_trans_huge(*pmd))) {
 701		spin_unlock(ptl);
 702		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
 703	}
 704	if (flags & FOLL_SPLIT_PMD) {
 705		int ret;
 706		page = pmd_page(*pmd);
 707		if (is_huge_zero_page(page)) {
 708			spin_unlock(ptl);
 709			ret = 0;
 710			split_huge_pmd(vma, pmd, address);
 711			if (pmd_trans_unstable(pmd))
 712				ret = -EBUSY;
 713		} else {
 
 714			spin_unlock(ptl);
 715			split_huge_pmd(vma, pmd, address);
 716			ret = pte_alloc(mm, pmd) ? -ENOMEM : 0;
 
 
 
 
 717		}
 718
 719		return ret ? ERR_PTR(ret) :
 720			follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
 721	}
 722	page = follow_trans_huge_pmd(vma, address, pmd, flags);
 723	spin_unlock(ptl);
 724	ctx->page_mask = HPAGE_PMD_NR - 1;
 725	return page;
 726}
 727
 
 728static struct page *follow_pud_mask(struct vm_area_struct *vma,
 729				    unsigned long address, p4d_t *p4dp,
 730				    unsigned int flags,
 731				    struct follow_page_context *ctx)
 732{
 733	pud_t *pud;
 734	spinlock_t *ptl;
 735	struct page *page;
 736	struct mm_struct *mm = vma->vm_mm;
 737
 738	pud = pud_offset(p4dp, address);
 739	if (pud_none(*pud))
 740		return no_page_table(vma, flags);
 741	if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) {
 742		page = follow_huge_pud(mm, address, pud, flags);
 743		if (page)
 744			return page;
 745		return no_page_table(vma, flags);
 746	}
 747	if (is_hugepd(__hugepd(pud_val(*pud)))) {
 748		page = follow_huge_pd(vma, address,
 749				      __hugepd(pud_val(*pud)), flags,
 750				      PUD_SHIFT);
 751		if (page)
 752			return page;
 753		return no_page_table(vma, flags);
 754	}
 755	if (pud_devmap(*pud)) {
 756		ptl = pud_lock(mm, pud);
 757		page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
 758		spin_unlock(ptl);
 759		if (page)
 760			return page;
 761	}
 762	if (unlikely(pud_bad(*pud)))
 763		return no_page_table(vma, flags);
 764
 765	return follow_pmd_mask(vma, address, pud, flags, ctx);
 766}
 767
 
 768static struct page *follow_p4d_mask(struct vm_area_struct *vma,
 769				    unsigned long address, pgd_t *pgdp,
 770				    unsigned int flags,
 771				    struct follow_page_context *ctx)
 772{
 773	p4d_t *p4d;
 774	struct page *page;
 775
 776	p4d = p4d_offset(pgdp, address);
 777	if (p4d_none(*p4d))
 778		return no_page_table(vma, flags);
 779	BUILD_BUG_ON(p4d_huge(*p4d));
 780	if (unlikely(p4d_bad(*p4d)))
 781		return no_page_table(vma, flags);
 782
 783	if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
 784		page = follow_huge_pd(vma, address,
 785				      __hugepd(p4d_val(*p4d)), flags,
 786				      P4D_SHIFT);
 787		if (page)
 788			return page;
 789		return no_page_table(vma, flags);
 790	}
 791	return follow_pud_mask(vma, address, p4d, flags, ctx);
 792}
 793
 794/**
 795 * follow_page_mask - look up a page descriptor from a user-virtual address
 796 * @vma: vm_area_struct mapping @address
 797 * @address: virtual address to look up
 798 * @flags: flags modifying lookup behaviour
 799 * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
 800 *       pointer to output page_mask
 801 *
 802 * @flags can have FOLL_ flags set, defined in <linux/mm.h>
 803 *
 804 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
 805 * the device's dev_pagemap metadata to avoid repeating expensive lookups.
 806 *
 807 * On output, the @ctx->page_mask is set according to the size of the page.
 808 *
 809 * Return: the mapped (struct page *), %NULL if no mapping exists, or
 810 * an error pointer if there is a mapping to something not represented
 811 * by a page descriptor (see also vm_normal_page()).
 812 */
 813static struct page *follow_page_mask(struct vm_area_struct *vma,
 814			      unsigned long address, unsigned int flags,
 815			      struct follow_page_context *ctx)
 816{
 817	pgd_t *pgd;
 818	struct page *page;
 819	struct mm_struct *mm = vma->vm_mm;
 820
 821	ctx->page_mask = 0;
 822
 823	/* make this handle hugepd */
 824	page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
 825	if (!IS_ERR(page)) {
 826		WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN));
 827		return page;
 828	}
 829
 830	pgd = pgd_offset(mm, address);
 831
 832	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
 833		return no_page_table(vma, flags);
 834
 835	if (pgd_huge(*pgd)) {
 836		page = follow_huge_pgd(mm, address, pgd, flags);
 837		if (page)
 838			return page;
 839		return no_page_table(vma, flags);
 840	}
 841	if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
 842		page = follow_huge_pd(vma, address,
 843				      __hugepd(pgd_val(*pgd)), flags,
 844				      PGDIR_SHIFT);
 845		if (page)
 846			return page;
 847		return no_page_table(vma, flags);
 848	}
 849
 850	return follow_p4d_mask(vma, address, pgd, flags, ctx);
 851}
 852
 853struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
 854			 unsigned int foll_flags)
 855{
 856	struct follow_page_context ctx = { NULL };
 857	struct page *page;
 858
 859	if (vma_is_secretmem(vma))
 860		return NULL;
 861
 862	page = follow_page_mask(vma, address, foll_flags, &ctx);
 863	if (ctx.pgmap)
 864		put_dev_pagemap(ctx.pgmap);
 865	return page;
 866}
 867
 868static int get_gate_page(struct mm_struct *mm, unsigned long address,
 869		unsigned int gup_flags, struct vm_area_struct **vma,
 870		struct page **page)
 871{
 872	pgd_t *pgd;
 873	p4d_t *p4d;
 874	pud_t *pud;
 875	pmd_t *pmd;
 876	pte_t *pte;
 877	int ret = -EFAULT;
 878
 879	/* user gate pages are read-only */
 880	if (gup_flags & FOLL_WRITE)
 881		return -EFAULT;
 882	if (address > TASK_SIZE)
 883		pgd = pgd_offset_k(address);
 884	else
 885		pgd = pgd_offset_gate(mm, address);
 886	if (pgd_none(*pgd))
 887		return -EFAULT;
 888	p4d = p4d_offset(pgd, address);
 889	if (p4d_none(*p4d))
 890		return -EFAULT;
 891	pud = pud_offset(p4d, address);
 892	if (pud_none(*pud))
 893		return -EFAULT;
 894	pmd = pmd_offset(pud, address);
 895	if (!pmd_present(*pmd))
 896		return -EFAULT;
 897	VM_BUG_ON(pmd_trans_huge(*pmd));
 898	pte = pte_offset_map(pmd, address);
 899	if (pte_none(*pte))
 900		goto unmap;
 901	*vma = get_gate_vma(mm);
 902	if (!page)
 903		goto out;
 904	*page = vm_normal_page(*vma, address, *pte);
 905	if (!*page) {
 906		if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
 907			goto unmap;
 908		*page = pte_page(*pte);
 
 
 
 
 
 
 
 909	}
 910	if (unlikely(!try_grab_page(*page, gup_flags))) {
 911		ret = -ENOMEM;
 912		goto unmap;
 913	}
 914out:
 915	ret = 0;
 916unmap:
 917	pte_unmap(pte);
 918	return ret;
 919}
 920
 921/*
 922 * mmap_lock must be held on entry.  If @locked != NULL and *@flags
 923 * does not include FOLL_NOWAIT, the mmap_lock may be released.  If it
 924 * is, *@locked will be set to 0 and -EBUSY returned.
 925 */
 926static int faultin_page(struct vm_area_struct *vma,
 927		unsigned long address, unsigned int *flags, int *locked)
 928{
 929	unsigned int fault_flags = 0;
 930	vm_fault_t ret;
 931
 932	/* mlock all present pages, but do not fault in new pages */
 933	if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
 934		return -ENOENT;
 935	if (*flags & FOLL_WRITE)
 936		fault_flags |= FAULT_FLAG_WRITE;
 937	if (*flags & FOLL_REMOTE)
 938		fault_flags |= FAULT_FLAG_REMOTE;
 939	if (locked)
 940		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 941	if (*flags & FOLL_NOWAIT)
 942		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
 943	if (*flags & FOLL_TRIED) {
 944		/*
 945		 * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
 946		 * can co-exist
 947		 */
 948		fault_flags |= FAULT_FLAG_TRIED;
 949	}
 950
 951	ret = handle_mm_fault(vma, address, fault_flags, NULL);
 952	if (ret & VM_FAULT_ERROR) {
 953		int err = vm_fault_to_errno(ret, *flags);
 954
 955		if (err)
 956			return err;
 957		BUG();
 958	}
 959
 
 
 
 
 
 
 
 960	if (ret & VM_FAULT_RETRY) {
 961		if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
 962			*locked = 0;
 963		return -EBUSY;
 964	}
 965
 966	/*
 967	 * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
 968	 * necessary, even if maybe_mkwrite decided not to set pte_write. We
 969	 * can thus safely do subsequent page lookups as if they were reads.
 970	 * But only do so when looping for pte_write is futile: in some cases
 971	 * userspace may also be wanting to write to the gotten user page,
 972	 * which a read fault here might prevent (a readonly page might get
 973	 * reCOWed by userspace write).
 974	 */
 975	if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
 976		*flags |= FOLL_COW;
 977	return 0;
 978}
 979
 980static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
 981{
 982	vm_flags_t vm_flags = vma->vm_flags;
 983	int write = (gup_flags & FOLL_WRITE);
 984	int foreign = (gup_flags & FOLL_REMOTE);
 985
 986	if (vm_flags & (VM_IO | VM_PFNMAP))
 987		return -EFAULT;
 988
 989	if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
 990		return -EFAULT;
 991
 992	if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma))
 993		return -EOPNOTSUPP;
 994
 995	if (vma_is_secretmem(vma))
 996		return -EFAULT;
 997
 998	if (write) {
 999		if (!(vm_flags & VM_WRITE)) {
1000			if (!(gup_flags & FOLL_FORCE))
1001				return -EFAULT;
1002			/*
1003			 * We used to let the write,force case do COW in a
1004			 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
1005			 * set a breakpoint in a read-only mapping of an
1006			 * executable, without corrupting the file (yet only
1007			 * when that file had been opened for writing!).
1008			 * Anon pages in shared mappings are surprising: now
1009			 * just reject it.
1010			 */
1011			if (!is_cow_mapping(vm_flags))
1012				return -EFAULT;
1013		}
1014	} else if (!(vm_flags & VM_READ)) {
1015		if (!(gup_flags & FOLL_FORCE))
1016			return -EFAULT;
1017		/*
1018		 * Is there actually any vma we can reach here which does not
1019		 * have VM_MAYREAD set?
1020		 */
1021		if (!(vm_flags & VM_MAYREAD))
1022			return -EFAULT;
1023	}
1024	/*
1025	 * gups are always data accesses, not instruction
1026	 * fetches, so execute=false here
1027	 */
1028	if (!arch_vma_access_permitted(vma, write, false, foreign))
1029		return -EFAULT;
1030	return 0;
1031}
1032
1033/**
1034 * __get_user_pages() - pin user pages in memory
 
1035 * @mm:		mm_struct of target mm
1036 * @start:	starting user address
1037 * @nr_pages:	number of pages from start to pin
1038 * @gup_flags:	flags modifying pin behaviour
1039 * @pages:	array that receives pointers to the pages pinned.
1040 *		Should be at least nr_pages long. Or NULL, if caller
1041 *		only intends to ensure the pages are faulted in.
1042 * @vmas:	array of pointers to vmas corresponding to each page.
1043 *		Or NULL if the caller does not require them.
1044 * @locked:     whether we're still with the mmap_lock held
1045 *
1046 * Returns either number of pages pinned (which may be less than the
1047 * number requested), or an error. Details about the return value:
 
 
 
1048 *
1049 * -- If nr_pages is 0, returns 0.
1050 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1051 * -- If nr_pages is >0, and some pages were pinned, returns the number of
1052 *    pages pinned. Again, this may be less than nr_pages.
1053 * -- 0 return value is possible when the fault would need to be retried.
1054 *
1055 * The caller is responsible for releasing returned @pages, via put_page().
1056 *
1057 * @vmas are valid only as long as mmap_lock is held.
1058 *
1059 * Must be called with mmap_lock held.  It may be released.  See below.
1060 *
1061 * __get_user_pages walks a process's page tables and takes a reference to
1062 * each struct page that each user address corresponds to at a given
1063 * instant. That is, it takes the page that would be accessed if a user
1064 * thread accesses the given user virtual address at that instant.
1065 *
1066 * This does not guarantee that the page exists in the user mappings when
1067 * __get_user_pages returns, and there may even be a completely different
1068 * page there in some cases (eg. if mmapped pagecache has been invalidated
1069 * and subsequently re faulted). However it does guarantee that the page
1070 * won't be freed completely. And mostly callers simply care that the page
1071 * contains data that was valid *at some point in time*. Typically, an IO
1072 * or similar operation cannot guarantee anything stronger anyway because
1073 * locks can't be held over the syscall boundary.
1074 *
1075 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
1076 * the page is written to, set_page_dirty (or set_page_dirty_lock, as
1077 * appropriate) must be called after the page is finished with, and
1078 * before put_page is called.
1079 *
1080 * If @locked != NULL, *@locked will be set to 0 when mmap_lock is
1081 * released by an up_read().  That can happen if @gup_flags does not
1082 * have FOLL_NOWAIT.
 
 
1083 *
1084 * A caller using such a combination of @locked and @gup_flags
1085 * must therefore hold the mmap_lock for reading only, and recognize
1086 * when it's been released.  Otherwise, it must be held for either
1087 * reading or writing and will not be released.
1088 *
1089 * In most cases, get_user_pages or get_user_pages_fast should be used
1090 * instead of __get_user_pages. __get_user_pages should be used only if
1091 * you need some special @gup_flags.
1092 */
1093static long __get_user_pages(struct mm_struct *mm,
1094		unsigned long start, unsigned long nr_pages,
1095		unsigned int gup_flags, struct page **pages,
1096		struct vm_area_struct **vmas, int *locked)
1097{
1098	long ret = 0, i = 0;
 
1099	struct vm_area_struct *vma = NULL;
1100	struct follow_page_context ctx = { NULL };
1101
1102	if (!nr_pages)
1103		return 0;
1104
1105	start = untagged_addr(start);
1106
1107	VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1108
1109	/*
1110	 * If FOLL_FORCE is set then do not force a full fault as the hinting
1111	 * fault information is unrelated to the reference behaviour of a task
1112	 * using the address space
1113	 */
1114	if (!(gup_flags & FOLL_FORCE))
1115		gup_flags |= FOLL_NUMA;
1116
1117	do {
1118		struct page *page;
1119		unsigned int foll_flags = gup_flags;
1120		unsigned int page_increm;
1121
1122		/* first iteration or cross vma bound */
1123		if (!vma || start >= vma->vm_end) {
1124			vma = find_extend_vma(mm, start);
1125			if (!vma && in_gate_area(mm, start)) {
 
1126				ret = get_gate_page(mm, start & PAGE_MASK,
1127						gup_flags, &vma,
1128						pages ? &pages[i] : NULL);
1129				if (ret)
1130					goto out;
1131				ctx.page_mask = 0;
1132				goto next_page;
1133			}
1134
1135			if (!vma) {
1136				ret = -EFAULT;
1137				goto out;
1138			}
1139			ret = check_vma_flags(vma, gup_flags);
1140			if (ret)
1141				goto out;
1142
1143			if (is_vm_hugetlb_page(vma)) {
1144				i = follow_hugetlb_page(mm, vma, pages, vmas,
1145						&start, &nr_pages, i,
1146						gup_flags, locked);
1147				if (locked && *locked == 0) {
1148					/*
1149					 * We've got a VM_FAULT_RETRY
1150					 * and we've lost mmap_lock.
1151					 * We must stop here.
1152					 */
1153					BUG_ON(gup_flags & FOLL_NOWAIT);
1154					BUG_ON(ret != 0);
1155					goto out;
1156				}
1157				continue;
1158			}
1159		}
1160retry:
1161		/*
1162		 * If we have a pending SIGKILL, don't keep faulting pages and
1163		 * potentially allocating memory.
1164		 */
1165		if (fatal_signal_pending(current)) {
1166			ret = -EINTR;
1167			goto out;
1168		}
1169		cond_resched();
1170
1171		page = follow_page_mask(vma, start, foll_flags, &ctx);
1172		if (!page) {
1173			ret = faultin_page(vma, start, &foll_flags, locked);
 
 
1174			switch (ret) {
1175			case 0:
1176				goto retry;
1177			case -EBUSY:
1178				ret = 0;
1179				fallthrough;
1180			case -EFAULT:
1181			case -ENOMEM:
1182			case -EHWPOISON:
1183				goto out;
 
 
1184			case -ENOENT:
1185				goto next_page;
1186			}
1187			BUG();
1188		} else if (PTR_ERR(page) == -EEXIST) {
1189			/*
1190			 * Proper page table entry exists, but no corresponding
1191			 * struct page.
1192			 */
1193			goto next_page;
1194		} else if (IS_ERR(page)) {
1195			ret = PTR_ERR(page);
1196			goto out;
1197		}
1198		if (pages) {
1199			pages[i] = page;
1200			flush_anon_page(vma, page, start);
1201			flush_dcache_page(page);
1202			ctx.page_mask = 0;
1203		}
1204next_page:
1205		if (vmas) {
1206			vmas[i] = vma;
1207			ctx.page_mask = 0;
1208		}
1209		page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
1210		if (page_increm > nr_pages)
1211			page_increm = nr_pages;
1212		i += page_increm;
1213		start += page_increm * PAGE_SIZE;
1214		nr_pages -= page_increm;
1215	} while (nr_pages);
1216out:
1217	if (ctx.pgmap)
1218		put_dev_pagemap(ctx.pgmap);
1219	return i ? i : ret;
1220}
1221
1222static bool vma_permits_fault(struct vm_area_struct *vma,
1223			      unsigned int fault_flags)
1224{
1225	bool write   = !!(fault_flags & FAULT_FLAG_WRITE);
1226	bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
1227	vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
1228
1229	if (!(vm_flags & vma->vm_flags))
1230		return false;
1231
1232	/*
1233	 * The architecture might have a hardware protection
1234	 * mechanism other than read/write that can deny access.
1235	 *
1236	 * gup always represents data access, not instruction
1237	 * fetches, so execute=false here:
1238	 */
1239	if (!arch_vma_access_permitted(vma, write, false, foreign))
1240		return false;
1241
1242	return true;
1243}
1244
1245/**
1246 * fixup_user_fault() - manually resolve a user page fault
 
 
1247 * @mm:		mm_struct of target mm
1248 * @address:	user address
1249 * @fault_flags:flags to pass down to handle_mm_fault()
1250 * @unlocked:	did we unlock the mmap_lock while retrying, maybe NULL if caller
1251 *		does not allow retry. If NULL, the caller must guarantee
1252 *		that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY.
1253 *
1254 * This is meant to be called in the specific scenario where for locking reasons
1255 * we try to access user memory in atomic context (within a pagefault_disable()
1256 * section), this returns -EFAULT, and we want to resolve the user fault before
1257 * trying again.
1258 *
1259 * Typically this is meant to be used by the futex code.
1260 *
1261 * The main difference with get_user_pages() is that this function will
1262 * unconditionally call handle_mm_fault() which will in turn perform all the
1263 * necessary SW fixup of the dirty and young bits in the PTE, while
1264 * get_user_pages() only guarantees to update these in the struct page.
1265 *
1266 * This is important for some architectures where those bits also gate the
1267 * access permission to the page because they are maintained in software.  On
1268 * such architectures, gup() will not be enough to make a subsequent access
1269 * succeed.
1270 *
1271 * This function will not return with an unlocked mmap_lock. So it has not the
1272 * same semantics wrt the @mm->mmap_lock as does filemap_fault().
1273 */
1274int fixup_user_fault(struct mm_struct *mm,
1275		     unsigned long address, unsigned int fault_flags,
1276		     bool *unlocked)
1277{
1278	struct vm_area_struct *vma;
1279	vm_fault_t ret, major = 0;
1280
1281	address = untagged_addr(address);
1282
1283	if (unlocked)
1284		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1285
1286retry:
1287	vma = find_extend_vma(mm, address);
1288	if (!vma || address < vma->vm_start)
1289		return -EFAULT;
1290
1291	if (!vma_permits_fault(vma, fault_flags))
1292		return -EFAULT;
1293
1294	if ((fault_flags & FAULT_FLAG_KILLABLE) &&
1295	    fatal_signal_pending(current))
1296		return -EINTR;
1297
1298	ret = handle_mm_fault(vma, address, fault_flags, NULL);
1299	major |= ret & VM_FAULT_MAJOR;
1300	if (ret & VM_FAULT_ERROR) {
1301		int err = vm_fault_to_errno(ret, 0);
1302
1303		if (err)
1304			return err;
1305		BUG();
1306	}
1307
1308	if (ret & VM_FAULT_RETRY) {
1309		mmap_read_lock(mm);
1310		*unlocked = true;
1311		fault_flags |= FAULT_FLAG_TRIED;
1312		goto retry;
 
 
 
1313	}
1314
 
 
 
 
 
 
1315	return 0;
1316}
1317EXPORT_SYMBOL_GPL(fixup_user_fault);
1318
1319/*
1320 * Please note that this function, unlike __get_user_pages will not
1321 * return 0 for nr_pages > 0 without FOLL_NOWAIT
1322 */
1323static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
1324						unsigned long start,
1325						unsigned long nr_pages,
1326						struct page **pages,
1327						struct vm_area_struct **vmas,
1328						int *locked,
1329						unsigned int flags)
1330{
1331	long ret, pages_done;
1332	bool lock_dropped;
1333
1334	if (locked) {
1335		/* if VM_FAULT_RETRY can be returned, vmas become invalid */
1336		BUG_ON(vmas);
1337		/* check caller initialized locked */
1338		BUG_ON(*locked != 1);
1339	}
1340
1341	if (flags & FOLL_PIN)
1342		mm_set_has_pinned_flag(&mm->flags);
1343
1344	/*
1345	 * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
1346	 * is to set FOLL_GET if the caller wants pages[] filled in (but has
1347	 * carelessly failed to specify FOLL_GET), so keep doing that, but only
1348	 * for FOLL_GET, not for the newer FOLL_PIN.
1349	 *
1350	 * FOLL_PIN always expects pages to be non-null, but no need to assert
1351	 * that here, as any failures will be obvious enough.
1352	 */
1353	if (pages && !(flags & FOLL_PIN))
1354		flags |= FOLL_GET;
1355
1356	pages_done = 0;
1357	lock_dropped = false;
1358	for (;;) {
1359		ret = __get_user_pages(mm, start, nr_pages, flags, pages,
1360				       vmas, locked);
1361		if (!locked)
1362			/* VM_FAULT_RETRY couldn't trigger, bypass */
1363			return ret;
1364
1365		/* VM_FAULT_RETRY cannot return errors */
1366		if (!*locked) {
1367			BUG_ON(ret < 0);
1368			BUG_ON(ret >= nr_pages);
1369		}
1370
 
 
 
 
1371		if (ret > 0) {
1372			nr_pages -= ret;
1373			pages_done += ret;
1374			if (!nr_pages)
1375				break;
1376		}
1377		if (*locked) {
1378			/*
1379			 * VM_FAULT_RETRY didn't trigger or it was a
1380			 * FOLL_NOWAIT.
1381			 */
1382			if (!pages_done)
1383				pages_done = ret;
1384			break;
1385		}
1386		/*
1387		 * VM_FAULT_RETRY triggered, so seek to the faulting offset.
1388		 * For the prefault case (!pages) we only update counts.
1389		 */
1390		if (likely(pages))
1391			pages += ret;
1392		start += ret << PAGE_SHIFT;
1393		lock_dropped = true;
1394
1395retry:
1396		/*
1397		 * Repeat on the address that fired VM_FAULT_RETRY
1398		 * with both FAULT_FLAG_ALLOW_RETRY and
1399		 * FAULT_FLAG_TRIED.  Note that GUP can be interrupted
1400		 * by fatal signals, so we need to check it before we
1401		 * start trying again otherwise it can loop forever.
1402		 */
1403
1404		if (fatal_signal_pending(current)) {
1405			if (!pages_done)
1406				pages_done = -EINTR;
1407			break;
1408		}
1409
1410		ret = mmap_read_lock_killable(mm);
1411		if (ret) {
1412			BUG_ON(ret > 0);
1413			if (!pages_done)
1414				pages_done = ret;
1415			break;
1416		}
1417
1418		*locked = 1;
1419		ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
1420				       pages, NULL, locked);
1421		if (!*locked) {
1422			/* Continue to retry until we succeeded */
1423			BUG_ON(ret != 0);
1424			goto retry;
1425		}
1426		if (ret != 1) {
1427			BUG_ON(ret > 1);
1428			if (!pages_done)
1429				pages_done = ret;
1430			break;
1431		}
1432		nr_pages--;
1433		pages_done++;
1434		if (!nr_pages)
1435			break;
1436		if (likely(pages))
1437			pages++;
1438		start += PAGE_SIZE;
1439	}
1440	if (lock_dropped && *locked) {
1441		/*
1442		 * We must let the caller know we temporarily dropped the lock
1443		 * and so the critical section protected by it was lost.
1444		 */
1445		mmap_read_unlock(mm);
1446		*locked = 0;
1447	}
1448	return pages_done;
1449}
1450
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1451/**
1452 * populate_vma_page_range() -  populate a range of pages in the vma.
1453 * @vma:   target vma
1454 * @start: start address
1455 * @end:   end address
1456 * @locked: whether the mmap_lock is still held
1457 *
1458 * This takes care of mlocking the pages too if VM_LOCKED is set.
1459 *
1460 * Return either number of pages pinned in the vma, or a negative error
1461 * code on error.
1462 *
1463 * vma->vm_mm->mmap_lock must be held.
1464 *
1465 * If @locked is NULL, it may be held for read or write and will
1466 * be unperturbed.
1467 *
1468 * If @locked is non-NULL, it must held for read only and may be
1469 * released.  If it's released, *@locked will be set to 0.
1470 */
1471long populate_vma_page_range(struct vm_area_struct *vma,
1472		unsigned long start, unsigned long end, int *locked)
1473{
1474	struct mm_struct *mm = vma->vm_mm;
1475	unsigned long nr_pages = (end - start) / PAGE_SIZE;
1476	int gup_flags;
1477
1478	VM_BUG_ON(start & ~PAGE_MASK);
1479	VM_BUG_ON(end   & ~PAGE_MASK);
1480	VM_BUG_ON_VMA(start < vma->vm_start, vma);
1481	VM_BUG_ON_VMA(end   > vma->vm_end, vma);
1482	mmap_assert_locked(mm);
1483
1484	gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1485	if (vma->vm_flags & VM_LOCKONFAULT)
1486		gup_flags &= ~FOLL_POPULATE;
1487	/*
1488	 * We want to touch writable mappings with a write fault in order
1489	 * to break COW, except for shared mappings because these don't COW
1490	 * and we would not want to dirty them for nothing.
1491	 */
1492	if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
1493		gup_flags |= FOLL_WRITE;
1494
1495	/*
1496	 * We want mlock to succeed for regions that have any permissions
1497	 * other than PROT_NONE.
1498	 */
1499	if (vma_is_accessible(vma))
1500		gup_flags |= FOLL_FORCE;
1501
1502	/*
1503	 * We made sure addr is within a VMA, so the following will
1504	 * not result in a stack expansion that recurses back here.
1505	 */
1506	return __get_user_pages(mm, start, nr_pages, gup_flags,
1507				NULL, NULL, locked);
1508}
1509
1510/*
1511 * faultin_vma_page_range() - populate (prefault) page tables inside the
1512 *			      given VMA range readable/writable
1513 *
1514 * This takes care of mlocking the pages, too, if VM_LOCKED is set.
1515 *
1516 * @vma: target vma
1517 * @start: start address
1518 * @end: end address
1519 * @write: whether to prefault readable or writable
1520 * @locked: whether the mmap_lock is still held
1521 *
1522 * Returns either number of processed pages in the vma, or a negative error
1523 * code on error (see __get_user_pages()).
1524 *
1525 * vma->vm_mm->mmap_lock must be held. The range must be page-aligned and
1526 * covered by the VMA.
1527 *
1528 * If @locked is NULL, it may be held for read or write and will be unperturbed.
1529 *
1530 * If @locked is non-NULL, it must held for read only and may be released.  If
1531 * it's released, *@locked will be set to 0.
1532 */
1533long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start,
1534			    unsigned long end, bool write, int *locked)
1535{
1536	struct mm_struct *mm = vma->vm_mm;
1537	unsigned long nr_pages = (end - start) / PAGE_SIZE;
1538	int gup_flags;
1539
1540	VM_BUG_ON(!PAGE_ALIGNED(start));
1541	VM_BUG_ON(!PAGE_ALIGNED(end));
1542	VM_BUG_ON_VMA(start < vma->vm_start, vma);
1543	VM_BUG_ON_VMA(end > vma->vm_end, vma);
1544	mmap_assert_locked(mm);
1545
1546	/*
1547	 * FOLL_TOUCH: Mark page accessed and thereby young; will also mark
1548	 *	       the page dirty with FOLL_WRITE -- which doesn't make a
1549	 *	       difference with !FOLL_FORCE, because the page is writable
1550	 *	       in the page table.
1551	 * FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit
1552	 *		  a poisoned page.
1553	 * FOLL_POPULATE: Always populate memory with VM_LOCKONFAULT.
1554	 * !FOLL_FORCE: Require proper access permissions.
1555	 */
1556	gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK | FOLL_HWPOISON;
1557	if (write)
1558		gup_flags |= FOLL_WRITE;
1559
1560	/*
1561	 * We want to report -EINVAL instead of -EFAULT for any permission
1562	 * problems or incompatible mappings.
1563	 */
1564	if (check_vma_flags(vma, gup_flags))
1565		return -EINVAL;
1566
1567	return __get_user_pages(mm, start, nr_pages, gup_flags,
1568				NULL, NULL, locked);
1569}
1570
1571/*
1572 * __mm_populate - populate and/or mlock pages within a range of address space.
1573 *
1574 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
1575 * flags. VMAs must be already marked with the desired vm_flags, and
1576 * mmap_lock must not be held.
1577 */
1578int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
1579{
1580	struct mm_struct *mm = current->mm;
1581	unsigned long end, nstart, nend;
1582	struct vm_area_struct *vma = NULL;
1583	int locked = 0;
1584	long ret = 0;
1585
 
 
1586	end = start + len;
1587
1588	for (nstart = start; nstart < end; nstart = nend) {
1589		/*
1590		 * We want to fault in pages for [nstart; end) address range.
1591		 * Find first corresponding VMA.
1592		 */
1593		if (!locked) {
1594			locked = 1;
1595			mmap_read_lock(mm);
1596			vma = find_vma(mm, nstart);
1597		} else if (nstart >= vma->vm_end)
1598			vma = vma->vm_next;
1599		if (!vma || vma->vm_start >= end)
1600			break;
1601		/*
1602		 * Set [nstart; nend) to intersection of desired address
1603		 * range with the first VMA. Also, skip undesirable VMA types.
1604		 */
1605		nend = min(end, vma->vm_end);
1606		if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1607			continue;
1608		if (nstart < vma->vm_start)
1609			nstart = vma->vm_start;
1610		/*
1611		 * Now fault in a range of pages. populate_vma_page_range()
1612		 * double checks the vma flags, so that it won't mlock pages
1613		 * if the vma was already munlocked.
1614		 */
1615		ret = populate_vma_page_range(vma, nstart, nend, &locked);
1616		if (ret < 0) {
1617			if (ignore_errors) {
1618				ret = 0;
1619				continue;	/* continue at next VMA */
1620			}
1621			break;
1622		}
1623		nend = nstart + ret * PAGE_SIZE;
1624		ret = 0;
1625	}
1626	if (locked)
1627		mmap_read_unlock(mm);
1628	return ret;	/* 0 or negative error code */
1629}
1630#else /* CONFIG_MMU */
1631static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
1632		unsigned long nr_pages, struct page **pages,
1633		struct vm_area_struct **vmas, int *locked,
1634		unsigned int foll_flags)
1635{
1636	struct vm_area_struct *vma;
1637	unsigned long vm_flags;
1638	long i;
1639
1640	/* calculate required read or write permissions.
1641	 * If FOLL_FORCE is set, we only require the "MAY" flags.
1642	 */
1643	vm_flags  = (foll_flags & FOLL_WRITE) ?
1644			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1645	vm_flags &= (foll_flags & FOLL_FORCE) ?
1646			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1647
1648	for (i = 0; i < nr_pages; i++) {
1649		vma = find_vma(mm, start);
1650		if (!vma)
1651			goto finish_or_fault;
1652
1653		/* protect what we can, including chardevs */
1654		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1655		    !(vm_flags & vma->vm_flags))
1656			goto finish_or_fault;
1657
1658		if (pages) {
1659			pages[i] = virt_to_page(start);
1660			if (pages[i])
1661				get_page(pages[i]);
1662		}
1663		if (vmas)
1664			vmas[i] = vma;
1665		start = (start + PAGE_SIZE) & PAGE_MASK;
1666	}
1667
1668	return i;
1669
1670finish_or_fault:
1671	return i ? : -EFAULT;
1672}
1673#endif /* !CONFIG_MMU */
1674
1675/**
1676 * get_dump_page() - pin user page in memory while writing it to core dump
1677 * @addr: user address
1678 *
1679 * Returns struct page pointer of user page pinned for dump,
1680 * to be freed afterwards by put_page().
1681 *
1682 * Returns NULL on any kind of failure - a hole must then be inserted into
1683 * the corefile, to preserve alignment with its headers; and also returns
1684 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
1685 * allowing a hole to be left in the corefile to save disk space.
1686 *
1687 * Called without mmap_lock (takes and releases the mmap_lock by itself).
1688 */
1689#ifdef CONFIG_ELF_CORE
1690struct page *get_dump_page(unsigned long addr)
1691{
1692	struct mm_struct *mm = current->mm;
1693	struct page *page;
1694	int locked = 1;
1695	int ret;
1696
1697	if (mmap_read_lock_killable(mm))
 
 
1698		return NULL;
1699	ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked,
1700				      FOLL_FORCE | FOLL_DUMP | FOLL_GET);
1701	if (locked)
1702		mmap_read_unlock(mm);
1703	return (ret == 1) ? page : NULL;
1704}
1705#endif /* CONFIG_ELF_CORE */
1706
1707#ifdef CONFIG_MIGRATION
1708/*
1709 * Check whether all pages are pinnable, if so return number of pages.  If some
1710 * pages are not pinnable, migrate them, and unpin all pages. Return zero if
1711 * pages were migrated, or if some pages were not successfully isolated.
1712 * Return negative error if migration fails.
1713 */
1714static long check_and_migrate_movable_pages(unsigned long nr_pages,
1715					    struct page **pages,
1716					    unsigned int gup_flags)
1717{
1718	unsigned long i;
1719	unsigned long isolation_error_count = 0;
1720	bool drain_allow = true;
1721	LIST_HEAD(movable_page_list);
1722	long ret = 0;
1723	struct page *prev_head = NULL;
1724	struct page *head;
1725	struct migration_target_control mtc = {
1726		.nid = NUMA_NO_NODE,
1727		.gfp_mask = GFP_USER | __GFP_NOWARN,
1728	};
1729
1730	for (i = 0; i < nr_pages; i++) {
1731		head = compound_head(pages[i]);
1732		if (head == prev_head)
1733			continue;
1734		prev_head = head;
1735		/*
1736		 * If we get a movable page, since we are going to be pinning
1737		 * these entries, try to move them out if possible.
1738		 */
1739		if (!is_pinnable_page(head)) {
1740			if (PageHuge(head)) {
1741				if (!isolate_huge_page(head, &movable_page_list))
1742					isolation_error_count++;
1743			} else {
1744				if (!PageLRU(head) && drain_allow) {
1745					lru_add_drain_all();
1746					drain_allow = false;
1747				}
1748
1749				if (isolate_lru_page(head)) {
1750					isolation_error_count++;
1751					continue;
1752				}
1753				list_add_tail(&head->lru, &movable_page_list);
1754				mod_node_page_state(page_pgdat(head),
1755						    NR_ISOLATED_ANON +
1756						    page_is_file_lru(head),
1757						    thp_nr_pages(head));
1758			}
1759		}
1760	}
1761
1762	/*
1763	 * If list is empty, and no isolation errors, means that all pages are
1764	 * in the correct zone.
1765	 */
1766	if (list_empty(&movable_page_list) && !isolation_error_count)
1767		return nr_pages;
1768
1769	if (gup_flags & FOLL_PIN) {
1770		unpin_user_pages(pages, nr_pages);
1771	} else {
1772		for (i = 0; i < nr_pages; i++)
1773			put_page(pages[i]);
1774	}
1775	if (!list_empty(&movable_page_list)) {
1776		ret = migrate_pages(&movable_page_list, alloc_migration_target,
1777				    NULL, (unsigned long)&mtc, MIGRATE_SYNC,
1778				    MR_LONGTERM_PIN);
1779		if (ret && !list_empty(&movable_page_list))
1780			putback_movable_pages(&movable_page_list);
1781	}
1782
1783	return ret > 0 ? -ENOMEM : ret;
1784}
1785#else
1786static long check_and_migrate_movable_pages(unsigned long nr_pages,
1787					    struct page **pages,
1788					    unsigned int gup_flags)
1789{
1790	return nr_pages;
1791}
1792#endif /* CONFIG_MIGRATION */
1793
1794/*
1795 * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
1796 * allows us to process the FOLL_LONGTERM flag.
1797 */
1798static long __gup_longterm_locked(struct mm_struct *mm,
1799				  unsigned long start,
1800				  unsigned long nr_pages,
1801				  struct page **pages,
1802				  struct vm_area_struct **vmas,
1803				  unsigned int gup_flags)
1804{
1805	unsigned int flags;
1806	long rc;
1807
1808	if (!(gup_flags & FOLL_LONGTERM))
1809		return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1810					       NULL, gup_flags);
1811	flags = memalloc_pin_save();
1812	do {
1813		rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1814					     NULL, gup_flags);
1815		if (rc <= 0)
1816			break;
1817		rc = check_and_migrate_movable_pages(rc, pages, gup_flags);
1818	} while (!rc);
1819	memalloc_pin_restore(flags);
1820
1821	return rc;
1822}
1823
1824static bool is_valid_gup_flags(unsigned int gup_flags)
1825{
1826	/*
1827	 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1828	 * never directly by the caller, so enforce that with an assertion:
1829	 */
1830	if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1831		return false;
1832	/*
1833	 * FOLL_PIN is a prerequisite to FOLL_LONGTERM. Another way of saying
1834	 * that is, FOLL_LONGTERM is a specific case, more restrictive case of
1835	 * FOLL_PIN.
1836	 */
1837	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1838		return false;
1839
1840	return true;
1841}
1842
1843#ifdef CONFIG_MMU
1844static long __get_user_pages_remote(struct mm_struct *mm,
1845				    unsigned long start, unsigned long nr_pages,
1846				    unsigned int gup_flags, struct page **pages,
1847				    struct vm_area_struct **vmas, int *locked)
1848{
1849	/*
1850	 * Parts of FOLL_LONGTERM behavior are incompatible with
1851	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1852	 * vmas. However, this only comes up if locked is set, and there are
1853	 * callers that do request FOLL_LONGTERM, but do not set locked. So,
1854	 * allow what we can.
1855	 */
1856	if (gup_flags & FOLL_LONGTERM) {
1857		if (WARN_ON_ONCE(locked))
1858			return -EINVAL;
1859		/*
1860		 * This will check the vmas (even if our vmas arg is NULL)
1861		 * and return -ENOTSUPP if DAX isn't allowed in this case:
1862		 */
1863		return __gup_longterm_locked(mm, start, nr_pages, pages,
1864					     vmas, gup_flags | FOLL_TOUCH |
1865					     FOLL_REMOTE);
1866	}
1867
1868	return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1869				       locked,
1870				       gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1871}
1872
1873/**
1874 * get_user_pages_remote() - pin user pages in memory
1875 * @mm:		mm_struct of target mm
1876 * @start:	starting user address
1877 * @nr_pages:	number of pages from start to pin
1878 * @gup_flags:	flags modifying lookup behaviour
1879 * @pages:	array that receives pointers to the pages pinned.
1880 *		Should be at least nr_pages long. Or NULL, if caller
1881 *		only intends to ensure the pages are faulted in.
1882 * @vmas:	array of pointers to vmas corresponding to each page.
1883 *		Or NULL if the caller does not require them.
1884 * @locked:	pointer to lock flag indicating whether lock is held and
1885 *		subsequently whether VM_FAULT_RETRY functionality can be
1886 *		utilised. Lock must initially be held.
1887 *
1888 * Returns either number of pages pinned (which may be less than the
1889 * number requested), or an error. Details about the return value:
1890 *
1891 * -- If nr_pages is 0, returns 0.
1892 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1893 * -- If nr_pages is >0, and some pages were pinned, returns the number of
1894 *    pages pinned. Again, this may be less than nr_pages.
1895 *
1896 * The caller is responsible for releasing returned @pages, via put_page().
1897 *
1898 * @vmas are valid only as long as mmap_lock is held.
1899 *
1900 * Must be called with mmap_lock held for read or write.
1901 *
1902 * get_user_pages_remote walks a process's page tables and takes a reference
1903 * to each struct page that each user address corresponds to at a given
1904 * instant. That is, it takes the page that would be accessed if a user
1905 * thread accesses the given user virtual address at that instant.
1906 *
1907 * This does not guarantee that the page exists in the user mappings when
1908 * get_user_pages_remote returns, and there may even be a completely different
1909 * page there in some cases (eg. if mmapped pagecache has been invalidated
1910 * and subsequently re faulted). However it does guarantee that the page
1911 * won't be freed completely. And mostly callers simply care that the page
1912 * contains data that was valid *at some point in time*. Typically, an IO
1913 * or similar operation cannot guarantee anything stronger anyway because
1914 * locks can't be held over the syscall boundary.
1915 *
1916 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1917 * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
1918 * be called after the page is finished with, and before put_page is called.
1919 *
1920 * get_user_pages_remote is typically used for fewer-copy IO operations,
1921 * to get a handle on the memory by some means other than accesses
1922 * via the user virtual addresses. The pages may be submitted for
1923 * DMA to devices or accessed via their kernel linear mapping (via the
1924 * kmap APIs). Care should be taken to use the correct cache flushing APIs.
1925 *
1926 * See also get_user_pages_fast, for performance critical applications.
1927 *
1928 * get_user_pages_remote should be phased out in favor of
1929 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1930 * should use get_user_pages_remote because it cannot pass
1931 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
1932 */
1933long get_user_pages_remote(struct mm_struct *mm,
1934		unsigned long start, unsigned long nr_pages,
1935		unsigned int gup_flags, struct page **pages,
1936		struct vm_area_struct **vmas, int *locked)
1937{
1938	if (!is_valid_gup_flags(gup_flags))
1939		return -EINVAL;
1940
1941	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
1942				       pages, vmas, locked);
1943}
1944EXPORT_SYMBOL(get_user_pages_remote);
1945
1946#else /* CONFIG_MMU */
1947long get_user_pages_remote(struct mm_struct *mm,
1948			   unsigned long start, unsigned long nr_pages,
1949			   unsigned int gup_flags, struct page **pages,
1950			   struct vm_area_struct **vmas, int *locked)
1951{
1952	return 0;
1953}
1954
1955static long __get_user_pages_remote(struct mm_struct *mm,
1956				    unsigned long start, unsigned long nr_pages,
1957				    unsigned int gup_flags, struct page **pages,
1958				    struct vm_area_struct **vmas, int *locked)
1959{
1960	return 0;
1961}
1962#endif /* !CONFIG_MMU */
1963
1964/**
1965 * get_user_pages() - pin user pages in memory
1966 * @start:      starting user address
1967 * @nr_pages:   number of pages from start to pin
1968 * @gup_flags:  flags modifying lookup behaviour
1969 * @pages:      array that receives pointers to the pages pinned.
1970 *              Should be at least nr_pages long. Or NULL, if caller
1971 *              only intends to ensure the pages are faulted in.
1972 * @vmas:       array of pointers to vmas corresponding to each page.
1973 *              Or NULL if the caller does not require them.
1974 *
1975 * This is the same as get_user_pages_remote(), just with a less-flexible
1976 * calling convention where we assume that the mm being operated on belongs to
1977 * the current task, and doesn't allow passing of a locked parameter.  We also
1978 * obviously don't pass FOLL_REMOTE in here.
1979 */
1980long get_user_pages(unsigned long start, unsigned long nr_pages,
1981		unsigned int gup_flags, struct page **pages,
1982		struct vm_area_struct **vmas)
1983{
1984	if (!is_valid_gup_flags(gup_flags))
1985		return -EINVAL;
1986
1987	return __gup_longterm_locked(current->mm, start, nr_pages,
1988				     pages, vmas, gup_flags | FOLL_TOUCH);
1989}
1990EXPORT_SYMBOL(get_user_pages);
1991
1992/**
1993 * get_user_pages_locked() - variant of get_user_pages()
1994 *
1995 * @start:      starting user address
1996 * @nr_pages:   number of pages from start to pin
1997 * @gup_flags:  flags modifying lookup behaviour
1998 * @pages:      array that receives pointers to the pages pinned.
1999 *              Should be at least nr_pages long. Or NULL, if caller
2000 *              only intends to ensure the pages are faulted in.
2001 * @locked:     pointer to lock flag indicating whether lock is held and
2002 *              subsequently whether VM_FAULT_RETRY functionality can be
2003 *              utilised. Lock must initially be held.
2004 *
2005 * It is suitable to replace the form:
2006 *
2007 *      mmap_read_lock(mm);
2008 *      do_something()
2009 *      get_user_pages(mm, ..., pages, NULL);
2010 *      mmap_read_unlock(mm);
2011 *
2012 *  to:
2013 *
2014 *      int locked = 1;
2015 *      mmap_read_lock(mm);
2016 *      do_something()
2017 *      get_user_pages_locked(mm, ..., pages, &locked);
2018 *      if (locked)
2019 *          mmap_read_unlock(mm);
2020 *
2021 * We can leverage the VM_FAULT_RETRY functionality in the page fault
2022 * paths better by using either get_user_pages_locked() or
2023 * get_user_pages_unlocked().
2024 *
2025 */
2026long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
2027			   unsigned int gup_flags, struct page **pages,
2028			   int *locked)
2029{
2030	/*
2031	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
2032	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
2033	 * vmas.  As there are no users of this flag in this call we simply
2034	 * disallow this option for now.
2035	 */
2036	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2037		return -EINVAL;
2038	/*
2039	 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
2040	 * never directly by the caller, so enforce that:
2041	 */
2042	if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
2043		return -EINVAL;
2044
2045	return __get_user_pages_locked(current->mm, start, nr_pages,
2046				       pages, NULL, locked,
2047				       gup_flags | FOLL_TOUCH);
2048}
2049EXPORT_SYMBOL(get_user_pages_locked);
2050
2051/*
2052 * get_user_pages_unlocked() is suitable to replace the form:
2053 *
2054 *      mmap_read_lock(mm);
2055 *      get_user_pages(mm, ..., pages, NULL);
2056 *      mmap_read_unlock(mm);
2057 *
2058 *  with:
2059 *
2060 *      get_user_pages_unlocked(mm, ..., pages);
2061 *
2062 * It is functionally equivalent to get_user_pages_fast so
2063 * get_user_pages_fast should be used instead if specific gup_flags
2064 * (e.g. FOLL_FORCE) are not required.
2065 */
2066long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2067			     struct page **pages, unsigned int gup_flags)
2068{
2069	struct mm_struct *mm = current->mm;
2070	int locked = 1;
2071	long ret;
2072
2073	/*
2074	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
2075	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
2076	 * vmas.  As there are no users of this flag in this call we simply
2077	 * disallow this option for now.
2078	 */
2079	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2080		return -EINVAL;
2081
2082	mmap_read_lock(mm);
2083	ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
2084				      &locked, gup_flags | FOLL_TOUCH);
2085	if (locked)
2086		mmap_read_unlock(mm);
2087	return ret;
2088}
2089EXPORT_SYMBOL(get_user_pages_unlocked);
2090
2091/*
2092 * Fast GUP
2093 *
2094 * get_user_pages_fast attempts to pin user pages by walking the page
2095 * tables directly and avoids taking locks. Thus the walker needs to be
2096 * protected from page table pages being freed from under it, and should
2097 * block any THP splits.
2098 *
2099 * One way to achieve this is to have the walker disable interrupts, and
2100 * rely on IPIs from the TLB flushing code blocking before the page table
2101 * pages are freed. This is unsuitable for architectures that do not need
2102 * to broadcast an IPI when invalidating TLBs.
2103 *
2104 * Another way to achieve this is to batch up page table containing pages
2105 * belonging to more than one mm_user, then rcu_sched a callback to free those
2106 * pages. Disabling interrupts will allow the fast_gup walker to both block
2107 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
2108 * (which is a relatively rare event). The code below adopts this strategy.
2109 *
2110 * Before activating this code, please be aware that the following assumptions
2111 * are currently made:
2112 *
2113 *  *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
2114 *  free pages containing page tables or TLB flushing requires IPI broadcast.
2115 *
2116 *  *) ptes can be read atomically by the architecture.
2117 *
2118 *  *) access_ok is sufficient to validate userspace address ranges.
2119 *
2120 * The last two assumptions can be relaxed by the addition of helper functions.
2121 *
2122 * This code is based heavily on the PowerPC implementation by Nick Piggin.
2123 */
2124#ifdef CONFIG_HAVE_FAST_GUP
2125
2126static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
2127					    unsigned int flags,
2128					    struct page **pages)
 
 
 
 
 
 
 
 
 
2129{
2130	while ((*nr) - nr_start) {
2131		struct page *page = pages[--(*nr)];
2132
2133		ClearPageReferenced(page);
2134		if (flags & FOLL_PIN)
2135			unpin_user_page(page);
2136		else
2137			put_page(page);
2138	}
2139}
2140
2141#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
2142static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2143			 unsigned int flags, struct page **pages, int *nr)
2144{
2145	struct dev_pagemap *pgmap = NULL;
2146	int nr_start = *nr, ret = 0;
2147	pte_t *ptep, *ptem;
2148
2149	ptem = ptep = pte_offset_map(&pmd, addr);
2150	do {
2151		pte_t pte = ptep_get_lockless(ptep);
2152		struct page *head, *page;
2153
2154		/*
2155		 * Similar to the PMD case below, NUMA hinting must take slow
2156		 * path using the pte_protnone check.
2157		 */
2158		if (pte_protnone(pte))
2159			goto pte_unmap;
2160
2161		if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2162			goto pte_unmap;
2163
2164		if (pte_devmap(pte)) {
2165			if (unlikely(flags & FOLL_LONGTERM))
2166				goto pte_unmap;
2167
2168			pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
2169			if (unlikely(!pgmap)) {
2170				undo_dev_pagemap(nr, nr_start, flags, pages);
2171				goto pte_unmap;
2172			}
2173		} else if (pte_special(pte))
2174			goto pte_unmap;
2175
2176		VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2177		page = pte_page(pte);
 
2178
2179		head = try_grab_compound_head(page, 1, flags);
2180		if (!head)
2181			goto pte_unmap;
2182
2183		if (unlikely(page_is_secretmem(page))) {
2184			put_compound_head(head, 1, flags);
2185			goto pte_unmap;
2186		}
2187
2188		if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2189			put_compound_head(head, 1, flags);
2190			goto pte_unmap;
2191		}
2192
2193		VM_BUG_ON_PAGE(compound_head(page) != head, page);
2194
2195		/*
2196		 * We need to make the page accessible if and only if we are
2197		 * going to access its content (the FOLL_PIN case).  Please
2198		 * see Documentation/core-api/pin_user_pages.rst for
2199		 * details.
2200		 */
2201		if (flags & FOLL_PIN) {
2202			ret = arch_make_page_accessible(page);
2203			if (ret) {
2204				unpin_user_page(page);
2205				goto pte_unmap;
2206			}
2207		}
2208		SetPageReferenced(page);
2209		pages[*nr] = page;
2210		(*nr)++;
2211
2212	} while (ptep++, addr += PAGE_SIZE, addr != end);
2213
2214	ret = 1;
2215
2216pte_unmap:
2217	if (pgmap)
2218		put_dev_pagemap(pgmap);
2219	pte_unmap(ptem);
2220	return ret;
2221}
2222#else
2223
2224/*
2225 * If we can't determine whether or not a pte is special, then fail immediately
2226 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
2227 * to be special.
2228 *
2229 * For a futex to be placed on a THP tail page, get_futex_key requires a
2230 * get_user_pages_fast_only implementation that can pin pages. Thus it's still
2231 * useful to have gup_huge_pmd even if we can't operate on ptes.
2232 */
2233static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2234			 unsigned int flags, struct page **pages, int *nr)
2235{
2236	return 0;
2237}
2238#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
2239
2240#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
2241static int __gup_device_huge(unsigned long pfn, unsigned long addr,
2242			     unsigned long end, unsigned int flags,
2243			     struct page **pages, int *nr)
2244{
2245	int nr_start = *nr;
2246	struct dev_pagemap *pgmap = NULL;
2247
2248	do {
2249		struct page *page = pfn_to_page(pfn);
2250
2251		pgmap = get_dev_pagemap(pfn, pgmap);
2252		if (unlikely(!pgmap)) {
2253			undo_dev_pagemap(nr, nr_start, flags, pages);
2254			return 0;
2255		}
2256		SetPageReferenced(page);
2257		pages[*nr] = page;
2258		if (unlikely(!try_grab_page(page, flags))) {
2259			undo_dev_pagemap(nr, nr_start, flags, pages);
2260			return 0;
2261		}
2262		(*nr)++;
2263		pfn++;
2264	} while (addr += PAGE_SIZE, addr != end);
2265
2266	if (pgmap)
2267		put_dev_pagemap(pgmap);
2268	return 1;
2269}
2270
2271static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2272				 unsigned long end, unsigned int flags,
2273				 struct page **pages, int *nr)
2274{
2275	unsigned long fault_pfn;
2276	int nr_start = *nr;
2277
2278	fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2279	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2280		return 0;
2281
2282	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2283		undo_dev_pagemap(nr, nr_start, flags, pages);
2284		return 0;
2285	}
2286	return 1;
2287}
2288
2289static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2290				 unsigned long end, unsigned int flags,
2291				 struct page **pages, int *nr)
2292{
2293	unsigned long fault_pfn;
2294	int nr_start = *nr;
2295
2296	fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2297	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2298		return 0;
2299
2300	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2301		undo_dev_pagemap(nr, nr_start, flags, pages);
2302		return 0;
2303	}
2304	return 1;
2305}
2306#else
2307static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2308				 unsigned long end, unsigned int flags,
2309				 struct page **pages, int *nr)
2310{
2311	BUILD_BUG();
2312	return 0;
2313}
2314
2315static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
2316				 unsigned long end, unsigned int flags,
2317				 struct page **pages, int *nr)
2318{
2319	BUILD_BUG();
2320	return 0;
2321}
2322#endif
2323
2324static int record_subpages(struct page *page, unsigned long addr,
2325			   unsigned long end, struct page **pages)
2326{
2327	int nr;
2328
2329	for (nr = 0; addr != end; addr += PAGE_SIZE)
2330		pages[nr++] = page++;
2331
2332	return nr;
2333}
2334
2335#ifdef CONFIG_ARCH_HAS_HUGEPD
2336static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2337				      unsigned long sz)
2338{
2339	unsigned long __boundary = (addr + sz) & ~(sz-1);
2340	return (__boundary - 1 < end - 1) ? __boundary : end;
2341}
2342
2343static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
2344		       unsigned long end, unsigned int flags,
2345		       struct page **pages, int *nr)
2346{
2347	unsigned long pte_end;
2348	struct page *head, *page;
2349	pte_t pte;
2350	int refs;
2351
2352	pte_end = (addr + sz) & ~(sz-1);
2353	if (pte_end < end)
2354		end = pte_end;
2355
2356	pte = huge_ptep_get(ptep);
2357
2358	if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2359		return 0;
2360
2361	/* hugepages are never "special" */
2362	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2363
2364	head = pte_page(pte);
2365	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2366	refs = record_subpages(page, addr, end, pages + *nr);
2367
2368	head = try_grab_compound_head(head, refs, flags);
2369	if (!head)
2370		return 0;
2371
2372	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2373		put_compound_head(head, refs, flags);
2374		return 0;
2375	}
2376
2377	*nr += refs;
2378	SetPageReferenced(head);
2379	return 1;
2380}
2381
2382static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2383		unsigned int pdshift, unsigned long end, unsigned int flags,
2384		struct page **pages, int *nr)
2385{
2386	pte_t *ptep;
2387	unsigned long sz = 1UL << hugepd_shift(hugepd);
2388	unsigned long next;
2389
2390	ptep = hugepte_offset(hugepd, addr, pdshift);
2391	do {
2392		next = hugepte_addr_end(addr, end, sz);
2393		if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2394			return 0;
2395	} while (ptep++, addr = next, addr != end);
2396
2397	return 1;
2398}
2399#else
2400static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2401		unsigned int pdshift, unsigned long end, unsigned int flags,
2402		struct page **pages, int *nr)
2403{
2404	return 0;
2405}
2406#endif /* CONFIG_ARCH_HAS_HUGEPD */
2407
2408static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2409			unsigned long end, unsigned int flags,
2410			struct page **pages, int *nr)
2411{
2412	struct page *head, *page;
2413	int refs;
2414
2415	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
2416		return 0;
2417
2418	if (pmd_devmap(orig)) {
2419		if (unlikely(flags & FOLL_LONGTERM))
2420			return 0;
2421		return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2422					     pages, nr);
2423	}
2424
 
2425	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2426	refs = record_subpages(page, addr, end, pages + *nr);
 
 
 
 
 
2427
2428	head = try_grab_compound_head(pmd_page(orig), refs, flags);
2429	if (!head)
 
2430		return 0;
 
2431
2432	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2433		put_compound_head(head, refs, flags);
 
 
2434		return 0;
2435	}
2436
2437	*nr += refs;
2438	SetPageReferenced(head);
2439	return 1;
2440}
2441
2442static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2443			unsigned long end, unsigned int flags,
2444			struct page **pages, int *nr)
2445{
2446	struct page *head, *page;
2447	int refs;
2448
2449	if (!pud_access_permitted(orig, flags & FOLL_WRITE))
2450		return 0;
2451
2452	if (pud_devmap(orig)) {
2453		if (unlikely(flags & FOLL_LONGTERM))
2454			return 0;
2455		return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2456					     pages, nr);
2457	}
2458
 
2459	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2460	refs = record_subpages(page, addr, end, pages + *nr);
 
 
 
 
 
2461
2462	head = try_grab_compound_head(pud_page(orig), refs, flags);
2463	if (!head)
 
2464		return 0;
 
2465
2466	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2467		put_compound_head(head, refs, flags);
 
 
2468		return 0;
2469	}
2470
2471	*nr += refs;
2472	SetPageReferenced(head);
2473	return 1;
2474}
2475
2476static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
2477			unsigned long end, unsigned int flags,
2478			struct page **pages, int *nr)
2479{
2480	int refs;
2481	struct page *head, *page;
2482
2483	if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
2484		return 0;
2485
2486	BUILD_BUG_ON(pgd_devmap(orig));
2487
2488	page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2489	refs = record_subpages(page, addr, end, pages + *nr);
 
 
 
 
 
2490
2491	head = try_grab_compound_head(pgd_page(orig), refs, flags);
2492	if (!head)
 
2493		return 0;
 
2494
2495	if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2496		put_compound_head(head, refs, flags);
 
 
2497		return 0;
2498	}
2499
2500	*nr += refs;
2501	SetPageReferenced(head);
2502	return 1;
2503}
2504
2505static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
2506		unsigned int flags, struct page **pages, int *nr)
2507{
2508	unsigned long next;
2509	pmd_t *pmdp;
2510
2511	pmdp = pmd_offset_lockless(pudp, pud, addr);
2512	do {
2513		pmd_t pmd = READ_ONCE(*pmdp);
2514
2515		next = pmd_addr_end(addr, end);
2516		if (!pmd_present(pmd))
2517			return 0;
2518
2519		if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
2520			     pmd_devmap(pmd))) {
2521			/*
2522			 * NUMA hinting faults need to be handled in the GUP
2523			 * slowpath for accounting purposes and so that they
2524			 * can be serialised against THP migration.
2525			 */
2526			if (pmd_protnone(pmd))
2527				return 0;
2528
2529			if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
2530				pages, nr))
2531				return 0;
2532
2533		} else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
2534			/*
2535			 * architecture have different format for hugetlbfs
2536			 * pmd format and THP pmd format
2537			 */
2538			if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
2539					 PMD_SHIFT, next, flags, pages, nr))
2540				return 0;
2541		} else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
2542			return 0;
2543	} while (pmdp++, addr = next, addr != end);
2544
2545	return 1;
2546}
2547
2548static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
2549			 unsigned int flags, struct page **pages, int *nr)
2550{
2551	unsigned long next;
2552	pud_t *pudp;
2553
2554	pudp = pud_offset_lockless(p4dp, p4d, addr);
2555	do {
2556		pud_t pud = READ_ONCE(*pudp);
2557
2558		next = pud_addr_end(addr, end);
2559		if (unlikely(!pud_present(pud)))
2560			return 0;
2561		if (unlikely(pud_huge(pud))) {
2562			if (!gup_huge_pud(pud, pudp, addr, next, flags,
2563					  pages, nr))
2564				return 0;
2565		} else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
2566			if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
2567					 PUD_SHIFT, next, flags, pages, nr))
2568				return 0;
2569		} else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
2570			return 0;
2571	} while (pudp++, addr = next, addr != end);
2572
2573	return 1;
2574}
2575
2576static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
2577			 unsigned int flags, struct page **pages, int *nr)
2578{
2579	unsigned long next;
2580	p4d_t *p4dp;
2581
2582	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
2583	do {
2584		p4d_t p4d = READ_ONCE(*p4dp);
2585
2586		next = p4d_addr_end(addr, end);
2587		if (p4d_none(p4d))
2588			return 0;
2589		BUILD_BUG_ON(p4d_huge(p4d));
2590		if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
2591			if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
2592					 P4D_SHIFT, next, flags, pages, nr))
2593				return 0;
2594		} else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
2595			return 0;
2596	} while (p4dp++, addr = next, addr != end);
2597
2598	return 1;
2599}
2600
2601static void gup_pgd_range(unsigned long addr, unsigned long end,
2602		unsigned int flags, struct page **pages, int *nr)
2603{
2604	unsigned long next;
2605	pgd_t *pgdp;
2606
2607	pgdp = pgd_offset(current->mm, addr);
2608	do {
2609		pgd_t pgd = READ_ONCE(*pgdp);
2610
2611		next = pgd_addr_end(addr, end);
2612		if (pgd_none(pgd))
2613			return;
2614		if (unlikely(pgd_huge(pgd))) {
2615			if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
2616					  pages, nr))
2617				return;
2618		} else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
2619			if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
2620					 PGDIR_SHIFT, next, flags, pages, nr))
2621				return;
2622		} else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
2623			return;
2624	} while (pgdp++, addr = next, addr != end);
2625}
2626#else
2627static inline void gup_pgd_range(unsigned long addr, unsigned long end,
2628		unsigned int flags, struct page **pages, int *nr)
2629{
2630}
2631#endif /* CONFIG_HAVE_FAST_GUP */
2632
2633#ifndef gup_fast_permitted
2634/*
2635 * Check if it's allowed to use get_user_pages_fast_only() for the range, or
2636 * we need to fall back to the slow version:
2637 */
2638static bool gup_fast_permitted(unsigned long start, unsigned long end)
2639{
2640	return true;
2641}
2642#endif
2643
2644static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
2645				   unsigned int gup_flags, struct page **pages)
2646{
2647	int ret;
2648
2649	/*
2650	 * FIXME: FOLL_LONGTERM does not work with
2651	 * get_user_pages_unlocked() (see comments in that function)
2652	 */
2653	if (gup_flags & FOLL_LONGTERM) {
2654		mmap_read_lock(current->mm);
2655		ret = __gup_longterm_locked(current->mm,
2656					    start, nr_pages,
2657					    pages, NULL, gup_flags);
2658		mmap_read_unlock(current->mm);
2659	} else {
2660		ret = get_user_pages_unlocked(start, nr_pages,
2661					      pages, gup_flags);
2662	}
2663
2664	return ret;
2665}
2666
2667static unsigned long lockless_pages_from_mm(unsigned long start,
2668					    unsigned long end,
2669					    unsigned int gup_flags,
2670					    struct page **pages)
2671{
2672	unsigned long flags;
2673	int nr_pinned = 0;
2674	unsigned seq;
2675
2676	if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) ||
2677	    !gup_fast_permitted(start, end))
2678		return 0;
2679
2680	if (gup_flags & FOLL_PIN) {
2681		seq = raw_read_seqcount(&current->mm->write_protect_seq);
2682		if (seq & 1)
2683			return 0;
2684	}
2685
2686	/*
2687	 * Disable interrupts. The nested form is used, in order to allow full,
2688	 * general purpose use of this routine.
2689	 *
2690	 * With interrupts disabled, we block page table pages from being freed
2691	 * from under us. See struct mmu_table_batch comments in
2692	 * include/asm-generic/tlb.h for more details.
2693	 *
2694	 * We do not adopt an rcu_read_lock() here as we also want to block IPIs
2695	 * that come from THPs splitting.
2696	 */
2697	local_irq_save(flags);
2698	gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
2699	local_irq_restore(flags);
2700
2701	/*
2702	 * When pinning pages for DMA there could be a concurrent write protect
2703	 * from fork() via copy_page_range(), in this case always fail fast GUP.
2704	 */
2705	if (gup_flags & FOLL_PIN) {
2706		if (read_seqcount_retry(&current->mm->write_protect_seq, seq)) {
2707			unpin_user_pages(pages, nr_pinned);
2708			return 0;
2709		}
2710	}
2711	return nr_pinned;
2712}
2713
2714static int internal_get_user_pages_fast(unsigned long start,
2715					unsigned long nr_pages,
2716					unsigned int gup_flags,
2717					struct page **pages)
2718{
2719	unsigned long len, end;
2720	unsigned long nr_pinned;
2721	int ret;
2722
2723	if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2724				       FOLL_FORCE | FOLL_PIN | FOLL_GET |
2725				       FOLL_FAST_ONLY)))
2726		return -EINVAL;
2727
2728	if (gup_flags & FOLL_PIN)
2729		mm_set_has_pinned_flag(&current->mm->flags);
2730
2731	if (!(gup_flags & FOLL_FAST_ONLY))
2732		might_lock_read(&current->mm->mmap_lock);
2733
2734	start = untagged_addr(start) & PAGE_MASK;
2735	len = nr_pages << PAGE_SHIFT;
2736	if (check_add_overflow(start, len, &end))
2737		return 0;
2738	if (unlikely(!access_ok((void __user *)start, len)))
2739		return -EFAULT;
2740
2741	nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
2742	if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
2743		return nr_pinned;
2744
2745	/* Slow path: try to get the remaining pages with get_user_pages */
2746	start += nr_pinned << PAGE_SHIFT;
2747	pages += nr_pinned;
2748	ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
2749				      pages);
2750	if (ret < 0) {
2751		/*
2752		 * The caller has to unpin the pages we already pinned so
2753		 * returning -errno is not an option
2754		 */
2755		if (nr_pinned)
2756			return nr_pinned;
2757		return ret;
2758	}
2759	return ret + nr_pinned;
2760}
 
2761
2762/**
2763 * get_user_pages_fast_only() - pin user pages in memory
2764 * @start:      starting user address
2765 * @nr_pages:   number of pages from start to pin
2766 * @gup_flags:  flags modifying pin behaviour
2767 * @pages:      array that receives pointers to the pages pinned.
2768 *              Should be at least nr_pages long.
2769 *
2770 * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
2771 * the regular GUP.
2772 * Note a difference with get_user_pages_fast: this always returns the
2773 * number of pages pinned, 0 if no pages were pinned.
2774 *
2775 * If the architecture does not support this function, simply return with no
2776 * pages pinned.
2777 *
2778 * Careful, careful! COW breaking can go either way, so a non-write
2779 * access can get ambiguous page results. If you call this function without
2780 * 'write' set, you'd better be sure that you're ok with that ambiguity.
2781 */
2782int get_user_pages_fast_only(unsigned long start, int nr_pages,
2783			     unsigned int gup_flags, struct page **pages)
2784{
2785	int nr_pinned;
2786	/*
2787	 * Internally (within mm/gup.c), gup fast variants must set FOLL_GET,
2788	 * because gup fast is always a "pin with a +1 page refcount" request.
2789	 *
2790	 * FOLL_FAST_ONLY is required in order to match the API description of
2791	 * this routine: no fall back to regular ("slow") GUP.
2792	 */
2793	gup_flags |= FOLL_GET | FOLL_FAST_ONLY;
2794
2795	nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2796						 pages);
2797
2798	/*
2799	 * As specified in the API description above, this routine is not
2800	 * allowed to return negative values. However, the common core
2801	 * routine internal_get_user_pages_fast() *can* return -errno.
2802	 * Therefore, correct for that here:
2803	 */
2804	if (nr_pinned < 0)
2805		nr_pinned = 0;
2806
2807	return nr_pinned;
2808}
2809EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
 
2810
2811/**
2812 * get_user_pages_fast() - pin user pages in memory
2813 * @start:      starting user address
2814 * @nr_pages:   number of pages from start to pin
2815 * @gup_flags:  flags modifying pin behaviour
2816 * @pages:      array that receives pointers to the pages pinned.
2817 *              Should be at least nr_pages long.
2818 *
2819 * Attempt to pin user pages in memory without taking mm->mmap_lock.
2820 * If not successful, it will fall back to taking the lock and
2821 * calling get_user_pages().
2822 *
2823 * Returns number of pages pinned. This may be fewer than the number requested.
2824 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
2825 * -errno.
2826 */
2827int get_user_pages_fast(unsigned long start, int nr_pages,
2828			unsigned int gup_flags, struct page **pages)
2829{
2830	if (!is_valid_gup_flags(gup_flags))
2831		return -EINVAL;
2832
2833	/*
2834	 * The caller may or may not have explicitly set FOLL_GET; either way is
2835	 * OK. However, internally (within mm/gup.c), gup fast variants must set
2836	 * FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
2837	 * request.
 
 
 
 
 
2838	 */
2839	gup_flags |= FOLL_GET;
2840	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2841}
2842EXPORT_SYMBOL_GPL(get_user_pages_fast);
2843
2844/**
2845 * pin_user_pages_fast() - pin user pages in memory without taking locks
2846 *
2847 * @start:      starting user address
2848 * @nr_pages:   number of pages from start to pin
2849 * @gup_flags:  flags modifying pin behaviour
2850 * @pages:      array that receives pointers to the pages pinned.
2851 *              Should be at least nr_pages long.
2852 *
2853 * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
2854 * get_user_pages_fast() for documentation on the function arguments, because
2855 * the arguments here are identical.
2856 *
2857 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2858 * see Documentation/core-api/pin_user_pages.rst for further details.
2859 */
2860int pin_user_pages_fast(unsigned long start, int nr_pages,
2861			unsigned int gup_flags, struct page **pages)
2862{
2863	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2864	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2865		return -EINVAL;
2866
2867	gup_flags |= FOLL_PIN;
2868	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2869}
2870EXPORT_SYMBOL_GPL(pin_user_pages_fast);
2871
2872/*
2873 * This is the FOLL_PIN equivalent of get_user_pages_fast_only(). Behavior
2874 * is the same, except that this one sets FOLL_PIN instead of FOLL_GET.
2875 *
2876 * The API rules are the same, too: no negative values may be returned.
2877 */
2878int pin_user_pages_fast_only(unsigned long start, int nr_pages,
2879			     unsigned int gup_flags, struct page **pages)
2880{
2881	int nr_pinned;
2882
2883	/*
2884	 * FOLL_GET and FOLL_PIN are mutually exclusive. Note that the API
2885	 * rules require returning 0, rather than -errno:
2886	 */
2887	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2888		return 0;
2889	/*
2890	 * FOLL_FAST_ONLY is required in order to match the API description of
2891	 * this routine: no fall back to regular ("slow") GUP.
2892	 */
2893	gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);
2894	nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2895						 pages);
2896	/*
2897	 * This routine is not allowed to return negative values. However,
2898	 * internal_get_user_pages_fast() *can* return -errno. Therefore,
2899	 * correct for that here:
2900	 */
2901	if (nr_pinned < 0)
2902		nr_pinned = 0;
2903
2904	return nr_pinned;
2905}
2906EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
2907
2908/**
2909 * pin_user_pages_remote() - pin pages of a remote process
2910 *
2911 * @mm:		mm_struct of target mm
2912 * @start:	starting user address
2913 * @nr_pages:	number of pages from start to pin
2914 * @gup_flags:	flags modifying lookup behaviour
2915 * @pages:	array that receives pointers to the pages pinned.
2916 *		Should be at least nr_pages long. Or NULL, if caller
2917 *		only intends to ensure the pages are faulted in.
2918 * @vmas:	array of pointers to vmas corresponding to each page.
2919 *		Or NULL if the caller does not require them.
2920 * @locked:	pointer to lock flag indicating whether lock is held and
2921 *		subsequently whether VM_FAULT_RETRY functionality can be
2922 *		utilised. Lock must initially be held.
2923 *
2924 * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
2925 * get_user_pages_remote() for documentation on the function arguments, because
2926 * the arguments here are identical.
2927 *
2928 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2929 * see Documentation/core-api/pin_user_pages.rst for details.
2930 */
2931long pin_user_pages_remote(struct mm_struct *mm,
2932			   unsigned long start, unsigned long nr_pages,
2933			   unsigned int gup_flags, struct page **pages,
2934			   struct vm_area_struct **vmas, int *locked)
2935{
2936	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2937	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2938		return -EINVAL;
2939
2940	gup_flags |= FOLL_PIN;
2941	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
2942				       pages, vmas, locked);
2943}
2944EXPORT_SYMBOL(pin_user_pages_remote);
2945
2946/**
2947 * pin_user_pages() - pin user pages in memory for use by other devices
2948 *
2949 * @start:	starting user address
2950 * @nr_pages:	number of pages from start to pin
2951 * @gup_flags:	flags modifying lookup behaviour
2952 * @pages:	array that receives pointers to the pages pinned.
2953 *		Should be at least nr_pages long. Or NULL, if caller
2954 *		only intends to ensure the pages are faulted in.
2955 * @vmas:	array of pointers to vmas corresponding to each page.
2956 *		Or NULL if the caller does not require them.
2957 *
2958 * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
2959 * FOLL_PIN is set.
2960 *
2961 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2962 * see Documentation/core-api/pin_user_pages.rst for details.
2963 */
2964long pin_user_pages(unsigned long start, unsigned long nr_pages,
2965		    unsigned int gup_flags, struct page **pages,
2966		    struct vm_area_struct **vmas)
2967{
2968	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2969	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2970		return -EINVAL;
2971
2972	gup_flags |= FOLL_PIN;
2973	return __gup_longterm_locked(current->mm, start, nr_pages,
2974				     pages, vmas, gup_flags);
2975}
2976EXPORT_SYMBOL(pin_user_pages);
2977
2978/*
2979 * pin_user_pages_unlocked() is the FOLL_PIN variant of
2980 * get_user_pages_unlocked(). Behavior is the same, except that this one sets
2981 * FOLL_PIN and rejects FOLL_GET.
2982 */
2983long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2984			     struct page **pages, unsigned int gup_flags)
2985{
2986	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2987	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2988		return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
2989
2990	gup_flags |= FOLL_PIN;
2991	return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
2992}
2993EXPORT_SYMBOL(pin_user_pages_unlocked);
2994
2995/*
2996 * pin_user_pages_locked() is the FOLL_PIN variant of get_user_pages_locked().
2997 * Behavior is the same, except that this one sets FOLL_PIN and rejects
2998 * FOLL_GET.
2999 */
3000long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
3001			   unsigned int gup_flags, struct page **pages,
3002			   int *locked)
3003{
3004	/*
3005	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
3006	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
3007	 * vmas.  As there are no users of this flag in this call we simply
3008	 * disallow this option for now.
3009	 */
3010	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
3011		return -EINVAL;
3012
3013	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
3014	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
3015		return -EINVAL;
3016
3017	gup_flags |= FOLL_PIN;
3018	return __get_user_pages_locked(current->mm, start, nr_pages,
3019				       pages, NULL, locked,
3020				       gup_flags | FOLL_TOUCH);
3021}
3022EXPORT_SYMBOL(pin_user_pages_locked);