Linux Audio

Check our new training course

Embedded Linux training

Mar 31-Apr 8, 2025
Register
Loading...
v5.4
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Memory Migration functionality - linux/mm/migrate.c
   4 *
   5 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
   6 *
   7 * Page migration was first developed in the context of the memory hotplug
   8 * project. The main authors of the migration code are:
   9 *
  10 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  11 * Hirokazu Takahashi <taka@valinux.co.jp>
  12 * Dave Hansen <haveblue@us.ibm.com>
  13 * Christoph Lameter
  14 */
  15
  16#include <linux/migrate.h>
  17#include <linux/export.h>
  18#include <linux/swap.h>
  19#include <linux/swapops.h>
  20#include <linux/pagemap.h>
  21#include <linux/buffer_head.h>
  22#include <linux/mm_inline.h>
  23#include <linux/nsproxy.h>
  24#include <linux/pagevec.h>
  25#include <linux/ksm.h>
  26#include <linux/rmap.h>
  27#include <linux/topology.h>
  28#include <linux/cpu.h>
  29#include <linux/cpuset.h>
  30#include <linux/writeback.h>
  31#include <linux/mempolicy.h>
  32#include <linux/vmalloc.h>
  33#include <linux/security.h>
  34#include <linux/backing-dev.h>
  35#include <linux/compaction.h>
  36#include <linux/syscalls.h>
  37#include <linux/compat.h>
  38#include <linux/hugetlb.h>
  39#include <linux/hugetlb_cgroup.h>
  40#include <linux/gfp.h>
  41#include <linux/pagewalk.h>
  42#include <linux/pfn_t.h>
  43#include <linux/memremap.h>
  44#include <linux/userfaultfd_k.h>
  45#include <linux/balloon_compaction.h>
  46#include <linux/mmu_notifier.h>
  47#include <linux/page_idle.h>
  48#include <linux/page_owner.h>
  49#include <linux/sched/mm.h>
  50#include <linux/ptrace.h>
 
  51
  52#include <asm/tlbflush.h>
  53
  54#define CREATE_TRACE_POINTS
  55#include <trace/events/migrate.h>
  56
  57#include "internal.h"
  58
  59/*
  60 * migrate_prep() needs to be called before we start compiling a list of pages
  61 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
  62 * undesirable, use migrate_prep_local()
  63 */
  64int migrate_prep(void)
  65{
  66	/*
  67	 * Clear the LRU lists so pages can be isolated.
  68	 * Note that pages may be moved off the LRU after we have
  69	 * drained them. Those pages will fail to migrate like other
  70	 * pages that may be busy.
  71	 */
  72	lru_add_drain_all();
  73
  74	return 0;
  75}
  76
  77/* Do the necessary work of migrate_prep but not if it involves other CPUs */
  78int migrate_prep_local(void)
  79{
  80	lru_add_drain();
  81
  82	return 0;
  83}
  84
  85int isolate_movable_page(struct page *page, isolate_mode_t mode)
  86{
  87	struct address_space *mapping;
  88
  89	/*
  90	 * Avoid burning cycles with pages that are yet under __free_pages(),
  91	 * or just got freed under us.
  92	 *
  93	 * In case we 'win' a race for a movable page being freed under us and
  94	 * raise its refcount preventing __free_pages() from doing its job
  95	 * the put_page() at the end of this block will take care of
  96	 * release this page, thus avoiding a nasty leakage.
  97	 */
  98	if (unlikely(!get_page_unless_zero(page)))
  99		goto out;
 100
 101	/*
 102	 * Check PageMovable before holding a PG_lock because page's owner
 103	 * assumes anybody doesn't touch PG_lock of newly allocated page
 104	 * so unconditionally grabbing the lock ruins page's owner side.
 105	 */
 106	if (unlikely(!__PageMovable(page)))
 107		goto out_putpage;
 108	/*
 109	 * As movable pages are not isolated from LRU lists, concurrent
 110	 * compaction threads can race against page migration functions
 111	 * as well as race against the releasing a page.
 112	 *
 113	 * In order to avoid having an already isolated movable page
 114	 * being (wrongly) re-isolated while it is under migration,
 115	 * or to avoid attempting to isolate pages being released,
 116	 * lets be sure we have the page lock
 117	 * before proceeding with the movable page isolation steps.
 118	 */
 119	if (unlikely(!trylock_page(page)))
 120		goto out_putpage;
 121
 122	if (!PageMovable(page) || PageIsolated(page))
 123		goto out_no_isolated;
 124
 125	mapping = page_mapping(page);
 126	VM_BUG_ON_PAGE(!mapping, page);
 127
 128	if (!mapping->a_ops->isolate_page(page, mode))
 129		goto out_no_isolated;
 130
 131	/* Driver shouldn't use PG_isolated bit of page->flags */
 132	WARN_ON_ONCE(PageIsolated(page));
 133	__SetPageIsolated(page);
 134	unlock_page(page);
 135
 136	return 0;
 137
 138out_no_isolated:
 139	unlock_page(page);
 140out_putpage:
 141	put_page(page);
 142out:
 143	return -EBUSY;
 144}
 145
 146/* It should be called on page which is PG_movable */
 147void putback_movable_page(struct page *page)
 148{
 149	struct address_space *mapping;
 150
 151	VM_BUG_ON_PAGE(!PageLocked(page), page);
 152	VM_BUG_ON_PAGE(!PageMovable(page), page);
 153	VM_BUG_ON_PAGE(!PageIsolated(page), page);
 154
 155	mapping = page_mapping(page);
 156	mapping->a_ops->putback_page(page);
 157	__ClearPageIsolated(page);
 158}
 159
 160/*
 161 * Put previously isolated pages back onto the appropriate lists
 162 * from where they were once taken off for compaction/migration.
 163 *
 164 * This function shall be used whenever the isolated pageset has been
 165 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
 166 * and isolate_huge_page().
 167 */
 168void putback_movable_pages(struct list_head *l)
 169{
 170	struct page *page;
 171	struct page *page2;
 172
 173	list_for_each_entry_safe(page, page2, l, lru) {
 174		if (unlikely(PageHuge(page))) {
 175			putback_active_hugepage(page);
 176			continue;
 177		}
 178		list_del(&page->lru);
 179		/*
 180		 * We isolated non-lru movable page so here we can use
 181		 * __PageMovable because LRU page's mapping cannot have
 182		 * PAGE_MAPPING_MOVABLE.
 183		 */
 184		if (unlikely(__PageMovable(page))) {
 185			VM_BUG_ON_PAGE(!PageIsolated(page), page);
 186			lock_page(page);
 187			if (PageMovable(page))
 188				putback_movable_page(page);
 189			else
 190				__ClearPageIsolated(page);
 191			unlock_page(page);
 192			put_page(page);
 193		} else {
 194			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
 195					page_is_file_cache(page), -hpage_nr_pages(page));
 196			putback_lru_page(page);
 197		}
 198	}
 199}
 200
 201/*
 202 * Restore a potential migration pte to a working pte entry
 203 */
 204static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
 205				 unsigned long addr, void *old)
 206{
 207	struct page_vma_mapped_walk pvmw = {
 208		.page = old,
 209		.vma = vma,
 210		.address = addr,
 211		.flags = PVMW_SYNC | PVMW_MIGRATION,
 212	};
 213	struct page *new;
 214	pte_t pte;
 215	swp_entry_t entry;
 216
 217	VM_BUG_ON_PAGE(PageTail(page), page);
 218	while (page_vma_mapped_walk(&pvmw)) {
 219		if (PageKsm(page))
 220			new = page;
 221		else
 222			new = page - pvmw.page->index +
 223				linear_page_index(vma, pvmw.address);
 224
 225#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 226		/* PMD-mapped THP migration entry */
 227		if (!pvmw.pte) {
 228			VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
 229			remove_migration_pmd(&pvmw, new);
 230			continue;
 231		}
 232#endif
 233
 234		get_page(new);
 235		pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
 236		if (pte_swp_soft_dirty(*pvmw.pte))
 237			pte = pte_mksoft_dirty(pte);
 238
 239		/*
 240		 * Recheck VMA as permissions can change since migration started
 241		 */
 242		entry = pte_to_swp_entry(*pvmw.pte);
 243		if (is_write_migration_entry(entry))
 244			pte = maybe_mkwrite(pte, vma);
 
 
 245
 246		if (unlikely(is_zone_device_page(new))) {
 247			if (is_device_private_page(new)) {
 248				entry = make_device_private_entry(new, pte_write(pte));
 249				pte = swp_entry_to_pte(entry);
 250			}
 
 
 
 
 
 
 
 251		}
 252
 253#ifdef CONFIG_HUGETLB_PAGE
 254		if (PageHuge(new)) {
 
 
 255			pte = pte_mkhuge(pte);
 256			pte = arch_make_huge_pte(pte, vma, new, 0);
 257			set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 258			if (PageAnon(new))
 259				hugepage_add_anon_rmap(new, vma, pvmw.address);
 260			else
 261				page_dup_rmap(new, true);
 262		} else
 263#endif
 264		{
 265			set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 266
 267			if (PageAnon(new))
 268				page_add_anon_rmap(new, vma, pvmw.address, false);
 269			else
 270				page_add_file_rmap(new, false);
 271		}
 272		if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
 273			mlock_vma_page(new);
 274
 275		if (PageTransHuge(page) && PageMlocked(page))
 276			clear_page_mlock(page);
 277
 278		/* No need to invalidate - it was non-present before */
 279		update_mmu_cache(vma, pvmw.address, pvmw.pte);
 280	}
 281
 282	return true;
 283}
 284
 285/*
 286 * Get rid of all migration entries and replace them by
 287 * references to the indicated page.
 288 */
 289void remove_migration_ptes(struct page *old, struct page *new, bool locked)
 290{
 291	struct rmap_walk_control rwc = {
 292		.rmap_one = remove_migration_pte,
 293		.arg = old,
 294	};
 295
 296	if (locked)
 297		rmap_walk_locked(new, &rwc);
 298	else
 299		rmap_walk(new, &rwc);
 300}
 301
 302/*
 303 * Something used the pte of a page under migration. We need to
 304 * get to the page and wait until migration is finished.
 305 * When we return from this function the fault will be retried.
 306 */
 307void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 308				spinlock_t *ptl)
 309{
 310	pte_t pte;
 311	swp_entry_t entry;
 312	struct page *page;
 313
 314	spin_lock(ptl);
 315	pte = *ptep;
 316	if (!is_swap_pte(pte))
 317		goto out;
 318
 319	entry = pte_to_swp_entry(pte);
 320	if (!is_migration_entry(entry))
 321		goto out;
 322
 323	page = migration_entry_to_page(entry);
 
 324
 325	/*
 326	 * Once page cache replacement of page migration started, page_count
 327	 * is zero; but we must not call put_and_wait_on_page_locked() without
 328	 * a ref. Use get_page_unless_zero(), and just fault again if it fails.
 329	 */
 330	if (!get_page_unless_zero(page))
 331		goto out;
 332	pte_unmap_unlock(ptep, ptl);
 333	put_and_wait_on_page_locked(page);
 334	return;
 335out:
 336	pte_unmap_unlock(ptep, ptl);
 337}
 338
 339void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
 340				unsigned long address)
 341{
 342	spinlock_t *ptl = pte_lockptr(mm, pmd);
 343	pte_t *ptep = pte_offset_map(pmd, address);
 344	__migration_entry_wait(mm, ptep, ptl);
 345}
 346
 347void migration_entry_wait_huge(struct vm_area_struct *vma,
 348		struct mm_struct *mm, pte_t *pte)
 349{
 350	spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
 351	__migration_entry_wait(mm, pte, ptl);
 352}
 353
 354#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 355void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
 356{
 357	spinlock_t *ptl;
 358	struct page *page;
 359
 360	ptl = pmd_lock(mm, pmd);
 361	if (!is_pmd_migration_entry(*pmd))
 362		goto unlock;
 363	page = migration_entry_to_page(pmd_to_swp_entry(*pmd));
 364	if (!get_page_unless_zero(page))
 365		goto unlock;
 366	spin_unlock(ptl);
 367	put_and_wait_on_page_locked(page);
 368	return;
 369unlock:
 370	spin_unlock(ptl);
 371}
 372#endif
 373
 374static int expected_page_refs(struct address_space *mapping, struct page *page)
 375{
 376	int expected_count = 1;
 377
 378	/*
 379	 * Device public or private pages have an extra refcount as they are
 380	 * ZONE_DEVICE pages.
 381	 */
 382	expected_count += is_device_private_page(page);
 383	if (mapping)
 384		expected_count += hpage_nr_pages(page) + page_has_private(page);
 385
 386	return expected_count;
 387}
 388
 389/*
 390 * Replace the page in the mapping.
 391 *
 392 * The number of remaining references must be:
 393 * 1 for anonymous pages without a mapping
 394 * 2 for pages with a mapping
 395 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
 396 */
 397int migrate_page_move_mapping(struct address_space *mapping,
 398		struct page *newpage, struct page *page, int extra_count)
 399{
 400	XA_STATE(xas, &mapping->i_pages, page_index(page));
 401	struct zone *oldzone, *newzone;
 402	int dirty;
 403	int expected_count = expected_page_refs(mapping, page) + extra_count;
 
 404
 405	if (!mapping) {
 406		/* Anonymous page without mapping */
 407		if (page_count(page) != expected_count)
 408			return -EAGAIN;
 409
 410		/* No turning back from here */
 411		newpage->index = page->index;
 412		newpage->mapping = page->mapping;
 413		if (PageSwapBacked(page))
 414			__SetPageSwapBacked(newpage);
 415
 416		return MIGRATEPAGE_SUCCESS;
 417	}
 418
 419	oldzone = page_zone(page);
 420	newzone = page_zone(newpage);
 421
 422	xas_lock_irq(&xas);
 423	if (page_count(page) != expected_count || xas_load(&xas) != page) {
 424		xas_unlock_irq(&xas);
 425		return -EAGAIN;
 426	}
 427
 428	if (!page_ref_freeze(page, expected_count)) {
 429		xas_unlock_irq(&xas);
 430		return -EAGAIN;
 431	}
 432
 433	/*
 434	 * Now we know that no one else is looking at the page:
 435	 * no turning back from here.
 436	 */
 437	newpage->index = page->index;
 438	newpage->mapping = page->mapping;
 439	page_ref_add(newpage, hpage_nr_pages(page)); /* add cache reference */
 440	if (PageSwapBacked(page)) {
 441		__SetPageSwapBacked(newpage);
 442		if (PageSwapCache(page)) {
 443			SetPageSwapCache(newpage);
 444			set_page_private(newpage, page_private(page));
 445		}
 446	} else {
 447		VM_BUG_ON_PAGE(PageSwapCache(page), page);
 448	}
 449
 450	/* Move dirty while page refs frozen and newpage not yet exposed */
 451	dirty = PageDirty(page);
 452	if (dirty) {
 453		ClearPageDirty(page);
 454		SetPageDirty(newpage);
 455	}
 456
 457	xas_store(&xas, newpage);
 458	if (PageTransHuge(page)) {
 459		int i;
 460
 461		for (i = 1; i < HPAGE_PMD_NR; i++) {
 462			xas_next(&xas);
 463			xas_store(&xas, newpage);
 464		}
 465	}
 466
 467	/*
 468	 * Drop cache reference from old page by unfreezing
 469	 * to one less reference.
 470	 * We know this isn't the last reference.
 471	 */
 472	page_ref_unfreeze(page, expected_count - hpage_nr_pages(page));
 473
 474	xas_unlock(&xas);
 475	/* Leave irq disabled to prevent preemption while updating stats */
 476
 477	/*
 478	 * If moved to a different zone then also account
 479	 * the page for that zone. Other VM counters will be
 480	 * taken care of when we establish references to the
 481	 * new page and drop references to the old page.
 482	 *
 483	 * Note that anonymous pages are accounted for
 484	 * via NR_FILE_PAGES and NR_ANON_MAPPED if they
 485	 * are mapped to swap space.
 486	 */
 487	if (newzone != oldzone) {
 488		__dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
 489		__inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
 
 
 
 
 
 
 
 490		if (PageSwapBacked(page) && !PageSwapCache(page)) {
 491			__dec_node_state(oldzone->zone_pgdat, NR_SHMEM);
 492			__inc_node_state(newzone->zone_pgdat, NR_SHMEM);
 493		}
 494		if (dirty && mapping_cap_account_dirty(mapping)) {
 495			__dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
 496			__dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
 497			__inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);
 498			__inc_zone_state(newzone, NR_ZONE_WRITE_PENDING);
 
 
 
 
 
 
 499		}
 500	}
 501	local_irq_enable();
 502
 503	return MIGRATEPAGE_SUCCESS;
 504}
 505EXPORT_SYMBOL(migrate_page_move_mapping);
 506
 507/*
 508 * The expected number of remaining references is the same as that
 509 * of migrate_page_move_mapping().
 510 */
 511int migrate_huge_page_move_mapping(struct address_space *mapping,
 512				   struct page *newpage, struct page *page)
 513{
 514	XA_STATE(xas, &mapping->i_pages, page_index(page));
 515	int expected_count;
 516
 517	xas_lock_irq(&xas);
 518	expected_count = 2 + page_has_private(page);
 519	if (page_count(page) != expected_count || xas_load(&xas) != page) {
 520		xas_unlock_irq(&xas);
 521		return -EAGAIN;
 522	}
 523
 524	if (!page_ref_freeze(page, expected_count)) {
 525		xas_unlock_irq(&xas);
 526		return -EAGAIN;
 527	}
 528
 529	newpage->index = page->index;
 530	newpage->mapping = page->mapping;
 531
 532	get_page(newpage);
 533
 534	xas_store(&xas, newpage);
 535
 536	page_ref_unfreeze(page, expected_count - 1);
 537
 538	xas_unlock_irq(&xas);
 539
 540	return MIGRATEPAGE_SUCCESS;
 541}
 542
 543/*
 544 * Gigantic pages are so large that we do not guarantee that page++ pointer
 545 * arithmetic will work across the entire page.  We need something more
 546 * specialized.
 547 */
 548static void __copy_gigantic_page(struct page *dst, struct page *src,
 549				int nr_pages)
 550{
 551	int i;
 552	struct page *dst_base = dst;
 553	struct page *src_base = src;
 554
 555	for (i = 0; i < nr_pages; ) {
 556		cond_resched();
 557		copy_highpage(dst, src);
 558
 559		i++;
 560		dst = mem_map_next(dst, dst_base, i);
 561		src = mem_map_next(src, src_base, i);
 562	}
 563}
 564
 565static void copy_huge_page(struct page *dst, struct page *src)
 566{
 567	int i;
 568	int nr_pages;
 569
 570	if (PageHuge(src)) {
 571		/* hugetlbfs page */
 572		struct hstate *h = page_hstate(src);
 573		nr_pages = pages_per_huge_page(h);
 574
 575		if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
 576			__copy_gigantic_page(dst, src, nr_pages);
 577			return;
 578		}
 579	} else {
 580		/* thp page */
 581		BUG_ON(!PageTransHuge(src));
 582		nr_pages = hpage_nr_pages(src);
 583	}
 584
 585	for (i = 0; i < nr_pages; i++) {
 586		cond_resched();
 587		copy_highpage(dst + i, src + i);
 588	}
 589}
 590
 591/*
 592 * Copy the page to its new location
 593 */
 594void migrate_page_states(struct page *newpage, struct page *page)
 595{
 596	int cpupid;
 597
 598	if (PageError(page))
 599		SetPageError(newpage);
 600	if (PageReferenced(page))
 601		SetPageReferenced(newpage);
 602	if (PageUptodate(page))
 603		SetPageUptodate(newpage);
 604	if (TestClearPageActive(page)) {
 605		VM_BUG_ON_PAGE(PageUnevictable(page), page);
 606		SetPageActive(newpage);
 607	} else if (TestClearPageUnevictable(page))
 608		SetPageUnevictable(newpage);
 609	if (PageWorkingset(page))
 610		SetPageWorkingset(newpage);
 611	if (PageChecked(page))
 612		SetPageChecked(newpage);
 613	if (PageMappedToDisk(page))
 614		SetPageMappedToDisk(newpage);
 615
 616	/* Move dirty on pages not done by migrate_page_move_mapping() */
 617	if (PageDirty(page))
 618		SetPageDirty(newpage);
 619
 620	if (page_is_young(page))
 621		set_page_young(newpage);
 622	if (page_is_idle(page))
 623		set_page_idle(newpage);
 624
 625	/*
 626	 * Copy NUMA information to the new page, to prevent over-eager
 627	 * future migrations of this same page.
 628	 */
 629	cpupid = page_cpupid_xchg_last(page, -1);
 630	page_cpupid_xchg_last(newpage, cpupid);
 631
 632	ksm_migrate_page(newpage, page);
 633	/*
 634	 * Please do not reorder this without considering how mm/ksm.c's
 635	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
 636	 */
 637	if (PageSwapCache(page))
 638		ClearPageSwapCache(page);
 639	ClearPagePrivate(page);
 640	set_page_private(page, 0);
 
 
 
 641
 642	/*
 643	 * If any waiters have accumulated on the new page then
 644	 * wake them up.
 645	 */
 646	if (PageWriteback(newpage))
 647		end_page_writeback(newpage);
 648
 
 
 
 
 
 
 
 
 649	copy_page_owner(page, newpage);
 650
 651	mem_cgroup_migrate(page, newpage);
 
 652}
 653EXPORT_SYMBOL(migrate_page_states);
 654
 655void migrate_page_copy(struct page *newpage, struct page *page)
 656{
 657	if (PageHuge(page) || PageTransHuge(page))
 658		copy_huge_page(newpage, page);
 659	else
 660		copy_highpage(newpage, page);
 661
 662	migrate_page_states(newpage, page);
 663}
 664EXPORT_SYMBOL(migrate_page_copy);
 665
 666/************************************************************
 667 *                    Migration functions
 668 ***********************************************************/
 669
 670/*
 671 * Common logic to directly migrate a single LRU page suitable for
 672 * pages that do not use PagePrivate/PagePrivate2.
 673 *
 674 * Pages are locked upon entry and exit.
 675 */
 676int migrate_page(struct address_space *mapping,
 677		struct page *newpage, struct page *page,
 678		enum migrate_mode mode)
 679{
 680	int rc;
 681
 682	BUG_ON(PageWriteback(page));	/* Writeback must be complete */
 683
 684	rc = migrate_page_move_mapping(mapping, newpage, page, 0);
 685
 686	if (rc != MIGRATEPAGE_SUCCESS)
 687		return rc;
 688
 689	if (mode != MIGRATE_SYNC_NO_COPY)
 690		migrate_page_copy(newpage, page);
 691	else
 692		migrate_page_states(newpage, page);
 693	return MIGRATEPAGE_SUCCESS;
 694}
 695EXPORT_SYMBOL(migrate_page);
 696
 697#ifdef CONFIG_BLOCK
 698/* Returns true if all buffers are successfully locked */
 699static bool buffer_migrate_lock_buffers(struct buffer_head *head,
 700							enum migrate_mode mode)
 701{
 702	struct buffer_head *bh = head;
 703
 704	/* Simple case, sync compaction */
 705	if (mode != MIGRATE_ASYNC) {
 706		do {
 707			lock_buffer(bh);
 708			bh = bh->b_this_page;
 709
 710		} while (bh != head);
 711
 712		return true;
 713	}
 714
 715	/* async case, we cannot block on lock_buffer so use trylock_buffer */
 716	do {
 717		if (!trylock_buffer(bh)) {
 718			/*
 719			 * We failed to lock the buffer and cannot stall in
 720			 * async migration. Release the taken locks
 721			 */
 722			struct buffer_head *failed_bh = bh;
 723			bh = head;
 724			while (bh != failed_bh) {
 725				unlock_buffer(bh);
 726				bh = bh->b_this_page;
 727			}
 728			return false;
 729		}
 730
 731		bh = bh->b_this_page;
 732	} while (bh != head);
 733	return true;
 734}
 735
 736static int __buffer_migrate_page(struct address_space *mapping,
 737		struct page *newpage, struct page *page, enum migrate_mode mode,
 738		bool check_refs)
 739{
 740	struct buffer_head *bh, *head;
 741	int rc;
 742	int expected_count;
 743
 744	if (!page_has_buffers(page))
 745		return migrate_page(mapping, newpage, page, mode);
 746
 747	/* Check whether page does not have extra refs before we do more work */
 748	expected_count = expected_page_refs(mapping, page);
 749	if (page_count(page) != expected_count)
 750		return -EAGAIN;
 751
 752	head = page_buffers(page);
 753	if (!buffer_migrate_lock_buffers(head, mode))
 754		return -EAGAIN;
 755
 756	if (check_refs) {
 757		bool busy;
 758		bool invalidated = false;
 759
 760recheck_buffers:
 761		busy = false;
 762		spin_lock(&mapping->private_lock);
 763		bh = head;
 764		do {
 765			if (atomic_read(&bh->b_count)) {
 766				busy = true;
 767				break;
 768			}
 769			bh = bh->b_this_page;
 770		} while (bh != head);
 771		if (busy) {
 772			if (invalidated) {
 773				rc = -EAGAIN;
 774				goto unlock_buffers;
 775			}
 776			spin_unlock(&mapping->private_lock);
 777			invalidate_bh_lrus();
 778			invalidated = true;
 779			goto recheck_buffers;
 780		}
 781	}
 782
 783	rc = migrate_page_move_mapping(mapping, newpage, page, 0);
 784	if (rc != MIGRATEPAGE_SUCCESS)
 785		goto unlock_buffers;
 786
 787	ClearPagePrivate(page);
 788	set_page_private(newpage, page_private(page));
 789	set_page_private(page, 0);
 790	put_page(page);
 791	get_page(newpage);
 792
 793	bh = head;
 794	do {
 795		set_bh_page(bh, newpage, bh_offset(bh));
 796		bh = bh->b_this_page;
 797
 798	} while (bh != head);
 799
 800	SetPagePrivate(newpage);
 801
 802	if (mode != MIGRATE_SYNC_NO_COPY)
 803		migrate_page_copy(newpage, page);
 804	else
 805		migrate_page_states(newpage, page);
 806
 807	rc = MIGRATEPAGE_SUCCESS;
 808unlock_buffers:
 809	if (check_refs)
 810		spin_unlock(&mapping->private_lock);
 811	bh = head;
 812	do {
 813		unlock_buffer(bh);
 814		bh = bh->b_this_page;
 815
 816	} while (bh != head);
 817
 818	return rc;
 819}
 820
 821/*
 822 * Migration function for pages with buffers. This function can only be used
 823 * if the underlying filesystem guarantees that no other references to "page"
 824 * exist. For example attached buffer heads are accessed only under page lock.
 825 */
 826int buffer_migrate_page(struct address_space *mapping,
 827		struct page *newpage, struct page *page, enum migrate_mode mode)
 828{
 829	return __buffer_migrate_page(mapping, newpage, page, mode, false);
 830}
 831EXPORT_SYMBOL(buffer_migrate_page);
 832
 833/*
 834 * Same as above except that this variant is more careful and checks that there
 835 * are also no buffer head references. This function is the right one for
 836 * mappings where buffer heads are directly looked up and referenced (such as
 837 * block device mappings).
 838 */
 839int buffer_migrate_page_norefs(struct address_space *mapping,
 840		struct page *newpage, struct page *page, enum migrate_mode mode)
 841{
 842	return __buffer_migrate_page(mapping, newpage, page, mode, true);
 843}
 844#endif
 845
 846/*
 847 * Writeback a page to clean the dirty state
 848 */
 849static int writeout(struct address_space *mapping, struct page *page)
 850{
 851	struct writeback_control wbc = {
 852		.sync_mode = WB_SYNC_NONE,
 853		.nr_to_write = 1,
 854		.range_start = 0,
 855		.range_end = LLONG_MAX,
 856		.for_reclaim = 1
 857	};
 858	int rc;
 859
 860	if (!mapping->a_ops->writepage)
 861		/* No write method for the address space */
 862		return -EINVAL;
 863
 864	if (!clear_page_dirty_for_io(page))
 865		/* Someone else already triggered a write */
 866		return -EAGAIN;
 867
 868	/*
 869	 * A dirty page may imply that the underlying filesystem has
 870	 * the page on some queue. So the page must be clean for
 871	 * migration. Writeout may mean we loose the lock and the
 872	 * page state is no longer what we checked for earlier.
 873	 * At this point we know that the migration attempt cannot
 874	 * be successful.
 875	 */
 876	remove_migration_ptes(page, page, false);
 877
 878	rc = mapping->a_ops->writepage(page, &wbc);
 879
 880	if (rc != AOP_WRITEPAGE_ACTIVATE)
 881		/* unlocked. Relock */
 882		lock_page(page);
 883
 884	return (rc < 0) ? -EIO : -EAGAIN;
 885}
 886
 887/*
 888 * Default handling if a filesystem does not provide a migration function.
 889 */
 890static int fallback_migrate_page(struct address_space *mapping,
 891	struct page *newpage, struct page *page, enum migrate_mode mode)
 892{
 893	if (PageDirty(page)) {
 894		/* Only writeback pages in full synchronous migration */
 895		switch (mode) {
 896		case MIGRATE_SYNC:
 897		case MIGRATE_SYNC_NO_COPY:
 898			break;
 899		default:
 900			return -EBUSY;
 901		}
 902		return writeout(mapping, page);
 903	}
 904
 905	/*
 906	 * Buffers may be managed in a filesystem specific way.
 907	 * We must have no buffers or drop them.
 908	 */
 909	if (page_has_private(page) &&
 910	    !try_to_release_page(page, GFP_KERNEL))
 911		return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
 912
 913	return migrate_page(mapping, newpage, page, mode);
 914}
 915
 916/*
 917 * Move a page to a newly allocated page
 918 * The page is locked and all ptes have been successfully removed.
 919 *
 920 * The new page will have replaced the old page if this function
 921 * is successful.
 922 *
 923 * Return value:
 924 *   < 0 - error code
 925 *  MIGRATEPAGE_SUCCESS - success
 926 */
 927static int move_to_new_page(struct page *newpage, struct page *page,
 928				enum migrate_mode mode)
 929{
 930	struct address_space *mapping;
 931	int rc = -EAGAIN;
 932	bool is_lru = !__PageMovable(page);
 933
 934	VM_BUG_ON_PAGE(!PageLocked(page), page);
 935	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
 936
 937	mapping = page_mapping(page);
 938
 939	if (likely(is_lru)) {
 940		if (!mapping)
 941			rc = migrate_page(mapping, newpage, page, mode);
 942		else if (mapping->a_ops->migratepage)
 943			/*
 944			 * Most pages have a mapping and most filesystems
 945			 * provide a migratepage callback. Anonymous pages
 946			 * are part of swap space which also has its own
 947			 * migratepage callback. This is the most common path
 948			 * for page migration.
 949			 */
 950			rc = mapping->a_ops->migratepage(mapping, newpage,
 951							page, mode);
 952		else
 953			rc = fallback_migrate_page(mapping, newpage,
 954							page, mode);
 955	} else {
 956		/*
 957		 * In case of non-lru page, it could be released after
 958		 * isolation step. In that case, we shouldn't try migration.
 959		 */
 960		VM_BUG_ON_PAGE(!PageIsolated(page), page);
 961		if (!PageMovable(page)) {
 962			rc = MIGRATEPAGE_SUCCESS;
 963			__ClearPageIsolated(page);
 964			goto out;
 965		}
 966
 967		rc = mapping->a_ops->migratepage(mapping, newpage,
 968						page, mode);
 969		WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
 970			!PageIsolated(page));
 971	}
 972
 973	/*
 974	 * When successful, old pagecache page->mapping must be cleared before
 975	 * page is freed; but stats require that PageAnon be left as PageAnon.
 976	 */
 977	if (rc == MIGRATEPAGE_SUCCESS) {
 978		if (__PageMovable(page)) {
 979			VM_BUG_ON_PAGE(!PageIsolated(page), page);
 980
 981			/*
 982			 * We clear PG_movable under page_lock so any compactor
 983			 * cannot try to migrate this page.
 984			 */
 985			__ClearPageIsolated(page);
 986		}
 987
 988		/*
 989		 * Anonymous and movable page->mapping will be cleard by
 990		 * free_pages_prepare so don't reset it here for keeping
 991		 * the type to work PageAnon, for example.
 992		 */
 993		if (!PageMappingFlags(page))
 994			page->mapping = NULL;
 995
 996		if (likely(!is_zone_device_page(newpage)))
 997			flush_dcache_page(newpage);
 998
 999	}
1000out:
1001	return rc;
1002}
1003
1004static int __unmap_and_move(struct page *page, struct page *newpage,
1005				int force, enum migrate_mode mode)
1006{
1007	int rc = -EAGAIN;
1008	int page_was_mapped = 0;
1009	struct anon_vma *anon_vma = NULL;
1010	bool is_lru = !__PageMovable(page);
1011
1012	if (!trylock_page(page)) {
1013		if (!force || mode == MIGRATE_ASYNC)
1014			goto out;
1015
1016		/*
1017		 * It's not safe for direct compaction to call lock_page.
1018		 * For example, during page readahead pages are added locked
1019		 * to the LRU. Later, when the IO completes the pages are
1020		 * marked uptodate and unlocked. However, the queueing
1021		 * could be merging multiple pages for one bio (e.g.
1022		 * mpage_readpages). If an allocation happens for the
1023		 * second or third page, the process can end up locking
1024		 * the same page twice and deadlocking. Rather than
1025		 * trying to be clever about what pages can be locked,
1026		 * avoid the use of lock_page for direct compaction
1027		 * altogether.
1028		 */
1029		if (current->flags & PF_MEMALLOC)
1030			goto out;
1031
1032		lock_page(page);
1033	}
1034
1035	if (PageWriteback(page)) {
1036		/*
1037		 * Only in the case of a full synchronous migration is it
1038		 * necessary to wait for PageWriteback. In the async case,
1039		 * the retry loop is too short and in the sync-light case,
1040		 * the overhead of stalling is too much
1041		 */
1042		switch (mode) {
1043		case MIGRATE_SYNC:
1044		case MIGRATE_SYNC_NO_COPY:
1045			break;
1046		default:
1047			rc = -EBUSY;
1048			goto out_unlock;
1049		}
1050		if (!force)
1051			goto out_unlock;
1052		wait_on_page_writeback(page);
1053	}
1054
1055	/*
1056	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
1057	 * we cannot notice that anon_vma is freed while we migrates a page.
1058	 * This get_anon_vma() delays freeing anon_vma pointer until the end
1059	 * of migration. File cache pages are no problem because of page_lock()
1060	 * File Caches may use write_page() or lock_page() in migration, then,
1061	 * just care Anon page here.
1062	 *
1063	 * Only page_get_anon_vma() understands the subtleties of
1064	 * getting a hold on an anon_vma from outside one of its mms.
1065	 * But if we cannot get anon_vma, then we won't need it anyway,
1066	 * because that implies that the anon page is no longer mapped
1067	 * (and cannot be remapped so long as we hold the page lock).
1068	 */
1069	if (PageAnon(page) && !PageKsm(page))
1070		anon_vma = page_get_anon_vma(page);
1071
1072	/*
1073	 * Block others from accessing the new page when we get around to
1074	 * establishing additional references. We are usually the only one
1075	 * holding a reference to newpage at this point. We used to have a BUG
1076	 * here if trylock_page(newpage) fails, but would like to allow for
1077	 * cases where there might be a race with the previous use of newpage.
1078	 * This is much like races on refcount of oldpage: just don't BUG().
1079	 */
1080	if (unlikely(!trylock_page(newpage)))
1081		goto out_unlock;
1082
1083	if (unlikely(!is_lru)) {
1084		rc = move_to_new_page(newpage, page, mode);
1085		goto out_unlock_both;
1086	}
1087
1088	/*
1089	 * Corner case handling:
1090	 * 1. When a new swap-cache page is read into, it is added to the LRU
1091	 * and treated as swapcache but it has no rmap yet.
1092	 * Calling try_to_unmap() against a page->mapping==NULL page will
1093	 * trigger a BUG.  So handle it here.
1094	 * 2. An orphaned page (see truncate_complete_page) might have
1095	 * fs-private metadata. The page can be picked up due to memory
1096	 * offlining.  Everywhere else except page reclaim, the page is
1097	 * invisible to the vm, so the page can not be migrated.  So try to
1098	 * free the metadata, so the page can be freed.
1099	 */
1100	if (!page->mapping) {
1101		VM_BUG_ON_PAGE(PageAnon(page), page);
1102		if (page_has_private(page)) {
1103			try_to_free_buffers(page);
1104			goto out_unlock_both;
1105		}
1106	} else if (page_mapped(page)) {
1107		/* Establish migration ptes */
1108		VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
1109				page);
1110		try_to_unmap(page,
1111			TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1112		page_was_mapped = 1;
1113	}
1114
1115	if (!page_mapped(page))
1116		rc = move_to_new_page(newpage, page, mode);
1117
1118	if (page_was_mapped)
1119		remove_migration_ptes(page,
1120			rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1121
1122out_unlock_both:
1123	unlock_page(newpage);
1124out_unlock:
1125	/* Drop an anon_vma reference if we took one */
1126	if (anon_vma)
1127		put_anon_vma(anon_vma);
1128	unlock_page(page);
1129out:
1130	/*
1131	 * If migration is successful, decrease refcount of the newpage
1132	 * which will not free the page because new page owner increased
1133	 * refcounter. As well, if it is LRU page, add the page to LRU
1134	 * list in here. Use the old state of the isolated source page to
1135	 * determine if we migrated a LRU page. newpage was already unlocked
1136	 * and possibly modified by its owner - don't rely on the page
1137	 * state.
1138	 */
1139	if (rc == MIGRATEPAGE_SUCCESS) {
1140		if (unlikely(!is_lru))
1141			put_page(newpage);
1142		else
1143			putback_lru_page(newpage);
1144	}
1145
1146	return rc;
1147}
1148
1149/*
1150 * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move().  Work
1151 * around it.
1152 */
1153#if defined(CONFIG_ARM) && \
1154	defined(GCC_VERSION) && GCC_VERSION < 40900 && GCC_VERSION >= 40700
1155#define ICE_noinline noinline
1156#else
1157#define ICE_noinline
1158#endif
1159
1160/*
1161 * Obtain the lock on page, remove all ptes and migrate the page
1162 * to the newly allocated page in newpage.
1163 */
1164static ICE_noinline int unmap_and_move(new_page_t get_new_page,
1165				   free_page_t put_new_page,
1166				   unsigned long private, struct page *page,
1167				   int force, enum migrate_mode mode,
1168				   enum migrate_reason reason)
 
1169{
1170	int rc = MIGRATEPAGE_SUCCESS;
1171	struct page *newpage;
1172
1173	if (!thp_migration_supported() && PageTransHuge(page))
1174		return -ENOMEM;
1175
1176	newpage = get_new_page(page, private);
1177	if (!newpage)
1178		return -ENOMEM;
1179
1180	if (page_count(page) == 1) {
1181		/* page was freed from under us. So we are done. */
1182		ClearPageActive(page);
1183		ClearPageUnevictable(page);
1184		if (unlikely(__PageMovable(page))) {
1185			lock_page(page);
1186			if (!PageMovable(page))
1187				__ClearPageIsolated(page);
1188			unlock_page(page);
1189		}
1190		if (put_new_page)
1191			put_new_page(newpage, private);
1192		else
1193			put_page(newpage);
1194		goto out;
1195	}
1196
 
 
 
 
1197	rc = __unmap_and_move(page, newpage, force, mode);
1198	if (rc == MIGRATEPAGE_SUCCESS)
1199		set_page_owner_migrate_reason(newpage, reason);
1200
1201out:
1202	if (rc != -EAGAIN) {
1203		/*
1204		 * A page that has been migrated has all references
1205		 * removed and will be freed. A page that has not been
1206		 * migrated will have kepts its references and be
1207		 * restored.
1208		 */
1209		list_del(&page->lru);
 
1210
 
 
 
 
 
 
1211		/*
1212		 * Compaction can migrate also non-LRU pages which are
1213		 * not accounted to NR_ISOLATED_*. They can be recognized
1214		 * as __PageMovable
1215		 */
1216		if (likely(!__PageMovable(page)))
1217			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1218					page_is_file_cache(page), -hpage_nr_pages(page));
1219	}
1220
1221	/*
1222	 * If migration is successful, releases reference grabbed during
1223	 * isolation. Otherwise, restore the page to right list unless
1224	 * we want to retry.
1225	 */
1226	if (rc == MIGRATEPAGE_SUCCESS) {
1227		put_page(page);
1228		if (reason == MR_MEMORY_FAILURE) {
1229			/*
1230			 * Set PG_HWPoison on just freed page
1231			 * intentionally. Although it's rather weird,
1232			 * it's how HWPoison flag works at the moment.
1233			 */
1234			if (set_hwpoison_free_buddy_page(page))
1235				num_poisoned_pages_inc();
1236		}
1237	} else {
1238		if (rc != -EAGAIN) {
1239			if (likely(!__PageMovable(page))) {
1240				putback_lru_page(page);
1241				goto put_new;
1242			}
1243
1244			lock_page(page);
1245			if (PageMovable(page))
1246				putback_movable_page(page);
1247			else
1248				__ClearPageIsolated(page);
1249			unlock_page(page);
1250			put_page(page);
1251		}
1252put_new:
1253		if (put_new_page)
1254			put_new_page(newpage, private);
1255		else
1256			put_page(newpage);
1257	}
1258
1259	return rc;
1260}
1261
1262/*
1263 * Counterpart of unmap_and_move_page() for hugepage migration.
1264 *
1265 * This function doesn't wait the completion of hugepage I/O
1266 * because there is no race between I/O and migration for hugepage.
1267 * Note that currently hugepage I/O occurs only in direct I/O
1268 * where no lock is held and PG_writeback is irrelevant,
1269 * and writeback status of all subpages are counted in the reference
1270 * count of the head page (i.e. if all subpages of a 2MB hugepage are
1271 * under direct I/O, the reference of the head page is 512 and a bit more.)
1272 * This means that when we try to migrate hugepage whose subpages are
1273 * doing direct I/O, some references remain after try_to_unmap() and
1274 * hugepage migration fails without data corruption.
1275 *
1276 * There is also no race when direct I/O is issued on the page under migration,
1277 * because then pte is replaced with migration swap entry and direct I/O code
1278 * will wait in the page fault for migration to complete.
1279 */
1280static int unmap_and_move_huge_page(new_page_t get_new_page,
1281				free_page_t put_new_page, unsigned long private,
1282				struct page *hpage, int force,
1283				enum migrate_mode mode, int reason)
 
1284{
1285	int rc = -EAGAIN;
1286	int page_was_mapped = 0;
1287	struct page *new_hpage;
1288	struct anon_vma *anon_vma = NULL;
 
1289
1290	/*
1291	 * Migratability of hugepages depends on architectures and their size.
1292	 * This check is necessary because some callers of hugepage migration
1293	 * like soft offline and memory hotremove don't walk through page
1294	 * tables or check whether the hugepage is pmd-based or not before
1295	 * kicking migration.
1296	 */
1297	if (!hugepage_migration_supported(page_hstate(hpage))) {
1298		putback_active_hugepage(hpage);
1299		return -ENOSYS;
1300	}
1301
 
 
 
 
 
 
1302	new_hpage = get_new_page(hpage, private);
1303	if (!new_hpage)
1304		return -ENOMEM;
1305
1306	if (!trylock_page(hpage)) {
1307		if (!force)
1308			goto out;
1309		switch (mode) {
1310		case MIGRATE_SYNC:
1311		case MIGRATE_SYNC_NO_COPY:
1312			break;
1313		default:
1314			goto out;
1315		}
1316		lock_page(hpage);
1317	}
1318
1319	/*
1320	 * Check for pages which are in the process of being freed.  Without
1321	 * page_mapping() set, hugetlbfs specific move page routine will not
1322	 * be called and we could leak usage counts for subpools.
1323	 */
1324	if (page_private(hpage) && !page_mapping(hpage)) {
1325		rc = -EBUSY;
1326		goto out_unlock;
1327	}
1328
1329	if (PageAnon(hpage))
1330		anon_vma = page_get_anon_vma(hpage);
1331
1332	if (unlikely(!trylock_page(new_hpage)))
1333		goto put_anon;
1334
1335	if (page_mapped(hpage)) {
1336		try_to_unmap(hpage,
1337			TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1338		page_was_mapped = 1;
 
 
 
1339	}
1340
1341	if (!page_mapped(hpage))
1342		rc = move_to_new_page(new_hpage, hpage, mode);
1343
1344	if (page_was_mapped)
1345		remove_migration_ptes(hpage,
1346			rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1347
 
1348	unlock_page(new_hpage);
1349
1350put_anon:
1351	if (anon_vma)
1352		put_anon_vma(anon_vma);
1353
1354	if (rc == MIGRATEPAGE_SUCCESS) {
1355		move_hugetlb_state(hpage, new_hpage, reason);
1356		put_new_page = NULL;
1357	}
1358
1359out_unlock:
1360	unlock_page(hpage);
1361out:
1362	if (rc != -EAGAIN)
1363		putback_active_hugepage(hpage);
 
 
1364
1365	/*
1366	 * If migration was not successful and there's a freeing callback, use
1367	 * it.  Otherwise, put_page() will drop the reference grabbed during
1368	 * isolation.
1369	 */
1370	if (put_new_page)
1371		put_new_page(new_hpage, private);
1372	else
1373		putback_active_hugepage(new_hpage);
1374
1375	return rc;
1376}
1377
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1378/*
1379 * migrate_pages - migrate the pages specified in a list, to the free pages
1380 *		   supplied as the target for the page migration
1381 *
1382 * @from:		The list of pages to be migrated.
1383 * @get_new_page:	The function used to allocate free pages to be used
1384 *			as the target of the page migration.
1385 * @put_new_page:	The function used to free target pages if migration
1386 *			fails, or NULL if no special handling is necessary.
1387 * @private:		Private data to be passed on to get_new_page()
1388 * @mode:		The migration mode that specifies the constraints for
1389 *			page migration, if any.
1390 * @reason:		The reason for page migration.
1391 *
1392 * The function returns after 10 attempts or if no pages are movable any more
1393 * because the list has become empty or no retryable pages exist any more.
1394 * The caller should call putback_movable_pages() to return pages to the LRU
1395 * or free list only if ret != 0.
1396 *
1397 * Returns the number of pages that were not migrated, or an error code.
1398 */
1399int migrate_pages(struct list_head *from, new_page_t get_new_page,
1400		free_page_t put_new_page, unsigned long private,
1401		enum migrate_mode mode, int reason)
1402{
1403	int retry = 1;
 
1404	int nr_failed = 0;
1405	int nr_succeeded = 0;
 
 
 
1406	int pass = 0;
 
1407	struct page *page;
1408	struct page *page2;
1409	int swapwrite = current->flags & PF_SWAPWRITE;
1410	int rc;
 
 
 
 
1411
1412	if (!swapwrite)
1413		current->flags |= PF_SWAPWRITE;
1414
1415	for(pass = 0; pass < 10 && retry; pass++) {
1416		retry = 0;
 
1417
1418		list_for_each_entry_safe(page, page2, from, lru) {
1419retry:
 
 
 
 
 
 
 
1420			cond_resched();
1421
1422			if (PageHuge(page))
1423				rc = unmap_and_move_huge_page(get_new_page,
1424						put_new_page, private, page,
1425						pass > 2, mode, reason);
 
1426			else
1427				rc = unmap_and_move(get_new_page, put_new_page,
1428						private, page, pass > 2, mode,
1429						reason);
1430
 
 
 
 
 
 
 
 
1431			switch(rc) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1432			case -ENOMEM:
1433				/*
1434				 * THP migration might be unsupported or the
1435				 * allocation could've failed so we should
1436				 * retry on the same page with the THP split
1437				 * to base pages.
1438				 *
1439				 * Head page is retried immediately and tail
1440				 * pages are added to the tail of the list so
1441				 * we encounter them after the rest of the list
1442				 * is processed.
1443				 */
1444				if (PageTransHuge(page) && !PageHuge(page)) {
1445					lock_page(page);
1446					rc = split_huge_page_to_list(page, from);
1447					unlock_page(page);
1448					if (!rc) {
1449						list_safe_reset_next(page, page2, lru);
1450						goto retry;
1451					}
 
 
 
 
1452				}
1453				nr_failed++;
1454				goto out;
1455			case -EAGAIN:
 
 
 
 
1456				retry++;
1457				break;
1458			case MIGRATEPAGE_SUCCESS:
 
 
 
 
 
1459				nr_succeeded++;
1460				break;
1461			default:
1462				/*
1463				 * Permanent failure (-EBUSY, -ENOSYS, etc.):
1464				 * unlike -EAGAIN case, the failed page is
1465				 * removed from migration page list and not
1466				 * retried in the next outer loop.
1467				 */
 
 
 
 
 
1468				nr_failed++;
1469				break;
1470			}
1471		}
1472	}
1473	nr_failed += retry;
 
1474	rc = nr_failed;
1475out:
1476	if (nr_succeeded)
1477		count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1478	if (nr_failed)
1479		count_vm_events(PGMIGRATE_FAIL, nr_failed);
1480	trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
 
 
 
 
 
 
 
 
1481
1482	if (!swapwrite)
1483		current->flags &= ~PF_SWAPWRITE;
1484
1485	return rc;
1486}
1487
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1488#ifdef CONFIG_NUMA
1489
1490static int store_status(int __user *status, int start, int value, int nr)
1491{
1492	while (nr-- > 0) {
1493		if (put_user(value, status + start))
1494			return -EFAULT;
1495		start++;
1496	}
1497
1498	return 0;
1499}
1500
1501static int do_move_pages_to_node(struct mm_struct *mm,
1502		struct list_head *pagelist, int node)
1503{
1504	int err;
 
 
 
 
1505
1506	if (list_empty(pagelist))
1507		return 0;
1508
1509	err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
1510			MIGRATE_SYNC, MR_SYSCALL);
1511	if (err)
1512		putback_movable_pages(pagelist);
1513	return err;
1514}
1515
1516/*
1517 * Resolves the given address to a struct page, isolates it from the LRU and
1518 * puts it to the given pagelist.
1519 * Returns -errno if the page cannot be found/isolated or 0 when it has been
1520 * queued or the page doesn't need to be migrated because it is already on
1521 * the target node
 
 
1522 */
1523static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
1524		int node, struct list_head *pagelist, bool migrate_all)
1525{
1526	struct vm_area_struct *vma;
1527	struct page *page;
1528	unsigned int follflags;
1529	int err;
1530
1531	down_read(&mm->mmap_sem);
1532	err = -EFAULT;
1533	vma = find_vma(mm, addr);
1534	if (!vma || addr < vma->vm_start || !vma_migratable(vma))
1535		goto out;
1536
1537	/* FOLL_DUMP to ignore special (like zero) pages */
1538	follflags = FOLL_GET | FOLL_DUMP;
1539	page = follow_page(vma, addr, follflags);
1540
1541	err = PTR_ERR(page);
1542	if (IS_ERR(page))
1543		goto out;
1544
1545	err = -ENOENT;
1546	if (!page)
1547		goto out;
1548
1549	err = 0;
1550	if (page_to_nid(page) == node)
1551		goto out_putpage;
1552
1553	err = -EACCES;
1554	if (page_mapcount(page) > 1 && !migrate_all)
1555		goto out_putpage;
1556
1557	if (PageHuge(page)) {
1558		if (PageHead(page)) {
1559			isolate_huge_page(page, pagelist);
1560			err = 0;
1561		}
1562	} else {
1563		struct page *head;
1564
1565		head = compound_head(page);
1566		err = isolate_lru_page(head);
1567		if (err)
1568			goto out_putpage;
1569
1570		err = 0;
1571		list_add_tail(&head->lru, pagelist);
1572		mod_node_page_state(page_pgdat(head),
1573			NR_ISOLATED_ANON + page_is_file_cache(head),
1574			hpage_nr_pages(head));
1575	}
1576out_putpage:
1577	/*
1578	 * Either remove the duplicate refcount from
1579	 * isolate_lru_page() or drop the page ref if it was
1580	 * not isolated.
1581	 */
1582	put_page(page);
1583out:
1584	up_read(&mm->mmap_sem);
1585	return err;
1586}
1587
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1588/*
1589 * Migrate an array of page address onto an array of nodes and fill
1590 * the corresponding array of status.
1591 */
1592static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1593			 unsigned long nr_pages,
1594			 const void __user * __user *pages,
1595			 const int __user *nodes,
1596			 int __user *status, int flags)
1597{
1598	int current_node = NUMA_NO_NODE;
1599	LIST_HEAD(pagelist);
1600	int start, i;
1601	int err = 0, err1;
1602
1603	migrate_prep();
1604
1605	for (i = start = 0; i < nr_pages; i++) {
1606		const void __user *p;
1607		unsigned long addr;
1608		int node;
1609
1610		err = -EFAULT;
1611		if (get_user(p, pages + i))
1612			goto out_flush;
1613		if (get_user(node, nodes + i))
1614			goto out_flush;
1615		addr = (unsigned long)untagged_addr(p);
1616
1617		err = -ENODEV;
1618		if (node < 0 || node >= MAX_NUMNODES)
1619			goto out_flush;
1620		if (!node_state(node, N_MEMORY))
1621			goto out_flush;
1622
1623		err = -EACCES;
1624		if (!node_isset(node, task_nodes))
1625			goto out_flush;
1626
1627		if (current_node == NUMA_NO_NODE) {
1628			current_node = node;
1629			start = i;
1630		} else if (node != current_node) {
1631			err = do_move_pages_to_node(mm, &pagelist, current_node);
1632			if (err)
1633				goto out;
1634			err = store_status(status, start, current_node, i - start);
1635			if (err)
1636				goto out;
1637			start = i;
1638			current_node = node;
1639		}
1640
1641		/*
1642		 * Errors in the page lookup or isolation are not fatal and we simply
1643		 * report them via status
1644		 */
1645		err = add_page_for_migration(mm, addr, current_node,
1646				&pagelist, flags & MPOL_MF_MOVE_ALL);
1647		if (!err)
 
 
1648			continue;
 
1649
1650		err = store_status(status, i, err, 1);
 
 
 
 
1651		if (err)
1652			goto out_flush;
1653
1654		err = do_move_pages_to_node(mm, &pagelist, current_node);
 
1655		if (err)
1656			goto out;
1657		if (i > start) {
1658			err = store_status(status, start, current_node, i - start);
1659			if (err)
1660				goto out;
1661		}
1662		current_node = NUMA_NO_NODE;
1663	}
1664out_flush:
1665	if (list_empty(&pagelist))
1666		return err;
1667
1668	/* Make sure we do not overwrite the existing error */
1669	err1 = do_move_pages_to_node(mm, &pagelist, current_node);
1670	if (!err1)
1671		err1 = store_status(status, start, current_node, i - start);
1672	if (!err)
1673		err = err1;
1674out:
 
1675	return err;
1676}
1677
1678/*
1679 * Determine the nodes of an array of pages and store it in an array of status.
1680 */
1681static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1682				const void __user **pages, int *status)
1683{
1684	unsigned long i;
1685
1686	down_read(&mm->mmap_sem);
1687
1688	for (i = 0; i < nr_pages; i++) {
1689		unsigned long addr = (unsigned long)(*pages);
1690		struct vm_area_struct *vma;
1691		struct page *page;
1692		int err = -EFAULT;
1693
1694		vma = find_vma(mm, addr);
1695		if (!vma || addr < vma->vm_start)
1696			goto set_status;
1697
1698		/* FOLL_DUMP to ignore special (like zero) pages */
1699		page = follow_page(vma, addr, FOLL_DUMP);
1700
1701		err = PTR_ERR(page);
1702		if (IS_ERR(page))
1703			goto set_status;
1704
1705		err = page ? page_to_nid(page) : -ENOENT;
1706set_status:
1707		*status = err;
1708
1709		pages++;
1710		status++;
1711	}
1712
1713	up_read(&mm->mmap_sem);
1714}
1715
1716/*
1717 * Determine the nodes of a user array of pages and store it in
1718 * a user array of status.
1719 */
1720static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1721			 const void __user * __user *pages,
1722			 int __user *status)
1723{
1724#define DO_PAGES_STAT_CHUNK_NR 16
1725	const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1726	int chunk_status[DO_PAGES_STAT_CHUNK_NR];
1727
1728	while (nr_pages) {
1729		unsigned long chunk_nr;
1730
1731		chunk_nr = nr_pages;
1732		if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1733			chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1734
1735		if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1736			break;
1737
1738		do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1739
1740		if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1741			break;
1742
1743		pages += chunk_nr;
1744		status += chunk_nr;
1745		nr_pages -= chunk_nr;
1746	}
1747	return nr_pages ? -EFAULT : 0;
1748}
1749
1750/*
1751 * Move a list of pages in the address space of the currently executing
1752 * process.
1753 */
1754static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
1755			     const void __user * __user *pages,
1756			     const int __user *nodes,
1757			     int __user *status, int flags)
1758{
1759	struct task_struct *task;
1760	struct mm_struct *mm;
1761	int err;
1762	nodemask_t task_nodes;
1763
1764	/* Check flags */
1765	if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1766		return -EINVAL;
1767
1768	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1769		return -EPERM;
 
 
 
 
 
 
 
1770
1771	/* Find the mm_struct */
1772	rcu_read_lock();
1773	task = pid ? find_task_by_vpid(pid) : current;
1774	if (!task) {
1775		rcu_read_unlock();
1776		return -ESRCH;
1777	}
1778	get_task_struct(task);
1779
1780	/*
1781	 * Check if this process has the right to modify the specified
1782	 * process. Use the regular "ptrace_may_access()" checks.
1783	 */
1784	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1785		rcu_read_unlock();
1786		err = -EPERM;
1787		goto out;
1788	}
1789	rcu_read_unlock();
1790
1791 	err = security_task_movememory(task);
1792 	if (err)
1793		goto out;
1794
1795	task_nodes = cpuset_mems_allowed(task);
1796	mm = get_task_mm(task);
 
1797	put_task_struct(task);
1798
1799	if (!mm)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1800		return -EINVAL;
1801
 
 
 
 
 
 
 
1802	if (nodes)
1803		err = do_pages_move(mm, task_nodes, nr_pages, pages,
1804				    nodes, status, flags);
1805	else
1806		err = do_pages_stat(mm, nr_pages, pages, status);
1807
1808	mmput(mm);
1809	return err;
1810
1811out:
1812	put_task_struct(task);
1813	return err;
1814}
1815
1816SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1817		const void __user * __user *, pages,
1818		const int __user *, nodes,
1819		int __user *, status, int, flags)
1820{
1821	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1822}
1823
1824#ifdef CONFIG_COMPAT
1825COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
1826		       compat_uptr_t __user *, pages32,
1827		       const int __user *, nodes,
1828		       int __user *, status,
1829		       int, flags)
1830{
1831	const void __user * __user *pages;
1832	int i;
1833
1834	pages = compat_alloc_user_space(nr_pages * sizeof(void *));
1835	for (i = 0; i < nr_pages; i++) {
1836		compat_uptr_t p;
1837
1838		if (get_user(p, pages32 + i) ||
1839			put_user(compat_ptr(p), pages + i))
1840			return -EFAULT;
1841	}
1842	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1843}
1844#endif /* CONFIG_COMPAT */
1845
1846#ifdef CONFIG_NUMA_BALANCING
1847/*
1848 * Returns true if this is a safe migration target node for misplaced NUMA
1849 * pages. Currently it only checks the watermarks which crude
1850 */
1851static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1852				   unsigned long nr_migrate_pages)
1853{
1854	int z;
1855
1856	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1857		struct zone *zone = pgdat->node_zones + z;
1858
1859		if (!populated_zone(zone))
1860			continue;
1861
1862		/* Avoid waking kswapd by allocating pages_to_migrate pages. */
1863		if (!zone_watermark_ok(zone, 0,
1864				       high_wmark_pages(zone) +
1865				       nr_migrate_pages,
1866				       0, 0))
1867			continue;
1868		return true;
1869	}
1870	return false;
1871}
1872
1873static struct page *alloc_misplaced_dst_page(struct page *page,
1874					   unsigned long data)
1875{
1876	int nid = (int) data;
1877	struct page *newpage;
1878
1879	newpage = __alloc_pages_node(nid,
1880					 (GFP_HIGHUSER_MOVABLE |
1881					  __GFP_THISNODE | __GFP_NOMEMALLOC |
1882					  __GFP_NORETRY | __GFP_NOWARN) &
1883					 ~__GFP_RECLAIM, 0);
1884
1885	return newpage;
1886}
1887
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1888static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1889{
1890	int page_lru;
1891
1892	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
1893
 
 
 
 
1894	/* Avoid migrating to a node that is nearly full */
1895	if (!migrate_balanced_pgdat(pgdat, compound_nr(page)))
1896		return 0;
1897
1898	if (isolate_lru_page(page))
1899		return 0;
1900
1901	/*
1902	 * migrate_misplaced_transhuge_page() skips page migration's usual
1903	 * check on page_count(), so we must do it here, now that the page
1904	 * has been isolated: a GUP pin, or any other pin, prevents migration.
1905	 * The expected page count is 3: 1 for page's mapcount and 1 for the
1906	 * caller's pin and 1 for the reference taken by isolate_lru_page().
1907	 */
1908	if (PageTransHuge(page) && page_count(page) != 3) {
1909		putback_lru_page(page);
1910		return 0;
1911	}
1912
1913	page_lru = page_is_file_cache(page);
1914	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
1915				hpage_nr_pages(page));
1916
1917	/*
1918	 * Isolating the page has taken another reference, so the
1919	 * caller's reference can be safely dropped without the page
1920	 * disappearing underneath us during migration.
1921	 */
1922	put_page(page);
1923	return 1;
1924}
1925
1926bool pmd_trans_migrating(pmd_t pmd)
1927{
1928	struct page *page = pmd_page(pmd);
1929	return PageLocked(page);
1930}
1931
1932/*
1933 * Attempt to migrate a misplaced page to the specified destination
1934 * node. Caller is expected to have an elevated reference count on
1935 * the page that will be dropped by this function before returning.
1936 */
1937int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
1938			   int node)
1939{
1940	pg_data_t *pgdat = NODE_DATA(node);
1941	int isolated;
1942	int nr_remaining;
1943	LIST_HEAD(migratepages);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1944
1945	/*
1946	 * Don't migrate file pages that are mapped in multiple processes
1947	 * with execute permissions as they are probably shared libraries.
1948	 */
1949	if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
1950	    (vma->vm_flags & VM_EXEC))
1951		goto out;
1952
1953	/*
1954	 * Also do not migrate dirty pages as not all filesystems can move
1955	 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
1956	 */
1957	if (page_is_file_cache(page) && PageDirty(page))
1958		goto out;
1959
1960	isolated = numamigrate_isolate_page(pgdat, page);
1961	if (!isolated)
1962		goto out;
1963
1964	list_add(&page->lru, &migratepages);
1965	nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
1966				     NULL, node, MIGRATE_ASYNC,
1967				     MR_NUMA_MISPLACED);
1968	if (nr_remaining) {
1969		if (!list_empty(&migratepages)) {
1970			list_del(&page->lru);
1971			dec_node_page_state(page, NR_ISOLATED_ANON +
1972					page_is_file_cache(page));
1973			putback_lru_page(page);
1974		}
1975		isolated = 0;
1976	} else
1977		count_vm_numa_event(NUMA_PAGE_MIGRATE);
1978	BUG_ON(!list_empty(&migratepages));
1979	return isolated;
1980
1981out:
1982	put_page(page);
1983	return 0;
1984}
1985#endif /* CONFIG_NUMA_BALANCING */
1986
1987#if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1988/*
1989 * Migrates a THP to a given target node. page must be locked and is unlocked
1990 * before returning.
1991 */
1992int migrate_misplaced_transhuge_page(struct mm_struct *mm,
1993				struct vm_area_struct *vma,
1994				pmd_t *pmd, pmd_t entry,
1995				unsigned long address,
1996				struct page *page, int node)
1997{
1998	spinlock_t *ptl;
1999	pg_data_t *pgdat = NODE_DATA(node);
2000	int isolated = 0;
2001	struct page *new_page = NULL;
2002	int page_lru = page_is_file_cache(page);
2003	unsigned long start = address & HPAGE_PMD_MASK;
2004
2005	new_page = alloc_pages_node(node,
2006		(GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
2007		HPAGE_PMD_ORDER);
2008	if (!new_page)
2009		goto out_fail;
2010	prep_transhuge_page(new_page);
2011
2012	isolated = numamigrate_isolate_page(pgdat, page);
2013	if (!isolated) {
2014		put_page(new_page);
2015		goto out_fail;
2016	}
2017
2018	/* Prepare a page as a migration target */
2019	__SetPageLocked(new_page);
2020	if (PageSwapBacked(page))
2021		__SetPageSwapBacked(new_page);
2022
2023	/* anon mapping, we can simply copy page->mapping to the new page: */
2024	new_page->mapping = page->mapping;
2025	new_page->index = page->index;
2026	/* flush the cache before copying using the kernel virtual address */
2027	flush_cache_range(vma, start, start + HPAGE_PMD_SIZE);
2028	migrate_page_copy(new_page, page);
2029	WARN_ON(PageLRU(new_page));
2030
2031	/* Recheck the target PMD */
2032	ptl = pmd_lock(mm, pmd);
2033	if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) {
2034		spin_unlock(ptl);
2035
2036		/* Reverse changes made by migrate_page_copy() */
2037		if (TestClearPageActive(new_page))
2038			SetPageActive(page);
2039		if (TestClearPageUnevictable(new_page))
2040			SetPageUnevictable(page);
2041
2042		unlock_page(new_page);
2043		put_page(new_page);		/* Free it */
2044
2045		/* Retake the callers reference and putback on LRU */
2046		get_page(page);
2047		putback_lru_page(page);
2048		mod_node_page_state(page_pgdat(page),
2049			 NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
2050
2051		goto out_unlock;
2052	}
2053
2054	entry = mk_huge_pmd(new_page, vma->vm_page_prot);
2055	entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
2056
2057	/*
2058	 * Overwrite the old entry under pagetable lock and establish
2059	 * the new PTE. Any parallel GUP will either observe the old
2060	 * page blocking on the page lock, block on the page table
2061	 * lock or observe the new page. The SetPageUptodate on the
2062	 * new page and page_add_new_anon_rmap guarantee the copy is
2063	 * visible before the pagetable update.
2064	 */
2065	page_add_anon_rmap(new_page, vma, start, true);
2066	/*
2067	 * At this point the pmd is numa/protnone (i.e. non present) and the TLB
2068	 * has already been flushed globally.  So no TLB can be currently
2069	 * caching this non present pmd mapping.  There's no need to clear the
2070	 * pmd before doing set_pmd_at(), nor to flush the TLB after
2071	 * set_pmd_at().  Clearing the pmd here would introduce a race
2072	 * condition against MADV_DONTNEED, because MADV_DONTNEED only holds the
2073	 * mmap_sem for reading.  If the pmd is set to NULL at any given time,
2074	 * MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this
2075	 * pmd.
2076	 */
2077	set_pmd_at(mm, start, pmd, entry);
2078	update_mmu_cache_pmd(vma, address, &entry);
2079
2080	page_ref_unfreeze(page, 2);
2081	mlock_migrate_page(new_page, page);
2082	page_remove_rmap(page, true);
2083	set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
2084
2085	spin_unlock(ptl);
2086
2087	/* Take an "isolate" reference and put new page on the LRU. */
2088	get_page(new_page);
2089	putback_lru_page(new_page);
2090
2091	unlock_page(new_page);
2092	unlock_page(page);
2093	put_page(page);			/* Drop the rmap reference */
2094	put_page(page);			/* Drop the LRU isolation reference */
2095
2096	count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
2097	count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
2098
2099	mod_node_page_state(page_pgdat(page),
2100			NR_ISOLATED_ANON + page_lru,
2101			-HPAGE_PMD_NR);
2102	return isolated;
2103
2104out_fail:
2105	count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
2106	ptl = pmd_lock(mm, pmd);
2107	if (pmd_same(*pmd, entry)) {
2108		entry = pmd_modify(entry, vma->vm_page_prot);
2109		set_pmd_at(mm, start, pmd, entry);
2110		update_mmu_cache_pmd(vma, address, &entry);
2111	}
2112	spin_unlock(ptl);
2113
2114out_unlock:
2115	unlock_page(page);
2116	put_page(page);
2117	return 0;
2118}
2119#endif /* CONFIG_NUMA_BALANCING */
2120
2121#endif /* CONFIG_NUMA */
2122
2123#ifdef CONFIG_DEVICE_PRIVATE
2124static int migrate_vma_collect_hole(unsigned long start,
2125				    unsigned long end,
2126				    struct mm_walk *walk)
2127{
2128	struct migrate_vma *migrate = walk->private;
2129	unsigned long addr;
2130
2131	for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2132		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2133		migrate->dst[migrate->npages] = 0;
2134		migrate->npages++;
2135		migrate->cpages++;
2136	}
2137
2138	return 0;
2139}
2140
2141static int migrate_vma_collect_skip(unsigned long start,
2142				    unsigned long end,
 
2143				    struct mm_walk *walk)
2144{
2145	struct migrate_vma *migrate = walk->private;
2146	unsigned long addr;
2147
2148	for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
 
 
 
 
 
2149		migrate->dst[migrate->npages] = 0;
2150		migrate->src[migrate->npages++] = 0;
 
2151	}
2152
2153	return 0;
2154}
2155
2156static int migrate_vma_collect_pmd(pmd_t *pmdp,
2157				   unsigned long start,
2158				   unsigned long end,
2159				   struct mm_walk *walk)
2160{
2161	struct migrate_vma *migrate = walk->private;
2162	struct vm_area_struct *vma = walk->vma;
2163	struct mm_struct *mm = vma->vm_mm;
2164	unsigned long addr = start, unmapped = 0;
2165	spinlock_t *ptl;
2166	pte_t *ptep;
2167
2168again:
2169	if (pmd_none(*pmdp))
2170		return migrate_vma_collect_hole(start, end, walk);
2171
2172	if (pmd_trans_huge(*pmdp)) {
2173		struct page *page;
2174
2175		ptl = pmd_lock(mm, pmdp);
2176		if (unlikely(!pmd_trans_huge(*pmdp))) {
2177			spin_unlock(ptl);
2178			goto again;
2179		}
2180
2181		page = pmd_page(*pmdp);
2182		if (is_huge_zero_page(page)) {
2183			spin_unlock(ptl);
2184			split_huge_pmd(vma, pmdp, addr);
2185			if (pmd_trans_unstable(pmdp))
2186				return migrate_vma_collect_skip(start, end,
2187								walk);
2188		} else {
2189			int ret;
2190
2191			get_page(page);
2192			spin_unlock(ptl);
2193			if (unlikely(!trylock_page(page)))
2194				return migrate_vma_collect_skip(start, end,
2195								walk);
2196			ret = split_huge_page(page);
2197			unlock_page(page);
2198			put_page(page);
2199			if (ret)
2200				return migrate_vma_collect_skip(start, end,
2201								walk);
2202			if (pmd_none(*pmdp))
2203				return migrate_vma_collect_hole(start, end,
2204								walk);
2205		}
2206	}
2207
2208	if (unlikely(pmd_bad(*pmdp)))
2209		return migrate_vma_collect_skip(start, end, walk);
2210
2211	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2212	arch_enter_lazy_mmu_mode();
2213
2214	for (; addr < end; addr += PAGE_SIZE, ptep++) {
2215		unsigned long mpfn, pfn;
2216		struct page *page;
2217		swp_entry_t entry;
2218		pte_t pte;
2219
2220		pte = *ptep;
2221
2222		if (pte_none(pte)) {
2223			mpfn = MIGRATE_PFN_MIGRATE;
2224			migrate->cpages++;
 
 
2225			goto next;
2226		}
2227
2228		if (!pte_present(pte)) {
2229			mpfn = 0;
2230
2231			/*
2232			 * Only care about unaddressable device page special
2233			 * page table entry. Other special swap entries are not
2234			 * migratable, and we ignore regular swapped page.
2235			 */
2236			entry = pte_to_swp_entry(pte);
2237			if (!is_device_private_entry(entry))
2238				goto next;
2239
2240			page = device_private_entry_to_page(entry);
 
 
 
 
 
2241			mpfn = migrate_pfn(page_to_pfn(page)) |
2242					MIGRATE_PFN_MIGRATE;
2243			if (is_write_device_private_entry(entry))
2244				mpfn |= MIGRATE_PFN_WRITE;
2245		} else {
 
 
2246			pfn = pte_pfn(pte);
2247			if (is_zero_pfn(pfn)) {
2248				mpfn = MIGRATE_PFN_MIGRATE;
2249				migrate->cpages++;
2250				goto next;
2251			}
2252			page = vm_normal_page(migrate->vma, addr, pte);
2253			mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
2254			mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2255		}
2256
2257		/* FIXME support THP */
2258		if (!page || !page->mapping || PageTransCompound(page)) {
2259			mpfn = 0;
2260			goto next;
2261		}
2262
2263		/*
2264		 * By getting a reference on the page we pin it and that blocks
2265		 * any kind of migration. Side effect is that it "freezes" the
2266		 * pte.
2267		 *
2268		 * We drop this reference after isolating the page from the lru
2269		 * for non device page (device page are not on the lru and thus
2270		 * can't be dropped from it).
2271		 */
2272		get_page(page);
2273		migrate->cpages++;
2274
2275		/*
2276		 * Optimize for the common case where page is only mapped once
2277		 * in one process. If we can lock the page, then we can safely
2278		 * set up a special migration page table entry now.
2279		 */
2280		if (trylock_page(page)) {
2281			pte_t swp_pte;
2282
2283			mpfn |= MIGRATE_PFN_LOCKED;
2284			ptep_get_and_clear(mm, addr, ptep);
2285
2286			/* Setup special migration page table entry */
2287			entry = make_migration_entry(page, mpfn &
2288						     MIGRATE_PFN_WRITE);
 
 
 
 
2289			swp_pte = swp_entry_to_pte(entry);
2290			if (pte_soft_dirty(pte))
2291				swp_pte = pte_swp_mksoft_dirty(swp_pte);
 
 
 
 
 
 
 
 
 
2292			set_pte_at(mm, addr, ptep, swp_pte);
2293
2294			/*
2295			 * This is like regular unmap: we remove the rmap and
2296			 * drop page refcount. Page won't be freed, as we took
2297			 * a reference just above.
2298			 */
2299			page_remove_rmap(page, false);
2300			put_page(page);
2301
2302			if (pte_present(pte))
2303				unmapped++;
2304		}
2305
2306next:
2307		migrate->dst[migrate->npages] = 0;
2308		migrate->src[migrate->npages++] = mpfn;
2309	}
2310	arch_leave_lazy_mmu_mode();
2311	pte_unmap_unlock(ptep - 1, ptl);
2312
2313	/* Only flush the TLB if we actually modified any entries */
2314	if (unmapped)
2315		flush_tlb_range(walk->vma, start, end);
2316
2317	return 0;
2318}
2319
2320static const struct mm_walk_ops migrate_vma_walk_ops = {
2321	.pmd_entry		= migrate_vma_collect_pmd,
2322	.pte_hole		= migrate_vma_collect_hole,
2323};
2324
2325/*
2326 * migrate_vma_collect() - collect pages over a range of virtual addresses
2327 * @migrate: migrate struct containing all migration information
2328 *
2329 * This will walk the CPU page table. For each virtual address backed by a
2330 * valid page, it updates the src array and takes a reference on the page, in
2331 * order to pin the page until we lock it and unmap it.
2332 */
2333static void migrate_vma_collect(struct migrate_vma *migrate)
2334{
2335	struct mmu_notifier_range range;
2336
2337	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL,
2338			migrate->vma->vm_mm, migrate->start, migrate->end);
 
 
 
 
 
 
2339	mmu_notifier_invalidate_range_start(&range);
2340
2341	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
2342			&migrate_vma_walk_ops, migrate);
2343
2344	mmu_notifier_invalidate_range_end(&range);
2345	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2346}
2347
2348/*
2349 * migrate_vma_check_page() - check if page is pinned or not
2350 * @page: struct page to check
2351 *
2352 * Pinned pages cannot be migrated. This is the same test as in
2353 * migrate_page_move_mapping(), except that here we allow migration of a
2354 * ZONE_DEVICE page.
2355 */
2356static bool migrate_vma_check_page(struct page *page)
2357{
2358	/*
2359	 * One extra ref because caller holds an extra reference, either from
2360	 * isolate_lru_page() for a regular page, or migrate_vma_collect() for
2361	 * a device page.
2362	 */
2363	int extra = 1;
2364
2365	/*
2366	 * FIXME support THP (transparent huge page), it is bit more complex to
2367	 * check them than regular pages, because they can be mapped with a pmd
2368	 * or with a pte (split pte mapping).
2369	 */
2370	if (PageCompound(page))
2371		return false;
2372
2373	/* Page from ZONE_DEVICE have one extra reference */
2374	if (is_zone_device_page(page)) {
2375		/*
2376		 * Private page can never be pin as they have no valid pte and
2377		 * GUP will fail for those. Yet if there is a pending migration
2378		 * a thread might try to wait on the pte migration entry and
2379		 * will bump the page reference count. Sadly there is no way to
2380		 * differentiate a regular pin from migration wait. Hence to
2381		 * avoid 2 racing thread trying to migrate back to CPU to enter
2382		 * infinite loop (one stoping migration because the other is
2383		 * waiting on pte migration entry). We always return true here.
2384		 *
2385		 * FIXME proper solution is to rework migration_entry_wait() so
2386		 * it does not need to take a reference on page.
2387		 */
2388		return is_device_private_page(page);
2389	}
2390
2391	/* For file back page */
2392	if (page_mapping(page))
2393		extra += 1 + page_has_private(page);
2394
2395	if ((page_count(page) - extra) > page_mapcount(page))
2396		return false;
2397
2398	return true;
2399}
2400
2401/*
2402 * migrate_vma_prepare() - lock pages and isolate them from the lru
2403 * @migrate: migrate struct containing all migration information
2404 *
2405 * This locks pages that have been collected by migrate_vma_collect(). Once each
2406 * page is locked it is isolated from the lru (for non-device pages). Finally,
2407 * the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
2408 * migrated by concurrent kernel threads.
2409 */
2410static void migrate_vma_prepare(struct migrate_vma *migrate)
2411{
2412	const unsigned long npages = migrate->npages;
2413	const unsigned long start = migrate->start;
2414	unsigned long addr, i, restore = 0;
2415	bool allow_drain = true;
2416
2417	lru_add_drain();
2418
2419	for (i = 0; (i < npages) && migrate->cpages; i++) {
2420		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2421		bool remap = true;
2422
2423		if (!page)
2424			continue;
2425
2426		if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
2427			/*
2428			 * Because we are migrating several pages there can be
2429			 * a deadlock between 2 concurrent migration where each
2430			 * are waiting on each other page lock.
2431			 *
2432			 * Make migrate_vma() a best effort thing and backoff
2433			 * for any page we can not lock right away.
2434			 */
2435			if (!trylock_page(page)) {
2436				migrate->src[i] = 0;
2437				migrate->cpages--;
2438				put_page(page);
2439				continue;
2440			}
2441			remap = false;
2442			migrate->src[i] |= MIGRATE_PFN_LOCKED;
2443		}
2444
2445		/* ZONE_DEVICE pages are not on LRU */
2446		if (!is_zone_device_page(page)) {
2447			if (!PageLRU(page) && allow_drain) {
2448				/* Drain CPU's pagevec */
2449				lru_add_drain_all();
2450				allow_drain = false;
2451			}
2452
2453			if (isolate_lru_page(page)) {
2454				if (remap) {
2455					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2456					migrate->cpages--;
2457					restore++;
2458				} else {
2459					migrate->src[i] = 0;
2460					unlock_page(page);
2461					migrate->cpages--;
2462					put_page(page);
2463				}
2464				continue;
2465			}
2466
2467			/* Drop the reference we took in collect */
2468			put_page(page);
2469		}
2470
2471		if (!migrate_vma_check_page(page)) {
2472			if (remap) {
2473				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2474				migrate->cpages--;
2475				restore++;
2476
2477				if (!is_zone_device_page(page)) {
2478					get_page(page);
2479					putback_lru_page(page);
2480				}
2481			} else {
2482				migrate->src[i] = 0;
2483				unlock_page(page);
2484				migrate->cpages--;
2485
2486				if (!is_zone_device_page(page))
2487					putback_lru_page(page);
2488				else
2489					put_page(page);
2490			}
2491		}
2492	}
2493
2494	for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
2495		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2496
2497		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2498			continue;
2499
2500		remove_migration_pte(page, migrate->vma, addr, page);
2501
2502		migrate->src[i] = 0;
2503		unlock_page(page);
2504		put_page(page);
2505		restore--;
2506	}
2507}
2508
2509/*
2510 * migrate_vma_unmap() - replace page mapping with special migration pte entry
2511 * @migrate: migrate struct containing all migration information
2512 *
2513 * Replace page mapping (CPU page table pte) with a special migration pte entry
2514 * and check again if it has been pinned. Pinned pages are restored because we
2515 * cannot migrate them.
2516 *
2517 * This is the last step before we call the device driver callback to allocate
2518 * destination memory and copy contents of original page over to new page.
2519 */
2520static void migrate_vma_unmap(struct migrate_vma *migrate)
2521{
2522	int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
2523	const unsigned long npages = migrate->npages;
2524	const unsigned long start = migrate->start;
2525	unsigned long addr, i, restore = 0;
2526
2527	for (i = 0; i < npages; i++) {
2528		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2529
2530		if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2531			continue;
2532
2533		if (page_mapped(page)) {
2534			try_to_unmap(page, flags);
2535			if (page_mapped(page))
2536				goto restore;
2537		}
2538
2539		if (migrate_vma_check_page(page))
2540			continue;
2541
2542restore:
2543		migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2544		migrate->cpages--;
2545		restore++;
2546	}
2547
2548	for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
2549		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2550
2551		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2552			continue;
2553
2554		remove_migration_ptes(page, page, false);
2555
2556		migrate->src[i] = 0;
2557		unlock_page(page);
2558		restore--;
2559
2560		if (is_zone_device_page(page))
2561			put_page(page);
2562		else
2563			putback_lru_page(page);
2564	}
2565}
2566
2567/**
2568 * migrate_vma_setup() - prepare to migrate a range of memory
2569 * @args: contains the vma, start, and and pfns arrays for the migration
2570 *
2571 * Returns: negative errno on failures, 0 when 0 or more pages were migrated
2572 * without an error.
2573 *
2574 * Prepare to migrate a range of memory virtual address range by collecting all
2575 * the pages backing each virtual address in the range, saving them inside the
2576 * src array.  Then lock those pages and unmap them. Once the pages are locked
2577 * and unmapped, check whether each page is pinned or not.  Pages that aren't
2578 * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
2579 * corresponding src array entry.  Then restores any pages that are pinned, by
2580 * remapping and unlocking those pages.
2581 *
2582 * The caller should then allocate destination memory and copy source memory to
2583 * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
2584 * flag set).  Once these are allocated and copied, the caller must update each
2585 * corresponding entry in the dst array with the pfn value of the destination
2586 * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
2587 * (destination pages must have their struct pages locked, via lock_page()).
2588 *
2589 * Note that the caller does not have to migrate all the pages that are marked
2590 * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
2591 * device memory to system memory.  If the caller cannot migrate a device page
2592 * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
2593 * consequences for the userspace process, so it must be avoided if at all
2594 * possible.
2595 *
2596 * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
2597 * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
2598 * allowing the caller to allocate device memory for those unback virtual
2599 * address.  For this the caller simply has to allocate device memory and
2600 * properly set the destination entry like for regular migration.  Note that
2601 * this can still fails and thus inside the device driver must check if the
2602 * migration was successful for those entries after calling migrate_vma_pages()
2603 * just like for regular migration.
2604 *
2605 * After that, the callers must call migrate_vma_pages() to go over each entry
2606 * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
2607 * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
2608 * then migrate_vma_pages() to migrate struct page information from the source
2609 * struct page to the destination struct page.  If it fails to migrate the
2610 * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
2611 * src array.
2612 *
2613 * At this point all successfully migrated pages have an entry in the src
2614 * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
2615 * array entry with MIGRATE_PFN_VALID flag set.
2616 *
2617 * Once migrate_vma_pages() returns the caller may inspect which pages were
2618 * successfully migrated, and which were not.  Successfully migrated pages will
2619 * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
2620 *
2621 * It is safe to update device page table after migrate_vma_pages() because
2622 * both destination and source page are still locked, and the mmap_sem is held
2623 * in read mode (hence no one can unmap the range being migrated).
2624 *
2625 * Once the caller is done cleaning up things and updating its page table (if it
2626 * chose to do so, this is not an obligation) it finally calls
2627 * migrate_vma_finalize() to update the CPU page table to point to new pages
2628 * for successfully migrated pages or otherwise restore the CPU page table to
2629 * point to the original source pages.
2630 */
2631int migrate_vma_setup(struct migrate_vma *args)
2632{
2633	long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
2634
2635	args->start &= PAGE_MASK;
2636	args->end &= PAGE_MASK;
2637	if (!args->vma || is_vm_hugetlb_page(args->vma) ||
2638	    (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
2639		return -EINVAL;
2640	if (nr_pages <= 0)
2641		return -EINVAL;
2642	if (args->start < args->vma->vm_start ||
2643	    args->start >= args->vma->vm_end)
2644		return -EINVAL;
2645	if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
2646		return -EINVAL;
2647	if (!args->src || !args->dst)
2648		return -EINVAL;
2649
2650	memset(args->src, 0, sizeof(*args->src) * nr_pages);
2651	args->cpages = 0;
2652	args->npages = 0;
2653
2654	migrate_vma_collect(args);
2655
2656	if (args->cpages)
2657		migrate_vma_prepare(args);
2658	if (args->cpages)
2659		migrate_vma_unmap(args);
2660
2661	/*
2662	 * At this point pages are locked and unmapped, and thus they have
2663	 * stable content and can safely be copied to destination memory that
2664	 * is allocated by the drivers.
2665	 */
2666	return 0;
2667
2668}
2669EXPORT_SYMBOL(migrate_vma_setup);
2670
 
 
 
 
 
 
 
 
2671static void migrate_vma_insert_page(struct migrate_vma *migrate,
2672				    unsigned long addr,
2673				    struct page *page,
2674				    unsigned long *src,
2675				    unsigned long *dst)
2676{
2677	struct vm_area_struct *vma = migrate->vma;
2678	struct mm_struct *mm = vma->vm_mm;
2679	struct mem_cgroup *memcg;
2680	bool flush = false;
2681	spinlock_t *ptl;
2682	pte_t entry;
2683	pgd_t *pgdp;
2684	p4d_t *p4dp;
2685	pud_t *pudp;
2686	pmd_t *pmdp;
2687	pte_t *ptep;
2688
2689	/* Only allow populating anonymous memory */
2690	if (!vma_is_anonymous(vma))
2691		goto abort;
2692
2693	pgdp = pgd_offset(mm, addr);
2694	p4dp = p4d_alloc(mm, pgdp, addr);
2695	if (!p4dp)
2696		goto abort;
2697	pudp = pud_alloc(mm, p4dp, addr);
2698	if (!pudp)
2699		goto abort;
2700	pmdp = pmd_alloc(mm, pudp, addr);
2701	if (!pmdp)
2702		goto abort;
2703
2704	if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
2705		goto abort;
2706
2707	/*
2708	 * Use pte_alloc() instead of pte_alloc_map().  We can't run
2709	 * pte_offset_map() on pmds where a huge pmd might be created
2710	 * from a different thread.
2711	 *
2712	 * pte_alloc_map() is safe to use under down_write(mmap_sem) or when
2713	 * parallel threads are excluded by other means.
2714	 *
2715	 * Here we only have down_read(mmap_sem).
2716	 */
2717	if (pte_alloc(mm, pmdp))
2718		goto abort;
2719
2720	/* See the comment in pte_alloc_one_map() */
2721	if (unlikely(pmd_trans_unstable(pmdp)))
2722		goto abort;
2723
2724	if (unlikely(anon_vma_prepare(vma)))
2725		goto abort;
2726	if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
2727		goto abort;
2728
2729	/*
2730	 * The memory barrier inside __SetPageUptodate makes sure that
2731	 * preceding stores to the page contents become visible before
2732	 * the set_pte_at() write.
2733	 */
2734	__SetPageUptodate(page);
2735
2736	if (is_zone_device_page(page)) {
2737		if (is_device_private_page(page)) {
2738			swp_entry_t swp_entry;
2739
2740			swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
 
 
 
 
 
2741			entry = swp_entry_to_pte(swp_entry);
 
 
 
 
 
 
 
2742		}
2743	} else {
2744		entry = mk_pte(page, vma->vm_page_prot);
2745		if (vma->vm_flags & VM_WRITE)
2746			entry = pte_mkwrite(pte_mkdirty(entry));
2747	}
2748
2749	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2750
 
 
 
2751	if (pte_present(*ptep)) {
2752		unsigned long pfn = pte_pfn(*ptep);
2753
2754		if (!is_zero_pfn(pfn)) {
2755			pte_unmap_unlock(ptep, ptl);
2756			mem_cgroup_cancel_charge(page, memcg, false);
2757			goto abort;
2758		}
2759		flush = true;
2760	} else if (!pte_none(*ptep)) {
2761		pte_unmap_unlock(ptep, ptl);
2762		mem_cgroup_cancel_charge(page, memcg, false);
2763		goto abort;
2764	}
2765
2766	/*
2767	 * Check for usefaultfd but do not deliver the fault. Instead,
2768	 * just back off.
2769	 */
2770	if (userfaultfd_missing(vma)) {
2771		pte_unmap_unlock(ptep, ptl);
2772		mem_cgroup_cancel_charge(page, memcg, false);
2773		goto abort;
2774	}
2775
2776	inc_mm_counter(mm, MM_ANONPAGES);
2777	page_add_new_anon_rmap(page, vma, addr, false);
2778	mem_cgroup_commit_charge(page, memcg, false, false);
2779	if (!is_zone_device_page(page))
2780		lru_cache_add_active_or_unevictable(page, vma);
2781	get_page(page);
2782
2783	if (flush) {
2784		flush_cache_page(vma, addr, pte_pfn(*ptep));
2785		ptep_clear_flush_notify(vma, addr, ptep);
2786		set_pte_at_notify(mm, addr, ptep, entry);
2787		update_mmu_cache(vma, addr, ptep);
2788	} else {
2789		/* No need to invalidate - it was non-present before */
2790		set_pte_at(mm, addr, ptep, entry);
2791		update_mmu_cache(vma, addr, ptep);
2792	}
2793
2794	pte_unmap_unlock(ptep, ptl);
2795	*src = MIGRATE_PFN_MIGRATE;
2796	return;
2797
 
 
2798abort:
2799	*src &= ~MIGRATE_PFN_MIGRATE;
2800}
2801
2802/**
2803 * migrate_vma_pages() - migrate meta-data from src page to dst page
2804 * @migrate: migrate struct containing all migration information
2805 *
2806 * This migrates struct page meta-data from source struct page to destination
2807 * struct page. This effectively finishes the migration from source page to the
2808 * destination page.
2809 */
2810void migrate_vma_pages(struct migrate_vma *migrate)
2811{
2812	const unsigned long npages = migrate->npages;
2813	const unsigned long start = migrate->start;
2814	struct mmu_notifier_range range;
2815	unsigned long addr, i;
2816	bool notified = false;
2817
2818	for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2819		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2820		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2821		struct address_space *mapping;
2822		int r;
2823
2824		if (!newpage) {
2825			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2826			continue;
2827		}
2828
2829		if (!page) {
2830			if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
2831				continue;
2832			}
2833			if (!notified) {
2834				notified = true;
2835
2836				mmu_notifier_range_init(&range,
2837							MMU_NOTIFY_CLEAR, 0,
2838							NULL,
2839							migrate->vma->vm_mm,
2840							addr, migrate->end);
2841				mmu_notifier_invalidate_range_start(&range);
2842			}
2843			migrate_vma_insert_page(migrate, addr, newpage,
2844						&migrate->src[i],
2845						&migrate->dst[i]);
2846			continue;
2847		}
2848
2849		mapping = page_mapping(page);
2850
2851		if (is_zone_device_page(newpage)) {
2852			if (is_device_private_page(newpage)) {
2853				/*
2854				 * For now only support private anonymous when
2855				 * migrating to un-addressable device memory.
2856				 */
2857				if (mapping) {
2858					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2859					continue;
2860				}
2861			} else {
2862				/*
2863				 * Other types of ZONE_DEVICE page are not
2864				 * supported.
2865				 */
2866				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2867				continue;
2868			}
2869		}
2870
2871		r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2872		if (r != MIGRATEPAGE_SUCCESS)
2873			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2874	}
2875
2876	/*
2877	 * No need to double call mmu_notifier->invalidate_range() callback as
2878	 * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
2879	 * did already call it.
2880	 */
2881	if (notified)
2882		mmu_notifier_invalidate_range_only_end(&range);
2883}
2884EXPORT_SYMBOL(migrate_vma_pages);
2885
2886/**
2887 * migrate_vma_finalize() - restore CPU page table entry
2888 * @migrate: migrate struct containing all migration information
2889 *
2890 * This replaces the special migration pte entry with either a mapping to the
2891 * new page if migration was successful for that page, or to the original page
2892 * otherwise.
2893 *
2894 * This also unlocks the pages and puts them back on the lru, or drops the extra
2895 * refcount, for device pages.
2896 */
2897void migrate_vma_finalize(struct migrate_vma *migrate)
2898{
2899	const unsigned long npages = migrate->npages;
2900	unsigned long i;
2901
2902	for (i = 0; i < npages; i++) {
2903		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2904		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2905
2906		if (!page) {
2907			if (newpage) {
2908				unlock_page(newpage);
2909				put_page(newpage);
2910			}
2911			continue;
2912		}
2913
2914		if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
2915			if (newpage) {
2916				unlock_page(newpage);
2917				put_page(newpage);
2918			}
2919			newpage = page;
2920		}
2921
2922		remove_migration_ptes(page, newpage, false);
2923		unlock_page(page);
2924		migrate->cpages--;
2925
2926		if (is_zone_device_page(page))
2927			put_page(page);
2928		else
2929			putback_lru_page(page);
2930
2931		if (newpage != page) {
2932			unlock_page(newpage);
2933			if (is_zone_device_page(newpage))
2934				put_page(newpage);
2935			else
2936				putback_lru_page(newpage);
2937		}
2938	}
2939}
2940EXPORT_SYMBOL(migrate_vma_finalize);
2941#endif /* CONFIG_DEVICE_PRIVATE */
v5.14.15
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Memory Migration functionality - linux/mm/migrate.c
   4 *
   5 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
   6 *
   7 * Page migration was first developed in the context of the memory hotplug
   8 * project. The main authors of the migration code are:
   9 *
  10 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  11 * Hirokazu Takahashi <taka@valinux.co.jp>
  12 * Dave Hansen <haveblue@us.ibm.com>
  13 * Christoph Lameter
  14 */
  15
  16#include <linux/migrate.h>
  17#include <linux/export.h>
  18#include <linux/swap.h>
  19#include <linux/swapops.h>
  20#include <linux/pagemap.h>
  21#include <linux/buffer_head.h>
  22#include <linux/mm_inline.h>
  23#include <linux/nsproxy.h>
  24#include <linux/pagevec.h>
  25#include <linux/ksm.h>
  26#include <linux/rmap.h>
  27#include <linux/topology.h>
  28#include <linux/cpu.h>
  29#include <linux/cpuset.h>
  30#include <linux/writeback.h>
  31#include <linux/mempolicy.h>
  32#include <linux/vmalloc.h>
  33#include <linux/security.h>
  34#include <linux/backing-dev.h>
  35#include <linux/compaction.h>
  36#include <linux/syscalls.h>
  37#include <linux/compat.h>
  38#include <linux/hugetlb.h>
  39#include <linux/hugetlb_cgroup.h>
  40#include <linux/gfp.h>
  41#include <linux/pagewalk.h>
  42#include <linux/pfn_t.h>
  43#include <linux/memremap.h>
  44#include <linux/userfaultfd_k.h>
  45#include <linux/balloon_compaction.h>
  46#include <linux/mmu_notifier.h>
  47#include <linux/page_idle.h>
  48#include <linux/page_owner.h>
  49#include <linux/sched/mm.h>
  50#include <linux/ptrace.h>
  51#include <linux/oom.h>
  52
  53#include <asm/tlbflush.h>
  54
  55#define CREATE_TRACE_POINTS
  56#include <trace/events/migrate.h>
  57
  58#include "internal.h"
  59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  60int isolate_movable_page(struct page *page, isolate_mode_t mode)
  61{
  62	struct address_space *mapping;
  63
  64	/*
  65	 * Avoid burning cycles with pages that are yet under __free_pages(),
  66	 * or just got freed under us.
  67	 *
  68	 * In case we 'win' a race for a movable page being freed under us and
  69	 * raise its refcount preventing __free_pages() from doing its job
  70	 * the put_page() at the end of this block will take care of
  71	 * release this page, thus avoiding a nasty leakage.
  72	 */
  73	if (unlikely(!get_page_unless_zero(page)))
  74		goto out;
  75
  76	/*
  77	 * Check PageMovable before holding a PG_lock because page's owner
  78	 * assumes anybody doesn't touch PG_lock of newly allocated page
  79	 * so unconditionally grabbing the lock ruins page's owner side.
  80	 */
  81	if (unlikely(!__PageMovable(page)))
  82		goto out_putpage;
  83	/*
  84	 * As movable pages are not isolated from LRU lists, concurrent
  85	 * compaction threads can race against page migration functions
  86	 * as well as race against the releasing a page.
  87	 *
  88	 * In order to avoid having an already isolated movable page
  89	 * being (wrongly) re-isolated while it is under migration,
  90	 * or to avoid attempting to isolate pages being released,
  91	 * lets be sure we have the page lock
  92	 * before proceeding with the movable page isolation steps.
  93	 */
  94	if (unlikely(!trylock_page(page)))
  95		goto out_putpage;
  96
  97	if (!PageMovable(page) || PageIsolated(page))
  98		goto out_no_isolated;
  99
 100	mapping = page_mapping(page);
 101	VM_BUG_ON_PAGE(!mapping, page);
 102
 103	if (!mapping->a_ops->isolate_page(page, mode))
 104		goto out_no_isolated;
 105
 106	/* Driver shouldn't use PG_isolated bit of page->flags */
 107	WARN_ON_ONCE(PageIsolated(page));
 108	__SetPageIsolated(page);
 109	unlock_page(page);
 110
 111	return 0;
 112
 113out_no_isolated:
 114	unlock_page(page);
 115out_putpage:
 116	put_page(page);
 117out:
 118	return -EBUSY;
 119}
 120
 121static void putback_movable_page(struct page *page)
 
 122{
 123	struct address_space *mapping;
 124
 
 
 
 
 125	mapping = page_mapping(page);
 126	mapping->a_ops->putback_page(page);
 127	__ClearPageIsolated(page);
 128}
 129
 130/*
 131 * Put previously isolated pages back onto the appropriate lists
 132 * from where they were once taken off for compaction/migration.
 133 *
 134 * This function shall be used whenever the isolated pageset has been
 135 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
 136 * and isolate_huge_page().
 137 */
 138void putback_movable_pages(struct list_head *l)
 139{
 140	struct page *page;
 141	struct page *page2;
 142
 143	list_for_each_entry_safe(page, page2, l, lru) {
 144		if (unlikely(PageHuge(page))) {
 145			putback_active_hugepage(page);
 146			continue;
 147		}
 148		list_del(&page->lru);
 149		/*
 150		 * We isolated non-lru movable page so here we can use
 151		 * __PageMovable because LRU page's mapping cannot have
 152		 * PAGE_MAPPING_MOVABLE.
 153		 */
 154		if (unlikely(__PageMovable(page))) {
 155			VM_BUG_ON_PAGE(!PageIsolated(page), page);
 156			lock_page(page);
 157			if (PageMovable(page))
 158				putback_movable_page(page);
 159			else
 160				__ClearPageIsolated(page);
 161			unlock_page(page);
 162			put_page(page);
 163		} else {
 164			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
 165					page_is_file_lru(page), -thp_nr_pages(page));
 166			putback_lru_page(page);
 167		}
 168	}
 169}
 170
 171/*
 172 * Restore a potential migration pte to a working pte entry
 173 */
 174static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
 175				 unsigned long addr, void *old)
 176{
 177	struct page_vma_mapped_walk pvmw = {
 178		.page = old,
 179		.vma = vma,
 180		.address = addr,
 181		.flags = PVMW_SYNC | PVMW_MIGRATION,
 182	};
 183	struct page *new;
 184	pte_t pte;
 185	swp_entry_t entry;
 186
 187	VM_BUG_ON_PAGE(PageTail(page), page);
 188	while (page_vma_mapped_walk(&pvmw)) {
 189		if (PageKsm(page))
 190			new = page;
 191		else
 192			new = page - pvmw.page->index +
 193				linear_page_index(vma, pvmw.address);
 194
 195#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 196		/* PMD-mapped THP migration entry */
 197		if (!pvmw.pte) {
 198			VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
 199			remove_migration_pmd(&pvmw, new);
 200			continue;
 201		}
 202#endif
 203
 204		get_page(new);
 205		pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
 206		if (pte_swp_soft_dirty(*pvmw.pte))
 207			pte = pte_mksoft_dirty(pte);
 208
 209		/*
 210		 * Recheck VMA as permissions can change since migration started
 211		 */
 212		entry = pte_to_swp_entry(*pvmw.pte);
 213		if (is_writable_migration_entry(entry))
 214			pte = maybe_mkwrite(pte, vma);
 215		else if (pte_swp_uffd_wp(*pvmw.pte))
 216			pte = pte_mkuffd_wp(pte);
 217
 218		if (unlikely(is_device_private_page(new))) {
 219			if (pte_write(pte))
 220				entry = make_writable_device_private_entry(
 221							page_to_pfn(new));
 222			else
 223				entry = make_readable_device_private_entry(
 224							page_to_pfn(new));
 225			pte = swp_entry_to_pte(entry);
 226			if (pte_swp_soft_dirty(*pvmw.pte))
 227				pte = pte_swp_mksoft_dirty(pte);
 228			if (pte_swp_uffd_wp(*pvmw.pte))
 229				pte = pte_swp_mkuffd_wp(pte);
 230		}
 231
 232#ifdef CONFIG_HUGETLB_PAGE
 233		if (PageHuge(new)) {
 234			unsigned int shift = huge_page_shift(hstate_vma(vma));
 235
 236			pte = pte_mkhuge(pte);
 237			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
 238			set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 239			if (PageAnon(new))
 240				hugepage_add_anon_rmap(new, vma, pvmw.address);
 241			else
 242				page_dup_rmap(new, true);
 243		} else
 244#endif
 245		{
 246			set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 247
 248			if (PageAnon(new))
 249				page_add_anon_rmap(new, vma, pvmw.address, false);
 250			else
 251				page_add_file_rmap(new, false);
 252		}
 253		if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
 254			mlock_vma_page(new);
 255
 256		if (PageTransHuge(page) && PageMlocked(page))
 257			clear_page_mlock(page);
 258
 259		/* No need to invalidate - it was non-present before */
 260		update_mmu_cache(vma, pvmw.address, pvmw.pte);
 261	}
 262
 263	return true;
 264}
 265
 266/*
 267 * Get rid of all migration entries and replace them by
 268 * references to the indicated page.
 269 */
 270void remove_migration_ptes(struct page *old, struct page *new, bool locked)
 271{
 272	struct rmap_walk_control rwc = {
 273		.rmap_one = remove_migration_pte,
 274		.arg = old,
 275	};
 276
 277	if (locked)
 278		rmap_walk_locked(new, &rwc);
 279	else
 280		rmap_walk(new, &rwc);
 281}
 282
 283/*
 284 * Something used the pte of a page under migration. We need to
 285 * get to the page and wait until migration is finished.
 286 * When we return from this function the fault will be retried.
 287 */
 288void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 289				spinlock_t *ptl)
 290{
 291	pte_t pte;
 292	swp_entry_t entry;
 293	struct page *page;
 294
 295	spin_lock(ptl);
 296	pte = *ptep;
 297	if (!is_swap_pte(pte))
 298		goto out;
 299
 300	entry = pte_to_swp_entry(pte);
 301	if (!is_migration_entry(entry))
 302		goto out;
 303
 304	page = pfn_swap_entry_to_page(entry);
 305	page = compound_head(page);
 306
 307	/*
 308	 * Once page cache replacement of page migration started, page_count
 309	 * is zero; but we must not call put_and_wait_on_page_locked() without
 310	 * a ref. Use get_page_unless_zero(), and just fault again if it fails.
 311	 */
 312	if (!get_page_unless_zero(page))
 313		goto out;
 314	pte_unmap_unlock(ptep, ptl);
 315	put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE);
 316	return;
 317out:
 318	pte_unmap_unlock(ptep, ptl);
 319}
 320
 321void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
 322				unsigned long address)
 323{
 324	spinlock_t *ptl = pte_lockptr(mm, pmd);
 325	pte_t *ptep = pte_offset_map(pmd, address);
 326	__migration_entry_wait(mm, ptep, ptl);
 327}
 328
 329void migration_entry_wait_huge(struct vm_area_struct *vma,
 330		struct mm_struct *mm, pte_t *pte)
 331{
 332	spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
 333	__migration_entry_wait(mm, pte, ptl);
 334}
 335
 336#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 337void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
 338{
 339	spinlock_t *ptl;
 340	struct page *page;
 341
 342	ptl = pmd_lock(mm, pmd);
 343	if (!is_pmd_migration_entry(*pmd))
 344		goto unlock;
 345	page = pfn_swap_entry_to_page(pmd_to_swp_entry(*pmd));
 346	if (!get_page_unless_zero(page))
 347		goto unlock;
 348	spin_unlock(ptl);
 349	put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE);
 350	return;
 351unlock:
 352	spin_unlock(ptl);
 353}
 354#endif
 355
 356static int expected_page_refs(struct address_space *mapping, struct page *page)
 357{
 358	int expected_count = 1;
 359
 360	/*
 361	 * Device private pages have an extra refcount as they are
 362	 * ZONE_DEVICE pages.
 363	 */
 364	expected_count += is_device_private_page(page);
 365	if (mapping)
 366		expected_count += thp_nr_pages(page) + page_has_private(page);
 367
 368	return expected_count;
 369}
 370
 371/*
 372 * Replace the page in the mapping.
 373 *
 374 * The number of remaining references must be:
 375 * 1 for anonymous pages without a mapping
 376 * 2 for pages with a mapping
 377 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
 378 */
 379int migrate_page_move_mapping(struct address_space *mapping,
 380		struct page *newpage, struct page *page, int extra_count)
 381{
 382	XA_STATE(xas, &mapping->i_pages, page_index(page));
 383	struct zone *oldzone, *newzone;
 384	int dirty;
 385	int expected_count = expected_page_refs(mapping, page) + extra_count;
 386	int nr = thp_nr_pages(page);
 387
 388	if (!mapping) {
 389		/* Anonymous page without mapping */
 390		if (page_count(page) != expected_count)
 391			return -EAGAIN;
 392
 393		/* No turning back from here */
 394		newpage->index = page->index;
 395		newpage->mapping = page->mapping;
 396		if (PageSwapBacked(page))
 397			__SetPageSwapBacked(newpage);
 398
 399		return MIGRATEPAGE_SUCCESS;
 400	}
 401
 402	oldzone = page_zone(page);
 403	newzone = page_zone(newpage);
 404
 405	xas_lock_irq(&xas);
 406	if (page_count(page) != expected_count || xas_load(&xas) != page) {
 407		xas_unlock_irq(&xas);
 408		return -EAGAIN;
 409	}
 410
 411	if (!page_ref_freeze(page, expected_count)) {
 412		xas_unlock_irq(&xas);
 413		return -EAGAIN;
 414	}
 415
 416	/*
 417	 * Now we know that no one else is looking at the page:
 418	 * no turning back from here.
 419	 */
 420	newpage->index = page->index;
 421	newpage->mapping = page->mapping;
 422	page_ref_add(newpage, nr); /* add cache reference */
 423	if (PageSwapBacked(page)) {
 424		__SetPageSwapBacked(newpage);
 425		if (PageSwapCache(page)) {
 426			SetPageSwapCache(newpage);
 427			set_page_private(newpage, page_private(page));
 428		}
 429	} else {
 430		VM_BUG_ON_PAGE(PageSwapCache(page), page);
 431	}
 432
 433	/* Move dirty while page refs frozen and newpage not yet exposed */
 434	dirty = PageDirty(page);
 435	if (dirty) {
 436		ClearPageDirty(page);
 437		SetPageDirty(newpage);
 438	}
 439
 440	xas_store(&xas, newpage);
 441	if (PageTransHuge(page)) {
 442		int i;
 443
 444		for (i = 1; i < nr; i++) {
 445			xas_next(&xas);
 446			xas_store(&xas, newpage);
 447		}
 448	}
 449
 450	/*
 451	 * Drop cache reference from old page by unfreezing
 452	 * to one less reference.
 453	 * We know this isn't the last reference.
 454	 */
 455	page_ref_unfreeze(page, expected_count - nr);
 456
 457	xas_unlock(&xas);
 458	/* Leave irq disabled to prevent preemption while updating stats */
 459
 460	/*
 461	 * If moved to a different zone then also account
 462	 * the page for that zone. Other VM counters will be
 463	 * taken care of when we establish references to the
 464	 * new page and drop references to the old page.
 465	 *
 466	 * Note that anonymous pages are accounted for
 467	 * via NR_FILE_PAGES and NR_ANON_MAPPED if they
 468	 * are mapped to swap space.
 469	 */
 470	if (newzone != oldzone) {
 471		struct lruvec *old_lruvec, *new_lruvec;
 472		struct mem_cgroup *memcg;
 473
 474		memcg = page_memcg(page);
 475		old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
 476		new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);
 477
 478		__mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
 479		__mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
 480		if (PageSwapBacked(page) && !PageSwapCache(page)) {
 481			__mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
 482			__mod_lruvec_state(new_lruvec, NR_SHMEM, nr);
 483		}
 484#ifdef CONFIG_SWAP
 485		if (PageSwapCache(page)) {
 486			__mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr);
 487			__mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr);
 488		}
 489#endif
 490		if (dirty && mapping_can_writeback(mapping)) {
 491			__mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
 492			__mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
 493			__mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
 494			__mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
 495		}
 496	}
 497	local_irq_enable();
 498
 499	return MIGRATEPAGE_SUCCESS;
 500}
 501EXPORT_SYMBOL(migrate_page_move_mapping);
 502
 503/*
 504 * The expected number of remaining references is the same as that
 505 * of migrate_page_move_mapping().
 506 */
 507int migrate_huge_page_move_mapping(struct address_space *mapping,
 508				   struct page *newpage, struct page *page)
 509{
 510	XA_STATE(xas, &mapping->i_pages, page_index(page));
 511	int expected_count;
 512
 513	xas_lock_irq(&xas);
 514	expected_count = 2 + page_has_private(page);
 515	if (page_count(page) != expected_count || xas_load(&xas) != page) {
 516		xas_unlock_irq(&xas);
 517		return -EAGAIN;
 518	}
 519
 520	if (!page_ref_freeze(page, expected_count)) {
 521		xas_unlock_irq(&xas);
 522		return -EAGAIN;
 523	}
 524
 525	newpage->index = page->index;
 526	newpage->mapping = page->mapping;
 527
 528	get_page(newpage);
 529
 530	xas_store(&xas, newpage);
 531
 532	page_ref_unfreeze(page, expected_count - 1);
 533
 534	xas_unlock_irq(&xas);
 535
 536	return MIGRATEPAGE_SUCCESS;
 537}
 538
 539/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 540 * Copy the page to its new location
 541 */
 542void migrate_page_states(struct page *newpage, struct page *page)
 543{
 544	int cpupid;
 545
 546	if (PageError(page))
 547		SetPageError(newpage);
 548	if (PageReferenced(page))
 549		SetPageReferenced(newpage);
 550	if (PageUptodate(page))
 551		SetPageUptodate(newpage);
 552	if (TestClearPageActive(page)) {
 553		VM_BUG_ON_PAGE(PageUnevictable(page), page);
 554		SetPageActive(newpage);
 555	} else if (TestClearPageUnevictable(page))
 556		SetPageUnevictable(newpage);
 557	if (PageWorkingset(page))
 558		SetPageWorkingset(newpage);
 559	if (PageChecked(page))
 560		SetPageChecked(newpage);
 561	if (PageMappedToDisk(page))
 562		SetPageMappedToDisk(newpage);
 563
 564	/* Move dirty on pages not done by migrate_page_move_mapping() */
 565	if (PageDirty(page))
 566		SetPageDirty(newpage);
 567
 568	if (page_is_young(page))
 569		set_page_young(newpage);
 570	if (page_is_idle(page))
 571		set_page_idle(newpage);
 572
 573	/*
 574	 * Copy NUMA information to the new page, to prevent over-eager
 575	 * future migrations of this same page.
 576	 */
 577	cpupid = page_cpupid_xchg_last(page, -1);
 578	page_cpupid_xchg_last(newpage, cpupid);
 579
 580	ksm_migrate_page(newpage, page);
 581	/*
 582	 * Please do not reorder this without considering how mm/ksm.c's
 583	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
 584	 */
 585	if (PageSwapCache(page))
 586		ClearPageSwapCache(page);
 587	ClearPagePrivate(page);
 588
 589	/* page->private contains hugetlb specific flags */
 590	if (!PageHuge(page))
 591		set_page_private(page, 0);
 592
 593	/*
 594	 * If any waiters have accumulated on the new page then
 595	 * wake them up.
 596	 */
 597	if (PageWriteback(newpage))
 598		end_page_writeback(newpage);
 599
 600	/*
 601	 * PG_readahead shares the same bit with PG_reclaim.  The above
 602	 * end_page_writeback() may clear PG_readahead mistakenly, so set the
 603	 * bit after that.
 604	 */
 605	if (PageReadahead(page))
 606		SetPageReadahead(newpage);
 607
 608	copy_page_owner(page, newpage);
 609
 610	if (!PageHuge(page))
 611		mem_cgroup_migrate(page, newpage);
 612}
 613EXPORT_SYMBOL(migrate_page_states);
 614
 615void migrate_page_copy(struct page *newpage, struct page *page)
 616{
 617	if (PageHuge(page) || PageTransHuge(page))
 618		copy_huge_page(newpage, page);
 619	else
 620		copy_highpage(newpage, page);
 621
 622	migrate_page_states(newpage, page);
 623}
 624EXPORT_SYMBOL(migrate_page_copy);
 625
 626/************************************************************
 627 *                    Migration functions
 628 ***********************************************************/
 629
 630/*
 631 * Common logic to directly migrate a single LRU page suitable for
 632 * pages that do not use PagePrivate/PagePrivate2.
 633 *
 634 * Pages are locked upon entry and exit.
 635 */
 636int migrate_page(struct address_space *mapping,
 637		struct page *newpage, struct page *page,
 638		enum migrate_mode mode)
 639{
 640	int rc;
 641
 642	BUG_ON(PageWriteback(page));	/* Writeback must be complete */
 643
 644	rc = migrate_page_move_mapping(mapping, newpage, page, 0);
 645
 646	if (rc != MIGRATEPAGE_SUCCESS)
 647		return rc;
 648
 649	if (mode != MIGRATE_SYNC_NO_COPY)
 650		migrate_page_copy(newpage, page);
 651	else
 652		migrate_page_states(newpage, page);
 653	return MIGRATEPAGE_SUCCESS;
 654}
 655EXPORT_SYMBOL(migrate_page);
 656
 657#ifdef CONFIG_BLOCK
 658/* Returns true if all buffers are successfully locked */
 659static bool buffer_migrate_lock_buffers(struct buffer_head *head,
 660							enum migrate_mode mode)
 661{
 662	struct buffer_head *bh = head;
 663
 664	/* Simple case, sync compaction */
 665	if (mode != MIGRATE_ASYNC) {
 666		do {
 667			lock_buffer(bh);
 668			bh = bh->b_this_page;
 669
 670		} while (bh != head);
 671
 672		return true;
 673	}
 674
 675	/* async case, we cannot block on lock_buffer so use trylock_buffer */
 676	do {
 677		if (!trylock_buffer(bh)) {
 678			/*
 679			 * We failed to lock the buffer and cannot stall in
 680			 * async migration. Release the taken locks
 681			 */
 682			struct buffer_head *failed_bh = bh;
 683			bh = head;
 684			while (bh != failed_bh) {
 685				unlock_buffer(bh);
 686				bh = bh->b_this_page;
 687			}
 688			return false;
 689		}
 690
 691		bh = bh->b_this_page;
 692	} while (bh != head);
 693	return true;
 694}
 695
 696static int __buffer_migrate_page(struct address_space *mapping,
 697		struct page *newpage, struct page *page, enum migrate_mode mode,
 698		bool check_refs)
 699{
 700	struct buffer_head *bh, *head;
 701	int rc;
 702	int expected_count;
 703
 704	if (!page_has_buffers(page))
 705		return migrate_page(mapping, newpage, page, mode);
 706
 707	/* Check whether page does not have extra refs before we do more work */
 708	expected_count = expected_page_refs(mapping, page);
 709	if (page_count(page) != expected_count)
 710		return -EAGAIN;
 711
 712	head = page_buffers(page);
 713	if (!buffer_migrate_lock_buffers(head, mode))
 714		return -EAGAIN;
 715
 716	if (check_refs) {
 717		bool busy;
 718		bool invalidated = false;
 719
 720recheck_buffers:
 721		busy = false;
 722		spin_lock(&mapping->private_lock);
 723		bh = head;
 724		do {
 725			if (atomic_read(&bh->b_count)) {
 726				busy = true;
 727				break;
 728			}
 729			bh = bh->b_this_page;
 730		} while (bh != head);
 731		if (busy) {
 732			if (invalidated) {
 733				rc = -EAGAIN;
 734				goto unlock_buffers;
 735			}
 736			spin_unlock(&mapping->private_lock);
 737			invalidate_bh_lrus();
 738			invalidated = true;
 739			goto recheck_buffers;
 740		}
 741	}
 742
 743	rc = migrate_page_move_mapping(mapping, newpage, page, 0);
 744	if (rc != MIGRATEPAGE_SUCCESS)
 745		goto unlock_buffers;
 746
 747	attach_page_private(newpage, detach_page_private(page));
 
 
 
 
 748
 749	bh = head;
 750	do {
 751		set_bh_page(bh, newpage, bh_offset(bh));
 752		bh = bh->b_this_page;
 753
 754	} while (bh != head);
 755
 
 
 756	if (mode != MIGRATE_SYNC_NO_COPY)
 757		migrate_page_copy(newpage, page);
 758	else
 759		migrate_page_states(newpage, page);
 760
 761	rc = MIGRATEPAGE_SUCCESS;
 762unlock_buffers:
 763	if (check_refs)
 764		spin_unlock(&mapping->private_lock);
 765	bh = head;
 766	do {
 767		unlock_buffer(bh);
 768		bh = bh->b_this_page;
 769
 770	} while (bh != head);
 771
 772	return rc;
 773}
 774
 775/*
 776 * Migration function for pages with buffers. This function can only be used
 777 * if the underlying filesystem guarantees that no other references to "page"
 778 * exist. For example attached buffer heads are accessed only under page lock.
 779 */
 780int buffer_migrate_page(struct address_space *mapping,
 781		struct page *newpage, struct page *page, enum migrate_mode mode)
 782{
 783	return __buffer_migrate_page(mapping, newpage, page, mode, false);
 784}
 785EXPORT_SYMBOL(buffer_migrate_page);
 786
 787/*
 788 * Same as above except that this variant is more careful and checks that there
 789 * are also no buffer head references. This function is the right one for
 790 * mappings where buffer heads are directly looked up and referenced (such as
 791 * block device mappings).
 792 */
 793int buffer_migrate_page_norefs(struct address_space *mapping,
 794		struct page *newpage, struct page *page, enum migrate_mode mode)
 795{
 796	return __buffer_migrate_page(mapping, newpage, page, mode, true);
 797}
 798#endif
 799
 800/*
 801 * Writeback a page to clean the dirty state
 802 */
 803static int writeout(struct address_space *mapping, struct page *page)
 804{
 805	struct writeback_control wbc = {
 806		.sync_mode = WB_SYNC_NONE,
 807		.nr_to_write = 1,
 808		.range_start = 0,
 809		.range_end = LLONG_MAX,
 810		.for_reclaim = 1
 811	};
 812	int rc;
 813
 814	if (!mapping->a_ops->writepage)
 815		/* No write method for the address space */
 816		return -EINVAL;
 817
 818	if (!clear_page_dirty_for_io(page))
 819		/* Someone else already triggered a write */
 820		return -EAGAIN;
 821
 822	/*
 823	 * A dirty page may imply that the underlying filesystem has
 824	 * the page on some queue. So the page must be clean for
 825	 * migration. Writeout may mean we loose the lock and the
 826	 * page state is no longer what we checked for earlier.
 827	 * At this point we know that the migration attempt cannot
 828	 * be successful.
 829	 */
 830	remove_migration_ptes(page, page, false);
 831
 832	rc = mapping->a_ops->writepage(page, &wbc);
 833
 834	if (rc != AOP_WRITEPAGE_ACTIVATE)
 835		/* unlocked. Relock */
 836		lock_page(page);
 837
 838	return (rc < 0) ? -EIO : -EAGAIN;
 839}
 840
 841/*
 842 * Default handling if a filesystem does not provide a migration function.
 843 */
 844static int fallback_migrate_page(struct address_space *mapping,
 845	struct page *newpage, struct page *page, enum migrate_mode mode)
 846{
 847	if (PageDirty(page)) {
 848		/* Only writeback pages in full synchronous migration */
 849		switch (mode) {
 850		case MIGRATE_SYNC:
 851		case MIGRATE_SYNC_NO_COPY:
 852			break;
 853		default:
 854			return -EBUSY;
 855		}
 856		return writeout(mapping, page);
 857	}
 858
 859	/*
 860	 * Buffers may be managed in a filesystem specific way.
 861	 * We must have no buffers or drop them.
 862	 */
 863	if (page_has_private(page) &&
 864	    !try_to_release_page(page, GFP_KERNEL))
 865		return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
 866
 867	return migrate_page(mapping, newpage, page, mode);
 868}
 869
 870/*
 871 * Move a page to a newly allocated page
 872 * The page is locked and all ptes have been successfully removed.
 873 *
 874 * The new page will have replaced the old page if this function
 875 * is successful.
 876 *
 877 * Return value:
 878 *   < 0 - error code
 879 *  MIGRATEPAGE_SUCCESS - success
 880 */
 881static int move_to_new_page(struct page *newpage, struct page *page,
 882				enum migrate_mode mode)
 883{
 884	struct address_space *mapping;
 885	int rc = -EAGAIN;
 886	bool is_lru = !__PageMovable(page);
 887
 888	VM_BUG_ON_PAGE(!PageLocked(page), page);
 889	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
 890
 891	mapping = page_mapping(page);
 892
 893	if (likely(is_lru)) {
 894		if (!mapping)
 895			rc = migrate_page(mapping, newpage, page, mode);
 896		else if (mapping->a_ops->migratepage)
 897			/*
 898			 * Most pages have a mapping and most filesystems
 899			 * provide a migratepage callback. Anonymous pages
 900			 * are part of swap space which also has its own
 901			 * migratepage callback. This is the most common path
 902			 * for page migration.
 903			 */
 904			rc = mapping->a_ops->migratepage(mapping, newpage,
 905							page, mode);
 906		else
 907			rc = fallback_migrate_page(mapping, newpage,
 908							page, mode);
 909	} else {
 910		/*
 911		 * In case of non-lru page, it could be released after
 912		 * isolation step. In that case, we shouldn't try migration.
 913		 */
 914		VM_BUG_ON_PAGE(!PageIsolated(page), page);
 915		if (!PageMovable(page)) {
 916			rc = MIGRATEPAGE_SUCCESS;
 917			__ClearPageIsolated(page);
 918			goto out;
 919		}
 920
 921		rc = mapping->a_ops->migratepage(mapping, newpage,
 922						page, mode);
 923		WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
 924			!PageIsolated(page));
 925	}
 926
 927	/*
 928	 * When successful, old pagecache page->mapping must be cleared before
 929	 * page is freed; but stats require that PageAnon be left as PageAnon.
 930	 */
 931	if (rc == MIGRATEPAGE_SUCCESS) {
 932		if (__PageMovable(page)) {
 933			VM_BUG_ON_PAGE(!PageIsolated(page), page);
 934
 935			/*
 936			 * We clear PG_movable under page_lock so any compactor
 937			 * cannot try to migrate this page.
 938			 */
 939			__ClearPageIsolated(page);
 940		}
 941
 942		/*
 943		 * Anonymous and movable page->mapping will be cleared by
 944		 * free_pages_prepare so don't reset it here for keeping
 945		 * the type to work PageAnon, for example.
 946		 */
 947		if (!PageMappingFlags(page))
 948			page->mapping = NULL;
 949
 950		if (likely(!is_zone_device_page(newpage)))
 951			flush_dcache_page(newpage);
 952
 953	}
 954out:
 955	return rc;
 956}
 957
 958static int __unmap_and_move(struct page *page, struct page *newpage,
 959				int force, enum migrate_mode mode)
 960{
 961	int rc = -EAGAIN;
 962	int page_was_mapped = 0;
 963	struct anon_vma *anon_vma = NULL;
 964	bool is_lru = !__PageMovable(page);
 965
 966	if (!trylock_page(page)) {
 967		if (!force || mode == MIGRATE_ASYNC)
 968			goto out;
 969
 970		/*
 971		 * It's not safe for direct compaction to call lock_page.
 972		 * For example, during page readahead pages are added locked
 973		 * to the LRU. Later, when the IO completes the pages are
 974		 * marked uptodate and unlocked. However, the queueing
 975		 * could be merging multiple pages for one bio (e.g.
 976		 * mpage_readahead). If an allocation happens for the
 977		 * second or third page, the process can end up locking
 978		 * the same page twice and deadlocking. Rather than
 979		 * trying to be clever about what pages can be locked,
 980		 * avoid the use of lock_page for direct compaction
 981		 * altogether.
 982		 */
 983		if (current->flags & PF_MEMALLOC)
 984			goto out;
 985
 986		lock_page(page);
 987	}
 988
 989	if (PageWriteback(page)) {
 990		/*
 991		 * Only in the case of a full synchronous migration is it
 992		 * necessary to wait for PageWriteback. In the async case,
 993		 * the retry loop is too short and in the sync-light case,
 994		 * the overhead of stalling is too much
 995		 */
 996		switch (mode) {
 997		case MIGRATE_SYNC:
 998		case MIGRATE_SYNC_NO_COPY:
 999			break;
1000		default:
1001			rc = -EBUSY;
1002			goto out_unlock;
1003		}
1004		if (!force)
1005			goto out_unlock;
1006		wait_on_page_writeback(page);
1007	}
1008
1009	/*
1010	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
1011	 * we cannot notice that anon_vma is freed while we migrates a page.
1012	 * This get_anon_vma() delays freeing anon_vma pointer until the end
1013	 * of migration. File cache pages are no problem because of page_lock()
1014	 * File Caches may use write_page() or lock_page() in migration, then,
1015	 * just care Anon page here.
1016	 *
1017	 * Only page_get_anon_vma() understands the subtleties of
1018	 * getting a hold on an anon_vma from outside one of its mms.
1019	 * But if we cannot get anon_vma, then we won't need it anyway,
1020	 * because that implies that the anon page is no longer mapped
1021	 * (and cannot be remapped so long as we hold the page lock).
1022	 */
1023	if (PageAnon(page) && !PageKsm(page))
1024		anon_vma = page_get_anon_vma(page);
1025
1026	/*
1027	 * Block others from accessing the new page when we get around to
1028	 * establishing additional references. We are usually the only one
1029	 * holding a reference to newpage at this point. We used to have a BUG
1030	 * here if trylock_page(newpage) fails, but would like to allow for
1031	 * cases where there might be a race with the previous use of newpage.
1032	 * This is much like races on refcount of oldpage: just don't BUG().
1033	 */
1034	if (unlikely(!trylock_page(newpage)))
1035		goto out_unlock;
1036
1037	if (unlikely(!is_lru)) {
1038		rc = move_to_new_page(newpage, page, mode);
1039		goto out_unlock_both;
1040	}
1041
1042	/*
1043	 * Corner case handling:
1044	 * 1. When a new swap-cache page is read into, it is added to the LRU
1045	 * and treated as swapcache but it has no rmap yet.
1046	 * Calling try_to_unmap() against a page->mapping==NULL page will
1047	 * trigger a BUG.  So handle it here.
1048	 * 2. An orphaned page (see truncate_cleanup_page) might have
1049	 * fs-private metadata. The page can be picked up due to memory
1050	 * offlining.  Everywhere else except page reclaim, the page is
1051	 * invisible to the vm, so the page can not be migrated.  So try to
1052	 * free the metadata, so the page can be freed.
1053	 */
1054	if (!page->mapping) {
1055		VM_BUG_ON_PAGE(PageAnon(page), page);
1056		if (page_has_private(page)) {
1057			try_to_free_buffers(page);
1058			goto out_unlock_both;
1059		}
1060	} else if (page_mapped(page)) {
1061		/* Establish migration ptes */
1062		VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
1063				page);
1064		try_to_migrate(page, 0);
 
1065		page_was_mapped = 1;
1066	}
1067
1068	if (!page_mapped(page))
1069		rc = move_to_new_page(newpage, page, mode);
1070
1071	if (page_was_mapped)
1072		remove_migration_ptes(page,
1073			rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1074
1075out_unlock_both:
1076	unlock_page(newpage);
1077out_unlock:
1078	/* Drop an anon_vma reference if we took one */
1079	if (anon_vma)
1080		put_anon_vma(anon_vma);
1081	unlock_page(page);
1082out:
1083	/*
1084	 * If migration is successful, decrease refcount of the newpage
1085	 * which will not free the page because new page owner increased
1086	 * refcounter. As well, if it is LRU page, add the page to LRU
1087	 * list in here. Use the old state of the isolated source page to
1088	 * determine if we migrated a LRU page. newpage was already unlocked
1089	 * and possibly modified by its owner - don't rely on the page
1090	 * state.
1091	 */
1092	if (rc == MIGRATEPAGE_SUCCESS) {
1093		if (unlikely(!is_lru))
1094			put_page(newpage);
1095		else
1096			putback_lru_page(newpage);
1097	}
1098
1099	return rc;
1100}
1101
1102/*
 
 
 
 
 
 
 
 
 
 
 
1103 * Obtain the lock on page, remove all ptes and migrate the page
1104 * to the newly allocated page in newpage.
1105 */
1106static int unmap_and_move(new_page_t get_new_page,
1107				   free_page_t put_new_page,
1108				   unsigned long private, struct page *page,
1109				   int force, enum migrate_mode mode,
1110				   enum migrate_reason reason,
1111				   struct list_head *ret)
1112{
1113	int rc = MIGRATEPAGE_SUCCESS;
1114	struct page *newpage = NULL;
1115
1116	if (!thp_migration_supported() && PageTransHuge(page))
1117		return -ENOSYS;
 
 
 
 
1118
1119	if (page_count(page) == 1) {
1120		/* page was freed from under us. So we are done. */
1121		ClearPageActive(page);
1122		ClearPageUnevictable(page);
1123		if (unlikely(__PageMovable(page))) {
1124			lock_page(page);
1125			if (!PageMovable(page))
1126				__ClearPageIsolated(page);
1127			unlock_page(page);
1128		}
 
 
 
 
1129		goto out;
1130	}
1131
1132	newpage = get_new_page(page, private);
1133	if (!newpage)
1134		return -ENOMEM;
1135
1136	rc = __unmap_and_move(page, newpage, force, mode);
1137	if (rc == MIGRATEPAGE_SUCCESS)
1138		set_page_owner_migrate_reason(newpage, reason);
1139
1140out:
1141	if (rc != -EAGAIN) {
1142		/*
1143		 * A page that has been migrated has all references
1144		 * removed and will be freed. A page that has not been
1145		 * migrated will have kept its references and be restored.
 
1146		 */
1147		list_del(&page->lru);
1148	}
1149
1150	/*
1151	 * If migration is successful, releases reference grabbed during
1152	 * isolation. Otherwise, restore the page to right list unless
1153	 * we want to retry.
1154	 */
1155	if (rc == MIGRATEPAGE_SUCCESS) {
1156		/*
1157		 * Compaction can migrate also non-LRU pages which are
1158		 * not accounted to NR_ISOLATED_*. They can be recognized
1159		 * as __PageMovable
1160		 */
1161		if (likely(!__PageMovable(page)))
1162			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1163					page_is_file_lru(page), -thp_nr_pages(page));
 
1164
1165		if (reason != MR_MEMORY_FAILURE)
 
 
 
 
 
 
 
1166			/*
1167			 * We release the page in page_handle_poison.
 
 
1168			 */
1169			put_page(page);
 
 
1170	} else {
1171		if (rc != -EAGAIN)
1172			list_add_tail(&page->lru, ret);
 
 
 
1173
 
 
 
 
 
 
 
 
 
1174		if (put_new_page)
1175			put_new_page(newpage, private);
1176		else
1177			put_page(newpage);
1178	}
1179
1180	return rc;
1181}
1182
1183/*
1184 * Counterpart of unmap_and_move_page() for hugepage migration.
1185 *
1186 * This function doesn't wait the completion of hugepage I/O
1187 * because there is no race between I/O and migration for hugepage.
1188 * Note that currently hugepage I/O occurs only in direct I/O
1189 * where no lock is held and PG_writeback is irrelevant,
1190 * and writeback status of all subpages are counted in the reference
1191 * count of the head page (i.e. if all subpages of a 2MB hugepage are
1192 * under direct I/O, the reference of the head page is 512 and a bit more.)
1193 * This means that when we try to migrate hugepage whose subpages are
1194 * doing direct I/O, some references remain after try_to_unmap() and
1195 * hugepage migration fails without data corruption.
1196 *
1197 * There is also no race when direct I/O is issued on the page under migration,
1198 * because then pte is replaced with migration swap entry and direct I/O code
1199 * will wait in the page fault for migration to complete.
1200 */
1201static int unmap_and_move_huge_page(new_page_t get_new_page,
1202				free_page_t put_new_page, unsigned long private,
1203				struct page *hpage, int force,
1204				enum migrate_mode mode, int reason,
1205				struct list_head *ret)
1206{
1207	int rc = -EAGAIN;
1208	int page_was_mapped = 0;
1209	struct page *new_hpage;
1210	struct anon_vma *anon_vma = NULL;
1211	struct address_space *mapping = NULL;
1212
1213	/*
1214	 * Migratability of hugepages depends on architectures and their size.
1215	 * This check is necessary because some callers of hugepage migration
1216	 * like soft offline and memory hotremove don't walk through page
1217	 * tables or check whether the hugepage is pmd-based or not before
1218	 * kicking migration.
1219	 */
1220	if (!hugepage_migration_supported(page_hstate(hpage))) {
1221		list_move_tail(&hpage->lru, ret);
1222		return -ENOSYS;
1223	}
1224
1225	if (page_count(hpage) == 1) {
1226		/* page was freed from under us. So we are done. */
1227		putback_active_hugepage(hpage);
1228		return MIGRATEPAGE_SUCCESS;
1229	}
1230
1231	new_hpage = get_new_page(hpage, private);
1232	if (!new_hpage)
1233		return -ENOMEM;
1234
1235	if (!trylock_page(hpage)) {
1236		if (!force)
1237			goto out;
1238		switch (mode) {
1239		case MIGRATE_SYNC:
1240		case MIGRATE_SYNC_NO_COPY:
1241			break;
1242		default:
1243			goto out;
1244		}
1245		lock_page(hpage);
1246	}
1247
1248	/*
1249	 * Check for pages which are in the process of being freed.  Without
1250	 * page_mapping() set, hugetlbfs specific move page routine will not
1251	 * be called and we could leak usage counts for subpools.
1252	 */
1253	if (hugetlb_page_subpool(hpage) && !page_mapping(hpage)) {
1254		rc = -EBUSY;
1255		goto out_unlock;
1256	}
1257
1258	if (PageAnon(hpage))
1259		anon_vma = page_get_anon_vma(hpage);
1260
1261	if (unlikely(!trylock_page(new_hpage)))
1262		goto put_anon;
1263
1264	if (page_mapped(hpage)) {
1265		bool mapping_locked = false;
1266		enum ttu_flags ttu = 0;
1267
1268		if (!PageAnon(hpage)) {
1269			/*
1270			 * In shared mappings, try_to_unmap could potentially
1271			 * call huge_pmd_unshare.  Because of this, take
1272			 * semaphore in write mode here and set TTU_RMAP_LOCKED
1273			 * to let lower levels know we have taken the lock.
1274			 */
1275			mapping = hugetlb_page_mapping_lock_write(hpage);
1276			if (unlikely(!mapping))
1277				goto unlock_put_anon;
1278
1279			mapping_locked = true;
1280			ttu |= TTU_RMAP_LOCKED;
1281		}
1282
1283		try_to_migrate(hpage, ttu);
1284		page_was_mapped = 1;
1285
1286		if (mapping_locked)
1287			i_mmap_unlock_write(mapping);
1288	}
1289
1290	if (!page_mapped(hpage))
1291		rc = move_to_new_page(new_hpage, hpage, mode);
1292
1293	if (page_was_mapped)
1294		remove_migration_ptes(hpage,
1295			rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1296
1297unlock_put_anon:
1298	unlock_page(new_hpage);
1299
1300put_anon:
1301	if (anon_vma)
1302		put_anon_vma(anon_vma);
1303
1304	if (rc == MIGRATEPAGE_SUCCESS) {
1305		move_hugetlb_state(hpage, new_hpage, reason);
1306		put_new_page = NULL;
1307	}
1308
1309out_unlock:
1310	unlock_page(hpage);
1311out:
1312	if (rc == MIGRATEPAGE_SUCCESS)
1313		putback_active_hugepage(hpage);
1314	else if (rc != -EAGAIN)
1315		list_move_tail(&hpage->lru, ret);
1316
1317	/*
1318	 * If migration was not successful and there's a freeing callback, use
1319	 * it.  Otherwise, put_page() will drop the reference grabbed during
1320	 * isolation.
1321	 */
1322	if (put_new_page)
1323		put_new_page(new_hpage, private);
1324	else
1325		putback_active_hugepage(new_hpage);
1326
1327	return rc;
1328}
1329
1330static inline int try_split_thp(struct page *page, struct page **page2,
1331				struct list_head *from)
1332{
1333	int rc = 0;
1334
1335	lock_page(page);
1336	rc = split_huge_page_to_list(page, from);
1337	unlock_page(page);
1338	if (!rc)
1339		list_safe_reset_next(page, *page2, lru);
1340
1341	return rc;
1342}
1343
1344/*
1345 * migrate_pages - migrate the pages specified in a list, to the free pages
1346 *		   supplied as the target for the page migration
1347 *
1348 * @from:		The list of pages to be migrated.
1349 * @get_new_page:	The function used to allocate free pages to be used
1350 *			as the target of the page migration.
1351 * @put_new_page:	The function used to free target pages if migration
1352 *			fails, or NULL if no special handling is necessary.
1353 * @private:		Private data to be passed on to get_new_page()
1354 * @mode:		The migration mode that specifies the constraints for
1355 *			page migration, if any.
1356 * @reason:		The reason for page migration.
1357 *
1358 * The function returns after 10 attempts or if no pages are movable any more
1359 * because the list has become empty or no retryable pages exist any more.
1360 * It is caller's responsibility to call putback_movable_pages() to return pages
1361 * to the LRU or free list only if ret != 0.
1362 *
1363 * Returns the number of pages that were not migrated, or an error code.
1364 */
1365int migrate_pages(struct list_head *from, new_page_t get_new_page,
1366		free_page_t put_new_page, unsigned long private,
1367		enum migrate_mode mode, int reason)
1368{
1369	int retry = 1;
1370	int thp_retry = 1;
1371	int nr_failed = 0;
1372	int nr_succeeded = 0;
1373	int nr_thp_succeeded = 0;
1374	int nr_thp_failed = 0;
1375	int nr_thp_split = 0;
1376	int pass = 0;
1377	bool is_thp = false;
1378	struct page *page;
1379	struct page *page2;
1380	int swapwrite = current->flags & PF_SWAPWRITE;
1381	int rc, nr_subpages;
1382	LIST_HEAD(ret_pages);
1383	bool nosplit = (reason == MR_NUMA_MISPLACED);
1384
1385	trace_mm_migrate_pages_start(mode, reason);
1386
1387	if (!swapwrite)
1388		current->flags |= PF_SWAPWRITE;
1389
1390	for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
1391		retry = 0;
1392		thp_retry = 0;
1393
1394		list_for_each_entry_safe(page, page2, from, lru) {
1395retry:
1396			/*
1397			 * THP statistics is based on the source huge page.
1398			 * Capture required information that might get lost
1399			 * during migration.
1400			 */
1401			is_thp = PageTransHuge(page) && !PageHuge(page);
1402			nr_subpages = thp_nr_pages(page);
1403			cond_resched();
1404
1405			if (PageHuge(page))
1406				rc = unmap_and_move_huge_page(get_new_page,
1407						put_new_page, private, page,
1408						pass > 2, mode, reason,
1409						&ret_pages);
1410			else
1411				rc = unmap_and_move(get_new_page, put_new_page,
1412						private, page, pass > 2, mode,
1413						reason, &ret_pages);
1414			/*
1415			 * The rules are:
1416			 *	Success: non hugetlb page will be freed, hugetlb
1417			 *		 page will be put back
1418			 *	-EAGAIN: stay on the from list
1419			 *	-ENOMEM: stay on the from list
1420			 *	Other errno: put on ret_pages list then splice to
1421			 *		     from list
1422			 */
1423			switch(rc) {
1424			/*
1425			 * THP migration might be unsupported or the
1426			 * allocation could've failed so we should
1427			 * retry on the same page with the THP split
1428			 * to base pages.
1429			 *
1430			 * Head page is retried immediately and tail
1431			 * pages are added to the tail of the list so
1432			 * we encounter them after the rest of the list
1433			 * is processed.
1434			 */
1435			case -ENOSYS:
1436				/* THP migration is unsupported */
1437				if (is_thp) {
1438					if (!try_split_thp(page, &page2, from)) {
1439						nr_thp_split++;
1440						goto retry;
1441					}
1442
1443					nr_thp_failed++;
1444					nr_failed += nr_subpages;
1445					break;
1446				}
1447
1448				/* Hugetlb migration is unsupported */
1449				nr_failed++;
1450				break;
1451			case -ENOMEM:
1452				/*
1453				 * When memory is low, don't bother to try to migrate
1454				 * other pages, just exit.
1455				 * THP NUMA faulting doesn't split THP to retry.
 
 
 
 
 
 
1456				 */
1457				if (is_thp && !nosplit) {
1458					if (!try_split_thp(page, &page2, from)) {
1459						nr_thp_split++;
 
 
 
1460						goto retry;
1461					}
1462
1463					nr_thp_failed++;
1464					nr_failed += nr_subpages;
1465					goto out;
1466				}
1467				nr_failed++;
1468				goto out;
1469			case -EAGAIN:
1470				if (is_thp) {
1471					thp_retry++;
1472					break;
1473				}
1474				retry++;
1475				break;
1476			case MIGRATEPAGE_SUCCESS:
1477				if (is_thp) {
1478					nr_thp_succeeded++;
1479					nr_succeeded += nr_subpages;
1480					break;
1481				}
1482				nr_succeeded++;
1483				break;
1484			default:
1485				/*
1486				 * Permanent failure (-EBUSY, etc.):
1487				 * unlike -EAGAIN case, the failed page is
1488				 * removed from migration page list and not
1489				 * retried in the next outer loop.
1490				 */
1491				if (is_thp) {
1492					nr_thp_failed++;
1493					nr_failed += nr_subpages;
1494					break;
1495				}
1496				nr_failed++;
1497				break;
1498			}
1499		}
1500	}
1501	nr_failed += retry + thp_retry;
1502	nr_thp_failed += thp_retry;
1503	rc = nr_failed;
1504out:
1505	/*
1506	 * Put the permanent failure page back to migration list, they
1507	 * will be put back to the right list by the caller.
1508	 */
1509	list_splice(&ret_pages, from);
1510
1511	count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1512	count_vm_events(PGMIGRATE_FAIL, nr_failed);
1513	count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
1514	count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
1515	count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
1516	trace_mm_migrate_pages(nr_succeeded, nr_failed, nr_thp_succeeded,
1517			       nr_thp_failed, nr_thp_split, mode, reason);
1518
1519	if (!swapwrite)
1520		current->flags &= ~PF_SWAPWRITE;
1521
1522	return rc;
1523}
1524
1525struct page *alloc_migration_target(struct page *page, unsigned long private)
1526{
1527	struct migration_target_control *mtc;
1528	gfp_t gfp_mask;
1529	unsigned int order = 0;
1530	struct page *new_page = NULL;
1531	int nid;
1532	int zidx;
1533
1534	mtc = (struct migration_target_control *)private;
1535	gfp_mask = mtc->gfp_mask;
1536	nid = mtc->nid;
1537	if (nid == NUMA_NO_NODE)
1538		nid = page_to_nid(page);
1539
1540	if (PageHuge(page)) {
1541		struct hstate *h = page_hstate(compound_head(page));
1542
1543		gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
1544		return alloc_huge_page_nodemask(h, nid, mtc->nmask, gfp_mask);
1545	}
1546
1547	if (PageTransHuge(page)) {
1548		/*
1549		 * clear __GFP_RECLAIM to make the migration callback
1550		 * consistent with regular THP allocations.
1551		 */
1552		gfp_mask &= ~__GFP_RECLAIM;
1553		gfp_mask |= GFP_TRANSHUGE;
1554		order = HPAGE_PMD_ORDER;
1555	}
1556	zidx = zone_idx(page_zone(page));
1557	if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
1558		gfp_mask |= __GFP_HIGHMEM;
1559
1560	new_page = __alloc_pages(gfp_mask, order, nid, mtc->nmask);
1561
1562	if (new_page && PageTransHuge(new_page))
1563		prep_transhuge_page(new_page);
1564
1565	return new_page;
1566}
1567
1568#ifdef CONFIG_NUMA
1569
1570static int store_status(int __user *status, int start, int value, int nr)
1571{
1572	while (nr-- > 0) {
1573		if (put_user(value, status + start))
1574			return -EFAULT;
1575		start++;
1576	}
1577
1578	return 0;
1579}
1580
1581static int do_move_pages_to_node(struct mm_struct *mm,
1582		struct list_head *pagelist, int node)
1583{
1584	int err;
1585	struct migration_target_control mtc = {
1586		.nid = node,
1587		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
1588	};
1589
1590	err = migrate_pages(pagelist, alloc_migration_target, NULL,
1591			(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL);
 
 
 
1592	if (err)
1593		putback_movable_pages(pagelist);
1594	return err;
1595}
1596
1597/*
1598 * Resolves the given address to a struct page, isolates it from the LRU and
1599 * puts it to the given pagelist.
1600 * Returns:
1601 *     errno - if the page cannot be found/isolated
1602 *     0 - when it doesn't have to be migrated because it is already on the
1603 *         target node
1604 *     1 - when it has been queued
1605 */
1606static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
1607		int node, struct list_head *pagelist, bool migrate_all)
1608{
1609	struct vm_area_struct *vma;
1610	struct page *page;
1611	unsigned int follflags;
1612	int err;
1613
1614	mmap_read_lock(mm);
1615	err = -EFAULT;
1616	vma = find_vma(mm, addr);
1617	if (!vma || addr < vma->vm_start || !vma_migratable(vma))
1618		goto out;
1619
1620	/* FOLL_DUMP to ignore special (like zero) pages */
1621	follflags = FOLL_GET | FOLL_DUMP;
1622	page = follow_page(vma, addr, follflags);
1623
1624	err = PTR_ERR(page);
1625	if (IS_ERR(page))
1626		goto out;
1627
1628	err = -ENOENT;
1629	if (!page)
1630		goto out;
1631
1632	err = 0;
1633	if (page_to_nid(page) == node)
1634		goto out_putpage;
1635
1636	err = -EACCES;
1637	if (page_mapcount(page) > 1 && !migrate_all)
1638		goto out_putpage;
1639
1640	if (PageHuge(page)) {
1641		if (PageHead(page)) {
1642			isolate_huge_page(page, pagelist);
1643			err = 1;
1644		}
1645	} else {
1646		struct page *head;
1647
1648		head = compound_head(page);
1649		err = isolate_lru_page(head);
1650		if (err)
1651			goto out_putpage;
1652
1653		err = 1;
1654		list_add_tail(&head->lru, pagelist);
1655		mod_node_page_state(page_pgdat(head),
1656			NR_ISOLATED_ANON + page_is_file_lru(head),
1657			thp_nr_pages(head));
1658	}
1659out_putpage:
1660	/*
1661	 * Either remove the duplicate refcount from
1662	 * isolate_lru_page() or drop the page ref if it was
1663	 * not isolated.
1664	 */
1665	put_page(page);
1666out:
1667	mmap_read_unlock(mm);
1668	return err;
1669}
1670
1671static int move_pages_and_store_status(struct mm_struct *mm, int node,
1672		struct list_head *pagelist, int __user *status,
1673		int start, int i, unsigned long nr_pages)
1674{
1675	int err;
1676
1677	if (list_empty(pagelist))
1678		return 0;
1679
1680	err = do_move_pages_to_node(mm, pagelist, node);
1681	if (err) {
1682		/*
1683		 * Positive err means the number of failed
1684		 * pages to migrate.  Since we are going to
1685		 * abort and return the number of non-migrated
1686		 * pages, so need to include the rest of the
1687		 * nr_pages that have not been attempted as
1688		 * well.
1689		 */
1690		if (err > 0)
1691			err += nr_pages - i - 1;
1692		return err;
1693	}
1694	return store_status(status, start, node, i - start);
1695}
1696
1697/*
1698 * Migrate an array of page address onto an array of nodes and fill
1699 * the corresponding array of status.
1700 */
1701static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1702			 unsigned long nr_pages,
1703			 const void __user * __user *pages,
1704			 const int __user *nodes,
1705			 int __user *status, int flags)
1706{
1707	int current_node = NUMA_NO_NODE;
1708	LIST_HEAD(pagelist);
1709	int start, i;
1710	int err = 0, err1;
1711
1712	lru_cache_disable();
1713
1714	for (i = start = 0; i < nr_pages; i++) {
1715		const void __user *p;
1716		unsigned long addr;
1717		int node;
1718
1719		err = -EFAULT;
1720		if (get_user(p, pages + i))
1721			goto out_flush;
1722		if (get_user(node, nodes + i))
1723			goto out_flush;
1724		addr = (unsigned long)untagged_addr(p);
1725
1726		err = -ENODEV;
1727		if (node < 0 || node >= MAX_NUMNODES)
1728			goto out_flush;
1729		if (!node_state(node, N_MEMORY))
1730			goto out_flush;
1731
1732		err = -EACCES;
1733		if (!node_isset(node, task_nodes))
1734			goto out_flush;
1735
1736		if (current_node == NUMA_NO_NODE) {
1737			current_node = node;
1738			start = i;
1739		} else if (node != current_node) {
1740			err = move_pages_and_store_status(mm, current_node,
1741					&pagelist, status, start, i, nr_pages);
 
 
1742			if (err)
1743				goto out;
1744			start = i;
1745			current_node = node;
1746		}
1747
1748		/*
1749		 * Errors in the page lookup or isolation are not fatal and we simply
1750		 * report them via status
1751		 */
1752		err = add_page_for_migration(mm, addr, current_node,
1753				&pagelist, flags & MPOL_MF_MOVE_ALL);
1754
1755		if (err > 0) {
1756			/* The page is successfully queued for migration */
1757			continue;
1758		}
1759
1760		/*
1761		 * If the page is already on the target node (!err), store the
1762		 * node, otherwise, store the err.
1763		 */
1764		err = store_status(status, i, err ? : current_node, 1);
1765		if (err)
1766			goto out_flush;
1767
1768		err = move_pages_and_store_status(mm, current_node, &pagelist,
1769				status, start, i, nr_pages);
1770		if (err)
1771			goto out;
 
 
 
 
 
1772		current_node = NUMA_NO_NODE;
1773	}
1774out_flush:
 
 
 
1775	/* Make sure we do not overwrite the existing error */
1776	err1 = move_pages_and_store_status(mm, current_node, &pagelist,
1777				status, start, i, nr_pages);
1778	if (err >= 0)
 
1779		err = err1;
1780out:
1781	lru_cache_enable();
1782	return err;
1783}
1784
1785/*
1786 * Determine the nodes of an array of pages and store it in an array of status.
1787 */
1788static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1789				const void __user **pages, int *status)
1790{
1791	unsigned long i;
1792
1793	mmap_read_lock(mm);
1794
1795	for (i = 0; i < nr_pages; i++) {
1796		unsigned long addr = (unsigned long)(*pages);
1797		struct vm_area_struct *vma;
1798		struct page *page;
1799		int err = -EFAULT;
1800
1801		vma = vma_lookup(mm, addr);
1802		if (!vma)
1803			goto set_status;
1804
1805		/* FOLL_DUMP to ignore special (like zero) pages */
1806		page = follow_page(vma, addr, FOLL_DUMP);
1807
1808		err = PTR_ERR(page);
1809		if (IS_ERR(page))
1810			goto set_status;
1811
1812		err = page ? page_to_nid(page) : -ENOENT;
1813set_status:
1814		*status = err;
1815
1816		pages++;
1817		status++;
1818	}
1819
1820	mmap_read_unlock(mm);
1821}
1822
1823/*
1824 * Determine the nodes of a user array of pages and store it in
1825 * a user array of status.
1826 */
1827static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1828			 const void __user * __user *pages,
1829			 int __user *status)
1830{
1831#define DO_PAGES_STAT_CHUNK_NR 16
1832	const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1833	int chunk_status[DO_PAGES_STAT_CHUNK_NR];
1834
1835	while (nr_pages) {
1836		unsigned long chunk_nr;
1837
1838		chunk_nr = nr_pages;
1839		if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1840			chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1841
1842		if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1843			break;
1844
1845		do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1846
1847		if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1848			break;
1849
1850		pages += chunk_nr;
1851		status += chunk_nr;
1852		nr_pages -= chunk_nr;
1853	}
1854	return nr_pages ? -EFAULT : 0;
1855}
1856
1857static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
 
 
 
 
 
 
 
1858{
1859	struct task_struct *task;
1860	struct mm_struct *mm;
 
 
 
 
 
 
1861
1862	/*
1863	 * There is no need to check if current process has the right to modify
1864	 * the specified process when they are same.
1865	 */
1866	if (!pid) {
1867		mmget(current->mm);
1868		*mem_nodes = cpuset_mems_allowed(current);
1869		return current->mm;
1870	}
1871
1872	/* Find the mm_struct */
1873	rcu_read_lock();
1874	task = find_task_by_vpid(pid);
1875	if (!task) {
1876		rcu_read_unlock();
1877		return ERR_PTR(-ESRCH);
1878	}
1879	get_task_struct(task);
1880
1881	/*
1882	 * Check if this process has the right to modify the specified
1883	 * process. Use the regular "ptrace_may_access()" checks.
1884	 */
1885	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1886		rcu_read_unlock();
1887		mm = ERR_PTR(-EPERM);
1888		goto out;
1889	}
1890	rcu_read_unlock();
1891
1892	mm = ERR_PTR(security_task_movememory(task));
1893	if (IS_ERR(mm))
1894		goto out;
1895	*mem_nodes = cpuset_mems_allowed(task);
 
1896	mm = get_task_mm(task);
1897out:
1898	put_task_struct(task);
 
1899	if (!mm)
1900		mm = ERR_PTR(-EINVAL);
1901	return mm;
1902}
1903
1904/*
1905 * Move a list of pages in the address space of the currently executing
1906 * process.
1907 */
1908static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
1909			     const void __user * __user *pages,
1910			     const int __user *nodes,
1911			     int __user *status, int flags)
1912{
1913	struct mm_struct *mm;
1914	int err;
1915	nodemask_t task_nodes;
1916
1917	/* Check flags */
1918	if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1919		return -EINVAL;
1920
1921	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1922		return -EPERM;
1923
1924	mm = find_mm_struct(pid, &task_nodes);
1925	if (IS_ERR(mm))
1926		return PTR_ERR(mm);
1927
1928	if (nodes)
1929		err = do_pages_move(mm, task_nodes, nr_pages, pages,
1930				    nodes, status, flags);
1931	else
1932		err = do_pages_stat(mm, nr_pages, pages, status);
1933
1934	mmput(mm);
1935	return err;
 
 
 
 
1936}
1937
1938SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1939		const void __user * __user *, pages,
1940		const int __user *, nodes,
1941		int __user *, status, int, flags)
1942{
1943	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1944}
1945
1946#ifdef CONFIG_COMPAT
1947COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
1948		       compat_uptr_t __user *, pages32,
1949		       const int __user *, nodes,
1950		       int __user *, status,
1951		       int, flags)
1952{
1953	const void __user * __user *pages;
1954	int i;
1955
1956	pages = compat_alloc_user_space(nr_pages * sizeof(void *));
1957	for (i = 0; i < nr_pages; i++) {
1958		compat_uptr_t p;
1959
1960		if (get_user(p, pages32 + i) ||
1961			put_user(compat_ptr(p), pages + i))
1962			return -EFAULT;
1963	}
1964	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1965}
1966#endif /* CONFIG_COMPAT */
1967
1968#ifdef CONFIG_NUMA_BALANCING
1969/*
1970 * Returns true if this is a safe migration target node for misplaced NUMA
1971 * pages. Currently it only checks the watermarks which crude
1972 */
1973static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1974				   unsigned long nr_migrate_pages)
1975{
1976	int z;
1977
1978	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1979		struct zone *zone = pgdat->node_zones + z;
1980
1981		if (!populated_zone(zone))
1982			continue;
1983
1984		/* Avoid waking kswapd by allocating pages_to_migrate pages. */
1985		if (!zone_watermark_ok(zone, 0,
1986				       high_wmark_pages(zone) +
1987				       nr_migrate_pages,
1988				       ZONE_MOVABLE, 0))
1989			continue;
1990		return true;
1991	}
1992	return false;
1993}
1994
1995static struct page *alloc_misplaced_dst_page(struct page *page,
1996					   unsigned long data)
1997{
1998	int nid = (int) data;
1999	struct page *newpage;
2000
2001	newpage = __alloc_pages_node(nid,
2002					 (GFP_HIGHUSER_MOVABLE |
2003					  __GFP_THISNODE | __GFP_NOMEMALLOC |
2004					  __GFP_NORETRY | __GFP_NOWARN) &
2005					 ~__GFP_RECLAIM, 0);
2006
2007	return newpage;
2008}
2009
2010static struct page *alloc_misplaced_dst_page_thp(struct page *page,
2011						 unsigned long data)
2012{
2013	int nid = (int) data;
2014	struct page *newpage;
2015
2016	newpage = alloc_pages_node(nid, (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
2017				   HPAGE_PMD_ORDER);
2018	if (!newpage)
2019		goto out;
2020
2021	prep_transhuge_page(newpage);
2022
2023out:
2024	return newpage;
2025}
2026
2027static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
2028{
2029	int page_lru;
2030
2031	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
2032
2033	/* Do not migrate THP mapped by multiple processes */
2034	if (PageTransHuge(page) && total_mapcount(page) > 1)
2035		return 0;
2036
2037	/* Avoid migrating to a node that is nearly full */
2038	if (!migrate_balanced_pgdat(pgdat, compound_nr(page)))
2039		return 0;
2040
2041	if (isolate_lru_page(page))
2042		return 0;
2043
2044	page_lru = page_is_file_lru(page);
 
 
 
 
 
 
 
 
 
 
 
 
2045	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
2046				thp_nr_pages(page));
2047
2048	/*
2049	 * Isolating the page has taken another reference, so the
2050	 * caller's reference can be safely dropped without the page
2051	 * disappearing underneath us during migration.
2052	 */
2053	put_page(page);
2054	return 1;
2055}
2056
 
 
 
 
 
 
2057/*
2058 * Attempt to migrate a misplaced page to the specified destination
2059 * node. Caller is expected to have an elevated reference count on
2060 * the page that will be dropped by this function before returning.
2061 */
2062int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
2063			   int node)
2064{
2065	pg_data_t *pgdat = NODE_DATA(node);
2066	int isolated;
2067	int nr_remaining;
2068	LIST_HEAD(migratepages);
2069	new_page_t *new;
2070	bool compound;
2071	int nr_pages = thp_nr_pages(page);
2072
2073	/*
2074	 * PTE mapped THP or HugeTLB page can't reach here so the page could
2075	 * be either base page or THP.  And it must be head page if it is
2076	 * THP.
2077	 */
2078	compound = PageTransHuge(page);
2079
2080	if (compound)
2081		new = alloc_misplaced_dst_page_thp;
2082	else
2083		new = alloc_misplaced_dst_page;
2084
2085	/*
2086	 * Don't migrate file pages that are mapped in multiple processes
2087	 * with execute permissions as they are probably shared libraries.
2088	 */
2089	if (page_mapcount(page) != 1 && page_is_file_lru(page) &&
2090	    (vma->vm_flags & VM_EXEC))
2091		goto out;
2092
2093	/*
2094	 * Also do not migrate dirty pages as not all filesystems can move
2095	 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
2096	 */
2097	if (page_is_file_lru(page) && PageDirty(page))
2098		goto out;
2099
2100	isolated = numamigrate_isolate_page(pgdat, page);
2101	if (!isolated)
2102		goto out;
2103
2104	list_add(&page->lru, &migratepages);
2105	nr_remaining = migrate_pages(&migratepages, *new, NULL, node,
2106				     MIGRATE_ASYNC, MR_NUMA_MISPLACED);
 
2107	if (nr_remaining) {
2108		if (!list_empty(&migratepages)) {
2109			list_del(&page->lru);
2110			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
2111					page_is_file_lru(page), -nr_pages);
2112			putback_lru_page(page);
2113		}
2114		isolated = 0;
2115	} else
2116		count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_pages);
2117	BUG_ON(!list_empty(&migratepages));
2118	return isolated;
2119
2120out:
2121	put_page(page);
2122	return 0;
2123}
2124#endif /* CONFIG_NUMA_BALANCING */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2125#endif /* CONFIG_NUMA */
2126
2127#ifdef CONFIG_DEVICE_PRIVATE
2128static int migrate_vma_collect_skip(unsigned long start,
2129				    unsigned long end,
2130				    struct mm_walk *walk)
2131{
2132	struct migrate_vma *migrate = walk->private;
2133	unsigned long addr;
2134
2135	for (addr = start; addr < end; addr += PAGE_SIZE) {
 
2136		migrate->dst[migrate->npages] = 0;
2137		migrate->src[migrate->npages++] = 0;
 
2138	}
2139
2140	return 0;
2141}
2142
2143static int migrate_vma_collect_hole(unsigned long start,
2144				    unsigned long end,
2145				    __always_unused int depth,
2146				    struct mm_walk *walk)
2147{
2148	struct migrate_vma *migrate = walk->private;
2149	unsigned long addr;
2150
2151	/* Only allow populating anonymous memory. */
2152	if (!vma_is_anonymous(walk->vma))
2153		return migrate_vma_collect_skip(start, end, walk);
2154
2155	for (addr = start; addr < end; addr += PAGE_SIZE) {
2156		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2157		migrate->dst[migrate->npages] = 0;
2158		migrate->npages++;
2159		migrate->cpages++;
2160	}
2161
2162	return 0;
2163}
2164
2165static int migrate_vma_collect_pmd(pmd_t *pmdp,
2166				   unsigned long start,
2167				   unsigned long end,
2168				   struct mm_walk *walk)
2169{
2170	struct migrate_vma *migrate = walk->private;
2171	struct vm_area_struct *vma = walk->vma;
2172	struct mm_struct *mm = vma->vm_mm;
2173	unsigned long addr = start, unmapped = 0;
2174	spinlock_t *ptl;
2175	pte_t *ptep;
2176
2177again:
2178	if (pmd_none(*pmdp))
2179		return migrate_vma_collect_hole(start, end, -1, walk);
2180
2181	if (pmd_trans_huge(*pmdp)) {
2182		struct page *page;
2183
2184		ptl = pmd_lock(mm, pmdp);
2185		if (unlikely(!pmd_trans_huge(*pmdp))) {
2186			spin_unlock(ptl);
2187			goto again;
2188		}
2189
2190		page = pmd_page(*pmdp);
2191		if (is_huge_zero_page(page)) {
2192			spin_unlock(ptl);
2193			split_huge_pmd(vma, pmdp, addr);
2194			if (pmd_trans_unstable(pmdp))
2195				return migrate_vma_collect_skip(start, end,
2196								walk);
2197		} else {
2198			int ret;
2199
2200			get_page(page);
2201			spin_unlock(ptl);
2202			if (unlikely(!trylock_page(page)))
2203				return migrate_vma_collect_skip(start, end,
2204								walk);
2205			ret = split_huge_page(page);
2206			unlock_page(page);
2207			put_page(page);
2208			if (ret)
2209				return migrate_vma_collect_skip(start, end,
2210								walk);
2211			if (pmd_none(*pmdp))
2212				return migrate_vma_collect_hole(start, end, -1,
2213								walk);
2214		}
2215	}
2216
2217	if (unlikely(pmd_bad(*pmdp)))
2218		return migrate_vma_collect_skip(start, end, walk);
2219
2220	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2221	arch_enter_lazy_mmu_mode();
2222
2223	for (; addr < end; addr += PAGE_SIZE, ptep++) {
2224		unsigned long mpfn = 0, pfn;
2225		struct page *page;
2226		swp_entry_t entry;
2227		pte_t pte;
2228
2229		pte = *ptep;
2230
2231		if (pte_none(pte)) {
2232			if (vma_is_anonymous(vma)) {
2233				mpfn = MIGRATE_PFN_MIGRATE;
2234				migrate->cpages++;
2235			}
2236			goto next;
2237		}
2238
2239		if (!pte_present(pte)) {
 
 
2240			/*
2241			 * Only care about unaddressable device page special
2242			 * page table entry. Other special swap entries are not
2243			 * migratable, and we ignore regular swapped page.
2244			 */
2245			entry = pte_to_swp_entry(pte);
2246			if (!is_device_private_entry(entry))
2247				goto next;
2248
2249			page = pfn_swap_entry_to_page(entry);
2250			if (!(migrate->flags &
2251				MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
2252			    page->pgmap->owner != migrate->pgmap_owner)
2253				goto next;
2254
2255			mpfn = migrate_pfn(page_to_pfn(page)) |
2256					MIGRATE_PFN_MIGRATE;
2257			if (is_writable_device_private_entry(entry))
2258				mpfn |= MIGRATE_PFN_WRITE;
2259		} else {
2260			if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
2261				goto next;
2262			pfn = pte_pfn(pte);
2263			if (is_zero_pfn(pfn)) {
2264				mpfn = MIGRATE_PFN_MIGRATE;
2265				migrate->cpages++;
2266				goto next;
2267			}
2268			page = vm_normal_page(migrate->vma, addr, pte);
2269			mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
2270			mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2271		}
2272
2273		/* FIXME support THP */
2274		if (!page || !page->mapping || PageTransCompound(page)) {
2275			mpfn = 0;
2276			goto next;
2277		}
2278
2279		/*
2280		 * By getting a reference on the page we pin it and that blocks
2281		 * any kind of migration. Side effect is that it "freezes" the
2282		 * pte.
2283		 *
2284		 * We drop this reference after isolating the page from the lru
2285		 * for non device page (device page are not on the lru and thus
2286		 * can't be dropped from it).
2287		 */
2288		get_page(page);
2289		migrate->cpages++;
2290
2291		/*
2292		 * Optimize for the common case where page is only mapped once
2293		 * in one process. If we can lock the page, then we can safely
2294		 * set up a special migration page table entry now.
2295		 */
2296		if (trylock_page(page)) {
2297			pte_t swp_pte;
2298
2299			mpfn |= MIGRATE_PFN_LOCKED;
2300			ptep_get_and_clear(mm, addr, ptep);
2301
2302			/* Setup special migration page table entry */
2303			if (mpfn & MIGRATE_PFN_WRITE)
2304				entry = make_writable_migration_entry(
2305							page_to_pfn(page));
2306			else
2307				entry = make_readable_migration_entry(
2308							page_to_pfn(page));
2309			swp_pte = swp_entry_to_pte(entry);
2310			if (pte_present(pte)) {
2311				if (pte_soft_dirty(pte))
2312					swp_pte = pte_swp_mksoft_dirty(swp_pte);
2313				if (pte_uffd_wp(pte))
2314					swp_pte = pte_swp_mkuffd_wp(swp_pte);
2315			} else {
2316				if (pte_swp_soft_dirty(pte))
2317					swp_pte = pte_swp_mksoft_dirty(swp_pte);
2318				if (pte_swp_uffd_wp(pte))
2319					swp_pte = pte_swp_mkuffd_wp(swp_pte);
2320			}
2321			set_pte_at(mm, addr, ptep, swp_pte);
2322
2323			/*
2324			 * This is like regular unmap: we remove the rmap and
2325			 * drop page refcount. Page won't be freed, as we took
2326			 * a reference just above.
2327			 */
2328			page_remove_rmap(page, false);
2329			put_page(page);
2330
2331			if (pte_present(pte))
2332				unmapped++;
2333		}
2334
2335next:
2336		migrate->dst[migrate->npages] = 0;
2337		migrate->src[migrate->npages++] = mpfn;
2338	}
2339	arch_leave_lazy_mmu_mode();
2340	pte_unmap_unlock(ptep - 1, ptl);
2341
2342	/* Only flush the TLB if we actually modified any entries */
2343	if (unmapped)
2344		flush_tlb_range(walk->vma, start, end);
2345
2346	return 0;
2347}
2348
2349static const struct mm_walk_ops migrate_vma_walk_ops = {
2350	.pmd_entry		= migrate_vma_collect_pmd,
2351	.pte_hole		= migrate_vma_collect_hole,
2352};
2353
2354/*
2355 * migrate_vma_collect() - collect pages over a range of virtual addresses
2356 * @migrate: migrate struct containing all migration information
2357 *
2358 * This will walk the CPU page table. For each virtual address backed by a
2359 * valid page, it updates the src array and takes a reference on the page, in
2360 * order to pin the page until we lock it and unmap it.
2361 */
2362static void migrate_vma_collect(struct migrate_vma *migrate)
2363{
2364	struct mmu_notifier_range range;
2365
2366	/*
2367	 * Note that the pgmap_owner is passed to the mmu notifier callback so
2368	 * that the registered device driver can skip invalidating device
2369	 * private page mappings that won't be migrated.
2370	 */
2371	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_MIGRATE, 0,
2372		migrate->vma, migrate->vma->vm_mm, migrate->start, migrate->end,
2373		migrate->pgmap_owner);
2374	mmu_notifier_invalidate_range_start(&range);
2375
2376	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
2377			&migrate_vma_walk_ops, migrate);
2378
2379	mmu_notifier_invalidate_range_end(&range);
2380	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2381}
2382
2383/*
2384 * migrate_vma_check_page() - check if page is pinned or not
2385 * @page: struct page to check
2386 *
2387 * Pinned pages cannot be migrated. This is the same test as in
2388 * migrate_page_move_mapping(), except that here we allow migration of a
2389 * ZONE_DEVICE page.
2390 */
2391static bool migrate_vma_check_page(struct page *page)
2392{
2393	/*
2394	 * One extra ref because caller holds an extra reference, either from
2395	 * isolate_lru_page() for a regular page, or migrate_vma_collect() for
2396	 * a device page.
2397	 */
2398	int extra = 1;
2399
2400	/*
2401	 * FIXME support THP (transparent huge page), it is bit more complex to
2402	 * check them than regular pages, because they can be mapped with a pmd
2403	 * or with a pte (split pte mapping).
2404	 */
2405	if (PageCompound(page))
2406		return false;
2407
2408	/* Page from ZONE_DEVICE have one extra reference */
2409	if (is_zone_device_page(page)) {
2410		/*
2411		 * Private page can never be pin as they have no valid pte and
2412		 * GUP will fail for those. Yet if there is a pending migration
2413		 * a thread might try to wait on the pte migration entry and
2414		 * will bump the page reference count. Sadly there is no way to
2415		 * differentiate a regular pin from migration wait. Hence to
2416		 * avoid 2 racing thread trying to migrate back to CPU to enter
2417		 * infinite loop (one stopping migration because the other is
2418		 * waiting on pte migration entry). We always return true here.
2419		 *
2420		 * FIXME proper solution is to rework migration_entry_wait() so
2421		 * it does not need to take a reference on page.
2422		 */
2423		return is_device_private_page(page);
2424	}
2425
2426	/* For file back page */
2427	if (page_mapping(page))
2428		extra += 1 + page_has_private(page);
2429
2430	if ((page_count(page) - extra) > page_mapcount(page))
2431		return false;
2432
2433	return true;
2434}
2435
2436/*
2437 * migrate_vma_prepare() - lock pages and isolate them from the lru
2438 * @migrate: migrate struct containing all migration information
2439 *
2440 * This locks pages that have been collected by migrate_vma_collect(). Once each
2441 * page is locked it is isolated from the lru (for non-device pages). Finally,
2442 * the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
2443 * migrated by concurrent kernel threads.
2444 */
2445static void migrate_vma_prepare(struct migrate_vma *migrate)
2446{
2447	const unsigned long npages = migrate->npages;
2448	const unsigned long start = migrate->start;
2449	unsigned long addr, i, restore = 0;
2450	bool allow_drain = true;
2451
2452	lru_add_drain();
2453
2454	for (i = 0; (i < npages) && migrate->cpages; i++) {
2455		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2456		bool remap = true;
2457
2458		if (!page)
2459			continue;
2460
2461		if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
2462			/*
2463			 * Because we are migrating several pages there can be
2464			 * a deadlock between 2 concurrent migration where each
2465			 * are waiting on each other page lock.
2466			 *
2467			 * Make migrate_vma() a best effort thing and backoff
2468			 * for any page we can not lock right away.
2469			 */
2470			if (!trylock_page(page)) {
2471				migrate->src[i] = 0;
2472				migrate->cpages--;
2473				put_page(page);
2474				continue;
2475			}
2476			remap = false;
2477			migrate->src[i] |= MIGRATE_PFN_LOCKED;
2478		}
2479
2480		/* ZONE_DEVICE pages are not on LRU */
2481		if (!is_zone_device_page(page)) {
2482			if (!PageLRU(page) && allow_drain) {
2483				/* Drain CPU's pagevec */
2484				lru_add_drain_all();
2485				allow_drain = false;
2486			}
2487
2488			if (isolate_lru_page(page)) {
2489				if (remap) {
2490					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2491					migrate->cpages--;
2492					restore++;
2493				} else {
2494					migrate->src[i] = 0;
2495					unlock_page(page);
2496					migrate->cpages--;
2497					put_page(page);
2498				}
2499				continue;
2500			}
2501
2502			/* Drop the reference we took in collect */
2503			put_page(page);
2504		}
2505
2506		if (!migrate_vma_check_page(page)) {
2507			if (remap) {
2508				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2509				migrate->cpages--;
2510				restore++;
2511
2512				if (!is_zone_device_page(page)) {
2513					get_page(page);
2514					putback_lru_page(page);
2515				}
2516			} else {
2517				migrate->src[i] = 0;
2518				unlock_page(page);
2519				migrate->cpages--;
2520
2521				if (!is_zone_device_page(page))
2522					putback_lru_page(page);
2523				else
2524					put_page(page);
2525			}
2526		}
2527	}
2528
2529	for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
2530		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2531
2532		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2533			continue;
2534
2535		remove_migration_pte(page, migrate->vma, addr, page);
2536
2537		migrate->src[i] = 0;
2538		unlock_page(page);
2539		put_page(page);
2540		restore--;
2541	}
2542}
2543
2544/*
2545 * migrate_vma_unmap() - replace page mapping with special migration pte entry
2546 * @migrate: migrate struct containing all migration information
2547 *
2548 * Replace page mapping (CPU page table pte) with a special migration pte entry
2549 * and check again if it has been pinned. Pinned pages are restored because we
2550 * cannot migrate them.
2551 *
2552 * This is the last step before we call the device driver callback to allocate
2553 * destination memory and copy contents of original page over to new page.
2554 */
2555static void migrate_vma_unmap(struct migrate_vma *migrate)
2556{
 
2557	const unsigned long npages = migrate->npages;
2558	const unsigned long start = migrate->start;
2559	unsigned long addr, i, restore = 0;
2560
2561	for (i = 0; i < npages; i++) {
2562		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2563
2564		if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2565			continue;
2566
2567		if (page_mapped(page)) {
2568			try_to_migrate(page, 0);
2569			if (page_mapped(page))
2570				goto restore;
2571		}
2572
2573		if (migrate_vma_check_page(page))
2574			continue;
2575
2576restore:
2577		migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2578		migrate->cpages--;
2579		restore++;
2580	}
2581
2582	for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
2583		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2584
2585		if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2586			continue;
2587
2588		remove_migration_ptes(page, page, false);
2589
2590		migrate->src[i] = 0;
2591		unlock_page(page);
2592		restore--;
2593
2594		if (is_zone_device_page(page))
2595			put_page(page);
2596		else
2597			putback_lru_page(page);
2598	}
2599}
2600
2601/**
2602 * migrate_vma_setup() - prepare to migrate a range of memory
2603 * @args: contains the vma, start, and pfns arrays for the migration
2604 *
2605 * Returns: negative errno on failures, 0 when 0 or more pages were migrated
2606 * without an error.
2607 *
2608 * Prepare to migrate a range of memory virtual address range by collecting all
2609 * the pages backing each virtual address in the range, saving them inside the
2610 * src array.  Then lock those pages and unmap them. Once the pages are locked
2611 * and unmapped, check whether each page is pinned or not.  Pages that aren't
2612 * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
2613 * corresponding src array entry.  Then restores any pages that are pinned, by
2614 * remapping and unlocking those pages.
2615 *
2616 * The caller should then allocate destination memory and copy source memory to
2617 * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
2618 * flag set).  Once these are allocated and copied, the caller must update each
2619 * corresponding entry in the dst array with the pfn value of the destination
2620 * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
2621 * (destination pages must have their struct pages locked, via lock_page()).
2622 *
2623 * Note that the caller does not have to migrate all the pages that are marked
2624 * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
2625 * device memory to system memory.  If the caller cannot migrate a device page
2626 * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
2627 * consequences for the userspace process, so it must be avoided if at all
2628 * possible.
2629 *
2630 * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
2631 * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
2632 * allowing the caller to allocate device memory for those unbacked virtual
2633 * addresses.  For this the caller simply has to allocate device memory and
2634 * properly set the destination entry like for regular migration.  Note that
2635 * this can still fail, and thus inside the device driver you must check if the
2636 * migration was successful for those entries after calling migrate_vma_pages(),
2637 * just like for regular migration.
2638 *
2639 * After that, the callers must call migrate_vma_pages() to go over each entry
2640 * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
2641 * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
2642 * then migrate_vma_pages() to migrate struct page information from the source
2643 * struct page to the destination struct page.  If it fails to migrate the
2644 * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
2645 * src array.
2646 *
2647 * At this point all successfully migrated pages have an entry in the src
2648 * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
2649 * array entry with MIGRATE_PFN_VALID flag set.
2650 *
2651 * Once migrate_vma_pages() returns the caller may inspect which pages were
2652 * successfully migrated, and which were not.  Successfully migrated pages will
2653 * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
2654 *
2655 * It is safe to update device page table after migrate_vma_pages() because
2656 * both destination and source page are still locked, and the mmap_lock is held
2657 * in read mode (hence no one can unmap the range being migrated).
2658 *
2659 * Once the caller is done cleaning up things and updating its page table (if it
2660 * chose to do so, this is not an obligation) it finally calls
2661 * migrate_vma_finalize() to update the CPU page table to point to new pages
2662 * for successfully migrated pages or otherwise restore the CPU page table to
2663 * point to the original source pages.
2664 */
2665int migrate_vma_setup(struct migrate_vma *args)
2666{
2667	long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
2668
2669	args->start &= PAGE_MASK;
2670	args->end &= PAGE_MASK;
2671	if (!args->vma || is_vm_hugetlb_page(args->vma) ||
2672	    (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
2673		return -EINVAL;
2674	if (nr_pages <= 0)
2675		return -EINVAL;
2676	if (args->start < args->vma->vm_start ||
2677	    args->start >= args->vma->vm_end)
2678		return -EINVAL;
2679	if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
2680		return -EINVAL;
2681	if (!args->src || !args->dst)
2682		return -EINVAL;
2683
2684	memset(args->src, 0, sizeof(*args->src) * nr_pages);
2685	args->cpages = 0;
2686	args->npages = 0;
2687
2688	migrate_vma_collect(args);
2689
2690	if (args->cpages)
2691		migrate_vma_prepare(args);
2692	if (args->cpages)
2693		migrate_vma_unmap(args);
2694
2695	/*
2696	 * At this point pages are locked and unmapped, and thus they have
2697	 * stable content and can safely be copied to destination memory that
2698	 * is allocated by the drivers.
2699	 */
2700	return 0;
2701
2702}
2703EXPORT_SYMBOL(migrate_vma_setup);
2704
2705/*
2706 * This code closely matches the code in:
2707 *   __handle_mm_fault()
2708 *     handle_pte_fault()
2709 *       do_anonymous_page()
2710 * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
2711 * private page.
2712 */
2713static void migrate_vma_insert_page(struct migrate_vma *migrate,
2714				    unsigned long addr,
2715				    struct page *page,
2716				    unsigned long *src)
 
2717{
2718	struct vm_area_struct *vma = migrate->vma;
2719	struct mm_struct *mm = vma->vm_mm;
 
2720	bool flush = false;
2721	spinlock_t *ptl;
2722	pte_t entry;
2723	pgd_t *pgdp;
2724	p4d_t *p4dp;
2725	pud_t *pudp;
2726	pmd_t *pmdp;
2727	pte_t *ptep;
2728
2729	/* Only allow populating anonymous memory */
2730	if (!vma_is_anonymous(vma))
2731		goto abort;
2732
2733	pgdp = pgd_offset(mm, addr);
2734	p4dp = p4d_alloc(mm, pgdp, addr);
2735	if (!p4dp)
2736		goto abort;
2737	pudp = pud_alloc(mm, p4dp, addr);
2738	if (!pudp)
2739		goto abort;
2740	pmdp = pmd_alloc(mm, pudp, addr);
2741	if (!pmdp)
2742		goto abort;
2743
2744	if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
2745		goto abort;
2746
2747	/*
2748	 * Use pte_alloc() instead of pte_alloc_map().  We can't run
2749	 * pte_offset_map() on pmds where a huge pmd might be created
2750	 * from a different thread.
2751	 *
2752	 * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
2753	 * parallel threads are excluded by other means.
2754	 *
2755	 * Here we only have mmap_read_lock(mm).
2756	 */
2757	if (pte_alloc(mm, pmdp))
2758		goto abort;
2759
2760	/* See the comment in pte_alloc_one_map() */
2761	if (unlikely(pmd_trans_unstable(pmdp)))
2762		goto abort;
2763
2764	if (unlikely(anon_vma_prepare(vma)))
2765		goto abort;
2766	if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
2767		goto abort;
2768
2769	/*
2770	 * The memory barrier inside __SetPageUptodate makes sure that
2771	 * preceding stores to the page contents become visible before
2772	 * the set_pte_at() write.
2773	 */
2774	__SetPageUptodate(page);
2775
2776	if (is_zone_device_page(page)) {
2777		if (is_device_private_page(page)) {
2778			swp_entry_t swp_entry;
2779
2780			if (vma->vm_flags & VM_WRITE)
2781				swp_entry = make_writable_device_private_entry(
2782							page_to_pfn(page));
2783			else
2784				swp_entry = make_readable_device_private_entry(
2785							page_to_pfn(page));
2786			entry = swp_entry_to_pte(swp_entry);
2787		} else {
2788			/*
2789			 * For now we only support migrating to un-addressable
2790			 * device memory.
2791			 */
2792			pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
2793			goto abort;
2794		}
2795	} else {
2796		entry = mk_pte(page, vma->vm_page_prot);
2797		if (vma->vm_flags & VM_WRITE)
2798			entry = pte_mkwrite(pte_mkdirty(entry));
2799	}
2800
2801	ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2802
2803	if (check_stable_address_space(mm))
2804		goto unlock_abort;
2805
2806	if (pte_present(*ptep)) {
2807		unsigned long pfn = pte_pfn(*ptep);
2808
2809		if (!is_zero_pfn(pfn))
2810			goto unlock_abort;
 
 
 
2811		flush = true;
2812	} else if (!pte_none(*ptep))
2813		goto unlock_abort;
 
 
 
2814
2815	/*
2816	 * Check for userfaultfd but do not deliver the fault. Instead,
2817	 * just back off.
2818	 */
2819	if (userfaultfd_missing(vma))
2820		goto unlock_abort;
 
 
 
2821
2822	inc_mm_counter(mm, MM_ANONPAGES);
2823	page_add_new_anon_rmap(page, vma, addr, false);
 
2824	if (!is_zone_device_page(page))
2825		lru_cache_add_inactive_or_unevictable(page, vma);
2826	get_page(page);
2827
2828	if (flush) {
2829		flush_cache_page(vma, addr, pte_pfn(*ptep));
2830		ptep_clear_flush_notify(vma, addr, ptep);
2831		set_pte_at_notify(mm, addr, ptep, entry);
2832		update_mmu_cache(vma, addr, ptep);
2833	} else {
2834		/* No need to invalidate - it was non-present before */
2835		set_pte_at(mm, addr, ptep, entry);
2836		update_mmu_cache(vma, addr, ptep);
2837	}
2838
2839	pte_unmap_unlock(ptep, ptl);
2840	*src = MIGRATE_PFN_MIGRATE;
2841	return;
2842
2843unlock_abort:
2844	pte_unmap_unlock(ptep, ptl);
2845abort:
2846	*src &= ~MIGRATE_PFN_MIGRATE;
2847}
2848
2849/**
2850 * migrate_vma_pages() - migrate meta-data from src page to dst page
2851 * @migrate: migrate struct containing all migration information
2852 *
2853 * This migrates struct page meta-data from source struct page to destination
2854 * struct page. This effectively finishes the migration from source page to the
2855 * destination page.
2856 */
2857void migrate_vma_pages(struct migrate_vma *migrate)
2858{
2859	const unsigned long npages = migrate->npages;
2860	const unsigned long start = migrate->start;
2861	struct mmu_notifier_range range;
2862	unsigned long addr, i;
2863	bool notified = false;
2864
2865	for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2866		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2867		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2868		struct address_space *mapping;
2869		int r;
2870
2871		if (!newpage) {
2872			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2873			continue;
2874		}
2875
2876		if (!page) {
2877			if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2878				continue;
 
2879			if (!notified) {
2880				notified = true;
2881
2882				mmu_notifier_range_init_owner(&range,
2883					MMU_NOTIFY_MIGRATE, 0, migrate->vma,
2884					migrate->vma->vm_mm, addr, migrate->end,
2885					migrate->pgmap_owner);
 
2886				mmu_notifier_invalidate_range_start(&range);
2887			}
2888			migrate_vma_insert_page(migrate, addr, newpage,
2889						&migrate->src[i]);
 
2890			continue;
2891		}
2892
2893		mapping = page_mapping(page);
2894
2895		if (is_zone_device_page(newpage)) {
2896			if (is_device_private_page(newpage)) {
2897				/*
2898				 * For now only support private anonymous when
2899				 * migrating to un-addressable device memory.
2900				 */
2901				if (mapping) {
2902					migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2903					continue;
2904				}
2905			} else {
2906				/*
2907				 * Other types of ZONE_DEVICE page are not
2908				 * supported.
2909				 */
2910				migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2911				continue;
2912			}
2913		}
2914
2915		r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2916		if (r != MIGRATEPAGE_SUCCESS)
2917			migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2918	}
2919
2920	/*
2921	 * No need to double call mmu_notifier->invalidate_range() callback as
2922	 * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
2923	 * did already call it.
2924	 */
2925	if (notified)
2926		mmu_notifier_invalidate_range_only_end(&range);
2927}
2928EXPORT_SYMBOL(migrate_vma_pages);
2929
2930/**
2931 * migrate_vma_finalize() - restore CPU page table entry
2932 * @migrate: migrate struct containing all migration information
2933 *
2934 * This replaces the special migration pte entry with either a mapping to the
2935 * new page if migration was successful for that page, or to the original page
2936 * otherwise.
2937 *
2938 * This also unlocks the pages and puts them back on the lru, or drops the extra
2939 * refcount, for device pages.
2940 */
2941void migrate_vma_finalize(struct migrate_vma *migrate)
2942{
2943	const unsigned long npages = migrate->npages;
2944	unsigned long i;
2945
2946	for (i = 0; i < npages; i++) {
2947		struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2948		struct page *page = migrate_pfn_to_page(migrate->src[i]);
2949
2950		if (!page) {
2951			if (newpage) {
2952				unlock_page(newpage);
2953				put_page(newpage);
2954			}
2955			continue;
2956		}
2957
2958		if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
2959			if (newpage) {
2960				unlock_page(newpage);
2961				put_page(newpage);
2962			}
2963			newpage = page;
2964		}
2965
2966		remove_migration_ptes(page, newpage, false);
2967		unlock_page(page);
 
2968
2969		if (is_zone_device_page(page))
2970			put_page(page);
2971		else
2972			putback_lru_page(page);
2973
2974		if (newpage != page) {
2975			unlock_page(newpage);
2976			if (is_zone_device_page(newpage))
2977				put_page(newpage);
2978			else
2979				putback_lru_page(newpage);
2980		}
2981	}
2982}
2983EXPORT_SYMBOL(migrate_vma_finalize);
2984#endif /* CONFIG_DEVICE_PRIVATE */