1/*
   2 * Copyright (C) 2008, 2009 Intel Corporation
   3 * Authors: Andi Kleen, Fengguang Wu
   4 *
   5 * This software may be redistributed and/or modified under the terms of
   6 * the GNU General Public License ("GPL") version 2 only as published by the
   7 * Free Software Foundation.
   8 *
   9 * High level machine check handler. Handles pages reported by the
  10 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
  11 * failure.
  12 * 
  13 * In addition there is a "soft offline" entry point that allows stop using
  14 * not-yet-corrupted-by-suspicious pages without killing anything.
  15 *
  16 * Handles page cache pages in various states.	The tricky part
  17 * here is that we can access any page asynchronously in respect to 
  18 * other VM users, because memory failures could happen anytime and 
  19 * anywhere. This could violate some of their assumptions. This is why 
  20 * this code has to be extremely careful. Generally it tries to use 
  21 * normal locking rules, as in get the standard locks, even if that means 
  22 * the error handling takes potentially a long time.
  23 * 
  24 * There are several operations here with exponential complexity because
  25 * of unsuitable VM data structures. For example the operation to map back 
  26 * from RMAP chains to processes has to walk the complete process list and 
  27 * has non linear complexity with the number. But since memory corruptions
  28 * are rare we hope to get away with this. This avoids impacting the core 
  29 * VM.
  30 */
  31
  32/*
  33 * Notebook:
  34 * - hugetlb needs more code
  35 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
  36 * - pass bad pages to kdump next kernel
  37 */
  38#include <linux/kernel.h>
  39#include <linux/mm.h>
  40#include <linux/page-flags.h>
  41#include <linux/kernel-page-flags.h>
  42#include <linux/sched.h>
  43#include <linux/ksm.h>
  44#include <linux/rmap.h>
  45#include <linux/export.h>
  46#include <linux/pagemap.h>
  47#include <linux/swap.h>
  48#include <linux/backing-dev.h>
  49#include <linux/migrate.h>
  50#include <linux/page-isolation.h>
  51#include <linux/suspend.h>
  52#include <linux/slab.h>
  53#include <linux/swapops.h>
  54#include <linux/hugetlb.h>
  55#include <linux/memory_hotplug.h>
  56#include <linux/mm_inline.h>
  57#include <linux/kfifo.h>
  58#include "internal.h"
  59
  60int sysctl_memory_failure_early_kill __read_mostly = 0;
  61
  62int sysctl_memory_failure_recovery __read_mostly = 1;
  63
  64atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
  65
  66#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
  67
  68u32 hwpoison_filter_enable = 0;
  69u32 hwpoison_filter_dev_major = ~0U;
  70u32 hwpoison_filter_dev_minor = ~0U;
  71u64 hwpoison_filter_flags_mask;
  72u64 hwpoison_filter_flags_value;
  73EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
  74EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
  75EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
  76EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
  77EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
  78
  79static int hwpoison_filter_dev(struct page *p)
  80{
  81	struct address_space *mapping;
  82	dev_t dev;
  83
  84	if (hwpoison_filter_dev_major == ~0U &&
  85	    hwpoison_filter_dev_minor == ~0U)
  86		return 0;
  87
  88	/*
  89	 * page_mapping() does not accept slab pages.
  90	 */
  91	if (PageSlab(p))
  92		return -EINVAL;
  93
  94	mapping = page_mapping(p);
  95	if (mapping == NULL || mapping->host == NULL)
  96		return -EINVAL;
  97
  98	dev = mapping->host->i_sb->s_dev;
  99	if (hwpoison_filter_dev_major != ~0U &&
 100	    hwpoison_filter_dev_major != MAJOR(dev))
 101		return -EINVAL;
 102	if (hwpoison_filter_dev_minor != ~0U &&
 103	    hwpoison_filter_dev_minor != MINOR(dev))
 104		return -EINVAL;
 105
 106	return 0;
 107}
 108
 109static int hwpoison_filter_flags(struct page *p)
 110{
 111	if (!hwpoison_filter_flags_mask)
 112		return 0;
 113
 114	if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
 115				    hwpoison_filter_flags_value)
 116		return 0;
 117	else
 118		return -EINVAL;
 119}
 120
 121/*
 122 * This allows stress tests to limit test scope to a collection of tasks
 123 * by putting them under some memcg. This prevents killing unrelated/important
 124 * processes such as /sbin/init. Note that the target task may share clean
 125 * pages with init (eg. libc text), which is harmless. If the target task
 126 * share _dirty_ pages with another task B, the test scheme must make sure B
 127 * is also included in the memcg. At last, due to race conditions this filter
 128 * can only guarantee that the page either belongs to the memcg tasks, or is
 129 * a freed page.
 130 */
 131#ifdef	CONFIG_MEMCG_SWAP
 132u64 hwpoison_filter_memcg;
 133EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
 134static int hwpoison_filter_task(struct page *p)
 135{
 136	struct mem_cgroup *mem;
 137	struct cgroup_subsys_state *css;
 138	unsigned long ino;
 139
 140	if (!hwpoison_filter_memcg)
 141		return 0;
 142
 143	mem = try_get_mem_cgroup_from_page(p);
 144	if (!mem)
 145		return -EINVAL;
 146
 147	css = mem_cgroup_css(mem);
 148	ino = cgroup_ino(css->cgroup);
 
 
 
 
 149	css_put(css);
 150
 151	if (!ino || ino != hwpoison_filter_memcg)
 152		return -EINVAL;
 153
 154	return 0;
 155}
 156#else
 157static int hwpoison_filter_task(struct page *p) { return 0; }
 158#endif
 159
 160int hwpoison_filter(struct page *p)
 161{
 162	if (!hwpoison_filter_enable)
 163		return 0;
 164
 165	if (hwpoison_filter_dev(p))
 166		return -EINVAL;
 167
 168	if (hwpoison_filter_flags(p))
 169		return -EINVAL;
 170
 171	if (hwpoison_filter_task(p))
 172		return -EINVAL;
 173
 174	return 0;
 175}
 176#else
 177int hwpoison_filter(struct page *p)
 178{
 179	return 0;
 180}
 181#endif
 182
 183EXPORT_SYMBOL_GPL(hwpoison_filter);
 184
 185/*
 186 * Send all the processes who have the page mapped a signal.
 187 * ``action optional'' if they are not immediately affected by the error
 188 * ``action required'' if error happened in current execution context
 189 */
 190static int kill_proc(struct task_struct *t, unsigned long addr, int trapno,
 191			unsigned long pfn, struct page *page, int flags)
 192{
 193	struct siginfo si;
 194	int ret;
 195
 196	printk(KERN_ERR
 197		"MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
 198		pfn, t->comm, t->pid);
 199	si.si_signo = SIGBUS;
 200	si.si_errno = 0;
 201	si.si_addr = (void *)addr;
 202#ifdef __ARCH_SI_TRAPNO
 203	si.si_trapno = trapno;
 204#endif
 205	si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT;
 206
 207	if ((flags & MF_ACTION_REQUIRED) && t == current) {
 208		si.si_code = BUS_MCEERR_AR;
 209		ret = force_sig_info(SIGBUS, &si, t);
 210	} else {
 211		/*
 212		 * Don't use force here, it's convenient if the signal
 213		 * can be temporarily blocked.
 214		 * This could cause a loop when the user sets SIGBUS
 215		 * to SIG_IGN, but hopefully no one will do that?
 216		 */
 217		si.si_code = BUS_MCEERR_AO;
 218		ret = send_sig_info(SIGBUS, &si, t);  /* synchronous? */
 219	}
 220	if (ret < 0)
 221		printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
 222		       t->comm, t->pid, ret);
 223	return ret;
 224}
 225
 226/*
 227 * When a unknown page type is encountered drain as many buffers as possible
 228 * in the hope to turn the page into a LRU or free page, which we can handle.
 229 */
 230void shake_page(struct page *p, int access)
 231{
 232	if (!PageSlab(p)) {
 233		lru_add_drain_all();
 234		if (PageLRU(p))
 235			return;
 236		drain_all_pages();
 237		if (PageLRU(p) || is_free_buddy_page(p))
 238			return;
 239	}
 240
 241	/*
 242	 * Only call shrink_slab here (which would also shrink other caches) if
 243	 * access is not potentially fatal.
 244	 */
 245	if (access) {
 246		int nr;
 247		int nid = page_to_nid(p);
 248		do {
 249			struct shrink_control shrink = {
 250				.gfp_mask = GFP_KERNEL,
 251			};
 252			node_set(nid, shrink.nodes_to_scan);
 253
 254			nr = shrink_slab(&shrink, 1000, 1000);
 255			if (page_count(p) == 1)
 256				break;
 257		} while (nr > 10);
 258	}
 259}
 260EXPORT_SYMBOL_GPL(shake_page);
 261
 262/*
 263 * Kill all processes that have a poisoned page mapped and then isolate
 264 * the page.
 265 *
 266 * General strategy:
 267 * Find all processes having the page mapped and kill them.
 268 * But we keep a page reference around so that the page is not
 269 * actually freed yet.
 270 * Then stash the page away
 271 *
 272 * There's no convenient way to get back to mapped processes
 273 * from the VMAs. So do a brute-force search over all
 274 * running processes.
 275 *
 276 * Remember that machine checks are not common (or rather
 277 * if they are common you have other problems), so this shouldn't
 278 * be a performance issue.
 279 *
 280 * Also there are some races possible while we get from the
 281 * error detection to actually handle it.
 282 */
 283
 284struct to_kill {
 285	struct list_head nd;
 286	struct task_struct *tsk;
 287	unsigned long addr;
 288	char addr_valid;
 289};
 290
 291/*
 292 * Failure handling: if we can't find or can't kill a process there's
 293 * not much we can do.	We just print a message and ignore otherwise.
 294 */
 295
 296/*
 297 * Schedule a process for later kill.
 298 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
 299 * TBD would GFP_NOIO be enough?
 300 */
 301static void add_to_kill(struct task_struct *tsk, struct page *p,
 302		       struct vm_area_struct *vma,
 303		       struct list_head *to_kill,
 304		       struct to_kill **tkc)
 305{
 306	struct to_kill *tk;
 307
 308	if (*tkc) {
 309		tk = *tkc;
 310		*tkc = NULL;
 311	} else {
 312		tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
 313		if (!tk) {
 314			printk(KERN_ERR
 315		"MCE: Out of memory while machine check handling\n");
 316			return;
 317		}
 318	}
 319	tk->addr = page_address_in_vma(p, vma);
 320	tk->addr_valid = 1;
 321
 322	/*
 323	 * In theory we don't have to kill when the page was
 324	 * munmaped. But it could be also a mremap. Since that's
 325	 * likely very rare kill anyways just out of paranoia, but use
 326	 * a SIGKILL because the error is not contained anymore.
 327	 */
 328	if (tk->addr == -EFAULT) {
 329		pr_info("MCE: Unable to find user space address %lx in %s\n",
 330			page_to_pfn(p), tsk->comm);
 331		tk->addr_valid = 0;
 332	}
 333	get_task_struct(tsk);
 334	tk->tsk = tsk;
 335	list_add_tail(&tk->nd, to_kill);
 336}
 337
 338/*
 339 * Kill the processes that have been collected earlier.
 340 *
 341 * Only do anything when DOIT is set, otherwise just free the list
 342 * (this is used for clean pages which do not need killing)
 343 * Also when FAIL is set do a force kill because something went
 344 * wrong earlier.
 345 */
 346static void kill_procs(struct list_head *to_kill, int forcekill, int trapno,
 347			  int fail, struct page *page, unsigned long pfn,
 348			  int flags)
 349{
 350	struct to_kill *tk, *next;
 351
 352	list_for_each_entry_safe (tk, next, to_kill, nd) {
 353		if (forcekill) {
 354			/*
 355			 * In case something went wrong with munmapping
 356			 * make sure the process doesn't catch the
 357			 * signal and then access the memory. Just kill it.
 358			 */
 359			if (fail || tk->addr_valid == 0) {
 360				printk(KERN_ERR
 361		"MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
 362					pfn, tk->tsk->comm, tk->tsk->pid);
 363				force_sig(SIGKILL, tk->tsk);
 364			}
 365
 366			/*
 367			 * In theory the process could have mapped
 368			 * something else on the address in-between. We could
 369			 * check for that, but we need to tell the
 370			 * process anyways.
 371			 */
 372			else if (kill_proc(tk->tsk, tk->addr, trapno,
 373					      pfn, page, flags) < 0)
 374				printk(KERN_ERR
 375		"MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
 376					pfn, tk->tsk->comm, tk->tsk->pid);
 377		}
 378		put_task_struct(tk->tsk);
 379		kfree(tk);
 380	}
 381}
 382
 383static int task_early_kill(struct task_struct *tsk)
 384{
 385	if (!tsk->mm)
 386		return 0;
 387	if (tsk->flags & PF_MCE_PROCESS)
 388		return !!(tsk->flags & PF_MCE_EARLY);
 389	return sysctl_memory_failure_early_kill;
 390}
 391
 392/*
 393 * Collect processes when the error hit an anonymous page.
 394 */
 395static void collect_procs_anon(struct page *page, struct list_head *to_kill,
 396			      struct to_kill **tkc)
 397{
 398	struct vm_area_struct *vma;
 399	struct task_struct *tsk;
 400	struct anon_vma *av;
 401	pgoff_t pgoff;
 402
 403	av = page_lock_anon_vma_read(page);
 404	if (av == NULL)	/* Not actually mapped anymore */
 405		return;
 406
 407	pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
 408	read_lock(&tasklist_lock);
 409	for_each_process (tsk) {
 410		struct anon_vma_chain *vmac;
 411
 412		if (!task_early_kill(tsk))
 413			continue;
 414		anon_vma_interval_tree_foreach(vmac, &av->rb_root,
 415					       pgoff, pgoff) {
 416			vma = vmac->vma;
 417			if (!page_mapped_in_vma(page, vma))
 418				continue;
 419			if (vma->vm_mm == tsk->mm)
 420				add_to_kill(tsk, page, vma, to_kill, tkc);
 421		}
 422	}
 423	read_unlock(&tasklist_lock);
 424	page_unlock_anon_vma_read(av);
 425}
 426
 427/*
 428 * Collect processes when the error hit a file mapped page.
 429 */
 430static void collect_procs_file(struct page *page, struct list_head *to_kill,
 431			      struct to_kill **tkc)
 432{
 433	struct vm_area_struct *vma;
 434	struct task_struct *tsk;
 
 435	struct address_space *mapping = page->mapping;
 436
 437	mutex_lock(&mapping->i_mmap_mutex);
 438	read_lock(&tasklist_lock);
 439	for_each_process(tsk) {
 440		pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
 441
 442		if (!task_early_kill(tsk))
 443			continue;
 444
 445		vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
 446				      pgoff) {
 447			/*
 448			 * Send early kill signal to tasks where a vma covers
 449			 * the page but the corrupted page is not necessarily
 450			 * mapped it in its pte.
 451			 * Assume applications who requested early kill want
 452			 * to be informed of all such data corruptions.
 453			 */
 454			if (vma->vm_mm == tsk->mm)
 455				add_to_kill(tsk, page, vma, to_kill, tkc);
 456		}
 457	}
 458	read_unlock(&tasklist_lock);
 459	mutex_unlock(&mapping->i_mmap_mutex);
 460}
 461
 462/*
 463 * Collect the processes who have the corrupted page mapped to kill.
 464 * This is done in two steps for locking reasons.
 465 * First preallocate one tokill structure outside the spin locks,
 466 * so that we can kill at least one process reasonably reliable.
 467 */
 468static void collect_procs(struct page *page, struct list_head *tokill)
 469{
 470	struct to_kill *tk;
 471
 472	if (!page->mapping)
 473		return;
 474
 475	tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
 476	if (!tk)
 477		return;
 478	if (PageAnon(page))
 479		collect_procs_anon(page, tokill, &tk);
 480	else
 481		collect_procs_file(page, tokill, &tk);
 482	kfree(tk);
 483}
 484
 485/*
 486 * Error handlers for various types of pages.
 487 */
 488
 489enum outcome {
 490	IGNORED,	/* Error: cannot be handled */
 491	FAILED,		/* Error: handling failed */
 492	DELAYED,	/* Will be handled later */
 493	RECOVERED,	/* Successfully recovered */
 494};
 495
 496static const char *action_name[] = {
 497	[IGNORED] = "Ignored",
 498	[FAILED] = "Failed",
 499	[DELAYED] = "Delayed",
 500	[RECOVERED] = "Recovered",
 501};
 502
 503/*
 504 * XXX: It is possible that a page is isolated from LRU cache,
 505 * and then kept in swap cache or failed to remove from page cache.
 506 * The page count will stop it from being freed by unpoison.
 507 * Stress tests should be aware of this memory leak problem.
 508 */
 509static int delete_from_lru_cache(struct page *p)
 510{
 511	if (!isolate_lru_page(p)) {
 512		/*
 513		 * Clear sensible page flags, so that the buddy system won't
 514		 * complain when the page is unpoison-and-freed.
 515		 */
 516		ClearPageActive(p);
 517		ClearPageUnevictable(p);
 518		/*
 519		 * drop the page count elevated by isolate_lru_page()
 520		 */
 521		page_cache_release(p);
 522		return 0;
 523	}
 524	return -EIO;
 525}
 526
 527/*
 528 * Error hit kernel page.
 529 * Do nothing, try to be lucky and not touch this instead. For a few cases we
 530 * could be more sophisticated.
 531 */
 532static int me_kernel(struct page *p, unsigned long pfn)
 533{
 534	return IGNORED;
 535}
 536
 537/*
 538 * Page in unknown state. Do nothing.
 539 */
 540static int me_unknown(struct page *p, unsigned long pfn)
 541{
 542	printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
 543	return FAILED;
 544}
 545
 546/*
 547 * Clean (or cleaned) page cache page.
 548 */
 549static int me_pagecache_clean(struct page *p, unsigned long pfn)
 550{
 551	int err;
 552	int ret = FAILED;
 553	struct address_space *mapping;
 554
 555	delete_from_lru_cache(p);
 556
 557	/*
 558	 * For anonymous pages we're done the only reference left
 559	 * should be the one m_f() holds.
 560	 */
 561	if (PageAnon(p))
 562		return RECOVERED;
 563
 564	/*
 565	 * Now truncate the page in the page cache. This is really
 566	 * more like a "temporary hole punch"
 567	 * Don't do this for block devices when someone else
 568	 * has a reference, because it could be file system metadata
 569	 * and that's not safe to truncate.
 570	 */
 571	mapping = page_mapping(p);
 572	if (!mapping) {
 573		/*
 574		 * Page has been teared down in the meanwhile
 575		 */
 576		return FAILED;
 577	}
 578
 579	/*
 580	 * Truncation is a bit tricky. Enable it per file system for now.
 581	 *
 582	 * Open: to take i_mutex or not for this? Right now we don't.
 583	 */
 584	if (mapping->a_ops->error_remove_page) {
 585		err = mapping->a_ops->error_remove_page(mapping, p);
 586		if (err != 0) {
 587			printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
 588					pfn, err);
 589		} else if (page_has_private(p) &&
 590				!try_to_release_page(p, GFP_NOIO)) {
 591			pr_info("MCE %#lx: failed to release buffers\n", pfn);
 592		} else {
 593			ret = RECOVERED;
 594		}
 595	} else {
 596		/*
 597		 * If the file system doesn't support it just invalidate
 598		 * This fails on dirty or anything with private pages
 599		 */
 600		if (invalidate_inode_page(p))
 601			ret = RECOVERED;
 602		else
 603			printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
 604				pfn);
 605	}
 606	return ret;
 607}
 608
 609/*
 610 * Dirty pagecache page
 611 * Issues: when the error hit a hole page the error is not properly
 612 * propagated.
 613 */
 614static int me_pagecache_dirty(struct page *p, unsigned long pfn)
 615{
 616	struct address_space *mapping = page_mapping(p);
 617
 618	SetPageError(p);
 619	/* TBD: print more information about the file. */
 620	if (mapping) {
 621		/*
 622		 * IO error will be reported by write(), fsync(), etc.
 623		 * who check the mapping.
 624		 * This way the application knows that something went
 625		 * wrong with its dirty file data.
 626		 *
 627		 * There's one open issue:
 628		 *
 629		 * The EIO will be only reported on the next IO
 630		 * operation and then cleared through the IO map.
 631		 * Normally Linux has two mechanisms to pass IO error
 632		 * first through the AS_EIO flag in the address space
 633		 * and then through the PageError flag in the page.
 634		 * Since we drop pages on memory failure handling the
 635		 * only mechanism open to use is through AS_AIO.
 636		 *
 637		 * This has the disadvantage that it gets cleared on
 638		 * the first operation that returns an error, while
 639		 * the PageError bit is more sticky and only cleared
 640		 * when the page is reread or dropped.  If an
 641		 * application assumes it will always get error on
 642		 * fsync, but does other operations on the fd before
 643		 * and the page is dropped between then the error
 644		 * will not be properly reported.
 645		 *
 646		 * This can already happen even without hwpoisoned
 647		 * pages: first on metadata IO errors (which only
 648		 * report through AS_EIO) or when the page is dropped
 649		 * at the wrong time.
 650		 *
 651		 * So right now we assume that the application DTRT on
 652		 * the first EIO, but we're not worse than other parts
 653		 * of the kernel.
 654		 */
 655		mapping_set_error(mapping, EIO);
 656	}
 657
 658	return me_pagecache_clean(p, pfn);
 659}
 660
 661/*
 662 * Clean and dirty swap cache.
 663 *
 664 * Dirty swap cache page is tricky to handle. The page could live both in page
 665 * cache and swap cache(ie. page is freshly swapped in). So it could be
 666 * referenced concurrently by 2 types of PTEs:
 667 * normal PTEs and swap PTEs. We try to handle them consistently by calling
 668 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
 669 * and then
 670 *      - clear dirty bit to prevent IO
 671 *      - remove from LRU
 672 *      - but keep in the swap cache, so that when we return to it on
 673 *        a later page fault, we know the application is accessing
 674 *        corrupted data and shall be killed (we installed simple
 675 *        interception code in do_swap_page to catch it).
 676 *
 677 * Clean swap cache pages can be directly isolated. A later page fault will
 678 * bring in the known good data from disk.
 679 */
 680static int me_swapcache_dirty(struct page *p, unsigned long pfn)
 681{
 682	ClearPageDirty(p);
 683	/* Trigger EIO in shmem: */
 684	ClearPageUptodate(p);
 685
 686	if (!delete_from_lru_cache(p))
 687		return DELAYED;
 688	else
 689		return FAILED;
 690}
 691
 692static int me_swapcache_clean(struct page *p, unsigned long pfn)
 693{
 694	delete_from_swap_cache(p);
 695
 696	if (!delete_from_lru_cache(p))
 697		return RECOVERED;
 698	else
 699		return FAILED;
 700}
 701
 702/*
 703 * Huge pages. Needs work.
 704 * Issues:
 705 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
 706 *   To narrow down kill region to one page, we need to break up pmd.
 707 */
 708static int me_huge_page(struct page *p, unsigned long pfn)
 709{
 710	int res = 0;
 711	struct page *hpage = compound_head(p);
 712	/*
 713	 * We can safely recover from error on free or reserved (i.e.
 714	 * not in-use) hugepage by dequeuing it from freelist.
 715	 * To check whether a hugepage is in-use or not, we can't use
 716	 * page->lru because it can be used in other hugepage operations,
 717	 * such as __unmap_hugepage_range() and gather_surplus_pages().
 718	 * So instead we use page_mapping() and PageAnon().
 719	 * We assume that this function is called with page lock held,
 720	 * so there is no race between isolation and mapping/unmapping.
 721	 */
 722	if (!(page_mapping(hpage) || PageAnon(hpage))) {
 723		res = dequeue_hwpoisoned_huge_page(hpage);
 724		if (!res)
 725			return RECOVERED;
 726	}
 727	return DELAYED;
 728}
 729
 730/*
 731 * Various page states we can handle.
 732 *
 733 * A page state is defined by its current page->flags bits.
 734 * The table matches them in order and calls the right handler.
 735 *
 736 * This is quite tricky because we can access page at any time
 737 * in its live cycle, so all accesses have to be extremely careful.
 738 *
 739 * This is not complete. More states could be added.
 740 * For any missing state don't attempt recovery.
 741 */
 742
 743#define dirty		(1UL << PG_dirty)
 744#define sc		(1UL << PG_swapcache)
 745#define unevict		(1UL << PG_unevictable)
 746#define mlock		(1UL << PG_mlocked)
 747#define writeback	(1UL << PG_writeback)
 748#define lru		(1UL << PG_lru)
 749#define swapbacked	(1UL << PG_swapbacked)
 750#define head		(1UL << PG_head)
 751#define tail		(1UL << PG_tail)
 752#define compound	(1UL << PG_compound)
 753#define slab		(1UL << PG_slab)
 754#define reserved	(1UL << PG_reserved)
 755
 756static struct page_state {
 757	unsigned long mask;
 758	unsigned long res;
 759	char *msg;
 760	int (*action)(struct page *p, unsigned long pfn);
 761} error_states[] = {
 762	{ reserved,	reserved,	"reserved kernel",	me_kernel },
 763	/*
 764	 * free pages are specially detected outside this table:
 765	 * PG_buddy pages only make a small fraction of all free pages.
 766	 */
 767
 768	/*
 769	 * Could in theory check if slab page is free or if we can drop
 770	 * currently unused objects without touching them. But just
 771	 * treat it as standard kernel for now.
 772	 */
 773	{ slab,		slab,		"kernel slab",	me_kernel },
 774
 775#ifdef CONFIG_PAGEFLAGS_EXTENDED
 776	{ head,		head,		"huge",		me_huge_page },
 777	{ tail,		tail,		"huge",		me_huge_page },
 778#else
 779	{ compound,	compound,	"huge",		me_huge_page },
 780#endif
 781
 782	{ sc|dirty,	sc|dirty,	"dirty swapcache",	me_swapcache_dirty },
 783	{ sc|dirty,	sc,		"clean swapcache",	me_swapcache_clean },
 784
 785	{ mlock|dirty,	mlock|dirty,	"dirty mlocked LRU",	me_pagecache_dirty },
 786	{ mlock|dirty,	mlock,		"clean mlocked LRU",	me_pagecache_clean },
 787
 788	{ unevict|dirty, unevict|dirty,	"dirty unevictable LRU", me_pagecache_dirty },
 789	{ unevict|dirty, unevict,	"clean unevictable LRU", me_pagecache_clean },
 790
 791	{ lru|dirty,	lru|dirty,	"dirty LRU",	me_pagecache_dirty },
 792	{ lru|dirty,	lru,		"clean LRU",	me_pagecache_clean },
 793
 794	/*
 795	 * Catchall entry: must be at end.
 796	 */
 797	{ 0,		0,		"unknown page state",	me_unknown },
 798};
 799
 800#undef dirty
 801#undef sc
 802#undef unevict
 803#undef mlock
 804#undef writeback
 805#undef lru
 806#undef swapbacked
 807#undef head
 808#undef tail
 809#undef compound
 810#undef slab
 811#undef reserved
 812
 813/*
 814 * "Dirty/Clean" indication is not 100% accurate due to the possibility of
 815 * setting PG_dirty outside page lock. See also comment above set_page_dirty().
 816 */
 817static void action_result(unsigned long pfn, char *msg, int result)
 818{
 819	pr_err("MCE %#lx: %s page recovery: %s\n",
 820		pfn, msg, action_name[result]);
 
 
 
 
 821}
 822
 823static int page_action(struct page_state *ps, struct page *p,
 824			unsigned long pfn)
 825{
 826	int result;
 827	int count;
 828
 829	result = ps->action(p, pfn);
 830	action_result(pfn, ps->msg, result);
 831
 832	count = page_count(p) - 1;
 833	if (ps->action == me_swapcache_dirty && result == DELAYED)
 834		count--;
 835	if (count != 0) {
 836		printk(KERN_ERR
 837		       "MCE %#lx: %s page still referenced by %d users\n",
 838		       pfn, ps->msg, count);
 839		result = FAILED;
 840	}
 841
 842	/* Could do more checks here if page looks ok */
 843	/*
 844	 * Could adjust zone counters here to correct for the missing page.
 845	 */
 846
 847	return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY;
 848}
 849
 850/*
 851 * Do all that is necessary to remove user space mappings. Unmap
 852 * the pages and send SIGBUS to the processes if the data was dirty.
 853 */
 854static int hwpoison_user_mappings(struct page *p, unsigned long pfn,
 855				  int trapno, int flags, struct page **hpagep)
 856{
 857	enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
 858	struct address_space *mapping;
 859	LIST_HEAD(tokill);
 860	int ret;
 861	int kill = 1, forcekill;
 862	struct page *hpage = *hpagep;
 863	struct page *ppage;
 864
 865	if (PageReserved(p) || PageSlab(p))
 866		return SWAP_SUCCESS;
 867
 868	/*
 869	 * This check implies we don't kill processes if their pages
 870	 * are in the swap cache early. Those are always late kills.
 871	 */
 872	if (!page_mapped(hpage))
 873		return SWAP_SUCCESS;
 874
 875	if (PageKsm(p))
 876		return SWAP_FAIL;
 877
 878	if (PageSwapCache(p)) {
 879		printk(KERN_ERR
 880		       "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
 881		ttu |= TTU_IGNORE_HWPOISON;
 882	}
 883
 884	/*
 885	 * Propagate the dirty bit from PTEs to struct page first, because we
 886	 * need this to decide if we should kill or just drop the page.
 887	 * XXX: the dirty test could be racy: set_page_dirty() may not always
 888	 * be called inside page lock (it's recommended but not enforced).
 889	 */
 890	mapping = page_mapping(hpage);
 891	if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
 892	    mapping_cap_writeback_dirty(mapping)) {
 893		if (page_mkclean(hpage)) {
 894			SetPageDirty(hpage);
 895		} else {
 896			kill = 0;
 897			ttu |= TTU_IGNORE_HWPOISON;
 898			printk(KERN_INFO
 899	"MCE %#lx: corrupted page was clean: dropped without side effects\n",
 900				pfn);
 901		}
 902	}
 903
 904	/*
 905	 * ppage: poisoned page
 906	 *   if p is regular page(4k page)
 907	 *        ppage == real poisoned page;
 908	 *   else p is hugetlb or THP, ppage == head page.
 909	 */
 910	ppage = hpage;
 911
 912	if (PageTransHuge(hpage)) {
 913		/*
 914		 * Verify that this isn't a hugetlbfs head page, the check for
 915		 * PageAnon is just for avoid tripping a split_huge_page
 916		 * internal debug check, as split_huge_page refuses to deal with
 917		 * anything that isn't an anon page. PageAnon can't go away fro
 918		 * under us because we hold a refcount on the hpage, without a
 919		 * refcount on the hpage. split_huge_page can't be safely called
 920		 * in the first place, having a refcount on the tail isn't
 921		 * enough * to be safe.
 922		 */
 923		if (!PageHuge(hpage) && PageAnon(hpage)) {
 924			if (unlikely(split_huge_page(hpage))) {
 925				/*
 926				 * FIXME: if splitting THP is failed, it is
 927				 * better to stop the following operation rather
 928				 * than causing panic by unmapping. System might
 929				 * survive if the page is freed later.
 930				 */
 931				printk(KERN_INFO
 932					"MCE %#lx: failed to split THP\n", pfn);
 933
 934				BUG_ON(!PageHWPoison(p));
 935				return SWAP_FAIL;
 936			}
 937			/*
 938			 * We pinned the head page for hwpoison handling,
 939			 * now we split the thp and we are interested in
 940			 * the hwpoisoned raw page, so move the refcount
 941			 * to it. Similarly, page lock is shifted.
 942			 */
 943			if (hpage != p) {
 944				if (!(flags & MF_COUNT_INCREASED)) {
 945					put_page(hpage);
 946					get_page(p);
 947				}
 948				lock_page(p);
 949				unlock_page(hpage);
 950				*hpagep = p;
 951			}
 952			/* THP is split, so ppage should be the real poisoned page. */
 953			ppage = p;
 954		}
 955	}
 956
 957	/*
 958	 * First collect all the processes that have the page
 959	 * mapped in dirty form.  This has to be done before try_to_unmap,
 960	 * because ttu takes the rmap data structures down.
 961	 *
 962	 * Error handling: We ignore errors here because
 963	 * there's nothing that can be done.
 964	 */
 965	if (kill)
 966		collect_procs(ppage, &tokill);
 967
 
 
 
 968	ret = try_to_unmap(ppage, ttu);
 969	if (ret != SWAP_SUCCESS)
 970		printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
 971				pfn, page_mapcount(ppage));
 972
 
 
 
 973	/*
 974	 * Now that the dirty bit has been propagated to the
 975	 * struct page and all unmaps done we can decide if
 976	 * killing is needed or not.  Only kill when the page
 977	 * was dirty or the process is not restartable,
 978	 * otherwise the tokill list is merely
 979	 * freed.  When there was a problem unmapping earlier
 980	 * use a more force-full uncatchable kill to prevent
 981	 * any accesses to the poisoned memory.
 982	 */
 983	forcekill = PageDirty(ppage) || (flags & MF_MUST_KILL);
 984	kill_procs(&tokill, forcekill, trapno,
 985		      ret != SWAP_SUCCESS, p, pfn, flags);
 986
 987	return ret;
 988}
 989
 990static void set_page_hwpoison_huge_page(struct page *hpage)
 991{
 992	int i;
 993	int nr_pages = 1 << compound_order(hpage);
 994	for (i = 0; i < nr_pages; i++)
 995		SetPageHWPoison(hpage + i);
 996}
 997
 998static void clear_page_hwpoison_huge_page(struct page *hpage)
 999{
1000	int i;
1001	int nr_pages = 1 << compound_order(hpage);
1002	for (i = 0; i < nr_pages; i++)
1003		ClearPageHWPoison(hpage + i);
1004}
1005
1006/**
1007 * memory_failure - Handle memory failure of a page.
1008 * @pfn: Page Number of the corrupted page
1009 * @trapno: Trap number reported in the signal to user space.
1010 * @flags: fine tune action taken
1011 *
1012 * This function is called by the low level machine check code
1013 * of an architecture when it detects hardware memory corruption
1014 * of a page. It tries its best to recover, which includes
1015 * dropping pages, killing processes etc.
1016 *
1017 * The function is primarily of use for corruptions that
1018 * happen outside the current execution context (e.g. when
1019 * detected by a background scrubber)
1020 *
1021 * Must run in process context (e.g. a work queue) with interrupts
1022 * enabled and no spinlocks hold.
1023 */
1024int memory_failure(unsigned long pfn, int trapno, int flags)
1025{
1026	struct page_state *ps;
1027	struct page *p;
1028	struct page *hpage;
1029	int res;
1030	unsigned int nr_pages;
1031	unsigned long page_flags;
1032
1033	if (!sysctl_memory_failure_recovery)
1034		panic("Memory failure from trap %d on page %lx", trapno, pfn);
1035
1036	if (!pfn_valid(pfn)) {
1037		printk(KERN_ERR
1038		       "MCE %#lx: memory outside kernel control\n",
1039		       pfn);
1040		return -ENXIO;
1041	}
1042
1043	p = pfn_to_page(pfn);
1044	hpage = compound_head(p);
1045	if (TestSetPageHWPoison(p)) {
1046		printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
1047		return 0;
1048	}
1049
1050	/*
1051	 * Currently errors on hugetlbfs pages are measured in hugepage units,
1052	 * so nr_pages should be 1 << compound_order.  OTOH when errors are on
1053	 * transparent hugepages, they are supposed to be split and error
1054	 * measurement is done in normal page units.  So nr_pages should be one
1055	 * in this case.
1056	 */
1057	if (PageHuge(p))
1058		nr_pages = 1 << compound_order(hpage);
1059	else /* normal page or thp */
1060		nr_pages = 1;
1061	atomic_long_add(nr_pages, &num_poisoned_pages);
1062
1063	/*
1064	 * We need/can do nothing about count=0 pages.
1065	 * 1) it's a free page, and therefore in safe hand:
1066	 *    prep_new_page() will be the gate keeper.
1067	 * 2) it's a free hugepage, which is also safe:
1068	 *    an affected hugepage will be dequeued from hugepage freelist,
1069	 *    so there's no concern about reusing it ever after.
1070	 * 3) it's part of a non-compound high order page.
1071	 *    Implies some kernel user: cannot stop them from
1072	 *    R/W the page; let's pray that the page has been
1073	 *    used and will be freed some time later.
1074	 * In fact it's dangerous to directly bump up page count from 0,
1075	 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
1076	 */
1077	if (!(flags & MF_COUNT_INCREASED) &&
1078		!get_page_unless_zero(hpage)) {
1079		if (is_free_buddy_page(p)) {
1080			action_result(pfn, "free buddy", DELAYED);
1081			return 0;
1082		} else if (PageHuge(hpage)) {
1083			/*
1084			 * Check "filter hit" and "race with other subpage."
 
1085			 */
1086			lock_page(hpage);
1087			if (PageHWPoison(hpage)) {
1088				if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
1089				    || (p != hpage && TestSetPageHWPoison(hpage))) {
1090					atomic_long_sub(nr_pages, &num_poisoned_pages);
1091					unlock_page(hpage);
1092					return 0;
1093				}
1094			}
1095			set_page_hwpoison_huge_page(hpage);
1096			res = dequeue_hwpoisoned_huge_page(hpage);
1097			action_result(pfn, "free huge",
1098				      res ? IGNORED : DELAYED);
1099			unlock_page(hpage);
1100			return res;
1101		} else {
1102			action_result(pfn, "high order kernel", IGNORED);
1103			return -EBUSY;
1104		}
1105	}
1106
1107	/*
1108	 * We ignore non-LRU pages for good reasons.
1109	 * - PG_locked is only well defined for LRU pages and a few others
1110	 * - to avoid races with __set_page_locked()
1111	 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
1112	 * The check (unnecessarily) ignores LRU pages being isolated and
1113	 * walked by the page reclaim code, however that's not a big loss.
1114	 */
1115	if (!PageHuge(p) && !PageTransTail(p)) {
1116		if (!PageLRU(p))
1117			shake_page(p, 0);
1118		if (!PageLRU(p)) {
1119			/*
1120			 * shake_page could have turned it free.
1121			 */
1122			if (is_free_buddy_page(p)) {
1123				if (flags & MF_COUNT_INCREASED)
1124					action_result(pfn, "free buddy", DELAYED);
1125				else
1126					action_result(pfn, "free buddy, 2nd try", DELAYED);
1127				return 0;
1128			}
1129			action_result(pfn, "non LRU", IGNORED);
1130			put_page(p);
1131			return -EBUSY;
1132		}
1133	}
1134
1135	/*
1136	 * Lock the page and wait for writeback to finish.
1137	 * It's very difficult to mess with pages currently under IO
1138	 * and in many cases impossible, so we just avoid it here.
1139	 */
1140	lock_page(hpage);
1141
1142	/*
1143	 * We use page flags to determine what action should be taken, but
1144	 * the flags can be modified by the error containment action.  One
1145	 * example is an mlocked page, where PG_mlocked is cleared by
1146	 * page_remove_rmap() in try_to_unmap_one(). So to determine page status
1147	 * correctly, we save a copy of the page flags at this time.
1148	 */
1149	page_flags = p->flags;
1150
1151	/*
1152	 * unpoison always clear PG_hwpoison inside page lock
1153	 */
1154	if (!PageHWPoison(p)) {
1155		printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
1156		atomic_long_sub(nr_pages, &num_poisoned_pages);
1157		put_page(hpage);
1158		res = 0;
1159		goto out;
1160	}
1161	if (hwpoison_filter(p)) {
1162		if (TestClearPageHWPoison(p))
1163			atomic_long_sub(nr_pages, &num_poisoned_pages);
1164		unlock_page(hpage);
1165		put_page(hpage);
1166		return 0;
1167	}
1168
1169	/*
1170	 * For error on the tail page, we should set PG_hwpoison
1171	 * on the head page to show that the hugepage is hwpoisoned
1172	 */
1173	if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
1174		action_result(pfn, "hugepage already hardware poisoned",
1175				IGNORED);
1176		unlock_page(hpage);
1177		put_page(hpage);
1178		return 0;
1179	}
1180	/*
1181	 * Set PG_hwpoison on all pages in an error hugepage,
1182	 * because containment is done in hugepage unit for now.
1183	 * Since we have done TestSetPageHWPoison() for the head page with
1184	 * page lock held, we can safely set PG_hwpoison bits on tail pages.
1185	 */
1186	if (PageHuge(p))
1187		set_page_hwpoison_huge_page(hpage);
1188
1189	wait_on_page_writeback(p);
1190
1191	/*
1192	 * Now take care of user space mappings.
1193	 * Abort on fail: __delete_from_page_cache() assumes unmapped page.
1194	 *
1195	 * When the raw error page is thp tail page, hpage points to the raw
1196	 * page after thp split.
1197	 */
1198	if (hwpoison_user_mappings(p, pfn, trapno, flags, &hpage)
1199	    != SWAP_SUCCESS) {
1200		printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn);
1201		res = -EBUSY;
1202		goto out;
1203	}
1204
1205	/*
1206	 * Torn down by someone else?
1207	 */
1208	if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
1209		action_result(pfn, "already truncated LRU", IGNORED);
1210		res = -EBUSY;
1211		goto out;
1212	}
1213
1214	res = -EBUSY;
1215	/*
1216	 * The first check uses the current page flags which may not have any
1217	 * relevant information. The second check with the saved page flagss is
1218	 * carried out only if the first check can't determine the page status.
1219	 */
1220	for (ps = error_states;; ps++)
1221		if ((p->flags & ps->mask) == ps->res)
1222			break;
1223
1224	page_flags |= (p->flags & (1UL << PG_dirty));
1225
1226	if (!ps->mask)
1227		for (ps = error_states;; ps++)
1228			if ((page_flags & ps->mask) == ps->res)
1229				break;
1230	res = page_action(ps, p, pfn);
1231out:
1232	unlock_page(hpage);
1233	return res;
1234}
1235EXPORT_SYMBOL_GPL(memory_failure);
1236
1237#define MEMORY_FAILURE_FIFO_ORDER	4
1238#define MEMORY_FAILURE_FIFO_SIZE	(1 << MEMORY_FAILURE_FIFO_ORDER)
1239
1240struct memory_failure_entry {
1241	unsigned long pfn;
1242	int trapno;
1243	int flags;
1244};
1245
1246struct memory_failure_cpu {
1247	DECLARE_KFIFO(fifo, struct memory_failure_entry,
1248		      MEMORY_FAILURE_FIFO_SIZE);
1249	spinlock_t lock;
1250	struct work_struct work;
1251};
1252
1253static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
1254
1255/**
1256 * memory_failure_queue - Schedule handling memory failure of a page.
1257 * @pfn: Page Number of the corrupted page
1258 * @trapno: Trap number reported in the signal to user space.
1259 * @flags: Flags for memory failure handling
1260 *
1261 * This function is called by the low level hardware error handler
1262 * when it detects hardware memory corruption of a page. It schedules
1263 * the recovering of error page, including dropping pages, killing
1264 * processes etc.
1265 *
1266 * The function is primarily of use for corruptions that
1267 * happen outside the current execution context (e.g. when
1268 * detected by a background scrubber)
1269 *
1270 * Can run in IRQ context.
1271 */
1272void memory_failure_queue(unsigned long pfn, int trapno, int flags)
1273{
1274	struct memory_failure_cpu *mf_cpu;
1275	unsigned long proc_flags;
1276	struct memory_failure_entry entry = {
1277		.pfn =		pfn,
1278		.trapno =	trapno,
1279		.flags =	flags,
1280	};
1281
1282	mf_cpu = &get_cpu_var(memory_failure_cpu);
1283	spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1284	if (kfifo_put(&mf_cpu->fifo, entry))
1285		schedule_work_on(smp_processor_id(), &mf_cpu->work);
1286	else
1287		pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n",
1288		       pfn);
1289	spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1290	put_cpu_var(memory_failure_cpu);
1291}
1292EXPORT_SYMBOL_GPL(memory_failure_queue);
1293
1294static void memory_failure_work_func(struct work_struct *work)
1295{
1296	struct memory_failure_cpu *mf_cpu;
1297	struct memory_failure_entry entry = { 0, };
1298	unsigned long proc_flags;
1299	int gotten;
1300
1301	mf_cpu = &__get_cpu_var(memory_failure_cpu);
1302	for (;;) {
1303		spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1304		gotten = kfifo_get(&mf_cpu->fifo, &entry);
1305		spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1306		if (!gotten)
1307			break;
1308		if (entry.flags & MF_SOFT_OFFLINE)
1309			soft_offline_page(pfn_to_page(entry.pfn), entry.flags);
1310		else
1311			memory_failure(entry.pfn, entry.trapno, entry.flags);
1312	}
1313}
1314
1315static int __init memory_failure_init(void)
1316{
1317	struct memory_failure_cpu *mf_cpu;
1318	int cpu;
1319
1320	for_each_possible_cpu(cpu) {
1321		mf_cpu = &per_cpu(memory_failure_cpu, cpu);
1322		spin_lock_init(&mf_cpu->lock);
1323		INIT_KFIFO(mf_cpu->fifo);
1324		INIT_WORK(&mf_cpu->work, memory_failure_work_func);
1325	}
1326
1327	return 0;
1328}
1329core_initcall(memory_failure_init);
1330
1331/**
1332 * unpoison_memory - Unpoison a previously poisoned page
1333 * @pfn: Page number of the to be unpoisoned page
1334 *
1335 * Software-unpoison a page that has been poisoned by
1336 * memory_failure() earlier.
1337 *
1338 * This is only done on the software-level, so it only works
1339 * for linux injected failures, not real hardware failures
1340 *
1341 * Returns 0 for success, otherwise -errno.
1342 */
1343int unpoison_memory(unsigned long pfn)
1344{
1345	struct page *page;
1346	struct page *p;
1347	int freeit = 0;
1348	unsigned int nr_pages;
1349
1350	if (!pfn_valid(pfn))
1351		return -ENXIO;
1352
1353	p = pfn_to_page(pfn);
1354	page = compound_head(p);
1355
1356	if (!PageHWPoison(p)) {
1357		pr_info("MCE: Page was already unpoisoned %#lx\n", pfn);
1358		return 0;
1359	}
1360
1361	/*
1362	 * unpoison_memory() can encounter thp only when the thp is being
1363	 * worked by memory_failure() and the page lock is not held yet.
1364	 * In such case, we yield to memory_failure() and make unpoison fail.
1365	 */
1366	if (!PageHuge(page) && PageTransHuge(page)) {
1367		pr_info("MCE: Memory failure is now running on %#lx\n", pfn);
1368			return 0;
1369	}
1370
1371	nr_pages = 1 << compound_order(page);
1372
1373	if (!get_page_unless_zero(page)) {
1374		/*
1375		 * Since HWPoisoned hugepage should have non-zero refcount,
1376		 * race between memory failure and unpoison seems to happen.
1377		 * In such case unpoison fails and memory failure runs
1378		 * to the end.
1379		 */
1380		if (PageHuge(page)) {
1381			pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn);
1382			return 0;
1383		}
1384		if (TestClearPageHWPoison(p))
1385			atomic_long_dec(&num_poisoned_pages);
1386		pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
1387		return 0;
1388	}
1389
1390	lock_page(page);
1391	/*
1392	 * This test is racy because PG_hwpoison is set outside of page lock.
1393	 * That's acceptable because that won't trigger kernel panic. Instead,
1394	 * the PG_hwpoison page will be caught and isolated on the entrance to
1395	 * the free buddy page pool.
1396	 */
1397	if (TestClearPageHWPoison(page)) {
1398		pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
1399		atomic_long_sub(nr_pages, &num_poisoned_pages);
1400		freeit = 1;
1401		if (PageHuge(page))
1402			clear_page_hwpoison_huge_page(page);
1403	}
1404	unlock_page(page);
1405
1406	put_page(page);
1407	if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1))
1408		put_page(page);
1409
1410	return 0;
1411}
1412EXPORT_SYMBOL(unpoison_memory);
1413
1414static struct page *new_page(struct page *p, unsigned long private, int **x)
1415{
1416	int nid = page_to_nid(p);
1417	if (PageHuge(p))
1418		return alloc_huge_page_node(page_hstate(compound_head(p)),
1419						   nid);
1420	else
1421		return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0);
1422}
1423
1424/*
1425 * Safely get reference count of an arbitrary page.
1426 * Returns 0 for a free page, -EIO for a zero refcount page
1427 * that is not free, and 1 for any other page type.
1428 * For 1 the page is returned with increased page count, otherwise not.
1429 */
1430static int __get_any_page(struct page *p, unsigned long pfn, int flags)
1431{
1432	int ret;
1433
1434	if (flags & MF_COUNT_INCREASED)
1435		return 1;
1436
1437	/*
 
 
 
 
 
 
 
 
 
 
 
1438	 * When the target page is a free hugepage, just remove it
1439	 * from free hugepage list.
1440	 */
1441	if (!get_page_unless_zero(compound_head(p))) {
1442		if (PageHuge(p)) {
1443			pr_info("%s: %#lx free huge page\n", __func__, pfn);
1444			ret = 0;
1445		} else if (is_free_buddy_page(p)) {
1446			pr_info("%s: %#lx free buddy page\n", __func__, pfn);
 
 
1447			ret = 0;
1448		} else {
1449			pr_info("%s: %#lx: unknown zero refcount page type %lx\n",
1450				__func__, pfn, p->flags);
1451			ret = -EIO;
1452		}
1453	} else {
1454		/* Not a free page */
1455		ret = 1;
1456	}
1457	return ret;
1458}
1459
1460static int get_any_page(struct page *page, unsigned long pfn, int flags)
1461{
1462	int ret = __get_any_page(page, pfn, flags);
1463
1464	if (ret == 1 && !PageHuge(page) && !PageLRU(page)) {
1465		/*
1466		 * Try to free it.
1467		 */
1468		put_page(page);
1469		shake_page(page, 1);
1470
1471		/*
1472		 * Did it turn free?
1473		 */
1474		ret = __get_any_page(page, pfn, 0);
1475		if (!PageLRU(page)) {
1476			pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n",
1477				pfn, page->flags);
1478			return -EIO;
1479		}
1480	}
1481	return ret;
1482}
1483
1484static int soft_offline_huge_page(struct page *page, int flags)
1485{
1486	int ret;
1487	unsigned long pfn = page_to_pfn(page);
1488	struct page *hpage = compound_head(page);
1489	LIST_HEAD(pagelist);
1490
1491	/*
1492	 * This double-check of PageHWPoison is to avoid the race with
1493	 * memory_failure(). See also comment in __soft_offline_page().
1494	 */
1495	lock_page(hpage);
 
1496	if (PageHWPoison(hpage)) {
1497		unlock_page(hpage);
1498		put_page(hpage);
1499		pr_info("soft offline: %#lx hugepage already poisoned\n", pfn);
1500		return -EBUSY;
1501	}
1502	unlock_page(hpage);
1503
1504	/* Keep page count to indicate a given hugepage is isolated. */
1505	list_move(&hpage->lru, &pagelist);
1506	ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
1507				MIGRATE_SYNC, MR_MEMORY_FAILURE);
 
1508	if (ret) {
 
 
 
 
1509		pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
1510			pfn, ret, page->flags);
1511		/*
1512		 * We know that soft_offline_huge_page() tries to migrate
1513		 * only one hugepage pointed to by hpage, so we need not
1514		 * run through the pagelist here.
1515		 */
1516		putback_active_hugepage(hpage);
1517		if (ret > 0)
1518			ret = -EIO;
1519	} else {
1520		/* overcommit hugetlb page will be freed to buddy */
1521		if (PageHuge(page)) {
1522			set_page_hwpoison_huge_page(hpage);
1523			dequeue_hwpoisoned_huge_page(hpage);
1524			atomic_long_add(1 << compound_order(hpage),
1525					&num_poisoned_pages);
1526		} else {
1527			SetPageHWPoison(page);
1528			atomic_long_inc(&num_poisoned_pages);
1529		}
1530	}
 
 
 
 
 
 
1531	return ret;
1532}
1533
1534static int __soft_offline_page(struct page *page, int flags)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1535{
1536	int ret;
1537	unsigned long pfn = page_to_pfn(page);
1538
 
 
 
 
 
 
 
 
 
1539	/*
1540	 * Check PageHWPoison again inside page lock because PageHWPoison
1541	 * is set by memory_failure() outside page lock. Note that
1542	 * memory_failure() also double-checks PageHWPoison inside page lock,
1543	 * so there's no race between soft_offline_page() and memory_failure().
1544	 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1545	lock_page(page);
1546	wait_on_page_writeback(page);
 
 
 
 
1547	if (PageHWPoison(page)) {
1548		unlock_page(page);
1549		put_page(page);
1550		pr_info("soft offline: %#lx page already poisoned\n", pfn);
1551		return -EBUSY;
1552	}
 
1553	/*
1554	 * Try to invalidate first. This should work for
1555	 * non dirty unmapped page cache pages.
1556	 */
1557	ret = invalidate_inode_page(page);
1558	unlock_page(page);
1559	/*
1560	 * RED-PEN would be better to keep it isolated here, but we
1561	 * would need to fix isolation locking first.
1562	 */
1563	if (ret == 1) {
1564		put_page(page);
 
1565		pr_info("soft_offline: %#lx: invalidated\n", pfn);
1566		SetPageHWPoison(page);
1567		atomic_long_inc(&num_poisoned_pages);
1568		return 0;
1569	}
1570
1571	/*
1572	 * Simple invalidation didn't work.
1573	 * Try to migrate to a new page instead. migrate.c
1574	 * handles a large number of cases for us.
1575	 */
1576	ret = isolate_lru_page(page);
1577	/*
1578	 * Drop page reference which is came from get_any_page()
1579	 * successful isolate_lru_page() already took another one.
1580	 */
1581	put_page(page);
1582	if (!ret) {
1583		LIST_HEAD(pagelist);
1584		inc_zone_page_state(page, NR_ISOLATED_ANON +
1585					page_is_file_cache(page));
1586		list_add(&page->lru, &pagelist);
1587		ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
1588					MIGRATE_SYNC, MR_MEMORY_FAILURE);
1589		if (ret) {
1590			if (!list_empty(&pagelist)) {
1591				list_del(&page->lru);
1592				dec_zone_page_state(page, NR_ISOLATED_ANON +
1593						page_is_file_cache(page));
1594				putback_lru_page(page);
1595			}
1596
1597			pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
1598				pfn, ret, page->flags);
1599			if (ret > 0)
1600				ret = -EIO;
1601		} else {
1602			/*
1603			 * After page migration succeeds, the source page can
1604			 * be trapped in pagevec and actual freeing is delayed.
1605			 * Freeing code works differently based on PG_hwpoison,
1606			 * so there's a race. We need to make sure that the
1607			 * source page should be freed back to buddy before
1608			 * setting PG_hwpoison.
1609			 */
1610			if (!is_free_buddy_page(page))
1611				lru_add_drain_all();
1612			if (!is_free_buddy_page(page))
1613				drain_all_pages();
1614			SetPageHWPoison(page);
1615			if (!is_free_buddy_page(page))
1616				pr_info("soft offline: %#lx: page leaked\n",
1617					pfn);
1618			atomic_long_inc(&num_poisoned_pages);
1619		}
1620	} else {
1621		pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
1622			pfn, ret, page_count(page), page->flags);
1623	}
1624	return ret;
1625}
1626
1627/**
1628 * soft_offline_page - Soft offline a page.
1629 * @page: page to offline
1630 * @flags: flags. Same as memory_failure().
1631 *
1632 * Returns 0 on success, otherwise negated errno.
1633 *
1634 * Soft offline a page, by migration or invalidation,
1635 * without killing anything. This is for the case when
1636 * a page is not corrupted yet (so it's still valid to access),
1637 * but has had a number of corrected errors and is better taken
1638 * out.
1639 *
1640 * The actual policy on when to do that is maintained by
1641 * user space.
1642 *
1643 * This should never impact any application or cause data loss,
1644 * however it might take some time.
1645 *
1646 * This is not a 100% solution for all memory, but tries to be
1647 * ``good enough'' for the majority of memory.
1648 */
1649int soft_offline_page(struct page *page, int flags)
1650{
1651	int ret;
1652	unsigned long pfn = page_to_pfn(page);
1653	struct page *hpage = compound_head(page);
1654
1655	if (PageHWPoison(page)) {
1656		pr_info("soft offline: %#lx page already poisoned\n", pfn);
1657		return -EBUSY;
1658	}
1659	if (!PageHuge(page) && PageTransHuge(hpage)) {
1660		if (PageAnon(hpage) && unlikely(split_huge_page(hpage))) {
1661			pr_info("soft offline: %#lx: failed to split THP\n",
1662				pfn);
1663			return -EBUSY;
1664		}
1665	}
1666
1667	/*
1668	 * The lock_memory_hotplug prevents a race with memory hotplug.
1669	 * This is a big hammer, a better would be nicer.
1670	 */
1671	lock_memory_hotplug();
1672
1673	/*
1674	 * Isolate the page, so that it doesn't get reallocated if it
1675	 * was free. This flag should be kept set until the source page
1676	 * is freed and PG_hwpoison on it is set.
1677	 */
1678	if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
1679		set_migratetype_isolate(page, true);
1680
1681	ret = get_any_page(page, pfn, flags);
1682	unlock_memory_hotplug();
1683	if (ret > 0) { /* for in-use pages */
1684		if (PageHuge(page))
1685			ret = soft_offline_huge_page(page, flags);
1686		else
1687			ret = __soft_offline_page(page, flags);
1688	} else if (ret == 0) { /* for free pages */
1689		if (PageHuge(page)) {
1690			set_page_hwpoison_huge_page(hpage);
1691			dequeue_hwpoisoned_huge_page(hpage);
1692			atomic_long_add(1 << compound_order(hpage),
1693					&num_poisoned_pages);
1694		} else {
1695			SetPageHWPoison(page);
1696			atomic_long_inc(&num_poisoned_pages);
1697		}
1698	}
1699	unset_migratetype_isolate(page, MIGRATE_MOVABLE);
1700	return ret;
1701}