1/*
   2 * Copyright (C) 2008, 2009 Intel Corporation
   3 * Authors: Andi Kleen, Fengguang Wu
   4 *
   5 * This software may be redistributed and/or modified under the terms of
   6 * the GNU General Public License ("GPL") version 2 only as published by the
   7 * Free Software Foundation.
   8 *
   9 * High level machine check handler. Handles pages reported by the
  10 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
  11 * failure.
  12 * 
  13 * In addition there is a "soft offline" entry point that allows stop using
  14 * not-yet-corrupted-by-suspicious pages without killing anything.
  15 *
  16 * Handles page cache pages in various states.	The tricky part
  17 * here is that we can access any page asynchronously in respect to 
  18 * other VM users, because memory failures could happen anytime and 
  19 * anywhere. This could violate some of their assumptions. This is why 
  20 * this code has to be extremely careful. Generally it tries to use 
  21 * normal locking rules, as in get the standard locks, even if that means 
  22 * the error handling takes potentially a long time.
 
 
 
 
 
 
 
 
  23 * 
  24 * There are several operations here with exponential complexity because
  25 * of unsuitable VM data structures. For example the operation to map back 
  26 * from RMAP chains to processes has to walk the complete process list and 
  27 * has non linear complexity with the number. But since memory corruptions
  28 * are rare we hope to get away with this. This avoids impacting the core 
  29 * VM.
  30 */
  31
  32/*
  33 * Notebook:
  34 * - hugetlb needs more code
  35 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
  36 * - pass bad pages to kdump next kernel
  37 */
  38#include <linux/kernel.h>
  39#include <linux/mm.h>
  40#include <linux/page-flags.h>
  41#include <linux/kernel-page-flags.h>
  42#include <linux/sched.h>
 
  43#include <linux/ksm.h>
  44#include <linux/rmap.h>
 
  45#include <linux/pagemap.h>
  46#include <linux/swap.h>
  47#include <linux/backing-dev.h>
  48#include <linux/migrate.h>
  49#include <linux/page-isolation.h>
  50#include <linux/suspend.h>
  51#include <linux/slab.h>
  52#include <linux/swapops.h>
  53#include <linux/hugetlb.h>
  54#include <linux/memory_hotplug.h>
  55#include <linux/mm_inline.h>
 
  56#include <linux/kfifo.h>
 
 
 
  57#include "internal.h"
 
  58
  59int sysctl_memory_failure_early_kill __read_mostly = 0;
  60
  61int sysctl_memory_failure_recovery __read_mostly = 1;
  62
  63atomic_long_t mce_bad_pages __read_mostly = ATOMIC_LONG_INIT(0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  64
  65#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
  66
  67u32 hwpoison_filter_enable = 0;
  68u32 hwpoison_filter_dev_major = ~0U;
  69u32 hwpoison_filter_dev_minor = ~0U;
  70u64 hwpoison_filter_flags_mask;
  71u64 hwpoison_filter_flags_value;
  72EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
  73EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
  74EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
  75EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
  76EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
  77
  78static int hwpoison_filter_dev(struct page *p)
  79{
  80	struct address_space *mapping;
  81	dev_t dev;
  82
  83	if (hwpoison_filter_dev_major == ~0U &&
  84	    hwpoison_filter_dev_minor == ~0U)
  85		return 0;
  86
  87	/*
  88	 * page_mapping() does not accept slab pages.
  89	 */
  90	if (PageSlab(p))
  91		return -EINVAL;
  92
  93	mapping = page_mapping(p);
  94	if (mapping == NULL || mapping->host == NULL)
  95		return -EINVAL;
  96
  97	dev = mapping->host->i_sb->s_dev;
  98	if (hwpoison_filter_dev_major != ~0U &&
  99	    hwpoison_filter_dev_major != MAJOR(dev))
 100		return -EINVAL;
 101	if (hwpoison_filter_dev_minor != ~0U &&
 102	    hwpoison_filter_dev_minor != MINOR(dev))
 103		return -EINVAL;
 104
 105	return 0;
 106}
 107
 108static int hwpoison_filter_flags(struct page *p)
 109{
 110	if (!hwpoison_filter_flags_mask)
 111		return 0;
 112
 113	if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
 114				    hwpoison_filter_flags_value)
 115		return 0;
 116	else
 117		return -EINVAL;
 118}
 119
 120/*
 121 * This allows stress tests to limit test scope to a collection of tasks
 122 * by putting them under some memcg. This prevents killing unrelated/important
 123 * processes such as /sbin/init. Note that the target task may share clean
 124 * pages with init (eg. libc text), which is harmless. If the target task
 125 * share _dirty_ pages with another task B, the test scheme must make sure B
 126 * is also included in the memcg. At last, due to race conditions this filter
 127 * can only guarantee that the page either belongs to the memcg tasks, or is
 128 * a freed page.
 129 */
 130#ifdef	CONFIG_CGROUP_MEM_RES_CTLR_SWAP
 131u64 hwpoison_filter_memcg;
 132EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
 133static int hwpoison_filter_task(struct page *p)
 134{
 135	struct mem_cgroup *mem;
 136	struct cgroup_subsys_state *css;
 137	unsigned long ino;
 138
 139	if (!hwpoison_filter_memcg)
 140		return 0;
 141
 142	mem = try_get_mem_cgroup_from_page(p);
 143	if (!mem)
 144		return -EINVAL;
 145
 146	css = mem_cgroup_css(mem);
 147	/* root_mem_cgroup has NULL dentries */
 148	if (!css->cgroup->dentry)
 149		return -EINVAL;
 150
 151	ino = css->cgroup->dentry->d_inode->i_ino;
 152	css_put(css);
 153
 154	if (ino != hwpoison_filter_memcg)
 155		return -EINVAL;
 156
 157	return 0;
 158}
 159#else
 160static int hwpoison_filter_task(struct page *p) { return 0; }
 161#endif
 162
 163int hwpoison_filter(struct page *p)
 164{
 165	if (!hwpoison_filter_enable)
 166		return 0;
 167
 168	if (hwpoison_filter_dev(p))
 169		return -EINVAL;
 170
 171	if (hwpoison_filter_flags(p))
 172		return -EINVAL;
 173
 174	if (hwpoison_filter_task(p))
 175		return -EINVAL;
 176
 177	return 0;
 178}
 179#else
 180int hwpoison_filter(struct page *p)
 181{
 182	return 0;
 183}
 184#endif
 185
 186EXPORT_SYMBOL_GPL(hwpoison_filter);
 187
 188/*
 189 * Send all the processes who have the page mapped an ``action optional''
 190 * signal.
 191 */
 192static int kill_proc_ao(struct task_struct *t, unsigned long addr, int trapno,
 193			unsigned long pfn, struct page *page)
 194{
 195	struct siginfo si;
 196	int ret;
 197
 198	printk(KERN_ERR
 199		"MCE %#lx: Killing %s:%d early due to hardware memory corruption\n",
 200		pfn, t->comm, t->pid);
 201	si.si_signo = SIGBUS;
 202	si.si_errno = 0;
 203	si.si_code = BUS_MCEERR_AO;
 204	si.si_addr = (void *)addr;
 205#ifdef __ARCH_SI_TRAPNO
 206	si.si_trapno = trapno;
 207#endif
 208	si.si_addr_lsb = compound_trans_order(compound_head(page)) + PAGE_SHIFT;
 209	/*
 210	 * Don't use force here, it's convenient if the signal
 211	 * can be temporarily blocked.
 212	 * This could cause a loop when the user sets SIGBUS
 213	 * to SIG_IGN, but hopefully no one will do that?
 214	 */
 215	ret = send_sig_info(SIGBUS, &si, t);  /* synchronous? */
 216	if (ret < 0)
 217		printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
 218		       t->comm, t->pid, ret);
 219	return ret;
 220}
 221
 222/*
 223 * When a unknown page type is encountered drain as many buffers as possible
 224 * in the hope to turn the page into a LRU or free page, which we can handle.
 225 */
 226void shake_page(struct page *p, int access)
 227{
 228	if (!PageSlab(p)) {
 229		lru_add_drain_all();
 230		if (PageLRU(p))
 231			return;
 232		drain_all_pages();
 233		if (PageLRU(p) || is_free_buddy_page(p))
 234			return;
 235	}
 236
 237	/*
 238	 * Only call shrink_slab here (which would also shrink other caches) if
 239	 * access is not potentially fatal.
 240	 */
 241	if (access) {
 242		int nr;
 243		do {
 244			struct shrink_control shrink = {
 245				.gfp_mask = GFP_KERNEL,
 246			};
 247
 248			nr = shrink_slab(&shrink, 1000, 1000);
 249			if (page_count(p) == 1)
 250				break;
 251		} while (nr > 10);
 252	}
 253}
 254EXPORT_SYMBOL_GPL(shake_page);
 255
 256/*
 257 * Kill all processes that have a poisoned page mapped and then isolate
 258 * the page.
 259 *
 260 * General strategy:
 261 * Find all processes having the page mapped and kill them.
 262 * But we keep a page reference around so that the page is not
 263 * actually freed yet.
 264 * Then stash the page away
 265 *
 266 * There's no convenient way to get back to mapped processes
 267 * from the VMAs. So do a brute-force search over all
 268 * running processes.
 269 *
 270 * Remember that machine checks are not common (or rather
 271 * if they are common you have other problems), so this shouldn't
 272 * be a performance issue.
 273 *
 274 * Also there are some races possible while we get from the
 275 * error detection to actually handle it.
 276 */
 277
 278struct to_kill {
 279	struct list_head nd;
 280	struct task_struct *tsk;
 281	unsigned long addr;
 282	char addr_valid;
 283};
 284
 285/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 286 * Failure handling: if we can't find or can't kill a process there's
 287 * not much we can do.	We just print a message and ignore otherwise.
 288 */
 289
 290/*
 291 * Schedule a process for later kill.
 292 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
 293 * TBD would GFP_NOIO be enough?
 294 */
 295static void add_to_kill(struct task_struct *tsk, struct page *p,
 296		       struct vm_area_struct *vma,
 297		       struct list_head *to_kill,
 298		       struct to_kill **tkc)
 299{
 300	struct to_kill *tk;
 301
 302	if (*tkc) {
 303		tk = *tkc;
 304		*tkc = NULL;
 305	} else {
 306		tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
 307		if (!tk) {
 308			printk(KERN_ERR
 309		"MCE: Out of memory while machine check handling\n");
 310			return;
 311		}
 312	}
 
 313	tk->addr = page_address_in_vma(p, vma);
 314	tk->addr_valid = 1;
 
 
 
 315
 316	/*
 317	 * In theory we don't have to kill when the page was
 318	 * munmaped. But it could be also a mremap. Since that's
 319	 * likely very rare kill anyways just out of paranoia, but use
 320	 * a SIGKILL because the error is not contained anymore.
 
 
 
 
 321	 */
 322	if (tk->addr == -EFAULT) {
 323		pr_info("MCE: Unable to find user space address %lx in %s\n",
 324			page_to_pfn(p), tsk->comm);
 325		tk->addr_valid = 0;
 
 
 326	}
 
 327	get_task_struct(tsk);
 328	tk->tsk = tsk;
 329	list_add_tail(&tk->nd, to_kill);
 330}
 331
 332/*
 333 * Kill the processes that have been collected earlier.
 334 *
 335 * Only do anything when DOIT is set, otherwise just free the list
 336 * (this is used for clean pages which do not need killing)
 337 * Also when FAIL is set do a force kill because something went
 338 * wrong earlier.
 339 */
 340static void kill_procs_ao(struct list_head *to_kill, int doit, int trapno,
 341			  int fail, struct page *page, unsigned long pfn)
 342{
 343	struct to_kill *tk, *next;
 344
 345	list_for_each_entry_safe (tk, next, to_kill, nd) {
 346		if (doit) {
 347			/*
 348			 * In case something went wrong with munmapping
 349			 * make sure the process doesn't catch the
 350			 * signal and then access the memory. Just kill it.
 351			 */
 352			if (fail || tk->addr_valid == 0) {
 353				printk(KERN_ERR
 354		"MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
 355					pfn, tk->tsk->comm, tk->tsk->pid);
 356				force_sig(SIGKILL, tk->tsk);
 357			}
 358
 359			/*
 360			 * In theory the process could have mapped
 361			 * something else on the address in-between. We could
 362			 * check for that, but we need to tell the
 363			 * process anyways.
 364			 */
 365			else if (kill_proc_ao(tk->tsk, tk->addr, trapno,
 366					      pfn, page) < 0)
 367				printk(KERN_ERR
 368		"MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
 369					pfn, tk->tsk->comm, tk->tsk->pid);
 370		}
 371		put_task_struct(tk->tsk);
 372		kfree(tk);
 373	}
 374}
 375
 376static int task_early_kill(struct task_struct *tsk)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 377{
 378	if (!tsk->mm)
 379		return 0;
 380	if (tsk->flags & PF_MCE_PROCESS)
 381		return !!(tsk->flags & PF_MCE_EARLY);
 382	return sysctl_memory_failure_early_kill;
 
 
 
 
 
 383}
 384
 385/*
 386 * Collect processes when the error hit an anonymous page.
 387 */
 388static void collect_procs_anon(struct page *page, struct list_head *to_kill,
 389			      struct to_kill **tkc)
 390{
 391	struct vm_area_struct *vma;
 392	struct task_struct *tsk;
 393	struct anon_vma *av;
 
 394
 395	av = page_lock_anon_vma(page);
 396	if (av == NULL)	/* Not actually mapped anymore */
 397		return;
 398
 
 399	read_lock(&tasklist_lock);
 400	for_each_process (tsk) {
 401		struct anon_vma_chain *vmac;
 
 402
 403		if (!task_early_kill(tsk))
 404			continue;
 405		list_for_each_entry(vmac, &av->head, same_anon_vma) {
 
 406			vma = vmac->vma;
 407			if (!page_mapped_in_vma(page, vma))
 408				continue;
 409			if (vma->vm_mm == tsk->mm)
 410				add_to_kill(tsk, page, vma, to_kill, tkc);
 411		}
 412	}
 413	read_unlock(&tasklist_lock);
 414	page_unlock_anon_vma(av);
 415}
 416
 417/*
 418 * Collect processes when the error hit a file mapped page.
 419 */
 420static void collect_procs_file(struct page *page, struct list_head *to_kill,
 421			      struct to_kill **tkc)
 422{
 423	struct vm_area_struct *vma;
 424	struct task_struct *tsk;
 425	struct prio_tree_iter iter;
 426	struct address_space *mapping = page->mapping;
 
 427
 428	mutex_lock(&mapping->i_mmap_mutex);
 429	read_lock(&tasklist_lock);
 
 430	for_each_process(tsk) {
 431		pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
 432
 433		if (!task_early_kill(tsk))
 434			continue;
 435
 436		vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff,
 437				      pgoff) {
 438			/*
 439			 * Send early kill signal to tasks where a vma covers
 440			 * the page but the corrupted page is not necessarily
 441			 * mapped it in its pte.
 442			 * Assume applications who requested early kill want
 443			 * to be informed of all such data corruptions.
 444			 */
 445			if (vma->vm_mm == tsk->mm)
 446				add_to_kill(tsk, page, vma, to_kill, tkc);
 447		}
 448	}
 449	read_unlock(&tasklist_lock);
 450	mutex_unlock(&mapping->i_mmap_mutex);
 451}
 452
 453/*
 454 * Collect the processes who have the corrupted page mapped to kill.
 455 * This is done in two steps for locking reasons.
 456 * First preallocate one tokill structure outside the spin locks,
 457 * so that we can kill at least one process reasonably reliable.
 458 */
 459static void collect_procs(struct page *page, struct list_head *tokill)
 
 460{
 461	struct to_kill *tk;
 462
 463	if (!page->mapping)
 464		return;
 465
 466	tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
 467	if (!tk)
 468		return;
 469	if (PageAnon(page))
 470		collect_procs_anon(page, tokill, &tk);
 471	else
 472		collect_procs_file(page, tokill, &tk);
 473	kfree(tk);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 474}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 475
 476/*
 477 * Error handlers for various types of pages.
 
 
 
 
 
 
 
 
 
 
 478 */
 
 
 
 
 
 
 
 
 479
 480enum outcome {
 481	IGNORED,	/* Error: cannot be handled */
 482	FAILED,		/* Error: handling failed */
 483	DELAYED,	/* Will be handled later */
 484	RECOVERED,	/* Successfully recovered */
 485};
 
 
 486
 487static const char *action_name[] = {
 488	[IGNORED] = "Ignored",
 489	[FAILED] = "Failed",
 490	[DELAYED] = "Delayed",
 491	[RECOVERED] = "Recovered",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 492};
 493
 494/*
 495 * XXX: It is possible that a page is isolated from LRU cache,
 496 * and then kept in swap cache or failed to remove from page cache.
 497 * The page count will stop it from being freed by unpoison.
 498 * Stress tests should be aware of this memory leak problem.
 499 */
 500static int delete_from_lru_cache(struct page *p)
 501{
 502	if (!isolate_lru_page(p)) {
 503		/*
 504		 * Clear sensible page flags, so that the buddy system won't
 505		 * complain when the page is unpoison-and-freed.
 506		 */
 507		ClearPageActive(p);
 508		ClearPageUnevictable(p);
 
 
 
 
 
 
 
 509		/*
 510		 * drop the page count elevated by isolate_lru_page()
 511		 */
 512		page_cache_release(p);
 513		return 0;
 514	}
 515	return -EIO;
 516}
 517
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 518/*
 519 * Error hit kernel page.
 520 * Do nothing, try to be lucky and not touch this instead. For a few cases we
 521 * could be more sophisticated.
 522 */
 523static int me_kernel(struct page *p, unsigned long pfn)
 524{
 525	return IGNORED;
 
 526}
 527
 528/*
 529 * Page in unknown state. Do nothing.
 530 */
 531static int me_unknown(struct page *p, unsigned long pfn)
 532{
 533	printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
 534	return FAILED;
 
 535}
 536
 537/*
 538 * Clean (or cleaned) page cache page.
 539 */
 540static int me_pagecache_clean(struct page *p, unsigned long pfn)
 541{
 542	int err;
 543	int ret = FAILED;
 544	struct address_space *mapping;
 545
 546	delete_from_lru_cache(p);
 547
 548	/*
 549	 * For anonymous pages we're done the only reference left
 550	 * should be the one m_f() holds.
 551	 */
 552	if (PageAnon(p))
 553		return RECOVERED;
 
 
 554
 555	/*
 556	 * Now truncate the page in the page cache. This is really
 557	 * more like a "temporary hole punch"
 558	 * Don't do this for block devices when someone else
 559	 * has a reference, because it could be file system metadata
 560	 * and that's not safe to truncate.
 561	 */
 562	mapping = page_mapping(p);
 563	if (!mapping) {
 564		/*
 565		 * Page has been teared down in the meanwhile
 566		 */
 567		return FAILED;
 
 568	}
 569
 570	/*
 571	 * Truncation is a bit tricky. Enable it per file system for now.
 572	 *
 573	 * Open: to take i_mutex or not for this? Right now we don't.
 574	 */
 575	if (mapping->a_ops->error_remove_page) {
 576		err = mapping->a_ops->error_remove_page(mapping, p);
 577		if (err != 0) {
 578			printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
 579					pfn, err);
 580		} else if (page_has_private(p) &&
 581				!try_to_release_page(p, GFP_NOIO)) {
 582			pr_info("MCE %#lx: failed to release buffers\n", pfn);
 583		} else {
 584			ret = RECOVERED;
 585		}
 586	} else {
 587		/*
 588		 * If the file system doesn't support it just invalidate
 589		 * This fails on dirty or anything with private pages
 590		 */
 591		if (invalidate_inode_page(p))
 592			ret = RECOVERED;
 593		else
 594			printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
 595				pfn);
 596	}
 597	return ret;
 598}
 599
 600/*
 601 * Dirty cache page page
 602 * Issues: when the error hit a hole page the error is not properly
 603 * propagated.
 604 */
 605static int me_pagecache_dirty(struct page *p, unsigned long pfn)
 606{
 607	struct address_space *mapping = page_mapping(p);
 608
 609	SetPageError(p);
 610	/* TBD: print more information about the file. */
 611	if (mapping) {
 612		/*
 613		 * IO error will be reported by write(), fsync(), etc.
 614		 * who check the mapping.
 615		 * This way the application knows that something went
 616		 * wrong with its dirty file data.
 617		 *
 618		 * There's one open issue:
 619		 *
 620		 * The EIO will be only reported on the next IO
 621		 * operation and then cleared through the IO map.
 622		 * Normally Linux has two mechanisms to pass IO error
 623		 * first through the AS_EIO flag in the address space
 624		 * and then through the PageError flag in the page.
 625		 * Since we drop pages on memory failure handling the
 626		 * only mechanism open to use is through AS_AIO.
 627		 *
 628		 * This has the disadvantage that it gets cleared on
 629		 * the first operation that returns an error, while
 630		 * the PageError bit is more sticky and only cleared
 631		 * when the page is reread or dropped.  If an
 632		 * application assumes it will always get error on
 633		 * fsync, but does other operations on the fd before
 634		 * and the page is dropped between then the error
 635		 * will not be properly reported.
 636		 *
 637		 * This can already happen even without hwpoisoned
 638		 * pages: first on metadata IO errors (which only
 639		 * report through AS_EIO) or when the page is dropped
 640		 * at the wrong time.
 641		 *
 642		 * So right now we assume that the application DTRT on
 643		 * the first EIO, but we're not worse than other parts
 644		 * of the kernel.
 645		 */
 646		mapping_set_error(mapping, EIO);
 647	}
 648
 649	return me_pagecache_clean(p, pfn);
 650}
 651
 652/*
 653 * Clean and dirty swap cache.
 654 *
 655 * Dirty swap cache page is tricky to handle. The page could live both in page
 656 * cache and swap cache(ie. page is freshly swapped in). So it could be
 657 * referenced concurrently by 2 types of PTEs:
 658 * normal PTEs and swap PTEs. We try to handle them consistently by calling
 659 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
 660 * and then
 661 *      - clear dirty bit to prevent IO
 662 *      - remove from LRU
 663 *      - but keep in the swap cache, so that when we return to it on
 664 *        a later page fault, we know the application is accessing
 665 *        corrupted data and shall be killed (we installed simple
 666 *        interception code in do_swap_page to catch it).
 667 *
 668 * Clean swap cache pages can be directly isolated. A later page fault will
 669 * bring in the known good data from disk.
 670 */
 671static int me_swapcache_dirty(struct page *p, unsigned long pfn)
 672{
 
 
 673	ClearPageDirty(p);
 674	/* Trigger EIO in shmem: */
 675	ClearPageUptodate(p);
 676
 677	if (!delete_from_lru_cache(p))
 678		return DELAYED;
 679	else
 680		return FAILED;
 681}
 682
 683static int me_swapcache_clean(struct page *p, unsigned long pfn)
 684{
 
 
 685	delete_from_swap_cache(p);
 686
 687	if (!delete_from_lru_cache(p))
 688		return RECOVERED;
 689	else
 690		return FAILED;
 691}
 692
 693/*
 694 * Huge pages. Needs work.
 695 * Issues:
 696 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
 697 *   To narrow down kill region to one page, we need to break up pmd.
 698 */
 699static int me_huge_page(struct page *p, unsigned long pfn)
 700{
 701	int res = 0;
 702	struct page *hpage = compound_head(p);
 703	/*
 704	 * We can safely recover from error on free or reserved (i.e.
 705	 * not in-use) hugepage by dequeuing it from freelist.
 706	 * To check whether a hugepage is in-use or not, we can't use
 707	 * page->lru because it can be used in other hugepage operations,
 708	 * such as __unmap_hugepage_range() and gather_surplus_pages().
 709	 * So instead we use page_mapping() and PageAnon().
 710	 * We assume that this function is called with page lock held,
 711	 * so there is no race between isolation and mapping/unmapping.
 712	 */
 713	if (!(page_mapping(hpage) || PageAnon(hpage))) {
 714		res = dequeue_hwpoisoned_huge_page(hpage);
 715		if (!res)
 716			return RECOVERED;
 
 
 
 
 
 
 
 
 
 717	}
 718	return DELAYED;
 
 719}
 720
 721/*
 722 * Various page states we can handle.
 723 *
 724 * A page state is defined by its current page->flags bits.
 725 * The table matches them in order and calls the right handler.
 726 *
 727 * This is quite tricky because we can access page at any time
 728 * in its live cycle, so all accesses have to be extremely careful.
 729 *
 730 * This is not complete. More states could be added.
 731 * For any missing state don't attempt recovery.
 732 */
 733
 734#define dirty		(1UL << PG_dirty)
 735#define sc		(1UL << PG_swapcache)
 736#define unevict		(1UL << PG_unevictable)
 737#define mlock		(1UL << PG_mlocked)
 738#define writeback	(1UL << PG_writeback)
 739#define lru		(1UL << PG_lru)
 740#define swapbacked	(1UL << PG_swapbacked)
 741#define head		(1UL << PG_head)
 742#define tail		(1UL << PG_tail)
 743#define compound	(1UL << PG_compound)
 744#define slab		(1UL << PG_slab)
 745#define reserved	(1UL << PG_reserved)
 746
 747static struct page_state {
 748	unsigned long mask;
 749	unsigned long res;
 750	char *msg;
 
 
 751	int (*action)(struct page *p, unsigned long pfn);
 752} error_states[] = {
 753	{ reserved,	reserved,	"reserved kernel",	me_kernel },
 754	/*
 755	 * free pages are specially detected outside this table:
 756	 * PG_buddy pages only make a small fraction of all free pages.
 757	 */
 758
 759	/*
 760	 * Could in theory check if slab page is free or if we can drop
 761	 * currently unused objects without touching them. But just
 762	 * treat it as standard kernel for now.
 763	 */
 764	{ slab,		slab,		"kernel slab",	me_kernel },
 765
 766#ifdef CONFIG_PAGEFLAGS_EXTENDED
 767	{ head,		head,		"huge",		me_huge_page },
 768	{ tail,		tail,		"huge",		me_huge_page },
 769#else
 770	{ compound,	compound,	"huge",		me_huge_page },
 771#endif
 772
 773	{ sc|dirty,	sc|dirty,	"swapcache",	me_swapcache_dirty },
 774	{ sc|dirty,	sc,		"swapcache",	me_swapcache_clean },
 775
 776	{ unevict|dirty, unevict|dirty,	"unevictable LRU", me_pagecache_dirty},
 777	{ unevict,	unevict,	"unevictable LRU", me_pagecache_clean},
 778
 779	{ mlock|dirty,	mlock|dirty,	"mlocked LRU",	me_pagecache_dirty },
 780	{ mlock,	mlock,		"mlocked LRU",	me_pagecache_clean },
 781
 782	{ lru|dirty,	lru|dirty,	"LRU",		me_pagecache_dirty },
 783	{ lru|dirty,	lru,		"clean LRU",	me_pagecache_clean },
 784
 785	/*
 786	 * Catchall entry: must be at end.
 787	 */
 788	{ 0,		0,		"unknown page state",	me_unknown },
 789};
 790
 791#undef dirty
 792#undef sc
 793#undef unevict
 794#undef mlock
 795#undef writeback
 796#undef lru
 797#undef swapbacked
 798#undef head
 799#undef tail
 800#undef compound
 801#undef slab
 802#undef reserved
 803
 804static void action_result(unsigned long pfn, char *msg, int result)
 
 
 
 
 
 805{
 806	struct page *page = pfn_to_page(pfn);
 807
 808	printk(KERN_ERR "MCE %#lx: %s%s page recovery: %s\n",
 809		pfn,
 810		PageDirty(page) ? "dirty " : "",
 811		msg, action_name[result]);
 812}
 813
 814static int page_action(struct page_state *ps, struct page *p,
 815			unsigned long pfn)
 816{
 817	int result;
 818	int count;
 819
 
 820	result = ps->action(p, pfn);
 821	action_result(pfn, ps->msg, result);
 822
 823	count = page_count(p) - 1;
 824	if (ps->action == me_swapcache_dirty && result == DELAYED)
 825		count--;
 826	if (count != 0) {
 827		printk(KERN_ERR
 828		       "MCE %#lx: %s page still referenced by %d users\n",
 829		       pfn, ps->msg, count);
 830		result = FAILED;
 831	}
 
 832
 833	/* Could do more checks here if page looks ok */
 834	/*
 835	 * Could adjust zone counters here to correct for the missing page.
 836	 */
 837
 838	return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 839}
 840
 841/*
 842 * Do all that is necessary to remove user space mappings. Unmap
 843 * the pages and send SIGBUS to the processes if the data was dirty.
 844 */
 845static int hwpoison_user_mappings(struct page *p, unsigned long pfn,
 846				  int trapno)
 847{
 848	enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
 849	struct address_space *mapping;
 850	LIST_HEAD(tokill);
 851	int ret;
 852	int kill = 1;
 853	struct page *hpage = compound_head(p);
 854	struct page *ppage;
 855
 
 
 
 
 856	if (PageReserved(p) || PageSlab(p))
 857		return SWAP_SUCCESS;
 
 
 858
 859	/*
 860	 * This check implies we don't kill processes if their pages
 861	 * are in the swap cache early. Those are always late kills.
 862	 */
 863	if (!page_mapped(hpage))
 864		return SWAP_SUCCESS;
 865
 866	if (PageKsm(p))
 867		return SWAP_FAIL;
 
 
 868
 869	if (PageSwapCache(p)) {
 870		printk(KERN_ERR
 871		       "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
 872		ttu |= TTU_IGNORE_HWPOISON;
 873	}
 874
 875	/*
 876	 * Propagate the dirty bit from PTEs to struct page first, because we
 877	 * need this to decide if we should kill or just drop the page.
 878	 * XXX: the dirty test could be racy: set_page_dirty() may not always
 879	 * be called inside page lock (it's recommended but not enforced).
 880	 */
 881	mapping = page_mapping(hpage);
 882	if (!PageDirty(hpage) && mapping &&
 883	    mapping_cap_writeback_dirty(mapping)) {
 884		if (page_mkclean(hpage)) {
 885			SetPageDirty(hpage);
 886		} else {
 887			kill = 0;
 888			ttu |= TTU_IGNORE_HWPOISON;
 889			printk(KERN_INFO
 890	"MCE %#lx: corrupted page was clean: dropped without side effects\n",
 891				pfn);
 892		}
 893	}
 894
 895	/*
 896	 * ppage: poisoned page
 897	 *   if p is regular page(4k page)
 898	 *        ppage == real poisoned page;
 899	 *   else p is hugetlb or THP, ppage == head page.
 900	 */
 901	ppage = hpage;
 902
 903	if (PageTransHuge(hpage)) {
 904		/*
 905		 * Verify that this isn't a hugetlbfs head page, the check for
 906		 * PageAnon is just for avoid tripping a split_huge_page
 907		 * internal debug check, as split_huge_page refuses to deal with
 908		 * anything that isn't an anon page. PageAnon can't go away fro
 909		 * under us because we hold a refcount on the hpage, without a
 910		 * refcount on the hpage. split_huge_page can't be safely called
 911		 * in the first place, having a refcount on the tail isn't
 912		 * enough * to be safe.
 913		 */
 914		if (!PageHuge(hpage) && PageAnon(hpage)) {
 915			if (unlikely(split_huge_page(hpage))) {
 916				/*
 917				 * FIXME: if splitting THP is failed, it is
 918				 * better to stop the following operation rather
 919				 * than causing panic by unmapping. System might
 920				 * survive if the page is freed later.
 921				 */
 922				printk(KERN_INFO
 923					"MCE %#lx: failed to split THP\n", pfn);
 924
 925				BUG_ON(!PageHWPoison(p));
 926				return SWAP_FAIL;
 927			}
 928			/* THP is split, so ppage should be the real poisoned page. */
 929			ppage = p;
 930		}
 931	}
 932
 933	/*
 934	 * First collect all the processes that have the page
 935	 * mapped in dirty form.  This has to be done before try_to_unmap,
 936	 * because ttu takes the rmap data structures down.
 937	 *
 938	 * Error handling: We ignore errors here because
 939	 * there's nothing that can be done.
 940	 */
 941	if (kill)
 942		collect_procs(ppage, &tokill);
 943
 944	if (hpage != ppage)
 945		lock_page(ppage);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 946
 947	ret = try_to_unmap(ppage, ttu);
 948	if (ret != SWAP_SUCCESS)
 949		printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
 950				pfn, page_mapcount(ppage));
 951
 952	if (hpage != ppage)
 953		unlock_page(ppage);
 
 
 
 
 954
 955	/*
 956	 * Now that the dirty bit has been propagated to the
 957	 * struct page and all unmaps done we can decide if
 958	 * killing is needed or not.  Only kill when the page
 959	 * was dirty, otherwise the tokill list is merely
 
 960	 * freed.  When there was a problem unmapping earlier
 961	 * use a more force-full uncatchable kill to prevent
 962	 * any accesses to the poisoned memory.
 963	 */
 964	kill_procs_ao(&tokill, !!PageDirty(ppage), trapno,
 965		      ret != SWAP_SUCCESS, p, pfn);
 966
 967	return ret;
 968}
 969
 970static void set_page_hwpoison_huge_page(struct page *hpage)
 
 971{
 972	int i;
 973	int nr_pages = 1 << compound_trans_order(hpage);
 974	for (i = 0; i < nr_pages; i++)
 975		SetPageHWPoison(hpage + i);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 976}
 977
 978static void clear_page_hwpoison_huge_page(struct page *hpage)
 979{
 980	int i;
 981	int nr_pages = 1 << compound_trans_order(hpage);
 982	for (i = 0; i < nr_pages; i++)
 983		ClearPageHWPoison(hpage + i);
 
 
 
 
 
 
 
 
 
 
 
 984}
 985
 986int __memory_failure(unsigned long pfn, int trapno, int flags)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 987{
 988	struct page_state *ps;
 989	struct page *p;
 990	struct page *hpage;
 991	int res;
 992	unsigned int nr_pages;
 
 
 
 
 993
 994	if (!sysctl_memory_failure_recovery)
 995		panic("Memory failure from trap %d on page %lx", trapno, pfn);
 996
 997	if (!pfn_valid(pfn)) {
 998		printk(KERN_ERR
 999		       "MCE %#lx: memory outside kernel control\n",
1000		       pfn);
 
 
 
 
 
 
1001		return -ENXIO;
1002	}
1003
1004	p = pfn_to_page(pfn);
1005	hpage = compound_head(p);
 
 
 
 
 
 
1006	if (TestSetPageHWPoison(p)) {
1007		printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
1008		return 0;
 
 
 
 
1009	}
1010
1011	nr_pages = 1 << compound_trans_order(hpage);
1012	atomic_long_add(nr_pages, &mce_bad_pages);
1013
1014	/*
1015	 * We need/can do nothing about count=0 pages.
1016	 * 1) it's a free page, and therefore in safe hand:
1017	 *    prep_new_page() will be the gate keeper.
1018	 * 2) it's a free hugepage, which is also safe:
1019	 *    an affected hugepage will be dequeued from hugepage freelist,
1020	 *    so there's no concern about reusing it ever after.
1021	 * 3) it's part of a non-compound high order page.
1022	 *    Implies some kernel user: cannot stop them from
1023	 *    R/W the page; let's pray that the page has been
1024	 *    used and will be freed some time later.
1025	 * In fact it's dangerous to directly bump up page count from 0,
1026	 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
1027	 */
1028	if (!(flags & MF_COUNT_INCREASED) &&
1029		!get_page_unless_zero(hpage)) {
1030		if (is_free_buddy_page(p)) {
1031			action_result(pfn, "free buddy", DELAYED);
1032			return 0;
1033		} else if (PageHuge(hpage)) {
1034			/*
1035			 * Check "just unpoisoned", "filter hit", and
1036			 * "race with other subpage."
1037			 */
1038			lock_page(hpage);
1039			if (!PageHWPoison(hpage)
1040			    || (hwpoison_filter(p) && TestClearPageHWPoison(p))
1041			    || (p != hpage && TestSetPageHWPoison(hpage))) {
1042				atomic_long_sub(nr_pages, &mce_bad_pages);
1043				return 0;
 
 
 
 
 
 
1044			}
1045			set_page_hwpoison_huge_page(hpage);
1046			res = dequeue_hwpoisoned_huge_page(hpage);
1047			action_result(pfn, "free huge",
1048				      res ? IGNORED : DELAYED);
1049			unlock_page(hpage);
1050			return res;
1051		} else {
1052			action_result(pfn, "high order kernel", IGNORED);
1053			return -EBUSY;
 
 
 
 
1054		}
 
1055	}
1056
1057	/*
1058	 * We ignore non-LRU pages for good reasons.
1059	 * - PG_locked is only well defined for LRU pages and a few others
1060	 * - to avoid races with __set_page_locked()
1061	 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
1062	 * The check (unnecessarily) ignores LRU pages being isolated and
1063	 * walked by the page reclaim code, however that's not a big loss.
1064	 */
1065	if (!PageHuge(p) && !PageTransCompound(p)) {
1066		if (!PageLRU(p))
1067			shake_page(p, 0);
1068		if (!PageLRU(p)) {
1069			/*
1070			 * shake_page could have turned it free.
1071			 */
1072			if (is_free_buddy_page(p)) {
1073				action_result(pfn, "free buddy, 2nd try",
1074						DELAYED);
1075				return 0;
1076			}
1077			action_result(pfn, "non LRU", IGNORED);
1078			put_page(p);
1079			return -EBUSY;
1080		}
1081	}
1082
1083	/*
1084	 * Lock the page and wait for writeback to finish.
1085	 * It's very difficult to mess with pages currently under IO
1086	 * and in many cases impossible, so we just avoid it here.
 
 
1087	 */
1088	lock_page(hpage);
1089
1090	/*
1091	 * unpoison always clear PG_hwpoison inside page lock
1092	 */
1093	if (!PageHWPoison(p)) {
1094		printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
1095		res = 0;
1096		goto out;
 
 
1097	}
1098	if (hwpoison_filter(p)) {
1099		if (TestClearPageHWPoison(p))
1100			atomic_long_sub(nr_pages, &mce_bad_pages);
1101		unlock_page(hpage);
1102		put_page(hpage);
1103		return 0;
1104	}
1105
1106	/*
1107	 * For error on the tail page, we should set PG_hwpoison
1108	 * on the head page to show that the hugepage is hwpoisoned
 
1109	 */
1110	if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
1111		action_result(pfn, "hugepage already hardware poisoned",
1112				IGNORED);
1113		unlock_page(hpage);
1114		put_page(hpage);
1115		return 0;
1116	}
1117	/*
1118	 * Set PG_hwpoison on all pages in an error hugepage,
1119	 * because containment is done in hugepage unit for now.
1120	 * Since we have done TestSetPageHWPoison() for the head page with
1121	 * page lock held, we can safely set PG_hwpoison bits on tail pages.
1122	 */
1123	if (PageHuge(p))
1124		set_page_hwpoison_huge_page(hpage);
1125
1126	wait_on_page_writeback(p);
1127
1128	/*
1129	 * Now take care of user space mappings.
1130	 * Abort on fail: __delete_from_page_cache() assumes unmapped page.
1131	 */
1132	if (hwpoison_user_mappings(p, pfn, trapno) != SWAP_SUCCESS) {
1133		printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn);
1134		res = -EBUSY;
1135		goto out;
1136	}
1137
1138	/*
1139	 * Torn down by someone else?
1140	 */
1141	if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
1142		action_result(pfn, "already truncated LRU", IGNORED);
1143		res = -EBUSY;
1144		goto out;
1145	}
1146
1147	res = -EBUSY;
1148	for (ps = error_states;; ps++) {
1149		if ((p->flags & ps->mask) == ps->res) {
1150			res = page_action(ps, p, pfn);
1151			break;
1152		}
1153	}
1154out:
1155	unlock_page(hpage);
1156	return res;
1157}
1158EXPORT_SYMBOL_GPL(__memory_failure);
1159
1160/**
1161 * memory_failure - Handle memory failure of a page.
1162 * @pfn: Page Number of the corrupted page
1163 * @trapno: Trap number reported in the signal to user space.
1164 *
1165 * This function is called by the low level machine check code
1166 * of an architecture when it detects hardware memory corruption
1167 * of a page. It tries its best to recover, which includes
1168 * dropping pages, killing processes etc.
1169 *
1170 * The function is primarily of use for corruptions that
1171 * happen outside the current execution context (e.g. when
1172 * detected by a background scrubber)
1173 *
1174 * Must run in process context (e.g. a work queue) with interrupts
1175 * enabled and no spinlocks hold.
1176 */
1177void memory_failure(unsigned long pfn, int trapno)
1178{
1179	__memory_failure(pfn, trapno, 0);
1180}
1181
1182#define MEMORY_FAILURE_FIFO_ORDER	4
1183#define MEMORY_FAILURE_FIFO_SIZE	(1 << MEMORY_FAILURE_FIFO_ORDER)
1184
1185struct memory_failure_entry {
1186	unsigned long pfn;
1187	int trapno;
1188	int flags;
1189};
1190
1191struct memory_failure_cpu {
1192	DECLARE_KFIFO(fifo, struct memory_failure_entry,
1193		      MEMORY_FAILURE_FIFO_SIZE);
1194	spinlock_t lock;
1195	struct work_struct work;
1196};
1197
1198static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
1199
1200/**
1201 * memory_failure_queue - Schedule handling memory failure of a page.
1202 * @pfn: Page Number of the corrupted page
1203 * @trapno: Trap number reported in the signal to user space.
1204 * @flags: Flags for memory failure handling
1205 *
1206 * This function is called by the low level hardware error handler
1207 * when it detects hardware memory corruption of a page. It schedules
1208 * the recovering of error page, including dropping pages, killing
1209 * processes etc.
1210 *
1211 * The function is primarily of use for corruptions that
1212 * happen outside the current execution context (e.g. when
1213 * detected by a background scrubber)
1214 *
1215 * Can run in IRQ context.
1216 */
1217void memory_failure_queue(unsigned long pfn, int trapno, int flags)
1218{
1219	struct memory_failure_cpu *mf_cpu;
1220	unsigned long proc_flags;
1221	struct memory_failure_entry entry = {
1222		.pfn =		pfn,
1223		.trapno =	trapno,
1224		.flags =	flags,
1225	};
1226
1227	mf_cpu = &get_cpu_var(memory_failure_cpu);
1228	spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1229	if (kfifo_put(&mf_cpu->fifo, &entry))
1230		schedule_work_on(smp_processor_id(), &mf_cpu->work);
1231	else
1232		pr_err("Memory failure: buffer overflow when queuing memory failure at 0x%#lx\n",
1233		       pfn);
1234	spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1235	put_cpu_var(memory_failure_cpu);
1236}
1237EXPORT_SYMBOL_GPL(memory_failure_queue);
1238
1239static void memory_failure_work_func(struct work_struct *work)
1240{
1241	struct memory_failure_cpu *mf_cpu;
1242	struct memory_failure_entry entry = { 0, };
1243	unsigned long proc_flags;
1244	int gotten;
1245
1246	mf_cpu = &__get_cpu_var(memory_failure_cpu);
1247	for (;;) {
1248		spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1249		gotten = kfifo_get(&mf_cpu->fifo, &entry);
1250		spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1251		if (!gotten)
1252			break;
1253		__memory_failure(entry.pfn, entry.trapno, entry.flags);
 
 
 
1254	}
1255}
1256
 
 
 
 
 
 
 
 
 
 
 
 
 
1257static int __init memory_failure_init(void)
1258{
1259	struct memory_failure_cpu *mf_cpu;
1260	int cpu;
1261
1262	for_each_possible_cpu(cpu) {
1263		mf_cpu = &per_cpu(memory_failure_cpu, cpu);
1264		spin_lock_init(&mf_cpu->lock);
1265		INIT_KFIFO(mf_cpu->fifo);
1266		INIT_WORK(&mf_cpu->work, memory_failure_work_func);
1267	}
1268
1269	return 0;
1270}
1271core_initcall(memory_failure_init);
1272
 
 
 
 
 
 
1273/**
1274 * unpoison_memory - Unpoison a previously poisoned page
1275 * @pfn: Page number of the to be unpoisoned page
1276 *
1277 * Software-unpoison a page that has been poisoned by
1278 * memory_failure() earlier.
1279 *
1280 * This is only done on the software-level, so it only works
1281 * for linux injected failures, not real hardware failures
1282 *
1283 * Returns 0 for success, otherwise -errno.
1284 */
1285int unpoison_memory(unsigned long pfn)
1286{
1287	struct page *page;
1288	struct page *p;
1289	int freeit = 0;
1290	unsigned int nr_pages;
 
 
1291
1292	if (!pfn_valid(pfn))
1293		return -ENXIO;
1294
1295	p = pfn_to_page(pfn);
1296	page = compound_head(p);
1297
1298	if (!PageHWPoison(p)) {
1299		pr_info("MCE: Page was already unpoisoned %#lx\n", pfn);
 
1300		return 0;
1301	}
1302
1303	nr_pages = 1 << compound_trans_order(page);
 
 
 
 
1304
1305	if (!get_page_unless_zero(page)) {
1306		/*
1307		 * Since HWPoisoned hugepage should have non-zero refcount,
1308		 * race between memory failure and unpoison seems to happen.
1309		 * In such case unpoison fails and memory failure runs
1310		 * to the end.
1311		 */
1312		if (PageHuge(page)) {
1313			pr_debug("MCE: Memory failure is now running on free hugepage %#lx\n", pfn);
1314			return 0;
1315		}
 
 
 
 
 
 
 
 
 
 
 
 
 
1316		if (TestClearPageHWPoison(p))
1317			atomic_long_sub(nr_pages, &mce_bad_pages);
1318		pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
 
1319		return 0;
1320	}
1321
1322	lock_page(page);
1323	/*
1324	 * This test is racy because PG_hwpoison is set outside of page lock.
1325	 * That's acceptable because that won't trigger kernel panic. Instead,
1326	 * the PG_hwpoison page will be caught and isolated on the entrance to
1327	 * the free buddy page pool.
1328	 */
1329	if (TestClearPageHWPoison(page)) {
1330		pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
1331		atomic_long_sub(nr_pages, &mce_bad_pages);
 
1332		freeit = 1;
1333		if (PageHuge(page))
1334			clear_page_hwpoison_huge_page(page);
1335	}
1336	unlock_page(page);
1337
1338	put_page(page);
1339	if (freeit)
1340		put_page(page);
1341
1342	return 0;
1343}
1344EXPORT_SYMBOL(unpoison_memory);
1345
1346static struct page *new_page(struct page *p, unsigned long private, int **x)
1347{
1348	int nid = page_to_nid(p);
1349	if (PageHuge(p))
1350		return alloc_huge_page_node(page_hstate(compound_head(p)),
1351						   nid);
1352	else
1353		return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1354}
1355
1356/*
1357 * Safely get reference count of an arbitrary page.
1358 * Returns 0 for a free page, -EIO for a zero refcount page
1359 * that is not free, and 1 for any other page type.
1360 * For 1 the page is returned with increased page count, otherwise not.
1361 */
1362static int get_any_page(struct page *p, unsigned long pfn, int flags)
1363{
1364	int ret;
1365
1366	if (flags & MF_COUNT_INCREASED)
1367		return 1;
 
 
 
 
 
 
1368
1369	/*
1370	 * The lock_memory_hotplug prevents a race with memory hotplug.
1371	 * This is a big hammer, a better would be nicer.
 
 
1372	 */
1373	lock_memory_hotplug();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1374
1375	/*
1376	 * Isolate the page, so that it doesn't get reallocated if it
1377	 * was free.
1378	 */
1379	set_migratetype_isolate(p);
1380	/*
1381	 * When the target page is a free hugepage, just remove it
1382	 * from free hugepage list.
1383	 */
1384	if (!get_page_unless_zero(compound_head(p))) {
1385		if (PageHuge(p)) {
1386			pr_info("get_any_page: %#lx free huge page\n", pfn);
1387			ret = dequeue_hwpoisoned_huge_page(compound_head(p));
1388		} else if (is_free_buddy_page(p)) {
1389			pr_info("get_any_page: %#lx free buddy page\n", pfn);
1390			/* Set hwpoison bit while page is still isolated */
1391			SetPageHWPoison(p);
1392			ret = 0;
 
 
 
 
 
1393		} else {
1394			pr_info("get_any_page: %#lx: unknown zero refcount page type %lx\n",
1395				pfn, p->flags);
1396			ret = -EIO;
 
 
 
 
1397		}
1398	} else {
1399		/* Not a free page */
1400		ret = 1;
 
1401	}
1402	unset_migratetype_isolate(p);
1403	unlock_memory_hotplug();
1404	return ret;
1405}
1406
1407static int soft_offline_huge_page(struct page *page, int flags)
1408{
1409	int ret;
1410	unsigned long pfn = page_to_pfn(page);
1411	struct page *hpage = compound_head(page);
1412	LIST_HEAD(pagelist);
1413
1414	ret = get_any_page(page, pfn, flags);
1415	if (ret < 0)
1416		return ret;
1417	if (ret == 0)
1418		goto done;
1419
1420	if (PageHWPoison(hpage)) {
1421		put_page(hpage);
1422		pr_debug("soft offline: %#lx hugepage already poisoned\n", pfn);
1423		return -EBUSY;
1424	}
1425
1426	/* Keep page count to indicate a given hugepage is isolated. */
 
1427
1428	list_add(&hpage->lru, &pagelist);
1429	ret = migrate_huge_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0,
1430				true);
1431	if (ret) {
1432		struct page *page1, *page2;
1433		list_for_each_entry_safe(page1, page2, &pagelist, lru)
1434			put_page(page1);
1435
1436		pr_debug("soft offline: %#lx: migration failed %d, type %lx\n",
1437			 pfn, ret, page->flags);
1438		if (ret > 0)
1439			ret = -EIO;
1440		return ret;
1441	}
1442done:
1443	if (!PageHWPoison(hpage))
1444		atomic_long_add(1 << compound_trans_order(hpage), &mce_bad_pages);
1445	set_page_hwpoison_huge_page(hpage);
1446	dequeue_hwpoisoned_huge_page(hpage);
1447	/* keep elevated page count for bad page */
1448	return ret;
1449}
1450
1451/**
1452 * soft_offline_page - Soft offline a page.
1453 * @page: page to offline
1454 * @flags: flags. Same as memory_failure().
1455 *
1456 * Returns 0 on success, otherwise negated errno.
1457 *
1458 * Soft offline a page, by migration or invalidation,
1459 * without killing anything. This is for the case when
1460 * a page is not corrupted yet (so it's still valid to access),
1461 * but has had a number of corrected errors and is better taken
1462 * out.
1463 *
1464 * The actual policy on when to do that is maintained by
1465 * user space.
1466 *
1467 * This should never impact any application or cause data loss,
1468 * however it might take some time.
1469 *
1470 * This is not a 100% solution for all memory, but tries to be
1471 * ``good enough'' for the majority of memory.
1472 */
1473int soft_offline_page(struct page *page, int flags)
1474{
1475	int ret;
1476	unsigned long pfn = page_to_pfn(page);
1477
1478	if (PageHuge(page))
1479		return soft_offline_huge_page(page, flags);
1480
1481	ret = get_any_page(page, pfn, flags);
1482	if (ret < 0)
1483		return ret;
1484	if (ret == 0)
1485		goto done;
1486
1487	/*
1488	 * Page cache page we can handle?
1489	 */
1490	if (!PageLRU(page)) {
1491		/*
1492		 * Try to free it.
1493		 */
1494		put_page(page);
1495		shake_page(page, 1);
1496
1497		/*
1498		 * Did it turn free?
1499		 */
1500		ret = get_any_page(page, pfn, 0);
1501		if (ret < 0)
1502			return ret;
1503		if (ret == 0)
1504			goto done;
1505	}
1506	if (!PageLRU(page)) {
1507		pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n",
1508				pfn, page->flags);
1509		return -EIO;
1510	}
1511
1512	lock_page(page);
1513	wait_on_page_writeback(page);
1514
1515	/*
1516	 * Synchronized using the page lock with memory_failure()
1517	 */
1518	if (PageHWPoison(page)) {
1519		unlock_page(page);
1520		put_page(page);
1521		pr_info("soft offline: %#lx page already poisoned\n", pfn);
1522		return -EBUSY;
1523	}
1524
1525	/*
1526	 * Try to invalidate first. This should work for
1527	 * non dirty unmapped page cache pages.
1528	 */
1529	ret = invalidate_inode_page(page);
1530	unlock_page(page);
1531	/*
1532	 * RED-PEN would be better to keep it isolated here, but we
1533	 * would need to fix isolation locking first.
1534	 */
1535	if (ret == 1) {
1536		put_page(page);
1537		ret = 0;
1538		pr_info("soft_offline: %#lx: invalidated\n", pfn);
1539		goto done;
1540	}
1541
1542	/*
1543	 * Simple invalidation didn't work.
1544	 * Try to migrate to a new page instead. migrate.c
1545	 * handles a large number of cases for us.
1546	 */
1547	ret = isolate_lru_page(page);
1548	/*
1549	 * Drop page reference which is came from get_any_page()
1550	 * successful isolate_lru_page() already took another one.
1551	 */
1552	put_page(page);
1553	if (!ret) {
1554		LIST_HEAD(pagelist);
1555		inc_zone_page_state(page, NR_ISOLATED_ANON +
1556					    page_is_file_cache(page));
1557		list_add(&page->lru, &pagelist);
1558		ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
1559								0, true);
1560		if (ret) {
1561			putback_lru_pages(&pagelist);
1562			pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
1563				pfn, ret, page->flags);
1564			if (ret > 0)
1565				ret = -EIO;
1566		}
1567	} else {
1568		pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
1569				pfn, ret, page_count(page), page->flags);
1570	}
1571	if (ret)
1572		return ret;
1573
1574done:
1575	atomic_long_add(1, &mce_bad_pages);
1576	SetPageHWPoison(page);
1577	/* keep elevated page count for bad page */
1578	return ret;
1579}