memory-failure.c revision 8c6c2ecb44667f7204e9d2b89c4c1f42edc5a196
1/* 2 * Copyright (C) 2008, 2009 Intel Corporation 3 * Authors: Andi Kleen, Fengguang Wu 4 * 5 * This software may be redistributed and/or modified under the terms of 6 * the GNU General Public License ("GPL") version 2 only as published by the 7 * Free Software Foundation. 8 * 9 * High level machine check handler. Handles pages reported by the 10 * hardware as being corrupted usually due to a 2bit ECC memory or cache 11 * failure. 12 * 13 * Handles page cache pages in various states. The tricky part 14 * here is that we can access any page asynchronous to other VM 15 * users, because memory failures could happen anytime and anywhere, 16 * possibly violating some of their assumptions. This is why this code 17 * has to be extremely careful. Generally it tries to use normal locking 18 * rules, as in get the standard locks, even if that means the 19 * error handling takes potentially a long time. 20 * 21 * The operation to map back from RMAP chains to processes has to walk 22 * the complete process list and has non linear complexity with the number 23 * mappings. In short it can be quite slow. But since memory corruptions 24 * are rare we hope to get away with this. 25 */ 26 27/* 28 * Notebook: 29 * - hugetlb needs more code 30 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages 31 * - pass bad pages to kdump next kernel 32 */ 33#define DEBUG 1 /* remove me in 2.6.34 */ 34#include <linux/kernel.h> 35#include <linux/mm.h> 36#include <linux/page-flags.h> 37#include <linux/kernel-page-flags.h> 38#include <linux/sched.h> 39#include <linux/ksm.h> 40#include <linux/rmap.h> 41#include <linux/pagemap.h> 42#include <linux/swap.h> 43#include <linux/backing-dev.h> 44#include <linux/migrate.h> 45#include <linux/page-isolation.h> 46#include <linux/suspend.h> 47#include <linux/slab.h> 48#include <linux/swapops.h> 49#include <linux/hugetlb.h> 50#include "internal.h" 51 52int sysctl_memory_failure_early_kill __read_mostly = 0; 53 54int sysctl_memory_failure_recovery __read_mostly = 1; 55 56atomic_long_t mce_bad_pages __read_mostly = ATOMIC_LONG_INIT(0); 57 58#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE) 59 60u32 hwpoison_filter_enable = 0; 61u32 hwpoison_filter_dev_major = ~0U; 62u32 hwpoison_filter_dev_minor = ~0U; 63u64 hwpoison_filter_flags_mask; 64u64 hwpoison_filter_flags_value; 65EXPORT_SYMBOL_GPL(hwpoison_filter_enable); 66EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major); 67EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor); 68EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask); 69EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value); 70 71static int hwpoison_filter_dev(struct page *p) 72{ 73 struct address_space *mapping; 74 dev_t dev; 75 76 if (hwpoison_filter_dev_major == ~0U && 77 hwpoison_filter_dev_minor == ~0U) 78 return 0; 79 80 /* 81 * page_mapping() does not accept slab page 82 */ 83 if (PageSlab(p)) 84 return -EINVAL; 85 86 mapping = page_mapping(p); 87 if (mapping == NULL || mapping->host == NULL) 88 return -EINVAL; 89 90 dev = mapping->host->i_sb->s_dev; 91 if (hwpoison_filter_dev_major != ~0U && 92 hwpoison_filter_dev_major != MAJOR(dev)) 93 return -EINVAL; 94 if (hwpoison_filter_dev_minor != ~0U && 95 hwpoison_filter_dev_minor != MINOR(dev)) 96 return -EINVAL; 97 98 return 0; 99} 100 101static int hwpoison_filter_flags(struct page *p) 102{ 103 if (!hwpoison_filter_flags_mask) 104 return 0; 105 106 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) == 107 hwpoison_filter_flags_value) 108 return 0; 109 else 110 return -EINVAL; 111} 112 113/* 114 * This allows stress tests to limit test scope to a collection of tasks 115 * by putting them under some memcg. This prevents killing unrelated/important 116 * processes such as /sbin/init. Note that the target task may share clean 117 * pages with init (eg. libc text), which is harmless. If the target task 118 * share _dirty_ pages with another task B, the test scheme must make sure B 119 * is also included in the memcg. At last, due to race conditions this filter 120 * can only guarantee that the page either belongs to the memcg tasks, or is 121 * a freed page. 122 */ 123#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP 124u64 hwpoison_filter_memcg; 125EXPORT_SYMBOL_GPL(hwpoison_filter_memcg); 126static int hwpoison_filter_task(struct page *p) 127{ 128 struct mem_cgroup *mem; 129 struct cgroup_subsys_state *css; 130 unsigned long ino; 131 132 if (!hwpoison_filter_memcg) 133 return 0; 134 135 mem = try_get_mem_cgroup_from_page(p); 136 if (!mem) 137 return -EINVAL; 138 139 css = mem_cgroup_css(mem); 140 /* root_mem_cgroup has NULL dentries */ 141 if (!css->cgroup->dentry) 142 return -EINVAL; 143 144 ino = css->cgroup->dentry->d_inode->i_ino; 145 css_put(css); 146 147 if (ino != hwpoison_filter_memcg) 148 return -EINVAL; 149 150 return 0; 151} 152#else 153static int hwpoison_filter_task(struct page *p) { return 0; } 154#endif 155 156int hwpoison_filter(struct page *p) 157{ 158 if (!hwpoison_filter_enable) 159 return 0; 160 161 if (hwpoison_filter_dev(p)) 162 return -EINVAL; 163 164 if (hwpoison_filter_flags(p)) 165 return -EINVAL; 166 167 if (hwpoison_filter_task(p)) 168 return -EINVAL; 169 170 return 0; 171} 172#else 173int hwpoison_filter(struct page *p) 174{ 175 return 0; 176} 177#endif 178 179EXPORT_SYMBOL_GPL(hwpoison_filter); 180 181/* 182 * Send all the processes who have the page mapped an ``action optional'' 183 * signal. 184 */ 185static int kill_proc_ao(struct task_struct *t, unsigned long addr, int trapno, 186 unsigned long pfn, struct page *page) 187{ 188 struct siginfo si; 189 int ret; 190 191 printk(KERN_ERR 192 "MCE %#lx: Killing %s:%d early due to hardware memory corruption\n", 193 pfn, t->comm, t->pid); 194 si.si_signo = SIGBUS; 195 si.si_errno = 0; 196 si.si_code = BUS_MCEERR_AO; 197 si.si_addr = (void *)addr; 198#ifdef __ARCH_SI_TRAPNO 199 si.si_trapno = trapno; 200#endif 201 si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT; 202 /* 203 * Don't use force here, it's convenient if the signal 204 * can be temporarily blocked. 205 * This could cause a loop when the user sets SIGBUS 206 * to SIG_IGN, but hopefully noone will do that? 207 */ 208 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */ 209 if (ret < 0) 210 printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n", 211 t->comm, t->pid, ret); 212 return ret; 213} 214 215/* 216 * When a unknown page type is encountered drain as many buffers as possible 217 * in the hope to turn the page into a LRU or free page, which we can handle. 218 */ 219void shake_page(struct page *p, int access) 220{ 221 if (!PageSlab(p)) { 222 lru_add_drain_all(); 223 if (PageLRU(p)) 224 return; 225 drain_all_pages(); 226 if (PageLRU(p) || is_free_buddy_page(p)) 227 return; 228 } 229 230 /* 231 * Only all shrink_slab here (which would also 232 * shrink other caches) if access is not potentially fatal. 233 */ 234 if (access) { 235 int nr; 236 do { 237 nr = shrink_slab(1000, GFP_KERNEL, 1000); 238 if (page_count(p) == 1) 239 break; 240 } while (nr > 10); 241 } 242} 243EXPORT_SYMBOL_GPL(shake_page); 244 245/* 246 * Kill all processes that have a poisoned page mapped and then isolate 247 * the page. 248 * 249 * General strategy: 250 * Find all processes having the page mapped and kill them. 251 * But we keep a page reference around so that the page is not 252 * actually freed yet. 253 * Then stash the page away 254 * 255 * There's no convenient way to get back to mapped processes 256 * from the VMAs. So do a brute-force search over all 257 * running processes. 258 * 259 * Remember that machine checks are not common (or rather 260 * if they are common you have other problems), so this shouldn't 261 * be a performance issue. 262 * 263 * Also there are some races possible while we get from the 264 * error detection to actually handle it. 265 */ 266 267struct to_kill { 268 struct list_head nd; 269 struct task_struct *tsk; 270 unsigned long addr; 271 unsigned addr_valid:1; 272}; 273 274/* 275 * Failure handling: if we can't find or can't kill a process there's 276 * not much we can do. We just print a message and ignore otherwise. 277 */ 278 279/* 280 * Schedule a process for later kill. 281 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. 282 * TBD would GFP_NOIO be enough? 283 */ 284static void add_to_kill(struct task_struct *tsk, struct page *p, 285 struct vm_area_struct *vma, 286 struct list_head *to_kill, 287 struct to_kill **tkc) 288{ 289 struct to_kill *tk; 290 291 if (*tkc) { 292 tk = *tkc; 293 *tkc = NULL; 294 } else { 295 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC); 296 if (!tk) { 297 printk(KERN_ERR 298 "MCE: Out of memory while machine check handling\n"); 299 return; 300 } 301 } 302 tk->addr = page_address_in_vma(p, vma); 303 tk->addr_valid = 1; 304 305 /* 306 * In theory we don't have to kill when the page was 307 * munmaped. But it could be also a mremap. Since that's 308 * likely very rare kill anyways just out of paranoia, but use 309 * a SIGKILL because the error is not contained anymore. 310 */ 311 if (tk->addr == -EFAULT) { 312 pr_debug("MCE: Unable to find user space address %lx in %s\n", 313 page_to_pfn(p), tsk->comm); 314 tk->addr_valid = 0; 315 } 316 get_task_struct(tsk); 317 tk->tsk = tsk; 318 list_add_tail(&tk->nd, to_kill); 319} 320 321/* 322 * Kill the processes that have been collected earlier. 323 * 324 * Only do anything when DOIT is set, otherwise just free the list 325 * (this is used for clean pages which do not need killing) 326 * Also when FAIL is set do a force kill because something went 327 * wrong earlier. 328 */ 329static void kill_procs_ao(struct list_head *to_kill, int doit, int trapno, 330 int fail, struct page *page, unsigned long pfn) 331{ 332 struct to_kill *tk, *next; 333 334 list_for_each_entry_safe (tk, next, to_kill, nd) { 335 if (doit) { 336 /* 337 * In case something went wrong with munmapping 338 * make sure the process doesn't catch the 339 * signal and then access the memory. Just kill it. 340 */ 341 if (fail || tk->addr_valid == 0) { 342 printk(KERN_ERR 343 "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", 344 pfn, tk->tsk->comm, tk->tsk->pid); 345 force_sig(SIGKILL, tk->tsk); 346 } 347 348 /* 349 * In theory the process could have mapped 350 * something else on the address in-between. We could 351 * check for that, but we need to tell the 352 * process anyways. 353 */ 354 else if (kill_proc_ao(tk->tsk, tk->addr, trapno, 355 pfn, page) < 0) 356 printk(KERN_ERR 357 "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n", 358 pfn, tk->tsk->comm, tk->tsk->pid); 359 } 360 put_task_struct(tk->tsk); 361 kfree(tk); 362 } 363} 364 365static int task_early_kill(struct task_struct *tsk) 366{ 367 if (!tsk->mm) 368 return 0; 369 if (tsk->flags & PF_MCE_PROCESS) 370 return !!(tsk->flags & PF_MCE_EARLY); 371 return sysctl_memory_failure_early_kill; 372} 373 374/* 375 * Collect processes when the error hit an anonymous page. 376 */ 377static void collect_procs_anon(struct page *page, struct list_head *to_kill, 378 struct to_kill **tkc) 379{ 380 struct vm_area_struct *vma; 381 struct task_struct *tsk; 382 struct anon_vma *av; 383 384 read_lock(&tasklist_lock); 385 av = page_lock_anon_vma(page); 386 if (av == NULL) /* Not actually mapped anymore */ 387 goto out; 388 for_each_process (tsk) { 389 struct anon_vma_chain *vmac; 390 391 if (!task_early_kill(tsk)) 392 continue; 393 list_for_each_entry(vmac, &av->head, same_anon_vma) { 394 vma = vmac->vma; 395 if (!page_mapped_in_vma(page, vma)) 396 continue; 397 if (vma->vm_mm == tsk->mm) 398 add_to_kill(tsk, page, vma, to_kill, tkc); 399 } 400 } 401 page_unlock_anon_vma(av); 402out: 403 read_unlock(&tasklist_lock); 404} 405 406/* 407 * Collect processes when the error hit a file mapped page. 408 */ 409static void collect_procs_file(struct page *page, struct list_head *to_kill, 410 struct to_kill **tkc) 411{ 412 struct vm_area_struct *vma; 413 struct task_struct *tsk; 414 struct prio_tree_iter iter; 415 struct address_space *mapping = page->mapping; 416 417 /* 418 * A note on the locking order between the two locks. 419 * We don't rely on this particular order. 420 * If you have some other code that needs a different order 421 * feel free to switch them around. Or add a reverse link 422 * from mm_struct to task_struct, then this could be all 423 * done without taking tasklist_lock and looping over all tasks. 424 */ 425 426 read_lock(&tasklist_lock); 427 spin_lock(&mapping->i_mmap_lock); 428 for_each_process(tsk) { 429 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 430 431 if (!task_early_kill(tsk)) 432 continue; 433 434 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, 435 pgoff) { 436 /* 437 * Send early kill signal to tasks where a vma covers 438 * the page but the corrupted page is not necessarily 439 * mapped it in its pte. 440 * Assume applications who requested early kill want 441 * to be informed of all such data corruptions. 442 */ 443 if (vma->vm_mm == tsk->mm) 444 add_to_kill(tsk, page, vma, to_kill, tkc); 445 } 446 } 447 spin_unlock(&mapping->i_mmap_lock); 448 read_unlock(&tasklist_lock); 449} 450 451/* 452 * Collect the processes who have the corrupted page mapped to kill. 453 * This is done in two steps for locking reasons. 454 * First preallocate one tokill structure outside the spin locks, 455 * so that we can kill at least one process reasonably reliable. 456 */ 457static void collect_procs(struct page *page, struct list_head *tokill) 458{ 459 struct to_kill *tk; 460 461 if (!page->mapping) 462 return; 463 464 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO); 465 if (!tk) 466 return; 467 if (PageAnon(page)) 468 collect_procs_anon(page, tokill, &tk); 469 else 470 collect_procs_file(page, tokill, &tk); 471 kfree(tk); 472} 473 474/* 475 * Error handlers for various types of pages. 476 */ 477 478enum outcome { 479 IGNORED, /* Error: cannot be handled */ 480 FAILED, /* Error: handling failed */ 481 DELAYED, /* Will be handled later */ 482 RECOVERED, /* Successfully recovered */ 483}; 484 485static const char *action_name[] = { 486 [IGNORED] = "Ignored", 487 [FAILED] = "Failed", 488 [DELAYED] = "Delayed", 489 [RECOVERED] = "Recovered", 490}; 491 492/* 493 * XXX: It is possible that a page is isolated from LRU cache, 494 * and then kept in swap cache or failed to remove from page cache. 495 * The page count will stop it from being freed by unpoison. 496 * Stress tests should be aware of this memory leak problem. 497 */ 498static int delete_from_lru_cache(struct page *p) 499{ 500 if (!isolate_lru_page(p)) { 501 /* 502 * Clear sensible page flags, so that the buddy system won't 503 * complain when the page is unpoison-and-freed. 504 */ 505 ClearPageActive(p); 506 ClearPageUnevictable(p); 507 /* 508 * drop the page count elevated by isolate_lru_page() 509 */ 510 page_cache_release(p); 511 return 0; 512 } 513 return -EIO; 514} 515 516/* 517 * Error hit kernel page. 518 * Do nothing, try to be lucky and not touch this instead. For a few cases we 519 * could be more sophisticated. 520 */ 521static int me_kernel(struct page *p, unsigned long pfn) 522{ 523 return IGNORED; 524} 525 526/* 527 * Page in unknown state. Do nothing. 528 */ 529static int me_unknown(struct page *p, unsigned long pfn) 530{ 531 printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn); 532 return FAILED; 533} 534 535/* 536 * Clean (or cleaned) page cache page. 537 */ 538static int me_pagecache_clean(struct page *p, unsigned long pfn) 539{ 540 int err; 541 int ret = FAILED; 542 struct address_space *mapping; 543 544 delete_from_lru_cache(p); 545 546 /* 547 * For anonymous pages we're done the only reference left 548 * should be the one m_f() holds. 549 */ 550 if (PageAnon(p)) 551 return RECOVERED; 552 553 /* 554 * Now truncate the page in the page cache. This is really 555 * more like a "temporary hole punch" 556 * Don't do this for block devices when someone else 557 * has a reference, because it could be file system metadata 558 * and that's not safe to truncate. 559 */ 560 mapping = page_mapping(p); 561 if (!mapping) { 562 /* 563 * Page has been teared down in the meanwhile 564 */ 565 return FAILED; 566 } 567 568 /* 569 * Truncation is a bit tricky. Enable it per file system for now. 570 * 571 * Open: to take i_mutex or not for this? Right now we don't. 572 */ 573 if (mapping->a_ops->error_remove_page) { 574 err = mapping->a_ops->error_remove_page(mapping, p); 575 if (err != 0) { 576 printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n", 577 pfn, err); 578 } else if (page_has_private(p) && 579 !try_to_release_page(p, GFP_NOIO)) { 580 pr_debug("MCE %#lx: failed to release buffers\n", pfn); 581 } else { 582 ret = RECOVERED; 583 } 584 } else { 585 /* 586 * If the file system doesn't support it just invalidate 587 * This fails on dirty or anything with private pages 588 */ 589 if (invalidate_inode_page(p)) 590 ret = RECOVERED; 591 else 592 printk(KERN_INFO "MCE %#lx: Failed to invalidate\n", 593 pfn); 594 } 595 return ret; 596} 597 598/* 599 * Dirty cache page page 600 * Issues: when the error hit a hole page the error is not properly 601 * propagated. 602 */ 603static int me_pagecache_dirty(struct page *p, unsigned long pfn) 604{ 605 struct address_space *mapping = page_mapping(p); 606 607 SetPageError(p); 608 /* TBD: print more information about the file. */ 609 if (mapping) { 610 /* 611 * IO error will be reported by write(), fsync(), etc. 612 * who check the mapping. 613 * This way the application knows that something went 614 * wrong with its dirty file data. 615 * 616 * There's one open issue: 617 * 618 * The EIO will be only reported on the next IO 619 * operation and then cleared through the IO map. 620 * Normally Linux has two mechanisms to pass IO error 621 * first through the AS_EIO flag in the address space 622 * and then through the PageError flag in the page. 623 * Since we drop pages on memory failure handling the 624 * only mechanism open to use is through AS_AIO. 625 * 626 * This has the disadvantage that it gets cleared on 627 * the first operation that returns an error, while 628 * the PageError bit is more sticky and only cleared 629 * when the page is reread or dropped. If an 630 * application assumes it will always get error on 631 * fsync, but does other operations on the fd before 632 * and the page is dropped inbetween then the error 633 * will not be properly reported. 634 * 635 * This can already happen even without hwpoisoned 636 * pages: first on metadata IO errors (which only 637 * report through AS_EIO) or when the page is dropped 638 * at the wrong time. 639 * 640 * So right now we assume that the application DTRT on 641 * the first EIO, but we're not worse than other parts 642 * of the kernel. 643 */ 644 mapping_set_error(mapping, EIO); 645 } 646 647 return me_pagecache_clean(p, pfn); 648} 649 650/* 651 * Clean and dirty swap cache. 652 * 653 * Dirty swap cache page is tricky to handle. The page could live both in page 654 * cache and swap cache(ie. page is freshly swapped in). So it could be 655 * referenced concurrently by 2 types of PTEs: 656 * normal PTEs and swap PTEs. We try to handle them consistently by calling 657 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs, 658 * and then 659 * - clear dirty bit to prevent IO 660 * - remove from LRU 661 * - but keep in the swap cache, so that when we return to it on 662 * a later page fault, we know the application is accessing 663 * corrupted data and shall be killed (we installed simple 664 * interception code in do_swap_page to catch it). 665 * 666 * Clean swap cache pages can be directly isolated. A later page fault will 667 * bring in the known good data from disk. 668 */ 669static int me_swapcache_dirty(struct page *p, unsigned long pfn) 670{ 671 ClearPageDirty(p); 672 /* Trigger EIO in shmem: */ 673 ClearPageUptodate(p); 674 675 if (!delete_from_lru_cache(p)) 676 return DELAYED; 677 else 678 return FAILED; 679} 680 681static int me_swapcache_clean(struct page *p, unsigned long pfn) 682{ 683 delete_from_swap_cache(p); 684 685 if (!delete_from_lru_cache(p)) 686 return RECOVERED; 687 else 688 return FAILED; 689} 690 691/* 692 * Huge pages. Needs work. 693 * Issues: 694 * - Error on hugepage is contained in hugepage unit (not in raw page unit.) 695 * To narrow down kill region to one page, we need to break up pmd. 696 * - To support soft-offlining for hugepage, we need to support hugepage 697 * migration. 698 */ 699static int me_huge_page(struct page *p, unsigned long pfn) 700{ 701 int res = 0; 702 struct page *hpage = compound_head(p); 703 /* 704 * We can safely recover from error on free or reserved (i.e. 705 * not in-use) hugepage by dequeuing it from freelist. 706 * To check whether a hugepage is in-use or not, we can't use 707 * page->lru because it can be used in other hugepage operations, 708 * such as __unmap_hugepage_range() and gather_surplus_pages(). 709 * So instead we use page_mapping() and PageAnon(). 710 * We assume that this function is called with page lock held, 711 * so there is no race between isolation and mapping/unmapping. 712 */ 713 if (!(page_mapping(hpage) || PageAnon(hpage))) { 714 res = dequeue_hwpoisoned_huge_page(hpage); 715 if (!res) 716 return RECOVERED; 717 } 718 return DELAYED; 719} 720 721/* 722 * Various page states we can handle. 723 * 724 * A page state is defined by its current page->flags bits. 725 * The table matches them in order and calls the right handler. 726 * 727 * This is quite tricky because we can access page at any time 728 * in its live cycle, so all accesses have to be extremly careful. 729 * 730 * This is not complete. More states could be added. 731 * For any missing state don't attempt recovery. 732 */ 733 734#define dirty (1UL << PG_dirty) 735#define sc (1UL << PG_swapcache) 736#define unevict (1UL << PG_unevictable) 737#define mlock (1UL << PG_mlocked) 738#define writeback (1UL << PG_writeback) 739#define lru (1UL << PG_lru) 740#define swapbacked (1UL << PG_swapbacked) 741#define head (1UL << PG_head) 742#define tail (1UL << PG_tail) 743#define compound (1UL << PG_compound) 744#define slab (1UL << PG_slab) 745#define reserved (1UL << PG_reserved) 746 747static struct page_state { 748 unsigned long mask; 749 unsigned long res; 750 char *msg; 751 int (*action)(struct page *p, unsigned long pfn); 752} error_states[] = { 753 { reserved, reserved, "reserved kernel", me_kernel }, 754 /* 755 * free pages are specially detected outside this table: 756 * PG_buddy pages only make a small fraction of all free pages. 757 */ 758 759 /* 760 * Could in theory check if slab page is free or if we can drop 761 * currently unused objects without touching them. But just 762 * treat it as standard kernel for now. 763 */ 764 { slab, slab, "kernel slab", me_kernel }, 765 766#ifdef CONFIG_PAGEFLAGS_EXTENDED 767 { head, head, "huge", me_huge_page }, 768 { tail, tail, "huge", me_huge_page }, 769#else 770 { compound, compound, "huge", me_huge_page }, 771#endif 772 773 { sc|dirty, sc|dirty, "swapcache", me_swapcache_dirty }, 774 { sc|dirty, sc, "swapcache", me_swapcache_clean }, 775 776 { unevict|dirty, unevict|dirty, "unevictable LRU", me_pagecache_dirty}, 777 { unevict, unevict, "unevictable LRU", me_pagecache_clean}, 778 779 { mlock|dirty, mlock|dirty, "mlocked LRU", me_pagecache_dirty }, 780 { mlock, mlock, "mlocked LRU", me_pagecache_clean }, 781 782 { lru|dirty, lru|dirty, "LRU", me_pagecache_dirty }, 783 { lru|dirty, lru, "clean LRU", me_pagecache_clean }, 784 785 /* 786 * Catchall entry: must be at end. 787 */ 788 { 0, 0, "unknown page state", me_unknown }, 789}; 790 791#undef dirty 792#undef sc 793#undef unevict 794#undef mlock 795#undef writeback 796#undef lru 797#undef swapbacked 798#undef head 799#undef tail 800#undef compound 801#undef slab 802#undef reserved 803 804static void action_result(unsigned long pfn, char *msg, int result) 805{ 806 struct page *page = pfn_to_page(pfn); 807 808 printk(KERN_ERR "MCE %#lx: %s%s page recovery: %s\n", 809 pfn, 810 PageDirty(page) ? "dirty " : "", 811 msg, action_name[result]); 812} 813 814static int page_action(struct page_state *ps, struct page *p, 815 unsigned long pfn) 816{ 817 int result; 818 int count; 819 820 result = ps->action(p, pfn); 821 action_result(pfn, ps->msg, result); 822 823 count = page_count(p) - 1; 824 if (ps->action == me_swapcache_dirty && result == DELAYED) 825 count--; 826 if (count != 0) { 827 printk(KERN_ERR 828 "MCE %#lx: %s page still referenced by %d users\n", 829 pfn, ps->msg, count); 830 result = FAILED; 831 } 832 833 /* Could do more checks here if page looks ok */ 834 /* 835 * Could adjust zone counters here to correct for the missing page. 836 */ 837 838 return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY; 839} 840 841#define N_UNMAP_TRIES 5 842 843/* 844 * Do all that is necessary to remove user space mappings. Unmap 845 * the pages and send SIGBUS to the processes if the data was dirty. 846 */ 847static int hwpoison_user_mappings(struct page *p, unsigned long pfn, 848 int trapno) 849{ 850 enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; 851 struct address_space *mapping; 852 LIST_HEAD(tokill); 853 int ret; 854 int i; 855 int kill = 1; 856 struct page *hpage = compound_head(p); 857 858 if (PageReserved(p) || PageSlab(p)) 859 return SWAP_SUCCESS; 860 861 /* 862 * This check implies we don't kill processes if their pages 863 * are in the swap cache early. Those are always late kills. 864 */ 865 if (!page_mapped(hpage)) 866 return SWAP_SUCCESS; 867 868 if (PageKsm(p)) 869 return SWAP_FAIL; 870 871 if (PageSwapCache(p)) { 872 printk(KERN_ERR 873 "MCE %#lx: keeping poisoned page in swap cache\n", pfn); 874 ttu |= TTU_IGNORE_HWPOISON; 875 } 876 877 /* 878 * Propagate the dirty bit from PTEs to struct page first, because we 879 * need this to decide if we should kill or just drop the page. 880 * XXX: the dirty test could be racy: set_page_dirty() may not always 881 * be called inside page lock (it's recommended but not enforced). 882 */ 883 mapping = page_mapping(hpage); 884 if (!PageDirty(hpage) && mapping && 885 mapping_cap_writeback_dirty(mapping)) { 886 if (page_mkclean(hpage)) { 887 SetPageDirty(hpage); 888 } else { 889 kill = 0; 890 ttu |= TTU_IGNORE_HWPOISON; 891 printk(KERN_INFO 892 "MCE %#lx: corrupted page was clean: dropped without side effects\n", 893 pfn); 894 } 895 } 896 897 /* 898 * First collect all the processes that have the page 899 * mapped in dirty form. This has to be done before try_to_unmap, 900 * because ttu takes the rmap data structures down. 901 * 902 * Error handling: We ignore errors here because 903 * there's nothing that can be done. 904 */ 905 if (kill) 906 collect_procs(hpage, &tokill); 907 908 /* 909 * try_to_unmap can fail temporarily due to races. 910 * Try a few times (RED-PEN better strategy?) 911 */ 912 for (i = 0; i < N_UNMAP_TRIES; i++) { 913 ret = try_to_unmap(hpage, ttu); 914 if (ret == SWAP_SUCCESS) 915 break; 916 pr_debug("MCE %#lx: try_to_unmap retry needed %d\n", pfn, ret); 917 } 918 919 if (ret != SWAP_SUCCESS) 920 printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n", 921 pfn, page_mapcount(hpage)); 922 923 /* 924 * Now that the dirty bit has been propagated to the 925 * struct page and all unmaps done we can decide if 926 * killing is needed or not. Only kill when the page 927 * was dirty, otherwise the tokill list is merely 928 * freed. When there was a problem unmapping earlier 929 * use a more force-full uncatchable kill to prevent 930 * any accesses to the poisoned memory. 931 */ 932 kill_procs_ao(&tokill, !!PageDirty(hpage), trapno, 933 ret != SWAP_SUCCESS, p, pfn); 934 935 return ret; 936} 937 938static void set_page_hwpoison_huge_page(struct page *hpage) 939{ 940 int i; 941 int nr_pages = 1 << compound_order(hpage); 942 for (i = 0; i < nr_pages; i++) 943 SetPageHWPoison(hpage + i); 944} 945 946static void clear_page_hwpoison_huge_page(struct page *hpage) 947{ 948 int i; 949 int nr_pages = 1 << compound_order(hpage); 950 for (i = 0; i < nr_pages; i++) 951 ClearPageHWPoison(hpage + i); 952} 953 954int __memory_failure(unsigned long pfn, int trapno, int flags) 955{ 956 struct page_state *ps; 957 struct page *p; 958 struct page *hpage; 959 int res; 960 unsigned int nr_pages; 961 962 if (!sysctl_memory_failure_recovery) 963 panic("Memory failure from trap %d on page %lx", trapno, pfn); 964 965 if (!pfn_valid(pfn)) { 966 printk(KERN_ERR 967 "MCE %#lx: memory outside kernel control\n", 968 pfn); 969 return -ENXIO; 970 } 971 972 p = pfn_to_page(pfn); 973 hpage = compound_head(p); 974 if (TestSetPageHWPoison(p)) { 975 printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn); 976 return 0; 977 } 978 979 nr_pages = 1 << compound_order(hpage); 980 atomic_long_add(nr_pages, &mce_bad_pages); 981 982 /* 983 * We need/can do nothing about count=0 pages. 984 * 1) it's a free page, and therefore in safe hand: 985 * prep_new_page() will be the gate keeper. 986 * 2) it's a free hugepage, which is also safe: 987 * an affected hugepage will be dequeued from hugepage freelist, 988 * so there's no concern about reusing it ever after. 989 * 3) it's part of a non-compound high order page. 990 * Implies some kernel user: cannot stop them from 991 * R/W the page; let's pray that the page has been 992 * used and will be freed some time later. 993 * In fact it's dangerous to directly bump up page count from 0, 994 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch. 995 */ 996 if (!(flags & MF_COUNT_INCREASED) && 997 !get_page_unless_zero(hpage)) { 998 if (is_free_buddy_page(p)) { 999 action_result(pfn, "free buddy", DELAYED); 1000 return 0; 1001 } else if (PageHuge(hpage)) { 1002 /* 1003 * Check "just unpoisoned", "filter hit", and 1004 * "race with other subpage." 1005 */ 1006 lock_page_nosync(hpage); 1007 if (!PageHWPoison(hpage) 1008 || (hwpoison_filter(p) && TestClearPageHWPoison(p)) 1009 || (p != hpage && TestSetPageHWPoison(hpage))) { 1010 atomic_long_sub(nr_pages, &mce_bad_pages); 1011 return 0; 1012 } 1013 set_page_hwpoison_huge_page(hpage); 1014 res = dequeue_hwpoisoned_huge_page(hpage); 1015 action_result(pfn, "free huge", 1016 res ? IGNORED : DELAYED); 1017 unlock_page(hpage); 1018 return res; 1019 } else { 1020 action_result(pfn, "high order kernel", IGNORED); 1021 return -EBUSY; 1022 } 1023 } 1024 1025 /* 1026 * We ignore non-LRU pages for good reasons. 1027 * - PG_locked is only well defined for LRU pages and a few others 1028 * - to avoid races with __set_page_locked() 1029 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) 1030 * The check (unnecessarily) ignores LRU pages being isolated and 1031 * walked by the page reclaim code, however that's not a big loss. 1032 */ 1033 if (!PageLRU(p) && !PageHuge(p)) 1034 shake_page(p, 0); 1035 if (!PageLRU(p) && !PageHuge(p)) { 1036 /* 1037 * shake_page could have turned it free. 1038 */ 1039 if (is_free_buddy_page(p)) { 1040 action_result(pfn, "free buddy, 2nd try", DELAYED); 1041 return 0; 1042 } 1043 action_result(pfn, "non LRU", IGNORED); 1044 put_page(p); 1045 return -EBUSY; 1046 } 1047 1048 /* 1049 * Lock the page and wait for writeback to finish. 1050 * It's very difficult to mess with pages currently under IO 1051 * and in many cases impossible, so we just avoid it here. 1052 */ 1053 lock_page_nosync(hpage); 1054 1055 /* 1056 * unpoison always clear PG_hwpoison inside page lock 1057 */ 1058 if (!PageHWPoison(p)) { 1059 printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn); 1060 res = 0; 1061 goto out; 1062 } 1063 if (hwpoison_filter(p)) { 1064 if (TestClearPageHWPoison(p)) 1065 atomic_long_sub(nr_pages, &mce_bad_pages); 1066 unlock_page(hpage); 1067 put_page(hpage); 1068 return 0; 1069 } 1070 1071 /* 1072 * For error on the tail page, we should set PG_hwpoison 1073 * on the head page to show that the hugepage is hwpoisoned 1074 */ 1075 if (PageTail(p) && TestSetPageHWPoison(hpage)) { 1076 action_result(pfn, "hugepage already hardware poisoned", 1077 IGNORED); 1078 unlock_page(hpage); 1079 put_page(hpage); 1080 return 0; 1081 } 1082 /* 1083 * Set PG_hwpoison on all pages in an error hugepage, 1084 * because containment is done in hugepage unit for now. 1085 * Since we have done TestSetPageHWPoison() for the head page with 1086 * page lock held, we can safely set PG_hwpoison bits on tail pages. 1087 */ 1088 if (PageHuge(p)) 1089 set_page_hwpoison_huge_page(hpage); 1090 1091 wait_on_page_writeback(p); 1092 1093 /* 1094 * Now take care of user space mappings. 1095 * Abort on fail: __remove_from_page_cache() assumes unmapped page. 1096 */ 1097 if (hwpoison_user_mappings(p, pfn, trapno) != SWAP_SUCCESS) { 1098 printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn); 1099 res = -EBUSY; 1100 goto out; 1101 } 1102 1103 /* 1104 * Torn down by someone else? 1105 */ 1106 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { 1107 action_result(pfn, "already truncated LRU", IGNORED); 1108 res = -EBUSY; 1109 goto out; 1110 } 1111 1112 res = -EBUSY; 1113 for (ps = error_states;; ps++) { 1114 if ((p->flags & ps->mask) == ps->res) { 1115 res = page_action(ps, p, pfn); 1116 break; 1117 } 1118 } 1119out: 1120 unlock_page(hpage); 1121 return res; 1122} 1123EXPORT_SYMBOL_GPL(__memory_failure); 1124 1125/** 1126 * memory_failure - Handle memory failure of a page. 1127 * @pfn: Page Number of the corrupted page 1128 * @trapno: Trap number reported in the signal to user space. 1129 * 1130 * This function is called by the low level machine check code 1131 * of an architecture when it detects hardware memory corruption 1132 * of a page. It tries its best to recover, which includes 1133 * dropping pages, killing processes etc. 1134 * 1135 * The function is primarily of use for corruptions that 1136 * happen outside the current execution context (e.g. when 1137 * detected by a background scrubber) 1138 * 1139 * Must run in process context (e.g. a work queue) with interrupts 1140 * enabled and no spinlocks hold. 1141 */ 1142void memory_failure(unsigned long pfn, int trapno) 1143{ 1144 __memory_failure(pfn, trapno, 0); 1145} 1146 1147/** 1148 * unpoison_memory - Unpoison a previously poisoned page 1149 * @pfn: Page number of the to be unpoisoned page 1150 * 1151 * Software-unpoison a page that has been poisoned by 1152 * memory_failure() earlier. 1153 * 1154 * This is only done on the software-level, so it only works 1155 * for linux injected failures, not real hardware failures 1156 * 1157 * Returns 0 for success, otherwise -errno. 1158 */ 1159int unpoison_memory(unsigned long pfn) 1160{ 1161 struct page *page; 1162 struct page *p; 1163 int freeit = 0; 1164 unsigned int nr_pages; 1165 1166 if (!pfn_valid(pfn)) 1167 return -ENXIO; 1168 1169 p = pfn_to_page(pfn); 1170 page = compound_head(p); 1171 1172 if (!PageHWPoison(p)) { 1173 pr_debug("MCE: Page was already unpoisoned %#lx\n", pfn); 1174 return 0; 1175 } 1176 1177 nr_pages = 1 << compound_order(page); 1178 1179 if (!get_page_unless_zero(page)) { 1180 /* 1181 * Since HWPoisoned hugepage should have non-zero refcount, 1182 * race between memory failure and unpoison seems to happen. 1183 * In such case unpoison fails and memory failure runs 1184 * to the end. 1185 */ 1186 if (PageHuge(page)) { 1187 pr_debug("MCE: Memory failure is now running on free hugepage %#lx\n", pfn); 1188 return 0; 1189 } 1190 if (TestClearPageHWPoison(p)) 1191 atomic_long_sub(nr_pages, &mce_bad_pages); 1192 pr_debug("MCE: Software-unpoisoned free page %#lx\n", pfn); 1193 return 0; 1194 } 1195 1196 lock_page_nosync(page); 1197 /* 1198 * This test is racy because PG_hwpoison is set outside of page lock. 1199 * That's acceptable because that won't trigger kernel panic. Instead, 1200 * the PG_hwpoison page will be caught and isolated on the entrance to 1201 * the free buddy page pool. 1202 */ 1203 if (TestClearPageHWPoison(page)) { 1204 pr_debug("MCE: Software-unpoisoned page %#lx\n", pfn); 1205 atomic_long_sub(nr_pages, &mce_bad_pages); 1206 freeit = 1; 1207 } 1208 if (PageHuge(p)) 1209 clear_page_hwpoison_huge_page(page); 1210 unlock_page(page); 1211 1212 put_page(page); 1213 if (freeit) 1214 put_page(page); 1215 1216 return 0; 1217} 1218EXPORT_SYMBOL(unpoison_memory); 1219 1220static struct page *new_page(struct page *p, unsigned long private, int **x) 1221{ 1222 int nid = page_to_nid(p); 1223 return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0); 1224} 1225 1226/* 1227 * Safely get reference count of an arbitrary page. 1228 * Returns 0 for a free page, -EIO for a zero refcount page 1229 * that is not free, and 1 for any other page type. 1230 * For 1 the page is returned with increased page count, otherwise not. 1231 */ 1232static int get_any_page(struct page *p, unsigned long pfn, int flags) 1233{ 1234 int ret; 1235 1236 if (flags & MF_COUNT_INCREASED) 1237 return 1; 1238 1239 /* 1240 * The lock_system_sleep prevents a race with memory hotplug, 1241 * because the isolation assumes there's only a single user. 1242 * This is a big hammer, a better would be nicer. 1243 */ 1244 lock_system_sleep(); 1245 1246 /* 1247 * Isolate the page, so that it doesn't get reallocated if it 1248 * was free. 1249 */ 1250 set_migratetype_isolate(p); 1251 if (!get_page_unless_zero(compound_head(p))) { 1252 if (is_free_buddy_page(p)) { 1253 pr_debug("get_any_page: %#lx free buddy page\n", pfn); 1254 /* Set hwpoison bit while page is still isolated */ 1255 SetPageHWPoison(p); 1256 ret = 0; 1257 } else { 1258 pr_debug("get_any_page: %#lx: unknown zero refcount page type %lx\n", 1259 pfn, p->flags); 1260 ret = -EIO; 1261 } 1262 } else { 1263 /* Not a free page */ 1264 ret = 1; 1265 } 1266 unset_migratetype_isolate(p); 1267 unlock_system_sleep(); 1268 return ret; 1269} 1270 1271/** 1272 * soft_offline_page - Soft offline a page. 1273 * @page: page to offline 1274 * @flags: flags. Same as memory_failure(). 1275 * 1276 * Returns 0 on success, otherwise negated errno. 1277 * 1278 * Soft offline a page, by migration or invalidation, 1279 * without killing anything. This is for the case when 1280 * a page is not corrupted yet (so it's still valid to access), 1281 * but has had a number of corrected errors and is better taken 1282 * out. 1283 * 1284 * The actual policy on when to do that is maintained by 1285 * user space. 1286 * 1287 * This should never impact any application or cause data loss, 1288 * however it might take some time. 1289 * 1290 * This is not a 100% solution for all memory, but tries to be 1291 * ``good enough'' for the majority of memory. 1292 */ 1293int soft_offline_page(struct page *page, int flags) 1294{ 1295 int ret; 1296 unsigned long pfn = page_to_pfn(page); 1297 1298 ret = get_any_page(page, pfn, flags); 1299 if (ret < 0) 1300 return ret; 1301 if (ret == 0) 1302 goto done; 1303 1304 /* 1305 * Page cache page we can handle? 1306 */ 1307 if (!PageLRU(page)) { 1308 /* 1309 * Try to free it. 1310 */ 1311 put_page(page); 1312 shake_page(page, 1); 1313 1314 /* 1315 * Did it turn free? 1316 */ 1317 ret = get_any_page(page, pfn, 0); 1318 if (ret < 0) 1319 return ret; 1320 if (ret == 0) 1321 goto done; 1322 } 1323 if (!PageLRU(page)) { 1324 pr_debug("soft_offline: %#lx: unknown non LRU page type %lx\n", 1325 pfn, page->flags); 1326 return -EIO; 1327 } 1328 1329 lock_page(page); 1330 wait_on_page_writeback(page); 1331 1332 /* 1333 * Synchronized using the page lock with memory_failure() 1334 */ 1335 if (PageHWPoison(page)) { 1336 unlock_page(page); 1337 put_page(page); 1338 pr_debug("soft offline: %#lx page already poisoned\n", pfn); 1339 return -EBUSY; 1340 } 1341 1342 /* 1343 * Try to invalidate first. This should work for 1344 * non dirty unmapped page cache pages. 1345 */ 1346 ret = invalidate_inode_page(page); 1347 unlock_page(page); 1348 1349 /* 1350 * Drop count because page migration doesn't like raised 1351 * counts. The page could get re-allocated, but if it becomes 1352 * LRU the isolation will just fail. 1353 * RED-PEN would be better to keep it isolated here, but we 1354 * would need to fix isolation locking first. 1355 */ 1356 put_page(page); 1357 if (ret == 1) { 1358 ret = 0; 1359 pr_debug("soft_offline: %#lx: invalidated\n", pfn); 1360 goto done; 1361 } 1362 1363 /* 1364 * Simple invalidation didn't work. 1365 * Try to migrate to a new page instead. migrate.c 1366 * handles a large number of cases for us. 1367 */ 1368 ret = isolate_lru_page(page); 1369 if (!ret) { 1370 LIST_HEAD(pagelist); 1371 1372 list_add(&page->lru, &pagelist); 1373 ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0); 1374 if (ret) { 1375 pr_debug("soft offline: %#lx: migration failed %d, type %lx\n", 1376 pfn, ret, page->flags); 1377 if (ret > 0) 1378 ret = -EIO; 1379 } 1380 } else { 1381 pr_debug("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n", 1382 pfn, ret, page_count(page), page->flags); 1383 } 1384 if (ret) 1385 return ret; 1386 1387done: 1388 atomic_long_add(1, &mce_bad_pages); 1389 SetPageHWPoison(page); 1390 /* keep elevated page count for bad page */ 1391 return ret; 1392} 1393 1394/* 1395 * The caller must hold current->mm->mmap_sem in read mode. 1396 */ 1397int is_hwpoison_address(unsigned long addr) 1398{ 1399 pgd_t *pgdp; 1400 pud_t pud, *pudp; 1401 pmd_t pmd, *pmdp; 1402 pte_t pte, *ptep; 1403 swp_entry_t entry; 1404 1405 pgdp = pgd_offset(current->mm, addr); 1406 if (!pgd_present(*pgdp)) 1407 return 0; 1408 pudp = pud_offset(pgdp, addr); 1409 pud = *pudp; 1410 if (!pud_present(pud) || pud_large(pud)) 1411 return 0; 1412 pmdp = pmd_offset(pudp, addr); 1413 pmd = *pmdp; 1414 if (!pmd_present(pmd) || pmd_large(pmd)) 1415 return 0; 1416 ptep = pte_offset_map(pmdp, addr); 1417 pte = *ptep; 1418 pte_unmap(ptep); 1419 if (!is_swap_pte(pte)) 1420 return 0; 1421 entry = pte_to_swp_entry(pte); 1422 return is_hwpoison_entry(entry); 1423} 1424EXPORT_SYMBOL_GPL(is_hwpoison_address); 1425