1 /* 2 * Copyright (C) 2008, 2009 Intel Corporation 3 * Authors: Andi Kleen, Fengguang Wu 4 * 5 * This software may be redistributed and/or modified under the terms of 6 * the GNU General Public License ("GPL") version 2 only as published by the 7 * Free Software Foundation. 8 * 9 * High level machine check handler. Handles pages reported by the 10 * hardware as being corrupted usually due to a multi-bit ECC memory or cache 11 * failure. 12 * 13 * In addition there is a "soft offline" entry point that allows stop using 14 * not-yet-corrupted-by-suspicious pages without killing anything. 15 * 16 * Handles page cache pages in various states. The tricky part 17 * here is that we can access any page asynchronously in respect to 18 * other VM users, because memory failures could happen anytime and 19 * anywhere. This could violate some of their assumptions. This is why 20 * this code has to be extremely careful. Generally it tries to use 21 * normal locking rules, as in get the standard locks, even if that means 22 * the error handling takes potentially a long time. 23 * 24 * There are several operations here with exponential complexity because 25 * of unsuitable VM data structures. For example the operation to map back 26 * from RMAP chains to processes has to walk the complete process list and 27 * has non linear complexity with the number. But since memory corruptions 28 * are rare we hope to get away with this. This avoids impacting the core 29 * VM. 30 */ 31 32 /* 33 * Notebook: 34 * - hugetlb needs more code 35 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages 36 * - pass bad pages to kdump next kernel 37 */ 38 #include <linux/kernel.h> 39 #include <linux/mm.h> 40 #include <linux/page-flags.h> 41 #include <linux/kernel-page-flags.h> 42 #include <linux/sched.h> 43 #include <linux/ksm.h> 44 #include <linux/rmap.h> 45 #include <linux/export.h> 46 #include <linux/pagemap.h> 47 #include <linux/swap.h> 48 #include <linux/backing-dev.h> 49 #include <linux/migrate.h> 50 #include <linux/page-isolation.h> 51 #include <linux/suspend.h> 52 #include <linux/slab.h> 53 #include <linux/swapops.h> 54 #include <linux/hugetlb.h> 55 #include <linux/memory_hotplug.h> 56 #include <linux/mm_inline.h> 57 #include <linux/kfifo.h> 58 #include "internal.h" 59 60 int sysctl_memory_failure_early_kill __read_mostly = 0; 61 62 int sysctl_memory_failure_recovery __read_mostly = 1; 63 64 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); 65 66 #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE) 67 68 u32 hwpoison_filter_enable = 0; 69 u32 hwpoison_filter_dev_major = ~0U; 70 u32 hwpoison_filter_dev_minor = ~0U; 71 u64 hwpoison_filter_flags_mask; 72 u64 hwpoison_filter_flags_value; 73 EXPORT_SYMBOL_GPL(hwpoison_filter_enable); 74 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major); 75 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor); 76 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask); 77 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value); 78 79 static int hwpoison_filter_dev(struct page *p) 80 { 81 struct address_space *mapping; 82 dev_t dev; 83 84 if (hwpoison_filter_dev_major == ~0U && 85 hwpoison_filter_dev_minor == ~0U) 86 return 0; 87 88 /* 89 * page_mapping() does not accept slab pages. 90 */ 91 if (PageSlab(p)) 92 return -EINVAL; 93 94 mapping = page_mapping(p); 95 if (mapping == NULL || mapping->host == NULL) 96 return -EINVAL; 97 98 dev = mapping->host->i_sb->s_dev; 99 if (hwpoison_filter_dev_major != ~0U && 100 hwpoison_filter_dev_major != MAJOR(dev)) 101 return -EINVAL; 102 if (hwpoison_filter_dev_minor != ~0U && 103 hwpoison_filter_dev_minor != MINOR(dev)) 104 return -EINVAL; 105 106 return 0; 107 } 108 109 static int hwpoison_filter_flags(struct page *p) 110 { 111 if (!hwpoison_filter_flags_mask) 112 return 0; 113 114 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) == 115 hwpoison_filter_flags_value) 116 return 0; 117 else 118 return -EINVAL; 119 } 120 121 /* 122 * This allows stress tests to limit test scope to a collection of tasks 123 * by putting them under some memcg. This prevents killing unrelated/important 124 * processes such as /sbin/init. Note that the target task may share clean 125 * pages with init (eg. libc text), which is harmless. If the target task 126 * share _dirty_ pages with another task B, the test scheme must make sure B 127 * is also included in the memcg. At last, due to race conditions this filter 128 * can only guarantee that the page either belongs to the memcg tasks, or is 129 * a freed page. 130 */ 131 #ifdef CONFIG_MEMCG_SWAP 132 u64 hwpoison_filter_memcg; 133 EXPORT_SYMBOL_GPL(hwpoison_filter_memcg); 134 static int hwpoison_filter_task(struct page *p) 135 { 136 struct mem_cgroup *mem; 137 struct cgroup_subsys_state *css; 138 unsigned long ino; 139 140 if (!hwpoison_filter_memcg) 141 return 0; 142 143 mem = try_get_mem_cgroup_from_page(p); 144 if (!mem) 145 return -EINVAL; 146 147 css = mem_cgroup_css(mem); 148 ino = cgroup_ino(css->cgroup); 149 css_put(css); 150 151 if (!ino || ino != hwpoison_filter_memcg) 152 return -EINVAL; 153 154 return 0; 155 } 156 #else 157 static int hwpoison_filter_task(struct page *p) { return 0; } 158 #endif 159 160 int hwpoison_filter(struct page *p) 161 { 162 if (!hwpoison_filter_enable) 163 return 0; 164 165 if (hwpoison_filter_dev(p)) 166 return -EINVAL; 167 168 if (hwpoison_filter_flags(p)) 169 return -EINVAL; 170 171 if (hwpoison_filter_task(p)) 172 return -EINVAL; 173 174 return 0; 175 } 176 #else 177 int hwpoison_filter(struct page *p) 178 { 179 return 0; 180 } 181 #endif 182 183 EXPORT_SYMBOL_GPL(hwpoison_filter); 184 185 /* 186 * Send all the processes who have the page mapped a signal. 187 * ``action optional'' if they are not immediately affected by the error 188 * ``action required'' if error happened in current execution context 189 */ 190 static int kill_proc(struct task_struct *t, unsigned long addr, int trapno, 191 unsigned long pfn, struct page *page, int flags) 192 { 193 struct siginfo si; 194 int ret; 195 196 printk(KERN_ERR 197 "MCE %#lx: Killing %s:%d due to hardware memory corruption\n", 198 pfn, t->comm, t->pid); 199 si.si_signo = SIGBUS; 200 si.si_errno = 0; 201 si.si_addr = (void *)addr; 202 #ifdef __ARCH_SI_TRAPNO 203 si.si_trapno = trapno; 204 #endif 205 si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT; 206 207 if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) { 208 si.si_code = BUS_MCEERR_AR; 209 ret = force_sig_info(SIGBUS, &si, current); 210 } else { 211 /* 212 * Don't use force here, it's convenient if the signal 213 * can be temporarily blocked. 214 * This could cause a loop when the user sets SIGBUS 215 * to SIG_IGN, but hopefully no one will do that? 216 */ 217 si.si_code = BUS_MCEERR_AO; 218 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */ 219 } 220 if (ret < 0) 221 printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n", 222 t->comm, t->pid, ret); 223 return ret; 224 } 225 226 /* 227 * When a unknown page type is encountered drain as many buffers as possible 228 * in the hope to turn the page into a LRU or free page, which we can handle. 229 */ 230 void shake_page(struct page *p, int access) 231 { 232 if (!PageSlab(p)) { 233 lru_add_drain_all(); 234 if (PageLRU(p)) 235 return; 236 drain_all_pages(); 237 if (PageLRU(p) || is_free_buddy_page(p)) 238 return; 239 } 240 241 /* 242 * Only call shrink_slab here (which would also shrink other caches) if 243 * access is not potentially fatal. 244 */ 245 if (access) { 246 int nr; 247 int nid = page_to_nid(p); 248 do { 249 struct shrink_control shrink = { 250 .gfp_mask = GFP_KERNEL, 251 }; 252 node_set(nid, shrink.nodes_to_scan); 253 254 nr = shrink_slab(&shrink, 1000, 1000); 255 if (page_count(p) == 1) 256 break; 257 } while (nr > 10); 258 } 259 } 260 EXPORT_SYMBOL_GPL(shake_page); 261 262 /* 263 * Kill all processes that have a poisoned page mapped and then isolate 264 * the page. 265 * 266 * General strategy: 267 * Find all processes having the page mapped and kill them. 268 * But we keep a page reference around so that the page is not 269 * actually freed yet. 270 * Then stash the page away 271 * 272 * There's no convenient way to get back to mapped processes 273 * from the VMAs. So do a brute-force search over all 274 * running processes. 275 * 276 * Remember that machine checks are not common (or rather 277 * if they are common you have other problems), so this shouldn't 278 * be a performance issue. 279 * 280 * Also there are some races possible while we get from the 281 * error detection to actually handle it. 282 */ 283 284 struct to_kill { 285 struct list_head nd; 286 struct task_struct *tsk; 287 unsigned long addr; 288 char addr_valid; 289 }; 290 291 /* 292 * Failure handling: if we can't find or can't kill a process there's 293 * not much we can do. We just print a message and ignore otherwise. 294 */ 295 296 /* 297 * Schedule a process for later kill. 298 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. 299 * TBD would GFP_NOIO be enough? 300 */ 301 static void add_to_kill(struct task_struct *tsk, struct page *p, 302 struct vm_area_struct *vma, 303 struct list_head *to_kill, 304 struct to_kill **tkc) 305 { 306 struct to_kill *tk; 307 308 if (*tkc) { 309 tk = *tkc; 310 *tkc = NULL; 311 } else { 312 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC); 313 if (!tk) { 314 printk(KERN_ERR 315 "MCE: Out of memory while machine check handling\n"); 316 return; 317 } 318 } 319 tk->addr = page_address_in_vma(p, vma); 320 tk->addr_valid = 1; 321 322 /* 323 * In theory we don't have to kill when the page was 324 * munmaped. But it could be also a mremap. Since that's 325 * likely very rare kill anyways just out of paranoia, but use 326 * a SIGKILL because the error is not contained anymore. 327 */ 328 if (tk->addr == -EFAULT) { 329 pr_info("MCE: Unable to find user space address %lx in %s\n", 330 page_to_pfn(p), tsk->comm); 331 tk->addr_valid = 0; 332 } 333 get_task_struct(tsk); 334 tk->tsk = tsk; 335 list_add_tail(&tk->nd, to_kill); 336 } 337 338 /* 339 * Kill the processes that have been collected earlier. 340 * 341 * Only do anything when DOIT is set, otherwise just free the list 342 * (this is used for clean pages which do not need killing) 343 * Also when FAIL is set do a force kill because something went 344 * wrong earlier. 345 */ 346 static void kill_procs(struct list_head *to_kill, int forcekill, int trapno, 347 int fail, struct page *page, unsigned long pfn, 348 int flags) 349 { 350 struct to_kill *tk, *next; 351 352 list_for_each_entry_safe (tk, next, to_kill, nd) { 353 if (forcekill) { 354 /* 355 * In case something went wrong with munmapping 356 * make sure the process doesn't catch the 357 * signal and then access the memory. Just kill it. 358 */ 359 if (fail || tk->addr_valid == 0) { 360 printk(KERN_ERR 361 "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", 362 pfn, tk->tsk->comm, tk->tsk->pid); 363 force_sig(SIGKILL, tk->tsk); 364 } 365 366 /* 367 * In theory the process could have mapped 368 * something else on the address in-between. We could 369 * check for that, but we need to tell the 370 * process anyways. 371 */ 372 else if (kill_proc(tk->tsk, tk->addr, trapno, 373 pfn, page, flags) < 0) 374 printk(KERN_ERR 375 "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n", 376 pfn, tk->tsk->comm, tk->tsk->pid); 377 } 378 put_task_struct(tk->tsk); 379 kfree(tk); 380 } 381 } 382 383 /* 384 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) 385 * on behalf of the thread group. Return task_struct of the (first found) 386 * dedicated thread if found, and return NULL otherwise. 387 * 388 * We already hold read_lock(&tasklist_lock) in the caller, so we don't 389 * have to call rcu_read_lock/unlock() in this function. 390 */ 391 static struct task_struct *find_early_kill_thread(struct task_struct *tsk) 392 { 393 struct task_struct *t; 394 395 for_each_thread(tsk, t) 396 if ((t->flags & PF_MCE_PROCESS) && (t->flags & PF_MCE_EARLY)) 397 return t; 398 return NULL; 399 } 400 401 /* 402 * Determine whether a given process is "early kill" process which expects 403 * to be signaled when some page under the process is hwpoisoned. 404 * Return task_struct of the dedicated thread (main thread unless explicitly 405 * specified) if the process is "early kill," and otherwise returns NULL. 406 */ 407 static struct task_struct *task_early_kill(struct task_struct *tsk, 408 int force_early) 409 { 410 struct task_struct *t; 411 if (!tsk->mm) 412 return NULL; 413 if (force_early) 414 return tsk; 415 t = find_early_kill_thread(tsk); 416 if (t) 417 return t; 418 if (sysctl_memory_failure_early_kill) 419 return tsk; 420 return NULL; 421 } 422 423 /* 424 * Collect processes when the error hit an anonymous page. 425 */ 426 static void collect_procs_anon(struct page *page, struct list_head *to_kill, 427 struct to_kill **tkc, int force_early) 428 { 429 struct vm_area_struct *vma; 430 struct task_struct *tsk; 431 struct anon_vma *av; 432 pgoff_t pgoff; 433 434 av = page_lock_anon_vma_read(page); 435 if (av == NULL) /* Not actually mapped anymore */ 436 return; 437 438 pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 439 read_lock(&tasklist_lock); 440 for_each_process (tsk) { 441 struct anon_vma_chain *vmac; 442 struct task_struct *t = task_early_kill(tsk, force_early); 443 444 if (!t) 445 continue; 446 anon_vma_interval_tree_foreach(vmac, &av->rb_root, 447 pgoff, pgoff) { 448 vma = vmac->vma; 449 if (!page_mapped_in_vma(page, vma)) 450 continue; 451 if (vma->vm_mm == t->mm) 452 add_to_kill(t, page, vma, to_kill, tkc); 453 } 454 } 455 read_unlock(&tasklist_lock); 456 page_unlock_anon_vma_read(av); 457 } 458 459 /* 460 * Collect processes when the error hit a file mapped page. 461 */ 462 static void collect_procs_file(struct page *page, struct list_head *to_kill, 463 struct to_kill **tkc, int force_early) 464 { 465 struct vm_area_struct *vma; 466 struct task_struct *tsk; 467 struct address_space *mapping = page->mapping; 468 469 mutex_lock(&mapping->i_mmap_mutex); 470 read_lock(&tasklist_lock); 471 for_each_process(tsk) { 472 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 473 struct task_struct *t = task_early_kill(tsk, force_early); 474 475 if (!t) 476 continue; 477 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, 478 pgoff) { 479 /* 480 * Send early kill signal to tasks where a vma covers 481 * the page but the corrupted page is not necessarily 482 * mapped it in its pte. 483 * Assume applications who requested early kill want 484 * to be informed of all such data corruptions. 485 */ 486 if (vma->vm_mm == t->mm) 487 add_to_kill(t, page, vma, to_kill, tkc); 488 } 489 } 490 read_unlock(&tasklist_lock); 491 mutex_unlock(&mapping->i_mmap_mutex); 492 } 493 494 /* 495 * Collect the processes who have the corrupted page mapped to kill. 496 * This is done in two steps for locking reasons. 497 * First preallocate one tokill structure outside the spin locks, 498 * so that we can kill at least one process reasonably reliable. 499 */ 500 static void collect_procs(struct page *page, struct list_head *tokill, 501 int force_early) 502 { 503 struct to_kill *tk; 504 505 if (!page->mapping) 506 return; 507 508 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO); 509 if (!tk) 510 return; 511 if (PageAnon(page)) 512 collect_procs_anon(page, tokill, &tk, force_early); 513 else 514 collect_procs_file(page, tokill, &tk, force_early); 515 kfree(tk); 516 } 517 518 /* 519 * Error handlers for various types of pages. 520 */ 521 522 enum outcome { 523 IGNORED, /* Error: cannot be handled */ 524 FAILED, /* Error: handling failed */ 525 DELAYED, /* Will be handled later */ 526 RECOVERED, /* Successfully recovered */ 527 }; 528 529 static const char *action_name[] = { 530 [IGNORED] = "Ignored", 531 [FAILED] = "Failed", 532 [DELAYED] = "Delayed", 533 [RECOVERED] = "Recovered", 534 }; 535 536 /* 537 * XXX: It is possible that a page is isolated from LRU cache, 538 * and then kept in swap cache or failed to remove from page cache. 539 * The page count will stop it from being freed by unpoison. 540 * Stress tests should be aware of this memory leak problem. 541 */ 542 static int delete_from_lru_cache(struct page *p) 543 { 544 if (!isolate_lru_page(p)) { 545 /* 546 * Clear sensible page flags, so that the buddy system won't 547 * complain when the page is unpoison-and-freed. 548 */ 549 ClearPageActive(p); 550 ClearPageUnevictable(p); 551 /* 552 * drop the page count elevated by isolate_lru_page() 553 */ 554 page_cache_release(p); 555 return 0; 556 } 557 return -EIO; 558 } 559 560 /* 561 * Error hit kernel page. 562 * Do nothing, try to be lucky and not touch this instead. For a few cases we 563 * could be more sophisticated. 564 */ 565 static int me_kernel(struct page *p, unsigned long pfn) 566 { 567 return IGNORED; 568 } 569 570 /* 571 * Page in unknown state. Do nothing. 572 */ 573 static int me_unknown(struct page *p, unsigned long pfn) 574 { 575 printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn); 576 return FAILED; 577 } 578 579 /* 580 * Clean (or cleaned) page cache page. 581 */ 582 static int me_pagecache_clean(struct page *p, unsigned long pfn) 583 { 584 int err; 585 int ret = FAILED; 586 struct address_space *mapping; 587 588 delete_from_lru_cache(p); 589 590 /* 591 * For anonymous pages we're done the only reference left 592 * should be the one m_f() holds. 593 */ 594 if (PageAnon(p)) 595 return RECOVERED; 596 597 /* 598 * Now truncate the page in the page cache. This is really 599 * more like a "temporary hole punch" 600 * Don't do this for block devices when someone else 601 * has a reference, because it could be file system metadata 602 * and that's not safe to truncate. 603 */ 604 mapping = page_mapping(p); 605 if (!mapping) { 606 /* 607 * Page has been teared down in the meanwhile 608 */ 609 return FAILED; 610 } 611 612 /* 613 * Truncation is a bit tricky. Enable it per file system for now. 614 * 615 * Open: to take i_mutex or not for this? Right now we don't. 616 */ 617 if (mapping->a_ops->error_remove_page) { 618 err = mapping->a_ops->error_remove_page(mapping, p); 619 if (err != 0) { 620 printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n", 621 pfn, err); 622 } else if (page_has_private(p) && 623 !try_to_release_page(p, GFP_NOIO)) { 624 pr_info("MCE %#lx: failed to release buffers\n", pfn); 625 } else { 626 ret = RECOVERED; 627 } 628 } else { 629 /* 630 * If the file system doesn't support it just invalidate 631 * This fails on dirty or anything with private pages 632 */ 633 if (invalidate_inode_page(p)) 634 ret = RECOVERED; 635 else 636 printk(KERN_INFO "MCE %#lx: Failed to invalidate\n", 637 pfn); 638 } 639 return ret; 640 } 641 642 /* 643 * Dirty pagecache page 644 * Issues: when the error hit a hole page the error is not properly 645 * propagated. 646 */ 647 static int me_pagecache_dirty(struct page *p, unsigned long pfn) 648 { 649 struct address_space *mapping = page_mapping(p); 650 651 SetPageError(p); 652 /* TBD: print more information about the file. */ 653 if (mapping) { 654 /* 655 * IO error will be reported by write(), fsync(), etc. 656 * who check the mapping. 657 * This way the application knows that something went 658 * wrong with its dirty file data. 659 * 660 * There's one open issue: 661 * 662 * The EIO will be only reported on the next IO 663 * operation and then cleared through the IO map. 664 * Normally Linux has two mechanisms to pass IO error 665 * first through the AS_EIO flag in the address space 666 * and then through the PageError flag in the page. 667 * Since we drop pages on memory failure handling the 668 * only mechanism open to use is through AS_AIO. 669 * 670 * This has the disadvantage that it gets cleared on 671 * the first operation that returns an error, while 672 * the PageError bit is more sticky and only cleared 673 * when the page is reread or dropped. If an 674 * application assumes it will always get error on 675 * fsync, but does other operations on the fd before 676 * and the page is dropped between then the error 677 * will not be properly reported. 678 * 679 * This can already happen even without hwpoisoned 680 * pages: first on metadata IO errors (which only 681 * report through AS_EIO) or when the page is dropped 682 * at the wrong time. 683 * 684 * So right now we assume that the application DTRT on 685 * the first EIO, but we're not worse than other parts 686 * of the kernel. 687 */ 688 mapping_set_error(mapping, EIO); 689 } 690 691 return me_pagecache_clean(p, pfn); 692 } 693 694 /* 695 * Clean and dirty swap cache. 696 * 697 * Dirty swap cache page is tricky to handle. The page could live both in page 698 * cache and swap cache(ie. page is freshly swapped in). So it could be 699 * referenced concurrently by 2 types of PTEs: 700 * normal PTEs and swap PTEs. We try to handle them consistently by calling 701 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs, 702 * and then 703 * - clear dirty bit to prevent IO 704 * - remove from LRU 705 * - but keep in the swap cache, so that when we return to it on 706 * a later page fault, we know the application is accessing 707 * corrupted data and shall be killed (we installed simple 708 * interception code in do_swap_page to catch it). 709 * 710 * Clean swap cache pages can be directly isolated. A later page fault will 711 * bring in the known good data from disk. 712 */ 713 static int me_swapcache_dirty(struct page *p, unsigned long pfn) 714 { 715 ClearPageDirty(p); 716 /* Trigger EIO in shmem: */ 717 ClearPageUptodate(p); 718 719 if (!delete_from_lru_cache(p)) 720 return DELAYED; 721 else 722 return FAILED; 723 } 724 725 static int me_swapcache_clean(struct page *p, unsigned long pfn) 726 { 727 delete_from_swap_cache(p); 728 729 if (!delete_from_lru_cache(p)) 730 return RECOVERED; 731 else 732 return FAILED; 733 } 734 735 /* 736 * Huge pages. Needs work. 737 * Issues: 738 * - Error on hugepage is contained in hugepage unit (not in raw page unit.) 739 * To narrow down kill region to one page, we need to break up pmd. 740 */ 741 static int me_huge_page(struct page *p, unsigned long pfn) 742 { 743 int res = 0; 744 struct page *hpage = compound_head(p); 745 /* 746 * We can safely recover from error on free or reserved (i.e. 747 * not in-use) hugepage by dequeuing it from freelist. 748 * To check whether a hugepage is in-use or not, we can't use 749 * page->lru because it can be used in other hugepage operations, 750 * such as __unmap_hugepage_range() and gather_surplus_pages(). 751 * So instead we use page_mapping() and PageAnon(). 752 * We assume that this function is called with page lock held, 753 * so there is no race between isolation and mapping/unmapping. 754 */ 755 if (!(page_mapping(hpage) || PageAnon(hpage))) { 756 res = dequeue_hwpoisoned_huge_page(hpage); 757 if (!res) 758 return RECOVERED; 759 } 760 return DELAYED; 761 } 762 763 /* 764 * Various page states we can handle. 765 * 766 * A page state is defined by its current page->flags bits. 767 * The table matches them in order and calls the right handler. 768 * 769 * This is quite tricky because we can access page at any time 770 * in its live cycle, so all accesses have to be extremely careful. 771 * 772 * This is not complete. More states could be added. 773 * For any missing state don't attempt recovery. 774 */ 775 776 #define dirty (1UL << PG_dirty) 777 #define sc (1UL << PG_swapcache) 778 #define unevict (1UL << PG_unevictable) 779 #define mlock (1UL << PG_mlocked) 780 #define writeback (1UL << PG_writeback) 781 #define lru (1UL << PG_lru) 782 #define swapbacked (1UL << PG_swapbacked) 783 #define head (1UL << PG_head) 784 #define tail (1UL << PG_tail) 785 #define compound (1UL << PG_compound) 786 #define slab (1UL << PG_slab) 787 #define reserved (1UL << PG_reserved) 788 789 static struct page_state { 790 unsigned long mask; 791 unsigned long res; 792 char *msg; 793 int (*action)(struct page *p, unsigned long pfn); 794 } error_states[] = { 795 { reserved, reserved, "reserved kernel", me_kernel }, 796 /* 797 * free pages are specially detected outside this table: 798 * PG_buddy pages only make a small fraction of all free pages. 799 */ 800 801 /* 802 * Could in theory check if slab page is free or if we can drop 803 * currently unused objects without touching them. But just 804 * treat it as standard kernel for now. 805 */ 806 { slab, slab, "kernel slab", me_kernel }, 807 808 #ifdef CONFIG_PAGEFLAGS_EXTENDED 809 { head, head, "huge", me_huge_page }, 810 { tail, tail, "huge", me_huge_page }, 811 #else 812 { compound, compound, "huge", me_huge_page }, 813 #endif 814 815 { sc|dirty, sc|dirty, "dirty swapcache", me_swapcache_dirty }, 816 { sc|dirty, sc, "clean swapcache", me_swapcache_clean }, 817 818 { mlock|dirty, mlock|dirty, "dirty mlocked LRU", me_pagecache_dirty }, 819 { mlock|dirty, mlock, "clean mlocked LRU", me_pagecache_clean }, 820 821 { unevict|dirty, unevict|dirty, "dirty unevictable LRU", me_pagecache_dirty }, 822 { unevict|dirty, unevict, "clean unevictable LRU", me_pagecache_clean }, 823 824 { lru|dirty, lru|dirty, "dirty LRU", me_pagecache_dirty }, 825 { lru|dirty, lru, "clean LRU", me_pagecache_clean }, 826 827 /* 828 * Catchall entry: must be at end. 829 */ 830 { 0, 0, "unknown page state", me_unknown }, 831 }; 832 833 #undef dirty 834 #undef sc 835 #undef unevict 836 #undef mlock 837 #undef writeback 838 #undef lru 839 #undef swapbacked 840 #undef head 841 #undef tail 842 #undef compound 843 #undef slab 844 #undef reserved 845 846 /* 847 * "Dirty/Clean" indication is not 100% accurate due to the possibility of 848 * setting PG_dirty outside page lock. See also comment above set_page_dirty(). 849 */ 850 static void action_result(unsigned long pfn, char *msg, int result) 851 { 852 pr_err("MCE %#lx: %s page recovery: %s\n", 853 pfn, msg, action_name[result]); 854 } 855 856 static int page_action(struct page_state *ps, struct page *p, 857 unsigned long pfn) 858 { 859 int result; 860 int count; 861 862 result = ps->action(p, pfn); 863 action_result(pfn, ps->msg, result); 864 865 count = page_count(p) - 1; 866 if (ps->action == me_swapcache_dirty && result == DELAYED) 867 count--; 868 if (count != 0) { 869 printk(KERN_ERR 870 "MCE %#lx: %s page still referenced by %d users\n", 871 pfn, ps->msg, count); 872 result = FAILED; 873 } 874 875 /* Could do more checks here if page looks ok */ 876 /* 877 * Could adjust zone counters here to correct for the missing page. 878 */ 879 880 return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY; 881 } 882 883 /* 884 * Do all that is necessary to remove user space mappings. Unmap 885 * the pages and send SIGBUS to the processes if the data was dirty. 886 */ 887 static int hwpoison_user_mappings(struct page *p, unsigned long pfn, 888 int trapno, int flags, struct page **hpagep) 889 { 890 enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; 891 struct address_space *mapping; 892 LIST_HEAD(tokill); 893 int ret; 894 int kill = 1, forcekill; 895 struct page *hpage = *hpagep; 896 struct page *ppage; 897 898 if (PageReserved(p) || PageSlab(p)) 899 return SWAP_SUCCESS; 900 901 /* 902 * This check implies we don't kill processes if their pages 903 * are in the swap cache early. Those are always late kills. 904 */ 905 if (!page_mapped(hpage)) 906 return SWAP_SUCCESS; 907 908 if (PageKsm(p)) 909 return SWAP_FAIL; 910 911 if (PageSwapCache(p)) { 912 printk(KERN_ERR 913 "MCE %#lx: keeping poisoned page in swap cache\n", pfn); 914 ttu |= TTU_IGNORE_HWPOISON; 915 } 916 917 /* 918 * Propagate the dirty bit from PTEs to struct page first, because we 919 * need this to decide if we should kill or just drop the page. 920 * XXX: the dirty test could be racy: set_page_dirty() may not always 921 * be called inside page lock (it's recommended but not enforced). 922 */ 923 mapping = page_mapping(hpage); 924 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping && 925 mapping_cap_writeback_dirty(mapping)) { 926 if (page_mkclean(hpage)) { 927 SetPageDirty(hpage); 928 } else { 929 kill = 0; 930 ttu |= TTU_IGNORE_HWPOISON; 931 printk(KERN_INFO 932 "MCE %#lx: corrupted page was clean: dropped without side effects\n", 933 pfn); 934 } 935 } 936 937 /* 938 * ppage: poisoned page 939 * if p is regular page(4k page) 940 * ppage == real poisoned page; 941 * else p is hugetlb or THP, ppage == head page. 942 */ 943 ppage = hpage; 944 945 if (PageTransHuge(hpage)) { 946 /* 947 * Verify that this isn't a hugetlbfs head page, the check for 948 * PageAnon is just for avoid tripping a split_huge_page 949 * internal debug check, as split_huge_page refuses to deal with 950 * anything that isn't an anon page. PageAnon can't go away fro 951 * under us because we hold a refcount on the hpage, without a 952 * refcount on the hpage. split_huge_page can't be safely called 953 * in the first place, having a refcount on the tail isn't 954 * enough * to be safe. 955 */ 956 if (!PageHuge(hpage) && PageAnon(hpage)) { 957 if (unlikely(split_huge_page(hpage))) { 958 /* 959 * FIXME: if splitting THP is failed, it is 960 * better to stop the following operation rather 961 * than causing panic by unmapping. System might 962 * survive if the page is freed later. 963 */ 964 printk(KERN_INFO 965 "MCE %#lx: failed to split THP\n", pfn); 966 967 BUG_ON(!PageHWPoison(p)); 968 return SWAP_FAIL; 969 } 970 /* 971 * We pinned the head page for hwpoison handling, 972 * now we split the thp and we are interested in 973 * the hwpoisoned raw page, so move the refcount 974 * to it. Similarly, page lock is shifted. 975 */ 976 if (hpage != p) { 977 if (!(flags & MF_COUNT_INCREASED)) { 978 put_page(hpage); 979 get_page(p); 980 } 981 lock_page(p); 982 unlock_page(hpage); 983 *hpagep = p; 984 } 985 /* THP is split, so ppage should be the real poisoned page. */ 986 ppage = p; 987 } 988 } 989 990 /* 991 * First collect all the processes that have the page 992 * mapped in dirty form. This has to be done before try_to_unmap, 993 * because ttu takes the rmap data structures down. 994 * 995 * Error handling: We ignore errors here because 996 * there's nothing that can be done. 997 */ 998 if (kill) 999 collect_procs(ppage, &tokill, flags & MF_ACTION_REQUIRED); 1000 1001 ret = try_to_unmap(ppage, ttu); 1002 if (ret != SWAP_SUCCESS) 1003 printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n", 1004 pfn, page_mapcount(ppage)); 1005 1006 /* 1007 * Now that the dirty bit has been propagated to the 1008 * struct page and all unmaps done we can decide if 1009 * killing is needed or not. Only kill when the page 1010 * was dirty or the process is not restartable, 1011 * otherwise the tokill list is merely 1012 * freed. When there was a problem unmapping earlier 1013 * use a more force-full uncatchable kill to prevent 1014 * any accesses to the poisoned memory. 1015 */ 1016 forcekill = PageDirty(ppage) || (flags & MF_MUST_KILL); 1017 kill_procs(&tokill, forcekill, trapno, 1018 ret != SWAP_SUCCESS, p, pfn, flags); 1019 1020 return ret; 1021 } 1022 1023 static void set_page_hwpoison_huge_page(struct page *hpage) 1024 { 1025 int i; 1026 int nr_pages = 1 << compound_order(hpage); 1027 for (i = 0; i < nr_pages; i++) 1028 SetPageHWPoison(hpage + i); 1029 } 1030 1031 static void clear_page_hwpoison_huge_page(struct page *hpage) 1032 { 1033 int i; 1034 int nr_pages = 1 << compound_order(hpage); 1035 for (i = 0; i < nr_pages; i++) 1036 ClearPageHWPoison(hpage + i); 1037 } 1038 1039 /** 1040 * memory_failure - Handle memory failure of a page. 1041 * @pfn: Page Number of the corrupted page 1042 * @trapno: Trap number reported in the signal to user space. 1043 * @flags: fine tune action taken 1044 * 1045 * This function is called by the low level machine check code 1046 * of an architecture when it detects hardware memory corruption 1047 * of a page. It tries its best to recover, which includes 1048 * dropping pages, killing processes etc. 1049 * 1050 * The function is primarily of use for corruptions that 1051 * happen outside the current execution context (e.g. when 1052 * detected by a background scrubber) 1053 * 1054 * Must run in process context (e.g. a work queue) with interrupts 1055 * enabled and no spinlocks hold. 1056 */ 1057 int memory_failure(unsigned long pfn, int trapno, int flags) 1058 { 1059 struct page_state *ps; 1060 struct page *p; 1061 struct page *hpage; 1062 int res; 1063 unsigned int nr_pages; 1064 unsigned long page_flags; 1065 1066 if (!sysctl_memory_failure_recovery) 1067 panic("Memory failure from trap %d on page %lx", trapno, pfn); 1068 1069 if (!pfn_valid(pfn)) { 1070 printk(KERN_ERR 1071 "MCE %#lx: memory outside kernel control\n", 1072 pfn); 1073 return -ENXIO; 1074 } 1075 1076 p = pfn_to_page(pfn); 1077 hpage = compound_head(p); 1078 if (TestSetPageHWPoison(p)) { 1079 printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn); 1080 return 0; 1081 } 1082 1083 /* 1084 * Currently errors on hugetlbfs pages are measured in hugepage units, 1085 * so nr_pages should be 1 << compound_order. OTOH when errors are on 1086 * transparent hugepages, they are supposed to be split and error 1087 * measurement is done in normal page units. So nr_pages should be one 1088 * in this case. 1089 */ 1090 if (PageHuge(p)) 1091 nr_pages = 1 << compound_order(hpage); 1092 else /* normal page or thp */ 1093 nr_pages = 1; 1094 atomic_long_add(nr_pages, &num_poisoned_pages); 1095 1096 /* 1097 * We need/can do nothing about count=0 pages. 1098 * 1) it's a free page, and therefore in safe hand: 1099 * prep_new_page() will be the gate keeper. 1100 * 2) it's a free hugepage, which is also safe: 1101 * an affected hugepage will be dequeued from hugepage freelist, 1102 * so there's no concern about reusing it ever after. 1103 * 3) it's part of a non-compound high order page. 1104 * Implies some kernel user: cannot stop them from 1105 * R/W the page; let's pray that the page has been 1106 * used and will be freed some time later. 1107 * In fact it's dangerous to directly bump up page count from 0, 1108 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch. 1109 */ 1110 if (!(flags & MF_COUNT_INCREASED) && 1111 !get_page_unless_zero(hpage)) { 1112 if (is_free_buddy_page(p)) { 1113 action_result(pfn, "free buddy", DELAYED); 1114 return 0; 1115 } else if (PageHuge(hpage)) { 1116 /* 1117 * Check "filter hit" and "race with other subpage." 1118 */ 1119 lock_page(hpage); 1120 if (PageHWPoison(hpage)) { 1121 if ((hwpoison_filter(p) && TestClearPageHWPoison(p)) 1122 || (p != hpage && TestSetPageHWPoison(hpage))) { 1123 atomic_long_sub(nr_pages, &num_poisoned_pages); 1124 unlock_page(hpage); 1125 return 0; 1126 } 1127 } 1128 set_page_hwpoison_huge_page(hpage); 1129 res = dequeue_hwpoisoned_huge_page(hpage); 1130 action_result(pfn, "free huge", 1131 res ? IGNORED : DELAYED); 1132 unlock_page(hpage); 1133 return res; 1134 } else { 1135 action_result(pfn, "high order kernel", IGNORED); 1136 return -EBUSY; 1137 } 1138 } 1139 1140 /* 1141 * We ignore non-LRU pages for good reasons. 1142 * - PG_locked is only well defined for LRU pages and a few others 1143 * - to avoid races with __set_page_locked() 1144 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) 1145 * The check (unnecessarily) ignores LRU pages being isolated and 1146 * walked by the page reclaim code, however that's not a big loss. 1147 */ 1148 if (!PageHuge(p) && !PageTransTail(p)) { 1149 if (!PageLRU(p)) 1150 shake_page(p, 0); 1151 if (!PageLRU(p)) { 1152 /* 1153 * shake_page could have turned it free. 1154 */ 1155 if (is_free_buddy_page(p)) { 1156 if (flags & MF_COUNT_INCREASED) 1157 action_result(pfn, "free buddy", DELAYED); 1158 else 1159 action_result(pfn, "free buddy, 2nd try", DELAYED); 1160 return 0; 1161 } 1162 action_result(pfn, "non LRU", IGNORED); 1163 put_page(p); 1164 return -EBUSY; 1165 } 1166 } 1167 1168 lock_page(hpage); 1169 1170 /* 1171 * We use page flags to determine what action should be taken, but 1172 * the flags can be modified by the error containment action. One 1173 * example is an mlocked page, where PG_mlocked is cleared by 1174 * page_remove_rmap() in try_to_unmap_one(). So to determine page status 1175 * correctly, we save a copy of the page flags at this time. 1176 */ 1177 page_flags = p->flags; 1178 1179 /* 1180 * unpoison always clear PG_hwpoison inside page lock 1181 */ 1182 if (!PageHWPoison(p)) { 1183 printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn); 1184 atomic_long_sub(nr_pages, &num_poisoned_pages); 1185 put_page(hpage); 1186 res = 0; 1187 goto out; 1188 } 1189 if (hwpoison_filter(p)) { 1190 if (TestClearPageHWPoison(p)) 1191 atomic_long_sub(nr_pages, &num_poisoned_pages); 1192 unlock_page(hpage); 1193 put_page(hpage); 1194 return 0; 1195 } 1196 1197 /* 1198 * For error on the tail page, we should set PG_hwpoison 1199 * on the head page to show that the hugepage is hwpoisoned 1200 */ 1201 if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) { 1202 action_result(pfn, "hugepage already hardware poisoned", 1203 IGNORED); 1204 unlock_page(hpage); 1205 put_page(hpage); 1206 return 0; 1207 } 1208 /* 1209 * Set PG_hwpoison on all pages in an error hugepage, 1210 * because containment is done in hugepage unit for now. 1211 * Since we have done TestSetPageHWPoison() for the head page with 1212 * page lock held, we can safely set PG_hwpoison bits on tail pages. 1213 */ 1214 if (PageHuge(p)) 1215 set_page_hwpoison_huge_page(hpage); 1216 1217 /* 1218 * It's very difficult to mess with pages currently under IO 1219 * and in many cases impossible, so we just avoid it here. 1220 */ 1221 wait_on_page_writeback(p); 1222 1223 /* 1224 * Now take care of user space mappings. 1225 * Abort on fail: __delete_from_page_cache() assumes unmapped page. 1226 * 1227 * When the raw error page is thp tail page, hpage points to the raw 1228 * page after thp split. 1229 */ 1230 if (hwpoison_user_mappings(p, pfn, trapno, flags, &hpage) 1231 != SWAP_SUCCESS) { 1232 printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn); 1233 res = -EBUSY; 1234 goto out; 1235 } 1236 1237 /* 1238 * Torn down by someone else? 1239 */ 1240 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { 1241 action_result(pfn, "already truncated LRU", IGNORED); 1242 res = -EBUSY; 1243 goto out; 1244 } 1245 1246 res = -EBUSY; 1247 /* 1248 * The first check uses the current page flags which may not have any 1249 * relevant information. The second check with the saved page flagss is 1250 * carried out only if the first check can't determine the page status. 1251 */ 1252 for (ps = error_states;; ps++) 1253 if ((p->flags & ps->mask) == ps->res) 1254 break; 1255 1256 page_flags |= (p->flags & (1UL << PG_dirty)); 1257 1258 if (!ps->mask) 1259 for (ps = error_states;; ps++) 1260 if ((page_flags & ps->mask) == ps->res) 1261 break; 1262 res = page_action(ps, p, pfn); 1263 out: 1264 unlock_page(hpage); 1265 return res; 1266 } 1267 EXPORT_SYMBOL_GPL(memory_failure); 1268 1269 #define MEMORY_FAILURE_FIFO_ORDER 4 1270 #define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER) 1271 1272 struct memory_failure_entry { 1273 unsigned long pfn; 1274 int trapno; 1275 int flags; 1276 }; 1277 1278 struct memory_failure_cpu { 1279 DECLARE_KFIFO(fifo, struct memory_failure_entry, 1280 MEMORY_FAILURE_FIFO_SIZE); 1281 spinlock_t lock; 1282 struct work_struct work; 1283 }; 1284 1285 static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu); 1286 1287 /** 1288 * memory_failure_queue - Schedule handling memory failure of a page. 1289 * @pfn: Page Number of the corrupted page 1290 * @trapno: Trap number reported in the signal to user space. 1291 * @flags: Flags for memory failure handling 1292 * 1293 * This function is called by the low level hardware error handler 1294 * when it detects hardware memory corruption of a page. It schedules 1295 * the recovering of error page, including dropping pages, killing 1296 * processes etc. 1297 * 1298 * The function is primarily of use for corruptions that 1299 * happen outside the current execution context (e.g. when 1300 * detected by a background scrubber) 1301 * 1302 * Can run in IRQ context. 1303 */ 1304 void memory_failure_queue(unsigned long pfn, int trapno, int flags) 1305 { 1306 struct memory_failure_cpu *mf_cpu; 1307 unsigned long proc_flags; 1308 struct memory_failure_entry entry = { 1309 .pfn = pfn, 1310 .trapno = trapno, 1311 .flags = flags, 1312 }; 1313 1314 mf_cpu = &get_cpu_var(memory_failure_cpu); 1315 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 1316 if (kfifo_put(&mf_cpu->fifo, entry)) 1317 schedule_work_on(smp_processor_id(), &mf_cpu->work); 1318 else 1319 pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n", 1320 pfn); 1321 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 1322 put_cpu_var(memory_failure_cpu); 1323 } 1324 EXPORT_SYMBOL_GPL(memory_failure_queue); 1325 1326 static void memory_failure_work_func(struct work_struct *work) 1327 { 1328 struct memory_failure_cpu *mf_cpu; 1329 struct memory_failure_entry entry = { 0, }; 1330 unsigned long proc_flags; 1331 int gotten; 1332 1333 mf_cpu = this_cpu_ptr(&memory_failure_cpu); 1334 for (;;) { 1335 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 1336 gotten = kfifo_get(&mf_cpu->fifo, &entry); 1337 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 1338 if (!gotten) 1339 break; 1340 if (entry.flags & MF_SOFT_OFFLINE) 1341 soft_offline_page(pfn_to_page(entry.pfn), entry.flags); 1342 else 1343 memory_failure(entry.pfn, entry.trapno, entry.flags); 1344 } 1345 } 1346 1347 static int __init memory_failure_init(void) 1348 { 1349 struct memory_failure_cpu *mf_cpu; 1350 int cpu; 1351 1352 for_each_possible_cpu(cpu) { 1353 mf_cpu = &per_cpu(memory_failure_cpu, cpu); 1354 spin_lock_init(&mf_cpu->lock); 1355 INIT_KFIFO(mf_cpu->fifo); 1356 INIT_WORK(&mf_cpu->work, memory_failure_work_func); 1357 } 1358 1359 return 0; 1360 } 1361 core_initcall(memory_failure_init); 1362 1363 /** 1364 * unpoison_memory - Unpoison a previously poisoned page 1365 * @pfn: Page number of the to be unpoisoned page 1366 * 1367 * Software-unpoison a page that has been poisoned by 1368 * memory_failure() earlier. 1369 * 1370 * This is only done on the software-level, so it only works 1371 * for linux injected failures, not real hardware failures 1372 * 1373 * Returns 0 for success, otherwise -errno. 1374 */ 1375 int unpoison_memory(unsigned long pfn) 1376 { 1377 struct page *page; 1378 struct page *p; 1379 int freeit = 0; 1380 unsigned int nr_pages; 1381 1382 if (!pfn_valid(pfn)) 1383 return -ENXIO; 1384 1385 p = pfn_to_page(pfn); 1386 page = compound_head(p); 1387 1388 if (!PageHWPoison(p)) { 1389 pr_info("MCE: Page was already unpoisoned %#lx\n", pfn); 1390 return 0; 1391 } 1392 1393 /* 1394 * unpoison_memory() can encounter thp only when the thp is being 1395 * worked by memory_failure() and the page lock is not held yet. 1396 * In such case, we yield to memory_failure() and make unpoison fail. 1397 */ 1398 if (!PageHuge(page) && PageTransHuge(page)) { 1399 pr_info("MCE: Memory failure is now running on %#lx\n", pfn); 1400 return 0; 1401 } 1402 1403 nr_pages = 1 << compound_order(page); 1404 1405 if (!get_page_unless_zero(page)) { 1406 /* 1407 * Since HWPoisoned hugepage should have non-zero refcount, 1408 * race between memory failure and unpoison seems to happen. 1409 * In such case unpoison fails and memory failure runs 1410 * to the end. 1411 */ 1412 if (PageHuge(page)) { 1413 pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn); 1414 return 0; 1415 } 1416 if (TestClearPageHWPoison(p)) 1417 atomic_long_dec(&num_poisoned_pages); 1418 pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn); 1419 return 0; 1420 } 1421 1422 lock_page(page); 1423 /* 1424 * This test is racy because PG_hwpoison is set outside of page lock. 1425 * That's acceptable because that won't trigger kernel panic. Instead, 1426 * the PG_hwpoison page will be caught and isolated on the entrance to 1427 * the free buddy page pool. 1428 */ 1429 if (TestClearPageHWPoison(page)) { 1430 pr_info("MCE: Software-unpoisoned page %#lx\n", pfn); 1431 atomic_long_sub(nr_pages, &num_poisoned_pages); 1432 freeit = 1; 1433 if (PageHuge(page)) 1434 clear_page_hwpoison_huge_page(page); 1435 } 1436 unlock_page(page); 1437 1438 put_page(page); 1439 if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) 1440 put_page(page); 1441 1442 return 0; 1443 } 1444 EXPORT_SYMBOL(unpoison_memory); 1445 1446 static struct page *new_page(struct page *p, unsigned long private, int **x) 1447 { 1448 int nid = page_to_nid(p); 1449 if (PageHuge(p)) 1450 return alloc_huge_page_node(page_hstate(compound_head(p)), 1451 nid); 1452 else 1453 return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0); 1454 } 1455 1456 /* 1457 * Safely get reference count of an arbitrary page. 1458 * Returns 0 for a free page, -EIO for a zero refcount page 1459 * that is not free, and 1 for any other page type. 1460 * For 1 the page is returned with increased page count, otherwise not. 1461 */ 1462 static int __get_any_page(struct page *p, unsigned long pfn, int flags) 1463 { 1464 int ret; 1465 1466 if (flags & MF_COUNT_INCREASED) 1467 return 1; 1468 1469 /* 1470 * When the target page is a free hugepage, just remove it 1471 * from free hugepage list. 1472 */ 1473 if (!get_page_unless_zero(compound_head(p))) { 1474 if (PageHuge(p)) { 1475 pr_info("%s: %#lx free huge page\n", __func__, pfn); 1476 ret = 0; 1477 } else if (is_free_buddy_page(p)) { 1478 pr_info("%s: %#lx free buddy page\n", __func__, pfn); 1479 ret = 0; 1480 } else { 1481 pr_info("%s: %#lx: unknown zero refcount page type %lx\n", 1482 __func__, pfn, p->flags); 1483 ret = -EIO; 1484 } 1485 } else { 1486 /* Not a free page */ 1487 ret = 1; 1488 } 1489 return ret; 1490 } 1491 1492 static int get_any_page(struct page *page, unsigned long pfn, int flags) 1493 { 1494 int ret = __get_any_page(page, pfn, flags); 1495 1496 if (ret == 1 && !PageHuge(page) && !PageLRU(page)) { 1497 /* 1498 * Try to free it. 1499 */ 1500 put_page(page); 1501 shake_page(page, 1); 1502 1503 /* 1504 * Did it turn free? 1505 */ 1506 ret = __get_any_page(page, pfn, 0); 1507 if (!PageLRU(page)) { 1508 pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n", 1509 pfn, page->flags); 1510 return -EIO; 1511 } 1512 } 1513 return ret; 1514 } 1515 1516 static int soft_offline_huge_page(struct page *page, int flags) 1517 { 1518 int ret; 1519 unsigned long pfn = page_to_pfn(page); 1520 struct page *hpage = compound_head(page); 1521 LIST_HEAD(pagelist); 1522 1523 /* 1524 * This double-check of PageHWPoison is to avoid the race with 1525 * memory_failure(). See also comment in __soft_offline_page(). 1526 */ 1527 lock_page(hpage); 1528 if (PageHWPoison(hpage)) { 1529 unlock_page(hpage); 1530 put_page(hpage); 1531 pr_info("soft offline: %#lx hugepage already poisoned\n", pfn); 1532 return -EBUSY; 1533 } 1534 unlock_page(hpage); 1535 1536 /* Keep page count to indicate a given hugepage is isolated. */ 1537 list_move(&hpage->lru, &pagelist); 1538 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1539 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1540 if (ret) { 1541 pr_info("soft offline: %#lx: migration failed %d, type %lx\n", 1542 pfn, ret, page->flags); 1543 /* 1544 * We know that soft_offline_huge_page() tries to migrate 1545 * only one hugepage pointed to by hpage, so we need not 1546 * run through the pagelist here. 1547 */ 1548 putback_active_hugepage(hpage); 1549 if (ret > 0) 1550 ret = -EIO; 1551 } else { 1552 /* overcommit hugetlb page will be freed to buddy */ 1553 if (PageHuge(page)) { 1554 set_page_hwpoison_huge_page(hpage); 1555 dequeue_hwpoisoned_huge_page(hpage); 1556 atomic_long_add(1 << compound_order(hpage), 1557 &num_poisoned_pages); 1558 } else { 1559 SetPageHWPoison(page); 1560 atomic_long_inc(&num_poisoned_pages); 1561 } 1562 } 1563 return ret; 1564 } 1565 1566 static int __soft_offline_page(struct page *page, int flags) 1567 { 1568 int ret; 1569 unsigned long pfn = page_to_pfn(page); 1570 1571 /* 1572 * Check PageHWPoison again inside page lock because PageHWPoison 1573 * is set by memory_failure() outside page lock. Note that 1574 * memory_failure() also double-checks PageHWPoison inside page lock, 1575 * so there's no race between soft_offline_page() and memory_failure(). 1576 */ 1577 lock_page(page); 1578 wait_on_page_writeback(page); 1579 if (PageHWPoison(page)) { 1580 unlock_page(page); 1581 put_page(page); 1582 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1583 return -EBUSY; 1584 } 1585 /* 1586 * Try to invalidate first. This should work for 1587 * non dirty unmapped page cache pages. 1588 */ 1589 ret = invalidate_inode_page(page); 1590 unlock_page(page); 1591 /* 1592 * RED-PEN would be better to keep it isolated here, but we 1593 * would need to fix isolation locking first. 1594 */ 1595 if (ret == 1) { 1596 put_page(page); 1597 pr_info("soft_offline: %#lx: invalidated\n", pfn); 1598 SetPageHWPoison(page); 1599 atomic_long_inc(&num_poisoned_pages); 1600 return 0; 1601 } 1602 1603 /* 1604 * Simple invalidation didn't work. 1605 * Try to migrate to a new page instead. migrate.c 1606 * handles a large number of cases for us. 1607 */ 1608 ret = isolate_lru_page(page); 1609 /* 1610 * Drop page reference which is came from get_any_page() 1611 * successful isolate_lru_page() already took another one. 1612 */ 1613 put_page(page); 1614 if (!ret) { 1615 LIST_HEAD(pagelist); 1616 inc_zone_page_state(page, NR_ISOLATED_ANON + 1617 page_is_file_cache(page)); 1618 list_add(&page->lru, &pagelist); 1619 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1620 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1621 if (ret) { 1622 if (!list_empty(&pagelist)) { 1623 list_del(&page->lru); 1624 dec_zone_page_state(page, NR_ISOLATED_ANON + 1625 page_is_file_cache(page)); 1626 putback_lru_page(page); 1627 } 1628 1629 pr_info("soft offline: %#lx: migration failed %d, type %lx\n", 1630 pfn, ret, page->flags); 1631 if (ret > 0) 1632 ret = -EIO; 1633 } else { 1634 /* 1635 * After page migration succeeds, the source page can 1636 * be trapped in pagevec and actual freeing is delayed. 1637 * Freeing code works differently based on PG_hwpoison, 1638 * so there's a race. We need to make sure that the 1639 * source page should be freed back to buddy before 1640 * setting PG_hwpoison. 1641 */ 1642 if (!is_free_buddy_page(page)) 1643 lru_add_drain_all(); 1644 if (!is_free_buddy_page(page)) 1645 drain_all_pages(); 1646 SetPageHWPoison(page); 1647 if (!is_free_buddy_page(page)) 1648 pr_info("soft offline: %#lx: page leaked\n", 1649 pfn); 1650 atomic_long_inc(&num_poisoned_pages); 1651 } 1652 } else { 1653 pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n", 1654 pfn, ret, page_count(page), page->flags); 1655 } 1656 return ret; 1657 } 1658 1659 /** 1660 * soft_offline_page - Soft offline a page. 1661 * @page: page to offline 1662 * @flags: flags. Same as memory_failure(). 1663 * 1664 * Returns 0 on success, otherwise negated errno. 1665 * 1666 * Soft offline a page, by migration or invalidation, 1667 * without killing anything. This is for the case when 1668 * a page is not corrupted yet (so it's still valid to access), 1669 * but has had a number of corrected errors and is better taken 1670 * out. 1671 * 1672 * The actual policy on when to do that is maintained by 1673 * user space. 1674 * 1675 * This should never impact any application or cause data loss, 1676 * however it might take some time. 1677 * 1678 * This is not a 100% solution for all memory, but tries to be 1679 * ``good enough'' for the majority of memory. 1680 */ 1681 int soft_offline_page(struct page *page, int flags) 1682 { 1683 int ret; 1684 unsigned long pfn = page_to_pfn(page); 1685 struct page *hpage = compound_head(page); 1686 1687 if (PageHWPoison(page)) { 1688 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1689 return -EBUSY; 1690 } 1691 if (!PageHuge(page) && PageTransHuge(hpage)) { 1692 if (PageAnon(hpage) && unlikely(split_huge_page(hpage))) { 1693 pr_info("soft offline: %#lx: failed to split THP\n", 1694 pfn); 1695 return -EBUSY; 1696 } 1697 } 1698 1699 get_online_mems(); 1700 1701 /* 1702 * Isolate the page, so that it doesn't get reallocated if it 1703 * was free. This flag should be kept set until the source page 1704 * is freed and PG_hwpoison on it is set. 1705 */ 1706 if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE) 1707 set_migratetype_isolate(page, true); 1708 1709 ret = get_any_page(page, pfn, flags); 1710 put_online_mems(); 1711 if (ret > 0) { /* for in-use pages */ 1712 if (PageHuge(page)) 1713 ret = soft_offline_huge_page(page, flags); 1714 else 1715 ret = __soft_offline_page(page, flags); 1716 } else if (ret == 0) { /* for free pages */ 1717 if (PageHuge(page)) { 1718 set_page_hwpoison_huge_page(hpage); 1719 dequeue_hwpoisoned_huge_page(hpage); 1720 atomic_long_add(1 << compound_order(hpage), 1721 &num_poisoned_pages); 1722 } else { 1723 SetPageHWPoison(page); 1724 atomic_long_inc(&num_poisoned_pages); 1725 } 1726 } 1727 unset_migratetype_isolate(page, MIGRATE_MOVABLE); 1728 return ret; 1729 } 1730