1 /* 2 * Copyright (C) 2008, 2009 Intel Corporation 3 * Authors: Andi Kleen, Fengguang Wu 4 * 5 * This software may be redistributed and/or modified under the terms of 6 * the GNU General Public License ("GPL") version 2 only as published by the 7 * Free Software Foundation. 8 * 9 * High level machine check handler. Handles pages reported by the 10 * hardware as being corrupted usually due to a multi-bit ECC memory or cache 11 * failure. 12 * 13 * In addition there is a "soft offline" entry point that allows stop using 14 * not-yet-corrupted-by-suspicious pages without killing anything. 15 * 16 * Handles page cache pages in various states. The tricky part 17 * here is that we can access any page asynchronously in respect to 18 * other VM users, because memory failures could happen anytime and 19 * anywhere. This could violate some of their assumptions. This is why 20 * this code has to be extremely careful. Generally it tries to use 21 * normal locking rules, as in get the standard locks, even if that means 22 * the error handling takes potentially a long time. 23 * 24 * It can be very tempting to add handling for obscure cases here. 25 * In general any code for handling new cases should only be added iff: 26 * - You know how to test it. 27 * - You have a test that can be added to mce-test 28 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/ 29 * - The case actually shows up as a frequent (top 10) page state in 30 * tools/vm/page-types when running a real workload. 31 * 32 * There are several operations here with exponential complexity because 33 * of unsuitable VM data structures. For example the operation to map back 34 * from RMAP chains to processes has to walk the complete process list and 35 * has non linear complexity with the number. But since memory corruptions 36 * are rare we hope to get away with this. This avoids impacting the core 37 * VM. 38 */ 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/page-flags.h> 42 #include <linux/kernel-page-flags.h> 43 #include <linux/sched.h> 44 #include <linux/ksm.h> 45 #include <linux/rmap.h> 46 #include <linux/export.h> 47 #include <linux/pagemap.h> 48 #include <linux/swap.h> 49 #include <linux/backing-dev.h> 50 #include <linux/migrate.h> 51 #include <linux/page-isolation.h> 52 #include <linux/suspend.h> 53 #include <linux/slab.h> 54 #include <linux/swapops.h> 55 #include <linux/hugetlb.h> 56 #include <linux/memory_hotplug.h> 57 #include <linux/mm_inline.h> 58 #include <linux/kfifo.h> 59 #include "internal.h" 60 #include "ras/ras_event.h" 61 62 int sysctl_memory_failure_early_kill __read_mostly = 0; 63 64 int sysctl_memory_failure_recovery __read_mostly = 1; 65 66 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); 67 68 #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE) 69 70 u32 hwpoison_filter_enable = 0; 71 u32 hwpoison_filter_dev_major = ~0U; 72 u32 hwpoison_filter_dev_minor = ~0U; 73 u64 hwpoison_filter_flags_mask; 74 u64 hwpoison_filter_flags_value; 75 EXPORT_SYMBOL_GPL(hwpoison_filter_enable); 76 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major); 77 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor); 78 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask); 79 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value); 80 81 static int hwpoison_filter_dev(struct page *p) 82 { 83 struct address_space *mapping; 84 dev_t dev; 85 86 if (hwpoison_filter_dev_major == ~0U && 87 hwpoison_filter_dev_minor == ~0U) 88 return 0; 89 90 /* 91 * page_mapping() does not accept slab pages. 92 */ 93 if (PageSlab(p)) 94 return -EINVAL; 95 96 mapping = page_mapping(p); 97 if (mapping == NULL || mapping->host == NULL) 98 return -EINVAL; 99 100 dev = mapping->host->i_sb->s_dev; 101 if (hwpoison_filter_dev_major != ~0U && 102 hwpoison_filter_dev_major != MAJOR(dev)) 103 return -EINVAL; 104 if (hwpoison_filter_dev_minor != ~0U && 105 hwpoison_filter_dev_minor != MINOR(dev)) 106 return -EINVAL; 107 108 return 0; 109 } 110 111 static int hwpoison_filter_flags(struct page *p) 112 { 113 if (!hwpoison_filter_flags_mask) 114 return 0; 115 116 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) == 117 hwpoison_filter_flags_value) 118 return 0; 119 else 120 return -EINVAL; 121 } 122 123 /* 124 * This allows stress tests to limit test scope to a collection of tasks 125 * by putting them under some memcg. This prevents killing unrelated/important 126 * processes such as /sbin/init. Note that the target task may share clean 127 * pages with init (eg. libc text), which is harmless. If the target task 128 * share _dirty_ pages with another task B, the test scheme must make sure B 129 * is also included in the memcg. At last, due to race conditions this filter 130 * can only guarantee that the page either belongs to the memcg tasks, or is 131 * a freed page. 132 */ 133 #ifdef CONFIG_MEMCG_SWAP 134 u64 hwpoison_filter_memcg; 135 EXPORT_SYMBOL_GPL(hwpoison_filter_memcg); 136 static int hwpoison_filter_task(struct page *p) 137 { 138 struct mem_cgroup *mem; 139 struct cgroup_subsys_state *css; 140 unsigned long ino; 141 142 if (!hwpoison_filter_memcg) 143 return 0; 144 145 mem = try_get_mem_cgroup_from_page(p); 146 if (!mem) 147 return -EINVAL; 148 149 css = mem_cgroup_css(mem); 150 ino = cgroup_ino(css->cgroup); 151 css_put(css); 152 153 if (ino != hwpoison_filter_memcg) 154 return -EINVAL; 155 156 return 0; 157 } 158 #else 159 static int hwpoison_filter_task(struct page *p) { return 0; } 160 #endif 161 162 int hwpoison_filter(struct page *p) 163 { 164 if (!hwpoison_filter_enable) 165 return 0; 166 167 if (hwpoison_filter_dev(p)) 168 return -EINVAL; 169 170 if (hwpoison_filter_flags(p)) 171 return -EINVAL; 172 173 if (hwpoison_filter_task(p)) 174 return -EINVAL; 175 176 return 0; 177 } 178 #else 179 int hwpoison_filter(struct page *p) 180 { 181 return 0; 182 } 183 #endif 184 185 EXPORT_SYMBOL_GPL(hwpoison_filter); 186 187 /* 188 * Send all the processes who have the page mapped a signal. 189 * ``action optional'' if they are not immediately affected by the error 190 * ``action required'' if error happened in current execution context 191 */ 192 static int kill_proc(struct task_struct *t, unsigned long addr, int trapno, 193 unsigned long pfn, struct page *page, int flags) 194 { 195 struct siginfo si; 196 int ret; 197 198 printk(KERN_ERR 199 "MCE %#lx: Killing %s:%d due to hardware memory corruption\n", 200 pfn, t->comm, t->pid); 201 si.si_signo = SIGBUS; 202 si.si_errno = 0; 203 si.si_addr = (void *)addr; 204 #ifdef __ARCH_SI_TRAPNO 205 si.si_trapno = trapno; 206 #endif 207 si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT; 208 209 if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) { 210 si.si_code = BUS_MCEERR_AR; 211 ret = force_sig_info(SIGBUS, &si, current); 212 } else { 213 /* 214 * Don't use force here, it's convenient if the signal 215 * can be temporarily blocked. 216 * This could cause a loop when the user sets SIGBUS 217 * to SIG_IGN, but hopefully no one will do that? 218 */ 219 si.si_code = BUS_MCEERR_AO; 220 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */ 221 } 222 if (ret < 0) 223 printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n", 224 t->comm, t->pid, ret); 225 return ret; 226 } 227 228 /* 229 * When a unknown page type is encountered drain as many buffers as possible 230 * in the hope to turn the page into a LRU or free page, which we can handle. 231 */ 232 void shake_page(struct page *p, int access) 233 { 234 if (!PageSlab(p)) { 235 lru_add_drain_all(); 236 if (PageLRU(p)) 237 return; 238 drain_all_pages(page_zone(p)); 239 if (PageLRU(p) || is_free_buddy_page(p)) 240 return; 241 } 242 243 /* 244 * Only call shrink_node_slabs here (which would also shrink 245 * other caches) if access is not potentially fatal. 246 */ 247 if (access) 248 drop_slab_node(page_to_nid(p)); 249 } 250 EXPORT_SYMBOL_GPL(shake_page); 251 252 /* 253 * Kill all processes that have a poisoned page mapped and then isolate 254 * the page. 255 * 256 * General strategy: 257 * Find all processes having the page mapped and kill them. 258 * But we keep a page reference around so that the page is not 259 * actually freed yet. 260 * Then stash the page away 261 * 262 * There's no convenient way to get back to mapped processes 263 * from the VMAs. So do a brute-force search over all 264 * running processes. 265 * 266 * Remember that machine checks are not common (or rather 267 * if they are common you have other problems), so this shouldn't 268 * be a performance issue. 269 * 270 * Also there are some races possible while we get from the 271 * error detection to actually handle it. 272 */ 273 274 struct to_kill { 275 struct list_head nd; 276 struct task_struct *tsk; 277 unsigned long addr; 278 char addr_valid; 279 }; 280 281 /* 282 * Failure handling: if we can't find or can't kill a process there's 283 * not much we can do. We just print a message and ignore otherwise. 284 */ 285 286 /* 287 * Schedule a process for later kill. 288 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. 289 * TBD would GFP_NOIO be enough? 290 */ 291 static void add_to_kill(struct task_struct *tsk, struct page *p, 292 struct vm_area_struct *vma, 293 struct list_head *to_kill, 294 struct to_kill **tkc) 295 { 296 struct to_kill *tk; 297 298 if (*tkc) { 299 tk = *tkc; 300 *tkc = NULL; 301 } else { 302 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC); 303 if (!tk) { 304 printk(KERN_ERR 305 "MCE: Out of memory while machine check handling\n"); 306 return; 307 } 308 } 309 tk->addr = page_address_in_vma(p, vma); 310 tk->addr_valid = 1; 311 312 /* 313 * In theory we don't have to kill when the page was 314 * munmaped. But it could be also a mremap. Since that's 315 * likely very rare kill anyways just out of paranoia, but use 316 * a SIGKILL because the error is not contained anymore. 317 */ 318 if (tk->addr == -EFAULT) { 319 pr_info("MCE: Unable to find user space address %lx in %s\n", 320 page_to_pfn(p), tsk->comm); 321 tk->addr_valid = 0; 322 } 323 get_task_struct(tsk); 324 tk->tsk = tsk; 325 list_add_tail(&tk->nd, to_kill); 326 } 327 328 /* 329 * Kill the processes that have been collected earlier. 330 * 331 * Only do anything when DOIT is set, otherwise just free the list 332 * (this is used for clean pages which do not need killing) 333 * Also when FAIL is set do a force kill because something went 334 * wrong earlier. 335 */ 336 static void kill_procs(struct list_head *to_kill, int forcekill, int trapno, 337 int fail, struct page *page, unsigned long pfn, 338 int flags) 339 { 340 struct to_kill *tk, *next; 341 342 list_for_each_entry_safe (tk, next, to_kill, nd) { 343 if (forcekill) { 344 /* 345 * In case something went wrong with munmapping 346 * make sure the process doesn't catch the 347 * signal and then access the memory. Just kill it. 348 */ 349 if (fail || tk->addr_valid == 0) { 350 printk(KERN_ERR 351 "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", 352 pfn, tk->tsk->comm, tk->tsk->pid); 353 force_sig(SIGKILL, tk->tsk); 354 } 355 356 /* 357 * In theory the process could have mapped 358 * something else on the address in-between. We could 359 * check for that, but we need to tell the 360 * process anyways. 361 */ 362 else if (kill_proc(tk->tsk, tk->addr, trapno, 363 pfn, page, flags) < 0) 364 printk(KERN_ERR 365 "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n", 366 pfn, tk->tsk->comm, tk->tsk->pid); 367 } 368 put_task_struct(tk->tsk); 369 kfree(tk); 370 } 371 } 372 373 /* 374 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) 375 * on behalf of the thread group. Return task_struct of the (first found) 376 * dedicated thread if found, and return NULL otherwise. 377 * 378 * We already hold read_lock(&tasklist_lock) in the caller, so we don't 379 * have to call rcu_read_lock/unlock() in this function. 380 */ 381 static struct task_struct *find_early_kill_thread(struct task_struct *tsk) 382 { 383 struct task_struct *t; 384 385 for_each_thread(tsk, t) 386 if ((t->flags & PF_MCE_PROCESS) && (t->flags & PF_MCE_EARLY)) 387 return t; 388 return NULL; 389 } 390 391 /* 392 * Determine whether a given process is "early kill" process which expects 393 * to be signaled when some page under the process is hwpoisoned. 394 * Return task_struct of the dedicated thread (main thread unless explicitly 395 * specified) if the process is "early kill," and otherwise returns NULL. 396 */ 397 static struct task_struct *task_early_kill(struct task_struct *tsk, 398 int force_early) 399 { 400 struct task_struct *t; 401 if (!tsk->mm) 402 return NULL; 403 if (force_early) 404 return tsk; 405 t = find_early_kill_thread(tsk); 406 if (t) 407 return t; 408 if (sysctl_memory_failure_early_kill) 409 return tsk; 410 return NULL; 411 } 412 413 /* 414 * Collect processes when the error hit an anonymous page. 415 */ 416 static void collect_procs_anon(struct page *page, struct list_head *to_kill, 417 struct to_kill **tkc, int force_early) 418 { 419 struct vm_area_struct *vma; 420 struct task_struct *tsk; 421 struct anon_vma *av; 422 pgoff_t pgoff; 423 424 av = page_lock_anon_vma_read(page); 425 if (av == NULL) /* Not actually mapped anymore */ 426 return; 427 428 pgoff = page_to_pgoff(page); 429 read_lock(&tasklist_lock); 430 for_each_process (tsk) { 431 struct anon_vma_chain *vmac; 432 struct task_struct *t = task_early_kill(tsk, force_early); 433 434 if (!t) 435 continue; 436 anon_vma_interval_tree_foreach(vmac, &av->rb_root, 437 pgoff, pgoff) { 438 vma = vmac->vma; 439 if (!page_mapped_in_vma(page, vma)) 440 continue; 441 if (vma->vm_mm == t->mm) 442 add_to_kill(t, page, vma, to_kill, tkc); 443 } 444 } 445 read_unlock(&tasklist_lock); 446 page_unlock_anon_vma_read(av); 447 } 448 449 /* 450 * Collect processes when the error hit a file mapped page. 451 */ 452 static void collect_procs_file(struct page *page, struct list_head *to_kill, 453 struct to_kill **tkc, int force_early) 454 { 455 struct vm_area_struct *vma; 456 struct task_struct *tsk; 457 struct address_space *mapping = page->mapping; 458 459 i_mmap_lock_read(mapping); 460 read_lock(&tasklist_lock); 461 for_each_process(tsk) { 462 pgoff_t pgoff = page_to_pgoff(page); 463 struct task_struct *t = task_early_kill(tsk, force_early); 464 465 if (!t) 466 continue; 467 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, 468 pgoff) { 469 /* 470 * Send early kill signal to tasks where a vma covers 471 * the page but the corrupted page is not necessarily 472 * mapped it in its pte. 473 * Assume applications who requested early kill want 474 * to be informed of all such data corruptions. 475 */ 476 if (vma->vm_mm == t->mm) 477 add_to_kill(t, page, vma, to_kill, tkc); 478 } 479 } 480 read_unlock(&tasklist_lock); 481 i_mmap_unlock_read(mapping); 482 } 483 484 /* 485 * Collect the processes who have the corrupted page mapped to kill. 486 * This is done in two steps for locking reasons. 487 * First preallocate one tokill structure outside the spin locks, 488 * so that we can kill at least one process reasonably reliable. 489 */ 490 static void collect_procs(struct page *page, struct list_head *tokill, 491 int force_early) 492 { 493 struct to_kill *tk; 494 495 if (!page->mapping) 496 return; 497 498 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO); 499 if (!tk) 500 return; 501 if (PageAnon(page)) 502 collect_procs_anon(page, tokill, &tk, force_early); 503 else 504 collect_procs_file(page, tokill, &tk, force_early); 505 kfree(tk); 506 } 507 508 static const char *action_name[] = { 509 [MF_IGNORED] = "Ignored", 510 [MF_FAILED] = "Failed", 511 [MF_DELAYED] = "Delayed", 512 [MF_RECOVERED] = "Recovered", 513 }; 514 515 static const char * const action_page_types[] = { 516 [MF_MSG_KERNEL] = "reserved kernel page", 517 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page", 518 [MF_MSG_SLAB] = "kernel slab page", 519 [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking", 520 [MF_MSG_POISONED_HUGE] = "huge page already hardware poisoned", 521 [MF_MSG_HUGE] = "huge page", 522 [MF_MSG_FREE_HUGE] = "free huge page", 523 [MF_MSG_UNMAP_FAILED] = "unmapping failed page", 524 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page", 525 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page", 526 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page", 527 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page", 528 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page", 529 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page", 530 [MF_MSG_DIRTY_LRU] = "dirty LRU page", 531 [MF_MSG_CLEAN_LRU] = "clean LRU page", 532 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", 533 [MF_MSG_BUDDY] = "free buddy page", 534 [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)", 535 [MF_MSG_UNKNOWN] = "unknown page", 536 }; 537 538 /* 539 * XXX: It is possible that a page is isolated from LRU cache, 540 * and then kept in swap cache or failed to remove from page cache. 541 * The page count will stop it from being freed by unpoison. 542 * Stress tests should be aware of this memory leak problem. 543 */ 544 static int delete_from_lru_cache(struct page *p) 545 { 546 if (!isolate_lru_page(p)) { 547 /* 548 * Clear sensible page flags, so that the buddy system won't 549 * complain when the page is unpoison-and-freed. 550 */ 551 ClearPageActive(p); 552 ClearPageUnevictable(p); 553 /* 554 * drop the page count elevated by isolate_lru_page() 555 */ 556 page_cache_release(p); 557 return 0; 558 } 559 return -EIO; 560 } 561 562 /* 563 * Error hit kernel page. 564 * Do nothing, try to be lucky and not touch this instead. For a few cases we 565 * could be more sophisticated. 566 */ 567 static int me_kernel(struct page *p, unsigned long pfn) 568 { 569 return MF_IGNORED; 570 } 571 572 /* 573 * Page in unknown state. Do nothing. 574 */ 575 static int me_unknown(struct page *p, unsigned long pfn) 576 { 577 printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn); 578 return MF_FAILED; 579 } 580 581 /* 582 * Clean (or cleaned) page cache page. 583 */ 584 static int me_pagecache_clean(struct page *p, unsigned long pfn) 585 { 586 int err; 587 int ret = MF_FAILED; 588 struct address_space *mapping; 589 590 delete_from_lru_cache(p); 591 592 /* 593 * For anonymous pages we're done the only reference left 594 * should be the one m_f() holds. 595 */ 596 if (PageAnon(p)) 597 return MF_RECOVERED; 598 599 /* 600 * Now truncate the page in the page cache. This is really 601 * more like a "temporary hole punch" 602 * Don't do this for block devices when someone else 603 * has a reference, because it could be file system metadata 604 * and that's not safe to truncate. 605 */ 606 mapping = page_mapping(p); 607 if (!mapping) { 608 /* 609 * Page has been teared down in the meanwhile 610 */ 611 return MF_FAILED; 612 } 613 614 /* 615 * Truncation is a bit tricky. Enable it per file system for now. 616 * 617 * Open: to take i_mutex or not for this? Right now we don't. 618 */ 619 if (mapping->a_ops->error_remove_page) { 620 err = mapping->a_ops->error_remove_page(mapping, p); 621 if (err != 0) { 622 printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n", 623 pfn, err); 624 } else if (page_has_private(p) && 625 !try_to_release_page(p, GFP_NOIO)) { 626 pr_info("MCE %#lx: failed to release buffers\n", pfn); 627 } else { 628 ret = MF_RECOVERED; 629 } 630 } else { 631 /* 632 * If the file system doesn't support it just invalidate 633 * This fails on dirty or anything with private pages 634 */ 635 if (invalidate_inode_page(p)) 636 ret = MF_RECOVERED; 637 else 638 printk(KERN_INFO "MCE %#lx: Failed to invalidate\n", 639 pfn); 640 } 641 return ret; 642 } 643 644 /* 645 * Dirty pagecache page 646 * Issues: when the error hit a hole page the error is not properly 647 * propagated. 648 */ 649 static int me_pagecache_dirty(struct page *p, unsigned long pfn) 650 { 651 struct address_space *mapping = page_mapping(p); 652 653 SetPageError(p); 654 /* TBD: print more information about the file. */ 655 if (mapping) { 656 /* 657 * IO error will be reported by write(), fsync(), etc. 658 * who check the mapping. 659 * This way the application knows that something went 660 * wrong with its dirty file data. 661 * 662 * There's one open issue: 663 * 664 * The EIO will be only reported on the next IO 665 * operation and then cleared through the IO map. 666 * Normally Linux has two mechanisms to pass IO error 667 * first through the AS_EIO flag in the address space 668 * and then through the PageError flag in the page. 669 * Since we drop pages on memory failure handling the 670 * only mechanism open to use is through AS_AIO. 671 * 672 * This has the disadvantage that it gets cleared on 673 * the first operation that returns an error, while 674 * the PageError bit is more sticky and only cleared 675 * when the page is reread or dropped. If an 676 * application assumes it will always get error on 677 * fsync, but does other operations on the fd before 678 * and the page is dropped between then the error 679 * will not be properly reported. 680 * 681 * This can already happen even without hwpoisoned 682 * pages: first on metadata IO errors (which only 683 * report through AS_EIO) or when the page is dropped 684 * at the wrong time. 685 * 686 * So right now we assume that the application DTRT on 687 * the first EIO, but we're not worse than other parts 688 * of the kernel. 689 */ 690 mapping_set_error(mapping, EIO); 691 } 692 693 return me_pagecache_clean(p, pfn); 694 } 695 696 /* 697 * Clean and dirty swap cache. 698 * 699 * Dirty swap cache page is tricky to handle. The page could live both in page 700 * cache and swap cache(ie. page is freshly swapped in). So it could be 701 * referenced concurrently by 2 types of PTEs: 702 * normal PTEs and swap PTEs. We try to handle them consistently by calling 703 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs, 704 * and then 705 * - clear dirty bit to prevent IO 706 * - remove from LRU 707 * - but keep in the swap cache, so that when we return to it on 708 * a later page fault, we know the application is accessing 709 * corrupted data and shall be killed (we installed simple 710 * interception code in do_swap_page to catch it). 711 * 712 * Clean swap cache pages can be directly isolated. A later page fault will 713 * bring in the known good data from disk. 714 */ 715 static int me_swapcache_dirty(struct page *p, unsigned long pfn) 716 { 717 ClearPageDirty(p); 718 /* Trigger EIO in shmem: */ 719 ClearPageUptodate(p); 720 721 if (!delete_from_lru_cache(p)) 722 return MF_DELAYED; 723 else 724 return MF_FAILED; 725 } 726 727 static int me_swapcache_clean(struct page *p, unsigned long pfn) 728 { 729 delete_from_swap_cache(p); 730 731 if (!delete_from_lru_cache(p)) 732 return MF_RECOVERED; 733 else 734 return MF_FAILED; 735 } 736 737 /* 738 * Huge pages. Needs work. 739 * Issues: 740 * - Error on hugepage is contained in hugepage unit (not in raw page unit.) 741 * To narrow down kill region to one page, we need to break up pmd. 742 */ 743 static int me_huge_page(struct page *p, unsigned long pfn) 744 { 745 int res = 0; 746 struct page *hpage = compound_head(p); 747 748 if (!PageHuge(hpage)) 749 return MF_DELAYED; 750 751 /* 752 * We can safely recover from error on free or reserved (i.e. 753 * not in-use) hugepage by dequeuing it from freelist. 754 * To check whether a hugepage is in-use or not, we can't use 755 * page->lru because it can be used in other hugepage operations, 756 * such as __unmap_hugepage_range() and gather_surplus_pages(). 757 * So instead we use page_mapping() and PageAnon(). 758 * We assume that this function is called with page lock held, 759 * so there is no race between isolation and mapping/unmapping. 760 */ 761 if (!(page_mapping(hpage) || PageAnon(hpage))) { 762 res = dequeue_hwpoisoned_huge_page(hpage); 763 if (!res) 764 return MF_RECOVERED; 765 } 766 return MF_DELAYED; 767 } 768 769 /* 770 * Various page states we can handle. 771 * 772 * A page state is defined by its current page->flags bits. 773 * The table matches them in order and calls the right handler. 774 * 775 * This is quite tricky because we can access page at any time 776 * in its live cycle, so all accesses have to be extremely careful. 777 * 778 * This is not complete. More states could be added. 779 * For any missing state don't attempt recovery. 780 */ 781 782 #define dirty (1UL << PG_dirty) 783 #define sc (1UL << PG_swapcache) 784 #define unevict (1UL << PG_unevictable) 785 #define mlock (1UL << PG_mlocked) 786 #define writeback (1UL << PG_writeback) 787 #define lru (1UL << PG_lru) 788 #define swapbacked (1UL << PG_swapbacked) 789 #define head (1UL << PG_head) 790 #define tail (1UL << PG_tail) 791 #define compound (1UL << PG_compound) 792 #define slab (1UL << PG_slab) 793 #define reserved (1UL << PG_reserved) 794 795 static struct page_state { 796 unsigned long mask; 797 unsigned long res; 798 enum mf_action_page_type type; 799 int (*action)(struct page *p, unsigned long pfn); 800 } error_states[] = { 801 { reserved, reserved, MF_MSG_KERNEL, me_kernel }, 802 /* 803 * free pages are specially detected outside this table: 804 * PG_buddy pages only make a small fraction of all free pages. 805 */ 806 807 /* 808 * Could in theory check if slab page is free or if we can drop 809 * currently unused objects without touching them. But just 810 * treat it as standard kernel for now. 811 */ 812 { slab, slab, MF_MSG_SLAB, me_kernel }, 813 814 #ifdef CONFIG_PAGEFLAGS_EXTENDED 815 { head, head, MF_MSG_HUGE, me_huge_page }, 816 { tail, tail, MF_MSG_HUGE, me_huge_page }, 817 #else 818 { compound, compound, MF_MSG_HUGE, me_huge_page }, 819 #endif 820 821 { sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty }, 822 { sc|dirty, sc, MF_MSG_CLEAN_SWAPCACHE, me_swapcache_clean }, 823 824 { mlock|dirty, mlock|dirty, MF_MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty }, 825 { mlock|dirty, mlock, MF_MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean }, 826 827 { unevict|dirty, unevict|dirty, MF_MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty }, 828 { unevict|dirty, unevict, MF_MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean }, 829 830 { lru|dirty, lru|dirty, MF_MSG_DIRTY_LRU, me_pagecache_dirty }, 831 { lru|dirty, lru, MF_MSG_CLEAN_LRU, me_pagecache_clean }, 832 833 /* 834 * Catchall entry: must be at end. 835 */ 836 { 0, 0, MF_MSG_UNKNOWN, me_unknown }, 837 }; 838 839 #undef dirty 840 #undef sc 841 #undef unevict 842 #undef mlock 843 #undef writeback 844 #undef lru 845 #undef swapbacked 846 #undef head 847 #undef tail 848 #undef compound 849 #undef slab 850 #undef reserved 851 852 /* 853 * "Dirty/Clean" indication is not 100% accurate due to the possibility of 854 * setting PG_dirty outside page lock. See also comment above set_page_dirty(). 855 */ 856 static void action_result(unsigned long pfn, enum mf_action_page_type type, 857 enum mf_result result) 858 { 859 trace_memory_failure_event(pfn, type, result); 860 861 pr_err("MCE %#lx: recovery action for %s: %s\n", 862 pfn, action_page_types[type], action_name[result]); 863 } 864 865 static int page_action(struct page_state *ps, struct page *p, 866 unsigned long pfn) 867 { 868 int result; 869 int count; 870 871 result = ps->action(p, pfn); 872 873 count = page_count(p) - 1; 874 if (ps->action == me_swapcache_dirty && result == MF_DELAYED) 875 count--; 876 if (count != 0) { 877 printk(KERN_ERR 878 "MCE %#lx: %s still referenced by %d users\n", 879 pfn, action_page_types[ps->type], count); 880 result = MF_FAILED; 881 } 882 action_result(pfn, ps->type, result); 883 884 /* Could do more checks here if page looks ok */ 885 /* 886 * Could adjust zone counters here to correct for the missing page. 887 */ 888 889 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; 890 } 891 892 /** 893 * get_hwpoison_page() - Get refcount for memory error handling: 894 * @page: raw error page (hit by memory error) 895 * 896 * Return: return 0 if failed to grab the refcount, otherwise true (some 897 * non-zero value.) 898 */ 899 int get_hwpoison_page(struct page *page) 900 { 901 struct page *head = compound_head(page); 902 903 if (PageHuge(head)) 904 return get_page_unless_zero(head); 905 906 /* 907 * Thp tail page has special refcounting rule (refcount of tail pages 908 * is stored in ->_mapcount,) so we can't call get_page_unless_zero() 909 * directly for tail pages. 910 */ 911 if (PageTransHuge(head)) { 912 /* 913 * Non anonymous thp exists only in allocation/free time. We 914 * can't handle such a case correctly, so let's give it up. 915 * This should be better than triggering BUG_ON when kernel 916 * tries to touch the "partially handled" page. 917 */ 918 if (!PageAnon(head)) { 919 pr_err("MCE: %#lx: non anonymous thp\n", 920 page_to_pfn(page)); 921 return 0; 922 } 923 924 if (get_page_unless_zero(head)) { 925 if (PageTail(page)) 926 get_page(page); 927 return 1; 928 } else { 929 return 0; 930 } 931 } 932 933 return get_page_unless_zero(page); 934 } 935 EXPORT_SYMBOL_GPL(get_hwpoison_page); 936 937 /* 938 * Do all that is necessary to remove user space mappings. Unmap 939 * the pages and send SIGBUS to the processes if the data was dirty. 940 */ 941 static int hwpoison_user_mappings(struct page *p, unsigned long pfn, 942 int trapno, int flags, struct page **hpagep) 943 { 944 enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; 945 struct address_space *mapping; 946 LIST_HEAD(tokill); 947 int ret; 948 int kill = 1, forcekill; 949 struct page *hpage = *hpagep; 950 951 /* 952 * Here we are interested only in user-mapped pages, so skip any 953 * other types of pages. 954 */ 955 if (PageReserved(p) || PageSlab(p)) 956 return SWAP_SUCCESS; 957 if (!(PageLRU(hpage) || PageHuge(p))) 958 return SWAP_SUCCESS; 959 960 /* 961 * This check implies we don't kill processes if their pages 962 * are in the swap cache early. Those are always late kills. 963 */ 964 if (!page_mapped(hpage)) 965 return SWAP_SUCCESS; 966 967 if (PageKsm(p)) { 968 pr_err("MCE %#lx: can't handle KSM pages.\n", pfn); 969 return SWAP_FAIL; 970 } 971 972 if (PageSwapCache(p)) { 973 printk(KERN_ERR 974 "MCE %#lx: keeping poisoned page in swap cache\n", pfn); 975 ttu |= TTU_IGNORE_HWPOISON; 976 } 977 978 /* 979 * Propagate the dirty bit from PTEs to struct page first, because we 980 * need this to decide if we should kill or just drop the page. 981 * XXX: the dirty test could be racy: set_page_dirty() may not always 982 * be called inside page lock (it's recommended but not enforced). 983 */ 984 mapping = page_mapping(hpage); 985 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping && 986 mapping_cap_writeback_dirty(mapping)) { 987 if (page_mkclean(hpage)) { 988 SetPageDirty(hpage); 989 } else { 990 kill = 0; 991 ttu |= TTU_IGNORE_HWPOISON; 992 printk(KERN_INFO 993 "MCE %#lx: corrupted page was clean: dropped without side effects\n", 994 pfn); 995 } 996 } 997 998 /* 999 * First collect all the processes that have the page 1000 * mapped in dirty form. This has to be done before try_to_unmap, 1001 * because ttu takes the rmap data structures down. 1002 * 1003 * Error handling: We ignore errors here because 1004 * there's nothing that can be done. 1005 */ 1006 if (kill) 1007 collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED); 1008 1009 ret = try_to_unmap(hpage, ttu); 1010 if (ret != SWAP_SUCCESS) 1011 printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n", 1012 pfn, page_mapcount(hpage)); 1013 1014 /* 1015 * Now that the dirty bit has been propagated to the 1016 * struct page and all unmaps done we can decide if 1017 * killing is needed or not. Only kill when the page 1018 * was dirty or the process is not restartable, 1019 * otherwise the tokill list is merely 1020 * freed. When there was a problem unmapping earlier 1021 * use a more force-full uncatchable kill to prevent 1022 * any accesses to the poisoned memory. 1023 */ 1024 forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL); 1025 kill_procs(&tokill, forcekill, trapno, 1026 ret != SWAP_SUCCESS, p, pfn, flags); 1027 1028 return ret; 1029 } 1030 1031 static void set_page_hwpoison_huge_page(struct page *hpage) 1032 { 1033 int i; 1034 int nr_pages = 1 << compound_order(hpage); 1035 for (i = 0; i < nr_pages; i++) 1036 SetPageHWPoison(hpage + i); 1037 } 1038 1039 static void clear_page_hwpoison_huge_page(struct page *hpage) 1040 { 1041 int i; 1042 int nr_pages = 1 << compound_order(hpage); 1043 for (i = 0; i < nr_pages; i++) 1044 ClearPageHWPoison(hpage + i); 1045 } 1046 1047 /** 1048 * memory_failure - Handle memory failure of a page. 1049 * @pfn: Page Number of the corrupted page 1050 * @trapno: Trap number reported in the signal to user space. 1051 * @flags: fine tune action taken 1052 * 1053 * This function is called by the low level machine check code 1054 * of an architecture when it detects hardware memory corruption 1055 * of a page. It tries its best to recover, which includes 1056 * dropping pages, killing processes etc. 1057 * 1058 * The function is primarily of use for corruptions that 1059 * happen outside the current execution context (e.g. when 1060 * detected by a background scrubber) 1061 * 1062 * Must run in process context (e.g. a work queue) with interrupts 1063 * enabled and no spinlocks hold. 1064 */ 1065 int memory_failure(unsigned long pfn, int trapno, int flags) 1066 { 1067 struct page_state *ps; 1068 struct page *p; 1069 struct page *hpage; 1070 struct page *orig_head; 1071 int res; 1072 unsigned int nr_pages; 1073 unsigned long page_flags; 1074 1075 if (!sysctl_memory_failure_recovery) 1076 panic("Memory failure from trap %d on page %lx", trapno, pfn); 1077 1078 if (!pfn_valid(pfn)) { 1079 printk(KERN_ERR 1080 "MCE %#lx: memory outside kernel control\n", 1081 pfn); 1082 return -ENXIO; 1083 } 1084 1085 p = pfn_to_page(pfn); 1086 orig_head = hpage = compound_head(p); 1087 if (TestSetPageHWPoison(p)) { 1088 printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn); 1089 return 0; 1090 } 1091 1092 /* 1093 * Currently errors on hugetlbfs pages are measured in hugepage units, 1094 * so nr_pages should be 1 << compound_order. OTOH when errors are on 1095 * transparent hugepages, they are supposed to be split and error 1096 * measurement is done in normal page units. So nr_pages should be one 1097 * in this case. 1098 */ 1099 if (PageHuge(p)) 1100 nr_pages = 1 << compound_order(hpage); 1101 else /* normal page or thp */ 1102 nr_pages = 1; 1103 atomic_long_add(nr_pages, &num_poisoned_pages); 1104 1105 /* 1106 * We need/can do nothing about count=0 pages. 1107 * 1) it's a free page, and therefore in safe hand: 1108 * prep_new_page() will be the gate keeper. 1109 * 2) it's a free hugepage, which is also safe: 1110 * an affected hugepage will be dequeued from hugepage freelist, 1111 * so there's no concern about reusing it ever after. 1112 * 3) it's part of a non-compound high order page. 1113 * Implies some kernel user: cannot stop them from 1114 * R/W the page; let's pray that the page has been 1115 * used and will be freed some time later. 1116 * In fact it's dangerous to directly bump up page count from 0, 1117 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch. 1118 */ 1119 if (!(flags & MF_COUNT_INCREASED) && !get_hwpoison_page(p)) { 1120 if (is_free_buddy_page(p)) { 1121 action_result(pfn, MF_MSG_BUDDY, MF_DELAYED); 1122 return 0; 1123 } else if (PageHuge(hpage)) { 1124 /* 1125 * Check "filter hit" and "race with other subpage." 1126 */ 1127 lock_page(hpage); 1128 if (PageHWPoison(hpage)) { 1129 if ((hwpoison_filter(p) && TestClearPageHWPoison(p)) 1130 || (p != hpage && TestSetPageHWPoison(hpage))) { 1131 atomic_long_sub(nr_pages, &num_poisoned_pages); 1132 unlock_page(hpage); 1133 return 0; 1134 } 1135 } 1136 set_page_hwpoison_huge_page(hpage); 1137 res = dequeue_hwpoisoned_huge_page(hpage); 1138 action_result(pfn, MF_MSG_FREE_HUGE, 1139 res ? MF_IGNORED : MF_DELAYED); 1140 unlock_page(hpage); 1141 return res; 1142 } else { 1143 action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); 1144 return -EBUSY; 1145 } 1146 } 1147 1148 if (!PageHuge(p) && PageTransHuge(hpage)) { 1149 if (!PageAnon(hpage) || unlikely(split_huge_page(hpage))) { 1150 if (!PageAnon(hpage)) 1151 pr_err("MCE: %#lx: non anonymous thp\n", pfn); 1152 else 1153 pr_err("MCE: %#lx: thp split failed\n", pfn); 1154 if (TestClearPageHWPoison(p)) 1155 atomic_long_sub(nr_pages, &num_poisoned_pages); 1156 put_page(p); 1157 if (p != hpage) 1158 put_page(hpage); 1159 return -EBUSY; 1160 } 1161 VM_BUG_ON_PAGE(!page_count(p), p); 1162 hpage = compound_head(p); 1163 } 1164 1165 /* 1166 * We ignore non-LRU pages for good reasons. 1167 * - PG_locked is only well defined for LRU pages and a few others 1168 * - to avoid races with __set_page_locked() 1169 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) 1170 * The check (unnecessarily) ignores LRU pages being isolated and 1171 * walked by the page reclaim code, however that's not a big loss. 1172 */ 1173 if (!PageHuge(p)) { 1174 if (!PageLRU(p)) 1175 shake_page(p, 0); 1176 if (!PageLRU(p)) { 1177 /* 1178 * shake_page could have turned it free. 1179 */ 1180 if (is_free_buddy_page(p)) { 1181 if (flags & MF_COUNT_INCREASED) 1182 action_result(pfn, MF_MSG_BUDDY, MF_DELAYED); 1183 else 1184 action_result(pfn, MF_MSG_BUDDY_2ND, 1185 MF_DELAYED); 1186 return 0; 1187 } 1188 } 1189 } 1190 1191 lock_page(hpage); 1192 1193 /* 1194 * The page could have changed compound pages during the locking. 1195 * If this happens just bail out. 1196 */ 1197 if (PageCompound(p) && compound_head(p) != orig_head) { 1198 action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED); 1199 res = -EBUSY; 1200 goto out; 1201 } 1202 1203 /* 1204 * We use page flags to determine what action should be taken, but 1205 * the flags can be modified by the error containment action. One 1206 * example is an mlocked page, where PG_mlocked is cleared by 1207 * page_remove_rmap() in try_to_unmap_one(). So to determine page status 1208 * correctly, we save a copy of the page flags at this time. 1209 */ 1210 page_flags = p->flags; 1211 1212 /* 1213 * unpoison always clear PG_hwpoison inside page lock 1214 */ 1215 if (!PageHWPoison(p)) { 1216 printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn); 1217 atomic_long_sub(nr_pages, &num_poisoned_pages); 1218 unlock_page(hpage); 1219 put_page(hpage); 1220 return 0; 1221 } 1222 if (hwpoison_filter(p)) { 1223 if (TestClearPageHWPoison(p)) 1224 atomic_long_sub(nr_pages, &num_poisoned_pages); 1225 unlock_page(hpage); 1226 put_page(hpage); 1227 return 0; 1228 } 1229 1230 if (!PageHuge(p) && !PageTransTail(p) && !PageLRU(p)) 1231 goto identify_page_state; 1232 1233 /* 1234 * For error on the tail page, we should set PG_hwpoison 1235 * on the head page to show that the hugepage is hwpoisoned 1236 */ 1237 if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) { 1238 action_result(pfn, MF_MSG_POISONED_HUGE, MF_IGNORED); 1239 unlock_page(hpage); 1240 put_page(hpage); 1241 return 0; 1242 } 1243 /* 1244 * Set PG_hwpoison on all pages in an error hugepage, 1245 * because containment is done in hugepage unit for now. 1246 * Since we have done TestSetPageHWPoison() for the head page with 1247 * page lock held, we can safely set PG_hwpoison bits on tail pages. 1248 */ 1249 if (PageHuge(p)) 1250 set_page_hwpoison_huge_page(hpage); 1251 1252 /* 1253 * It's very difficult to mess with pages currently under IO 1254 * and in many cases impossible, so we just avoid it here. 1255 */ 1256 wait_on_page_writeback(p); 1257 1258 /* 1259 * Now take care of user space mappings. 1260 * Abort on fail: __delete_from_page_cache() assumes unmapped page. 1261 * 1262 * When the raw error page is thp tail page, hpage points to the raw 1263 * page after thp split. 1264 */ 1265 if (hwpoison_user_mappings(p, pfn, trapno, flags, &hpage) 1266 != SWAP_SUCCESS) { 1267 action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); 1268 res = -EBUSY; 1269 goto out; 1270 } 1271 1272 /* 1273 * Torn down by someone else? 1274 */ 1275 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { 1276 action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED); 1277 res = -EBUSY; 1278 goto out; 1279 } 1280 1281 identify_page_state: 1282 res = -EBUSY; 1283 /* 1284 * The first check uses the current page flags which may not have any 1285 * relevant information. The second check with the saved page flagss is 1286 * carried out only if the first check can't determine the page status. 1287 */ 1288 for (ps = error_states;; ps++) 1289 if ((p->flags & ps->mask) == ps->res) 1290 break; 1291 1292 page_flags |= (p->flags & (1UL << PG_dirty)); 1293 1294 if (!ps->mask) 1295 for (ps = error_states;; ps++) 1296 if ((page_flags & ps->mask) == ps->res) 1297 break; 1298 res = page_action(ps, p, pfn); 1299 out: 1300 unlock_page(hpage); 1301 return res; 1302 } 1303 EXPORT_SYMBOL_GPL(memory_failure); 1304 1305 #define MEMORY_FAILURE_FIFO_ORDER 4 1306 #define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER) 1307 1308 struct memory_failure_entry { 1309 unsigned long pfn; 1310 int trapno; 1311 int flags; 1312 }; 1313 1314 struct memory_failure_cpu { 1315 DECLARE_KFIFO(fifo, struct memory_failure_entry, 1316 MEMORY_FAILURE_FIFO_SIZE); 1317 spinlock_t lock; 1318 struct work_struct work; 1319 }; 1320 1321 static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu); 1322 1323 /** 1324 * memory_failure_queue - Schedule handling memory failure of a page. 1325 * @pfn: Page Number of the corrupted page 1326 * @trapno: Trap number reported in the signal to user space. 1327 * @flags: Flags for memory failure handling 1328 * 1329 * This function is called by the low level hardware error handler 1330 * when it detects hardware memory corruption of a page. It schedules 1331 * the recovering of error page, including dropping pages, killing 1332 * processes etc. 1333 * 1334 * The function is primarily of use for corruptions that 1335 * happen outside the current execution context (e.g. when 1336 * detected by a background scrubber) 1337 * 1338 * Can run in IRQ context. 1339 */ 1340 void memory_failure_queue(unsigned long pfn, int trapno, int flags) 1341 { 1342 struct memory_failure_cpu *mf_cpu; 1343 unsigned long proc_flags; 1344 struct memory_failure_entry entry = { 1345 .pfn = pfn, 1346 .trapno = trapno, 1347 .flags = flags, 1348 }; 1349 1350 mf_cpu = &get_cpu_var(memory_failure_cpu); 1351 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 1352 if (kfifo_put(&mf_cpu->fifo, entry)) 1353 schedule_work_on(smp_processor_id(), &mf_cpu->work); 1354 else 1355 pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n", 1356 pfn); 1357 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 1358 put_cpu_var(memory_failure_cpu); 1359 } 1360 EXPORT_SYMBOL_GPL(memory_failure_queue); 1361 1362 static void memory_failure_work_func(struct work_struct *work) 1363 { 1364 struct memory_failure_cpu *mf_cpu; 1365 struct memory_failure_entry entry = { 0, }; 1366 unsigned long proc_flags; 1367 int gotten; 1368 1369 mf_cpu = this_cpu_ptr(&memory_failure_cpu); 1370 for (;;) { 1371 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 1372 gotten = kfifo_get(&mf_cpu->fifo, &entry); 1373 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 1374 if (!gotten) 1375 break; 1376 if (entry.flags & MF_SOFT_OFFLINE) 1377 soft_offline_page(pfn_to_page(entry.pfn), entry.flags); 1378 else 1379 memory_failure(entry.pfn, entry.trapno, entry.flags); 1380 } 1381 } 1382 1383 static int __init memory_failure_init(void) 1384 { 1385 struct memory_failure_cpu *mf_cpu; 1386 int cpu; 1387 1388 for_each_possible_cpu(cpu) { 1389 mf_cpu = &per_cpu(memory_failure_cpu, cpu); 1390 spin_lock_init(&mf_cpu->lock); 1391 INIT_KFIFO(mf_cpu->fifo); 1392 INIT_WORK(&mf_cpu->work, memory_failure_work_func); 1393 } 1394 1395 return 0; 1396 } 1397 core_initcall(memory_failure_init); 1398 1399 /** 1400 * unpoison_memory - Unpoison a previously poisoned page 1401 * @pfn: Page number of the to be unpoisoned page 1402 * 1403 * Software-unpoison a page that has been poisoned by 1404 * memory_failure() earlier. 1405 * 1406 * This is only done on the software-level, so it only works 1407 * for linux injected failures, not real hardware failures 1408 * 1409 * Returns 0 for success, otherwise -errno. 1410 */ 1411 int unpoison_memory(unsigned long pfn) 1412 { 1413 struct page *page; 1414 struct page *p; 1415 int freeit = 0; 1416 unsigned int nr_pages; 1417 1418 if (!pfn_valid(pfn)) 1419 return -ENXIO; 1420 1421 p = pfn_to_page(pfn); 1422 page = compound_head(p); 1423 1424 if (!PageHWPoison(p)) { 1425 pr_info("MCE: Page was already unpoisoned %#lx\n", pfn); 1426 return 0; 1427 } 1428 1429 /* 1430 * unpoison_memory() can encounter thp only when the thp is being 1431 * worked by memory_failure() and the page lock is not held yet. 1432 * In such case, we yield to memory_failure() and make unpoison fail. 1433 */ 1434 if (!PageHuge(page) && PageTransHuge(page)) { 1435 pr_info("MCE: Memory failure is now running on %#lx\n", pfn); 1436 return 0; 1437 } 1438 1439 nr_pages = 1 << compound_order(page); 1440 1441 if (!get_hwpoison_page(p)) { 1442 /* 1443 * Since HWPoisoned hugepage should have non-zero refcount, 1444 * race between memory failure and unpoison seems to happen. 1445 * In such case unpoison fails and memory failure runs 1446 * to the end. 1447 */ 1448 if (PageHuge(page)) { 1449 pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn); 1450 return 0; 1451 } 1452 if (TestClearPageHWPoison(p)) 1453 atomic_long_dec(&num_poisoned_pages); 1454 pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn); 1455 return 0; 1456 } 1457 1458 lock_page(page); 1459 /* 1460 * This test is racy because PG_hwpoison is set outside of page lock. 1461 * That's acceptable because that won't trigger kernel panic. Instead, 1462 * the PG_hwpoison page will be caught and isolated on the entrance to 1463 * the free buddy page pool. 1464 */ 1465 if (TestClearPageHWPoison(page)) { 1466 pr_info("MCE: Software-unpoisoned page %#lx\n", pfn); 1467 atomic_long_sub(nr_pages, &num_poisoned_pages); 1468 freeit = 1; 1469 if (PageHuge(page)) 1470 clear_page_hwpoison_huge_page(page); 1471 } 1472 unlock_page(page); 1473 1474 put_page(page); 1475 if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) 1476 put_page(page); 1477 1478 return 0; 1479 } 1480 EXPORT_SYMBOL(unpoison_memory); 1481 1482 static struct page *new_page(struct page *p, unsigned long private, int **x) 1483 { 1484 int nid = page_to_nid(p); 1485 if (PageHuge(p)) 1486 return alloc_huge_page_node(page_hstate(compound_head(p)), 1487 nid); 1488 else 1489 return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0); 1490 } 1491 1492 /* 1493 * Safely get reference count of an arbitrary page. 1494 * Returns 0 for a free page, -EIO for a zero refcount page 1495 * that is not free, and 1 for any other page type. 1496 * For 1 the page is returned with increased page count, otherwise not. 1497 */ 1498 static int __get_any_page(struct page *p, unsigned long pfn, int flags) 1499 { 1500 int ret; 1501 1502 if (flags & MF_COUNT_INCREASED) 1503 return 1; 1504 1505 /* 1506 * When the target page is a free hugepage, just remove it 1507 * from free hugepage list. 1508 */ 1509 if (!get_hwpoison_page(p)) { 1510 if (PageHuge(p)) { 1511 pr_info("%s: %#lx free huge page\n", __func__, pfn); 1512 ret = 0; 1513 } else if (is_free_buddy_page(p)) { 1514 pr_info("%s: %#lx free buddy page\n", __func__, pfn); 1515 ret = 0; 1516 } else { 1517 pr_info("%s: %#lx: unknown zero refcount page type %lx\n", 1518 __func__, pfn, p->flags); 1519 ret = -EIO; 1520 } 1521 } else { 1522 /* Not a free page */ 1523 ret = 1; 1524 } 1525 return ret; 1526 } 1527 1528 static int get_any_page(struct page *page, unsigned long pfn, int flags) 1529 { 1530 int ret = __get_any_page(page, pfn, flags); 1531 1532 if (ret == 1 && !PageHuge(page) && !PageLRU(page)) { 1533 /* 1534 * Try to free it. 1535 */ 1536 put_page(page); 1537 shake_page(page, 1); 1538 1539 /* 1540 * Did it turn free? 1541 */ 1542 ret = __get_any_page(page, pfn, 0); 1543 if (!PageLRU(page)) { 1544 /* Drop page reference which is from __get_any_page() */ 1545 put_page(page); 1546 pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n", 1547 pfn, page->flags); 1548 return -EIO; 1549 } 1550 } 1551 return ret; 1552 } 1553 1554 static int soft_offline_huge_page(struct page *page, int flags) 1555 { 1556 int ret; 1557 unsigned long pfn = page_to_pfn(page); 1558 struct page *hpage = compound_head(page); 1559 LIST_HEAD(pagelist); 1560 1561 /* 1562 * This double-check of PageHWPoison is to avoid the race with 1563 * memory_failure(). See also comment in __soft_offline_page(). 1564 */ 1565 lock_page(hpage); 1566 if (PageHWPoison(hpage)) { 1567 unlock_page(hpage); 1568 put_page(hpage); 1569 pr_info("soft offline: %#lx hugepage already poisoned\n", pfn); 1570 return -EBUSY; 1571 } 1572 unlock_page(hpage); 1573 1574 ret = isolate_huge_page(hpage, &pagelist); 1575 /* 1576 * get_any_page() and isolate_huge_page() takes a refcount each, 1577 * so need to drop one here. 1578 */ 1579 put_page(hpage); 1580 if (!ret) { 1581 pr_info("soft offline: %#lx hugepage failed to isolate\n", pfn); 1582 return -EBUSY; 1583 } 1584 1585 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1586 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1587 if (ret) { 1588 pr_info("soft offline: %#lx: migration failed %d, type %lx\n", 1589 pfn, ret, page->flags); 1590 /* 1591 * We know that soft_offline_huge_page() tries to migrate 1592 * only one hugepage pointed to by hpage, so we need not 1593 * run through the pagelist here. 1594 */ 1595 putback_active_hugepage(hpage); 1596 if (ret > 0) 1597 ret = -EIO; 1598 } else { 1599 /* overcommit hugetlb page will be freed to buddy */ 1600 if (PageHuge(page)) { 1601 set_page_hwpoison_huge_page(hpage); 1602 dequeue_hwpoisoned_huge_page(hpage); 1603 atomic_long_add(1 << compound_order(hpage), 1604 &num_poisoned_pages); 1605 } else { 1606 SetPageHWPoison(page); 1607 atomic_long_inc(&num_poisoned_pages); 1608 } 1609 } 1610 return ret; 1611 } 1612 1613 static int __soft_offline_page(struct page *page, int flags) 1614 { 1615 int ret; 1616 unsigned long pfn = page_to_pfn(page); 1617 1618 /* 1619 * Check PageHWPoison again inside page lock because PageHWPoison 1620 * is set by memory_failure() outside page lock. Note that 1621 * memory_failure() also double-checks PageHWPoison inside page lock, 1622 * so there's no race between soft_offline_page() and memory_failure(). 1623 */ 1624 lock_page(page); 1625 wait_on_page_writeback(page); 1626 if (PageHWPoison(page)) { 1627 unlock_page(page); 1628 put_page(page); 1629 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1630 return -EBUSY; 1631 } 1632 /* 1633 * Try to invalidate first. This should work for 1634 * non dirty unmapped page cache pages. 1635 */ 1636 ret = invalidate_inode_page(page); 1637 unlock_page(page); 1638 /* 1639 * RED-PEN would be better to keep it isolated here, but we 1640 * would need to fix isolation locking first. 1641 */ 1642 if (ret == 1) { 1643 put_page(page); 1644 pr_info("soft_offline: %#lx: invalidated\n", pfn); 1645 SetPageHWPoison(page); 1646 atomic_long_inc(&num_poisoned_pages); 1647 return 0; 1648 } 1649 1650 /* 1651 * Simple invalidation didn't work. 1652 * Try to migrate to a new page instead. migrate.c 1653 * handles a large number of cases for us. 1654 */ 1655 ret = isolate_lru_page(page); 1656 /* 1657 * Drop page reference which is came from get_any_page() 1658 * successful isolate_lru_page() already took another one. 1659 */ 1660 put_page(page); 1661 if (!ret) { 1662 LIST_HEAD(pagelist); 1663 inc_zone_page_state(page, NR_ISOLATED_ANON + 1664 page_is_file_cache(page)); 1665 list_add(&page->lru, &pagelist); 1666 if (!TestSetPageHWPoison(page)) 1667 atomic_long_inc(&num_poisoned_pages); 1668 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL, 1669 MIGRATE_SYNC, MR_MEMORY_FAILURE); 1670 if (ret) { 1671 if (!list_empty(&pagelist)) { 1672 list_del(&page->lru); 1673 dec_zone_page_state(page, NR_ISOLATED_ANON + 1674 page_is_file_cache(page)); 1675 putback_lru_page(page); 1676 } 1677 1678 pr_info("soft offline: %#lx: migration failed %d, type %lx\n", 1679 pfn, ret, page->flags); 1680 if (ret > 0) 1681 ret = -EIO; 1682 if (TestClearPageHWPoison(page)) 1683 atomic_long_dec(&num_poisoned_pages); 1684 } 1685 } else { 1686 pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n", 1687 pfn, ret, page_count(page), page->flags); 1688 } 1689 return ret; 1690 } 1691 1692 /** 1693 * soft_offline_page - Soft offline a page. 1694 * @page: page to offline 1695 * @flags: flags. Same as memory_failure(). 1696 * 1697 * Returns 0 on success, otherwise negated errno. 1698 * 1699 * Soft offline a page, by migration or invalidation, 1700 * without killing anything. This is for the case when 1701 * a page is not corrupted yet (so it's still valid to access), 1702 * but has had a number of corrected errors and is better taken 1703 * out. 1704 * 1705 * The actual policy on when to do that is maintained by 1706 * user space. 1707 * 1708 * This should never impact any application or cause data loss, 1709 * however it might take some time. 1710 * 1711 * This is not a 100% solution for all memory, but tries to be 1712 * ``good enough'' for the majority of memory. 1713 */ 1714 int soft_offline_page(struct page *page, int flags) 1715 { 1716 int ret; 1717 unsigned long pfn = page_to_pfn(page); 1718 struct page *hpage = compound_head(page); 1719 1720 if (PageHWPoison(page)) { 1721 pr_info("soft offline: %#lx page already poisoned\n", pfn); 1722 return -EBUSY; 1723 } 1724 if (!PageHuge(page) && PageTransHuge(hpage)) { 1725 if (PageAnon(hpage) && unlikely(split_huge_page(hpage))) { 1726 pr_info("soft offline: %#lx: failed to split THP\n", 1727 pfn); 1728 return -EBUSY; 1729 } 1730 } 1731 1732 get_online_mems(); 1733 1734 ret = get_any_page(page, pfn, flags); 1735 put_online_mems(); 1736 if (ret > 0) { /* for in-use pages */ 1737 if (PageHuge(page)) 1738 ret = soft_offline_huge_page(page, flags); 1739 else 1740 ret = __soft_offline_page(page, flags); 1741 } else if (ret == 0) { /* for free pages */ 1742 if (PageHuge(page)) { 1743 set_page_hwpoison_huge_page(hpage); 1744 if (!dequeue_hwpoisoned_huge_page(hpage)) 1745 atomic_long_add(1 << compound_order(hpage), 1746 &num_poisoned_pages); 1747 } else { 1748 if (!TestSetPageHWPoison(page)) 1749 atomic_long_inc(&num_poisoned_pages); 1750 } 1751 } 1752 return ret; 1753 } 1754