1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2008, 2009 Intel Corporation 4 * Authors: Andi Kleen, Fengguang Wu 5 * 6 * High level machine check handler. Handles pages reported by the 7 * hardware as being corrupted usually due to a multi-bit ECC memory or cache 8 * failure. 9 * 10 * In addition there is a "soft offline" entry point that allows stop using 11 * not-yet-corrupted-by-suspicious pages without killing anything. 12 * 13 * Handles page cache pages in various states. The tricky part 14 * here is that we can access any page asynchronously in respect to 15 * other VM users, because memory failures could happen anytime and 16 * anywhere. This could violate some of their assumptions. This is why 17 * this code has to be extremely careful. Generally it tries to use 18 * normal locking rules, as in get the standard locks, even if that means 19 * the error handling takes potentially a long time. 20 * 21 * It can be very tempting to add handling for obscure cases here. 22 * In general any code for handling new cases should only be added iff: 23 * - You know how to test it. 24 * - You have a test that can be added to mce-test 25 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/ 26 * - The case actually shows up as a frequent (top 10) page state in 27 * tools/vm/page-types when running a real workload. 28 * 29 * There are several operations here with exponential complexity because 30 * of unsuitable VM data structures. For example the operation to map back 31 * from RMAP chains to processes has to walk the complete process list and 32 * has non linear complexity with the number. But since memory corruptions 33 * are rare we hope to get away with this. This avoids impacting the core 34 * VM. 35 */ 36 37 #define pr_fmt(fmt) "Memory failure: " fmt 38 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/page-flags.h> 42 #include <linux/kernel-page-flags.h> 43 #include <linux/sched/signal.h> 44 #include <linux/sched/task.h> 45 #include <linux/dax.h> 46 #include <linux/ksm.h> 47 #include <linux/rmap.h> 48 #include <linux/export.h> 49 #include <linux/pagemap.h> 50 #include <linux/swap.h> 51 #include <linux/backing-dev.h> 52 #include <linux/migrate.h> 53 #include <linux/suspend.h> 54 #include <linux/slab.h> 55 #include <linux/swapops.h> 56 #include <linux/hugetlb.h> 57 #include <linux/memory_hotplug.h> 58 #include <linux/mm_inline.h> 59 #include <linux/memremap.h> 60 #include <linux/kfifo.h> 61 #include <linux/ratelimit.h> 62 #include <linux/page-isolation.h> 63 #include <linux/pagewalk.h> 64 #include <linux/shmem_fs.h> 65 #include "swap.h" 66 #include "internal.h" 67 #include "ras/ras_event.h" 68 69 int sysctl_memory_failure_early_kill __read_mostly = 0; 70 71 int sysctl_memory_failure_recovery __read_mostly = 1; 72 73 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); 74 75 static bool hw_memory_failure __read_mostly = false; 76 77 /* 78 * Return values: 79 * 1: the page is dissolved (if needed) and taken off from buddy, 80 * 0: the page is dissolved (if needed) and not taken off from buddy, 81 * < 0: failed to dissolve. 82 */ 83 static int __page_handle_poison(struct page *page) 84 { 85 int ret; 86 87 zone_pcp_disable(page_zone(page)); 88 ret = dissolve_free_huge_page(page); 89 if (!ret) 90 ret = take_page_off_buddy(page); 91 zone_pcp_enable(page_zone(page)); 92 93 return ret; 94 } 95 96 static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release) 97 { 98 if (hugepage_or_freepage) { 99 /* 100 * Doing this check for free pages is also fine since dissolve_free_huge_page 101 * returns 0 for non-hugetlb pages as well. 102 */ 103 if (__page_handle_poison(page) <= 0) 104 /* 105 * We could fail to take off the target page from buddy 106 * for example due to racy page allocation, but that's 107 * acceptable because soft-offlined page is not broken 108 * and if someone really want to use it, they should 109 * take it. 110 */ 111 return false; 112 } 113 114 SetPageHWPoison(page); 115 if (release) 116 put_page(page); 117 page_ref_inc(page); 118 num_poisoned_pages_inc(); 119 120 return true; 121 } 122 123 #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE) 124 125 u32 hwpoison_filter_enable = 0; 126 u32 hwpoison_filter_dev_major = ~0U; 127 u32 hwpoison_filter_dev_minor = ~0U; 128 u64 hwpoison_filter_flags_mask; 129 u64 hwpoison_filter_flags_value; 130 EXPORT_SYMBOL_GPL(hwpoison_filter_enable); 131 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major); 132 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor); 133 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask); 134 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value); 135 136 static int hwpoison_filter_dev(struct page *p) 137 { 138 struct address_space *mapping; 139 dev_t dev; 140 141 if (hwpoison_filter_dev_major == ~0U && 142 hwpoison_filter_dev_minor == ~0U) 143 return 0; 144 145 mapping = page_mapping(p); 146 if (mapping == NULL || mapping->host == NULL) 147 return -EINVAL; 148 149 dev = mapping->host->i_sb->s_dev; 150 if (hwpoison_filter_dev_major != ~0U && 151 hwpoison_filter_dev_major != MAJOR(dev)) 152 return -EINVAL; 153 if (hwpoison_filter_dev_minor != ~0U && 154 hwpoison_filter_dev_minor != MINOR(dev)) 155 return -EINVAL; 156 157 return 0; 158 } 159 160 static int hwpoison_filter_flags(struct page *p) 161 { 162 if (!hwpoison_filter_flags_mask) 163 return 0; 164 165 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) == 166 hwpoison_filter_flags_value) 167 return 0; 168 else 169 return -EINVAL; 170 } 171 172 /* 173 * This allows stress tests to limit test scope to a collection of tasks 174 * by putting them under some memcg. This prevents killing unrelated/important 175 * processes such as /sbin/init. Note that the target task may share clean 176 * pages with init (eg. libc text), which is harmless. If the target task 177 * share _dirty_ pages with another task B, the test scheme must make sure B 178 * is also included in the memcg. At last, due to race conditions this filter 179 * can only guarantee that the page either belongs to the memcg tasks, or is 180 * a freed page. 181 */ 182 #ifdef CONFIG_MEMCG 183 u64 hwpoison_filter_memcg; 184 EXPORT_SYMBOL_GPL(hwpoison_filter_memcg); 185 static int hwpoison_filter_task(struct page *p) 186 { 187 if (!hwpoison_filter_memcg) 188 return 0; 189 190 if (page_cgroup_ino(p) != hwpoison_filter_memcg) 191 return -EINVAL; 192 193 return 0; 194 } 195 #else 196 static int hwpoison_filter_task(struct page *p) { return 0; } 197 #endif 198 199 int hwpoison_filter(struct page *p) 200 { 201 if (!hwpoison_filter_enable) 202 return 0; 203 204 if (hwpoison_filter_dev(p)) 205 return -EINVAL; 206 207 if (hwpoison_filter_flags(p)) 208 return -EINVAL; 209 210 if (hwpoison_filter_task(p)) 211 return -EINVAL; 212 213 return 0; 214 } 215 #else 216 int hwpoison_filter(struct page *p) 217 { 218 return 0; 219 } 220 #endif 221 222 EXPORT_SYMBOL_GPL(hwpoison_filter); 223 224 /* 225 * Kill all processes that have a poisoned page mapped and then isolate 226 * the page. 227 * 228 * General strategy: 229 * Find all processes having the page mapped and kill them. 230 * But we keep a page reference around so that the page is not 231 * actually freed yet. 232 * Then stash the page away 233 * 234 * There's no convenient way to get back to mapped processes 235 * from the VMAs. So do a brute-force search over all 236 * running processes. 237 * 238 * Remember that machine checks are not common (or rather 239 * if they are common you have other problems), so this shouldn't 240 * be a performance issue. 241 * 242 * Also there are some races possible while we get from the 243 * error detection to actually handle it. 244 */ 245 246 struct to_kill { 247 struct list_head nd; 248 struct task_struct *tsk; 249 unsigned long addr; 250 short size_shift; 251 }; 252 253 /* 254 * Send all the processes who have the page mapped a signal. 255 * ``action optional'' if they are not immediately affected by the error 256 * ``action required'' if error happened in current execution context 257 */ 258 static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) 259 { 260 struct task_struct *t = tk->tsk; 261 short addr_lsb = tk->size_shift; 262 int ret = 0; 263 264 pr_err("%#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n", 265 pfn, t->comm, t->pid); 266 267 if ((flags & MF_ACTION_REQUIRED) && (t == current)) 268 ret = force_sig_mceerr(BUS_MCEERR_AR, 269 (void __user *)tk->addr, addr_lsb); 270 else 271 /* 272 * Signal other processes sharing the page if they have 273 * PF_MCE_EARLY set. 274 * Don't use force here, it's convenient if the signal 275 * can be temporarily blocked. 276 * This could cause a loop when the user sets SIGBUS 277 * to SIG_IGN, but hopefully no one will do that? 278 */ 279 ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, 280 addr_lsb, t); 281 if (ret < 0) 282 pr_info("Error sending signal to %s:%d: %d\n", 283 t->comm, t->pid, ret); 284 return ret; 285 } 286 287 /* 288 * Unknown page type encountered. Try to check whether it can turn PageLRU by 289 * lru_add_drain_all. 290 */ 291 void shake_page(struct page *p) 292 { 293 if (PageHuge(p)) 294 return; 295 296 if (!PageSlab(p)) { 297 lru_add_drain_all(); 298 if (PageLRU(p) || is_free_buddy_page(p)) 299 return; 300 } 301 302 /* 303 * TODO: Could shrink slab caches here if a lightweight range-based 304 * shrinker will be available. 305 */ 306 } 307 EXPORT_SYMBOL_GPL(shake_page); 308 309 static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, 310 unsigned long address) 311 { 312 unsigned long ret = 0; 313 pgd_t *pgd; 314 p4d_t *p4d; 315 pud_t *pud; 316 pmd_t *pmd; 317 pte_t *pte; 318 319 VM_BUG_ON_VMA(address == -EFAULT, vma); 320 pgd = pgd_offset(vma->vm_mm, address); 321 if (!pgd_present(*pgd)) 322 return 0; 323 p4d = p4d_offset(pgd, address); 324 if (!p4d_present(*p4d)) 325 return 0; 326 pud = pud_offset(p4d, address); 327 if (!pud_present(*pud)) 328 return 0; 329 if (pud_devmap(*pud)) 330 return PUD_SHIFT; 331 pmd = pmd_offset(pud, address); 332 if (!pmd_present(*pmd)) 333 return 0; 334 if (pmd_devmap(*pmd)) 335 return PMD_SHIFT; 336 pte = pte_offset_map(pmd, address); 337 if (pte_present(*pte) && pte_devmap(*pte)) 338 ret = PAGE_SHIFT; 339 pte_unmap(pte); 340 return ret; 341 } 342 343 /* 344 * Failure handling: if we can't find or can't kill a process there's 345 * not much we can do. We just print a message and ignore otherwise. 346 */ 347 348 #define FSDAX_INVALID_PGOFF ULONG_MAX 349 350 /* 351 * Schedule a process for later kill. 352 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. 353 * 354 * Note: @fsdax_pgoff is used only when @p is a fsdax page and a 355 * filesystem with a memory failure handler has claimed the 356 * memory_failure event. In all other cases, page->index and 357 * page->mapping are sufficient for mapping the page back to its 358 * corresponding user virtual address. 359 */ 360 static void add_to_kill(struct task_struct *tsk, struct page *p, 361 pgoff_t fsdax_pgoff, struct vm_area_struct *vma, 362 struct list_head *to_kill) 363 { 364 struct to_kill *tk; 365 366 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC); 367 if (!tk) { 368 pr_err("Out of memory while machine check handling\n"); 369 return; 370 } 371 372 tk->addr = page_address_in_vma(p, vma); 373 if (is_zone_device_page(p)) { 374 if (fsdax_pgoff != FSDAX_INVALID_PGOFF) 375 tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma); 376 tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); 377 } else 378 tk->size_shift = page_shift(compound_head(p)); 379 380 /* 381 * Send SIGKILL if "tk->addr == -EFAULT". Also, as 382 * "tk->size_shift" is always non-zero for !is_zone_device_page(), 383 * so "tk->size_shift == 0" effectively checks no mapping on 384 * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times 385 * to a process' address space, it's possible not all N VMAs 386 * contain mappings for the page, but at least one VMA does. 387 * Only deliver SIGBUS with payload derived from the VMA that 388 * has a mapping for the page. 389 */ 390 if (tk->addr == -EFAULT) { 391 pr_info("Unable to find user space address %lx in %s\n", 392 page_to_pfn(p), tsk->comm); 393 } else if (tk->size_shift == 0) { 394 kfree(tk); 395 return; 396 } 397 398 get_task_struct(tsk); 399 tk->tsk = tsk; 400 list_add_tail(&tk->nd, to_kill); 401 } 402 403 /* 404 * Kill the processes that have been collected earlier. 405 * 406 * Only do anything when FORCEKILL is set, otherwise just free the 407 * list (this is used for clean pages which do not need killing) 408 * Also when FAIL is set do a force kill because something went 409 * wrong earlier. 410 */ 411 static void kill_procs(struct list_head *to_kill, int forcekill, bool fail, 412 unsigned long pfn, int flags) 413 { 414 struct to_kill *tk, *next; 415 416 list_for_each_entry_safe(tk, next, to_kill, nd) { 417 if (forcekill) { 418 /* 419 * In case something went wrong with munmapping 420 * make sure the process doesn't catch the 421 * signal and then access the memory. Just kill it. 422 */ 423 if (fail || tk->addr == -EFAULT) { 424 pr_err("%#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", 425 pfn, tk->tsk->comm, tk->tsk->pid); 426 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, 427 tk->tsk, PIDTYPE_PID); 428 } 429 430 /* 431 * In theory the process could have mapped 432 * something else on the address in-between. We could 433 * check for that, but we need to tell the 434 * process anyways. 435 */ 436 else if (kill_proc(tk, pfn, flags) < 0) 437 pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n", 438 pfn, tk->tsk->comm, tk->tsk->pid); 439 } 440 list_del(&tk->nd); 441 put_task_struct(tk->tsk); 442 kfree(tk); 443 } 444 } 445 446 /* 447 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) 448 * on behalf of the thread group. Return task_struct of the (first found) 449 * dedicated thread if found, and return NULL otherwise. 450 * 451 * We already hold read_lock(&tasklist_lock) in the caller, so we don't 452 * have to call rcu_read_lock/unlock() in this function. 453 */ 454 static struct task_struct *find_early_kill_thread(struct task_struct *tsk) 455 { 456 struct task_struct *t; 457 458 for_each_thread(tsk, t) { 459 if (t->flags & PF_MCE_PROCESS) { 460 if (t->flags & PF_MCE_EARLY) 461 return t; 462 } else { 463 if (sysctl_memory_failure_early_kill) 464 return t; 465 } 466 } 467 return NULL; 468 } 469 470 /* 471 * Determine whether a given process is "early kill" process which expects 472 * to be signaled when some page under the process is hwpoisoned. 473 * Return task_struct of the dedicated thread (main thread unless explicitly 474 * specified) if the process is "early kill" and otherwise returns NULL. 475 * 476 * Note that the above is true for Action Optional case. For Action Required 477 * case, it's only meaningful to the current thread which need to be signaled 478 * with SIGBUS, this error is Action Optional for other non current 479 * processes sharing the same error page,if the process is "early kill", the 480 * task_struct of the dedicated thread will also be returned. 481 */ 482 static struct task_struct *task_early_kill(struct task_struct *tsk, 483 int force_early) 484 { 485 if (!tsk->mm) 486 return NULL; 487 /* 488 * Comparing ->mm here because current task might represent 489 * a subthread, while tsk always points to the main thread. 490 */ 491 if (force_early && tsk->mm == current->mm) 492 return current; 493 494 return find_early_kill_thread(tsk); 495 } 496 497 /* 498 * Collect processes when the error hit an anonymous page. 499 */ 500 static void collect_procs_anon(struct page *page, struct list_head *to_kill, 501 int force_early) 502 { 503 struct folio *folio = page_folio(page); 504 struct vm_area_struct *vma; 505 struct task_struct *tsk; 506 struct anon_vma *av; 507 pgoff_t pgoff; 508 509 av = folio_lock_anon_vma_read(folio, NULL); 510 if (av == NULL) /* Not actually mapped anymore */ 511 return; 512 513 pgoff = page_to_pgoff(page); 514 read_lock(&tasklist_lock); 515 for_each_process (tsk) { 516 struct anon_vma_chain *vmac; 517 struct task_struct *t = task_early_kill(tsk, force_early); 518 519 if (!t) 520 continue; 521 anon_vma_interval_tree_foreach(vmac, &av->rb_root, 522 pgoff, pgoff) { 523 vma = vmac->vma; 524 if (vma->vm_mm != t->mm) 525 continue; 526 if (!page_mapped_in_vma(page, vma)) 527 continue; 528 add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma, to_kill); 529 } 530 } 531 read_unlock(&tasklist_lock); 532 anon_vma_unlock_read(av); 533 } 534 535 /* 536 * Collect processes when the error hit a file mapped page. 537 */ 538 static void collect_procs_file(struct page *page, struct list_head *to_kill, 539 int force_early) 540 { 541 struct vm_area_struct *vma; 542 struct task_struct *tsk; 543 struct address_space *mapping = page->mapping; 544 pgoff_t pgoff; 545 546 i_mmap_lock_read(mapping); 547 read_lock(&tasklist_lock); 548 pgoff = page_to_pgoff(page); 549 for_each_process(tsk) { 550 struct task_struct *t = task_early_kill(tsk, force_early); 551 552 if (!t) 553 continue; 554 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, 555 pgoff) { 556 /* 557 * Send early kill signal to tasks where a vma covers 558 * the page but the corrupted page is not necessarily 559 * mapped it in its pte. 560 * Assume applications who requested early kill want 561 * to be informed of all such data corruptions. 562 */ 563 if (vma->vm_mm == t->mm) 564 add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma, 565 to_kill); 566 } 567 } 568 read_unlock(&tasklist_lock); 569 i_mmap_unlock_read(mapping); 570 } 571 572 #ifdef CONFIG_FS_DAX 573 /* 574 * Collect processes when the error hit a fsdax page. 575 */ 576 static void collect_procs_fsdax(struct page *page, 577 struct address_space *mapping, pgoff_t pgoff, 578 struct list_head *to_kill) 579 { 580 struct vm_area_struct *vma; 581 struct task_struct *tsk; 582 583 i_mmap_lock_read(mapping); 584 read_lock(&tasklist_lock); 585 for_each_process(tsk) { 586 struct task_struct *t = task_early_kill(tsk, true); 587 588 if (!t) 589 continue; 590 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 591 if (vma->vm_mm == t->mm) 592 add_to_kill(t, page, pgoff, vma, to_kill); 593 } 594 } 595 read_unlock(&tasklist_lock); 596 i_mmap_unlock_read(mapping); 597 } 598 #endif /* CONFIG_FS_DAX */ 599 600 /* 601 * Collect the processes who have the corrupted page mapped to kill. 602 */ 603 static void collect_procs(struct page *page, struct list_head *tokill, 604 int force_early) 605 { 606 if (!page->mapping) 607 return; 608 609 if (PageAnon(page)) 610 collect_procs_anon(page, tokill, force_early); 611 else 612 collect_procs_file(page, tokill, force_early); 613 } 614 615 struct hwp_walk { 616 struct to_kill tk; 617 unsigned long pfn; 618 int flags; 619 }; 620 621 static void set_to_kill(struct to_kill *tk, unsigned long addr, short shift) 622 { 623 tk->addr = addr; 624 tk->size_shift = shift; 625 } 626 627 static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, 628 unsigned long poisoned_pfn, struct to_kill *tk) 629 { 630 unsigned long pfn = 0; 631 632 if (pte_present(pte)) { 633 pfn = pte_pfn(pte); 634 } else { 635 swp_entry_t swp = pte_to_swp_entry(pte); 636 637 if (is_hwpoison_entry(swp)) 638 pfn = swp_offset_pfn(swp); 639 } 640 641 if (!pfn || pfn != poisoned_pfn) 642 return 0; 643 644 set_to_kill(tk, addr, shift); 645 return 1; 646 } 647 648 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 649 static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, 650 struct hwp_walk *hwp) 651 { 652 pmd_t pmd = *pmdp; 653 unsigned long pfn; 654 unsigned long hwpoison_vaddr; 655 656 if (!pmd_present(pmd)) 657 return 0; 658 pfn = pmd_pfn(pmd); 659 if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) { 660 hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT); 661 set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT); 662 return 1; 663 } 664 return 0; 665 } 666 #else 667 static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, 668 struct hwp_walk *hwp) 669 { 670 return 0; 671 } 672 #endif 673 674 static int hwpoison_pte_range(pmd_t *pmdp, unsigned long addr, 675 unsigned long end, struct mm_walk *walk) 676 { 677 struct hwp_walk *hwp = walk->private; 678 int ret = 0; 679 pte_t *ptep, *mapped_pte; 680 spinlock_t *ptl; 681 682 ptl = pmd_trans_huge_lock(pmdp, walk->vma); 683 if (ptl) { 684 ret = check_hwpoisoned_pmd_entry(pmdp, addr, hwp); 685 spin_unlock(ptl); 686 goto out; 687 } 688 689 if (pmd_trans_unstable(pmdp)) 690 goto out; 691 692 mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp, 693 addr, &ptl); 694 for (; addr != end; ptep++, addr += PAGE_SIZE) { 695 ret = check_hwpoisoned_entry(*ptep, addr, PAGE_SHIFT, 696 hwp->pfn, &hwp->tk); 697 if (ret == 1) 698 break; 699 } 700 pte_unmap_unlock(mapped_pte, ptl); 701 out: 702 cond_resched(); 703 return ret; 704 } 705 706 #ifdef CONFIG_HUGETLB_PAGE 707 static int hwpoison_hugetlb_range(pte_t *ptep, unsigned long hmask, 708 unsigned long addr, unsigned long end, 709 struct mm_walk *walk) 710 { 711 struct hwp_walk *hwp = walk->private; 712 pte_t pte = huge_ptep_get(ptep); 713 struct hstate *h = hstate_vma(walk->vma); 714 715 return check_hwpoisoned_entry(pte, addr, huge_page_shift(h), 716 hwp->pfn, &hwp->tk); 717 } 718 #else 719 #define hwpoison_hugetlb_range NULL 720 #endif 721 722 static const struct mm_walk_ops hwp_walk_ops = { 723 .pmd_entry = hwpoison_pte_range, 724 .hugetlb_entry = hwpoison_hugetlb_range, 725 }; 726 727 /* 728 * Sends SIGBUS to the current process with error info. 729 * 730 * This function is intended to handle "Action Required" MCEs on already 731 * hardware poisoned pages. They could happen, for example, when 732 * memory_failure() failed to unmap the error page at the first call, or 733 * when multiple local machine checks happened on different CPUs. 734 * 735 * MCE handler currently has no easy access to the error virtual address, 736 * so this function walks page table to find it. The returned virtual address 737 * is proper in most cases, but it could be wrong when the application 738 * process has multiple entries mapping the error page. 739 */ 740 static int kill_accessing_process(struct task_struct *p, unsigned long pfn, 741 int flags) 742 { 743 int ret; 744 struct hwp_walk priv = { 745 .pfn = pfn, 746 }; 747 priv.tk.tsk = p; 748 749 if (!p->mm) 750 return -EFAULT; 751 752 mmap_read_lock(p->mm); 753 ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwp_walk_ops, 754 (void *)&priv); 755 if (ret == 1 && priv.tk.addr) 756 kill_proc(&priv.tk, pfn, flags); 757 else 758 ret = 0; 759 mmap_read_unlock(p->mm); 760 return ret > 0 ? -EHWPOISON : -EFAULT; 761 } 762 763 static const char *action_name[] = { 764 [MF_IGNORED] = "Ignored", 765 [MF_FAILED] = "Failed", 766 [MF_DELAYED] = "Delayed", 767 [MF_RECOVERED] = "Recovered", 768 }; 769 770 static const char * const action_page_types[] = { 771 [MF_MSG_KERNEL] = "reserved kernel page", 772 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page", 773 [MF_MSG_SLAB] = "kernel slab page", 774 [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking", 775 [MF_MSG_HUGE] = "huge page", 776 [MF_MSG_FREE_HUGE] = "free huge page", 777 [MF_MSG_UNMAP_FAILED] = "unmapping failed page", 778 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page", 779 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page", 780 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page", 781 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page", 782 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page", 783 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page", 784 [MF_MSG_DIRTY_LRU] = "dirty LRU page", 785 [MF_MSG_CLEAN_LRU] = "clean LRU page", 786 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", 787 [MF_MSG_BUDDY] = "free buddy page", 788 [MF_MSG_DAX] = "dax page", 789 [MF_MSG_UNSPLIT_THP] = "unsplit thp", 790 [MF_MSG_UNKNOWN] = "unknown page", 791 }; 792 793 /* 794 * XXX: It is possible that a page is isolated from LRU cache, 795 * and then kept in swap cache or failed to remove from page cache. 796 * The page count will stop it from being freed by unpoison. 797 * Stress tests should be aware of this memory leak problem. 798 */ 799 static int delete_from_lru_cache(struct page *p) 800 { 801 if (!isolate_lru_page(p)) { 802 /* 803 * Clear sensible page flags, so that the buddy system won't 804 * complain when the page is unpoison-and-freed. 805 */ 806 ClearPageActive(p); 807 ClearPageUnevictable(p); 808 809 /* 810 * Poisoned page might never drop its ref count to 0 so we have 811 * to uncharge it manually from its memcg. 812 */ 813 mem_cgroup_uncharge(page_folio(p)); 814 815 /* 816 * drop the page count elevated by isolate_lru_page() 817 */ 818 put_page(p); 819 return 0; 820 } 821 return -EIO; 822 } 823 824 static int truncate_error_page(struct page *p, unsigned long pfn, 825 struct address_space *mapping) 826 { 827 int ret = MF_FAILED; 828 829 if (mapping->a_ops->error_remove_page) { 830 int err = mapping->a_ops->error_remove_page(mapping, p); 831 832 if (err != 0) { 833 pr_info("%#lx: Failed to punch page: %d\n", pfn, err); 834 } else if (page_has_private(p) && 835 !try_to_release_page(p, GFP_NOIO)) { 836 pr_info("%#lx: failed to release buffers\n", pfn); 837 } else { 838 ret = MF_RECOVERED; 839 } 840 } else { 841 /* 842 * If the file system doesn't support it just invalidate 843 * This fails on dirty or anything with private pages 844 */ 845 if (invalidate_inode_page(p)) 846 ret = MF_RECOVERED; 847 else 848 pr_info("%#lx: Failed to invalidate\n", pfn); 849 } 850 851 return ret; 852 } 853 854 struct page_state { 855 unsigned long mask; 856 unsigned long res; 857 enum mf_action_page_type type; 858 859 /* Callback ->action() has to unlock the relevant page inside it. */ 860 int (*action)(struct page_state *ps, struct page *p); 861 }; 862 863 /* 864 * Return true if page is still referenced by others, otherwise return 865 * false. 866 * 867 * The extra_pins is true when one extra refcount is expected. 868 */ 869 static bool has_extra_refcount(struct page_state *ps, struct page *p, 870 bool extra_pins) 871 { 872 int count = page_count(p) - 1; 873 874 if (extra_pins) 875 count -= 1; 876 877 if (count > 0) { 878 pr_err("%#lx: %s still referenced by %d users\n", 879 page_to_pfn(p), action_page_types[ps->type], count); 880 return true; 881 } 882 883 return false; 884 } 885 886 /* 887 * Error hit kernel page. 888 * Do nothing, try to be lucky and not touch this instead. For a few cases we 889 * could be more sophisticated. 890 */ 891 static int me_kernel(struct page_state *ps, struct page *p) 892 { 893 unlock_page(p); 894 return MF_IGNORED; 895 } 896 897 /* 898 * Page in unknown state. Do nothing. 899 */ 900 static int me_unknown(struct page_state *ps, struct page *p) 901 { 902 pr_err("%#lx: Unknown page state\n", page_to_pfn(p)); 903 unlock_page(p); 904 return MF_FAILED; 905 } 906 907 /* 908 * Clean (or cleaned) page cache page. 909 */ 910 static int me_pagecache_clean(struct page_state *ps, struct page *p) 911 { 912 int ret; 913 struct address_space *mapping; 914 bool extra_pins; 915 916 delete_from_lru_cache(p); 917 918 /* 919 * For anonymous pages we're done the only reference left 920 * should be the one m_f() holds. 921 */ 922 if (PageAnon(p)) { 923 ret = MF_RECOVERED; 924 goto out; 925 } 926 927 /* 928 * Now truncate the page in the page cache. This is really 929 * more like a "temporary hole punch" 930 * Don't do this for block devices when someone else 931 * has a reference, because it could be file system metadata 932 * and that's not safe to truncate. 933 */ 934 mapping = page_mapping(p); 935 if (!mapping) { 936 /* 937 * Page has been teared down in the meanwhile 938 */ 939 ret = MF_FAILED; 940 goto out; 941 } 942 943 /* 944 * The shmem page is kept in page cache instead of truncating 945 * so is expected to have an extra refcount after error-handling. 946 */ 947 extra_pins = shmem_mapping(mapping); 948 949 /* 950 * Truncation is a bit tricky. Enable it per file system for now. 951 * 952 * Open: to take i_rwsem or not for this? Right now we don't. 953 */ 954 ret = truncate_error_page(p, page_to_pfn(p), mapping); 955 if (has_extra_refcount(ps, p, extra_pins)) 956 ret = MF_FAILED; 957 958 out: 959 unlock_page(p); 960 961 return ret; 962 } 963 964 /* 965 * Dirty pagecache page 966 * Issues: when the error hit a hole page the error is not properly 967 * propagated. 968 */ 969 static int me_pagecache_dirty(struct page_state *ps, struct page *p) 970 { 971 struct address_space *mapping = page_mapping(p); 972 973 SetPageError(p); 974 /* TBD: print more information about the file. */ 975 if (mapping) { 976 /* 977 * IO error will be reported by write(), fsync(), etc. 978 * who check the mapping. 979 * This way the application knows that something went 980 * wrong with its dirty file data. 981 * 982 * There's one open issue: 983 * 984 * The EIO will be only reported on the next IO 985 * operation and then cleared through the IO map. 986 * Normally Linux has two mechanisms to pass IO error 987 * first through the AS_EIO flag in the address space 988 * and then through the PageError flag in the page. 989 * Since we drop pages on memory failure handling the 990 * only mechanism open to use is through AS_AIO. 991 * 992 * This has the disadvantage that it gets cleared on 993 * the first operation that returns an error, while 994 * the PageError bit is more sticky and only cleared 995 * when the page is reread or dropped. If an 996 * application assumes it will always get error on 997 * fsync, but does other operations on the fd before 998 * and the page is dropped between then the error 999 * will not be properly reported. 1000 * 1001 * This can already happen even without hwpoisoned 1002 * pages: first on metadata IO errors (which only 1003 * report through AS_EIO) or when the page is dropped 1004 * at the wrong time. 1005 * 1006 * So right now we assume that the application DTRT on 1007 * the first EIO, but we're not worse than other parts 1008 * of the kernel. 1009 */ 1010 mapping_set_error(mapping, -EIO); 1011 } 1012 1013 return me_pagecache_clean(ps, p); 1014 } 1015 1016 /* 1017 * Clean and dirty swap cache. 1018 * 1019 * Dirty swap cache page is tricky to handle. The page could live both in page 1020 * cache and swap cache(ie. page is freshly swapped in). So it could be 1021 * referenced concurrently by 2 types of PTEs: 1022 * normal PTEs and swap PTEs. We try to handle them consistently by calling 1023 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs, 1024 * and then 1025 * - clear dirty bit to prevent IO 1026 * - remove from LRU 1027 * - but keep in the swap cache, so that when we return to it on 1028 * a later page fault, we know the application is accessing 1029 * corrupted data and shall be killed (we installed simple 1030 * interception code in do_swap_page to catch it). 1031 * 1032 * Clean swap cache pages can be directly isolated. A later page fault will 1033 * bring in the known good data from disk. 1034 */ 1035 static int me_swapcache_dirty(struct page_state *ps, struct page *p) 1036 { 1037 int ret; 1038 bool extra_pins = false; 1039 1040 ClearPageDirty(p); 1041 /* Trigger EIO in shmem: */ 1042 ClearPageUptodate(p); 1043 1044 ret = delete_from_lru_cache(p) ? MF_FAILED : MF_DELAYED; 1045 unlock_page(p); 1046 1047 if (ret == MF_DELAYED) 1048 extra_pins = true; 1049 1050 if (has_extra_refcount(ps, p, extra_pins)) 1051 ret = MF_FAILED; 1052 1053 return ret; 1054 } 1055 1056 static int me_swapcache_clean(struct page_state *ps, struct page *p) 1057 { 1058 struct folio *folio = page_folio(p); 1059 int ret; 1060 1061 delete_from_swap_cache(folio); 1062 1063 ret = delete_from_lru_cache(p) ? MF_FAILED : MF_RECOVERED; 1064 folio_unlock(folio); 1065 1066 if (has_extra_refcount(ps, p, false)) 1067 ret = MF_FAILED; 1068 1069 return ret; 1070 } 1071 1072 /* 1073 * Huge pages. Needs work. 1074 * Issues: 1075 * - Error on hugepage is contained in hugepage unit (not in raw page unit.) 1076 * To narrow down kill region to one page, we need to break up pmd. 1077 */ 1078 static int me_huge_page(struct page_state *ps, struct page *p) 1079 { 1080 int res; 1081 struct page *hpage = compound_head(p); 1082 struct address_space *mapping; 1083 bool extra_pins = false; 1084 1085 if (!PageHuge(hpage)) 1086 return MF_DELAYED; 1087 1088 mapping = page_mapping(hpage); 1089 if (mapping) { 1090 res = truncate_error_page(hpage, page_to_pfn(p), mapping); 1091 /* The page is kept in page cache. */ 1092 extra_pins = true; 1093 unlock_page(hpage); 1094 } else { 1095 unlock_page(hpage); 1096 /* 1097 * migration entry prevents later access on error hugepage, 1098 * so we can free and dissolve it into buddy to save healthy 1099 * subpages. 1100 */ 1101 put_page(hpage); 1102 if (__page_handle_poison(p) >= 0) { 1103 page_ref_inc(p); 1104 res = MF_RECOVERED; 1105 } else { 1106 res = MF_FAILED; 1107 } 1108 } 1109 1110 if (has_extra_refcount(ps, p, extra_pins)) 1111 res = MF_FAILED; 1112 1113 return res; 1114 } 1115 1116 /* 1117 * Various page states we can handle. 1118 * 1119 * A page state is defined by its current page->flags bits. 1120 * The table matches them in order and calls the right handler. 1121 * 1122 * This is quite tricky because we can access page at any time 1123 * in its live cycle, so all accesses have to be extremely careful. 1124 * 1125 * This is not complete. More states could be added. 1126 * For any missing state don't attempt recovery. 1127 */ 1128 1129 #define dirty (1UL << PG_dirty) 1130 #define sc ((1UL << PG_swapcache) | (1UL << PG_swapbacked)) 1131 #define unevict (1UL << PG_unevictable) 1132 #define mlock (1UL << PG_mlocked) 1133 #define lru (1UL << PG_lru) 1134 #define head (1UL << PG_head) 1135 #define slab (1UL << PG_slab) 1136 #define reserved (1UL << PG_reserved) 1137 1138 static struct page_state error_states[] = { 1139 { reserved, reserved, MF_MSG_KERNEL, me_kernel }, 1140 /* 1141 * free pages are specially detected outside this table: 1142 * PG_buddy pages only make a small fraction of all free pages. 1143 */ 1144 1145 /* 1146 * Could in theory check if slab page is free or if we can drop 1147 * currently unused objects without touching them. But just 1148 * treat it as standard kernel for now. 1149 */ 1150 { slab, slab, MF_MSG_SLAB, me_kernel }, 1151 1152 { head, head, MF_MSG_HUGE, me_huge_page }, 1153 1154 { sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty }, 1155 { sc|dirty, sc, MF_MSG_CLEAN_SWAPCACHE, me_swapcache_clean }, 1156 1157 { mlock|dirty, mlock|dirty, MF_MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty }, 1158 { mlock|dirty, mlock, MF_MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean }, 1159 1160 { unevict|dirty, unevict|dirty, MF_MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty }, 1161 { unevict|dirty, unevict, MF_MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean }, 1162 1163 { lru|dirty, lru|dirty, MF_MSG_DIRTY_LRU, me_pagecache_dirty }, 1164 { lru|dirty, lru, MF_MSG_CLEAN_LRU, me_pagecache_clean }, 1165 1166 /* 1167 * Catchall entry: must be at end. 1168 */ 1169 { 0, 0, MF_MSG_UNKNOWN, me_unknown }, 1170 }; 1171 1172 #undef dirty 1173 #undef sc 1174 #undef unevict 1175 #undef mlock 1176 #undef lru 1177 #undef head 1178 #undef slab 1179 #undef reserved 1180 1181 /* 1182 * "Dirty/Clean" indication is not 100% accurate due to the possibility of 1183 * setting PG_dirty outside page lock. See also comment above set_page_dirty(). 1184 */ 1185 static void action_result(unsigned long pfn, enum mf_action_page_type type, 1186 enum mf_result result) 1187 { 1188 trace_memory_failure_event(pfn, type, result); 1189 1190 num_poisoned_pages_inc(); 1191 pr_err("%#lx: recovery action for %s: %s\n", 1192 pfn, action_page_types[type], action_name[result]); 1193 } 1194 1195 static int page_action(struct page_state *ps, struct page *p, 1196 unsigned long pfn) 1197 { 1198 int result; 1199 1200 /* page p should be unlocked after returning from ps->action(). */ 1201 result = ps->action(ps, p); 1202 1203 action_result(pfn, ps->type, result); 1204 1205 /* Could do more checks here if page looks ok */ 1206 /* 1207 * Could adjust zone counters here to correct for the missing page. 1208 */ 1209 1210 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; 1211 } 1212 1213 static inline bool PageHWPoisonTakenOff(struct page *page) 1214 { 1215 return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON; 1216 } 1217 1218 void SetPageHWPoisonTakenOff(struct page *page) 1219 { 1220 set_page_private(page, MAGIC_HWPOISON); 1221 } 1222 1223 void ClearPageHWPoisonTakenOff(struct page *page) 1224 { 1225 if (PageHWPoison(page)) 1226 set_page_private(page, 0); 1227 } 1228 1229 /* 1230 * Return true if a page type of a given page is supported by hwpoison 1231 * mechanism (while handling could fail), otherwise false. This function 1232 * does not return true for hugetlb or device memory pages, so it's assumed 1233 * to be called only in the context where we never have such pages. 1234 */ 1235 static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) 1236 { 1237 /* Soft offline could migrate non-LRU movable pages */ 1238 if ((flags & MF_SOFT_OFFLINE) && __PageMovable(page)) 1239 return true; 1240 1241 return PageLRU(page) || is_free_buddy_page(page); 1242 } 1243 1244 static int __get_hwpoison_page(struct page *page, unsigned long flags) 1245 { 1246 struct page *head = compound_head(page); 1247 int ret = 0; 1248 bool hugetlb = false; 1249 1250 ret = get_hwpoison_huge_page(head, &hugetlb); 1251 if (hugetlb) 1252 return ret; 1253 1254 /* 1255 * This check prevents from calling get_page_unless_zero() for any 1256 * unsupported type of page in order to reduce the risk of unexpected 1257 * races caused by taking a page refcount. 1258 */ 1259 if (!HWPoisonHandlable(head, flags)) 1260 return -EBUSY; 1261 1262 if (get_page_unless_zero(head)) { 1263 if (head == compound_head(page)) 1264 return 1; 1265 1266 pr_info("%#lx cannot catch tail\n", page_to_pfn(page)); 1267 put_page(head); 1268 } 1269 1270 return 0; 1271 } 1272 1273 static int get_any_page(struct page *p, unsigned long flags) 1274 { 1275 int ret = 0, pass = 0; 1276 bool count_increased = false; 1277 1278 if (flags & MF_COUNT_INCREASED) 1279 count_increased = true; 1280 1281 try_again: 1282 if (!count_increased) { 1283 ret = __get_hwpoison_page(p, flags); 1284 if (!ret) { 1285 if (page_count(p)) { 1286 /* We raced with an allocation, retry. */ 1287 if (pass++ < 3) 1288 goto try_again; 1289 ret = -EBUSY; 1290 } else if (!PageHuge(p) && !is_free_buddy_page(p)) { 1291 /* We raced with put_page, retry. */ 1292 if (pass++ < 3) 1293 goto try_again; 1294 ret = -EIO; 1295 } 1296 goto out; 1297 } else if (ret == -EBUSY) { 1298 /* 1299 * We raced with (possibly temporary) unhandlable 1300 * page, retry. 1301 */ 1302 if (pass++ < 3) { 1303 shake_page(p); 1304 goto try_again; 1305 } 1306 ret = -EIO; 1307 goto out; 1308 } 1309 } 1310 1311 if (PageHuge(p) || HWPoisonHandlable(p, flags)) { 1312 ret = 1; 1313 } else { 1314 /* 1315 * A page we cannot handle. Check whether we can turn 1316 * it into something we can handle. 1317 */ 1318 if (pass++ < 3) { 1319 put_page(p); 1320 shake_page(p); 1321 count_increased = false; 1322 goto try_again; 1323 } 1324 put_page(p); 1325 ret = -EIO; 1326 } 1327 out: 1328 if (ret == -EIO) 1329 pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); 1330 1331 return ret; 1332 } 1333 1334 static int __get_unpoison_page(struct page *page) 1335 { 1336 struct page *head = compound_head(page); 1337 int ret = 0; 1338 bool hugetlb = false; 1339 1340 ret = get_hwpoison_huge_page(head, &hugetlb); 1341 if (hugetlb) 1342 return ret; 1343 1344 /* 1345 * PageHWPoisonTakenOff pages are not only marked as PG_hwpoison, 1346 * but also isolated from buddy freelist, so need to identify the 1347 * state and have to cancel both operations to unpoison. 1348 */ 1349 if (PageHWPoisonTakenOff(page)) 1350 return -EHWPOISON; 1351 1352 return get_page_unless_zero(page) ? 1 : 0; 1353 } 1354 1355 /** 1356 * get_hwpoison_page() - Get refcount for memory error handling 1357 * @p: Raw error page (hit by memory error) 1358 * @flags: Flags controlling behavior of error handling 1359 * 1360 * get_hwpoison_page() takes a page refcount of an error page to handle memory 1361 * error on it, after checking that the error page is in a well-defined state 1362 * (defined as a page-type we can successfully handle the memory error on it, 1363 * such as LRU page and hugetlb page). 1364 * 1365 * Memory error handling could be triggered at any time on any type of page, 1366 * so it's prone to race with typical memory management lifecycle (like 1367 * allocation and free). So to avoid such races, get_hwpoison_page() takes 1368 * extra care for the error page's state (as done in __get_hwpoison_page()), 1369 * and has some retry logic in get_any_page(). 1370 * 1371 * When called from unpoison_memory(), the caller should already ensure that 1372 * the given page has PG_hwpoison. So it's never reused for other page 1373 * allocations, and __get_unpoison_page() never races with them. 1374 * 1375 * Return: 0 on failure, 1376 * 1 on success for in-use pages in a well-defined state, 1377 * -EIO for pages on which we can not handle memory errors, 1378 * -EBUSY when get_hwpoison_page() has raced with page lifecycle 1379 * operations like allocation and free, 1380 * -EHWPOISON when the page is hwpoisoned and taken off from buddy. 1381 */ 1382 static int get_hwpoison_page(struct page *p, unsigned long flags) 1383 { 1384 int ret; 1385 1386 zone_pcp_disable(page_zone(p)); 1387 if (flags & MF_UNPOISON) 1388 ret = __get_unpoison_page(p); 1389 else 1390 ret = get_any_page(p, flags); 1391 zone_pcp_enable(page_zone(p)); 1392 1393 return ret; 1394 } 1395 1396 /* 1397 * Do all that is necessary to remove user space mappings. Unmap 1398 * the pages and send SIGBUS to the processes if the data was dirty. 1399 */ 1400 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, 1401 int flags, struct page *hpage) 1402 { 1403 struct folio *folio = page_folio(hpage); 1404 enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC; 1405 struct address_space *mapping; 1406 LIST_HEAD(tokill); 1407 bool unmap_success; 1408 int forcekill; 1409 bool mlocked = PageMlocked(hpage); 1410 1411 /* 1412 * Here we are interested only in user-mapped pages, so skip any 1413 * other types of pages. 1414 */ 1415 if (PageReserved(p) || PageSlab(p) || PageTable(p)) 1416 return true; 1417 if (!(PageLRU(hpage) || PageHuge(p))) 1418 return true; 1419 1420 /* 1421 * This check implies we don't kill processes if their pages 1422 * are in the swap cache early. Those are always late kills. 1423 */ 1424 if (!page_mapped(hpage)) 1425 return true; 1426 1427 if (PageKsm(p)) { 1428 pr_err("%#lx: can't handle KSM pages.\n", pfn); 1429 return false; 1430 } 1431 1432 if (PageSwapCache(p)) { 1433 pr_err("%#lx: keeping poisoned page in swap cache\n", pfn); 1434 ttu |= TTU_IGNORE_HWPOISON; 1435 } 1436 1437 /* 1438 * Propagate the dirty bit from PTEs to struct page first, because we 1439 * need this to decide if we should kill or just drop the page. 1440 * XXX: the dirty test could be racy: set_page_dirty() may not always 1441 * be called inside page lock (it's recommended but not enforced). 1442 */ 1443 mapping = page_mapping(hpage); 1444 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping && 1445 mapping_can_writeback(mapping)) { 1446 if (page_mkclean(hpage)) { 1447 SetPageDirty(hpage); 1448 } else { 1449 ttu |= TTU_IGNORE_HWPOISON; 1450 pr_info("%#lx: corrupted page was clean: dropped without side effects\n", 1451 pfn); 1452 } 1453 } 1454 1455 /* 1456 * First collect all the processes that have the page 1457 * mapped in dirty form. This has to be done before try_to_unmap, 1458 * because ttu takes the rmap data structures down. 1459 */ 1460 collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED); 1461 1462 if (PageHuge(hpage) && !PageAnon(hpage)) { 1463 /* 1464 * For hugetlb pages in shared mappings, try_to_unmap 1465 * could potentially call huge_pmd_unshare. Because of 1466 * this, take semaphore in write mode here and set 1467 * TTU_RMAP_LOCKED to indicate we have taken the lock 1468 * at this higher level. 1469 */ 1470 mapping = hugetlb_page_mapping_lock_write(hpage); 1471 if (mapping) { 1472 try_to_unmap(folio, ttu|TTU_RMAP_LOCKED); 1473 i_mmap_unlock_write(mapping); 1474 } else 1475 pr_info("%#lx: could not lock mapping for mapped huge page\n", pfn); 1476 } else { 1477 try_to_unmap(folio, ttu); 1478 } 1479 1480 unmap_success = !page_mapped(hpage); 1481 if (!unmap_success) 1482 pr_err("%#lx: failed to unmap page (mapcount=%d)\n", 1483 pfn, page_mapcount(hpage)); 1484 1485 /* 1486 * try_to_unmap() might put mlocked page in lru cache, so call 1487 * shake_page() again to ensure that it's flushed. 1488 */ 1489 if (mlocked) 1490 shake_page(hpage); 1491 1492 /* 1493 * Now that the dirty bit has been propagated to the 1494 * struct page and all unmaps done we can decide if 1495 * killing is needed or not. Only kill when the page 1496 * was dirty or the process is not restartable, 1497 * otherwise the tokill list is merely 1498 * freed. When there was a problem unmapping earlier 1499 * use a more force-full uncatchable kill to prevent 1500 * any accesses to the poisoned memory. 1501 */ 1502 forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL) || 1503 !unmap_success; 1504 kill_procs(&tokill, forcekill, !unmap_success, pfn, flags); 1505 1506 return unmap_success; 1507 } 1508 1509 static int identify_page_state(unsigned long pfn, struct page *p, 1510 unsigned long page_flags) 1511 { 1512 struct page_state *ps; 1513 1514 /* 1515 * The first check uses the current page flags which may not have any 1516 * relevant information. The second check with the saved page flags is 1517 * carried out only if the first check can't determine the page status. 1518 */ 1519 for (ps = error_states;; ps++) 1520 if ((p->flags & ps->mask) == ps->res) 1521 break; 1522 1523 page_flags |= (p->flags & (1UL << PG_dirty)); 1524 1525 if (!ps->mask) 1526 for (ps = error_states;; ps++) 1527 if ((page_flags & ps->mask) == ps->res) 1528 break; 1529 return page_action(ps, p, pfn); 1530 } 1531 1532 static int try_to_split_thp_page(struct page *page) 1533 { 1534 int ret; 1535 1536 lock_page(page); 1537 ret = split_huge_page(page); 1538 unlock_page(page); 1539 1540 if (unlikely(ret)) 1541 put_page(page); 1542 1543 return ret; 1544 } 1545 1546 static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn, 1547 struct address_space *mapping, pgoff_t index, int flags) 1548 { 1549 struct to_kill *tk; 1550 unsigned long size = 0; 1551 1552 list_for_each_entry(tk, to_kill, nd) 1553 if (tk->size_shift) 1554 size = max(size, 1UL << tk->size_shift); 1555 1556 if (size) { 1557 /* 1558 * Unmap the largest mapping to avoid breaking up device-dax 1559 * mappings which are constant size. The actual size of the 1560 * mapping being torn down is communicated in siginfo, see 1561 * kill_proc() 1562 */ 1563 loff_t start = (index << PAGE_SHIFT) & ~(size - 1); 1564 1565 unmap_mapping_range(mapping, start, size, 0); 1566 } 1567 1568 kill_procs(to_kill, flags & MF_MUST_KILL, false, pfn, flags); 1569 } 1570 1571 static int mf_generic_kill_procs(unsigned long long pfn, int flags, 1572 struct dev_pagemap *pgmap) 1573 { 1574 struct page *page = pfn_to_page(pfn); 1575 LIST_HEAD(to_kill); 1576 dax_entry_t cookie; 1577 int rc = 0; 1578 1579 /* 1580 * Pages instantiated by device-dax (not filesystem-dax) 1581 * may be compound pages. 1582 */ 1583 page = compound_head(page); 1584 1585 /* 1586 * Prevent the inode from being freed while we are interrogating 1587 * the address_space, typically this would be handled by 1588 * lock_page(), but dax pages do not use the page lock. This 1589 * also prevents changes to the mapping of this pfn until 1590 * poison signaling is complete. 1591 */ 1592 cookie = dax_lock_page(page); 1593 if (!cookie) 1594 return -EBUSY; 1595 1596 if (hwpoison_filter(page)) { 1597 rc = -EOPNOTSUPP; 1598 goto unlock; 1599 } 1600 1601 switch (pgmap->type) { 1602 case MEMORY_DEVICE_PRIVATE: 1603 case MEMORY_DEVICE_COHERENT: 1604 /* 1605 * TODO: Handle device pages which may need coordination 1606 * with device-side memory. 1607 */ 1608 rc = -ENXIO; 1609 goto unlock; 1610 default: 1611 break; 1612 } 1613 1614 /* 1615 * Use this flag as an indication that the dax page has been 1616 * remapped UC to prevent speculative consumption of poison. 1617 */ 1618 SetPageHWPoison(page); 1619 1620 /* 1621 * Unlike System-RAM there is no possibility to swap in a 1622 * different physical page at a given virtual address, so all 1623 * userspace consumption of ZONE_DEVICE memory necessitates 1624 * SIGBUS (i.e. MF_MUST_KILL) 1625 */ 1626 flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; 1627 collect_procs(page, &to_kill, true); 1628 1629 unmap_and_kill(&to_kill, pfn, page->mapping, page->index, flags); 1630 unlock: 1631 dax_unlock_page(page, cookie); 1632 return rc; 1633 } 1634 1635 #ifdef CONFIG_FS_DAX 1636 /** 1637 * mf_dax_kill_procs - Collect and kill processes who are using this file range 1638 * @mapping: address_space of the file in use 1639 * @index: start pgoff of the range within the file 1640 * @count: length of the range, in unit of PAGE_SIZE 1641 * @mf_flags: memory failure flags 1642 */ 1643 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, 1644 unsigned long count, int mf_flags) 1645 { 1646 LIST_HEAD(to_kill); 1647 dax_entry_t cookie; 1648 struct page *page; 1649 size_t end = index + count; 1650 1651 mf_flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; 1652 1653 for (; index < end; index++) { 1654 page = NULL; 1655 cookie = dax_lock_mapping_entry(mapping, index, &page); 1656 if (!cookie) 1657 return -EBUSY; 1658 if (!page) 1659 goto unlock; 1660 1661 SetPageHWPoison(page); 1662 1663 collect_procs_fsdax(page, mapping, index, &to_kill); 1664 unmap_and_kill(&to_kill, page_to_pfn(page), mapping, 1665 index, mf_flags); 1666 unlock: 1667 dax_unlock_mapping_entry(mapping, index, cookie); 1668 } 1669 return 0; 1670 } 1671 EXPORT_SYMBOL_GPL(mf_dax_kill_procs); 1672 #endif /* CONFIG_FS_DAX */ 1673 1674 #ifdef CONFIG_HUGETLB_PAGE 1675 /* 1676 * Struct raw_hwp_page represents information about "raw error page", 1677 * constructing singly linked list originated from ->private field of 1678 * SUBPAGE_INDEX_HWPOISON-th tail page. 1679 */ 1680 struct raw_hwp_page { 1681 struct llist_node node; 1682 struct page *page; 1683 }; 1684 1685 static inline struct llist_head *raw_hwp_list_head(struct page *hpage) 1686 { 1687 return (struct llist_head *)&page_private(hpage + SUBPAGE_INDEX_HWPOISON); 1688 } 1689 1690 static unsigned long __free_raw_hwp_pages(struct page *hpage, bool move_flag) 1691 { 1692 struct llist_head *head; 1693 struct llist_node *t, *tnode; 1694 unsigned long count = 0; 1695 1696 head = raw_hwp_list_head(hpage); 1697 llist_for_each_safe(tnode, t, head->first) { 1698 struct raw_hwp_page *p = container_of(tnode, struct raw_hwp_page, node); 1699 1700 if (move_flag) 1701 SetPageHWPoison(p->page); 1702 kfree(p); 1703 count++; 1704 } 1705 llist_del_all(head); 1706 return count; 1707 } 1708 1709 static int hugetlb_set_page_hwpoison(struct page *hpage, struct page *page) 1710 { 1711 struct llist_head *head; 1712 struct raw_hwp_page *raw_hwp; 1713 struct llist_node *t, *tnode; 1714 int ret = TestSetPageHWPoison(hpage) ? -EHWPOISON : 0; 1715 1716 /* 1717 * Once the hwpoison hugepage has lost reliable raw error info, 1718 * there is little meaning to keep additional error info precisely, 1719 * so skip to add additional raw error info. 1720 */ 1721 if (HPageRawHwpUnreliable(hpage)) 1722 return -EHWPOISON; 1723 head = raw_hwp_list_head(hpage); 1724 llist_for_each_safe(tnode, t, head->first) { 1725 struct raw_hwp_page *p = container_of(tnode, struct raw_hwp_page, node); 1726 1727 if (p->page == page) 1728 return -EHWPOISON; 1729 } 1730 1731 raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC); 1732 if (raw_hwp) { 1733 raw_hwp->page = page; 1734 llist_add(&raw_hwp->node, head); 1735 /* the first error event will be counted in action_result(). */ 1736 if (ret) 1737 num_poisoned_pages_inc(); 1738 } else { 1739 /* 1740 * Failed to save raw error info. We no longer trace all 1741 * hwpoisoned subpages, and we need refuse to free/dissolve 1742 * this hwpoisoned hugepage. 1743 */ 1744 SetHPageRawHwpUnreliable(hpage); 1745 /* 1746 * Once HPageRawHwpUnreliable is set, raw_hwp_page is not 1747 * used any more, so free it. 1748 */ 1749 __free_raw_hwp_pages(hpage, false); 1750 } 1751 return ret; 1752 } 1753 1754 static unsigned long free_raw_hwp_pages(struct page *hpage, bool move_flag) 1755 { 1756 /* 1757 * HPageVmemmapOptimized hugepages can't be freed because struct 1758 * pages for tail pages are required but they don't exist. 1759 */ 1760 if (move_flag && HPageVmemmapOptimized(hpage)) 1761 return 0; 1762 1763 /* 1764 * HPageRawHwpUnreliable hugepages shouldn't be unpoisoned by 1765 * definition. 1766 */ 1767 if (HPageRawHwpUnreliable(hpage)) 1768 return 0; 1769 1770 return __free_raw_hwp_pages(hpage, move_flag); 1771 } 1772 1773 void hugetlb_clear_page_hwpoison(struct page *hpage) 1774 { 1775 if (HPageRawHwpUnreliable(hpage)) 1776 return; 1777 ClearPageHWPoison(hpage); 1778 free_raw_hwp_pages(hpage, true); 1779 } 1780 1781 /* 1782 * Called from hugetlb code with hugetlb_lock held. 1783 * 1784 * Return values: 1785 * 0 - free hugepage 1786 * 1 - in-use hugepage 1787 * 2 - not a hugepage 1788 * -EBUSY - the hugepage is busy (try to retry) 1789 * -EHWPOISON - the hugepage is already hwpoisoned 1790 */ 1791 int __get_huge_page_for_hwpoison(unsigned long pfn, int flags) 1792 { 1793 struct page *page = pfn_to_page(pfn); 1794 struct page *head = compound_head(page); 1795 int ret = 2; /* fallback to normal page handling */ 1796 bool count_increased = false; 1797 1798 if (!PageHeadHuge(head)) 1799 goto out; 1800 1801 if (flags & MF_COUNT_INCREASED) { 1802 ret = 1; 1803 count_increased = true; 1804 } else if (HPageFreed(head)) { 1805 ret = 0; 1806 } else if (HPageMigratable(head)) { 1807 ret = get_page_unless_zero(head); 1808 if (ret) 1809 count_increased = true; 1810 } else { 1811 ret = -EBUSY; 1812 if (!(flags & MF_NO_RETRY)) 1813 goto out; 1814 } 1815 1816 if (hugetlb_set_page_hwpoison(head, page)) { 1817 ret = -EHWPOISON; 1818 goto out; 1819 } 1820 1821 return ret; 1822 out: 1823 if (count_increased) 1824 put_page(head); 1825 return ret; 1826 } 1827 1828 /* 1829 * Taking refcount of hugetlb pages needs extra care about race conditions 1830 * with basic operations like hugepage allocation/free/demotion. 1831 * So some of prechecks for hwpoison (pinning, and testing/setting 1832 * PageHWPoison) should be done in single hugetlb_lock range. 1833 */ 1834 static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb) 1835 { 1836 int res; 1837 struct page *p = pfn_to_page(pfn); 1838 struct page *head; 1839 unsigned long page_flags; 1840 1841 *hugetlb = 1; 1842 retry: 1843 res = get_huge_page_for_hwpoison(pfn, flags); 1844 if (res == 2) { /* fallback to normal page handling */ 1845 *hugetlb = 0; 1846 return 0; 1847 } else if (res == -EHWPOISON) { 1848 pr_err("%#lx: already hardware poisoned\n", pfn); 1849 if (flags & MF_ACTION_REQUIRED) { 1850 head = compound_head(p); 1851 res = kill_accessing_process(current, page_to_pfn(head), flags); 1852 } 1853 return res; 1854 } else if (res == -EBUSY) { 1855 if (!(flags & MF_NO_RETRY)) { 1856 flags |= MF_NO_RETRY; 1857 goto retry; 1858 } 1859 action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED); 1860 return res; 1861 } 1862 1863 head = compound_head(p); 1864 lock_page(head); 1865 1866 if (hwpoison_filter(p)) { 1867 hugetlb_clear_page_hwpoison(head); 1868 unlock_page(head); 1869 if (res == 1) 1870 put_page(head); 1871 return -EOPNOTSUPP; 1872 } 1873 1874 /* 1875 * Handling free hugepage. The possible race with hugepage allocation 1876 * or demotion can be prevented by PageHWPoison flag. 1877 */ 1878 if (res == 0) { 1879 unlock_page(head); 1880 if (__page_handle_poison(p) >= 0) { 1881 page_ref_inc(p); 1882 res = MF_RECOVERED; 1883 } else { 1884 res = MF_FAILED; 1885 } 1886 action_result(pfn, MF_MSG_FREE_HUGE, res); 1887 return res == MF_RECOVERED ? 0 : -EBUSY; 1888 } 1889 1890 page_flags = head->flags; 1891 1892 if (!hwpoison_user_mappings(p, pfn, flags, head)) { 1893 action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); 1894 res = -EBUSY; 1895 goto out; 1896 } 1897 1898 return identify_page_state(pfn, p, page_flags); 1899 out: 1900 unlock_page(head); 1901 return res; 1902 } 1903 1904 #else 1905 static inline int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb) 1906 { 1907 return 0; 1908 } 1909 1910 static inline unsigned long free_raw_hwp_pages(struct page *hpage, bool flag) 1911 { 1912 return 0; 1913 } 1914 #endif /* CONFIG_HUGETLB_PAGE */ 1915 1916 static int memory_failure_dev_pagemap(unsigned long pfn, int flags, 1917 struct dev_pagemap *pgmap) 1918 { 1919 struct page *page = pfn_to_page(pfn); 1920 int rc = -ENXIO; 1921 1922 if (flags & MF_COUNT_INCREASED) 1923 /* 1924 * Drop the extra refcount in case we come from madvise(). 1925 */ 1926 put_page(page); 1927 1928 /* device metadata space is not recoverable */ 1929 if (!pgmap_pfn_valid(pgmap, pfn)) 1930 goto out; 1931 1932 /* 1933 * Call driver's implementation to handle the memory failure, otherwise 1934 * fall back to generic handler. 1935 */ 1936 if (pgmap_has_memory_failure(pgmap)) { 1937 rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); 1938 /* 1939 * Fall back to generic handler too if operation is not 1940 * supported inside the driver/device/filesystem. 1941 */ 1942 if (rc != -EOPNOTSUPP) 1943 goto out; 1944 } 1945 1946 rc = mf_generic_kill_procs(pfn, flags, pgmap); 1947 out: 1948 /* drop pgmap ref acquired in caller */ 1949 put_dev_pagemap(pgmap); 1950 action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED); 1951 return rc; 1952 } 1953 1954 static DEFINE_MUTEX(mf_mutex); 1955 1956 /** 1957 * memory_failure - Handle memory failure of a page. 1958 * @pfn: Page Number of the corrupted page 1959 * @flags: fine tune action taken 1960 * 1961 * This function is called by the low level machine check code 1962 * of an architecture when it detects hardware memory corruption 1963 * of a page. It tries its best to recover, which includes 1964 * dropping pages, killing processes etc. 1965 * 1966 * The function is primarily of use for corruptions that 1967 * happen outside the current execution context (e.g. when 1968 * detected by a background scrubber) 1969 * 1970 * Must run in process context (e.g. a work queue) with interrupts 1971 * enabled and no spinlocks hold. 1972 * 1973 * Return: 0 for successfully handled the memory error, 1974 * -EOPNOTSUPP for hwpoison_filter() filtered the error event, 1975 * < 0(except -EOPNOTSUPP) on failure. 1976 */ 1977 int memory_failure(unsigned long pfn, int flags) 1978 { 1979 struct page *p; 1980 struct page *hpage; 1981 struct dev_pagemap *pgmap; 1982 int res = 0; 1983 unsigned long page_flags; 1984 bool retry = true; 1985 int hugetlb = 0; 1986 1987 if (!sysctl_memory_failure_recovery) 1988 panic("Memory failure on page %lx", pfn); 1989 1990 mutex_lock(&mf_mutex); 1991 1992 if (!(flags & MF_SW_SIMULATED)) 1993 hw_memory_failure = true; 1994 1995 p = pfn_to_online_page(pfn); 1996 if (!p) { 1997 res = arch_memory_failure(pfn, flags); 1998 if (res == 0) 1999 goto unlock_mutex; 2000 2001 if (pfn_valid(pfn)) { 2002 pgmap = get_dev_pagemap(pfn, NULL); 2003 if (pgmap) { 2004 res = memory_failure_dev_pagemap(pfn, flags, 2005 pgmap); 2006 goto unlock_mutex; 2007 } 2008 } 2009 pr_err("%#lx: memory outside kernel control\n", pfn); 2010 res = -ENXIO; 2011 goto unlock_mutex; 2012 } 2013 2014 try_again: 2015 res = try_memory_failure_hugetlb(pfn, flags, &hugetlb); 2016 if (hugetlb) 2017 goto unlock_mutex; 2018 2019 if (TestSetPageHWPoison(p)) { 2020 pr_err("%#lx: already hardware poisoned\n", pfn); 2021 res = -EHWPOISON; 2022 if (flags & MF_ACTION_REQUIRED) 2023 res = kill_accessing_process(current, pfn, flags); 2024 if (flags & MF_COUNT_INCREASED) 2025 put_page(p); 2026 goto unlock_mutex; 2027 } 2028 2029 hpage = compound_head(p); 2030 2031 /* 2032 * We need/can do nothing about count=0 pages. 2033 * 1) it's a free page, and therefore in safe hand: 2034 * check_new_page() will be the gate keeper. 2035 * 2) it's part of a non-compound high order page. 2036 * Implies some kernel user: cannot stop them from 2037 * R/W the page; let's pray that the page has been 2038 * used and will be freed some time later. 2039 * In fact it's dangerous to directly bump up page count from 0, 2040 * that may make page_ref_freeze()/page_ref_unfreeze() mismatch. 2041 */ 2042 if (!(flags & MF_COUNT_INCREASED)) { 2043 res = get_hwpoison_page(p, flags); 2044 if (!res) { 2045 if (is_free_buddy_page(p)) { 2046 if (take_page_off_buddy(p)) { 2047 page_ref_inc(p); 2048 res = MF_RECOVERED; 2049 } else { 2050 /* We lost the race, try again */ 2051 if (retry) { 2052 ClearPageHWPoison(p); 2053 retry = false; 2054 goto try_again; 2055 } 2056 res = MF_FAILED; 2057 } 2058 action_result(pfn, MF_MSG_BUDDY, res); 2059 res = res == MF_RECOVERED ? 0 : -EBUSY; 2060 } else { 2061 action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); 2062 res = -EBUSY; 2063 } 2064 goto unlock_mutex; 2065 } else if (res < 0) { 2066 action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED); 2067 res = -EBUSY; 2068 goto unlock_mutex; 2069 } 2070 } 2071 2072 if (PageTransHuge(hpage)) { 2073 /* 2074 * The flag must be set after the refcount is bumped 2075 * otherwise it may race with THP split. 2076 * And the flag can't be set in get_hwpoison_page() since 2077 * it is called by soft offline too and it is just called 2078 * for !MF_COUNT_INCREASE. So here seems to be the best 2079 * place. 2080 * 2081 * Don't need care about the above error handling paths for 2082 * get_hwpoison_page() since they handle either free page 2083 * or unhandlable page. The refcount is bumped iff the 2084 * page is a valid handlable page. 2085 */ 2086 SetPageHasHWPoisoned(hpage); 2087 if (try_to_split_thp_page(p) < 0) { 2088 action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED); 2089 res = -EBUSY; 2090 goto unlock_mutex; 2091 } 2092 VM_BUG_ON_PAGE(!page_count(p), p); 2093 } 2094 2095 /* 2096 * We ignore non-LRU pages for good reasons. 2097 * - PG_locked is only well defined for LRU pages and a few others 2098 * - to avoid races with __SetPageLocked() 2099 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) 2100 * The check (unnecessarily) ignores LRU pages being isolated and 2101 * walked by the page reclaim code, however that's not a big loss. 2102 */ 2103 shake_page(p); 2104 2105 lock_page(p); 2106 2107 /* 2108 * We're only intended to deal with the non-Compound page here. 2109 * However, the page could have changed compound pages due to 2110 * race window. If this happens, we could try again to hopefully 2111 * handle the page next round. 2112 */ 2113 if (PageCompound(p)) { 2114 if (retry) { 2115 ClearPageHWPoison(p); 2116 unlock_page(p); 2117 put_page(p); 2118 flags &= ~MF_COUNT_INCREASED; 2119 retry = false; 2120 goto try_again; 2121 } 2122 action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED); 2123 res = -EBUSY; 2124 goto unlock_page; 2125 } 2126 2127 /* 2128 * We use page flags to determine what action should be taken, but 2129 * the flags can be modified by the error containment action. One 2130 * example is an mlocked page, where PG_mlocked is cleared by 2131 * page_remove_rmap() in try_to_unmap_one(). So to determine page status 2132 * correctly, we save a copy of the page flags at this time. 2133 */ 2134 page_flags = p->flags; 2135 2136 if (hwpoison_filter(p)) { 2137 ClearPageHWPoison(p); 2138 unlock_page(p); 2139 put_page(p); 2140 res = -EOPNOTSUPP; 2141 goto unlock_mutex; 2142 } 2143 2144 /* 2145 * __munlock_pagevec may clear a writeback page's LRU flag without 2146 * page_lock. We need wait writeback completion for this page or it 2147 * may trigger vfs BUG while evict inode. 2148 */ 2149 if (!PageLRU(p) && !PageWriteback(p)) 2150 goto identify_page_state; 2151 2152 /* 2153 * It's very difficult to mess with pages currently under IO 2154 * and in many cases impossible, so we just avoid it here. 2155 */ 2156 wait_on_page_writeback(p); 2157 2158 /* 2159 * Now take care of user space mappings. 2160 * Abort on fail: __filemap_remove_folio() assumes unmapped page. 2161 */ 2162 if (!hwpoison_user_mappings(p, pfn, flags, p)) { 2163 action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); 2164 res = -EBUSY; 2165 goto unlock_page; 2166 } 2167 2168 /* 2169 * Torn down by someone else? 2170 */ 2171 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { 2172 action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED); 2173 res = -EBUSY; 2174 goto unlock_page; 2175 } 2176 2177 identify_page_state: 2178 res = identify_page_state(pfn, p, page_flags); 2179 mutex_unlock(&mf_mutex); 2180 return res; 2181 unlock_page: 2182 unlock_page(p); 2183 unlock_mutex: 2184 mutex_unlock(&mf_mutex); 2185 return res; 2186 } 2187 EXPORT_SYMBOL_GPL(memory_failure); 2188 2189 #define MEMORY_FAILURE_FIFO_ORDER 4 2190 #define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER) 2191 2192 struct memory_failure_entry { 2193 unsigned long pfn; 2194 int flags; 2195 }; 2196 2197 struct memory_failure_cpu { 2198 DECLARE_KFIFO(fifo, struct memory_failure_entry, 2199 MEMORY_FAILURE_FIFO_SIZE); 2200 spinlock_t lock; 2201 struct work_struct work; 2202 }; 2203 2204 static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu); 2205 2206 /** 2207 * memory_failure_queue - Schedule handling memory failure of a page. 2208 * @pfn: Page Number of the corrupted page 2209 * @flags: Flags for memory failure handling 2210 * 2211 * This function is called by the low level hardware error handler 2212 * when it detects hardware memory corruption of a page. It schedules 2213 * the recovering of error page, including dropping pages, killing 2214 * processes etc. 2215 * 2216 * The function is primarily of use for corruptions that 2217 * happen outside the current execution context (e.g. when 2218 * detected by a background scrubber) 2219 * 2220 * Can run in IRQ context. 2221 */ 2222 void memory_failure_queue(unsigned long pfn, int flags) 2223 { 2224 struct memory_failure_cpu *mf_cpu; 2225 unsigned long proc_flags; 2226 struct memory_failure_entry entry = { 2227 .pfn = pfn, 2228 .flags = flags, 2229 }; 2230 2231 mf_cpu = &get_cpu_var(memory_failure_cpu); 2232 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 2233 if (kfifo_put(&mf_cpu->fifo, entry)) 2234 schedule_work_on(smp_processor_id(), &mf_cpu->work); 2235 else 2236 pr_err("buffer overflow when queuing memory failure at %#lx\n", 2237 pfn); 2238 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 2239 put_cpu_var(memory_failure_cpu); 2240 } 2241 EXPORT_SYMBOL_GPL(memory_failure_queue); 2242 2243 static void memory_failure_work_func(struct work_struct *work) 2244 { 2245 struct memory_failure_cpu *mf_cpu; 2246 struct memory_failure_entry entry = { 0, }; 2247 unsigned long proc_flags; 2248 int gotten; 2249 2250 mf_cpu = container_of(work, struct memory_failure_cpu, work); 2251 for (;;) { 2252 spin_lock_irqsave(&mf_cpu->lock, proc_flags); 2253 gotten = kfifo_get(&mf_cpu->fifo, &entry); 2254 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 2255 if (!gotten) 2256 break; 2257 if (entry.flags & MF_SOFT_OFFLINE) 2258 soft_offline_page(entry.pfn, entry.flags); 2259 else 2260 memory_failure(entry.pfn, entry.flags); 2261 } 2262 } 2263 2264 /* 2265 * Process memory_failure work queued on the specified CPU. 2266 * Used to avoid return-to-userspace racing with the memory_failure workqueue. 2267 */ 2268 void memory_failure_queue_kick(int cpu) 2269 { 2270 struct memory_failure_cpu *mf_cpu; 2271 2272 mf_cpu = &per_cpu(memory_failure_cpu, cpu); 2273 cancel_work_sync(&mf_cpu->work); 2274 memory_failure_work_func(&mf_cpu->work); 2275 } 2276 2277 static int __init memory_failure_init(void) 2278 { 2279 struct memory_failure_cpu *mf_cpu; 2280 int cpu; 2281 2282 for_each_possible_cpu(cpu) { 2283 mf_cpu = &per_cpu(memory_failure_cpu, cpu); 2284 spin_lock_init(&mf_cpu->lock); 2285 INIT_KFIFO(mf_cpu->fifo); 2286 INIT_WORK(&mf_cpu->work, memory_failure_work_func); 2287 } 2288 2289 return 0; 2290 } 2291 core_initcall(memory_failure_init); 2292 2293 #undef pr_fmt 2294 #define pr_fmt(fmt) "" fmt 2295 #define unpoison_pr_info(fmt, pfn, rs) \ 2296 ({ \ 2297 if (__ratelimit(rs)) \ 2298 pr_info(fmt, pfn); \ 2299 }) 2300 2301 /** 2302 * unpoison_memory - Unpoison a previously poisoned page 2303 * @pfn: Page number of the to be unpoisoned page 2304 * 2305 * Software-unpoison a page that has been poisoned by 2306 * memory_failure() earlier. 2307 * 2308 * This is only done on the software-level, so it only works 2309 * for linux injected failures, not real hardware failures 2310 * 2311 * Returns 0 for success, otherwise -errno. 2312 */ 2313 int unpoison_memory(unsigned long pfn) 2314 { 2315 struct page *page; 2316 struct page *p; 2317 int ret = -EBUSY; 2318 int freeit = 0; 2319 unsigned long count = 1; 2320 static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL, 2321 DEFAULT_RATELIMIT_BURST); 2322 2323 if (!pfn_valid(pfn)) 2324 return -ENXIO; 2325 2326 p = pfn_to_page(pfn); 2327 page = compound_head(p); 2328 2329 mutex_lock(&mf_mutex); 2330 2331 if (hw_memory_failure) { 2332 unpoison_pr_info("Unpoison: Disabled after HW memory failure %#lx\n", 2333 pfn, &unpoison_rs); 2334 ret = -EOPNOTSUPP; 2335 goto unlock_mutex; 2336 } 2337 2338 if (!PageHWPoison(p)) { 2339 unpoison_pr_info("Unpoison: Page was already unpoisoned %#lx\n", 2340 pfn, &unpoison_rs); 2341 goto unlock_mutex; 2342 } 2343 2344 if (page_count(page) > 1) { 2345 unpoison_pr_info("Unpoison: Someone grabs the hwpoison page %#lx\n", 2346 pfn, &unpoison_rs); 2347 goto unlock_mutex; 2348 } 2349 2350 if (page_mapped(page)) { 2351 unpoison_pr_info("Unpoison: Someone maps the hwpoison page %#lx\n", 2352 pfn, &unpoison_rs); 2353 goto unlock_mutex; 2354 } 2355 2356 if (page_mapping(page)) { 2357 unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n", 2358 pfn, &unpoison_rs); 2359 goto unlock_mutex; 2360 } 2361 2362 if (PageSlab(page) || PageTable(page) || PageReserved(page)) 2363 goto unlock_mutex; 2364 2365 ret = get_hwpoison_page(p, MF_UNPOISON); 2366 if (!ret) { 2367 if (PageHuge(p)) { 2368 count = free_raw_hwp_pages(page, false); 2369 if (count == 0) { 2370 ret = -EBUSY; 2371 goto unlock_mutex; 2372 } 2373 } 2374 ret = TestClearPageHWPoison(page) ? 0 : -EBUSY; 2375 } else if (ret < 0) { 2376 if (ret == -EHWPOISON) { 2377 ret = put_page_back_buddy(p) ? 0 : -EBUSY; 2378 } else 2379 unpoison_pr_info("Unpoison: failed to grab page %#lx\n", 2380 pfn, &unpoison_rs); 2381 } else { 2382 if (PageHuge(p)) { 2383 count = free_raw_hwp_pages(page, false); 2384 if (count == 0) { 2385 ret = -EBUSY; 2386 put_page(page); 2387 goto unlock_mutex; 2388 } 2389 } 2390 freeit = !!TestClearPageHWPoison(p); 2391 2392 put_page(page); 2393 if (freeit) { 2394 put_page(page); 2395 ret = 0; 2396 } 2397 } 2398 2399 unlock_mutex: 2400 mutex_unlock(&mf_mutex); 2401 if (!ret || freeit) { 2402 num_poisoned_pages_sub(count); 2403 unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n", 2404 page_to_pfn(p), &unpoison_rs); 2405 } 2406 return ret; 2407 } 2408 EXPORT_SYMBOL(unpoison_memory); 2409 2410 static bool isolate_page(struct page *page, struct list_head *pagelist) 2411 { 2412 bool isolated = false; 2413 2414 if (PageHuge(page)) { 2415 isolated = !isolate_hugetlb(page, pagelist); 2416 } else { 2417 bool lru = !__PageMovable(page); 2418 2419 if (lru) 2420 isolated = !isolate_lru_page(page); 2421 else 2422 isolated = !isolate_movable_page(page, 2423 ISOLATE_UNEVICTABLE); 2424 2425 if (isolated) { 2426 list_add(&page->lru, pagelist); 2427 if (lru) 2428 inc_node_page_state(page, NR_ISOLATED_ANON + 2429 page_is_file_lru(page)); 2430 } 2431 } 2432 2433 /* 2434 * If we succeed to isolate the page, we grabbed another refcount on 2435 * the page, so we can safely drop the one we got from get_any_pages(). 2436 * If we failed to isolate the page, it means that we cannot go further 2437 * and we will return an error, so drop the reference we got from 2438 * get_any_pages() as well. 2439 */ 2440 put_page(page); 2441 return isolated; 2442 } 2443 2444 /* 2445 * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages. 2446 * If the page is a non-dirty unmapped page-cache page, it simply invalidates. 2447 * If the page is mapped, it migrates the contents over. 2448 */ 2449 static int soft_offline_in_use_page(struct page *page) 2450 { 2451 long ret = 0; 2452 unsigned long pfn = page_to_pfn(page); 2453 struct page *hpage = compound_head(page); 2454 char const *msg_page[] = {"page", "hugepage"}; 2455 bool huge = PageHuge(page); 2456 LIST_HEAD(pagelist); 2457 struct migration_target_control mtc = { 2458 .nid = NUMA_NO_NODE, 2459 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 2460 }; 2461 2462 if (!huge && PageTransHuge(hpage)) { 2463 if (try_to_split_thp_page(page)) { 2464 pr_info("soft offline: %#lx: thp split failed\n", pfn); 2465 return -EBUSY; 2466 } 2467 hpage = page; 2468 } 2469 2470 lock_page(page); 2471 if (!PageHuge(page)) 2472 wait_on_page_writeback(page); 2473 if (PageHWPoison(page)) { 2474 unlock_page(page); 2475 put_page(page); 2476 pr_info("soft offline: %#lx page already poisoned\n", pfn); 2477 return 0; 2478 } 2479 2480 if (!PageHuge(page) && PageLRU(page) && !PageSwapCache(page)) 2481 /* 2482 * Try to invalidate first. This should work for 2483 * non dirty unmapped page cache pages. 2484 */ 2485 ret = invalidate_inode_page(page); 2486 unlock_page(page); 2487 2488 if (ret) { 2489 pr_info("soft_offline: %#lx: invalidated\n", pfn); 2490 page_handle_poison(page, false, true); 2491 return 0; 2492 } 2493 2494 if (isolate_page(hpage, &pagelist)) { 2495 ret = migrate_pages(&pagelist, alloc_migration_target, NULL, 2496 (unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_FAILURE, NULL); 2497 if (!ret) { 2498 bool release = !huge; 2499 2500 if (!page_handle_poison(page, huge, release)) 2501 ret = -EBUSY; 2502 } else { 2503 if (!list_empty(&pagelist)) 2504 putback_movable_pages(&pagelist); 2505 2506 pr_info("soft offline: %#lx: %s migration failed %ld, type %pGp\n", 2507 pfn, msg_page[huge], ret, &page->flags); 2508 if (ret > 0) 2509 ret = -EBUSY; 2510 } 2511 } else { 2512 pr_info("soft offline: %#lx: %s isolation failed, page count %d, type %pGp\n", 2513 pfn, msg_page[huge], page_count(page), &page->flags); 2514 ret = -EBUSY; 2515 } 2516 return ret; 2517 } 2518 2519 static void put_ref_page(struct page *page) 2520 { 2521 if (page) 2522 put_page(page); 2523 } 2524 2525 /** 2526 * soft_offline_page - Soft offline a page. 2527 * @pfn: pfn to soft-offline 2528 * @flags: flags. Same as memory_failure(). 2529 * 2530 * Returns 0 on success 2531 * -EOPNOTSUPP for hwpoison_filter() filtered the error event 2532 * < 0 otherwise negated errno. 2533 * 2534 * Soft offline a page, by migration or invalidation, 2535 * without killing anything. This is for the case when 2536 * a page is not corrupted yet (so it's still valid to access), 2537 * but has had a number of corrected errors and is better taken 2538 * out. 2539 * 2540 * The actual policy on when to do that is maintained by 2541 * user space. 2542 * 2543 * This should never impact any application or cause data loss, 2544 * however it might take some time. 2545 * 2546 * This is not a 100% solution for all memory, but tries to be 2547 * ``good enough'' for the majority of memory. 2548 */ 2549 int soft_offline_page(unsigned long pfn, int flags) 2550 { 2551 int ret; 2552 bool try_again = true; 2553 struct page *page, *ref_page = NULL; 2554 2555 WARN_ON_ONCE(!pfn_valid(pfn) && (flags & MF_COUNT_INCREASED)); 2556 2557 if (!pfn_valid(pfn)) 2558 return -ENXIO; 2559 if (flags & MF_COUNT_INCREASED) 2560 ref_page = pfn_to_page(pfn); 2561 2562 /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */ 2563 page = pfn_to_online_page(pfn); 2564 if (!page) { 2565 put_ref_page(ref_page); 2566 return -EIO; 2567 } 2568 2569 mutex_lock(&mf_mutex); 2570 2571 if (PageHWPoison(page)) { 2572 pr_info("%s: %#lx page already poisoned\n", __func__, pfn); 2573 put_ref_page(ref_page); 2574 mutex_unlock(&mf_mutex); 2575 return 0; 2576 } 2577 2578 retry: 2579 get_online_mems(); 2580 ret = get_hwpoison_page(page, flags | MF_SOFT_OFFLINE); 2581 put_online_mems(); 2582 2583 if (hwpoison_filter(page)) { 2584 if (ret > 0) 2585 put_page(page); 2586 2587 mutex_unlock(&mf_mutex); 2588 return -EOPNOTSUPP; 2589 } 2590 2591 if (ret > 0) { 2592 ret = soft_offline_in_use_page(page); 2593 } else if (ret == 0) { 2594 if (!page_handle_poison(page, true, false) && try_again) { 2595 try_again = false; 2596 flags &= ~MF_COUNT_INCREASED; 2597 goto retry; 2598 } 2599 } 2600 2601 mutex_unlock(&mf_mutex); 2602 2603 return ret; 2604 } 2605 2606 void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) 2607 { 2608 int i, total = 0; 2609 2610 /* 2611 * A further optimization is to have per section refcounted 2612 * num_poisoned_pages. But that would need more space per memmap, so 2613 * for now just do a quick global check to speed up this routine in the 2614 * absence of bad pages. 2615 */ 2616 if (atomic_long_read(&num_poisoned_pages) == 0) 2617 return; 2618 2619 for (i = 0; i < nr_pages; i++) { 2620 if (PageHWPoison(&memmap[i])) { 2621 total++; 2622 ClearPageHWPoison(&memmap[i]); 2623 } 2624 } 2625 if (total) 2626 num_poisoned_pages_sub(total); 2627 } 2628