xref: /openbmc/linux/mm/memory-failure.c (revision 9a234a2a)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2008, 2009 Intel Corporation
4  * Authors: Andi Kleen, Fengguang Wu
5  *
6  * High level machine check handler. Handles pages reported by the
7  * hardware as being corrupted usually due to a multi-bit ECC memory or cache
8  * failure.
9  *
10  * In addition there is a "soft offline" entry point that allows stop using
11  * not-yet-corrupted-by-suspicious pages without killing anything.
12  *
13  * Handles page cache pages in various states.	The tricky part
14  * here is that we can access any page asynchronously in respect to
15  * other VM users, because memory failures could happen anytime and
16  * anywhere. This could violate some of their assumptions. This is why
17  * this code has to be extremely careful. Generally it tries to use
18  * normal locking rules, as in get the standard locks, even if that means
19  * the error handling takes potentially a long time.
20  *
21  * It can be very tempting to add handling for obscure cases here.
22  * In general any code for handling new cases should only be added iff:
23  * - You know how to test it.
24  * - You have a test that can be added to mce-test
25  *   https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/
26  * - The case actually shows up as a frequent (top 10) page state in
27  *   tools/vm/page-types when running a real workload.
28  *
29  * There are several operations here with exponential complexity because
30  * of unsuitable VM data structures. For example the operation to map back
31  * from RMAP chains to processes has to walk the complete process list and
32  * has non linear complexity with the number. But since memory corruptions
33  * are rare we hope to get away with this. This avoids impacting the core
34  * VM.
35  */
36 
37 #define pr_fmt(fmt) "Memory failure: " fmt
38 
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/page-flags.h>
42 #include <linux/kernel-page-flags.h>
43 #include <linux/sched/signal.h>
44 #include <linux/sched/task.h>
45 #include <linux/dax.h>
46 #include <linux/ksm.h>
47 #include <linux/rmap.h>
48 #include <linux/export.h>
49 #include <linux/pagemap.h>
50 #include <linux/swap.h>
51 #include <linux/backing-dev.h>
52 #include <linux/migrate.h>
53 #include <linux/suspend.h>
54 #include <linux/slab.h>
55 #include <linux/swapops.h>
56 #include <linux/hugetlb.h>
57 #include <linux/memory_hotplug.h>
58 #include <linux/mm_inline.h>
59 #include <linux/memremap.h>
60 #include <linux/kfifo.h>
61 #include <linux/ratelimit.h>
62 #include <linux/page-isolation.h>
63 #include <linux/pagewalk.h>
64 #include <linux/shmem_fs.h>
65 #include "swap.h"
66 #include "internal.h"
67 #include "ras/ras_event.h"
68 
69 int sysctl_memory_failure_early_kill __read_mostly = 0;
70 
71 int sysctl_memory_failure_recovery __read_mostly = 1;
72 
73 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
74 
75 static bool hw_memory_failure __read_mostly = false;
76 
77 /*
78  * Return values:
79  *   1:   the page is dissolved (if needed) and taken off from buddy,
80  *   0:   the page is dissolved (if needed) and not taken off from buddy,
81  *   < 0: failed to dissolve.
82  */
83 static int __page_handle_poison(struct page *page)
84 {
85 	int ret;
86 
87 	zone_pcp_disable(page_zone(page));
88 	ret = dissolve_free_huge_page(page);
89 	if (!ret)
90 		ret = take_page_off_buddy(page);
91 	zone_pcp_enable(page_zone(page));
92 
93 	return ret;
94 }
95 
96 static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release)
97 {
98 	if (hugepage_or_freepage) {
99 		/*
100 		 * Doing this check for free pages is also fine since dissolve_free_huge_page
101 		 * returns 0 for non-hugetlb pages as well.
102 		 */
103 		if (__page_handle_poison(page) <= 0)
104 			/*
105 			 * We could fail to take off the target page from buddy
106 			 * for example due to racy page allocation, but that's
107 			 * acceptable because soft-offlined page is not broken
108 			 * and if someone really want to use it, they should
109 			 * take it.
110 			 */
111 			return false;
112 	}
113 
114 	SetPageHWPoison(page);
115 	if (release)
116 		put_page(page);
117 	page_ref_inc(page);
118 	num_poisoned_pages_inc();
119 
120 	return true;
121 }
122 
123 #if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
124 
125 u32 hwpoison_filter_enable = 0;
126 u32 hwpoison_filter_dev_major = ~0U;
127 u32 hwpoison_filter_dev_minor = ~0U;
128 u64 hwpoison_filter_flags_mask;
129 u64 hwpoison_filter_flags_value;
130 EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
131 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
132 EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
133 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
134 EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
135 
136 static int hwpoison_filter_dev(struct page *p)
137 {
138 	struct address_space *mapping;
139 	dev_t dev;
140 
141 	if (hwpoison_filter_dev_major == ~0U &&
142 	    hwpoison_filter_dev_minor == ~0U)
143 		return 0;
144 
145 	mapping = page_mapping(p);
146 	if (mapping == NULL || mapping->host == NULL)
147 		return -EINVAL;
148 
149 	dev = mapping->host->i_sb->s_dev;
150 	if (hwpoison_filter_dev_major != ~0U &&
151 	    hwpoison_filter_dev_major != MAJOR(dev))
152 		return -EINVAL;
153 	if (hwpoison_filter_dev_minor != ~0U &&
154 	    hwpoison_filter_dev_minor != MINOR(dev))
155 		return -EINVAL;
156 
157 	return 0;
158 }
159 
160 static int hwpoison_filter_flags(struct page *p)
161 {
162 	if (!hwpoison_filter_flags_mask)
163 		return 0;
164 
165 	if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
166 				    hwpoison_filter_flags_value)
167 		return 0;
168 	else
169 		return -EINVAL;
170 }
171 
172 /*
173  * This allows stress tests to limit test scope to a collection of tasks
174  * by putting them under some memcg. This prevents killing unrelated/important
175  * processes such as /sbin/init. Note that the target task may share clean
176  * pages with init (eg. libc text), which is harmless. If the target task
177  * share _dirty_ pages with another task B, the test scheme must make sure B
178  * is also included in the memcg. At last, due to race conditions this filter
179  * can only guarantee that the page either belongs to the memcg tasks, or is
180  * a freed page.
181  */
182 #ifdef CONFIG_MEMCG
183 u64 hwpoison_filter_memcg;
184 EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
185 static int hwpoison_filter_task(struct page *p)
186 {
187 	if (!hwpoison_filter_memcg)
188 		return 0;
189 
190 	if (page_cgroup_ino(p) != hwpoison_filter_memcg)
191 		return -EINVAL;
192 
193 	return 0;
194 }
195 #else
196 static int hwpoison_filter_task(struct page *p) { return 0; }
197 #endif
198 
199 int hwpoison_filter(struct page *p)
200 {
201 	if (!hwpoison_filter_enable)
202 		return 0;
203 
204 	if (hwpoison_filter_dev(p))
205 		return -EINVAL;
206 
207 	if (hwpoison_filter_flags(p))
208 		return -EINVAL;
209 
210 	if (hwpoison_filter_task(p))
211 		return -EINVAL;
212 
213 	return 0;
214 }
215 #else
216 int hwpoison_filter(struct page *p)
217 {
218 	return 0;
219 }
220 #endif
221 
222 EXPORT_SYMBOL_GPL(hwpoison_filter);
223 
224 /*
225  * Kill all processes that have a poisoned page mapped and then isolate
226  * the page.
227  *
228  * General strategy:
229  * Find all processes having the page mapped and kill them.
230  * But we keep a page reference around so that the page is not
231  * actually freed yet.
232  * Then stash the page away
233  *
234  * There's no convenient way to get back to mapped processes
235  * from the VMAs. So do a brute-force search over all
236  * running processes.
237  *
238  * Remember that machine checks are not common (or rather
239  * if they are common you have other problems), so this shouldn't
240  * be a performance issue.
241  *
242  * Also there are some races possible while we get from the
243  * error detection to actually handle it.
244  */
245 
246 struct to_kill {
247 	struct list_head nd;
248 	struct task_struct *tsk;
249 	unsigned long addr;
250 	short size_shift;
251 };
252 
253 /*
254  * Send all the processes who have the page mapped a signal.
255  * ``action optional'' if they are not immediately affected by the error
256  * ``action required'' if error happened in current execution context
257  */
258 static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags)
259 {
260 	struct task_struct *t = tk->tsk;
261 	short addr_lsb = tk->size_shift;
262 	int ret = 0;
263 
264 	pr_err("%#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n",
265 			pfn, t->comm, t->pid);
266 
267 	if ((flags & MF_ACTION_REQUIRED) && (t == current))
268 		ret = force_sig_mceerr(BUS_MCEERR_AR,
269 				 (void __user *)tk->addr, addr_lsb);
270 	else
271 		/*
272 		 * Signal other processes sharing the page if they have
273 		 * PF_MCE_EARLY set.
274 		 * Don't use force here, it's convenient if the signal
275 		 * can be temporarily blocked.
276 		 * This could cause a loop when the user sets SIGBUS
277 		 * to SIG_IGN, but hopefully no one will do that?
278 		 */
279 		ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr,
280 				      addr_lsb, t);
281 	if (ret < 0)
282 		pr_info("Error sending signal to %s:%d: %d\n",
283 			t->comm, t->pid, ret);
284 	return ret;
285 }
286 
287 /*
288  * Unknown page type encountered. Try to check whether it can turn PageLRU by
289  * lru_add_drain_all.
290  */
291 void shake_page(struct page *p)
292 {
293 	if (PageHuge(p))
294 		return;
295 
296 	if (!PageSlab(p)) {
297 		lru_add_drain_all();
298 		if (PageLRU(p) || is_free_buddy_page(p))
299 			return;
300 	}
301 
302 	/*
303 	 * TODO: Could shrink slab caches here if a lightweight range-based
304 	 * shrinker will be available.
305 	 */
306 }
307 EXPORT_SYMBOL_GPL(shake_page);
308 
309 static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma,
310 		unsigned long address)
311 {
312 	unsigned long ret = 0;
313 	pgd_t *pgd;
314 	p4d_t *p4d;
315 	pud_t *pud;
316 	pmd_t *pmd;
317 	pte_t *pte;
318 
319 	VM_BUG_ON_VMA(address == -EFAULT, vma);
320 	pgd = pgd_offset(vma->vm_mm, address);
321 	if (!pgd_present(*pgd))
322 		return 0;
323 	p4d = p4d_offset(pgd, address);
324 	if (!p4d_present(*p4d))
325 		return 0;
326 	pud = pud_offset(p4d, address);
327 	if (!pud_present(*pud))
328 		return 0;
329 	if (pud_devmap(*pud))
330 		return PUD_SHIFT;
331 	pmd = pmd_offset(pud, address);
332 	if (!pmd_present(*pmd))
333 		return 0;
334 	if (pmd_devmap(*pmd))
335 		return PMD_SHIFT;
336 	pte = pte_offset_map(pmd, address);
337 	if (pte_present(*pte) && pte_devmap(*pte))
338 		ret = PAGE_SHIFT;
339 	pte_unmap(pte);
340 	return ret;
341 }
342 
343 /*
344  * Failure handling: if we can't find or can't kill a process there's
345  * not much we can do.	We just print a message and ignore otherwise.
346  */
347 
348 #define FSDAX_INVALID_PGOFF ULONG_MAX
349 
350 /*
351  * Schedule a process for later kill.
352  * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
353  *
354  * Note: @fsdax_pgoff is used only when @p is a fsdax page and a
355  * filesystem with a memory failure handler has claimed the
356  * memory_failure event. In all other cases, page->index and
357  * page->mapping are sufficient for mapping the page back to its
358  * corresponding user virtual address.
359  */
360 static void add_to_kill(struct task_struct *tsk, struct page *p,
361 			pgoff_t fsdax_pgoff, struct vm_area_struct *vma,
362 			struct list_head *to_kill)
363 {
364 	struct to_kill *tk;
365 
366 	tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
367 	if (!tk) {
368 		pr_err("Out of memory while machine check handling\n");
369 		return;
370 	}
371 
372 	tk->addr = page_address_in_vma(p, vma);
373 	if (is_zone_device_page(p)) {
374 		if (fsdax_pgoff != FSDAX_INVALID_PGOFF)
375 			tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma);
376 		tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr);
377 	} else
378 		tk->size_shift = page_shift(compound_head(p));
379 
380 	/*
381 	 * Send SIGKILL if "tk->addr == -EFAULT". Also, as
382 	 * "tk->size_shift" is always non-zero for !is_zone_device_page(),
383 	 * so "tk->size_shift == 0" effectively checks no mapping on
384 	 * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times
385 	 * to a process' address space, it's possible not all N VMAs
386 	 * contain mappings for the page, but at least one VMA does.
387 	 * Only deliver SIGBUS with payload derived from the VMA that
388 	 * has a mapping for the page.
389 	 */
390 	if (tk->addr == -EFAULT) {
391 		pr_info("Unable to find user space address %lx in %s\n",
392 			page_to_pfn(p), tsk->comm);
393 	} else if (tk->size_shift == 0) {
394 		kfree(tk);
395 		return;
396 	}
397 
398 	get_task_struct(tsk);
399 	tk->tsk = tsk;
400 	list_add_tail(&tk->nd, to_kill);
401 }
402 
403 /*
404  * Kill the processes that have been collected earlier.
405  *
406  * Only do anything when FORCEKILL is set, otherwise just free the
407  * list (this is used for clean pages which do not need killing)
408  * Also when FAIL is set do a force kill because something went
409  * wrong earlier.
410  */
411 static void kill_procs(struct list_head *to_kill, int forcekill, bool fail,
412 		unsigned long pfn, int flags)
413 {
414 	struct to_kill *tk, *next;
415 
416 	list_for_each_entry_safe(tk, next, to_kill, nd) {
417 		if (forcekill) {
418 			/*
419 			 * In case something went wrong with munmapping
420 			 * make sure the process doesn't catch the
421 			 * signal and then access the memory. Just kill it.
422 			 */
423 			if (fail || tk->addr == -EFAULT) {
424 				pr_err("%#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
425 				       pfn, tk->tsk->comm, tk->tsk->pid);
426 				do_send_sig_info(SIGKILL, SEND_SIG_PRIV,
427 						 tk->tsk, PIDTYPE_PID);
428 			}
429 
430 			/*
431 			 * In theory the process could have mapped
432 			 * something else on the address in-between. We could
433 			 * check for that, but we need to tell the
434 			 * process anyways.
435 			 */
436 			else if (kill_proc(tk, pfn, flags) < 0)
437 				pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n",
438 				       pfn, tk->tsk->comm, tk->tsk->pid);
439 		}
440 		list_del(&tk->nd);
441 		put_task_struct(tk->tsk);
442 		kfree(tk);
443 	}
444 }
445 
446 /*
447  * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO)
448  * on behalf of the thread group. Return task_struct of the (first found)
449  * dedicated thread if found, and return NULL otherwise.
450  *
451  * We already hold read_lock(&tasklist_lock) in the caller, so we don't
452  * have to call rcu_read_lock/unlock() in this function.
453  */
454 static struct task_struct *find_early_kill_thread(struct task_struct *tsk)
455 {
456 	struct task_struct *t;
457 
458 	for_each_thread(tsk, t) {
459 		if (t->flags & PF_MCE_PROCESS) {
460 			if (t->flags & PF_MCE_EARLY)
461 				return t;
462 		} else {
463 			if (sysctl_memory_failure_early_kill)
464 				return t;
465 		}
466 	}
467 	return NULL;
468 }
469 
470 /*
471  * Determine whether a given process is "early kill" process which expects
472  * to be signaled when some page under the process is hwpoisoned.
473  * Return task_struct of the dedicated thread (main thread unless explicitly
474  * specified) if the process is "early kill" and otherwise returns NULL.
475  *
476  * Note that the above is true for Action Optional case. For Action Required
477  * case, it's only meaningful to the current thread which need to be signaled
478  * with SIGBUS, this error is Action Optional for other non current
479  * processes sharing the same error page,if the process is "early kill", the
480  * task_struct of the dedicated thread will also be returned.
481  */
482 static struct task_struct *task_early_kill(struct task_struct *tsk,
483 					   int force_early)
484 {
485 	if (!tsk->mm)
486 		return NULL;
487 	/*
488 	 * Comparing ->mm here because current task might represent
489 	 * a subthread, while tsk always points to the main thread.
490 	 */
491 	if (force_early && tsk->mm == current->mm)
492 		return current;
493 
494 	return find_early_kill_thread(tsk);
495 }
496 
497 /*
498  * Collect processes when the error hit an anonymous page.
499  */
500 static void collect_procs_anon(struct page *page, struct list_head *to_kill,
501 				int force_early)
502 {
503 	struct folio *folio = page_folio(page);
504 	struct vm_area_struct *vma;
505 	struct task_struct *tsk;
506 	struct anon_vma *av;
507 	pgoff_t pgoff;
508 
509 	av = folio_lock_anon_vma_read(folio, NULL);
510 	if (av == NULL)	/* Not actually mapped anymore */
511 		return;
512 
513 	pgoff = page_to_pgoff(page);
514 	read_lock(&tasklist_lock);
515 	for_each_process (tsk) {
516 		struct anon_vma_chain *vmac;
517 		struct task_struct *t = task_early_kill(tsk, force_early);
518 
519 		if (!t)
520 			continue;
521 		anon_vma_interval_tree_foreach(vmac, &av->rb_root,
522 					       pgoff, pgoff) {
523 			vma = vmac->vma;
524 			if (vma->vm_mm != t->mm)
525 				continue;
526 			if (!page_mapped_in_vma(page, vma))
527 				continue;
528 			add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma, to_kill);
529 		}
530 	}
531 	read_unlock(&tasklist_lock);
532 	anon_vma_unlock_read(av);
533 }
534 
535 /*
536  * Collect processes when the error hit a file mapped page.
537  */
538 static void collect_procs_file(struct page *page, struct list_head *to_kill,
539 				int force_early)
540 {
541 	struct vm_area_struct *vma;
542 	struct task_struct *tsk;
543 	struct address_space *mapping = page->mapping;
544 	pgoff_t pgoff;
545 
546 	i_mmap_lock_read(mapping);
547 	read_lock(&tasklist_lock);
548 	pgoff = page_to_pgoff(page);
549 	for_each_process(tsk) {
550 		struct task_struct *t = task_early_kill(tsk, force_early);
551 
552 		if (!t)
553 			continue;
554 		vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
555 				      pgoff) {
556 			/*
557 			 * Send early kill signal to tasks where a vma covers
558 			 * the page but the corrupted page is not necessarily
559 			 * mapped it in its pte.
560 			 * Assume applications who requested early kill want
561 			 * to be informed of all such data corruptions.
562 			 */
563 			if (vma->vm_mm == t->mm)
564 				add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma,
565 					    to_kill);
566 		}
567 	}
568 	read_unlock(&tasklist_lock);
569 	i_mmap_unlock_read(mapping);
570 }
571 
572 #ifdef CONFIG_FS_DAX
573 /*
574  * Collect processes when the error hit a fsdax page.
575  */
576 static void collect_procs_fsdax(struct page *page,
577 		struct address_space *mapping, pgoff_t pgoff,
578 		struct list_head *to_kill)
579 {
580 	struct vm_area_struct *vma;
581 	struct task_struct *tsk;
582 
583 	i_mmap_lock_read(mapping);
584 	read_lock(&tasklist_lock);
585 	for_each_process(tsk) {
586 		struct task_struct *t = task_early_kill(tsk, true);
587 
588 		if (!t)
589 			continue;
590 		vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
591 			if (vma->vm_mm == t->mm)
592 				add_to_kill(t, page, pgoff, vma, to_kill);
593 		}
594 	}
595 	read_unlock(&tasklist_lock);
596 	i_mmap_unlock_read(mapping);
597 }
598 #endif /* CONFIG_FS_DAX */
599 
600 /*
601  * Collect the processes who have the corrupted page mapped to kill.
602  */
603 static void collect_procs(struct page *page, struct list_head *tokill,
604 				int force_early)
605 {
606 	if (!page->mapping)
607 		return;
608 
609 	if (PageAnon(page))
610 		collect_procs_anon(page, tokill, force_early);
611 	else
612 		collect_procs_file(page, tokill, force_early);
613 }
614 
615 struct hwp_walk {
616 	struct to_kill tk;
617 	unsigned long pfn;
618 	int flags;
619 };
620 
621 static void set_to_kill(struct to_kill *tk, unsigned long addr, short shift)
622 {
623 	tk->addr = addr;
624 	tk->size_shift = shift;
625 }
626 
627 static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift,
628 				unsigned long poisoned_pfn, struct to_kill *tk)
629 {
630 	unsigned long pfn = 0;
631 
632 	if (pte_present(pte)) {
633 		pfn = pte_pfn(pte);
634 	} else {
635 		swp_entry_t swp = pte_to_swp_entry(pte);
636 
637 		if (is_hwpoison_entry(swp))
638 			pfn = swp_offset_pfn(swp);
639 	}
640 
641 	if (!pfn || pfn != poisoned_pfn)
642 		return 0;
643 
644 	set_to_kill(tk, addr, shift);
645 	return 1;
646 }
647 
648 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
649 static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr,
650 				      struct hwp_walk *hwp)
651 {
652 	pmd_t pmd = *pmdp;
653 	unsigned long pfn;
654 	unsigned long hwpoison_vaddr;
655 
656 	if (!pmd_present(pmd))
657 		return 0;
658 	pfn = pmd_pfn(pmd);
659 	if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) {
660 		hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT);
661 		set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT);
662 		return 1;
663 	}
664 	return 0;
665 }
666 #else
667 static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr,
668 				      struct hwp_walk *hwp)
669 {
670 	return 0;
671 }
672 #endif
673 
674 static int hwpoison_pte_range(pmd_t *pmdp, unsigned long addr,
675 			      unsigned long end, struct mm_walk *walk)
676 {
677 	struct hwp_walk *hwp = walk->private;
678 	int ret = 0;
679 	pte_t *ptep, *mapped_pte;
680 	spinlock_t *ptl;
681 
682 	ptl = pmd_trans_huge_lock(pmdp, walk->vma);
683 	if (ptl) {
684 		ret = check_hwpoisoned_pmd_entry(pmdp, addr, hwp);
685 		spin_unlock(ptl);
686 		goto out;
687 	}
688 
689 	if (pmd_trans_unstable(pmdp))
690 		goto out;
691 
692 	mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp,
693 						addr, &ptl);
694 	for (; addr != end; ptep++, addr += PAGE_SIZE) {
695 		ret = check_hwpoisoned_entry(*ptep, addr, PAGE_SHIFT,
696 					     hwp->pfn, &hwp->tk);
697 		if (ret == 1)
698 			break;
699 	}
700 	pte_unmap_unlock(mapped_pte, ptl);
701 out:
702 	cond_resched();
703 	return ret;
704 }
705 
706 #ifdef CONFIG_HUGETLB_PAGE
707 static int hwpoison_hugetlb_range(pte_t *ptep, unsigned long hmask,
708 			    unsigned long addr, unsigned long end,
709 			    struct mm_walk *walk)
710 {
711 	struct hwp_walk *hwp = walk->private;
712 	pte_t pte = huge_ptep_get(ptep);
713 	struct hstate *h = hstate_vma(walk->vma);
714 
715 	return check_hwpoisoned_entry(pte, addr, huge_page_shift(h),
716 				      hwp->pfn, &hwp->tk);
717 }
718 #else
719 #define hwpoison_hugetlb_range	NULL
720 #endif
721 
722 static const struct mm_walk_ops hwp_walk_ops = {
723 	.pmd_entry = hwpoison_pte_range,
724 	.hugetlb_entry = hwpoison_hugetlb_range,
725 };
726 
727 /*
728  * Sends SIGBUS to the current process with error info.
729  *
730  * This function is intended to handle "Action Required" MCEs on already
731  * hardware poisoned pages. They could happen, for example, when
732  * memory_failure() failed to unmap the error page at the first call, or
733  * when multiple local machine checks happened on different CPUs.
734  *
735  * MCE handler currently has no easy access to the error virtual address,
736  * so this function walks page table to find it. The returned virtual address
737  * is proper in most cases, but it could be wrong when the application
738  * process has multiple entries mapping the error page.
739  */
740 static int kill_accessing_process(struct task_struct *p, unsigned long pfn,
741 				  int flags)
742 {
743 	int ret;
744 	struct hwp_walk priv = {
745 		.pfn = pfn,
746 	};
747 	priv.tk.tsk = p;
748 
749 	if (!p->mm)
750 		return -EFAULT;
751 
752 	mmap_read_lock(p->mm);
753 	ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwp_walk_ops,
754 			      (void *)&priv);
755 	if (ret == 1 && priv.tk.addr)
756 		kill_proc(&priv.tk, pfn, flags);
757 	else
758 		ret = 0;
759 	mmap_read_unlock(p->mm);
760 	return ret > 0 ? -EHWPOISON : -EFAULT;
761 }
762 
763 static const char *action_name[] = {
764 	[MF_IGNORED] = "Ignored",
765 	[MF_FAILED] = "Failed",
766 	[MF_DELAYED] = "Delayed",
767 	[MF_RECOVERED] = "Recovered",
768 };
769 
770 static const char * const action_page_types[] = {
771 	[MF_MSG_KERNEL]			= "reserved kernel page",
772 	[MF_MSG_KERNEL_HIGH_ORDER]	= "high-order kernel page",
773 	[MF_MSG_SLAB]			= "kernel slab page",
774 	[MF_MSG_DIFFERENT_COMPOUND]	= "different compound page after locking",
775 	[MF_MSG_HUGE]			= "huge page",
776 	[MF_MSG_FREE_HUGE]		= "free huge page",
777 	[MF_MSG_UNMAP_FAILED]		= "unmapping failed page",
778 	[MF_MSG_DIRTY_SWAPCACHE]	= "dirty swapcache page",
779 	[MF_MSG_CLEAN_SWAPCACHE]	= "clean swapcache page",
780 	[MF_MSG_DIRTY_MLOCKED_LRU]	= "dirty mlocked LRU page",
781 	[MF_MSG_CLEAN_MLOCKED_LRU]	= "clean mlocked LRU page",
782 	[MF_MSG_DIRTY_UNEVICTABLE_LRU]	= "dirty unevictable LRU page",
783 	[MF_MSG_CLEAN_UNEVICTABLE_LRU]	= "clean unevictable LRU page",
784 	[MF_MSG_DIRTY_LRU]		= "dirty LRU page",
785 	[MF_MSG_CLEAN_LRU]		= "clean LRU page",
786 	[MF_MSG_TRUNCATED_LRU]		= "already truncated LRU page",
787 	[MF_MSG_BUDDY]			= "free buddy page",
788 	[MF_MSG_DAX]			= "dax page",
789 	[MF_MSG_UNSPLIT_THP]		= "unsplit thp",
790 	[MF_MSG_UNKNOWN]		= "unknown page",
791 };
792 
793 /*
794  * XXX: It is possible that a page is isolated from LRU cache,
795  * and then kept in swap cache or failed to remove from page cache.
796  * The page count will stop it from being freed by unpoison.
797  * Stress tests should be aware of this memory leak problem.
798  */
799 static int delete_from_lru_cache(struct page *p)
800 {
801 	if (!isolate_lru_page(p)) {
802 		/*
803 		 * Clear sensible page flags, so that the buddy system won't
804 		 * complain when the page is unpoison-and-freed.
805 		 */
806 		ClearPageActive(p);
807 		ClearPageUnevictable(p);
808 
809 		/*
810 		 * Poisoned page might never drop its ref count to 0 so we have
811 		 * to uncharge it manually from its memcg.
812 		 */
813 		mem_cgroup_uncharge(page_folio(p));
814 
815 		/*
816 		 * drop the page count elevated by isolate_lru_page()
817 		 */
818 		put_page(p);
819 		return 0;
820 	}
821 	return -EIO;
822 }
823 
824 static int truncate_error_page(struct page *p, unsigned long pfn,
825 				struct address_space *mapping)
826 {
827 	int ret = MF_FAILED;
828 
829 	if (mapping->a_ops->error_remove_page) {
830 		int err = mapping->a_ops->error_remove_page(mapping, p);
831 
832 		if (err != 0) {
833 			pr_info("%#lx: Failed to punch page: %d\n", pfn, err);
834 		} else if (page_has_private(p) &&
835 			   !try_to_release_page(p, GFP_NOIO)) {
836 			pr_info("%#lx: failed to release buffers\n", pfn);
837 		} else {
838 			ret = MF_RECOVERED;
839 		}
840 	} else {
841 		/*
842 		 * If the file system doesn't support it just invalidate
843 		 * This fails on dirty or anything with private pages
844 		 */
845 		if (invalidate_inode_page(p))
846 			ret = MF_RECOVERED;
847 		else
848 			pr_info("%#lx: Failed to invalidate\n",	pfn);
849 	}
850 
851 	return ret;
852 }
853 
854 struct page_state {
855 	unsigned long mask;
856 	unsigned long res;
857 	enum mf_action_page_type type;
858 
859 	/* Callback ->action() has to unlock the relevant page inside it. */
860 	int (*action)(struct page_state *ps, struct page *p);
861 };
862 
863 /*
864  * Return true if page is still referenced by others, otherwise return
865  * false.
866  *
867  * The extra_pins is true when one extra refcount is expected.
868  */
869 static bool has_extra_refcount(struct page_state *ps, struct page *p,
870 			       bool extra_pins)
871 {
872 	int count = page_count(p) - 1;
873 
874 	if (extra_pins)
875 		count -= 1;
876 
877 	if (count > 0) {
878 		pr_err("%#lx: %s still referenced by %d users\n",
879 		       page_to_pfn(p), action_page_types[ps->type], count);
880 		return true;
881 	}
882 
883 	return false;
884 }
885 
886 /*
887  * Error hit kernel page.
888  * Do nothing, try to be lucky and not touch this instead. For a few cases we
889  * could be more sophisticated.
890  */
891 static int me_kernel(struct page_state *ps, struct page *p)
892 {
893 	unlock_page(p);
894 	return MF_IGNORED;
895 }
896 
897 /*
898  * Page in unknown state. Do nothing.
899  */
900 static int me_unknown(struct page_state *ps, struct page *p)
901 {
902 	pr_err("%#lx: Unknown page state\n", page_to_pfn(p));
903 	unlock_page(p);
904 	return MF_FAILED;
905 }
906 
907 /*
908  * Clean (or cleaned) page cache page.
909  */
910 static int me_pagecache_clean(struct page_state *ps, struct page *p)
911 {
912 	int ret;
913 	struct address_space *mapping;
914 	bool extra_pins;
915 
916 	delete_from_lru_cache(p);
917 
918 	/*
919 	 * For anonymous pages we're done the only reference left
920 	 * should be the one m_f() holds.
921 	 */
922 	if (PageAnon(p)) {
923 		ret = MF_RECOVERED;
924 		goto out;
925 	}
926 
927 	/*
928 	 * Now truncate the page in the page cache. This is really
929 	 * more like a "temporary hole punch"
930 	 * Don't do this for block devices when someone else
931 	 * has a reference, because it could be file system metadata
932 	 * and that's not safe to truncate.
933 	 */
934 	mapping = page_mapping(p);
935 	if (!mapping) {
936 		/*
937 		 * Page has been teared down in the meanwhile
938 		 */
939 		ret = MF_FAILED;
940 		goto out;
941 	}
942 
943 	/*
944 	 * The shmem page is kept in page cache instead of truncating
945 	 * so is expected to have an extra refcount after error-handling.
946 	 */
947 	extra_pins = shmem_mapping(mapping);
948 
949 	/*
950 	 * Truncation is a bit tricky. Enable it per file system for now.
951 	 *
952 	 * Open: to take i_rwsem or not for this? Right now we don't.
953 	 */
954 	ret = truncate_error_page(p, page_to_pfn(p), mapping);
955 	if (has_extra_refcount(ps, p, extra_pins))
956 		ret = MF_FAILED;
957 
958 out:
959 	unlock_page(p);
960 
961 	return ret;
962 }
963 
964 /*
965  * Dirty pagecache page
966  * Issues: when the error hit a hole page the error is not properly
967  * propagated.
968  */
969 static int me_pagecache_dirty(struct page_state *ps, struct page *p)
970 {
971 	struct address_space *mapping = page_mapping(p);
972 
973 	SetPageError(p);
974 	/* TBD: print more information about the file. */
975 	if (mapping) {
976 		/*
977 		 * IO error will be reported by write(), fsync(), etc.
978 		 * who check the mapping.
979 		 * This way the application knows that something went
980 		 * wrong with its dirty file data.
981 		 *
982 		 * There's one open issue:
983 		 *
984 		 * The EIO will be only reported on the next IO
985 		 * operation and then cleared through the IO map.
986 		 * Normally Linux has two mechanisms to pass IO error
987 		 * first through the AS_EIO flag in the address space
988 		 * and then through the PageError flag in the page.
989 		 * Since we drop pages on memory failure handling the
990 		 * only mechanism open to use is through AS_AIO.
991 		 *
992 		 * This has the disadvantage that it gets cleared on
993 		 * the first operation that returns an error, while
994 		 * the PageError bit is more sticky and only cleared
995 		 * when the page is reread or dropped.  If an
996 		 * application assumes it will always get error on
997 		 * fsync, but does other operations on the fd before
998 		 * and the page is dropped between then the error
999 		 * will not be properly reported.
1000 		 *
1001 		 * This can already happen even without hwpoisoned
1002 		 * pages: first on metadata IO errors (which only
1003 		 * report through AS_EIO) or when the page is dropped
1004 		 * at the wrong time.
1005 		 *
1006 		 * So right now we assume that the application DTRT on
1007 		 * the first EIO, but we're not worse than other parts
1008 		 * of the kernel.
1009 		 */
1010 		mapping_set_error(mapping, -EIO);
1011 	}
1012 
1013 	return me_pagecache_clean(ps, p);
1014 }
1015 
1016 /*
1017  * Clean and dirty swap cache.
1018  *
1019  * Dirty swap cache page is tricky to handle. The page could live both in page
1020  * cache and swap cache(ie. page is freshly swapped in). So it could be
1021  * referenced concurrently by 2 types of PTEs:
1022  * normal PTEs and swap PTEs. We try to handle them consistently by calling
1023  * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
1024  * and then
1025  *      - clear dirty bit to prevent IO
1026  *      - remove from LRU
1027  *      - but keep in the swap cache, so that when we return to it on
1028  *        a later page fault, we know the application is accessing
1029  *        corrupted data and shall be killed (we installed simple
1030  *        interception code in do_swap_page to catch it).
1031  *
1032  * Clean swap cache pages can be directly isolated. A later page fault will
1033  * bring in the known good data from disk.
1034  */
1035 static int me_swapcache_dirty(struct page_state *ps, struct page *p)
1036 {
1037 	int ret;
1038 	bool extra_pins = false;
1039 
1040 	ClearPageDirty(p);
1041 	/* Trigger EIO in shmem: */
1042 	ClearPageUptodate(p);
1043 
1044 	ret = delete_from_lru_cache(p) ? MF_FAILED : MF_DELAYED;
1045 	unlock_page(p);
1046 
1047 	if (ret == MF_DELAYED)
1048 		extra_pins = true;
1049 
1050 	if (has_extra_refcount(ps, p, extra_pins))
1051 		ret = MF_FAILED;
1052 
1053 	return ret;
1054 }
1055 
1056 static int me_swapcache_clean(struct page_state *ps, struct page *p)
1057 {
1058 	struct folio *folio = page_folio(p);
1059 	int ret;
1060 
1061 	delete_from_swap_cache(folio);
1062 
1063 	ret = delete_from_lru_cache(p) ? MF_FAILED : MF_RECOVERED;
1064 	folio_unlock(folio);
1065 
1066 	if (has_extra_refcount(ps, p, false))
1067 		ret = MF_FAILED;
1068 
1069 	return ret;
1070 }
1071 
1072 /*
1073  * Huge pages. Needs work.
1074  * Issues:
1075  * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
1076  *   To narrow down kill region to one page, we need to break up pmd.
1077  */
1078 static int me_huge_page(struct page_state *ps, struct page *p)
1079 {
1080 	int res;
1081 	struct page *hpage = compound_head(p);
1082 	struct address_space *mapping;
1083 	bool extra_pins = false;
1084 
1085 	if (!PageHuge(hpage))
1086 		return MF_DELAYED;
1087 
1088 	mapping = page_mapping(hpage);
1089 	if (mapping) {
1090 		res = truncate_error_page(hpage, page_to_pfn(p), mapping);
1091 		/* The page is kept in page cache. */
1092 		extra_pins = true;
1093 		unlock_page(hpage);
1094 	} else {
1095 		unlock_page(hpage);
1096 		/*
1097 		 * migration entry prevents later access on error hugepage,
1098 		 * so we can free and dissolve it into buddy to save healthy
1099 		 * subpages.
1100 		 */
1101 		put_page(hpage);
1102 		if (__page_handle_poison(p) >= 0) {
1103 			page_ref_inc(p);
1104 			res = MF_RECOVERED;
1105 		} else {
1106 			res = MF_FAILED;
1107 		}
1108 	}
1109 
1110 	if (has_extra_refcount(ps, p, extra_pins))
1111 		res = MF_FAILED;
1112 
1113 	return res;
1114 }
1115 
1116 /*
1117  * Various page states we can handle.
1118  *
1119  * A page state is defined by its current page->flags bits.
1120  * The table matches them in order and calls the right handler.
1121  *
1122  * This is quite tricky because we can access page at any time
1123  * in its live cycle, so all accesses have to be extremely careful.
1124  *
1125  * This is not complete. More states could be added.
1126  * For any missing state don't attempt recovery.
1127  */
1128 
1129 #define dirty		(1UL << PG_dirty)
1130 #define sc		((1UL << PG_swapcache) | (1UL << PG_swapbacked))
1131 #define unevict		(1UL << PG_unevictable)
1132 #define mlock		(1UL << PG_mlocked)
1133 #define lru		(1UL << PG_lru)
1134 #define head		(1UL << PG_head)
1135 #define slab		(1UL << PG_slab)
1136 #define reserved	(1UL << PG_reserved)
1137 
1138 static struct page_state error_states[] = {
1139 	{ reserved,	reserved,	MF_MSG_KERNEL,	me_kernel },
1140 	/*
1141 	 * free pages are specially detected outside this table:
1142 	 * PG_buddy pages only make a small fraction of all free pages.
1143 	 */
1144 
1145 	/*
1146 	 * Could in theory check if slab page is free or if we can drop
1147 	 * currently unused objects without touching them. But just
1148 	 * treat it as standard kernel for now.
1149 	 */
1150 	{ slab,		slab,		MF_MSG_SLAB,	me_kernel },
1151 
1152 	{ head,		head,		MF_MSG_HUGE,		me_huge_page },
1153 
1154 	{ sc|dirty,	sc|dirty,	MF_MSG_DIRTY_SWAPCACHE,	me_swapcache_dirty },
1155 	{ sc|dirty,	sc,		MF_MSG_CLEAN_SWAPCACHE,	me_swapcache_clean },
1156 
1157 	{ mlock|dirty,	mlock|dirty,	MF_MSG_DIRTY_MLOCKED_LRU,	me_pagecache_dirty },
1158 	{ mlock|dirty,	mlock,		MF_MSG_CLEAN_MLOCKED_LRU,	me_pagecache_clean },
1159 
1160 	{ unevict|dirty, unevict|dirty,	MF_MSG_DIRTY_UNEVICTABLE_LRU,	me_pagecache_dirty },
1161 	{ unevict|dirty, unevict,	MF_MSG_CLEAN_UNEVICTABLE_LRU,	me_pagecache_clean },
1162 
1163 	{ lru|dirty,	lru|dirty,	MF_MSG_DIRTY_LRU,	me_pagecache_dirty },
1164 	{ lru|dirty,	lru,		MF_MSG_CLEAN_LRU,	me_pagecache_clean },
1165 
1166 	/*
1167 	 * Catchall entry: must be at end.
1168 	 */
1169 	{ 0,		0,		MF_MSG_UNKNOWN,	me_unknown },
1170 };
1171 
1172 #undef dirty
1173 #undef sc
1174 #undef unevict
1175 #undef mlock
1176 #undef lru
1177 #undef head
1178 #undef slab
1179 #undef reserved
1180 
1181 /*
1182  * "Dirty/Clean" indication is not 100% accurate due to the possibility of
1183  * setting PG_dirty outside page lock. See also comment above set_page_dirty().
1184  */
1185 static void action_result(unsigned long pfn, enum mf_action_page_type type,
1186 			  enum mf_result result)
1187 {
1188 	trace_memory_failure_event(pfn, type, result);
1189 
1190 	num_poisoned_pages_inc();
1191 	pr_err("%#lx: recovery action for %s: %s\n",
1192 		pfn, action_page_types[type], action_name[result]);
1193 }
1194 
1195 static int page_action(struct page_state *ps, struct page *p,
1196 			unsigned long pfn)
1197 {
1198 	int result;
1199 
1200 	/* page p should be unlocked after returning from ps->action().  */
1201 	result = ps->action(ps, p);
1202 
1203 	action_result(pfn, ps->type, result);
1204 
1205 	/* Could do more checks here if page looks ok */
1206 	/*
1207 	 * Could adjust zone counters here to correct for the missing page.
1208 	 */
1209 
1210 	return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY;
1211 }
1212 
1213 static inline bool PageHWPoisonTakenOff(struct page *page)
1214 {
1215 	return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON;
1216 }
1217 
1218 void SetPageHWPoisonTakenOff(struct page *page)
1219 {
1220 	set_page_private(page, MAGIC_HWPOISON);
1221 }
1222 
1223 void ClearPageHWPoisonTakenOff(struct page *page)
1224 {
1225 	if (PageHWPoison(page))
1226 		set_page_private(page, 0);
1227 }
1228 
1229 /*
1230  * Return true if a page type of a given page is supported by hwpoison
1231  * mechanism (while handling could fail), otherwise false.  This function
1232  * does not return true for hugetlb or device memory pages, so it's assumed
1233  * to be called only in the context where we never have such pages.
1234  */
1235 static inline bool HWPoisonHandlable(struct page *page, unsigned long flags)
1236 {
1237 	/* Soft offline could migrate non-LRU movable pages */
1238 	if ((flags & MF_SOFT_OFFLINE) && __PageMovable(page))
1239 		return true;
1240 
1241 	return PageLRU(page) || is_free_buddy_page(page);
1242 }
1243 
1244 static int __get_hwpoison_page(struct page *page, unsigned long flags)
1245 {
1246 	struct page *head = compound_head(page);
1247 	int ret = 0;
1248 	bool hugetlb = false;
1249 
1250 	ret = get_hwpoison_huge_page(head, &hugetlb);
1251 	if (hugetlb)
1252 		return ret;
1253 
1254 	/*
1255 	 * This check prevents from calling get_page_unless_zero() for any
1256 	 * unsupported type of page in order to reduce the risk of unexpected
1257 	 * races caused by taking a page refcount.
1258 	 */
1259 	if (!HWPoisonHandlable(head, flags))
1260 		return -EBUSY;
1261 
1262 	if (get_page_unless_zero(head)) {
1263 		if (head == compound_head(page))
1264 			return 1;
1265 
1266 		pr_info("%#lx cannot catch tail\n", page_to_pfn(page));
1267 		put_page(head);
1268 	}
1269 
1270 	return 0;
1271 }
1272 
1273 static int get_any_page(struct page *p, unsigned long flags)
1274 {
1275 	int ret = 0, pass = 0;
1276 	bool count_increased = false;
1277 
1278 	if (flags & MF_COUNT_INCREASED)
1279 		count_increased = true;
1280 
1281 try_again:
1282 	if (!count_increased) {
1283 		ret = __get_hwpoison_page(p, flags);
1284 		if (!ret) {
1285 			if (page_count(p)) {
1286 				/* We raced with an allocation, retry. */
1287 				if (pass++ < 3)
1288 					goto try_again;
1289 				ret = -EBUSY;
1290 			} else if (!PageHuge(p) && !is_free_buddy_page(p)) {
1291 				/* We raced with put_page, retry. */
1292 				if (pass++ < 3)
1293 					goto try_again;
1294 				ret = -EIO;
1295 			}
1296 			goto out;
1297 		} else if (ret == -EBUSY) {
1298 			/*
1299 			 * We raced with (possibly temporary) unhandlable
1300 			 * page, retry.
1301 			 */
1302 			if (pass++ < 3) {
1303 				shake_page(p);
1304 				goto try_again;
1305 			}
1306 			ret = -EIO;
1307 			goto out;
1308 		}
1309 	}
1310 
1311 	if (PageHuge(p) || HWPoisonHandlable(p, flags)) {
1312 		ret = 1;
1313 	} else {
1314 		/*
1315 		 * A page we cannot handle. Check whether we can turn
1316 		 * it into something we can handle.
1317 		 */
1318 		if (pass++ < 3) {
1319 			put_page(p);
1320 			shake_page(p);
1321 			count_increased = false;
1322 			goto try_again;
1323 		}
1324 		put_page(p);
1325 		ret = -EIO;
1326 	}
1327 out:
1328 	if (ret == -EIO)
1329 		pr_err("%#lx: unhandlable page.\n", page_to_pfn(p));
1330 
1331 	return ret;
1332 }
1333 
1334 static int __get_unpoison_page(struct page *page)
1335 {
1336 	struct page *head = compound_head(page);
1337 	int ret = 0;
1338 	bool hugetlb = false;
1339 
1340 	ret = get_hwpoison_huge_page(head, &hugetlb);
1341 	if (hugetlb)
1342 		return ret;
1343 
1344 	/*
1345 	 * PageHWPoisonTakenOff pages are not only marked as PG_hwpoison,
1346 	 * but also isolated from buddy freelist, so need to identify the
1347 	 * state and have to cancel both operations to unpoison.
1348 	 */
1349 	if (PageHWPoisonTakenOff(page))
1350 		return -EHWPOISON;
1351 
1352 	return get_page_unless_zero(page) ? 1 : 0;
1353 }
1354 
1355 /**
1356  * get_hwpoison_page() - Get refcount for memory error handling
1357  * @p:		Raw error page (hit by memory error)
1358  * @flags:	Flags controlling behavior of error handling
1359  *
1360  * get_hwpoison_page() takes a page refcount of an error page to handle memory
1361  * error on it, after checking that the error page is in a well-defined state
1362  * (defined as a page-type we can successfully handle the memory error on it,
1363  * such as LRU page and hugetlb page).
1364  *
1365  * Memory error handling could be triggered at any time on any type of page,
1366  * so it's prone to race with typical memory management lifecycle (like
1367  * allocation and free).  So to avoid such races, get_hwpoison_page() takes
1368  * extra care for the error page's state (as done in __get_hwpoison_page()),
1369  * and has some retry logic in get_any_page().
1370  *
1371  * When called from unpoison_memory(), the caller should already ensure that
1372  * the given page has PG_hwpoison. So it's never reused for other page
1373  * allocations, and __get_unpoison_page() never races with them.
1374  *
1375  * Return: 0 on failure,
1376  *         1 on success for in-use pages in a well-defined state,
1377  *         -EIO for pages on which we can not handle memory errors,
1378  *         -EBUSY when get_hwpoison_page() has raced with page lifecycle
1379  *         operations like allocation and free,
1380  *         -EHWPOISON when the page is hwpoisoned and taken off from buddy.
1381  */
1382 static int get_hwpoison_page(struct page *p, unsigned long flags)
1383 {
1384 	int ret;
1385 
1386 	zone_pcp_disable(page_zone(p));
1387 	if (flags & MF_UNPOISON)
1388 		ret = __get_unpoison_page(p);
1389 	else
1390 		ret = get_any_page(p, flags);
1391 	zone_pcp_enable(page_zone(p));
1392 
1393 	return ret;
1394 }
1395 
1396 /*
1397  * Do all that is necessary to remove user space mappings. Unmap
1398  * the pages and send SIGBUS to the processes if the data was dirty.
1399  */
1400 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
1401 				  int flags, struct page *hpage)
1402 {
1403 	struct folio *folio = page_folio(hpage);
1404 	enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC;
1405 	struct address_space *mapping;
1406 	LIST_HEAD(tokill);
1407 	bool unmap_success;
1408 	int forcekill;
1409 	bool mlocked = PageMlocked(hpage);
1410 
1411 	/*
1412 	 * Here we are interested only in user-mapped pages, so skip any
1413 	 * other types of pages.
1414 	 */
1415 	if (PageReserved(p) || PageSlab(p) || PageTable(p))
1416 		return true;
1417 	if (!(PageLRU(hpage) || PageHuge(p)))
1418 		return true;
1419 
1420 	/*
1421 	 * This check implies we don't kill processes if their pages
1422 	 * are in the swap cache early. Those are always late kills.
1423 	 */
1424 	if (!page_mapped(hpage))
1425 		return true;
1426 
1427 	if (PageKsm(p)) {
1428 		pr_err("%#lx: can't handle KSM pages.\n", pfn);
1429 		return false;
1430 	}
1431 
1432 	if (PageSwapCache(p)) {
1433 		pr_err("%#lx: keeping poisoned page in swap cache\n", pfn);
1434 		ttu |= TTU_IGNORE_HWPOISON;
1435 	}
1436 
1437 	/*
1438 	 * Propagate the dirty bit from PTEs to struct page first, because we
1439 	 * need this to decide if we should kill or just drop the page.
1440 	 * XXX: the dirty test could be racy: set_page_dirty() may not always
1441 	 * be called inside page lock (it's recommended but not enforced).
1442 	 */
1443 	mapping = page_mapping(hpage);
1444 	if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
1445 	    mapping_can_writeback(mapping)) {
1446 		if (page_mkclean(hpage)) {
1447 			SetPageDirty(hpage);
1448 		} else {
1449 			ttu |= TTU_IGNORE_HWPOISON;
1450 			pr_info("%#lx: corrupted page was clean: dropped without side effects\n",
1451 				pfn);
1452 		}
1453 	}
1454 
1455 	/*
1456 	 * First collect all the processes that have the page
1457 	 * mapped in dirty form.  This has to be done before try_to_unmap,
1458 	 * because ttu takes the rmap data structures down.
1459 	 */
1460 	collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);
1461 
1462 	if (PageHuge(hpage) && !PageAnon(hpage)) {
1463 		/*
1464 		 * For hugetlb pages in shared mappings, try_to_unmap
1465 		 * could potentially call huge_pmd_unshare.  Because of
1466 		 * this, take semaphore in write mode here and set
1467 		 * TTU_RMAP_LOCKED to indicate we have taken the lock
1468 		 * at this higher level.
1469 		 */
1470 		mapping = hugetlb_page_mapping_lock_write(hpage);
1471 		if (mapping) {
1472 			try_to_unmap(folio, ttu|TTU_RMAP_LOCKED);
1473 			i_mmap_unlock_write(mapping);
1474 		} else
1475 			pr_info("%#lx: could not lock mapping for mapped huge page\n", pfn);
1476 	} else {
1477 		try_to_unmap(folio, ttu);
1478 	}
1479 
1480 	unmap_success = !page_mapped(hpage);
1481 	if (!unmap_success)
1482 		pr_err("%#lx: failed to unmap page (mapcount=%d)\n",
1483 		       pfn, page_mapcount(hpage));
1484 
1485 	/*
1486 	 * try_to_unmap() might put mlocked page in lru cache, so call
1487 	 * shake_page() again to ensure that it's flushed.
1488 	 */
1489 	if (mlocked)
1490 		shake_page(hpage);
1491 
1492 	/*
1493 	 * Now that the dirty bit has been propagated to the
1494 	 * struct page and all unmaps done we can decide if
1495 	 * killing is needed or not.  Only kill when the page
1496 	 * was dirty or the process is not restartable,
1497 	 * otherwise the tokill list is merely
1498 	 * freed.  When there was a problem unmapping earlier
1499 	 * use a more force-full uncatchable kill to prevent
1500 	 * any accesses to the poisoned memory.
1501 	 */
1502 	forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL) ||
1503 		    !unmap_success;
1504 	kill_procs(&tokill, forcekill, !unmap_success, pfn, flags);
1505 
1506 	return unmap_success;
1507 }
1508 
1509 static int identify_page_state(unsigned long pfn, struct page *p,
1510 				unsigned long page_flags)
1511 {
1512 	struct page_state *ps;
1513 
1514 	/*
1515 	 * The first check uses the current page flags which may not have any
1516 	 * relevant information. The second check with the saved page flags is
1517 	 * carried out only if the first check can't determine the page status.
1518 	 */
1519 	for (ps = error_states;; ps++)
1520 		if ((p->flags & ps->mask) == ps->res)
1521 			break;
1522 
1523 	page_flags |= (p->flags & (1UL << PG_dirty));
1524 
1525 	if (!ps->mask)
1526 		for (ps = error_states;; ps++)
1527 			if ((page_flags & ps->mask) == ps->res)
1528 				break;
1529 	return page_action(ps, p, pfn);
1530 }
1531 
1532 static int try_to_split_thp_page(struct page *page)
1533 {
1534 	int ret;
1535 
1536 	lock_page(page);
1537 	ret = split_huge_page(page);
1538 	unlock_page(page);
1539 
1540 	if (unlikely(ret))
1541 		put_page(page);
1542 
1543 	return ret;
1544 }
1545 
1546 static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,
1547 		struct address_space *mapping, pgoff_t index, int flags)
1548 {
1549 	struct to_kill *tk;
1550 	unsigned long size = 0;
1551 
1552 	list_for_each_entry(tk, to_kill, nd)
1553 		if (tk->size_shift)
1554 			size = max(size, 1UL << tk->size_shift);
1555 
1556 	if (size) {
1557 		/*
1558 		 * Unmap the largest mapping to avoid breaking up device-dax
1559 		 * mappings which are constant size. The actual size of the
1560 		 * mapping being torn down is communicated in siginfo, see
1561 		 * kill_proc()
1562 		 */
1563 		loff_t start = (index << PAGE_SHIFT) & ~(size - 1);
1564 
1565 		unmap_mapping_range(mapping, start, size, 0);
1566 	}
1567 
1568 	kill_procs(to_kill, flags & MF_MUST_KILL, false, pfn, flags);
1569 }
1570 
1571 static int mf_generic_kill_procs(unsigned long long pfn, int flags,
1572 		struct dev_pagemap *pgmap)
1573 {
1574 	struct page *page = pfn_to_page(pfn);
1575 	LIST_HEAD(to_kill);
1576 	dax_entry_t cookie;
1577 	int rc = 0;
1578 
1579 	/*
1580 	 * Pages instantiated by device-dax (not filesystem-dax)
1581 	 * may be compound pages.
1582 	 */
1583 	page = compound_head(page);
1584 
1585 	/*
1586 	 * Prevent the inode from being freed while we are interrogating
1587 	 * the address_space, typically this would be handled by
1588 	 * lock_page(), but dax pages do not use the page lock. This
1589 	 * also prevents changes to the mapping of this pfn until
1590 	 * poison signaling is complete.
1591 	 */
1592 	cookie = dax_lock_page(page);
1593 	if (!cookie)
1594 		return -EBUSY;
1595 
1596 	if (hwpoison_filter(page)) {
1597 		rc = -EOPNOTSUPP;
1598 		goto unlock;
1599 	}
1600 
1601 	switch (pgmap->type) {
1602 	case MEMORY_DEVICE_PRIVATE:
1603 	case MEMORY_DEVICE_COHERENT:
1604 		/*
1605 		 * TODO: Handle device pages which may need coordination
1606 		 * with device-side memory.
1607 		 */
1608 		rc = -ENXIO;
1609 		goto unlock;
1610 	default:
1611 		break;
1612 	}
1613 
1614 	/*
1615 	 * Use this flag as an indication that the dax page has been
1616 	 * remapped UC to prevent speculative consumption of poison.
1617 	 */
1618 	SetPageHWPoison(page);
1619 
1620 	/*
1621 	 * Unlike System-RAM there is no possibility to swap in a
1622 	 * different physical page at a given virtual address, so all
1623 	 * userspace consumption of ZONE_DEVICE memory necessitates
1624 	 * SIGBUS (i.e. MF_MUST_KILL)
1625 	 */
1626 	flags |= MF_ACTION_REQUIRED | MF_MUST_KILL;
1627 	collect_procs(page, &to_kill, true);
1628 
1629 	unmap_and_kill(&to_kill, pfn, page->mapping, page->index, flags);
1630 unlock:
1631 	dax_unlock_page(page, cookie);
1632 	return rc;
1633 }
1634 
1635 #ifdef CONFIG_FS_DAX
1636 /**
1637  * mf_dax_kill_procs - Collect and kill processes who are using this file range
1638  * @mapping:	address_space of the file in use
1639  * @index:	start pgoff of the range within the file
1640  * @count:	length of the range, in unit of PAGE_SIZE
1641  * @mf_flags:	memory failure flags
1642  */
1643 int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index,
1644 		unsigned long count, int mf_flags)
1645 {
1646 	LIST_HEAD(to_kill);
1647 	dax_entry_t cookie;
1648 	struct page *page;
1649 	size_t end = index + count;
1650 
1651 	mf_flags |= MF_ACTION_REQUIRED | MF_MUST_KILL;
1652 
1653 	for (; index < end; index++) {
1654 		page = NULL;
1655 		cookie = dax_lock_mapping_entry(mapping, index, &page);
1656 		if (!cookie)
1657 			return -EBUSY;
1658 		if (!page)
1659 			goto unlock;
1660 
1661 		SetPageHWPoison(page);
1662 
1663 		collect_procs_fsdax(page, mapping, index, &to_kill);
1664 		unmap_and_kill(&to_kill, page_to_pfn(page), mapping,
1665 				index, mf_flags);
1666 unlock:
1667 		dax_unlock_mapping_entry(mapping, index, cookie);
1668 	}
1669 	return 0;
1670 }
1671 EXPORT_SYMBOL_GPL(mf_dax_kill_procs);
1672 #endif /* CONFIG_FS_DAX */
1673 
1674 #ifdef CONFIG_HUGETLB_PAGE
1675 /*
1676  * Struct raw_hwp_page represents information about "raw error page",
1677  * constructing singly linked list originated from ->private field of
1678  * SUBPAGE_INDEX_HWPOISON-th tail page.
1679  */
1680 struct raw_hwp_page {
1681 	struct llist_node node;
1682 	struct page *page;
1683 };
1684 
1685 static inline struct llist_head *raw_hwp_list_head(struct page *hpage)
1686 {
1687 	return (struct llist_head *)&page_private(hpage + SUBPAGE_INDEX_HWPOISON);
1688 }
1689 
1690 static unsigned long __free_raw_hwp_pages(struct page *hpage, bool move_flag)
1691 {
1692 	struct llist_head *head;
1693 	struct llist_node *t, *tnode;
1694 	unsigned long count = 0;
1695 
1696 	head = raw_hwp_list_head(hpage);
1697 	llist_for_each_safe(tnode, t, head->first) {
1698 		struct raw_hwp_page *p = container_of(tnode, struct raw_hwp_page, node);
1699 
1700 		if (move_flag)
1701 			SetPageHWPoison(p->page);
1702 		kfree(p);
1703 		count++;
1704 	}
1705 	llist_del_all(head);
1706 	return count;
1707 }
1708 
1709 static int hugetlb_set_page_hwpoison(struct page *hpage, struct page *page)
1710 {
1711 	struct llist_head *head;
1712 	struct raw_hwp_page *raw_hwp;
1713 	struct llist_node *t, *tnode;
1714 	int ret = TestSetPageHWPoison(hpage) ? -EHWPOISON : 0;
1715 
1716 	/*
1717 	 * Once the hwpoison hugepage has lost reliable raw error info,
1718 	 * there is little meaning to keep additional error info precisely,
1719 	 * so skip to add additional raw error info.
1720 	 */
1721 	if (HPageRawHwpUnreliable(hpage))
1722 		return -EHWPOISON;
1723 	head = raw_hwp_list_head(hpage);
1724 	llist_for_each_safe(tnode, t, head->first) {
1725 		struct raw_hwp_page *p = container_of(tnode, struct raw_hwp_page, node);
1726 
1727 		if (p->page == page)
1728 			return -EHWPOISON;
1729 	}
1730 
1731 	raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC);
1732 	if (raw_hwp) {
1733 		raw_hwp->page = page;
1734 		llist_add(&raw_hwp->node, head);
1735 		/* the first error event will be counted in action_result(). */
1736 		if (ret)
1737 			num_poisoned_pages_inc();
1738 	} else {
1739 		/*
1740 		 * Failed to save raw error info.  We no longer trace all
1741 		 * hwpoisoned subpages, and we need refuse to free/dissolve
1742 		 * this hwpoisoned hugepage.
1743 		 */
1744 		SetHPageRawHwpUnreliable(hpage);
1745 		/*
1746 		 * Once HPageRawHwpUnreliable is set, raw_hwp_page is not
1747 		 * used any more, so free it.
1748 		 */
1749 		__free_raw_hwp_pages(hpage, false);
1750 	}
1751 	return ret;
1752 }
1753 
1754 static unsigned long free_raw_hwp_pages(struct page *hpage, bool move_flag)
1755 {
1756 	/*
1757 	 * HPageVmemmapOptimized hugepages can't be freed because struct
1758 	 * pages for tail pages are required but they don't exist.
1759 	 */
1760 	if (move_flag && HPageVmemmapOptimized(hpage))
1761 		return 0;
1762 
1763 	/*
1764 	 * HPageRawHwpUnreliable hugepages shouldn't be unpoisoned by
1765 	 * definition.
1766 	 */
1767 	if (HPageRawHwpUnreliable(hpage))
1768 		return 0;
1769 
1770 	return __free_raw_hwp_pages(hpage, move_flag);
1771 }
1772 
1773 void hugetlb_clear_page_hwpoison(struct page *hpage)
1774 {
1775 	if (HPageRawHwpUnreliable(hpage))
1776 		return;
1777 	ClearPageHWPoison(hpage);
1778 	free_raw_hwp_pages(hpage, true);
1779 }
1780 
1781 /*
1782  * Called from hugetlb code with hugetlb_lock held.
1783  *
1784  * Return values:
1785  *   0             - free hugepage
1786  *   1             - in-use hugepage
1787  *   2             - not a hugepage
1788  *   -EBUSY        - the hugepage is busy (try to retry)
1789  *   -EHWPOISON    - the hugepage is already hwpoisoned
1790  */
1791 int __get_huge_page_for_hwpoison(unsigned long pfn, int flags)
1792 {
1793 	struct page *page = pfn_to_page(pfn);
1794 	struct page *head = compound_head(page);
1795 	int ret = 2;	/* fallback to normal page handling */
1796 	bool count_increased = false;
1797 
1798 	if (!PageHeadHuge(head))
1799 		goto out;
1800 
1801 	if (flags & MF_COUNT_INCREASED) {
1802 		ret = 1;
1803 		count_increased = true;
1804 	} else if (HPageFreed(head)) {
1805 		ret = 0;
1806 	} else if (HPageMigratable(head)) {
1807 		ret = get_page_unless_zero(head);
1808 		if (ret)
1809 			count_increased = true;
1810 	} else {
1811 		ret = -EBUSY;
1812 		if (!(flags & MF_NO_RETRY))
1813 			goto out;
1814 	}
1815 
1816 	if (hugetlb_set_page_hwpoison(head, page)) {
1817 		ret = -EHWPOISON;
1818 		goto out;
1819 	}
1820 
1821 	return ret;
1822 out:
1823 	if (count_increased)
1824 		put_page(head);
1825 	return ret;
1826 }
1827 
1828 /*
1829  * Taking refcount of hugetlb pages needs extra care about race conditions
1830  * with basic operations like hugepage allocation/free/demotion.
1831  * So some of prechecks for hwpoison (pinning, and testing/setting
1832  * PageHWPoison) should be done in single hugetlb_lock range.
1833  */
1834 static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)
1835 {
1836 	int res;
1837 	struct page *p = pfn_to_page(pfn);
1838 	struct page *head;
1839 	unsigned long page_flags;
1840 
1841 	*hugetlb = 1;
1842 retry:
1843 	res = get_huge_page_for_hwpoison(pfn, flags);
1844 	if (res == 2) { /* fallback to normal page handling */
1845 		*hugetlb = 0;
1846 		return 0;
1847 	} else if (res == -EHWPOISON) {
1848 		pr_err("%#lx: already hardware poisoned\n", pfn);
1849 		if (flags & MF_ACTION_REQUIRED) {
1850 			head = compound_head(p);
1851 			res = kill_accessing_process(current, page_to_pfn(head), flags);
1852 		}
1853 		return res;
1854 	} else if (res == -EBUSY) {
1855 		if (!(flags & MF_NO_RETRY)) {
1856 			flags |= MF_NO_RETRY;
1857 			goto retry;
1858 		}
1859 		action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED);
1860 		return res;
1861 	}
1862 
1863 	head = compound_head(p);
1864 	lock_page(head);
1865 
1866 	if (hwpoison_filter(p)) {
1867 		hugetlb_clear_page_hwpoison(head);
1868 		unlock_page(head);
1869 		if (res == 1)
1870 			put_page(head);
1871 		return -EOPNOTSUPP;
1872 	}
1873 
1874 	/*
1875 	 * Handling free hugepage.  The possible race with hugepage allocation
1876 	 * or demotion can be prevented by PageHWPoison flag.
1877 	 */
1878 	if (res == 0) {
1879 		unlock_page(head);
1880 		if (__page_handle_poison(p) >= 0) {
1881 			page_ref_inc(p);
1882 			res = MF_RECOVERED;
1883 		} else {
1884 			res = MF_FAILED;
1885 		}
1886 		action_result(pfn, MF_MSG_FREE_HUGE, res);
1887 		return res == MF_RECOVERED ? 0 : -EBUSY;
1888 	}
1889 
1890 	page_flags = head->flags;
1891 
1892 	if (!hwpoison_user_mappings(p, pfn, flags, head)) {
1893 		action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
1894 		res = -EBUSY;
1895 		goto out;
1896 	}
1897 
1898 	return identify_page_state(pfn, p, page_flags);
1899 out:
1900 	unlock_page(head);
1901 	return res;
1902 }
1903 
1904 #else
1905 static inline int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)
1906 {
1907 	return 0;
1908 }
1909 
1910 static inline unsigned long free_raw_hwp_pages(struct page *hpage, bool flag)
1911 {
1912 	return 0;
1913 }
1914 #endif	/* CONFIG_HUGETLB_PAGE */
1915 
1916 static int memory_failure_dev_pagemap(unsigned long pfn, int flags,
1917 		struct dev_pagemap *pgmap)
1918 {
1919 	struct page *page = pfn_to_page(pfn);
1920 	int rc = -ENXIO;
1921 
1922 	if (flags & MF_COUNT_INCREASED)
1923 		/*
1924 		 * Drop the extra refcount in case we come from madvise().
1925 		 */
1926 		put_page(page);
1927 
1928 	/* device metadata space is not recoverable */
1929 	if (!pgmap_pfn_valid(pgmap, pfn))
1930 		goto out;
1931 
1932 	/*
1933 	 * Call driver's implementation to handle the memory failure, otherwise
1934 	 * fall back to generic handler.
1935 	 */
1936 	if (pgmap_has_memory_failure(pgmap)) {
1937 		rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags);
1938 		/*
1939 		 * Fall back to generic handler too if operation is not
1940 		 * supported inside the driver/device/filesystem.
1941 		 */
1942 		if (rc != -EOPNOTSUPP)
1943 			goto out;
1944 	}
1945 
1946 	rc = mf_generic_kill_procs(pfn, flags, pgmap);
1947 out:
1948 	/* drop pgmap ref acquired in caller */
1949 	put_dev_pagemap(pgmap);
1950 	action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED);
1951 	return rc;
1952 }
1953 
1954 static DEFINE_MUTEX(mf_mutex);
1955 
1956 /**
1957  * memory_failure - Handle memory failure of a page.
1958  * @pfn: Page Number of the corrupted page
1959  * @flags: fine tune action taken
1960  *
1961  * This function is called by the low level machine check code
1962  * of an architecture when it detects hardware memory corruption
1963  * of a page. It tries its best to recover, which includes
1964  * dropping pages, killing processes etc.
1965  *
1966  * The function is primarily of use for corruptions that
1967  * happen outside the current execution context (e.g. when
1968  * detected by a background scrubber)
1969  *
1970  * Must run in process context (e.g. a work queue) with interrupts
1971  * enabled and no spinlocks hold.
1972  *
1973  * Return: 0 for successfully handled the memory error,
1974  *         -EOPNOTSUPP for hwpoison_filter() filtered the error event,
1975  *         < 0(except -EOPNOTSUPP) on failure.
1976  */
1977 int memory_failure(unsigned long pfn, int flags)
1978 {
1979 	struct page *p;
1980 	struct page *hpage;
1981 	struct dev_pagemap *pgmap;
1982 	int res = 0;
1983 	unsigned long page_flags;
1984 	bool retry = true;
1985 	int hugetlb = 0;
1986 
1987 	if (!sysctl_memory_failure_recovery)
1988 		panic("Memory failure on page %lx", pfn);
1989 
1990 	mutex_lock(&mf_mutex);
1991 
1992 	if (!(flags & MF_SW_SIMULATED))
1993 		hw_memory_failure = true;
1994 
1995 	p = pfn_to_online_page(pfn);
1996 	if (!p) {
1997 		res = arch_memory_failure(pfn, flags);
1998 		if (res == 0)
1999 			goto unlock_mutex;
2000 
2001 		if (pfn_valid(pfn)) {
2002 			pgmap = get_dev_pagemap(pfn, NULL);
2003 			if (pgmap) {
2004 				res = memory_failure_dev_pagemap(pfn, flags,
2005 								 pgmap);
2006 				goto unlock_mutex;
2007 			}
2008 		}
2009 		pr_err("%#lx: memory outside kernel control\n", pfn);
2010 		res = -ENXIO;
2011 		goto unlock_mutex;
2012 	}
2013 
2014 try_again:
2015 	res = try_memory_failure_hugetlb(pfn, flags, &hugetlb);
2016 	if (hugetlb)
2017 		goto unlock_mutex;
2018 
2019 	if (TestSetPageHWPoison(p)) {
2020 		pr_err("%#lx: already hardware poisoned\n", pfn);
2021 		res = -EHWPOISON;
2022 		if (flags & MF_ACTION_REQUIRED)
2023 			res = kill_accessing_process(current, pfn, flags);
2024 		if (flags & MF_COUNT_INCREASED)
2025 			put_page(p);
2026 		goto unlock_mutex;
2027 	}
2028 
2029 	hpage = compound_head(p);
2030 
2031 	/*
2032 	 * We need/can do nothing about count=0 pages.
2033 	 * 1) it's a free page, and therefore in safe hand:
2034 	 *    check_new_page() will be the gate keeper.
2035 	 * 2) it's part of a non-compound high order page.
2036 	 *    Implies some kernel user: cannot stop them from
2037 	 *    R/W the page; let's pray that the page has been
2038 	 *    used and will be freed some time later.
2039 	 * In fact it's dangerous to directly bump up page count from 0,
2040 	 * that may make page_ref_freeze()/page_ref_unfreeze() mismatch.
2041 	 */
2042 	if (!(flags & MF_COUNT_INCREASED)) {
2043 		res = get_hwpoison_page(p, flags);
2044 		if (!res) {
2045 			if (is_free_buddy_page(p)) {
2046 				if (take_page_off_buddy(p)) {
2047 					page_ref_inc(p);
2048 					res = MF_RECOVERED;
2049 				} else {
2050 					/* We lost the race, try again */
2051 					if (retry) {
2052 						ClearPageHWPoison(p);
2053 						retry = false;
2054 						goto try_again;
2055 					}
2056 					res = MF_FAILED;
2057 				}
2058 				action_result(pfn, MF_MSG_BUDDY, res);
2059 				res = res == MF_RECOVERED ? 0 : -EBUSY;
2060 			} else {
2061 				action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);
2062 				res = -EBUSY;
2063 			}
2064 			goto unlock_mutex;
2065 		} else if (res < 0) {
2066 			action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED);
2067 			res = -EBUSY;
2068 			goto unlock_mutex;
2069 		}
2070 	}
2071 
2072 	if (PageTransHuge(hpage)) {
2073 		/*
2074 		 * The flag must be set after the refcount is bumped
2075 		 * otherwise it may race with THP split.
2076 		 * And the flag can't be set in get_hwpoison_page() since
2077 		 * it is called by soft offline too and it is just called
2078 		 * for !MF_COUNT_INCREASE.  So here seems to be the best
2079 		 * place.
2080 		 *
2081 		 * Don't need care about the above error handling paths for
2082 		 * get_hwpoison_page() since they handle either free page
2083 		 * or unhandlable page.  The refcount is bumped iff the
2084 		 * page is a valid handlable page.
2085 		 */
2086 		SetPageHasHWPoisoned(hpage);
2087 		if (try_to_split_thp_page(p) < 0) {
2088 			action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED);
2089 			res = -EBUSY;
2090 			goto unlock_mutex;
2091 		}
2092 		VM_BUG_ON_PAGE(!page_count(p), p);
2093 	}
2094 
2095 	/*
2096 	 * We ignore non-LRU pages for good reasons.
2097 	 * - PG_locked is only well defined for LRU pages and a few others
2098 	 * - to avoid races with __SetPageLocked()
2099 	 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
2100 	 * The check (unnecessarily) ignores LRU pages being isolated and
2101 	 * walked by the page reclaim code, however that's not a big loss.
2102 	 */
2103 	shake_page(p);
2104 
2105 	lock_page(p);
2106 
2107 	/*
2108 	 * We're only intended to deal with the non-Compound page here.
2109 	 * However, the page could have changed compound pages due to
2110 	 * race window. If this happens, we could try again to hopefully
2111 	 * handle the page next round.
2112 	 */
2113 	if (PageCompound(p)) {
2114 		if (retry) {
2115 			ClearPageHWPoison(p);
2116 			unlock_page(p);
2117 			put_page(p);
2118 			flags &= ~MF_COUNT_INCREASED;
2119 			retry = false;
2120 			goto try_again;
2121 		}
2122 		action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED);
2123 		res = -EBUSY;
2124 		goto unlock_page;
2125 	}
2126 
2127 	/*
2128 	 * We use page flags to determine what action should be taken, but
2129 	 * the flags can be modified by the error containment action.  One
2130 	 * example is an mlocked page, where PG_mlocked is cleared by
2131 	 * page_remove_rmap() in try_to_unmap_one(). So to determine page status
2132 	 * correctly, we save a copy of the page flags at this time.
2133 	 */
2134 	page_flags = p->flags;
2135 
2136 	if (hwpoison_filter(p)) {
2137 		ClearPageHWPoison(p);
2138 		unlock_page(p);
2139 		put_page(p);
2140 		res = -EOPNOTSUPP;
2141 		goto unlock_mutex;
2142 	}
2143 
2144 	/*
2145 	 * __munlock_pagevec may clear a writeback page's LRU flag without
2146 	 * page_lock. We need wait writeback completion for this page or it
2147 	 * may trigger vfs BUG while evict inode.
2148 	 */
2149 	if (!PageLRU(p) && !PageWriteback(p))
2150 		goto identify_page_state;
2151 
2152 	/*
2153 	 * It's very difficult to mess with pages currently under IO
2154 	 * and in many cases impossible, so we just avoid it here.
2155 	 */
2156 	wait_on_page_writeback(p);
2157 
2158 	/*
2159 	 * Now take care of user space mappings.
2160 	 * Abort on fail: __filemap_remove_folio() assumes unmapped page.
2161 	 */
2162 	if (!hwpoison_user_mappings(p, pfn, flags, p)) {
2163 		action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
2164 		res = -EBUSY;
2165 		goto unlock_page;
2166 	}
2167 
2168 	/*
2169 	 * Torn down by someone else?
2170 	 */
2171 	if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
2172 		action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED);
2173 		res = -EBUSY;
2174 		goto unlock_page;
2175 	}
2176 
2177 identify_page_state:
2178 	res = identify_page_state(pfn, p, page_flags);
2179 	mutex_unlock(&mf_mutex);
2180 	return res;
2181 unlock_page:
2182 	unlock_page(p);
2183 unlock_mutex:
2184 	mutex_unlock(&mf_mutex);
2185 	return res;
2186 }
2187 EXPORT_SYMBOL_GPL(memory_failure);
2188 
2189 #define MEMORY_FAILURE_FIFO_ORDER	4
2190 #define MEMORY_FAILURE_FIFO_SIZE	(1 << MEMORY_FAILURE_FIFO_ORDER)
2191 
2192 struct memory_failure_entry {
2193 	unsigned long pfn;
2194 	int flags;
2195 };
2196 
2197 struct memory_failure_cpu {
2198 	DECLARE_KFIFO(fifo, struct memory_failure_entry,
2199 		      MEMORY_FAILURE_FIFO_SIZE);
2200 	spinlock_t lock;
2201 	struct work_struct work;
2202 };
2203 
2204 static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
2205 
2206 /**
2207  * memory_failure_queue - Schedule handling memory failure of a page.
2208  * @pfn: Page Number of the corrupted page
2209  * @flags: Flags for memory failure handling
2210  *
2211  * This function is called by the low level hardware error handler
2212  * when it detects hardware memory corruption of a page. It schedules
2213  * the recovering of error page, including dropping pages, killing
2214  * processes etc.
2215  *
2216  * The function is primarily of use for corruptions that
2217  * happen outside the current execution context (e.g. when
2218  * detected by a background scrubber)
2219  *
2220  * Can run in IRQ context.
2221  */
2222 void memory_failure_queue(unsigned long pfn, int flags)
2223 {
2224 	struct memory_failure_cpu *mf_cpu;
2225 	unsigned long proc_flags;
2226 	struct memory_failure_entry entry = {
2227 		.pfn =		pfn,
2228 		.flags =	flags,
2229 	};
2230 
2231 	mf_cpu = &get_cpu_var(memory_failure_cpu);
2232 	spin_lock_irqsave(&mf_cpu->lock, proc_flags);
2233 	if (kfifo_put(&mf_cpu->fifo, entry))
2234 		schedule_work_on(smp_processor_id(), &mf_cpu->work);
2235 	else
2236 		pr_err("buffer overflow when queuing memory failure at %#lx\n",
2237 		       pfn);
2238 	spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
2239 	put_cpu_var(memory_failure_cpu);
2240 }
2241 EXPORT_SYMBOL_GPL(memory_failure_queue);
2242 
2243 static void memory_failure_work_func(struct work_struct *work)
2244 {
2245 	struct memory_failure_cpu *mf_cpu;
2246 	struct memory_failure_entry entry = { 0, };
2247 	unsigned long proc_flags;
2248 	int gotten;
2249 
2250 	mf_cpu = container_of(work, struct memory_failure_cpu, work);
2251 	for (;;) {
2252 		spin_lock_irqsave(&mf_cpu->lock, proc_flags);
2253 		gotten = kfifo_get(&mf_cpu->fifo, &entry);
2254 		spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
2255 		if (!gotten)
2256 			break;
2257 		if (entry.flags & MF_SOFT_OFFLINE)
2258 			soft_offline_page(entry.pfn, entry.flags);
2259 		else
2260 			memory_failure(entry.pfn, entry.flags);
2261 	}
2262 }
2263 
2264 /*
2265  * Process memory_failure work queued on the specified CPU.
2266  * Used to avoid return-to-userspace racing with the memory_failure workqueue.
2267  */
2268 void memory_failure_queue_kick(int cpu)
2269 {
2270 	struct memory_failure_cpu *mf_cpu;
2271 
2272 	mf_cpu = &per_cpu(memory_failure_cpu, cpu);
2273 	cancel_work_sync(&mf_cpu->work);
2274 	memory_failure_work_func(&mf_cpu->work);
2275 }
2276 
2277 static int __init memory_failure_init(void)
2278 {
2279 	struct memory_failure_cpu *mf_cpu;
2280 	int cpu;
2281 
2282 	for_each_possible_cpu(cpu) {
2283 		mf_cpu = &per_cpu(memory_failure_cpu, cpu);
2284 		spin_lock_init(&mf_cpu->lock);
2285 		INIT_KFIFO(mf_cpu->fifo);
2286 		INIT_WORK(&mf_cpu->work, memory_failure_work_func);
2287 	}
2288 
2289 	return 0;
2290 }
2291 core_initcall(memory_failure_init);
2292 
2293 #undef pr_fmt
2294 #define pr_fmt(fmt)	"" fmt
2295 #define unpoison_pr_info(fmt, pfn, rs)			\
2296 ({							\
2297 	if (__ratelimit(rs))				\
2298 		pr_info(fmt, pfn);			\
2299 })
2300 
2301 /**
2302  * unpoison_memory - Unpoison a previously poisoned page
2303  * @pfn: Page number of the to be unpoisoned page
2304  *
2305  * Software-unpoison a page that has been poisoned by
2306  * memory_failure() earlier.
2307  *
2308  * This is only done on the software-level, so it only works
2309  * for linux injected failures, not real hardware failures
2310  *
2311  * Returns 0 for success, otherwise -errno.
2312  */
2313 int unpoison_memory(unsigned long pfn)
2314 {
2315 	struct page *page;
2316 	struct page *p;
2317 	int ret = -EBUSY;
2318 	int freeit = 0;
2319 	unsigned long count = 1;
2320 	static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL,
2321 					DEFAULT_RATELIMIT_BURST);
2322 
2323 	if (!pfn_valid(pfn))
2324 		return -ENXIO;
2325 
2326 	p = pfn_to_page(pfn);
2327 	page = compound_head(p);
2328 
2329 	mutex_lock(&mf_mutex);
2330 
2331 	if (hw_memory_failure) {
2332 		unpoison_pr_info("Unpoison: Disabled after HW memory failure %#lx\n",
2333 				 pfn, &unpoison_rs);
2334 		ret = -EOPNOTSUPP;
2335 		goto unlock_mutex;
2336 	}
2337 
2338 	if (!PageHWPoison(p)) {
2339 		unpoison_pr_info("Unpoison: Page was already unpoisoned %#lx\n",
2340 				 pfn, &unpoison_rs);
2341 		goto unlock_mutex;
2342 	}
2343 
2344 	if (page_count(page) > 1) {
2345 		unpoison_pr_info("Unpoison: Someone grabs the hwpoison page %#lx\n",
2346 				 pfn, &unpoison_rs);
2347 		goto unlock_mutex;
2348 	}
2349 
2350 	if (page_mapped(page)) {
2351 		unpoison_pr_info("Unpoison: Someone maps the hwpoison page %#lx\n",
2352 				 pfn, &unpoison_rs);
2353 		goto unlock_mutex;
2354 	}
2355 
2356 	if (page_mapping(page)) {
2357 		unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n",
2358 				 pfn, &unpoison_rs);
2359 		goto unlock_mutex;
2360 	}
2361 
2362 	if (PageSlab(page) || PageTable(page) || PageReserved(page))
2363 		goto unlock_mutex;
2364 
2365 	ret = get_hwpoison_page(p, MF_UNPOISON);
2366 	if (!ret) {
2367 		if (PageHuge(p)) {
2368 			count = free_raw_hwp_pages(page, false);
2369 			if (count == 0) {
2370 				ret = -EBUSY;
2371 				goto unlock_mutex;
2372 			}
2373 		}
2374 		ret = TestClearPageHWPoison(page) ? 0 : -EBUSY;
2375 	} else if (ret < 0) {
2376 		if (ret == -EHWPOISON) {
2377 			ret = put_page_back_buddy(p) ? 0 : -EBUSY;
2378 		} else
2379 			unpoison_pr_info("Unpoison: failed to grab page %#lx\n",
2380 					 pfn, &unpoison_rs);
2381 	} else {
2382 		if (PageHuge(p)) {
2383 			count = free_raw_hwp_pages(page, false);
2384 			if (count == 0) {
2385 				ret = -EBUSY;
2386 				put_page(page);
2387 				goto unlock_mutex;
2388 			}
2389 		}
2390 		freeit = !!TestClearPageHWPoison(p);
2391 
2392 		put_page(page);
2393 		if (freeit) {
2394 			put_page(page);
2395 			ret = 0;
2396 		}
2397 	}
2398 
2399 unlock_mutex:
2400 	mutex_unlock(&mf_mutex);
2401 	if (!ret || freeit) {
2402 		num_poisoned_pages_sub(count);
2403 		unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n",
2404 				 page_to_pfn(p), &unpoison_rs);
2405 	}
2406 	return ret;
2407 }
2408 EXPORT_SYMBOL(unpoison_memory);
2409 
2410 static bool isolate_page(struct page *page, struct list_head *pagelist)
2411 {
2412 	bool isolated = false;
2413 
2414 	if (PageHuge(page)) {
2415 		isolated = !isolate_hugetlb(page, pagelist);
2416 	} else {
2417 		bool lru = !__PageMovable(page);
2418 
2419 		if (lru)
2420 			isolated = !isolate_lru_page(page);
2421 		else
2422 			isolated = !isolate_movable_page(page,
2423 							 ISOLATE_UNEVICTABLE);
2424 
2425 		if (isolated) {
2426 			list_add(&page->lru, pagelist);
2427 			if (lru)
2428 				inc_node_page_state(page, NR_ISOLATED_ANON +
2429 						    page_is_file_lru(page));
2430 		}
2431 	}
2432 
2433 	/*
2434 	 * If we succeed to isolate the page, we grabbed another refcount on
2435 	 * the page, so we can safely drop the one we got from get_any_pages().
2436 	 * If we failed to isolate the page, it means that we cannot go further
2437 	 * and we will return an error, so drop the reference we got from
2438 	 * get_any_pages() as well.
2439 	 */
2440 	put_page(page);
2441 	return isolated;
2442 }
2443 
2444 /*
2445  * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages.
2446  * If the page is a non-dirty unmapped page-cache page, it simply invalidates.
2447  * If the page is mapped, it migrates the contents over.
2448  */
2449 static int soft_offline_in_use_page(struct page *page)
2450 {
2451 	long ret = 0;
2452 	unsigned long pfn = page_to_pfn(page);
2453 	struct page *hpage = compound_head(page);
2454 	char const *msg_page[] = {"page", "hugepage"};
2455 	bool huge = PageHuge(page);
2456 	LIST_HEAD(pagelist);
2457 	struct migration_target_control mtc = {
2458 		.nid = NUMA_NO_NODE,
2459 		.gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
2460 	};
2461 
2462 	if (!huge && PageTransHuge(hpage)) {
2463 		if (try_to_split_thp_page(page)) {
2464 			pr_info("soft offline: %#lx: thp split failed\n", pfn);
2465 			return -EBUSY;
2466 		}
2467 		hpage = page;
2468 	}
2469 
2470 	lock_page(page);
2471 	if (!PageHuge(page))
2472 		wait_on_page_writeback(page);
2473 	if (PageHWPoison(page)) {
2474 		unlock_page(page);
2475 		put_page(page);
2476 		pr_info("soft offline: %#lx page already poisoned\n", pfn);
2477 		return 0;
2478 	}
2479 
2480 	if (!PageHuge(page) && PageLRU(page) && !PageSwapCache(page))
2481 		/*
2482 		 * Try to invalidate first. This should work for
2483 		 * non dirty unmapped page cache pages.
2484 		 */
2485 		ret = invalidate_inode_page(page);
2486 	unlock_page(page);
2487 
2488 	if (ret) {
2489 		pr_info("soft_offline: %#lx: invalidated\n", pfn);
2490 		page_handle_poison(page, false, true);
2491 		return 0;
2492 	}
2493 
2494 	if (isolate_page(hpage, &pagelist)) {
2495 		ret = migrate_pages(&pagelist, alloc_migration_target, NULL,
2496 			(unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_FAILURE, NULL);
2497 		if (!ret) {
2498 			bool release = !huge;
2499 
2500 			if (!page_handle_poison(page, huge, release))
2501 				ret = -EBUSY;
2502 		} else {
2503 			if (!list_empty(&pagelist))
2504 				putback_movable_pages(&pagelist);
2505 
2506 			pr_info("soft offline: %#lx: %s migration failed %ld, type %pGp\n",
2507 				pfn, msg_page[huge], ret, &page->flags);
2508 			if (ret > 0)
2509 				ret = -EBUSY;
2510 		}
2511 	} else {
2512 		pr_info("soft offline: %#lx: %s isolation failed, page count %d, type %pGp\n",
2513 			pfn, msg_page[huge], page_count(page), &page->flags);
2514 		ret = -EBUSY;
2515 	}
2516 	return ret;
2517 }
2518 
2519 static void put_ref_page(struct page *page)
2520 {
2521 	if (page)
2522 		put_page(page);
2523 }
2524 
2525 /**
2526  * soft_offline_page - Soft offline a page.
2527  * @pfn: pfn to soft-offline
2528  * @flags: flags. Same as memory_failure().
2529  *
2530  * Returns 0 on success
2531  *         -EOPNOTSUPP for hwpoison_filter() filtered the error event
2532  *         < 0 otherwise negated errno.
2533  *
2534  * Soft offline a page, by migration or invalidation,
2535  * without killing anything. This is for the case when
2536  * a page is not corrupted yet (so it's still valid to access),
2537  * but has had a number of corrected errors and is better taken
2538  * out.
2539  *
2540  * The actual policy on when to do that is maintained by
2541  * user space.
2542  *
2543  * This should never impact any application or cause data loss,
2544  * however it might take some time.
2545  *
2546  * This is not a 100% solution for all memory, but tries to be
2547  * ``good enough'' for the majority of memory.
2548  */
2549 int soft_offline_page(unsigned long pfn, int flags)
2550 {
2551 	int ret;
2552 	bool try_again = true;
2553 	struct page *page, *ref_page = NULL;
2554 
2555 	WARN_ON_ONCE(!pfn_valid(pfn) && (flags & MF_COUNT_INCREASED));
2556 
2557 	if (!pfn_valid(pfn))
2558 		return -ENXIO;
2559 	if (flags & MF_COUNT_INCREASED)
2560 		ref_page = pfn_to_page(pfn);
2561 
2562 	/* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
2563 	page = pfn_to_online_page(pfn);
2564 	if (!page) {
2565 		put_ref_page(ref_page);
2566 		return -EIO;
2567 	}
2568 
2569 	mutex_lock(&mf_mutex);
2570 
2571 	if (PageHWPoison(page)) {
2572 		pr_info("%s: %#lx page already poisoned\n", __func__, pfn);
2573 		put_ref_page(ref_page);
2574 		mutex_unlock(&mf_mutex);
2575 		return 0;
2576 	}
2577 
2578 retry:
2579 	get_online_mems();
2580 	ret = get_hwpoison_page(page, flags | MF_SOFT_OFFLINE);
2581 	put_online_mems();
2582 
2583 	if (hwpoison_filter(page)) {
2584 		if (ret > 0)
2585 			put_page(page);
2586 
2587 		mutex_unlock(&mf_mutex);
2588 		return -EOPNOTSUPP;
2589 	}
2590 
2591 	if (ret > 0) {
2592 		ret = soft_offline_in_use_page(page);
2593 	} else if (ret == 0) {
2594 		if (!page_handle_poison(page, true, false) && try_again) {
2595 			try_again = false;
2596 			flags &= ~MF_COUNT_INCREASED;
2597 			goto retry;
2598 		}
2599 	}
2600 
2601 	mutex_unlock(&mf_mutex);
2602 
2603 	return ret;
2604 }
2605 
2606 void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
2607 {
2608 	int i, total = 0;
2609 
2610 	/*
2611 	 * A further optimization is to have per section refcounted
2612 	 * num_poisoned_pages.  But that would need more space per memmap, so
2613 	 * for now just do a quick global check to speed up this routine in the
2614 	 * absence of bad pages.
2615 	 */
2616 	if (atomic_long_read(&num_poisoned_pages) == 0)
2617 		return;
2618 
2619 	for (i = 0; i < nr_pages; i++) {
2620 		if (PageHWPoison(&memmap[i])) {
2621 			total++;
2622 			ClearPageHWPoison(&memmap[i]);
2623 		}
2624 	}
2625 	if (total)
2626 		num_poisoned_pages_sub(total);
2627 }
2628