182869ac5SJames Morse /*: 282869ac5SJames Morse * Hibernate support specific for ARM64 382869ac5SJames Morse * 482869ac5SJames Morse * Derived from work on ARM hibernation support by: 582869ac5SJames Morse * 682869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove 782869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) 882869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) 982869ac5SJames Morse * https://lkml.org/lkml/2010/6/18/4 1082869ac5SJames Morse * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html 1182869ac5SJames Morse * https://patchwork.kernel.org/patch/96442/ 1282869ac5SJames Morse * 1382869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 1482869ac5SJames Morse * 1582869ac5SJames Morse * License terms: GNU General Public License (GPL) version 2 1682869ac5SJames Morse */ 1782869ac5SJames Morse #define pr_fmt(x) "hibernate: " x 1882869ac5SJames Morse #include <linux/kvm_host.h> 1982869ac5SJames Morse #include <linux/mm.h> 201fe492ceSJames Morse #include <linux/notifier.h> 2182869ac5SJames Morse #include <linux/pm.h> 2282869ac5SJames Morse #include <linux/sched.h> 2382869ac5SJames Morse #include <linux/suspend.h> 2482869ac5SJames Morse #include <linux/utsname.h> 2582869ac5SJames Morse #include <linux/version.h> 2682869ac5SJames Morse 2782869ac5SJames Morse #include <asm/barrier.h> 2882869ac5SJames Morse #include <asm/cacheflush.h> 2982869ac5SJames Morse #include <asm/irqflags.h> 3082869ac5SJames Morse #include <asm/memory.h> 3182869ac5SJames Morse #include <asm/mmu_context.h> 3282869ac5SJames Morse #include <asm/pgalloc.h> 3382869ac5SJames Morse #include <asm/pgtable.h> 3482869ac5SJames Morse #include <asm/pgtable-hwdef.h> 3582869ac5SJames Morse #include <asm/sections.h> 36d74b4e4fSJames Morse #include <asm/smp.h> 3782869ac5SJames Morse #include <asm/suspend.h> 3882869ac5SJames Morse #include <asm/virt.h> 3982869ac5SJames Morse 4082869ac5SJames Morse /* 4182869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it 4282869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This 4382869ac5SJames Morse * doesn't happen with either KASLR. 4482869ac5SJames Morse * 4582869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in 4682869ac5SJames Morse * kernel/power/hibernate.c 4782869ac5SJames Morse */ 4882869ac5SJames Morse extern int in_suspend; 4982869ac5SJames Morse 5082869ac5SJames Morse /* Find a symbols alias in the linear map */ 5182869ac5SJames Morse #define LMADDR(x) phys_to_virt(virt_to_phys(x)) 5282869ac5SJames Morse 5382869ac5SJames Morse /* Do we need to reset el2? */ 5482869ac5SJames Morse #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) 5582869ac5SJames Morse 5682869ac5SJames Morse /* 5782869ac5SJames Morse * Start/end of the hibernate exit code, this must be copied to a 'safe' 5882869ac5SJames Morse * location in memory, and executed from there. 5982869ac5SJames Morse */ 6082869ac5SJames Morse extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[]; 6182869ac5SJames Morse 6282869ac5SJames Morse /* temporary el2 vectors in the __hibernate_exit_text section. */ 6382869ac5SJames Morse extern char hibernate_el2_vectors[]; 6482869ac5SJames Morse 6582869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ 6682869ac5SJames Morse extern char __hyp_stub_vectors[]; 6782869ac5SJames Morse 6882869ac5SJames Morse /* 6982869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number 7082869ac5SJames Morse * and date in here so that we guarantee not to resume with a different 7182869ac5SJames Morse * kernel. 7282869ac5SJames Morse */ 7382869ac5SJames Morse struct arch_hibernate_hdr_invariants { 7482869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1]; 7582869ac5SJames Morse }; 7682869ac5SJames Morse 7782869ac5SJames Morse /* These values need to be know across a hibernate/restore. */ 7882869ac5SJames Morse static struct arch_hibernate_hdr { 7982869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 8082869ac5SJames Morse 8182869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */ 8282869ac5SJames Morse phys_addr_t ttbr1_el1; 8382869ac5SJames Morse void (*reenter_kernel)(void); 8482869ac5SJames Morse 8582869ac5SJames Morse /* 8682869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to 8782869ac5SJames Morse * re-configure el2. 8882869ac5SJames Morse */ 8982869ac5SJames Morse phys_addr_t __hyp_stub_vectors; 9082869ac5SJames Morse } resume_hdr; 9182869ac5SJames Morse 9282869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) 9382869ac5SJames Morse { 9482869ac5SJames Morse memset(i, 0, sizeof(*i)); 9582869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); 9682869ac5SJames Morse } 9782869ac5SJames Morse 9882869ac5SJames Morse int pfn_is_nosave(unsigned long pfn) 9982869ac5SJames Morse { 10082869ac5SJames Morse unsigned long nosave_begin_pfn = virt_to_pfn(&__nosave_begin); 10182869ac5SJames Morse unsigned long nosave_end_pfn = virt_to_pfn(&__nosave_end - 1); 10282869ac5SJames Morse 10382869ac5SJames Morse return (pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn); 10482869ac5SJames Morse } 10582869ac5SJames Morse 10682869ac5SJames Morse void notrace save_processor_state(void) 10782869ac5SJames Morse { 10882869ac5SJames Morse WARN_ON(num_online_cpus() != 1); 10982869ac5SJames Morse } 11082869ac5SJames Morse 11182869ac5SJames Morse void notrace restore_processor_state(void) 11282869ac5SJames Morse { 11382869ac5SJames Morse } 11482869ac5SJames Morse 11582869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size) 11682869ac5SJames Morse { 11782869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 11882869ac5SJames Morse 11982869ac5SJames Morse if (max_size < sizeof(*hdr)) 12082869ac5SJames Morse return -EOVERFLOW; 12182869ac5SJames Morse 12282869ac5SJames Morse arch_hdr_invariants(&hdr->invariants); 12382869ac5SJames Morse hdr->ttbr1_el1 = virt_to_phys(swapper_pg_dir); 12482869ac5SJames Morse hdr->reenter_kernel = _cpu_resume; 12582869ac5SJames Morse 12682869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */ 12782869ac5SJames Morse if (el2_reset_needed()) 12882869ac5SJames Morse hdr->__hyp_stub_vectors = virt_to_phys(__hyp_stub_vectors); 12982869ac5SJames Morse else 13082869ac5SJames Morse hdr->__hyp_stub_vectors = 0; 13182869ac5SJames Morse 13282869ac5SJames Morse return 0; 13382869ac5SJames Morse } 13482869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save); 13582869ac5SJames Morse 13682869ac5SJames Morse int arch_hibernation_header_restore(void *addr) 13782869ac5SJames Morse { 13882869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 13982869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 14082869ac5SJames Morse 14182869ac5SJames Morse arch_hdr_invariants(&invariants); 14282869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) { 14382869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n"); 14482869ac5SJames Morse return -EINVAL; 14582869ac5SJames Morse } 14682869ac5SJames Morse 14782869ac5SJames Morse resume_hdr = *hdr; 14882869ac5SJames Morse 14982869ac5SJames Morse return 0; 15082869ac5SJames Morse } 15182869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore); 15282869ac5SJames Morse 15382869ac5SJames Morse /* 15482869ac5SJames Morse * Copies length bytes, starting at src_start into an new page, 15582869ac5SJames Morse * perform cache maintentance, then maps it at the specified address low 15682869ac5SJames Morse * address as executable. 15782869ac5SJames Morse * 15882869ac5SJames Morse * This is used by hibernate to copy the code it needs to execute when 15982869ac5SJames Morse * overwriting the kernel text. This function generates a new set of page 16082869ac5SJames Morse * tables, which it loads into ttbr0. 16182869ac5SJames Morse * 16282869ac5SJames Morse * Length is provided as we probably only want 4K of data, even on a 64K 16382869ac5SJames Morse * page system. 16482869ac5SJames Morse */ 16582869ac5SJames Morse static int create_safe_exec_page(void *src_start, size_t length, 16682869ac5SJames Morse unsigned long dst_addr, 16782869ac5SJames Morse phys_addr_t *phys_dst_addr, 16882869ac5SJames Morse void *(*allocator)(gfp_t mask), 16982869ac5SJames Morse gfp_t mask) 17082869ac5SJames Morse { 17182869ac5SJames Morse int rc = 0; 17282869ac5SJames Morse pgd_t *pgd; 17382869ac5SJames Morse pud_t *pud; 17482869ac5SJames Morse pmd_t *pmd; 17582869ac5SJames Morse pte_t *pte; 17682869ac5SJames Morse unsigned long dst = (unsigned long)allocator(mask); 17782869ac5SJames Morse 17882869ac5SJames Morse if (!dst) { 17982869ac5SJames Morse rc = -ENOMEM; 18082869ac5SJames Morse goto out; 18182869ac5SJames Morse } 18282869ac5SJames Morse 18382869ac5SJames Morse memcpy((void *)dst, src_start, length); 18482869ac5SJames Morse flush_icache_range(dst, dst + length); 18582869ac5SJames Morse 18682869ac5SJames Morse pgd = pgd_offset_raw(allocator(mask), dst_addr); 18782869ac5SJames Morse if (pgd_none(*pgd)) { 18882869ac5SJames Morse pud = allocator(mask); 18982869ac5SJames Morse if (!pud) { 19082869ac5SJames Morse rc = -ENOMEM; 19182869ac5SJames Morse goto out; 19282869ac5SJames Morse } 19382869ac5SJames Morse pgd_populate(&init_mm, pgd, pud); 19482869ac5SJames Morse } 19582869ac5SJames Morse 19682869ac5SJames Morse pud = pud_offset(pgd, dst_addr); 19782869ac5SJames Morse if (pud_none(*pud)) { 19882869ac5SJames Morse pmd = allocator(mask); 19982869ac5SJames Morse if (!pmd) { 20082869ac5SJames Morse rc = -ENOMEM; 20182869ac5SJames Morse goto out; 20282869ac5SJames Morse } 20382869ac5SJames Morse pud_populate(&init_mm, pud, pmd); 20482869ac5SJames Morse } 20582869ac5SJames Morse 20682869ac5SJames Morse pmd = pmd_offset(pud, dst_addr); 20782869ac5SJames Morse if (pmd_none(*pmd)) { 20882869ac5SJames Morse pte = allocator(mask); 20982869ac5SJames Morse if (!pte) { 21082869ac5SJames Morse rc = -ENOMEM; 21182869ac5SJames Morse goto out; 21282869ac5SJames Morse } 21382869ac5SJames Morse pmd_populate_kernel(&init_mm, pmd, pte); 21482869ac5SJames Morse } 21582869ac5SJames Morse 21682869ac5SJames Morse pte = pte_offset_kernel(pmd, dst_addr); 21782869ac5SJames Morse set_pte(pte, __pte(virt_to_phys((void *)dst) | 21882869ac5SJames Morse pgprot_val(PAGE_KERNEL_EXEC))); 21982869ac5SJames Morse 22082869ac5SJames Morse /* Load our new page tables */ 22182869ac5SJames Morse asm volatile("msr ttbr0_el1, %0;" 22282869ac5SJames Morse "isb;" 22382869ac5SJames Morse "tlbi vmalle1is;" 22482869ac5SJames Morse "dsb ish;" 22582869ac5SJames Morse "isb" : : "r"(virt_to_phys(pgd))); 22682869ac5SJames Morse 22782869ac5SJames Morse *phys_dst_addr = virt_to_phys((void *)dst); 22882869ac5SJames Morse 22982869ac5SJames Morse out: 23082869ac5SJames Morse return rc; 23182869ac5SJames Morse } 23282869ac5SJames Morse 23382869ac5SJames Morse 23482869ac5SJames Morse int swsusp_arch_suspend(void) 23582869ac5SJames Morse { 23682869ac5SJames Morse int ret = 0; 23782869ac5SJames Morse unsigned long flags; 23882869ac5SJames Morse struct sleep_stack_data state; 23982869ac5SJames Morse 240d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) { 241d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); 242d74b4e4fSJames Morse return -EBUSY; 243d74b4e4fSJames Morse } 244d74b4e4fSJames Morse 24582869ac5SJames Morse local_dbg_save(flags); 24682869ac5SJames Morse 24782869ac5SJames Morse if (__cpu_suspend_enter(&state)) { 24882869ac5SJames Morse ret = swsusp_save(); 24982869ac5SJames Morse } else { 25082869ac5SJames Morse /* Clean kernel to PoC for secondary core startup */ 25182869ac5SJames Morse __flush_dcache_area(LMADDR(KERNEL_START), KERNEL_END - KERNEL_START); 25282869ac5SJames Morse 25382869ac5SJames Morse /* 25482869ac5SJames Morse * Tell the hibernation core that we've just restored 25582869ac5SJames Morse * the memory 25682869ac5SJames Morse */ 25782869ac5SJames Morse in_suspend = 0; 25882869ac5SJames Morse 25982869ac5SJames Morse __cpu_suspend_exit(); 26082869ac5SJames Morse } 26182869ac5SJames Morse 26282869ac5SJames Morse local_dbg_restore(flags); 26382869ac5SJames Morse 26482869ac5SJames Morse return ret; 26582869ac5SJames Morse } 26682869ac5SJames Morse 26782869ac5SJames Morse static int copy_pte(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long start, 26882869ac5SJames Morse unsigned long end) 26982869ac5SJames Morse { 27082869ac5SJames Morse pte_t *src_pte; 27182869ac5SJames Morse pte_t *dst_pte; 27282869ac5SJames Morse unsigned long addr = start; 27382869ac5SJames Morse 27482869ac5SJames Morse dst_pte = (pte_t *)get_safe_page(GFP_ATOMIC); 27582869ac5SJames Morse if (!dst_pte) 27682869ac5SJames Morse return -ENOMEM; 27782869ac5SJames Morse pmd_populate_kernel(&init_mm, dst_pmd, dst_pte); 27882869ac5SJames Morse dst_pte = pte_offset_kernel(dst_pmd, start); 27982869ac5SJames Morse 28082869ac5SJames Morse src_pte = pte_offset_kernel(src_pmd, start); 28182869ac5SJames Morse do { 28282869ac5SJames Morse if (!pte_none(*src_pte)) 28382869ac5SJames Morse /* 28482869ac5SJames Morse * Resume will overwrite areas that may be marked 28582869ac5SJames Morse * read only (code, rodata). Clear the RDONLY bit from 28682869ac5SJames Morse * the temporary mappings we use during restore. 28782869ac5SJames Morse */ 28882869ac5SJames Morse set_pte(dst_pte, __pte(pte_val(*src_pte) & ~PTE_RDONLY)); 28982869ac5SJames Morse } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 29082869ac5SJames Morse 29182869ac5SJames Morse return 0; 29282869ac5SJames Morse } 29382869ac5SJames Morse 29482869ac5SJames Morse static int copy_pmd(pud_t *dst_pud, pud_t *src_pud, unsigned long start, 29582869ac5SJames Morse unsigned long end) 29682869ac5SJames Morse { 29782869ac5SJames Morse pmd_t *src_pmd; 29882869ac5SJames Morse pmd_t *dst_pmd; 29982869ac5SJames Morse unsigned long next; 30082869ac5SJames Morse unsigned long addr = start; 30182869ac5SJames Morse 30282869ac5SJames Morse if (pud_none(*dst_pud)) { 30382869ac5SJames Morse dst_pmd = (pmd_t *)get_safe_page(GFP_ATOMIC); 30482869ac5SJames Morse if (!dst_pmd) 30582869ac5SJames Morse return -ENOMEM; 30682869ac5SJames Morse pud_populate(&init_mm, dst_pud, dst_pmd); 30782869ac5SJames Morse } 30882869ac5SJames Morse dst_pmd = pmd_offset(dst_pud, start); 30982869ac5SJames Morse 31082869ac5SJames Morse src_pmd = pmd_offset(src_pud, start); 31182869ac5SJames Morse do { 31282869ac5SJames Morse next = pmd_addr_end(addr, end); 31382869ac5SJames Morse if (pmd_none(*src_pmd)) 31482869ac5SJames Morse continue; 31582869ac5SJames Morse if (pmd_table(*src_pmd)) { 31682869ac5SJames Morse if (copy_pte(dst_pmd, src_pmd, addr, next)) 31782869ac5SJames Morse return -ENOMEM; 31882869ac5SJames Morse } else { 31982869ac5SJames Morse set_pmd(dst_pmd, 32082869ac5SJames Morse __pmd(pmd_val(*src_pmd) & ~PMD_SECT_RDONLY)); 32182869ac5SJames Morse } 32282869ac5SJames Morse } while (dst_pmd++, src_pmd++, addr = next, addr != end); 32382869ac5SJames Morse 32482869ac5SJames Morse return 0; 32582869ac5SJames Morse } 32682869ac5SJames Morse 32782869ac5SJames Morse static int copy_pud(pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long start, 32882869ac5SJames Morse unsigned long end) 32982869ac5SJames Morse { 33082869ac5SJames Morse pud_t *dst_pud; 33182869ac5SJames Morse pud_t *src_pud; 33282869ac5SJames Morse unsigned long next; 33382869ac5SJames Morse unsigned long addr = start; 33482869ac5SJames Morse 33582869ac5SJames Morse if (pgd_none(*dst_pgd)) { 33682869ac5SJames Morse dst_pud = (pud_t *)get_safe_page(GFP_ATOMIC); 33782869ac5SJames Morse if (!dst_pud) 33882869ac5SJames Morse return -ENOMEM; 33982869ac5SJames Morse pgd_populate(&init_mm, dst_pgd, dst_pud); 34082869ac5SJames Morse } 34182869ac5SJames Morse dst_pud = pud_offset(dst_pgd, start); 34282869ac5SJames Morse 34382869ac5SJames Morse src_pud = pud_offset(src_pgd, start); 34482869ac5SJames Morse do { 34582869ac5SJames Morse next = pud_addr_end(addr, end); 34682869ac5SJames Morse if (pud_none(*src_pud)) 34782869ac5SJames Morse continue; 34882869ac5SJames Morse if (pud_table(*(src_pud))) { 34982869ac5SJames Morse if (copy_pmd(dst_pud, src_pud, addr, next)) 35082869ac5SJames Morse return -ENOMEM; 35182869ac5SJames Morse } else { 35282869ac5SJames Morse set_pud(dst_pud, 35382869ac5SJames Morse __pud(pud_val(*src_pud) & ~PMD_SECT_RDONLY)); 35482869ac5SJames Morse } 35582869ac5SJames Morse } while (dst_pud++, src_pud++, addr = next, addr != end); 35682869ac5SJames Morse 35782869ac5SJames Morse return 0; 35882869ac5SJames Morse } 35982869ac5SJames Morse 36082869ac5SJames Morse static int copy_page_tables(pgd_t *dst_pgd, unsigned long start, 36182869ac5SJames Morse unsigned long end) 36282869ac5SJames Morse { 36382869ac5SJames Morse unsigned long next; 36482869ac5SJames Morse unsigned long addr = start; 36582869ac5SJames Morse pgd_t *src_pgd = pgd_offset_k(start); 36682869ac5SJames Morse 36782869ac5SJames Morse dst_pgd = pgd_offset_raw(dst_pgd, start); 36882869ac5SJames Morse do { 36982869ac5SJames Morse next = pgd_addr_end(addr, end); 37082869ac5SJames Morse if (pgd_none(*src_pgd)) 37182869ac5SJames Morse continue; 37282869ac5SJames Morse if (copy_pud(dst_pgd, src_pgd, addr, next)) 37382869ac5SJames Morse return -ENOMEM; 37482869ac5SJames Morse } while (dst_pgd++, src_pgd++, addr = next, addr != end); 37582869ac5SJames Morse 37682869ac5SJames Morse return 0; 37782869ac5SJames Morse } 37882869ac5SJames Morse 37982869ac5SJames Morse /* 38082869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit(). 38182869ac5SJames Morse * 38282869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code, 38382869ac5SJames Morse * we don't need to free it here. 38482869ac5SJames Morse */ 38582869ac5SJames Morse int swsusp_arch_resume(void) 38682869ac5SJames Morse { 38782869ac5SJames Morse int rc = 0; 38882869ac5SJames Morse void *zero_page; 38982869ac5SJames Morse size_t exit_size; 39082869ac5SJames Morse pgd_t *tmp_pg_dir; 39182869ac5SJames Morse void *lm_restore_pblist; 39282869ac5SJames Morse phys_addr_t phys_hibernate_exit; 39382869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, 39482869ac5SJames Morse void *, phys_addr_t, phys_addr_t); 39582869ac5SJames Morse 39682869ac5SJames Morse /* 39782869ac5SJames Morse * Locate the exit code in the bottom-but-one page, so that *NULL 39882869ac5SJames Morse * still has disastrous affects. 39982869ac5SJames Morse */ 40082869ac5SJames Morse hibernate_exit = (void *)PAGE_SIZE; 40182869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; 40282869ac5SJames Morse /* 40382869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate 40482869ac5SJames Morse * a new set of ttbr0 page tables and load them. 40582869ac5SJames Morse */ 40682869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size, 40782869ac5SJames Morse (unsigned long)hibernate_exit, 40882869ac5SJames Morse &phys_hibernate_exit, 40982869ac5SJames Morse (void *)get_safe_page, GFP_ATOMIC); 41082869ac5SJames Morse if (rc) { 41182869ac5SJames Morse pr_err("Failed to create safe executable page for hibernate_exit code."); 41282869ac5SJames Morse goto out; 41382869ac5SJames Morse } 41482869ac5SJames Morse 41582869ac5SJames Morse /* 41682869ac5SJames Morse * The hibernate exit text contains a set of el2 vectors, that will 41782869ac5SJames Morse * be executed at el2 with the mmu off in order to reload hyp-stub. 41882869ac5SJames Morse */ 41982869ac5SJames Morse __flush_dcache_area(hibernate_exit, exit_size); 42082869ac5SJames Morse 42182869ac5SJames Morse /* 42282869ac5SJames Morse * Restoring the memory image will overwrite the ttbr1 page tables. 42382869ac5SJames Morse * Create a second copy of just the linear map, and use this when 42482869ac5SJames Morse * restoring. 42582869ac5SJames Morse */ 42682869ac5SJames Morse tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC); 42782869ac5SJames Morse if (!tmp_pg_dir) { 42882869ac5SJames Morse pr_err("Failed to allocate memory for temporary page tables."); 42982869ac5SJames Morse rc = -ENOMEM; 43082869ac5SJames Morse goto out; 43182869ac5SJames Morse } 43282869ac5SJames Morse rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, 0); 43382869ac5SJames Morse if (rc) 43482869ac5SJames Morse goto out; 43582869ac5SJames Morse 43682869ac5SJames Morse /* 43782869ac5SJames Morse * Since we only copied the linear map, we need to find restore_pblist's 43882869ac5SJames Morse * linear map address. 43982869ac5SJames Morse */ 44082869ac5SJames Morse lm_restore_pblist = LMADDR(restore_pblist); 44182869ac5SJames Morse 44282869ac5SJames Morse /* 44382869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in 44482869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2. 44582869ac5SJames Morse * 44682869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE. 44782869ac5SJames Morse */ 44882869ac5SJames Morse if (el2_reset_needed()) { 44982869ac5SJames Morse phys_addr_t el2_vectors = phys_hibernate_exit; /* base */ 45082869ac5SJames Morse el2_vectors += hibernate_el2_vectors - 45182869ac5SJames Morse __hibernate_exit_text_start; /* offset */ 45282869ac5SJames Morse 45382869ac5SJames Morse __hyp_set_vectors(el2_vectors); 45482869ac5SJames Morse } 45582869ac5SJames Morse 45682869ac5SJames Morse /* 45782869ac5SJames Morse * We need a zero page that is zero before & after resume in order to 45882869ac5SJames Morse * to break before make on the ttbr1 page tables. 45982869ac5SJames Morse */ 46082869ac5SJames Morse zero_page = (void *)get_safe_page(GFP_ATOMIC); 46182869ac5SJames Morse 46282869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, 46382869ac5SJames Morse resume_hdr.reenter_kernel, lm_restore_pblist, 46482869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); 46582869ac5SJames Morse 46682869ac5SJames Morse out: 46782869ac5SJames Morse return rc; 46882869ac5SJames Morse } 4691fe492ceSJames Morse 4701fe492ceSJames Morse static int check_boot_cpu_online_pm_callback(struct notifier_block *nb, 4711fe492ceSJames Morse unsigned long action, void *ptr) 4721fe492ceSJames Morse { 4731fe492ceSJames Morse if (action == PM_HIBERNATION_PREPARE && 4741fe492ceSJames Morse cpumask_first(cpu_online_mask) != 0) { 4751fe492ceSJames Morse pr_warn("CPU0 is offline.\n"); 4761fe492ceSJames Morse return notifier_from_errno(-ENODEV); 4771fe492ceSJames Morse } 4781fe492ceSJames Morse 4791fe492ceSJames Morse return NOTIFY_OK; 4801fe492ceSJames Morse } 4811fe492ceSJames Morse 4821fe492ceSJames Morse static int __init check_boot_cpu_online_init(void) 4831fe492ceSJames Morse { 4841fe492ceSJames Morse /* 4851fe492ceSJames Morse * Set this pm_notifier callback with a lower priority than 4861fe492ceSJames Morse * cpu_hotplug_pm_callback, so that cpu_hotplug_pm_callback will be 4871fe492ceSJames Morse * called earlier to disable cpu hotplug before the cpu online check. 4881fe492ceSJames Morse */ 4891fe492ceSJames Morse pm_notifier(check_boot_cpu_online_pm_callback, -INT_MAX); 4901fe492ceSJames Morse 4911fe492ceSJames Morse return 0; 4921fe492ceSJames Morse } 4931fe492ceSJames Morse core_initcall(check_boot_cpu_online_init); 494