182869ac5SJames Morse /*: 282869ac5SJames Morse * Hibernate support specific for ARM64 382869ac5SJames Morse * 482869ac5SJames Morse * Derived from work on ARM hibernation support by: 582869ac5SJames Morse * 682869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove 782869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) 882869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) 982869ac5SJames Morse * https://lkml.org/lkml/2010/6/18/4 1082869ac5SJames Morse * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html 1182869ac5SJames Morse * https://patchwork.kernel.org/patch/96442/ 1282869ac5SJames Morse * 1382869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 1482869ac5SJames Morse * 1582869ac5SJames Morse * License terms: GNU General Public License (GPL) version 2 1682869ac5SJames Morse */ 1782869ac5SJames Morse #define pr_fmt(x) "hibernate: " x 188ec058fdSJames Morse #include <linux/cpu.h> 1982869ac5SJames Morse #include <linux/kvm_host.h> 2082869ac5SJames Morse #include <linux/mm.h> 2182869ac5SJames Morse #include <linux/pm.h> 2282869ac5SJames Morse #include <linux/sched.h> 2382869ac5SJames Morse #include <linux/suspend.h> 2482869ac5SJames Morse #include <linux/utsname.h> 2582869ac5SJames Morse #include <linux/version.h> 2682869ac5SJames Morse 2782869ac5SJames Morse #include <asm/barrier.h> 2882869ac5SJames Morse #include <asm/cacheflush.h> 298ec058fdSJames Morse #include <asm/cputype.h> 300fbeb318SJames Morse #include <asm/daifflags.h> 3182869ac5SJames Morse #include <asm/irqflags.h> 32254a41c0SAKASHI Takahiro #include <asm/kexec.h> 3382869ac5SJames Morse #include <asm/memory.h> 3482869ac5SJames Morse #include <asm/mmu_context.h> 3582869ac5SJames Morse #include <asm/pgalloc.h> 3682869ac5SJames Morse #include <asm/pgtable.h> 3782869ac5SJames Morse #include <asm/pgtable-hwdef.h> 3882869ac5SJames Morse #include <asm/sections.h> 39d74b4e4fSJames Morse #include <asm/smp.h> 408ec058fdSJames Morse #include <asm/smp_plat.h> 4182869ac5SJames Morse #include <asm/suspend.h> 420194e760SMark Rutland #include <asm/sysreg.h> 4382869ac5SJames Morse #include <asm/virt.h> 4482869ac5SJames Morse 4582869ac5SJames Morse /* 4682869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it 4782869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This 4882869ac5SJames Morse * doesn't happen with either KASLR. 4982869ac5SJames Morse * 5082869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in 5182869ac5SJames Morse * kernel/power/hibernate.c 5282869ac5SJames Morse */ 5382869ac5SJames Morse extern int in_suspend; 5482869ac5SJames Morse 5582869ac5SJames Morse /* Do we need to reset el2? */ 5682869ac5SJames Morse #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) 5782869ac5SJames Morse 5882869ac5SJames Morse /* temporary el2 vectors in the __hibernate_exit_text section. */ 5982869ac5SJames Morse extern char hibernate_el2_vectors[]; 6082869ac5SJames Morse 6182869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ 6282869ac5SJames Morse extern char __hyp_stub_vectors[]; 6382869ac5SJames Morse 6482869ac5SJames Morse /* 658ec058fdSJames Morse * The logical cpu number we should resume on, initialised to a non-cpu 668ec058fdSJames Morse * number. 678ec058fdSJames Morse */ 688ec058fdSJames Morse static int sleep_cpu = -EINVAL; 698ec058fdSJames Morse 708ec058fdSJames Morse /* 7182869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number 7282869ac5SJames Morse * and date in here so that we guarantee not to resume with a different 7382869ac5SJames Morse * kernel. 7482869ac5SJames Morse */ 7582869ac5SJames Morse struct arch_hibernate_hdr_invariants { 7682869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1]; 7782869ac5SJames Morse }; 7882869ac5SJames Morse 7982869ac5SJames Morse /* These values need to be know across a hibernate/restore. */ 8082869ac5SJames Morse static struct arch_hibernate_hdr { 8182869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 8282869ac5SJames Morse 8382869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */ 8482869ac5SJames Morse phys_addr_t ttbr1_el1; 8582869ac5SJames Morse void (*reenter_kernel)(void); 8682869ac5SJames Morse 8782869ac5SJames Morse /* 8882869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to 8982869ac5SJames Morse * re-configure el2. 9082869ac5SJames Morse */ 9182869ac5SJames Morse phys_addr_t __hyp_stub_vectors; 928ec058fdSJames Morse 938ec058fdSJames Morse u64 sleep_cpu_mpidr; 9482869ac5SJames Morse } resume_hdr; 9582869ac5SJames Morse 9682869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) 9782869ac5SJames Morse { 9882869ac5SJames Morse memset(i, 0, sizeof(*i)); 9982869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); 10082869ac5SJames Morse } 10182869ac5SJames Morse 10282869ac5SJames Morse int pfn_is_nosave(unsigned long pfn) 10382869ac5SJames Morse { 1042077be67SLaura Abbott unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); 1052077be67SLaura Abbott unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); 10682869ac5SJames Morse 107254a41c0SAKASHI Takahiro return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || 108254a41c0SAKASHI Takahiro crash_is_nosave(pfn); 10982869ac5SJames Morse } 11082869ac5SJames Morse 11182869ac5SJames Morse void notrace save_processor_state(void) 11282869ac5SJames Morse { 11382869ac5SJames Morse WARN_ON(num_online_cpus() != 1); 11482869ac5SJames Morse } 11582869ac5SJames Morse 11682869ac5SJames Morse void notrace restore_processor_state(void) 11782869ac5SJames Morse { 11882869ac5SJames Morse } 11982869ac5SJames Morse 12082869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size) 12182869ac5SJames Morse { 12282869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 12382869ac5SJames Morse 12482869ac5SJames Morse if (max_size < sizeof(*hdr)) 12582869ac5SJames Morse return -EOVERFLOW; 12682869ac5SJames Morse 12782869ac5SJames Morse arch_hdr_invariants(&hdr->invariants); 1282077be67SLaura Abbott hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir); 12982869ac5SJames Morse hdr->reenter_kernel = _cpu_resume; 13082869ac5SJames Morse 13182869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */ 13282869ac5SJames Morse if (el2_reset_needed()) 1332077be67SLaura Abbott hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors); 13482869ac5SJames Morse else 13582869ac5SJames Morse hdr->__hyp_stub_vectors = 0; 13682869ac5SJames Morse 1378ec058fdSJames Morse /* Save the mpidr of the cpu we called cpu_suspend() on... */ 1388ec058fdSJames Morse if (sleep_cpu < 0) { 1399165dabbSMasanari Iida pr_err("Failing to hibernate on an unknown CPU.\n"); 1408ec058fdSJames Morse return -ENODEV; 1418ec058fdSJames Morse } 1428ec058fdSJames Morse hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); 1438ec058fdSJames Morse pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1448ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1458ec058fdSJames Morse 14682869ac5SJames Morse return 0; 14782869ac5SJames Morse } 14882869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save); 14982869ac5SJames Morse 15082869ac5SJames Morse int arch_hibernation_header_restore(void *addr) 15182869ac5SJames Morse { 1528ec058fdSJames Morse int ret; 15382869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 15482869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 15582869ac5SJames Morse 15682869ac5SJames Morse arch_hdr_invariants(&invariants); 15782869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) { 15882869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n"); 15982869ac5SJames Morse return -EINVAL; 16082869ac5SJames Morse } 16182869ac5SJames Morse 1628ec058fdSJames Morse sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); 1638ec058fdSJames Morse pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1648ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1658ec058fdSJames Morse if (sleep_cpu < 0) { 1668ec058fdSJames Morse pr_crit("Hibernated on a CPU not known to this kernel!\n"); 1678ec058fdSJames Morse sleep_cpu = -EINVAL; 1688ec058fdSJames Morse return -EINVAL; 1698ec058fdSJames Morse } 1708ec058fdSJames Morse if (!cpu_online(sleep_cpu)) { 1718ec058fdSJames Morse pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n"); 1728ec058fdSJames Morse ret = cpu_up(sleep_cpu); 1738ec058fdSJames Morse if (ret) { 1748ec058fdSJames Morse pr_err("Failed to bring hibernate-CPU up!\n"); 1758ec058fdSJames Morse sleep_cpu = -EINVAL; 1768ec058fdSJames Morse return ret; 1778ec058fdSJames Morse } 1788ec058fdSJames Morse } 1798ec058fdSJames Morse 18082869ac5SJames Morse resume_hdr = *hdr; 18182869ac5SJames Morse 18282869ac5SJames Morse return 0; 18382869ac5SJames Morse } 18482869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore); 18582869ac5SJames Morse 18682869ac5SJames Morse /* 18782869ac5SJames Morse * Copies length bytes, starting at src_start into an new page, 18882869ac5SJames Morse * perform cache maintentance, then maps it at the specified address low 18982869ac5SJames Morse * address as executable. 19082869ac5SJames Morse * 19182869ac5SJames Morse * This is used by hibernate to copy the code it needs to execute when 19282869ac5SJames Morse * overwriting the kernel text. This function generates a new set of page 19382869ac5SJames Morse * tables, which it loads into ttbr0. 19482869ac5SJames Morse * 19582869ac5SJames Morse * Length is provided as we probably only want 4K of data, even on a 64K 19682869ac5SJames Morse * page system. 19782869ac5SJames Morse */ 19882869ac5SJames Morse static int create_safe_exec_page(void *src_start, size_t length, 19982869ac5SJames Morse unsigned long dst_addr, 20082869ac5SJames Morse phys_addr_t *phys_dst_addr, 20182869ac5SJames Morse void *(*allocator)(gfp_t mask), 20282869ac5SJames Morse gfp_t mask) 20382869ac5SJames Morse { 20482869ac5SJames Morse int rc = 0; 20520a004e7SWill Deacon pgd_t *pgdp; 20620a004e7SWill Deacon pud_t *pudp; 20720a004e7SWill Deacon pmd_t *pmdp; 20820a004e7SWill Deacon pte_t *ptep; 20982869ac5SJames Morse unsigned long dst = (unsigned long)allocator(mask); 21082869ac5SJames Morse 21182869ac5SJames Morse if (!dst) { 21282869ac5SJames Morse rc = -ENOMEM; 21382869ac5SJames Morse goto out; 21482869ac5SJames Morse } 21582869ac5SJames Morse 21682869ac5SJames Morse memcpy((void *)dst, src_start, length); 217b4aecf78SWill Deacon __flush_icache_range(dst, dst + length); 21882869ac5SJames Morse 21920a004e7SWill Deacon pgdp = pgd_offset_raw(allocator(mask), dst_addr); 22020a004e7SWill Deacon if (pgd_none(READ_ONCE(*pgdp))) { 22120a004e7SWill Deacon pudp = allocator(mask); 22220a004e7SWill Deacon if (!pudp) { 22382869ac5SJames Morse rc = -ENOMEM; 22482869ac5SJames Morse goto out; 22582869ac5SJames Morse } 22620a004e7SWill Deacon pgd_populate(&init_mm, pgdp, pudp); 22782869ac5SJames Morse } 22882869ac5SJames Morse 22920a004e7SWill Deacon pudp = pud_offset(pgdp, dst_addr); 23020a004e7SWill Deacon if (pud_none(READ_ONCE(*pudp))) { 23120a004e7SWill Deacon pmdp = allocator(mask); 23220a004e7SWill Deacon if (!pmdp) { 23382869ac5SJames Morse rc = -ENOMEM; 23482869ac5SJames Morse goto out; 23582869ac5SJames Morse } 23620a004e7SWill Deacon pud_populate(&init_mm, pudp, pmdp); 23782869ac5SJames Morse } 23882869ac5SJames Morse 23920a004e7SWill Deacon pmdp = pmd_offset(pudp, dst_addr); 24020a004e7SWill Deacon if (pmd_none(READ_ONCE(*pmdp))) { 24120a004e7SWill Deacon ptep = allocator(mask); 24220a004e7SWill Deacon if (!ptep) { 24382869ac5SJames Morse rc = -ENOMEM; 24482869ac5SJames Morse goto out; 24582869ac5SJames Morse } 24620a004e7SWill Deacon pmd_populate_kernel(&init_mm, pmdp, ptep); 24782869ac5SJames Morse } 24882869ac5SJames Morse 24920a004e7SWill Deacon ptep = pte_offset_kernel(pmdp, dst_addr); 25020a004e7SWill Deacon set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC)); 25182869ac5SJames Morse 2520194e760SMark Rutland /* 2530194e760SMark Rutland * Load our new page tables. A strict BBM approach requires that we 2540194e760SMark Rutland * ensure that TLBs are free of any entries that may overlap with the 2550194e760SMark Rutland * global mappings we are about to install. 2560194e760SMark Rutland * 2570194e760SMark Rutland * For a real hibernate/resume cycle TTBR0 currently points to a zero 2580194e760SMark Rutland * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI 2590194e760SMark Rutland * runtime services), while for a userspace-driven test_resume cycle it 2600194e760SMark Rutland * points to userspace page tables (and we must point it at a zero page 2610194e760SMark Rutland * ourselves). Elsewhere we only (un)install the idmap with preemption 2620194e760SMark Rutland * disabled, so T0SZ should be as required regardless. 2630194e760SMark Rutland */ 2640194e760SMark Rutland cpu_set_reserved_ttbr0(); 2650194e760SMark Rutland local_flush_tlb_all(); 26620a004e7SWill Deacon write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1); 2670194e760SMark Rutland isb(); 26882869ac5SJames Morse 26982869ac5SJames Morse *phys_dst_addr = virt_to_phys((void *)dst); 27082869ac5SJames Morse 27182869ac5SJames Morse out: 27282869ac5SJames Morse return rc; 27382869ac5SJames Morse } 27482869ac5SJames Morse 2755ebe3a44SJames Morse #define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) 27682869ac5SJames Morse 27782869ac5SJames Morse int swsusp_arch_suspend(void) 27882869ac5SJames Morse { 27982869ac5SJames Morse int ret = 0; 28082869ac5SJames Morse unsigned long flags; 28182869ac5SJames Morse struct sleep_stack_data state; 28282869ac5SJames Morse 283d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) { 284d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); 285d74b4e4fSJames Morse return -EBUSY; 286d74b4e4fSJames Morse } 287d74b4e4fSJames Morse 2880fbeb318SJames Morse flags = local_daif_save(); 28982869ac5SJames Morse 29082869ac5SJames Morse if (__cpu_suspend_enter(&state)) { 291254a41c0SAKASHI Takahiro /* make the crash dump kernel image visible/saveable */ 292254a41c0SAKASHI Takahiro crash_prepare_suspend(); 293254a41c0SAKASHI Takahiro 2948ec058fdSJames Morse sleep_cpu = smp_processor_id(); 29582869ac5SJames Morse ret = swsusp_save(); 29682869ac5SJames Morse } else { 2975ebe3a44SJames Morse /* Clean kernel core startup/idle code to PoC*/ 2985ebe3a44SJames Morse dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); 2995ebe3a44SJames Morse dcache_clean_range(__idmap_text_start, __idmap_text_end); 3005ebe3a44SJames Morse 3015ebe3a44SJames Morse /* Clean kvm setup code to PoC? */ 3025ebe3a44SJames Morse if (el2_reset_needed()) 3035ebe3a44SJames Morse dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); 30482869ac5SJames Morse 305254a41c0SAKASHI Takahiro /* make the crash dump kernel image protected again */ 306254a41c0SAKASHI Takahiro crash_post_resume(); 307254a41c0SAKASHI Takahiro 30882869ac5SJames Morse /* 30982869ac5SJames Morse * Tell the hibernation core that we've just restored 31082869ac5SJames Morse * the memory 31182869ac5SJames Morse */ 31282869ac5SJames Morse in_suspend = 0; 31382869ac5SJames Morse 3148ec058fdSJames Morse sleep_cpu = -EINVAL; 31582869ac5SJames Morse __cpu_suspend_exit(); 316647d0519SMarc Zyngier 317647d0519SMarc Zyngier /* 318647d0519SMarc Zyngier * Just in case the boot kernel did turn the SSBD 319647d0519SMarc Zyngier * mitigation off behind our back, let's set the state 320647d0519SMarc Zyngier * to what we expect it to be. 321647d0519SMarc Zyngier */ 322647d0519SMarc Zyngier switch (arm64_get_ssbd_state()) { 323647d0519SMarc Zyngier case ARM64_SSBD_FORCE_ENABLE: 324647d0519SMarc Zyngier case ARM64_SSBD_KERNEL: 325647d0519SMarc Zyngier arm64_set_ssbd_mitigation(true); 326647d0519SMarc Zyngier } 32782869ac5SJames Morse } 32882869ac5SJames Morse 3290fbeb318SJames Morse local_daif_restore(flags); 33082869ac5SJames Morse 33182869ac5SJames Morse return ret; 33282869ac5SJames Morse } 33382869ac5SJames Morse 33420a004e7SWill Deacon static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 3355ebe3a44SJames Morse { 33620a004e7SWill Deacon pte_t pte = READ_ONCE(*src_ptep); 3375ebe3a44SJames Morse 3385ebe3a44SJames Morse if (pte_valid(pte)) { 3395ebe3a44SJames Morse /* 3405ebe3a44SJames Morse * Resume will overwrite areas that may be marked 3415ebe3a44SJames Morse * read only (code, rodata). Clear the RDONLY bit from 3425ebe3a44SJames Morse * the temporary mappings we use during restore. 3435ebe3a44SJames Morse */ 34420a004e7SWill Deacon set_pte(dst_ptep, pte_mkwrite(pte)); 3455ebe3a44SJames Morse } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 3465ebe3a44SJames Morse /* 3475ebe3a44SJames Morse * debug_pagealloc will removed the PTE_VALID bit if 3485ebe3a44SJames Morse * the page isn't in use by the resume kernel. It may have 3495ebe3a44SJames Morse * been in use by the original kernel, in which case we need 3505ebe3a44SJames Morse * to put it back in our copy to do the restore. 3515ebe3a44SJames Morse * 3525ebe3a44SJames Morse * Before marking this entry valid, check the pfn should 3535ebe3a44SJames Morse * be mapped. 3545ebe3a44SJames Morse */ 3555ebe3a44SJames Morse BUG_ON(!pfn_valid(pte_pfn(pte))); 3565ebe3a44SJames Morse 35720a004e7SWill Deacon set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 3585ebe3a44SJames Morse } 3595ebe3a44SJames Morse } 3605ebe3a44SJames Morse 36120a004e7SWill Deacon static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, 36282869ac5SJames Morse unsigned long end) 36382869ac5SJames Morse { 36420a004e7SWill Deacon pte_t *src_ptep; 36520a004e7SWill Deacon pte_t *dst_ptep; 36682869ac5SJames Morse unsigned long addr = start; 36782869ac5SJames Morse 36820a004e7SWill Deacon dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); 36920a004e7SWill Deacon if (!dst_ptep) 37082869ac5SJames Morse return -ENOMEM; 37120a004e7SWill Deacon pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); 37220a004e7SWill Deacon dst_ptep = pte_offset_kernel(dst_pmdp, start); 37382869ac5SJames Morse 37420a004e7SWill Deacon src_ptep = pte_offset_kernel(src_pmdp, start); 37582869ac5SJames Morse do { 37620a004e7SWill Deacon _copy_pte(dst_ptep, src_ptep, addr); 37720a004e7SWill Deacon } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 37882869ac5SJames Morse 37982869ac5SJames Morse return 0; 38082869ac5SJames Morse } 38182869ac5SJames Morse 38220a004e7SWill Deacon static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start, 38382869ac5SJames Morse unsigned long end) 38482869ac5SJames Morse { 38520a004e7SWill Deacon pmd_t *src_pmdp; 38620a004e7SWill Deacon pmd_t *dst_pmdp; 38782869ac5SJames Morse unsigned long next; 38882869ac5SJames Morse unsigned long addr = start; 38982869ac5SJames Morse 39020a004e7SWill Deacon if (pud_none(READ_ONCE(*dst_pudp))) { 39120a004e7SWill Deacon dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); 39220a004e7SWill Deacon if (!dst_pmdp) 39382869ac5SJames Morse return -ENOMEM; 39420a004e7SWill Deacon pud_populate(&init_mm, dst_pudp, dst_pmdp); 39582869ac5SJames Morse } 39620a004e7SWill Deacon dst_pmdp = pmd_offset(dst_pudp, start); 39782869ac5SJames Morse 39820a004e7SWill Deacon src_pmdp = pmd_offset(src_pudp, start); 39982869ac5SJames Morse do { 40020a004e7SWill Deacon pmd_t pmd = READ_ONCE(*src_pmdp); 40120a004e7SWill Deacon 40282869ac5SJames Morse next = pmd_addr_end(addr, end); 40320a004e7SWill Deacon if (pmd_none(pmd)) 40482869ac5SJames Morse continue; 40520a004e7SWill Deacon if (pmd_table(pmd)) { 40620a004e7SWill Deacon if (copy_pte(dst_pmdp, src_pmdp, addr, next)) 40782869ac5SJames Morse return -ENOMEM; 40882869ac5SJames Morse } else { 40920a004e7SWill Deacon set_pmd(dst_pmdp, 41020a004e7SWill Deacon __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 41182869ac5SJames Morse } 41220a004e7SWill Deacon } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 41382869ac5SJames Morse 41482869ac5SJames Morse return 0; 41582869ac5SJames Morse } 41682869ac5SJames Morse 41720a004e7SWill Deacon static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start, 41882869ac5SJames Morse unsigned long end) 41982869ac5SJames Morse { 42020a004e7SWill Deacon pud_t *dst_pudp; 42120a004e7SWill Deacon pud_t *src_pudp; 42282869ac5SJames Morse unsigned long next; 42382869ac5SJames Morse unsigned long addr = start; 42482869ac5SJames Morse 42520a004e7SWill Deacon if (pgd_none(READ_ONCE(*dst_pgdp))) { 42620a004e7SWill Deacon dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); 42720a004e7SWill Deacon if (!dst_pudp) 42882869ac5SJames Morse return -ENOMEM; 42920a004e7SWill Deacon pgd_populate(&init_mm, dst_pgdp, dst_pudp); 43082869ac5SJames Morse } 43120a004e7SWill Deacon dst_pudp = pud_offset(dst_pgdp, start); 43282869ac5SJames Morse 43320a004e7SWill Deacon src_pudp = pud_offset(src_pgdp, start); 43482869ac5SJames Morse do { 43520a004e7SWill Deacon pud_t pud = READ_ONCE(*src_pudp); 43620a004e7SWill Deacon 43782869ac5SJames Morse next = pud_addr_end(addr, end); 43820a004e7SWill Deacon if (pud_none(pud)) 43982869ac5SJames Morse continue; 44020a004e7SWill Deacon if (pud_table(pud)) { 44120a004e7SWill Deacon if (copy_pmd(dst_pudp, src_pudp, addr, next)) 44282869ac5SJames Morse return -ENOMEM; 44382869ac5SJames Morse } else { 44420a004e7SWill Deacon set_pud(dst_pudp, 44520a004e7SWill Deacon __pud(pud_val(pud) & ~PMD_SECT_RDONLY)); 44682869ac5SJames Morse } 44720a004e7SWill Deacon } while (dst_pudp++, src_pudp++, addr = next, addr != end); 44882869ac5SJames Morse 44982869ac5SJames Morse return 0; 45082869ac5SJames Morse } 45182869ac5SJames Morse 45220a004e7SWill Deacon static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, 45382869ac5SJames Morse unsigned long end) 45482869ac5SJames Morse { 45582869ac5SJames Morse unsigned long next; 45682869ac5SJames Morse unsigned long addr = start; 45720a004e7SWill Deacon pgd_t *src_pgdp = pgd_offset_k(start); 45882869ac5SJames Morse 45920a004e7SWill Deacon dst_pgdp = pgd_offset_raw(dst_pgdp, start); 46082869ac5SJames Morse do { 46182869ac5SJames Morse next = pgd_addr_end(addr, end); 46220a004e7SWill Deacon if (pgd_none(READ_ONCE(*src_pgdp))) 46382869ac5SJames Morse continue; 46420a004e7SWill Deacon if (copy_pud(dst_pgdp, src_pgdp, addr, next)) 46582869ac5SJames Morse return -ENOMEM; 46620a004e7SWill Deacon } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 46782869ac5SJames Morse 46882869ac5SJames Morse return 0; 46982869ac5SJames Morse } 47082869ac5SJames Morse 47182869ac5SJames Morse /* 47282869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit(). 47382869ac5SJames Morse * 47482869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code, 47582869ac5SJames Morse * we don't need to free it here. 47682869ac5SJames Morse */ 47782869ac5SJames Morse int swsusp_arch_resume(void) 47882869ac5SJames Morse { 47982869ac5SJames Morse int rc = 0; 48082869ac5SJames Morse void *zero_page; 48182869ac5SJames Morse size_t exit_size; 48282869ac5SJames Morse pgd_t *tmp_pg_dir; 48382869ac5SJames Morse phys_addr_t phys_hibernate_exit; 48482869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, 48582869ac5SJames Morse void *, phys_addr_t, phys_addr_t); 48682869ac5SJames Morse 48782869ac5SJames Morse /* 488dfbca61aSMark Rutland * Restoring the memory image will overwrite the ttbr1 page tables. 489dfbca61aSMark Rutland * Create a second copy of just the linear map, and use this when 490dfbca61aSMark Rutland * restoring. 491dfbca61aSMark Rutland */ 492dfbca61aSMark Rutland tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC); 493dfbca61aSMark Rutland if (!tmp_pg_dir) { 494117f5727SMark Rutland pr_err("Failed to allocate memory for temporary page tables.\n"); 495dfbca61aSMark Rutland rc = -ENOMEM; 496dfbca61aSMark Rutland goto out; 497dfbca61aSMark Rutland } 498dfbca61aSMark Rutland rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, 0); 499dfbca61aSMark Rutland if (rc) 500dfbca61aSMark Rutland goto out; 501dfbca61aSMark Rutland 502dfbca61aSMark Rutland /* 503dfbca61aSMark Rutland * We need a zero page that is zero before & after resume in order to 504dfbca61aSMark Rutland * to break before make on the ttbr1 page tables. 505dfbca61aSMark Rutland */ 506dfbca61aSMark Rutland zero_page = (void *)get_safe_page(GFP_ATOMIC); 507dfbca61aSMark Rutland if (!zero_page) { 508117f5727SMark Rutland pr_err("Failed to allocate zero page.\n"); 509dfbca61aSMark Rutland rc = -ENOMEM; 510dfbca61aSMark Rutland goto out; 511dfbca61aSMark Rutland } 512dfbca61aSMark Rutland 513dfbca61aSMark Rutland /* 51482869ac5SJames Morse * Locate the exit code in the bottom-but-one page, so that *NULL 51582869ac5SJames Morse * still has disastrous affects. 51682869ac5SJames Morse */ 51782869ac5SJames Morse hibernate_exit = (void *)PAGE_SIZE; 51882869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; 51982869ac5SJames Morse /* 52082869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate 52182869ac5SJames Morse * a new set of ttbr0 page tables and load them. 52282869ac5SJames Morse */ 52382869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size, 52482869ac5SJames Morse (unsigned long)hibernate_exit, 52582869ac5SJames Morse &phys_hibernate_exit, 52682869ac5SJames Morse (void *)get_safe_page, GFP_ATOMIC); 52782869ac5SJames Morse if (rc) { 528117f5727SMark Rutland pr_err("Failed to create safe executable page for hibernate_exit code.\n"); 52982869ac5SJames Morse goto out; 53082869ac5SJames Morse } 53182869ac5SJames Morse 53282869ac5SJames Morse /* 53382869ac5SJames Morse * The hibernate exit text contains a set of el2 vectors, that will 53482869ac5SJames Morse * be executed at el2 with the mmu off in order to reload hyp-stub. 53582869ac5SJames Morse */ 53682869ac5SJames Morse __flush_dcache_area(hibernate_exit, exit_size); 53782869ac5SJames Morse 53882869ac5SJames Morse /* 53982869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in 54082869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2. 54182869ac5SJames Morse * 54282869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE. 54382869ac5SJames Morse */ 54482869ac5SJames Morse if (el2_reset_needed()) { 54582869ac5SJames Morse phys_addr_t el2_vectors = phys_hibernate_exit; /* base */ 54682869ac5SJames Morse el2_vectors += hibernate_el2_vectors - 54782869ac5SJames Morse __hibernate_exit_text_start; /* offset */ 54882869ac5SJames Morse 54982869ac5SJames Morse __hyp_set_vectors(el2_vectors); 55082869ac5SJames Morse } 55182869ac5SJames Morse 55282869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, 5532077be67SLaura Abbott resume_hdr.reenter_kernel, restore_pblist, 55482869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); 55582869ac5SJames Morse 55682869ac5SJames Morse out: 55782869ac5SJames Morse return rc; 55882869ac5SJames Morse } 5591fe492ceSJames Morse 5608ec058fdSJames Morse int hibernate_resume_nonboot_cpu_disable(void) 5618ec058fdSJames Morse { 5628ec058fdSJames Morse if (sleep_cpu < 0) { 5639165dabbSMasanari Iida pr_err("Failing to resume from hibernate on an unknown CPU.\n"); 5648ec058fdSJames Morse return -ENODEV; 5658ec058fdSJames Morse } 5668ec058fdSJames Morse 5678ec058fdSJames Morse return freeze_secondary_cpus(sleep_cpu); 5688ec058fdSJames Morse } 569