1af873fceSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 282869ac5SJames Morse /*: 382869ac5SJames Morse * Hibernate support specific for ARM64 482869ac5SJames Morse * 582869ac5SJames Morse * Derived from work on ARM hibernation support by: 682869ac5SJames Morse * 782869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove 882869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) 982869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) 1082869ac5SJames Morse * https://lkml.org/lkml/2010/6/18/4 1182869ac5SJames Morse * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html 1282869ac5SJames Morse * https://patchwork.kernel.org/patch/96442/ 1382869ac5SJames Morse * 1482869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 1582869ac5SJames Morse */ 1682869ac5SJames Morse #define pr_fmt(x) "hibernate: " x 178ec058fdSJames Morse #include <linux/cpu.h> 1882869ac5SJames Morse #include <linux/kvm_host.h> 1982869ac5SJames Morse #include <linux/mm.h> 2082869ac5SJames Morse #include <linux/pm.h> 2182869ac5SJames Morse #include <linux/sched.h> 2282869ac5SJames Morse #include <linux/suspend.h> 2382869ac5SJames Morse #include <linux/utsname.h> 2482869ac5SJames Morse #include <linux/version.h> 2582869ac5SJames Morse 2682869ac5SJames Morse #include <asm/barrier.h> 2782869ac5SJames Morse #include <asm/cacheflush.h> 288ec058fdSJames Morse #include <asm/cputype.h> 290fbeb318SJames Morse #include <asm/daifflags.h> 3082869ac5SJames Morse #include <asm/irqflags.h> 31254a41c0SAKASHI Takahiro #include <asm/kexec.h> 3282869ac5SJames Morse #include <asm/memory.h> 3382869ac5SJames Morse #include <asm/mmu_context.h> 3482869ac5SJames Morse #include <asm/pgalloc.h> 3582869ac5SJames Morse #include <asm/pgtable.h> 3682869ac5SJames Morse #include <asm/pgtable-hwdef.h> 3782869ac5SJames Morse #include <asm/sections.h> 38d74b4e4fSJames Morse #include <asm/smp.h> 398ec058fdSJames Morse #include <asm/smp_plat.h> 4082869ac5SJames Morse #include <asm/suspend.h> 410194e760SMark Rutland #include <asm/sysreg.h> 4282869ac5SJames Morse #include <asm/virt.h> 4382869ac5SJames Morse 4482869ac5SJames Morse /* 4582869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it 4682869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This 4782869ac5SJames Morse * doesn't happen with either KASLR. 4882869ac5SJames Morse * 4982869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in 5082869ac5SJames Morse * kernel/power/hibernate.c 5182869ac5SJames Morse */ 5282869ac5SJames Morse extern int in_suspend; 5382869ac5SJames Morse 5482869ac5SJames Morse /* Do we need to reset el2? */ 5582869ac5SJames Morse #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) 5682869ac5SJames Morse 5782869ac5SJames Morse /* temporary el2 vectors in the __hibernate_exit_text section. */ 5882869ac5SJames Morse extern char hibernate_el2_vectors[]; 5982869ac5SJames Morse 6082869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ 6182869ac5SJames Morse extern char __hyp_stub_vectors[]; 6282869ac5SJames Morse 6382869ac5SJames Morse /* 648ec058fdSJames Morse * The logical cpu number we should resume on, initialised to a non-cpu 658ec058fdSJames Morse * number. 668ec058fdSJames Morse */ 678ec058fdSJames Morse static int sleep_cpu = -EINVAL; 688ec058fdSJames Morse 698ec058fdSJames Morse /* 7082869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number 7182869ac5SJames Morse * and date in here so that we guarantee not to resume with a different 7282869ac5SJames Morse * kernel. 7382869ac5SJames Morse */ 7482869ac5SJames Morse struct arch_hibernate_hdr_invariants { 7582869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1]; 7682869ac5SJames Morse }; 7782869ac5SJames Morse 7882869ac5SJames Morse /* These values need to be know across a hibernate/restore. */ 7982869ac5SJames Morse static struct arch_hibernate_hdr { 8082869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 8182869ac5SJames Morse 8282869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */ 8382869ac5SJames Morse phys_addr_t ttbr1_el1; 8482869ac5SJames Morse void (*reenter_kernel)(void); 8582869ac5SJames Morse 8682869ac5SJames Morse /* 8782869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to 8882869ac5SJames Morse * re-configure el2. 8982869ac5SJames Morse */ 9082869ac5SJames Morse phys_addr_t __hyp_stub_vectors; 918ec058fdSJames Morse 928ec058fdSJames Morse u64 sleep_cpu_mpidr; 9382869ac5SJames Morse } resume_hdr; 9482869ac5SJames Morse 9582869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) 9682869ac5SJames Morse { 9782869ac5SJames Morse memset(i, 0, sizeof(*i)); 9882869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); 9982869ac5SJames Morse } 10082869ac5SJames Morse 10182869ac5SJames Morse int pfn_is_nosave(unsigned long pfn) 10282869ac5SJames Morse { 1032077be67SLaura Abbott unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); 1042077be67SLaura Abbott unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); 10582869ac5SJames Morse 106254a41c0SAKASHI Takahiro return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || 107254a41c0SAKASHI Takahiro crash_is_nosave(pfn); 10882869ac5SJames Morse } 10982869ac5SJames Morse 11082869ac5SJames Morse void notrace save_processor_state(void) 11182869ac5SJames Morse { 11282869ac5SJames Morse WARN_ON(num_online_cpus() != 1); 11382869ac5SJames Morse } 11482869ac5SJames Morse 11582869ac5SJames Morse void notrace restore_processor_state(void) 11682869ac5SJames Morse { 11782869ac5SJames Morse } 11882869ac5SJames Morse 11982869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size) 12082869ac5SJames Morse { 12182869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 12282869ac5SJames Morse 12382869ac5SJames Morse if (max_size < sizeof(*hdr)) 12482869ac5SJames Morse return -EOVERFLOW; 12582869ac5SJames Morse 12682869ac5SJames Morse arch_hdr_invariants(&hdr->invariants); 1272077be67SLaura Abbott hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir); 12882869ac5SJames Morse hdr->reenter_kernel = _cpu_resume; 12982869ac5SJames Morse 13082869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */ 13182869ac5SJames Morse if (el2_reset_needed()) 1322077be67SLaura Abbott hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors); 13382869ac5SJames Morse else 13482869ac5SJames Morse hdr->__hyp_stub_vectors = 0; 13582869ac5SJames Morse 1368ec058fdSJames Morse /* Save the mpidr of the cpu we called cpu_suspend() on... */ 1378ec058fdSJames Morse if (sleep_cpu < 0) { 1389165dabbSMasanari Iida pr_err("Failing to hibernate on an unknown CPU.\n"); 1398ec058fdSJames Morse return -ENODEV; 1408ec058fdSJames Morse } 1418ec058fdSJames Morse hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); 1428ec058fdSJames Morse pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1438ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1448ec058fdSJames Morse 14582869ac5SJames Morse return 0; 14682869ac5SJames Morse } 14782869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save); 14882869ac5SJames Morse 14982869ac5SJames Morse int arch_hibernation_header_restore(void *addr) 15082869ac5SJames Morse { 1518ec058fdSJames Morse int ret; 15282869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 15382869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 15482869ac5SJames Morse 15582869ac5SJames Morse arch_hdr_invariants(&invariants); 15682869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) { 15782869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n"); 15882869ac5SJames Morse return -EINVAL; 15982869ac5SJames Morse } 16082869ac5SJames Morse 1618ec058fdSJames Morse sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); 1628ec058fdSJames Morse pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1638ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1648ec058fdSJames Morse if (sleep_cpu < 0) { 1658ec058fdSJames Morse pr_crit("Hibernated on a CPU not known to this kernel!\n"); 1668ec058fdSJames Morse sleep_cpu = -EINVAL; 1678ec058fdSJames Morse return -EINVAL; 1688ec058fdSJames Morse } 1698ec058fdSJames Morse if (!cpu_online(sleep_cpu)) { 1708ec058fdSJames Morse pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n"); 1718ec058fdSJames Morse ret = cpu_up(sleep_cpu); 1728ec058fdSJames Morse if (ret) { 1738ec058fdSJames Morse pr_err("Failed to bring hibernate-CPU up!\n"); 1748ec058fdSJames Morse sleep_cpu = -EINVAL; 1758ec058fdSJames Morse return ret; 1768ec058fdSJames Morse } 1778ec058fdSJames Morse } 1788ec058fdSJames Morse 17982869ac5SJames Morse resume_hdr = *hdr; 18082869ac5SJames Morse 18182869ac5SJames Morse return 0; 18282869ac5SJames Morse } 18382869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore); 18482869ac5SJames Morse 18582869ac5SJames Morse /* 18682869ac5SJames Morse * Copies length bytes, starting at src_start into an new page, 18782869ac5SJames Morse * perform cache maintentance, then maps it at the specified address low 18882869ac5SJames Morse * address as executable. 18982869ac5SJames Morse * 19082869ac5SJames Morse * This is used by hibernate to copy the code it needs to execute when 19182869ac5SJames Morse * overwriting the kernel text. This function generates a new set of page 19282869ac5SJames Morse * tables, which it loads into ttbr0. 19382869ac5SJames Morse * 19482869ac5SJames Morse * Length is provided as we probably only want 4K of data, even on a 64K 19582869ac5SJames Morse * page system. 19682869ac5SJames Morse */ 19782869ac5SJames Morse static int create_safe_exec_page(void *src_start, size_t length, 19882869ac5SJames Morse unsigned long dst_addr, 199051a7a94SPavel Tatashin phys_addr_t *phys_dst_addr) 20082869ac5SJames Morse { 20182869ac5SJames Morse int rc = 0; 2028c551f91SPavel Tatashin pgd_t *trans_pgd; 20320a004e7SWill Deacon pgd_t *pgdp; 20420a004e7SWill Deacon pud_t *pudp; 20520a004e7SWill Deacon pmd_t *pmdp; 20620a004e7SWill Deacon pte_t *ptep; 207051a7a94SPavel Tatashin unsigned long dst = get_safe_page(GFP_ATOMIC); 20882869ac5SJames Morse 20982869ac5SJames Morse if (!dst) { 21082869ac5SJames Morse rc = -ENOMEM; 21182869ac5SJames Morse goto out; 21282869ac5SJames Morse } 21382869ac5SJames Morse 21482869ac5SJames Morse memcpy((void *)dst, src_start, length); 215b4aecf78SWill Deacon __flush_icache_range(dst, dst + length); 21682869ac5SJames Morse 217051a7a94SPavel Tatashin trans_pgd = (void *)get_safe_page(GFP_ATOMIC); 2188c551f91SPavel Tatashin if (!trans_pgd) { 2198c551f91SPavel Tatashin rc = -ENOMEM; 2208c551f91SPavel Tatashin goto out; 2218c551f91SPavel Tatashin } 2228c551f91SPavel Tatashin 2238c551f91SPavel Tatashin pgdp = pgd_offset_raw(trans_pgd, dst_addr); 22420a004e7SWill Deacon if (pgd_none(READ_ONCE(*pgdp))) { 225051a7a94SPavel Tatashin pudp = (void *)get_safe_page(GFP_ATOMIC); 22620a004e7SWill Deacon if (!pudp) { 22782869ac5SJames Morse rc = -ENOMEM; 22882869ac5SJames Morse goto out; 22982869ac5SJames Morse } 23020a004e7SWill Deacon pgd_populate(&init_mm, pgdp, pudp); 23182869ac5SJames Morse } 23282869ac5SJames Morse 23320a004e7SWill Deacon pudp = pud_offset(pgdp, dst_addr); 23420a004e7SWill Deacon if (pud_none(READ_ONCE(*pudp))) { 235051a7a94SPavel Tatashin pmdp = (void *)get_safe_page(GFP_ATOMIC); 23620a004e7SWill Deacon if (!pmdp) { 23782869ac5SJames Morse rc = -ENOMEM; 23882869ac5SJames Morse goto out; 23982869ac5SJames Morse } 24020a004e7SWill Deacon pud_populate(&init_mm, pudp, pmdp); 24182869ac5SJames Morse } 24282869ac5SJames Morse 24320a004e7SWill Deacon pmdp = pmd_offset(pudp, dst_addr); 24420a004e7SWill Deacon if (pmd_none(READ_ONCE(*pmdp))) { 245051a7a94SPavel Tatashin ptep = (void *)get_safe_page(GFP_ATOMIC); 24620a004e7SWill Deacon if (!ptep) { 24782869ac5SJames Morse rc = -ENOMEM; 24882869ac5SJames Morse goto out; 24982869ac5SJames Morse } 25020a004e7SWill Deacon pmd_populate_kernel(&init_mm, pmdp, ptep); 25182869ac5SJames Morse } 25282869ac5SJames Morse 25320a004e7SWill Deacon ptep = pte_offset_kernel(pmdp, dst_addr); 25420a004e7SWill Deacon set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC)); 25582869ac5SJames Morse 2560194e760SMark Rutland /* 2570194e760SMark Rutland * Load our new page tables. A strict BBM approach requires that we 2580194e760SMark Rutland * ensure that TLBs are free of any entries that may overlap with the 2590194e760SMark Rutland * global mappings we are about to install. 2600194e760SMark Rutland * 2610194e760SMark Rutland * For a real hibernate/resume cycle TTBR0 currently points to a zero 2620194e760SMark Rutland * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI 2630194e760SMark Rutland * runtime services), while for a userspace-driven test_resume cycle it 2640194e760SMark Rutland * points to userspace page tables (and we must point it at a zero page 2650194e760SMark Rutland * ourselves). Elsewhere we only (un)install the idmap with preemption 2660194e760SMark Rutland * disabled, so T0SZ should be as required regardless. 2670194e760SMark Rutland */ 2680194e760SMark Rutland cpu_set_reserved_ttbr0(); 2690194e760SMark Rutland local_flush_tlb_all(); 270d234332cSPavel Tatashin write_sysreg(phys_to_ttbr(virt_to_phys(trans_pgd)), ttbr0_el1); 2710194e760SMark Rutland isb(); 27282869ac5SJames Morse 27382869ac5SJames Morse *phys_dst_addr = virt_to_phys((void *)dst); 27482869ac5SJames Morse 27582869ac5SJames Morse out: 27682869ac5SJames Morse return rc; 27782869ac5SJames Morse } 27882869ac5SJames Morse 2795ebe3a44SJames Morse #define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) 28082869ac5SJames Morse 28182869ac5SJames Morse int swsusp_arch_suspend(void) 28282869ac5SJames Morse { 28382869ac5SJames Morse int ret = 0; 28482869ac5SJames Morse unsigned long flags; 28582869ac5SJames Morse struct sleep_stack_data state; 28682869ac5SJames Morse 287d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) { 288d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); 289d74b4e4fSJames Morse return -EBUSY; 290d74b4e4fSJames Morse } 291d74b4e4fSJames Morse 2920fbeb318SJames Morse flags = local_daif_save(); 29382869ac5SJames Morse 29482869ac5SJames Morse if (__cpu_suspend_enter(&state)) { 295254a41c0SAKASHI Takahiro /* make the crash dump kernel image visible/saveable */ 296254a41c0SAKASHI Takahiro crash_prepare_suspend(); 297254a41c0SAKASHI Takahiro 2988ec058fdSJames Morse sleep_cpu = smp_processor_id(); 29982869ac5SJames Morse ret = swsusp_save(); 30082869ac5SJames Morse } else { 3015ebe3a44SJames Morse /* Clean kernel core startup/idle code to PoC*/ 3025ebe3a44SJames Morse dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); 3035ebe3a44SJames Morse dcache_clean_range(__idmap_text_start, __idmap_text_end); 3045ebe3a44SJames Morse 3055ebe3a44SJames Morse /* Clean kvm setup code to PoC? */ 306f7daa9c8SJames Morse if (el2_reset_needed()) { 3075ebe3a44SJames Morse dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); 308f7daa9c8SJames Morse dcache_clean_range(__hyp_text_start, __hyp_text_end); 309f7daa9c8SJames Morse } 31082869ac5SJames Morse 311254a41c0SAKASHI Takahiro /* make the crash dump kernel image protected again */ 312254a41c0SAKASHI Takahiro crash_post_resume(); 313254a41c0SAKASHI Takahiro 31482869ac5SJames Morse /* 31582869ac5SJames Morse * Tell the hibernation core that we've just restored 31682869ac5SJames Morse * the memory 31782869ac5SJames Morse */ 31882869ac5SJames Morse in_suspend = 0; 31982869ac5SJames Morse 3208ec058fdSJames Morse sleep_cpu = -EINVAL; 32182869ac5SJames Morse __cpu_suspend_exit(); 322647d0519SMarc Zyngier 323647d0519SMarc Zyngier /* 324647d0519SMarc Zyngier * Just in case the boot kernel did turn the SSBD 325647d0519SMarc Zyngier * mitigation off behind our back, let's set the state 326647d0519SMarc Zyngier * to what we expect it to be. 327647d0519SMarc Zyngier */ 328647d0519SMarc Zyngier switch (arm64_get_ssbd_state()) { 329647d0519SMarc Zyngier case ARM64_SSBD_FORCE_ENABLE: 330647d0519SMarc Zyngier case ARM64_SSBD_KERNEL: 331647d0519SMarc Zyngier arm64_set_ssbd_mitigation(true); 332647d0519SMarc Zyngier } 33382869ac5SJames Morse } 33482869ac5SJames Morse 3350fbeb318SJames Morse local_daif_restore(flags); 33682869ac5SJames Morse 33782869ac5SJames Morse return ret; 33882869ac5SJames Morse } 33982869ac5SJames Morse 34020a004e7SWill Deacon static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 3415ebe3a44SJames Morse { 34220a004e7SWill Deacon pte_t pte = READ_ONCE(*src_ptep); 3435ebe3a44SJames Morse 3445ebe3a44SJames Morse if (pte_valid(pte)) { 3455ebe3a44SJames Morse /* 3465ebe3a44SJames Morse * Resume will overwrite areas that may be marked 3475ebe3a44SJames Morse * read only (code, rodata). Clear the RDONLY bit from 3485ebe3a44SJames Morse * the temporary mappings we use during restore. 3495ebe3a44SJames Morse */ 35020a004e7SWill Deacon set_pte(dst_ptep, pte_mkwrite(pte)); 3515ebe3a44SJames Morse } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 3525ebe3a44SJames Morse /* 3535ebe3a44SJames Morse * debug_pagealloc will removed the PTE_VALID bit if 3545ebe3a44SJames Morse * the page isn't in use by the resume kernel. It may have 3555ebe3a44SJames Morse * been in use by the original kernel, in which case we need 3565ebe3a44SJames Morse * to put it back in our copy to do the restore. 3575ebe3a44SJames Morse * 3585ebe3a44SJames Morse * Before marking this entry valid, check the pfn should 3595ebe3a44SJames Morse * be mapped. 3605ebe3a44SJames Morse */ 3615ebe3a44SJames Morse BUG_ON(!pfn_valid(pte_pfn(pte))); 3625ebe3a44SJames Morse 36320a004e7SWill Deacon set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 3645ebe3a44SJames Morse } 3655ebe3a44SJames Morse } 3665ebe3a44SJames Morse 36720a004e7SWill Deacon static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, 36882869ac5SJames Morse unsigned long end) 36982869ac5SJames Morse { 37020a004e7SWill Deacon pte_t *src_ptep; 37120a004e7SWill Deacon pte_t *dst_ptep; 37282869ac5SJames Morse unsigned long addr = start; 37382869ac5SJames Morse 37420a004e7SWill Deacon dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); 37520a004e7SWill Deacon if (!dst_ptep) 37682869ac5SJames Morse return -ENOMEM; 37720a004e7SWill Deacon pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); 37820a004e7SWill Deacon dst_ptep = pte_offset_kernel(dst_pmdp, start); 37982869ac5SJames Morse 38020a004e7SWill Deacon src_ptep = pte_offset_kernel(src_pmdp, start); 38182869ac5SJames Morse do { 38220a004e7SWill Deacon _copy_pte(dst_ptep, src_ptep, addr); 38320a004e7SWill Deacon } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 38482869ac5SJames Morse 38582869ac5SJames Morse return 0; 38682869ac5SJames Morse } 38782869ac5SJames Morse 38820a004e7SWill Deacon static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start, 38982869ac5SJames Morse unsigned long end) 39082869ac5SJames Morse { 39120a004e7SWill Deacon pmd_t *src_pmdp; 39220a004e7SWill Deacon pmd_t *dst_pmdp; 39382869ac5SJames Morse unsigned long next; 39482869ac5SJames Morse unsigned long addr = start; 39582869ac5SJames Morse 39620a004e7SWill Deacon if (pud_none(READ_ONCE(*dst_pudp))) { 39720a004e7SWill Deacon dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); 39820a004e7SWill Deacon if (!dst_pmdp) 39982869ac5SJames Morse return -ENOMEM; 40020a004e7SWill Deacon pud_populate(&init_mm, dst_pudp, dst_pmdp); 40182869ac5SJames Morse } 40220a004e7SWill Deacon dst_pmdp = pmd_offset(dst_pudp, start); 40382869ac5SJames Morse 40420a004e7SWill Deacon src_pmdp = pmd_offset(src_pudp, start); 40582869ac5SJames Morse do { 40620a004e7SWill Deacon pmd_t pmd = READ_ONCE(*src_pmdp); 40720a004e7SWill Deacon 40882869ac5SJames Morse next = pmd_addr_end(addr, end); 40920a004e7SWill Deacon if (pmd_none(pmd)) 41082869ac5SJames Morse continue; 41120a004e7SWill Deacon if (pmd_table(pmd)) { 41220a004e7SWill Deacon if (copy_pte(dst_pmdp, src_pmdp, addr, next)) 41382869ac5SJames Morse return -ENOMEM; 41482869ac5SJames Morse } else { 41520a004e7SWill Deacon set_pmd(dst_pmdp, 41620a004e7SWill Deacon __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 41782869ac5SJames Morse } 41820a004e7SWill Deacon } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 41982869ac5SJames Morse 42082869ac5SJames Morse return 0; 42182869ac5SJames Morse } 42282869ac5SJames Morse 42320a004e7SWill Deacon static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start, 42482869ac5SJames Morse unsigned long end) 42582869ac5SJames Morse { 42620a004e7SWill Deacon pud_t *dst_pudp; 42720a004e7SWill Deacon pud_t *src_pudp; 42882869ac5SJames Morse unsigned long next; 42982869ac5SJames Morse unsigned long addr = start; 43082869ac5SJames Morse 43120a004e7SWill Deacon if (pgd_none(READ_ONCE(*dst_pgdp))) { 43220a004e7SWill Deacon dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); 43320a004e7SWill Deacon if (!dst_pudp) 43482869ac5SJames Morse return -ENOMEM; 43520a004e7SWill Deacon pgd_populate(&init_mm, dst_pgdp, dst_pudp); 43682869ac5SJames Morse } 43720a004e7SWill Deacon dst_pudp = pud_offset(dst_pgdp, start); 43882869ac5SJames Morse 43920a004e7SWill Deacon src_pudp = pud_offset(src_pgdp, start); 44082869ac5SJames Morse do { 44120a004e7SWill Deacon pud_t pud = READ_ONCE(*src_pudp); 44220a004e7SWill Deacon 44382869ac5SJames Morse next = pud_addr_end(addr, end); 44420a004e7SWill Deacon if (pud_none(pud)) 44582869ac5SJames Morse continue; 44620a004e7SWill Deacon if (pud_table(pud)) { 44720a004e7SWill Deacon if (copy_pmd(dst_pudp, src_pudp, addr, next)) 44882869ac5SJames Morse return -ENOMEM; 44982869ac5SJames Morse } else { 45020a004e7SWill Deacon set_pud(dst_pudp, 45120a004e7SWill Deacon __pud(pud_val(pud) & ~PMD_SECT_RDONLY)); 45282869ac5SJames Morse } 45320a004e7SWill Deacon } while (dst_pudp++, src_pudp++, addr = next, addr != end); 45482869ac5SJames Morse 45582869ac5SJames Morse return 0; 45682869ac5SJames Morse } 45782869ac5SJames Morse 45820a004e7SWill Deacon static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, 45982869ac5SJames Morse unsigned long end) 46082869ac5SJames Morse { 46182869ac5SJames Morse unsigned long next; 46282869ac5SJames Morse unsigned long addr = start; 46320a004e7SWill Deacon pgd_t *src_pgdp = pgd_offset_k(start); 46482869ac5SJames Morse 46520a004e7SWill Deacon dst_pgdp = pgd_offset_raw(dst_pgdp, start); 46682869ac5SJames Morse do { 46782869ac5SJames Morse next = pgd_addr_end(addr, end); 46820a004e7SWill Deacon if (pgd_none(READ_ONCE(*src_pgdp))) 46982869ac5SJames Morse continue; 47020a004e7SWill Deacon if (copy_pud(dst_pgdp, src_pgdp, addr, next)) 47182869ac5SJames Morse return -ENOMEM; 47220a004e7SWill Deacon } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 47382869ac5SJames Morse 47482869ac5SJames Morse return 0; 47582869ac5SJames Morse } 47682869ac5SJames Morse 47782869ac5SJames Morse /* 47882869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit(). 47982869ac5SJames Morse * 48082869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code, 48182869ac5SJames Morse * we don't need to free it here. 48282869ac5SJames Morse */ 48382869ac5SJames Morse int swsusp_arch_resume(void) 48482869ac5SJames Morse { 48582869ac5SJames Morse int rc = 0; 48682869ac5SJames Morse void *zero_page; 48782869ac5SJames Morse size_t exit_size; 48882869ac5SJames Morse pgd_t *tmp_pg_dir; 48982869ac5SJames Morse phys_addr_t phys_hibernate_exit; 49082869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, 49182869ac5SJames Morse void *, phys_addr_t, phys_addr_t); 49282869ac5SJames Morse 49382869ac5SJames Morse /* 494dfbca61aSMark Rutland * Restoring the memory image will overwrite the ttbr1 page tables. 495dfbca61aSMark Rutland * Create a second copy of just the linear map, and use this when 496dfbca61aSMark Rutland * restoring. 497dfbca61aSMark Rutland */ 498dfbca61aSMark Rutland tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC); 499dfbca61aSMark Rutland if (!tmp_pg_dir) { 500117f5727SMark Rutland pr_err("Failed to allocate memory for temporary page tables.\n"); 501dfbca61aSMark Rutland rc = -ENOMEM; 502dfbca61aSMark Rutland goto out; 503dfbca61aSMark Rutland } 50477ad4ce6SMark Rutland rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, PAGE_END); 505dfbca61aSMark Rutland if (rc) 506dfbca61aSMark Rutland goto out; 507dfbca61aSMark Rutland 508dfbca61aSMark Rutland /* 509dfbca61aSMark Rutland * We need a zero page that is zero before & after resume in order to 510dfbca61aSMark Rutland * to break before make on the ttbr1 page tables. 511dfbca61aSMark Rutland */ 512dfbca61aSMark Rutland zero_page = (void *)get_safe_page(GFP_ATOMIC); 513dfbca61aSMark Rutland if (!zero_page) { 514117f5727SMark Rutland pr_err("Failed to allocate zero page.\n"); 515dfbca61aSMark Rutland rc = -ENOMEM; 516dfbca61aSMark Rutland goto out; 517dfbca61aSMark Rutland } 518dfbca61aSMark Rutland 519dfbca61aSMark Rutland /* 52082869ac5SJames Morse * Locate the exit code in the bottom-but-one page, so that *NULL 52182869ac5SJames Morse * still has disastrous affects. 52282869ac5SJames Morse */ 52382869ac5SJames Morse hibernate_exit = (void *)PAGE_SIZE; 52482869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; 52582869ac5SJames Morse /* 52682869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate 52782869ac5SJames Morse * a new set of ttbr0 page tables and load them. 52882869ac5SJames Morse */ 52982869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size, 53082869ac5SJames Morse (unsigned long)hibernate_exit, 531051a7a94SPavel Tatashin &phys_hibernate_exit); 53282869ac5SJames Morse if (rc) { 533117f5727SMark Rutland pr_err("Failed to create safe executable page for hibernate_exit code.\n"); 53482869ac5SJames Morse goto out; 53582869ac5SJames Morse } 53682869ac5SJames Morse 53782869ac5SJames Morse /* 53882869ac5SJames Morse * The hibernate exit text contains a set of el2 vectors, that will 53982869ac5SJames Morse * be executed at el2 with the mmu off in order to reload hyp-stub. 54082869ac5SJames Morse */ 54182869ac5SJames Morse __flush_dcache_area(hibernate_exit, exit_size); 54282869ac5SJames Morse 54382869ac5SJames Morse /* 54482869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in 54582869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2. 54682869ac5SJames Morse * 54782869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE. 54882869ac5SJames Morse */ 54982869ac5SJames Morse if (el2_reset_needed()) { 55082869ac5SJames Morse phys_addr_t el2_vectors = phys_hibernate_exit; /* base */ 55182869ac5SJames Morse el2_vectors += hibernate_el2_vectors - 55282869ac5SJames Morse __hibernate_exit_text_start; /* offset */ 55382869ac5SJames Morse 55482869ac5SJames Morse __hyp_set_vectors(el2_vectors); 55582869ac5SJames Morse } 55682869ac5SJames Morse 55782869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, 5582077be67SLaura Abbott resume_hdr.reenter_kernel, restore_pblist, 55982869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); 56082869ac5SJames Morse 56182869ac5SJames Morse out: 56282869ac5SJames Morse return rc; 56382869ac5SJames Morse } 5641fe492ceSJames Morse 5658ec058fdSJames Morse int hibernate_resume_nonboot_cpu_disable(void) 5668ec058fdSJames Morse { 5678ec058fdSJames Morse if (sleep_cpu < 0) { 5689165dabbSMasanari Iida pr_err("Failing to resume from hibernate on an unknown CPU.\n"); 5698ec058fdSJames Morse return -ENODEV; 5708ec058fdSJames Morse } 5718ec058fdSJames Morse 5728ec058fdSJames Morse return freeze_secondary_cpus(sleep_cpu); 5738ec058fdSJames Morse } 574