1af873fceSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 282869ac5SJames Morse /*: 382869ac5SJames Morse * Hibernate support specific for ARM64 482869ac5SJames Morse * 582869ac5SJames Morse * Derived from work on ARM hibernation support by: 682869ac5SJames Morse * 782869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove 882869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) 982869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) 1082869ac5SJames Morse * https://lkml.org/lkml/2010/6/18/4 1182869ac5SJames Morse * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html 1282869ac5SJames Morse * https://patchwork.kernel.org/patch/96442/ 1382869ac5SJames Morse * 1482869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 1582869ac5SJames Morse */ 1682869ac5SJames Morse #define pr_fmt(x) "hibernate: " x 178ec058fdSJames Morse #include <linux/cpu.h> 1882869ac5SJames Morse #include <linux/kvm_host.h> 1982869ac5SJames Morse #include <linux/mm.h> 2082869ac5SJames Morse #include <linux/pm.h> 2182869ac5SJames Morse #include <linux/sched.h> 2282869ac5SJames Morse #include <linux/suspend.h> 2382869ac5SJames Morse #include <linux/utsname.h> 2482869ac5SJames Morse #include <linux/version.h> 2582869ac5SJames Morse 2682869ac5SJames Morse #include <asm/barrier.h> 2782869ac5SJames Morse #include <asm/cacheflush.h> 288ec058fdSJames Morse #include <asm/cputype.h> 290fbeb318SJames Morse #include <asm/daifflags.h> 3082869ac5SJames Morse #include <asm/irqflags.h> 31254a41c0SAKASHI Takahiro #include <asm/kexec.h> 3282869ac5SJames Morse #include <asm/memory.h> 3382869ac5SJames Morse #include <asm/mmu_context.h> 3482869ac5SJames Morse #include <asm/pgalloc.h> 3582869ac5SJames Morse #include <asm/pgtable.h> 3682869ac5SJames Morse #include <asm/pgtable-hwdef.h> 3782869ac5SJames Morse #include <asm/sections.h> 38d74b4e4fSJames Morse #include <asm/smp.h> 398ec058fdSJames Morse #include <asm/smp_plat.h> 4082869ac5SJames Morse #include <asm/suspend.h> 410194e760SMark Rutland #include <asm/sysreg.h> 4282869ac5SJames Morse #include <asm/virt.h> 4382869ac5SJames Morse 4482869ac5SJames Morse /* 4582869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it 4682869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This 4782869ac5SJames Morse * doesn't happen with either KASLR. 4882869ac5SJames Morse * 4982869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in 5082869ac5SJames Morse * kernel/power/hibernate.c 5182869ac5SJames Morse */ 5282869ac5SJames Morse extern int in_suspend; 5382869ac5SJames Morse 5482869ac5SJames Morse /* Do we need to reset el2? */ 5582869ac5SJames Morse #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) 5682869ac5SJames Morse 5782869ac5SJames Morse /* temporary el2 vectors in the __hibernate_exit_text section. */ 5882869ac5SJames Morse extern char hibernate_el2_vectors[]; 5982869ac5SJames Morse 6082869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ 6182869ac5SJames Morse extern char __hyp_stub_vectors[]; 6282869ac5SJames Morse 6382869ac5SJames Morse /* 648ec058fdSJames Morse * The logical cpu number we should resume on, initialised to a non-cpu 658ec058fdSJames Morse * number. 668ec058fdSJames Morse */ 678ec058fdSJames Morse static int sleep_cpu = -EINVAL; 688ec058fdSJames Morse 698ec058fdSJames Morse /* 7082869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number 7182869ac5SJames Morse * and date in here so that we guarantee not to resume with a different 7282869ac5SJames Morse * kernel. 7382869ac5SJames Morse */ 7482869ac5SJames Morse struct arch_hibernate_hdr_invariants { 7582869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1]; 7682869ac5SJames Morse }; 7782869ac5SJames Morse 7882869ac5SJames Morse /* These values need to be know across a hibernate/restore. */ 7982869ac5SJames Morse static struct arch_hibernate_hdr { 8082869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 8182869ac5SJames Morse 8282869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */ 8382869ac5SJames Morse phys_addr_t ttbr1_el1; 8482869ac5SJames Morse void (*reenter_kernel)(void); 8582869ac5SJames Morse 8682869ac5SJames Morse /* 8782869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to 8882869ac5SJames Morse * re-configure el2. 8982869ac5SJames Morse */ 9082869ac5SJames Morse phys_addr_t __hyp_stub_vectors; 918ec058fdSJames Morse 928ec058fdSJames Morse u64 sleep_cpu_mpidr; 9382869ac5SJames Morse } resume_hdr; 9482869ac5SJames Morse 9582869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) 9682869ac5SJames Morse { 9782869ac5SJames Morse memset(i, 0, sizeof(*i)); 9882869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); 9982869ac5SJames Morse } 10082869ac5SJames Morse 10182869ac5SJames Morse int pfn_is_nosave(unsigned long pfn) 10282869ac5SJames Morse { 1032077be67SLaura Abbott unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); 1042077be67SLaura Abbott unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); 10582869ac5SJames Morse 106254a41c0SAKASHI Takahiro return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || 107254a41c0SAKASHI Takahiro crash_is_nosave(pfn); 10882869ac5SJames Morse } 10982869ac5SJames Morse 11082869ac5SJames Morse void notrace save_processor_state(void) 11182869ac5SJames Morse { 11282869ac5SJames Morse WARN_ON(num_online_cpus() != 1); 11382869ac5SJames Morse } 11482869ac5SJames Morse 11582869ac5SJames Morse void notrace restore_processor_state(void) 11682869ac5SJames Morse { 11782869ac5SJames Morse } 11882869ac5SJames Morse 11982869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size) 12082869ac5SJames Morse { 12182869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 12282869ac5SJames Morse 12382869ac5SJames Morse if (max_size < sizeof(*hdr)) 12482869ac5SJames Morse return -EOVERFLOW; 12582869ac5SJames Morse 12682869ac5SJames Morse arch_hdr_invariants(&hdr->invariants); 1272077be67SLaura Abbott hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir); 12882869ac5SJames Morse hdr->reenter_kernel = _cpu_resume; 12982869ac5SJames Morse 13082869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */ 13182869ac5SJames Morse if (el2_reset_needed()) 1322077be67SLaura Abbott hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors); 13382869ac5SJames Morse else 13482869ac5SJames Morse hdr->__hyp_stub_vectors = 0; 13582869ac5SJames Morse 1368ec058fdSJames Morse /* Save the mpidr of the cpu we called cpu_suspend() on... */ 1378ec058fdSJames Morse if (sleep_cpu < 0) { 1389165dabbSMasanari Iida pr_err("Failing to hibernate on an unknown CPU.\n"); 1398ec058fdSJames Morse return -ENODEV; 1408ec058fdSJames Morse } 1418ec058fdSJames Morse hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); 1428ec058fdSJames Morse pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1438ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1448ec058fdSJames Morse 14582869ac5SJames Morse return 0; 14682869ac5SJames Morse } 14782869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save); 14882869ac5SJames Morse 14982869ac5SJames Morse int arch_hibernation_header_restore(void *addr) 15082869ac5SJames Morse { 1518ec058fdSJames Morse int ret; 15282869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 15382869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 15482869ac5SJames Morse 15582869ac5SJames Morse arch_hdr_invariants(&invariants); 15682869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) { 15782869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n"); 15882869ac5SJames Morse return -EINVAL; 15982869ac5SJames Morse } 16082869ac5SJames Morse 1618ec058fdSJames Morse sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); 1628ec058fdSJames Morse pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1638ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1648ec058fdSJames Morse if (sleep_cpu < 0) { 1658ec058fdSJames Morse pr_crit("Hibernated on a CPU not known to this kernel!\n"); 1668ec058fdSJames Morse sleep_cpu = -EINVAL; 1678ec058fdSJames Morse return -EINVAL; 1688ec058fdSJames Morse } 1698ec058fdSJames Morse if (!cpu_online(sleep_cpu)) { 1708ec058fdSJames Morse pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n"); 1718ec058fdSJames Morse ret = cpu_up(sleep_cpu); 1728ec058fdSJames Morse if (ret) { 1738ec058fdSJames Morse pr_err("Failed to bring hibernate-CPU up!\n"); 1748ec058fdSJames Morse sleep_cpu = -EINVAL; 1758ec058fdSJames Morse return ret; 1768ec058fdSJames Morse } 1778ec058fdSJames Morse } 1788ec058fdSJames Morse 17982869ac5SJames Morse resume_hdr = *hdr; 18082869ac5SJames Morse 18182869ac5SJames Morse return 0; 18282869ac5SJames Morse } 18382869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore); 18482869ac5SJames Morse 18582869ac5SJames Morse /* 18682869ac5SJames Morse * Copies length bytes, starting at src_start into an new page, 18782869ac5SJames Morse * perform cache maintentance, then maps it at the specified address low 18882869ac5SJames Morse * address as executable. 18982869ac5SJames Morse * 19082869ac5SJames Morse * This is used by hibernate to copy the code it needs to execute when 19182869ac5SJames Morse * overwriting the kernel text. This function generates a new set of page 19282869ac5SJames Morse * tables, which it loads into ttbr0. 19382869ac5SJames Morse * 19482869ac5SJames Morse * Length is provided as we probably only want 4K of data, even on a 64K 19582869ac5SJames Morse * page system. 19682869ac5SJames Morse */ 19782869ac5SJames Morse static int create_safe_exec_page(void *src_start, size_t length, 19882869ac5SJames Morse unsigned long dst_addr, 199051a7a94SPavel Tatashin phys_addr_t *phys_dst_addr) 20082869ac5SJames Morse { 20113373f0eSPavel Tatashin void *page = (void *)get_safe_page(GFP_ATOMIC); 2028c551f91SPavel Tatashin pgd_t *trans_pgd; 20320a004e7SWill Deacon pgd_t *pgdp; 20420a004e7SWill Deacon pud_t *pudp; 20520a004e7SWill Deacon pmd_t *pmdp; 20620a004e7SWill Deacon pte_t *ptep; 20782869ac5SJames Morse 20813373f0eSPavel Tatashin if (!page) 209a89d7ff9SPavel Tatashin return -ENOMEM; 21082869ac5SJames Morse 21113373f0eSPavel Tatashin memcpy(page, src_start, length); 21213373f0eSPavel Tatashin __flush_icache_range((unsigned long)page, (unsigned long)page + length); 21382869ac5SJames Morse 214051a7a94SPavel Tatashin trans_pgd = (void *)get_safe_page(GFP_ATOMIC); 215a89d7ff9SPavel Tatashin if (!trans_pgd) 216a89d7ff9SPavel Tatashin return -ENOMEM; 2178c551f91SPavel Tatashin 2188c551f91SPavel Tatashin pgdp = pgd_offset_raw(trans_pgd, dst_addr); 21920a004e7SWill Deacon if (pgd_none(READ_ONCE(*pgdp))) { 220051a7a94SPavel Tatashin pudp = (void *)get_safe_page(GFP_ATOMIC); 221a89d7ff9SPavel Tatashin if (!pudp) 222a89d7ff9SPavel Tatashin return -ENOMEM; 22320a004e7SWill Deacon pgd_populate(&init_mm, pgdp, pudp); 22482869ac5SJames Morse } 22582869ac5SJames Morse 22620a004e7SWill Deacon pudp = pud_offset(pgdp, dst_addr); 22720a004e7SWill Deacon if (pud_none(READ_ONCE(*pudp))) { 228051a7a94SPavel Tatashin pmdp = (void *)get_safe_page(GFP_ATOMIC); 229a89d7ff9SPavel Tatashin if (!pmdp) 230a89d7ff9SPavel Tatashin return -ENOMEM; 23120a004e7SWill Deacon pud_populate(&init_mm, pudp, pmdp); 23282869ac5SJames Morse } 23382869ac5SJames Morse 23420a004e7SWill Deacon pmdp = pmd_offset(pudp, dst_addr); 23520a004e7SWill Deacon if (pmd_none(READ_ONCE(*pmdp))) { 236051a7a94SPavel Tatashin ptep = (void *)get_safe_page(GFP_ATOMIC); 237a89d7ff9SPavel Tatashin if (!ptep) 238a89d7ff9SPavel Tatashin return -ENOMEM; 23920a004e7SWill Deacon pmd_populate_kernel(&init_mm, pmdp, ptep); 24082869ac5SJames Morse } 24182869ac5SJames Morse 24220a004e7SWill Deacon ptep = pte_offset_kernel(pmdp, dst_addr); 24313373f0eSPavel Tatashin set_pte(ptep, pfn_pte(virt_to_pfn(page), PAGE_KERNEL_EXEC)); 24482869ac5SJames Morse 2450194e760SMark Rutland /* 2460194e760SMark Rutland * Load our new page tables. A strict BBM approach requires that we 2470194e760SMark Rutland * ensure that TLBs are free of any entries that may overlap with the 2480194e760SMark Rutland * global mappings we are about to install. 2490194e760SMark Rutland * 2500194e760SMark Rutland * For a real hibernate/resume cycle TTBR0 currently points to a zero 2510194e760SMark Rutland * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI 2520194e760SMark Rutland * runtime services), while for a userspace-driven test_resume cycle it 2530194e760SMark Rutland * points to userspace page tables (and we must point it at a zero page 2540194e760SMark Rutland * ourselves). Elsewhere we only (un)install the idmap with preemption 2550194e760SMark Rutland * disabled, so T0SZ should be as required regardless. 2560194e760SMark Rutland */ 2570194e760SMark Rutland cpu_set_reserved_ttbr0(); 2580194e760SMark Rutland local_flush_tlb_all(); 259d234332cSPavel Tatashin write_sysreg(phys_to_ttbr(virt_to_phys(trans_pgd)), ttbr0_el1); 2600194e760SMark Rutland isb(); 26182869ac5SJames Morse 26213373f0eSPavel Tatashin *phys_dst_addr = virt_to_phys(page); 26382869ac5SJames Morse 264a89d7ff9SPavel Tatashin return 0; 26582869ac5SJames Morse } 26682869ac5SJames Morse 2675ebe3a44SJames Morse #define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) 26882869ac5SJames Morse 26982869ac5SJames Morse int swsusp_arch_suspend(void) 27082869ac5SJames Morse { 27182869ac5SJames Morse int ret = 0; 27282869ac5SJames Morse unsigned long flags; 27382869ac5SJames Morse struct sleep_stack_data state; 27482869ac5SJames Morse 275d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) { 276d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); 277d74b4e4fSJames Morse return -EBUSY; 278d74b4e4fSJames Morse } 279d74b4e4fSJames Morse 2800fbeb318SJames Morse flags = local_daif_save(); 28182869ac5SJames Morse 28282869ac5SJames Morse if (__cpu_suspend_enter(&state)) { 283254a41c0SAKASHI Takahiro /* make the crash dump kernel image visible/saveable */ 284254a41c0SAKASHI Takahiro crash_prepare_suspend(); 285254a41c0SAKASHI Takahiro 2868ec058fdSJames Morse sleep_cpu = smp_processor_id(); 28782869ac5SJames Morse ret = swsusp_save(); 28882869ac5SJames Morse } else { 2895ebe3a44SJames Morse /* Clean kernel core startup/idle code to PoC*/ 2905ebe3a44SJames Morse dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); 2915ebe3a44SJames Morse dcache_clean_range(__idmap_text_start, __idmap_text_end); 2925ebe3a44SJames Morse 2935ebe3a44SJames Morse /* Clean kvm setup code to PoC? */ 294f7daa9c8SJames Morse if (el2_reset_needed()) { 2955ebe3a44SJames Morse dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); 296f7daa9c8SJames Morse dcache_clean_range(__hyp_text_start, __hyp_text_end); 297f7daa9c8SJames Morse } 29882869ac5SJames Morse 299254a41c0SAKASHI Takahiro /* make the crash dump kernel image protected again */ 300254a41c0SAKASHI Takahiro crash_post_resume(); 301254a41c0SAKASHI Takahiro 30282869ac5SJames Morse /* 30382869ac5SJames Morse * Tell the hibernation core that we've just restored 30482869ac5SJames Morse * the memory 30582869ac5SJames Morse */ 30682869ac5SJames Morse in_suspend = 0; 30782869ac5SJames Morse 3088ec058fdSJames Morse sleep_cpu = -EINVAL; 30982869ac5SJames Morse __cpu_suspend_exit(); 310647d0519SMarc Zyngier 311647d0519SMarc Zyngier /* 312647d0519SMarc Zyngier * Just in case the boot kernel did turn the SSBD 313647d0519SMarc Zyngier * mitigation off behind our back, let's set the state 314647d0519SMarc Zyngier * to what we expect it to be. 315647d0519SMarc Zyngier */ 316647d0519SMarc Zyngier switch (arm64_get_ssbd_state()) { 317647d0519SMarc Zyngier case ARM64_SSBD_FORCE_ENABLE: 318647d0519SMarc Zyngier case ARM64_SSBD_KERNEL: 319647d0519SMarc Zyngier arm64_set_ssbd_mitigation(true); 320647d0519SMarc Zyngier } 32182869ac5SJames Morse } 32282869ac5SJames Morse 3230fbeb318SJames Morse local_daif_restore(flags); 32482869ac5SJames Morse 32582869ac5SJames Morse return ret; 32682869ac5SJames Morse } 32782869ac5SJames Morse 32820a004e7SWill Deacon static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 3295ebe3a44SJames Morse { 33020a004e7SWill Deacon pte_t pte = READ_ONCE(*src_ptep); 3315ebe3a44SJames Morse 3325ebe3a44SJames Morse if (pte_valid(pte)) { 3335ebe3a44SJames Morse /* 3345ebe3a44SJames Morse * Resume will overwrite areas that may be marked 3355ebe3a44SJames Morse * read only (code, rodata). Clear the RDONLY bit from 3365ebe3a44SJames Morse * the temporary mappings we use during restore. 3375ebe3a44SJames Morse */ 33820a004e7SWill Deacon set_pte(dst_ptep, pte_mkwrite(pte)); 3395ebe3a44SJames Morse } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 3405ebe3a44SJames Morse /* 3415ebe3a44SJames Morse * debug_pagealloc will removed the PTE_VALID bit if 3425ebe3a44SJames Morse * the page isn't in use by the resume kernel. It may have 3435ebe3a44SJames Morse * been in use by the original kernel, in which case we need 3445ebe3a44SJames Morse * to put it back in our copy to do the restore. 3455ebe3a44SJames Morse * 3465ebe3a44SJames Morse * Before marking this entry valid, check the pfn should 3475ebe3a44SJames Morse * be mapped. 3485ebe3a44SJames Morse */ 3495ebe3a44SJames Morse BUG_ON(!pfn_valid(pte_pfn(pte))); 3505ebe3a44SJames Morse 35120a004e7SWill Deacon set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 3525ebe3a44SJames Morse } 3535ebe3a44SJames Morse } 3545ebe3a44SJames Morse 35520a004e7SWill Deacon static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, 35682869ac5SJames Morse unsigned long end) 35782869ac5SJames Morse { 35820a004e7SWill Deacon pte_t *src_ptep; 35920a004e7SWill Deacon pte_t *dst_ptep; 36082869ac5SJames Morse unsigned long addr = start; 36182869ac5SJames Morse 36220a004e7SWill Deacon dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); 36320a004e7SWill Deacon if (!dst_ptep) 36482869ac5SJames Morse return -ENOMEM; 36520a004e7SWill Deacon pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); 36620a004e7SWill Deacon dst_ptep = pte_offset_kernel(dst_pmdp, start); 36782869ac5SJames Morse 36820a004e7SWill Deacon src_ptep = pte_offset_kernel(src_pmdp, start); 36982869ac5SJames Morse do { 37020a004e7SWill Deacon _copy_pte(dst_ptep, src_ptep, addr); 37120a004e7SWill Deacon } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 37282869ac5SJames Morse 37382869ac5SJames Morse return 0; 37482869ac5SJames Morse } 37582869ac5SJames Morse 37620a004e7SWill Deacon static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start, 37782869ac5SJames Morse unsigned long end) 37882869ac5SJames Morse { 37920a004e7SWill Deacon pmd_t *src_pmdp; 38020a004e7SWill Deacon pmd_t *dst_pmdp; 38182869ac5SJames Morse unsigned long next; 38282869ac5SJames Morse unsigned long addr = start; 38382869ac5SJames Morse 38420a004e7SWill Deacon if (pud_none(READ_ONCE(*dst_pudp))) { 38520a004e7SWill Deacon dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); 38620a004e7SWill Deacon if (!dst_pmdp) 38782869ac5SJames Morse return -ENOMEM; 38820a004e7SWill Deacon pud_populate(&init_mm, dst_pudp, dst_pmdp); 38982869ac5SJames Morse } 39020a004e7SWill Deacon dst_pmdp = pmd_offset(dst_pudp, start); 39182869ac5SJames Morse 39220a004e7SWill Deacon src_pmdp = pmd_offset(src_pudp, start); 39382869ac5SJames Morse do { 39420a004e7SWill Deacon pmd_t pmd = READ_ONCE(*src_pmdp); 39520a004e7SWill Deacon 39682869ac5SJames Morse next = pmd_addr_end(addr, end); 39720a004e7SWill Deacon if (pmd_none(pmd)) 39882869ac5SJames Morse continue; 39920a004e7SWill Deacon if (pmd_table(pmd)) { 40020a004e7SWill Deacon if (copy_pte(dst_pmdp, src_pmdp, addr, next)) 40182869ac5SJames Morse return -ENOMEM; 40282869ac5SJames Morse } else { 40320a004e7SWill Deacon set_pmd(dst_pmdp, 40420a004e7SWill Deacon __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 40582869ac5SJames Morse } 40620a004e7SWill Deacon } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 40782869ac5SJames Morse 40882869ac5SJames Morse return 0; 40982869ac5SJames Morse } 41082869ac5SJames Morse 41120a004e7SWill Deacon static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start, 41282869ac5SJames Morse unsigned long end) 41382869ac5SJames Morse { 41420a004e7SWill Deacon pud_t *dst_pudp; 41520a004e7SWill Deacon pud_t *src_pudp; 41682869ac5SJames Morse unsigned long next; 41782869ac5SJames Morse unsigned long addr = start; 41882869ac5SJames Morse 41920a004e7SWill Deacon if (pgd_none(READ_ONCE(*dst_pgdp))) { 42020a004e7SWill Deacon dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); 42120a004e7SWill Deacon if (!dst_pudp) 42282869ac5SJames Morse return -ENOMEM; 42320a004e7SWill Deacon pgd_populate(&init_mm, dst_pgdp, dst_pudp); 42482869ac5SJames Morse } 42520a004e7SWill Deacon dst_pudp = pud_offset(dst_pgdp, start); 42682869ac5SJames Morse 42720a004e7SWill Deacon src_pudp = pud_offset(src_pgdp, start); 42882869ac5SJames Morse do { 42920a004e7SWill Deacon pud_t pud = READ_ONCE(*src_pudp); 43020a004e7SWill Deacon 43182869ac5SJames Morse next = pud_addr_end(addr, end); 43220a004e7SWill Deacon if (pud_none(pud)) 43382869ac5SJames Morse continue; 43420a004e7SWill Deacon if (pud_table(pud)) { 43520a004e7SWill Deacon if (copy_pmd(dst_pudp, src_pudp, addr, next)) 43682869ac5SJames Morse return -ENOMEM; 43782869ac5SJames Morse } else { 43820a004e7SWill Deacon set_pud(dst_pudp, 4397ea40889SPavel Tatashin __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); 44082869ac5SJames Morse } 44120a004e7SWill Deacon } while (dst_pudp++, src_pudp++, addr = next, addr != end); 44282869ac5SJames Morse 44382869ac5SJames Morse return 0; 44482869ac5SJames Morse } 44582869ac5SJames Morse 44620a004e7SWill Deacon static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, 44782869ac5SJames Morse unsigned long end) 44882869ac5SJames Morse { 44982869ac5SJames Morse unsigned long next; 45082869ac5SJames Morse unsigned long addr = start; 45120a004e7SWill Deacon pgd_t *src_pgdp = pgd_offset_k(start); 45282869ac5SJames Morse 45320a004e7SWill Deacon dst_pgdp = pgd_offset_raw(dst_pgdp, start); 45482869ac5SJames Morse do { 45582869ac5SJames Morse next = pgd_addr_end(addr, end); 45620a004e7SWill Deacon if (pgd_none(READ_ONCE(*src_pgdp))) 45782869ac5SJames Morse continue; 45820a004e7SWill Deacon if (copy_pud(dst_pgdp, src_pgdp, addr, next)) 45982869ac5SJames Morse return -ENOMEM; 46020a004e7SWill Deacon } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 46182869ac5SJames Morse 46282869ac5SJames Morse return 0; 46382869ac5SJames Morse } 46482869ac5SJames Morse 46582869ac5SJames Morse /* 46682869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit(). 46782869ac5SJames Morse * 46882869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code, 46982869ac5SJames Morse * we don't need to free it here. 47082869ac5SJames Morse */ 47182869ac5SJames Morse int swsusp_arch_resume(void) 47282869ac5SJames Morse { 473a89d7ff9SPavel Tatashin int rc; 47482869ac5SJames Morse void *zero_page; 47582869ac5SJames Morse size_t exit_size; 47682869ac5SJames Morse pgd_t *tmp_pg_dir; 47782869ac5SJames Morse phys_addr_t phys_hibernate_exit; 47882869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, 47982869ac5SJames Morse void *, phys_addr_t, phys_addr_t); 48082869ac5SJames Morse 48182869ac5SJames Morse /* 482dfbca61aSMark Rutland * Restoring the memory image will overwrite the ttbr1 page tables. 483dfbca61aSMark Rutland * Create a second copy of just the linear map, and use this when 484dfbca61aSMark Rutland * restoring. 485dfbca61aSMark Rutland */ 486dfbca61aSMark Rutland tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC); 487dfbca61aSMark Rutland if (!tmp_pg_dir) { 488117f5727SMark Rutland pr_err("Failed to allocate memory for temporary page tables.\n"); 489a89d7ff9SPavel Tatashin return -ENOMEM; 490dfbca61aSMark Rutland } 49177ad4ce6SMark Rutland rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, PAGE_END); 492dfbca61aSMark Rutland if (rc) 493a89d7ff9SPavel Tatashin return rc; 494dfbca61aSMark Rutland 495dfbca61aSMark Rutland /* 496dfbca61aSMark Rutland * We need a zero page that is zero before & after resume in order to 497dfbca61aSMark Rutland * to break before make on the ttbr1 page tables. 498dfbca61aSMark Rutland */ 499dfbca61aSMark Rutland zero_page = (void *)get_safe_page(GFP_ATOMIC); 500dfbca61aSMark Rutland if (!zero_page) { 501117f5727SMark Rutland pr_err("Failed to allocate zero page.\n"); 502a89d7ff9SPavel Tatashin return -ENOMEM; 503dfbca61aSMark Rutland } 504dfbca61aSMark Rutland 505dfbca61aSMark Rutland /* 50682869ac5SJames Morse * Locate the exit code in the bottom-but-one page, so that *NULL 50782869ac5SJames Morse * still has disastrous affects. 50882869ac5SJames Morse */ 50982869ac5SJames Morse hibernate_exit = (void *)PAGE_SIZE; 51082869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; 51182869ac5SJames Morse /* 51282869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate 51382869ac5SJames Morse * a new set of ttbr0 page tables and load them. 51482869ac5SJames Morse */ 51582869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size, 51682869ac5SJames Morse (unsigned long)hibernate_exit, 517051a7a94SPavel Tatashin &phys_hibernate_exit); 51882869ac5SJames Morse if (rc) { 519117f5727SMark Rutland pr_err("Failed to create safe executable page for hibernate_exit code.\n"); 520a89d7ff9SPavel Tatashin return rc; 52182869ac5SJames Morse } 52282869ac5SJames Morse 52382869ac5SJames Morse /* 52482869ac5SJames Morse * The hibernate exit text contains a set of el2 vectors, that will 52582869ac5SJames Morse * be executed at el2 with the mmu off in order to reload hyp-stub. 52682869ac5SJames Morse */ 52782869ac5SJames Morse __flush_dcache_area(hibernate_exit, exit_size); 52882869ac5SJames Morse 52982869ac5SJames Morse /* 53082869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in 53182869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2. 53282869ac5SJames Morse * 53382869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE. 53482869ac5SJames Morse */ 53582869ac5SJames Morse if (el2_reset_needed()) { 53682869ac5SJames Morse phys_addr_t el2_vectors = phys_hibernate_exit; /* base */ 53782869ac5SJames Morse el2_vectors += hibernate_el2_vectors - 53882869ac5SJames Morse __hibernate_exit_text_start; /* offset */ 53982869ac5SJames Morse 54082869ac5SJames Morse __hyp_set_vectors(el2_vectors); 54182869ac5SJames Morse } 54282869ac5SJames Morse 54382869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, 5442077be67SLaura Abbott resume_hdr.reenter_kernel, restore_pblist, 54582869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); 54682869ac5SJames Morse 547a89d7ff9SPavel Tatashin return 0; 54882869ac5SJames Morse } 5491fe492ceSJames Morse 5508ec058fdSJames Morse int hibernate_resume_nonboot_cpu_disable(void) 5518ec058fdSJames Morse { 5528ec058fdSJames Morse if (sleep_cpu < 0) { 5539165dabbSMasanari Iida pr_err("Failing to resume from hibernate on an unknown CPU.\n"); 5548ec058fdSJames Morse return -ENODEV; 5558ec058fdSJames Morse } 5568ec058fdSJames Morse 5578ec058fdSJames Morse return freeze_secondary_cpus(sleep_cpu); 5588ec058fdSJames Morse } 559