1af873fceSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 282869ac5SJames Morse /*: 382869ac5SJames Morse * Hibernate support specific for ARM64 482869ac5SJames Morse * 582869ac5SJames Morse * Derived from work on ARM hibernation support by: 682869ac5SJames Morse * 782869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove 882869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) 982869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) 1082869ac5SJames Morse * https://lkml.org/lkml/2010/6/18/4 1182869ac5SJames Morse * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html 1282869ac5SJames Morse * https://patchwork.kernel.org/patch/96442/ 1382869ac5SJames Morse * 1482869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 1582869ac5SJames Morse */ 1682869ac5SJames Morse #define pr_fmt(x) "hibernate: " x 178ec058fdSJames Morse #include <linux/cpu.h> 1882869ac5SJames Morse #include <linux/kvm_host.h> 1982869ac5SJames Morse #include <linux/mm.h> 2082869ac5SJames Morse #include <linux/pm.h> 2182869ac5SJames Morse #include <linux/sched.h> 2282869ac5SJames Morse #include <linux/suspend.h> 2382869ac5SJames Morse #include <linux/utsname.h> 2482869ac5SJames Morse #include <linux/version.h> 2582869ac5SJames Morse 2682869ac5SJames Morse #include <asm/barrier.h> 2782869ac5SJames Morse #include <asm/cacheflush.h> 288ec058fdSJames Morse #include <asm/cputype.h> 290fbeb318SJames Morse #include <asm/daifflags.h> 3082869ac5SJames Morse #include <asm/irqflags.h> 31254a41c0SAKASHI Takahiro #include <asm/kexec.h> 3282869ac5SJames Morse #include <asm/memory.h> 3382869ac5SJames Morse #include <asm/mmu_context.h> 3482869ac5SJames Morse #include <asm/pgalloc.h> 3582869ac5SJames Morse #include <asm/pgtable-hwdef.h> 3682869ac5SJames Morse #include <asm/sections.h> 37d74b4e4fSJames Morse #include <asm/smp.h> 388ec058fdSJames Morse #include <asm/smp_plat.h> 3982869ac5SJames Morse #include <asm/suspend.h> 400194e760SMark Rutland #include <asm/sysreg.h> 4182869ac5SJames Morse #include <asm/virt.h> 4282869ac5SJames Morse 4382869ac5SJames Morse /* 4482869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it 4582869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This 4682869ac5SJames Morse * doesn't happen with either KASLR. 4782869ac5SJames Morse * 4882869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in 4982869ac5SJames Morse * kernel/power/hibernate.c 5082869ac5SJames Morse */ 5182869ac5SJames Morse extern int in_suspend; 5282869ac5SJames Morse 5382869ac5SJames Morse /* Do we need to reset el2? */ 5482869ac5SJames Morse #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) 5582869ac5SJames Morse 5682869ac5SJames Morse /* temporary el2 vectors in the __hibernate_exit_text section. */ 5782869ac5SJames Morse extern char hibernate_el2_vectors[]; 5882869ac5SJames Morse 5982869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ 6082869ac5SJames Morse extern char __hyp_stub_vectors[]; 6182869ac5SJames Morse 6282869ac5SJames Morse /* 638ec058fdSJames Morse * The logical cpu number we should resume on, initialised to a non-cpu 648ec058fdSJames Morse * number. 658ec058fdSJames Morse */ 668ec058fdSJames Morse static int sleep_cpu = -EINVAL; 678ec058fdSJames Morse 688ec058fdSJames Morse /* 6982869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number 7082869ac5SJames Morse * and date in here so that we guarantee not to resume with a different 7182869ac5SJames Morse * kernel. 7282869ac5SJames Morse */ 7382869ac5SJames Morse struct arch_hibernate_hdr_invariants { 7482869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1]; 7582869ac5SJames Morse }; 7682869ac5SJames Morse 7782869ac5SJames Morse /* These values need to be know across a hibernate/restore. */ 7882869ac5SJames Morse static struct arch_hibernate_hdr { 7982869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 8082869ac5SJames Morse 8182869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */ 8282869ac5SJames Morse phys_addr_t ttbr1_el1; 8382869ac5SJames Morse void (*reenter_kernel)(void); 8482869ac5SJames Morse 8582869ac5SJames Morse /* 8682869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to 8782869ac5SJames Morse * re-configure el2. 8882869ac5SJames Morse */ 8982869ac5SJames Morse phys_addr_t __hyp_stub_vectors; 908ec058fdSJames Morse 918ec058fdSJames Morse u64 sleep_cpu_mpidr; 9282869ac5SJames Morse } resume_hdr; 9382869ac5SJames Morse 9482869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) 9582869ac5SJames Morse { 9682869ac5SJames Morse memset(i, 0, sizeof(*i)); 9782869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); 9882869ac5SJames Morse } 9982869ac5SJames Morse 10082869ac5SJames Morse int pfn_is_nosave(unsigned long pfn) 10182869ac5SJames Morse { 1022077be67SLaura Abbott unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); 1032077be67SLaura Abbott unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); 10482869ac5SJames Morse 105254a41c0SAKASHI Takahiro return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || 106254a41c0SAKASHI Takahiro crash_is_nosave(pfn); 10782869ac5SJames Morse } 10882869ac5SJames Morse 10982869ac5SJames Morse void notrace save_processor_state(void) 11082869ac5SJames Morse { 11182869ac5SJames Morse WARN_ON(num_online_cpus() != 1); 11282869ac5SJames Morse } 11382869ac5SJames Morse 11482869ac5SJames Morse void notrace restore_processor_state(void) 11582869ac5SJames Morse { 11682869ac5SJames Morse } 11782869ac5SJames Morse 11882869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size) 11982869ac5SJames Morse { 12082869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 12182869ac5SJames Morse 12282869ac5SJames Morse if (max_size < sizeof(*hdr)) 12382869ac5SJames Morse return -EOVERFLOW; 12482869ac5SJames Morse 12582869ac5SJames Morse arch_hdr_invariants(&hdr->invariants); 1262077be67SLaura Abbott hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir); 12782869ac5SJames Morse hdr->reenter_kernel = _cpu_resume; 12882869ac5SJames Morse 12982869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */ 13082869ac5SJames Morse if (el2_reset_needed()) 1312077be67SLaura Abbott hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors); 13282869ac5SJames Morse else 13382869ac5SJames Morse hdr->__hyp_stub_vectors = 0; 13482869ac5SJames Morse 1358ec058fdSJames Morse /* Save the mpidr of the cpu we called cpu_suspend() on... */ 1368ec058fdSJames Morse if (sleep_cpu < 0) { 1379165dabbSMasanari Iida pr_err("Failing to hibernate on an unknown CPU.\n"); 1388ec058fdSJames Morse return -ENODEV; 1398ec058fdSJames Morse } 1408ec058fdSJames Morse hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); 1418ec058fdSJames Morse pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1428ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1438ec058fdSJames Morse 14482869ac5SJames Morse return 0; 14582869ac5SJames Morse } 14682869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save); 14782869ac5SJames Morse 14882869ac5SJames Morse int arch_hibernation_header_restore(void *addr) 14982869ac5SJames Morse { 1508ec058fdSJames Morse int ret; 15182869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 15282869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 15382869ac5SJames Morse 15482869ac5SJames Morse arch_hdr_invariants(&invariants); 15582869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) { 15682869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n"); 15782869ac5SJames Morse return -EINVAL; 15882869ac5SJames Morse } 15982869ac5SJames Morse 1608ec058fdSJames Morse sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); 1618ec058fdSJames Morse pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1628ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1638ec058fdSJames Morse if (sleep_cpu < 0) { 1648ec058fdSJames Morse pr_crit("Hibernated on a CPU not known to this kernel!\n"); 1658ec058fdSJames Morse sleep_cpu = -EINVAL; 1668ec058fdSJames Morse return -EINVAL; 1678ec058fdSJames Morse } 168e646ac5bSQais Yousef 169e646ac5bSQais Yousef ret = bringup_hibernate_cpu(sleep_cpu); 1708ec058fdSJames Morse if (ret) { 1718ec058fdSJames Morse sleep_cpu = -EINVAL; 1728ec058fdSJames Morse return ret; 1738ec058fdSJames Morse } 1748ec058fdSJames Morse 17582869ac5SJames Morse resume_hdr = *hdr; 17682869ac5SJames Morse 17782869ac5SJames Morse return 0; 17882869ac5SJames Morse } 17982869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore); 18082869ac5SJames Morse 181a2c2e679SPavel Tatashin static int trans_pgd_map_page(pgd_t *trans_pgd, void *page, 18282869ac5SJames Morse unsigned long dst_addr, 183a2c2e679SPavel Tatashin pgprot_t pgprot) 18482869ac5SJames Morse { 18520a004e7SWill Deacon pgd_t *pgdp; 186e9f63768SMike Rapoport p4d_t *p4dp; 18720a004e7SWill Deacon pud_t *pudp; 18820a004e7SWill Deacon pmd_t *pmdp; 18920a004e7SWill Deacon pte_t *ptep; 19082869ac5SJames Morse 191974b9b2cSMike Rapoport pgdp = pgd_offset_pgd(trans_pgd, dst_addr); 19220a004e7SWill Deacon if (pgd_none(READ_ONCE(*pgdp))) { 193051a7a94SPavel Tatashin pudp = (void *)get_safe_page(GFP_ATOMIC); 194a89d7ff9SPavel Tatashin if (!pudp) 195a89d7ff9SPavel Tatashin return -ENOMEM; 19620a004e7SWill Deacon pgd_populate(&init_mm, pgdp, pudp); 19782869ac5SJames Morse } 19882869ac5SJames Morse 199e9f63768SMike Rapoport p4dp = p4d_offset(pgdp, dst_addr); 200e9f63768SMike Rapoport if (p4d_none(READ_ONCE(*p4dp))) { 201e9f63768SMike Rapoport pudp = (void *)get_safe_page(GFP_ATOMIC); 202e9f63768SMike Rapoport if (!pudp) 203e9f63768SMike Rapoport return -ENOMEM; 204e9f63768SMike Rapoport p4d_populate(&init_mm, p4dp, pudp); 205e9f63768SMike Rapoport } 206e9f63768SMike Rapoport 207e9f63768SMike Rapoport pudp = pud_offset(p4dp, dst_addr); 20820a004e7SWill Deacon if (pud_none(READ_ONCE(*pudp))) { 209051a7a94SPavel Tatashin pmdp = (void *)get_safe_page(GFP_ATOMIC); 210a89d7ff9SPavel Tatashin if (!pmdp) 211a89d7ff9SPavel Tatashin return -ENOMEM; 21220a004e7SWill Deacon pud_populate(&init_mm, pudp, pmdp); 21382869ac5SJames Morse } 21482869ac5SJames Morse 21520a004e7SWill Deacon pmdp = pmd_offset(pudp, dst_addr); 21620a004e7SWill Deacon if (pmd_none(READ_ONCE(*pmdp))) { 217051a7a94SPavel Tatashin ptep = (void *)get_safe_page(GFP_ATOMIC); 218a89d7ff9SPavel Tatashin if (!ptep) 219a89d7ff9SPavel Tatashin return -ENOMEM; 22020a004e7SWill Deacon pmd_populate_kernel(&init_mm, pmdp, ptep); 22182869ac5SJames Morse } 22282869ac5SJames Morse 22320a004e7SWill Deacon ptep = pte_offset_kernel(pmdp, dst_addr); 22413373f0eSPavel Tatashin set_pte(ptep, pfn_pte(virt_to_pfn(page), PAGE_KERNEL_EXEC)); 22582869ac5SJames Morse 226a2c2e679SPavel Tatashin return 0; 227a2c2e679SPavel Tatashin } 228a2c2e679SPavel Tatashin 229a2c2e679SPavel Tatashin /* 230a2c2e679SPavel Tatashin * Copies length bytes, starting at src_start into an new page, 231a2c2e679SPavel Tatashin * perform cache maintenance, then maps it at the specified address low 232a2c2e679SPavel Tatashin * address as executable. 233a2c2e679SPavel Tatashin * 234a2c2e679SPavel Tatashin * This is used by hibernate to copy the code it needs to execute when 235a2c2e679SPavel Tatashin * overwriting the kernel text. This function generates a new set of page 236a2c2e679SPavel Tatashin * tables, which it loads into ttbr0. 237a2c2e679SPavel Tatashin * 238a2c2e679SPavel Tatashin * Length is provided as we probably only want 4K of data, even on a 64K 239a2c2e679SPavel Tatashin * page system. 240a2c2e679SPavel Tatashin */ 241a2c2e679SPavel Tatashin static int create_safe_exec_page(void *src_start, size_t length, 242a2c2e679SPavel Tatashin unsigned long dst_addr, 243a2c2e679SPavel Tatashin phys_addr_t *phys_dst_addr) 244a2c2e679SPavel Tatashin { 245a2c2e679SPavel Tatashin void *page = (void *)get_safe_page(GFP_ATOMIC); 246a2c2e679SPavel Tatashin pgd_t *trans_pgd; 247a2c2e679SPavel Tatashin int rc; 248a2c2e679SPavel Tatashin 249a2c2e679SPavel Tatashin if (!page) 250a2c2e679SPavel Tatashin return -ENOMEM; 251a2c2e679SPavel Tatashin 252a2c2e679SPavel Tatashin memcpy(page, src_start, length); 253a2c2e679SPavel Tatashin __flush_icache_range((unsigned long)page, (unsigned long)page + length); 254a2c2e679SPavel Tatashin 255a2c2e679SPavel Tatashin trans_pgd = (void *)get_safe_page(GFP_ATOMIC); 256a2c2e679SPavel Tatashin if (!trans_pgd) 257a2c2e679SPavel Tatashin return -ENOMEM; 258a2c2e679SPavel Tatashin 259a2c2e679SPavel Tatashin rc = trans_pgd_map_page(trans_pgd, page, dst_addr, 260a2c2e679SPavel Tatashin PAGE_KERNEL_EXEC); 261a2c2e679SPavel Tatashin if (rc) 262a2c2e679SPavel Tatashin return rc; 263a2c2e679SPavel Tatashin 2640194e760SMark Rutland /* 2650194e760SMark Rutland * Load our new page tables. A strict BBM approach requires that we 2660194e760SMark Rutland * ensure that TLBs are free of any entries that may overlap with the 2670194e760SMark Rutland * global mappings we are about to install. 2680194e760SMark Rutland * 2690194e760SMark Rutland * For a real hibernate/resume cycle TTBR0 currently points to a zero 2700194e760SMark Rutland * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI 2710194e760SMark Rutland * runtime services), while for a userspace-driven test_resume cycle it 2720194e760SMark Rutland * points to userspace page tables (and we must point it at a zero page 2730194e760SMark Rutland * ourselves). Elsewhere we only (un)install the idmap with preemption 2740194e760SMark Rutland * disabled, so T0SZ should be as required regardless. 2750194e760SMark Rutland */ 2760194e760SMark Rutland cpu_set_reserved_ttbr0(); 2770194e760SMark Rutland local_flush_tlb_all(); 278d234332cSPavel Tatashin write_sysreg(phys_to_ttbr(virt_to_phys(trans_pgd)), ttbr0_el1); 2790194e760SMark Rutland isb(); 28082869ac5SJames Morse 28113373f0eSPavel Tatashin *phys_dst_addr = virt_to_phys(page); 28282869ac5SJames Morse 283a89d7ff9SPavel Tatashin return 0; 28482869ac5SJames Morse } 28582869ac5SJames Morse 2865ebe3a44SJames Morse #define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) 28782869ac5SJames Morse 28882869ac5SJames Morse int swsusp_arch_suspend(void) 28982869ac5SJames Morse { 29082869ac5SJames Morse int ret = 0; 29182869ac5SJames Morse unsigned long flags; 29282869ac5SJames Morse struct sleep_stack_data state; 29382869ac5SJames Morse 294d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) { 295d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); 296d74b4e4fSJames Morse return -EBUSY; 297d74b4e4fSJames Morse } 298d74b4e4fSJames Morse 2990fbeb318SJames Morse flags = local_daif_save(); 30082869ac5SJames Morse 30182869ac5SJames Morse if (__cpu_suspend_enter(&state)) { 302254a41c0SAKASHI Takahiro /* make the crash dump kernel image visible/saveable */ 303254a41c0SAKASHI Takahiro crash_prepare_suspend(); 304254a41c0SAKASHI Takahiro 3058ec058fdSJames Morse sleep_cpu = smp_processor_id(); 30682869ac5SJames Morse ret = swsusp_save(); 30782869ac5SJames Morse } else { 3085ebe3a44SJames Morse /* Clean kernel core startup/idle code to PoC*/ 3095ebe3a44SJames Morse dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); 3105ebe3a44SJames Morse dcache_clean_range(__idmap_text_start, __idmap_text_end); 3115ebe3a44SJames Morse 3125ebe3a44SJames Morse /* Clean kvm setup code to PoC? */ 313f7daa9c8SJames Morse if (el2_reset_needed()) { 3145ebe3a44SJames Morse dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); 315f7daa9c8SJames Morse dcache_clean_range(__hyp_text_start, __hyp_text_end); 316f7daa9c8SJames Morse } 31782869ac5SJames Morse 318254a41c0SAKASHI Takahiro /* make the crash dump kernel image protected again */ 319254a41c0SAKASHI Takahiro crash_post_resume(); 320254a41c0SAKASHI Takahiro 32182869ac5SJames Morse /* 32282869ac5SJames Morse * Tell the hibernation core that we've just restored 32382869ac5SJames Morse * the memory 32482869ac5SJames Morse */ 32582869ac5SJames Morse in_suspend = 0; 32682869ac5SJames Morse 3278ec058fdSJames Morse sleep_cpu = -EINVAL; 32882869ac5SJames Morse __cpu_suspend_exit(); 329647d0519SMarc Zyngier 330647d0519SMarc Zyngier /* 331647d0519SMarc Zyngier * Just in case the boot kernel did turn the SSBD 332647d0519SMarc Zyngier * mitigation off behind our back, let's set the state 333647d0519SMarc Zyngier * to what we expect it to be. 334647d0519SMarc Zyngier */ 335647d0519SMarc Zyngier switch (arm64_get_ssbd_state()) { 336647d0519SMarc Zyngier case ARM64_SSBD_FORCE_ENABLE: 337647d0519SMarc Zyngier case ARM64_SSBD_KERNEL: 338647d0519SMarc Zyngier arm64_set_ssbd_mitigation(true); 339647d0519SMarc Zyngier } 34082869ac5SJames Morse } 34182869ac5SJames Morse 3420fbeb318SJames Morse local_daif_restore(flags); 34382869ac5SJames Morse 34482869ac5SJames Morse return ret; 34582869ac5SJames Morse } 34682869ac5SJames Morse 34720a004e7SWill Deacon static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 3485ebe3a44SJames Morse { 34920a004e7SWill Deacon pte_t pte = READ_ONCE(*src_ptep); 3505ebe3a44SJames Morse 3515ebe3a44SJames Morse if (pte_valid(pte)) { 3525ebe3a44SJames Morse /* 3535ebe3a44SJames Morse * Resume will overwrite areas that may be marked 3545ebe3a44SJames Morse * read only (code, rodata). Clear the RDONLY bit from 3555ebe3a44SJames Morse * the temporary mappings we use during restore. 3565ebe3a44SJames Morse */ 35720a004e7SWill Deacon set_pte(dst_ptep, pte_mkwrite(pte)); 3585ebe3a44SJames Morse } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 3595ebe3a44SJames Morse /* 3605ebe3a44SJames Morse * debug_pagealloc will removed the PTE_VALID bit if 3615ebe3a44SJames Morse * the page isn't in use by the resume kernel. It may have 3625ebe3a44SJames Morse * been in use by the original kernel, in which case we need 3635ebe3a44SJames Morse * to put it back in our copy to do the restore. 3645ebe3a44SJames Morse * 3655ebe3a44SJames Morse * Before marking this entry valid, check the pfn should 3665ebe3a44SJames Morse * be mapped. 3675ebe3a44SJames Morse */ 3685ebe3a44SJames Morse BUG_ON(!pfn_valid(pte_pfn(pte))); 3695ebe3a44SJames Morse 37020a004e7SWill Deacon set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 3715ebe3a44SJames Morse } 3725ebe3a44SJames Morse } 3735ebe3a44SJames Morse 37420a004e7SWill Deacon static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, 37582869ac5SJames Morse unsigned long end) 37682869ac5SJames Morse { 37720a004e7SWill Deacon pte_t *src_ptep; 37820a004e7SWill Deacon pte_t *dst_ptep; 37982869ac5SJames Morse unsigned long addr = start; 38082869ac5SJames Morse 38120a004e7SWill Deacon dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); 38220a004e7SWill Deacon if (!dst_ptep) 38382869ac5SJames Morse return -ENOMEM; 38420a004e7SWill Deacon pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); 38520a004e7SWill Deacon dst_ptep = pte_offset_kernel(dst_pmdp, start); 38682869ac5SJames Morse 38720a004e7SWill Deacon src_ptep = pte_offset_kernel(src_pmdp, start); 38882869ac5SJames Morse do { 38920a004e7SWill Deacon _copy_pte(dst_ptep, src_ptep, addr); 39020a004e7SWill Deacon } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 39182869ac5SJames Morse 39282869ac5SJames Morse return 0; 39382869ac5SJames Morse } 39482869ac5SJames Morse 39520a004e7SWill Deacon static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start, 39682869ac5SJames Morse unsigned long end) 39782869ac5SJames Morse { 39820a004e7SWill Deacon pmd_t *src_pmdp; 39920a004e7SWill Deacon pmd_t *dst_pmdp; 40082869ac5SJames Morse unsigned long next; 40182869ac5SJames Morse unsigned long addr = start; 40282869ac5SJames Morse 40320a004e7SWill Deacon if (pud_none(READ_ONCE(*dst_pudp))) { 40420a004e7SWill Deacon dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); 40520a004e7SWill Deacon if (!dst_pmdp) 40682869ac5SJames Morse return -ENOMEM; 40720a004e7SWill Deacon pud_populate(&init_mm, dst_pudp, dst_pmdp); 40882869ac5SJames Morse } 40920a004e7SWill Deacon dst_pmdp = pmd_offset(dst_pudp, start); 41082869ac5SJames Morse 41120a004e7SWill Deacon src_pmdp = pmd_offset(src_pudp, start); 41282869ac5SJames Morse do { 41320a004e7SWill Deacon pmd_t pmd = READ_ONCE(*src_pmdp); 41420a004e7SWill Deacon 41582869ac5SJames Morse next = pmd_addr_end(addr, end); 41620a004e7SWill Deacon if (pmd_none(pmd)) 41782869ac5SJames Morse continue; 41820a004e7SWill Deacon if (pmd_table(pmd)) { 41920a004e7SWill Deacon if (copy_pte(dst_pmdp, src_pmdp, addr, next)) 42082869ac5SJames Morse return -ENOMEM; 42182869ac5SJames Morse } else { 42220a004e7SWill Deacon set_pmd(dst_pmdp, 42320a004e7SWill Deacon __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 42482869ac5SJames Morse } 42520a004e7SWill Deacon } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 42682869ac5SJames Morse 42782869ac5SJames Morse return 0; 42882869ac5SJames Morse } 42982869ac5SJames Morse 430e9f63768SMike Rapoport static int copy_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start, 43182869ac5SJames Morse unsigned long end) 43282869ac5SJames Morse { 43320a004e7SWill Deacon pud_t *dst_pudp; 43420a004e7SWill Deacon pud_t *src_pudp; 43582869ac5SJames Morse unsigned long next; 43682869ac5SJames Morse unsigned long addr = start; 43782869ac5SJames Morse 438e9f63768SMike Rapoport if (p4d_none(READ_ONCE(*dst_p4dp))) { 43920a004e7SWill Deacon dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); 44020a004e7SWill Deacon if (!dst_pudp) 44182869ac5SJames Morse return -ENOMEM; 442e9f63768SMike Rapoport p4d_populate(&init_mm, dst_p4dp, dst_pudp); 44382869ac5SJames Morse } 444e9f63768SMike Rapoport dst_pudp = pud_offset(dst_p4dp, start); 44582869ac5SJames Morse 446e9f63768SMike Rapoport src_pudp = pud_offset(src_p4dp, start); 44782869ac5SJames Morse do { 44820a004e7SWill Deacon pud_t pud = READ_ONCE(*src_pudp); 44920a004e7SWill Deacon 45082869ac5SJames Morse next = pud_addr_end(addr, end); 45120a004e7SWill Deacon if (pud_none(pud)) 45282869ac5SJames Morse continue; 45320a004e7SWill Deacon if (pud_table(pud)) { 45420a004e7SWill Deacon if (copy_pmd(dst_pudp, src_pudp, addr, next)) 45582869ac5SJames Morse return -ENOMEM; 45682869ac5SJames Morse } else { 45720a004e7SWill Deacon set_pud(dst_pudp, 4587ea40889SPavel Tatashin __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); 45982869ac5SJames Morse } 46020a004e7SWill Deacon } while (dst_pudp++, src_pudp++, addr = next, addr != end); 46182869ac5SJames Morse 46282869ac5SJames Morse return 0; 46382869ac5SJames Morse } 46482869ac5SJames Morse 465e9f63768SMike Rapoport static int copy_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start, 466e9f63768SMike Rapoport unsigned long end) 467e9f63768SMike Rapoport { 468e9f63768SMike Rapoport p4d_t *dst_p4dp; 469e9f63768SMike Rapoport p4d_t *src_p4dp; 470e9f63768SMike Rapoport unsigned long next; 471e9f63768SMike Rapoport unsigned long addr = start; 472e9f63768SMike Rapoport 473e9f63768SMike Rapoport dst_p4dp = p4d_offset(dst_pgdp, start); 474e9f63768SMike Rapoport src_p4dp = p4d_offset(src_pgdp, start); 475e9f63768SMike Rapoport do { 476e9f63768SMike Rapoport next = p4d_addr_end(addr, end); 477e9f63768SMike Rapoport if (p4d_none(READ_ONCE(*src_p4dp))) 478e9f63768SMike Rapoport continue; 479e9f63768SMike Rapoport if (copy_pud(dst_p4dp, src_p4dp, addr, next)) 480e9f63768SMike Rapoport return -ENOMEM; 481e9f63768SMike Rapoport } while (dst_p4dp++, src_p4dp++, addr = next, addr != end); 482e9f63768SMike Rapoport 483e9f63768SMike Rapoport return 0; 484e9f63768SMike Rapoport } 485e9f63768SMike Rapoport 48620a004e7SWill Deacon static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, 48782869ac5SJames Morse unsigned long end) 48882869ac5SJames Morse { 48982869ac5SJames Morse unsigned long next; 49082869ac5SJames Morse unsigned long addr = start; 49120a004e7SWill Deacon pgd_t *src_pgdp = pgd_offset_k(start); 49282869ac5SJames Morse 493974b9b2cSMike Rapoport dst_pgdp = pgd_offset_pgd(dst_pgdp, start); 49482869ac5SJames Morse do { 49582869ac5SJames Morse next = pgd_addr_end(addr, end); 49620a004e7SWill Deacon if (pgd_none(READ_ONCE(*src_pgdp))) 49782869ac5SJames Morse continue; 498e9f63768SMike Rapoport if (copy_p4d(dst_pgdp, src_pgdp, addr, next)) 49982869ac5SJames Morse return -ENOMEM; 50020a004e7SWill Deacon } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 50182869ac5SJames Morse 50282869ac5SJames Morse return 0; 50382869ac5SJames Morse } 50482869ac5SJames Morse 505a2c2e679SPavel Tatashin static int trans_pgd_create_copy(pgd_t **dst_pgdp, unsigned long start, 506a2c2e679SPavel Tatashin unsigned long end) 507a2c2e679SPavel Tatashin { 508a2c2e679SPavel Tatashin int rc; 509a2c2e679SPavel Tatashin pgd_t *trans_pgd = (pgd_t *)get_safe_page(GFP_ATOMIC); 510a2c2e679SPavel Tatashin 511a2c2e679SPavel Tatashin if (!trans_pgd) { 512a2c2e679SPavel Tatashin pr_err("Failed to allocate memory for temporary page tables.\n"); 513a2c2e679SPavel Tatashin return -ENOMEM; 514a2c2e679SPavel Tatashin } 515a2c2e679SPavel Tatashin 516a2c2e679SPavel Tatashin rc = copy_page_tables(trans_pgd, start, end); 517a2c2e679SPavel Tatashin if (!rc) 518a2c2e679SPavel Tatashin *dst_pgdp = trans_pgd; 519a2c2e679SPavel Tatashin 520a2c2e679SPavel Tatashin return rc; 521a2c2e679SPavel Tatashin } 522a2c2e679SPavel Tatashin 52382869ac5SJames Morse /* 52482869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit(). 52582869ac5SJames Morse * 52682869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code, 52782869ac5SJames Morse * we don't need to free it here. 52882869ac5SJames Morse */ 52982869ac5SJames Morse int swsusp_arch_resume(void) 53082869ac5SJames Morse { 531a89d7ff9SPavel Tatashin int rc; 53282869ac5SJames Morse void *zero_page; 53382869ac5SJames Morse size_t exit_size; 53482869ac5SJames Morse pgd_t *tmp_pg_dir; 53582869ac5SJames Morse phys_addr_t phys_hibernate_exit; 53682869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, 53782869ac5SJames Morse void *, phys_addr_t, phys_addr_t); 53882869ac5SJames Morse 53982869ac5SJames Morse /* 540dfbca61aSMark Rutland * Restoring the memory image will overwrite the ttbr1 page tables. 541dfbca61aSMark Rutland * Create a second copy of just the linear map, and use this when 542dfbca61aSMark Rutland * restoring. 543dfbca61aSMark Rutland */ 544a2c2e679SPavel Tatashin rc = trans_pgd_create_copy(&tmp_pg_dir, PAGE_OFFSET, PAGE_END); 545dfbca61aSMark Rutland if (rc) 546a89d7ff9SPavel Tatashin return rc; 547dfbca61aSMark Rutland 548dfbca61aSMark Rutland /* 549dfbca61aSMark Rutland * We need a zero page that is zero before & after resume in order to 550dfbca61aSMark Rutland * to break before make on the ttbr1 page tables. 551dfbca61aSMark Rutland */ 552dfbca61aSMark Rutland zero_page = (void *)get_safe_page(GFP_ATOMIC); 553dfbca61aSMark Rutland if (!zero_page) { 554117f5727SMark Rutland pr_err("Failed to allocate zero page.\n"); 555a89d7ff9SPavel Tatashin return -ENOMEM; 556dfbca61aSMark Rutland } 557dfbca61aSMark Rutland 558dfbca61aSMark Rutland /* 55982869ac5SJames Morse * Locate the exit code in the bottom-but-one page, so that *NULL 56082869ac5SJames Morse * still has disastrous affects. 56182869ac5SJames Morse */ 56282869ac5SJames Morse hibernate_exit = (void *)PAGE_SIZE; 56382869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; 56482869ac5SJames Morse /* 56582869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate 56682869ac5SJames Morse * a new set of ttbr0 page tables and load them. 56782869ac5SJames Morse */ 56882869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size, 56982869ac5SJames Morse (unsigned long)hibernate_exit, 570051a7a94SPavel Tatashin &phys_hibernate_exit); 57182869ac5SJames Morse if (rc) { 572117f5727SMark Rutland pr_err("Failed to create safe executable page for hibernate_exit code.\n"); 573a89d7ff9SPavel Tatashin return rc; 57482869ac5SJames Morse } 57582869ac5SJames Morse 57682869ac5SJames Morse /* 57782869ac5SJames Morse * The hibernate exit text contains a set of el2 vectors, that will 57882869ac5SJames Morse * be executed at el2 with the mmu off in order to reload hyp-stub. 57982869ac5SJames Morse */ 58082869ac5SJames Morse __flush_dcache_area(hibernate_exit, exit_size); 58182869ac5SJames Morse 58282869ac5SJames Morse /* 58382869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in 58482869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2. 58582869ac5SJames Morse * 58682869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE. 58782869ac5SJames Morse */ 58882869ac5SJames Morse if (el2_reset_needed()) { 58982869ac5SJames Morse phys_addr_t el2_vectors = phys_hibernate_exit; /* base */ 59082869ac5SJames Morse el2_vectors += hibernate_el2_vectors - 59182869ac5SJames Morse __hibernate_exit_text_start; /* offset */ 59282869ac5SJames Morse 59382869ac5SJames Morse __hyp_set_vectors(el2_vectors); 59482869ac5SJames Morse } 59582869ac5SJames Morse 59682869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, 5972077be67SLaura Abbott resume_hdr.reenter_kernel, restore_pblist, 59882869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); 59982869ac5SJames Morse 600a89d7ff9SPavel Tatashin return 0; 60182869ac5SJames Morse } 6021fe492ceSJames Morse 6038ec058fdSJames Morse int hibernate_resume_nonboot_cpu_disable(void) 6048ec058fdSJames Morse { 6058ec058fdSJames Morse if (sleep_cpu < 0) { 6069165dabbSMasanari Iida pr_err("Failing to resume from hibernate on an unknown CPU.\n"); 6078ec058fdSJames Morse return -ENODEV; 6088ec058fdSJames Morse } 6098ec058fdSJames Morse 6108ec058fdSJames Morse return freeze_secondary_cpus(sleep_cpu); 6118ec058fdSJames Morse } 612