1af873fceSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 282869ac5SJames Morse /*: 382869ac5SJames Morse * Hibernate support specific for ARM64 482869ac5SJames Morse * 582869ac5SJames Morse * Derived from work on ARM hibernation support by: 682869ac5SJames Morse * 782869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove 882869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) 982869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) 1082869ac5SJames Morse * https://lkml.org/lkml/2010/6/18/4 1182869ac5SJames Morse * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html 1282869ac5SJames Morse * https://patchwork.kernel.org/patch/96442/ 1382869ac5SJames Morse * 1482869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 1582869ac5SJames Morse */ 1682869ac5SJames Morse #define pr_fmt(x) "hibernate: " x 178ec058fdSJames Morse #include <linux/cpu.h> 1882869ac5SJames Morse #include <linux/kvm_host.h> 1982869ac5SJames Morse #include <linux/mm.h> 2082869ac5SJames Morse #include <linux/pm.h> 2182869ac5SJames Morse #include <linux/sched.h> 2282869ac5SJames Morse #include <linux/suspend.h> 2382869ac5SJames Morse #include <linux/utsname.h> 2482869ac5SJames Morse 2582869ac5SJames Morse #include <asm/barrier.h> 2682869ac5SJames Morse #include <asm/cacheflush.h> 278ec058fdSJames Morse #include <asm/cputype.h> 280fbeb318SJames Morse #include <asm/daifflags.h> 2982869ac5SJames Morse #include <asm/irqflags.h> 30254a41c0SAKASHI Takahiro #include <asm/kexec.h> 3182869ac5SJames Morse #include <asm/memory.h> 3282869ac5SJames Morse #include <asm/mmu_context.h> 33ee11f332SSteven Price #include <asm/mte.h> 3482869ac5SJames Morse #include <asm/pgalloc.h> 3582869ac5SJames Morse #include <asm/pgtable-hwdef.h> 3682869ac5SJames Morse #include <asm/sections.h> 37d74b4e4fSJames Morse #include <asm/smp.h> 388ec058fdSJames Morse #include <asm/smp_plat.h> 3982869ac5SJames Morse #include <asm/suspend.h> 400194e760SMark Rutland #include <asm/sysreg.h> 4182869ac5SJames Morse #include <asm/virt.h> 4282869ac5SJames Morse 4382869ac5SJames Morse /* 4482869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it 4582869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This 4682869ac5SJames Morse * doesn't happen with either KASLR. 4782869ac5SJames Morse * 4882869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in 4982869ac5SJames Morse * kernel/power/hibernate.c 5082869ac5SJames Morse */ 5182869ac5SJames Morse extern int in_suspend; 5282869ac5SJames Morse 5382869ac5SJames Morse /* Do we need to reset el2? */ 5482869ac5SJames Morse #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) 5582869ac5SJames Morse 5682869ac5SJames Morse /* temporary el2 vectors in the __hibernate_exit_text section. */ 5782869ac5SJames Morse extern char hibernate_el2_vectors[]; 5882869ac5SJames Morse 5982869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ 6082869ac5SJames Morse extern char __hyp_stub_vectors[]; 6182869ac5SJames Morse 6282869ac5SJames Morse /* 638ec058fdSJames Morse * The logical cpu number we should resume on, initialised to a non-cpu 648ec058fdSJames Morse * number. 658ec058fdSJames Morse */ 668ec058fdSJames Morse static int sleep_cpu = -EINVAL; 678ec058fdSJames Morse 688ec058fdSJames Morse /* 6982869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number 7082869ac5SJames Morse * and date in here so that we guarantee not to resume with a different 7182869ac5SJames Morse * kernel. 7282869ac5SJames Morse */ 7382869ac5SJames Morse struct arch_hibernate_hdr_invariants { 7482869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1]; 7582869ac5SJames Morse }; 7682869ac5SJames Morse 7782869ac5SJames Morse /* These values need to be know across a hibernate/restore. */ 7882869ac5SJames Morse static struct arch_hibernate_hdr { 7982869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 8082869ac5SJames Morse 8182869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */ 8282869ac5SJames Morse phys_addr_t ttbr1_el1; 8382869ac5SJames Morse void (*reenter_kernel)(void); 8482869ac5SJames Morse 8582869ac5SJames Morse /* 8682869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to 8782869ac5SJames Morse * re-configure el2. 8882869ac5SJames Morse */ 8982869ac5SJames Morse phys_addr_t __hyp_stub_vectors; 908ec058fdSJames Morse 918ec058fdSJames Morse u64 sleep_cpu_mpidr; 9282869ac5SJames Morse } resume_hdr; 9382869ac5SJames Morse 9482869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) 9582869ac5SJames Morse { 9682869ac5SJames Morse memset(i, 0, sizeof(*i)); 9782869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); 9882869ac5SJames Morse } 9982869ac5SJames Morse 10082869ac5SJames Morse int pfn_is_nosave(unsigned long pfn) 10182869ac5SJames Morse { 1022077be67SLaura Abbott unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); 1032077be67SLaura Abbott unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); 10482869ac5SJames Morse 105254a41c0SAKASHI Takahiro return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || 106254a41c0SAKASHI Takahiro crash_is_nosave(pfn); 10782869ac5SJames Morse } 10882869ac5SJames Morse 10982869ac5SJames Morse void notrace save_processor_state(void) 11082869ac5SJames Morse { 11182869ac5SJames Morse WARN_ON(num_online_cpus() != 1); 11282869ac5SJames Morse } 11382869ac5SJames Morse 11482869ac5SJames Morse void notrace restore_processor_state(void) 11582869ac5SJames Morse { 11682869ac5SJames Morse } 11782869ac5SJames Morse 11882869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size) 11982869ac5SJames Morse { 12082869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 12182869ac5SJames Morse 12282869ac5SJames Morse if (max_size < sizeof(*hdr)) 12382869ac5SJames Morse return -EOVERFLOW; 12482869ac5SJames Morse 12582869ac5SJames Morse arch_hdr_invariants(&hdr->invariants); 1262077be67SLaura Abbott hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir); 12782869ac5SJames Morse hdr->reenter_kernel = _cpu_resume; 12882869ac5SJames Morse 12982869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */ 13082869ac5SJames Morse if (el2_reset_needed()) 1312077be67SLaura Abbott hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors); 13282869ac5SJames Morse else 13382869ac5SJames Morse hdr->__hyp_stub_vectors = 0; 13482869ac5SJames Morse 1358ec058fdSJames Morse /* Save the mpidr of the cpu we called cpu_suspend() on... */ 1368ec058fdSJames Morse if (sleep_cpu < 0) { 1379165dabbSMasanari Iida pr_err("Failing to hibernate on an unknown CPU.\n"); 1388ec058fdSJames Morse return -ENODEV; 1398ec058fdSJames Morse } 1408ec058fdSJames Morse hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); 1418ec058fdSJames Morse pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1428ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1438ec058fdSJames Morse 14482869ac5SJames Morse return 0; 14582869ac5SJames Morse } 14682869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save); 14782869ac5SJames Morse 14882869ac5SJames Morse int arch_hibernation_header_restore(void *addr) 14982869ac5SJames Morse { 1508ec058fdSJames Morse int ret; 15182869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants; 15282869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr; 15382869ac5SJames Morse 15482869ac5SJames Morse arch_hdr_invariants(&invariants); 15582869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) { 15682869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n"); 15782869ac5SJames Morse return -EINVAL; 15882869ac5SJames Morse } 15982869ac5SJames Morse 1608ec058fdSJames Morse sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); 1618ec058fdSJames Morse pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, 1628ec058fdSJames Morse hdr->sleep_cpu_mpidr); 1638ec058fdSJames Morse if (sleep_cpu < 0) { 1648ec058fdSJames Morse pr_crit("Hibernated on a CPU not known to this kernel!\n"); 1658ec058fdSJames Morse sleep_cpu = -EINVAL; 1668ec058fdSJames Morse return -EINVAL; 1678ec058fdSJames Morse } 168e646ac5bSQais Yousef 169e646ac5bSQais Yousef ret = bringup_hibernate_cpu(sleep_cpu); 1708ec058fdSJames Morse if (ret) { 1718ec058fdSJames Morse sleep_cpu = -EINVAL; 1728ec058fdSJames Morse return ret; 1738ec058fdSJames Morse } 1748ec058fdSJames Morse 17582869ac5SJames Morse resume_hdr = *hdr; 17682869ac5SJames Morse 17782869ac5SJames Morse return 0; 17882869ac5SJames Morse } 17982869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore); 18082869ac5SJames Morse 181a2c2e679SPavel Tatashin static int trans_pgd_map_page(pgd_t *trans_pgd, void *page, 18282869ac5SJames Morse unsigned long dst_addr, 183a2c2e679SPavel Tatashin pgprot_t pgprot) 18482869ac5SJames Morse { 18520a004e7SWill Deacon pgd_t *pgdp; 186e9f63768SMike Rapoport p4d_t *p4dp; 18720a004e7SWill Deacon pud_t *pudp; 18820a004e7SWill Deacon pmd_t *pmdp; 18920a004e7SWill Deacon pte_t *ptep; 19082869ac5SJames Morse 191974b9b2cSMike Rapoport pgdp = pgd_offset_pgd(trans_pgd, dst_addr); 19220a004e7SWill Deacon if (pgd_none(READ_ONCE(*pgdp))) { 193051a7a94SPavel Tatashin pudp = (void *)get_safe_page(GFP_ATOMIC); 194a89d7ff9SPavel Tatashin if (!pudp) 195a89d7ff9SPavel Tatashin return -ENOMEM; 19620a004e7SWill Deacon pgd_populate(&init_mm, pgdp, pudp); 19782869ac5SJames Morse } 19882869ac5SJames Morse 199e9f63768SMike Rapoport p4dp = p4d_offset(pgdp, dst_addr); 200e9f63768SMike Rapoport if (p4d_none(READ_ONCE(*p4dp))) { 201e9f63768SMike Rapoport pudp = (void *)get_safe_page(GFP_ATOMIC); 202e9f63768SMike Rapoport if (!pudp) 203e9f63768SMike Rapoport return -ENOMEM; 204e9f63768SMike Rapoport p4d_populate(&init_mm, p4dp, pudp); 205e9f63768SMike Rapoport } 206e9f63768SMike Rapoport 207e9f63768SMike Rapoport pudp = pud_offset(p4dp, dst_addr); 20820a004e7SWill Deacon if (pud_none(READ_ONCE(*pudp))) { 209051a7a94SPavel Tatashin pmdp = (void *)get_safe_page(GFP_ATOMIC); 210a89d7ff9SPavel Tatashin if (!pmdp) 211a89d7ff9SPavel Tatashin return -ENOMEM; 21220a004e7SWill Deacon pud_populate(&init_mm, pudp, pmdp); 21382869ac5SJames Morse } 21482869ac5SJames Morse 21520a004e7SWill Deacon pmdp = pmd_offset(pudp, dst_addr); 21620a004e7SWill Deacon if (pmd_none(READ_ONCE(*pmdp))) { 217051a7a94SPavel Tatashin ptep = (void *)get_safe_page(GFP_ATOMIC); 218a89d7ff9SPavel Tatashin if (!ptep) 219a89d7ff9SPavel Tatashin return -ENOMEM; 22020a004e7SWill Deacon pmd_populate_kernel(&init_mm, pmdp, ptep); 22182869ac5SJames Morse } 22282869ac5SJames Morse 22320a004e7SWill Deacon ptep = pte_offset_kernel(pmdp, dst_addr); 22413373f0eSPavel Tatashin set_pte(ptep, pfn_pte(virt_to_pfn(page), PAGE_KERNEL_EXEC)); 22582869ac5SJames Morse 226a2c2e679SPavel Tatashin return 0; 227a2c2e679SPavel Tatashin } 228a2c2e679SPavel Tatashin 229a2c2e679SPavel Tatashin /* 230a2c2e679SPavel Tatashin * Copies length bytes, starting at src_start into an new page, 231a2c2e679SPavel Tatashin * perform cache maintenance, then maps it at the specified address low 232a2c2e679SPavel Tatashin * address as executable. 233a2c2e679SPavel Tatashin * 234a2c2e679SPavel Tatashin * This is used by hibernate to copy the code it needs to execute when 235a2c2e679SPavel Tatashin * overwriting the kernel text. This function generates a new set of page 236a2c2e679SPavel Tatashin * tables, which it loads into ttbr0. 237a2c2e679SPavel Tatashin * 238a2c2e679SPavel Tatashin * Length is provided as we probably only want 4K of data, even on a 64K 239a2c2e679SPavel Tatashin * page system. 240a2c2e679SPavel Tatashin */ 241a2c2e679SPavel Tatashin static int create_safe_exec_page(void *src_start, size_t length, 242a2c2e679SPavel Tatashin unsigned long dst_addr, 243a2c2e679SPavel Tatashin phys_addr_t *phys_dst_addr) 244a2c2e679SPavel Tatashin { 245a2c2e679SPavel Tatashin void *page = (void *)get_safe_page(GFP_ATOMIC); 246a2c2e679SPavel Tatashin pgd_t *trans_pgd; 247a2c2e679SPavel Tatashin int rc; 248a2c2e679SPavel Tatashin 249a2c2e679SPavel Tatashin if (!page) 250a2c2e679SPavel Tatashin return -ENOMEM; 251a2c2e679SPavel Tatashin 252a2c2e679SPavel Tatashin memcpy(page, src_start, length); 253a2c2e679SPavel Tatashin __flush_icache_range((unsigned long)page, (unsigned long)page + length); 254a2c2e679SPavel Tatashin 255a2c2e679SPavel Tatashin trans_pgd = (void *)get_safe_page(GFP_ATOMIC); 256a2c2e679SPavel Tatashin if (!trans_pgd) 257a2c2e679SPavel Tatashin return -ENOMEM; 258a2c2e679SPavel Tatashin 259a2c2e679SPavel Tatashin rc = trans_pgd_map_page(trans_pgd, page, dst_addr, 260a2c2e679SPavel Tatashin PAGE_KERNEL_EXEC); 261a2c2e679SPavel Tatashin if (rc) 262a2c2e679SPavel Tatashin return rc; 263a2c2e679SPavel Tatashin 2640194e760SMark Rutland /* 2650194e760SMark Rutland * Load our new page tables. A strict BBM approach requires that we 2660194e760SMark Rutland * ensure that TLBs are free of any entries that may overlap with the 2670194e760SMark Rutland * global mappings we are about to install. 2680194e760SMark Rutland * 2690194e760SMark Rutland * For a real hibernate/resume cycle TTBR0 currently points to a zero 2700194e760SMark Rutland * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI 2710194e760SMark Rutland * runtime services), while for a userspace-driven test_resume cycle it 2720194e760SMark Rutland * points to userspace page tables (and we must point it at a zero page 2730194e760SMark Rutland * ourselves). Elsewhere we only (un)install the idmap with preemption 2740194e760SMark Rutland * disabled, so T0SZ should be as required regardless. 2750194e760SMark Rutland */ 2760194e760SMark Rutland cpu_set_reserved_ttbr0(); 2770194e760SMark Rutland local_flush_tlb_all(); 278d234332cSPavel Tatashin write_sysreg(phys_to_ttbr(virt_to_phys(trans_pgd)), ttbr0_el1); 2790194e760SMark Rutland isb(); 28082869ac5SJames Morse 28113373f0eSPavel Tatashin *phys_dst_addr = virt_to_phys(page); 28282869ac5SJames Morse 283a89d7ff9SPavel Tatashin return 0; 28482869ac5SJames Morse } 28582869ac5SJames Morse 2865ebe3a44SJames Morse #define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) 28782869ac5SJames Morse 288ee11f332SSteven Price #ifdef CONFIG_ARM64_MTE 289ee11f332SSteven Price 290ee11f332SSteven Price static DEFINE_XARRAY(mte_pages); 291ee11f332SSteven Price 292ee11f332SSteven Price static int save_tags(struct page *page, unsigned long pfn) 293ee11f332SSteven Price { 294ee11f332SSteven Price void *tag_storage, *ret; 295ee11f332SSteven Price 296ee11f332SSteven Price tag_storage = mte_allocate_tag_storage(); 297ee11f332SSteven Price if (!tag_storage) 298ee11f332SSteven Price return -ENOMEM; 299ee11f332SSteven Price 300ee11f332SSteven Price mte_save_page_tags(page_address(page), tag_storage); 301ee11f332SSteven Price 302ee11f332SSteven Price ret = xa_store(&mte_pages, pfn, tag_storage, GFP_KERNEL); 303ee11f332SSteven Price if (WARN(xa_is_err(ret), "Failed to store MTE tags")) { 304ee11f332SSteven Price mte_free_tag_storage(tag_storage); 305ee11f332SSteven Price return xa_err(ret); 306ee11f332SSteven Price } else if (WARN(ret, "swsusp: %s: Duplicate entry", __func__)) { 307ee11f332SSteven Price mte_free_tag_storage(ret); 308ee11f332SSteven Price } 309ee11f332SSteven Price 310ee11f332SSteven Price return 0; 311ee11f332SSteven Price } 312ee11f332SSteven Price 313ee11f332SSteven Price static void swsusp_mte_free_storage(void) 314ee11f332SSteven Price { 315ee11f332SSteven Price XA_STATE(xa_state, &mte_pages, 0); 316ee11f332SSteven Price void *tags; 317ee11f332SSteven Price 318ee11f332SSteven Price xa_lock(&mte_pages); 319ee11f332SSteven Price xas_for_each(&xa_state, tags, ULONG_MAX) { 320ee11f332SSteven Price mte_free_tag_storage(tags); 321ee11f332SSteven Price } 322ee11f332SSteven Price xa_unlock(&mte_pages); 323ee11f332SSteven Price 324ee11f332SSteven Price xa_destroy(&mte_pages); 325ee11f332SSteven Price } 326ee11f332SSteven Price 327ee11f332SSteven Price static int swsusp_mte_save_tags(void) 328ee11f332SSteven Price { 329ee11f332SSteven Price struct zone *zone; 330ee11f332SSteven Price unsigned long pfn, max_zone_pfn; 331ee11f332SSteven Price int ret = 0; 332ee11f332SSteven Price int n = 0; 333ee11f332SSteven Price 334ee11f332SSteven Price if (!system_supports_mte()) 335ee11f332SSteven Price return 0; 336ee11f332SSteven Price 337ee11f332SSteven Price for_each_populated_zone(zone) { 338ee11f332SSteven Price max_zone_pfn = zone_end_pfn(zone); 339ee11f332SSteven Price for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 340ee11f332SSteven Price struct page *page = pfn_to_online_page(pfn); 341ee11f332SSteven Price 342ee11f332SSteven Price if (!page) 343ee11f332SSteven Price continue; 344ee11f332SSteven Price 345ee11f332SSteven Price if (!test_bit(PG_mte_tagged, &page->flags)) 346ee11f332SSteven Price continue; 347ee11f332SSteven Price 348ee11f332SSteven Price ret = save_tags(page, pfn); 349ee11f332SSteven Price if (ret) { 350ee11f332SSteven Price swsusp_mte_free_storage(); 351ee11f332SSteven Price goto out; 352ee11f332SSteven Price } 353ee11f332SSteven Price 354ee11f332SSteven Price n++; 355ee11f332SSteven Price } 356ee11f332SSteven Price } 357ee11f332SSteven Price pr_info("Saved %d MTE pages\n", n); 358ee11f332SSteven Price 359ee11f332SSteven Price out: 360ee11f332SSteven Price return ret; 361ee11f332SSteven Price } 362ee11f332SSteven Price 363ee11f332SSteven Price static void swsusp_mte_restore_tags(void) 364ee11f332SSteven Price { 365ee11f332SSteven Price XA_STATE(xa_state, &mte_pages, 0); 366ee11f332SSteven Price int n = 0; 367ee11f332SSteven Price void *tags; 368ee11f332SSteven Price 369ee11f332SSteven Price xa_lock(&mte_pages); 370ee11f332SSteven Price xas_for_each(&xa_state, tags, ULONG_MAX) { 371ee11f332SSteven Price unsigned long pfn = xa_state.xa_index; 372ee11f332SSteven Price struct page *page = pfn_to_online_page(pfn); 373ee11f332SSteven Price 374*e5b8d921SVincenzo Frascino /* 375*e5b8d921SVincenzo Frascino * It is not required to invoke page_kasan_tag_reset(page) 376*e5b8d921SVincenzo Frascino * at this point since the tags stored in page->flags are 377*e5b8d921SVincenzo Frascino * already restored. 378*e5b8d921SVincenzo Frascino */ 379ee11f332SSteven Price mte_restore_page_tags(page_address(page), tags); 380ee11f332SSteven Price 381ee11f332SSteven Price mte_free_tag_storage(tags); 382ee11f332SSteven Price n++; 383ee11f332SSteven Price } 384ee11f332SSteven Price xa_unlock(&mte_pages); 385ee11f332SSteven Price 386ee11f332SSteven Price pr_info("Restored %d MTE pages\n", n); 387ee11f332SSteven Price 388ee11f332SSteven Price xa_destroy(&mte_pages); 389ee11f332SSteven Price } 390ee11f332SSteven Price 391ee11f332SSteven Price #else /* CONFIG_ARM64_MTE */ 392ee11f332SSteven Price 393ee11f332SSteven Price static int swsusp_mte_save_tags(void) 394ee11f332SSteven Price { 395ee11f332SSteven Price return 0; 396ee11f332SSteven Price } 397ee11f332SSteven Price 398ee11f332SSteven Price static void swsusp_mte_restore_tags(void) 399ee11f332SSteven Price { 400ee11f332SSteven Price } 401ee11f332SSteven Price 402ee11f332SSteven Price #endif /* CONFIG_ARM64_MTE */ 403ee11f332SSteven Price 40482869ac5SJames Morse int swsusp_arch_suspend(void) 40582869ac5SJames Morse { 40682869ac5SJames Morse int ret = 0; 40782869ac5SJames Morse unsigned long flags; 40882869ac5SJames Morse struct sleep_stack_data state; 40982869ac5SJames Morse 410d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) { 411d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); 412d74b4e4fSJames Morse return -EBUSY; 413d74b4e4fSJames Morse } 414d74b4e4fSJames Morse 4150fbeb318SJames Morse flags = local_daif_save(); 41682869ac5SJames Morse 41782869ac5SJames Morse if (__cpu_suspend_enter(&state)) { 418254a41c0SAKASHI Takahiro /* make the crash dump kernel image visible/saveable */ 419254a41c0SAKASHI Takahiro crash_prepare_suspend(); 420254a41c0SAKASHI Takahiro 421ee11f332SSteven Price ret = swsusp_mte_save_tags(); 422ee11f332SSteven Price if (ret) 423ee11f332SSteven Price return ret; 424ee11f332SSteven Price 4258ec058fdSJames Morse sleep_cpu = smp_processor_id(); 42682869ac5SJames Morse ret = swsusp_save(); 42782869ac5SJames Morse } else { 4285ebe3a44SJames Morse /* Clean kernel core startup/idle code to PoC*/ 4295ebe3a44SJames Morse dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); 4305ebe3a44SJames Morse dcache_clean_range(__idmap_text_start, __idmap_text_end); 4315ebe3a44SJames Morse 4325ebe3a44SJames Morse /* Clean kvm setup code to PoC? */ 433f7daa9c8SJames Morse if (el2_reset_needed()) { 4345ebe3a44SJames Morse dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); 435f7daa9c8SJames Morse dcache_clean_range(__hyp_text_start, __hyp_text_end); 436f7daa9c8SJames Morse } 43782869ac5SJames Morse 438ee11f332SSteven Price swsusp_mte_restore_tags(); 439ee11f332SSteven Price 440254a41c0SAKASHI Takahiro /* make the crash dump kernel image protected again */ 441254a41c0SAKASHI Takahiro crash_post_resume(); 442254a41c0SAKASHI Takahiro 44382869ac5SJames Morse /* 44482869ac5SJames Morse * Tell the hibernation core that we've just restored 44582869ac5SJames Morse * the memory 44682869ac5SJames Morse */ 44782869ac5SJames Morse in_suspend = 0; 44882869ac5SJames Morse 4498ec058fdSJames Morse sleep_cpu = -EINVAL; 45082869ac5SJames Morse __cpu_suspend_exit(); 451647d0519SMarc Zyngier 452647d0519SMarc Zyngier /* 453647d0519SMarc Zyngier * Just in case the boot kernel did turn the SSBD 454647d0519SMarc Zyngier * mitigation off behind our back, let's set the state 455647d0519SMarc Zyngier * to what we expect it to be. 456647d0519SMarc Zyngier */ 457c2876207SWill Deacon spectre_v4_enable_mitigation(NULL); 45882869ac5SJames Morse } 45982869ac5SJames Morse 4600fbeb318SJames Morse local_daif_restore(flags); 46182869ac5SJames Morse 46282869ac5SJames Morse return ret; 46382869ac5SJames Morse } 46482869ac5SJames Morse 46520a004e7SWill Deacon static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) 4665ebe3a44SJames Morse { 46720a004e7SWill Deacon pte_t pte = READ_ONCE(*src_ptep); 4685ebe3a44SJames Morse 4695ebe3a44SJames Morse if (pte_valid(pte)) { 4705ebe3a44SJames Morse /* 4715ebe3a44SJames Morse * Resume will overwrite areas that may be marked 4725ebe3a44SJames Morse * read only (code, rodata). Clear the RDONLY bit from 4735ebe3a44SJames Morse * the temporary mappings we use during restore. 4745ebe3a44SJames Morse */ 47520a004e7SWill Deacon set_pte(dst_ptep, pte_mkwrite(pte)); 4765ebe3a44SJames Morse } else if (debug_pagealloc_enabled() && !pte_none(pte)) { 4775ebe3a44SJames Morse /* 4785ebe3a44SJames Morse * debug_pagealloc will removed the PTE_VALID bit if 4795ebe3a44SJames Morse * the page isn't in use by the resume kernel. It may have 4805ebe3a44SJames Morse * been in use by the original kernel, in which case we need 4815ebe3a44SJames Morse * to put it back in our copy to do the restore. 4825ebe3a44SJames Morse * 4835ebe3a44SJames Morse * Before marking this entry valid, check the pfn should 4845ebe3a44SJames Morse * be mapped. 4855ebe3a44SJames Morse */ 4865ebe3a44SJames Morse BUG_ON(!pfn_valid(pte_pfn(pte))); 4875ebe3a44SJames Morse 48820a004e7SWill Deacon set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); 4895ebe3a44SJames Morse } 4905ebe3a44SJames Morse } 4915ebe3a44SJames Morse 49220a004e7SWill Deacon static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, 49382869ac5SJames Morse unsigned long end) 49482869ac5SJames Morse { 49520a004e7SWill Deacon pte_t *src_ptep; 49620a004e7SWill Deacon pte_t *dst_ptep; 49782869ac5SJames Morse unsigned long addr = start; 49882869ac5SJames Morse 49920a004e7SWill Deacon dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); 50020a004e7SWill Deacon if (!dst_ptep) 50182869ac5SJames Morse return -ENOMEM; 50220a004e7SWill Deacon pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); 50320a004e7SWill Deacon dst_ptep = pte_offset_kernel(dst_pmdp, start); 50482869ac5SJames Morse 50520a004e7SWill Deacon src_ptep = pte_offset_kernel(src_pmdp, start); 50682869ac5SJames Morse do { 50720a004e7SWill Deacon _copy_pte(dst_ptep, src_ptep, addr); 50820a004e7SWill Deacon } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); 50982869ac5SJames Morse 51082869ac5SJames Morse return 0; 51182869ac5SJames Morse } 51282869ac5SJames Morse 51320a004e7SWill Deacon static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start, 51482869ac5SJames Morse unsigned long end) 51582869ac5SJames Morse { 51620a004e7SWill Deacon pmd_t *src_pmdp; 51720a004e7SWill Deacon pmd_t *dst_pmdp; 51882869ac5SJames Morse unsigned long next; 51982869ac5SJames Morse unsigned long addr = start; 52082869ac5SJames Morse 52120a004e7SWill Deacon if (pud_none(READ_ONCE(*dst_pudp))) { 52220a004e7SWill Deacon dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); 52320a004e7SWill Deacon if (!dst_pmdp) 52482869ac5SJames Morse return -ENOMEM; 52520a004e7SWill Deacon pud_populate(&init_mm, dst_pudp, dst_pmdp); 52682869ac5SJames Morse } 52720a004e7SWill Deacon dst_pmdp = pmd_offset(dst_pudp, start); 52882869ac5SJames Morse 52920a004e7SWill Deacon src_pmdp = pmd_offset(src_pudp, start); 53082869ac5SJames Morse do { 53120a004e7SWill Deacon pmd_t pmd = READ_ONCE(*src_pmdp); 53220a004e7SWill Deacon 53382869ac5SJames Morse next = pmd_addr_end(addr, end); 53420a004e7SWill Deacon if (pmd_none(pmd)) 53582869ac5SJames Morse continue; 53620a004e7SWill Deacon if (pmd_table(pmd)) { 53720a004e7SWill Deacon if (copy_pte(dst_pmdp, src_pmdp, addr, next)) 53882869ac5SJames Morse return -ENOMEM; 53982869ac5SJames Morse } else { 54020a004e7SWill Deacon set_pmd(dst_pmdp, 54120a004e7SWill Deacon __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); 54282869ac5SJames Morse } 54320a004e7SWill Deacon } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); 54482869ac5SJames Morse 54582869ac5SJames Morse return 0; 54682869ac5SJames Morse } 54782869ac5SJames Morse 548e9f63768SMike Rapoport static int copy_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start, 54982869ac5SJames Morse unsigned long end) 55082869ac5SJames Morse { 55120a004e7SWill Deacon pud_t *dst_pudp; 55220a004e7SWill Deacon pud_t *src_pudp; 55382869ac5SJames Morse unsigned long next; 55482869ac5SJames Morse unsigned long addr = start; 55582869ac5SJames Morse 556e9f63768SMike Rapoport if (p4d_none(READ_ONCE(*dst_p4dp))) { 55720a004e7SWill Deacon dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); 55820a004e7SWill Deacon if (!dst_pudp) 55982869ac5SJames Morse return -ENOMEM; 560e9f63768SMike Rapoport p4d_populate(&init_mm, dst_p4dp, dst_pudp); 56182869ac5SJames Morse } 562e9f63768SMike Rapoport dst_pudp = pud_offset(dst_p4dp, start); 56382869ac5SJames Morse 564e9f63768SMike Rapoport src_pudp = pud_offset(src_p4dp, start); 56582869ac5SJames Morse do { 56620a004e7SWill Deacon pud_t pud = READ_ONCE(*src_pudp); 56720a004e7SWill Deacon 56882869ac5SJames Morse next = pud_addr_end(addr, end); 56920a004e7SWill Deacon if (pud_none(pud)) 57082869ac5SJames Morse continue; 57120a004e7SWill Deacon if (pud_table(pud)) { 57220a004e7SWill Deacon if (copy_pmd(dst_pudp, src_pudp, addr, next)) 57382869ac5SJames Morse return -ENOMEM; 57482869ac5SJames Morse } else { 57520a004e7SWill Deacon set_pud(dst_pudp, 5767ea40889SPavel Tatashin __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); 57782869ac5SJames Morse } 57820a004e7SWill Deacon } while (dst_pudp++, src_pudp++, addr = next, addr != end); 57982869ac5SJames Morse 58082869ac5SJames Morse return 0; 58182869ac5SJames Morse } 58282869ac5SJames Morse 583e9f63768SMike Rapoport static int copy_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start, 584e9f63768SMike Rapoport unsigned long end) 585e9f63768SMike Rapoport { 586e9f63768SMike Rapoport p4d_t *dst_p4dp; 587e9f63768SMike Rapoport p4d_t *src_p4dp; 588e9f63768SMike Rapoport unsigned long next; 589e9f63768SMike Rapoport unsigned long addr = start; 590e9f63768SMike Rapoport 591e9f63768SMike Rapoport dst_p4dp = p4d_offset(dst_pgdp, start); 592e9f63768SMike Rapoport src_p4dp = p4d_offset(src_pgdp, start); 593e9f63768SMike Rapoport do { 594e9f63768SMike Rapoport next = p4d_addr_end(addr, end); 595e9f63768SMike Rapoport if (p4d_none(READ_ONCE(*src_p4dp))) 596e9f63768SMike Rapoport continue; 597e9f63768SMike Rapoport if (copy_pud(dst_p4dp, src_p4dp, addr, next)) 598e9f63768SMike Rapoport return -ENOMEM; 599e9f63768SMike Rapoport } while (dst_p4dp++, src_p4dp++, addr = next, addr != end); 600e9f63768SMike Rapoport 601e9f63768SMike Rapoport return 0; 602e9f63768SMike Rapoport } 603e9f63768SMike Rapoport 60420a004e7SWill Deacon static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, 60582869ac5SJames Morse unsigned long end) 60682869ac5SJames Morse { 60782869ac5SJames Morse unsigned long next; 60882869ac5SJames Morse unsigned long addr = start; 60920a004e7SWill Deacon pgd_t *src_pgdp = pgd_offset_k(start); 61082869ac5SJames Morse 611974b9b2cSMike Rapoport dst_pgdp = pgd_offset_pgd(dst_pgdp, start); 61282869ac5SJames Morse do { 61382869ac5SJames Morse next = pgd_addr_end(addr, end); 61420a004e7SWill Deacon if (pgd_none(READ_ONCE(*src_pgdp))) 61582869ac5SJames Morse continue; 616e9f63768SMike Rapoport if (copy_p4d(dst_pgdp, src_pgdp, addr, next)) 61782869ac5SJames Morse return -ENOMEM; 61820a004e7SWill Deacon } while (dst_pgdp++, src_pgdp++, addr = next, addr != end); 61982869ac5SJames Morse 62082869ac5SJames Morse return 0; 62182869ac5SJames Morse } 62282869ac5SJames Morse 623a2c2e679SPavel Tatashin static int trans_pgd_create_copy(pgd_t **dst_pgdp, unsigned long start, 624a2c2e679SPavel Tatashin unsigned long end) 625a2c2e679SPavel Tatashin { 626a2c2e679SPavel Tatashin int rc; 627a2c2e679SPavel Tatashin pgd_t *trans_pgd = (pgd_t *)get_safe_page(GFP_ATOMIC); 628a2c2e679SPavel Tatashin 629a2c2e679SPavel Tatashin if (!trans_pgd) { 630a2c2e679SPavel Tatashin pr_err("Failed to allocate memory for temporary page tables.\n"); 631a2c2e679SPavel Tatashin return -ENOMEM; 632a2c2e679SPavel Tatashin } 633a2c2e679SPavel Tatashin 634a2c2e679SPavel Tatashin rc = copy_page_tables(trans_pgd, start, end); 635a2c2e679SPavel Tatashin if (!rc) 636a2c2e679SPavel Tatashin *dst_pgdp = trans_pgd; 637a2c2e679SPavel Tatashin 638a2c2e679SPavel Tatashin return rc; 639a2c2e679SPavel Tatashin } 640a2c2e679SPavel Tatashin 64182869ac5SJames Morse /* 64282869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit(). 64382869ac5SJames Morse * 64482869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code, 64582869ac5SJames Morse * we don't need to free it here. 64682869ac5SJames Morse */ 64782869ac5SJames Morse int swsusp_arch_resume(void) 64882869ac5SJames Morse { 649a89d7ff9SPavel Tatashin int rc; 65082869ac5SJames Morse void *zero_page; 65182869ac5SJames Morse size_t exit_size; 65282869ac5SJames Morse pgd_t *tmp_pg_dir; 65382869ac5SJames Morse phys_addr_t phys_hibernate_exit; 65482869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, 65582869ac5SJames Morse void *, phys_addr_t, phys_addr_t); 65682869ac5SJames Morse 65782869ac5SJames Morse /* 658dfbca61aSMark Rutland * Restoring the memory image will overwrite the ttbr1 page tables. 659dfbca61aSMark Rutland * Create a second copy of just the linear map, and use this when 660dfbca61aSMark Rutland * restoring. 661dfbca61aSMark Rutland */ 662a2c2e679SPavel Tatashin rc = trans_pgd_create_copy(&tmp_pg_dir, PAGE_OFFSET, PAGE_END); 663dfbca61aSMark Rutland if (rc) 664a89d7ff9SPavel Tatashin return rc; 665dfbca61aSMark Rutland 666dfbca61aSMark Rutland /* 667dfbca61aSMark Rutland * We need a zero page that is zero before & after resume in order to 668dfbca61aSMark Rutland * to break before make on the ttbr1 page tables. 669dfbca61aSMark Rutland */ 670dfbca61aSMark Rutland zero_page = (void *)get_safe_page(GFP_ATOMIC); 671dfbca61aSMark Rutland if (!zero_page) { 672117f5727SMark Rutland pr_err("Failed to allocate zero page.\n"); 673a89d7ff9SPavel Tatashin return -ENOMEM; 674dfbca61aSMark Rutland } 675dfbca61aSMark Rutland 676dfbca61aSMark Rutland /* 67782869ac5SJames Morse * Locate the exit code in the bottom-but-one page, so that *NULL 67882869ac5SJames Morse * still has disastrous affects. 67982869ac5SJames Morse */ 68082869ac5SJames Morse hibernate_exit = (void *)PAGE_SIZE; 68182869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; 68282869ac5SJames Morse /* 68382869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate 68482869ac5SJames Morse * a new set of ttbr0 page tables and load them. 68582869ac5SJames Morse */ 68682869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size, 68782869ac5SJames Morse (unsigned long)hibernate_exit, 688051a7a94SPavel Tatashin &phys_hibernate_exit); 68982869ac5SJames Morse if (rc) { 690117f5727SMark Rutland pr_err("Failed to create safe executable page for hibernate_exit code.\n"); 691a89d7ff9SPavel Tatashin return rc; 69282869ac5SJames Morse } 69382869ac5SJames Morse 69482869ac5SJames Morse /* 69582869ac5SJames Morse * The hibernate exit text contains a set of el2 vectors, that will 69682869ac5SJames Morse * be executed at el2 with the mmu off in order to reload hyp-stub. 69782869ac5SJames Morse */ 69882869ac5SJames Morse __flush_dcache_area(hibernate_exit, exit_size); 69982869ac5SJames Morse 70082869ac5SJames Morse /* 70182869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in 70282869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2. 70382869ac5SJames Morse * 70482869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE. 70582869ac5SJames Morse */ 70682869ac5SJames Morse if (el2_reset_needed()) { 70782869ac5SJames Morse phys_addr_t el2_vectors = phys_hibernate_exit; /* base */ 70882869ac5SJames Morse el2_vectors += hibernate_el2_vectors - 70982869ac5SJames Morse __hibernate_exit_text_start; /* offset */ 71082869ac5SJames Morse 71182869ac5SJames Morse __hyp_set_vectors(el2_vectors); 71282869ac5SJames Morse } 71382869ac5SJames Morse 71482869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, 7152077be67SLaura Abbott resume_hdr.reenter_kernel, restore_pblist, 71682869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); 71782869ac5SJames Morse 718a89d7ff9SPavel Tatashin return 0; 71982869ac5SJames Morse } 7201fe492ceSJames Morse 7218ec058fdSJames Morse int hibernate_resume_nonboot_cpu_disable(void) 7228ec058fdSJames Morse { 7238ec058fdSJames Morse if (sleep_cpu < 0) { 7249165dabbSMasanari Iida pr_err("Failing to resume from hibernate on an unknown CPU.\n"); 7258ec058fdSJames Morse return -ENODEV; 7268ec058fdSJames Morse } 7278ec058fdSJames Morse 7288ec058fdSJames Morse return freeze_secondary_cpus(sleep_cpu); 7298ec058fdSJames Morse } 730