1af873fceSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
282869ac5SJames Morse /*:
382869ac5SJames Morse * Hibernate support specific for ARM64
482869ac5SJames Morse *
582869ac5SJames Morse * Derived from work on ARM hibernation support by:
682869ac5SJames Morse *
782869ac5SJames Morse * Ubuntu project, hibernation support for mach-dove
882869ac5SJames Morse * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu)
982869ac5SJames Morse * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.)
1082869ac5SJames Morse * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
1182869ac5SJames Morse */
1282869ac5SJames Morse #define pr_fmt(x) "hibernate: " x
138ec058fdSJames Morse #include <linux/cpu.h>
1482869ac5SJames Morse #include <linux/kvm_host.h>
1582869ac5SJames Morse #include <linux/pm.h>
1682869ac5SJames Morse #include <linux/sched.h>
1782869ac5SJames Morse #include <linux/suspend.h>
1882869ac5SJames Morse #include <linux/utsname.h>
1982869ac5SJames Morse
2082869ac5SJames Morse #include <asm/barrier.h>
2182869ac5SJames Morse #include <asm/cacheflush.h>
228ec058fdSJames Morse #include <asm/cputype.h>
230fbeb318SJames Morse #include <asm/daifflags.h>
2482869ac5SJames Morse #include <asm/irqflags.h>
25254a41c0SAKASHI Takahiro #include <asm/kexec.h>
2682869ac5SJames Morse #include <asm/memory.h>
2782869ac5SJames Morse #include <asm/mmu_context.h>
28ee11f332SSteven Price #include <asm/mte.h>
2982869ac5SJames Morse #include <asm/sections.h>
30d74b4e4fSJames Morse #include <asm/smp.h>
318ec058fdSJames Morse #include <asm/smp_plat.h>
3282869ac5SJames Morse #include <asm/suspend.h>
330194e760SMark Rutland #include <asm/sysreg.h>
34072e3d96SPavel Tatashin #include <asm/trans_pgd.h>
3582869ac5SJames Morse #include <asm/virt.h>
3682869ac5SJames Morse
3782869ac5SJames Morse /*
3882869ac5SJames Morse * Hibernate core relies on this value being 0 on resume, and marks it
3982869ac5SJames Morse * __nosavedata assuming it will keep the resume kernel's '0' value. This
4082869ac5SJames Morse * doesn't happen with either KASLR.
4182869ac5SJames Morse *
4282869ac5SJames Morse * defined as "__visible int in_suspend __nosavedata" in
4382869ac5SJames Morse * kernel/power/hibernate.c
4482869ac5SJames Morse */
4582869ac5SJames Morse extern int in_suspend;
4682869ac5SJames Morse
4782869ac5SJames Morse /* Do we need to reset el2? */
48094a3684SPasha Tatashin #define el2_reset_needed() (is_hyp_nvhe())
4982869ac5SJames Morse
5082869ac5SJames Morse /* hyp-stub vectors, used to restore el2 during resume from hibernate. */
5182869ac5SJames Morse extern char __hyp_stub_vectors[];
5282869ac5SJames Morse
5382869ac5SJames Morse /*
548ec058fdSJames Morse * The logical cpu number we should resume on, initialised to a non-cpu
558ec058fdSJames Morse * number.
568ec058fdSJames Morse */
578ec058fdSJames Morse static int sleep_cpu = -EINVAL;
588ec058fdSJames Morse
598ec058fdSJames Morse /*
6082869ac5SJames Morse * Values that may not change over hibernate/resume. We put the build number
6182869ac5SJames Morse * and date in here so that we guarantee not to resume with a different
6282869ac5SJames Morse * kernel.
6382869ac5SJames Morse */
6482869ac5SJames Morse struct arch_hibernate_hdr_invariants {
6582869ac5SJames Morse char uts_version[__NEW_UTS_LEN + 1];
6682869ac5SJames Morse };
6782869ac5SJames Morse
6882869ac5SJames Morse /* These values need to be know across a hibernate/restore. */
6982869ac5SJames Morse static struct arch_hibernate_hdr {
7082869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants;
7182869ac5SJames Morse
7282869ac5SJames Morse /* These are needed to find the relocated kernel if built with kaslr */
7382869ac5SJames Morse phys_addr_t ttbr1_el1;
7482869ac5SJames Morse void (*reenter_kernel)(void);
7582869ac5SJames Morse
7682869ac5SJames Morse /*
7782869ac5SJames Morse * We need to know where the __hyp_stub_vectors are after restore to
7882869ac5SJames Morse * re-configure el2.
7982869ac5SJames Morse */
8082869ac5SJames Morse phys_addr_t __hyp_stub_vectors;
818ec058fdSJames Morse
828ec058fdSJames Morse u64 sleep_cpu_mpidr;
8382869ac5SJames Morse } resume_hdr;
8482869ac5SJames Morse
arch_hdr_invariants(struct arch_hibernate_hdr_invariants * i)8582869ac5SJames Morse static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
8682869ac5SJames Morse {
8782869ac5SJames Morse memset(i, 0, sizeof(*i));
8882869ac5SJames Morse memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
8982869ac5SJames Morse }
9082869ac5SJames Morse
pfn_is_nosave(unsigned long pfn)9182869ac5SJames Morse int pfn_is_nosave(unsigned long pfn)
9282869ac5SJames Morse {
932077be67SLaura Abbott unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
942077be67SLaura Abbott unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
9582869ac5SJames Morse
96254a41c0SAKASHI Takahiro return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) ||
97254a41c0SAKASHI Takahiro crash_is_nosave(pfn);
9882869ac5SJames Morse }
9982869ac5SJames Morse
save_processor_state(void)10082869ac5SJames Morse void notrace save_processor_state(void)
10182869ac5SJames Morse {
10282869ac5SJames Morse }
10382869ac5SJames Morse
restore_processor_state(void)10482869ac5SJames Morse void notrace restore_processor_state(void)
10582869ac5SJames Morse {
10682869ac5SJames Morse }
10782869ac5SJames Morse
arch_hibernation_header_save(void * addr,unsigned int max_size)10882869ac5SJames Morse int arch_hibernation_header_save(void *addr, unsigned int max_size)
10982869ac5SJames Morse {
11082869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr;
11182869ac5SJames Morse
11282869ac5SJames Morse if (max_size < sizeof(*hdr))
11382869ac5SJames Morse return -EOVERFLOW;
11482869ac5SJames Morse
11582869ac5SJames Morse arch_hdr_invariants(&hdr->invariants);
1162077be67SLaura Abbott hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir);
11782869ac5SJames Morse hdr->reenter_kernel = _cpu_resume;
11882869ac5SJames Morse
11982869ac5SJames Morse /* We can't use __hyp_get_vectors() because kvm may still be loaded */
12082869ac5SJames Morse if (el2_reset_needed())
1212077be67SLaura Abbott hdr->__hyp_stub_vectors = __pa_symbol(__hyp_stub_vectors);
12282869ac5SJames Morse else
12382869ac5SJames Morse hdr->__hyp_stub_vectors = 0;
12482869ac5SJames Morse
1258ec058fdSJames Morse /* Save the mpidr of the cpu we called cpu_suspend() on... */
1268ec058fdSJames Morse if (sleep_cpu < 0) {
1279165dabbSMasanari Iida pr_err("Failing to hibernate on an unknown CPU.\n");
1288ec058fdSJames Morse return -ENODEV;
1298ec058fdSJames Morse }
1308ec058fdSJames Morse hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu);
1318ec058fdSJames Morse pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu,
1328ec058fdSJames Morse hdr->sleep_cpu_mpidr);
1338ec058fdSJames Morse
13482869ac5SJames Morse return 0;
13582869ac5SJames Morse }
13682869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_save);
13782869ac5SJames Morse
arch_hibernation_header_restore(void * addr)13882869ac5SJames Morse int arch_hibernation_header_restore(void *addr)
13982869ac5SJames Morse {
1408ec058fdSJames Morse int ret;
14182869ac5SJames Morse struct arch_hibernate_hdr_invariants invariants;
14282869ac5SJames Morse struct arch_hibernate_hdr *hdr = addr;
14382869ac5SJames Morse
14482869ac5SJames Morse arch_hdr_invariants(&invariants);
14582869ac5SJames Morse if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
14682869ac5SJames Morse pr_crit("Hibernate image not generated by this kernel!\n");
14782869ac5SJames Morse return -EINVAL;
14882869ac5SJames Morse }
14982869ac5SJames Morse
1508ec058fdSJames Morse sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr);
1518ec058fdSJames Morse pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu,
1528ec058fdSJames Morse hdr->sleep_cpu_mpidr);
1538ec058fdSJames Morse if (sleep_cpu < 0) {
1548ec058fdSJames Morse pr_crit("Hibernated on a CPU not known to this kernel!\n");
1558ec058fdSJames Morse sleep_cpu = -EINVAL;
1568ec058fdSJames Morse return -EINVAL;
1578ec058fdSJames Morse }
158e646ac5bSQais Yousef
159e646ac5bSQais Yousef ret = bringup_hibernate_cpu(sleep_cpu);
1608ec058fdSJames Morse if (ret) {
1618ec058fdSJames Morse sleep_cpu = -EINVAL;
1628ec058fdSJames Morse return ret;
1638ec058fdSJames Morse }
1648ec058fdSJames Morse
16582869ac5SJames Morse resume_hdr = *hdr;
16682869ac5SJames Morse
16782869ac5SJames Morse return 0;
16882869ac5SJames Morse }
16982869ac5SJames Morse EXPORT_SYMBOL(arch_hibernation_header_restore);
17082869ac5SJames Morse
hibernate_page_alloc(void * arg)17150f53fb7SPavel Tatashin static void *hibernate_page_alloc(void *arg)
17250f53fb7SPavel Tatashin {
173d1bbc35fSPavel Tatashin return (void *)get_safe_page((__force gfp_t)(unsigned long)arg);
17450f53fb7SPavel Tatashin }
17550f53fb7SPavel Tatashin
176a2c2e679SPavel Tatashin /*
177a2c2e679SPavel Tatashin * Copies length bytes, starting at src_start into an new page,
178a2c2e679SPavel Tatashin * perform cache maintenance, then maps it at the specified address low
179a2c2e679SPavel Tatashin * address as executable.
180a2c2e679SPavel Tatashin *
181a2c2e679SPavel Tatashin * This is used by hibernate to copy the code it needs to execute when
182a2c2e679SPavel Tatashin * overwriting the kernel text. This function generates a new set of page
183a2c2e679SPavel Tatashin * tables, which it loads into ttbr0.
184a2c2e679SPavel Tatashin *
185a2c2e679SPavel Tatashin * Length is provided as we probably only want 4K of data, even on a 64K
186a2c2e679SPavel Tatashin * page system.
187a2c2e679SPavel Tatashin */
create_safe_exec_page(void * src_start,size_t length,phys_addr_t * phys_dst_addr)188a2c2e679SPavel Tatashin static int create_safe_exec_page(void *src_start, size_t length,
189a2c2e679SPavel Tatashin phys_addr_t *phys_dst_addr)
190a2c2e679SPavel Tatashin {
19150f53fb7SPavel Tatashin struct trans_pgd_info trans_info = {
19250f53fb7SPavel Tatashin .trans_alloc_page = hibernate_page_alloc,
193d1bbc35fSPavel Tatashin .trans_alloc_arg = (__force void *)GFP_ATOMIC,
19450f53fb7SPavel Tatashin };
19550f53fb7SPavel Tatashin
196a2c2e679SPavel Tatashin void *page = (void *)get_safe_page(GFP_ATOMIC);
1977018d467SJames Morse phys_addr_t trans_ttbr0;
1987018d467SJames Morse unsigned long t0sz;
199a2c2e679SPavel Tatashin int rc;
200a2c2e679SPavel Tatashin
201a2c2e679SPavel Tatashin if (!page)
202a2c2e679SPavel Tatashin return -ENOMEM;
203a2c2e679SPavel Tatashin
204a2c2e679SPavel Tatashin memcpy(page, src_start, length);
205fade9c2cSFuad Tabba caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length);
2067018d467SJames Morse rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
207a2c2e679SPavel Tatashin if (rc)
208a2c2e679SPavel Tatashin return rc;
209a2c2e679SPavel Tatashin
210a347f601SPasha Tatashin cpu_install_ttbr0(trans_ttbr0, t0sz);
21113373f0eSPavel Tatashin *phys_dst_addr = virt_to_phys(page);
21282869ac5SJames Morse
213a89d7ff9SPavel Tatashin return 0;
21482869ac5SJames Morse }
21582869ac5SJames Morse
216ee11f332SSteven Price #ifdef CONFIG_ARM64_MTE
217ee11f332SSteven Price
218ee11f332SSteven Price static DEFINE_XARRAY(mte_pages);
219ee11f332SSteven Price
save_tags(struct page * page,unsigned long pfn)220ee11f332SSteven Price static int save_tags(struct page *page, unsigned long pfn)
221ee11f332SSteven Price {
222ee11f332SSteven Price void *tag_storage, *ret;
223ee11f332SSteven Price
224ee11f332SSteven Price tag_storage = mte_allocate_tag_storage();
225ee11f332SSteven Price if (!tag_storage)
226ee11f332SSteven Price return -ENOMEM;
227ee11f332SSteven Price
228ee11f332SSteven Price mte_save_page_tags(page_address(page), tag_storage);
229ee11f332SSteven Price
230ee11f332SSteven Price ret = xa_store(&mte_pages, pfn, tag_storage, GFP_KERNEL);
231ee11f332SSteven Price if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
232ee11f332SSteven Price mte_free_tag_storage(tag_storage);
233ee11f332SSteven Price return xa_err(ret);
234ee11f332SSteven Price } else if (WARN(ret, "swsusp: %s: Duplicate entry", __func__)) {
235ee11f332SSteven Price mte_free_tag_storage(ret);
236ee11f332SSteven Price }
237ee11f332SSteven Price
238ee11f332SSteven Price return 0;
239ee11f332SSteven Price }
240ee11f332SSteven Price
swsusp_mte_free_storage(void)241ee11f332SSteven Price static void swsusp_mte_free_storage(void)
242ee11f332SSteven Price {
243ee11f332SSteven Price XA_STATE(xa_state, &mte_pages, 0);
244ee11f332SSteven Price void *tags;
245ee11f332SSteven Price
246ee11f332SSteven Price xa_lock(&mte_pages);
247ee11f332SSteven Price xas_for_each(&xa_state, tags, ULONG_MAX) {
248ee11f332SSteven Price mte_free_tag_storage(tags);
249ee11f332SSteven Price }
250ee11f332SSteven Price xa_unlock(&mte_pages);
251ee11f332SSteven Price
252ee11f332SSteven Price xa_destroy(&mte_pages);
253ee11f332SSteven Price }
254ee11f332SSteven Price
swsusp_mte_save_tags(void)255ee11f332SSteven Price static int swsusp_mte_save_tags(void)
256ee11f332SSteven Price {
257ee11f332SSteven Price struct zone *zone;
258ee11f332SSteven Price unsigned long pfn, max_zone_pfn;
259ee11f332SSteven Price int ret = 0;
260ee11f332SSteven Price int n = 0;
261ee11f332SSteven Price
262ee11f332SSteven Price if (!system_supports_mte())
263ee11f332SSteven Price return 0;
264ee11f332SSteven Price
265ee11f332SSteven Price for_each_populated_zone(zone) {
266ee11f332SSteven Price max_zone_pfn = zone_end_pfn(zone);
267ee11f332SSteven Price for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
268ee11f332SSteven Price struct page *page = pfn_to_online_page(pfn);
269ee11f332SSteven Price
270ee11f332SSteven Price if (!page)
271ee11f332SSteven Price continue;
272ee11f332SSteven Price
273*e059853dSCatalin Marinas if (!page_mte_tagged(page))
274ee11f332SSteven Price continue;
275ee11f332SSteven Price
276ee11f332SSteven Price ret = save_tags(page, pfn);
277ee11f332SSteven Price if (ret) {
278ee11f332SSteven Price swsusp_mte_free_storage();
279ee11f332SSteven Price goto out;
280ee11f332SSteven Price }
281ee11f332SSteven Price
282ee11f332SSteven Price n++;
283ee11f332SSteven Price }
284ee11f332SSteven Price }
285ee11f332SSteven Price pr_info("Saved %d MTE pages\n", n);
286ee11f332SSteven Price
287ee11f332SSteven Price out:
288ee11f332SSteven Price return ret;
289ee11f332SSteven Price }
290ee11f332SSteven Price
swsusp_mte_restore_tags(void)291ee11f332SSteven Price static void swsusp_mte_restore_tags(void)
292ee11f332SSteven Price {
293ee11f332SSteven Price XA_STATE(xa_state, &mte_pages, 0);
294ee11f332SSteven Price int n = 0;
295ee11f332SSteven Price void *tags;
296ee11f332SSteven Price
297ee11f332SSteven Price xa_lock(&mte_pages);
298ee11f332SSteven Price xas_for_each(&xa_state, tags, ULONG_MAX) {
299ee11f332SSteven Price unsigned long pfn = xa_state.xa_index;
300ee11f332SSteven Price struct page *page = pfn_to_online_page(pfn);
301ee11f332SSteven Price
302ee11f332SSteven Price mte_restore_page_tags(page_address(page), tags);
303ee11f332SSteven Price
304ee11f332SSteven Price mte_free_tag_storage(tags);
305ee11f332SSteven Price n++;
306ee11f332SSteven Price }
307ee11f332SSteven Price xa_unlock(&mte_pages);
308ee11f332SSteven Price
309ee11f332SSteven Price pr_info("Restored %d MTE pages\n", n);
310ee11f332SSteven Price
311ee11f332SSteven Price xa_destroy(&mte_pages);
312ee11f332SSteven Price }
313ee11f332SSteven Price
314ee11f332SSteven Price #else /* CONFIG_ARM64_MTE */
315ee11f332SSteven Price
swsusp_mte_save_tags(void)316ee11f332SSteven Price static int swsusp_mte_save_tags(void)
317ee11f332SSteven Price {
318ee11f332SSteven Price return 0;
319ee11f332SSteven Price }
320ee11f332SSteven Price
swsusp_mte_restore_tags(void)321ee11f332SSteven Price static void swsusp_mte_restore_tags(void)
322ee11f332SSteven Price {
323ee11f332SSteven Price }
324ee11f332SSteven Price
325ee11f332SSteven Price #endif /* CONFIG_ARM64_MTE */
326ee11f332SSteven Price
swsusp_arch_suspend(void)32782869ac5SJames Morse int swsusp_arch_suspend(void)
32882869ac5SJames Morse {
32982869ac5SJames Morse int ret = 0;
33082869ac5SJames Morse unsigned long flags;
33182869ac5SJames Morse struct sleep_stack_data state;
33282869ac5SJames Morse
333d74b4e4fSJames Morse if (cpus_are_stuck_in_kernel()) {
334d74b4e4fSJames Morse pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n");
335d74b4e4fSJames Morse return -EBUSY;
336d74b4e4fSJames Morse }
337d74b4e4fSJames Morse
3380fbeb318SJames Morse flags = local_daif_save();
33982869ac5SJames Morse
34082869ac5SJames Morse if (__cpu_suspend_enter(&state)) {
341254a41c0SAKASHI Takahiro /* make the crash dump kernel image visible/saveable */
342254a41c0SAKASHI Takahiro crash_prepare_suspend();
343254a41c0SAKASHI Takahiro
344ee11f332SSteven Price ret = swsusp_mte_save_tags();
345ee11f332SSteven Price if (ret)
346ee11f332SSteven Price return ret;
347ee11f332SSteven Price
3488ec058fdSJames Morse sleep_cpu = smp_processor_id();
34982869ac5SJames Morse ret = swsusp_save();
35082869ac5SJames Morse } else {
3515ebe3a44SJames Morse /* Clean kernel core startup/idle code to PoC*/
352fade9c2cSFuad Tabba dcache_clean_inval_poc((unsigned long)__mmuoff_data_start,
353814b1860SFuad Tabba (unsigned long)__mmuoff_data_end);
354fade9c2cSFuad Tabba dcache_clean_inval_poc((unsigned long)__idmap_text_start,
355814b1860SFuad Tabba (unsigned long)__idmap_text_end);
3565ebe3a44SJames Morse
3575ebe3a44SJames Morse /* Clean kvm setup code to PoC? */
358f7daa9c8SJames Morse if (el2_reset_needed()) {
359fade9c2cSFuad Tabba dcache_clean_inval_poc(
360814b1860SFuad Tabba (unsigned long)__hyp_idmap_text_start,
361814b1860SFuad Tabba (unsigned long)__hyp_idmap_text_end);
362fade9c2cSFuad Tabba dcache_clean_inval_poc((unsigned long)__hyp_text_start,
363814b1860SFuad Tabba (unsigned long)__hyp_text_end);
364f7daa9c8SJames Morse }
36582869ac5SJames Morse
366ee11f332SSteven Price swsusp_mte_restore_tags();
367ee11f332SSteven Price
368254a41c0SAKASHI Takahiro /* make the crash dump kernel image protected again */
369254a41c0SAKASHI Takahiro crash_post_resume();
370254a41c0SAKASHI Takahiro
37182869ac5SJames Morse /*
37282869ac5SJames Morse * Tell the hibernation core that we've just restored
37382869ac5SJames Morse * the memory
37482869ac5SJames Morse */
37582869ac5SJames Morse in_suspend = 0;
37682869ac5SJames Morse
3778ec058fdSJames Morse sleep_cpu = -EINVAL;
37882869ac5SJames Morse __cpu_suspend_exit();
379647d0519SMarc Zyngier
380647d0519SMarc Zyngier /*
381647d0519SMarc Zyngier * Just in case the boot kernel did turn the SSBD
382647d0519SMarc Zyngier * mitigation off behind our back, let's set the state
383647d0519SMarc Zyngier * to what we expect it to be.
384647d0519SMarc Zyngier */
385c2876207SWill Deacon spectre_v4_enable_mitigation(NULL);
38682869ac5SJames Morse }
38782869ac5SJames Morse
3880fbeb318SJames Morse local_daif_restore(flags);
38982869ac5SJames Morse
39082869ac5SJames Morse return ret;
39182869ac5SJames Morse }
39282869ac5SJames Morse
39382869ac5SJames Morse /*
39482869ac5SJames Morse * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit().
39582869ac5SJames Morse *
39682869ac5SJames Morse * Memory allocated by get_safe_page() will be dealt with by the hibernate code,
39782869ac5SJames Morse * we don't need to free it here.
39882869ac5SJames Morse */
swsusp_arch_resume(void)39982869ac5SJames Morse int swsusp_arch_resume(void)
40082869ac5SJames Morse {
401a89d7ff9SPavel Tatashin int rc;
40282869ac5SJames Morse void *zero_page;
40382869ac5SJames Morse size_t exit_size;
40482869ac5SJames Morse pgd_t *tmp_pg_dir;
405788bfdd9SPasha Tatashin phys_addr_t el2_vectors;
40682869ac5SJames Morse void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *,
40782869ac5SJames Morse void *, phys_addr_t, phys_addr_t);
40889d1410fSPavel Tatashin struct trans_pgd_info trans_info = {
40989d1410fSPavel Tatashin .trans_alloc_page = hibernate_page_alloc,
41089d1410fSPavel Tatashin .trans_alloc_arg = (void *)GFP_ATOMIC,
41189d1410fSPavel Tatashin };
41282869ac5SJames Morse
41382869ac5SJames Morse /*
414dfbca61aSMark Rutland * Restoring the memory image will overwrite the ttbr1 page tables.
415dfbca61aSMark Rutland * Create a second copy of just the linear map, and use this when
416dfbca61aSMark Rutland * restoring.
417dfbca61aSMark Rutland */
41889d1410fSPavel Tatashin rc = trans_pgd_create_copy(&trans_info, &tmp_pg_dir, PAGE_OFFSET,
41989d1410fSPavel Tatashin PAGE_END);
420dfbca61aSMark Rutland if (rc)
421a89d7ff9SPavel Tatashin return rc;
422dfbca61aSMark Rutland
423dfbca61aSMark Rutland /*
424189129aeSXiang wangx * We need a zero page that is zero before & after resume in order
425dfbca61aSMark Rutland * to break before make on the ttbr1 page tables.
426dfbca61aSMark Rutland */
427dfbca61aSMark Rutland zero_page = (void *)get_safe_page(GFP_ATOMIC);
428dfbca61aSMark Rutland if (!zero_page) {
429117f5727SMark Rutland pr_err("Failed to allocate zero page.\n");
430a89d7ff9SPavel Tatashin return -ENOMEM;
431dfbca61aSMark Rutland }
432dfbca61aSMark Rutland
433788bfdd9SPasha Tatashin if (el2_reset_needed()) {
434788bfdd9SPasha Tatashin rc = trans_pgd_copy_el2_vectors(&trans_info, &el2_vectors);
435788bfdd9SPasha Tatashin if (rc) {
436788bfdd9SPasha Tatashin pr_err("Failed to setup el2 vectors\n");
437788bfdd9SPasha Tatashin return rc;
438788bfdd9SPasha Tatashin }
439788bfdd9SPasha Tatashin }
440788bfdd9SPasha Tatashin
44182869ac5SJames Morse exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start;
44282869ac5SJames Morse /*
44382869ac5SJames Morse * Copy swsusp_arch_suspend_exit() to a safe page. This will generate
44482869ac5SJames Morse * a new set of ttbr0 page tables and load them.
44582869ac5SJames Morse */
44682869ac5SJames Morse rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size,
4477018d467SJames Morse (phys_addr_t *)&hibernate_exit);
44882869ac5SJames Morse if (rc) {
449117f5727SMark Rutland pr_err("Failed to create safe executable page for hibernate_exit code.\n");
450a89d7ff9SPavel Tatashin return rc;
45182869ac5SJames Morse }
45282869ac5SJames Morse
45382869ac5SJames Morse /*
45482869ac5SJames Morse * KASLR will cause the el2 vectors to be in a different location in
45582869ac5SJames Morse * the resumed kernel. Load hibernate's temporary copy into el2.
45682869ac5SJames Morse *
45782869ac5SJames Morse * We can skip this step if we booted at EL1, or are running with VHE.
45882869ac5SJames Morse */
459788bfdd9SPasha Tatashin if (el2_reset_needed())
46082869ac5SJames Morse __hyp_set_vectors(el2_vectors);
46182869ac5SJames Morse
46282869ac5SJames Morse hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1,
4632077be67SLaura Abbott resume_hdr.reenter_kernel, restore_pblist,
46482869ac5SJames Morse resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page));
46582869ac5SJames Morse
466a89d7ff9SPavel Tatashin return 0;
46782869ac5SJames Morse }
4681fe492ceSJames Morse
hibernate_resume_nonboot_cpu_disable(void)4698ec058fdSJames Morse int hibernate_resume_nonboot_cpu_disable(void)
4708ec058fdSJames Morse {
4718ec058fdSJames Morse if (sleep_cpu < 0) {
4729165dabbSMasanari Iida pr_err("Failing to resume from hibernate on an unknown CPU.\n");
4738ec058fdSJames Morse return -ENODEV;
4748ec058fdSJames Morse }
4758ec058fdSJames Morse
4768ec058fdSJames Morse return freeze_secondary_cpus(sleep_cpu);
4778ec058fdSJames Morse }
478