1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Hibernation support for RISCV
4 *
5 * Copyright (C) 2023 StarFive Technology Co., Ltd.
6 *
7 * Author: Jee Heng Sia <jeeheng.sia@starfivetech.com>
8 */
9
10 #include <asm/barrier.h>
11 #include <asm/cacheflush.h>
12 #include <asm/mmu_context.h>
13 #include <asm/page.h>
14 #include <asm/pgalloc.h>
15 #include <asm/pgtable.h>
16 #include <asm/sections.h>
17 #include <asm/set_memory.h>
18 #include <asm/smp.h>
19 #include <asm/suspend.h>
20
21 #include <linux/cpu.h>
22 #include <linux/memblock.h>
23 #include <linux/pm.h>
24 #include <linux/sched.h>
25 #include <linux/suspend.h>
26 #include <linux/utsname.h>
27
28 /* The logical cpu number we should resume on, initialised to a non-cpu number. */
29 static int sleep_cpu = -EINVAL;
30
31 /* Pointer to the temporary resume page table. */
32 static pgd_t *resume_pg_dir;
33
34 /* CPU context to be saved. */
35 struct suspend_context *hibernate_cpu_context;
36 EXPORT_SYMBOL_GPL(hibernate_cpu_context);
37
38 unsigned long relocated_restore_code;
39 EXPORT_SYMBOL_GPL(relocated_restore_code);
40
41 /**
42 * struct arch_hibernate_hdr_invariants - container to store kernel build version.
43 * @uts_version: to save the build number and date so that we do not resume with
44 * a different kernel.
45 */
46 struct arch_hibernate_hdr_invariants {
47 char uts_version[__NEW_UTS_LEN + 1];
48 };
49
50 /**
51 * struct arch_hibernate_hdr - helper parameters that help us to restore the image.
52 * @invariants: container to store kernel build version.
53 * @hartid: to make sure same boot_cpu executes the hibernate/restore code.
54 * @saved_satp: original page table used by the hibernated image.
55 * @restore_cpu_addr: the kernel's image address to restore the CPU context.
56 */
57 static struct arch_hibernate_hdr {
58 struct arch_hibernate_hdr_invariants invariants;
59 unsigned long hartid;
60 unsigned long saved_satp;
61 unsigned long restore_cpu_addr;
62 } resume_hdr;
63
arch_hdr_invariants(struct arch_hibernate_hdr_invariants * i)64 static void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
65 {
66 memset(i, 0, sizeof(*i));
67 memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
68 }
69
70 /*
71 * Check if the given pfn is in the 'nosave' section.
72 */
pfn_is_nosave(unsigned long pfn)73 int pfn_is_nosave(unsigned long pfn)
74 {
75 unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
76 unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
77
78 return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn));
79 }
80
save_processor_state(void)81 void notrace save_processor_state(void)
82 {
83 }
84
restore_processor_state(void)85 void notrace restore_processor_state(void)
86 {
87 }
88
89 /*
90 * Helper parameters need to be saved to the hibernation image header.
91 */
arch_hibernation_header_save(void * addr,unsigned int max_size)92 int arch_hibernation_header_save(void *addr, unsigned int max_size)
93 {
94 struct arch_hibernate_hdr *hdr = addr;
95
96 if (max_size < sizeof(*hdr))
97 return -EOVERFLOW;
98
99 arch_hdr_invariants(&hdr->invariants);
100
101 hdr->hartid = cpuid_to_hartid_map(sleep_cpu);
102 hdr->saved_satp = csr_read(CSR_SATP);
103 hdr->restore_cpu_addr = (unsigned long)__hibernate_cpu_resume;
104
105 return 0;
106 }
107 EXPORT_SYMBOL_GPL(arch_hibernation_header_save);
108
109 /*
110 * Retrieve the helper parameters from the hibernation image header.
111 */
arch_hibernation_header_restore(void * addr)112 int arch_hibernation_header_restore(void *addr)
113 {
114 struct arch_hibernate_hdr_invariants invariants;
115 struct arch_hibernate_hdr *hdr = addr;
116 int ret = 0;
117
118 arch_hdr_invariants(&invariants);
119
120 if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
121 pr_crit("Hibernate image not generated by this kernel!\n");
122 return -EINVAL;
123 }
124
125 sleep_cpu = riscv_hartid_to_cpuid(hdr->hartid);
126 if (sleep_cpu < 0) {
127 pr_crit("Hibernated on a CPU not known to this kernel!\n");
128 sleep_cpu = -EINVAL;
129 return -EINVAL;
130 }
131
132 #ifdef CONFIG_SMP
133 ret = bringup_hibernate_cpu(sleep_cpu);
134 if (ret) {
135 sleep_cpu = -EINVAL;
136 return ret;
137 }
138 #endif
139 resume_hdr = *hdr;
140
141 return ret;
142 }
143 EXPORT_SYMBOL_GPL(arch_hibernation_header_restore);
144
swsusp_arch_suspend(void)145 int swsusp_arch_suspend(void)
146 {
147 int ret = 0;
148
149 if (__cpu_suspend_enter(hibernate_cpu_context)) {
150 sleep_cpu = smp_processor_id();
151 suspend_save_csrs(hibernate_cpu_context);
152 ret = swsusp_save();
153 } else {
154 suspend_restore_csrs(hibernate_cpu_context);
155 flush_tlb_all();
156 flush_icache_all();
157
158 /*
159 * Tell the hibernation core that we've just restored the memory.
160 */
161 in_suspend = 0;
162 sleep_cpu = -EINVAL;
163 }
164
165 return ret;
166 }
167
temp_pgtable_map_pte(pmd_t * dst_pmdp,pmd_t * src_pmdp,unsigned long start,unsigned long end,pgprot_t prot)168 static int temp_pgtable_map_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start,
169 unsigned long end, pgprot_t prot)
170 {
171 pte_t *src_ptep;
172 pte_t *dst_ptep;
173
174 if (pmd_none(READ_ONCE(*dst_pmdp))) {
175 dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC);
176 if (!dst_ptep)
177 return -ENOMEM;
178
179 pmd_populate_kernel(NULL, dst_pmdp, dst_ptep);
180 }
181
182 dst_ptep = pte_offset_kernel(dst_pmdp, start);
183 src_ptep = pte_offset_kernel(src_pmdp, start);
184
185 do {
186 pte_t pte = READ_ONCE(*src_ptep);
187
188 if (pte_present(pte))
189 set_pte(dst_ptep, __pte(pte_val(pte) | pgprot_val(prot)));
190 } while (dst_ptep++, src_ptep++, start += PAGE_SIZE, start < end);
191
192 return 0;
193 }
194
temp_pgtable_map_pmd(pud_t * dst_pudp,pud_t * src_pudp,unsigned long start,unsigned long end,pgprot_t prot)195 static int temp_pgtable_map_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
196 unsigned long end, pgprot_t prot)
197 {
198 unsigned long next;
199 unsigned long ret;
200 pmd_t *src_pmdp;
201 pmd_t *dst_pmdp;
202
203 if (pud_none(READ_ONCE(*dst_pudp))) {
204 dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC);
205 if (!dst_pmdp)
206 return -ENOMEM;
207
208 pud_populate(NULL, dst_pudp, dst_pmdp);
209 }
210
211 dst_pmdp = pmd_offset(dst_pudp, start);
212 src_pmdp = pmd_offset(src_pudp, start);
213
214 do {
215 pmd_t pmd = READ_ONCE(*src_pmdp);
216
217 next = pmd_addr_end(start, end);
218
219 if (pmd_none(pmd))
220 continue;
221
222 if (pmd_leaf(pmd)) {
223 set_pmd(dst_pmdp, __pmd(pmd_val(pmd) | pgprot_val(prot)));
224 } else {
225 ret = temp_pgtable_map_pte(dst_pmdp, src_pmdp, start, next, prot);
226 if (ret)
227 return -ENOMEM;
228 }
229 } while (dst_pmdp++, src_pmdp++, start = next, start != end);
230
231 return 0;
232 }
233
temp_pgtable_map_pud(p4d_t * dst_p4dp,p4d_t * src_p4dp,unsigned long start,unsigned long end,pgprot_t prot)234 static int temp_pgtable_map_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start,
235 unsigned long end, pgprot_t prot)
236 {
237 unsigned long next;
238 unsigned long ret;
239 pud_t *dst_pudp;
240 pud_t *src_pudp;
241
242 if (p4d_none(READ_ONCE(*dst_p4dp))) {
243 dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
244 if (!dst_pudp)
245 return -ENOMEM;
246
247 p4d_populate(NULL, dst_p4dp, dst_pudp);
248 }
249
250 dst_pudp = pud_offset(dst_p4dp, start);
251 src_pudp = pud_offset(src_p4dp, start);
252
253 do {
254 pud_t pud = READ_ONCE(*src_pudp);
255
256 next = pud_addr_end(start, end);
257
258 if (pud_none(pud))
259 continue;
260
261 if (pud_leaf(pud)) {
262 set_pud(dst_pudp, __pud(pud_val(pud) | pgprot_val(prot)));
263 } else {
264 ret = temp_pgtable_map_pmd(dst_pudp, src_pudp, start, next, prot);
265 if (ret)
266 return -ENOMEM;
267 }
268 } while (dst_pudp++, src_pudp++, start = next, start != end);
269
270 return 0;
271 }
272
temp_pgtable_map_p4d(pgd_t * dst_pgdp,pgd_t * src_pgdp,unsigned long start,unsigned long end,pgprot_t prot)273 static int temp_pgtable_map_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
274 unsigned long end, pgprot_t prot)
275 {
276 unsigned long next;
277 unsigned long ret;
278 p4d_t *dst_p4dp;
279 p4d_t *src_p4dp;
280
281 if (pgd_none(READ_ONCE(*dst_pgdp))) {
282 dst_p4dp = (p4d_t *)get_safe_page(GFP_ATOMIC);
283 if (!dst_p4dp)
284 return -ENOMEM;
285
286 pgd_populate(NULL, dst_pgdp, dst_p4dp);
287 }
288
289 dst_p4dp = p4d_offset(dst_pgdp, start);
290 src_p4dp = p4d_offset(src_pgdp, start);
291
292 do {
293 p4d_t p4d = READ_ONCE(*src_p4dp);
294
295 next = p4d_addr_end(start, end);
296
297 if (p4d_none(p4d))
298 continue;
299
300 if (p4d_leaf(p4d)) {
301 set_p4d(dst_p4dp, __p4d(p4d_val(p4d) | pgprot_val(prot)));
302 } else {
303 ret = temp_pgtable_map_pud(dst_p4dp, src_p4dp, start, next, prot);
304 if (ret)
305 return -ENOMEM;
306 }
307 } while (dst_p4dp++, src_p4dp++, start = next, start != end);
308
309 return 0;
310 }
311
temp_pgtable_mapping(pgd_t * pgdp,unsigned long start,unsigned long end,pgprot_t prot)312 static int temp_pgtable_mapping(pgd_t *pgdp, unsigned long start, unsigned long end, pgprot_t prot)
313 {
314 pgd_t *dst_pgdp = pgd_offset_pgd(pgdp, start);
315 pgd_t *src_pgdp = pgd_offset_k(start);
316 unsigned long next;
317 unsigned long ret;
318
319 do {
320 pgd_t pgd = READ_ONCE(*src_pgdp);
321
322 next = pgd_addr_end(start, end);
323
324 if (pgd_none(pgd))
325 continue;
326
327 if (pgd_leaf(pgd)) {
328 set_pgd(dst_pgdp, __pgd(pgd_val(pgd) | pgprot_val(prot)));
329 } else {
330 ret = temp_pgtable_map_p4d(dst_pgdp, src_pgdp, start, next, prot);
331 if (ret)
332 return -ENOMEM;
333 }
334 } while (dst_pgdp++, src_pgdp++, start = next, start != end);
335
336 return 0;
337 }
338
relocate_restore_code(void)339 static unsigned long relocate_restore_code(void)
340 {
341 void *page = (void *)get_safe_page(GFP_ATOMIC);
342
343 if (!page)
344 return -ENOMEM;
345
346 copy_page(page, hibernate_core_restore_code);
347
348 /* Make the page containing the relocated code executable. */
349 set_memory_x((unsigned long)page, 1);
350
351 return (unsigned long)page;
352 }
353
swsusp_arch_resume(void)354 int swsusp_arch_resume(void)
355 {
356 unsigned long end = (unsigned long)pfn_to_virt(max_low_pfn);
357 unsigned long start = PAGE_OFFSET;
358 int ret;
359
360 /*
361 * Memory allocated by get_safe_page() will be dealt with by the hibernation core,
362 * we don't need to free it here.
363 */
364 resume_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
365 if (!resume_pg_dir)
366 return -ENOMEM;
367
368 /*
369 * Create a temporary page table and map the whole linear region as executable and
370 * writable.
371 */
372 ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE | _PAGE_EXEC));
373 if (ret)
374 return ret;
375
376 /* Move the restore code to a new page so that it doesn't get overwritten by itself. */
377 relocated_restore_code = relocate_restore_code();
378 if (relocated_restore_code == -ENOMEM)
379 return -ENOMEM;
380
381 /*
382 * Map the __hibernate_cpu_resume() address to the temporary page table so that the
383 * restore code can jumps to it after finished restore the image. The next execution
384 * code doesn't find itself in a different address space after switching over to the
385 * original page table used by the hibernated image.
386 * The __hibernate_cpu_resume() mapping is unnecessary for RV32 since the kernel and
387 * linear addresses are identical, but different for RV64. To ensure consistency, we
388 * map it for both RV32 and RV64 kernels.
389 * Additionally, we should ensure that the page is writable before restoring the image.
390 */
391 start = (unsigned long)resume_hdr.restore_cpu_addr;
392 end = start + PAGE_SIZE;
393
394 ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE));
395 if (ret)
396 return ret;
397
398 hibernate_restore_image(resume_hdr.saved_satp, (PFN_DOWN(__pa(resume_pg_dir)) | satp_mode),
399 resume_hdr.restore_cpu_addr);
400
401 return 0;
402 }
403
404 #ifdef CONFIG_PM_SLEEP_SMP
hibernate_resume_nonboot_cpu_disable(void)405 int hibernate_resume_nonboot_cpu_disable(void)
406 {
407 if (sleep_cpu < 0) {
408 pr_err("Failing to resume from hibernate on an unknown CPU\n");
409 return -ENODEV;
410 }
411
412 return freeze_secondary_cpus(sleep_cpu);
413 }
414 #endif
415
riscv_hibernate_init(void)416 static int __init riscv_hibernate_init(void)
417 {
418 hibernate_cpu_context = kzalloc(sizeof(*hibernate_cpu_context), GFP_KERNEL);
419
420 if (WARN_ON(!hibernate_cpu_context))
421 return -ENOMEM;
422
423 return 0;
424 }
425
426 early_initcall(riscv_hibernate_init);
427