1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * kexec for arm64
4  *
5  * Copyright (C) Linaro.
6  * Copyright (C) Huawei Futurewei Technologies.
7  */
8 
9 #include <linux/interrupt.h>
10 #include <linux/irq.h>
11 #include <linux/kernel.h>
12 #include <linux/kexec.h>
13 #include <linux/page-flags.h>
14 #include <linux/smp.h>
15 
16 #include <asm/cacheflush.h>
17 #include <asm/cpu_ops.h>
18 #include <asm/daifflags.h>
19 #include <asm/memory.h>
20 #include <asm/mmu.h>
21 #include <asm/mmu_context.h>
22 #include <asm/page.h>
23 
24 #include "cpu-reset.h"
25 
26 /* Global variables for the arm64_relocate_new_kernel routine. */
27 extern const unsigned char arm64_relocate_new_kernel[];
28 extern const unsigned long arm64_relocate_new_kernel_size;
29 
30 /**
31  * kexec_image_info - For debugging output.
32  */
33 #define kexec_image_info(_i) _kexec_image_info(__func__, __LINE__, _i)
34 static void _kexec_image_info(const char *func, int line,
35 	const struct kimage *kimage)
36 {
37 	unsigned long i;
38 
39 	pr_debug("%s:%d:\n", func, line);
40 	pr_debug("  kexec kimage info:\n");
41 	pr_debug("    type:        %d\n", kimage->type);
42 	pr_debug("    start:       %lx\n", kimage->start);
43 	pr_debug("    head:        %lx\n", kimage->head);
44 	pr_debug("    nr_segments: %lu\n", kimage->nr_segments);
45 
46 	for (i = 0; i < kimage->nr_segments; i++) {
47 		pr_debug("      segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n",
48 			i,
49 			kimage->segment[i].mem,
50 			kimage->segment[i].mem + kimage->segment[i].memsz,
51 			kimage->segment[i].memsz,
52 			kimage->segment[i].memsz /  PAGE_SIZE);
53 	}
54 }
55 
56 void machine_kexec_cleanup(struct kimage *kimage)
57 {
58 	/* Empty routine needed to avoid build errors. */
59 }
60 
61 /**
62  * machine_kexec_prepare - Prepare for a kexec reboot.
63  *
64  * Called from the core kexec code when a kernel image is loaded.
65  * Forbid loading a kexec kernel if we have no way of hotplugging cpus or cpus
66  * are stuck in the kernel. This avoids a panic once we hit machine_kexec().
67  */
68 int machine_kexec_prepare(struct kimage *kimage)
69 {
70 	kexec_image_info(kimage);
71 
72 	if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) {
73 		pr_err("Can't kexec: CPUs are stuck in the kernel.\n");
74 		return -EBUSY;
75 	}
76 
77 	return 0;
78 }
79 
80 /**
81  * kexec_list_flush - Helper to flush the kimage list and source pages to PoC.
82  */
83 static void kexec_list_flush(struct kimage *kimage)
84 {
85 	kimage_entry_t *entry;
86 
87 	for (entry = &kimage->head; ; entry++) {
88 		unsigned int flag;
89 		void *addr;
90 
91 		/* flush the list entries. */
92 		__flush_dcache_area(entry, sizeof(kimage_entry_t));
93 
94 		flag = *entry & IND_FLAGS;
95 		if (flag == IND_DONE)
96 			break;
97 
98 		addr = phys_to_virt(*entry & PAGE_MASK);
99 
100 		switch (flag) {
101 		case IND_INDIRECTION:
102 			/* Set entry point just before the new list page. */
103 			entry = (kimage_entry_t *)addr - 1;
104 			break;
105 		case IND_SOURCE:
106 			/* flush the source pages. */
107 			__flush_dcache_area(addr, PAGE_SIZE);
108 			break;
109 		case IND_DESTINATION:
110 			break;
111 		default:
112 			BUG();
113 		}
114 	}
115 }
116 
117 /**
118  * kexec_segment_flush - Helper to flush the kimage segments to PoC.
119  */
120 static void kexec_segment_flush(const struct kimage *kimage)
121 {
122 	unsigned long i;
123 
124 	pr_debug("%s:\n", __func__);
125 
126 	for (i = 0; i < kimage->nr_segments; i++) {
127 		pr_debug("  segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n",
128 			i,
129 			kimage->segment[i].mem,
130 			kimage->segment[i].mem + kimage->segment[i].memsz,
131 			kimage->segment[i].memsz,
132 			kimage->segment[i].memsz /  PAGE_SIZE);
133 
134 		__flush_dcache_area(phys_to_virt(kimage->segment[i].mem),
135 			kimage->segment[i].memsz);
136 	}
137 }
138 
139 /**
140  * machine_kexec - Do the kexec reboot.
141  *
142  * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC.
143  */
144 void machine_kexec(struct kimage *kimage)
145 {
146 	phys_addr_t reboot_code_buffer_phys;
147 	void *reboot_code_buffer;
148 	bool in_kexec_crash = (kimage == kexec_crash_image);
149 	bool stuck_cpus = cpus_are_stuck_in_kernel();
150 
151 	/*
152 	 * New cpus may have become stuck_in_kernel after we loaded the image.
153 	 */
154 	BUG_ON(!in_kexec_crash && (stuck_cpus || (num_online_cpus() > 1)));
155 	WARN(in_kexec_crash && (stuck_cpus || smp_crash_stop_failed()),
156 		"Some CPUs may be stale, kdump will be unreliable.\n");
157 
158 	reboot_code_buffer_phys = page_to_phys(kimage->control_code_page);
159 	reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys);
160 
161 	kexec_image_info(kimage);
162 
163 	/*
164 	 * Copy arm64_relocate_new_kernel to the reboot_code_buffer for use
165 	 * after the kernel is shut down.
166 	 */
167 	memcpy(reboot_code_buffer, arm64_relocate_new_kernel,
168 		arm64_relocate_new_kernel_size);
169 
170 	/* Flush the reboot_code_buffer in preparation for its execution. */
171 	__flush_dcache_area(reboot_code_buffer, arm64_relocate_new_kernel_size);
172 
173 	/*
174 	 * Although we've killed off the secondary CPUs, we don't update
175 	 * the online mask if we're handling a crash kernel and consequently
176 	 * need to avoid flush_icache_range(), which will attempt to IPI
177 	 * the offline CPUs. Therefore, we must use the __* variant here.
178 	 */
179 	__flush_icache_range((uintptr_t)reboot_code_buffer,
180 			     arm64_relocate_new_kernel_size);
181 
182 	/* Flush the kimage list and its buffers. */
183 	kexec_list_flush(kimage);
184 
185 	/* Flush the new image if already in place. */
186 	if ((kimage != kexec_crash_image) && (kimage->head & IND_DONE))
187 		kexec_segment_flush(kimage);
188 
189 	pr_info("Bye!\n");
190 
191 	local_daif_mask();
192 
193 	/*
194 	 * cpu_soft_restart will shutdown the MMU, disable data caches, then
195 	 * transfer control to the reboot_code_buffer which contains a copy of
196 	 * the arm64_relocate_new_kernel routine.  arm64_relocate_new_kernel
197 	 * uses physical addressing to relocate the new image to its final
198 	 * position and transfers control to the image entry point when the
199 	 * relocation is complete.
200 	 * In kexec case, kimage->start points to purgatory assuming that
201 	 * kernel entry and dtb address are embedded in purgatory by
202 	 * userspace (kexec-tools).
203 	 * In kexec_file case, the kernel starts directly without purgatory.
204 	 */
205 	cpu_soft_restart(reboot_code_buffer_phys, kimage->head, kimage->start,
206 #ifdef CONFIG_KEXEC_FILE
207 						kimage->arch.dtb_mem);
208 #else
209 						0);
210 #endif
211 
212 	BUG(); /* Should never get here. */
213 }
214 
215 static void machine_kexec_mask_interrupts(void)
216 {
217 	unsigned int i;
218 	struct irq_desc *desc;
219 
220 	for_each_irq_desc(i, desc) {
221 		struct irq_chip *chip;
222 		int ret;
223 
224 		chip = irq_desc_get_chip(desc);
225 		if (!chip)
226 			continue;
227 
228 		/*
229 		 * First try to remove the active state. If this
230 		 * fails, try to EOI the interrupt.
231 		 */
232 		ret = irq_set_irqchip_state(i, IRQCHIP_STATE_ACTIVE, false);
233 
234 		if (ret && irqd_irq_inprogress(&desc->irq_data) &&
235 		    chip->irq_eoi)
236 			chip->irq_eoi(&desc->irq_data);
237 
238 		if (chip->irq_mask)
239 			chip->irq_mask(&desc->irq_data);
240 
241 		if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data))
242 			chip->irq_disable(&desc->irq_data);
243 	}
244 }
245 
246 /**
247  * machine_crash_shutdown - shutdown non-crashing cpus and save registers
248  */
249 void machine_crash_shutdown(struct pt_regs *regs)
250 {
251 	local_irq_disable();
252 
253 	/* shutdown non-crashing cpus */
254 	crash_smp_send_stop();
255 
256 	/* for crashing cpu */
257 	crash_save_cpu(regs, smp_processor_id());
258 	machine_kexec_mask_interrupts();
259 
260 	pr_info("Starting crashdump kernel...\n");
261 }
262 
263 void arch_kexec_protect_crashkres(void)
264 {
265 	int i;
266 
267 	kexec_segment_flush(kexec_crash_image);
268 
269 	for (i = 0; i < kexec_crash_image->nr_segments; i++)
270 		set_memory_valid(
271 			__phys_to_virt(kexec_crash_image->segment[i].mem),
272 			kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 0);
273 }
274 
275 void arch_kexec_unprotect_crashkres(void)
276 {
277 	int i;
278 
279 	for (i = 0; i < kexec_crash_image->nr_segments; i++)
280 		set_memory_valid(
281 			__phys_to_virt(kexec_crash_image->segment[i].mem),
282 			kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 1);
283 }
284 
285 #ifdef CONFIG_HIBERNATION
286 /*
287  * To preserve the crash dump kernel image, the relevant memory segments
288  * should be mapped again around the hibernation.
289  */
290 void crash_prepare_suspend(void)
291 {
292 	if (kexec_crash_image)
293 		arch_kexec_unprotect_crashkres();
294 }
295 
296 void crash_post_resume(void)
297 {
298 	if (kexec_crash_image)
299 		arch_kexec_protect_crashkres();
300 }
301 
302 /*
303  * crash_is_nosave
304  *
305  * Return true only if a page is part of reserved memory for crash dump kernel,
306  * but does not hold any data of loaded kernel image.
307  *
308  * Note that all the pages in crash dump kernel memory have been initially
309  * marked as Reserved as memory was allocated via memblock_reserve().
310  *
311  * In hibernation, the pages which are Reserved and yet "nosave" are excluded
312  * from the hibernation iamge. crash_is_nosave() does thich check for crash
313  * dump kernel and will reduce the total size of hibernation image.
314  */
315 
316 bool crash_is_nosave(unsigned long pfn)
317 {
318 	int i;
319 	phys_addr_t addr;
320 
321 	if (!crashk_res.end)
322 		return false;
323 
324 	/* in reserved memory? */
325 	addr = __pfn_to_phys(pfn);
326 	if ((addr < crashk_res.start) || (crashk_res.end < addr))
327 		return false;
328 
329 	if (!kexec_crash_image)
330 		return true;
331 
332 	/* not part of loaded kernel image? */
333 	for (i = 0; i < kexec_crash_image->nr_segments; i++)
334 		if (addr >= kexec_crash_image->segment[i].mem &&
335 				addr < (kexec_crash_image->segment[i].mem +
336 					kexec_crash_image->segment[i].memsz))
337 			return false;
338 
339 	return true;
340 }
341 
342 void crash_free_reserved_phys_range(unsigned long begin, unsigned long end)
343 {
344 	unsigned long addr;
345 	struct page *page;
346 
347 	for (addr = begin; addr < end; addr += PAGE_SIZE) {
348 		page = phys_to_page(addr);
349 		free_reserved_page(page);
350 	}
351 }
352 #endif /* CONFIG_HIBERNATION */
353