1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * PPC64 code to handle Linux booting another kernel.
4 *
5 * Copyright (C) 2004-2005, IBM Corp.
6 *
7 * Created by: Milton D Miller II
8 */
9
10
11 #include <linux/kexec.h>
12 #include <linux/smp.h>
13 #include <linux/thread_info.h>
14 #include <linux/init_task.h>
15 #include <linux/errno.h>
16 #include <linux/kernel.h>
17 #include <linux/cpu.h>
18 #include <linux/hardirq.h>
19 #include <linux/of.h>
20
21 #include <asm/page.h>
22 #include <asm/current.h>
23 #include <asm/machdep.h>
24 #include <asm/cacheflush.h>
25 #include <asm/firmware.h>
26 #include <asm/paca.h>
27 #include <asm/mmu.h>
28 #include <asm/sections.h> /* _end */
29 #include <asm/setup.h>
30 #include <asm/smp.h>
31 #include <asm/hw_breakpoint.h>
32 #include <asm/svm.h>
33 #include <asm/ultravisor.h>
34
machine_kexec_prepare(struct kimage * image)35 int machine_kexec_prepare(struct kimage *image)
36 {
37 int i;
38 unsigned long begin, end; /* limits of segment */
39 unsigned long low, high; /* limits of blocked memory range */
40 struct device_node *node;
41 const unsigned long *basep;
42 const unsigned int *sizep;
43
44 /*
45 * Since we use the kernel fault handlers and paging code to
46 * handle the virtual mode, we must make sure no destination
47 * overlaps kernel static data or bss.
48 */
49 for (i = 0; i < image->nr_segments; i++)
50 if (image->segment[i].mem < __pa(_end))
51 return -ETXTBSY;
52
53 /* We also should not overwrite the tce tables */
54 for_each_node_by_type(node, "pci") {
55 basep = of_get_property(node, "linux,tce-base", NULL);
56 sizep = of_get_property(node, "linux,tce-size", NULL);
57 if (basep == NULL || sizep == NULL)
58 continue;
59
60 low = *basep;
61 high = low + (*sizep);
62
63 for (i = 0; i < image->nr_segments; i++) {
64 begin = image->segment[i].mem;
65 end = begin + image->segment[i].memsz;
66
67 if ((begin < high) && (end > low)) {
68 of_node_put(node);
69 return -ETXTBSY;
70 }
71 }
72 }
73
74 return 0;
75 }
76
77 /* Called during kexec sequence with MMU off */
copy_segments(unsigned long ind)78 static notrace void copy_segments(unsigned long ind)
79 {
80 unsigned long entry;
81 unsigned long *ptr;
82 void *dest;
83 void *addr;
84
85 /*
86 * We rely on kexec_load to create a lists that properly
87 * initializes these pointers before they are used.
88 * We will still crash if the list is wrong, but at least
89 * the compiler will be quiet.
90 */
91 ptr = NULL;
92 dest = NULL;
93
94 for (entry = ind; !(entry & IND_DONE); entry = *ptr++) {
95 addr = __va(entry & PAGE_MASK);
96
97 switch (entry & IND_FLAGS) {
98 case IND_DESTINATION:
99 dest = addr;
100 break;
101 case IND_INDIRECTION:
102 ptr = addr;
103 break;
104 case IND_SOURCE:
105 copy_page(dest, addr);
106 dest += PAGE_SIZE;
107 }
108 }
109 }
110
111 /* Called during kexec sequence with MMU off */
kexec_copy_flush(struct kimage * image)112 notrace void kexec_copy_flush(struct kimage *image)
113 {
114 long i, nr_segments = image->nr_segments;
115 struct kexec_segment ranges[KEXEC_SEGMENT_MAX];
116
117 /* save the ranges on the stack to efficiently flush the icache */
118 memcpy(ranges, image->segment, sizeof(ranges));
119
120 /*
121 * After this call we may not use anything allocated in dynamic
122 * memory, including *image.
123 *
124 * Only globals and the stack are allowed.
125 */
126 copy_segments(image->head);
127
128 /*
129 * we need to clear the icache for all dest pages sometime,
130 * including ones that were in place on the original copy
131 */
132 for (i = 0; i < nr_segments; i++)
133 flush_icache_range((unsigned long)__va(ranges[i].mem),
134 (unsigned long)__va(ranges[i].mem + ranges[i].memsz));
135 }
136
137 #ifdef CONFIG_SMP
138
139 static int kexec_all_irq_disabled = 0;
140
kexec_smp_down(void * arg)141 static void kexec_smp_down(void *arg)
142 {
143 local_irq_disable();
144 hard_irq_disable();
145
146 mb(); /* make sure our irqs are disabled before we say they are */
147 get_paca()->kexec_state = KEXEC_STATE_IRQS_OFF;
148 while(kexec_all_irq_disabled == 0)
149 cpu_relax();
150 mb(); /* make sure all irqs are disabled before this */
151 hw_breakpoint_disable();
152 /*
153 * Now every CPU has IRQs off, we can clear out any pending
154 * IPIs and be sure that no more will come in after this.
155 */
156 if (ppc_md.kexec_cpu_down)
157 ppc_md.kexec_cpu_down(0, 1);
158
159 reset_sprs();
160
161 kexec_smp_wait();
162 /* NOTREACHED */
163 }
164
kexec_prepare_cpus_wait(int wait_state)165 static void kexec_prepare_cpus_wait(int wait_state)
166 {
167 int my_cpu, i, notified=-1;
168
169 hw_breakpoint_disable();
170 my_cpu = get_cpu();
171 /* Make sure each CPU has at least made it to the state we need.
172 *
173 * FIXME: There is a (slim) chance of a problem if not all of the CPUs
174 * are correctly onlined. If somehow we start a CPU on boot with RTAS
175 * start-cpu, but somehow that CPU doesn't write callin_cpu_map[] in
176 * time, the boot CPU will timeout. If it does eventually execute
177 * stuff, the secondary will start up (paca_ptrs[]->cpu_start was
178 * written) and get into a peculiar state.
179 * If the platform supports smp_ops->take_timebase(), the secondary CPU
180 * will probably be spinning in there. If not (i.e. pseries), the
181 * secondary will continue on and try to online itself/idle/etc. If it
182 * survives that, we need to find these
183 * possible-but-not-online-but-should-be CPUs and chaperone them into
184 * kexec_smp_wait().
185 */
186 for_each_online_cpu(i) {
187 if (i == my_cpu)
188 continue;
189
190 while (paca_ptrs[i]->kexec_state < wait_state) {
191 barrier();
192 if (i != notified) {
193 printk(KERN_INFO "kexec: waiting for cpu %d "
194 "(physical %d) to enter %i state\n",
195 i, paca_ptrs[i]->hw_cpu_id, wait_state);
196 notified = i;
197 }
198 }
199 }
200 mb();
201 }
202
203 /*
204 * We need to make sure each present CPU is online. The next kernel will scan
205 * the device tree and assume primary threads are online and query secondary
206 * threads via RTAS to online them if required. If we don't online primary
207 * threads, they will be stuck. However, we also online secondary threads as we
208 * may be using 'cede offline'. In this case RTAS doesn't see the secondary
209 * threads as offline -- and again, these CPUs will be stuck.
210 *
211 * So, we online all CPUs that should be running, including secondary threads.
212 */
wake_offline_cpus(void)213 static void wake_offline_cpus(void)
214 {
215 int cpu = 0;
216
217 for_each_present_cpu(cpu) {
218 if (!cpu_online(cpu)) {
219 printk(KERN_INFO "kexec: Waking offline cpu %d.\n",
220 cpu);
221 WARN_ON(add_cpu(cpu));
222 }
223 }
224 }
225
kexec_prepare_cpus(void)226 static void kexec_prepare_cpus(void)
227 {
228 wake_offline_cpus();
229 smp_call_function(kexec_smp_down, NULL, /* wait */0);
230 local_irq_disable();
231 hard_irq_disable();
232
233 mb(); /* make sure IRQs are disabled before we say they are */
234 get_paca()->kexec_state = KEXEC_STATE_IRQS_OFF;
235
236 kexec_prepare_cpus_wait(KEXEC_STATE_IRQS_OFF);
237 /* we are sure every CPU has IRQs off at this point */
238 kexec_all_irq_disabled = 1;
239
240 /*
241 * Before removing MMU mappings make sure all CPUs have entered real
242 * mode:
243 */
244 kexec_prepare_cpus_wait(KEXEC_STATE_REAL_MODE);
245
246 /* after we tell the others to go down */
247 if (ppc_md.kexec_cpu_down)
248 ppc_md.kexec_cpu_down(0, 0);
249
250 put_cpu();
251 }
252
253 #else /* ! SMP */
254
kexec_prepare_cpus(void)255 static void kexec_prepare_cpus(void)
256 {
257 /*
258 * move the secondarys to us so that we can copy
259 * the new kernel 0-0x100 safely
260 *
261 * do this if kexec in setup.c ?
262 *
263 * We need to release the cpus if we are ever going from an
264 * UP to an SMP kernel.
265 */
266 smp_release_cpus();
267 if (ppc_md.kexec_cpu_down)
268 ppc_md.kexec_cpu_down(0, 0);
269 local_irq_disable();
270 hard_irq_disable();
271 }
272
273 #endif /* SMP */
274
275 /*
276 * kexec thread structure and stack.
277 *
278 * We need to make sure that this is 16384-byte aligned due to the
279 * way process stacks are handled. It also must be statically allocated
280 * or allocated as part of the kimage, because everything else may be
281 * overwritten when we copy the kexec image. We piggyback on the
282 * "init_task" linker section here to statically allocate a stack.
283 *
284 * We could use a smaller stack if we don't care about anything using
285 * current, but that audit has not been performed.
286 */
287 static union thread_union kexec_stack __init_task_data =
288 { };
289
290 /*
291 * For similar reasons to the stack above, the kexecing CPU needs to be on a
292 * static PACA; we switch to kexec_paca.
293 */
294 static struct paca_struct kexec_paca;
295
296 /* Our assembly helper, in misc_64.S */
297 extern void kexec_sequence(void *newstack, unsigned long start,
298 void *image, void *control,
299 void (*clear_all)(void),
300 bool copy_with_mmu_off) __noreturn;
301
302 /* too late to fail here */
default_machine_kexec(struct kimage * image)303 void default_machine_kexec(struct kimage *image)
304 {
305 bool copy_with_mmu_off;
306
307 /* prepare control code if any */
308
309 /*
310 * If the kexec boot is the normal one, need to shutdown other cpus
311 * into our wait loop and quiesce interrupts.
312 * Otherwise, in the case of crashed mode (crashing_cpu >= 0),
313 * stopping other CPUs and collecting their pt_regs is done before
314 * using debugger IPI.
315 */
316
317 if (!kdump_in_progress())
318 kexec_prepare_cpus();
319
320 #ifdef CONFIG_PPC_PSERIES
321 /*
322 * This must be done after other CPUs have shut down, otherwise they
323 * could execute the 'scv' instruction, which is not supported with
324 * reloc disabled (see configure_exceptions()).
325 */
326 if (firmware_has_feature(FW_FEATURE_SET_MODE))
327 pseries_disable_reloc_on_exc();
328 #endif
329
330 printk("kexec: Starting switchover sequence.\n");
331
332 /* switch to a staticly allocated stack. Based on irq stack code.
333 * We setup preempt_count to avoid using VMX in memcpy.
334 * XXX: the task struct will likely be invalid once we do the copy!
335 */
336 current_thread_info()->flags = 0;
337 current_thread_info()->preempt_count = HARDIRQ_OFFSET;
338
339 /* We need a static PACA, too; copy this CPU's PACA over and switch to
340 * it. Also poison per_cpu_offset and NULL lppaca to catch anyone using
341 * non-static data.
342 */
343 memcpy(&kexec_paca, get_paca(), sizeof(struct paca_struct));
344 kexec_paca.data_offset = 0xedeaddeadeeeeeeeUL;
345 #ifdef CONFIG_PPC_PSERIES
346 kexec_paca.lppaca_ptr = NULL;
347 #endif
348
349 if (is_secure_guest() && !(image->preserve_context ||
350 image->type == KEXEC_TYPE_CRASH)) {
351 uv_unshare_all_pages();
352 printk("kexec: Unshared all shared pages.\n");
353 }
354
355 paca_ptrs[kexec_paca.paca_index] = &kexec_paca;
356
357 setup_paca(&kexec_paca);
358
359 /*
360 * The lppaca should be unregistered at this point so the HV won't
361 * touch it. In the case of a crash, none of the lppacas are
362 * unregistered so there is not much we can do about it here.
363 */
364
365 /*
366 * On Book3S, the copy must happen with the MMU off if we are either
367 * using Radix page tables or we are not in an LPAR since we can
368 * overwrite the page tables while copying.
369 *
370 * In an LPAR, we keep the MMU on otherwise we can't access beyond
371 * the RMA. On BookE there is no real MMU off mode, so we have to
372 * keep it enabled as well (but then we have bolted TLB entries).
373 */
374 #ifdef CONFIG_PPC_BOOK3E_64
375 copy_with_mmu_off = false;
376 #else
377 copy_with_mmu_off = radix_enabled() ||
378 !(firmware_has_feature(FW_FEATURE_LPAR) ||
379 firmware_has_feature(FW_FEATURE_PS3_LV1));
380 #endif
381
382 /* Some things are best done in assembly. Finding globals with
383 * a toc is easier in C, so pass in what we can.
384 */
385 kexec_sequence(&kexec_stack, image->start, image,
386 page_address(image->control_code_page),
387 mmu_cleanup_all, copy_with_mmu_off);
388 /* NOTREACHED */
389 }
390
391 #ifdef CONFIG_PPC_64S_HASH_MMU
392 /* Values we need to export to the second kernel via the device tree. */
393 static unsigned long htab_base;
394 static unsigned long htab_size;
395
396 static struct property htab_base_prop = {
397 .name = "linux,htab-base",
398 .length = sizeof(unsigned long),
399 .value = &htab_base,
400 };
401
402 static struct property htab_size_prop = {
403 .name = "linux,htab-size",
404 .length = sizeof(unsigned long),
405 .value = &htab_size,
406 };
407
export_htab_values(void)408 static int __init export_htab_values(void)
409 {
410 struct device_node *node;
411
412 /* On machines with no htab htab_address is NULL */
413 if (!htab_address)
414 return -ENODEV;
415
416 node = of_find_node_by_path("/chosen");
417 if (!node)
418 return -ENODEV;
419
420 /* remove any stale properties so ours can be found */
421 of_remove_property(node, of_find_property(node, htab_base_prop.name, NULL));
422 of_remove_property(node, of_find_property(node, htab_size_prop.name, NULL));
423
424 htab_base = cpu_to_be64(__pa(htab_address));
425 of_add_property(node, &htab_base_prop);
426 htab_size = cpu_to_be64(htab_size_bytes);
427 of_add_property(node, &htab_size_prop);
428
429 of_node_put(node);
430 return 0;
431 }
432 late_initcall(export_htab_values);
433 #endif /* CONFIG_PPC_64S_HASH_MMU */
434