1 /*
2  * handle transition of Linux booting another kernel
3  * Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
4  *
5  * This source code is licensed under the GNU General Public License,
6  * Version 2.  See the file COPYING for more details.
7  */
8 
9 #define pr_fmt(fmt)	"kexec: " fmt
10 
11 #include <linux/mm.h>
12 #include <linux/kexec.h>
13 #include <linux/string.h>
14 #include <linux/gfp.h>
15 #include <linux/reboot.h>
16 #include <linux/numa.h>
17 #include <linux/ftrace.h>
18 #include <linux/io.h>
19 #include <linux/suspend.h>
20 #include <linux/vmalloc.h>
21 
22 #include <asm/init.h>
23 #include <asm/pgtable.h>
24 #include <asm/tlbflush.h>
25 #include <asm/mmu_context.h>
26 #include <asm/io_apic.h>
27 #include <asm/debugreg.h>
28 #include <asm/kexec-bzimage64.h>
29 #include <asm/setup.h>
30 
31 #ifdef CONFIG_KEXEC_FILE
32 static struct kexec_file_ops *kexec_file_loaders[] = {
33 		&kexec_bzImage64_ops,
34 };
35 #endif
36 
37 static void free_transition_pgtable(struct kimage *image)
38 {
39 	free_page((unsigned long)image->arch.p4d);
40 	free_page((unsigned long)image->arch.pud);
41 	free_page((unsigned long)image->arch.pmd);
42 	free_page((unsigned long)image->arch.pte);
43 }
44 
45 static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
46 {
47 	p4d_t *p4d;
48 	pud_t *pud;
49 	pmd_t *pmd;
50 	pte_t *pte;
51 	unsigned long vaddr, paddr;
52 	int result = -ENOMEM;
53 
54 	vaddr = (unsigned long)relocate_kernel;
55 	paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE);
56 	pgd += pgd_index(vaddr);
57 	if (!pgd_present(*pgd)) {
58 		p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
59 		if (!p4d)
60 			goto err;
61 		image->arch.p4d = p4d;
62 		set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
63 	}
64 	p4d = p4d_offset(pgd, vaddr);
65 	if (!p4d_present(*p4d)) {
66 		pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
67 		if (!pud)
68 			goto err;
69 		image->arch.pud = pud;
70 		set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
71 	}
72 	pud = pud_offset(p4d, vaddr);
73 	if (!pud_present(*pud)) {
74 		pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
75 		if (!pmd)
76 			goto err;
77 		image->arch.pmd = pmd;
78 		set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
79 	}
80 	pmd = pmd_offset(pud, vaddr);
81 	if (!pmd_present(*pmd)) {
82 		pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
83 		if (!pte)
84 			goto err;
85 		image->arch.pte = pte;
86 		set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
87 	}
88 	pte = pte_offset_kernel(pmd, vaddr);
89 	set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
90 	return 0;
91 err:
92 	free_transition_pgtable(image);
93 	return result;
94 }
95 
96 static void *alloc_pgt_page(void *data)
97 {
98 	struct kimage *image = (struct kimage *)data;
99 	struct page *page;
100 	void *p = NULL;
101 
102 	page = kimage_alloc_control_pages(image, 0);
103 	if (page) {
104 		p = page_address(page);
105 		clear_page(p);
106 	}
107 
108 	return p;
109 }
110 
111 static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
112 {
113 	struct x86_mapping_info info = {
114 		.alloc_pgt_page	= alloc_pgt_page,
115 		.context	= image,
116 		.pmd_flag	= __PAGE_KERNEL_LARGE_EXEC,
117 	};
118 	unsigned long mstart, mend;
119 	pgd_t *level4p;
120 	int result;
121 	int i;
122 
123 	level4p = (pgd_t *)__va(start_pgtable);
124 	clear_page(level4p);
125 	for (i = 0; i < nr_pfn_mapped; i++) {
126 		mstart = pfn_mapped[i].start << PAGE_SHIFT;
127 		mend   = pfn_mapped[i].end << PAGE_SHIFT;
128 
129 		result = kernel_ident_mapping_init(&info,
130 						 level4p, mstart, mend);
131 		if (result)
132 			return result;
133 	}
134 
135 	/*
136 	 * segments's mem ranges could be outside 0 ~ max_pfn,
137 	 * for example when jump back to original kernel from kexeced kernel.
138 	 * or first kernel is booted with user mem map, and second kernel
139 	 * could be loaded out of that range.
140 	 */
141 	for (i = 0; i < image->nr_segments; i++) {
142 		mstart = image->segment[i].mem;
143 		mend   = mstart + image->segment[i].memsz;
144 
145 		result = kernel_ident_mapping_init(&info,
146 						 level4p, mstart, mend);
147 
148 		if (result)
149 			return result;
150 	}
151 
152 	return init_transition_pgtable(image, level4p);
153 }
154 
155 static void set_idt(void *newidt, u16 limit)
156 {
157 	struct desc_ptr curidt;
158 
159 	/* x86-64 supports unaliged loads & stores */
160 	curidt.size    = limit;
161 	curidt.address = (unsigned long)newidt;
162 
163 	__asm__ __volatile__ (
164 		"lidtq %0\n"
165 		: : "m" (curidt)
166 		);
167 };
168 
169 
170 static void set_gdt(void *newgdt, u16 limit)
171 {
172 	struct desc_ptr curgdt;
173 
174 	/* x86-64 supports unaligned loads & stores */
175 	curgdt.size    = limit;
176 	curgdt.address = (unsigned long)newgdt;
177 
178 	__asm__ __volatile__ (
179 		"lgdtq %0\n"
180 		: : "m" (curgdt)
181 		);
182 };
183 
184 static void load_segments(void)
185 {
186 	__asm__ __volatile__ (
187 		"\tmovl %0,%%ds\n"
188 		"\tmovl %0,%%es\n"
189 		"\tmovl %0,%%ss\n"
190 		"\tmovl %0,%%fs\n"
191 		"\tmovl %0,%%gs\n"
192 		: : "a" (__KERNEL_DS) : "memory"
193 		);
194 }
195 
196 #ifdef CONFIG_KEXEC_FILE
197 /* Update purgatory as needed after various image segments have been prepared */
198 static int arch_update_purgatory(struct kimage *image)
199 {
200 	int ret = 0;
201 
202 	if (!image->file_mode)
203 		return 0;
204 
205 	/* Setup copying of backup region */
206 	if (image->type == KEXEC_TYPE_CRASH) {
207 		ret = kexec_purgatory_get_set_symbol(image,
208 				"purgatory_backup_dest",
209 				&image->arch.backup_load_addr,
210 				sizeof(image->arch.backup_load_addr), 0);
211 		if (ret)
212 			return ret;
213 
214 		ret = kexec_purgatory_get_set_symbol(image,
215 				"purgatory_backup_src",
216 				&image->arch.backup_src_start,
217 				sizeof(image->arch.backup_src_start), 0);
218 		if (ret)
219 			return ret;
220 
221 		ret = kexec_purgatory_get_set_symbol(image,
222 				"purgatory_backup_sz",
223 				&image->arch.backup_src_sz,
224 				sizeof(image->arch.backup_src_sz), 0);
225 		if (ret)
226 			return ret;
227 	}
228 
229 	return ret;
230 }
231 #else /* !CONFIG_KEXEC_FILE */
232 static inline int arch_update_purgatory(struct kimage *image)
233 {
234 	return 0;
235 }
236 #endif /* CONFIG_KEXEC_FILE */
237 
238 int machine_kexec_prepare(struct kimage *image)
239 {
240 	unsigned long start_pgtable;
241 	int result;
242 
243 	/* Calculate the offsets */
244 	start_pgtable = page_to_pfn(image->control_code_page) << PAGE_SHIFT;
245 
246 	/* Setup the identity mapped 64bit page table */
247 	result = init_pgtable(image, start_pgtable);
248 	if (result)
249 		return result;
250 
251 	/* update purgatory as needed */
252 	result = arch_update_purgatory(image);
253 	if (result)
254 		return result;
255 
256 	return 0;
257 }
258 
259 void machine_kexec_cleanup(struct kimage *image)
260 {
261 	free_transition_pgtable(image);
262 }
263 
264 /*
265  * Do not allocate memory (or fail in any way) in machine_kexec().
266  * We are past the point of no return, committed to rebooting now.
267  */
268 void machine_kexec(struct kimage *image)
269 {
270 	unsigned long page_list[PAGES_NR];
271 	void *control_page;
272 	int save_ftrace_enabled;
273 
274 #ifdef CONFIG_KEXEC_JUMP
275 	if (image->preserve_context)
276 		save_processor_state();
277 #endif
278 
279 	save_ftrace_enabled = __ftrace_enabled_save();
280 
281 	/* Interrupts aren't acceptable while we reboot */
282 	local_irq_disable();
283 	hw_breakpoint_disable();
284 
285 	if (image->preserve_context) {
286 #ifdef CONFIG_X86_IO_APIC
287 		/*
288 		 * We need to put APICs in legacy mode so that we can
289 		 * get timer interrupts in second kernel. kexec/kdump
290 		 * paths already have calls to disable_IO_APIC() in
291 		 * one form or other. kexec jump path also need
292 		 * one.
293 		 */
294 		disable_IO_APIC();
295 #endif
296 	}
297 
298 	control_page = page_address(image->control_code_page) + PAGE_SIZE;
299 	memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
300 
301 	page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
302 	page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
303 	page_list[PA_TABLE_PAGE] =
304 	  (unsigned long)__pa(page_address(image->control_code_page));
305 
306 	if (image->type == KEXEC_TYPE_DEFAULT)
307 		page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
308 						<< PAGE_SHIFT);
309 
310 	/*
311 	 * The segment registers are funny things, they have both a
312 	 * visible and an invisible part.  Whenever the visible part is
313 	 * set to a specific selector, the invisible part is loaded
314 	 * with from a table in memory.  At no other time is the
315 	 * descriptor table in memory accessed.
316 	 *
317 	 * I take advantage of this here by force loading the
318 	 * segments, before I zap the gdt with an invalid value.
319 	 */
320 	load_segments();
321 	/*
322 	 * The gdt & idt are now invalid.
323 	 * If you want to load them you must set up your own idt & gdt.
324 	 */
325 	set_gdt(phys_to_virt(0), 0);
326 	set_idt(phys_to_virt(0), 0);
327 
328 	/* now call it */
329 	image->start = relocate_kernel((unsigned long)image->head,
330 				       (unsigned long)page_list,
331 				       image->start,
332 				       image->preserve_context);
333 
334 #ifdef CONFIG_KEXEC_JUMP
335 	if (image->preserve_context)
336 		restore_processor_state();
337 #endif
338 
339 	__ftrace_enabled_restore(save_ftrace_enabled);
340 }
341 
342 void arch_crash_save_vmcoreinfo(void)
343 {
344 	VMCOREINFO_NUMBER(phys_base);
345 	VMCOREINFO_SYMBOL(init_level4_pgt);
346 
347 #ifdef CONFIG_NUMA
348 	VMCOREINFO_SYMBOL(node_data);
349 	VMCOREINFO_LENGTH(node_data, MAX_NUMNODES);
350 #endif
351 	vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
352 			      kaslr_offset());
353 	VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE);
354 }
355 
356 /* arch-dependent functionality related to kexec file-based syscall */
357 
358 #ifdef CONFIG_KEXEC_FILE
359 int arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
360 				  unsigned long buf_len)
361 {
362 	int i, ret = -ENOEXEC;
363 	struct kexec_file_ops *fops;
364 
365 	for (i = 0; i < ARRAY_SIZE(kexec_file_loaders); i++) {
366 		fops = kexec_file_loaders[i];
367 		if (!fops || !fops->probe)
368 			continue;
369 
370 		ret = fops->probe(buf, buf_len);
371 		if (!ret) {
372 			image->fops = fops;
373 			return ret;
374 		}
375 	}
376 
377 	return ret;
378 }
379 
380 void *arch_kexec_kernel_image_load(struct kimage *image)
381 {
382 	vfree(image->arch.elf_headers);
383 	image->arch.elf_headers = NULL;
384 
385 	if (!image->fops || !image->fops->load)
386 		return ERR_PTR(-ENOEXEC);
387 
388 	return image->fops->load(image, image->kernel_buf,
389 				 image->kernel_buf_len, image->initrd_buf,
390 				 image->initrd_buf_len, image->cmdline_buf,
391 				 image->cmdline_buf_len);
392 }
393 
394 int arch_kimage_file_post_load_cleanup(struct kimage *image)
395 {
396 	if (!image->fops || !image->fops->cleanup)
397 		return 0;
398 
399 	return image->fops->cleanup(image->image_loader_data);
400 }
401 
402 #ifdef CONFIG_KEXEC_VERIFY_SIG
403 int arch_kexec_kernel_verify_sig(struct kimage *image, void *kernel,
404 				 unsigned long kernel_len)
405 {
406 	if (!image->fops || !image->fops->verify_sig) {
407 		pr_debug("kernel loader does not support signature verification.");
408 		return -EKEYREJECTED;
409 	}
410 
411 	return image->fops->verify_sig(kernel, kernel_len);
412 }
413 #endif
414 
415 /*
416  * Apply purgatory relocations.
417  *
418  * ehdr: Pointer to elf headers
419  * sechdrs: Pointer to section headers.
420  * relsec: section index of SHT_RELA section.
421  *
422  * TODO: Some of the code belongs to generic code. Move that in kexec.c.
423  */
424 int arch_kexec_apply_relocations_add(const Elf64_Ehdr *ehdr,
425 				     Elf64_Shdr *sechdrs, unsigned int relsec)
426 {
427 	unsigned int i;
428 	Elf64_Rela *rel;
429 	Elf64_Sym *sym;
430 	void *location;
431 	Elf64_Shdr *section, *symtabsec;
432 	unsigned long address, sec_base, value;
433 	const char *strtab, *name, *shstrtab;
434 
435 	/*
436 	 * ->sh_offset has been modified to keep the pointer to section
437 	 * contents in memory
438 	 */
439 	rel = (void *)sechdrs[relsec].sh_offset;
440 
441 	/* Section to which relocations apply */
442 	section = &sechdrs[sechdrs[relsec].sh_info];
443 
444 	pr_debug("Applying relocate section %u to %u\n", relsec,
445 		 sechdrs[relsec].sh_info);
446 
447 	/* Associated symbol table */
448 	symtabsec = &sechdrs[sechdrs[relsec].sh_link];
449 
450 	/* String table */
451 	if (symtabsec->sh_link >= ehdr->e_shnum) {
452 		/* Invalid strtab section number */
453 		pr_err("Invalid string table section index %d\n",
454 		       symtabsec->sh_link);
455 		return -ENOEXEC;
456 	}
457 
458 	strtab = (char *)sechdrs[symtabsec->sh_link].sh_offset;
459 
460 	/* section header string table */
461 	shstrtab = (char *)sechdrs[ehdr->e_shstrndx].sh_offset;
462 
463 	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
464 
465 		/*
466 		 * rel[i].r_offset contains byte offset from beginning
467 		 * of section to the storage unit affected.
468 		 *
469 		 * This is location to update (->sh_offset). This is temporary
470 		 * buffer where section is currently loaded. This will finally
471 		 * be loaded to a different address later, pointed to by
472 		 * ->sh_addr. kexec takes care of moving it
473 		 *  (kexec_load_segment()).
474 		 */
475 		location = (void *)(section->sh_offset + rel[i].r_offset);
476 
477 		/* Final address of the location */
478 		address = section->sh_addr + rel[i].r_offset;
479 
480 		/*
481 		 * rel[i].r_info contains information about symbol table index
482 		 * w.r.t which relocation must be made and type of relocation
483 		 * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
484 		 * these respectively.
485 		 */
486 		sym = (Elf64_Sym *)symtabsec->sh_offset +
487 				ELF64_R_SYM(rel[i].r_info);
488 
489 		if (sym->st_name)
490 			name = strtab + sym->st_name;
491 		else
492 			name = shstrtab + sechdrs[sym->st_shndx].sh_name;
493 
494 		pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
495 			 name, sym->st_info, sym->st_shndx, sym->st_value,
496 			 sym->st_size);
497 
498 		if (sym->st_shndx == SHN_UNDEF) {
499 			pr_err("Undefined symbol: %s\n", name);
500 			return -ENOEXEC;
501 		}
502 
503 		if (sym->st_shndx == SHN_COMMON) {
504 			pr_err("symbol '%s' in common section\n", name);
505 			return -ENOEXEC;
506 		}
507 
508 		if (sym->st_shndx == SHN_ABS)
509 			sec_base = 0;
510 		else if (sym->st_shndx >= ehdr->e_shnum) {
511 			pr_err("Invalid section %d for symbol %s\n",
512 			       sym->st_shndx, name);
513 			return -ENOEXEC;
514 		} else
515 			sec_base = sechdrs[sym->st_shndx].sh_addr;
516 
517 		value = sym->st_value;
518 		value += sec_base;
519 		value += rel[i].r_addend;
520 
521 		switch (ELF64_R_TYPE(rel[i].r_info)) {
522 		case R_X86_64_NONE:
523 			break;
524 		case R_X86_64_64:
525 			*(u64 *)location = value;
526 			break;
527 		case R_X86_64_32:
528 			*(u32 *)location = value;
529 			if (value != *(u32 *)location)
530 				goto overflow;
531 			break;
532 		case R_X86_64_32S:
533 			*(s32 *)location = value;
534 			if ((s64)value != *(s32 *)location)
535 				goto overflow;
536 			break;
537 		case R_X86_64_PC32:
538 			value -= (u64)address;
539 			*(u32 *)location = value;
540 			break;
541 		default:
542 			pr_err("Unknown rela relocation: %llu\n",
543 			       ELF64_R_TYPE(rel[i].r_info));
544 			return -ENOEXEC;
545 		}
546 	}
547 	return 0;
548 
549 overflow:
550 	pr_err("Overflow in relocation type %d value 0x%lx\n",
551 	       (int)ELF64_R_TYPE(rel[i].r_info), value);
552 	return -ENOEXEC;
553 }
554 #endif /* CONFIG_KEXEC_FILE */
555 
556 static int
557 kexec_mark_range(unsigned long start, unsigned long end, bool protect)
558 {
559 	struct page *page;
560 	unsigned int nr_pages;
561 
562 	/*
563 	 * For physical range: [start, end]. We must skip the unassigned
564 	 * crashk resource with zero-valued "end" member.
565 	 */
566 	if (!end || start > end)
567 		return 0;
568 
569 	page = pfn_to_page(start >> PAGE_SHIFT);
570 	nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
571 	if (protect)
572 		return set_pages_ro(page, nr_pages);
573 	else
574 		return set_pages_rw(page, nr_pages);
575 }
576 
577 static void kexec_mark_crashkres(bool protect)
578 {
579 	unsigned long control;
580 
581 	kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect);
582 
583 	/* Don't touch the control code page used in crash_kexec().*/
584 	control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page));
585 	/* Control code page is located in the 2nd page. */
586 	kexec_mark_range(crashk_res.start, control + PAGE_SIZE - 1, protect);
587 	control += KEXEC_CONTROL_PAGE_SIZE;
588 	kexec_mark_range(control, crashk_res.end, protect);
589 }
590 
591 void arch_kexec_protect_crashkres(void)
592 {
593 	kexec_mark_crashkres(true);
594 }
595 
596 void arch_kexec_unprotect_crashkres(void)
597 {
598 	kexec_mark_crashkres(false);
599 }
600