1 /* 2 * kexec for arm64 3 * 4 * Copyright (C) Linaro. 5 * Copyright (C) Huawei Futurewei Technologies. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <linux/interrupt.h> 13 #include <linux/irq.h> 14 #include <linux/kernel.h> 15 #include <linux/kexec.h> 16 #include <linux/page-flags.h> 17 #include <linux/smp.h> 18 19 #include <asm/cacheflush.h> 20 #include <asm/cpu_ops.h> 21 #include <asm/daifflags.h> 22 #include <asm/memory.h> 23 #include <asm/mmu.h> 24 #include <asm/mmu_context.h> 25 #include <asm/page.h> 26 27 #include "cpu-reset.h" 28 29 /* Global variables for the arm64_relocate_new_kernel routine. */ 30 extern const unsigned char arm64_relocate_new_kernel[]; 31 extern const unsigned long arm64_relocate_new_kernel_size; 32 33 /** 34 * kexec_image_info - For debugging output. 35 */ 36 #define kexec_image_info(_i) _kexec_image_info(__func__, __LINE__, _i) 37 static void _kexec_image_info(const char *func, int line, 38 const struct kimage *kimage) 39 { 40 unsigned long i; 41 42 pr_debug("%s:%d:\n", func, line); 43 pr_debug(" kexec kimage info:\n"); 44 pr_debug(" type: %d\n", kimage->type); 45 pr_debug(" start: %lx\n", kimage->start); 46 pr_debug(" head: %lx\n", kimage->head); 47 pr_debug(" nr_segments: %lu\n", kimage->nr_segments); 48 49 for (i = 0; i < kimage->nr_segments; i++) { 50 pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n", 51 i, 52 kimage->segment[i].mem, 53 kimage->segment[i].mem + kimage->segment[i].memsz, 54 kimage->segment[i].memsz, 55 kimage->segment[i].memsz / PAGE_SIZE); 56 } 57 } 58 59 void machine_kexec_cleanup(struct kimage *kimage) 60 { 61 /* Empty routine needed to avoid build errors. */ 62 } 63 64 /** 65 * machine_kexec_prepare - Prepare for a kexec reboot. 66 * 67 * Called from the core kexec code when a kernel image is loaded. 68 * Forbid loading a kexec kernel if we have no way of hotplugging cpus or cpus 69 * are stuck in the kernel. This avoids a panic once we hit machine_kexec(). 70 */ 71 int machine_kexec_prepare(struct kimage *kimage) 72 { 73 kexec_image_info(kimage); 74 75 if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { 76 pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); 77 return -EBUSY; 78 } 79 80 return 0; 81 } 82 83 /** 84 * kexec_list_flush - Helper to flush the kimage list and source pages to PoC. 85 */ 86 static void kexec_list_flush(struct kimage *kimage) 87 { 88 kimage_entry_t *entry; 89 90 for (entry = &kimage->head; ; entry++) { 91 unsigned int flag; 92 void *addr; 93 94 /* flush the list entries. */ 95 __flush_dcache_area(entry, sizeof(kimage_entry_t)); 96 97 flag = *entry & IND_FLAGS; 98 if (flag == IND_DONE) 99 break; 100 101 addr = phys_to_virt(*entry & PAGE_MASK); 102 103 switch (flag) { 104 case IND_INDIRECTION: 105 /* Set entry point just before the new list page. */ 106 entry = (kimage_entry_t *)addr - 1; 107 break; 108 case IND_SOURCE: 109 /* flush the source pages. */ 110 __flush_dcache_area(addr, PAGE_SIZE); 111 break; 112 case IND_DESTINATION: 113 break; 114 default: 115 BUG(); 116 } 117 } 118 } 119 120 /** 121 * kexec_segment_flush - Helper to flush the kimage segments to PoC. 122 */ 123 static void kexec_segment_flush(const struct kimage *kimage) 124 { 125 unsigned long i; 126 127 pr_debug("%s:\n", __func__); 128 129 for (i = 0; i < kimage->nr_segments; i++) { 130 pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n", 131 i, 132 kimage->segment[i].mem, 133 kimage->segment[i].mem + kimage->segment[i].memsz, 134 kimage->segment[i].memsz, 135 kimage->segment[i].memsz / PAGE_SIZE); 136 137 __flush_dcache_area(phys_to_virt(kimage->segment[i].mem), 138 kimage->segment[i].memsz); 139 } 140 } 141 142 /** 143 * machine_kexec - Do the kexec reboot. 144 * 145 * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC. 146 */ 147 void machine_kexec(struct kimage *kimage) 148 { 149 phys_addr_t reboot_code_buffer_phys; 150 void *reboot_code_buffer; 151 bool in_kexec_crash = (kimage == kexec_crash_image); 152 bool stuck_cpus = cpus_are_stuck_in_kernel(); 153 154 /* 155 * New cpus may have become stuck_in_kernel after we loaded the image. 156 */ 157 BUG_ON(!in_kexec_crash && (stuck_cpus || (num_online_cpus() > 1))); 158 WARN(in_kexec_crash && (stuck_cpus || smp_crash_stop_failed()), 159 "Some CPUs may be stale, kdump will be unreliable.\n"); 160 161 reboot_code_buffer_phys = page_to_phys(kimage->control_code_page); 162 reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys); 163 164 kexec_image_info(kimage); 165 166 pr_debug("%s:%d: control_code_page: %p\n", __func__, __LINE__, 167 kimage->control_code_page); 168 pr_debug("%s:%d: reboot_code_buffer_phys: %pa\n", __func__, __LINE__, 169 &reboot_code_buffer_phys); 170 pr_debug("%s:%d: reboot_code_buffer: %p\n", __func__, __LINE__, 171 reboot_code_buffer); 172 pr_debug("%s:%d: relocate_new_kernel: %p\n", __func__, __LINE__, 173 arm64_relocate_new_kernel); 174 pr_debug("%s:%d: relocate_new_kernel_size: 0x%lx(%lu) bytes\n", 175 __func__, __LINE__, arm64_relocate_new_kernel_size, 176 arm64_relocate_new_kernel_size); 177 178 /* 179 * Copy arm64_relocate_new_kernel to the reboot_code_buffer for use 180 * after the kernel is shut down. 181 */ 182 memcpy(reboot_code_buffer, arm64_relocate_new_kernel, 183 arm64_relocate_new_kernel_size); 184 185 /* Flush the reboot_code_buffer in preparation for its execution. */ 186 __flush_dcache_area(reboot_code_buffer, arm64_relocate_new_kernel_size); 187 188 /* 189 * Although we've killed off the secondary CPUs, we don't update 190 * the online mask if we're handling a crash kernel and consequently 191 * need to avoid flush_icache_range(), which will attempt to IPI 192 * the offline CPUs. Therefore, we must use the __* variant here. 193 */ 194 __flush_icache_range((uintptr_t)reboot_code_buffer, 195 arm64_relocate_new_kernel_size); 196 197 /* Flush the kimage list and its buffers. */ 198 kexec_list_flush(kimage); 199 200 /* Flush the new image if already in place. */ 201 if ((kimage != kexec_crash_image) && (kimage->head & IND_DONE)) 202 kexec_segment_flush(kimage); 203 204 pr_info("Bye!\n"); 205 206 local_daif_mask(); 207 208 /* 209 * cpu_soft_restart will shutdown the MMU, disable data caches, then 210 * transfer control to the reboot_code_buffer which contains a copy of 211 * the arm64_relocate_new_kernel routine. arm64_relocate_new_kernel 212 * uses physical addressing to relocate the new image to its final 213 * position and transfers control to the image entry point when the 214 * relocation is complete. 215 * In kexec case, kimage->start points to purgatory assuming that 216 * kernel entry and dtb address are embedded in purgatory by 217 * userspace (kexec-tools). 218 * In kexec_file case, the kernel starts directly without purgatory. 219 */ 220 cpu_soft_restart(reboot_code_buffer_phys, kimage->head, kimage->start, 221 #ifdef CONFIG_KEXEC_FILE 222 kimage->arch.dtb_mem); 223 #else 224 0); 225 #endif 226 227 BUG(); /* Should never get here. */ 228 } 229 230 static void machine_kexec_mask_interrupts(void) 231 { 232 unsigned int i; 233 struct irq_desc *desc; 234 235 for_each_irq_desc(i, desc) { 236 struct irq_chip *chip; 237 int ret; 238 239 chip = irq_desc_get_chip(desc); 240 if (!chip) 241 continue; 242 243 /* 244 * First try to remove the active state. If this 245 * fails, try to EOI the interrupt. 246 */ 247 ret = irq_set_irqchip_state(i, IRQCHIP_STATE_ACTIVE, false); 248 249 if (ret && irqd_irq_inprogress(&desc->irq_data) && 250 chip->irq_eoi) 251 chip->irq_eoi(&desc->irq_data); 252 253 if (chip->irq_mask) 254 chip->irq_mask(&desc->irq_data); 255 256 if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data)) 257 chip->irq_disable(&desc->irq_data); 258 } 259 } 260 261 /** 262 * machine_crash_shutdown - shutdown non-crashing cpus and save registers 263 */ 264 void machine_crash_shutdown(struct pt_regs *regs) 265 { 266 local_irq_disable(); 267 268 /* shutdown non-crashing cpus */ 269 crash_smp_send_stop(); 270 271 /* for crashing cpu */ 272 crash_save_cpu(regs, smp_processor_id()); 273 machine_kexec_mask_interrupts(); 274 275 pr_info("Starting crashdump kernel...\n"); 276 } 277 278 void arch_kexec_protect_crashkres(void) 279 { 280 int i; 281 282 kexec_segment_flush(kexec_crash_image); 283 284 for (i = 0; i < kexec_crash_image->nr_segments; i++) 285 set_memory_valid( 286 __phys_to_virt(kexec_crash_image->segment[i].mem), 287 kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 0); 288 } 289 290 void arch_kexec_unprotect_crashkres(void) 291 { 292 int i; 293 294 for (i = 0; i < kexec_crash_image->nr_segments; i++) 295 set_memory_valid( 296 __phys_to_virt(kexec_crash_image->segment[i].mem), 297 kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 1); 298 } 299 300 #ifdef CONFIG_HIBERNATION 301 /* 302 * To preserve the crash dump kernel image, the relevant memory segments 303 * should be mapped again around the hibernation. 304 */ 305 void crash_prepare_suspend(void) 306 { 307 if (kexec_crash_image) 308 arch_kexec_unprotect_crashkres(); 309 } 310 311 void crash_post_resume(void) 312 { 313 if (kexec_crash_image) 314 arch_kexec_protect_crashkres(); 315 } 316 317 /* 318 * crash_is_nosave 319 * 320 * Return true only if a page is part of reserved memory for crash dump kernel, 321 * but does not hold any data of loaded kernel image. 322 * 323 * Note that all the pages in crash dump kernel memory have been initially 324 * marked as Reserved as memory was allocated via memblock_reserve(). 325 * 326 * In hibernation, the pages which are Reserved and yet "nosave" are excluded 327 * from the hibernation iamge. crash_is_nosave() does thich check for crash 328 * dump kernel and will reduce the total size of hibernation image. 329 */ 330 331 bool crash_is_nosave(unsigned long pfn) 332 { 333 int i; 334 phys_addr_t addr; 335 336 if (!crashk_res.end) 337 return false; 338 339 /* in reserved memory? */ 340 addr = __pfn_to_phys(pfn); 341 if ((addr < crashk_res.start) || (crashk_res.end < addr)) 342 return false; 343 344 if (!kexec_crash_image) 345 return true; 346 347 /* not part of loaded kernel image? */ 348 for (i = 0; i < kexec_crash_image->nr_segments; i++) 349 if (addr >= kexec_crash_image->segment[i].mem && 350 addr < (kexec_crash_image->segment[i].mem + 351 kexec_crash_image->segment[i].memsz)) 352 return false; 353 354 return true; 355 } 356 357 void crash_free_reserved_phys_range(unsigned long begin, unsigned long end) 358 { 359 unsigned long addr; 360 struct page *page; 361 362 for (addr = begin; addr < end; addr += PAGE_SIZE) { 363 page = phys_to_page(addr); 364 free_reserved_page(page); 365 } 366 } 367 #endif /* CONFIG_HIBERNATION */ 368