1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Architecture specific (i386/x86_64) functions for kexec based crash dumps. 4 * 5 * Created by: Hariprasad Nellitheertha (hari@in.ibm.com) 6 * 7 * Copyright (C) IBM Corporation, 2004. All rights reserved. 8 * Copyright (C) Red Hat Inc., 2014. All rights reserved. 9 * Authors: 10 * Vivek Goyal <vgoyal@redhat.com> 11 * 12 */ 13 14 #define pr_fmt(fmt) "kexec: " fmt 15 16 #include <linux/types.h> 17 #include <linux/kernel.h> 18 #include <linux/smp.h> 19 #include <linux/reboot.h> 20 #include <linux/kexec.h> 21 #include <linux/delay.h> 22 #include <linux/elf.h> 23 #include <linux/elfcore.h> 24 #include <linux/export.h> 25 #include <linux/slab.h> 26 #include <linux/vmalloc.h> 27 28 #include <asm/processor.h> 29 #include <asm/hardirq.h> 30 #include <asm/nmi.h> 31 #include <asm/hw_irq.h> 32 #include <asm/apic.h> 33 #include <asm/e820/types.h> 34 #include <asm/io_apic.h> 35 #include <asm/hpet.h> 36 #include <linux/kdebug.h> 37 #include <asm/cpu.h> 38 #include <asm/reboot.h> 39 #include <asm/virtext.h> 40 #include <asm/intel_pt.h> 41 #include <asm/crash.h> 42 43 /* Used while preparing memory map entries for second kernel */ 44 struct crash_memmap_data { 45 struct boot_params *params; 46 /* Type of memory */ 47 unsigned int type; 48 }; 49 50 /* 51 * This is used to VMCLEAR all VMCSs loaded on the 52 * processor. And when loading kvm_intel module, the 53 * callback function pointer will be assigned. 54 * 55 * protected by rcu. 56 */ 57 crash_vmclear_fn __rcu *crash_vmclear_loaded_vmcss = NULL; 58 EXPORT_SYMBOL_GPL(crash_vmclear_loaded_vmcss); 59 unsigned long crash_zero_bytes; 60 61 static inline void cpu_crash_vmclear_loaded_vmcss(void) 62 { 63 crash_vmclear_fn *do_vmclear_operation = NULL; 64 65 rcu_read_lock(); 66 do_vmclear_operation = rcu_dereference(crash_vmclear_loaded_vmcss); 67 if (do_vmclear_operation) 68 do_vmclear_operation(); 69 rcu_read_unlock(); 70 } 71 72 #if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC) 73 74 static void kdump_nmi_callback(int cpu, struct pt_regs *regs) 75 { 76 #ifdef CONFIG_X86_32 77 struct pt_regs fixed_regs; 78 79 if (!user_mode(regs)) { 80 crash_fixup_ss_esp(&fixed_regs, regs); 81 regs = &fixed_regs; 82 } 83 #endif 84 crash_save_cpu(regs, cpu); 85 86 /* 87 * VMCLEAR VMCSs loaded on all cpus if needed. 88 */ 89 cpu_crash_vmclear_loaded_vmcss(); 90 91 /* Disable VMX or SVM if needed. 92 * 93 * We need to disable virtualization on all CPUs. 94 * Having VMX or SVM enabled on any CPU may break rebooting 95 * after the kdump kernel has finished its task. 96 */ 97 cpu_emergency_vmxoff(); 98 cpu_emergency_svm_disable(); 99 100 /* 101 * Disable Intel PT to stop its logging 102 */ 103 cpu_emergency_stop_pt(); 104 105 disable_local_APIC(); 106 } 107 108 void kdump_nmi_shootdown_cpus(void) 109 { 110 nmi_shootdown_cpus(kdump_nmi_callback); 111 112 disable_local_APIC(); 113 } 114 115 /* Override the weak function in kernel/panic.c */ 116 void crash_smp_send_stop(void) 117 { 118 static int cpus_stopped; 119 120 if (cpus_stopped) 121 return; 122 123 if (smp_ops.crash_stop_other_cpus) 124 smp_ops.crash_stop_other_cpus(); 125 else 126 smp_send_stop(); 127 128 cpus_stopped = 1; 129 } 130 131 #else 132 void crash_smp_send_stop(void) 133 { 134 /* There are no cpus to shootdown */ 135 } 136 #endif 137 138 void native_machine_crash_shutdown(struct pt_regs *regs) 139 { 140 /* This function is only called after the system 141 * has panicked or is otherwise in a critical state. 142 * The minimum amount of code to allow a kexec'd kernel 143 * to run successfully needs to happen here. 144 * 145 * In practice this means shooting down the other cpus in 146 * an SMP system. 147 */ 148 /* The kernel is broken so disable interrupts */ 149 local_irq_disable(); 150 151 crash_smp_send_stop(); 152 153 /* 154 * VMCLEAR VMCSs loaded on this cpu if needed. 155 */ 156 cpu_crash_vmclear_loaded_vmcss(); 157 158 /* Booting kdump kernel with VMX or SVM enabled won't work, 159 * because (among other limitations) we can't disable paging 160 * with the virt flags. 161 */ 162 cpu_emergency_vmxoff(); 163 cpu_emergency_svm_disable(); 164 165 /* 166 * Disable Intel PT to stop its logging 167 */ 168 cpu_emergency_stop_pt(); 169 170 #ifdef CONFIG_X86_IO_APIC 171 /* Prevent crash_kexec() from deadlocking on ioapic_lock. */ 172 ioapic_zap_locks(); 173 clear_IO_APIC(); 174 #endif 175 lapic_shutdown(); 176 restore_boot_irq_mode(); 177 #ifdef CONFIG_HPET_TIMER 178 hpet_disable(); 179 #endif 180 crash_save_cpu(regs, safe_smp_processor_id()); 181 } 182 183 #ifdef CONFIG_KEXEC_FILE 184 static int get_nr_ram_ranges_callback(struct resource *res, void *arg) 185 { 186 unsigned int *nr_ranges = arg; 187 188 (*nr_ranges)++; 189 return 0; 190 } 191 192 /* Gather all the required information to prepare elf headers for ram regions */ 193 static struct crash_mem *fill_up_crash_elf_data(void) 194 { 195 unsigned int nr_ranges = 0; 196 struct crash_mem *cmem; 197 198 walk_system_ram_res(0, -1, &nr_ranges, 199 get_nr_ram_ranges_callback); 200 if (!nr_ranges) 201 return NULL; 202 203 /* 204 * Exclusion of crash region and/or crashk_low_res may cause 205 * another range split. So add extra two slots here. 206 */ 207 nr_ranges += 2; 208 cmem = vzalloc(struct_size(cmem, ranges, nr_ranges)); 209 if (!cmem) 210 return NULL; 211 212 cmem->max_nr_ranges = nr_ranges; 213 cmem->nr_ranges = 0; 214 215 return cmem; 216 } 217 218 /* 219 * Look for any unwanted ranges between mstart, mend and remove them. This 220 * might lead to split and split ranges are put in cmem->ranges[] array 221 */ 222 static int elf_header_exclude_ranges(struct crash_mem *cmem) 223 { 224 int ret = 0; 225 226 /* Exclude crashkernel region */ 227 ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); 228 if (ret) 229 return ret; 230 231 if (crashk_low_res.end) { 232 ret = crash_exclude_mem_range(cmem, crashk_low_res.start, 233 crashk_low_res.end); 234 if (ret) 235 return ret; 236 } 237 238 return ret; 239 } 240 241 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) 242 { 243 struct crash_mem *cmem = arg; 244 245 cmem->ranges[cmem->nr_ranges].start = res->start; 246 cmem->ranges[cmem->nr_ranges].end = res->end; 247 cmem->nr_ranges++; 248 249 return 0; 250 } 251 252 /* Prepare elf headers. Return addr and size */ 253 static int prepare_elf_headers(struct kimage *image, void **addr, 254 unsigned long *sz) 255 { 256 struct crash_mem *cmem; 257 Elf64_Ehdr *ehdr; 258 Elf64_Phdr *phdr; 259 int ret, i; 260 261 cmem = fill_up_crash_elf_data(); 262 if (!cmem) 263 return -ENOMEM; 264 265 ret = walk_system_ram_res(0, -1, cmem, 266 prepare_elf64_ram_headers_callback); 267 if (ret) 268 goto out; 269 270 /* Exclude unwanted mem ranges */ 271 ret = elf_header_exclude_ranges(cmem); 272 if (ret) 273 goto out; 274 275 /* By default prepare 64bit headers */ 276 ret = crash_prepare_elf64_headers(cmem, 277 IS_ENABLED(CONFIG_X86_64), addr, sz); 278 if (ret) 279 goto out; 280 281 /* 282 * If a range matches backup region, adjust offset to backup 283 * segment. 284 */ 285 ehdr = (Elf64_Ehdr *)*addr; 286 phdr = (Elf64_Phdr *)(ehdr + 1); 287 for (i = 0; i < ehdr->e_phnum; phdr++, i++) 288 if (phdr->p_type == PT_LOAD && 289 phdr->p_paddr == image->arch.backup_src_start && 290 phdr->p_memsz == image->arch.backup_src_sz) { 291 phdr->p_offset = image->arch.backup_load_addr; 292 break; 293 } 294 out: 295 vfree(cmem); 296 return ret; 297 } 298 299 static int add_e820_entry(struct boot_params *params, struct e820_entry *entry) 300 { 301 unsigned int nr_e820_entries; 302 303 nr_e820_entries = params->e820_entries; 304 if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) 305 return 1; 306 307 memcpy(¶ms->e820_table[nr_e820_entries], entry, 308 sizeof(struct e820_entry)); 309 params->e820_entries++; 310 return 0; 311 } 312 313 static int memmap_entry_callback(struct resource *res, void *arg) 314 { 315 struct crash_memmap_data *cmd = arg; 316 struct boot_params *params = cmd->params; 317 struct e820_entry ei; 318 319 ei.addr = res->start; 320 ei.size = resource_size(res); 321 ei.type = cmd->type; 322 add_e820_entry(params, &ei); 323 324 return 0; 325 } 326 327 static int memmap_exclude_ranges(struct kimage *image, struct crash_mem *cmem, 328 unsigned long long mstart, 329 unsigned long long mend) 330 { 331 unsigned long start, end; 332 int ret = 0; 333 334 cmem->ranges[0].start = mstart; 335 cmem->ranges[0].end = mend; 336 cmem->nr_ranges = 1; 337 338 /* Exclude Backup region */ 339 start = image->arch.backup_load_addr; 340 end = start + image->arch.backup_src_sz - 1; 341 ret = crash_exclude_mem_range(cmem, start, end); 342 if (ret) 343 return ret; 344 345 /* Exclude elf header region */ 346 start = image->arch.elf_load_addr; 347 end = start + image->arch.elf_headers_sz - 1; 348 return crash_exclude_mem_range(cmem, start, end); 349 } 350 351 /* Prepare memory map for crash dump kernel */ 352 int crash_setup_memmap_entries(struct kimage *image, struct boot_params *params) 353 { 354 int i, ret = 0; 355 unsigned long flags; 356 struct e820_entry ei; 357 struct crash_memmap_data cmd; 358 struct crash_mem *cmem; 359 360 cmem = vzalloc(sizeof(struct crash_mem)); 361 if (!cmem) 362 return -ENOMEM; 363 364 memset(&cmd, 0, sizeof(struct crash_memmap_data)); 365 cmd.params = params; 366 367 /* Add first 640K segment */ 368 ei.addr = image->arch.backup_src_start; 369 ei.size = image->arch.backup_src_sz; 370 ei.type = E820_TYPE_RAM; 371 add_e820_entry(params, &ei); 372 373 /* Add ACPI tables */ 374 cmd.type = E820_TYPE_ACPI; 375 flags = IORESOURCE_MEM | IORESOURCE_BUSY; 376 walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, &cmd, 377 memmap_entry_callback); 378 379 /* Add ACPI Non-volatile Storage */ 380 cmd.type = E820_TYPE_NVS; 381 walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, &cmd, 382 memmap_entry_callback); 383 384 /* Add crashk_low_res region */ 385 if (crashk_low_res.end) { 386 ei.addr = crashk_low_res.start; 387 ei.size = crashk_low_res.end - crashk_low_res.start + 1; 388 ei.type = E820_TYPE_RAM; 389 add_e820_entry(params, &ei); 390 } 391 392 /* Exclude some ranges from crashk_res and add rest to memmap */ 393 ret = memmap_exclude_ranges(image, cmem, crashk_res.start, 394 crashk_res.end); 395 if (ret) 396 goto out; 397 398 for (i = 0; i < cmem->nr_ranges; i++) { 399 ei.size = cmem->ranges[i].end - cmem->ranges[i].start + 1; 400 401 /* If entry is less than a page, skip it */ 402 if (ei.size < PAGE_SIZE) 403 continue; 404 ei.addr = cmem->ranges[i].start; 405 ei.type = E820_TYPE_RAM; 406 add_e820_entry(params, &ei); 407 } 408 409 out: 410 vfree(cmem); 411 return ret; 412 } 413 414 static int determine_backup_region(struct resource *res, void *arg) 415 { 416 struct kimage *image = arg; 417 418 image->arch.backup_src_start = res->start; 419 image->arch.backup_src_sz = resource_size(res); 420 421 /* Expecting only one range for backup region */ 422 return 1; 423 } 424 425 int crash_load_segments(struct kimage *image) 426 { 427 int ret; 428 struct kexec_buf kbuf = { .image = image, .buf_min = 0, 429 .buf_max = ULONG_MAX, .top_down = false }; 430 431 /* 432 * Determine and load a segment for backup area. First 640K RAM 433 * region is backup source 434 */ 435 436 ret = walk_system_ram_res(KEXEC_BACKUP_SRC_START, KEXEC_BACKUP_SRC_END, 437 image, determine_backup_region); 438 439 /* Zero or postive return values are ok */ 440 if (ret < 0) 441 return ret; 442 443 /* Add backup segment. */ 444 if (image->arch.backup_src_sz) { 445 kbuf.buffer = &crash_zero_bytes; 446 kbuf.bufsz = sizeof(crash_zero_bytes); 447 kbuf.memsz = image->arch.backup_src_sz; 448 kbuf.buf_align = PAGE_SIZE; 449 /* 450 * Ideally there is no source for backup segment. This is 451 * copied in purgatory after crash. Just add a zero filled 452 * segment for now to make sure checksum logic works fine. 453 */ 454 ret = kexec_add_buffer(&kbuf); 455 if (ret) 456 return ret; 457 image->arch.backup_load_addr = kbuf.mem; 458 pr_debug("Loaded backup region at 0x%lx backup_start=0x%lx memsz=0x%lx\n", 459 image->arch.backup_load_addr, 460 image->arch.backup_src_start, kbuf.memsz); 461 } 462 463 /* Prepare elf headers and add a segment */ 464 ret = prepare_elf_headers(image, &kbuf.buffer, &kbuf.bufsz); 465 if (ret) 466 return ret; 467 468 image->arch.elf_headers = kbuf.buffer; 469 image->arch.elf_headers_sz = kbuf.bufsz; 470 471 kbuf.memsz = kbuf.bufsz; 472 kbuf.buf_align = ELF_CORE_HEADER_ALIGN; 473 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 474 ret = kexec_add_buffer(&kbuf); 475 if (ret) { 476 vfree((void *)image->arch.elf_headers); 477 return ret; 478 } 479 image->arch.elf_load_addr = kbuf.mem; 480 pr_debug("Loaded ELF headers at 0x%lx bufsz=0x%lx memsz=0x%lx\n", 481 image->arch.elf_load_addr, kbuf.bufsz, kbuf.bufsz); 482 483 return ret; 484 } 485 #endif /* CONFIG_KEXEC_FILE */ 486