1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Architecture specific (i386/x86_64) functions for kexec based crash dumps. 4 * 5 * Created by: Hariprasad Nellitheertha (hari@in.ibm.com) 6 * 7 * Copyright (C) IBM Corporation, 2004. All rights reserved. 8 * Copyright (C) Red Hat Inc., 2014. All rights reserved. 9 * Authors: 10 * Vivek Goyal <vgoyal@redhat.com> 11 * 12 */ 13 14 #define pr_fmt(fmt) "kexec: " fmt 15 16 #include <linux/types.h> 17 #include <linux/kernel.h> 18 #include <linux/smp.h> 19 #include <linux/reboot.h> 20 #include <linux/kexec.h> 21 #include <linux/delay.h> 22 #include <linux/elf.h> 23 #include <linux/elfcore.h> 24 #include <linux/export.h> 25 #include <linux/slab.h> 26 #include <linux/vmalloc.h> 27 28 #include <asm/processor.h> 29 #include <asm/hardirq.h> 30 #include <asm/nmi.h> 31 #include <asm/hw_irq.h> 32 #include <asm/apic.h> 33 #include <asm/e820/types.h> 34 #include <asm/io_apic.h> 35 #include <asm/hpet.h> 36 #include <linux/kdebug.h> 37 #include <asm/cpu.h> 38 #include <asm/reboot.h> 39 #include <asm/virtext.h> 40 #include <asm/intel_pt.h> 41 #include <asm/crash.h> 42 43 /* Used while preparing memory map entries for second kernel */ 44 struct crash_memmap_data { 45 struct boot_params *params; 46 /* Type of memory */ 47 unsigned int type; 48 }; 49 50 /* 51 * This is used to VMCLEAR all VMCSs loaded on the 52 * processor. And when loading kvm_intel module, the 53 * callback function pointer will be assigned. 54 * 55 * protected by rcu. 56 */ 57 crash_vmclear_fn __rcu *crash_vmclear_loaded_vmcss = NULL; 58 EXPORT_SYMBOL_GPL(crash_vmclear_loaded_vmcss); 59 60 static inline void cpu_crash_vmclear_loaded_vmcss(void) 61 { 62 crash_vmclear_fn *do_vmclear_operation = NULL; 63 64 rcu_read_lock(); 65 do_vmclear_operation = rcu_dereference(crash_vmclear_loaded_vmcss); 66 if (do_vmclear_operation) 67 do_vmclear_operation(); 68 rcu_read_unlock(); 69 } 70 71 #if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC) 72 73 static void kdump_nmi_callback(int cpu, struct pt_regs *regs) 74 { 75 crash_save_cpu(regs, cpu); 76 77 /* 78 * VMCLEAR VMCSs loaded on all cpus if needed. 79 */ 80 cpu_crash_vmclear_loaded_vmcss(); 81 82 /* Disable VMX or SVM if needed. 83 * 84 * We need to disable virtualization on all CPUs. 85 * Having VMX or SVM enabled on any CPU may break rebooting 86 * after the kdump kernel has finished its task. 87 */ 88 cpu_emergency_vmxoff(); 89 cpu_emergency_svm_disable(); 90 91 /* 92 * Disable Intel PT to stop its logging 93 */ 94 cpu_emergency_stop_pt(); 95 96 disable_local_APIC(); 97 } 98 99 void kdump_nmi_shootdown_cpus(void) 100 { 101 nmi_shootdown_cpus(kdump_nmi_callback); 102 103 disable_local_APIC(); 104 } 105 106 /* Override the weak function in kernel/panic.c */ 107 void crash_smp_send_stop(void) 108 { 109 static int cpus_stopped; 110 111 if (cpus_stopped) 112 return; 113 114 if (smp_ops.crash_stop_other_cpus) 115 smp_ops.crash_stop_other_cpus(); 116 else 117 smp_send_stop(); 118 119 cpus_stopped = 1; 120 } 121 122 #else 123 void crash_smp_send_stop(void) 124 { 125 /* There are no cpus to shootdown */ 126 } 127 #endif 128 129 void native_machine_crash_shutdown(struct pt_regs *regs) 130 { 131 /* This function is only called after the system 132 * has panicked or is otherwise in a critical state. 133 * The minimum amount of code to allow a kexec'd kernel 134 * to run successfully needs to happen here. 135 * 136 * In practice this means shooting down the other cpus in 137 * an SMP system. 138 */ 139 /* The kernel is broken so disable interrupts */ 140 local_irq_disable(); 141 142 crash_smp_send_stop(); 143 144 /* 145 * VMCLEAR VMCSs loaded on this cpu if needed. 146 */ 147 cpu_crash_vmclear_loaded_vmcss(); 148 149 /* Booting kdump kernel with VMX or SVM enabled won't work, 150 * because (among other limitations) we can't disable paging 151 * with the virt flags. 152 */ 153 cpu_emergency_vmxoff(); 154 cpu_emergency_svm_disable(); 155 156 /* 157 * Disable Intel PT to stop its logging 158 */ 159 cpu_emergency_stop_pt(); 160 161 #ifdef CONFIG_X86_IO_APIC 162 /* Prevent crash_kexec() from deadlocking on ioapic_lock. */ 163 ioapic_zap_locks(); 164 clear_IO_APIC(); 165 #endif 166 lapic_shutdown(); 167 restore_boot_irq_mode(); 168 #ifdef CONFIG_HPET_TIMER 169 hpet_disable(); 170 #endif 171 crash_save_cpu(regs, safe_smp_processor_id()); 172 } 173 174 #ifdef CONFIG_KEXEC_FILE 175 176 static unsigned long crash_zero_bytes; 177 178 static int get_nr_ram_ranges_callback(struct resource *res, void *arg) 179 { 180 unsigned int *nr_ranges = arg; 181 182 (*nr_ranges)++; 183 return 0; 184 } 185 186 /* Gather all the required information to prepare elf headers for ram regions */ 187 static struct crash_mem *fill_up_crash_elf_data(void) 188 { 189 unsigned int nr_ranges = 0; 190 struct crash_mem *cmem; 191 192 walk_system_ram_res(0, -1, &nr_ranges, 193 get_nr_ram_ranges_callback); 194 if (!nr_ranges) 195 return NULL; 196 197 /* 198 * Exclusion of crash region and/or crashk_low_res may cause 199 * another range split. So add extra two slots here. 200 */ 201 nr_ranges += 2; 202 cmem = vzalloc(struct_size(cmem, ranges, nr_ranges)); 203 if (!cmem) 204 return NULL; 205 206 cmem->max_nr_ranges = nr_ranges; 207 cmem->nr_ranges = 0; 208 209 return cmem; 210 } 211 212 /* 213 * Look for any unwanted ranges between mstart, mend and remove them. This 214 * might lead to split and split ranges are put in cmem->ranges[] array 215 */ 216 static int elf_header_exclude_ranges(struct crash_mem *cmem) 217 { 218 int ret = 0; 219 220 /* Exclude crashkernel region */ 221 ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); 222 if (ret) 223 return ret; 224 225 if (crashk_low_res.end) { 226 ret = crash_exclude_mem_range(cmem, crashk_low_res.start, 227 crashk_low_res.end); 228 if (ret) 229 return ret; 230 } 231 232 return ret; 233 } 234 235 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) 236 { 237 struct crash_mem *cmem = arg; 238 239 cmem->ranges[cmem->nr_ranges].start = res->start; 240 cmem->ranges[cmem->nr_ranges].end = res->end; 241 cmem->nr_ranges++; 242 243 return 0; 244 } 245 246 /* Prepare elf headers. Return addr and size */ 247 static int prepare_elf_headers(struct kimage *image, void **addr, 248 unsigned long *sz) 249 { 250 struct crash_mem *cmem; 251 Elf64_Ehdr *ehdr; 252 Elf64_Phdr *phdr; 253 int ret, i; 254 255 cmem = fill_up_crash_elf_data(); 256 if (!cmem) 257 return -ENOMEM; 258 259 ret = walk_system_ram_res(0, -1, cmem, 260 prepare_elf64_ram_headers_callback); 261 if (ret) 262 goto out; 263 264 /* Exclude unwanted mem ranges */ 265 ret = elf_header_exclude_ranges(cmem); 266 if (ret) 267 goto out; 268 269 /* By default prepare 64bit headers */ 270 ret = crash_prepare_elf64_headers(cmem, 271 IS_ENABLED(CONFIG_X86_64), addr, sz); 272 if (ret) 273 goto out; 274 275 /* 276 * If a range matches backup region, adjust offset to backup 277 * segment. 278 */ 279 ehdr = (Elf64_Ehdr *)*addr; 280 phdr = (Elf64_Phdr *)(ehdr + 1); 281 for (i = 0; i < ehdr->e_phnum; phdr++, i++) 282 if (phdr->p_type == PT_LOAD && 283 phdr->p_paddr == image->arch.backup_src_start && 284 phdr->p_memsz == image->arch.backup_src_sz) { 285 phdr->p_offset = image->arch.backup_load_addr; 286 break; 287 } 288 out: 289 vfree(cmem); 290 return ret; 291 } 292 293 static int add_e820_entry(struct boot_params *params, struct e820_entry *entry) 294 { 295 unsigned int nr_e820_entries; 296 297 nr_e820_entries = params->e820_entries; 298 if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) 299 return 1; 300 301 memcpy(¶ms->e820_table[nr_e820_entries], entry, 302 sizeof(struct e820_entry)); 303 params->e820_entries++; 304 return 0; 305 } 306 307 static int memmap_entry_callback(struct resource *res, void *arg) 308 { 309 struct crash_memmap_data *cmd = arg; 310 struct boot_params *params = cmd->params; 311 struct e820_entry ei; 312 313 ei.addr = res->start; 314 ei.size = resource_size(res); 315 ei.type = cmd->type; 316 add_e820_entry(params, &ei); 317 318 return 0; 319 } 320 321 static int memmap_exclude_ranges(struct kimage *image, struct crash_mem *cmem, 322 unsigned long long mstart, 323 unsigned long long mend) 324 { 325 unsigned long start, end; 326 int ret = 0; 327 328 cmem->ranges[0].start = mstart; 329 cmem->ranges[0].end = mend; 330 cmem->nr_ranges = 1; 331 332 /* Exclude Backup region */ 333 start = image->arch.backup_load_addr; 334 end = start + image->arch.backup_src_sz - 1; 335 ret = crash_exclude_mem_range(cmem, start, end); 336 if (ret) 337 return ret; 338 339 /* Exclude elf header region */ 340 start = image->arch.elf_load_addr; 341 end = start + image->arch.elf_headers_sz - 1; 342 return crash_exclude_mem_range(cmem, start, end); 343 } 344 345 /* Prepare memory map for crash dump kernel */ 346 int crash_setup_memmap_entries(struct kimage *image, struct boot_params *params) 347 { 348 int i, ret = 0; 349 unsigned long flags; 350 struct e820_entry ei; 351 struct crash_memmap_data cmd; 352 struct crash_mem *cmem; 353 354 cmem = vzalloc(sizeof(struct crash_mem)); 355 if (!cmem) 356 return -ENOMEM; 357 358 memset(&cmd, 0, sizeof(struct crash_memmap_data)); 359 cmd.params = params; 360 361 /* Add first 640K segment */ 362 ei.addr = image->arch.backup_src_start; 363 ei.size = image->arch.backup_src_sz; 364 ei.type = E820_TYPE_RAM; 365 add_e820_entry(params, &ei); 366 367 /* Add ACPI tables */ 368 cmd.type = E820_TYPE_ACPI; 369 flags = IORESOURCE_MEM | IORESOURCE_BUSY; 370 walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, &cmd, 371 memmap_entry_callback); 372 373 /* Add ACPI Non-volatile Storage */ 374 cmd.type = E820_TYPE_NVS; 375 walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, &cmd, 376 memmap_entry_callback); 377 378 /* Add e820 reserved ranges */ 379 cmd.type = E820_TYPE_RESERVED; 380 flags = IORESOURCE_MEM; 381 walk_iomem_res_desc(IORES_DESC_RESERVED, flags, 0, -1, &cmd, 382 memmap_entry_callback); 383 384 /* Add crashk_low_res region */ 385 if (crashk_low_res.end) { 386 ei.addr = crashk_low_res.start; 387 ei.size = crashk_low_res.end - crashk_low_res.start + 1; 388 ei.type = E820_TYPE_RAM; 389 add_e820_entry(params, &ei); 390 } 391 392 /* Exclude some ranges from crashk_res and add rest to memmap */ 393 ret = memmap_exclude_ranges(image, cmem, crashk_res.start, 394 crashk_res.end); 395 if (ret) 396 goto out; 397 398 for (i = 0; i < cmem->nr_ranges; i++) { 399 ei.size = cmem->ranges[i].end - cmem->ranges[i].start + 1; 400 401 /* If entry is less than a page, skip it */ 402 if (ei.size < PAGE_SIZE) 403 continue; 404 ei.addr = cmem->ranges[i].start; 405 ei.type = E820_TYPE_RAM; 406 add_e820_entry(params, &ei); 407 } 408 409 out: 410 vfree(cmem); 411 return ret; 412 } 413 414 static int determine_backup_region(struct resource *res, void *arg) 415 { 416 struct kimage *image = arg; 417 418 image->arch.backup_src_start = res->start; 419 image->arch.backup_src_sz = resource_size(res); 420 421 /* Expecting only one range for backup region */ 422 return 1; 423 } 424 425 int crash_load_segments(struct kimage *image) 426 { 427 int ret; 428 struct kexec_buf kbuf = { .image = image, .buf_min = 0, 429 .buf_max = ULONG_MAX, .top_down = false }; 430 431 /* 432 * Determine and load a segment for backup area. First 640K RAM 433 * region is backup source 434 */ 435 436 ret = walk_system_ram_res(KEXEC_BACKUP_SRC_START, KEXEC_BACKUP_SRC_END, 437 image, determine_backup_region); 438 439 /* Zero or postive return values are ok */ 440 if (ret < 0) 441 return ret; 442 443 /* Add backup segment. */ 444 if (image->arch.backup_src_sz) { 445 kbuf.buffer = &crash_zero_bytes; 446 kbuf.bufsz = sizeof(crash_zero_bytes); 447 kbuf.memsz = image->arch.backup_src_sz; 448 kbuf.buf_align = PAGE_SIZE; 449 /* 450 * Ideally there is no source for backup segment. This is 451 * copied in purgatory after crash. Just add a zero filled 452 * segment for now to make sure checksum logic works fine. 453 */ 454 ret = kexec_add_buffer(&kbuf); 455 if (ret) 456 return ret; 457 image->arch.backup_load_addr = kbuf.mem; 458 pr_debug("Loaded backup region at 0x%lx backup_start=0x%lx memsz=0x%lx\n", 459 image->arch.backup_load_addr, 460 image->arch.backup_src_start, kbuf.memsz); 461 } 462 463 /* Prepare elf headers and add a segment */ 464 ret = prepare_elf_headers(image, &kbuf.buffer, &kbuf.bufsz); 465 if (ret) 466 return ret; 467 468 image->arch.elf_headers = kbuf.buffer; 469 image->arch.elf_headers_sz = kbuf.bufsz; 470 471 kbuf.memsz = kbuf.bufsz; 472 kbuf.buf_align = ELF_CORE_HEADER_ALIGN; 473 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 474 ret = kexec_add_buffer(&kbuf); 475 if (ret) { 476 vfree((void *)image->arch.elf_headers); 477 return ret; 478 } 479 image->arch.elf_load_addr = kbuf.mem; 480 pr_debug("Loaded ELF headers at 0x%lx bufsz=0x%lx memsz=0x%lx\n", 481 image->arch.elf_load_addr, kbuf.bufsz, kbuf.bufsz); 482 483 return ret; 484 } 485 #endif /* CONFIG_KEXEC_FILE */ 486