1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Load ELF vmlinux file for the kexec_file_load syscall. 4 * 5 * Copyright (C) 2021 Huawei Technologies Co, Ltd. 6 * 7 * Author: Liao Chang (liaochang1@huawei.com) 8 * 9 * Based on kexec-tools' kexec-elf-riscv.c, heavily modified 10 * for kernel. 11 */ 12 13 #define pr_fmt(fmt) "kexec_image: " fmt 14 15 #include <linux/elf.h> 16 #include <linux/kexec.h> 17 #include <linux/slab.h> 18 #include <linux/of.h> 19 #include <linux/libfdt.h> 20 #include <linux/types.h> 21 #include <linux/memblock.h> 22 #include <asm/setup.h> 23 24 int arch_kimage_file_post_load_cleanup(struct kimage *image) 25 { 26 kvfree(image->arch.fdt); 27 image->arch.fdt = NULL; 28 29 vfree(image->elf_headers); 30 image->elf_headers = NULL; 31 image->elf_headers_sz = 0; 32 33 return kexec_image_post_load_cleanup_default(image); 34 } 35 36 static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr, 37 struct kexec_elf_info *elf_info, unsigned long old_pbase, 38 unsigned long new_pbase) 39 { 40 int i; 41 int ret = 0; 42 size_t size; 43 struct kexec_buf kbuf; 44 const struct elf_phdr *phdr; 45 46 kbuf.image = image; 47 48 for (i = 0; i < ehdr->e_phnum; i++) { 49 phdr = &elf_info->proghdrs[i]; 50 if (phdr->p_type != PT_LOAD) 51 continue; 52 53 size = phdr->p_filesz; 54 if (size > phdr->p_memsz) 55 size = phdr->p_memsz; 56 57 kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset; 58 kbuf.bufsz = size; 59 kbuf.buf_align = phdr->p_align; 60 kbuf.mem = phdr->p_paddr - old_pbase + new_pbase; 61 kbuf.memsz = phdr->p_memsz; 62 kbuf.top_down = false; 63 ret = kexec_add_buffer(&kbuf); 64 if (ret) 65 break; 66 } 67 68 return ret; 69 } 70 71 /* 72 * Go through the available phsyical memory regions and find one that hold 73 * an image of the specified size. 74 */ 75 static int elf_find_pbase(struct kimage *image, unsigned long kernel_len, 76 struct elfhdr *ehdr, struct kexec_elf_info *elf_info, 77 unsigned long *old_pbase, unsigned long *new_pbase) 78 { 79 int i; 80 int ret; 81 struct kexec_buf kbuf; 82 const struct elf_phdr *phdr; 83 unsigned long lowest_paddr = ULONG_MAX; 84 unsigned long lowest_vaddr = ULONG_MAX; 85 86 for (i = 0; i < ehdr->e_phnum; i++) { 87 phdr = &elf_info->proghdrs[i]; 88 if (phdr->p_type != PT_LOAD) 89 continue; 90 91 if (lowest_paddr > phdr->p_paddr) 92 lowest_paddr = phdr->p_paddr; 93 94 if (lowest_vaddr > phdr->p_vaddr) 95 lowest_vaddr = phdr->p_vaddr; 96 } 97 98 kbuf.image = image; 99 kbuf.buf_min = lowest_paddr; 100 kbuf.buf_max = ULONG_MAX; 101 kbuf.buf_align = PAGE_SIZE; 102 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 103 kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE); 104 kbuf.top_down = false; 105 ret = arch_kexec_locate_mem_hole(&kbuf); 106 if (!ret) { 107 *old_pbase = lowest_paddr; 108 *new_pbase = kbuf.mem; 109 image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem; 110 } 111 return ret; 112 } 113 114 static int get_nr_ram_ranges_callback(struct resource *res, void *arg) 115 { 116 unsigned int *nr_ranges = arg; 117 118 (*nr_ranges)++; 119 return 0; 120 } 121 122 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) 123 { 124 struct crash_mem *cmem = arg; 125 126 cmem->ranges[cmem->nr_ranges].start = res->start; 127 cmem->ranges[cmem->nr_ranges].end = res->end; 128 cmem->nr_ranges++; 129 130 return 0; 131 } 132 133 static int prepare_elf_headers(void **addr, unsigned long *sz) 134 { 135 struct crash_mem *cmem; 136 unsigned int nr_ranges; 137 int ret; 138 139 nr_ranges = 1; /* For exclusion of crashkernel region */ 140 walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); 141 142 cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL); 143 if (!cmem) 144 return -ENOMEM; 145 146 cmem->max_nr_ranges = nr_ranges; 147 cmem->nr_ranges = 0; 148 ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); 149 if (ret) 150 goto out; 151 152 /* Exclude crashkernel region */ 153 ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); 154 if (!ret) 155 ret = crash_prepare_elf64_headers(cmem, true, addr, sz); 156 157 out: 158 kfree(cmem); 159 return ret; 160 } 161 162 static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, 163 unsigned long cmdline_len) 164 { 165 int elfcorehdr_strlen; 166 char *cmdline_ptr; 167 168 cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL); 169 if (!cmdline_ptr) 170 return NULL; 171 172 elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ", 173 image->elf_load_addr); 174 175 if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) { 176 pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n"); 177 kfree(cmdline_ptr); 178 return NULL; 179 } 180 181 memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len); 182 /* Ensure it's nul terminated */ 183 cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0'; 184 return cmdline_ptr; 185 } 186 187 static void *elf_kexec_load(struct kimage *image, char *kernel_buf, 188 unsigned long kernel_len, char *initrd, 189 unsigned long initrd_len, char *cmdline, 190 unsigned long cmdline_len) 191 { 192 int ret; 193 unsigned long old_kernel_pbase = ULONG_MAX; 194 unsigned long new_kernel_pbase = 0UL; 195 unsigned long initrd_pbase = 0UL; 196 unsigned long headers_sz; 197 unsigned long kernel_start; 198 void *fdt, *headers; 199 struct elfhdr ehdr; 200 struct kexec_buf kbuf; 201 struct kexec_elf_info elf_info; 202 char *modified_cmdline = NULL; 203 204 ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info); 205 if (ret) 206 return ERR_PTR(ret); 207 208 ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info, 209 &old_kernel_pbase, &new_kernel_pbase); 210 if (ret) 211 goto out; 212 kernel_start = image->start; 213 pr_notice("The entry point of kernel at 0x%lx\n", image->start); 214 215 /* Add the kernel binary to the image */ 216 ret = riscv_kexec_elf_load(image, &ehdr, &elf_info, 217 old_kernel_pbase, new_kernel_pbase); 218 if (ret) 219 goto out; 220 221 kbuf.image = image; 222 kbuf.buf_min = new_kernel_pbase + kernel_len; 223 kbuf.buf_max = ULONG_MAX; 224 225 /* Add elfcorehdr */ 226 if (image->type == KEXEC_TYPE_CRASH) { 227 ret = prepare_elf_headers(&headers, &headers_sz); 228 if (ret) { 229 pr_err("Preparing elf core header failed\n"); 230 goto out; 231 } 232 233 kbuf.buffer = headers; 234 kbuf.bufsz = headers_sz; 235 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 236 kbuf.memsz = headers_sz; 237 kbuf.buf_align = ELF_CORE_HEADER_ALIGN; 238 kbuf.top_down = true; 239 240 ret = kexec_add_buffer(&kbuf); 241 if (ret) { 242 vfree(headers); 243 goto out; 244 } 245 image->elf_headers = headers; 246 image->elf_load_addr = kbuf.mem; 247 image->elf_headers_sz = headers_sz; 248 249 pr_debug("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n", 250 image->elf_load_addr, kbuf.bufsz, kbuf.memsz); 251 252 /* Setup cmdline for kdump kernel case */ 253 modified_cmdline = setup_kdump_cmdline(image, cmdline, 254 cmdline_len); 255 if (!modified_cmdline) { 256 pr_err("Setting up cmdline for kdump kernel failed\n"); 257 ret = -EINVAL; 258 goto out; 259 } 260 cmdline = modified_cmdline; 261 } 262 263 #ifdef CONFIG_ARCH_HAS_KEXEC_PURGATORY 264 /* Add purgatory to the image */ 265 kbuf.top_down = true; 266 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 267 ret = kexec_load_purgatory(image, &kbuf); 268 if (ret) { 269 pr_err("Error loading purgatory ret=%d\n", ret); 270 goto out; 271 } 272 ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry", 273 &kernel_start, 274 sizeof(kernel_start), 0); 275 if (ret) 276 pr_err("Error update purgatory ret=%d\n", ret); 277 #endif /* CONFIG_ARCH_HAS_KEXEC_PURGATORY */ 278 279 /* Add the initrd to the image */ 280 if (initrd != NULL) { 281 kbuf.buffer = initrd; 282 kbuf.bufsz = kbuf.memsz = initrd_len; 283 kbuf.buf_align = PAGE_SIZE; 284 kbuf.top_down = false; 285 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 286 ret = kexec_add_buffer(&kbuf); 287 if (ret) 288 goto out; 289 initrd_pbase = kbuf.mem; 290 pr_notice("Loaded initrd at 0x%lx\n", initrd_pbase); 291 } 292 293 /* Add the DTB to the image */ 294 fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase, 295 initrd_len, cmdline, 0); 296 if (!fdt) { 297 pr_err("Error setting up the new device tree.\n"); 298 ret = -EINVAL; 299 goto out; 300 } 301 302 fdt_pack(fdt); 303 kbuf.buffer = fdt; 304 kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt); 305 kbuf.buf_align = PAGE_SIZE; 306 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; 307 kbuf.top_down = true; 308 ret = kexec_add_buffer(&kbuf); 309 if (ret) { 310 pr_err("Error add DTB kbuf ret=%d\n", ret); 311 goto out_free_fdt; 312 } 313 /* Cache the fdt buffer address for memory cleanup */ 314 image->arch.fdt = fdt; 315 pr_notice("Loaded device tree at 0x%lx\n", kbuf.mem); 316 goto out; 317 318 out_free_fdt: 319 kvfree(fdt); 320 out: 321 kfree(modified_cmdline); 322 kexec_free_elf_info(&elf_info); 323 return ret ? ERR_PTR(ret) : NULL; 324 } 325 326 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 327 #define RISCV_IMM_BITS 12 328 #define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS) 329 #define RISCV_CONST_HIGH_PART(x) \ 330 (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1)) 331 #define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x)) 332 333 #define ENCODE_ITYPE_IMM(x) \ 334 (RV_X(x, 0, 12) << 20) 335 #define ENCODE_BTYPE_IMM(x) \ 336 ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \ 337 (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31)) 338 #define ENCODE_UTYPE_IMM(x) \ 339 (RV_X(x, 12, 20) << 12) 340 #define ENCODE_JTYPE_IMM(x) \ 341 ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \ 342 (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31)) 343 #define ENCODE_CBTYPE_IMM(x) \ 344 ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \ 345 (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12)) 346 #define ENCODE_CJTYPE_IMM(x) \ 347 ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \ 348 (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \ 349 (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12)) 350 #define ENCODE_UJTYPE_IMM(x) \ 351 (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \ 352 (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32)) 353 #define ENCODE_UITYPE_IMM(x) \ 354 (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32)) 355 356 #define CLEAN_IMM(type, x) \ 357 ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x)) 358 359 int arch_kexec_apply_relocations_add(struct purgatory_info *pi, 360 Elf_Shdr *section, 361 const Elf_Shdr *relsec, 362 const Elf_Shdr *symtab) 363 { 364 const char *strtab, *name, *shstrtab; 365 const Elf_Shdr *sechdrs; 366 Elf64_Rela *relas; 367 int i, r_type; 368 369 /* String & section header string table */ 370 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff; 371 strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset; 372 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset; 373 374 relas = (void *)pi->ehdr + relsec->sh_offset; 375 376 for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) { 377 const Elf_Sym *sym; /* symbol to relocate */ 378 unsigned long addr; /* final location after relocation */ 379 unsigned long val; /* relocated symbol value */ 380 unsigned long sec_base; /* relocated symbol value */ 381 void *loc; /* tmp location to modify */ 382 383 sym = (void *)pi->ehdr + symtab->sh_offset; 384 sym += ELF64_R_SYM(relas[i].r_info); 385 386 if (sym->st_name) 387 name = strtab + sym->st_name; 388 else 389 name = shstrtab + sechdrs[sym->st_shndx].sh_name; 390 391 loc = pi->purgatory_buf; 392 loc += section->sh_offset; 393 loc += relas[i].r_offset; 394 395 if (sym->st_shndx == SHN_ABS) 396 sec_base = 0; 397 else if (sym->st_shndx >= pi->ehdr->e_shnum) { 398 pr_err("Invalid section %d for symbol %s\n", 399 sym->st_shndx, name); 400 return -ENOEXEC; 401 } else 402 sec_base = pi->sechdrs[sym->st_shndx].sh_addr; 403 404 val = sym->st_value; 405 val += sec_base; 406 val += relas[i].r_addend; 407 408 addr = section->sh_addr + relas[i].r_offset; 409 410 r_type = ELF64_R_TYPE(relas[i].r_info); 411 412 switch (r_type) { 413 case R_RISCV_BRANCH: 414 *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) | 415 ENCODE_BTYPE_IMM(val - addr); 416 break; 417 case R_RISCV_JAL: 418 *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) | 419 ENCODE_JTYPE_IMM(val - addr); 420 break; 421 /* 422 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I 423 * sym is expected to be next to R_RISCV_PCREL_HI20 424 * in purgatory relsec. Handle it like R_RISCV_CALL 425 * sym, instead of searching the whole relsec. 426 */ 427 case R_RISCV_PCREL_HI20: 428 case R_RISCV_CALL: 429 *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) | 430 ENCODE_UJTYPE_IMM(val - addr); 431 break; 432 case R_RISCV_RVC_BRANCH: 433 *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) | 434 ENCODE_CBTYPE_IMM(val - addr); 435 break; 436 case R_RISCV_RVC_JUMP: 437 *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) | 438 ENCODE_CJTYPE_IMM(val - addr); 439 break; 440 case R_RISCV_ADD32: 441 *(u32 *)loc += val; 442 break; 443 case R_RISCV_SUB32: 444 *(u32 *)loc -= val; 445 break; 446 /* It has been applied by R_RISCV_PCREL_HI20 sym */ 447 case R_RISCV_PCREL_LO12_I: 448 case R_RISCV_ALIGN: 449 case R_RISCV_RELAX: 450 break; 451 default: 452 pr_err("Unknown rela relocation: %d\n", r_type); 453 return -ENOEXEC; 454 } 455 } 456 return 0; 457 } 458 459 const struct kexec_file_ops elf_kexec_ops = { 460 .probe = kexec_elf_probe, 461 .load = elf_kexec_load, 462 }; 463