1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * kaslr.c 4 * 5 * This contains the routines needed to generate a reasonable level of 6 * entropy to choose a randomized kernel base address offset in support 7 * of Kernel Address Space Layout Randomization (KASLR). Additionally 8 * handles walking the physical memory maps (and tracking memory regions 9 * to avoid) in order to select a physical memory location that can 10 * contain the entire properly aligned running kernel image. 11 * 12 */ 13 14 /* 15 * isspace() in linux/ctype.h is expected by next_args() to filter 16 * out "space/lf/tab". While boot/ctype.h conflicts with linux/ctype.h, 17 * since isdigit() is implemented in both of them. Hence disable it 18 * here. 19 */ 20 #define BOOT_CTYPE_H 21 22 /* 23 * _ctype[] in lib/ctype.c is needed by isspace() of linux/ctype.h. 24 * While both lib/ctype.c and lib/cmdline.c will bring EXPORT_SYMBOL 25 * which is meaningless and will cause compiling error in some cases. 26 */ 27 #define __DISABLE_EXPORTS 28 29 #include "misc.h" 30 #include "error.h" 31 #include "../string.h" 32 33 #include <generated/compile.h> 34 #include <linux/module.h> 35 #include <linux/uts.h> 36 #include <linux/utsname.h> 37 #include <linux/ctype.h> 38 #include <linux/efi.h> 39 #include <generated/utsrelease.h> 40 #include <asm/efi.h> 41 42 /* Macros used by the included decompressor code below. */ 43 #define STATIC 44 #include <linux/decompress/mm.h> 45 46 #ifdef CONFIG_X86_5LEVEL 47 unsigned int __pgtable_l5_enabled; 48 unsigned int pgdir_shift __ro_after_init = 39; 49 unsigned int ptrs_per_p4d __ro_after_init = 1; 50 #endif 51 52 extern unsigned long get_cmd_line_ptr(void); 53 54 /* Used by PAGE_KERN* macros: */ 55 pteval_t __default_kernel_pte_mask __read_mostly = ~0; 56 57 /* Simplified build-specific string for starting entropy. */ 58 static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@" 59 LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION; 60 61 static unsigned long rotate_xor(unsigned long hash, const void *area, 62 size_t size) 63 { 64 size_t i; 65 unsigned long *ptr = (unsigned long *)area; 66 67 for (i = 0; i < size / sizeof(hash); i++) { 68 /* Rotate by odd number of bits and XOR. */ 69 hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); 70 hash ^= ptr[i]; 71 } 72 73 return hash; 74 } 75 76 /* Attempt to create a simple but unpredictable starting entropy. */ 77 static unsigned long get_boot_seed(void) 78 { 79 unsigned long hash = 0; 80 81 hash = rotate_xor(hash, build_str, sizeof(build_str)); 82 hash = rotate_xor(hash, boot_params, sizeof(*boot_params)); 83 84 return hash; 85 } 86 87 #define KASLR_COMPRESSED_BOOT 88 #include "../../lib/kaslr.c" 89 90 struct mem_vector { 91 unsigned long long start; 92 unsigned long long size; 93 }; 94 95 /* Only supporting at most 4 unusable memmap regions with kaslr */ 96 #define MAX_MEMMAP_REGIONS 4 97 98 static bool memmap_too_large; 99 100 101 /* Store memory limit specified by "mem=nn[KMG]" or "memmap=nn[KMG]" */ 102 static unsigned long long mem_limit = ULLONG_MAX; 103 104 105 enum mem_avoid_index { 106 MEM_AVOID_ZO_RANGE = 0, 107 MEM_AVOID_INITRD, 108 MEM_AVOID_CMDLINE, 109 MEM_AVOID_BOOTPARAMS, 110 MEM_AVOID_MEMMAP_BEGIN, 111 MEM_AVOID_MEMMAP_END = MEM_AVOID_MEMMAP_BEGIN + MAX_MEMMAP_REGIONS - 1, 112 MEM_AVOID_MAX, 113 }; 114 115 static struct mem_vector mem_avoid[MEM_AVOID_MAX]; 116 117 static bool mem_overlaps(struct mem_vector *one, struct mem_vector *two) 118 { 119 /* Item one is entirely before item two. */ 120 if (one->start + one->size <= two->start) 121 return false; 122 /* Item one is entirely after item two. */ 123 if (one->start >= two->start + two->size) 124 return false; 125 return true; 126 } 127 128 char *skip_spaces(const char *str) 129 { 130 while (isspace(*str)) 131 ++str; 132 return (char *)str; 133 } 134 #include "../../../../lib/ctype.c" 135 #include "../../../../lib/cmdline.c" 136 137 static int 138 parse_memmap(char *p, unsigned long long *start, unsigned long long *size) 139 { 140 char *oldp; 141 142 if (!p) 143 return -EINVAL; 144 145 /* We don't care about this option here */ 146 if (!strncmp(p, "exactmap", 8)) 147 return -EINVAL; 148 149 oldp = p; 150 *size = memparse(p, &p); 151 if (p == oldp) 152 return -EINVAL; 153 154 switch (*p) { 155 case '#': 156 case '$': 157 case '!': 158 *start = memparse(p + 1, &p); 159 return 0; 160 case '@': 161 /* memmap=nn@ss specifies usable region, should be skipped */ 162 *size = 0; 163 /* Fall through */ 164 default: 165 /* 166 * If w/o offset, only size specified, memmap=nn[KMG] has the 167 * same behaviour as mem=nn[KMG]. It limits the max address 168 * system can use. Region above the limit should be avoided. 169 */ 170 *start = 0; 171 return 0; 172 } 173 174 return -EINVAL; 175 } 176 177 static void mem_avoid_memmap(char *str) 178 { 179 static int i; 180 181 if (i >= MAX_MEMMAP_REGIONS) 182 return; 183 184 while (str && (i < MAX_MEMMAP_REGIONS)) { 185 int rc; 186 unsigned long long start, size; 187 char *k = strchr(str, ','); 188 189 if (k) 190 *k++ = 0; 191 192 rc = parse_memmap(str, &start, &size); 193 if (rc < 0) 194 break; 195 str = k; 196 197 if (start == 0) { 198 /* Store the specified memory limit if size > 0 */ 199 if (size > 0) 200 mem_limit = size; 201 202 continue; 203 } 204 205 mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].start = start; 206 mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].size = size; 207 i++; 208 } 209 210 /* More than 4 memmaps, fail kaslr */ 211 if ((i >= MAX_MEMMAP_REGIONS) && str) 212 memmap_too_large = true; 213 } 214 215 /* Store the number of 1GB huge pages which users specified: */ 216 static unsigned long max_gb_huge_pages; 217 218 static void parse_gb_huge_pages(char *param, char *val) 219 { 220 static bool gbpage_sz; 221 char *p; 222 223 if (!strcmp(param, "hugepagesz")) { 224 p = val; 225 if (memparse(p, &p) != PUD_SIZE) { 226 gbpage_sz = false; 227 return; 228 } 229 230 if (gbpage_sz) 231 warn("Repeatedly set hugeTLB page size of 1G!\n"); 232 gbpage_sz = true; 233 return; 234 } 235 236 if (!strcmp(param, "hugepages") && gbpage_sz) { 237 p = val; 238 max_gb_huge_pages = simple_strtoull(p, &p, 0); 239 return; 240 } 241 } 242 243 244 static int handle_mem_options(void) 245 { 246 char *args = (char *)get_cmd_line_ptr(); 247 size_t len = strlen((char *)args); 248 char *tmp_cmdline; 249 char *param, *val; 250 u64 mem_size; 251 252 if (!strstr(args, "memmap=") && !strstr(args, "mem=") && 253 !strstr(args, "hugepages")) 254 return 0; 255 256 tmp_cmdline = malloc(len + 1); 257 if (!tmp_cmdline) 258 error("Failed to allocate space for tmp_cmdline"); 259 260 memcpy(tmp_cmdline, args, len); 261 tmp_cmdline[len] = 0; 262 args = tmp_cmdline; 263 264 /* Chew leading spaces */ 265 args = skip_spaces(args); 266 267 while (*args) { 268 args = next_arg(args, ¶m, &val); 269 /* Stop at -- */ 270 if (!val && strcmp(param, "--") == 0) { 271 warn("Only '--' specified in cmdline"); 272 free(tmp_cmdline); 273 return -1; 274 } 275 276 if (!strcmp(param, "memmap")) { 277 mem_avoid_memmap(val); 278 } else if (strstr(param, "hugepages")) { 279 parse_gb_huge_pages(param, val); 280 } else if (!strcmp(param, "mem")) { 281 char *p = val; 282 283 if (!strcmp(p, "nopentium")) 284 continue; 285 mem_size = memparse(p, &p); 286 if (mem_size == 0) { 287 free(tmp_cmdline); 288 return -EINVAL; 289 } 290 mem_limit = mem_size; 291 } 292 } 293 294 free(tmp_cmdline); 295 return 0; 296 } 297 298 /* 299 * In theory, KASLR can put the kernel anywhere in the range of [16M, 64T). 300 * The mem_avoid array is used to store the ranges that need to be avoided 301 * when KASLR searches for an appropriate random address. We must avoid any 302 * regions that are unsafe to overlap with during decompression, and other 303 * things like the initrd, cmdline and boot_params. This comment seeks to 304 * explain mem_avoid as clearly as possible since incorrect mem_avoid 305 * memory ranges lead to really hard to debug boot failures. 306 * 307 * The initrd, cmdline, and boot_params are trivial to identify for 308 * avoiding. They are MEM_AVOID_INITRD, MEM_AVOID_CMDLINE, and 309 * MEM_AVOID_BOOTPARAMS respectively below. 310 * 311 * What is not obvious how to avoid is the range of memory that is used 312 * during decompression (MEM_AVOID_ZO_RANGE below). This range must cover 313 * the compressed kernel (ZO) and its run space, which is used to extract 314 * the uncompressed kernel (VO) and relocs. 315 * 316 * ZO's full run size sits against the end of the decompression buffer, so 317 * we can calculate where text, data, bss, etc of ZO are positioned more 318 * easily. 319 * 320 * For additional background, the decompression calculations can be found 321 * in header.S, and the memory diagram is based on the one found in misc.c. 322 * 323 * The following conditions are already enforced by the image layouts and 324 * associated code: 325 * - input + input_size >= output + output_size 326 * - kernel_total_size <= init_size 327 * - kernel_total_size <= output_size (see Note below) 328 * - output + init_size >= output + output_size 329 * 330 * (Note that kernel_total_size and output_size have no fundamental 331 * relationship, but output_size is passed to choose_random_location 332 * as a maximum of the two. The diagram is showing a case where 333 * kernel_total_size is larger than output_size, but this case is 334 * handled by bumping output_size.) 335 * 336 * The above conditions can be illustrated by a diagram: 337 * 338 * 0 output input input+input_size output+init_size 339 * | | | | | 340 * | | | | | 341 * |-----|--------|--------|--------------|-----------|--|-------------| 342 * | | | 343 * | | | 344 * output+init_size-ZO_INIT_SIZE output+output_size output+kernel_total_size 345 * 346 * [output, output+init_size) is the entire memory range used for 347 * extracting the compressed image. 348 * 349 * [output, output+kernel_total_size) is the range needed for the 350 * uncompressed kernel (VO) and its run size (bss, brk, etc). 351 * 352 * [output, output+output_size) is VO plus relocs (i.e. the entire 353 * uncompressed payload contained by ZO). This is the area of the buffer 354 * written to during decompression. 355 * 356 * [output+init_size-ZO_INIT_SIZE, output+init_size) is the worst-case 357 * range of the copied ZO and decompression code. (i.e. the range 358 * covered backwards of size ZO_INIT_SIZE, starting from output+init_size.) 359 * 360 * [input, input+input_size) is the original copied compressed image (ZO) 361 * (i.e. it does not include its run size). This range must be avoided 362 * because it contains the data used for decompression. 363 * 364 * [input+input_size, output+init_size) is [_text, _end) for ZO. This 365 * range includes ZO's heap and stack, and must be avoided since it 366 * performs the decompression. 367 * 368 * Since the above two ranges need to be avoided and they are adjacent, 369 * they can be merged, resulting in: [input, output+init_size) which 370 * becomes the MEM_AVOID_ZO_RANGE below. 371 */ 372 static void mem_avoid_init(unsigned long input, unsigned long input_size, 373 unsigned long output) 374 { 375 unsigned long init_size = boot_params->hdr.init_size; 376 u64 initrd_start, initrd_size; 377 u64 cmd_line, cmd_line_size; 378 char *ptr; 379 380 /* 381 * Avoid the region that is unsafe to overlap during 382 * decompression. 383 */ 384 mem_avoid[MEM_AVOID_ZO_RANGE].start = input; 385 mem_avoid[MEM_AVOID_ZO_RANGE].size = (output + init_size) - input; 386 add_identity_map(mem_avoid[MEM_AVOID_ZO_RANGE].start, 387 mem_avoid[MEM_AVOID_ZO_RANGE].size); 388 389 /* Avoid initrd. */ 390 initrd_start = (u64)boot_params->ext_ramdisk_image << 32; 391 initrd_start |= boot_params->hdr.ramdisk_image; 392 initrd_size = (u64)boot_params->ext_ramdisk_size << 32; 393 initrd_size |= boot_params->hdr.ramdisk_size; 394 mem_avoid[MEM_AVOID_INITRD].start = initrd_start; 395 mem_avoid[MEM_AVOID_INITRD].size = initrd_size; 396 /* No need to set mapping for initrd, it will be handled in VO. */ 397 398 /* Avoid kernel command line. */ 399 cmd_line = (u64)boot_params->ext_cmd_line_ptr << 32; 400 cmd_line |= boot_params->hdr.cmd_line_ptr; 401 /* Calculate size of cmd_line. */ 402 ptr = (char *)(unsigned long)cmd_line; 403 for (cmd_line_size = 0; ptr[cmd_line_size++];) 404 ; 405 mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line; 406 mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size; 407 add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start, 408 mem_avoid[MEM_AVOID_CMDLINE].size); 409 410 /* Avoid boot parameters. */ 411 mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params; 412 mem_avoid[MEM_AVOID_BOOTPARAMS].size = sizeof(*boot_params); 413 add_identity_map(mem_avoid[MEM_AVOID_BOOTPARAMS].start, 414 mem_avoid[MEM_AVOID_BOOTPARAMS].size); 415 416 /* We don't need to set a mapping for setup_data. */ 417 418 /* Mark the memmap regions we need to avoid */ 419 handle_mem_options(); 420 421 #ifdef CONFIG_X86_VERBOSE_BOOTUP 422 /* Make sure video RAM can be used. */ 423 add_identity_map(0, PMD_SIZE); 424 #endif 425 } 426 427 /* 428 * Does this memory vector overlap a known avoided area? If so, record the 429 * overlap region with the lowest address. 430 */ 431 static bool mem_avoid_overlap(struct mem_vector *img, 432 struct mem_vector *overlap) 433 { 434 int i; 435 struct setup_data *ptr; 436 unsigned long earliest = img->start + img->size; 437 bool is_overlapping = false; 438 439 for (i = 0; i < MEM_AVOID_MAX; i++) { 440 if (mem_overlaps(img, &mem_avoid[i]) && 441 mem_avoid[i].start < earliest) { 442 *overlap = mem_avoid[i]; 443 earliest = overlap->start; 444 is_overlapping = true; 445 } 446 } 447 448 /* Avoid all entries in the setup_data linked list. */ 449 ptr = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data; 450 while (ptr) { 451 struct mem_vector avoid; 452 453 avoid.start = (unsigned long)ptr; 454 avoid.size = sizeof(*ptr) + ptr->len; 455 456 if (mem_overlaps(img, &avoid) && (avoid.start < earliest)) { 457 *overlap = avoid; 458 earliest = overlap->start; 459 is_overlapping = true; 460 } 461 462 ptr = (struct setup_data *)(unsigned long)ptr->next; 463 } 464 465 return is_overlapping; 466 } 467 468 struct slot_area { 469 unsigned long addr; 470 int num; 471 }; 472 473 #define MAX_SLOT_AREA 100 474 475 static struct slot_area slot_areas[MAX_SLOT_AREA]; 476 477 static unsigned long slot_max; 478 479 static unsigned long slot_area_index; 480 481 static void store_slot_info(struct mem_vector *region, unsigned long image_size) 482 { 483 struct slot_area slot_area; 484 485 if (slot_area_index == MAX_SLOT_AREA) 486 return; 487 488 slot_area.addr = region->start; 489 slot_area.num = (region->size - image_size) / 490 CONFIG_PHYSICAL_ALIGN + 1; 491 492 if (slot_area.num > 0) { 493 slot_areas[slot_area_index++] = slot_area; 494 slot_max += slot_area.num; 495 } 496 } 497 498 /* 499 * Skip as many 1GB huge pages as possible in the passed region 500 * according to the number which users specified: 501 */ 502 static void 503 process_gb_huge_pages(struct mem_vector *region, unsigned long image_size) 504 { 505 unsigned long addr, size = 0; 506 struct mem_vector tmp; 507 int i = 0; 508 509 if (!max_gb_huge_pages) { 510 store_slot_info(region, image_size); 511 return; 512 } 513 514 addr = ALIGN(region->start, PUD_SIZE); 515 /* Did we raise the address above the passed in memory entry? */ 516 if (addr < region->start + region->size) 517 size = region->size - (addr - region->start); 518 519 /* Check how many 1GB huge pages can be filtered out: */ 520 while (size > PUD_SIZE && max_gb_huge_pages) { 521 size -= PUD_SIZE; 522 max_gb_huge_pages--; 523 i++; 524 } 525 526 /* No good 1GB huge pages found: */ 527 if (!i) { 528 store_slot_info(region, image_size); 529 return; 530 } 531 532 /* 533 * Skip those 'i'*1GB good huge pages, and continue checking and 534 * processing the remaining head or tail part of the passed region 535 * if available. 536 */ 537 538 if (addr >= region->start + image_size) { 539 tmp.start = region->start; 540 tmp.size = addr - region->start; 541 store_slot_info(&tmp, image_size); 542 } 543 544 size = region->size - (addr - region->start) - i * PUD_SIZE; 545 if (size >= image_size) { 546 tmp.start = addr + i * PUD_SIZE; 547 tmp.size = size; 548 store_slot_info(&tmp, image_size); 549 } 550 } 551 552 static unsigned long slots_fetch_random(void) 553 { 554 unsigned long slot; 555 int i; 556 557 /* Handle case of no slots stored. */ 558 if (slot_max == 0) 559 return 0; 560 561 slot = kaslr_get_random_long("Physical") % slot_max; 562 563 for (i = 0; i < slot_area_index; i++) { 564 if (slot >= slot_areas[i].num) { 565 slot -= slot_areas[i].num; 566 continue; 567 } 568 return slot_areas[i].addr + slot * CONFIG_PHYSICAL_ALIGN; 569 } 570 571 if (i == slot_area_index) 572 debug_putstr("slots_fetch_random() failed!?\n"); 573 return 0; 574 } 575 576 static void process_mem_region(struct mem_vector *entry, 577 unsigned long minimum, 578 unsigned long image_size) 579 { 580 struct mem_vector region, overlap; 581 struct slot_area slot_area; 582 unsigned long start_orig, end; 583 struct mem_vector cur_entry; 584 585 /* On 32-bit, ignore entries entirely above our maximum. */ 586 if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE) 587 return; 588 589 /* Ignore entries entirely below our minimum. */ 590 if (entry->start + entry->size < minimum) 591 return; 592 593 /* Ignore entries above memory limit */ 594 end = min(entry->size + entry->start, mem_limit); 595 if (entry->start >= end) 596 return; 597 cur_entry.start = entry->start; 598 cur_entry.size = end - entry->start; 599 600 region.start = cur_entry.start; 601 region.size = cur_entry.size; 602 603 /* Give up if slot area array is full. */ 604 while (slot_area_index < MAX_SLOT_AREA) { 605 start_orig = region.start; 606 607 /* Potentially raise address to minimum location. */ 608 if (region.start < minimum) 609 region.start = minimum; 610 611 /* Potentially raise address to meet alignment needs. */ 612 region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN); 613 614 /* Did we raise the address above the passed in memory entry? */ 615 if (region.start > cur_entry.start + cur_entry.size) 616 return; 617 618 /* Reduce size by any delta from the original address. */ 619 region.size -= region.start - start_orig; 620 621 /* On 32-bit, reduce region size to fit within max size. */ 622 if (IS_ENABLED(CONFIG_X86_32) && 623 region.start + region.size > KERNEL_IMAGE_SIZE) 624 region.size = KERNEL_IMAGE_SIZE - region.start; 625 626 /* Return if region can't contain decompressed kernel */ 627 if (region.size < image_size) 628 return; 629 630 /* If nothing overlaps, store the region and return. */ 631 if (!mem_avoid_overlap(®ion, &overlap)) { 632 process_gb_huge_pages(®ion, image_size); 633 return; 634 } 635 636 /* Store beginning of region if holds at least image_size. */ 637 if (overlap.start > region.start + image_size) { 638 struct mem_vector beginning; 639 640 beginning.start = region.start; 641 beginning.size = overlap.start - region.start; 642 process_gb_huge_pages(&beginning, image_size); 643 } 644 645 /* Return if overlap extends to or past end of region. */ 646 if (overlap.start + overlap.size >= region.start + region.size) 647 return; 648 649 /* Clip off the overlapping region and start over. */ 650 region.size -= overlap.start - region.start + overlap.size; 651 region.start = overlap.start + overlap.size; 652 } 653 } 654 655 #ifdef CONFIG_EFI 656 /* 657 * Returns true if mirror region found (and must have been processed 658 * for slots adding) 659 */ 660 static bool 661 process_efi_entries(unsigned long minimum, unsigned long image_size) 662 { 663 struct efi_info *e = &boot_params->efi_info; 664 bool efi_mirror_found = false; 665 struct mem_vector region; 666 efi_memory_desc_t *md; 667 unsigned long pmap; 668 char *signature; 669 u32 nr_desc; 670 int i; 671 672 signature = (char *)&e->efi_loader_signature; 673 if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) && 674 strncmp(signature, EFI64_LOADER_SIGNATURE, 4)) 675 return false; 676 677 #ifdef CONFIG_X86_32 678 /* Can't handle data above 4GB at this time */ 679 if (e->efi_memmap_hi) { 680 warn("EFI memmap is above 4GB, can't be handled now on x86_32. EFI should be disabled.\n"); 681 return false; 682 } 683 pmap = e->efi_memmap; 684 #else 685 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); 686 #endif 687 688 nr_desc = e->efi_memmap_size / e->efi_memdesc_size; 689 for (i = 0; i < nr_desc; i++) { 690 md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i); 691 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { 692 efi_mirror_found = true; 693 break; 694 } 695 } 696 697 for (i = 0; i < nr_desc; i++) { 698 md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i); 699 700 /* 701 * Here we are more conservative in picking free memory than 702 * the EFI spec allows: 703 * 704 * According to the spec, EFI_BOOT_SERVICES_{CODE|DATA} are also 705 * free memory and thus available to place the kernel image into, 706 * but in practice there's firmware where using that memory leads 707 * to crashes. 708 * 709 * Only EFI_CONVENTIONAL_MEMORY is guaranteed to be free. 710 */ 711 if (md->type != EFI_CONVENTIONAL_MEMORY) 712 continue; 713 714 if (efi_mirror_found && 715 !(md->attribute & EFI_MEMORY_MORE_RELIABLE)) 716 continue; 717 718 region.start = md->phys_addr; 719 region.size = md->num_pages << EFI_PAGE_SHIFT; 720 process_mem_region(®ion, minimum, image_size); 721 if (slot_area_index == MAX_SLOT_AREA) { 722 debug_putstr("Aborted EFI scan (slot_areas full)!\n"); 723 break; 724 } 725 } 726 return true; 727 } 728 #else 729 static inline bool 730 process_efi_entries(unsigned long minimum, unsigned long image_size) 731 { 732 return false; 733 } 734 #endif 735 736 static void process_e820_entries(unsigned long minimum, 737 unsigned long image_size) 738 { 739 int i; 740 struct mem_vector region; 741 struct boot_e820_entry *entry; 742 743 /* Verify potential e820 positions, appending to slots list. */ 744 for (i = 0; i < boot_params->e820_entries; i++) { 745 entry = &boot_params->e820_table[i]; 746 /* Skip non-RAM entries. */ 747 if (entry->type != E820_TYPE_RAM) 748 continue; 749 region.start = entry->addr; 750 region.size = entry->size; 751 process_mem_region(®ion, minimum, image_size); 752 if (slot_area_index == MAX_SLOT_AREA) { 753 debug_putstr("Aborted e820 scan (slot_areas full)!\n"); 754 break; 755 } 756 } 757 } 758 759 static unsigned long find_random_phys_addr(unsigned long minimum, 760 unsigned long image_size) 761 { 762 /* Check if we had too many memmaps. */ 763 if (memmap_too_large) { 764 debug_putstr("Aborted memory entries scan (more than 4 memmap= args)!\n"); 765 return 0; 766 } 767 768 /* Make sure minimum is aligned. */ 769 minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN); 770 771 if (process_efi_entries(minimum, image_size)) 772 return slots_fetch_random(); 773 774 process_e820_entries(minimum, image_size); 775 return slots_fetch_random(); 776 } 777 778 static unsigned long find_random_virt_addr(unsigned long minimum, 779 unsigned long image_size) 780 { 781 unsigned long slots, random_addr; 782 783 /* Make sure minimum is aligned. */ 784 minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN); 785 /* Align image_size for easy slot calculations. */ 786 image_size = ALIGN(image_size, CONFIG_PHYSICAL_ALIGN); 787 788 /* 789 * There are how many CONFIG_PHYSICAL_ALIGN-sized slots 790 * that can hold image_size within the range of minimum to 791 * KERNEL_IMAGE_SIZE? 792 */ 793 slots = (KERNEL_IMAGE_SIZE - minimum - image_size) / 794 CONFIG_PHYSICAL_ALIGN + 1; 795 796 random_addr = kaslr_get_random_long("Virtual") % slots; 797 798 return random_addr * CONFIG_PHYSICAL_ALIGN + minimum; 799 } 800 801 /* 802 * Since this function examines addresses much more numerically, 803 * it takes the input and output pointers as 'unsigned long'. 804 */ 805 void choose_random_location(unsigned long input, 806 unsigned long input_size, 807 unsigned long *output, 808 unsigned long output_size, 809 unsigned long *virt_addr) 810 { 811 unsigned long random_addr, min_addr; 812 813 if (cmdline_find_option_bool("nokaslr")) { 814 warn("KASLR disabled: 'nokaslr' on cmdline."); 815 return; 816 } 817 818 #ifdef CONFIG_X86_5LEVEL 819 if (__read_cr4() & X86_CR4_LA57) { 820 __pgtable_l5_enabled = 1; 821 pgdir_shift = 48; 822 ptrs_per_p4d = 512; 823 } 824 #endif 825 826 boot_params->hdr.loadflags |= KASLR_FLAG; 827 828 /* Prepare to add new identity pagetables on demand. */ 829 initialize_identity_maps(); 830 831 /* Record the various known unsafe memory ranges. */ 832 mem_avoid_init(input, input_size, *output); 833 834 /* 835 * Low end of the randomization range should be the 836 * smaller of 512M or the initial kernel image 837 * location: 838 */ 839 min_addr = min(*output, 512UL << 20); 840 841 /* Walk available memory entries to find a random address. */ 842 random_addr = find_random_phys_addr(min_addr, output_size); 843 if (!random_addr) { 844 warn("Physical KASLR disabled: no suitable memory region!"); 845 } else { 846 /* Update the new physical address location. */ 847 if (*output != random_addr) { 848 add_identity_map(random_addr, output_size); 849 *output = random_addr; 850 } 851 852 /* 853 * This loads the identity mapping page table. 854 * This should only be done if a new physical address 855 * is found for the kernel, otherwise we should keep 856 * the old page table to make it be like the "nokaslr" 857 * case. 858 */ 859 finalize_identity_maps(); 860 } 861 862 863 /* Pick random virtual address starting from LOAD_PHYSICAL_ADDR. */ 864 if (IS_ENABLED(CONFIG_X86_64)) 865 random_addr = find_random_virt_addr(LOAD_PHYSICAL_ADDR, output_size); 866 *virt_addr = random_addr; 867 } 868