1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/init/main.c 4 * 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 * 7 * GK 2/5/95 - Changed to support mounting root fs via NFS 8 * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96 9 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96 10 * Simplified starting of init: Michael A. Griffith <grif@acm.org> 11 */ 12 13 #define DEBUG /* Enable initcall_debug */ 14 15 #include <linux/types.h> 16 #include <linux/extable.h> 17 #include <linux/module.h> 18 #include <linux/proc_fs.h> 19 #include <linux/binfmts.h> 20 #include <linux/kernel.h> 21 #include <linux/syscalls.h> 22 #include <linux/stackprotector.h> 23 #include <linux/string.h> 24 #include <linux/ctype.h> 25 #include <linux/delay.h> 26 #include <linux/ioport.h> 27 #include <linux/init.h> 28 #include <linux/initrd.h> 29 #include <linux/memblock.h> 30 #include <linux/acpi.h> 31 #include <linux/bootconfig.h> 32 #include <linux/console.h> 33 #include <linux/nmi.h> 34 #include <linux/percpu.h> 35 #include <linux/kmod.h> 36 #include <linux/vmalloc.h> 37 #include <linux/kernel_stat.h> 38 #include <linux/start_kernel.h> 39 #include <linux/security.h> 40 #include <linux/smp.h> 41 #include <linux/profile.h> 42 #include <linux/rcupdate.h> 43 #include <linux/moduleparam.h> 44 #include <linux/kallsyms.h> 45 #include <linux/writeback.h> 46 #include <linux/cpu.h> 47 #include <linux/cpuset.h> 48 #include <linux/cgroup.h> 49 #include <linux/efi.h> 50 #include <linux/tick.h> 51 #include <linux/sched/isolation.h> 52 #include <linux/interrupt.h> 53 #include <linux/taskstats_kern.h> 54 #include <linux/delayacct.h> 55 #include <linux/unistd.h> 56 #include <linux/utsname.h> 57 #include <linux/rmap.h> 58 #include <linux/mempolicy.h> 59 #include <linux/key.h> 60 #include <linux/buffer_head.h> 61 #include <linux/page_ext.h> 62 #include <linux/debug_locks.h> 63 #include <linux/debugobjects.h> 64 #include <linux/lockdep.h> 65 #include <linux/kmemleak.h> 66 #include <linux/padata.h> 67 #include <linux/pid_namespace.h> 68 #include <linux/device/driver.h> 69 #include <linux/kthread.h> 70 #include <linux/sched.h> 71 #include <linux/sched/init.h> 72 #include <linux/signal.h> 73 #include <linux/idr.h> 74 #include <linux/kgdb.h> 75 #include <linux/ftrace.h> 76 #include <linux/async.h> 77 #include <linux/sfi.h> 78 #include <linux/shmem_fs.h> 79 #include <linux/slab.h> 80 #include <linux/perf_event.h> 81 #include <linux/ptrace.h> 82 #include <linux/pti.h> 83 #include <linux/blkdev.h> 84 #include <linux/elevator.h> 85 #include <linux/sched/clock.h> 86 #include <linux/sched/task.h> 87 #include <linux/sched/task_stack.h> 88 #include <linux/context_tracking.h> 89 #include <linux/random.h> 90 #include <linux/list.h> 91 #include <linux/integrity.h> 92 #include <linux/proc_ns.h> 93 #include <linux/io.h> 94 #include <linux/cache.h> 95 #include <linux/rodata_test.h> 96 #include <linux/jump_label.h> 97 #include <linux/mem_encrypt.h> 98 99 #include <asm/io.h> 100 #include <asm/bugs.h> 101 #include <asm/setup.h> 102 #include <asm/sections.h> 103 #include <asm/cacheflush.h> 104 105 #define CREATE_TRACE_POINTS 106 #include <trace/events/initcall.h> 107 108 static int kernel_init(void *); 109 110 extern void init_IRQ(void); 111 extern void radix_tree_init(void); 112 113 /* 114 * Debug helper: via this flag we know that we are in 'early bootup code' 115 * where only the boot processor is running with IRQ disabled. This means 116 * two things - IRQ must not be enabled before the flag is cleared and some 117 * operations which are not allowed with IRQ disabled are allowed while the 118 * flag is set. 119 */ 120 bool early_boot_irqs_disabled __read_mostly; 121 122 enum system_states system_state __read_mostly; 123 EXPORT_SYMBOL(system_state); 124 125 /* 126 * Boot command-line arguments 127 */ 128 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT 129 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT 130 131 extern void time_init(void); 132 /* Default late time init is NULL. archs can override this later. */ 133 void (*__initdata late_time_init)(void); 134 135 /* Untouched command line saved by arch-specific code. */ 136 char __initdata boot_command_line[COMMAND_LINE_SIZE]; 137 /* Untouched saved command line (eg. for /proc) */ 138 char *saved_command_line; 139 /* Command line for parameter parsing */ 140 static char *static_command_line; 141 /* Untouched extra command line */ 142 static char *extra_command_line; 143 /* Extra init arguments */ 144 static char *extra_init_args; 145 146 #ifdef CONFIG_BOOT_CONFIG 147 /* Is bootconfig on command line? */ 148 static bool bootconfig_found; 149 static bool initargs_found; 150 #else 151 # define bootconfig_found false 152 # define initargs_found false 153 #endif 154 155 static char *execute_command; 156 static char *ramdisk_execute_command; 157 158 /* 159 * Used to generate warnings if static_key manipulation functions are used 160 * before jump_label_init is called. 161 */ 162 bool static_key_initialized __read_mostly; 163 EXPORT_SYMBOL_GPL(static_key_initialized); 164 165 /* 166 * If set, this is an indication to the drivers that reset the underlying 167 * device before going ahead with the initialization otherwise driver might 168 * rely on the BIOS and skip the reset operation. 169 * 170 * This is useful if kernel is booting in an unreliable environment. 171 * For ex. kdump situation where previous kernel has crashed, BIOS has been 172 * skipped and devices will be in unknown state. 173 */ 174 unsigned int reset_devices; 175 EXPORT_SYMBOL(reset_devices); 176 177 static int __init set_reset_devices(char *str) 178 { 179 reset_devices = 1; 180 return 1; 181 } 182 183 __setup("reset_devices", set_reset_devices); 184 185 static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, }; 186 const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, }; 187 static const char *panic_later, *panic_param; 188 189 extern const struct obs_kernel_param __setup_start[], __setup_end[]; 190 191 static bool __init obsolete_checksetup(char *line) 192 { 193 const struct obs_kernel_param *p; 194 bool had_early_param = false; 195 196 p = __setup_start; 197 do { 198 int n = strlen(p->str); 199 if (parameqn(line, p->str, n)) { 200 if (p->early) { 201 /* Already done in parse_early_param? 202 * (Needs exact match on param part). 203 * Keep iterating, as we can have early 204 * params and __setups of same names 8( */ 205 if (line[n] == '\0' || line[n] == '=') 206 had_early_param = true; 207 } else if (!p->setup_func) { 208 pr_warn("Parameter %s is obsolete, ignored\n", 209 p->str); 210 return true; 211 } else if (p->setup_func(line + n)) 212 return true; 213 } 214 p++; 215 } while (p < __setup_end); 216 217 return had_early_param; 218 } 219 220 /* 221 * This should be approx 2 Bo*oMips to start (note initial shift), and will 222 * still work even if initially too large, it will just take slightly longer 223 */ 224 unsigned long loops_per_jiffy = (1<<12); 225 EXPORT_SYMBOL(loops_per_jiffy); 226 227 static int __init debug_kernel(char *str) 228 { 229 console_loglevel = CONSOLE_LOGLEVEL_DEBUG; 230 return 0; 231 } 232 233 static int __init quiet_kernel(char *str) 234 { 235 console_loglevel = CONSOLE_LOGLEVEL_QUIET; 236 return 0; 237 } 238 239 early_param("debug", debug_kernel); 240 early_param("quiet", quiet_kernel); 241 242 static int __init loglevel(char *str) 243 { 244 int newlevel; 245 246 /* 247 * Only update loglevel value when a correct setting was passed, 248 * to prevent blind crashes (when loglevel being set to 0) that 249 * are quite hard to debug 250 */ 251 if (get_option(&str, &newlevel)) { 252 console_loglevel = newlevel; 253 return 0; 254 } 255 256 return -EINVAL; 257 } 258 259 early_param("loglevel", loglevel); 260 261 #ifdef CONFIG_BLK_DEV_INITRD 262 static void * __init get_boot_config_from_initrd(u32 *_size, u32 *_csum) 263 { 264 u32 size, csum; 265 char *data; 266 u32 *hdr; 267 268 if (!initrd_end) 269 return NULL; 270 271 data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN; 272 if (memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN)) 273 return NULL; 274 275 hdr = (u32 *)(data - 8); 276 size = hdr[0]; 277 csum = hdr[1]; 278 279 data = ((void *)hdr) - size; 280 if ((unsigned long)data < initrd_start) { 281 pr_err("bootconfig size %d is greater than initrd size %ld\n", 282 size, initrd_end - initrd_start); 283 return NULL; 284 } 285 286 /* Remove bootconfig from initramfs/initrd */ 287 initrd_end = (unsigned long)data; 288 if (_size) 289 *_size = size; 290 if (_csum) 291 *_csum = csum; 292 293 return data; 294 } 295 #else 296 static void * __init get_boot_config_from_initrd(u32 *_size, u32 *_csum) 297 { 298 return NULL; 299 } 300 #endif 301 302 #ifdef CONFIG_BOOT_CONFIG 303 304 char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; 305 306 #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) 307 308 static int __init xbc_snprint_cmdline(char *buf, size_t size, 309 struct xbc_node *root) 310 { 311 struct xbc_node *knode, *vnode; 312 char *end = buf + size; 313 const char *val; 314 int ret; 315 316 xbc_node_for_each_key_value(root, knode, val) { 317 ret = xbc_node_compose_key_after(root, knode, 318 xbc_namebuf, XBC_KEYLEN_MAX); 319 if (ret < 0) 320 return ret; 321 322 vnode = xbc_node_get_child(knode); 323 if (!vnode) { 324 ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf); 325 if (ret < 0) 326 return ret; 327 buf += ret; 328 continue; 329 } 330 xbc_array_for_each_value(vnode, val) { 331 ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ", 332 xbc_namebuf, val); 333 if (ret < 0) 334 return ret; 335 buf += ret; 336 } 337 } 338 339 return buf - (end - size); 340 } 341 #undef rest 342 343 /* Make an extra command line under given key word */ 344 static char * __init xbc_make_cmdline(const char *key) 345 { 346 struct xbc_node *root; 347 char *new_cmdline; 348 int ret, len = 0; 349 350 root = xbc_find_node(key); 351 if (!root) 352 return NULL; 353 354 /* Count required buffer size */ 355 len = xbc_snprint_cmdline(NULL, 0, root); 356 if (len <= 0) 357 return NULL; 358 359 new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES); 360 if (!new_cmdline) { 361 pr_err("Failed to allocate memory for extra kernel cmdline.\n"); 362 return NULL; 363 } 364 365 ret = xbc_snprint_cmdline(new_cmdline, len + 1, root); 366 if (ret < 0 || ret > len) { 367 pr_err("Failed to print extra kernel cmdline.\n"); 368 return NULL; 369 } 370 371 return new_cmdline; 372 } 373 374 static u32 boot_config_checksum(unsigned char *p, u32 size) 375 { 376 u32 ret = 0; 377 378 while (size--) 379 ret += *p++; 380 381 return ret; 382 } 383 384 static int __init bootconfig_params(char *param, char *val, 385 const char *unused, void *arg) 386 { 387 if (strcmp(param, "bootconfig") == 0) { 388 bootconfig_found = true; 389 } else if (strcmp(param, "--") == 0) { 390 initargs_found = true; 391 } 392 return 0; 393 } 394 395 static void __init setup_boot_config(const char *cmdline) 396 { 397 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; 398 const char *msg; 399 int pos; 400 u32 size, csum; 401 char *data, *copy; 402 int ret; 403 404 /* Cut out the bootconfig data even if we have no bootconfig option */ 405 data = get_boot_config_from_initrd(&size, &csum); 406 407 strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); 408 parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL, 409 bootconfig_params); 410 411 if (!bootconfig_found) 412 return; 413 414 if (!data) { 415 pr_err("'bootconfig' found on command line, but no bootconfig found\n"); 416 return; 417 } 418 419 if (size >= XBC_DATA_MAX) { 420 pr_err("bootconfig size %d greater than max size %d\n", 421 size, XBC_DATA_MAX); 422 return; 423 } 424 425 if (boot_config_checksum((unsigned char *)data, size) != csum) { 426 pr_err("bootconfig checksum failed\n"); 427 return; 428 } 429 430 copy = memblock_alloc(size + 1, SMP_CACHE_BYTES); 431 if (!copy) { 432 pr_err("Failed to allocate memory for bootconfig\n"); 433 return; 434 } 435 436 memcpy(copy, data, size); 437 copy[size] = '\0'; 438 439 ret = xbc_init(copy, &msg, &pos); 440 if (ret < 0) { 441 if (pos < 0) 442 pr_err("Failed to init bootconfig: %s.\n", msg); 443 else 444 pr_err("Failed to parse bootconfig: %s at %d.\n", 445 msg, pos); 446 } else { 447 pr_info("Load bootconfig: %d bytes %d nodes\n", size, ret); 448 /* keys starting with "kernel." are passed via cmdline */ 449 extra_command_line = xbc_make_cmdline("kernel"); 450 /* Also, "init." keys are init arguments */ 451 extra_init_args = xbc_make_cmdline("init"); 452 } 453 return; 454 } 455 456 #else 457 458 static void __init setup_boot_config(const char *cmdline) 459 { 460 /* Remove bootconfig data from initrd */ 461 get_boot_config_from_initrd(NULL, NULL); 462 } 463 464 static int __init warn_bootconfig(char *str) 465 { 466 pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOTCONFIG is not set.\n"); 467 return 0; 468 } 469 early_param("bootconfig", warn_bootconfig); 470 471 #endif 472 473 /* Change NUL term back to "=", to make "param" the whole string. */ 474 static void __init repair_env_string(char *param, char *val) 475 { 476 if (val) { 477 /* param=val or param="val"? */ 478 if (val == param+strlen(param)+1) 479 val[-1] = '='; 480 else if (val == param+strlen(param)+2) { 481 val[-2] = '='; 482 memmove(val-1, val, strlen(val)+1); 483 } else 484 BUG(); 485 } 486 } 487 488 /* Anything after -- gets handed straight to init. */ 489 static int __init set_init_arg(char *param, char *val, 490 const char *unused, void *arg) 491 { 492 unsigned int i; 493 494 if (panic_later) 495 return 0; 496 497 repair_env_string(param, val); 498 499 for (i = 0; argv_init[i]; i++) { 500 if (i == MAX_INIT_ARGS) { 501 panic_later = "init"; 502 panic_param = param; 503 return 0; 504 } 505 } 506 argv_init[i] = param; 507 return 0; 508 } 509 510 /* 511 * Unknown boot options get handed to init, unless they look like 512 * unused parameters (modprobe will find them in /proc/cmdline). 513 */ 514 static int __init unknown_bootoption(char *param, char *val, 515 const char *unused, void *arg) 516 { 517 size_t len = strlen(param); 518 519 repair_env_string(param, val); 520 521 /* Handle obsolete-style parameters */ 522 if (obsolete_checksetup(param)) 523 return 0; 524 525 /* Unused module parameter. */ 526 if (strnchr(param, len, '.')) 527 return 0; 528 529 if (panic_later) 530 return 0; 531 532 if (val) { 533 /* Environment option */ 534 unsigned int i; 535 for (i = 0; envp_init[i]; i++) { 536 if (i == MAX_INIT_ENVS) { 537 panic_later = "env"; 538 panic_param = param; 539 } 540 if (!strncmp(param, envp_init[i], len+1)) 541 break; 542 } 543 envp_init[i] = param; 544 } else { 545 /* Command line option */ 546 unsigned int i; 547 for (i = 0; argv_init[i]; i++) { 548 if (i == MAX_INIT_ARGS) { 549 panic_later = "init"; 550 panic_param = param; 551 } 552 } 553 argv_init[i] = param; 554 } 555 return 0; 556 } 557 558 static int __init init_setup(char *str) 559 { 560 unsigned int i; 561 562 execute_command = str; 563 /* 564 * In case LILO is going to boot us with default command line, 565 * it prepends "auto" before the whole cmdline which makes 566 * the shell think it should execute a script with such name. 567 * So we ignore all arguments entered _before_ init=... [MJ] 568 */ 569 for (i = 1; i < MAX_INIT_ARGS; i++) 570 argv_init[i] = NULL; 571 return 1; 572 } 573 __setup("init=", init_setup); 574 575 static int __init rdinit_setup(char *str) 576 { 577 unsigned int i; 578 579 ramdisk_execute_command = str; 580 /* See "auto" comment in init_setup */ 581 for (i = 1; i < MAX_INIT_ARGS; i++) 582 argv_init[i] = NULL; 583 return 1; 584 } 585 __setup("rdinit=", rdinit_setup); 586 587 #ifndef CONFIG_SMP 588 static const unsigned int setup_max_cpus = NR_CPUS; 589 static inline void setup_nr_cpu_ids(void) { } 590 static inline void smp_prepare_cpus(unsigned int maxcpus) { } 591 #endif 592 593 /* 594 * We need to store the untouched command line for future reference. 595 * We also need to store the touched command line since the parameter 596 * parsing is performed in place, and we should allow a component to 597 * store reference of name/value for future reference. 598 */ 599 static void __init setup_command_line(char *command_line) 600 { 601 size_t len, xlen = 0, ilen = 0; 602 603 if (extra_command_line) 604 xlen = strlen(extra_command_line); 605 if (extra_init_args) 606 ilen = strlen(extra_init_args) + 4; /* for " -- " */ 607 608 len = xlen + strlen(boot_command_line) + 1; 609 610 saved_command_line = memblock_alloc(len + ilen, SMP_CACHE_BYTES); 611 if (!saved_command_line) 612 panic("%s: Failed to allocate %zu bytes\n", __func__, len + ilen); 613 614 static_command_line = memblock_alloc(len, SMP_CACHE_BYTES); 615 if (!static_command_line) 616 panic("%s: Failed to allocate %zu bytes\n", __func__, len); 617 618 if (xlen) { 619 /* 620 * We have to put extra_command_line before boot command 621 * lines because there could be dashes (separator of init 622 * command line) in the command lines. 623 */ 624 strcpy(saved_command_line, extra_command_line); 625 strcpy(static_command_line, extra_command_line); 626 } 627 strcpy(saved_command_line + xlen, boot_command_line); 628 strcpy(static_command_line + xlen, command_line); 629 630 if (ilen) { 631 /* 632 * Append supplemental init boot args to saved_command_line 633 * so that user can check what command line options passed 634 * to init. 635 */ 636 len = strlen(saved_command_line); 637 if (initargs_found) { 638 saved_command_line[len++] = ' '; 639 } else { 640 strcpy(saved_command_line + len, " -- "); 641 len += 4; 642 } 643 644 strcpy(saved_command_line + len, extra_init_args); 645 } 646 } 647 648 /* 649 * We need to finalize in a non-__init function or else race conditions 650 * between the root thread and the init thread may cause start_kernel to 651 * be reaped by free_initmem before the root thread has proceeded to 652 * cpu_idle. 653 * 654 * gcc-3.4 accidentally inlines this function, so use noinline. 655 */ 656 657 static __initdata DECLARE_COMPLETION(kthreadd_done); 658 659 noinline void __ref rest_init(void) 660 { 661 struct task_struct *tsk; 662 int pid; 663 664 rcu_scheduler_starting(); 665 /* 666 * We need to spawn init first so that it obtains pid 1, however 667 * the init task will end up wanting to create kthreads, which, if 668 * we schedule it before we create kthreadd, will OOPS. 669 */ 670 pid = kernel_thread(kernel_init, NULL, CLONE_FS); 671 /* 672 * Pin init on the boot CPU. Task migration is not properly working 673 * until sched_init_smp() has been run. It will set the allowed 674 * CPUs for init to the non isolated CPUs. 675 */ 676 rcu_read_lock(); 677 tsk = find_task_by_pid_ns(pid, &init_pid_ns); 678 set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id())); 679 rcu_read_unlock(); 680 681 numa_default_policy(); 682 pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES); 683 rcu_read_lock(); 684 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns); 685 rcu_read_unlock(); 686 687 /* 688 * Enable might_sleep() and smp_processor_id() checks. 689 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y 690 * kernel_thread() would trigger might_sleep() splats. With 691 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled 692 * already, but it's stuck on the kthreadd_done completion. 693 */ 694 system_state = SYSTEM_SCHEDULING; 695 696 complete(&kthreadd_done); 697 698 /* 699 * The boot idle thread must execute schedule() 700 * at least once to get things moving: 701 */ 702 schedule_preempt_disabled(); 703 /* Call into cpu_idle with preempt disabled */ 704 cpu_startup_entry(CPUHP_ONLINE); 705 } 706 707 /* Check for early params. */ 708 static int __init do_early_param(char *param, char *val, 709 const char *unused, void *arg) 710 { 711 const struct obs_kernel_param *p; 712 713 for (p = __setup_start; p < __setup_end; p++) { 714 if ((p->early && parameq(param, p->str)) || 715 (strcmp(param, "console") == 0 && 716 strcmp(p->str, "earlycon") == 0) 717 ) { 718 if (p->setup_func(val) != 0) 719 pr_warn("Malformed early option '%s'\n", param); 720 } 721 } 722 /* We accept everything at this stage. */ 723 return 0; 724 } 725 726 void __init parse_early_options(char *cmdline) 727 { 728 parse_args("early options", cmdline, NULL, 0, 0, 0, NULL, 729 do_early_param); 730 } 731 732 /* Arch code calls this early on, or if not, just before other parsing. */ 733 void __init parse_early_param(void) 734 { 735 static int done __initdata; 736 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; 737 738 if (done) 739 return; 740 741 /* All fall through to do_early_param. */ 742 strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); 743 parse_early_options(tmp_cmdline); 744 done = 1; 745 } 746 747 void __init __weak arch_post_acpi_subsys_init(void) { } 748 749 void __init __weak smp_setup_processor_id(void) 750 { 751 } 752 753 # if THREAD_SIZE >= PAGE_SIZE 754 void __init __weak thread_stack_cache_init(void) 755 { 756 } 757 #endif 758 759 void __init __weak mem_encrypt_init(void) { } 760 761 void __init __weak poking_init(void) { } 762 763 void __init __weak pgtable_cache_init(void) { } 764 765 bool initcall_debug; 766 core_param(initcall_debug, initcall_debug, bool, 0644); 767 768 #ifdef TRACEPOINTS_ENABLED 769 static void __init initcall_debug_enable(void); 770 #else 771 static inline void initcall_debug_enable(void) 772 { 773 } 774 #endif 775 776 /* Report memory auto-initialization states for this boot. */ 777 static void __init report_meminit(void) 778 { 779 const char *stack; 780 781 if (IS_ENABLED(CONFIG_INIT_STACK_ALL)) 782 stack = "all"; 783 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) 784 stack = "byref_all"; 785 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) 786 stack = "byref"; 787 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) 788 stack = "__user"; 789 else 790 stack = "off"; 791 792 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", 793 stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", 794 want_init_on_free() ? "on" : "off"); 795 if (want_init_on_free()) 796 pr_info("mem auto-init: clearing system memory may take some time...\n"); 797 } 798 799 /* 800 * Set up kernel memory allocators 801 */ 802 static void __init mm_init(void) 803 { 804 /* 805 * page_ext requires contiguous pages, 806 * bigger than MAX_ORDER unless SPARSEMEM. 807 */ 808 page_ext_init_flatmem(); 809 init_debug_pagealloc(); 810 report_meminit(); 811 mem_init(); 812 kmem_cache_init(); 813 kmemleak_init(); 814 pgtable_init(); 815 debug_objects_mem_init(); 816 vmalloc_init(); 817 ioremap_huge_init(); 818 /* Should be run before the first non-init thread is created */ 819 init_espfix_bsp(); 820 /* Should be run after espfix64 is set up. */ 821 pti_init(); 822 } 823 824 void __init __weak arch_call_rest_init(void) 825 { 826 rest_init(); 827 } 828 829 asmlinkage __visible void __init start_kernel(void) 830 { 831 char *command_line; 832 char *after_dashes; 833 834 set_task_stack_end_magic(&init_task); 835 smp_setup_processor_id(); 836 debug_objects_early_init(); 837 838 cgroup_init_early(); 839 840 local_irq_disable(); 841 early_boot_irqs_disabled = true; 842 843 /* 844 * Interrupts are still disabled. Do necessary setups, then 845 * enable them. 846 */ 847 boot_cpu_init(); 848 page_address_init(); 849 pr_notice("%s", linux_banner); 850 early_security_init(); 851 setup_arch(&command_line); 852 setup_boot_config(command_line); 853 setup_command_line(command_line); 854 setup_nr_cpu_ids(); 855 setup_per_cpu_areas(); 856 smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ 857 boot_cpu_hotplug_init(); 858 859 build_all_zonelists(NULL); 860 page_alloc_init(); 861 862 pr_notice("Kernel command line: %s\n", saved_command_line); 863 /* parameters may set static keys */ 864 jump_label_init(); 865 parse_early_param(); 866 after_dashes = parse_args("Booting kernel", 867 static_command_line, __start___param, 868 __stop___param - __start___param, 869 -1, -1, NULL, &unknown_bootoption); 870 if (!IS_ERR_OR_NULL(after_dashes)) 871 parse_args("Setting init args", after_dashes, NULL, 0, -1, -1, 872 NULL, set_init_arg); 873 if (extra_init_args) 874 parse_args("Setting extra init args", extra_init_args, 875 NULL, 0, -1, -1, NULL, set_init_arg); 876 877 /* 878 * These use large bootmem allocations and must precede 879 * kmem_cache_init() 880 */ 881 setup_log_buf(0); 882 vfs_caches_init_early(); 883 sort_main_extable(); 884 trap_init(); 885 mm_init(); 886 887 ftrace_init(); 888 889 /* trace_printk can be enabled here */ 890 early_trace_init(); 891 892 /* 893 * Set up the scheduler prior starting any interrupts (such as the 894 * timer interrupt). Full topology setup happens at smp_init() 895 * time - but meanwhile we still have a functioning scheduler. 896 */ 897 sched_init(); 898 /* 899 * Disable preemption - early bootup scheduling is extremely 900 * fragile until we cpu_idle() for the first time. 901 */ 902 preempt_disable(); 903 if (WARN(!irqs_disabled(), 904 "Interrupts were enabled *very* early, fixing it\n")) 905 local_irq_disable(); 906 radix_tree_init(); 907 908 /* 909 * Set up housekeeping before setting up workqueues to allow the unbound 910 * workqueue to take non-housekeeping into account. 911 */ 912 housekeeping_init(); 913 914 /* 915 * Allow workqueue creation and work item queueing/cancelling 916 * early. Work item execution depends on kthreads and starts after 917 * workqueue_init(). 918 */ 919 workqueue_init_early(); 920 921 rcu_init(); 922 923 /* Trace events are available after this */ 924 trace_init(); 925 926 if (initcall_debug) 927 initcall_debug_enable(); 928 929 context_tracking_init(); 930 /* init some links before init_ISA_irqs() */ 931 early_irq_init(); 932 init_IRQ(); 933 tick_init(); 934 rcu_init_nohz(); 935 init_timers(); 936 hrtimers_init(); 937 softirq_init(); 938 timekeeping_init(); 939 940 /* 941 * For best initial stack canary entropy, prepare it after: 942 * - setup_arch() for any UEFI RNG entropy and boot cmdline access 943 * - timekeeping_init() for ktime entropy used in rand_initialize() 944 * - rand_initialize() to get any arch-specific entropy like RDRAND 945 * - add_latent_entropy() to get any latent entropy 946 * - adding command line entropy 947 */ 948 rand_initialize(); 949 add_latent_entropy(); 950 add_device_randomness(command_line, strlen(command_line)); 951 boot_init_stack_canary(); 952 953 time_init(); 954 perf_event_init(); 955 profile_init(); 956 call_function_init(); 957 WARN(!irqs_disabled(), "Interrupts were enabled early\n"); 958 959 early_boot_irqs_disabled = false; 960 local_irq_enable(); 961 962 kmem_cache_init_late(); 963 964 /* 965 * HACK ALERT! This is early. We're enabling the console before 966 * we've done PCI setups etc, and console_init() must be aware of 967 * this. But we do want output early, in case something goes wrong. 968 */ 969 console_init(); 970 if (panic_later) 971 panic("Too many boot %s vars at `%s'", panic_later, 972 panic_param); 973 974 lockdep_init(); 975 976 /* 977 * Need to run this when irqs are enabled, because it wants 978 * to self-test [hard/soft]-irqs on/off lock inversion bugs 979 * too: 980 */ 981 locking_selftest(); 982 983 /* 984 * This needs to be called before any devices perform DMA 985 * operations that might use the SWIOTLB bounce buffers. It will 986 * mark the bounce buffers as decrypted so that their usage will 987 * not cause "plain-text" data to be decrypted when accessed. 988 */ 989 mem_encrypt_init(); 990 991 #ifdef CONFIG_BLK_DEV_INITRD 992 if (initrd_start && !initrd_below_start_ok && 993 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) { 994 pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n", 995 page_to_pfn(virt_to_page((void *)initrd_start)), 996 min_low_pfn); 997 initrd_start = 0; 998 } 999 #endif 1000 setup_per_cpu_pageset(); 1001 numa_policy_init(); 1002 acpi_early_init(); 1003 if (late_time_init) 1004 late_time_init(); 1005 sched_clock_init(); 1006 calibrate_delay(); 1007 pid_idr_init(); 1008 anon_vma_init(); 1009 #ifdef CONFIG_X86 1010 if (efi_enabled(EFI_RUNTIME_SERVICES)) 1011 efi_enter_virtual_mode(); 1012 #endif 1013 thread_stack_cache_init(); 1014 cred_init(); 1015 fork_init(); 1016 proc_caches_init(); 1017 uts_ns_init(); 1018 buffer_init(); 1019 key_init(); 1020 security_init(); 1021 dbg_late_init(); 1022 vfs_caches_init(); 1023 pagecache_init(); 1024 signals_init(); 1025 seq_file_init(); 1026 proc_root_init(); 1027 nsfs_init(); 1028 cpuset_init(); 1029 cgroup_init(); 1030 taskstats_init_early(); 1031 delayacct_init(); 1032 1033 poking_init(); 1034 check_bugs(); 1035 1036 acpi_subsystem_init(); 1037 arch_post_acpi_subsys_init(); 1038 sfi_init_late(); 1039 1040 /* Do the rest non-__init'ed, we're now alive */ 1041 arch_call_rest_init(); 1042 1043 prevent_tail_call_optimization(); 1044 } 1045 1046 /* Call all constructor functions linked into the kernel. */ 1047 static void __init do_ctors(void) 1048 { 1049 #ifdef CONFIG_CONSTRUCTORS 1050 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start; 1051 1052 for (; fn < (ctor_fn_t *) __ctors_end; fn++) 1053 (*fn)(); 1054 #endif 1055 } 1056 1057 #ifdef CONFIG_KALLSYMS 1058 struct blacklist_entry { 1059 struct list_head next; 1060 char *buf; 1061 }; 1062 1063 static __initdata_or_module LIST_HEAD(blacklisted_initcalls); 1064 1065 static int __init initcall_blacklist(char *str) 1066 { 1067 char *str_entry; 1068 struct blacklist_entry *entry; 1069 1070 /* str argument is a comma-separated list of functions */ 1071 do { 1072 str_entry = strsep(&str, ","); 1073 if (str_entry) { 1074 pr_debug("blacklisting initcall %s\n", str_entry); 1075 entry = memblock_alloc(sizeof(*entry), 1076 SMP_CACHE_BYTES); 1077 if (!entry) 1078 panic("%s: Failed to allocate %zu bytes\n", 1079 __func__, sizeof(*entry)); 1080 entry->buf = memblock_alloc(strlen(str_entry) + 1, 1081 SMP_CACHE_BYTES); 1082 if (!entry->buf) 1083 panic("%s: Failed to allocate %zu bytes\n", 1084 __func__, strlen(str_entry) + 1); 1085 strcpy(entry->buf, str_entry); 1086 list_add(&entry->next, &blacklisted_initcalls); 1087 } 1088 } while (str_entry); 1089 1090 return 0; 1091 } 1092 1093 static bool __init_or_module initcall_blacklisted(initcall_t fn) 1094 { 1095 struct blacklist_entry *entry; 1096 char fn_name[KSYM_SYMBOL_LEN]; 1097 unsigned long addr; 1098 1099 if (list_empty(&blacklisted_initcalls)) 1100 return false; 1101 1102 addr = (unsigned long) dereference_function_descriptor(fn); 1103 sprint_symbol_no_offset(fn_name, addr); 1104 1105 /* 1106 * fn will be "function_name [module_name]" where [module_name] is not 1107 * displayed for built-in init functions. Strip off the [module_name]. 1108 */ 1109 strreplace(fn_name, ' ', '\0'); 1110 1111 list_for_each_entry(entry, &blacklisted_initcalls, next) { 1112 if (!strcmp(fn_name, entry->buf)) { 1113 pr_debug("initcall %s blacklisted\n", fn_name); 1114 return true; 1115 } 1116 } 1117 1118 return false; 1119 } 1120 #else 1121 static int __init initcall_blacklist(char *str) 1122 { 1123 pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n"); 1124 return 0; 1125 } 1126 1127 static bool __init_or_module initcall_blacklisted(initcall_t fn) 1128 { 1129 return false; 1130 } 1131 #endif 1132 __setup("initcall_blacklist=", initcall_blacklist); 1133 1134 static __init_or_module void 1135 trace_initcall_start_cb(void *data, initcall_t fn) 1136 { 1137 ktime_t *calltime = (ktime_t *)data; 1138 1139 printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current)); 1140 *calltime = ktime_get(); 1141 } 1142 1143 static __init_or_module void 1144 trace_initcall_finish_cb(void *data, initcall_t fn, int ret) 1145 { 1146 ktime_t *calltime = (ktime_t *)data; 1147 ktime_t delta, rettime; 1148 unsigned long long duration; 1149 1150 rettime = ktime_get(); 1151 delta = ktime_sub(rettime, *calltime); 1152 duration = (unsigned long long) ktime_to_ns(delta) >> 10; 1153 printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n", 1154 fn, ret, duration); 1155 } 1156 1157 static ktime_t initcall_calltime; 1158 1159 #ifdef TRACEPOINTS_ENABLED 1160 static void __init initcall_debug_enable(void) 1161 { 1162 int ret; 1163 1164 ret = register_trace_initcall_start(trace_initcall_start_cb, 1165 &initcall_calltime); 1166 ret |= register_trace_initcall_finish(trace_initcall_finish_cb, 1167 &initcall_calltime); 1168 WARN(ret, "Failed to register initcall tracepoints\n"); 1169 } 1170 # define do_trace_initcall_start trace_initcall_start 1171 # define do_trace_initcall_finish trace_initcall_finish 1172 #else 1173 static inline void do_trace_initcall_start(initcall_t fn) 1174 { 1175 if (!initcall_debug) 1176 return; 1177 trace_initcall_start_cb(&initcall_calltime, fn); 1178 } 1179 static inline void do_trace_initcall_finish(initcall_t fn, int ret) 1180 { 1181 if (!initcall_debug) 1182 return; 1183 trace_initcall_finish_cb(&initcall_calltime, fn, ret); 1184 } 1185 #endif /* !TRACEPOINTS_ENABLED */ 1186 1187 int __init_or_module do_one_initcall(initcall_t fn) 1188 { 1189 int count = preempt_count(); 1190 char msgbuf[64]; 1191 int ret; 1192 1193 if (initcall_blacklisted(fn)) 1194 return -EPERM; 1195 1196 do_trace_initcall_start(fn); 1197 ret = fn(); 1198 do_trace_initcall_finish(fn, ret); 1199 1200 msgbuf[0] = 0; 1201 1202 if (preempt_count() != count) { 1203 sprintf(msgbuf, "preemption imbalance "); 1204 preempt_count_set(count); 1205 } 1206 if (irqs_disabled()) { 1207 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf)); 1208 local_irq_enable(); 1209 } 1210 WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf); 1211 1212 add_latent_entropy(); 1213 return ret; 1214 } 1215 1216 1217 extern initcall_entry_t __initcall_start[]; 1218 extern initcall_entry_t __initcall0_start[]; 1219 extern initcall_entry_t __initcall1_start[]; 1220 extern initcall_entry_t __initcall2_start[]; 1221 extern initcall_entry_t __initcall3_start[]; 1222 extern initcall_entry_t __initcall4_start[]; 1223 extern initcall_entry_t __initcall5_start[]; 1224 extern initcall_entry_t __initcall6_start[]; 1225 extern initcall_entry_t __initcall7_start[]; 1226 extern initcall_entry_t __initcall_end[]; 1227 1228 static initcall_entry_t *initcall_levels[] __initdata = { 1229 __initcall0_start, 1230 __initcall1_start, 1231 __initcall2_start, 1232 __initcall3_start, 1233 __initcall4_start, 1234 __initcall5_start, 1235 __initcall6_start, 1236 __initcall7_start, 1237 __initcall_end, 1238 }; 1239 1240 /* Keep these in sync with initcalls in include/linux/init.h */ 1241 static const char *initcall_level_names[] __initdata = { 1242 "pure", 1243 "core", 1244 "postcore", 1245 "arch", 1246 "subsys", 1247 "fs", 1248 "device", 1249 "late", 1250 }; 1251 1252 static int __init ignore_unknown_bootoption(char *param, char *val, 1253 const char *unused, void *arg) 1254 { 1255 return 0; 1256 } 1257 1258 static void __init do_initcall_level(int level, char *command_line) 1259 { 1260 initcall_entry_t *fn; 1261 1262 parse_args(initcall_level_names[level], 1263 command_line, __start___param, 1264 __stop___param - __start___param, 1265 level, level, 1266 NULL, ignore_unknown_bootoption); 1267 1268 trace_initcall_level(initcall_level_names[level]); 1269 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++) 1270 do_one_initcall(initcall_from_entry(fn)); 1271 } 1272 1273 static void __init do_initcalls(void) 1274 { 1275 int level; 1276 size_t len = strlen(saved_command_line) + 1; 1277 char *command_line; 1278 1279 command_line = kzalloc(len, GFP_KERNEL); 1280 if (!command_line) 1281 panic("%s: Failed to allocate %zu bytes\n", __func__, len); 1282 1283 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) { 1284 /* Parser modifies command_line, restore it each time */ 1285 strcpy(command_line, saved_command_line); 1286 do_initcall_level(level, command_line); 1287 } 1288 1289 kfree(command_line); 1290 } 1291 1292 /* 1293 * Ok, the machine is now initialized. None of the devices 1294 * have been touched yet, but the CPU subsystem is up and 1295 * running, and memory and process management works. 1296 * 1297 * Now we can finally start doing some real work.. 1298 */ 1299 static void __init do_basic_setup(void) 1300 { 1301 cpuset_init_smp(); 1302 driver_init(); 1303 init_irq_proc(); 1304 do_ctors(); 1305 usermodehelper_enable(); 1306 do_initcalls(); 1307 } 1308 1309 static void __init do_pre_smp_initcalls(void) 1310 { 1311 initcall_entry_t *fn; 1312 1313 trace_initcall_level("early"); 1314 for (fn = __initcall_start; fn < __initcall0_start; fn++) 1315 do_one_initcall(initcall_from_entry(fn)); 1316 } 1317 1318 static int run_init_process(const char *init_filename) 1319 { 1320 const char *const *p; 1321 1322 argv_init[0] = init_filename; 1323 pr_info("Run %s as init process\n", init_filename); 1324 pr_debug(" with arguments:\n"); 1325 for (p = argv_init; *p; p++) 1326 pr_debug(" %s\n", *p); 1327 pr_debug(" with environment:\n"); 1328 for (p = envp_init; *p; p++) 1329 pr_debug(" %s\n", *p); 1330 return do_execve(getname_kernel(init_filename), 1331 (const char __user *const __user *)argv_init, 1332 (const char __user *const __user *)envp_init); 1333 } 1334 1335 static int try_to_run_init_process(const char *init_filename) 1336 { 1337 int ret; 1338 1339 ret = run_init_process(init_filename); 1340 1341 if (ret && ret != -ENOENT) { 1342 pr_err("Starting init: %s exists but couldn't execute it (error %d)\n", 1343 init_filename, ret); 1344 } 1345 1346 return ret; 1347 } 1348 1349 static noinline void __init kernel_init_freeable(void); 1350 1351 #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX) 1352 bool rodata_enabled __ro_after_init = true; 1353 static int __init set_debug_rodata(char *str) 1354 { 1355 return strtobool(str, &rodata_enabled); 1356 } 1357 __setup("rodata=", set_debug_rodata); 1358 #endif 1359 1360 #ifdef CONFIG_STRICT_KERNEL_RWX 1361 static void mark_readonly(void) 1362 { 1363 if (rodata_enabled) { 1364 /* 1365 * load_module() results in W+X mappings, which are cleaned 1366 * up with call_rcu(). Let's make sure that queued work is 1367 * flushed so that we don't hit false positives looking for 1368 * insecure pages which are W+X. 1369 */ 1370 rcu_barrier(); 1371 mark_rodata_ro(); 1372 rodata_test(); 1373 } else 1374 pr_info("Kernel memory protection disabled.\n"); 1375 } 1376 #elif defined(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX) 1377 static inline void mark_readonly(void) 1378 { 1379 pr_warn("Kernel memory protection not selected by kernel config.\n"); 1380 } 1381 #else 1382 static inline void mark_readonly(void) 1383 { 1384 pr_warn("This architecture does not have kernel memory protection.\n"); 1385 } 1386 #endif 1387 1388 void __weak free_initmem(void) 1389 { 1390 free_initmem_default(POISON_FREE_INITMEM); 1391 } 1392 1393 static int __ref kernel_init(void *unused) 1394 { 1395 int ret; 1396 1397 kernel_init_freeable(); 1398 /* need to finish all async __init code before freeing the memory */ 1399 async_synchronize_full(); 1400 ftrace_free_init_mem(); 1401 free_initmem(); 1402 mark_readonly(); 1403 1404 /* 1405 * Kernel mappings are now finalized - update the userspace page-table 1406 * to finalize PTI. 1407 */ 1408 pti_finalize(); 1409 1410 system_state = SYSTEM_RUNNING; 1411 numa_default_policy(); 1412 1413 rcu_end_inkernel_boot(); 1414 1415 do_sysctl_args(); 1416 1417 if (ramdisk_execute_command) { 1418 ret = run_init_process(ramdisk_execute_command); 1419 if (!ret) 1420 return 0; 1421 pr_err("Failed to execute %s (error %d)\n", 1422 ramdisk_execute_command, ret); 1423 } 1424 1425 /* 1426 * We try each of these until one succeeds. 1427 * 1428 * The Bourne shell can be used instead of init if we are 1429 * trying to recover a really broken machine. 1430 */ 1431 if (execute_command) { 1432 ret = run_init_process(execute_command); 1433 if (!ret) 1434 return 0; 1435 panic("Requested init %s failed (error %d).", 1436 execute_command, ret); 1437 } 1438 1439 if (CONFIG_DEFAULT_INIT[0] != '\0') { 1440 ret = run_init_process(CONFIG_DEFAULT_INIT); 1441 if (ret) 1442 pr_err("Default init %s failed (error %d)\n", 1443 CONFIG_DEFAULT_INIT, ret); 1444 else 1445 return 0; 1446 } 1447 1448 if (!try_to_run_init_process("/sbin/init") || 1449 !try_to_run_init_process("/etc/init") || 1450 !try_to_run_init_process("/bin/init") || 1451 !try_to_run_init_process("/bin/sh")) 1452 return 0; 1453 1454 panic("No working init found. Try passing init= option to kernel. " 1455 "See Linux Documentation/admin-guide/init.rst for guidance."); 1456 } 1457 1458 void console_on_rootfs(void) 1459 { 1460 /* Open the /dev/console as stdin, this should never fail */ 1461 if (ksys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0) 1462 pr_err("Warning: unable to open an initial console.\n"); 1463 1464 /* create stdout/stderr */ 1465 (void) ksys_dup(0); 1466 (void) ksys_dup(0); 1467 } 1468 1469 static noinline void __init kernel_init_freeable(void) 1470 { 1471 /* 1472 * Wait until kthreadd is all set-up. 1473 */ 1474 wait_for_completion(&kthreadd_done); 1475 1476 /* Now the scheduler is fully set up and can do blocking allocations */ 1477 gfp_allowed_mask = __GFP_BITS_MASK; 1478 1479 /* 1480 * init can allocate pages on any node 1481 */ 1482 set_mems_allowed(node_states[N_MEMORY]); 1483 1484 cad_pid = task_pid(current); 1485 1486 smp_prepare_cpus(setup_max_cpus); 1487 1488 workqueue_init(); 1489 1490 init_mm_internals(); 1491 1492 do_pre_smp_initcalls(); 1493 lockup_detector_init(); 1494 1495 smp_init(); 1496 sched_init_smp(); 1497 1498 padata_init(); 1499 page_alloc_init_late(); 1500 /* Initialize page ext after all struct pages are initialized. */ 1501 page_ext_init(); 1502 1503 do_basic_setup(); 1504 1505 console_on_rootfs(); 1506 1507 /* 1508 * check if there is an early userspace init. If yes, let it do all 1509 * the work 1510 */ 1511 1512 if (!ramdisk_execute_command) 1513 ramdisk_execute_command = "/init"; 1514 1515 if (ksys_access((const char __user *) 1516 ramdisk_execute_command, 0) != 0) { 1517 ramdisk_execute_command = NULL; 1518 prepare_namespace(); 1519 } 1520 1521 /* 1522 * Ok, we have completed the initial bootup, and 1523 * we're essentially up and running. Get rid of the 1524 * initmem segments and start the user-mode stuff.. 1525 * 1526 * rootfs is available now, try loading the public keys 1527 * and default modules 1528 */ 1529 1530 integrity_load_keys(); 1531 } 1532