1 /* 2 * linux/init/main.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 * 6 * GK 2/5/95 - Changed to support mounting root fs via NFS 7 * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96 8 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96 9 * Simplified starting of init: Michael A. Griffith <grif@acm.org> 10 */ 11 12 #define DEBUG /* Enable initcall_debug */ 13 14 #include <linux/types.h> 15 #include <linux/extable.h> 16 #include <linux/module.h> 17 #include <linux/proc_fs.h> 18 #include <linux/binfmts.h> 19 #include <linux/kernel.h> 20 #include <linux/syscalls.h> 21 #include <linux/stackprotector.h> 22 #include <linux/string.h> 23 #include <linux/ctype.h> 24 #include <linux/delay.h> 25 #include <linux/ioport.h> 26 #include <linux/init.h> 27 #include <linux/initrd.h> 28 #include <linux/bootmem.h> 29 #include <linux/acpi.h> 30 #include <linux/console.h> 31 #include <linux/nmi.h> 32 #include <linux/percpu.h> 33 #include <linux/kmod.h> 34 #include <linux/vmalloc.h> 35 #include <linux/kernel_stat.h> 36 #include <linux/start_kernel.h> 37 #include <linux/security.h> 38 #include <linux/smp.h> 39 #include <linux/profile.h> 40 #include <linux/rcupdate.h> 41 #include <linux/moduleparam.h> 42 #include <linux/kallsyms.h> 43 #include <linux/writeback.h> 44 #include <linux/cpu.h> 45 #include <linux/cpuset.h> 46 #include <linux/cgroup.h> 47 #include <linux/efi.h> 48 #include <linux/tick.h> 49 #include <linux/sched/isolation.h> 50 #include <linux/interrupt.h> 51 #include <linux/taskstats_kern.h> 52 #include <linux/delayacct.h> 53 #include <linux/unistd.h> 54 #include <linux/rmap.h> 55 #include <linux/mempolicy.h> 56 #include <linux/key.h> 57 #include <linux/buffer_head.h> 58 #include <linux/page_ext.h> 59 #include <linux/debug_locks.h> 60 #include <linux/debugobjects.h> 61 #include <linux/lockdep.h> 62 #include <linux/kmemleak.h> 63 #include <linux/pid_namespace.h> 64 #include <linux/device.h> 65 #include <linux/kthread.h> 66 #include <linux/sched.h> 67 #include <linux/sched/init.h> 68 #include <linux/signal.h> 69 #include <linux/idr.h> 70 #include <linux/kgdb.h> 71 #include <linux/ftrace.h> 72 #include <linux/async.h> 73 #include <linux/kmemcheck.h> 74 #include <linux/sfi.h> 75 #include <linux/shmem_fs.h> 76 #include <linux/slab.h> 77 #include <linux/perf_event.h> 78 #include <linux/ptrace.h> 79 #include <linux/blkdev.h> 80 #include <linux/elevator.h> 81 #include <linux/sched_clock.h> 82 #include <linux/sched/task.h> 83 #include <linux/sched/task_stack.h> 84 #include <linux/context_tracking.h> 85 #include <linux/random.h> 86 #include <linux/list.h> 87 #include <linux/integrity.h> 88 #include <linux/proc_ns.h> 89 #include <linux/io.h> 90 #include <linux/cache.h> 91 #include <linux/rodata_test.h> 92 93 #include <asm/io.h> 94 #include <asm/bugs.h> 95 #include <asm/setup.h> 96 #include <asm/sections.h> 97 #include <asm/cacheflush.h> 98 99 static int kernel_init(void *); 100 101 extern void init_IRQ(void); 102 extern void fork_init(void); 103 extern void radix_tree_init(void); 104 105 /* 106 * Debug helper: via this flag we know that we are in 'early bootup code' 107 * where only the boot processor is running with IRQ disabled. This means 108 * two things - IRQ must not be enabled before the flag is cleared and some 109 * operations which are not allowed with IRQ disabled are allowed while the 110 * flag is set. 111 */ 112 bool early_boot_irqs_disabled __read_mostly; 113 114 enum system_states system_state __read_mostly; 115 EXPORT_SYMBOL(system_state); 116 117 /* 118 * Boot command-line arguments 119 */ 120 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT 121 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT 122 123 extern void time_init(void); 124 /* Default late time init is NULL. archs can override this later. */ 125 void (*__initdata late_time_init)(void); 126 127 /* Untouched command line saved by arch-specific code. */ 128 char __initdata boot_command_line[COMMAND_LINE_SIZE]; 129 /* Untouched saved command line (eg. for /proc) */ 130 char *saved_command_line; 131 /* Command line for parameter parsing */ 132 static char *static_command_line; 133 /* Command line for per-initcall parameter parsing */ 134 static char *initcall_command_line; 135 136 static char *execute_command; 137 static char *ramdisk_execute_command; 138 139 /* 140 * Used to generate warnings if static_key manipulation functions are used 141 * before jump_label_init is called. 142 */ 143 bool static_key_initialized __read_mostly; 144 EXPORT_SYMBOL_GPL(static_key_initialized); 145 146 /* 147 * If set, this is an indication to the drivers that reset the underlying 148 * device before going ahead with the initialization otherwise driver might 149 * rely on the BIOS and skip the reset operation. 150 * 151 * This is useful if kernel is booting in an unreliable environment. 152 * For ex. kdump situation where previous kernel has crashed, BIOS has been 153 * skipped and devices will be in unknown state. 154 */ 155 unsigned int reset_devices; 156 EXPORT_SYMBOL(reset_devices); 157 158 static int __init set_reset_devices(char *str) 159 { 160 reset_devices = 1; 161 return 1; 162 } 163 164 __setup("reset_devices", set_reset_devices); 165 166 static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, }; 167 const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, }; 168 static const char *panic_later, *panic_param; 169 170 extern const struct obs_kernel_param __setup_start[], __setup_end[]; 171 172 static bool __init obsolete_checksetup(char *line) 173 { 174 const struct obs_kernel_param *p; 175 bool had_early_param = false; 176 177 p = __setup_start; 178 do { 179 int n = strlen(p->str); 180 if (parameqn(line, p->str, n)) { 181 if (p->early) { 182 /* Already done in parse_early_param? 183 * (Needs exact match on param part). 184 * Keep iterating, as we can have early 185 * params and __setups of same names 8( */ 186 if (line[n] == '\0' || line[n] == '=') 187 had_early_param = true; 188 } else if (!p->setup_func) { 189 pr_warn("Parameter %s is obsolete, ignored\n", 190 p->str); 191 return true; 192 } else if (p->setup_func(line + n)) 193 return true; 194 } 195 p++; 196 } while (p < __setup_end); 197 198 return had_early_param; 199 } 200 201 /* 202 * This should be approx 2 Bo*oMips to start (note initial shift), and will 203 * still work even if initially too large, it will just take slightly longer 204 */ 205 unsigned long loops_per_jiffy = (1<<12); 206 EXPORT_SYMBOL(loops_per_jiffy); 207 208 static int __init debug_kernel(char *str) 209 { 210 console_loglevel = CONSOLE_LOGLEVEL_DEBUG; 211 return 0; 212 } 213 214 static int __init quiet_kernel(char *str) 215 { 216 console_loglevel = CONSOLE_LOGLEVEL_QUIET; 217 return 0; 218 } 219 220 early_param("debug", debug_kernel); 221 early_param("quiet", quiet_kernel); 222 223 static int __init loglevel(char *str) 224 { 225 int newlevel; 226 227 /* 228 * Only update loglevel value when a correct setting was passed, 229 * to prevent blind crashes (when loglevel being set to 0) that 230 * are quite hard to debug 231 */ 232 if (get_option(&str, &newlevel)) { 233 console_loglevel = newlevel; 234 return 0; 235 } 236 237 return -EINVAL; 238 } 239 240 early_param("loglevel", loglevel); 241 242 /* Change NUL term back to "=", to make "param" the whole string. */ 243 static int __init repair_env_string(char *param, char *val, 244 const char *unused, void *arg) 245 { 246 if (val) { 247 /* param=val or param="val"? */ 248 if (val == param+strlen(param)+1) 249 val[-1] = '='; 250 else if (val == param+strlen(param)+2) { 251 val[-2] = '='; 252 memmove(val-1, val, strlen(val)+1); 253 val--; 254 } else 255 BUG(); 256 } 257 return 0; 258 } 259 260 /* Anything after -- gets handed straight to init. */ 261 static int __init set_init_arg(char *param, char *val, 262 const char *unused, void *arg) 263 { 264 unsigned int i; 265 266 if (panic_later) 267 return 0; 268 269 repair_env_string(param, val, unused, NULL); 270 271 for (i = 0; argv_init[i]; i++) { 272 if (i == MAX_INIT_ARGS) { 273 panic_later = "init"; 274 panic_param = param; 275 return 0; 276 } 277 } 278 argv_init[i] = param; 279 return 0; 280 } 281 282 /* 283 * Unknown boot options get handed to init, unless they look like 284 * unused parameters (modprobe will find them in /proc/cmdline). 285 */ 286 static int __init unknown_bootoption(char *param, char *val, 287 const char *unused, void *arg) 288 { 289 repair_env_string(param, val, unused, NULL); 290 291 /* Handle obsolete-style parameters */ 292 if (obsolete_checksetup(param)) 293 return 0; 294 295 /* Unused module parameter. */ 296 if (strchr(param, '.') && (!val || strchr(param, '.') < val)) 297 return 0; 298 299 if (panic_later) 300 return 0; 301 302 if (val) { 303 /* Environment option */ 304 unsigned int i; 305 for (i = 0; envp_init[i]; i++) { 306 if (i == MAX_INIT_ENVS) { 307 panic_later = "env"; 308 panic_param = param; 309 } 310 if (!strncmp(param, envp_init[i], val - param)) 311 break; 312 } 313 envp_init[i] = param; 314 } else { 315 /* Command line option */ 316 unsigned int i; 317 for (i = 0; argv_init[i]; i++) { 318 if (i == MAX_INIT_ARGS) { 319 panic_later = "init"; 320 panic_param = param; 321 } 322 } 323 argv_init[i] = param; 324 } 325 return 0; 326 } 327 328 static int __init init_setup(char *str) 329 { 330 unsigned int i; 331 332 execute_command = str; 333 /* 334 * In case LILO is going to boot us with default command line, 335 * it prepends "auto" before the whole cmdline which makes 336 * the shell think it should execute a script with such name. 337 * So we ignore all arguments entered _before_ init=... [MJ] 338 */ 339 for (i = 1; i < MAX_INIT_ARGS; i++) 340 argv_init[i] = NULL; 341 return 1; 342 } 343 __setup("init=", init_setup); 344 345 static int __init rdinit_setup(char *str) 346 { 347 unsigned int i; 348 349 ramdisk_execute_command = str; 350 /* See "auto" comment in init_setup */ 351 for (i = 1; i < MAX_INIT_ARGS; i++) 352 argv_init[i] = NULL; 353 return 1; 354 } 355 __setup("rdinit=", rdinit_setup); 356 357 #ifndef CONFIG_SMP 358 static const unsigned int setup_max_cpus = NR_CPUS; 359 static inline void setup_nr_cpu_ids(void) { } 360 static inline void smp_prepare_cpus(unsigned int maxcpus) { } 361 #endif 362 363 /* 364 * We need to store the untouched command line for future reference. 365 * We also need to store the touched command line since the parameter 366 * parsing is performed in place, and we should allow a component to 367 * store reference of name/value for future reference. 368 */ 369 static void __init setup_command_line(char *command_line) 370 { 371 saved_command_line = 372 memblock_virt_alloc(strlen(boot_command_line) + 1, 0); 373 initcall_command_line = 374 memblock_virt_alloc(strlen(boot_command_line) + 1, 0); 375 static_command_line = memblock_virt_alloc(strlen(command_line) + 1, 0); 376 strcpy(saved_command_line, boot_command_line); 377 strcpy(static_command_line, command_line); 378 } 379 380 /* 381 * We need to finalize in a non-__init function or else race conditions 382 * between the root thread and the init thread may cause start_kernel to 383 * be reaped by free_initmem before the root thread has proceeded to 384 * cpu_idle. 385 * 386 * gcc-3.4 accidentally inlines this function, so use noinline. 387 */ 388 389 static __initdata DECLARE_COMPLETION(kthreadd_done); 390 391 static noinline void __ref rest_init(void) 392 { 393 struct task_struct *tsk; 394 int pid; 395 396 rcu_scheduler_starting(); 397 /* 398 * We need to spawn init first so that it obtains pid 1, however 399 * the init task will end up wanting to create kthreads, which, if 400 * we schedule it before we create kthreadd, will OOPS. 401 */ 402 pid = kernel_thread(kernel_init, NULL, CLONE_FS); 403 /* 404 * Pin init on the boot CPU. Task migration is not properly working 405 * until sched_init_smp() has been run. It will set the allowed 406 * CPUs for init to the non isolated CPUs. 407 */ 408 rcu_read_lock(); 409 tsk = find_task_by_pid_ns(pid, &init_pid_ns); 410 set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id())); 411 rcu_read_unlock(); 412 413 numa_default_policy(); 414 pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES); 415 rcu_read_lock(); 416 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns); 417 rcu_read_unlock(); 418 419 /* 420 * Enable might_sleep() and smp_processor_id() checks. 421 * They cannot be enabled earlier because with CONFIG_PRREMPT=y 422 * kernel_thread() would trigger might_sleep() splats. With 423 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled 424 * already, but it's stuck on the kthreadd_done completion. 425 */ 426 system_state = SYSTEM_SCHEDULING; 427 428 complete(&kthreadd_done); 429 430 /* 431 * The boot idle thread must execute schedule() 432 * at least once to get things moving: 433 */ 434 schedule_preempt_disabled(); 435 /* Call into cpu_idle with preempt disabled */ 436 cpu_startup_entry(CPUHP_ONLINE); 437 } 438 439 /* Check for early params. */ 440 static int __init do_early_param(char *param, char *val, 441 const char *unused, void *arg) 442 { 443 const struct obs_kernel_param *p; 444 445 for (p = __setup_start; p < __setup_end; p++) { 446 if ((p->early && parameq(param, p->str)) || 447 (strcmp(param, "console") == 0 && 448 strcmp(p->str, "earlycon") == 0) 449 ) { 450 if (p->setup_func(val) != 0) 451 pr_warn("Malformed early option '%s'\n", param); 452 } 453 } 454 /* We accept everything at this stage. */ 455 return 0; 456 } 457 458 void __init parse_early_options(char *cmdline) 459 { 460 parse_args("early options", cmdline, NULL, 0, 0, 0, NULL, 461 do_early_param); 462 } 463 464 /* Arch code calls this early on, or if not, just before other parsing. */ 465 void __init parse_early_param(void) 466 { 467 static int done __initdata; 468 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; 469 470 if (done) 471 return; 472 473 /* All fall through to do_early_param. */ 474 strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); 475 parse_early_options(tmp_cmdline); 476 done = 1; 477 } 478 479 void __init __weak arch_post_acpi_subsys_init(void) { } 480 481 void __init __weak smp_setup_processor_id(void) 482 { 483 } 484 485 # if THREAD_SIZE >= PAGE_SIZE 486 void __init __weak thread_stack_cache_init(void) 487 { 488 } 489 #endif 490 491 void __init __weak mem_encrypt_init(void) { } 492 493 /* 494 * Set up kernel memory allocators 495 */ 496 static void __init mm_init(void) 497 { 498 /* 499 * page_ext requires contiguous pages, 500 * bigger than MAX_ORDER unless SPARSEMEM. 501 */ 502 page_ext_init_flatmem(); 503 mem_init(); 504 kmem_cache_init(); 505 pgtable_init(); 506 vmalloc_init(); 507 ioremap_huge_init(); 508 } 509 510 asmlinkage __visible void __init start_kernel(void) 511 { 512 char *command_line; 513 char *after_dashes; 514 515 set_task_stack_end_magic(&init_task); 516 smp_setup_processor_id(); 517 debug_objects_early_init(); 518 519 cgroup_init_early(); 520 521 local_irq_disable(); 522 early_boot_irqs_disabled = true; 523 524 /* 525 * Interrupts are still disabled. Do necessary setups, then 526 * enable them. 527 */ 528 boot_cpu_init(); 529 page_address_init(); 530 pr_notice("%s", linux_banner); 531 setup_arch(&command_line); 532 /* 533 * Set up the the initial canary and entropy after arch 534 * and after adding latent and command line entropy. 535 */ 536 add_latent_entropy(); 537 add_device_randomness(command_line, strlen(command_line)); 538 boot_init_stack_canary(); 539 mm_init_cpumask(&init_mm); 540 setup_command_line(command_line); 541 setup_nr_cpu_ids(); 542 setup_per_cpu_areas(); 543 boot_cpu_state_init(); 544 smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ 545 546 build_all_zonelists(NULL); 547 page_alloc_init(); 548 549 pr_notice("Kernel command line: %s\n", boot_command_line); 550 parse_early_param(); 551 after_dashes = parse_args("Booting kernel", 552 static_command_line, __start___param, 553 __stop___param - __start___param, 554 -1, -1, NULL, &unknown_bootoption); 555 if (!IS_ERR_OR_NULL(after_dashes)) 556 parse_args("Setting init args", after_dashes, NULL, 0, -1, -1, 557 NULL, set_init_arg); 558 559 jump_label_init(); 560 561 /* 562 * These use large bootmem allocations and must precede 563 * kmem_cache_init() 564 */ 565 setup_log_buf(0); 566 pidhash_init(); 567 vfs_caches_init_early(); 568 sort_main_extable(); 569 trap_init(); 570 mm_init(); 571 572 ftrace_init(); 573 574 /* trace_printk can be enabled here */ 575 early_trace_init(); 576 577 /* 578 * Set up the scheduler prior starting any interrupts (such as the 579 * timer interrupt). Full topology setup happens at smp_init() 580 * time - but meanwhile we still have a functioning scheduler. 581 */ 582 sched_init(); 583 /* 584 * Disable preemption - early bootup scheduling is extremely 585 * fragile until we cpu_idle() for the first time. 586 */ 587 preempt_disable(); 588 if (WARN(!irqs_disabled(), 589 "Interrupts were enabled *very* early, fixing it\n")) 590 local_irq_disable(); 591 radix_tree_init(); 592 593 /* 594 * Allow workqueue creation and work item queueing/cancelling 595 * early. Work item execution depends on kthreads and starts after 596 * workqueue_init(). 597 */ 598 workqueue_init_early(); 599 600 rcu_init(); 601 602 /* Trace events are available after this */ 603 trace_init(); 604 605 context_tracking_init(); 606 /* init some links before init_ISA_irqs() */ 607 early_irq_init(); 608 init_IRQ(); 609 tick_init(); 610 housekeeping_init(); 611 rcu_init_nohz(); 612 init_timers(); 613 hrtimers_init(); 614 softirq_init(); 615 timekeeping_init(); 616 time_init(); 617 sched_clock_postinit(); 618 printk_safe_init(); 619 perf_event_init(); 620 profile_init(); 621 call_function_init(); 622 WARN(!irqs_disabled(), "Interrupts were enabled early\n"); 623 early_boot_irqs_disabled = false; 624 local_irq_enable(); 625 626 kmem_cache_init_late(); 627 628 /* 629 * HACK ALERT! This is early. We're enabling the console before 630 * we've done PCI setups etc, and console_init() must be aware of 631 * this. But we do want output early, in case something goes wrong. 632 */ 633 console_init(); 634 if (panic_later) 635 panic("Too many boot %s vars at `%s'", panic_later, 636 panic_param); 637 638 lockdep_info(); 639 640 /* 641 * Need to run this when irqs are enabled, because it wants 642 * to self-test [hard/soft]-irqs on/off lock inversion bugs 643 * too: 644 */ 645 locking_selftest(); 646 647 /* 648 * This needs to be called before any devices perform DMA 649 * operations that might use the SWIOTLB bounce buffers. It will 650 * mark the bounce buffers as decrypted so that their usage will 651 * not cause "plain-text" data to be decrypted when accessed. 652 */ 653 mem_encrypt_init(); 654 655 #ifdef CONFIG_BLK_DEV_INITRD 656 if (initrd_start && !initrd_below_start_ok && 657 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) { 658 pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n", 659 page_to_pfn(virt_to_page((void *)initrd_start)), 660 min_low_pfn); 661 initrd_start = 0; 662 } 663 #endif 664 page_ext_init(); 665 kmemleak_init(); 666 debug_objects_mem_init(); 667 setup_per_cpu_pageset(); 668 numa_policy_init(); 669 acpi_early_init(); 670 if (late_time_init) 671 late_time_init(); 672 calibrate_delay(); 673 pidmap_init(); 674 anon_vma_init(); 675 #ifdef CONFIG_X86 676 if (efi_enabled(EFI_RUNTIME_SERVICES)) 677 efi_enter_virtual_mode(); 678 #endif 679 #ifdef CONFIG_X86_ESPFIX64 680 /* Should be run before the first non-init thread is created */ 681 init_espfix_bsp(); 682 #endif 683 thread_stack_cache_init(); 684 cred_init(); 685 fork_init(); 686 proc_caches_init(); 687 buffer_init(); 688 key_init(); 689 security_init(); 690 dbg_late_init(); 691 vfs_caches_init(); 692 pagecache_init(); 693 signals_init(); 694 proc_root_init(); 695 nsfs_init(); 696 cpuset_init(); 697 cgroup_init(); 698 taskstats_init_early(); 699 delayacct_init(); 700 701 check_bugs(); 702 703 acpi_subsystem_init(); 704 arch_post_acpi_subsys_init(); 705 sfi_init_late(); 706 707 if (efi_enabled(EFI_RUNTIME_SERVICES)) { 708 efi_free_boot_services(); 709 } 710 711 /* Do the rest non-__init'ed, we're now alive */ 712 rest_init(); 713 } 714 715 /* Call all constructor functions linked into the kernel. */ 716 static void __init do_ctors(void) 717 { 718 #ifdef CONFIG_CONSTRUCTORS 719 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start; 720 721 for (; fn < (ctor_fn_t *) __ctors_end; fn++) 722 (*fn)(); 723 #endif 724 } 725 726 bool initcall_debug; 727 core_param(initcall_debug, initcall_debug, bool, 0644); 728 729 #ifdef CONFIG_KALLSYMS 730 struct blacklist_entry { 731 struct list_head next; 732 char *buf; 733 }; 734 735 static __initdata_or_module LIST_HEAD(blacklisted_initcalls); 736 737 static int __init initcall_blacklist(char *str) 738 { 739 char *str_entry; 740 struct blacklist_entry *entry; 741 742 /* str argument is a comma-separated list of functions */ 743 do { 744 str_entry = strsep(&str, ","); 745 if (str_entry) { 746 pr_debug("blacklisting initcall %s\n", str_entry); 747 entry = alloc_bootmem(sizeof(*entry)); 748 entry->buf = alloc_bootmem(strlen(str_entry) + 1); 749 strcpy(entry->buf, str_entry); 750 list_add(&entry->next, &blacklisted_initcalls); 751 } 752 } while (str_entry); 753 754 return 0; 755 } 756 757 static bool __init_or_module initcall_blacklisted(initcall_t fn) 758 { 759 struct blacklist_entry *entry; 760 char fn_name[KSYM_SYMBOL_LEN]; 761 unsigned long addr; 762 763 if (list_empty(&blacklisted_initcalls)) 764 return false; 765 766 addr = (unsigned long) dereference_function_descriptor(fn); 767 sprint_symbol_no_offset(fn_name, addr); 768 769 /* 770 * fn will be "function_name [module_name]" where [module_name] is not 771 * displayed for built-in init functions. Strip off the [module_name]. 772 */ 773 strreplace(fn_name, ' ', '\0'); 774 775 list_for_each_entry(entry, &blacklisted_initcalls, next) { 776 if (!strcmp(fn_name, entry->buf)) { 777 pr_debug("initcall %s blacklisted\n", fn_name); 778 return true; 779 } 780 } 781 782 return false; 783 } 784 #else 785 static int __init initcall_blacklist(char *str) 786 { 787 pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n"); 788 return 0; 789 } 790 791 static bool __init_or_module initcall_blacklisted(initcall_t fn) 792 { 793 return false; 794 } 795 #endif 796 __setup("initcall_blacklist=", initcall_blacklist); 797 798 static int __init_or_module do_one_initcall_debug(initcall_t fn) 799 { 800 ktime_t calltime, delta, rettime; 801 unsigned long long duration; 802 int ret; 803 804 printk(KERN_DEBUG "calling %pF @ %i\n", fn, task_pid_nr(current)); 805 calltime = ktime_get(); 806 ret = fn(); 807 rettime = ktime_get(); 808 delta = ktime_sub(rettime, calltime); 809 duration = (unsigned long long) ktime_to_ns(delta) >> 10; 810 printk(KERN_DEBUG "initcall %pF returned %d after %lld usecs\n", 811 fn, ret, duration); 812 813 return ret; 814 } 815 816 int __init_or_module do_one_initcall(initcall_t fn) 817 { 818 int count = preempt_count(); 819 int ret; 820 char msgbuf[64]; 821 822 if (initcall_blacklisted(fn)) 823 return -EPERM; 824 825 if (initcall_debug) 826 ret = do_one_initcall_debug(fn); 827 else 828 ret = fn(); 829 830 msgbuf[0] = 0; 831 832 if (preempt_count() != count) { 833 sprintf(msgbuf, "preemption imbalance "); 834 preempt_count_set(count); 835 } 836 if (irqs_disabled()) { 837 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf)); 838 local_irq_enable(); 839 } 840 WARN(msgbuf[0], "initcall %pF returned with %s\n", fn, msgbuf); 841 842 add_latent_entropy(); 843 return ret; 844 } 845 846 847 extern initcall_t __initcall_start[]; 848 extern initcall_t __initcall0_start[]; 849 extern initcall_t __initcall1_start[]; 850 extern initcall_t __initcall2_start[]; 851 extern initcall_t __initcall3_start[]; 852 extern initcall_t __initcall4_start[]; 853 extern initcall_t __initcall5_start[]; 854 extern initcall_t __initcall6_start[]; 855 extern initcall_t __initcall7_start[]; 856 extern initcall_t __initcall_end[]; 857 858 static initcall_t *initcall_levels[] __initdata = { 859 __initcall0_start, 860 __initcall1_start, 861 __initcall2_start, 862 __initcall3_start, 863 __initcall4_start, 864 __initcall5_start, 865 __initcall6_start, 866 __initcall7_start, 867 __initcall_end, 868 }; 869 870 /* Keep these in sync with initcalls in include/linux/init.h */ 871 static char *initcall_level_names[] __initdata = { 872 "early", 873 "core", 874 "postcore", 875 "arch", 876 "subsys", 877 "fs", 878 "device", 879 "late", 880 }; 881 882 static void __init do_initcall_level(int level) 883 { 884 initcall_t *fn; 885 886 strcpy(initcall_command_line, saved_command_line); 887 parse_args(initcall_level_names[level], 888 initcall_command_line, __start___param, 889 __stop___param - __start___param, 890 level, level, 891 NULL, &repair_env_string); 892 893 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++) 894 do_one_initcall(*fn); 895 } 896 897 static void __init do_initcalls(void) 898 { 899 int level; 900 901 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) 902 do_initcall_level(level); 903 } 904 905 /* 906 * Ok, the machine is now initialized. None of the devices 907 * have been touched yet, but the CPU subsystem is up and 908 * running, and memory and process management works. 909 * 910 * Now we can finally start doing some real work.. 911 */ 912 static void __init do_basic_setup(void) 913 { 914 cpuset_init_smp(); 915 shmem_init(); 916 driver_init(); 917 init_irq_proc(); 918 do_ctors(); 919 usermodehelper_enable(); 920 do_initcalls(); 921 } 922 923 static void __init do_pre_smp_initcalls(void) 924 { 925 initcall_t *fn; 926 927 for (fn = __initcall_start; fn < __initcall0_start; fn++) 928 do_one_initcall(*fn); 929 } 930 931 /* 932 * This function requests modules which should be loaded by default and is 933 * called twice right after initrd is mounted and right before init is 934 * exec'd. If such modules are on either initrd or rootfs, they will be 935 * loaded before control is passed to userland. 936 */ 937 void __init load_default_modules(void) 938 { 939 load_default_elevator_module(); 940 } 941 942 static int run_init_process(const char *init_filename) 943 { 944 argv_init[0] = init_filename; 945 return do_execve(getname_kernel(init_filename), 946 (const char __user *const __user *)argv_init, 947 (const char __user *const __user *)envp_init); 948 } 949 950 static int try_to_run_init_process(const char *init_filename) 951 { 952 int ret; 953 954 ret = run_init_process(init_filename); 955 956 if (ret && ret != -ENOENT) { 957 pr_err("Starting init: %s exists but couldn't execute it (error %d)\n", 958 init_filename, ret); 959 } 960 961 return ret; 962 } 963 964 static noinline void __init kernel_init_freeable(void); 965 966 #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX) 967 bool rodata_enabled __ro_after_init = true; 968 static int __init set_debug_rodata(char *str) 969 { 970 return strtobool(str, &rodata_enabled); 971 } 972 __setup("rodata=", set_debug_rodata); 973 #endif 974 975 #ifdef CONFIG_STRICT_KERNEL_RWX 976 static void mark_readonly(void) 977 { 978 if (rodata_enabled) { 979 mark_rodata_ro(); 980 rodata_test(); 981 } else 982 pr_info("Kernel memory protection disabled.\n"); 983 } 984 #else 985 static inline void mark_readonly(void) 986 { 987 pr_warn("This architecture does not have kernel memory protection.\n"); 988 } 989 #endif 990 991 static int __ref kernel_init(void *unused) 992 { 993 int ret; 994 995 kernel_init_freeable(); 996 /* need to finish all async __init code before freeing the memory */ 997 async_synchronize_full(); 998 ftrace_free_init_mem(); 999 free_initmem(); 1000 mark_readonly(); 1001 system_state = SYSTEM_RUNNING; 1002 numa_default_policy(); 1003 1004 rcu_end_inkernel_boot(); 1005 1006 if (ramdisk_execute_command) { 1007 ret = run_init_process(ramdisk_execute_command); 1008 if (!ret) 1009 return 0; 1010 pr_err("Failed to execute %s (error %d)\n", 1011 ramdisk_execute_command, ret); 1012 } 1013 1014 /* 1015 * We try each of these until one succeeds. 1016 * 1017 * The Bourne shell can be used instead of init if we are 1018 * trying to recover a really broken machine. 1019 */ 1020 if (execute_command) { 1021 ret = run_init_process(execute_command); 1022 if (!ret) 1023 return 0; 1024 panic("Requested init %s failed (error %d).", 1025 execute_command, ret); 1026 } 1027 if (!try_to_run_init_process("/sbin/init") || 1028 !try_to_run_init_process("/etc/init") || 1029 !try_to_run_init_process("/bin/init") || 1030 !try_to_run_init_process("/bin/sh")) 1031 return 0; 1032 1033 panic("No working init found. Try passing init= option to kernel. " 1034 "See Linux Documentation/admin-guide/init.rst for guidance."); 1035 } 1036 1037 static noinline void __init kernel_init_freeable(void) 1038 { 1039 /* 1040 * Wait until kthreadd is all set-up. 1041 */ 1042 wait_for_completion(&kthreadd_done); 1043 1044 /* Now the scheduler is fully set up and can do blocking allocations */ 1045 gfp_allowed_mask = __GFP_BITS_MASK; 1046 1047 /* 1048 * init can allocate pages on any node 1049 */ 1050 set_mems_allowed(node_states[N_MEMORY]); 1051 1052 cad_pid = task_pid(current); 1053 1054 smp_prepare_cpus(setup_max_cpus); 1055 1056 workqueue_init(); 1057 1058 init_mm_internals(); 1059 1060 do_pre_smp_initcalls(); 1061 lockup_detector_init(); 1062 1063 smp_init(); 1064 sched_init_smp(); 1065 1066 page_alloc_init_late(); 1067 1068 do_basic_setup(); 1069 1070 /* Open the /dev/console on the rootfs, this should never fail */ 1071 if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0) 1072 pr_err("Warning: unable to open an initial console.\n"); 1073 1074 (void) sys_dup(0); 1075 (void) sys_dup(0); 1076 /* 1077 * check if there is an early userspace init. If yes, let it do all 1078 * the work 1079 */ 1080 1081 if (!ramdisk_execute_command) 1082 ramdisk_execute_command = "/init"; 1083 1084 if (sys_access((const char __user *) ramdisk_execute_command, 0) != 0) { 1085 ramdisk_execute_command = NULL; 1086 prepare_namespace(); 1087 } 1088 1089 /* 1090 * Ok, we have completed the initial bootup, and 1091 * we're essentially up and running. Get rid of the 1092 * initmem segments and start the user-mode stuff.. 1093 * 1094 * rootfs is available now, try loading the public keys 1095 * and default modules 1096 */ 1097 1098 integrity_load_keys(); 1099 load_default_modules(); 1100 } 1101