xref: /openbmc/linux/init/main.c (revision ec5c05e5ac8bcb4a6bcd92970e15494a85400d34)
1  // SPDX-License-Identifier: GPL-2.0-only
2  /*
3   *  linux/init/main.c
4   *
5   *  Copyright (C) 1991, 1992  Linus Torvalds
6   *
7   *  GK 2/5/95  -  Changed to support mounting root fs via NFS
8   *  Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
9   *  Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
10   *  Simplified starting of init:  Michael A. Griffith <grif@acm.org>
11   */
12  
13  #define DEBUG		/* Enable initcall_debug */
14  
15  #include <linux/types.h>
16  #include <linux/extable.h>
17  #include <linux/module.h>
18  #include <linux/proc_fs.h>
19  #include <linux/binfmts.h>
20  #include <linux/kernel.h>
21  #include <linux/syscalls.h>
22  #include <linux/stackprotector.h>
23  #include <linux/string.h>
24  #include <linux/ctype.h>
25  #include <linux/delay.h>
26  #include <linux/ioport.h>
27  #include <linux/init.h>
28  #include <linux/initrd.h>
29  #include <linux/memblock.h>
30  #include <linux/acpi.h>
31  #include <linux/bootconfig.h>
32  #include <linux/console.h>
33  #include <linux/nmi.h>
34  #include <linux/percpu.h>
35  #include <linux/kmod.h>
36  #include <linux/kprobes.h>
37  #include <linux/kmsan.h>
38  #include <linux/vmalloc.h>
39  #include <linux/kernel_stat.h>
40  #include <linux/start_kernel.h>
41  #include <linux/security.h>
42  #include <linux/smp.h>
43  #include <linux/profile.h>
44  #include <linux/kfence.h>
45  #include <linux/rcupdate.h>
46  #include <linux/srcu.h>
47  #include <linux/moduleparam.h>
48  #include <linux/kallsyms.h>
49  #include <linux/buildid.h>
50  #include <linux/writeback.h>
51  #include <linux/cpu.h>
52  #include <linux/cpuset.h>
53  #include <linux/cgroup.h>
54  #include <linux/efi.h>
55  #include <linux/tick.h>
56  #include <linux/sched/isolation.h>
57  #include <linux/interrupt.h>
58  #include <linux/taskstats_kern.h>
59  #include <linux/delayacct.h>
60  #include <linux/unistd.h>
61  #include <linux/utsname.h>
62  #include <linux/rmap.h>
63  #include <linux/mempolicy.h>
64  #include <linux/key.h>
65  #include <linux/page_ext.h>
66  #include <linux/debug_locks.h>
67  #include <linux/debugobjects.h>
68  #include <linux/lockdep.h>
69  #include <linux/kmemleak.h>
70  #include <linux/padata.h>
71  #include <linux/pid_namespace.h>
72  #include <linux/device/driver.h>
73  #include <linux/kthread.h>
74  #include <linux/sched.h>
75  #include <linux/sched/init.h>
76  #include <linux/signal.h>
77  #include <linux/idr.h>
78  #include <linux/kgdb.h>
79  #include <linux/ftrace.h>
80  #include <linux/async.h>
81  #include <linux/shmem_fs.h>
82  #include <linux/slab.h>
83  #include <linux/perf_event.h>
84  #include <linux/ptrace.h>
85  #include <linux/pti.h>
86  #include <linux/blkdev.h>
87  #include <linux/sched/clock.h>
88  #include <linux/sched/task.h>
89  #include <linux/sched/task_stack.h>
90  #include <linux/context_tracking.h>
91  #include <linux/random.h>
92  #include <linux/list.h>
93  #include <linux/integrity.h>
94  #include <linux/proc_ns.h>
95  #include <linux/io.h>
96  #include <linux/cache.h>
97  #include <linux/rodata_test.h>
98  #include <linux/jump_label.h>
99  #include <linux/mem_encrypt.h>
100  #include <linux/kcsan.h>
101  #include <linux/init_syscalls.h>
102  #include <linux/stackdepot.h>
103  #include <linux/randomize_kstack.h>
104  #include <net/net_namespace.h>
105  
106  #include <asm/io.h>
107  #include <asm/bugs.h>
108  #include <asm/setup.h>
109  #include <asm/sections.h>
110  #include <asm/cacheflush.h>
111  
112  #define CREATE_TRACE_POINTS
113  #include <trace/events/initcall.h>
114  
115  #include <kunit/test.h>
116  
117  static int kernel_init(void *);
118  
119  extern void init_IRQ(void);
120  extern void radix_tree_init(void);
121  extern void maple_tree_init(void);
122  
123  /*
124   * Debug helper: via this flag we know that we are in 'early bootup code'
125   * where only the boot processor is running with IRQ disabled.  This means
126   * two things - IRQ must not be enabled before the flag is cleared and some
127   * operations which are not allowed with IRQ disabled are allowed while the
128   * flag is set.
129   */
130  bool early_boot_irqs_disabled __read_mostly;
131  
132  enum system_states system_state __read_mostly;
133  EXPORT_SYMBOL(system_state);
134  
135  /*
136   * Boot command-line arguments
137   */
138  #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
139  #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
140  
141  extern void time_init(void);
142  /* Default late time init is NULL. archs can override this later. */
143  void (*__initdata late_time_init)(void);
144  
145  /* Untouched command line saved by arch-specific code. */
146  char __initdata boot_command_line[COMMAND_LINE_SIZE];
147  /* Untouched saved command line (eg. for /proc) */
148  char *saved_command_line __ro_after_init;
149  unsigned int saved_command_line_len __ro_after_init;
150  /* Command line for parameter parsing */
151  static char *static_command_line;
152  /* Untouched extra command line */
153  static char *extra_command_line;
154  /* Extra init arguments */
155  static char *extra_init_args;
156  
157  #ifdef CONFIG_BOOT_CONFIG
158  /* Is bootconfig on command line? */
159  static bool bootconfig_found;
160  static size_t initargs_offs;
161  #else
162  # define bootconfig_found false
163  # define initargs_offs 0
164  #endif
165  
166  static char *execute_command;
167  static char *ramdisk_execute_command = "/init";
168  
169  /*
170   * Used to generate warnings if static_key manipulation functions are used
171   * before jump_label_init is called.
172   */
173  bool static_key_initialized __read_mostly;
174  EXPORT_SYMBOL_GPL(static_key_initialized);
175  
176  /*
177   * If set, this is an indication to the drivers that reset the underlying
178   * device before going ahead with the initialization otherwise driver might
179   * rely on the BIOS and skip the reset operation.
180   *
181   * This is useful if kernel is booting in an unreliable environment.
182   * For ex. kdump situation where previous kernel has crashed, BIOS has been
183   * skipped and devices will be in unknown state.
184   */
185  unsigned int reset_devices;
186  EXPORT_SYMBOL(reset_devices);
187  
188  static int __init set_reset_devices(char *str)
189  {
190  	reset_devices = 1;
191  	return 1;
192  }
193  
194  __setup("reset_devices", set_reset_devices);
195  
196  static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
197  const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
198  static const char *panic_later, *panic_param;
199  
200  extern const struct obs_kernel_param __setup_start[], __setup_end[];
201  
202  static bool __init obsolete_checksetup(char *line)
203  {
204  	const struct obs_kernel_param *p;
205  	bool had_early_param = false;
206  
207  	p = __setup_start;
208  	do {
209  		int n = strlen(p->str);
210  		if (parameqn(line, p->str, n)) {
211  			if (p->early) {
212  				/* Already done in parse_early_param?
213  				 * (Needs exact match on param part).
214  				 * Keep iterating, as we can have early
215  				 * params and __setups of same names 8( */
216  				if (line[n] == '\0' || line[n] == '=')
217  					had_early_param = true;
218  			} else if (!p->setup_func) {
219  				pr_warn("Parameter %s is obsolete, ignored\n",
220  					p->str);
221  				return true;
222  			} else if (p->setup_func(line + n))
223  				return true;
224  		}
225  		p++;
226  	} while (p < __setup_end);
227  
228  	return had_early_param;
229  }
230  
231  /*
232   * This should be approx 2 Bo*oMips to start (note initial shift), and will
233   * still work even if initially too large, it will just take slightly longer
234   */
235  unsigned long loops_per_jiffy = (1<<12);
236  EXPORT_SYMBOL(loops_per_jiffy);
237  
238  static int __init debug_kernel(char *str)
239  {
240  	console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
241  	return 0;
242  }
243  
244  static int __init quiet_kernel(char *str)
245  {
246  	console_loglevel = CONSOLE_LOGLEVEL_QUIET;
247  	return 0;
248  }
249  
250  early_param("debug", debug_kernel);
251  early_param("quiet", quiet_kernel);
252  
253  static int __init loglevel(char *str)
254  {
255  	int newlevel;
256  
257  	/*
258  	 * Only update loglevel value when a correct setting was passed,
259  	 * to prevent blind crashes (when loglevel being set to 0) that
260  	 * are quite hard to debug
261  	 */
262  	if (get_option(&str, &newlevel)) {
263  		console_loglevel = newlevel;
264  		return 0;
265  	}
266  
267  	return -EINVAL;
268  }
269  
270  early_param("loglevel", loglevel);
271  
272  #ifdef CONFIG_BLK_DEV_INITRD
273  static void * __init get_boot_config_from_initrd(size_t *_size)
274  {
275  	u32 size, csum;
276  	char *data;
277  	u32 *hdr;
278  	int i;
279  
280  	if (!initrd_end)
281  		return NULL;
282  
283  	data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;
284  	/*
285  	 * Since Grub may align the size of initrd to 4, we must
286  	 * check the preceding 3 bytes as well.
287  	 */
288  	for (i = 0; i < 4; i++) {
289  		if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))
290  			goto found;
291  		data--;
292  	}
293  	return NULL;
294  
295  found:
296  	hdr = (u32 *)(data - 8);
297  	size = le32_to_cpu(hdr[0]);
298  	csum = le32_to_cpu(hdr[1]);
299  
300  	data = ((void *)hdr) - size;
301  	if ((unsigned long)data < initrd_start) {
302  		pr_err("bootconfig size %d is greater than initrd size %ld\n",
303  			size, initrd_end - initrd_start);
304  		return NULL;
305  	}
306  
307  	if (xbc_calc_checksum(data, size) != csum) {
308  		pr_err("bootconfig checksum failed\n");
309  		return NULL;
310  	}
311  
312  	/* Remove bootconfig from initramfs/initrd */
313  	initrd_end = (unsigned long)data;
314  	if (_size)
315  		*_size = size;
316  
317  	return data;
318  }
319  #else
320  static void * __init get_boot_config_from_initrd(size_t *_size)
321  {
322  	return NULL;
323  }
324  #endif
325  
326  #ifdef CONFIG_BOOT_CONFIG
327  
328  static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
329  
330  #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
331  
332  static int __init xbc_snprint_cmdline(char *buf, size_t size,
333  				      struct xbc_node *root)
334  {
335  	struct xbc_node *knode, *vnode;
336  	char *end = buf + size;
337  	const char *val;
338  	int ret;
339  
340  	xbc_node_for_each_key_value(root, knode, val) {
341  		ret = xbc_node_compose_key_after(root, knode,
342  					xbc_namebuf, XBC_KEYLEN_MAX);
343  		if (ret < 0)
344  			return ret;
345  
346  		vnode = xbc_node_get_child(knode);
347  		if (!vnode) {
348  			ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
349  			if (ret < 0)
350  				return ret;
351  			buf += ret;
352  			continue;
353  		}
354  		xbc_array_for_each_value(vnode, val) {
355  			ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ",
356  				       xbc_namebuf, val);
357  			if (ret < 0)
358  				return ret;
359  			buf += ret;
360  		}
361  	}
362  
363  	return buf - (end - size);
364  }
365  #undef rest
366  
367  /* Make an extra command line under given key word */
368  static char * __init xbc_make_cmdline(const char *key)
369  {
370  	struct xbc_node *root;
371  	char *new_cmdline;
372  	int ret, len = 0;
373  
374  	root = xbc_find_node(key);
375  	if (!root)
376  		return NULL;
377  
378  	/* Count required buffer size */
379  	len = xbc_snprint_cmdline(NULL, 0, root);
380  	if (len <= 0)
381  		return NULL;
382  
383  	new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);
384  	if (!new_cmdline) {
385  		pr_err("Failed to allocate memory for extra kernel cmdline.\n");
386  		return NULL;
387  	}
388  
389  	ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);
390  	if (ret < 0 || ret > len) {
391  		pr_err("Failed to print extra kernel cmdline.\n");
392  		memblock_free(new_cmdline, len + 1);
393  		return NULL;
394  	}
395  
396  	return new_cmdline;
397  }
398  
399  static int __init bootconfig_params(char *param, char *val,
400  				    const char *unused, void *arg)
401  {
402  	if (strcmp(param, "bootconfig") == 0) {
403  		bootconfig_found = true;
404  	}
405  	return 0;
406  }
407  
408  static int __init warn_bootconfig(char *str)
409  {
410  	/* The 'bootconfig' has been handled by bootconfig_params(). */
411  	return 0;
412  }
413  
414  static void __init setup_boot_config(void)
415  {
416  	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
417  	const char *msg, *data;
418  	int pos, ret;
419  	size_t size;
420  	char *err;
421  
422  	/* Cut out the bootconfig data even if we have no bootconfig option */
423  	data = get_boot_config_from_initrd(&size);
424  	/* If there is no bootconfig in initrd, try embedded one. */
425  	if (!data)
426  		data = xbc_get_embedded_bootconfig(&size);
427  
428  	strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
429  	err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
430  			 bootconfig_params);
431  
432  	if (IS_ERR(err) || !bootconfig_found)
433  		return;
434  
435  	/* parse_args() stops at the next param of '--' and returns an address */
436  	if (err)
437  		initargs_offs = err - tmp_cmdline;
438  
439  	if (!data) {
440  		pr_err("'bootconfig' found on command line, but no bootconfig found\n");
441  		return;
442  	}
443  
444  	if (size >= XBC_DATA_MAX) {
445  		pr_err("bootconfig size %ld greater than max size %d\n",
446  			(long)size, XBC_DATA_MAX);
447  		return;
448  	}
449  
450  	ret = xbc_init(data, size, &msg, &pos);
451  	if (ret < 0) {
452  		if (pos < 0)
453  			pr_err("Failed to init bootconfig: %s.\n", msg);
454  		else
455  			pr_err("Failed to parse bootconfig: %s at %d.\n",
456  				msg, pos);
457  	} else {
458  		xbc_get_info(&ret, NULL);
459  		pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);
460  		/* keys starting with "kernel." are passed via cmdline */
461  		extra_command_line = xbc_make_cmdline("kernel");
462  		/* Also, "init." keys are init arguments */
463  		extra_init_args = xbc_make_cmdline("init");
464  	}
465  	return;
466  }
467  
468  static void __init exit_boot_config(void)
469  {
470  	xbc_exit();
471  }
472  
473  #else	/* !CONFIG_BOOT_CONFIG */
474  
475  static void __init setup_boot_config(void)
476  {
477  	/* Remove bootconfig data from initrd */
478  	get_boot_config_from_initrd(NULL);
479  }
480  
481  static int __init warn_bootconfig(char *str)
482  {
483  	pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");
484  	return 0;
485  }
486  
487  #define exit_boot_config()	do {} while (0)
488  
489  #endif	/* CONFIG_BOOT_CONFIG */
490  
491  early_param("bootconfig", warn_bootconfig);
492  
493  /* Change NUL term back to "=", to make "param" the whole string. */
494  static void __init repair_env_string(char *param, char *val)
495  {
496  	if (val) {
497  		/* param=val or param="val"? */
498  		if (val == param+strlen(param)+1)
499  			val[-1] = '=';
500  		else if (val == param+strlen(param)+2) {
501  			val[-2] = '=';
502  			memmove(val-1, val, strlen(val)+1);
503  		} else
504  			BUG();
505  	}
506  }
507  
508  /* Anything after -- gets handed straight to init. */
509  static int __init set_init_arg(char *param, char *val,
510  			       const char *unused, void *arg)
511  {
512  	unsigned int i;
513  
514  	if (panic_later)
515  		return 0;
516  
517  	repair_env_string(param, val);
518  
519  	for (i = 0; argv_init[i]; i++) {
520  		if (i == MAX_INIT_ARGS) {
521  			panic_later = "init";
522  			panic_param = param;
523  			return 0;
524  		}
525  	}
526  	argv_init[i] = param;
527  	return 0;
528  }
529  
530  /*
531   * Unknown boot options get handed to init, unless they look like
532   * unused parameters (modprobe will find them in /proc/cmdline).
533   */
534  static int __init unknown_bootoption(char *param, char *val,
535  				     const char *unused, void *arg)
536  {
537  	size_t len = strlen(param);
538  
539  	repair_env_string(param, val);
540  
541  	/* Handle obsolete-style parameters */
542  	if (obsolete_checksetup(param))
543  		return 0;
544  
545  	/* Unused module parameter. */
546  	if (strnchr(param, len, '.'))
547  		return 0;
548  
549  	if (panic_later)
550  		return 0;
551  
552  	if (val) {
553  		/* Environment option */
554  		unsigned int i;
555  		for (i = 0; envp_init[i]; i++) {
556  			if (i == MAX_INIT_ENVS) {
557  				panic_later = "env";
558  				panic_param = param;
559  			}
560  			if (!strncmp(param, envp_init[i], len+1))
561  				break;
562  		}
563  		envp_init[i] = param;
564  	} else {
565  		/* Command line option */
566  		unsigned int i;
567  		for (i = 0; argv_init[i]; i++) {
568  			if (i == MAX_INIT_ARGS) {
569  				panic_later = "init";
570  				panic_param = param;
571  			}
572  		}
573  		argv_init[i] = param;
574  	}
575  	return 0;
576  }
577  
578  static int __init init_setup(char *str)
579  {
580  	unsigned int i;
581  
582  	execute_command = str;
583  	/*
584  	 * In case LILO is going to boot us with default command line,
585  	 * it prepends "auto" before the whole cmdline which makes
586  	 * the shell think it should execute a script with such name.
587  	 * So we ignore all arguments entered _before_ init=... [MJ]
588  	 */
589  	for (i = 1; i < MAX_INIT_ARGS; i++)
590  		argv_init[i] = NULL;
591  	return 1;
592  }
593  __setup("init=", init_setup);
594  
595  static int __init rdinit_setup(char *str)
596  {
597  	unsigned int i;
598  
599  	ramdisk_execute_command = str;
600  	/* See "auto" comment in init_setup */
601  	for (i = 1; i < MAX_INIT_ARGS; i++)
602  		argv_init[i] = NULL;
603  	return 1;
604  }
605  __setup("rdinit=", rdinit_setup);
606  
607  #ifndef CONFIG_SMP
608  static const unsigned int setup_max_cpus = NR_CPUS;
609  static inline void setup_nr_cpu_ids(void) { }
610  static inline void smp_prepare_cpus(unsigned int maxcpus) { }
611  #endif
612  
613  /*
614   * We need to store the untouched command line for future reference.
615   * We also need to store the touched command line since the parameter
616   * parsing is performed in place, and we should allow a component to
617   * store reference of name/value for future reference.
618   */
619  static void __init setup_command_line(char *command_line)
620  {
621  	size_t len, xlen = 0, ilen = 0;
622  
623  	if (extra_command_line)
624  		xlen = strlen(extra_command_line);
625  	if (extra_init_args)
626  		ilen = strlen(extra_init_args) + 4; /* for " -- " */
627  
628  	len = xlen + strlen(boot_command_line) + 1;
629  
630  	saved_command_line = memblock_alloc(len + ilen, SMP_CACHE_BYTES);
631  	if (!saved_command_line)
632  		panic("%s: Failed to allocate %zu bytes\n", __func__, len + ilen);
633  
634  	static_command_line = memblock_alloc(len, SMP_CACHE_BYTES);
635  	if (!static_command_line)
636  		panic("%s: Failed to allocate %zu bytes\n", __func__, len);
637  
638  	if (xlen) {
639  		/*
640  		 * We have to put extra_command_line before boot command
641  		 * lines because there could be dashes (separator of init
642  		 * command line) in the command lines.
643  		 */
644  		strcpy(saved_command_line, extra_command_line);
645  		strcpy(static_command_line, extra_command_line);
646  	}
647  	strcpy(saved_command_line + xlen, boot_command_line);
648  	strcpy(static_command_line + xlen, command_line);
649  
650  	if (ilen) {
651  		/*
652  		 * Append supplemental init boot args to saved_command_line
653  		 * so that user can check what command line options passed
654  		 * to init.
655  		 * The order should always be
656  		 * " -- "[bootconfig init-param][cmdline init-param]
657  		 */
658  		if (initargs_offs) {
659  			len = xlen + initargs_offs;
660  			strcpy(saved_command_line + len, extra_init_args);
661  			len += ilen - 4;	/* strlen(extra_init_args) */
662  			strcpy(saved_command_line + len,
663  				boot_command_line + initargs_offs - 1);
664  		} else {
665  			len = strlen(saved_command_line);
666  			strcpy(saved_command_line + len, " -- ");
667  			len += 4;
668  			strcpy(saved_command_line + len, extra_init_args);
669  		}
670  	}
671  
672  	saved_command_line_len = strlen(saved_command_line);
673  }
674  
675  /*
676   * We need to finalize in a non-__init function or else race conditions
677   * between the root thread and the init thread may cause start_kernel to
678   * be reaped by free_initmem before the root thread has proceeded to
679   * cpu_idle.
680   *
681   * gcc-3.4 accidentally inlines this function, so use noinline.
682   */
683  
684  static __initdata DECLARE_COMPLETION(kthreadd_done);
685  
686  noinline void __ref rest_init(void)
687  {
688  	struct task_struct *tsk;
689  	int pid;
690  
691  	rcu_scheduler_starting();
692  	/*
693  	 * We need to spawn init first so that it obtains pid 1, however
694  	 * the init task will end up wanting to create kthreads, which, if
695  	 * we schedule it before we create kthreadd, will OOPS.
696  	 */
697  	pid = user_mode_thread(kernel_init, NULL, CLONE_FS);
698  	/*
699  	 * Pin init on the boot CPU. Task migration is not properly working
700  	 * until sched_init_smp() has been run. It will set the allowed
701  	 * CPUs for init to the non isolated CPUs.
702  	 */
703  	rcu_read_lock();
704  	tsk = find_task_by_pid_ns(pid, &init_pid_ns);
705  	tsk->flags |= PF_NO_SETAFFINITY;
706  	set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));
707  	rcu_read_unlock();
708  
709  	numa_default_policy();
710  	pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES);
711  	rcu_read_lock();
712  	kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
713  	rcu_read_unlock();
714  
715  	/*
716  	 * Enable might_sleep() and smp_processor_id() checks.
717  	 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y
718  	 * kernel_thread() would trigger might_sleep() splats. With
719  	 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled
720  	 * already, but it's stuck on the kthreadd_done completion.
721  	 */
722  	system_state = SYSTEM_SCHEDULING;
723  
724  	complete(&kthreadd_done);
725  
726  	/*
727  	 * The boot idle thread must execute schedule()
728  	 * at least once to get things moving:
729  	 */
730  	schedule_preempt_disabled();
731  	/* Call into cpu_idle with preempt disabled */
732  	cpu_startup_entry(CPUHP_ONLINE);
733  }
734  
735  /* Check for early params. */
736  static int __init do_early_param(char *param, char *val,
737  				 const char *unused, void *arg)
738  {
739  	const struct obs_kernel_param *p;
740  
741  	for (p = __setup_start; p < __setup_end; p++) {
742  		if ((p->early && parameq(param, p->str)) ||
743  		    (strcmp(param, "console") == 0 &&
744  		     strcmp(p->str, "earlycon") == 0)
745  		) {
746  			if (p->setup_func(val) != 0)
747  				pr_warn("Malformed early option '%s'\n", param);
748  		}
749  	}
750  	/* We accept everything at this stage. */
751  	return 0;
752  }
753  
754  void __init parse_early_options(char *cmdline)
755  {
756  	parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
757  		   do_early_param);
758  }
759  
760  /* Arch code calls this early on, or if not, just before other parsing. */
761  void __init parse_early_param(void)
762  {
763  	static int done __initdata;
764  	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
765  
766  	if (done)
767  		return;
768  
769  	/* All fall through to do_early_param. */
770  	strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
771  	parse_early_options(tmp_cmdline);
772  	done = 1;
773  }
774  
775  void __init __weak arch_post_acpi_subsys_init(void) { }
776  
777  void __init __weak smp_setup_processor_id(void)
778  {
779  }
780  
781  # if THREAD_SIZE >= PAGE_SIZE
782  void __init __weak thread_stack_cache_init(void)
783  {
784  }
785  #endif
786  
787  void __init __weak mem_encrypt_init(void) { }
788  
789  void __init __weak poking_init(void) { }
790  
791  void __init __weak pgtable_cache_init(void) { }
792  
793  void __init __weak trap_init(void) { }
794  
795  bool initcall_debug;
796  core_param(initcall_debug, initcall_debug, bool, 0644);
797  
798  #ifdef TRACEPOINTS_ENABLED
799  static void __init initcall_debug_enable(void);
800  #else
801  static inline void initcall_debug_enable(void)
802  {
803  }
804  #endif
805  
806  /* Report memory auto-initialization states for this boot. */
807  static void __init report_meminit(void)
808  {
809  	const char *stack;
810  
811  	if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN))
812  		stack = "all(pattern)";
813  	else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO))
814  		stack = "all(zero)";
815  	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL))
816  		stack = "byref_all(zero)";
817  	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF))
818  		stack = "byref(zero)";
819  	else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER))
820  		stack = "__user(zero)";
821  	else
822  		stack = "off";
823  
824  	pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",
825  		stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",
826  		want_init_on_free() ? "on" : "off");
827  	if (want_init_on_free())
828  		pr_info("mem auto-init: clearing system memory may take some time...\n");
829  }
830  
831  /*
832   * Set up kernel memory allocators
833   */
834  static void __init mm_init(void)
835  {
836  	/*
837  	 * page_ext requires contiguous pages,
838  	 * bigger than MAX_ORDER unless SPARSEMEM.
839  	 */
840  	page_ext_init_flatmem();
841  	init_mem_debugging_and_hardening();
842  	kfence_alloc_pool();
843  	report_meminit();
844  	kmsan_init_shadow();
845  	stack_depot_early_init();
846  	mem_init();
847  	mem_init_print_info();
848  	kmem_cache_init();
849  	/*
850  	 * page_owner must be initialized after buddy is ready, and also after
851  	 * slab is ready so that stack_depot_init() works properly
852  	 */
853  	page_ext_init_flatmem_late();
854  	kmemleak_init();
855  	pgtable_init();
856  	debug_objects_mem_init();
857  	vmalloc_init();
858  	/* Should be run after vmap initialization */
859  	if (early_page_ext_enabled())
860  		page_ext_init();
861  	/* Should be run before the first non-init thread is created */
862  	init_espfix_bsp();
863  	/* Should be run after espfix64 is set up. */
864  	pti_init();
865  	kmsan_init_runtime();
866  	mm_cache_init();
867  }
868  
869  #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
870  DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,
871  			   randomize_kstack_offset);
872  DEFINE_PER_CPU(u32, kstack_offset);
873  
874  static int __init early_randomize_kstack_offset(char *buf)
875  {
876  	int ret;
877  	bool bool_result;
878  
879  	ret = kstrtobool(buf, &bool_result);
880  	if (ret)
881  		return ret;
882  
883  	if (bool_result)
884  		static_branch_enable(&randomize_kstack_offset);
885  	else
886  		static_branch_disable(&randomize_kstack_offset);
887  	return 0;
888  }
889  early_param("randomize_kstack_offset", early_randomize_kstack_offset);
890  #endif
891  
892  void __init __weak arch_call_rest_init(void)
893  {
894  	rest_init();
895  }
896  
897  static void __init print_unknown_bootoptions(void)
898  {
899  	char *unknown_options;
900  	char *end;
901  	const char *const *p;
902  	size_t len;
903  
904  	if (panic_later || (!argv_init[1] && !envp_init[2]))
905  		return;
906  
907  	/*
908  	 * Determine how many options we have to print out, plus a space
909  	 * before each
910  	 */
911  	len = 1; /* null terminator */
912  	for (p = &argv_init[1]; *p; p++) {
913  		len++;
914  		len += strlen(*p);
915  	}
916  	for (p = &envp_init[2]; *p; p++) {
917  		len++;
918  		len += strlen(*p);
919  	}
920  
921  	unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);
922  	if (!unknown_options) {
923  		pr_err("%s: Failed to allocate %zu bytes\n",
924  			__func__, len);
925  		return;
926  	}
927  	end = unknown_options;
928  
929  	for (p = &argv_init[1]; *p; p++)
930  		end += sprintf(end, " %s", *p);
931  	for (p = &envp_init[2]; *p; p++)
932  		end += sprintf(end, " %s", *p);
933  
934  	/* Start at unknown_options[1] to skip the initial space */
935  	pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
936  		&unknown_options[1]);
937  	memblock_free(unknown_options, len);
938  }
939  
940  asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
941  {
942  	char *command_line;
943  	char *after_dashes;
944  
945  	set_task_stack_end_magic(&init_task);
946  	smp_setup_processor_id();
947  	debug_objects_early_init();
948  	init_vmlinux_build_id();
949  
950  	cgroup_init_early();
951  
952  	local_irq_disable();
953  	early_boot_irqs_disabled = true;
954  
955  	/*
956  	 * Interrupts are still disabled. Do necessary setups, then
957  	 * enable them.
958  	 */
959  	boot_cpu_init();
960  	page_address_init();
961  	pr_notice("%s", linux_banner);
962  	early_security_init();
963  	setup_arch(&command_line);
964  	setup_boot_config();
965  	setup_command_line(command_line);
966  	setup_nr_cpu_ids();
967  	setup_per_cpu_areas();
968  	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
969  	boot_cpu_hotplug_init();
970  
971  	build_all_zonelists(NULL);
972  	page_alloc_init();
973  
974  	pr_notice("Kernel command line: %s\n", saved_command_line);
975  	/* parameters may set static keys */
976  	jump_label_init();
977  	parse_early_param();
978  	after_dashes = parse_args("Booting kernel",
979  				  static_command_line, __start___param,
980  				  __stop___param - __start___param,
981  				  -1, -1, NULL, &unknown_bootoption);
982  	print_unknown_bootoptions();
983  	if (!IS_ERR_OR_NULL(after_dashes))
984  		parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
985  			   NULL, set_init_arg);
986  	if (extra_init_args)
987  		parse_args("Setting extra init args", extra_init_args,
988  			   NULL, 0, -1, -1, NULL, set_init_arg);
989  
990  	/* Architectural and non-timekeeping rng init, before allocator init */
991  	random_init_early(command_line);
992  
993  	/*
994  	 * These use large bootmem allocations and must precede
995  	 * kmem_cache_init()
996  	 */
997  	setup_log_buf(0);
998  	vfs_caches_init_early();
999  	sort_main_extable();
1000  	trap_init();
1001  	mm_init();
1002  	poking_init();
1003  	ftrace_init();
1004  
1005  	/* trace_printk can be enabled here */
1006  	early_trace_init();
1007  
1008  	/*
1009  	 * Set up the scheduler prior starting any interrupts (such as the
1010  	 * timer interrupt). Full topology setup happens at smp_init()
1011  	 * time - but meanwhile we still have a functioning scheduler.
1012  	 */
1013  	sched_init();
1014  
1015  	if (WARN(!irqs_disabled(),
1016  		 "Interrupts were enabled *very* early, fixing it\n"))
1017  		local_irq_disable();
1018  	radix_tree_init();
1019  	maple_tree_init();
1020  
1021  	/*
1022  	 * Set up housekeeping before setting up workqueues to allow the unbound
1023  	 * workqueue to take non-housekeeping into account.
1024  	 */
1025  	housekeeping_init();
1026  
1027  	/*
1028  	 * Allow workqueue creation and work item queueing/cancelling
1029  	 * early.  Work item execution depends on kthreads and starts after
1030  	 * workqueue_init().
1031  	 */
1032  	workqueue_init_early();
1033  
1034  	rcu_init();
1035  
1036  	/* Trace events are available after this */
1037  	trace_init();
1038  
1039  	if (initcall_debug)
1040  		initcall_debug_enable();
1041  
1042  	context_tracking_init();
1043  	/* init some links before init_ISA_irqs() */
1044  	early_irq_init();
1045  	init_IRQ();
1046  	tick_init();
1047  	rcu_init_nohz();
1048  	init_timers();
1049  	srcu_init();
1050  	hrtimers_init();
1051  	softirq_init();
1052  	timekeeping_init();
1053  	time_init();
1054  
1055  	/* This must be after timekeeping is initialized */
1056  	random_init();
1057  
1058  	/* These make use of the fully initialized rng */
1059  	kfence_init();
1060  	boot_init_stack_canary();
1061  
1062  	perf_event_init();
1063  	profile_init();
1064  	call_function_init();
1065  	WARN(!irqs_disabled(), "Interrupts were enabled early\n");
1066  
1067  	early_boot_irqs_disabled = false;
1068  	local_irq_enable();
1069  
1070  	kmem_cache_init_late();
1071  
1072  	/*
1073  	 * HACK ALERT! This is early. We're enabling the console before
1074  	 * we've done PCI setups etc, and console_init() must be aware of
1075  	 * this. But we do want output early, in case something goes wrong.
1076  	 */
1077  	console_init();
1078  	if (panic_later)
1079  		panic("Too many boot %s vars at `%s'", panic_later,
1080  		      panic_param);
1081  
1082  	lockdep_init();
1083  
1084  	/*
1085  	 * Need to run this when irqs are enabled, because it wants
1086  	 * to self-test [hard/soft]-irqs on/off lock inversion bugs
1087  	 * too:
1088  	 */
1089  	locking_selftest();
1090  
1091  	/*
1092  	 * This needs to be called before any devices perform DMA
1093  	 * operations that might use the SWIOTLB bounce buffers. It will
1094  	 * mark the bounce buffers as decrypted so that their usage will
1095  	 * not cause "plain-text" data to be decrypted when accessed.
1096  	 */
1097  	mem_encrypt_init();
1098  
1099  #ifdef CONFIG_BLK_DEV_INITRD
1100  	if (initrd_start && !initrd_below_start_ok &&
1101  	    page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
1102  		pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
1103  		    page_to_pfn(virt_to_page((void *)initrd_start)),
1104  		    min_low_pfn);
1105  		initrd_start = 0;
1106  	}
1107  #endif
1108  	setup_per_cpu_pageset();
1109  	numa_policy_init();
1110  	acpi_early_init();
1111  	if (late_time_init)
1112  		late_time_init();
1113  	sched_clock_init();
1114  	calibrate_delay();
1115  	pid_idr_init();
1116  	anon_vma_init();
1117  #ifdef CONFIG_X86
1118  	if (efi_enabled(EFI_RUNTIME_SERVICES))
1119  		efi_enter_virtual_mode();
1120  #endif
1121  	thread_stack_cache_init();
1122  	cred_init();
1123  	fork_init();
1124  	proc_caches_init();
1125  	uts_ns_init();
1126  	key_init();
1127  	security_init();
1128  	dbg_late_init();
1129  	net_ns_init();
1130  	vfs_caches_init();
1131  	pagecache_init();
1132  	signals_init();
1133  	seq_file_init();
1134  	proc_root_init();
1135  	nsfs_init();
1136  	cpuset_init();
1137  	cgroup_init();
1138  	taskstats_init_early();
1139  	delayacct_init();
1140  
1141  	check_bugs();
1142  
1143  	acpi_subsystem_init();
1144  	arch_post_acpi_subsys_init();
1145  	kcsan_init();
1146  
1147  	/* Do the rest non-__init'ed, we're now alive */
1148  	arch_call_rest_init();
1149  
1150  	prevent_tail_call_optimization();
1151  }
1152  
1153  /* Call all constructor functions linked into the kernel. */
1154  static void __init do_ctors(void)
1155  {
1156  /*
1157   * For UML, the constructors have already been called by the
1158   * normal setup code as it's just a normal ELF binary, so we
1159   * cannot do it again - but we do need CONFIG_CONSTRUCTORS
1160   * even on UML for modules.
1161   */
1162  #if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)
1163  	ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
1164  
1165  	for (; fn < (ctor_fn_t *) __ctors_end; fn++)
1166  		(*fn)();
1167  #endif
1168  }
1169  
1170  #ifdef CONFIG_KALLSYMS
1171  struct blacklist_entry {
1172  	struct list_head next;
1173  	char *buf;
1174  };
1175  
1176  static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
1177  
1178  static int __init initcall_blacklist(char *str)
1179  {
1180  	char *str_entry;
1181  	struct blacklist_entry *entry;
1182  
1183  	/* str argument is a comma-separated list of functions */
1184  	do {
1185  		str_entry = strsep(&str, ",");
1186  		if (str_entry) {
1187  			pr_debug("blacklisting initcall %s\n", str_entry);
1188  			entry = memblock_alloc(sizeof(*entry),
1189  					       SMP_CACHE_BYTES);
1190  			if (!entry)
1191  				panic("%s: Failed to allocate %zu bytes\n",
1192  				      __func__, sizeof(*entry));
1193  			entry->buf = memblock_alloc(strlen(str_entry) + 1,
1194  						    SMP_CACHE_BYTES);
1195  			if (!entry->buf)
1196  				panic("%s: Failed to allocate %zu bytes\n",
1197  				      __func__, strlen(str_entry) + 1);
1198  			strcpy(entry->buf, str_entry);
1199  			list_add(&entry->next, &blacklisted_initcalls);
1200  		}
1201  	} while (str_entry);
1202  
1203  	return 1;
1204  }
1205  
1206  static bool __init_or_module initcall_blacklisted(initcall_t fn)
1207  {
1208  	struct blacklist_entry *entry;
1209  	char fn_name[KSYM_SYMBOL_LEN];
1210  	unsigned long addr;
1211  
1212  	if (list_empty(&blacklisted_initcalls))
1213  		return false;
1214  
1215  	addr = (unsigned long) dereference_function_descriptor(fn);
1216  	sprint_symbol_no_offset(fn_name, addr);
1217  
1218  	/*
1219  	 * fn will be "function_name [module_name]" where [module_name] is not
1220  	 * displayed for built-in init functions.  Strip off the [module_name].
1221  	 */
1222  	strreplace(fn_name, ' ', '\0');
1223  
1224  	list_for_each_entry(entry, &blacklisted_initcalls, next) {
1225  		if (!strcmp(fn_name, entry->buf)) {
1226  			pr_debug("initcall %s blacklisted\n", fn_name);
1227  			return true;
1228  		}
1229  	}
1230  
1231  	return false;
1232  }
1233  #else
1234  static int __init initcall_blacklist(char *str)
1235  {
1236  	pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");
1237  	return 0;
1238  }
1239  
1240  static bool __init_or_module initcall_blacklisted(initcall_t fn)
1241  {
1242  	return false;
1243  }
1244  #endif
1245  __setup("initcall_blacklist=", initcall_blacklist);
1246  
1247  static __init_or_module void
1248  trace_initcall_start_cb(void *data, initcall_t fn)
1249  {
1250  	ktime_t *calltime = data;
1251  
1252  	printk(KERN_DEBUG "calling  %pS @ %i\n", fn, task_pid_nr(current));
1253  	*calltime = ktime_get();
1254  }
1255  
1256  static __init_or_module void
1257  trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
1258  {
1259  	ktime_t rettime, *calltime = data;
1260  
1261  	rettime = ktime_get();
1262  	printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
1263  		 fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));
1264  }
1265  
1266  static ktime_t initcall_calltime;
1267  
1268  #ifdef TRACEPOINTS_ENABLED
1269  static void __init initcall_debug_enable(void)
1270  {
1271  	int ret;
1272  
1273  	ret = register_trace_initcall_start(trace_initcall_start_cb,
1274  					    &initcall_calltime);
1275  	ret |= register_trace_initcall_finish(trace_initcall_finish_cb,
1276  					      &initcall_calltime);
1277  	WARN(ret, "Failed to register initcall tracepoints\n");
1278  }
1279  # define do_trace_initcall_start	trace_initcall_start
1280  # define do_trace_initcall_finish	trace_initcall_finish
1281  #else
1282  static inline void do_trace_initcall_start(initcall_t fn)
1283  {
1284  	if (!initcall_debug)
1285  		return;
1286  	trace_initcall_start_cb(&initcall_calltime, fn);
1287  }
1288  static inline void do_trace_initcall_finish(initcall_t fn, int ret)
1289  {
1290  	if (!initcall_debug)
1291  		return;
1292  	trace_initcall_finish_cb(&initcall_calltime, fn, ret);
1293  }
1294  #endif /* !TRACEPOINTS_ENABLED */
1295  
1296  int __init_or_module do_one_initcall(initcall_t fn)
1297  {
1298  	int count = preempt_count();
1299  	char msgbuf[64];
1300  	int ret;
1301  
1302  	if (initcall_blacklisted(fn))
1303  		return -EPERM;
1304  
1305  	do_trace_initcall_start(fn);
1306  	ret = fn();
1307  	do_trace_initcall_finish(fn, ret);
1308  
1309  	msgbuf[0] = 0;
1310  
1311  	if (preempt_count() != count) {
1312  		sprintf(msgbuf, "preemption imbalance ");
1313  		preempt_count_set(count);
1314  	}
1315  	if (irqs_disabled()) {
1316  		strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
1317  		local_irq_enable();
1318  	}
1319  	WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
1320  
1321  	add_latent_entropy();
1322  	return ret;
1323  }
1324  
1325  
1326  extern initcall_entry_t __initcall_start[];
1327  extern initcall_entry_t __initcall0_start[];
1328  extern initcall_entry_t __initcall1_start[];
1329  extern initcall_entry_t __initcall2_start[];
1330  extern initcall_entry_t __initcall3_start[];
1331  extern initcall_entry_t __initcall4_start[];
1332  extern initcall_entry_t __initcall5_start[];
1333  extern initcall_entry_t __initcall6_start[];
1334  extern initcall_entry_t __initcall7_start[];
1335  extern initcall_entry_t __initcall_end[];
1336  
1337  static initcall_entry_t *initcall_levels[] __initdata = {
1338  	__initcall0_start,
1339  	__initcall1_start,
1340  	__initcall2_start,
1341  	__initcall3_start,
1342  	__initcall4_start,
1343  	__initcall5_start,
1344  	__initcall6_start,
1345  	__initcall7_start,
1346  	__initcall_end,
1347  };
1348  
1349  /* Keep these in sync with initcalls in include/linux/init.h */
1350  static const char *initcall_level_names[] __initdata = {
1351  	"pure",
1352  	"core",
1353  	"postcore",
1354  	"arch",
1355  	"subsys",
1356  	"fs",
1357  	"device",
1358  	"late",
1359  };
1360  
1361  static int __init ignore_unknown_bootoption(char *param, char *val,
1362  			       const char *unused, void *arg)
1363  {
1364  	return 0;
1365  }
1366  
1367  static void __init do_initcall_level(int level, char *command_line)
1368  {
1369  	initcall_entry_t *fn;
1370  
1371  	parse_args(initcall_level_names[level],
1372  		   command_line, __start___param,
1373  		   __stop___param - __start___param,
1374  		   level, level,
1375  		   NULL, ignore_unknown_bootoption);
1376  
1377  	trace_initcall_level(initcall_level_names[level]);
1378  	for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
1379  		do_one_initcall(initcall_from_entry(fn));
1380  }
1381  
1382  static void __init do_initcalls(void)
1383  {
1384  	int level;
1385  	size_t len = saved_command_line_len + 1;
1386  	char *command_line;
1387  
1388  	command_line = kzalloc(len, GFP_KERNEL);
1389  	if (!command_line)
1390  		panic("%s: Failed to allocate %zu bytes\n", __func__, len);
1391  
1392  	for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {
1393  		/* Parser modifies command_line, restore it each time */
1394  		strcpy(command_line, saved_command_line);
1395  		do_initcall_level(level, command_line);
1396  	}
1397  
1398  	kfree(command_line);
1399  }
1400  
1401  /*
1402   * Ok, the machine is now initialized. None of the devices
1403   * have been touched yet, but the CPU subsystem is up and
1404   * running, and memory and process management works.
1405   *
1406   * Now we can finally start doing some real work..
1407   */
1408  static void __init do_basic_setup(void)
1409  {
1410  	cpuset_init_smp();
1411  	driver_init();
1412  	init_irq_proc();
1413  	do_ctors();
1414  	do_initcalls();
1415  }
1416  
1417  static void __init do_pre_smp_initcalls(void)
1418  {
1419  	initcall_entry_t *fn;
1420  
1421  	trace_initcall_level("early");
1422  	for (fn = __initcall_start; fn < __initcall0_start; fn++)
1423  		do_one_initcall(initcall_from_entry(fn));
1424  }
1425  
1426  static int run_init_process(const char *init_filename)
1427  {
1428  	const char *const *p;
1429  
1430  	argv_init[0] = init_filename;
1431  	pr_info("Run %s as init process\n", init_filename);
1432  	pr_debug("  with arguments:\n");
1433  	for (p = argv_init; *p; p++)
1434  		pr_debug("    %s\n", *p);
1435  	pr_debug("  with environment:\n");
1436  	for (p = envp_init; *p; p++)
1437  		pr_debug("    %s\n", *p);
1438  	return kernel_execve(init_filename, argv_init, envp_init);
1439  }
1440  
1441  static int try_to_run_init_process(const char *init_filename)
1442  {
1443  	int ret;
1444  
1445  	ret = run_init_process(init_filename);
1446  
1447  	if (ret && ret != -ENOENT) {
1448  		pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",
1449  		       init_filename, ret);
1450  	}
1451  
1452  	return ret;
1453  }
1454  
1455  static noinline void __init kernel_init_freeable(void);
1456  
1457  #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
1458  bool rodata_enabled __ro_after_init = true;
1459  
1460  #ifndef arch_parse_debug_rodata
1461  static inline bool arch_parse_debug_rodata(char *str) { return false; }
1462  #endif
1463  
1464  static int __init set_debug_rodata(char *str)
1465  {
1466  	if (arch_parse_debug_rodata(str))
1467  		return 0;
1468  
1469  	if (str && !strcmp(str, "on"))
1470  		rodata_enabled = true;
1471  	else if (str && !strcmp(str, "off"))
1472  		rodata_enabled = false;
1473  	else
1474  		pr_warn("Invalid option string for rodata: '%s'\n", str);
1475  	return 0;
1476  }
1477  early_param("rodata", set_debug_rodata);
1478  #endif
1479  
1480  #ifdef CONFIG_STRICT_KERNEL_RWX
1481  static void mark_readonly(void)
1482  {
1483  	if (rodata_enabled) {
1484  		/*
1485  		 * load_module() results in W+X mappings, which are cleaned
1486  		 * up with call_rcu().  Let's make sure that queued work is
1487  		 * flushed so that we don't hit false positives looking for
1488  		 * insecure pages which are W+X.
1489  		 */
1490  		rcu_barrier();
1491  		mark_rodata_ro();
1492  		rodata_test();
1493  	} else
1494  		pr_info("Kernel memory protection disabled.\n");
1495  }
1496  #elif defined(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)
1497  static inline void mark_readonly(void)
1498  {
1499  	pr_warn("Kernel memory protection not selected by kernel config.\n");
1500  }
1501  #else
1502  static inline void mark_readonly(void)
1503  {
1504  	pr_warn("This architecture does not have kernel memory protection.\n");
1505  }
1506  #endif
1507  
1508  void __weak free_initmem(void)
1509  {
1510  	free_initmem_default(POISON_FREE_INITMEM);
1511  }
1512  
1513  static int __ref kernel_init(void *unused)
1514  {
1515  	int ret;
1516  
1517  	/*
1518  	 * Wait until kthreadd is all set-up.
1519  	 */
1520  	wait_for_completion(&kthreadd_done);
1521  
1522  	kernel_init_freeable();
1523  	/* need to finish all async __init code before freeing the memory */
1524  	async_synchronize_full();
1525  
1526  	system_state = SYSTEM_FREEING_INITMEM;
1527  	kprobe_free_init_mem();
1528  	ftrace_free_init_mem();
1529  	kgdb_free_init_mem();
1530  	exit_boot_config();
1531  	free_initmem();
1532  	mark_readonly();
1533  
1534  	/*
1535  	 * Kernel mappings are now finalized - update the userspace page-table
1536  	 * to finalize PTI.
1537  	 */
1538  	pti_finalize();
1539  
1540  	system_state = SYSTEM_RUNNING;
1541  	numa_default_policy();
1542  
1543  	rcu_end_inkernel_boot();
1544  
1545  	do_sysctl_args();
1546  
1547  	if (ramdisk_execute_command) {
1548  		ret = run_init_process(ramdisk_execute_command);
1549  		if (!ret)
1550  			return 0;
1551  		pr_err("Failed to execute %s (error %d)\n",
1552  		       ramdisk_execute_command, ret);
1553  	}
1554  
1555  	/*
1556  	 * We try each of these until one succeeds.
1557  	 *
1558  	 * The Bourne shell can be used instead of init if we are
1559  	 * trying to recover a really broken machine.
1560  	 */
1561  	if (execute_command) {
1562  		ret = run_init_process(execute_command);
1563  		if (!ret)
1564  			return 0;
1565  		panic("Requested init %s failed (error %d).",
1566  		      execute_command, ret);
1567  	}
1568  
1569  	if (CONFIG_DEFAULT_INIT[0] != '\0') {
1570  		ret = run_init_process(CONFIG_DEFAULT_INIT);
1571  		if (ret)
1572  			pr_err("Default init %s failed (error %d)\n",
1573  			       CONFIG_DEFAULT_INIT, ret);
1574  		else
1575  			return 0;
1576  	}
1577  
1578  	if (!try_to_run_init_process("/sbin/init") ||
1579  	    !try_to_run_init_process("/etc/init") ||
1580  	    !try_to_run_init_process("/bin/init") ||
1581  	    !try_to_run_init_process("/bin/sh"))
1582  		return 0;
1583  
1584  	panic("No working init found.  Try passing init= option to kernel. "
1585  	      "See Linux Documentation/admin-guide/init.rst for guidance.");
1586  }
1587  
1588  /* Open /dev/console, for stdin/stdout/stderr, this should never fail */
1589  void __init console_on_rootfs(void)
1590  {
1591  	struct file *file = filp_open("/dev/console", O_RDWR, 0);
1592  
1593  	if (IS_ERR(file)) {
1594  		pr_err("Warning: unable to open an initial console.\n");
1595  		return;
1596  	}
1597  	init_dup(file);
1598  	init_dup(file);
1599  	init_dup(file);
1600  	fput(file);
1601  }
1602  
1603  static noinline void __init kernel_init_freeable(void)
1604  {
1605  	/* Now the scheduler is fully set up and can do blocking allocations */
1606  	gfp_allowed_mask = __GFP_BITS_MASK;
1607  
1608  	/*
1609  	 * init can allocate pages on any node
1610  	 */
1611  	set_mems_allowed(node_states[N_MEMORY]);
1612  
1613  	cad_pid = get_pid(task_pid(current));
1614  
1615  	smp_prepare_cpus(setup_max_cpus);
1616  
1617  	workqueue_init();
1618  
1619  	init_mm_internals();
1620  
1621  	rcu_init_tasks_generic();
1622  	do_pre_smp_initcalls();
1623  	lockup_detector_init();
1624  
1625  	smp_init();
1626  	sched_init_smp();
1627  
1628  	padata_init();
1629  	page_alloc_init_late();
1630  	/* Initialize page ext after all struct pages are initialized. */
1631  	if (!early_page_ext_enabled())
1632  		page_ext_init();
1633  
1634  	do_basic_setup();
1635  
1636  	kunit_run_all_tests();
1637  
1638  	wait_for_initramfs();
1639  	console_on_rootfs();
1640  
1641  	/*
1642  	 * check if there is an early userspace init.  If yes, let it do all
1643  	 * the work
1644  	 */
1645  	if (init_eaccess(ramdisk_execute_command) != 0) {
1646  		ramdisk_execute_command = NULL;
1647  		prepare_namespace();
1648  	}
1649  
1650  	/*
1651  	 * Ok, we have completed the initial bootup, and
1652  	 * we're essentially up and running. Get rid of the
1653  	 * initmem segments and start the user-mode stuff..
1654  	 *
1655  	 * rootfs is available now, try loading the public keys
1656  	 * and default modules
1657  	 */
1658  
1659  	integrity_load_keys();
1660  }
1661