xref: /openbmc/linux/kernel/module/main.c (revision 8a0a6c9af053fb93c0edf4581518c77fd131803e)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2002 Richard Henderson
4  * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5  * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6  */
7 
8 #define INCLUDE_VERMAGIC
9 
10 #include <linux/export.h>
11 #include <linux/extable.h>
12 #include <linux/moduleloader.h>
13 #include <linux/module_signature.h>
14 #include <linux/trace_events.h>
15 #include <linux/init.h>
16 #include <linux/kallsyms.h>
17 #include <linux/buildid.h>
18 #include <linux/fs.h>
19 #include <linux/kernel.h>
20 #include <linux/kernel_read_file.h>
21 #include <linux/kstrtox.h>
22 #include <linux/slab.h>
23 #include <linux/vmalloc.h>
24 #include <linux/elf.h>
25 #include <linux/seq_file.h>
26 #include <linux/syscalls.h>
27 #include <linux/fcntl.h>
28 #include <linux/rcupdate.h>
29 #include <linux/capability.h>
30 #include <linux/cpu.h>
31 #include <linux/moduleparam.h>
32 #include <linux/errno.h>
33 #include <linux/err.h>
34 #include <linux/vermagic.h>
35 #include <linux/notifier.h>
36 #include <linux/sched.h>
37 #include <linux/device.h>
38 #include <linux/string.h>
39 #include <linux/mutex.h>
40 #include <linux/rculist.h>
41 #include <linux/uaccess.h>
42 #include <asm/cacheflush.h>
43 #include <linux/set_memory.h>
44 #include <asm/mmu_context.h>
45 #include <linux/license.h>
46 #include <asm/sections.h>
47 #include <linux/tracepoint.h>
48 #include <linux/ftrace.h>
49 #include <linux/livepatch.h>
50 #include <linux/async.h>
51 #include <linux/percpu.h>
52 #include <linux/kmemleak.h>
53 #include <linux/jump_label.h>
54 #include <linux/pfn.h>
55 #include <linux/bsearch.h>
56 #include <linux/dynamic_debug.h>
57 #include <linux/audit.h>
58 #include <linux/cfi.h>
59 #include <uapi/linux/module.h>
60 #include "internal.h"
61 
62 #define CREATE_TRACE_POINTS
63 #include <trace/events/module.h>
64 
65 /*
66  * Mutex protects:
67  * 1) List of modules (also safely readable with preempt_disable),
68  * 2) module_use links,
69  * 3) mod_tree.addr_min/mod_tree.addr_max.
70  * (delete and add uses RCU list operations).
71  */
72 DEFINE_MUTEX(module_mutex);
73 LIST_HEAD(modules);
74 
75 /* Work queue for freeing init sections in success case */
76 static void do_free_init(struct work_struct *w);
77 static DECLARE_WORK(init_free_wq, do_free_init);
78 static LLIST_HEAD(init_free_list);
79 
80 struct mod_tree_root mod_tree __cacheline_aligned = {
81 	.addr_min = -1UL,
82 };
83 
84 struct symsearch {
85 	const struct kernel_symbol *start, *stop;
86 	const s32 *crcs;
87 	enum mod_license license;
88 };
89 
90 /*
91  * Bounds of module memory, for speeding up __module_address.
92  * Protected by module_mutex.
93  */
94 static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,
95 				unsigned int size, struct mod_tree_root *tree)
96 {
97 	unsigned long min = (unsigned long)base;
98 	unsigned long max = min + size;
99 
100 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
101 	if (mod_mem_type_is_core_data(type)) {
102 		if (min < tree->data_addr_min)
103 			tree->data_addr_min = min;
104 		if (max > tree->data_addr_max)
105 			tree->data_addr_max = max;
106 		return;
107 	}
108 #endif
109 	if (min < tree->addr_min)
110 		tree->addr_min = min;
111 	if (max > tree->addr_max)
112 		tree->addr_max = max;
113 }
114 
115 static void mod_update_bounds(struct module *mod)
116 {
117 	for_each_mod_mem_type(type) {
118 		struct module_memory *mod_mem = &mod->mem[type];
119 
120 		if (mod_mem->size)
121 			__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);
122 	}
123 }
124 
125 /* Block module loading/unloading? */
126 int modules_disabled;
127 core_param(nomodule, modules_disabled, bint, 0);
128 
129 /* Waiting for a module to finish initializing? */
130 static DECLARE_WAIT_QUEUE_HEAD(module_wq);
131 
132 static BLOCKING_NOTIFIER_HEAD(module_notify_list);
133 
134 int register_module_notifier(struct notifier_block *nb)
135 {
136 	return blocking_notifier_chain_register(&module_notify_list, nb);
137 }
138 EXPORT_SYMBOL(register_module_notifier);
139 
140 int unregister_module_notifier(struct notifier_block *nb)
141 {
142 	return blocking_notifier_chain_unregister(&module_notify_list, nb);
143 }
144 EXPORT_SYMBOL(unregister_module_notifier);
145 
146 /*
147  * We require a truly strong try_module_get(): 0 means success.
148  * Otherwise an error is returned due to ongoing or failed
149  * initialization etc.
150  */
151 static inline int strong_try_module_get(struct module *mod)
152 {
153 	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
154 	if (mod && mod->state == MODULE_STATE_COMING)
155 		return -EBUSY;
156 	if (try_module_get(mod))
157 		return 0;
158 	else
159 		return -ENOENT;
160 }
161 
162 static inline void add_taint_module(struct module *mod, unsigned flag,
163 				    enum lockdep_ok lockdep_ok)
164 {
165 	add_taint(flag, lockdep_ok);
166 	set_bit(flag, &mod->taints);
167 }
168 
169 /*
170  * A thread that wants to hold a reference to a module only while it
171  * is running can call this to safely exit.
172  */
173 void __noreturn __module_put_and_kthread_exit(struct module *mod, long code)
174 {
175 	module_put(mod);
176 	kthread_exit(code);
177 }
178 EXPORT_SYMBOL(__module_put_and_kthread_exit);
179 
180 /* Find a module section: 0 means not found. */
181 static unsigned int find_sec(const struct load_info *info, const char *name)
182 {
183 	unsigned int i;
184 
185 	for (i = 1; i < info->hdr->e_shnum; i++) {
186 		Elf_Shdr *shdr = &info->sechdrs[i];
187 		/* Alloc bit cleared means "ignore it." */
188 		if ((shdr->sh_flags & SHF_ALLOC)
189 		    && strcmp(info->secstrings + shdr->sh_name, name) == 0)
190 			return i;
191 	}
192 	return 0;
193 }
194 
195 /* Find a module section, or NULL. */
196 static void *section_addr(const struct load_info *info, const char *name)
197 {
198 	/* Section 0 has sh_addr 0. */
199 	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
200 }
201 
202 /* Find a module section, or NULL.  Fill in number of "objects" in section. */
203 static void *section_objs(const struct load_info *info,
204 			  const char *name,
205 			  size_t object_size,
206 			  unsigned int *num)
207 {
208 	unsigned int sec = find_sec(info, name);
209 
210 	/* Section 0 has sh_addr 0 and sh_size 0. */
211 	*num = info->sechdrs[sec].sh_size / object_size;
212 	return (void *)info->sechdrs[sec].sh_addr;
213 }
214 
215 /* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */
216 static unsigned int find_any_sec(const struct load_info *info, const char *name)
217 {
218 	unsigned int i;
219 
220 	for (i = 1; i < info->hdr->e_shnum; i++) {
221 		Elf_Shdr *shdr = &info->sechdrs[i];
222 		if (strcmp(info->secstrings + shdr->sh_name, name) == 0)
223 			return i;
224 	}
225 	return 0;
226 }
227 
228 /*
229  * Find a module section, or NULL. Fill in number of "objects" in section.
230  * Ignores SHF_ALLOC flag.
231  */
232 static __maybe_unused void *any_section_objs(const struct load_info *info,
233 					     const char *name,
234 					     size_t object_size,
235 					     unsigned int *num)
236 {
237 	unsigned int sec = find_any_sec(info, name);
238 
239 	/* Section 0 has sh_addr 0 and sh_size 0. */
240 	*num = info->sechdrs[sec].sh_size / object_size;
241 	return (void *)info->sechdrs[sec].sh_addr;
242 }
243 
244 #ifndef CONFIG_MODVERSIONS
245 #define symversion(base, idx) NULL
246 #else
247 #define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
248 #endif
249 
250 static const char *kernel_symbol_name(const struct kernel_symbol *sym)
251 {
252 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
253 	return offset_to_ptr(&sym->name_offset);
254 #else
255 	return sym->name;
256 #endif
257 }
258 
259 static const char *kernel_symbol_namespace(const struct kernel_symbol *sym)
260 {
261 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
262 	if (!sym->namespace_offset)
263 		return NULL;
264 	return offset_to_ptr(&sym->namespace_offset);
265 #else
266 	return sym->namespace;
267 #endif
268 }
269 
270 int cmp_name(const void *name, const void *sym)
271 {
272 	return strcmp(name, kernel_symbol_name(sym));
273 }
274 
275 static bool find_exported_symbol_in_section(const struct symsearch *syms,
276 					    struct module *owner,
277 					    struct find_symbol_arg *fsa)
278 {
279 	struct kernel_symbol *sym;
280 
281 	if (!fsa->gplok && syms->license == GPL_ONLY)
282 		return false;
283 
284 	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
285 			sizeof(struct kernel_symbol), cmp_name);
286 	if (!sym)
287 		return false;
288 
289 	fsa->owner = owner;
290 	fsa->crc = symversion(syms->crcs, sym - syms->start);
291 	fsa->sym = sym;
292 	fsa->license = syms->license;
293 
294 	return true;
295 }
296 
297 /*
298  * Find an exported symbol and return it, along with, (optional) crc and
299  * (optional) module which owns it.  Needs preempt disabled or module_mutex.
300  */
301 bool find_symbol(struct find_symbol_arg *fsa)
302 {
303 	static const struct symsearch arr[] = {
304 		{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
305 		  NOT_GPL_ONLY },
306 		{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
307 		  __start___kcrctab_gpl,
308 		  GPL_ONLY },
309 	};
310 	struct module *mod;
311 	unsigned int i;
312 
313 	module_assert_mutex_or_preempt();
314 
315 	for (i = 0; i < ARRAY_SIZE(arr); i++)
316 		if (find_exported_symbol_in_section(&arr[i], NULL, fsa))
317 			return true;
318 
319 	list_for_each_entry_rcu(mod, &modules, list,
320 				lockdep_is_held(&module_mutex)) {
321 		struct symsearch arr[] = {
322 			{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
323 			  NOT_GPL_ONLY },
324 			{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
325 			  mod->gpl_crcs,
326 			  GPL_ONLY },
327 		};
328 
329 		if (mod->state == MODULE_STATE_UNFORMED)
330 			continue;
331 
332 		for (i = 0; i < ARRAY_SIZE(arr); i++)
333 			if (find_exported_symbol_in_section(&arr[i], mod, fsa))
334 				return true;
335 	}
336 
337 	pr_debug("Failed to find symbol %s\n", fsa->name);
338 	return false;
339 }
340 
341 /*
342  * Search for module by name: must hold module_mutex (or preempt disabled
343  * for read-only access).
344  */
345 struct module *find_module_all(const char *name, size_t len,
346 			       bool even_unformed)
347 {
348 	struct module *mod;
349 
350 	module_assert_mutex_or_preempt();
351 
352 	list_for_each_entry_rcu(mod, &modules, list,
353 				lockdep_is_held(&module_mutex)) {
354 		if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
355 			continue;
356 		if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
357 			return mod;
358 	}
359 	return NULL;
360 }
361 
362 struct module *find_module(const char *name)
363 {
364 	return find_module_all(name, strlen(name), false);
365 }
366 
367 #ifdef CONFIG_SMP
368 
369 static inline void __percpu *mod_percpu(struct module *mod)
370 {
371 	return mod->percpu;
372 }
373 
374 static int percpu_modalloc(struct module *mod, struct load_info *info)
375 {
376 	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
377 	unsigned long align = pcpusec->sh_addralign;
378 
379 	if (!pcpusec->sh_size)
380 		return 0;
381 
382 	if (align > PAGE_SIZE) {
383 		pr_warn("%s: per-cpu alignment %li > %li\n",
384 			mod->name, align, PAGE_SIZE);
385 		align = PAGE_SIZE;
386 	}
387 
388 	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
389 	if (!mod->percpu) {
390 		pr_warn("%s: Could not allocate %lu bytes percpu data\n",
391 			mod->name, (unsigned long)pcpusec->sh_size);
392 		return -ENOMEM;
393 	}
394 	mod->percpu_size = pcpusec->sh_size;
395 	return 0;
396 }
397 
398 static void percpu_modfree(struct module *mod)
399 {
400 	free_percpu(mod->percpu);
401 }
402 
403 static unsigned int find_pcpusec(struct load_info *info)
404 {
405 	return find_sec(info, ".data..percpu");
406 }
407 
408 static void percpu_modcopy(struct module *mod,
409 			   const void *from, unsigned long size)
410 {
411 	int cpu;
412 
413 	for_each_possible_cpu(cpu)
414 		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
415 }
416 
417 bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
418 {
419 	struct module *mod;
420 	unsigned int cpu;
421 
422 	preempt_disable();
423 
424 	list_for_each_entry_rcu(mod, &modules, list) {
425 		if (mod->state == MODULE_STATE_UNFORMED)
426 			continue;
427 		if (!mod->percpu_size)
428 			continue;
429 		for_each_possible_cpu(cpu) {
430 			void *start = per_cpu_ptr(mod->percpu, cpu);
431 			void *va = (void *)addr;
432 
433 			if (va >= start && va < start + mod->percpu_size) {
434 				if (can_addr) {
435 					*can_addr = (unsigned long) (va - start);
436 					*can_addr += (unsigned long)
437 						per_cpu_ptr(mod->percpu,
438 							    get_boot_cpu_id());
439 				}
440 				preempt_enable();
441 				return true;
442 			}
443 		}
444 	}
445 
446 	preempt_enable();
447 	return false;
448 }
449 
450 /**
451  * is_module_percpu_address() - test whether address is from module static percpu
452  * @addr: address to test
453  *
454  * Test whether @addr belongs to module static percpu area.
455  *
456  * Return: %true if @addr is from module static percpu area
457  */
458 bool is_module_percpu_address(unsigned long addr)
459 {
460 	return __is_module_percpu_address(addr, NULL);
461 }
462 
463 #else /* ... !CONFIG_SMP */
464 
465 static inline void __percpu *mod_percpu(struct module *mod)
466 {
467 	return NULL;
468 }
469 static int percpu_modalloc(struct module *mod, struct load_info *info)
470 {
471 	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
472 	if (info->sechdrs[info->index.pcpu].sh_size != 0)
473 		return -ENOMEM;
474 	return 0;
475 }
476 static inline void percpu_modfree(struct module *mod)
477 {
478 }
479 static unsigned int find_pcpusec(struct load_info *info)
480 {
481 	return 0;
482 }
483 static inline void percpu_modcopy(struct module *mod,
484 				  const void *from, unsigned long size)
485 {
486 	/* pcpusec should be 0, and size of that section should be 0. */
487 	BUG_ON(size != 0);
488 }
489 bool is_module_percpu_address(unsigned long addr)
490 {
491 	return false;
492 }
493 
494 bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
495 {
496 	return false;
497 }
498 
499 #endif /* CONFIG_SMP */
500 
501 #define MODINFO_ATTR(field)	\
502 static void setup_modinfo_##field(struct module *mod, const char *s)  \
503 {                                                                     \
504 	mod->field = kstrdup(s, GFP_KERNEL);                          \
505 }                                                                     \
506 static ssize_t show_modinfo_##field(struct module_attribute *mattr,   \
507 			struct module_kobject *mk, char *buffer)      \
508 {                                                                     \
509 	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field);  \
510 }                                                                     \
511 static int modinfo_##field##_exists(struct module *mod)               \
512 {                                                                     \
513 	return mod->field != NULL;                                    \
514 }                                                                     \
515 static void free_modinfo_##field(struct module *mod)                  \
516 {                                                                     \
517 	kfree(mod->field);                                            \
518 	mod->field = NULL;                                            \
519 }                                                                     \
520 static struct module_attribute modinfo_##field = {                    \
521 	.attr = { .name = __stringify(field), .mode = 0444 },         \
522 	.show = show_modinfo_##field,                                 \
523 	.setup = setup_modinfo_##field,                               \
524 	.test = modinfo_##field##_exists,                             \
525 	.free = free_modinfo_##field,                                 \
526 };
527 
528 MODINFO_ATTR(version);
529 MODINFO_ATTR(srcversion);
530 
531 static struct {
532 	char name[MODULE_NAME_LEN + 1];
533 	char taints[MODULE_FLAGS_BUF_SIZE];
534 } last_unloaded_module;
535 
536 #ifdef CONFIG_MODULE_UNLOAD
537 
538 EXPORT_TRACEPOINT_SYMBOL(module_get);
539 
540 /* MODULE_REF_BASE is the base reference count by kmodule loader. */
541 #define MODULE_REF_BASE	1
542 
543 /* Init the unload section of the module. */
544 static int module_unload_init(struct module *mod)
545 {
546 	/*
547 	 * Initialize reference counter to MODULE_REF_BASE.
548 	 * refcnt == 0 means module is going.
549 	 */
550 	atomic_set(&mod->refcnt, MODULE_REF_BASE);
551 
552 	INIT_LIST_HEAD(&mod->source_list);
553 	INIT_LIST_HEAD(&mod->target_list);
554 
555 	/* Hold reference count during initialization. */
556 	atomic_inc(&mod->refcnt);
557 
558 	return 0;
559 }
560 
561 /* Does a already use b? */
562 static int already_uses(struct module *a, struct module *b)
563 {
564 	struct module_use *use;
565 
566 	list_for_each_entry(use, &b->source_list, source_list) {
567 		if (use->source == a)
568 			return 1;
569 	}
570 	pr_debug("%s does not use %s!\n", a->name, b->name);
571 	return 0;
572 }
573 
574 /*
575  * Module a uses b
576  *  - we add 'a' as a "source", 'b' as a "target" of module use
577  *  - the module_use is added to the list of 'b' sources (so
578  *    'b' can walk the list to see who sourced them), and of 'a'
579  *    targets (so 'a' can see what modules it targets).
580  */
581 static int add_module_usage(struct module *a, struct module *b)
582 {
583 	struct module_use *use;
584 
585 	pr_debug("Allocating new usage for %s.\n", a->name);
586 	use = kmalloc(sizeof(*use), GFP_ATOMIC);
587 	if (!use)
588 		return -ENOMEM;
589 
590 	use->source = a;
591 	use->target = b;
592 	list_add(&use->source_list, &b->source_list);
593 	list_add(&use->target_list, &a->target_list);
594 	return 0;
595 }
596 
597 /* Module a uses b: caller needs module_mutex() */
598 static int ref_module(struct module *a, struct module *b)
599 {
600 	int err;
601 
602 	if (b == NULL || already_uses(a, b))
603 		return 0;
604 
605 	/* If module isn't available, we fail. */
606 	err = strong_try_module_get(b);
607 	if (err)
608 		return err;
609 
610 	err = add_module_usage(a, b);
611 	if (err) {
612 		module_put(b);
613 		return err;
614 	}
615 	return 0;
616 }
617 
618 /* Clear the unload stuff of the module. */
619 static void module_unload_free(struct module *mod)
620 {
621 	struct module_use *use, *tmp;
622 
623 	mutex_lock(&module_mutex);
624 	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
625 		struct module *i = use->target;
626 		pr_debug("%s unusing %s\n", mod->name, i->name);
627 		module_put(i);
628 		list_del(&use->source_list);
629 		list_del(&use->target_list);
630 		kfree(use);
631 	}
632 	mutex_unlock(&module_mutex);
633 }
634 
635 #ifdef CONFIG_MODULE_FORCE_UNLOAD
636 static inline int try_force_unload(unsigned int flags)
637 {
638 	int ret = (flags & O_TRUNC);
639 	if (ret)
640 		add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
641 	return ret;
642 }
643 #else
644 static inline int try_force_unload(unsigned int flags)
645 {
646 	return 0;
647 }
648 #endif /* CONFIG_MODULE_FORCE_UNLOAD */
649 
650 /* Try to release refcount of module, 0 means success. */
651 static int try_release_module_ref(struct module *mod)
652 {
653 	int ret;
654 
655 	/* Try to decrement refcnt which we set at loading */
656 	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
657 	BUG_ON(ret < 0);
658 	if (ret)
659 		/* Someone can put this right now, recover with checking */
660 		ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
661 
662 	return ret;
663 }
664 
665 static int try_stop_module(struct module *mod, int flags, int *forced)
666 {
667 	/* If it's not unused, quit unless we're forcing. */
668 	if (try_release_module_ref(mod) != 0) {
669 		*forced = try_force_unload(flags);
670 		if (!(*forced))
671 			return -EWOULDBLOCK;
672 	}
673 
674 	/* Mark it as dying. */
675 	mod->state = MODULE_STATE_GOING;
676 
677 	return 0;
678 }
679 
680 /**
681  * module_refcount() - return the refcount or -1 if unloading
682  * @mod:	the module we're checking
683  *
684  * Return:
685  *	-1 if the module is in the process of unloading
686  *	otherwise the number of references in the kernel to the module
687  */
688 int module_refcount(struct module *mod)
689 {
690 	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
691 }
692 EXPORT_SYMBOL(module_refcount);
693 
694 /* This exists whether we can unload or not */
695 static void free_module(struct module *mod);
696 
697 SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
698 		unsigned int, flags)
699 {
700 	struct module *mod;
701 	char name[MODULE_NAME_LEN];
702 	char buf[MODULE_FLAGS_BUF_SIZE];
703 	int ret, forced = 0;
704 
705 	if (!capable(CAP_SYS_MODULE) || modules_disabled)
706 		return -EPERM;
707 
708 	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
709 		return -EFAULT;
710 	name[MODULE_NAME_LEN-1] = '\0';
711 
712 	audit_log_kern_module(name);
713 
714 	if (mutex_lock_interruptible(&module_mutex) != 0)
715 		return -EINTR;
716 
717 	mod = find_module(name);
718 	if (!mod) {
719 		ret = -ENOENT;
720 		goto out;
721 	}
722 
723 	if (!list_empty(&mod->source_list)) {
724 		/* Other modules depend on us: get rid of them first. */
725 		ret = -EWOULDBLOCK;
726 		goto out;
727 	}
728 
729 	/* Doing init or already dying? */
730 	if (mod->state != MODULE_STATE_LIVE) {
731 		/* FIXME: if (force), slam module count damn the torpedoes */
732 		pr_debug("%s already dying\n", mod->name);
733 		ret = -EBUSY;
734 		goto out;
735 	}
736 
737 	/* If it has an init func, it must have an exit func to unload */
738 	if (mod->init && !mod->exit) {
739 		forced = try_force_unload(flags);
740 		if (!forced) {
741 			/* This module can't be removed */
742 			ret = -EBUSY;
743 			goto out;
744 		}
745 	}
746 
747 	ret = try_stop_module(mod, flags, &forced);
748 	if (ret != 0)
749 		goto out;
750 
751 	mutex_unlock(&module_mutex);
752 	/* Final destruction now no one is using it. */
753 	if (mod->exit != NULL)
754 		mod->exit();
755 	blocking_notifier_call_chain(&module_notify_list,
756 				     MODULE_STATE_GOING, mod);
757 	klp_module_going(mod);
758 	ftrace_release_mod(mod);
759 
760 	async_synchronize_full();
761 
762 	/* Store the name and taints of the last unloaded module for diagnostic purposes */
763 	strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name));
764 	strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints));
765 
766 	free_module(mod);
767 	/* someone could wait for the module in add_unformed_module() */
768 	wake_up_all(&module_wq);
769 	return 0;
770 out:
771 	mutex_unlock(&module_mutex);
772 	return ret;
773 }
774 
775 void __symbol_put(const char *symbol)
776 {
777 	struct find_symbol_arg fsa = {
778 		.name	= symbol,
779 		.gplok	= true,
780 	};
781 
782 	preempt_disable();
783 	BUG_ON(!find_symbol(&fsa));
784 	module_put(fsa.owner);
785 	preempt_enable();
786 }
787 EXPORT_SYMBOL(__symbol_put);
788 
789 /* Note this assumes addr is a function, which it currently always is. */
790 void symbol_put_addr(void *addr)
791 {
792 	struct module *modaddr;
793 	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
794 
795 	if (core_kernel_text(a))
796 		return;
797 
798 	/*
799 	 * Even though we hold a reference on the module; we still need to
800 	 * disable preemption in order to safely traverse the data structure.
801 	 */
802 	preempt_disable();
803 	modaddr = __module_text_address(a);
804 	BUG_ON(!modaddr);
805 	module_put(modaddr);
806 	preempt_enable();
807 }
808 EXPORT_SYMBOL_GPL(symbol_put_addr);
809 
810 static ssize_t show_refcnt(struct module_attribute *mattr,
811 			   struct module_kobject *mk, char *buffer)
812 {
813 	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
814 }
815 
816 static struct module_attribute modinfo_refcnt =
817 	__ATTR(refcnt, 0444, show_refcnt, NULL);
818 
819 void __module_get(struct module *module)
820 {
821 	if (module) {
822 		preempt_disable();
823 		atomic_inc(&module->refcnt);
824 		trace_module_get(module, _RET_IP_);
825 		preempt_enable();
826 	}
827 }
828 EXPORT_SYMBOL(__module_get);
829 
830 bool try_module_get(struct module *module)
831 {
832 	bool ret = true;
833 
834 	if (module) {
835 		preempt_disable();
836 		/* Note: here, we can fail to get a reference */
837 		if (likely(module_is_live(module) &&
838 			   atomic_inc_not_zero(&module->refcnt) != 0))
839 			trace_module_get(module, _RET_IP_);
840 		else
841 			ret = false;
842 
843 		preempt_enable();
844 	}
845 	return ret;
846 }
847 EXPORT_SYMBOL(try_module_get);
848 
849 void module_put(struct module *module)
850 {
851 	int ret;
852 
853 	if (module) {
854 		preempt_disable();
855 		ret = atomic_dec_if_positive(&module->refcnt);
856 		WARN_ON(ret < 0);	/* Failed to put refcount */
857 		trace_module_put(module, _RET_IP_);
858 		preempt_enable();
859 	}
860 }
861 EXPORT_SYMBOL(module_put);
862 
863 #else /* !CONFIG_MODULE_UNLOAD */
864 static inline void module_unload_free(struct module *mod)
865 {
866 }
867 
868 static int ref_module(struct module *a, struct module *b)
869 {
870 	return strong_try_module_get(b);
871 }
872 
873 static inline int module_unload_init(struct module *mod)
874 {
875 	return 0;
876 }
877 #endif /* CONFIG_MODULE_UNLOAD */
878 
879 size_t module_flags_taint(unsigned long taints, char *buf)
880 {
881 	size_t l = 0;
882 	int i;
883 
884 	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
885 		if (taint_flags[i].module && test_bit(i, &taints))
886 			buf[l++] = taint_flags[i].c_true;
887 	}
888 
889 	return l;
890 }
891 
892 static ssize_t show_initstate(struct module_attribute *mattr,
893 			      struct module_kobject *mk, char *buffer)
894 {
895 	const char *state = "unknown";
896 
897 	switch (mk->mod->state) {
898 	case MODULE_STATE_LIVE:
899 		state = "live";
900 		break;
901 	case MODULE_STATE_COMING:
902 		state = "coming";
903 		break;
904 	case MODULE_STATE_GOING:
905 		state = "going";
906 		break;
907 	default:
908 		BUG();
909 	}
910 	return sprintf(buffer, "%s\n", state);
911 }
912 
913 static struct module_attribute modinfo_initstate =
914 	__ATTR(initstate, 0444, show_initstate, NULL);
915 
916 static ssize_t store_uevent(struct module_attribute *mattr,
917 			    struct module_kobject *mk,
918 			    const char *buffer, size_t count)
919 {
920 	int rc;
921 
922 	rc = kobject_synth_uevent(&mk->kobj, buffer, count);
923 	return rc ? rc : count;
924 }
925 
926 struct module_attribute module_uevent =
927 	__ATTR(uevent, 0200, NULL, store_uevent);
928 
929 static ssize_t show_coresize(struct module_attribute *mattr,
930 			     struct module_kobject *mk, char *buffer)
931 {
932 	unsigned int size = mk->mod->mem[MOD_TEXT].size;
933 
934 	if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {
935 		for_class_mod_mem_type(type, core_data)
936 			size += mk->mod->mem[type].size;
937 	}
938 	return sprintf(buffer, "%u\n", size);
939 }
940 
941 static struct module_attribute modinfo_coresize =
942 	__ATTR(coresize, 0444, show_coresize, NULL);
943 
944 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
945 static ssize_t show_datasize(struct module_attribute *mattr,
946 			     struct module_kobject *mk, char *buffer)
947 {
948 	unsigned int size = 0;
949 
950 	for_class_mod_mem_type(type, core_data)
951 		size += mk->mod->mem[type].size;
952 	return sprintf(buffer, "%u\n", size);
953 }
954 
955 static struct module_attribute modinfo_datasize =
956 	__ATTR(datasize, 0444, show_datasize, NULL);
957 #endif
958 
959 static ssize_t show_initsize(struct module_attribute *mattr,
960 			     struct module_kobject *mk, char *buffer)
961 {
962 	unsigned int size = 0;
963 
964 	for_class_mod_mem_type(type, init)
965 		size += mk->mod->mem[type].size;
966 	return sprintf(buffer, "%u\n", size);
967 }
968 
969 static struct module_attribute modinfo_initsize =
970 	__ATTR(initsize, 0444, show_initsize, NULL);
971 
972 static ssize_t show_taint(struct module_attribute *mattr,
973 			  struct module_kobject *mk, char *buffer)
974 {
975 	size_t l;
976 
977 	l = module_flags_taint(mk->mod->taints, buffer);
978 	buffer[l++] = '\n';
979 	return l;
980 }
981 
982 static struct module_attribute modinfo_taint =
983 	__ATTR(taint, 0444, show_taint, NULL);
984 
985 struct module_attribute *modinfo_attrs[] = {
986 	&module_uevent,
987 	&modinfo_version,
988 	&modinfo_srcversion,
989 	&modinfo_initstate,
990 	&modinfo_coresize,
991 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
992 	&modinfo_datasize,
993 #endif
994 	&modinfo_initsize,
995 	&modinfo_taint,
996 #ifdef CONFIG_MODULE_UNLOAD
997 	&modinfo_refcnt,
998 #endif
999 	NULL,
1000 };
1001 
1002 size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs);
1003 
1004 static const char vermagic[] = VERMAGIC_STRING;
1005 
1006 int try_to_force_load(struct module *mod, const char *reason)
1007 {
1008 #ifdef CONFIG_MODULE_FORCE_LOAD
1009 	if (!test_taint(TAINT_FORCED_MODULE))
1010 		pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1011 	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1012 	return 0;
1013 #else
1014 	return -ENOEXEC;
1015 #endif
1016 }
1017 
1018 /* Parse tag=value strings from .modinfo section */
1019 char *module_next_tag_pair(char *string, unsigned long *secsize)
1020 {
1021 	/* Skip non-zero chars */
1022 	while (string[0]) {
1023 		string++;
1024 		if ((*secsize)-- <= 1)
1025 			return NULL;
1026 	}
1027 
1028 	/* Skip any zero padding. */
1029 	while (!string[0]) {
1030 		string++;
1031 		if ((*secsize)-- <= 1)
1032 			return NULL;
1033 	}
1034 	return string;
1035 }
1036 
1037 static char *get_next_modinfo(const struct load_info *info, const char *tag,
1038 			      char *prev)
1039 {
1040 	char *p;
1041 	unsigned int taglen = strlen(tag);
1042 	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
1043 	unsigned long size = infosec->sh_size;
1044 
1045 	/*
1046 	 * get_modinfo() calls made before rewrite_section_headers()
1047 	 * must use sh_offset, as sh_addr isn't set!
1048 	 */
1049 	char *modinfo = (char *)info->hdr + infosec->sh_offset;
1050 
1051 	if (prev) {
1052 		size -= prev - modinfo;
1053 		modinfo = module_next_tag_pair(prev, &size);
1054 	}
1055 
1056 	for (p = modinfo; p; p = module_next_tag_pair(p, &size)) {
1057 		if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
1058 			return p + taglen + 1;
1059 	}
1060 	return NULL;
1061 }
1062 
1063 static char *get_modinfo(const struct load_info *info, const char *tag)
1064 {
1065 	return get_next_modinfo(info, tag, NULL);
1066 }
1067 
1068 static int verify_namespace_is_imported(const struct load_info *info,
1069 					const struct kernel_symbol *sym,
1070 					struct module *mod)
1071 {
1072 	const char *namespace;
1073 	char *imported_namespace;
1074 
1075 	namespace = kernel_symbol_namespace(sym);
1076 	if (namespace && namespace[0]) {
1077 		for_each_modinfo_entry(imported_namespace, info, "import_ns") {
1078 			if (strcmp(namespace, imported_namespace) == 0)
1079 				return 0;
1080 		}
1081 #ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1082 		pr_warn(
1083 #else
1084 		pr_err(
1085 #endif
1086 			"%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
1087 			mod->name, kernel_symbol_name(sym), namespace);
1088 #ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
1089 		return -EINVAL;
1090 #endif
1091 	}
1092 	return 0;
1093 }
1094 
1095 static bool inherit_taint(struct module *mod, struct module *owner, const char *name)
1096 {
1097 	if (!owner || !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
1098 		return true;
1099 
1100 	if (mod->using_gplonly_symbols) {
1101 		pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n",
1102 			mod->name, name, owner->name);
1103 		return false;
1104 	}
1105 
1106 	if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
1107 		pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n",
1108 			mod->name, name, owner->name);
1109 		set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
1110 	}
1111 	return true;
1112 }
1113 
1114 /* Resolve a symbol for this module.  I.e. if we find one, record usage. */
1115 static const struct kernel_symbol *resolve_symbol(struct module *mod,
1116 						  const struct load_info *info,
1117 						  const char *name,
1118 						  char ownername[])
1119 {
1120 	struct find_symbol_arg fsa = {
1121 		.name	= name,
1122 		.gplok	= !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)),
1123 		.warn	= true,
1124 	};
1125 	int err;
1126 
1127 	/*
1128 	 * The module_mutex should not be a heavily contended lock;
1129 	 * if we get the occasional sleep here, we'll go an extra iteration
1130 	 * in the wait_event_interruptible(), which is harmless.
1131 	 */
1132 	sched_annotate_sleep();
1133 	mutex_lock(&module_mutex);
1134 	if (!find_symbol(&fsa))
1135 		goto unlock;
1136 
1137 	if (fsa.license == GPL_ONLY)
1138 		mod->using_gplonly_symbols = true;
1139 
1140 	if (!inherit_taint(mod, fsa.owner, name)) {
1141 		fsa.sym = NULL;
1142 		goto getname;
1143 	}
1144 
1145 	if (!check_version(info, name, mod, fsa.crc)) {
1146 		fsa.sym = ERR_PTR(-EINVAL);
1147 		goto getname;
1148 	}
1149 
1150 	err = verify_namespace_is_imported(info, fsa.sym, mod);
1151 	if (err) {
1152 		fsa.sym = ERR_PTR(err);
1153 		goto getname;
1154 	}
1155 
1156 	err = ref_module(mod, fsa.owner);
1157 	if (err) {
1158 		fsa.sym = ERR_PTR(err);
1159 		goto getname;
1160 	}
1161 
1162 getname:
1163 	/* We must make copy under the lock if we failed to get ref. */
1164 	strncpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN);
1165 unlock:
1166 	mutex_unlock(&module_mutex);
1167 	return fsa.sym;
1168 }
1169 
1170 static const struct kernel_symbol *
1171 resolve_symbol_wait(struct module *mod,
1172 		    const struct load_info *info,
1173 		    const char *name)
1174 {
1175 	const struct kernel_symbol *ksym;
1176 	char owner[MODULE_NAME_LEN];
1177 
1178 	if (wait_event_interruptible_timeout(module_wq,
1179 			!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1180 			|| PTR_ERR(ksym) != -EBUSY,
1181 					     30 * HZ) <= 0) {
1182 		pr_warn("%s: gave up waiting for init of module %s.\n",
1183 			mod->name, owner);
1184 	}
1185 	return ksym;
1186 }
1187 
1188 void __weak module_memfree(void *module_region)
1189 {
1190 	/*
1191 	 * This memory may be RO, and freeing RO memory in an interrupt is not
1192 	 * supported by vmalloc.
1193 	 */
1194 	WARN_ON(in_interrupt());
1195 	vfree(module_region);
1196 }
1197 
1198 void __weak module_arch_cleanup(struct module *mod)
1199 {
1200 }
1201 
1202 void __weak module_arch_freeing_init(struct module *mod)
1203 {
1204 }
1205 
1206 static bool mod_mem_use_vmalloc(enum mod_mem_type type)
1207 {
1208 	return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) &&
1209 		mod_mem_type_is_core_data(type);
1210 }
1211 
1212 static void *module_memory_alloc(unsigned int size, enum mod_mem_type type)
1213 {
1214 	if (mod_mem_use_vmalloc(type))
1215 		return vzalloc(size);
1216 	return module_alloc(size);
1217 }
1218 
1219 static void module_memory_free(void *ptr, enum mod_mem_type type)
1220 {
1221 	if (mod_mem_use_vmalloc(type))
1222 		vfree(ptr);
1223 	else
1224 		module_memfree(ptr);
1225 }
1226 
1227 static void free_mod_mem(struct module *mod)
1228 {
1229 	for_each_mod_mem_type(type) {
1230 		struct module_memory *mod_mem = &mod->mem[type];
1231 
1232 		if (type == MOD_DATA)
1233 			continue;
1234 
1235 		/* Free lock-classes; relies on the preceding sync_rcu(). */
1236 		lockdep_free_key_range(mod_mem->base, mod_mem->size);
1237 		if (mod_mem->size)
1238 			module_memory_free(mod_mem->base, type);
1239 	}
1240 
1241 	/* MOD_DATA hosts mod, so free it at last */
1242 	lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
1243 	module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA);
1244 }
1245 
1246 /* Free a module, remove from lists, etc. */
1247 static void free_module(struct module *mod)
1248 {
1249 	trace_module_free(mod);
1250 
1251 	mod_sysfs_teardown(mod);
1252 
1253 	/*
1254 	 * We leave it in list to prevent duplicate loads, but make sure
1255 	 * that noone uses it while it's being deconstructed.
1256 	 */
1257 	mutex_lock(&module_mutex);
1258 	mod->state = MODULE_STATE_UNFORMED;
1259 	mutex_unlock(&module_mutex);
1260 
1261 	/* Arch-specific cleanup. */
1262 	module_arch_cleanup(mod);
1263 
1264 	/* Module unload stuff */
1265 	module_unload_free(mod);
1266 
1267 	/* Free any allocated parameters. */
1268 	destroy_params(mod->kp, mod->num_kp);
1269 
1270 	if (is_livepatch_module(mod))
1271 		free_module_elf(mod);
1272 
1273 	/* Now we can delete it from the lists */
1274 	mutex_lock(&module_mutex);
1275 	/* Unlink carefully: kallsyms could be walking list. */
1276 	list_del_rcu(&mod->list);
1277 	mod_tree_remove(mod);
1278 	/* Remove this module from bug list, this uses list_del_rcu */
1279 	module_bug_cleanup(mod);
1280 	/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
1281 	synchronize_rcu();
1282 	if (try_add_tainted_module(mod))
1283 		pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n",
1284 		       mod->name);
1285 	mutex_unlock(&module_mutex);
1286 
1287 	/* This may be empty, but that's OK */
1288 	module_arch_freeing_init(mod);
1289 	kfree(mod->args);
1290 	percpu_modfree(mod);
1291 
1292 	free_mod_mem(mod);
1293 }
1294 
1295 void *__symbol_get(const char *symbol)
1296 {
1297 	struct find_symbol_arg fsa = {
1298 		.name	= symbol,
1299 		.gplok	= true,
1300 		.warn	= true,
1301 	};
1302 
1303 	preempt_disable();
1304 	if (!find_symbol(&fsa) || strong_try_module_get(fsa.owner)) {
1305 		preempt_enable();
1306 		return NULL;
1307 	}
1308 	preempt_enable();
1309 	return (void *)kernel_symbol_value(fsa.sym);
1310 }
1311 EXPORT_SYMBOL_GPL(__symbol_get);
1312 
1313 /*
1314  * Ensure that an exported symbol [global namespace] does not already exist
1315  * in the kernel or in some other module's exported symbol table.
1316  *
1317  * You must hold the module_mutex.
1318  */
1319 static int verify_exported_symbols(struct module *mod)
1320 {
1321 	unsigned int i;
1322 	const struct kernel_symbol *s;
1323 	struct {
1324 		const struct kernel_symbol *sym;
1325 		unsigned int num;
1326 	} arr[] = {
1327 		{ mod->syms, mod->num_syms },
1328 		{ mod->gpl_syms, mod->num_gpl_syms },
1329 	};
1330 
1331 	for (i = 0; i < ARRAY_SIZE(arr); i++) {
1332 		for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
1333 			struct find_symbol_arg fsa = {
1334 				.name	= kernel_symbol_name(s),
1335 				.gplok	= true,
1336 			};
1337 			if (find_symbol(&fsa)) {
1338 				pr_err("%s: exports duplicate symbol %s"
1339 				       " (owned by %s)\n",
1340 				       mod->name, kernel_symbol_name(s),
1341 				       module_name(fsa.owner));
1342 				return -ENOEXEC;
1343 			}
1344 		}
1345 	}
1346 	return 0;
1347 }
1348 
1349 static bool ignore_undef_symbol(Elf_Half emachine, const char *name)
1350 {
1351 	/*
1352 	 * On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as
1353 	 * before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64.
1354 	 * i386 has a similar problem but may not deserve a fix.
1355 	 *
1356 	 * If we ever have to ignore many symbols, consider refactoring the code to
1357 	 * only warn if referenced by a relocation.
1358 	 */
1359 	if (emachine == EM_386 || emachine == EM_X86_64)
1360 		return !strcmp(name, "_GLOBAL_OFFSET_TABLE_");
1361 	return false;
1362 }
1363 
1364 /* Change all symbols so that st_value encodes the pointer directly. */
1365 static int simplify_symbols(struct module *mod, const struct load_info *info)
1366 {
1367 	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1368 	Elf_Sym *sym = (void *)symsec->sh_addr;
1369 	unsigned long secbase;
1370 	unsigned int i;
1371 	int ret = 0;
1372 	const struct kernel_symbol *ksym;
1373 
1374 	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
1375 		const char *name = info->strtab + sym[i].st_name;
1376 
1377 		switch (sym[i].st_shndx) {
1378 		case SHN_COMMON:
1379 			/* Ignore common symbols */
1380 			if (!strncmp(name, "__gnu_lto", 9))
1381 				break;
1382 
1383 			/*
1384 			 * We compiled with -fno-common.  These are not
1385 			 * supposed to happen.
1386 			 */
1387 			pr_debug("Common symbol: %s\n", name);
1388 			pr_warn("%s: please compile with -fno-common\n",
1389 			       mod->name);
1390 			ret = -ENOEXEC;
1391 			break;
1392 
1393 		case SHN_ABS:
1394 			/* Don't need to do anything */
1395 			pr_debug("Absolute symbol: 0x%08lx %s\n",
1396 				 (long)sym[i].st_value, name);
1397 			break;
1398 
1399 		case SHN_LIVEPATCH:
1400 			/* Livepatch symbols are resolved by livepatch */
1401 			break;
1402 
1403 		case SHN_UNDEF:
1404 			ksym = resolve_symbol_wait(mod, info, name);
1405 			/* Ok if resolved.  */
1406 			if (ksym && !IS_ERR(ksym)) {
1407 				sym[i].st_value = kernel_symbol_value(ksym);
1408 				break;
1409 			}
1410 
1411 			/* Ok if weak or ignored.  */
1412 			if (!ksym &&
1413 			    (ELF_ST_BIND(sym[i].st_info) == STB_WEAK ||
1414 			     ignore_undef_symbol(info->hdr->e_machine, name)))
1415 				break;
1416 
1417 			ret = PTR_ERR(ksym) ?: -ENOENT;
1418 			pr_warn("%s: Unknown symbol %s (err %d)\n",
1419 				mod->name, name, ret);
1420 			break;
1421 
1422 		default:
1423 			/* Divert to percpu allocation if a percpu var. */
1424 			if (sym[i].st_shndx == info->index.pcpu)
1425 				secbase = (unsigned long)mod_percpu(mod);
1426 			else
1427 				secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
1428 			sym[i].st_value += secbase;
1429 			break;
1430 		}
1431 	}
1432 
1433 	return ret;
1434 }
1435 
1436 static int apply_relocations(struct module *mod, const struct load_info *info)
1437 {
1438 	unsigned int i;
1439 	int err = 0;
1440 
1441 	/* Now do relocations. */
1442 	for (i = 1; i < info->hdr->e_shnum; i++) {
1443 		unsigned int infosec = info->sechdrs[i].sh_info;
1444 
1445 		/* Not a valid relocation section? */
1446 		if (infosec >= info->hdr->e_shnum)
1447 			continue;
1448 
1449 		/* Don't bother with non-allocated sections */
1450 		if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
1451 			continue;
1452 
1453 		if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
1454 			err = klp_apply_section_relocs(mod, info->sechdrs,
1455 						       info->secstrings,
1456 						       info->strtab,
1457 						       info->index.sym, i,
1458 						       NULL);
1459 		else if (info->sechdrs[i].sh_type == SHT_REL)
1460 			err = apply_relocate(info->sechdrs, info->strtab,
1461 					     info->index.sym, i, mod);
1462 		else if (info->sechdrs[i].sh_type == SHT_RELA)
1463 			err = apply_relocate_add(info->sechdrs, info->strtab,
1464 						 info->index.sym, i, mod);
1465 		if (err < 0)
1466 			break;
1467 	}
1468 	return err;
1469 }
1470 
1471 /* Additional bytes needed by arch in front of individual sections */
1472 unsigned int __weak arch_mod_section_prepend(struct module *mod,
1473 					     unsigned int section)
1474 {
1475 	/* default implementation just returns zero */
1476 	return 0;
1477 }
1478 
1479 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
1480 				Elf_Shdr *sechdr, unsigned int section)
1481 {
1482 	long offset;
1483 	long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT;
1484 
1485 	mod->mem[type].size += arch_mod_section_prepend(mod, section);
1486 	offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1);
1487 	mod->mem[type].size = offset + sechdr->sh_size;
1488 
1489 	WARN_ON_ONCE(offset & mask);
1490 	return offset | mask;
1491 }
1492 
1493 static bool module_init_layout_section(const char *sname)
1494 {
1495 #ifndef CONFIG_MODULE_UNLOAD
1496 	if (module_exit_section(sname))
1497 		return true;
1498 #endif
1499 	return module_init_section(sname);
1500 }
1501 
1502 static void __layout_sections(struct module *mod, struct load_info *info, bool is_init)
1503 {
1504 	unsigned int m, i;
1505 
1506 	static const unsigned long masks[][2] = {
1507 		/*
1508 		 * NOTE: all executable code must be the first section
1509 		 * in this array; otherwise modify the text_size
1510 		 * finder in the two loops below
1511 		 */
1512 		{ SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
1513 		{ SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
1514 		{ SHF_RO_AFTER_INIT | SHF_ALLOC, ARCH_SHF_SMALL },
1515 		{ SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
1516 		{ ARCH_SHF_SMALL | SHF_ALLOC, 0 }
1517 	};
1518 	static const int core_m_to_mem_type[] = {
1519 		MOD_TEXT,
1520 		MOD_RODATA,
1521 		MOD_RO_AFTER_INIT,
1522 		MOD_DATA,
1523 		MOD_INVALID,	/* This is needed to match the masks array */
1524 	};
1525 	static const int init_m_to_mem_type[] = {
1526 		MOD_INIT_TEXT,
1527 		MOD_INIT_RODATA,
1528 		MOD_INVALID,
1529 		MOD_INIT_DATA,
1530 		MOD_INVALID,	/* This is needed to match the masks array */
1531 	};
1532 
1533 	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1534 		enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m];
1535 
1536 		for (i = 0; i < info->hdr->e_shnum; ++i) {
1537 			Elf_Shdr *s = &info->sechdrs[i];
1538 			const char *sname = info->secstrings + s->sh_name;
1539 
1540 			if ((s->sh_flags & masks[m][0]) != masks[m][0]
1541 			    || (s->sh_flags & masks[m][1])
1542 			    || s->sh_entsize != ~0UL
1543 			    || is_init != module_init_layout_section(sname))
1544 				continue;
1545 
1546 			if (WARN_ON_ONCE(type == MOD_INVALID))
1547 				continue;
1548 
1549 			s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
1550 			pr_debug("\t%s\n", sname);
1551 		}
1552 	}
1553 }
1554 
1555 /*
1556  * Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
1557  * might -- code, read-only data, read-write data, small data.  Tally
1558  * sizes, and place the offsets into sh_entsize fields: high bit means it
1559  * belongs in init.
1560  */
1561 static void layout_sections(struct module *mod, struct load_info *info)
1562 {
1563 	unsigned int i;
1564 
1565 	for (i = 0; i < info->hdr->e_shnum; i++)
1566 		info->sechdrs[i].sh_entsize = ~0UL;
1567 
1568 	pr_debug("Core section allocation order for %s:\n", mod->name);
1569 	__layout_sections(mod, info, false);
1570 
1571 	pr_debug("Init section allocation order for %s:\n", mod->name);
1572 	__layout_sections(mod, info, true);
1573 }
1574 
1575 static void module_license_taint_check(struct module *mod, const char *license)
1576 {
1577 	if (!license)
1578 		license = "unspecified";
1579 
1580 	if (!license_is_gpl_compatible(license)) {
1581 		if (!test_taint(TAINT_PROPRIETARY_MODULE))
1582 			pr_warn("%s: module license '%s' taints kernel.\n",
1583 				mod->name, license);
1584 		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
1585 				 LOCKDEP_NOW_UNRELIABLE);
1586 	}
1587 }
1588 
1589 static void setup_modinfo(struct module *mod, struct load_info *info)
1590 {
1591 	struct module_attribute *attr;
1592 	int i;
1593 
1594 	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1595 		if (attr->setup)
1596 			attr->setup(mod, get_modinfo(info, attr->attr.name));
1597 	}
1598 }
1599 
1600 static void free_modinfo(struct module *mod)
1601 {
1602 	struct module_attribute *attr;
1603 	int i;
1604 
1605 	for (i = 0; (attr = modinfo_attrs[i]); i++) {
1606 		if (attr->free)
1607 			attr->free(mod);
1608 	}
1609 }
1610 
1611 void * __weak module_alloc(unsigned long size)
1612 {
1613 	return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
1614 			GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
1615 			NUMA_NO_NODE, __builtin_return_address(0));
1616 }
1617 
1618 bool __weak module_init_section(const char *name)
1619 {
1620 	return strstarts(name, ".init");
1621 }
1622 
1623 bool __weak module_exit_section(const char *name)
1624 {
1625 	return strstarts(name, ".exit");
1626 }
1627 
1628 static int validate_section_offset(struct load_info *info, Elf_Shdr *shdr)
1629 {
1630 #if defined(CONFIG_64BIT)
1631 	unsigned long long secend;
1632 #else
1633 	unsigned long secend;
1634 #endif
1635 
1636 	/*
1637 	 * Check for both overflow and offset/size being
1638 	 * too large.
1639 	 */
1640 	secend = shdr->sh_offset + shdr->sh_size;
1641 	if (secend < shdr->sh_offset || secend > info->len)
1642 		return -ENOEXEC;
1643 
1644 	return 0;
1645 }
1646 
1647 /*
1648  * Check userspace passed ELF module against our expectations, and cache
1649  * useful variables for further processing as we go.
1650  *
1651  * This does basic validity checks against section offsets and sizes, the
1652  * section name string table, and the indices used for it (sh_name).
1653  *
1654  * As a last step, since we're already checking the ELF sections we cache
1655  * useful variables which will be used later for our convenience:
1656  *
1657  * 	o pointers to section headers
1658  * 	o cache the modinfo symbol section
1659  * 	o cache the string symbol section
1660  * 	o cache the module section
1661  *
1662  * As a last step we set info->mod to the temporary copy of the module in
1663  * info->hdr. The final one will be allocated in move_module(). Any
1664  * modifications we make to our copy of the module will be carried over
1665  * to the final minted module.
1666  */
1667 static int elf_validity_cache_copy(struct load_info *info, int flags)
1668 {
1669 	unsigned int i;
1670 	Elf_Shdr *shdr, *strhdr;
1671 	int err;
1672 	unsigned int num_mod_secs = 0, mod_idx;
1673 	unsigned int num_info_secs = 0, info_idx;
1674 	unsigned int num_sym_secs = 0, sym_idx;
1675 
1676 	if (info->len < sizeof(*(info->hdr))) {
1677 		pr_err("Invalid ELF header len %lu\n", info->len);
1678 		goto no_exec;
1679 	}
1680 
1681 	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) {
1682 		pr_err("Invalid ELF header magic: != %s\n", ELFMAG);
1683 		goto no_exec;
1684 	}
1685 	if (info->hdr->e_type != ET_REL) {
1686 		pr_err("Invalid ELF header type: %u != %u\n",
1687 		       info->hdr->e_type, ET_REL);
1688 		goto no_exec;
1689 	}
1690 	if (!elf_check_arch(info->hdr)) {
1691 		pr_err("Invalid architecture in ELF header: %u\n",
1692 		       info->hdr->e_machine);
1693 		goto no_exec;
1694 	}
1695 	if (!module_elf_check_arch(info->hdr)) {
1696 		pr_err("Invalid module architecture in ELF header: %u\n",
1697 		       info->hdr->e_machine);
1698 		goto no_exec;
1699 	}
1700 	if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) {
1701 		pr_err("Invalid ELF section header size\n");
1702 		goto no_exec;
1703 	}
1704 
1705 	/*
1706 	 * e_shnum is 16 bits, and sizeof(Elf_Shdr) is
1707 	 * known and small. So e_shnum * sizeof(Elf_Shdr)
1708 	 * will not overflow unsigned long on any platform.
1709 	 */
1710 	if (info->hdr->e_shoff >= info->len
1711 	    || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
1712 		info->len - info->hdr->e_shoff)) {
1713 		pr_err("Invalid ELF section header overflow\n");
1714 		goto no_exec;
1715 	}
1716 
1717 	info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
1718 
1719 	/*
1720 	 * Verify if the section name table index is valid.
1721 	 */
1722 	if (info->hdr->e_shstrndx == SHN_UNDEF
1723 	    || info->hdr->e_shstrndx >= info->hdr->e_shnum) {
1724 		pr_err("Invalid ELF section name index: %d || e_shstrndx (%d) >= e_shnum (%d)\n",
1725 		       info->hdr->e_shstrndx, info->hdr->e_shstrndx,
1726 		       info->hdr->e_shnum);
1727 		goto no_exec;
1728 	}
1729 
1730 	strhdr = &info->sechdrs[info->hdr->e_shstrndx];
1731 	err = validate_section_offset(info, strhdr);
1732 	if (err < 0) {
1733 		pr_err("Invalid ELF section hdr(type %u)\n", strhdr->sh_type);
1734 		return err;
1735 	}
1736 
1737 	/*
1738 	 * The section name table must be NUL-terminated, as required
1739 	 * by the spec. This makes strcmp and pr_* calls that access
1740 	 * strings in the section safe.
1741 	 */
1742 	info->secstrings = (void *)info->hdr + strhdr->sh_offset;
1743 	if (strhdr->sh_size == 0) {
1744 		pr_err("empty section name table\n");
1745 		goto no_exec;
1746 	}
1747 	if (info->secstrings[strhdr->sh_size - 1] != '\0') {
1748 		pr_err("ELF Spec violation: section name table isn't null terminated\n");
1749 		goto no_exec;
1750 	}
1751 
1752 	/*
1753 	 * The code assumes that section 0 has a length of zero and
1754 	 * an addr of zero, so check for it.
1755 	 */
1756 	if (info->sechdrs[0].sh_type != SHT_NULL
1757 	    || info->sechdrs[0].sh_size != 0
1758 	    || info->sechdrs[0].sh_addr != 0) {
1759 		pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n",
1760 		       info->sechdrs[0].sh_type);
1761 		goto no_exec;
1762 	}
1763 
1764 	for (i = 1; i < info->hdr->e_shnum; i++) {
1765 		shdr = &info->sechdrs[i];
1766 		switch (shdr->sh_type) {
1767 		case SHT_NULL:
1768 		case SHT_NOBITS:
1769 			continue;
1770 		case SHT_SYMTAB:
1771 			if (shdr->sh_link == SHN_UNDEF
1772 			    || shdr->sh_link >= info->hdr->e_shnum) {
1773 				pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n",
1774 				       shdr->sh_link, shdr->sh_link,
1775 				       info->hdr->e_shnum);
1776 				goto no_exec;
1777 			}
1778 			num_sym_secs++;
1779 			sym_idx = i;
1780 			fallthrough;
1781 		default:
1782 			err = validate_section_offset(info, shdr);
1783 			if (err < 0) {
1784 				pr_err("Invalid ELF section in module (section %u type %u)\n",
1785 					i, shdr->sh_type);
1786 				return err;
1787 			}
1788 			if (strcmp(info->secstrings + shdr->sh_name,
1789 				   ".gnu.linkonce.this_module") == 0) {
1790 				num_mod_secs++;
1791 				mod_idx = i;
1792 			} else if (strcmp(info->secstrings + shdr->sh_name,
1793 				   ".modinfo") == 0) {
1794 				num_info_secs++;
1795 				info_idx = i;
1796 			}
1797 
1798 			if (shdr->sh_flags & SHF_ALLOC) {
1799 				if (shdr->sh_name >= strhdr->sh_size) {
1800 					pr_err("Invalid ELF section name in module (section %u type %u)\n",
1801 					       i, shdr->sh_type);
1802 					return -ENOEXEC;
1803 				}
1804 			}
1805 			break;
1806 		}
1807 	}
1808 
1809 	if (num_info_secs > 1) {
1810 		pr_err("Only one .modinfo section must exist.\n");
1811 		goto no_exec;
1812 	} else if (num_info_secs == 1) {
1813 		/* Try to find a name early so we can log errors with a module name */
1814 		info->index.info = info_idx;
1815 		info->name = get_modinfo(info, "name");
1816 	}
1817 
1818 	if (num_sym_secs != 1) {
1819 		pr_warn("%s: module has no symbols (stripped?)\n",
1820 			info->name ?: "(missing .modinfo section or name field)");
1821 		goto no_exec;
1822 	}
1823 
1824 	/* Sets internal symbols and strings. */
1825 	info->index.sym = sym_idx;
1826 	shdr = &info->sechdrs[sym_idx];
1827 	info->index.str = shdr->sh_link;
1828 	info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset;
1829 
1830 	/*
1831 	 * The ".gnu.linkonce.this_module" ELF section is special. It is
1832 	 * what modpost uses to refer to __this_module and let's use rely
1833 	 * on THIS_MODULE to point to &__this_module properly. The kernel's
1834 	 * modpost declares it on each modules's *.mod.c file. If the struct
1835 	 * module of the kernel changes a full kernel rebuild is required.
1836 	 *
1837 	 * We have a few expectaions for this special section, the following
1838 	 * code validates all this for us:
1839 	 *
1840 	 *   o Only one section must exist
1841 	 *   o We expect the kernel to always have to allocate it: SHF_ALLOC
1842 	 *   o The section size must match the kernel's run time's struct module
1843 	 *     size
1844 	 */
1845 	if (num_mod_secs != 1) {
1846 		pr_err("module %s: Only one .gnu.linkonce.this_module section must exist.\n",
1847 		       info->name ?: "(missing .modinfo section or name field)");
1848 		goto no_exec;
1849 	}
1850 
1851 	shdr = &info->sechdrs[mod_idx];
1852 
1853 	/*
1854 	 * This is already implied on the switch above, however let's be
1855 	 * pedantic about it.
1856 	 */
1857 	if (shdr->sh_type == SHT_NOBITS) {
1858 		pr_err("module %s: .gnu.linkonce.this_module section must have a size set\n",
1859 		       info->name ?: "(missing .modinfo section or name field)");
1860 		goto no_exec;
1861 	}
1862 
1863 	if (!(shdr->sh_flags & SHF_ALLOC)) {
1864 		pr_err("module %s: .gnu.linkonce.this_module must occupy memory during process execution\n",
1865 		       info->name ?: "(missing .modinfo section or name field)");
1866 		goto no_exec;
1867 	}
1868 
1869 	if (shdr->sh_size != sizeof(struct module)) {
1870 		pr_err("module %s: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time\n",
1871 		       info->name ?: "(missing .modinfo section or name field)");
1872 		goto no_exec;
1873 	}
1874 
1875 	info->index.mod = mod_idx;
1876 
1877 	/* This is temporary: point mod into copy of data. */
1878 	info->mod = (void *)info->hdr + shdr->sh_offset;
1879 
1880 	/*
1881 	 * If we didn't load the .modinfo 'name' field earlier, fall back to
1882 	 * on-disk struct mod 'name' field.
1883 	 */
1884 	if (!info->name)
1885 		info->name = info->mod->name;
1886 
1887 	if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
1888 		info->index.vers = 0; /* Pretend no __versions section! */
1889 	else
1890 		info->index.vers = find_sec(info, "__versions");
1891 
1892 	info->index.pcpu = find_pcpusec(info);
1893 
1894 	return 0;
1895 
1896 no_exec:
1897 	return -ENOEXEC;
1898 }
1899 
1900 #define COPY_CHUNK_SIZE (16*PAGE_SIZE)
1901 
1902 static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len)
1903 {
1904 	do {
1905 		unsigned long n = min(len, COPY_CHUNK_SIZE);
1906 
1907 		if (copy_from_user(dst, usrc, n) != 0)
1908 			return -EFAULT;
1909 		cond_resched();
1910 		dst += n;
1911 		usrc += n;
1912 		len -= n;
1913 	} while (len);
1914 	return 0;
1915 }
1916 
1917 static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
1918 {
1919 	if (!get_modinfo(info, "livepatch"))
1920 		/* Nothing more to do */
1921 		return 0;
1922 
1923 	if (set_livepatch_module(mod))
1924 		return 0;
1925 
1926 	pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
1927 	       mod->name);
1928 	return -ENOEXEC;
1929 }
1930 
1931 static void check_modinfo_retpoline(struct module *mod, struct load_info *info)
1932 {
1933 	if (retpoline_module_ok(get_modinfo(info, "retpoline")))
1934 		return;
1935 
1936 	pr_warn("%s: loading module not compiled with retpoline compiler.\n",
1937 		mod->name);
1938 }
1939 
1940 /* Sets info->hdr and info->len. */
1941 static int copy_module_from_user(const void __user *umod, unsigned long len,
1942 				  struct load_info *info)
1943 {
1944 	int err;
1945 
1946 	info->len = len;
1947 	if (info->len < sizeof(*(info->hdr)))
1948 		return -ENOEXEC;
1949 
1950 	err = security_kernel_load_data(LOADING_MODULE, true);
1951 	if (err)
1952 		return err;
1953 
1954 	/* Suck in entire file: we'll want most of it. */
1955 	info->hdr = __vmalloc(info->len, GFP_KERNEL | __GFP_NOWARN);
1956 	if (!info->hdr)
1957 		return -ENOMEM;
1958 
1959 	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
1960 		err = -EFAULT;
1961 		goto out;
1962 	}
1963 
1964 	err = security_kernel_post_load_data((char *)info->hdr, info->len,
1965 					     LOADING_MODULE, "init_module");
1966 out:
1967 	if (err)
1968 		vfree(info->hdr);
1969 
1970 	return err;
1971 }
1972 
1973 static void free_copy(struct load_info *info, int flags)
1974 {
1975 	if (flags & MODULE_INIT_COMPRESSED_FILE)
1976 		module_decompress_cleanup(info);
1977 	else
1978 		vfree(info->hdr);
1979 }
1980 
1981 static int rewrite_section_headers(struct load_info *info, int flags)
1982 {
1983 	unsigned int i;
1984 
1985 	/* This should always be true, but let's be sure. */
1986 	info->sechdrs[0].sh_addr = 0;
1987 
1988 	for (i = 1; i < info->hdr->e_shnum; i++) {
1989 		Elf_Shdr *shdr = &info->sechdrs[i];
1990 
1991 		/*
1992 		 * Mark all sections sh_addr with their address in the
1993 		 * temporary image.
1994 		 */
1995 		shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
1996 
1997 	}
1998 
1999 	/* Track but don't keep modinfo and version sections. */
2000 	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2001 	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2002 
2003 	return 0;
2004 }
2005 
2006 /*
2007  * These calls taint the kernel depending certain module circumstances */
2008 static void module_augment_kernel_taints(struct module *mod, struct load_info *info)
2009 {
2010 	int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);
2011 
2012 	if (!get_modinfo(info, "intree")) {
2013 		if (!test_taint(TAINT_OOT_MODULE))
2014 			pr_warn("%s: loading out-of-tree module taints kernel.\n",
2015 				mod->name);
2016 		add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
2017 	}
2018 
2019 	check_modinfo_retpoline(mod, info);
2020 
2021 	if (get_modinfo(info, "staging")) {
2022 		add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
2023 		pr_warn("%s: module is from the staging directory, the quality "
2024 			"is unknown, you have been warned.\n", mod->name);
2025 	}
2026 
2027 	if (is_livepatch_module(mod)) {
2028 		add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
2029 		pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
2030 				mod->name);
2031 	}
2032 
2033 	module_license_taint_check(mod, get_modinfo(info, "license"));
2034 
2035 	if (get_modinfo(info, "test")) {
2036 		if (!test_taint(TAINT_TEST))
2037 			pr_warn("%s: loading test module taints kernel.\n",
2038 				mod->name);
2039 		add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK);
2040 	}
2041 #ifdef CONFIG_MODULE_SIG
2042 	mod->sig_ok = info->sig_ok;
2043 	if (!mod->sig_ok) {
2044 		pr_notice_once("%s: module verification failed: signature "
2045 			       "and/or required key missing - tainting "
2046 			       "kernel\n", mod->name);
2047 		add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
2048 	}
2049 #endif
2050 
2051 	/*
2052 	 * ndiswrapper is under GPL by itself, but loads proprietary modules.
2053 	 * Don't use add_taint_module(), as it would prevent ndiswrapper from
2054 	 * using GPL-only symbols it needs.
2055 	 */
2056 	if (strcmp(mod->name, "ndiswrapper") == 0)
2057 		add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
2058 
2059 	/* driverloader was caught wrongly pretending to be under GPL */
2060 	if (strcmp(mod->name, "driverloader") == 0)
2061 		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2062 				 LOCKDEP_NOW_UNRELIABLE);
2063 
2064 	/* lve claims to be GPL but upstream won't provide source */
2065 	if (strcmp(mod->name, "lve") == 0)
2066 		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2067 				 LOCKDEP_NOW_UNRELIABLE);
2068 
2069 	if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
2070 		pr_warn("%s: module license taints kernel.\n", mod->name);
2071 
2072 }
2073 
2074 static int check_modinfo(struct module *mod, struct load_info *info, int flags)
2075 {
2076 	const char *modmagic = get_modinfo(info, "vermagic");
2077 	int err;
2078 
2079 	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
2080 		modmagic = NULL;
2081 
2082 	/* This is allowed: modprobe --force will invalidate it. */
2083 	if (!modmagic) {
2084 		err = try_to_force_load(mod, "bad vermagic");
2085 		if (err)
2086 			return err;
2087 	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
2088 		pr_err("%s: version magic '%s' should be '%s'\n",
2089 		       info->name, modmagic, vermagic);
2090 		return -ENOEXEC;
2091 	}
2092 
2093 	err = check_modinfo_livepatch(mod, info);
2094 	if (err)
2095 		return err;
2096 
2097 	return 0;
2098 }
2099 
2100 static int find_module_sections(struct module *mod, struct load_info *info)
2101 {
2102 	mod->kp = section_objs(info, "__param",
2103 			       sizeof(*mod->kp), &mod->num_kp);
2104 	mod->syms = section_objs(info, "__ksymtab",
2105 				 sizeof(*mod->syms), &mod->num_syms);
2106 	mod->crcs = section_addr(info, "__kcrctab");
2107 	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
2108 				     sizeof(*mod->gpl_syms),
2109 				     &mod->num_gpl_syms);
2110 	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2111 
2112 #ifdef CONFIG_CONSTRUCTORS
2113 	mod->ctors = section_objs(info, ".ctors",
2114 				  sizeof(*mod->ctors), &mod->num_ctors);
2115 	if (!mod->ctors)
2116 		mod->ctors = section_objs(info, ".init_array",
2117 				sizeof(*mod->ctors), &mod->num_ctors);
2118 	else if (find_sec(info, ".init_array")) {
2119 		/*
2120 		 * This shouldn't happen with same compiler and binutils
2121 		 * building all parts of the module.
2122 		 */
2123 		pr_warn("%s: has both .ctors and .init_array.\n",
2124 		       mod->name);
2125 		return -EINVAL;
2126 	}
2127 #endif
2128 
2129 	mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
2130 						&mod->noinstr_text_size);
2131 
2132 #ifdef CONFIG_TRACEPOINTS
2133 	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
2134 					     sizeof(*mod->tracepoints_ptrs),
2135 					     &mod->num_tracepoints);
2136 #endif
2137 #ifdef CONFIG_TREE_SRCU
2138 	mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
2139 					     sizeof(*mod->srcu_struct_ptrs),
2140 					     &mod->num_srcu_structs);
2141 #endif
2142 #ifdef CONFIG_BPF_EVENTS
2143 	mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
2144 					   sizeof(*mod->bpf_raw_events),
2145 					   &mod->num_bpf_raw_events);
2146 #endif
2147 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2148 	mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);
2149 #endif
2150 #ifdef CONFIG_JUMP_LABEL
2151 	mod->jump_entries = section_objs(info, "__jump_table",
2152 					sizeof(*mod->jump_entries),
2153 					&mod->num_jump_entries);
2154 #endif
2155 #ifdef CONFIG_EVENT_TRACING
2156 	mod->trace_events = section_objs(info, "_ftrace_events",
2157 					 sizeof(*mod->trace_events),
2158 					 &mod->num_trace_events);
2159 	mod->trace_evals = section_objs(info, "_ftrace_eval_map",
2160 					sizeof(*mod->trace_evals),
2161 					&mod->num_trace_evals);
2162 #endif
2163 #ifdef CONFIG_TRACING
2164 	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
2165 					 sizeof(*mod->trace_bprintk_fmt_start),
2166 					 &mod->num_trace_bprintk_fmt);
2167 #endif
2168 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
2169 	/* sechdrs[0].sh_size is always zero */
2170 	mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
2171 					     sizeof(*mod->ftrace_callsites),
2172 					     &mod->num_ftrace_callsites);
2173 #endif
2174 #ifdef CONFIG_FUNCTION_ERROR_INJECTION
2175 	mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
2176 					    sizeof(*mod->ei_funcs),
2177 					    &mod->num_ei_funcs);
2178 #endif
2179 #ifdef CONFIG_KPROBES
2180 	mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
2181 						&mod->kprobes_text_size);
2182 	mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
2183 						sizeof(unsigned long),
2184 						&mod->num_kprobe_blacklist);
2185 #endif
2186 #ifdef CONFIG_PRINTK_INDEX
2187 	mod->printk_index_start = section_objs(info, ".printk_index",
2188 					       sizeof(*mod->printk_index_start),
2189 					       &mod->printk_index_size);
2190 #endif
2191 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
2192 	mod->static_call_sites = section_objs(info, ".static_call_sites",
2193 					      sizeof(*mod->static_call_sites),
2194 					      &mod->num_static_call_sites);
2195 #endif
2196 #if IS_ENABLED(CONFIG_KUNIT)
2197 	mod->kunit_suites = section_objs(info, ".kunit_test_suites",
2198 					      sizeof(*mod->kunit_suites),
2199 					      &mod->num_kunit_suites);
2200 #endif
2201 
2202 	mod->extable = section_objs(info, "__ex_table",
2203 				    sizeof(*mod->extable), &mod->num_exentries);
2204 
2205 	if (section_addr(info, "__obsparm"))
2206 		pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
2207 
2208 #ifdef CONFIG_DYNAMIC_DEBUG_CORE
2209 	mod->dyndbg_info.descs = section_objs(info, "__dyndbg",
2210 					      sizeof(*mod->dyndbg_info.descs),
2211 					      &mod->dyndbg_info.num_descs);
2212 	mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes",
2213 						sizeof(*mod->dyndbg_info.classes),
2214 						&mod->dyndbg_info.num_classes);
2215 #endif
2216 
2217 	return 0;
2218 }
2219 
2220 static int move_module(struct module *mod, struct load_info *info)
2221 {
2222 	int i;
2223 	void *ptr;
2224 	enum mod_mem_type t = 0;
2225 	int ret = -ENOMEM;
2226 
2227 	for_each_mod_mem_type(type) {
2228 		if (!mod->mem[type].size) {
2229 			mod->mem[type].base = NULL;
2230 			continue;
2231 		}
2232 		mod->mem[type].size = PAGE_ALIGN(mod->mem[type].size);
2233 		ptr = module_memory_alloc(mod->mem[type].size, type);
2234 
2235 		/*
2236 		 * The pointer to this block is stored in the module structure
2237 		 * which is inside the block. Just mark it as not being a
2238 		 * leak.
2239 		 */
2240 		kmemleak_ignore(ptr);
2241 		if (!ptr) {
2242 			t = type;
2243 			goto out_enomem;
2244 		}
2245 		memset(ptr, 0, mod->mem[type].size);
2246 		mod->mem[type].base = ptr;
2247 	}
2248 
2249 	/* Transfer each section which specifies SHF_ALLOC */
2250 	pr_debug("Final section addresses for %s:\n", mod->name);
2251 	for (i = 0; i < info->hdr->e_shnum; i++) {
2252 		void *dest;
2253 		Elf_Shdr *shdr = &info->sechdrs[i];
2254 		enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
2255 
2256 		if (!(shdr->sh_flags & SHF_ALLOC))
2257 			continue;
2258 
2259 		dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK);
2260 
2261 		if (shdr->sh_type != SHT_NOBITS) {
2262 			/*
2263 			 * Our ELF checker already validated this, but let's
2264 			 * be pedantic and make the goal clearer. We actually
2265 			 * end up copying over all modifications made to the
2266 			 * userspace copy of the entire struct module.
2267 			 */
2268 			if (i == info->index.mod &&
2269 			   (WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) {
2270 				ret = -ENOEXEC;
2271 				goto out_enomem;
2272 			}
2273 			memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
2274 		}
2275 		/*
2276 		 * Update the userspace copy's ELF section address to point to
2277 		 * our newly allocated memory as a pure convenience so that
2278 		 * users of info can keep taking advantage and using the newly
2279 		 * minted official memory area.
2280 		 */
2281 		shdr->sh_addr = (unsigned long)dest;
2282 		pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr,
2283 			 (long)shdr->sh_size, info->secstrings + shdr->sh_name);
2284 	}
2285 
2286 	return 0;
2287 out_enomem:
2288 	for (t--; t >= 0; t--)
2289 		module_memory_free(mod->mem[t].base, t);
2290 	return ret;
2291 }
2292 
2293 static int check_export_symbol_versions(struct module *mod)
2294 {
2295 #ifdef CONFIG_MODVERSIONS
2296 	if ((mod->num_syms && !mod->crcs) ||
2297 	    (mod->num_gpl_syms && !mod->gpl_crcs)) {
2298 		return try_to_force_load(mod,
2299 					 "no versions for exported symbols");
2300 	}
2301 #endif
2302 	return 0;
2303 }
2304 
2305 static void flush_module_icache(const struct module *mod)
2306 {
2307 	/*
2308 	 * Flush the instruction cache, since we've played with text.
2309 	 * Do it before processing of module parameters, so the module
2310 	 * can provide parameter accessor functions of its own.
2311 	 */
2312 	for_each_mod_mem_type(type) {
2313 		const struct module_memory *mod_mem = &mod->mem[type];
2314 
2315 		if (mod_mem->size) {
2316 			flush_icache_range((unsigned long)mod_mem->base,
2317 					   (unsigned long)mod_mem->base + mod_mem->size);
2318 		}
2319 	}
2320 }
2321 
2322 bool __weak module_elf_check_arch(Elf_Ehdr *hdr)
2323 {
2324 	return true;
2325 }
2326 
2327 int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
2328 				     Elf_Shdr *sechdrs,
2329 				     char *secstrings,
2330 				     struct module *mod)
2331 {
2332 	return 0;
2333 }
2334 
2335 /* module_blacklist is a comma-separated list of module names */
2336 static char *module_blacklist;
2337 static bool blacklisted(const char *module_name)
2338 {
2339 	const char *p;
2340 	size_t len;
2341 
2342 	if (!module_blacklist)
2343 		return false;
2344 
2345 	for (p = module_blacklist; *p; p += len) {
2346 		len = strcspn(p, ",");
2347 		if (strlen(module_name) == len && !memcmp(module_name, p, len))
2348 			return true;
2349 		if (p[len] == ',')
2350 			len++;
2351 	}
2352 	return false;
2353 }
2354 core_param(module_blacklist, module_blacklist, charp, 0400);
2355 
2356 static struct module *layout_and_allocate(struct load_info *info, int flags)
2357 {
2358 	struct module *mod;
2359 	unsigned int ndx;
2360 	int err;
2361 
2362 	/* Allow arches to frob section contents and sizes.  */
2363 	err = module_frob_arch_sections(info->hdr, info->sechdrs,
2364 					info->secstrings, info->mod);
2365 	if (err < 0)
2366 		return ERR_PTR(err);
2367 
2368 	err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
2369 					  info->secstrings, info->mod);
2370 	if (err < 0)
2371 		return ERR_PTR(err);
2372 
2373 	/* We will do a special allocation for per-cpu sections later. */
2374 	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
2375 
2376 	/*
2377 	 * Mark ro_after_init section with SHF_RO_AFTER_INIT so that
2378 	 * layout_sections() can put it in the right place.
2379 	 * Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
2380 	 */
2381 	ndx = find_sec(info, ".data..ro_after_init");
2382 	if (ndx)
2383 		info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
2384 	/*
2385 	 * Mark the __jump_table section as ro_after_init as well: these data
2386 	 * structures are never modified, with the exception of entries that
2387 	 * refer to code in the __init section, which are annotated as such
2388 	 * at module load time.
2389 	 */
2390 	ndx = find_sec(info, "__jump_table");
2391 	if (ndx)
2392 		info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
2393 
2394 	/*
2395 	 * Determine total sizes, and put offsets in sh_entsize.  For now
2396 	 * this is done generically; there doesn't appear to be any
2397 	 * special cases for the architectures.
2398 	 */
2399 	layout_sections(info->mod, info);
2400 	layout_symtab(info->mod, info);
2401 
2402 	/* Allocate and move to the final place */
2403 	err = move_module(info->mod, info);
2404 	if (err)
2405 		return ERR_PTR(err);
2406 
2407 	/* Module has been copied to its final place now: return it. */
2408 	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2409 	kmemleak_load_module(mod, info);
2410 	return mod;
2411 }
2412 
2413 /* mod is no longer valid after this! */
2414 static void module_deallocate(struct module *mod, struct load_info *info)
2415 {
2416 	percpu_modfree(mod);
2417 	module_arch_freeing_init(mod);
2418 
2419 	free_mod_mem(mod);
2420 }
2421 
2422 int __weak module_finalize(const Elf_Ehdr *hdr,
2423 			   const Elf_Shdr *sechdrs,
2424 			   struct module *me)
2425 {
2426 	return 0;
2427 }
2428 
2429 static int post_relocation(struct module *mod, const struct load_info *info)
2430 {
2431 	/* Sort exception table now relocations are done. */
2432 	sort_extable(mod->extable, mod->extable + mod->num_exentries);
2433 
2434 	/* Copy relocated percpu area over. */
2435 	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
2436 		       info->sechdrs[info->index.pcpu].sh_size);
2437 
2438 	/* Setup kallsyms-specific fields. */
2439 	add_kallsyms(mod, info);
2440 
2441 	/* Arch-specific module finalizing. */
2442 	return module_finalize(info->hdr, info->sechdrs, mod);
2443 }
2444 
2445 /* Is this module of this name done loading?  No locks held. */
2446 static bool finished_loading(const char *name)
2447 {
2448 	struct module *mod;
2449 	bool ret;
2450 
2451 	/*
2452 	 * The module_mutex should not be a heavily contended lock;
2453 	 * if we get the occasional sleep here, we'll go an extra iteration
2454 	 * in the wait_event_interruptible(), which is harmless.
2455 	 */
2456 	sched_annotate_sleep();
2457 	mutex_lock(&module_mutex);
2458 	mod = find_module_all(name, strlen(name), true);
2459 	ret = !mod || mod->state == MODULE_STATE_LIVE
2460 		|| mod->state == MODULE_STATE_GOING;
2461 	mutex_unlock(&module_mutex);
2462 
2463 	return ret;
2464 }
2465 
2466 /* Call module constructors. */
2467 static void do_mod_ctors(struct module *mod)
2468 {
2469 #ifdef CONFIG_CONSTRUCTORS
2470 	unsigned long i;
2471 
2472 	for (i = 0; i < mod->num_ctors; i++)
2473 		mod->ctors[i]();
2474 #endif
2475 }
2476 
2477 /* For freeing module_init on success, in case kallsyms traversing */
2478 struct mod_initfree {
2479 	struct llist_node node;
2480 	void *init_text;
2481 	void *init_data;
2482 	void *init_rodata;
2483 };
2484 
2485 static void do_free_init(struct work_struct *w)
2486 {
2487 	struct llist_node *pos, *n, *list;
2488 	struct mod_initfree *initfree;
2489 
2490 	list = llist_del_all(&init_free_list);
2491 
2492 	synchronize_rcu();
2493 
2494 	llist_for_each_safe(pos, n, list) {
2495 		initfree = container_of(pos, struct mod_initfree, node);
2496 		module_memfree(initfree->init_text);
2497 		module_memfree(initfree->init_data);
2498 		module_memfree(initfree->init_rodata);
2499 		kfree(initfree);
2500 	}
2501 }
2502 
2503 #undef MODULE_PARAM_PREFIX
2504 #define MODULE_PARAM_PREFIX "module."
2505 /* Default value for module->async_probe_requested */
2506 static bool async_probe;
2507 module_param(async_probe, bool, 0644);
2508 
2509 /*
2510  * This is where the real work happens.
2511  *
2512  * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
2513  * helper command 'lx-symbols'.
2514  */
2515 static noinline int do_init_module(struct module *mod)
2516 {
2517 	int ret = 0;
2518 	struct mod_initfree *freeinit;
2519 
2520 	freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
2521 	if (!freeinit) {
2522 		ret = -ENOMEM;
2523 		goto fail;
2524 	}
2525 	freeinit->init_text = mod->mem[MOD_INIT_TEXT].base;
2526 	freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
2527 	freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
2528 
2529 	do_mod_ctors(mod);
2530 	/* Start the module */
2531 	if (mod->init != NULL)
2532 		ret = do_one_initcall(mod->init);
2533 	if (ret < 0) {
2534 		goto fail_free_freeinit;
2535 	}
2536 	if (ret > 0) {
2537 		pr_warn("%s: '%s'->init suspiciously returned %d, it should "
2538 			"follow 0/-E convention\n"
2539 			"%s: loading module anyway...\n",
2540 			__func__, mod->name, ret, __func__);
2541 		dump_stack();
2542 	}
2543 
2544 	/* Now it's a first class citizen! */
2545 	mod->state = MODULE_STATE_LIVE;
2546 	blocking_notifier_call_chain(&module_notify_list,
2547 				     MODULE_STATE_LIVE, mod);
2548 
2549 	/* Delay uevent until module has finished its init routine */
2550 	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
2551 
2552 	/*
2553 	 * We need to finish all async code before the module init sequence
2554 	 * is done. This has potential to deadlock if synchronous module
2555 	 * loading is requested from async (which is not allowed!).
2556 	 *
2557 	 * See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
2558 	 * request_module() from async workers") for more details.
2559 	 */
2560 	if (!mod->async_probe_requested)
2561 		async_synchronize_full();
2562 
2563 	ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base,
2564 			mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size);
2565 	mutex_lock(&module_mutex);
2566 	/* Drop initial reference. */
2567 	module_put(mod);
2568 	trim_init_extable(mod);
2569 #ifdef CONFIG_KALLSYMS
2570 	/* Switch to core kallsyms now init is done: kallsyms may be walking! */
2571 	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
2572 #endif
2573 	module_enable_ro(mod, true);
2574 	mod_tree_remove_init(mod);
2575 	module_arch_freeing_init(mod);
2576 	for_class_mod_mem_type(type, init) {
2577 		mod->mem[type].base = NULL;
2578 		mod->mem[type].size = 0;
2579 	}
2580 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2581 	/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */
2582 	mod->btf_data = NULL;
2583 #endif
2584 	/*
2585 	 * We want to free module_init, but be aware that kallsyms may be
2586 	 * walking this with preempt disabled.  In all the failure paths, we
2587 	 * call synchronize_rcu(), but we don't want to slow down the success
2588 	 * path. module_memfree() cannot be called in an interrupt, so do the
2589 	 * work and call synchronize_rcu() in a work queue.
2590 	 *
2591 	 * Note that module_alloc() on most architectures creates W+X page
2592 	 * mappings which won't be cleaned up until do_free_init() runs.  Any
2593 	 * code such as mark_rodata_ro() which depends on those mappings to
2594 	 * be cleaned up needs to sync with the queued work - ie
2595 	 * rcu_barrier()
2596 	 */
2597 	if (llist_add(&freeinit->node, &init_free_list))
2598 		schedule_work(&init_free_wq);
2599 
2600 	mutex_unlock(&module_mutex);
2601 	wake_up_all(&module_wq);
2602 
2603 	return 0;
2604 
2605 fail_free_freeinit:
2606 	kfree(freeinit);
2607 fail:
2608 	/* Try to protect us from buggy refcounters. */
2609 	mod->state = MODULE_STATE_GOING;
2610 	synchronize_rcu();
2611 	module_put(mod);
2612 	blocking_notifier_call_chain(&module_notify_list,
2613 				     MODULE_STATE_GOING, mod);
2614 	klp_module_going(mod);
2615 	ftrace_release_mod(mod);
2616 	free_module(mod);
2617 	wake_up_all(&module_wq);
2618 	return ret;
2619 }
2620 
2621 static int may_init_module(void)
2622 {
2623 	if (!capable(CAP_SYS_MODULE) || modules_disabled)
2624 		return -EPERM;
2625 
2626 	return 0;
2627 }
2628 
2629 /*
2630  * We try to place it in the list now to make sure it's unique before
2631  * we dedicate too many resources.  In particular, temporary percpu
2632  * memory exhaustion.
2633  */
2634 static int add_unformed_module(struct module *mod)
2635 {
2636 	int err;
2637 	struct module *old;
2638 
2639 	mod->state = MODULE_STATE_UNFORMED;
2640 
2641 	mutex_lock(&module_mutex);
2642 	old = find_module_all(mod->name, strlen(mod->name), true);
2643 	if (old != NULL) {
2644 		if (old->state == MODULE_STATE_COMING
2645 		    || old->state == MODULE_STATE_UNFORMED) {
2646 			/* Wait in case it fails to load. */
2647 			mutex_unlock(&module_mutex);
2648 			err = wait_event_interruptible(module_wq,
2649 					       finished_loading(mod->name));
2650 			if (err)
2651 				goto out_unlocked;
2652 
2653 			/* The module might have gone in the meantime. */
2654 			mutex_lock(&module_mutex);
2655 			old = find_module_all(mod->name, strlen(mod->name),
2656 					      true);
2657 		}
2658 
2659 		/*
2660 		 * We are here only when the same module was being loaded. Do
2661 		 * not try to load it again right now. It prevents long delays
2662 		 * caused by serialized module load failures. It might happen
2663 		 * when more devices of the same type trigger load of
2664 		 * a particular module.
2665 		 */
2666 		if (old && old->state == MODULE_STATE_LIVE)
2667 			err = -EEXIST;
2668 		else
2669 			err = -EBUSY;
2670 		goto out;
2671 	}
2672 	mod_update_bounds(mod);
2673 	list_add_rcu(&mod->list, &modules);
2674 	mod_tree_insert(mod);
2675 	err = 0;
2676 
2677 out:
2678 	mutex_unlock(&module_mutex);
2679 out_unlocked:
2680 	return err;
2681 }
2682 
2683 static int complete_formation(struct module *mod, struct load_info *info)
2684 {
2685 	int err;
2686 
2687 	mutex_lock(&module_mutex);
2688 
2689 	/* Find duplicate symbols (must be called under lock). */
2690 	err = verify_exported_symbols(mod);
2691 	if (err < 0)
2692 		goto out;
2693 
2694 	/* These rely on module_mutex for list integrity. */
2695 	module_bug_finalize(info->hdr, info->sechdrs, mod);
2696 	module_cfi_finalize(info->hdr, info->sechdrs, mod);
2697 
2698 	module_enable_ro(mod, false);
2699 	module_enable_nx(mod);
2700 	module_enable_x(mod);
2701 
2702 	/*
2703 	 * Mark state as coming so strong_try_module_get() ignores us,
2704 	 * but kallsyms etc. can see us.
2705 	 */
2706 	mod->state = MODULE_STATE_COMING;
2707 	mutex_unlock(&module_mutex);
2708 
2709 	return 0;
2710 
2711 out:
2712 	mutex_unlock(&module_mutex);
2713 	return err;
2714 }
2715 
2716 static int prepare_coming_module(struct module *mod)
2717 {
2718 	int err;
2719 
2720 	ftrace_module_enable(mod);
2721 	err = klp_module_coming(mod);
2722 	if (err)
2723 		return err;
2724 
2725 	err = blocking_notifier_call_chain_robust(&module_notify_list,
2726 			MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
2727 	err = notifier_to_errno(err);
2728 	if (err)
2729 		klp_module_going(mod);
2730 
2731 	return err;
2732 }
2733 
2734 static int unknown_module_param_cb(char *param, char *val, const char *modname,
2735 				   void *arg)
2736 {
2737 	struct module *mod = arg;
2738 	int ret;
2739 
2740 	if (strcmp(param, "async_probe") == 0) {
2741 		if (kstrtobool(val, &mod->async_probe_requested))
2742 			mod->async_probe_requested = true;
2743 		return 0;
2744 	}
2745 
2746 	/* Check for magic 'dyndbg' arg */
2747 	ret = ddebug_dyndbg_module_param_cb(param, val, modname);
2748 	if (ret != 0)
2749 		pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
2750 	return 0;
2751 }
2752 
2753 /* Module within temporary copy, this doesn't do any allocation  */
2754 static int early_mod_check(struct load_info *info, int flags)
2755 {
2756 	int err;
2757 
2758 	/*
2759 	 * Now that we know we have the correct module name, check
2760 	 * if it's blacklisted.
2761 	 */
2762 	if (blacklisted(info->name)) {
2763 		pr_err("Module %s is blacklisted\n", info->name);
2764 		return -EPERM;
2765 	}
2766 
2767 	err = rewrite_section_headers(info, flags);
2768 	if (err)
2769 		return err;
2770 
2771 	/* Check module struct version now, before we try to use module. */
2772 	if (!check_modstruct_version(info, info->mod))
2773 		return -ENOEXEC;
2774 
2775 	err = check_modinfo(info->mod, info, flags);
2776 	if (err)
2777 		return err;
2778 
2779 	return 0;
2780 }
2781 
2782 /*
2783  * Allocate and load the module: note that size of section 0 is always
2784  * zero, and we rely on this for optional sections.
2785  */
2786 static int load_module(struct load_info *info, const char __user *uargs,
2787 		       int flags)
2788 {
2789 	struct module *mod;
2790 	long err = 0;
2791 	char *after_dashes;
2792 
2793 	/*
2794 	 * Do the signature check (if any) first. All that
2795 	 * the signature check needs is info->len, it does
2796 	 * not need any of the section info. That can be
2797 	 * set up later. This will minimize the chances
2798 	 * of a corrupt module causing problems before
2799 	 * we even get to the signature check.
2800 	 *
2801 	 * The check will also adjust info->len by stripping
2802 	 * off the sig length at the end of the module, making
2803 	 * checks against info->len more correct.
2804 	 */
2805 	err = module_sig_check(info, flags);
2806 	if (err)
2807 		goto free_copy;
2808 
2809 	/*
2810 	 * Do basic sanity checks against the ELF header and
2811 	 * sections. Cache useful sections and set the
2812 	 * info->mod to the userspace passed struct module.
2813 	 */
2814 	err = elf_validity_cache_copy(info, flags);
2815 	if (err)
2816 		goto free_copy;
2817 
2818 	err = early_mod_check(info, flags);
2819 	if (err)
2820 		goto free_copy;
2821 
2822 	/* Figure out module layout, and allocate all the memory. */
2823 	mod = layout_and_allocate(info, flags);
2824 	if (IS_ERR(mod)) {
2825 		err = PTR_ERR(mod);
2826 		goto free_copy;
2827 	}
2828 
2829 	audit_log_kern_module(mod->name);
2830 
2831 	/* Reserve our place in the list. */
2832 	err = add_unformed_module(mod);
2833 	if (err)
2834 		goto free_module;
2835 
2836 	/*
2837 	 * We are tainting your kernel if your module gets into
2838 	 * the modules linked list somehow.
2839 	 */
2840 	module_augment_kernel_taints(mod, info);
2841 
2842 	/* To avoid stressing percpu allocator, do this once we're unique. */
2843 	err = percpu_modalloc(mod, info);
2844 	if (err)
2845 		goto unlink_mod;
2846 
2847 	/* Now module is in final location, initialize linked lists, etc. */
2848 	err = module_unload_init(mod);
2849 	if (err)
2850 		goto unlink_mod;
2851 
2852 	init_param_lock(mod);
2853 
2854 	/*
2855 	 * Now we've got everything in the final locations, we can
2856 	 * find optional sections.
2857 	 */
2858 	err = find_module_sections(mod, info);
2859 	if (err)
2860 		goto free_unload;
2861 
2862 	err = check_export_symbol_versions(mod);
2863 	if (err)
2864 		goto free_unload;
2865 
2866 	/* Set up MODINFO_ATTR fields */
2867 	setup_modinfo(mod, info);
2868 
2869 	/* Fix up syms, so that st_value is a pointer to location. */
2870 	err = simplify_symbols(mod, info);
2871 	if (err < 0)
2872 		goto free_modinfo;
2873 
2874 	err = apply_relocations(mod, info);
2875 	if (err < 0)
2876 		goto free_modinfo;
2877 
2878 	err = post_relocation(mod, info);
2879 	if (err < 0)
2880 		goto free_modinfo;
2881 
2882 	flush_module_icache(mod);
2883 
2884 	/* Now copy in args */
2885 	mod->args = strndup_user(uargs, ~0UL >> 1);
2886 	if (IS_ERR(mod->args)) {
2887 		err = PTR_ERR(mod->args);
2888 		goto free_arch_cleanup;
2889 	}
2890 
2891 	init_build_id(mod, info);
2892 
2893 	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
2894 	ftrace_module_init(mod);
2895 
2896 	/* Finally it's fully formed, ready to start executing. */
2897 	err = complete_formation(mod, info);
2898 	if (err)
2899 		goto ddebug_cleanup;
2900 
2901 	err = prepare_coming_module(mod);
2902 	if (err)
2903 		goto bug_cleanup;
2904 
2905 	mod->async_probe_requested = async_probe;
2906 
2907 	/* Module is ready to execute: parsing args may do that. */
2908 	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
2909 				  -32768, 32767, mod,
2910 				  unknown_module_param_cb);
2911 	if (IS_ERR(after_dashes)) {
2912 		err = PTR_ERR(after_dashes);
2913 		goto coming_cleanup;
2914 	} else if (after_dashes) {
2915 		pr_warn("%s: parameters '%s' after `--' ignored\n",
2916 		       mod->name, after_dashes);
2917 	}
2918 
2919 	/* Link in to sysfs. */
2920 	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
2921 	if (err < 0)
2922 		goto coming_cleanup;
2923 
2924 	if (is_livepatch_module(mod)) {
2925 		err = copy_module_elf(mod, info);
2926 		if (err < 0)
2927 			goto sysfs_cleanup;
2928 	}
2929 
2930 	/* Get rid of temporary copy. */
2931 	free_copy(info, flags);
2932 
2933 	/* Done! */
2934 	trace_module_load(mod);
2935 
2936 	return do_init_module(mod);
2937 
2938  sysfs_cleanup:
2939 	mod_sysfs_teardown(mod);
2940  coming_cleanup:
2941 	mod->state = MODULE_STATE_GOING;
2942 	destroy_params(mod->kp, mod->num_kp);
2943 	blocking_notifier_call_chain(&module_notify_list,
2944 				     MODULE_STATE_GOING, mod);
2945 	klp_module_going(mod);
2946  bug_cleanup:
2947 	mod->state = MODULE_STATE_GOING;
2948 	/* module_bug_cleanup needs module_mutex protection */
2949 	mutex_lock(&module_mutex);
2950 	module_bug_cleanup(mod);
2951 	mutex_unlock(&module_mutex);
2952 
2953  ddebug_cleanup:
2954 	ftrace_release_mod(mod);
2955 	synchronize_rcu();
2956 	kfree(mod->args);
2957  free_arch_cleanup:
2958 	module_arch_cleanup(mod);
2959  free_modinfo:
2960 	free_modinfo(mod);
2961  free_unload:
2962 	module_unload_free(mod);
2963  unlink_mod:
2964 	mutex_lock(&module_mutex);
2965 	/* Unlink carefully: kallsyms could be walking list. */
2966 	list_del_rcu(&mod->list);
2967 	mod_tree_remove(mod);
2968 	wake_up_all(&module_wq);
2969 	/* Wait for RCU-sched synchronizing before releasing mod->list. */
2970 	synchronize_rcu();
2971 	mutex_unlock(&module_mutex);
2972  free_module:
2973 	/* Free lock-classes; relies on the preceding sync_rcu() */
2974 	for_class_mod_mem_type(type, core_data) {
2975 		lockdep_free_key_range(mod->mem[type].base,
2976 				       mod->mem[type].size);
2977 	}
2978 
2979 	module_deallocate(mod, info);
2980  free_copy:
2981 	free_copy(info, flags);
2982 	return err;
2983 }
2984 
2985 SYSCALL_DEFINE3(init_module, void __user *, umod,
2986 		unsigned long, len, const char __user *, uargs)
2987 {
2988 	int err;
2989 	struct load_info info = { };
2990 
2991 	err = may_init_module();
2992 	if (err)
2993 		return err;
2994 
2995 	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
2996 	       umod, len, uargs);
2997 
2998 	err = copy_module_from_user(umod, len, &info);
2999 	if (err)
3000 		return err;
3001 
3002 	return load_module(&info, uargs, 0);
3003 }
3004 
3005 SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3006 {
3007 	struct load_info info = { };
3008 	void *buf = NULL;
3009 	int len;
3010 	int err;
3011 
3012 	err = may_init_module();
3013 	if (err)
3014 		return err;
3015 
3016 	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3017 
3018 	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
3019 		      |MODULE_INIT_IGNORE_VERMAGIC
3020 		      |MODULE_INIT_COMPRESSED_FILE))
3021 		return -EINVAL;
3022 
3023 	len = kernel_read_file_from_fd(fd, 0, &buf, INT_MAX, NULL,
3024 				       READING_MODULE);
3025 	if (len < 0)
3026 		return len;
3027 
3028 	if (flags & MODULE_INIT_COMPRESSED_FILE) {
3029 		err = module_decompress(&info, buf, len);
3030 		vfree(buf); /* compressed data is no longer needed */
3031 		if (err)
3032 			return err;
3033 	} else {
3034 		info.hdr = buf;
3035 		info.len = len;
3036 	}
3037 
3038 	return load_module(&info, uargs, flags);
3039 }
3040 
3041 /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
3042 char *module_flags(struct module *mod, char *buf, bool show_state)
3043 {
3044 	int bx = 0;
3045 
3046 	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
3047 	if (!mod->taints && !show_state)
3048 		goto out;
3049 	if (mod->taints ||
3050 	    mod->state == MODULE_STATE_GOING ||
3051 	    mod->state == MODULE_STATE_COMING) {
3052 		buf[bx++] = '(';
3053 		bx += module_flags_taint(mod->taints, buf + bx);
3054 		/* Show a - for module-is-being-unloaded */
3055 		if (mod->state == MODULE_STATE_GOING && show_state)
3056 			buf[bx++] = '-';
3057 		/* Show a + for module-is-being-loaded */
3058 		if (mod->state == MODULE_STATE_COMING && show_state)
3059 			buf[bx++] = '+';
3060 		buf[bx++] = ')';
3061 	}
3062 out:
3063 	buf[bx] = '\0';
3064 
3065 	return buf;
3066 }
3067 
3068 /* Given an address, look for it in the module exception tables. */
3069 const struct exception_table_entry *search_module_extables(unsigned long addr)
3070 {
3071 	const struct exception_table_entry *e = NULL;
3072 	struct module *mod;
3073 
3074 	preempt_disable();
3075 	mod = __module_address(addr);
3076 	if (!mod)
3077 		goto out;
3078 
3079 	if (!mod->num_exentries)
3080 		goto out;
3081 
3082 	e = search_extable(mod->extable,
3083 			   mod->num_exentries,
3084 			   addr);
3085 out:
3086 	preempt_enable();
3087 
3088 	/*
3089 	 * Now, if we found one, we are running inside it now, hence
3090 	 * we cannot unload the module, hence no refcnt needed.
3091 	 */
3092 	return e;
3093 }
3094 
3095 /**
3096  * is_module_address() - is this address inside a module?
3097  * @addr: the address to check.
3098  *
3099  * See is_module_text_address() if you simply want to see if the address
3100  * is code (not data).
3101  */
3102 bool is_module_address(unsigned long addr)
3103 {
3104 	bool ret;
3105 
3106 	preempt_disable();
3107 	ret = __module_address(addr) != NULL;
3108 	preempt_enable();
3109 
3110 	return ret;
3111 }
3112 
3113 /**
3114  * __module_address() - get the module which contains an address.
3115  * @addr: the address.
3116  *
3117  * Must be called with preempt disabled or module mutex held so that
3118  * module doesn't get freed during this.
3119  */
3120 struct module *__module_address(unsigned long addr)
3121 {
3122 	struct module *mod;
3123 
3124 	if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max)
3125 		goto lookup;
3126 
3127 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
3128 	if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max)
3129 		goto lookup;
3130 #endif
3131 
3132 	return NULL;
3133 
3134 lookup:
3135 	module_assert_mutex_or_preempt();
3136 
3137 	mod = mod_find(addr, &mod_tree);
3138 	if (mod) {
3139 		BUG_ON(!within_module(addr, mod));
3140 		if (mod->state == MODULE_STATE_UNFORMED)
3141 			mod = NULL;
3142 	}
3143 	return mod;
3144 }
3145 
3146 /**
3147  * is_module_text_address() - is this address inside module code?
3148  * @addr: the address to check.
3149  *
3150  * See is_module_address() if you simply want to see if the address is
3151  * anywhere in a module.  See kernel_text_address() for testing if an
3152  * address corresponds to kernel or module code.
3153  */
3154 bool is_module_text_address(unsigned long addr)
3155 {
3156 	bool ret;
3157 
3158 	preempt_disable();
3159 	ret = __module_text_address(addr) != NULL;
3160 	preempt_enable();
3161 
3162 	return ret;
3163 }
3164 
3165 /**
3166  * __module_text_address() - get the module whose code contains an address.
3167  * @addr: the address.
3168  *
3169  * Must be called with preempt disabled or module mutex held so that
3170  * module doesn't get freed during this.
3171  */
3172 struct module *__module_text_address(unsigned long addr)
3173 {
3174 	struct module *mod = __module_address(addr);
3175 	if (mod) {
3176 		/* Make sure it's within the text section. */
3177 		if (!within_module_mem_type(addr, mod, MOD_TEXT) &&
3178 		    !within_module_mem_type(addr, mod, MOD_INIT_TEXT))
3179 			mod = NULL;
3180 	}
3181 	return mod;
3182 }
3183 
3184 /* Don't grab lock, we're oopsing. */
3185 void print_modules(void)
3186 {
3187 	struct module *mod;
3188 	char buf[MODULE_FLAGS_BUF_SIZE];
3189 
3190 	printk(KERN_DEFAULT "Modules linked in:");
3191 	/* Most callers should already have preempt disabled, but make sure */
3192 	preempt_disable();
3193 	list_for_each_entry_rcu(mod, &modules, list) {
3194 		if (mod->state == MODULE_STATE_UNFORMED)
3195 			continue;
3196 		pr_cont(" %s%s", mod->name, module_flags(mod, buf, true));
3197 	}
3198 
3199 	print_unloaded_tainted_modules();
3200 	preempt_enable();
3201 	if (last_unloaded_module.name[0])
3202 		pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name,
3203 			last_unloaded_module.taints);
3204 	pr_cont("\n");
3205 }
3206