1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2002 Richard Henderson 4 * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM. 5 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org> 6 */ 7 8 #define INCLUDE_VERMAGIC 9 10 #include <linux/export.h> 11 #include <linux/extable.h> 12 #include <linux/moduleloader.h> 13 #include <linux/module_signature.h> 14 #include <linux/trace_events.h> 15 #include <linux/init.h> 16 #include <linux/kallsyms.h> 17 #include <linux/buildid.h> 18 #include <linux/fs.h> 19 #include <linux/kernel.h> 20 #include <linux/kernel_read_file.h> 21 #include <linux/kstrtox.h> 22 #include <linux/slab.h> 23 #include <linux/vmalloc.h> 24 #include <linux/elf.h> 25 #include <linux/seq_file.h> 26 #include <linux/syscalls.h> 27 #include <linux/fcntl.h> 28 #include <linux/rcupdate.h> 29 #include <linux/capability.h> 30 #include <linux/cpu.h> 31 #include <linux/moduleparam.h> 32 #include <linux/errno.h> 33 #include <linux/err.h> 34 #include <linux/vermagic.h> 35 #include <linux/notifier.h> 36 #include <linux/sched.h> 37 #include <linux/device.h> 38 #include <linux/string.h> 39 #include <linux/mutex.h> 40 #include <linux/rculist.h> 41 #include <linux/uaccess.h> 42 #include <asm/cacheflush.h> 43 #include <linux/set_memory.h> 44 #include <asm/mmu_context.h> 45 #include <linux/license.h> 46 #include <asm/sections.h> 47 #include <linux/tracepoint.h> 48 #include <linux/ftrace.h> 49 #include <linux/livepatch.h> 50 #include <linux/async.h> 51 #include <linux/percpu.h> 52 #include <linux/kmemleak.h> 53 #include <linux/jump_label.h> 54 #include <linux/pfn.h> 55 #include <linux/bsearch.h> 56 #include <linux/dynamic_debug.h> 57 #include <linux/audit.h> 58 #include <linux/cfi.h> 59 #include <uapi/linux/module.h> 60 #include "internal.h" 61 62 #define CREATE_TRACE_POINTS 63 #include <trace/events/module.h> 64 65 /* 66 * Mutex protects: 67 * 1) List of modules (also safely readable with preempt_disable), 68 * 2) module_use links, 69 * 3) mod_tree.addr_min/mod_tree.addr_max. 70 * (delete and add uses RCU list operations). 71 */ 72 DEFINE_MUTEX(module_mutex); 73 LIST_HEAD(modules); 74 75 /* Work queue for freeing init sections in success case */ 76 static void do_free_init(struct work_struct *w); 77 static DECLARE_WORK(init_free_wq, do_free_init); 78 static LLIST_HEAD(init_free_list); 79 80 struct mod_tree_root mod_tree __cacheline_aligned = { 81 .addr_min = -1UL, 82 }; 83 84 struct symsearch { 85 const struct kernel_symbol *start, *stop; 86 const s32 *crcs; 87 enum mod_license license; 88 }; 89 90 /* 91 * Bounds of module memory, for speeding up __module_address. 92 * Protected by module_mutex. 93 */ 94 static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base, 95 unsigned int size, struct mod_tree_root *tree) 96 { 97 unsigned long min = (unsigned long)base; 98 unsigned long max = min + size; 99 100 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 101 if (mod_mem_type_is_core_data(type)) { 102 if (min < tree->data_addr_min) 103 tree->data_addr_min = min; 104 if (max > tree->data_addr_max) 105 tree->data_addr_max = max; 106 return; 107 } 108 #endif 109 if (min < tree->addr_min) 110 tree->addr_min = min; 111 if (max > tree->addr_max) 112 tree->addr_max = max; 113 } 114 115 static void mod_update_bounds(struct module *mod) 116 { 117 for_each_mod_mem_type(type) { 118 struct module_memory *mod_mem = &mod->mem[type]; 119 120 if (mod_mem->size) 121 __mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree); 122 } 123 } 124 125 /* Block module loading/unloading? */ 126 int modules_disabled; 127 core_param(nomodule, modules_disabled, bint, 0); 128 129 /* Waiting for a module to finish initializing? */ 130 static DECLARE_WAIT_QUEUE_HEAD(module_wq); 131 132 static BLOCKING_NOTIFIER_HEAD(module_notify_list); 133 134 int register_module_notifier(struct notifier_block *nb) 135 { 136 return blocking_notifier_chain_register(&module_notify_list, nb); 137 } 138 EXPORT_SYMBOL(register_module_notifier); 139 140 int unregister_module_notifier(struct notifier_block *nb) 141 { 142 return blocking_notifier_chain_unregister(&module_notify_list, nb); 143 } 144 EXPORT_SYMBOL(unregister_module_notifier); 145 146 /* 147 * We require a truly strong try_module_get(): 0 means success. 148 * Otherwise an error is returned due to ongoing or failed 149 * initialization etc. 150 */ 151 static inline int strong_try_module_get(struct module *mod) 152 { 153 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED); 154 if (mod && mod->state == MODULE_STATE_COMING) 155 return -EBUSY; 156 if (try_module_get(mod)) 157 return 0; 158 else 159 return -ENOENT; 160 } 161 162 static inline void add_taint_module(struct module *mod, unsigned flag, 163 enum lockdep_ok lockdep_ok) 164 { 165 add_taint(flag, lockdep_ok); 166 set_bit(flag, &mod->taints); 167 } 168 169 /* 170 * A thread that wants to hold a reference to a module only while it 171 * is running can call this to safely exit. 172 */ 173 void __noreturn __module_put_and_kthread_exit(struct module *mod, long code) 174 { 175 module_put(mod); 176 kthread_exit(code); 177 } 178 EXPORT_SYMBOL(__module_put_and_kthread_exit); 179 180 /* Find a module section: 0 means not found. */ 181 static unsigned int find_sec(const struct load_info *info, const char *name) 182 { 183 unsigned int i; 184 185 for (i = 1; i < info->hdr->e_shnum; i++) { 186 Elf_Shdr *shdr = &info->sechdrs[i]; 187 /* Alloc bit cleared means "ignore it." */ 188 if ((shdr->sh_flags & SHF_ALLOC) 189 && strcmp(info->secstrings + shdr->sh_name, name) == 0) 190 return i; 191 } 192 return 0; 193 } 194 195 /* Find a module section, or NULL. */ 196 static void *section_addr(const struct load_info *info, const char *name) 197 { 198 /* Section 0 has sh_addr 0. */ 199 return (void *)info->sechdrs[find_sec(info, name)].sh_addr; 200 } 201 202 /* Find a module section, or NULL. Fill in number of "objects" in section. */ 203 static void *section_objs(const struct load_info *info, 204 const char *name, 205 size_t object_size, 206 unsigned int *num) 207 { 208 unsigned int sec = find_sec(info, name); 209 210 /* Section 0 has sh_addr 0 and sh_size 0. */ 211 *num = info->sechdrs[sec].sh_size / object_size; 212 return (void *)info->sechdrs[sec].sh_addr; 213 } 214 215 /* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */ 216 static unsigned int find_any_sec(const struct load_info *info, const char *name) 217 { 218 unsigned int i; 219 220 for (i = 1; i < info->hdr->e_shnum; i++) { 221 Elf_Shdr *shdr = &info->sechdrs[i]; 222 if (strcmp(info->secstrings + shdr->sh_name, name) == 0) 223 return i; 224 } 225 return 0; 226 } 227 228 /* 229 * Find a module section, or NULL. Fill in number of "objects" in section. 230 * Ignores SHF_ALLOC flag. 231 */ 232 static __maybe_unused void *any_section_objs(const struct load_info *info, 233 const char *name, 234 size_t object_size, 235 unsigned int *num) 236 { 237 unsigned int sec = find_any_sec(info, name); 238 239 /* Section 0 has sh_addr 0 and sh_size 0. */ 240 *num = info->sechdrs[sec].sh_size / object_size; 241 return (void *)info->sechdrs[sec].sh_addr; 242 } 243 244 #ifndef CONFIG_MODVERSIONS 245 #define symversion(base, idx) NULL 246 #else 247 #define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL) 248 #endif 249 250 static const char *kernel_symbol_name(const struct kernel_symbol *sym) 251 { 252 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 253 return offset_to_ptr(&sym->name_offset); 254 #else 255 return sym->name; 256 #endif 257 } 258 259 static const char *kernel_symbol_namespace(const struct kernel_symbol *sym) 260 { 261 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 262 if (!sym->namespace_offset) 263 return NULL; 264 return offset_to_ptr(&sym->namespace_offset); 265 #else 266 return sym->namespace; 267 #endif 268 } 269 270 int cmp_name(const void *name, const void *sym) 271 { 272 return strcmp(name, kernel_symbol_name(sym)); 273 } 274 275 static bool find_exported_symbol_in_section(const struct symsearch *syms, 276 struct module *owner, 277 struct find_symbol_arg *fsa) 278 { 279 struct kernel_symbol *sym; 280 281 if (!fsa->gplok && syms->license == GPL_ONLY) 282 return false; 283 284 sym = bsearch(fsa->name, syms->start, syms->stop - syms->start, 285 sizeof(struct kernel_symbol), cmp_name); 286 if (!sym) 287 return false; 288 289 fsa->owner = owner; 290 fsa->crc = symversion(syms->crcs, sym - syms->start); 291 fsa->sym = sym; 292 fsa->license = syms->license; 293 294 return true; 295 } 296 297 /* 298 * Find an exported symbol and return it, along with, (optional) crc and 299 * (optional) module which owns it. Needs preempt disabled or module_mutex. 300 */ 301 bool find_symbol(struct find_symbol_arg *fsa) 302 { 303 static const struct symsearch arr[] = { 304 { __start___ksymtab, __stop___ksymtab, __start___kcrctab, 305 NOT_GPL_ONLY }, 306 { __start___ksymtab_gpl, __stop___ksymtab_gpl, 307 __start___kcrctab_gpl, 308 GPL_ONLY }, 309 }; 310 struct module *mod; 311 unsigned int i; 312 313 module_assert_mutex_or_preempt(); 314 315 for (i = 0; i < ARRAY_SIZE(arr); i++) 316 if (find_exported_symbol_in_section(&arr[i], NULL, fsa)) 317 return true; 318 319 list_for_each_entry_rcu(mod, &modules, list, 320 lockdep_is_held(&module_mutex)) { 321 struct symsearch arr[] = { 322 { mod->syms, mod->syms + mod->num_syms, mod->crcs, 323 NOT_GPL_ONLY }, 324 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, 325 mod->gpl_crcs, 326 GPL_ONLY }, 327 }; 328 329 if (mod->state == MODULE_STATE_UNFORMED) 330 continue; 331 332 for (i = 0; i < ARRAY_SIZE(arr); i++) 333 if (find_exported_symbol_in_section(&arr[i], mod, fsa)) 334 return true; 335 } 336 337 pr_debug("Failed to find symbol %s\n", fsa->name); 338 return false; 339 } 340 341 /* 342 * Search for module by name: must hold module_mutex (or preempt disabled 343 * for read-only access). 344 */ 345 struct module *find_module_all(const char *name, size_t len, 346 bool even_unformed) 347 { 348 struct module *mod; 349 350 module_assert_mutex_or_preempt(); 351 352 list_for_each_entry_rcu(mod, &modules, list, 353 lockdep_is_held(&module_mutex)) { 354 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) 355 continue; 356 if (strlen(mod->name) == len && !memcmp(mod->name, name, len)) 357 return mod; 358 } 359 return NULL; 360 } 361 362 struct module *find_module(const char *name) 363 { 364 return find_module_all(name, strlen(name), false); 365 } 366 367 #ifdef CONFIG_SMP 368 369 static inline void __percpu *mod_percpu(struct module *mod) 370 { 371 return mod->percpu; 372 } 373 374 static int percpu_modalloc(struct module *mod, struct load_info *info) 375 { 376 Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu]; 377 unsigned long align = pcpusec->sh_addralign; 378 379 if (!pcpusec->sh_size) 380 return 0; 381 382 if (align > PAGE_SIZE) { 383 pr_warn("%s: per-cpu alignment %li > %li\n", 384 mod->name, align, PAGE_SIZE); 385 align = PAGE_SIZE; 386 } 387 388 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align); 389 if (!mod->percpu) { 390 pr_warn("%s: Could not allocate %lu bytes percpu data\n", 391 mod->name, (unsigned long)pcpusec->sh_size); 392 return -ENOMEM; 393 } 394 mod->percpu_size = pcpusec->sh_size; 395 return 0; 396 } 397 398 static void percpu_modfree(struct module *mod) 399 { 400 free_percpu(mod->percpu); 401 } 402 403 static unsigned int find_pcpusec(struct load_info *info) 404 { 405 return find_sec(info, ".data..percpu"); 406 } 407 408 static void percpu_modcopy(struct module *mod, 409 const void *from, unsigned long size) 410 { 411 int cpu; 412 413 for_each_possible_cpu(cpu) 414 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); 415 } 416 417 bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr) 418 { 419 struct module *mod; 420 unsigned int cpu; 421 422 preempt_disable(); 423 424 list_for_each_entry_rcu(mod, &modules, list) { 425 if (mod->state == MODULE_STATE_UNFORMED) 426 continue; 427 if (!mod->percpu_size) 428 continue; 429 for_each_possible_cpu(cpu) { 430 void *start = per_cpu_ptr(mod->percpu, cpu); 431 void *va = (void *)addr; 432 433 if (va >= start && va < start + mod->percpu_size) { 434 if (can_addr) { 435 *can_addr = (unsigned long) (va - start); 436 *can_addr += (unsigned long) 437 per_cpu_ptr(mod->percpu, 438 get_boot_cpu_id()); 439 } 440 preempt_enable(); 441 return true; 442 } 443 } 444 } 445 446 preempt_enable(); 447 return false; 448 } 449 450 /** 451 * is_module_percpu_address() - test whether address is from module static percpu 452 * @addr: address to test 453 * 454 * Test whether @addr belongs to module static percpu area. 455 * 456 * Return: %true if @addr is from module static percpu area 457 */ 458 bool is_module_percpu_address(unsigned long addr) 459 { 460 return __is_module_percpu_address(addr, NULL); 461 } 462 463 #else /* ... !CONFIG_SMP */ 464 465 static inline void __percpu *mod_percpu(struct module *mod) 466 { 467 return NULL; 468 } 469 static int percpu_modalloc(struct module *mod, struct load_info *info) 470 { 471 /* UP modules shouldn't have this section: ENOMEM isn't quite right */ 472 if (info->sechdrs[info->index.pcpu].sh_size != 0) 473 return -ENOMEM; 474 return 0; 475 } 476 static inline void percpu_modfree(struct module *mod) 477 { 478 } 479 static unsigned int find_pcpusec(struct load_info *info) 480 { 481 return 0; 482 } 483 static inline void percpu_modcopy(struct module *mod, 484 const void *from, unsigned long size) 485 { 486 /* pcpusec should be 0, and size of that section should be 0. */ 487 BUG_ON(size != 0); 488 } 489 bool is_module_percpu_address(unsigned long addr) 490 { 491 return false; 492 } 493 494 bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr) 495 { 496 return false; 497 } 498 499 #endif /* CONFIG_SMP */ 500 501 #define MODINFO_ATTR(field) \ 502 static void setup_modinfo_##field(struct module *mod, const char *s) \ 503 { \ 504 mod->field = kstrdup(s, GFP_KERNEL); \ 505 } \ 506 static ssize_t show_modinfo_##field(struct module_attribute *mattr, \ 507 struct module_kobject *mk, char *buffer) \ 508 { \ 509 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \ 510 } \ 511 static int modinfo_##field##_exists(struct module *mod) \ 512 { \ 513 return mod->field != NULL; \ 514 } \ 515 static void free_modinfo_##field(struct module *mod) \ 516 { \ 517 kfree(mod->field); \ 518 mod->field = NULL; \ 519 } \ 520 static struct module_attribute modinfo_##field = { \ 521 .attr = { .name = __stringify(field), .mode = 0444 }, \ 522 .show = show_modinfo_##field, \ 523 .setup = setup_modinfo_##field, \ 524 .test = modinfo_##field##_exists, \ 525 .free = free_modinfo_##field, \ 526 }; 527 528 MODINFO_ATTR(version); 529 MODINFO_ATTR(srcversion); 530 531 static struct { 532 char name[MODULE_NAME_LEN + 1]; 533 char taints[MODULE_FLAGS_BUF_SIZE]; 534 } last_unloaded_module; 535 536 #ifdef CONFIG_MODULE_UNLOAD 537 538 EXPORT_TRACEPOINT_SYMBOL(module_get); 539 540 /* MODULE_REF_BASE is the base reference count by kmodule loader. */ 541 #define MODULE_REF_BASE 1 542 543 /* Init the unload section of the module. */ 544 static int module_unload_init(struct module *mod) 545 { 546 /* 547 * Initialize reference counter to MODULE_REF_BASE. 548 * refcnt == 0 means module is going. 549 */ 550 atomic_set(&mod->refcnt, MODULE_REF_BASE); 551 552 INIT_LIST_HEAD(&mod->source_list); 553 INIT_LIST_HEAD(&mod->target_list); 554 555 /* Hold reference count during initialization. */ 556 atomic_inc(&mod->refcnt); 557 558 return 0; 559 } 560 561 /* Does a already use b? */ 562 static int already_uses(struct module *a, struct module *b) 563 { 564 struct module_use *use; 565 566 list_for_each_entry(use, &b->source_list, source_list) { 567 if (use->source == a) 568 return 1; 569 } 570 pr_debug("%s does not use %s!\n", a->name, b->name); 571 return 0; 572 } 573 574 /* 575 * Module a uses b 576 * - we add 'a' as a "source", 'b' as a "target" of module use 577 * - the module_use is added to the list of 'b' sources (so 578 * 'b' can walk the list to see who sourced them), and of 'a' 579 * targets (so 'a' can see what modules it targets). 580 */ 581 static int add_module_usage(struct module *a, struct module *b) 582 { 583 struct module_use *use; 584 585 pr_debug("Allocating new usage for %s.\n", a->name); 586 use = kmalloc(sizeof(*use), GFP_ATOMIC); 587 if (!use) 588 return -ENOMEM; 589 590 use->source = a; 591 use->target = b; 592 list_add(&use->source_list, &b->source_list); 593 list_add(&use->target_list, &a->target_list); 594 return 0; 595 } 596 597 /* Module a uses b: caller needs module_mutex() */ 598 static int ref_module(struct module *a, struct module *b) 599 { 600 int err; 601 602 if (b == NULL || already_uses(a, b)) 603 return 0; 604 605 /* If module isn't available, we fail. */ 606 err = strong_try_module_get(b); 607 if (err) 608 return err; 609 610 err = add_module_usage(a, b); 611 if (err) { 612 module_put(b); 613 return err; 614 } 615 return 0; 616 } 617 618 /* Clear the unload stuff of the module. */ 619 static void module_unload_free(struct module *mod) 620 { 621 struct module_use *use, *tmp; 622 623 mutex_lock(&module_mutex); 624 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) { 625 struct module *i = use->target; 626 pr_debug("%s unusing %s\n", mod->name, i->name); 627 module_put(i); 628 list_del(&use->source_list); 629 list_del(&use->target_list); 630 kfree(use); 631 } 632 mutex_unlock(&module_mutex); 633 } 634 635 #ifdef CONFIG_MODULE_FORCE_UNLOAD 636 static inline int try_force_unload(unsigned int flags) 637 { 638 int ret = (flags & O_TRUNC); 639 if (ret) 640 add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE); 641 return ret; 642 } 643 #else 644 static inline int try_force_unload(unsigned int flags) 645 { 646 return 0; 647 } 648 #endif /* CONFIG_MODULE_FORCE_UNLOAD */ 649 650 /* Try to release refcount of module, 0 means success. */ 651 static int try_release_module_ref(struct module *mod) 652 { 653 int ret; 654 655 /* Try to decrement refcnt which we set at loading */ 656 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt); 657 BUG_ON(ret < 0); 658 if (ret) 659 /* Someone can put this right now, recover with checking */ 660 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0); 661 662 return ret; 663 } 664 665 static int try_stop_module(struct module *mod, int flags, int *forced) 666 { 667 /* If it's not unused, quit unless we're forcing. */ 668 if (try_release_module_ref(mod) != 0) { 669 *forced = try_force_unload(flags); 670 if (!(*forced)) 671 return -EWOULDBLOCK; 672 } 673 674 /* Mark it as dying. */ 675 mod->state = MODULE_STATE_GOING; 676 677 return 0; 678 } 679 680 /** 681 * module_refcount() - return the refcount or -1 if unloading 682 * @mod: the module we're checking 683 * 684 * Return: 685 * -1 if the module is in the process of unloading 686 * otherwise the number of references in the kernel to the module 687 */ 688 int module_refcount(struct module *mod) 689 { 690 return atomic_read(&mod->refcnt) - MODULE_REF_BASE; 691 } 692 EXPORT_SYMBOL(module_refcount); 693 694 /* This exists whether we can unload or not */ 695 static void free_module(struct module *mod); 696 697 SYSCALL_DEFINE2(delete_module, const char __user *, name_user, 698 unsigned int, flags) 699 { 700 struct module *mod; 701 char name[MODULE_NAME_LEN]; 702 char buf[MODULE_FLAGS_BUF_SIZE]; 703 int ret, forced = 0; 704 705 if (!capable(CAP_SYS_MODULE) || modules_disabled) 706 return -EPERM; 707 708 if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0) 709 return -EFAULT; 710 name[MODULE_NAME_LEN-1] = '\0'; 711 712 audit_log_kern_module(name); 713 714 if (mutex_lock_interruptible(&module_mutex) != 0) 715 return -EINTR; 716 717 mod = find_module(name); 718 if (!mod) { 719 ret = -ENOENT; 720 goto out; 721 } 722 723 if (!list_empty(&mod->source_list)) { 724 /* Other modules depend on us: get rid of them first. */ 725 ret = -EWOULDBLOCK; 726 goto out; 727 } 728 729 /* Doing init or already dying? */ 730 if (mod->state != MODULE_STATE_LIVE) { 731 /* FIXME: if (force), slam module count damn the torpedoes */ 732 pr_debug("%s already dying\n", mod->name); 733 ret = -EBUSY; 734 goto out; 735 } 736 737 /* If it has an init func, it must have an exit func to unload */ 738 if (mod->init && !mod->exit) { 739 forced = try_force_unload(flags); 740 if (!forced) { 741 /* This module can't be removed */ 742 ret = -EBUSY; 743 goto out; 744 } 745 } 746 747 ret = try_stop_module(mod, flags, &forced); 748 if (ret != 0) 749 goto out; 750 751 mutex_unlock(&module_mutex); 752 /* Final destruction now no one is using it. */ 753 if (mod->exit != NULL) 754 mod->exit(); 755 blocking_notifier_call_chain(&module_notify_list, 756 MODULE_STATE_GOING, mod); 757 klp_module_going(mod); 758 ftrace_release_mod(mod); 759 760 async_synchronize_full(); 761 762 /* Store the name and taints of the last unloaded module for diagnostic purposes */ 763 strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name)); 764 strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints)); 765 766 free_module(mod); 767 /* someone could wait for the module in add_unformed_module() */ 768 wake_up_all(&module_wq); 769 return 0; 770 out: 771 mutex_unlock(&module_mutex); 772 return ret; 773 } 774 775 void __symbol_put(const char *symbol) 776 { 777 struct find_symbol_arg fsa = { 778 .name = symbol, 779 .gplok = true, 780 }; 781 782 preempt_disable(); 783 BUG_ON(!find_symbol(&fsa)); 784 module_put(fsa.owner); 785 preempt_enable(); 786 } 787 EXPORT_SYMBOL(__symbol_put); 788 789 /* Note this assumes addr is a function, which it currently always is. */ 790 void symbol_put_addr(void *addr) 791 { 792 struct module *modaddr; 793 unsigned long a = (unsigned long)dereference_function_descriptor(addr); 794 795 if (core_kernel_text(a)) 796 return; 797 798 /* 799 * Even though we hold a reference on the module; we still need to 800 * disable preemption in order to safely traverse the data structure. 801 */ 802 preempt_disable(); 803 modaddr = __module_text_address(a); 804 BUG_ON(!modaddr); 805 module_put(modaddr); 806 preempt_enable(); 807 } 808 EXPORT_SYMBOL_GPL(symbol_put_addr); 809 810 static ssize_t show_refcnt(struct module_attribute *mattr, 811 struct module_kobject *mk, char *buffer) 812 { 813 return sprintf(buffer, "%i\n", module_refcount(mk->mod)); 814 } 815 816 static struct module_attribute modinfo_refcnt = 817 __ATTR(refcnt, 0444, show_refcnt, NULL); 818 819 void __module_get(struct module *module) 820 { 821 if (module) { 822 preempt_disable(); 823 atomic_inc(&module->refcnt); 824 trace_module_get(module, _RET_IP_); 825 preempt_enable(); 826 } 827 } 828 EXPORT_SYMBOL(__module_get); 829 830 bool try_module_get(struct module *module) 831 { 832 bool ret = true; 833 834 if (module) { 835 preempt_disable(); 836 /* Note: here, we can fail to get a reference */ 837 if (likely(module_is_live(module) && 838 atomic_inc_not_zero(&module->refcnt) != 0)) 839 trace_module_get(module, _RET_IP_); 840 else 841 ret = false; 842 843 preempt_enable(); 844 } 845 return ret; 846 } 847 EXPORT_SYMBOL(try_module_get); 848 849 void module_put(struct module *module) 850 { 851 int ret; 852 853 if (module) { 854 preempt_disable(); 855 ret = atomic_dec_if_positive(&module->refcnt); 856 WARN_ON(ret < 0); /* Failed to put refcount */ 857 trace_module_put(module, _RET_IP_); 858 preempt_enable(); 859 } 860 } 861 EXPORT_SYMBOL(module_put); 862 863 #else /* !CONFIG_MODULE_UNLOAD */ 864 static inline void module_unload_free(struct module *mod) 865 { 866 } 867 868 static int ref_module(struct module *a, struct module *b) 869 { 870 return strong_try_module_get(b); 871 } 872 873 static inline int module_unload_init(struct module *mod) 874 { 875 return 0; 876 } 877 #endif /* CONFIG_MODULE_UNLOAD */ 878 879 size_t module_flags_taint(unsigned long taints, char *buf) 880 { 881 size_t l = 0; 882 int i; 883 884 for (i = 0; i < TAINT_FLAGS_COUNT; i++) { 885 if (taint_flags[i].module && test_bit(i, &taints)) 886 buf[l++] = taint_flags[i].c_true; 887 } 888 889 return l; 890 } 891 892 static ssize_t show_initstate(struct module_attribute *mattr, 893 struct module_kobject *mk, char *buffer) 894 { 895 const char *state = "unknown"; 896 897 switch (mk->mod->state) { 898 case MODULE_STATE_LIVE: 899 state = "live"; 900 break; 901 case MODULE_STATE_COMING: 902 state = "coming"; 903 break; 904 case MODULE_STATE_GOING: 905 state = "going"; 906 break; 907 default: 908 BUG(); 909 } 910 return sprintf(buffer, "%s\n", state); 911 } 912 913 static struct module_attribute modinfo_initstate = 914 __ATTR(initstate, 0444, show_initstate, NULL); 915 916 static ssize_t store_uevent(struct module_attribute *mattr, 917 struct module_kobject *mk, 918 const char *buffer, size_t count) 919 { 920 int rc; 921 922 rc = kobject_synth_uevent(&mk->kobj, buffer, count); 923 return rc ? rc : count; 924 } 925 926 struct module_attribute module_uevent = 927 __ATTR(uevent, 0200, NULL, store_uevent); 928 929 static ssize_t show_coresize(struct module_attribute *mattr, 930 struct module_kobject *mk, char *buffer) 931 { 932 unsigned int size = mk->mod->mem[MOD_TEXT].size; 933 934 if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) { 935 for_class_mod_mem_type(type, core_data) 936 size += mk->mod->mem[type].size; 937 } 938 return sprintf(buffer, "%u\n", size); 939 } 940 941 static struct module_attribute modinfo_coresize = 942 __ATTR(coresize, 0444, show_coresize, NULL); 943 944 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 945 static ssize_t show_datasize(struct module_attribute *mattr, 946 struct module_kobject *mk, char *buffer) 947 { 948 unsigned int size = 0; 949 950 for_class_mod_mem_type(type, core_data) 951 size += mk->mod->mem[type].size; 952 return sprintf(buffer, "%u\n", size); 953 } 954 955 static struct module_attribute modinfo_datasize = 956 __ATTR(datasize, 0444, show_datasize, NULL); 957 #endif 958 959 static ssize_t show_initsize(struct module_attribute *mattr, 960 struct module_kobject *mk, char *buffer) 961 { 962 unsigned int size = 0; 963 964 for_class_mod_mem_type(type, init) 965 size += mk->mod->mem[type].size; 966 return sprintf(buffer, "%u\n", size); 967 } 968 969 static struct module_attribute modinfo_initsize = 970 __ATTR(initsize, 0444, show_initsize, NULL); 971 972 static ssize_t show_taint(struct module_attribute *mattr, 973 struct module_kobject *mk, char *buffer) 974 { 975 size_t l; 976 977 l = module_flags_taint(mk->mod->taints, buffer); 978 buffer[l++] = '\n'; 979 return l; 980 } 981 982 static struct module_attribute modinfo_taint = 983 __ATTR(taint, 0444, show_taint, NULL); 984 985 struct module_attribute *modinfo_attrs[] = { 986 &module_uevent, 987 &modinfo_version, 988 &modinfo_srcversion, 989 &modinfo_initstate, 990 &modinfo_coresize, 991 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 992 &modinfo_datasize, 993 #endif 994 &modinfo_initsize, 995 &modinfo_taint, 996 #ifdef CONFIG_MODULE_UNLOAD 997 &modinfo_refcnt, 998 #endif 999 NULL, 1000 }; 1001 1002 size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs); 1003 1004 static const char vermagic[] = VERMAGIC_STRING; 1005 1006 int try_to_force_load(struct module *mod, const char *reason) 1007 { 1008 #ifdef CONFIG_MODULE_FORCE_LOAD 1009 if (!test_taint(TAINT_FORCED_MODULE)) 1010 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason); 1011 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE); 1012 return 0; 1013 #else 1014 return -ENOEXEC; 1015 #endif 1016 } 1017 1018 /* Parse tag=value strings from .modinfo section */ 1019 char *module_next_tag_pair(char *string, unsigned long *secsize) 1020 { 1021 /* Skip non-zero chars */ 1022 while (string[0]) { 1023 string++; 1024 if ((*secsize)-- <= 1) 1025 return NULL; 1026 } 1027 1028 /* Skip any zero padding. */ 1029 while (!string[0]) { 1030 string++; 1031 if ((*secsize)-- <= 1) 1032 return NULL; 1033 } 1034 return string; 1035 } 1036 1037 static char *get_next_modinfo(const struct load_info *info, const char *tag, 1038 char *prev) 1039 { 1040 char *p; 1041 unsigned int taglen = strlen(tag); 1042 Elf_Shdr *infosec = &info->sechdrs[info->index.info]; 1043 unsigned long size = infosec->sh_size; 1044 1045 /* 1046 * get_modinfo() calls made before rewrite_section_headers() 1047 * must use sh_offset, as sh_addr isn't set! 1048 */ 1049 char *modinfo = (char *)info->hdr + infosec->sh_offset; 1050 1051 if (prev) { 1052 size -= prev - modinfo; 1053 modinfo = module_next_tag_pair(prev, &size); 1054 } 1055 1056 for (p = modinfo; p; p = module_next_tag_pair(p, &size)) { 1057 if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=') 1058 return p + taglen + 1; 1059 } 1060 return NULL; 1061 } 1062 1063 static char *get_modinfo(const struct load_info *info, const char *tag) 1064 { 1065 return get_next_modinfo(info, tag, NULL); 1066 } 1067 1068 static int verify_namespace_is_imported(const struct load_info *info, 1069 const struct kernel_symbol *sym, 1070 struct module *mod) 1071 { 1072 const char *namespace; 1073 char *imported_namespace; 1074 1075 namespace = kernel_symbol_namespace(sym); 1076 if (namespace && namespace[0]) { 1077 for_each_modinfo_entry(imported_namespace, info, "import_ns") { 1078 if (strcmp(namespace, imported_namespace) == 0) 1079 return 0; 1080 } 1081 #ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS 1082 pr_warn( 1083 #else 1084 pr_err( 1085 #endif 1086 "%s: module uses symbol (%s) from namespace %s, but does not import it.\n", 1087 mod->name, kernel_symbol_name(sym), namespace); 1088 #ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS 1089 return -EINVAL; 1090 #endif 1091 } 1092 return 0; 1093 } 1094 1095 static bool inherit_taint(struct module *mod, struct module *owner, const char *name) 1096 { 1097 if (!owner || !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints)) 1098 return true; 1099 1100 if (mod->using_gplonly_symbols) { 1101 pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n", 1102 mod->name, name, owner->name); 1103 return false; 1104 } 1105 1106 if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) { 1107 pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n", 1108 mod->name, name, owner->name); 1109 set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints); 1110 } 1111 return true; 1112 } 1113 1114 /* Resolve a symbol for this module. I.e. if we find one, record usage. */ 1115 static const struct kernel_symbol *resolve_symbol(struct module *mod, 1116 const struct load_info *info, 1117 const char *name, 1118 char ownername[]) 1119 { 1120 struct find_symbol_arg fsa = { 1121 .name = name, 1122 .gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), 1123 .warn = true, 1124 }; 1125 int err; 1126 1127 /* 1128 * The module_mutex should not be a heavily contended lock; 1129 * if we get the occasional sleep here, we'll go an extra iteration 1130 * in the wait_event_interruptible(), which is harmless. 1131 */ 1132 sched_annotate_sleep(); 1133 mutex_lock(&module_mutex); 1134 if (!find_symbol(&fsa)) 1135 goto unlock; 1136 1137 if (fsa.license == GPL_ONLY) 1138 mod->using_gplonly_symbols = true; 1139 1140 if (!inherit_taint(mod, fsa.owner, name)) { 1141 fsa.sym = NULL; 1142 goto getname; 1143 } 1144 1145 if (!check_version(info, name, mod, fsa.crc)) { 1146 fsa.sym = ERR_PTR(-EINVAL); 1147 goto getname; 1148 } 1149 1150 err = verify_namespace_is_imported(info, fsa.sym, mod); 1151 if (err) { 1152 fsa.sym = ERR_PTR(err); 1153 goto getname; 1154 } 1155 1156 err = ref_module(mod, fsa.owner); 1157 if (err) { 1158 fsa.sym = ERR_PTR(err); 1159 goto getname; 1160 } 1161 1162 getname: 1163 /* We must make copy under the lock if we failed to get ref. */ 1164 strncpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN); 1165 unlock: 1166 mutex_unlock(&module_mutex); 1167 return fsa.sym; 1168 } 1169 1170 static const struct kernel_symbol * 1171 resolve_symbol_wait(struct module *mod, 1172 const struct load_info *info, 1173 const char *name) 1174 { 1175 const struct kernel_symbol *ksym; 1176 char owner[MODULE_NAME_LEN]; 1177 1178 if (wait_event_interruptible_timeout(module_wq, 1179 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner)) 1180 || PTR_ERR(ksym) != -EBUSY, 1181 30 * HZ) <= 0) { 1182 pr_warn("%s: gave up waiting for init of module %s.\n", 1183 mod->name, owner); 1184 } 1185 return ksym; 1186 } 1187 1188 void __weak module_memfree(void *module_region) 1189 { 1190 /* 1191 * This memory may be RO, and freeing RO memory in an interrupt is not 1192 * supported by vmalloc. 1193 */ 1194 WARN_ON(in_interrupt()); 1195 vfree(module_region); 1196 } 1197 1198 void __weak module_arch_cleanup(struct module *mod) 1199 { 1200 } 1201 1202 void __weak module_arch_freeing_init(struct module *mod) 1203 { 1204 } 1205 1206 static bool mod_mem_use_vmalloc(enum mod_mem_type type) 1207 { 1208 return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) && 1209 mod_mem_type_is_core_data(type); 1210 } 1211 1212 static void *module_memory_alloc(unsigned int size, enum mod_mem_type type) 1213 { 1214 if (mod_mem_use_vmalloc(type)) 1215 return vzalloc(size); 1216 return module_alloc(size); 1217 } 1218 1219 static void module_memory_free(void *ptr, enum mod_mem_type type) 1220 { 1221 if (mod_mem_use_vmalloc(type)) 1222 vfree(ptr); 1223 else 1224 module_memfree(ptr); 1225 } 1226 1227 static void free_mod_mem(struct module *mod) 1228 { 1229 for_each_mod_mem_type(type) { 1230 struct module_memory *mod_mem = &mod->mem[type]; 1231 1232 if (type == MOD_DATA) 1233 continue; 1234 1235 /* Free lock-classes; relies on the preceding sync_rcu(). */ 1236 lockdep_free_key_range(mod_mem->base, mod_mem->size); 1237 if (mod_mem->size) 1238 module_memory_free(mod_mem->base, type); 1239 } 1240 1241 /* MOD_DATA hosts mod, so free it at last */ 1242 lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size); 1243 module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA); 1244 } 1245 1246 /* Free a module, remove from lists, etc. */ 1247 static void free_module(struct module *mod) 1248 { 1249 trace_module_free(mod); 1250 1251 mod_sysfs_teardown(mod); 1252 1253 /* 1254 * We leave it in list to prevent duplicate loads, but make sure 1255 * that noone uses it while it's being deconstructed. 1256 */ 1257 mutex_lock(&module_mutex); 1258 mod->state = MODULE_STATE_UNFORMED; 1259 mutex_unlock(&module_mutex); 1260 1261 /* Arch-specific cleanup. */ 1262 module_arch_cleanup(mod); 1263 1264 /* Module unload stuff */ 1265 module_unload_free(mod); 1266 1267 /* Free any allocated parameters. */ 1268 destroy_params(mod->kp, mod->num_kp); 1269 1270 if (is_livepatch_module(mod)) 1271 free_module_elf(mod); 1272 1273 /* Now we can delete it from the lists */ 1274 mutex_lock(&module_mutex); 1275 /* Unlink carefully: kallsyms could be walking list. */ 1276 list_del_rcu(&mod->list); 1277 mod_tree_remove(mod); 1278 /* Remove this module from bug list, this uses list_del_rcu */ 1279 module_bug_cleanup(mod); 1280 /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */ 1281 synchronize_rcu(); 1282 if (try_add_tainted_module(mod)) 1283 pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n", 1284 mod->name); 1285 mutex_unlock(&module_mutex); 1286 1287 /* This may be empty, but that's OK */ 1288 module_arch_freeing_init(mod); 1289 kfree(mod->args); 1290 percpu_modfree(mod); 1291 1292 free_mod_mem(mod); 1293 } 1294 1295 void *__symbol_get(const char *symbol) 1296 { 1297 struct find_symbol_arg fsa = { 1298 .name = symbol, 1299 .gplok = true, 1300 .warn = true, 1301 }; 1302 1303 preempt_disable(); 1304 if (!find_symbol(&fsa) || strong_try_module_get(fsa.owner)) { 1305 preempt_enable(); 1306 return NULL; 1307 } 1308 preempt_enable(); 1309 return (void *)kernel_symbol_value(fsa.sym); 1310 } 1311 EXPORT_SYMBOL_GPL(__symbol_get); 1312 1313 /* 1314 * Ensure that an exported symbol [global namespace] does not already exist 1315 * in the kernel or in some other module's exported symbol table. 1316 * 1317 * You must hold the module_mutex. 1318 */ 1319 static int verify_exported_symbols(struct module *mod) 1320 { 1321 unsigned int i; 1322 const struct kernel_symbol *s; 1323 struct { 1324 const struct kernel_symbol *sym; 1325 unsigned int num; 1326 } arr[] = { 1327 { mod->syms, mod->num_syms }, 1328 { mod->gpl_syms, mod->num_gpl_syms }, 1329 }; 1330 1331 for (i = 0; i < ARRAY_SIZE(arr); i++) { 1332 for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) { 1333 struct find_symbol_arg fsa = { 1334 .name = kernel_symbol_name(s), 1335 .gplok = true, 1336 }; 1337 if (find_symbol(&fsa)) { 1338 pr_err("%s: exports duplicate symbol %s" 1339 " (owned by %s)\n", 1340 mod->name, kernel_symbol_name(s), 1341 module_name(fsa.owner)); 1342 return -ENOEXEC; 1343 } 1344 } 1345 } 1346 return 0; 1347 } 1348 1349 static bool ignore_undef_symbol(Elf_Half emachine, const char *name) 1350 { 1351 /* 1352 * On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as 1353 * before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64. 1354 * i386 has a similar problem but may not deserve a fix. 1355 * 1356 * If we ever have to ignore many symbols, consider refactoring the code to 1357 * only warn if referenced by a relocation. 1358 */ 1359 if (emachine == EM_386 || emachine == EM_X86_64) 1360 return !strcmp(name, "_GLOBAL_OFFSET_TABLE_"); 1361 return false; 1362 } 1363 1364 /* Change all symbols so that st_value encodes the pointer directly. */ 1365 static int simplify_symbols(struct module *mod, const struct load_info *info) 1366 { 1367 Elf_Shdr *symsec = &info->sechdrs[info->index.sym]; 1368 Elf_Sym *sym = (void *)symsec->sh_addr; 1369 unsigned long secbase; 1370 unsigned int i; 1371 int ret = 0; 1372 const struct kernel_symbol *ksym; 1373 1374 for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) { 1375 const char *name = info->strtab + sym[i].st_name; 1376 1377 switch (sym[i].st_shndx) { 1378 case SHN_COMMON: 1379 /* Ignore common symbols */ 1380 if (!strncmp(name, "__gnu_lto", 9)) 1381 break; 1382 1383 /* 1384 * We compiled with -fno-common. These are not 1385 * supposed to happen. 1386 */ 1387 pr_debug("Common symbol: %s\n", name); 1388 pr_warn("%s: please compile with -fno-common\n", 1389 mod->name); 1390 ret = -ENOEXEC; 1391 break; 1392 1393 case SHN_ABS: 1394 /* Don't need to do anything */ 1395 pr_debug("Absolute symbol: 0x%08lx %s\n", 1396 (long)sym[i].st_value, name); 1397 break; 1398 1399 case SHN_LIVEPATCH: 1400 /* Livepatch symbols are resolved by livepatch */ 1401 break; 1402 1403 case SHN_UNDEF: 1404 ksym = resolve_symbol_wait(mod, info, name); 1405 /* Ok if resolved. */ 1406 if (ksym && !IS_ERR(ksym)) { 1407 sym[i].st_value = kernel_symbol_value(ksym); 1408 break; 1409 } 1410 1411 /* Ok if weak or ignored. */ 1412 if (!ksym && 1413 (ELF_ST_BIND(sym[i].st_info) == STB_WEAK || 1414 ignore_undef_symbol(info->hdr->e_machine, name))) 1415 break; 1416 1417 ret = PTR_ERR(ksym) ?: -ENOENT; 1418 pr_warn("%s: Unknown symbol %s (err %d)\n", 1419 mod->name, name, ret); 1420 break; 1421 1422 default: 1423 /* Divert to percpu allocation if a percpu var. */ 1424 if (sym[i].st_shndx == info->index.pcpu) 1425 secbase = (unsigned long)mod_percpu(mod); 1426 else 1427 secbase = info->sechdrs[sym[i].st_shndx].sh_addr; 1428 sym[i].st_value += secbase; 1429 break; 1430 } 1431 } 1432 1433 return ret; 1434 } 1435 1436 static int apply_relocations(struct module *mod, const struct load_info *info) 1437 { 1438 unsigned int i; 1439 int err = 0; 1440 1441 /* Now do relocations. */ 1442 for (i = 1; i < info->hdr->e_shnum; i++) { 1443 unsigned int infosec = info->sechdrs[i].sh_info; 1444 1445 /* Not a valid relocation section? */ 1446 if (infosec >= info->hdr->e_shnum) 1447 continue; 1448 1449 /* Don't bother with non-allocated sections */ 1450 if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC)) 1451 continue; 1452 1453 if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH) 1454 err = klp_apply_section_relocs(mod, info->sechdrs, 1455 info->secstrings, 1456 info->strtab, 1457 info->index.sym, i, 1458 NULL); 1459 else if (info->sechdrs[i].sh_type == SHT_REL) 1460 err = apply_relocate(info->sechdrs, info->strtab, 1461 info->index.sym, i, mod); 1462 else if (info->sechdrs[i].sh_type == SHT_RELA) 1463 err = apply_relocate_add(info->sechdrs, info->strtab, 1464 info->index.sym, i, mod); 1465 if (err < 0) 1466 break; 1467 } 1468 return err; 1469 } 1470 1471 /* Additional bytes needed by arch in front of individual sections */ 1472 unsigned int __weak arch_mod_section_prepend(struct module *mod, 1473 unsigned int section) 1474 { 1475 /* default implementation just returns zero */ 1476 return 0; 1477 } 1478 1479 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, 1480 Elf_Shdr *sechdr, unsigned int section) 1481 { 1482 long offset; 1483 long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT; 1484 1485 mod->mem[type].size += arch_mod_section_prepend(mod, section); 1486 offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1); 1487 mod->mem[type].size = offset + sechdr->sh_size; 1488 1489 WARN_ON_ONCE(offset & mask); 1490 return offset | mask; 1491 } 1492 1493 static bool module_init_layout_section(const char *sname) 1494 { 1495 #ifndef CONFIG_MODULE_UNLOAD 1496 if (module_exit_section(sname)) 1497 return true; 1498 #endif 1499 return module_init_section(sname); 1500 } 1501 1502 static void __layout_sections(struct module *mod, struct load_info *info, bool is_init) 1503 { 1504 unsigned int m, i; 1505 1506 static const unsigned long masks[][2] = { 1507 /* 1508 * NOTE: all executable code must be the first section 1509 * in this array; otherwise modify the text_size 1510 * finder in the two loops below 1511 */ 1512 { SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL }, 1513 { SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL }, 1514 { SHF_RO_AFTER_INIT | SHF_ALLOC, ARCH_SHF_SMALL }, 1515 { SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL }, 1516 { ARCH_SHF_SMALL | SHF_ALLOC, 0 } 1517 }; 1518 static const int core_m_to_mem_type[] = { 1519 MOD_TEXT, 1520 MOD_RODATA, 1521 MOD_RO_AFTER_INIT, 1522 MOD_DATA, 1523 MOD_INVALID, /* This is needed to match the masks array */ 1524 }; 1525 static const int init_m_to_mem_type[] = { 1526 MOD_INIT_TEXT, 1527 MOD_INIT_RODATA, 1528 MOD_INVALID, 1529 MOD_INIT_DATA, 1530 MOD_INVALID, /* This is needed to match the masks array */ 1531 }; 1532 1533 for (m = 0; m < ARRAY_SIZE(masks); ++m) { 1534 enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m]; 1535 1536 for (i = 0; i < info->hdr->e_shnum; ++i) { 1537 Elf_Shdr *s = &info->sechdrs[i]; 1538 const char *sname = info->secstrings + s->sh_name; 1539 1540 if ((s->sh_flags & masks[m][0]) != masks[m][0] 1541 || (s->sh_flags & masks[m][1]) 1542 || s->sh_entsize != ~0UL 1543 || is_init != module_init_layout_section(sname)) 1544 continue; 1545 1546 if (WARN_ON_ONCE(type == MOD_INVALID)) 1547 continue; 1548 1549 s->sh_entsize = module_get_offset_and_type(mod, type, s, i); 1550 pr_debug("\t%s\n", sname); 1551 } 1552 } 1553 } 1554 1555 /* 1556 * Lay out the SHF_ALLOC sections in a way not dissimilar to how ld 1557 * might -- code, read-only data, read-write data, small data. Tally 1558 * sizes, and place the offsets into sh_entsize fields: high bit means it 1559 * belongs in init. 1560 */ 1561 static void layout_sections(struct module *mod, struct load_info *info) 1562 { 1563 unsigned int i; 1564 1565 for (i = 0; i < info->hdr->e_shnum; i++) 1566 info->sechdrs[i].sh_entsize = ~0UL; 1567 1568 pr_debug("Core section allocation order for %s:\n", mod->name); 1569 __layout_sections(mod, info, false); 1570 1571 pr_debug("Init section allocation order for %s:\n", mod->name); 1572 __layout_sections(mod, info, true); 1573 } 1574 1575 static void module_license_taint_check(struct module *mod, const char *license) 1576 { 1577 if (!license) 1578 license = "unspecified"; 1579 1580 if (!license_is_gpl_compatible(license)) { 1581 if (!test_taint(TAINT_PROPRIETARY_MODULE)) 1582 pr_warn("%s: module license '%s' taints kernel.\n", 1583 mod->name, license); 1584 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, 1585 LOCKDEP_NOW_UNRELIABLE); 1586 } 1587 } 1588 1589 static void setup_modinfo(struct module *mod, struct load_info *info) 1590 { 1591 struct module_attribute *attr; 1592 int i; 1593 1594 for (i = 0; (attr = modinfo_attrs[i]); i++) { 1595 if (attr->setup) 1596 attr->setup(mod, get_modinfo(info, attr->attr.name)); 1597 } 1598 } 1599 1600 static void free_modinfo(struct module *mod) 1601 { 1602 struct module_attribute *attr; 1603 int i; 1604 1605 for (i = 0; (attr = modinfo_attrs[i]); i++) { 1606 if (attr->free) 1607 attr->free(mod); 1608 } 1609 } 1610 1611 void * __weak module_alloc(unsigned long size) 1612 { 1613 return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, 1614 GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, 1615 NUMA_NO_NODE, __builtin_return_address(0)); 1616 } 1617 1618 bool __weak module_init_section(const char *name) 1619 { 1620 return strstarts(name, ".init"); 1621 } 1622 1623 bool __weak module_exit_section(const char *name) 1624 { 1625 return strstarts(name, ".exit"); 1626 } 1627 1628 static int validate_section_offset(struct load_info *info, Elf_Shdr *shdr) 1629 { 1630 #if defined(CONFIG_64BIT) 1631 unsigned long long secend; 1632 #else 1633 unsigned long secend; 1634 #endif 1635 1636 /* 1637 * Check for both overflow and offset/size being 1638 * too large. 1639 */ 1640 secend = shdr->sh_offset + shdr->sh_size; 1641 if (secend < shdr->sh_offset || secend > info->len) 1642 return -ENOEXEC; 1643 1644 return 0; 1645 } 1646 1647 /* 1648 * Check userspace passed ELF module against our expectations, and cache 1649 * useful variables for further processing as we go. 1650 * 1651 * This does basic validity checks against section offsets and sizes, the 1652 * section name string table, and the indices used for it (sh_name). 1653 * 1654 * As a last step, since we're already checking the ELF sections we cache 1655 * useful variables which will be used later for our convenience: 1656 * 1657 * o pointers to section headers 1658 * o cache the modinfo symbol section 1659 * o cache the string symbol section 1660 * o cache the module section 1661 * 1662 * As a last step we set info->mod to the temporary copy of the module in 1663 * info->hdr. The final one will be allocated in move_module(). Any 1664 * modifications we make to our copy of the module will be carried over 1665 * to the final minted module. 1666 */ 1667 static int elf_validity_cache_copy(struct load_info *info, int flags) 1668 { 1669 unsigned int i; 1670 Elf_Shdr *shdr, *strhdr; 1671 int err; 1672 unsigned int num_mod_secs = 0, mod_idx; 1673 unsigned int num_info_secs = 0, info_idx; 1674 unsigned int num_sym_secs = 0, sym_idx; 1675 1676 if (info->len < sizeof(*(info->hdr))) { 1677 pr_err("Invalid ELF header len %lu\n", info->len); 1678 goto no_exec; 1679 } 1680 1681 if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) { 1682 pr_err("Invalid ELF header magic: != %s\n", ELFMAG); 1683 goto no_exec; 1684 } 1685 if (info->hdr->e_type != ET_REL) { 1686 pr_err("Invalid ELF header type: %u != %u\n", 1687 info->hdr->e_type, ET_REL); 1688 goto no_exec; 1689 } 1690 if (!elf_check_arch(info->hdr)) { 1691 pr_err("Invalid architecture in ELF header: %u\n", 1692 info->hdr->e_machine); 1693 goto no_exec; 1694 } 1695 if (!module_elf_check_arch(info->hdr)) { 1696 pr_err("Invalid module architecture in ELF header: %u\n", 1697 info->hdr->e_machine); 1698 goto no_exec; 1699 } 1700 if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) { 1701 pr_err("Invalid ELF section header size\n"); 1702 goto no_exec; 1703 } 1704 1705 /* 1706 * e_shnum is 16 bits, and sizeof(Elf_Shdr) is 1707 * known and small. So e_shnum * sizeof(Elf_Shdr) 1708 * will not overflow unsigned long on any platform. 1709 */ 1710 if (info->hdr->e_shoff >= info->len 1711 || (info->hdr->e_shnum * sizeof(Elf_Shdr) > 1712 info->len - info->hdr->e_shoff)) { 1713 pr_err("Invalid ELF section header overflow\n"); 1714 goto no_exec; 1715 } 1716 1717 info->sechdrs = (void *)info->hdr + info->hdr->e_shoff; 1718 1719 /* 1720 * Verify if the section name table index is valid. 1721 */ 1722 if (info->hdr->e_shstrndx == SHN_UNDEF 1723 || info->hdr->e_shstrndx >= info->hdr->e_shnum) { 1724 pr_err("Invalid ELF section name index: %d || e_shstrndx (%d) >= e_shnum (%d)\n", 1725 info->hdr->e_shstrndx, info->hdr->e_shstrndx, 1726 info->hdr->e_shnum); 1727 goto no_exec; 1728 } 1729 1730 strhdr = &info->sechdrs[info->hdr->e_shstrndx]; 1731 err = validate_section_offset(info, strhdr); 1732 if (err < 0) { 1733 pr_err("Invalid ELF section hdr(type %u)\n", strhdr->sh_type); 1734 return err; 1735 } 1736 1737 /* 1738 * The section name table must be NUL-terminated, as required 1739 * by the spec. This makes strcmp and pr_* calls that access 1740 * strings in the section safe. 1741 */ 1742 info->secstrings = (void *)info->hdr + strhdr->sh_offset; 1743 if (strhdr->sh_size == 0) { 1744 pr_err("empty section name table\n"); 1745 goto no_exec; 1746 } 1747 if (info->secstrings[strhdr->sh_size - 1] != '\0') { 1748 pr_err("ELF Spec violation: section name table isn't null terminated\n"); 1749 goto no_exec; 1750 } 1751 1752 /* 1753 * The code assumes that section 0 has a length of zero and 1754 * an addr of zero, so check for it. 1755 */ 1756 if (info->sechdrs[0].sh_type != SHT_NULL 1757 || info->sechdrs[0].sh_size != 0 1758 || info->sechdrs[0].sh_addr != 0) { 1759 pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n", 1760 info->sechdrs[0].sh_type); 1761 goto no_exec; 1762 } 1763 1764 for (i = 1; i < info->hdr->e_shnum; i++) { 1765 shdr = &info->sechdrs[i]; 1766 switch (shdr->sh_type) { 1767 case SHT_NULL: 1768 case SHT_NOBITS: 1769 continue; 1770 case SHT_SYMTAB: 1771 if (shdr->sh_link == SHN_UNDEF 1772 || shdr->sh_link >= info->hdr->e_shnum) { 1773 pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n", 1774 shdr->sh_link, shdr->sh_link, 1775 info->hdr->e_shnum); 1776 goto no_exec; 1777 } 1778 num_sym_secs++; 1779 sym_idx = i; 1780 fallthrough; 1781 default: 1782 err = validate_section_offset(info, shdr); 1783 if (err < 0) { 1784 pr_err("Invalid ELF section in module (section %u type %u)\n", 1785 i, shdr->sh_type); 1786 return err; 1787 } 1788 if (strcmp(info->secstrings + shdr->sh_name, 1789 ".gnu.linkonce.this_module") == 0) { 1790 num_mod_secs++; 1791 mod_idx = i; 1792 } else if (strcmp(info->secstrings + shdr->sh_name, 1793 ".modinfo") == 0) { 1794 num_info_secs++; 1795 info_idx = i; 1796 } 1797 1798 if (shdr->sh_flags & SHF_ALLOC) { 1799 if (shdr->sh_name >= strhdr->sh_size) { 1800 pr_err("Invalid ELF section name in module (section %u type %u)\n", 1801 i, shdr->sh_type); 1802 return -ENOEXEC; 1803 } 1804 } 1805 break; 1806 } 1807 } 1808 1809 if (num_info_secs > 1) { 1810 pr_err("Only one .modinfo section must exist.\n"); 1811 goto no_exec; 1812 } else if (num_info_secs == 1) { 1813 /* Try to find a name early so we can log errors with a module name */ 1814 info->index.info = info_idx; 1815 info->name = get_modinfo(info, "name"); 1816 } 1817 1818 if (num_sym_secs != 1) { 1819 pr_warn("%s: module has no symbols (stripped?)\n", 1820 info->name ?: "(missing .modinfo section or name field)"); 1821 goto no_exec; 1822 } 1823 1824 /* Sets internal symbols and strings. */ 1825 info->index.sym = sym_idx; 1826 shdr = &info->sechdrs[sym_idx]; 1827 info->index.str = shdr->sh_link; 1828 info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset; 1829 1830 /* 1831 * The ".gnu.linkonce.this_module" ELF section is special. It is 1832 * what modpost uses to refer to __this_module and let's use rely 1833 * on THIS_MODULE to point to &__this_module properly. The kernel's 1834 * modpost declares it on each modules's *.mod.c file. If the struct 1835 * module of the kernel changes a full kernel rebuild is required. 1836 * 1837 * We have a few expectaions for this special section, the following 1838 * code validates all this for us: 1839 * 1840 * o Only one section must exist 1841 * o We expect the kernel to always have to allocate it: SHF_ALLOC 1842 * o The section size must match the kernel's run time's struct module 1843 * size 1844 */ 1845 if (num_mod_secs != 1) { 1846 pr_err("module %s: Only one .gnu.linkonce.this_module section must exist.\n", 1847 info->name ?: "(missing .modinfo section or name field)"); 1848 goto no_exec; 1849 } 1850 1851 shdr = &info->sechdrs[mod_idx]; 1852 1853 /* 1854 * This is already implied on the switch above, however let's be 1855 * pedantic about it. 1856 */ 1857 if (shdr->sh_type == SHT_NOBITS) { 1858 pr_err("module %s: .gnu.linkonce.this_module section must have a size set\n", 1859 info->name ?: "(missing .modinfo section or name field)"); 1860 goto no_exec; 1861 } 1862 1863 if (!(shdr->sh_flags & SHF_ALLOC)) { 1864 pr_err("module %s: .gnu.linkonce.this_module must occupy memory during process execution\n", 1865 info->name ?: "(missing .modinfo section or name field)"); 1866 goto no_exec; 1867 } 1868 1869 if (shdr->sh_size != sizeof(struct module)) { 1870 pr_err("module %s: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time\n", 1871 info->name ?: "(missing .modinfo section or name field)"); 1872 goto no_exec; 1873 } 1874 1875 info->index.mod = mod_idx; 1876 1877 /* This is temporary: point mod into copy of data. */ 1878 info->mod = (void *)info->hdr + shdr->sh_offset; 1879 1880 /* 1881 * If we didn't load the .modinfo 'name' field earlier, fall back to 1882 * on-disk struct mod 'name' field. 1883 */ 1884 if (!info->name) 1885 info->name = info->mod->name; 1886 1887 if (flags & MODULE_INIT_IGNORE_MODVERSIONS) 1888 info->index.vers = 0; /* Pretend no __versions section! */ 1889 else 1890 info->index.vers = find_sec(info, "__versions"); 1891 1892 info->index.pcpu = find_pcpusec(info); 1893 1894 return 0; 1895 1896 no_exec: 1897 return -ENOEXEC; 1898 } 1899 1900 #define COPY_CHUNK_SIZE (16*PAGE_SIZE) 1901 1902 static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len) 1903 { 1904 do { 1905 unsigned long n = min(len, COPY_CHUNK_SIZE); 1906 1907 if (copy_from_user(dst, usrc, n) != 0) 1908 return -EFAULT; 1909 cond_resched(); 1910 dst += n; 1911 usrc += n; 1912 len -= n; 1913 } while (len); 1914 return 0; 1915 } 1916 1917 static int check_modinfo_livepatch(struct module *mod, struct load_info *info) 1918 { 1919 if (!get_modinfo(info, "livepatch")) 1920 /* Nothing more to do */ 1921 return 0; 1922 1923 if (set_livepatch_module(mod)) 1924 return 0; 1925 1926 pr_err("%s: module is marked as livepatch module, but livepatch support is disabled", 1927 mod->name); 1928 return -ENOEXEC; 1929 } 1930 1931 static void check_modinfo_retpoline(struct module *mod, struct load_info *info) 1932 { 1933 if (retpoline_module_ok(get_modinfo(info, "retpoline"))) 1934 return; 1935 1936 pr_warn("%s: loading module not compiled with retpoline compiler.\n", 1937 mod->name); 1938 } 1939 1940 /* Sets info->hdr and info->len. */ 1941 static int copy_module_from_user(const void __user *umod, unsigned long len, 1942 struct load_info *info) 1943 { 1944 int err; 1945 1946 info->len = len; 1947 if (info->len < sizeof(*(info->hdr))) 1948 return -ENOEXEC; 1949 1950 err = security_kernel_load_data(LOADING_MODULE, true); 1951 if (err) 1952 return err; 1953 1954 /* Suck in entire file: we'll want most of it. */ 1955 info->hdr = __vmalloc(info->len, GFP_KERNEL | __GFP_NOWARN); 1956 if (!info->hdr) 1957 return -ENOMEM; 1958 1959 if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) { 1960 err = -EFAULT; 1961 goto out; 1962 } 1963 1964 err = security_kernel_post_load_data((char *)info->hdr, info->len, 1965 LOADING_MODULE, "init_module"); 1966 out: 1967 if (err) 1968 vfree(info->hdr); 1969 1970 return err; 1971 } 1972 1973 static void free_copy(struct load_info *info, int flags) 1974 { 1975 if (flags & MODULE_INIT_COMPRESSED_FILE) 1976 module_decompress_cleanup(info); 1977 else 1978 vfree(info->hdr); 1979 } 1980 1981 static int rewrite_section_headers(struct load_info *info, int flags) 1982 { 1983 unsigned int i; 1984 1985 /* This should always be true, but let's be sure. */ 1986 info->sechdrs[0].sh_addr = 0; 1987 1988 for (i = 1; i < info->hdr->e_shnum; i++) { 1989 Elf_Shdr *shdr = &info->sechdrs[i]; 1990 1991 /* 1992 * Mark all sections sh_addr with their address in the 1993 * temporary image. 1994 */ 1995 shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset; 1996 1997 } 1998 1999 /* Track but don't keep modinfo and version sections. */ 2000 info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC; 2001 info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC; 2002 2003 return 0; 2004 } 2005 2006 /* 2007 * These calls taint the kernel depending certain module circumstances */ 2008 static void module_augment_kernel_taints(struct module *mod, struct load_info *info) 2009 { 2010 int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE); 2011 2012 if (!get_modinfo(info, "intree")) { 2013 if (!test_taint(TAINT_OOT_MODULE)) 2014 pr_warn("%s: loading out-of-tree module taints kernel.\n", 2015 mod->name); 2016 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK); 2017 } 2018 2019 check_modinfo_retpoline(mod, info); 2020 2021 if (get_modinfo(info, "staging")) { 2022 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK); 2023 pr_warn("%s: module is from the staging directory, the quality " 2024 "is unknown, you have been warned.\n", mod->name); 2025 } 2026 2027 if (is_livepatch_module(mod)) { 2028 add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK); 2029 pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n", 2030 mod->name); 2031 } 2032 2033 module_license_taint_check(mod, get_modinfo(info, "license")); 2034 2035 if (get_modinfo(info, "test")) { 2036 if (!test_taint(TAINT_TEST)) 2037 pr_warn("%s: loading test module taints kernel.\n", 2038 mod->name); 2039 add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK); 2040 } 2041 #ifdef CONFIG_MODULE_SIG 2042 mod->sig_ok = info->sig_ok; 2043 if (!mod->sig_ok) { 2044 pr_notice_once("%s: module verification failed: signature " 2045 "and/or required key missing - tainting " 2046 "kernel\n", mod->name); 2047 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK); 2048 } 2049 #endif 2050 2051 /* 2052 * ndiswrapper is under GPL by itself, but loads proprietary modules. 2053 * Don't use add_taint_module(), as it would prevent ndiswrapper from 2054 * using GPL-only symbols it needs. 2055 */ 2056 if (strcmp(mod->name, "ndiswrapper") == 0) 2057 add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE); 2058 2059 /* driverloader was caught wrongly pretending to be under GPL */ 2060 if (strcmp(mod->name, "driverloader") == 0) 2061 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, 2062 LOCKDEP_NOW_UNRELIABLE); 2063 2064 /* lve claims to be GPL but upstream won't provide source */ 2065 if (strcmp(mod->name, "lve") == 0) 2066 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, 2067 LOCKDEP_NOW_UNRELIABLE); 2068 2069 if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE)) 2070 pr_warn("%s: module license taints kernel.\n", mod->name); 2071 2072 } 2073 2074 static int check_modinfo(struct module *mod, struct load_info *info, int flags) 2075 { 2076 const char *modmagic = get_modinfo(info, "vermagic"); 2077 int err; 2078 2079 if (flags & MODULE_INIT_IGNORE_VERMAGIC) 2080 modmagic = NULL; 2081 2082 /* This is allowed: modprobe --force will invalidate it. */ 2083 if (!modmagic) { 2084 err = try_to_force_load(mod, "bad vermagic"); 2085 if (err) 2086 return err; 2087 } else if (!same_magic(modmagic, vermagic, info->index.vers)) { 2088 pr_err("%s: version magic '%s' should be '%s'\n", 2089 info->name, modmagic, vermagic); 2090 return -ENOEXEC; 2091 } 2092 2093 err = check_modinfo_livepatch(mod, info); 2094 if (err) 2095 return err; 2096 2097 return 0; 2098 } 2099 2100 static int find_module_sections(struct module *mod, struct load_info *info) 2101 { 2102 mod->kp = section_objs(info, "__param", 2103 sizeof(*mod->kp), &mod->num_kp); 2104 mod->syms = section_objs(info, "__ksymtab", 2105 sizeof(*mod->syms), &mod->num_syms); 2106 mod->crcs = section_addr(info, "__kcrctab"); 2107 mod->gpl_syms = section_objs(info, "__ksymtab_gpl", 2108 sizeof(*mod->gpl_syms), 2109 &mod->num_gpl_syms); 2110 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl"); 2111 2112 #ifdef CONFIG_CONSTRUCTORS 2113 mod->ctors = section_objs(info, ".ctors", 2114 sizeof(*mod->ctors), &mod->num_ctors); 2115 if (!mod->ctors) 2116 mod->ctors = section_objs(info, ".init_array", 2117 sizeof(*mod->ctors), &mod->num_ctors); 2118 else if (find_sec(info, ".init_array")) { 2119 /* 2120 * This shouldn't happen with same compiler and binutils 2121 * building all parts of the module. 2122 */ 2123 pr_warn("%s: has both .ctors and .init_array.\n", 2124 mod->name); 2125 return -EINVAL; 2126 } 2127 #endif 2128 2129 mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1, 2130 &mod->noinstr_text_size); 2131 2132 #ifdef CONFIG_TRACEPOINTS 2133 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs", 2134 sizeof(*mod->tracepoints_ptrs), 2135 &mod->num_tracepoints); 2136 #endif 2137 #ifdef CONFIG_TREE_SRCU 2138 mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs", 2139 sizeof(*mod->srcu_struct_ptrs), 2140 &mod->num_srcu_structs); 2141 #endif 2142 #ifdef CONFIG_BPF_EVENTS 2143 mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map", 2144 sizeof(*mod->bpf_raw_events), 2145 &mod->num_bpf_raw_events); 2146 #endif 2147 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 2148 mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size); 2149 #endif 2150 #ifdef CONFIG_JUMP_LABEL 2151 mod->jump_entries = section_objs(info, "__jump_table", 2152 sizeof(*mod->jump_entries), 2153 &mod->num_jump_entries); 2154 #endif 2155 #ifdef CONFIG_EVENT_TRACING 2156 mod->trace_events = section_objs(info, "_ftrace_events", 2157 sizeof(*mod->trace_events), 2158 &mod->num_trace_events); 2159 mod->trace_evals = section_objs(info, "_ftrace_eval_map", 2160 sizeof(*mod->trace_evals), 2161 &mod->num_trace_evals); 2162 #endif 2163 #ifdef CONFIG_TRACING 2164 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt", 2165 sizeof(*mod->trace_bprintk_fmt_start), 2166 &mod->num_trace_bprintk_fmt); 2167 #endif 2168 #ifdef CONFIG_FTRACE_MCOUNT_RECORD 2169 /* sechdrs[0].sh_size is always zero */ 2170 mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION, 2171 sizeof(*mod->ftrace_callsites), 2172 &mod->num_ftrace_callsites); 2173 #endif 2174 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 2175 mod->ei_funcs = section_objs(info, "_error_injection_whitelist", 2176 sizeof(*mod->ei_funcs), 2177 &mod->num_ei_funcs); 2178 #endif 2179 #ifdef CONFIG_KPROBES 2180 mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1, 2181 &mod->kprobes_text_size); 2182 mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist", 2183 sizeof(unsigned long), 2184 &mod->num_kprobe_blacklist); 2185 #endif 2186 #ifdef CONFIG_PRINTK_INDEX 2187 mod->printk_index_start = section_objs(info, ".printk_index", 2188 sizeof(*mod->printk_index_start), 2189 &mod->printk_index_size); 2190 #endif 2191 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE 2192 mod->static_call_sites = section_objs(info, ".static_call_sites", 2193 sizeof(*mod->static_call_sites), 2194 &mod->num_static_call_sites); 2195 #endif 2196 #if IS_ENABLED(CONFIG_KUNIT) 2197 mod->kunit_suites = section_objs(info, ".kunit_test_suites", 2198 sizeof(*mod->kunit_suites), 2199 &mod->num_kunit_suites); 2200 #endif 2201 2202 mod->extable = section_objs(info, "__ex_table", 2203 sizeof(*mod->extable), &mod->num_exentries); 2204 2205 if (section_addr(info, "__obsparm")) 2206 pr_warn("%s: Ignoring obsolete parameters\n", mod->name); 2207 2208 #ifdef CONFIG_DYNAMIC_DEBUG_CORE 2209 mod->dyndbg_info.descs = section_objs(info, "__dyndbg", 2210 sizeof(*mod->dyndbg_info.descs), 2211 &mod->dyndbg_info.num_descs); 2212 mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes", 2213 sizeof(*mod->dyndbg_info.classes), 2214 &mod->dyndbg_info.num_classes); 2215 #endif 2216 2217 return 0; 2218 } 2219 2220 static int move_module(struct module *mod, struct load_info *info) 2221 { 2222 int i; 2223 void *ptr; 2224 enum mod_mem_type t = 0; 2225 int ret = -ENOMEM; 2226 2227 for_each_mod_mem_type(type) { 2228 if (!mod->mem[type].size) { 2229 mod->mem[type].base = NULL; 2230 continue; 2231 } 2232 mod->mem[type].size = PAGE_ALIGN(mod->mem[type].size); 2233 ptr = module_memory_alloc(mod->mem[type].size, type); 2234 2235 /* 2236 * The pointer to this block is stored in the module structure 2237 * which is inside the block. Just mark it as not being a 2238 * leak. 2239 */ 2240 kmemleak_ignore(ptr); 2241 if (!ptr) { 2242 t = type; 2243 goto out_enomem; 2244 } 2245 memset(ptr, 0, mod->mem[type].size); 2246 mod->mem[type].base = ptr; 2247 } 2248 2249 /* Transfer each section which specifies SHF_ALLOC */ 2250 pr_debug("Final section addresses for %s:\n", mod->name); 2251 for (i = 0; i < info->hdr->e_shnum; i++) { 2252 void *dest; 2253 Elf_Shdr *shdr = &info->sechdrs[i]; 2254 enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT; 2255 2256 if (!(shdr->sh_flags & SHF_ALLOC)) 2257 continue; 2258 2259 dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK); 2260 2261 if (shdr->sh_type != SHT_NOBITS) { 2262 /* 2263 * Our ELF checker already validated this, but let's 2264 * be pedantic and make the goal clearer. We actually 2265 * end up copying over all modifications made to the 2266 * userspace copy of the entire struct module. 2267 */ 2268 if (i == info->index.mod && 2269 (WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) { 2270 ret = -ENOEXEC; 2271 goto out_enomem; 2272 } 2273 memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size); 2274 } 2275 /* 2276 * Update the userspace copy's ELF section address to point to 2277 * our newly allocated memory as a pure convenience so that 2278 * users of info can keep taking advantage and using the newly 2279 * minted official memory area. 2280 */ 2281 shdr->sh_addr = (unsigned long)dest; 2282 pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr, 2283 (long)shdr->sh_size, info->secstrings + shdr->sh_name); 2284 } 2285 2286 return 0; 2287 out_enomem: 2288 for (t--; t >= 0; t--) 2289 module_memory_free(mod->mem[t].base, t); 2290 return ret; 2291 } 2292 2293 static int check_export_symbol_versions(struct module *mod) 2294 { 2295 #ifdef CONFIG_MODVERSIONS 2296 if ((mod->num_syms && !mod->crcs) || 2297 (mod->num_gpl_syms && !mod->gpl_crcs)) { 2298 return try_to_force_load(mod, 2299 "no versions for exported symbols"); 2300 } 2301 #endif 2302 return 0; 2303 } 2304 2305 static void flush_module_icache(const struct module *mod) 2306 { 2307 /* 2308 * Flush the instruction cache, since we've played with text. 2309 * Do it before processing of module parameters, so the module 2310 * can provide parameter accessor functions of its own. 2311 */ 2312 for_each_mod_mem_type(type) { 2313 const struct module_memory *mod_mem = &mod->mem[type]; 2314 2315 if (mod_mem->size) { 2316 flush_icache_range((unsigned long)mod_mem->base, 2317 (unsigned long)mod_mem->base + mod_mem->size); 2318 } 2319 } 2320 } 2321 2322 bool __weak module_elf_check_arch(Elf_Ehdr *hdr) 2323 { 2324 return true; 2325 } 2326 2327 int __weak module_frob_arch_sections(Elf_Ehdr *hdr, 2328 Elf_Shdr *sechdrs, 2329 char *secstrings, 2330 struct module *mod) 2331 { 2332 return 0; 2333 } 2334 2335 /* module_blacklist is a comma-separated list of module names */ 2336 static char *module_blacklist; 2337 static bool blacklisted(const char *module_name) 2338 { 2339 const char *p; 2340 size_t len; 2341 2342 if (!module_blacklist) 2343 return false; 2344 2345 for (p = module_blacklist; *p; p += len) { 2346 len = strcspn(p, ","); 2347 if (strlen(module_name) == len && !memcmp(module_name, p, len)) 2348 return true; 2349 if (p[len] == ',') 2350 len++; 2351 } 2352 return false; 2353 } 2354 core_param(module_blacklist, module_blacklist, charp, 0400); 2355 2356 static struct module *layout_and_allocate(struct load_info *info, int flags) 2357 { 2358 struct module *mod; 2359 unsigned int ndx; 2360 int err; 2361 2362 /* Allow arches to frob section contents and sizes. */ 2363 err = module_frob_arch_sections(info->hdr, info->sechdrs, 2364 info->secstrings, info->mod); 2365 if (err < 0) 2366 return ERR_PTR(err); 2367 2368 err = module_enforce_rwx_sections(info->hdr, info->sechdrs, 2369 info->secstrings, info->mod); 2370 if (err < 0) 2371 return ERR_PTR(err); 2372 2373 /* We will do a special allocation for per-cpu sections later. */ 2374 info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC; 2375 2376 /* 2377 * Mark ro_after_init section with SHF_RO_AFTER_INIT so that 2378 * layout_sections() can put it in the right place. 2379 * Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set. 2380 */ 2381 ndx = find_sec(info, ".data..ro_after_init"); 2382 if (ndx) 2383 info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT; 2384 /* 2385 * Mark the __jump_table section as ro_after_init as well: these data 2386 * structures are never modified, with the exception of entries that 2387 * refer to code in the __init section, which are annotated as such 2388 * at module load time. 2389 */ 2390 ndx = find_sec(info, "__jump_table"); 2391 if (ndx) 2392 info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT; 2393 2394 /* 2395 * Determine total sizes, and put offsets in sh_entsize. For now 2396 * this is done generically; there doesn't appear to be any 2397 * special cases for the architectures. 2398 */ 2399 layout_sections(info->mod, info); 2400 layout_symtab(info->mod, info); 2401 2402 /* Allocate and move to the final place */ 2403 err = move_module(info->mod, info); 2404 if (err) 2405 return ERR_PTR(err); 2406 2407 /* Module has been copied to its final place now: return it. */ 2408 mod = (void *)info->sechdrs[info->index.mod].sh_addr; 2409 kmemleak_load_module(mod, info); 2410 return mod; 2411 } 2412 2413 /* mod is no longer valid after this! */ 2414 static void module_deallocate(struct module *mod, struct load_info *info) 2415 { 2416 percpu_modfree(mod); 2417 module_arch_freeing_init(mod); 2418 2419 free_mod_mem(mod); 2420 } 2421 2422 int __weak module_finalize(const Elf_Ehdr *hdr, 2423 const Elf_Shdr *sechdrs, 2424 struct module *me) 2425 { 2426 return 0; 2427 } 2428 2429 static int post_relocation(struct module *mod, const struct load_info *info) 2430 { 2431 /* Sort exception table now relocations are done. */ 2432 sort_extable(mod->extable, mod->extable + mod->num_exentries); 2433 2434 /* Copy relocated percpu area over. */ 2435 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr, 2436 info->sechdrs[info->index.pcpu].sh_size); 2437 2438 /* Setup kallsyms-specific fields. */ 2439 add_kallsyms(mod, info); 2440 2441 /* Arch-specific module finalizing. */ 2442 return module_finalize(info->hdr, info->sechdrs, mod); 2443 } 2444 2445 /* Is this module of this name done loading? No locks held. */ 2446 static bool finished_loading(const char *name) 2447 { 2448 struct module *mod; 2449 bool ret; 2450 2451 /* 2452 * The module_mutex should not be a heavily contended lock; 2453 * if we get the occasional sleep here, we'll go an extra iteration 2454 * in the wait_event_interruptible(), which is harmless. 2455 */ 2456 sched_annotate_sleep(); 2457 mutex_lock(&module_mutex); 2458 mod = find_module_all(name, strlen(name), true); 2459 ret = !mod || mod->state == MODULE_STATE_LIVE 2460 || mod->state == MODULE_STATE_GOING; 2461 mutex_unlock(&module_mutex); 2462 2463 return ret; 2464 } 2465 2466 /* Call module constructors. */ 2467 static void do_mod_ctors(struct module *mod) 2468 { 2469 #ifdef CONFIG_CONSTRUCTORS 2470 unsigned long i; 2471 2472 for (i = 0; i < mod->num_ctors; i++) 2473 mod->ctors[i](); 2474 #endif 2475 } 2476 2477 /* For freeing module_init on success, in case kallsyms traversing */ 2478 struct mod_initfree { 2479 struct llist_node node; 2480 void *init_text; 2481 void *init_data; 2482 void *init_rodata; 2483 }; 2484 2485 static void do_free_init(struct work_struct *w) 2486 { 2487 struct llist_node *pos, *n, *list; 2488 struct mod_initfree *initfree; 2489 2490 list = llist_del_all(&init_free_list); 2491 2492 synchronize_rcu(); 2493 2494 llist_for_each_safe(pos, n, list) { 2495 initfree = container_of(pos, struct mod_initfree, node); 2496 module_memfree(initfree->init_text); 2497 module_memfree(initfree->init_data); 2498 module_memfree(initfree->init_rodata); 2499 kfree(initfree); 2500 } 2501 } 2502 2503 #undef MODULE_PARAM_PREFIX 2504 #define MODULE_PARAM_PREFIX "module." 2505 /* Default value for module->async_probe_requested */ 2506 static bool async_probe; 2507 module_param(async_probe, bool, 0644); 2508 2509 /* 2510 * This is where the real work happens. 2511 * 2512 * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb 2513 * helper command 'lx-symbols'. 2514 */ 2515 static noinline int do_init_module(struct module *mod) 2516 { 2517 int ret = 0; 2518 struct mod_initfree *freeinit; 2519 2520 freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL); 2521 if (!freeinit) { 2522 ret = -ENOMEM; 2523 goto fail; 2524 } 2525 freeinit->init_text = mod->mem[MOD_INIT_TEXT].base; 2526 freeinit->init_data = mod->mem[MOD_INIT_DATA].base; 2527 freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base; 2528 2529 do_mod_ctors(mod); 2530 /* Start the module */ 2531 if (mod->init != NULL) 2532 ret = do_one_initcall(mod->init); 2533 if (ret < 0) { 2534 goto fail_free_freeinit; 2535 } 2536 if (ret > 0) { 2537 pr_warn("%s: '%s'->init suspiciously returned %d, it should " 2538 "follow 0/-E convention\n" 2539 "%s: loading module anyway...\n", 2540 __func__, mod->name, ret, __func__); 2541 dump_stack(); 2542 } 2543 2544 /* Now it's a first class citizen! */ 2545 mod->state = MODULE_STATE_LIVE; 2546 blocking_notifier_call_chain(&module_notify_list, 2547 MODULE_STATE_LIVE, mod); 2548 2549 /* Delay uevent until module has finished its init routine */ 2550 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); 2551 2552 /* 2553 * We need to finish all async code before the module init sequence 2554 * is done. This has potential to deadlock if synchronous module 2555 * loading is requested from async (which is not allowed!). 2556 * 2557 * See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous 2558 * request_module() from async workers") for more details. 2559 */ 2560 if (!mod->async_probe_requested) 2561 async_synchronize_full(); 2562 2563 ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base, 2564 mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size); 2565 mutex_lock(&module_mutex); 2566 /* Drop initial reference. */ 2567 module_put(mod); 2568 trim_init_extable(mod); 2569 #ifdef CONFIG_KALLSYMS 2570 /* Switch to core kallsyms now init is done: kallsyms may be walking! */ 2571 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); 2572 #endif 2573 module_enable_ro(mod, true); 2574 mod_tree_remove_init(mod); 2575 module_arch_freeing_init(mod); 2576 for_class_mod_mem_type(type, init) { 2577 mod->mem[type].base = NULL; 2578 mod->mem[type].size = 0; 2579 } 2580 #ifdef CONFIG_DEBUG_INFO_BTF_MODULES 2581 /* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */ 2582 mod->btf_data = NULL; 2583 #endif 2584 /* 2585 * We want to free module_init, but be aware that kallsyms may be 2586 * walking this with preempt disabled. In all the failure paths, we 2587 * call synchronize_rcu(), but we don't want to slow down the success 2588 * path. module_memfree() cannot be called in an interrupt, so do the 2589 * work and call synchronize_rcu() in a work queue. 2590 * 2591 * Note that module_alloc() on most architectures creates W+X page 2592 * mappings which won't be cleaned up until do_free_init() runs. Any 2593 * code such as mark_rodata_ro() which depends on those mappings to 2594 * be cleaned up needs to sync with the queued work - ie 2595 * rcu_barrier() 2596 */ 2597 if (llist_add(&freeinit->node, &init_free_list)) 2598 schedule_work(&init_free_wq); 2599 2600 mutex_unlock(&module_mutex); 2601 wake_up_all(&module_wq); 2602 2603 return 0; 2604 2605 fail_free_freeinit: 2606 kfree(freeinit); 2607 fail: 2608 /* Try to protect us from buggy refcounters. */ 2609 mod->state = MODULE_STATE_GOING; 2610 synchronize_rcu(); 2611 module_put(mod); 2612 blocking_notifier_call_chain(&module_notify_list, 2613 MODULE_STATE_GOING, mod); 2614 klp_module_going(mod); 2615 ftrace_release_mod(mod); 2616 free_module(mod); 2617 wake_up_all(&module_wq); 2618 return ret; 2619 } 2620 2621 static int may_init_module(void) 2622 { 2623 if (!capable(CAP_SYS_MODULE) || modules_disabled) 2624 return -EPERM; 2625 2626 return 0; 2627 } 2628 2629 /* 2630 * We try to place it in the list now to make sure it's unique before 2631 * we dedicate too many resources. In particular, temporary percpu 2632 * memory exhaustion. 2633 */ 2634 static int add_unformed_module(struct module *mod) 2635 { 2636 int err; 2637 struct module *old; 2638 2639 mod->state = MODULE_STATE_UNFORMED; 2640 2641 mutex_lock(&module_mutex); 2642 old = find_module_all(mod->name, strlen(mod->name), true); 2643 if (old != NULL) { 2644 if (old->state == MODULE_STATE_COMING 2645 || old->state == MODULE_STATE_UNFORMED) { 2646 /* Wait in case it fails to load. */ 2647 mutex_unlock(&module_mutex); 2648 err = wait_event_interruptible(module_wq, 2649 finished_loading(mod->name)); 2650 if (err) 2651 goto out_unlocked; 2652 2653 /* The module might have gone in the meantime. */ 2654 mutex_lock(&module_mutex); 2655 old = find_module_all(mod->name, strlen(mod->name), 2656 true); 2657 } 2658 2659 /* 2660 * We are here only when the same module was being loaded. Do 2661 * not try to load it again right now. It prevents long delays 2662 * caused by serialized module load failures. It might happen 2663 * when more devices of the same type trigger load of 2664 * a particular module. 2665 */ 2666 if (old && old->state == MODULE_STATE_LIVE) 2667 err = -EEXIST; 2668 else 2669 err = -EBUSY; 2670 goto out; 2671 } 2672 mod_update_bounds(mod); 2673 list_add_rcu(&mod->list, &modules); 2674 mod_tree_insert(mod); 2675 err = 0; 2676 2677 out: 2678 mutex_unlock(&module_mutex); 2679 out_unlocked: 2680 return err; 2681 } 2682 2683 static int complete_formation(struct module *mod, struct load_info *info) 2684 { 2685 int err; 2686 2687 mutex_lock(&module_mutex); 2688 2689 /* Find duplicate symbols (must be called under lock). */ 2690 err = verify_exported_symbols(mod); 2691 if (err < 0) 2692 goto out; 2693 2694 /* These rely on module_mutex for list integrity. */ 2695 module_bug_finalize(info->hdr, info->sechdrs, mod); 2696 module_cfi_finalize(info->hdr, info->sechdrs, mod); 2697 2698 module_enable_ro(mod, false); 2699 module_enable_nx(mod); 2700 module_enable_x(mod); 2701 2702 /* 2703 * Mark state as coming so strong_try_module_get() ignores us, 2704 * but kallsyms etc. can see us. 2705 */ 2706 mod->state = MODULE_STATE_COMING; 2707 mutex_unlock(&module_mutex); 2708 2709 return 0; 2710 2711 out: 2712 mutex_unlock(&module_mutex); 2713 return err; 2714 } 2715 2716 static int prepare_coming_module(struct module *mod) 2717 { 2718 int err; 2719 2720 ftrace_module_enable(mod); 2721 err = klp_module_coming(mod); 2722 if (err) 2723 return err; 2724 2725 err = blocking_notifier_call_chain_robust(&module_notify_list, 2726 MODULE_STATE_COMING, MODULE_STATE_GOING, mod); 2727 err = notifier_to_errno(err); 2728 if (err) 2729 klp_module_going(mod); 2730 2731 return err; 2732 } 2733 2734 static int unknown_module_param_cb(char *param, char *val, const char *modname, 2735 void *arg) 2736 { 2737 struct module *mod = arg; 2738 int ret; 2739 2740 if (strcmp(param, "async_probe") == 0) { 2741 if (kstrtobool(val, &mod->async_probe_requested)) 2742 mod->async_probe_requested = true; 2743 return 0; 2744 } 2745 2746 /* Check for magic 'dyndbg' arg */ 2747 ret = ddebug_dyndbg_module_param_cb(param, val, modname); 2748 if (ret != 0) 2749 pr_warn("%s: unknown parameter '%s' ignored\n", modname, param); 2750 return 0; 2751 } 2752 2753 /* Module within temporary copy, this doesn't do any allocation */ 2754 static int early_mod_check(struct load_info *info, int flags) 2755 { 2756 int err; 2757 2758 /* 2759 * Now that we know we have the correct module name, check 2760 * if it's blacklisted. 2761 */ 2762 if (blacklisted(info->name)) { 2763 pr_err("Module %s is blacklisted\n", info->name); 2764 return -EPERM; 2765 } 2766 2767 err = rewrite_section_headers(info, flags); 2768 if (err) 2769 return err; 2770 2771 /* Check module struct version now, before we try to use module. */ 2772 if (!check_modstruct_version(info, info->mod)) 2773 return -ENOEXEC; 2774 2775 err = check_modinfo(info->mod, info, flags); 2776 if (err) 2777 return err; 2778 2779 return 0; 2780 } 2781 2782 /* 2783 * Allocate and load the module: note that size of section 0 is always 2784 * zero, and we rely on this for optional sections. 2785 */ 2786 static int load_module(struct load_info *info, const char __user *uargs, 2787 int flags) 2788 { 2789 struct module *mod; 2790 long err = 0; 2791 char *after_dashes; 2792 2793 /* 2794 * Do the signature check (if any) first. All that 2795 * the signature check needs is info->len, it does 2796 * not need any of the section info. That can be 2797 * set up later. This will minimize the chances 2798 * of a corrupt module causing problems before 2799 * we even get to the signature check. 2800 * 2801 * The check will also adjust info->len by stripping 2802 * off the sig length at the end of the module, making 2803 * checks against info->len more correct. 2804 */ 2805 err = module_sig_check(info, flags); 2806 if (err) 2807 goto free_copy; 2808 2809 /* 2810 * Do basic sanity checks against the ELF header and 2811 * sections. Cache useful sections and set the 2812 * info->mod to the userspace passed struct module. 2813 */ 2814 err = elf_validity_cache_copy(info, flags); 2815 if (err) 2816 goto free_copy; 2817 2818 err = early_mod_check(info, flags); 2819 if (err) 2820 goto free_copy; 2821 2822 /* Figure out module layout, and allocate all the memory. */ 2823 mod = layout_and_allocate(info, flags); 2824 if (IS_ERR(mod)) { 2825 err = PTR_ERR(mod); 2826 goto free_copy; 2827 } 2828 2829 audit_log_kern_module(mod->name); 2830 2831 /* Reserve our place in the list. */ 2832 err = add_unformed_module(mod); 2833 if (err) 2834 goto free_module; 2835 2836 /* 2837 * We are tainting your kernel if your module gets into 2838 * the modules linked list somehow. 2839 */ 2840 module_augment_kernel_taints(mod, info); 2841 2842 /* To avoid stressing percpu allocator, do this once we're unique. */ 2843 err = percpu_modalloc(mod, info); 2844 if (err) 2845 goto unlink_mod; 2846 2847 /* Now module is in final location, initialize linked lists, etc. */ 2848 err = module_unload_init(mod); 2849 if (err) 2850 goto unlink_mod; 2851 2852 init_param_lock(mod); 2853 2854 /* 2855 * Now we've got everything in the final locations, we can 2856 * find optional sections. 2857 */ 2858 err = find_module_sections(mod, info); 2859 if (err) 2860 goto free_unload; 2861 2862 err = check_export_symbol_versions(mod); 2863 if (err) 2864 goto free_unload; 2865 2866 /* Set up MODINFO_ATTR fields */ 2867 setup_modinfo(mod, info); 2868 2869 /* Fix up syms, so that st_value is a pointer to location. */ 2870 err = simplify_symbols(mod, info); 2871 if (err < 0) 2872 goto free_modinfo; 2873 2874 err = apply_relocations(mod, info); 2875 if (err < 0) 2876 goto free_modinfo; 2877 2878 err = post_relocation(mod, info); 2879 if (err < 0) 2880 goto free_modinfo; 2881 2882 flush_module_icache(mod); 2883 2884 /* Now copy in args */ 2885 mod->args = strndup_user(uargs, ~0UL >> 1); 2886 if (IS_ERR(mod->args)) { 2887 err = PTR_ERR(mod->args); 2888 goto free_arch_cleanup; 2889 } 2890 2891 init_build_id(mod, info); 2892 2893 /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */ 2894 ftrace_module_init(mod); 2895 2896 /* Finally it's fully formed, ready to start executing. */ 2897 err = complete_formation(mod, info); 2898 if (err) 2899 goto ddebug_cleanup; 2900 2901 err = prepare_coming_module(mod); 2902 if (err) 2903 goto bug_cleanup; 2904 2905 mod->async_probe_requested = async_probe; 2906 2907 /* Module is ready to execute: parsing args may do that. */ 2908 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, 2909 -32768, 32767, mod, 2910 unknown_module_param_cb); 2911 if (IS_ERR(after_dashes)) { 2912 err = PTR_ERR(after_dashes); 2913 goto coming_cleanup; 2914 } else if (after_dashes) { 2915 pr_warn("%s: parameters '%s' after `--' ignored\n", 2916 mod->name, after_dashes); 2917 } 2918 2919 /* Link in to sysfs. */ 2920 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp); 2921 if (err < 0) 2922 goto coming_cleanup; 2923 2924 if (is_livepatch_module(mod)) { 2925 err = copy_module_elf(mod, info); 2926 if (err < 0) 2927 goto sysfs_cleanup; 2928 } 2929 2930 /* Get rid of temporary copy. */ 2931 free_copy(info, flags); 2932 2933 /* Done! */ 2934 trace_module_load(mod); 2935 2936 return do_init_module(mod); 2937 2938 sysfs_cleanup: 2939 mod_sysfs_teardown(mod); 2940 coming_cleanup: 2941 mod->state = MODULE_STATE_GOING; 2942 destroy_params(mod->kp, mod->num_kp); 2943 blocking_notifier_call_chain(&module_notify_list, 2944 MODULE_STATE_GOING, mod); 2945 klp_module_going(mod); 2946 bug_cleanup: 2947 mod->state = MODULE_STATE_GOING; 2948 /* module_bug_cleanup needs module_mutex protection */ 2949 mutex_lock(&module_mutex); 2950 module_bug_cleanup(mod); 2951 mutex_unlock(&module_mutex); 2952 2953 ddebug_cleanup: 2954 ftrace_release_mod(mod); 2955 synchronize_rcu(); 2956 kfree(mod->args); 2957 free_arch_cleanup: 2958 module_arch_cleanup(mod); 2959 free_modinfo: 2960 free_modinfo(mod); 2961 free_unload: 2962 module_unload_free(mod); 2963 unlink_mod: 2964 mutex_lock(&module_mutex); 2965 /* Unlink carefully: kallsyms could be walking list. */ 2966 list_del_rcu(&mod->list); 2967 mod_tree_remove(mod); 2968 wake_up_all(&module_wq); 2969 /* Wait for RCU-sched synchronizing before releasing mod->list. */ 2970 synchronize_rcu(); 2971 mutex_unlock(&module_mutex); 2972 free_module: 2973 /* Free lock-classes; relies on the preceding sync_rcu() */ 2974 for_class_mod_mem_type(type, core_data) { 2975 lockdep_free_key_range(mod->mem[type].base, 2976 mod->mem[type].size); 2977 } 2978 2979 module_deallocate(mod, info); 2980 free_copy: 2981 free_copy(info, flags); 2982 return err; 2983 } 2984 2985 SYSCALL_DEFINE3(init_module, void __user *, umod, 2986 unsigned long, len, const char __user *, uargs) 2987 { 2988 int err; 2989 struct load_info info = { }; 2990 2991 err = may_init_module(); 2992 if (err) 2993 return err; 2994 2995 pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n", 2996 umod, len, uargs); 2997 2998 err = copy_module_from_user(umod, len, &info); 2999 if (err) 3000 return err; 3001 3002 return load_module(&info, uargs, 0); 3003 } 3004 3005 SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) 3006 { 3007 struct load_info info = { }; 3008 void *buf = NULL; 3009 int len; 3010 int err; 3011 3012 err = may_init_module(); 3013 if (err) 3014 return err; 3015 3016 pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags); 3017 3018 if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS 3019 |MODULE_INIT_IGNORE_VERMAGIC 3020 |MODULE_INIT_COMPRESSED_FILE)) 3021 return -EINVAL; 3022 3023 len = kernel_read_file_from_fd(fd, 0, &buf, INT_MAX, NULL, 3024 READING_MODULE); 3025 if (len < 0) 3026 return len; 3027 3028 if (flags & MODULE_INIT_COMPRESSED_FILE) { 3029 err = module_decompress(&info, buf, len); 3030 vfree(buf); /* compressed data is no longer needed */ 3031 if (err) 3032 return err; 3033 } else { 3034 info.hdr = buf; 3035 info.len = len; 3036 } 3037 3038 return load_module(&info, uargs, flags); 3039 } 3040 3041 /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ 3042 char *module_flags(struct module *mod, char *buf, bool show_state) 3043 { 3044 int bx = 0; 3045 3046 BUG_ON(mod->state == MODULE_STATE_UNFORMED); 3047 if (!mod->taints && !show_state) 3048 goto out; 3049 if (mod->taints || 3050 mod->state == MODULE_STATE_GOING || 3051 mod->state == MODULE_STATE_COMING) { 3052 buf[bx++] = '('; 3053 bx += module_flags_taint(mod->taints, buf + bx); 3054 /* Show a - for module-is-being-unloaded */ 3055 if (mod->state == MODULE_STATE_GOING && show_state) 3056 buf[bx++] = '-'; 3057 /* Show a + for module-is-being-loaded */ 3058 if (mod->state == MODULE_STATE_COMING && show_state) 3059 buf[bx++] = '+'; 3060 buf[bx++] = ')'; 3061 } 3062 out: 3063 buf[bx] = '\0'; 3064 3065 return buf; 3066 } 3067 3068 /* Given an address, look for it in the module exception tables. */ 3069 const struct exception_table_entry *search_module_extables(unsigned long addr) 3070 { 3071 const struct exception_table_entry *e = NULL; 3072 struct module *mod; 3073 3074 preempt_disable(); 3075 mod = __module_address(addr); 3076 if (!mod) 3077 goto out; 3078 3079 if (!mod->num_exentries) 3080 goto out; 3081 3082 e = search_extable(mod->extable, 3083 mod->num_exentries, 3084 addr); 3085 out: 3086 preempt_enable(); 3087 3088 /* 3089 * Now, if we found one, we are running inside it now, hence 3090 * we cannot unload the module, hence no refcnt needed. 3091 */ 3092 return e; 3093 } 3094 3095 /** 3096 * is_module_address() - is this address inside a module? 3097 * @addr: the address to check. 3098 * 3099 * See is_module_text_address() if you simply want to see if the address 3100 * is code (not data). 3101 */ 3102 bool is_module_address(unsigned long addr) 3103 { 3104 bool ret; 3105 3106 preempt_disable(); 3107 ret = __module_address(addr) != NULL; 3108 preempt_enable(); 3109 3110 return ret; 3111 } 3112 3113 /** 3114 * __module_address() - get the module which contains an address. 3115 * @addr: the address. 3116 * 3117 * Must be called with preempt disabled or module mutex held so that 3118 * module doesn't get freed during this. 3119 */ 3120 struct module *__module_address(unsigned long addr) 3121 { 3122 struct module *mod; 3123 3124 if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max) 3125 goto lookup; 3126 3127 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 3128 if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max) 3129 goto lookup; 3130 #endif 3131 3132 return NULL; 3133 3134 lookup: 3135 module_assert_mutex_or_preempt(); 3136 3137 mod = mod_find(addr, &mod_tree); 3138 if (mod) { 3139 BUG_ON(!within_module(addr, mod)); 3140 if (mod->state == MODULE_STATE_UNFORMED) 3141 mod = NULL; 3142 } 3143 return mod; 3144 } 3145 3146 /** 3147 * is_module_text_address() - is this address inside module code? 3148 * @addr: the address to check. 3149 * 3150 * See is_module_address() if you simply want to see if the address is 3151 * anywhere in a module. See kernel_text_address() for testing if an 3152 * address corresponds to kernel or module code. 3153 */ 3154 bool is_module_text_address(unsigned long addr) 3155 { 3156 bool ret; 3157 3158 preempt_disable(); 3159 ret = __module_text_address(addr) != NULL; 3160 preempt_enable(); 3161 3162 return ret; 3163 } 3164 3165 /** 3166 * __module_text_address() - get the module whose code contains an address. 3167 * @addr: the address. 3168 * 3169 * Must be called with preempt disabled or module mutex held so that 3170 * module doesn't get freed during this. 3171 */ 3172 struct module *__module_text_address(unsigned long addr) 3173 { 3174 struct module *mod = __module_address(addr); 3175 if (mod) { 3176 /* Make sure it's within the text section. */ 3177 if (!within_module_mem_type(addr, mod, MOD_TEXT) && 3178 !within_module_mem_type(addr, mod, MOD_INIT_TEXT)) 3179 mod = NULL; 3180 } 3181 return mod; 3182 } 3183 3184 /* Don't grab lock, we're oopsing. */ 3185 void print_modules(void) 3186 { 3187 struct module *mod; 3188 char buf[MODULE_FLAGS_BUF_SIZE]; 3189 3190 printk(KERN_DEFAULT "Modules linked in:"); 3191 /* Most callers should already have preempt disabled, but make sure */ 3192 preempt_disable(); 3193 list_for_each_entry_rcu(mod, &modules, list) { 3194 if (mod->state == MODULE_STATE_UNFORMED) 3195 continue; 3196 pr_cont(" %s%s", mod->name, module_flags(mod, buf, true)); 3197 } 3198 3199 print_unloaded_tainted_modules(); 3200 preempt_enable(); 3201 if (last_unloaded_module.name[0]) 3202 pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name, 3203 last_unloaded_module.taints); 3204 pr_cont("\n"); 3205 } 3206