1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * core.c - Kernel Live Patching Core
31  * accesses to klp-related variables and structures must have mutex protection,
34  * - klp_ftrace_handler()
35  * - klp_update_patch_state()
36  * - __klp_sched_try_switch()
41  * Actively used patches: enabled or in transition. Note that replaced
51 	return obj->name;  in klp_is_module()
54 /* sets obj->mod if object is not vmlinux and module is found */
68 	mod = find_module(obj->name);  in klp_find_object_module()
73 	 * until mod->exit() finishes. This is especially important for  in klp_find_object_module()
76 	if (mod && mod->klp_alive)  in klp_find_object_module()
77 		obj->mod = mod;  in klp_find_object_module()
93 		if ((strcmp(old_func->old_name, func->old_name) == 0) &&  in klp_find_func()
94 		    (old_func->old_sympos == func->old_sympos)) {  in klp_find_func()
110 			    strcmp(old_obj->name, obj->name) == 0) {  in klp_find_object()
132 	args->addr = addr;  in klp_match_callback()
133 	args->count++;  in klp_match_callback()
137 	 * or the position is not defined for a non-unique symbol.  in klp_match_callback()
139 	if ((args->pos && (args->count == args->pos)) ||  in klp_match_callback()
140 	    (!args->pos && (args->count > 1)))  in klp_match_callback()
150 	if (strcmp(args->name, name))  in klp_find_callback()
189 	return -EINVAL;  in klp_find_object_symbol()
207 	 * call are hard-coded and correspond to MODULE_NAME_LEN and  in klp_resolve_symbols()
217 	relas = (Elf_Rela *) relasec->sh_addr;  in klp_resolve_symbols()
219 	for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {  in klp_resolve_symbols()
221 		if (sym->st_shndx != SHN_LIVEPATCH) {  in klp_resolve_symbols()
223 			       strtab + sym->st_name);  in klp_resolve_symbols()
224 			return -EINVAL;  in klp_resolve_symbols()
228 		cnt = sscanf(strtab + sym->st_name,  in klp_resolve_symbols()
233 			       strtab + sym->st_name);  in klp_resolve_symbols()
234 			return -EINVAL;  in klp_resolve_symbols()
240 		 * Prevent module-specific KLP rela sections from referencing  in klp_resolve_symbols()
246 …pr_err("invalid access to vmlinux symbol '%s' from module-specific livepatch relocation section\n",  in klp_resolve_symbols()
248 			return -EINVAL;  in klp_resolve_symbols()
257 		sym->st_value = addr;  in klp_resolve_symbols()
272  * At a high-level, there are two types of klp relocation sections: those which
277  *    write vmlinux-specific klp relocations (.klp.rela.vmlinux.* sections).
284  * 2) When a to-be-patched module loads -- or is already loaded when a
285  *    corresponding klp module loads -- klp code calls this function to write
286  *    module-specific klp relocations (.klp.rela.{module}.* sections).  These
288  *    reference symbols which live in the to-be-patched module or one of its
291  *    the to-be-patched module to be loaded and patched sometime *after* the
294 static int klp_write_section_relocs(struct module *pmod, Elf_Shdr *sechdrs,  in klp_write_section_relocs()  argument
308 	cnt = sscanf(shstrtab + sec->sh_name, ".klp.rela.%55[^.]",  in klp_write_section_relocs()
312 		       shstrtab + sec->sh_name);  in klp_write_section_relocs()
313 		return -EINVAL;  in klp_write_section_relocs()
325 		return apply_relocate_add(sechdrs, strtab, symndx, secndx, pmod);  in klp_write_section_relocs()
328 	clear_relocate_add(sechdrs, strtab, symndx, secndx, pmod);  in klp_write_section_relocs()
332 int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs,  in klp_apply_section_relocs()  argument
337 	return klp_write_section_relocs(pmod, sechdrs, shstrtab, strtab, symndx,  in klp_apply_section_relocs()
346  * /sys/kernel/livepatch/<patch>/enabled
360 	bool enabled;  in enabled_store()  local
362 	ret = kstrtobool(buf, &enabled);  in enabled_store()
370 	if (patch->enabled == enabled) {  in enabled_store()
372 		ret = -EINVAL;  in enabled_store()
381 	 * Do not allow to re-enable a disabled patch.  in enabled_store()
385 	else if (!enabled)  in enabled_store()
388 		ret = -EINVAL;  in enabled_store()
404 	return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);  in enabled_show()
413 	return snprintf(buf, PAGE_SIZE-1, "%d\n",  in transition_show()
436 		return -EINVAL;  in force_store()
446 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
463 	return sysfs_emit(buf, "%d\n", obj->patched);  in patched_show()
475 	kfree(obj->name);  in klp_free_object_dynamic()
494 		obj->name = kstrdup(name, GFP_KERNEL);  in klp_alloc_object_dynamic()
495 		if (!obj->name) {  in klp_alloc_object_dynamic()
502 	obj->dynamic = true;  in klp_alloc_object_dynamic()
509 	kfree(func->old_name);  in klp_free_func_nop()
522 	if (old_func->old_name) {  in klp_alloc_func_nop()
523 		func->old_name = kstrdup(old_func->old_name, GFP_KERNEL);  in klp_alloc_func_nop()
524 		if (!func->old_name) {  in klp_alloc_func_nop()
532 	 * func->new_func is same as func->old_func. These addresses are  in klp_alloc_func_nop()
535 	func->old_sympos = old_func->old_sympos;  in klp_alloc_func_nop()
536 	func->nop = true;  in klp_alloc_func_nop()
550 		obj = klp_alloc_object_dynamic(old_obj->name, patch);  in klp_add_object_nops()
552 			return -ENOMEM;  in klp_add_object_nops()
562 			return -ENOMEM;  in klp_add_object_nops()
596 	complete(&patch->finish);  in klp_kobj_release_patch()
611 	if (obj->dynamic)  in klp_kobj_release_object()
627 	if (func->nop)  in klp_kobj_release_func()
641 		if (nops_only && !func->nop)  in __klp_free_funcs()
644 		list_del(&func->node);  in __klp_free_funcs()
645 		kobject_put(&func->kobj);  in __klp_free_funcs()
654 	obj->mod = NULL;  in klp_free_object_loaded()
657 		func->old_func = NULL;  in klp_free_object_loaded()
659 		if (func->nop)  in klp_free_object_loaded()
660 			func->new_func = NULL;  in klp_free_object_loaded()
671 		if (nops_only && !obj->dynamic)  in __klp_free_objects()
674 		list_del(&obj->node);  in __klp_free_objects()
675 		kobject_put(&obj->kobj);  in __klp_free_objects()
698 	if (!list_empty(&patch->list))  in klp_free_patch_start()
699 		list_del(&patch->list);  in klp_free_patch_start()
718 	 * cannot get enabled again.  in klp_free_patch_finish()
720 	kobject_put(&patch->kobj);  in klp_free_patch_finish()
721 	wait_for_completion(&patch->finish);  in klp_free_patch_finish()
724 	if (!patch->forced)  in klp_free_patch_finish()
725 		module_put(patch->mod);  in klp_free_patch_finish()
744 	schedule_work(&patch->free_work);  in klp_free_patch_async()
760 	if (!func->old_name)  in klp_init_func()
761 		return -EINVAL;  in klp_init_func()
767 	if (!func->new_func && !func->nop)  in klp_init_func()
768 		return -EINVAL;  in klp_init_func()
770 	if (strlen(func->old_name) >= KSYM_NAME_LEN)  in klp_init_func()
771 		return -EINVAL;  in klp_init_func()
773 	INIT_LIST_HEAD(&func->stack_node);  in klp_init_func()
774 	func->patched = false;  in klp_init_func()
775 	func->transition = false;  in klp_init_func()
782 	return kobject_add(&func->kobj, &obj->kobj, "%s,%lu",  in klp_init_func()
783 			   func->old_name,  in klp_init_func()
784 			   func->old_sympos ? func->old_sympos : 1);  in klp_init_func()
792 	struct klp_modinfo *info = patch->mod->klp_info;  in klp_write_object_relocs()
794 	for (i = 1; i < info->hdr.e_shnum; i++) {  in klp_write_object_relocs()
795 		Elf_Shdr *sec = info->sechdrs + i;  in klp_write_object_relocs()
797 		if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))  in klp_write_object_relocs()
800 		ret = klp_write_section_relocs(patch->mod, info->sechdrs,  in klp_write_object_relocs()
801 					       info->secstrings,  in klp_write_object_relocs()
802 					       patch->mod->core_kallsyms.strtab,  in klp_write_object_relocs()
803 					       info->symndx, i, obj->name, apply);  in klp_write_object_relocs()
832 		 * Only write module-specific relocations here  in klp_init_object_loaded()
833 		 * (.klp.rela.{module}.*).  vmlinux-specific relocations were  in klp_init_object_loaded()
843 		ret = klp_find_object_symbol(obj->name, func->old_name,  in klp_init_object_loaded()
844 					     func->old_sympos,  in klp_init_object_loaded()
845 					     (unsigned long *)&func->old_func);  in klp_init_object_loaded()
849 		ret = kallsyms_lookup_size_offset((unsigned long)func->old_func,  in klp_init_object_loaded()
850 						  &func->old_size, NULL);  in klp_init_object_loaded()
853 			       func->old_name);  in klp_init_object_loaded()
854 			return -ENOENT;  in klp_init_object_loaded()
857 		if (func->nop)  in klp_init_object_loaded()
858 			func->new_func = func->old_func;  in klp_init_object_loaded()
860 		ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,  in klp_init_object_loaded()
861 						  &func->new_size, NULL);  in klp_init_object_loaded()
864 			       func->old_name);  in klp_init_object_loaded()
865 			return -ENOENT;  in klp_init_object_loaded()
878 	if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN)  in klp_init_object()
879 		return -EINVAL;  in klp_init_object()
881 	obj->patched = false;  in klp_init_object()
882 	obj->mod = NULL;  in klp_init_object()
886 	name = klp_is_module(obj) ? obj->name : "vmlinux";  in klp_init_object()
887 	ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name);  in klp_init_object()
906 	kobject_init(&func->kobj, &klp_ktype_func);  in klp_init_func_early()
907 	list_add_tail(&func->node, &obj->func_list);  in klp_init_func_early()
913 	INIT_LIST_HEAD(&obj->func_list);  in klp_init_object_early()
914 	kobject_init(&obj->kobj, &klp_ktype_object);  in klp_init_object_early()
915 	list_add_tail(&obj->node, &patch->obj_list);  in klp_init_object_early()
923 	INIT_LIST_HEAD(&patch->list);  in klp_init_patch_early()
924 	INIT_LIST_HEAD(&patch->obj_list);  in klp_init_patch_early()
925 	kobject_init(&patch->kobj, &klp_ktype_patch);  in klp_init_patch_early()
926 	patch->enabled = false;  in klp_init_patch_early()
927 	patch->forced = false;  in klp_init_patch_early()
928 	INIT_WORK(&patch->free_work, klp_free_patch_work_fn);  in klp_init_patch_early()
929 	init_completion(&patch->finish);  in klp_init_patch_early()
945 	ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name);  in klp_init_patch()
949 	if (patch->replace) {  in klp_init_patch()
961 	list_add_tail(&patch->list, &klp_patches);  in klp_init_patch()
970 	if (WARN_ON(!patch->enabled))  in __klp_disable_patch()
971 		return -EINVAL;  in __klp_disable_patch()
974 		return -EBUSY;  in __klp_disable_patch()
979 		if (obj->patched)  in __klp_disable_patch()
983 	 * Enforce the order of the func->transition writes in  in __klp_disable_patch()
987 	 * this ensures the handler sees that func->transition is set.  in __klp_disable_patch()
992 	patch->enabled = false;  in __klp_disable_patch()
1004 		return -EBUSY;  in __klp_enable_patch()
1006 	if (WARN_ON(patch->enabled))  in __klp_enable_patch()
1007 		return -EINVAL;  in __klp_enable_patch()
1009 	pr_notice("enabling patch '%s'\n", patch->mod->name);  in __klp_enable_patch()
1014 	 * Enforce the order of the func->transition writes in  in __klp_enable_patch()
1015 	 * klp_init_transition() and the ops->func_stack writes in  in __klp_enable_patch()
1017 	 * func->transition updates before the handler is registered and the  in __klp_enable_patch()
1028 			pr_warn("pre-patch callback failed for object '%s'\n",  in __klp_enable_patch()
1029 				klp_is_module(obj) ? obj->name : "vmlinux");  in __klp_enable_patch()
1036 				klp_is_module(obj) ? obj->name : "vmlinux");  in __klp_enable_patch()
1042 	patch->enabled = true;  in __klp_enable_patch()
1047 	pr_warn("failed to enable patch '%s'\n", patch->mod->name);  in __klp_enable_patch()
1054  * klp_enable_patch() - enable the livepatch
1055  * @patch:	patch to be enabled
1071 	if (!patch || !patch->mod || !patch->objs)  in klp_enable_patch()
1072 		return -EINVAL;  in klp_enable_patch()
1075 		if (!obj->funcs)  in klp_enable_patch()
1076 			return -EINVAL;  in klp_enable_patch()
1080 	if (!is_livepatch_module(patch->mod)) {  in klp_enable_patch()
1082 		       patch->mod->name);  in klp_enable_patch()
1083 		return -EINVAL;  in klp_enable_patch()
1087 		return -ENODEV;  in klp_enable_patch()
1098 			patch->mod->name);  in klp_enable_patch()
1100 		return -EINVAL;  in klp_enable_patch()
1103 	if (!try_module_get(patch->mod)) {  in klp_enable_patch()
1105 		return -ENODEV;  in klp_enable_patch()
1146  * this is handled transparently by patch->module_put.
1156 		old_patch->enabled = false;  in klp_unpatch_replaced_patches()
1168  * will see a valid ops->func_stack entry thanks to RCU.
1171  * in ops->func_stack. Therefore unregister_ftrace_function() is called.
1199 			if (!klp_is_module(obj) || strcmp(obj->name, mod->name))  in klp_cleanup_module_patches_limited()
1206 				  patch->mod->name, obj->mod->name);  in klp_cleanup_module_patches_limited()
1223 	if (WARN_ON(mod->state != MODULE_STATE_COMING))  in klp_module_coming()
1224 		return -EINVAL;  in klp_module_coming()
1226 	if (!strcmp(mod->name, "vmlinux")) {  in klp_module_coming()
1228 		return -EINVAL;  in klp_module_coming()
1237 	mod->klp_alive = true;  in klp_module_coming()
1241 			if (!klp_is_module(obj) || strcmp(obj->name, mod->name))  in klp_module_coming()
1244 			obj->mod = mod;  in klp_module_coming()
1249 					patch->mod->name, obj->mod->name, ret);  in klp_module_coming()
1254 				  patch->mod->name, obj->mod->name);  in klp_module_coming()
1258 				pr_warn("pre-patch callback failed for object '%s'\n",  in klp_module_coming()
1259 					obj->name);  in klp_module_coming()
1266 					patch->mod->name, obj->mod->name, ret);  in klp_module_coming()
1289 		patch->mod->name, obj->mod->name, obj->mod->name);  in klp_module_coming()
1290 	mod->klp_alive = false;  in klp_module_coming()
1291 	obj->mod = NULL;  in klp_module_coming()
1300 	if (WARN_ON(mod->state != MODULE_STATE_GOING &&  in klp_module_going()
1301 		    mod->state != MODULE_STATE_COMING))  in klp_module_going()
1310 	mod->klp_alive = false;  in klp_module_going()
1321 		return -ENOMEM;  in klp_init()