1 /* 2 * patch.c - livepatch patching functions 3 * 4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> 5 * Copyright (C) 2014 SUSE 6 * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com> 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License 10 * as published by the Free Software Foundation; either version 2 11 * of the License, or (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 23 24 #include <linux/livepatch.h> 25 #include <linux/list.h> 26 #include <linux/ftrace.h> 27 #include <linux/rculist.h> 28 #include <linux/slab.h> 29 #include <linux/bug.h> 30 #include <linux/printk.h> 31 #include "core.h" 32 #include "patch.h" 33 #include "transition.h" 34 35 static LIST_HEAD(klp_ops); 36 37 struct klp_ops *klp_find_ops(void *old_func) 38 { 39 struct klp_ops *ops; 40 struct klp_func *func; 41 42 list_for_each_entry(ops, &klp_ops, node) { 43 func = list_first_entry(&ops->func_stack, struct klp_func, 44 stack_node); 45 if (func->old_func == old_func) 46 return ops; 47 } 48 49 return NULL; 50 } 51 52 static void notrace klp_ftrace_handler(unsigned long ip, 53 unsigned long parent_ip, 54 struct ftrace_ops *fops, 55 struct pt_regs *regs) 56 { 57 struct klp_ops *ops; 58 struct klp_func *func; 59 int patch_state; 60 61 ops = container_of(fops, struct klp_ops, fops); 62 63 /* 64 * A variant of synchronize_rcu() is used to allow patching functions 65 * where RCU is not watching, see klp_synchronize_transition(). 66 */ 67 preempt_disable_notrace(); 68 69 func = list_first_or_null_rcu(&ops->func_stack, struct klp_func, 70 stack_node); 71 72 /* 73 * func should never be NULL because preemption should be disabled here 74 * and unregister_ftrace_function() does the equivalent of a 75 * synchronize_rcu() before the func_stack removal. 76 */ 77 if (WARN_ON_ONCE(!func)) 78 goto unlock; 79 80 /* 81 * In the enable path, enforce the order of the ops->func_stack and 82 * func->transition reads. The corresponding write barrier is in 83 * __klp_enable_patch(). 84 * 85 * (Note that this barrier technically isn't needed in the disable 86 * path. In the rare case where klp_update_patch_state() runs before 87 * this handler, its TIF_PATCH_PENDING read and this func->transition 88 * read need to be ordered. But klp_update_patch_state() already 89 * enforces that.) 90 */ 91 smp_rmb(); 92 93 if (unlikely(func->transition)) { 94 95 /* 96 * Enforce the order of the func->transition and 97 * current->patch_state reads. Otherwise we could read an 98 * out-of-date task state and pick the wrong function. The 99 * corresponding write barrier is in klp_init_transition(). 100 */ 101 smp_rmb(); 102 103 patch_state = current->patch_state; 104 105 WARN_ON_ONCE(patch_state == KLP_UNDEFINED); 106 107 if (patch_state == KLP_UNPATCHED) { 108 /* 109 * Use the previously patched version of the function. 110 * If no previous patches exist, continue with the 111 * original function. 112 */ 113 func = list_entry_rcu(func->stack_node.next, 114 struct klp_func, stack_node); 115 116 if (&func->stack_node == &ops->func_stack) 117 goto unlock; 118 } 119 } 120 121 klp_arch_set_pc(regs, (unsigned long)func->new_func); 122 unlock: 123 preempt_enable_notrace(); 124 } 125 126 /* 127 * Convert a function address into the appropriate ftrace location. 128 * 129 * Usually this is just the address of the function, but on some architectures 130 * it's more complicated so allow them to provide a custom behaviour. 131 */ 132 #ifndef klp_get_ftrace_location 133 static unsigned long klp_get_ftrace_location(unsigned long faddr) 134 { 135 return faddr; 136 } 137 #endif 138 139 static void klp_unpatch_func(struct klp_func *func) 140 { 141 struct klp_ops *ops; 142 143 if (WARN_ON(!func->patched)) 144 return; 145 if (WARN_ON(!func->old_func)) 146 return; 147 148 ops = klp_find_ops(func->old_func); 149 if (WARN_ON(!ops)) 150 return; 151 152 if (list_is_singular(&ops->func_stack)) { 153 unsigned long ftrace_loc; 154 155 ftrace_loc = 156 klp_get_ftrace_location((unsigned long)func->old_func); 157 if (WARN_ON(!ftrace_loc)) 158 return; 159 160 WARN_ON(unregister_ftrace_function(&ops->fops)); 161 WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0)); 162 163 list_del_rcu(&func->stack_node); 164 list_del(&ops->node); 165 kfree(ops); 166 } else { 167 list_del_rcu(&func->stack_node); 168 } 169 170 func->patched = false; 171 } 172 173 static int klp_patch_func(struct klp_func *func) 174 { 175 struct klp_ops *ops; 176 int ret; 177 178 if (WARN_ON(!func->old_func)) 179 return -EINVAL; 180 181 if (WARN_ON(func->patched)) 182 return -EINVAL; 183 184 ops = klp_find_ops(func->old_func); 185 if (!ops) { 186 unsigned long ftrace_loc; 187 188 ftrace_loc = 189 klp_get_ftrace_location((unsigned long)func->old_func); 190 if (!ftrace_loc) { 191 pr_err("failed to find location for function '%s'\n", 192 func->old_name); 193 return -EINVAL; 194 } 195 196 ops = kzalloc(sizeof(*ops), GFP_KERNEL); 197 if (!ops) 198 return -ENOMEM; 199 200 ops->fops.func = klp_ftrace_handler; 201 ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS | 202 FTRACE_OPS_FL_DYNAMIC | 203 FTRACE_OPS_FL_IPMODIFY; 204 205 list_add(&ops->node, &klp_ops); 206 207 INIT_LIST_HEAD(&ops->func_stack); 208 list_add_rcu(&func->stack_node, &ops->func_stack); 209 210 ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0); 211 if (ret) { 212 pr_err("failed to set ftrace filter for function '%s' (%d)\n", 213 func->old_name, ret); 214 goto err; 215 } 216 217 ret = register_ftrace_function(&ops->fops); 218 if (ret) { 219 pr_err("failed to register ftrace handler for function '%s' (%d)\n", 220 func->old_name, ret); 221 ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0); 222 goto err; 223 } 224 225 226 } else { 227 list_add_rcu(&func->stack_node, &ops->func_stack); 228 } 229 230 func->patched = true; 231 232 return 0; 233 234 err: 235 list_del_rcu(&func->stack_node); 236 list_del(&ops->node); 237 kfree(ops); 238 return ret; 239 } 240 241 void klp_unpatch_object(struct klp_object *obj) 242 { 243 struct klp_func *func; 244 245 klp_for_each_func(obj, func) 246 if (func->patched) 247 klp_unpatch_func(func); 248 249 obj->patched = false; 250 } 251 252 int klp_patch_object(struct klp_object *obj) 253 { 254 struct klp_func *func; 255 int ret; 256 257 if (WARN_ON(obj->patched)) 258 return -EINVAL; 259 260 klp_for_each_func(obj, func) { 261 ret = klp_patch_func(func); 262 if (ret) { 263 klp_unpatch_object(obj); 264 return ret; 265 } 266 } 267 obj->patched = true; 268 269 return 0; 270 } 271 272 void klp_unpatch_objects(struct klp_patch *patch) 273 { 274 struct klp_object *obj; 275 276 klp_for_each_object(patch, obj) 277 if (obj->patched) 278 klp_unpatch_object(obj); 279 } 280