1 /* 2 * shadow.c - Shadow Variables 3 * 4 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com> 5 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> 6 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com> 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License 10 * as published by the Free Software Foundation; either version 2 11 * of the License, or (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 /** 23 * DOC: Shadow variable API concurrency notes: 24 * 25 * The shadow variable API provides a simple relationship between an 26 * <obj, id> pair and a pointer value. It is the responsibility of the 27 * caller to provide any mutual exclusion required of the shadow data. 28 * 29 * Once a shadow variable is attached to its parent object via the 30 * klp_shadow_*alloc() API calls, it is considered live: any subsequent 31 * call to klp_shadow_get() may then return the shadow variable's data 32 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data 33 * accordingly. 34 * 35 * The klp_shadow_*alloc() API calls may allocate memory for new shadow 36 * variable structures. Their implementation does not call kmalloc 37 * inside any spinlocks, but API callers should pass GFP flags according 38 * to their specific needs. 39 * 40 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against 41 * concurrent klp_shadow_free() and klp_shadow_get() operations. 42 */ 43 44 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 45 46 #include <linux/hashtable.h> 47 #include <linux/slab.h> 48 #include <linux/livepatch.h> 49 50 static DEFINE_HASHTABLE(klp_shadow_hash, 12); 51 52 /* 53 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and 54 * the shadow variables it references. 55 */ 56 static DEFINE_SPINLOCK(klp_shadow_lock); 57 58 /** 59 * struct klp_shadow - shadow variable structure 60 * @node: klp_shadow_hash hash table node 61 * @rcu_head: RCU is used to safely free this structure 62 * @obj: pointer to parent object 63 * @id: data identifier 64 * @data: data area 65 */ 66 struct klp_shadow { 67 struct hlist_node node; 68 struct rcu_head rcu_head; 69 void *obj; 70 unsigned long id; 71 char data[]; 72 }; 73 74 /** 75 * klp_shadow_match() - verify a shadow variable matches given <obj, id> 76 * @shadow: shadow variable to match 77 * @obj: pointer to parent object 78 * @id: data identifier 79 * 80 * Return: true if the shadow variable matches. 81 */ 82 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj, 83 unsigned long id) 84 { 85 return shadow->obj == obj && shadow->id == id; 86 } 87 88 /** 89 * klp_shadow_get() - retrieve a shadow variable data pointer 90 * @obj: pointer to parent object 91 * @id: data identifier 92 * 93 * Return: the shadow variable data element, NULL on failure. 94 */ 95 void *klp_shadow_get(void *obj, unsigned long id) 96 { 97 struct klp_shadow *shadow; 98 99 rcu_read_lock(); 100 101 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, 102 (unsigned long)obj) { 103 104 if (klp_shadow_match(shadow, obj, id)) { 105 rcu_read_unlock(); 106 return shadow->data; 107 } 108 } 109 110 rcu_read_unlock(); 111 112 return NULL; 113 } 114 EXPORT_SYMBOL_GPL(klp_shadow_get); 115 116 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, 117 size_t size, gfp_t gfp_flags, 118 klp_shadow_ctor_t ctor, void *ctor_data, 119 bool warn_on_exist) 120 { 121 struct klp_shadow *new_shadow; 122 void *shadow_data; 123 unsigned long flags; 124 125 /* Check if the shadow variable already exists */ 126 shadow_data = klp_shadow_get(obj, id); 127 if (shadow_data) 128 goto exists; 129 130 /* 131 * Allocate a new shadow variable. Fill it with zeroes by default. 132 * More complex setting can be done by @ctor function. But it is 133 * called only when the buffer is really used (under klp_shadow_lock). 134 */ 135 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags); 136 if (!new_shadow) 137 return NULL; 138 139 /* Look for <obj, id> again under the lock */ 140 spin_lock_irqsave(&klp_shadow_lock, flags); 141 shadow_data = klp_shadow_get(obj, id); 142 if (unlikely(shadow_data)) { 143 /* 144 * Shadow variable was found, throw away speculative 145 * allocation. 146 */ 147 spin_unlock_irqrestore(&klp_shadow_lock, flags); 148 kfree(new_shadow); 149 goto exists; 150 } 151 152 new_shadow->obj = obj; 153 new_shadow->id = id; 154 155 if (ctor) { 156 int err; 157 158 err = ctor(obj, new_shadow->data, ctor_data); 159 if (err) { 160 spin_unlock_irqrestore(&klp_shadow_lock, flags); 161 kfree(new_shadow); 162 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n", 163 obj, id, err); 164 return NULL; 165 } 166 } 167 168 /* No <obj, id> found, so attach the newly allocated one */ 169 hash_add_rcu(klp_shadow_hash, &new_shadow->node, 170 (unsigned long)new_shadow->obj); 171 spin_unlock_irqrestore(&klp_shadow_lock, flags); 172 173 return new_shadow->data; 174 175 exists: 176 if (warn_on_exist) { 177 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id); 178 return NULL; 179 } 180 181 return shadow_data; 182 } 183 184 /** 185 * klp_shadow_alloc() - allocate and add a new shadow variable 186 * @obj: pointer to parent object 187 * @id: data identifier 188 * @size: size of attached data 189 * @gfp_flags: GFP mask for allocation 190 * @ctor: custom constructor to initialize the shadow data (optional) 191 * @ctor_data: pointer to any data needed by @ctor (optional) 192 * 193 * Allocates @size bytes for new shadow variable data using @gfp_flags. 194 * The data are zeroed by default. They are further initialized by @ctor 195 * function if it is not NULL. The new shadow variable is then added 196 * to the global hashtable. 197 * 198 * If an existing <obj, id> shadow variable can be found, this routine will 199 * issue a WARN, exit early and return NULL. 200 * 201 * This function guarantees that the constructor function is called only when 202 * the variable did not exist before. The cost is that @ctor is called 203 * in atomic context under a spin lock. 204 * 205 * Return: the shadow variable data element, NULL on duplicate or 206 * failure. 207 */ 208 void *klp_shadow_alloc(void *obj, unsigned long id, 209 size_t size, gfp_t gfp_flags, 210 klp_shadow_ctor_t ctor, void *ctor_data) 211 { 212 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, 213 ctor, ctor_data, true); 214 } 215 EXPORT_SYMBOL_GPL(klp_shadow_alloc); 216 217 /** 218 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable 219 * @obj: pointer to parent object 220 * @id: data identifier 221 * @size: size of attached data 222 * @gfp_flags: GFP mask for allocation 223 * @ctor: custom constructor to initialize the shadow data (optional) 224 * @ctor_data: pointer to any data needed by @ctor (optional) 225 * 226 * Returns a pointer to existing shadow data if an <obj, id> shadow 227 * variable is already present. Otherwise, it creates a new shadow 228 * variable like klp_shadow_alloc(). 229 * 230 * This function guarantees that only one shadow variable exists with the given 231 * @id for the given @obj. It also guarantees that the constructor function 232 * will be called only when the variable did not exist before. The cost is 233 * that @ctor is called in atomic context under a spin lock. 234 * 235 * Return: the shadow variable data element, NULL on failure. 236 */ 237 void *klp_shadow_get_or_alloc(void *obj, unsigned long id, 238 size_t size, gfp_t gfp_flags, 239 klp_shadow_ctor_t ctor, void *ctor_data) 240 { 241 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, 242 ctor, ctor_data, false); 243 } 244 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); 245 246 static void klp_shadow_free_struct(struct klp_shadow *shadow, 247 klp_shadow_dtor_t dtor) 248 { 249 hash_del_rcu(&shadow->node); 250 if (dtor) 251 dtor(shadow->obj, shadow->data); 252 kfree_rcu(shadow, rcu_head); 253 } 254 255 /** 256 * klp_shadow_free() - detach and free a <obj, id> shadow variable 257 * @obj: pointer to parent object 258 * @id: data identifier 259 * @dtor: custom callback that can be used to unregister the variable 260 * and/or free data that the shadow variable points to (optional) 261 * 262 * This function releases the memory for this <obj, id> shadow variable 263 * instance, callers should stop referencing it accordingly. 264 */ 265 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) 266 { 267 struct klp_shadow *shadow; 268 unsigned long flags; 269 270 spin_lock_irqsave(&klp_shadow_lock, flags); 271 272 /* Delete <obj, id> from hash */ 273 hash_for_each_possible(klp_shadow_hash, shadow, node, 274 (unsigned long)obj) { 275 276 if (klp_shadow_match(shadow, obj, id)) { 277 klp_shadow_free_struct(shadow, dtor); 278 break; 279 } 280 } 281 282 spin_unlock_irqrestore(&klp_shadow_lock, flags); 283 } 284 EXPORT_SYMBOL_GPL(klp_shadow_free); 285 286 /** 287 * klp_shadow_free_all() - detach and free all <*, id> shadow variables 288 * @id: data identifier 289 * @dtor: custom callback that can be used to unregister the variable 290 * and/or free data that the shadow variable points to (optional) 291 * 292 * This function releases the memory for all <*, id> shadow variable 293 * instances, callers should stop referencing them accordingly. 294 */ 295 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) 296 { 297 struct klp_shadow *shadow; 298 unsigned long flags; 299 int i; 300 301 spin_lock_irqsave(&klp_shadow_lock, flags); 302 303 /* Delete all <*, id> from hash */ 304 hash_for_each(klp_shadow_hash, i, shadow, node) { 305 if (klp_shadow_match(shadow, shadow->obj, id)) 306 klp_shadow_free_struct(shadow, dtor); 307 } 308 309 spin_unlock_irqrestore(&klp_shadow_lock, flags); 310 } 311 EXPORT_SYMBOL_GPL(klp_shadow_free_all); 312