1 /* 2 * shadow.c - Shadow Variables 3 * 4 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com> 5 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> 6 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com> 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License 10 * as published by the Free Software Foundation; either version 2 11 * of the License, or (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 /** 23 * DOC: Shadow variable API concurrency notes: 24 * 25 * The shadow variable API provides a simple relationship between an 26 * <obj, id> pair and a pointer value. It is the responsibility of the 27 * caller to provide any mutual exclusion required of the shadow data. 28 * 29 * Once a shadow variable is attached to its parent object via the 30 * klp_shadow_*alloc() API calls, it is considered live: any subsequent 31 * call to klp_shadow_get() may then return the shadow variable's data 32 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data 33 * accordingly. 34 * 35 * The klp_shadow_*alloc() API calls may allocate memory for new shadow 36 * variable structures. Their implementation does not call kmalloc 37 * inside any spinlocks, but API callers should pass GFP flags according 38 * to their specific needs. 39 * 40 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against 41 * concurrent klp_shadow_free() and klp_shadow_get() operations. 42 */ 43 44 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 45 46 #include <linux/hashtable.h> 47 #include <linux/slab.h> 48 #include <linux/livepatch.h> 49 50 static DEFINE_HASHTABLE(klp_shadow_hash, 12); 51 52 /* 53 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and 54 * the shadow variables it references. 55 */ 56 static DEFINE_SPINLOCK(klp_shadow_lock); 57 58 /** 59 * struct klp_shadow - shadow variable structure 60 * @node: klp_shadow_hash hash table node 61 * @rcu_head: RCU is used to safely free this structure 62 * @obj: pointer to parent object 63 * @id: data identifier 64 * @data: data area 65 */ 66 struct klp_shadow { 67 struct hlist_node node; 68 struct rcu_head rcu_head; 69 void *obj; 70 unsigned long id; 71 char data[]; 72 }; 73 74 /** 75 * klp_shadow_match() - verify a shadow variable matches given <obj, id> 76 * @shadow: shadow variable to match 77 * @obj: pointer to parent object 78 * @id: data identifier 79 * 80 * Return: true if the shadow variable matches. 81 */ 82 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj, 83 unsigned long id) 84 { 85 return shadow->obj == obj && shadow->id == id; 86 } 87 88 /** 89 * klp_shadow_get() - retrieve a shadow variable data pointer 90 * @obj: pointer to parent object 91 * @id: data identifier 92 * 93 * Return: the shadow variable data element, NULL on failure. 94 */ 95 void *klp_shadow_get(void *obj, unsigned long id) 96 { 97 struct klp_shadow *shadow; 98 99 rcu_read_lock(); 100 101 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, 102 (unsigned long)obj) { 103 104 if (klp_shadow_match(shadow, obj, id)) { 105 rcu_read_unlock(); 106 return shadow->data; 107 } 108 } 109 110 rcu_read_unlock(); 111 112 return NULL; 113 } 114 EXPORT_SYMBOL_GPL(klp_shadow_get); 115 116 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data, 117 size_t size, gfp_t gfp_flags, bool warn_on_exist) 118 { 119 struct klp_shadow *new_shadow; 120 void *shadow_data; 121 unsigned long flags; 122 123 /* Check if the shadow variable already exists */ 124 shadow_data = klp_shadow_get(obj, id); 125 if (shadow_data) 126 goto exists; 127 128 /* Allocate a new shadow variable for use inside the lock below */ 129 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags); 130 if (!new_shadow) 131 return NULL; 132 133 new_shadow->obj = obj; 134 new_shadow->id = id; 135 136 /* Initialize the shadow variable if data provided */ 137 if (data) 138 memcpy(new_shadow->data, data, size); 139 140 /* Look for <obj, id> again under the lock */ 141 spin_lock_irqsave(&klp_shadow_lock, flags); 142 shadow_data = klp_shadow_get(obj, id); 143 if (unlikely(shadow_data)) { 144 /* 145 * Shadow variable was found, throw away speculative 146 * allocation. 147 */ 148 spin_unlock_irqrestore(&klp_shadow_lock, flags); 149 kfree(new_shadow); 150 goto exists; 151 } 152 153 /* No <obj, id> found, so attach the newly allocated one */ 154 hash_add_rcu(klp_shadow_hash, &new_shadow->node, 155 (unsigned long)new_shadow->obj); 156 spin_unlock_irqrestore(&klp_shadow_lock, flags); 157 158 return new_shadow->data; 159 160 exists: 161 if (warn_on_exist) { 162 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id); 163 return NULL; 164 } 165 166 return shadow_data; 167 } 168 169 /** 170 * klp_shadow_alloc() - allocate and add a new shadow variable 171 * @obj: pointer to parent object 172 * @id: data identifier 173 * @data: pointer to data to attach to parent 174 * @size: size of attached data 175 * @gfp_flags: GFP mask for allocation 176 * 177 * Allocates @size bytes for new shadow variable data using @gfp_flags 178 * and copies @size bytes from @data into the new shadow variable's own 179 * data space. If @data is NULL, @size bytes are still allocated, but 180 * no copy is performed. The new shadow variable is then added to the 181 * global hashtable. 182 * 183 * If an existing <obj, id> shadow variable can be found, this routine 184 * will issue a WARN, exit early and return NULL. 185 * 186 * Return: the shadow variable data element, NULL on duplicate or 187 * failure. 188 */ 189 void *klp_shadow_alloc(void *obj, unsigned long id, void *data, 190 size_t size, gfp_t gfp_flags) 191 { 192 return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, true); 193 } 194 EXPORT_SYMBOL_GPL(klp_shadow_alloc); 195 196 /** 197 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable 198 * @obj: pointer to parent object 199 * @id: data identifier 200 * @data: pointer to data to attach to parent 201 * @size: size of attached data 202 * @gfp_flags: GFP mask for allocation 203 * 204 * Returns a pointer to existing shadow data if an <obj, id> shadow 205 * variable is already present. Otherwise, it creates a new shadow 206 * variable like klp_shadow_alloc(). 207 * 208 * This function guarantees that only one shadow variable exists with 209 * the given @id for the given @obj. It also guarantees that the shadow 210 * variable will be initialized by the given @data only when it did not 211 * exist before. 212 * 213 * Return: the shadow variable data element, NULL on failure. 214 */ 215 void *klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data, 216 size_t size, gfp_t gfp_flags) 217 { 218 return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, false); 219 } 220 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); 221 222 /** 223 * klp_shadow_free() - detach and free a <obj, id> shadow variable 224 * @obj: pointer to parent object 225 * @id: data identifier 226 * 227 * This function releases the memory for this <obj, id> shadow variable 228 * instance, callers should stop referencing it accordingly. 229 */ 230 void klp_shadow_free(void *obj, unsigned long id) 231 { 232 struct klp_shadow *shadow; 233 unsigned long flags; 234 235 spin_lock_irqsave(&klp_shadow_lock, flags); 236 237 /* Delete <obj, id> from hash */ 238 hash_for_each_possible(klp_shadow_hash, shadow, node, 239 (unsigned long)obj) { 240 241 if (klp_shadow_match(shadow, obj, id)) { 242 hash_del_rcu(&shadow->node); 243 kfree_rcu(shadow, rcu_head); 244 break; 245 } 246 } 247 248 spin_unlock_irqrestore(&klp_shadow_lock, flags); 249 } 250 EXPORT_SYMBOL_GPL(klp_shadow_free); 251 252 /** 253 * klp_shadow_free_all() - detach and free all <*, id> shadow variables 254 * @id: data identifier 255 * 256 * This function releases the memory for all <*, id> shadow variable 257 * instances, callers should stop referencing them accordingly. 258 */ 259 void klp_shadow_free_all(unsigned long id) 260 { 261 struct klp_shadow *shadow; 262 unsigned long flags; 263 int i; 264 265 spin_lock_irqsave(&klp_shadow_lock, flags); 266 267 /* Delete all <*, id> from hash */ 268 hash_for_each(klp_shadow_hash, i, shadow, node) { 269 if (klp_shadow_match(shadow, shadow->obj, id)) { 270 hash_del_rcu(&shadow->node); 271 kfree_rcu(shadow, rcu_head); 272 } 273 } 274 275 spin_unlock_irqrestore(&klp_shadow_lock, flags); 276 } 277 EXPORT_SYMBOL_GPL(klp_shadow_free_all); 278