1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/err.h> 14 #include <linux/vmalloc.h> 15 #include <linux/slab.h> 16 #include <linux/mm.h> 17 #include <linux/filter.h> 18 #include <linux/perf_event.h> 19 20 /* Called from syscall */ 21 static struct bpf_map *array_map_alloc(union bpf_attr *attr) 22 { 23 struct bpf_array *array; 24 u32 elem_size, array_size; 25 26 /* check sanity of attributes */ 27 if (attr->max_entries == 0 || attr->key_size != 4 || 28 attr->value_size == 0) 29 return ERR_PTR(-EINVAL); 30 31 elem_size = round_up(attr->value_size, 8); 32 33 /* check round_up into zero and u32 overflow */ 34 if (elem_size == 0 || 35 attr->max_entries > (U32_MAX - sizeof(*array)) / elem_size) 36 return ERR_PTR(-ENOMEM); 37 38 array_size = sizeof(*array) + attr->max_entries * elem_size; 39 40 /* allocate all map elements and zero-initialize them */ 41 array = kzalloc(array_size, GFP_USER | __GFP_NOWARN); 42 if (!array) { 43 array = vzalloc(array_size); 44 if (!array) 45 return ERR_PTR(-ENOMEM); 46 } 47 48 /* copy mandatory map attributes */ 49 array->map.key_size = attr->key_size; 50 array->map.value_size = attr->value_size; 51 array->map.max_entries = attr->max_entries; 52 array->map.pages = round_up(array_size, PAGE_SIZE) >> PAGE_SHIFT; 53 array->elem_size = elem_size; 54 55 return &array->map; 56 } 57 58 /* Called from syscall or from eBPF program */ 59 static void *array_map_lookup_elem(struct bpf_map *map, void *key) 60 { 61 struct bpf_array *array = container_of(map, struct bpf_array, map); 62 u32 index = *(u32 *)key; 63 64 if (index >= array->map.max_entries) 65 return NULL; 66 67 return array->value + array->elem_size * index; 68 } 69 70 /* Called from syscall */ 71 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) 72 { 73 struct bpf_array *array = container_of(map, struct bpf_array, map); 74 u32 index = *(u32 *)key; 75 u32 *next = (u32 *)next_key; 76 77 if (index >= array->map.max_entries) { 78 *next = 0; 79 return 0; 80 } 81 82 if (index == array->map.max_entries - 1) 83 return -ENOENT; 84 85 *next = index + 1; 86 return 0; 87 } 88 89 /* Called from syscall or from eBPF program */ 90 static int array_map_update_elem(struct bpf_map *map, void *key, void *value, 91 u64 map_flags) 92 { 93 struct bpf_array *array = container_of(map, struct bpf_array, map); 94 u32 index = *(u32 *)key; 95 96 if (map_flags > BPF_EXIST) 97 /* unknown flags */ 98 return -EINVAL; 99 100 if (index >= array->map.max_entries) 101 /* all elements were pre-allocated, cannot insert a new one */ 102 return -E2BIG; 103 104 if (map_flags == BPF_NOEXIST) 105 /* all elements already exist */ 106 return -EEXIST; 107 108 memcpy(array->value + array->elem_size * index, value, array->elem_size); 109 return 0; 110 } 111 112 /* Called from syscall or from eBPF program */ 113 static int array_map_delete_elem(struct bpf_map *map, void *key) 114 { 115 return -EINVAL; 116 } 117 118 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */ 119 static void array_map_free(struct bpf_map *map) 120 { 121 struct bpf_array *array = container_of(map, struct bpf_array, map); 122 123 /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, 124 * so the programs (can be more than one that used this map) were 125 * disconnected from events. Wait for outstanding programs to complete 126 * and free the array 127 */ 128 synchronize_rcu(); 129 130 kvfree(array); 131 } 132 133 static const struct bpf_map_ops array_ops = { 134 .map_alloc = array_map_alloc, 135 .map_free = array_map_free, 136 .map_get_next_key = array_map_get_next_key, 137 .map_lookup_elem = array_map_lookup_elem, 138 .map_update_elem = array_map_update_elem, 139 .map_delete_elem = array_map_delete_elem, 140 }; 141 142 static struct bpf_map_type_list array_type __read_mostly = { 143 .ops = &array_ops, 144 .type = BPF_MAP_TYPE_ARRAY, 145 }; 146 147 static int __init register_array_map(void) 148 { 149 bpf_register_map_type(&array_type); 150 return 0; 151 } 152 late_initcall(register_array_map); 153 154 static struct bpf_map *fd_array_map_alloc(union bpf_attr *attr) 155 { 156 /* only file descriptors can be stored in this type of map */ 157 if (attr->value_size != sizeof(u32)) 158 return ERR_PTR(-EINVAL); 159 return array_map_alloc(attr); 160 } 161 162 static void fd_array_map_free(struct bpf_map *map) 163 { 164 struct bpf_array *array = container_of(map, struct bpf_array, map); 165 int i; 166 167 synchronize_rcu(); 168 169 /* make sure it's empty */ 170 for (i = 0; i < array->map.max_entries; i++) 171 BUG_ON(array->ptrs[i] != NULL); 172 kvfree(array); 173 } 174 175 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) 176 { 177 return NULL; 178 } 179 180 /* only called from syscall */ 181 static int fd_array_map_update_elem(struct bpf_map *map, void *key, 182 void *value, u64 map_flags) 183 { 184 struct bpf_array *array = container_of(map, struct bpf_array, map); 185 void *new_ptr, *old_ptr; 186 u32 index = *(u32 *)key, ufd; 187 188 if (map_flags != BPF_ANY) 189 return -EINVAL; 190 191 if (index >= array->map.max_entries) 192 return -E2BIG; 193 194 ufd = *(u32 *)value; 195 new_ptr = map->ops->map_fd_get_ptr(map, ufd); 196 if (IS_ERR(new_ptr)) 197 return PTR_ERR(new_ptr); 198 199 old_ptr = xchg(array->ptrs + index, new_ptr); 200 if (old_ptr) 201 map->ops->map_fd_put_ptr(old_ptr); 202 203 return 0; 204 } 205 206 static int fd_array_map_delete_elem(struct bpf_map *map, void *key) 207 { 208 struct bpf_array *array = container_of(map, struct bpf_array, map); 209 void *old_ptr; 210 u32 index = *(u32 *)key; 211 212 if (index >= array->map.max_entries) 213 return -E2BIG; 214 215 old_ptr = xchg(array->ptrs + index, NULL); 216 if (old_ptr) { 217 map->ops->map_fd_put_ptr(old_ptr); 218 return 0; 219 } else { 220 return -ENOENT; 221 } 222 } 223 224 static void *prog_fd_array_get_ptr(struct bpf_map *map, int fd) 225 { 226 struct bpf_array *array = container_of(map, struct bpf_array, map); 227 struct bpf_prog *prog = bpf_prog_get(fd); 228 if (IS_ERR(prog)) 229 return prog; 230 231 if (!bpf_prog_array_compatible(array, prog)) { 232 bpf_prog_put(prog); 233 return ERR_PTR(-EINVAL); 234 } 235 return prog; 236 } 237 238 static void prog_fd_array_put_ptr(void *ptr) 239 { 240 struct bpf_prog *prog = ptr; 241 242 bpf_prog_put_rcu(prog); 243 } 244 245 /* decrement refcnt of all bpf_progs that are stored in this map */ 246 void bpf_fd_array_map_clear(struct bpf_map *map) 247 { 248 struct bpf_array *array = container_of(map, struct bpf_array, map); 249 int i; 250 251 for (i = 0; i < array->map.max_entries; i++) 252 fd_array_map_delete_elem(map, &i); 253 } 254 255 static const struct bpf_map_ops prog_array_ops = { 256 .map_alloc = fd_array_map_alloc, 257 .map_free = fd_array_map_free, 258 .map_get_next_key = array_map_get_next_key, 259 .map_lookup_elem = fd_array_map_lookup_elem, 260 .map_update_elem = fd_array_map_update_elem, 261 .map_delete_elem = fd_array_map_delete_elem, 262 .map_fd_get_ptr = prog_fd_array_get_ptr, 263 .map_fd_put_ptr = prog_fd_array_put_ptr, 264 }; 265 266 static struct bpf_map_type_list prog_array_type __read_mostly = { 267 .ops = &prog_array_ops, 268 .type = BPF_MAP_TYPE_PROG_ARRAY, 269 }; 270 271 static int __init register_prog_array_map(void) 272 { 273 bpf_register_map_type(&prog_array_type); 274 return 0; 275 } 276 late_initcall(register_prog_array_map); 277 278 static void perf_event_array_map_free(struct bpf_map *map) 279 { 280 bpf_fd_array_map_clear(map); 281 fd_array_map_free(map); 282 } 283 284 static void *perf_event_fd_array_get_ptr(struct bpf_map *map, int fd) 285 { 286 struct perf_event *event; 287 const struct perf_event_attr *attr; 288 289 event = perf_event_get(fd); 290 if (IS_ERR(event)) 291 return event; 292 293 attr = perf_event_attrs(event); 294 if (IS_ERR(attr)) 295 goto err; 296 297 if (attr->inherit) 298 goto err; 299 300 if (attr->type == PERF_TYPE_RAW) 301 return event; 302 303 if (attr->type == PERF_TYPE_HARDWARE) 304 return event; 305 306 if (attr->type == PERF_TYPE_SOFTWARE && 307 attr->config == PERF_COUNT_SW_BPF_OUTPUT) 308 return event; 309 err: 310 perf_event_release_kernel(event); 311 return ERR_PTR(-EINVAL); 312 } 313 314 static void perf_event_fd_array_put_ptr(void *ptr) 315 { 316 struct perf_event *event = ptr; 317 318 perf_event_release_kernel(event); 319 } 320 321 static const struct bpf_map_ops perf_event_array_ops = { 322 .map_alloc = fd_array_map_alloc, 323 .map_free = perf_event_array_map_free, 324 .map_get_next_key = array_map_get_next_key, 325 .map_lookup_elem = fd_array_map_lookup_elem, 326 .map_update_elem = fd_array_map_update_elem, 327 .map_delete_elem = fd_array_map_delete_elem, 328 .map_fd_get_ptr = perf_event_fd_array_get_ptr, 329 .map_fd_put_ptr = perf_event_fd_array_put_ptr, 330 }; 331 332 static struct bpf_map_type_list perf_event_array_type __read_mostly = { 333 .ops = &perf_event_array_ops, 334 .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, 335 }; 336 337 static int __init register_perf_event_array_map(void) 338 { 339 bpf_register_map_type(&perf_event_array_type); 340 return 0; 341 } 342 late_initcall(register_perf_event_array_map); 343