1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * Copyright (c) 2016,2017 Facebook 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of version 2 of the GNU General Public 6 * License as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 #include <linux/bpf.h> 14 #include <linux/err.h> 15 #include <linux/slab.h> 16 #include <linux/mm.h> 17 #include <linux/filter.h> 18 #include <linux/perf_event.h> 19 20 #include "map_in_map.h" 21 22 #define ARRAY_CREATE_FLAG_MASK \ 23 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) 24 25 static void bpf_array_free_percpu(struct bpf_array *array) 26 { 27 int i; 28 29 for (i = 0; i < array->map.max_entries; i++) { 30 free_percpu(array->pptrs[i]); 31 cond_resched(); 32 } 33 } 34 35 static int bpf_array_alloc_percpu(struct bpf_array *array) 36 { 37 void __percpu *ptr; 38 int i; 39 40 for (i = 0; i < array->map.max_entries; i++) { 41 ptr = __alloc_percpu_gfp(array->elem_size, 8, 42 GFP_USER | __GFP_NOWARN); 43 if (!ptr) { 44 bpf_array_free_percpu(array); 45 return -ENOMEM; 46 } 47 array->pptrs[i] = ptr; 48 cond_resched(); 49 } 50 51 return 0; 52 } 53 54 /* Called from syscall */ 55 static int array_map_alloc_check(union bpf_attr *attr) 56 { 57 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; 58 int numa_node = bpf_map_attr_numa_node(attr); 59 60 /* check sanity of attributes */ 61 if (attr->max_entries == 0 || attr->key_size != 4 || 62 attr->value_size == 0 || 63 attr->map_flags & ~ARRAY_CREATE_FLAG_MASK || 64 (percpu && numa_node != NUMA_NO_NODE)) 65 return -EINVAL; 66 67 if (attr->value_size > KMALLOC_MAX_SIZE) 68 /* if value_size is bigger, the user space won't be able to 69 * access the elements. 70 */ 71 return -E2BIG; 72 73 return 0; 74 } 75 76 static struct bpf_map *array_map_alloc(union bpf_attr *attr) 77 { 78 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; 79 int ret, numa_node = bpf_map_attr_numa_node(attr); 80 u32 elem_size, index_mask, max_entries; 81 bool unpriv = !capable(CAP_SYS_ADMIN); 82 u64 cost, array_size, mask64; 83 struct bpf_array *array; 84 85 elem_size = round_up(attr->value_size, 8); 86 87 max_entries = attr->max_entries; 88 89 /* On 32 bit archs roundup_pow_of_two() with max_entries that has 90 * upper most bit set in u32 space is undefined behavior due to 91 * resulting 1U << 32, so do it manually here in u64 space. 92 */ 93 mask64 = fls_long(max_entries - 1); 94 mask64 = 1ULL << mask64; 95 mask64 -= 1; 96 97 index_mask = mask64; 98 if (unpriv) { 99 /* round up array size to nearest power of 2, 100 * since cpu will speculate within index_mask limits 101 */ 102 max_entries = index_mask + 1; 103 /* Check for overflows. */ 104 if (max_entries < attr->max_entries) 105 return ERR_PTR(-E2BIG); 106 } 107 108 array_size = sizeof(*array); 109 if (percpu) 110 array_size += (u64) max_entries * sizeof(void *); 111 else 112 array_size += (u64) max_entries * elem_size; 113 114 /* make sure there is no u32 overflow later in round_up() */ 115 cost = array_size; 116 if (cost >= U32_MAX - PAGE_SIZE) 117 return ERR_PTR(-ENOMEM); 118 if (percpu) { 119 cost += (u64)attr->max_entries * elem_size * num_possible_cpus(); 120 if (cost >= U32_MAX - PAGE_SIZE) 121 return ERR_PTR(-ENOMEM); 122 } 123 cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT; 124 125 ret = bpf_map_precharge_memlock(cost); 126 if (ret < 0) 127 return ERR_PTR(ret); 128 129 /* allocate all map elements and zero-initialize them */ 130 array = bpf_map_area_alloc(array_size, numa_node); 131 if (!array) 132 return ERR_PTR(-ENOMEM); 133 array->index_mask = index_mask; 134 array->map.unpriv_array = unpriv; 135 136 /* copy mandatory map attributes */ 137 bpf_map_init_from_attr(&array->map, attr); 138 array->map.pages = cost; 139 array->elem_size = elem_size; 140 141 if (percpu && bpf_array_alloc_percpu(array)) { 142 bpf_map_area_free(array); 143 return ERR_PTR(-ENOMEM); 144 } 145 146 return &array->map; 147 } 148 149 /* Called from syscall or from eBPF program */ 150 static void *array_map_lookup_elem(struct bpf_map *map, void *key) 151 { 152 struct bpf_array *array = container_of(map, struct bpf_array, map); 153 u32 index = *(u32 *)key; 154 155 if (unlikely(index >= array->map.max_entries)) 156 return NULL; 157 158 return array->value + array->elem_size * (index & array->index_mask); 159 } 160 161 /* emit BPF instructions equivalent to C code of array_map_lookup_elem() */ 162 static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) 163 { 164 struct bpf_array *array = container_of(map, struct bpf_array, map); 165 struct bpf_insn *insn = insn_buf; 166 u32 elem_size = round_up(map->value_size, 8); 167 const int ret = BPF_REG_0; 168 const int map_ptr = BPF_REG_1; 169 const int index = BPF_REG_2; 170 171 *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); 172 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); 173 if (map->unpriv_array) { 174 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); 175 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); 176 } else { 177 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); 178 } 179 180 if (is_power_of_2(elem_size)) { 181 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size)); 182 } else { 183 *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size); 184 } 185 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr); 186 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 187 *insn++ = BPF_MOV64_IMM(ret, 0); 188 return insn - insn_buf; 189 } 190 191 /* Called from eBPF program */ 192 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) 193 { 194 struct bpf_array *array = container_of(map, struct bpf_array, map); 195 u32 index = *(u32 *)key; 196 197 if (unlikely(index >= array->map.max_entries)) 198 return NULL; 199 200 return this_cpu_ptr(array->pptrs[index & array->index_mask]); 201 } 202 203 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) 204 { 205 struct bpf_array *array = container_of(map, struct bpf_array, map); 206 u32 index = *(u32 *)key; 207 void __percpu *pptr; 208 int cpu, off = 0; 209 u32 size; 210 211 if (unlikely(index >= array->map.max_entries)) 212 return -ENOENT; 213 214 /* per_cpu areas are zero-filled and bpf programs can only 215 * access 'value_size' of them, so copying rounded areas 216 * will not leak any kernel data 217 */ 218 size = round_up(map->value_size, 8); 219 rcu_read_lock(); 220 pptr = array->pptrs[index & array->index_mask]; 221 for_each_possible_cpu(cpu) { 222 bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size); 223 off += size; 224 } 225 rcu_read_unlock(); 226 return 0; 227 } 228 229 /* Called from syscall */ 230 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) 231 { 232 struct bpf_array *array = container_of(map, struct bpf_array, map); 233 u32 index = key ? *(u32 *)key : U32_MAX; 234 u32 *next = (u32 *)next_key; 235 236 if (index >= array->map.max_entries) { 237 *next = 0; 238 return 0; 239 } 240 241 if (index == array->map.max_entries - 1) 242 return -ENOENT; 243 244 *next = index + 1; 245 return 0; 246 } 247 248 /* Called from syscall or from eBPF program */ 249 static int array_map_update_elem(struct bpf_map *map, void *key, void *value, 250 u64 map_flags) 251 { 252 struct bpf_array *array = container_of(map, struct bpf_array, map); 253 u32 index = *(u32 *)key; 254 255 if (unlikely(map_flags > BPF_EXIST)) 256 /* unknown flags */ 257 return -EINVAL; 258 259 if (unlikely(index >= array->map.max_entries)) 260 /* all elements were pre-allocated, cannot insert a new one */ 261 return -E2BIG; 262 263 if (unlikely(map_flags == BPF_NOEXIST)) 264 /* all elements already exist */ 265 return -EEXIST; 266 267 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 268 memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]), 269 value, map->value_size); 270 else 271 memcpy(array->value + 272 array->elem_size * (index & array->index_mask), 273 value, map->value_size); 274 return 0; 275 } 276 277 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 278 u64 map_flags) 279 { 280 struct bpf_array *array = container_of(map, struct bpf_array, map); 281 u32 index = *(u32 *)key; 282 void __percpu *pptr; 283 int cpu, off = 0; 284 u32 size; 285 286 if (unlikely(map_flags > BPF_EXIST)) 287 /* unknown flags */ 288 return -EINVAL; 289 290 if (unlikely(index >= array->map.max_entries)) 291 /* all elements were pre-allocated, cannot insert a new one */ 292 return -E2BIG; 293 294 if (unlikely(map_flags == BPF_NOEXIST)) 295 /* all elements already exist */ 296 return -EEXIST; 297 298 /* the user space will provide round_up(value_size, 8) bytes that 299 * will be copied into per-cpu area. bpf programs can only access 300 * value_size of it. During lookup the same extra bytes will be 301 * returned or zeros which were zero-filled by percpu_alloc, 302 * so no kernel data leaks possible 303 */ 304 size = round_up(map->value_size, 8); 305 rcu_read_lock(); 306 pptr = array->pptrs[index & array->index_mask]; 307 for_each_possible_cpu(cpu) { 308 bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size); 309 off += size; 310 } 311 rcu_read_unlock(); 312 return 0; 313 } 314 315 /* Called from syscall or from eBPF program */ 316 static int array_map_delete_elem(struct bpf_map *map, void *key) 317 { 318 return -EINVAL; 319 } 320 321 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */ 322 static void array_map_free(struct bpf_map *map) 323 { 324 struct bpf_array *array = container_of(map, struct bpf_array, map); 325 326 /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, 327 * so the programs (can be more than one that used this map) were 328 * disconnected from events. Wait for outstanding programs to complete 329 * and free the array 330 */ 331 synchronize_rcu(); 332 333 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 334 bpf_array_free_percpu(array); 335 336 bpf_map_area_free(array); 337 } 338 339 const struct bpf_map_ops array_map_ops = { 340 .map_alloc_check = array_map_alloc_check, 341 .map_alloc = array_map_alloc, 342 .map_free = array_map_free, 343 .map_get_next_key = array_map_get_next_key, 344 .map_lookup_elem = array_map_lookup_elem, 345 .map_update_elem = array_map_update_elem, 346 .map_delete_elem = array_map_delete_elem, 347 .map_gen_lookup = array_map_gen_lookup, 348 }; 349 350 const struct bpf_map_ops percpu_array_map_ops = { 351 .map_alloc_check = array_map_alloc_check, 352 .map_alloc = array_map_alloc, 353 .map_free = array_map_free, 354 .map_get_next_key = array_map_get_next_key, 355 .map_lookup_elem = percpu_array_map_lookup_elem, 356 .map_update_elem = array_map_update_elem, 357 .map_delete_elem = array_map_delete_elem, 358 }; 359 360 static int fd_array_map_alloc_check(union bpf_attr *attr) 361 { 362 /* only file descriptors can be stored in this type of map */ 363 if (attr->value_size != sizeof(u32)) 364 return -EINVAL; 365 return array_map_alloc_check(attr); 366 } 367 368 static void fd_array_map_free(struct bpf_map *map) 369 { 370 struct bpf_array *array = container_of(map, struct bpf_array, map); 371 int i; 372 373 synchronize_rcu(); 374 375 /* make sure it's empty */ 376 for (i = 0; i < array->map.max_entries; i++) 377 BUG_ON(array->ptrs[i] != NULL); 378 379 bpf_map_area_free(array); 380 } 381 382 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) 383 { 384 return NULL; 385 } 386 387 /* only called from syscall */ 388 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) 389 { 390 void **elem, *ptr; 391 int ret = 0; 392 393 if (!map->ops->map_fd_sys_lookup_elem) 394 return -ENOTSUPP; 395 396 rcu_read_lock(); 397 elem = array_map_lookup_elem(map, key); 398 if (elem && (ptr = READ_ONCE(*elem))) 399 *value = map->ops->map_fd_sys_lookup_elem(ptr); 400 else 401 ret = -ENOENT; 402 rcu_read_unlock(); 403 404 return ret; 405 } 406 407 /* only called from syscall */ 408 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 409 void *key, void *value, u64 map_flags) 410 { 411 struct bpf_array *array = container_of(map, struct bpf_array, map); 412 void *new_ptr, *old_ptr; 413 u32 index = *(u32 *)key, ufd; 414 415 if (map_flags != BPF_ANY) 416 return -EINVAL; 417 418 if (index >= array->map.max_entries) 419 return -E2BIG; 420 421 ufd = *(u32 *)value; 422 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); 423 if (IS_ERR(new_ptr)) 424 return PTR_ERR(new_ptr); 425 426 old_ptr = xchg(array->ptrs + index, new_ptr); 427 if (old_ptr) 428 map->ops->map_fd_put_ptr(old_ptr); 429 430 return 0; 431 } 432 433 static int fd_array_map_delete_elem(struct bpf_map *map, void *key) 434 { 435 struct bpf_array *array = container_of(map, struct bpf_array, map); 436 void *old_ptr; 437 u32 index = *(u32 *)key; 438 439 if (index >= array->map.max_entries) 440 return -E2BIG; 441 442 old_ptr = xchg(array->ptrs + index, NULL); 443 if (old_ptr) { 444 map->ops->map_fd_put_ptr(old_ptr); 445 return 0; 446 } else { 447 return -ENOENT; 448 } 449 } 450 451 static void *prog_fd_array_get_ptr(struct bpf_map *map, 452 struct file *map_file, int fd) 453 { 454 struct bpf_array *array = container_of(map, struct bpf_array, map); 455 struct bpf_prog *prog = bpf_prog_get(fd); 456 457 if (IS_ERR(prog)) 458 return prog; 459 460 if (!bpf_prog_array_compatible(array, prog)) { 461 bpf_prog_put(prog); 462 return ERR_PTR(-EINVAL); 463 } 464 465 return prog; 466 } 467 468 static void prog_fd_array_put_ptr(void *ptr) 469 { 470 bpf_prog_put(ptr); 471 } 472 473 static u32 prog_fd_array_sys_lookup_elem(void *ptr) 474 { 475 return ((struct bpf_prog *)ptr)->aux->id; 476 } 477 478 /* decrement refcnt of all bpf_progs that are stored in this map */ 479 static void bpf_fd_array_map_clear(struct bpf_map *map) 480 { 481 struct bpf_array *array = container_of(map, struct bpf_array, map); 482 int i; 483 484 for (i = 0; i < array->map.max_entries; i++) 485 fd_array_map_delete_elem(map, &i); 486 } 487 488 const struct bpf_map_ops prog_array_map_ops = { 489 .map_alloc_check = fd_array_map_alloc_check, 490 .map_alloc = array_map_alloc, 491 .map_free = fd_array_map_free, 492 .map_get_next_key = array_map_get_next_key, 493 .map_lookup_elem = fd_array_map_lookup_elem, 494 .map_delete_elem = fd_array_map_delete_elem, 495 .map_fd_get_ptr = prog_fd_array_get_ptr, 496 .map_fd_put_ptr = prog_fd_array_put_ptr, 497 .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem, 498 .map_release_uref = bpf_fd_array_map_clear, 499 }; 500 501 static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, 502 struct file *map_file) 503 { 504 struct bpf_event_entry *ee; 505 506 ee = kzalloc(sizeof(*ee), GFP_ATOMIC); 507 if (ee) { 508 ee->event = perf_file->private_data; 509 ee->perf_file = perf_file; 510 ee->map_file = map_file; 511 } 512 513 return ee; 514 } 515 516 static void __bpf_event_entry_free(struct rcu_head *rcu) 517 { 518 struct bpf_event_entry *ee; 519 520 ee = container_of(rcu, struct bpf_event_entry, rcu); 521 fput(ee->perf_file); 522 kfree(ee); 523 } 524 525 static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee) 526 { 527 call_rcu(&ee->rcu, __bpf_event_entry_free); 528 } 529 530 static void *perf_event_fd_array_get_ptr(struct bpf_map *map, 531 struct file *map_file, int fd) 532 { 533 struct bpf_event_entry *ee; 534 struct perf_event *event; 535 struct file *perf_file; 536 u64 value; 537 538 perf_file = perf_event_get(fd); 539 if (IS_ERR(perf_file)) 540 return perf_file; 541 542 ee = ERR_PTR(-EOPNOTSUPP); 543 event = perf_file->private_data; 544 if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP) 545 goto err_out; 546 547 ee = bpf_event_entry_gen(perf_file, map_file); 548 if (ee) 549 return ee; 550 ee = ERR_PTR(-ENOMEM); 551 err_out: 552 fput(perf_file); 553 return ee; 554 } 555 556 static void perf_event_fd_array_put_ptr(void *ptr) 557 { 558 bpf_event_entry_free_rcu(ptr); 559 } 560 561 static void perf_event_fd_array_release(struct bpf_map *map, 562 struct file *map_file) 563 { 564 struct bpf_array *array = container_of(map, struct bpf_array, map); 565 struct bpf_event_entry *ee; 566 int i; 567 568 rcu_read_lock(); 569 for (i = 0; i < array->map.max_entries; i++) { 570 ee = READ_ONCE(array->ptrs[i]); 571 if (ee && ee->map_file == map_file) 572 fd_array_map_delete_elem(map, &i); 573 } 574 rcu_read_unlock(); 575 } 576 577 const struct bpf_map_ops perf_event_array_map_ops = { 578 .map_alloc_check = fd_array_map_alloc_check, 579 .map_alloc = array_map_alloc, 580 .map_free = fd_array_map_free, 581 .map_get_next_key = array_map_get_next_key, 582 .map_lookup_elem = fd_array_map_lookup_elem, 583 .map_delete_elem = fd_array_map_delete_elem, 584 .map_fd_get_ptr = perf_event_fd_array_get_ptr, 585 .map_fd_put_ptr = perf_event_fd_array_put_ptr, 586 .map_release = perf_event_fd_array_release, 587 }; 588 589 #ifdef CONFIG_CGROUPS 590 static void *cgroup_fd_array_get_ptr(struct bpf_map *map, 591 struct file *map_file /* not used */, 592 int fd) 593 { 594 return cgroup_get_from_fd(fd); 595 } 596 597 static void cgroup_fd_array_put_ptr(void *ptr) 598 { 599 /* cgroup_put free cgrp after a rcu grace period */ 600 cgroup_put(ptr); 601 } 602 603 static void cgroup_fd_array_free(struct bpf_map *map) 604 { 605 bpf_fd_array_map_clear(map); 606 fd_array_map_free(map); 607 } 608 609 const struct bpf_map_ops cgroup_array_map_ops = { 610 .map_alloc_check = fd_array_map_alloc_check, 611 .map_alloc = array_map_alloc, 612 .map_free = cgroup_fd_array_free, 613 .map_get_next_key = array_map_get_next_key, 614 .map_lookup_elem = fd_array_map_lookup_elem, 615 .map_delete_elem = fd_array_map_delete_elem, 616 .map_fd_get_ptr = cgroup_fd_array_get_ptr, 617 .map_fd_put_ptr = cgroup_fd_array_put_ptr, 618 }; 619 #endif 620 621 static struct bpf_map *array_of_map_alloc(union bpf_attr *attr) 622 { 623 struct bpf_map *map, *inner_map_meta; 624 625 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); 626 if (IS_ERR(inner_map_meta)) 627 return inner_map_meta; 628 629 map = array_map_alloc(attr); 630 if (IS_ERR(map)) { 631 bpf_map_meta_free(inner_map_meta); 632 return map; 633 } 634 635 map->inner_map_meta = inner_map_meta; 636 637 return map; 638 } 639 640 static void array_of_map_free(struct bpf_map *map) 641 { 642 /* map->inner_map_meta is only accessed by syscall which 643 * is protected by fdget/fdput. 644 */ 645 bpf_map_meta_free(map->inner_map_meta); 646 bpf_fd_array_map_clear(map); 647 fd_array_map_free(map); 648 } 649 650 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) 651 { 652 struct bpf_map **inner_map = array_map_lookup_elem(map, key); 653 654 if (!inner_map) 655 return NULL; 656 657 return READ_ONCE(*inner_map); 658 } 659 660 static u32 array_of_map_gen_lookup(struct bpf_map *map, 661 struct bpf_insn *insn_buf) 662 { 663 struct bpf_array *array = container_of(map, struct bpf_array, map); 664 u32 elem_size = round_up(map->value_size, 8); 665 struct bpf_insn *insn = insn_buf; 666 const int ret = BPF_REG_0; 667 const int map_ptr = BPF_REG_1; 668 const int index = BPF_REG_2; 669 670 *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); 671 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); 672 if (map->unpriv_array) { 673 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); 674 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); 675 } else { 676 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); 677 } 678 if (is_power_of_2(elem_size)) 679 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size)); 680 else 681 *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size); 682 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr); 683 *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); 684 *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); 685 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 686 *insn++ = BPF_MOV64_IMM(ret, 0); 687 688 return insn - insn_buf; 689 } 690 691 const struct bpf_map_ops array_of_maps_map_ops = { 692 .map_alloc_check = fd_array_map_alloc_check, 693 .map_alloc = array_of_map_alloc, 694 .map_free = array_of_map_free, 695 .map_get_next_key = array_map_get_next_key, 696 .map_lookup_elem = array_of_map_lookup_elem, 697 .map_delete_elem = fd_array_map_delete_elem, 698 .map_fd_get_ptr = bpf_map_fd_get_ptr, 699 .map_fd_put_ptr = bpf_map_fd_put_ptr, 700 .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, 701 .map_gen_lookup = array_of_map_gen_lookup, 702 }; 703