1 // SPDX-License-Identifier: LGPL-2.1 2 3 /* 4 * common eBPF ELF operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU Lesser General Public 12 * License as published by the Free Software Foundation; 13 * version 2.1 of the License (not later!) 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU Lesser General Public License for more details. 19 * 20 * You should have received a copy of the GNU Lesser General Public 21 * License along with this program; if not, see <http://www.gnu.org/licenses> 22 */ 23 24 #include <stdlib.h> 25 #include <memory.h> 26 #include <unistd.h> 27 #include <asm/unistd.h> 28 #include <linux/bpf.h> 29 #include "bpf.h" 30 #include "libbpf.h" 31 #include "nlattr.h" 32 #include <linux/rtnetlink.h> 33 #include <linux/if_link.h> 34 #include <sys/socket.h> 35 #include <errno.h> 36 37 #ifndef SOL_NETLINK 38 #define SOL_NETLINK 270 39 #endif 40 41 /* 42 * When building perf, unistd.h is overridden. __NR_bpf is 43 * required to be defined explicitly. 44 */ 45 #ifndef __NR_bpf 46 # if defined(__i386__) 47 # define __NR_bpf 357 48 # elif defined(__x86_64__) 49 # define __NR_bpf 321 50 # elif defined(__aarch64__) 51 # define __NR_bpf 280 52 # elif defined(__sparc__) 53 # define __NR_bpf 349 54 # elif defined(__s390__) 55 # define __NR_bpf 351 56 # else 57 # error __NR_bpf not defined. libbpf does not support your arch. 58 # endif 59 #endif 60 61 #ifndef min 62 #define min(x, y) ((x) < (y) ? (x) : (y)) 63 #endif 64 65 static inline __u64 ptr_to_u64(const void *ptr) 66 { 67 return (__u64) (unsigned long) ptr; 68 } 69 70 static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, 71 unsigned int size) 72 { 73 return syscall(__NR_bpf, cmd, attr, size); 74 } 75 76 int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr) 77 { 78 __u32 name_len = create_attr->name ? strlen(create_attr->name) : 0; 79 union bpf_attr attr; 80 81 memset(&attr, '\0', sizeof(attr)); 82 83 attr.map_type = create_attr->map_type; 84 attr.key_size = create_attr->key_size; 85 attr.value_size = create_attr->value_size; 86 attr.max_entries = create_attr->max_entries; 87 attr.map_flags = create_attr->map_flags; 88 memcpy(attr.map_name, create_attr->name, 89 min(name_len, BPF_OBJ_NAME_LEN - 1)); 90 attr.numa_node = create_attr->numa_node; 91 attr.btf_fd = create_attr->btf_fd; 92 attr.btf_key_type_id = create_attr->btf_key_type_id; 93 attr.btf_value_type_id = create_attr->btf_value_type_id; 94 attr.map_ifindex = create_attr->map_ifindex; 95 attr.inner_map_fd = create_attr->inner_map_fd; 96 97 return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); 98 } 99 100 int bpf_create_map_node(enum bpf_map_type map_type, const char *name, 101 int key_size, int value_size, int max_entries, 102 __u32 map_flags, int node) 103 { 104 struct bpf_create_map_attr map_attr = {}; 105 106 map_attr.name = name; 107 map_attr.map_type = map_type; 108 map_attr.map_flags = map_flags; 109 map_attr.key_size = key_size; 110 map_attr.value_size = value_size; 111 map_attr.max_entries = max_entries; 112 if (node >= 0) { 113 map_attr.numa_node = node; 114 map_attr.map_flags |= BPF_F_NUMA_NODE; 115 } 116 117 return bpf_create_map_xattr(&map_attr); 118 } 119 120 int bpf_create_map(enum bpf_map_type map_type, int key_size, 121 int value_size, int max_entries, __u32 map_flags) 122 { 123 struct bpf_create_map_attr map_attr = {}; 124 125 map_attr.map_type = map_type; 126 map_attr.map_flags = map_flags; 127 map_attr.key_size = key_size; 128 map_attr.value_size = value_size; 129 map_attr.max_entries = max_entries; 130 131 return bpf_create_map_xattr(&map_attr); 132 } 133 134 int bpf_create_map_name(enum bpf_map_type map_type, const char *name, 135 int key_size, int value_size, int max_entries, 136 __u32 map_flags) 137 { 138 struct bpf_create_map_attr map_attr = {}; 139 140 map_attr.name = name; 141 map_attr.map_type = map_type; 142 map_attr.map_flags = map_flags; 143 map_attr.key_size = key_size; 144 map_attr.value_size = value_size; 145 map_attr.max_entries = max_entries; 146 147 return bpf_create_map_xattr(&map_attr); 148 } 149 150 int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name, 151 int key_size, int inner_map_fd, int max_entries, 152 __u32 map_flags, int node) 153 { 154 __u32 name_len = name ? strlen(name) : 0; 155 union bpf_attr attr; 156 157 memset(&attr, '\0', sizeof(attr)); 158 159 attr.map_type = map_type; 160 attr.key_size = key_size; 161 attr.value_size = 4; 162 attr.inner_map_fd = inner_map_fd; 163 attr.max_entries = max_entries; 164 attr.map_flags = map_flags; 165 memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1)); 166 167 if (node >= 0) { 168 attr.map_flags |= BPF_F_NUMA_NODE; 169 attr.numa_node = node; 170 } 171 172 return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); 173 } 174 175 int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name, 176 int key_size, int inner_map_fd, int max_entries, 177 __u32 map_flags) 178 { 179 return bpf_create_map_in_map_node(map_type, name, key_size, 180 inner_map_fd, max_entries, map_flags, 181 -1); 182 } 183 184 int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, 185 char *log_buf, size_t log_buf_sz) 186 { 187 union bpf_attr attr; 188 __u32 name_len; 189 int fd; 190 191 if (!load_attr) 192 return -EINVAL; 193 194 name_len = load_attr->name ? strlen(load_attr->name) : 0; 195 196 bzero(&attr, sizeof(attr)); 197 attr.prog_type = load_attr->prog_type; 198 attr.expected_attach_type = load_attr->expected_attach_type; 199 attr.insn_cnt = (__u32)load_attr->insns_cnt; 200 attr.insns = ptr_to_u64(load_attr->insns); 201 attr.license = ptr_to_u64(load_attr->license); 202 attr.log_buf = ptr_to_u64(NULL); 203 attr.log_size = 0; 204 attr.log_level = 0; 205 attr.kern_version = load_attr->kern_version; 206 attr.prog_ifindex = load_attr->prog_ifindex; 207 memcpy(attr.prog_name, load_attr->name, 208 min(name_len, BPF_OBJ_NAME_LEN - 1)); 209 210 fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); 211 if (fd >= 0 || !log_buf || !log_buf_sz) 212 return fd; 213 214 /* Try again with log */ 215 attr.log_buf = ptr_to_u64(log_buf); 216 attr.log_size = log_buf_sz; 217 attr.log_level = 1; 218 log_buf[0] = 0; 219 return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); 220 } 221 222 int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 223 size_t insns_cnt, const char *license, 224 __u32 kern_version, char *log_buf, 225 size_t log_buf_sz) 226 { 227 struct bpf_load_program_attr load_attr; 228 229 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 230 load_attr.prog_type = type; 231 load_attr.expected_attach_type = 0; 232 load_attr.name = NULL; 233 load_attr.insns = insns; 234 load_attr.insns_cnt = insns_cnt; 235 load_attr.license = license; 236 load_attr.kern_version = kern_version; 237 238 return bpf_load_program_xattr(&load_attr, log_buf, log_buf_sz); 239 } 240 241 int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, 242 size_t insns_cnt, int strict_alignment, 243 const char *license, __u32 kern_version, 244 char *log_buf, size_t log_buf_sz, int log_level) 245 { 246 union bpf_attr attr; 247 248 bzero(&attr, sizeof(attr)); 249 attr.prog_type = type; 250 attr.insn_cnt = (__u32)insns_cnt; 251 attr.insns = ptr_to_u64(insns); 252 attr.license = ptr_to_u64(license); 253 attr.log_buf = ptr_to_u64(log_buf); 254 attr.log_size = log_buf_sz; 255 attr.log_level = log_level; 256 log_buf[0] = 0; 257 attr.kern_version = kern_version; 258 attr.prog_flags = strict_alignment ? BPF_F_STRICT_ALIGNMENT : 0; 259 260 return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); 261 } 262 263 int bpf_map_update_elem(int fd, const void *key, const void *value, 264 __u64 flags) 265 { 266 union bpf_attr attr; 267 268 bzero(&attr, sizeof(attr)); 269 attr.map_fd = fd; 270 attr.key = ptr_to_u64(key); 271 attr.value = ptr_to_u64(value); 272 attr.flags = flags; 273 274 return sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); 275 } 276 277 int bpf_map_lookup_elem(int fd, const void *key, void *value) 278 { 279 union bpf_attr attr; 280 281 bzero(&attr, sizeof(attr)); 282 attr.map_fd = fd; 283 attr.key = ptr_to_u64(key); 284 attr.value = ptr_to_u64(value); 285 286 return sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)); 287 } 288 289 int bpf_map_delete_elem(int fd, const void *key) 290 { 291 union bpf_attr attr; 292 293 bzero(&attr, sizeof(attr)); 294 attr.map_fd = fd; 295 attr.key = ptr_to_u64(key); 296 297 return sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr)); 298 } 299 300 int bpf_map_get_next_key(int fd, const void *key, void *next_key) 301 { 302 union bpf_attr attr; 303 304 bzero(&attr, sizeof(attr)); 305 attr.map_fd = fd; 306 attr.key = ptr_to_u64(key); 307 attr.next_key = ptr_to_u64(next_key); 308 309 return sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr)); 310 } 311 312 int bpf_obj_pin(int fd, const char *pathname) 313 { 314 union bpf_attr attr; 315 316 bzero(&attr, sizeof(attr)); 317 attr.pathname = ptr_to_u64((void *)pathname); 318 attr.bpf_fd = fd; 319 320 return sys_bpf(BPF_OBJ_PIN, &attr, sizeof(attr)); 321 } 322 323 int bpf_obj_get(const char *pathname) 324 { 325 union bpf_attr attr; 326 327 bzero(&attr, sizeof(attr)); 328 attr.pathname = ptr_to_u64((void *)pathname); 329 330 return sys_bpf(BPF_OBJ_GET, &attr, sizeof(attr)); 331 } 332 333 int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type, 334 unsigned int flags) 335 { 336 union bpf_attr attr; 337 338 bzero(&attr, sizeof(attr)); 339 attr.target_fd = target_fd; 340 attr.attach_bpf_fd = prog_fd; 341 attr.attach_type = type; 342 attr.attach_flags = flags; 343 344 return sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr)); 345 } 346 347 int bpf_prog_detach(int target_fd, enum bpf_attach_type type) 348 { 349 union bpf_attr attr; 350 351 bzero(&attr, sizeof(attr)); 352 attr.target_fd = target_fd; 353 attr.attach_type = type; 354 355 return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr)); 356 } 357 358 int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type) 359 { 360 union bpf_attr attr; 361 362 bzero(&attr, sizeof(attr)); 363 attr.target_fd = target_fd; 364 attr.attach_bpf_fd = prog_fd; 365 attr.attach_type = type; 366 367 return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr)); 368 } 369 370 int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, 371 __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt) 372 { 373 union bpf_attr attr; 374 int ret; 375 376 bzero(&attr, sizeof(attr)); 377 attr.query.target_fd = target_fd; 378 attr.query.attach_type = type; 379 attr.query.query_flags = query_flags; 380 attr.query.prog_cnt = *prog_cnt; 381 attr.query.prog_ids = ptr_to_u64(prog_ids); 382 383 ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr)); 384 if (attach_flags) 385 *attach_flags = attr.query.attach_flags; 386 *prog_cnt = attr.query.prog_cnt; 387 return ret; 388 } 389 390 int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size, 391 void *data_out, __u32 *size_out, __u32 *retval, 392 __u32 *duration) 393 { 394 union bpf_attr attr; 395 int ret; 396 397 bzero(&attr, sizeof(attr)); 398 attr.test.prog_fd = prog_fd; 399 attr.test.data_in = ptr_to_u64(data); 400 attr.test.data_out = ptr_to_u64(data_out); 401 attr.test.data_size_in = size; 402 attr.test.repeat = repeat; 403 404 ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); 405 if (size_out) 406 *size_out = attr.test.data_size_out; 407 if (retval) 408 *retval = attr.test.retval; 409 if (duration) 410 *duration = attr.test.duration; 411 return ret; 412 } 413 414 int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id) 415 { 416 union bpf_attr attr; 417 int err; 418 419 bzero(&attr, sizeof(attr)); 420 attr.start_id = start_id; 421 422 err = sys_bpf(BPF_PROG_GET_NEXT_ID, &attr, sizeof(attr)); 423 if (!err) 424 *next_id = attr.next_id; 425 426 return err; 427 } 428 429 int bpf_map_get_next_id(__u32 start_id, __u32 *next_id) 430 { 431 union bpf_attr attr; 432 int err; 433 434 bzero(&attr, sizeof(attr)); 435 attr.start_id = start_id; 436 437 err = sys_bpf(BPF_MAP_GET_NEXT_ID, &attr, sizeof(attr)); 438 if (!err) 439 *next_id = attr.next_id; 440 441 return err; 442 } 443 444 int bpf_prog_get_fd_by_id(__u32 id) 445 { 446 union bpf_attr attr; 447 448 bzero(&attr, sizeof(attr)); 449 attr.prog_id = id; 450 451 return sys_bpf(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr)); 452 } 453 454 int bpf_map_get_fd_by_id(__u32 id) 455 { 456 union bpf_attr attr; 457 458 bzero(&attr, sizeof(attr)); 459 attr.map_id = id; 460 461 return sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr)); 462 } 463 464 int bpf_btf_get_fd_by_id(__u32 id) 465 { 466 union bpf_attr attr; 467 468 bzero(&attr, sizeof(attr)); 469 attr.btf_id = id; 470 471 return sys_bpf(BPF_BTF_GET_FD_BY_ID, &attr, sizeof(attr)); 472 } 473 474 int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len) 475 { 476 union bpf_attr attr; 477 int err; 478 479 bzero(&attr, sizeof(attr)); 480 attr.info.bpf_fd = prog_fd; 481 attr.info.info_len = *info_len; 482 attr.info.info = ptr_to_u64(info); 483 484 err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)); 485 if (!err) 486 *info_len = attr.info.info_len; 487 488 return err; 489 } 490 491 int bpf_raw_tracepoint_open(const char *name, int prog_fd) 492 { 493 union bpf_attr attr; 494 495 bzero(&attr, sizeof(attr)); 496 attr.raw_tracepoint.name = ptr_to_u64(name); 497 attr.raw_tracepoint.prog_fd = prog_fd; 498 499 return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr)); 500 } 501 502 int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags) 503 { 504 struct sockaddr_nl sa; 505 int sock, seq = 0, len, ret = -1; 506 char buf[4096]; 507 struct nlattr *nla, *nla_xdp; 508 struct { 509 struct nlmsghdr nh; 510 struct ifinfomsg ifinfo; 511 char attrbuf[64]; 512 } req; 513 struct nlmsghdr *nh; 514 struct nlmsgerr *err; 515 socklen_t addrlen; 516 int one = 1; 517 518 memset(&sa, 0, sizeof(sa)); 519 sa.nl_family = AF_NETLINK; 520 521 sock = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); 522 if (sock < 0) { 523 return -errno; 524 } 525 526 if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK, 527 &one, sizeof(one)) < 0) { 528 fprintf(stderr, "Netlink error reporting not supported\n"); 529 } 530 531 if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) { 532 ret = -errno; 533 goto cleanup; 534 } 535 536 addrlen = sizeof(sa); 537 if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) { 538 ret = -errno; 539 goto cleanup; 540 } 541 542 if (addrlen != sizeof(sa)) { 543 ret = -LIBBPF_ERRNO__INTERNAL; 544 goto cleanup; 545 } 546 547 memset(&req, 0, sizeof(req)); 548 req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)); 549 req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; 550 req.nh.nlmsg_type = RTM_SETLINK; 551 req.nh.nlmsg_pid = 0; 552 req.nh.nlmsg_seq = ++seq; 553 req.ifinfo.ifi_family = AF_UNSPEC; 554 req.ifinfo.ifi_index = ifindex; 555 556 /* started nested attribute for XDP */ 557 nla = (struct nlattr *)(((char *)&req) 558 + NLMSG_ALIGN(req.nh.nlmsg_len)); 559 nla->nla_type = NLA_F_NESTED | IFLA_XDP; 560 nla->nla_len = NLA_HDRLEN; 561 562 /* add XDP fd */ 563 nla_xdp = (struct nlattr *)((char *)nla + nla->nla_len); 564 nla_xdp->nla_type = IFLA_XDP_FD; 565 nla_xdp->nla_len = NLA_HDRLEN + sizeof(int); 566 memcpy((char *)nla_xdp + NLA_HDRLEN, &fd, sizeof(fd)); 567 nla->nla_len += nla_xdp->nla_len; 568 569 /* if user passed in any flags, add those too */ 570 if (flags) { 571 nla_xdp = (struct nlattr *)((char *)nla + nla->nla_len); 572 nla_xdp->nla_type = IFLA_XDP_FLAGS; 573 nla_xdp->nla_len = NLA_HDRLEN + sizeof(flags); 574 memcpy((char *)nla_xdp + NLA_HDRLEN, &flags, sizeof(flags)); 575 nla->nla_len += nla_xdp->nla_len; 576 } 577 578 req.nh.nlmsg_len += NLA_ALIGN(nla->nla_len); 579 580 if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { 581 ret = -errno; 582 goto cleanup; 583 } 584 585 len = recv(sock, buf, sizeof(buf), 0); 586 if (len < 0) { 587 ret = -errno; 588 goto cleanup; 589 } 590 591 for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, len); 592 nh = NLMSG_NEXT(nh, len)) { 593 if (nh->nlmsg_pid != sa.nl_pid) { 594 ret = -LIBBPF_ERRNO__WRNGPID; 595 goto cleanup; 596 } 597 if (nh->nlmsg_seq != seq) { 598 ret = -LIBBPF_ERRNO__INVSEQ; 599 goto cleanup; 600 } 601 switch (nh->nlmsg_type) { 602 case NLMSG_ERROR: 603 err = (struct nlmsgerr *)NLMSG_DATA(nh); 604 if (!err->error) 605 continue; 606 ret = err->error; 607 nla_dump_errormsg(nh); 608 goto cleanup; 609 case NLMSG_DONE: 610 break; 611 default: 612 break; 613 } 614 } 615 616 ret = 0; 617 618 cleanup: 619 close(sock); 620 return ret; 621 } 622 623 int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, 624 bool do_log) 625 { 626 union bpf_attr attr = {}; 627 int fd; 628 629 attr.btf = ptr_to_u64(btf); 630 attr.btf_size = btf_size; 631 632 retry: 633 if (do_log && log_buf && log_buf_size) { 634 attr.btf_log_level = 1; 635 attr.btf_log_size = log_buf_size; 636 attr.btf_log_buf = ptr_to_u64(log_buf); 637 } 638 639 fd = sys_bpf(BPF_BTF_LOAD, &attr, sizeof(attr)); 640 if (fd == -1 && !do_log && log_buf && log_buf_size) { 641 do_log = true; 642 goto retry; 643 } 644 645 return fd; 646 } 647 648 int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, 649 __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, 650 __u64 *probe_addr) 651 { 652 union bpf_attr attr = {}; 653 int err; 654 655 attr.task_fd_query.pid = pid; 656 attr.task_fd_query.fd = fd; 657 attr.task_fd_query.flags = flags; 658 attr.task_fd_query.buf = ptr_to_u64(buf); 659 attr.task_fd_query.buf_len = *buf_len; 660 661 err = sys_bpf(BPF_TASK_FD_QUERY, &attr, sizeof(attr)); 662 *buf_len = attr.task_fd_query.buf_len; 663 *prog_id = attr.task_fd_query.prog_id; 664 *fd_type = attr.task_fd_query.fd_type; 665 *probe_offset = attr.task_fd_query.probe_offset; 666 *probe_addr = attr.task_fd_query.probe_addr; 667 668 return err; 669 } 670