1 // SPDX-License-Identifier: GPL-2.0-only 2 #define _GNU_SOURCE 3 4 #include <arpa/inet.h> 5 #include <bpf/bpf.h> 6 #include <bpf/libbpf.h> 7 #include <errno.h> 8 #include <fcntl.h> 9 #include <getopt.h> 10 #include <linux/ethtool.h> 11 #include <linux/hashtable.h> 12 #include <linux/if_link.h> 13 #include <linux/jhash.h> 14 #include <linux/limits.h> 15 #include <linux/list.h> 16 #include <linux/sockios.h> 17 #include <locale.h> 18 #include <math.h> 19 #include <net/if.h> 20 #include <poll.h> 21 #include <signal.h> 22 #include <stdbool.h> 23 #include <stdio.h> 24 #include <stdlib.h> 25 #include <string.h> 26 #include <sys/ioctl.h> 27 #include <sys/mman.h> 28 #include <sys/resource.h> 29 #include <sys/signalfd.h> 30 #include <sys/sysinfo.h> 31 #include <sys/timerfd.h> 32 #include <sys/utsname.h> 33 #include <time.h> 34 #include <unistd.h> 35 36 #include "bpf_util.h" 37 #include "xdp_sample_user.h" 38 39 #define __sample_print(fmt, cond, ...) \ 40 ({ \ 41 if (cond) \ 42 printf(fmt, ##__VA_ARGS__); \ 43 }) 44 45 #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__) 46 #define print_default(fmt, ...) \ 47 __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__) 48 #define __print_err(err, fmt, ...) \ 49 ({ \ 50 __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \ 51 ##__VA_ARGS__); \ 52 sample_err_exp = sample_err_exp ? true : err > 0; \ 53 }) 54 #define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__) 55 56 #define __COLUMN(x) "%'10" x " %-13s" 57 #define FMT_COLUMNf __COLUMN(".0f") 58 #define FMT_COLUMNd __COLUMN("d") 59 #define FMT_COLUMNl __COLUMN("llu") 60 #define RX(rx) rx, "rx/s" 61 #define PPS(pps) pps, "pkt/s" 62 #define DROP(drop) drop, "drop/s" 63 #define ERR(err) err, "error/s" 64 #define HITS(hits) hits, "hit/s" 65 #define XMIT(xmit) xmit, "xmit/s" 66 #define PASS(pass) pass, "pass/s" 67 #define REDIR(redir) redir, "redir/s" 68 #define NANOSEC_PER_SEC 1000000000 /* 10^9 */ 69 70 #define XDP_UNKNOWN (XDP_REDIRECT + 1) 71 #define XDP_ACTION_MAX (XDP_UNKNOWN + 1) 72 #define XDP_REDIRECT_ERR_MAX 7 73 74 enum map_type { 75 MAP_RX, 76 MAP_REDIRECT_ERR, 77 MAP_CPUMAP_ENQUEUE, 78 MAP_CPUMAP_KTHREAD, 79 MAP_EXCEPTION, 80 MAP_DEVMAP_XMIT, 81 MAP_DEVMAP_XMIT_MULTI, 82 NUM_MAP, 83 }; 84 85 enum log_level { 86 LL_DEFAULT = 1U << 0, 87 LL_SIMPLE = 1U << 1, 88 LL_DEBUG = 1U << 2, 89 }; 90 91 struct record { 92 __u64 timestamp; 93 struct datarec total; 94 struct datarec *cpu; 95 }; 96 97 struct map_entry { 98 struct hlist_node node; 99 __u64 pair; 100 struct record val; 101 }; 102 103 struct stats_record { 104 struct record rx_cnt; 105 struct record redir_err[XDP_REDIRECT_ERR_MAX]; 106 struct record kthread; 107 struct record exception[XDP_ACTION_MAX]; 108 struct record devmap_xmit; 109 DECLARE_HASHTABLE(xmit_map, 5); 110 struct record enq[]; 111 }; 112 113 struct sample_output { 114 struct { 115 __u64 rx; 116 __u64 redir; 117 __u64 drop; 118 __u64 drop_xmit; 119 __u64 err; 120 __u64 xmit; 121 } totals; 122 struct { 123 union { 124 __u64 pps; 125 __u64 num; 126 }; 127 __u64 drop; 128 __u64 err; 129 } rx_cnt; 130 struct { 131 __u64 suc; 132 __u64 err; 133 } redir_cnt; 134 struct { 135 __u64 hits; 136 } except_cnt; 137 struct { 138 __u64 pps; 139 __u64 drop; 140 __u64 err; 141 double bavg; 142 } xmit_cnt; 143 }; 144 145 struct xdp_desc { 146 int ifindex; 147 __u32 prog_id; 148 int flags; 149 } sample_xdp_progs[32]; 150 151 struct datarec *sample_mmap[NUM_MAP]; 152 struct bpf_map *sample_map[NUM_MAP]; 153 size_t sample_map_count[NUM_MAP]; 154 enum log_level sample_log_level; 155 struct sample_output sample_out; 156 unsigned long sample_interval; 157 bool sample_err_exp; 158 int sample_xdp_cnt; 159 int sample_n_cpus; 160 int sample_sig_fd; 161 int sample_mask; 162 163 static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = { 164 /* Key=1 keeps unknown errors */ 165 "Success", 166 "Unknown", 167 "EINVAL", 168 "ENETDOWN", 169 "EMSGSIZE", 170 "EOPNOTSUPP", 171 "ENOSPC", 172 }; 173 174 /* Keyed from Unknown */ 175 static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = { 176 "Unknown error", 177 "Invalid redirection", 178 "Device being redirected to is down", 179 "Packet length too large for device", 180 "Operation not supported", 181 "No space in ptr_ring of cpumap kthread", 182 }; 183 184 static const char *xdp_action_names[XDP_ACTION_MAX] = { 185 [XDP_ABORTED] = "XDP_ABORTED", 186 [XDP_DROP] = "XDP_DROP", 187 [XDP_PASS] = "XDP_PASS", 188 [XDP_TX] = "XDP_TX", 189 [XDP_REDIRECT] = "XDP_REDIRECT", 190 [XDP_UNKNOWN] = "XDP_UNKNOWN", 191 }; 192 193 static __u64 gettime(void) 194 { 195 struct timespec t; 196 int res; 197 198 res = clock_gettime(CLOCK_MONOTONIC, &t); 199 if (res < 0) { 200 fprintf(stderr, "Error with gettimeofday! (%i)\n", res); 201 return UINT64_MAX; 202 } 203 return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec; 204 } 205 206 static const char *action2str(int action) 207 { 208 if (action < XDP_ACTION_MAX) 209 return xdp_action_names[action]; 210 return NULL; 211 } 212 213 static void sample_print_help(int mask) 214 { 215 printf("Output format description\n\n" 216 "By default, redirect success statistics are disabled, use -s to enable.\n" 217 "The terse output mode is default, verbose mode can be activated using -v\n" 218 "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n" 219 "Terse mode displays at most the following fields:\n" 220 " rx/s Number of packets received per second\n" 221 " redir/s Number of packets successfully redirected per second\n" 222 " err,drop/s Aggregated count of errors per second (including dropped packets)\n" 223 " xmit/s Number of packets transmitted on the output device per second\n\n" 224 "Output description for verbose mode:\n" 225 " FIELD DESCRIPTION\n"); 226 227 if (mask & SAMPLE_RX_CNT) { 228 printf(" receive\t\tDisplays the number of packets received & errors encountered\n" 229 " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n" 230 " \t\t\tand drop statistics will be expanded inline in terse mode.\n" 231 " \t\t\t\tpkt/s - Packets received per second\n" 232 " \t\t\t\tdrop/s - Packets dropped per second\n" 233 " \t\t\t\terror/s - Errors encountered per second\n\n"); 234 } 235 if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { 236 printf(" redirect\t\tDisplays the number of packets successfully redirected\n" 237 " \t\t\tErrors encountered are expanded under redirect_err field\n" 238 " \t\t\tNote that passing -s to enable it has a per packet overhead\n" 239 " \t\t\t\tredir/s - Packets redirected successfully per second\n\n" 240 " redirect_err\t\tDisplays the number of packets that failed redirection\n" 241 " \t\t\tThe errno is expanded under this field with per CPU count\n" 242 " \t\t\tThe recognized errors are:\n"); 243 244 for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++) 245 printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i], 246 xdp_redirect_err_help[i - 1]); 247 248 printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n"); 249 } 250 251 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) { 252 printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n" 253 " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n" 254 " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n" 255 " \t\t\tpackets to.\n" 256 " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n" 257 " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n" 258 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); 259 } 260 261 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) { 262 printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n" 263 " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n" 264 " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n" 265 " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n" 266 " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n" 267 " \t\t\t\tdrop/s - Packets dropped per second in kthread\n" 268 " \t\t\t\tsched - Number of times kthread called schedule()\n\n" 269 " \t\t\txdp_stats (also expands to per-CPU counts)\n" 270 " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n" 271 " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n" 272 " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n"); 273 } 274 275 if (mask & SAMPLE_EXCEPTION_CNT) { 276 printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n" 277 " \t\t\tThis can occur due to internal driver errors, unrecognized\n" 278 " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n" 279 " \t\t\tEach action is expanded below this field with its count\n" 280 " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n"); 281 } 282 283 if (mask & SAMPLE_DEVMAP_XMIT_CNT) { 284 printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n" 285 " \t\t\tThis tracepoint is invoked for successful transmissions on output\n" 286 " \t\t\tdevice but these statistics are not available for generic XDP mode,\n" 287 " \t\t\thence they will be omitted from the output when using SKB mode\n" 288 " \t\t\t\txmit/s - Number of packets that were transmitted per second\n" 289 " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n" 290 " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n" 291 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); 292 } 293 } 294 295 void sample_usage(char *argv[], const struct option *long_options, 296 const char *doc, int mask, bool error) 297 { 298 int i; 299 300 if (!error) 301 sample_print_help(mask); 302 303 printf("\n%s\nOption for %s:\n", doc, argv[0]); 304 for (i = 0; long_options[i].name != 0; i++) { 305 printf(" --%-15s", long_options[i].name); 306 if (long_options[i].flag != NULL) 307 printf(" flag (internal value: %d)", 308 *long_options[i].flag); 309 else 310 printf("\t short-option: -%c", long_options[i].val); 311 printf("\n"); 312 } 313 printf("\n"); 314 } 315 316 static struct datarec *alloc_record_per_cpu(void) 317 { 318 unsigned int nr_cpus = libbpf_num_possible_cpus(); 319 struct datarec *array; 320 321 array = calloc(nr_cpus, sizeof(*array)); 322 if (!array) { 323 fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n", 324 nr_cpus); 325 return NULL; 326 } 327 return array; 328 } 329 330 static int map_entry_init(struct map_entry *e, __u64 pair) 331 { 332 e->pair = pair; 333 INIT_HLIST_NODE(&e->node); 334 e->val.timestamp = gettime(); 335 e->val.cpu = alloc_record_per_cpu(); 336 if (!e->val.cpu) 337 return -ENOMEM; 338 return 0; 339 } 340 341 static void map_collect_percpu(struct datarec *values, struct record *rec) 342 { 343 /* For percpu maps, userspace gets a value per possible CPU */ 344 unsigned int nr_cpus = libbpf_num_possible_cpus(); 345 __u64 sum_xdp_redirect = 0; 346 __u64 sum_processed = 0; 347 __u64 sum_xdp_pass = 0; 348 __u64 sum_xdp_drop = 0; 349 __u64 sum_dropped = 0; 350 __u64 sum_issue = 0; 351 int i; 352 353 /* Get time as close as possible to reading map contents */ 354 rec->timestamp = gettime(); 355 356 /* Record and sum values from each CPU */ 357 for (i = 0; i < nr_cpus; i++) { 358 rec->cpu[i].processed = READ_ONCE(values[i].processed); 359 rec->cpu[i].dropped = READ_ONCE(values[i].dropped); 360 rec->cpu[i].issue = READ_ONCE(values[i].issue); 361 rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass); 362 rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop); 363 rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect); 364 365 sum_processed += rec->cpu[i].processed; 366 sum_dropped += rec->cpu[i].dropped; 367 sum_issue += rec->cpu[i].issue; 368 sum_xdp_pass += rec->cpu[i].xdp_pass; 369 sum_xdp_drop += rec->cpu[i].xdp_drop; 370 sum_xdp_redirect += rec->cpu[i].xdp_redirect; 371 } 372 373 rec->total.processed = sum_processed; 374 rec->total.dropped = sum_dropped; 375 rec->total.issue = sum_issue; 376 rec->total.xdp_pass = sum_xdp_pass; 377 rec->total.xdp_drop = sum_xdp_drop; 378 rec->total.xdp_redirect = sum_xdp_redirect; 379 } 380 381 static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec) 382 { 383 unsigned int nr_cpus = bpf_num_possible_cpus(); 384 __u32 batch, count = 32; 385 struct datarec *values; 386 bool init = false; 387 __u64 *keys; 388 int i, ret; 389 390 keys = calloc(count, sizeof(__u64)); 391 if (!keys) 392 return -ENOMEM; 393 values = calloc(count * nr_cpus, sizeof(struct datarec)); 394 if (!values) { 395 free(keys); 396 return -ENOMEM; 397 } 398 399 for (;;) { 400 bool exit = false; 401 402 ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch, 403 keys, values, &count, NULL); 404 if (ret < 0 && errno != ENOENT) 405 break; 406 if (errno == ENOENT) 407 exit = true; 408 409 init = true; 410 for (i = 0; i < count; i++) { 411 struct map_entry *e, *x = NULL; 412 __u64 pair = keys[i]; 413 struct datarec *arr; 414 415 arr = &values[i * nr_cpus]; 416 hash_for_each_possible(rec->xmit_map, e, node, pair) { 417 if (e->pair == pair) { 418 x = e; 419 break; 420 } 421 } 422 if (!x) { 423 x = calloc(1, sizeof(*x)); 424 if (!x) 425 goto cleanup; 426 if (map_entry_init(x, pair) < 0) { 427 free(x); 428 goto cleanup; 429 } 430 hash_add(rec->xmit_map, &x->node, pair); 431 } 432 map_collect_percpu(arr, &x->val); 433 } 434 435 if (exit) 436 break; 437 count = 32; 438 } 439 440 free(values); 441 free(keys); 442 return 0; 443 cleanup: 444 free(values); 445 free(keys); 446 return -ENOMEM; 447 } 448 449 static struct stats_record *alloc_stats_record(void) 450 { 451 struct stats_record *rec; 452 int i; 453 454 rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record)); 455 if (!rec) { 456 fprintf(stderr, "Failed to allocate memory\n"); 457 return NULL; 458 } 459 460 if (sample_mask & SAMPLE_RX_CNT) { 461 rec->rx_cnt.cpu = alloc_record_per_cpu(); 462 if (!rec->rx_cnt.cpu) { 463 fprintf(stderr, 464 "Failed to allocate rx_cnt per-CPU array\n"); 465 goto end_rec; 466 } 467 } 468 if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { 469 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) { 470 rec->redir_err[i].cpu = alloc_record_per_cpu(); 471 if (!rec->redir_err[i].cpu) { 472 fprintf(stderr, 473 "Failed to allocate redir_err per-CPU array for " 474 "\"%s\" case\n", 475 xdp_redirect_err_names[i]); 476 while (i--) 477 free(rec->redir_err[i].cpu); 478 goto end_rx_cnt; 479 } 480 } 481 } 482 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) { 483 rec->kthread.cpu = alloc_record_per_cpu(); 484 if (!rec->kthread.cpu) { 485 fprintf(stderr, 486 "Failed to allocate kthread per-CPU array\n"); 487 goto end_redir; 488 } 489 } 490 if (sample_mask & SAMPLE_EXCEPTION_CNT) { 491 for (i = 0; i < XDP_ACTION_MAX; i++) { 492 rec->exception[i].cpu = alloc_record_per_cpu(); 493 if (!rec->exception[i].cpu) { 494 fprintf(stderr, 495 "Failed to allocate exception per-CPU array for " 496 "\"%s\" case\n", 497 action2str(i)); 498 while (i--) 499 free(rec->exception[i].cpu); 500 goto end_kthread; 501 } 502 } 503 } 504 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) { 505 rec->devmap_xmit.cpu = alloc_record_per_cpu(); 506 if (!rec->devmap_xmit.cpu) { 507 fprintf(stderr, 508 "Failed to allocate devmap_xmit per-CPU array\n"); 509 goto end_exception; 510 } 511 } 512 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 513 hash_init(rec->xmit_map); 514 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) { 515 for (i = 0; i < sample_n_cpus; i++) { 516 rec->enq[i].cpu = alloc_record_per_cpu(); 517 if (!rec->enq[i].cpu) { 518 fprintf(stderr, 519 "Failed to allocate enqueue per-CPU array for " 520 "CPU %d\n", 521 i); 522 while (i--) 523 free(rec->enq[i].cpu); 524 goto end_devmap_xmit; 525 } 526 } 527 } 528 529 return rec; 530 531 end_devmap_xmit: 532 free(rec->devmap_xmit.cpu); 533 end_exception: 534 for (i = 0; i < XDP_ACTION_MAX; i++) 535 free(rec->exception[i].cpu); 536 end_kthread: 537 free(rec->kthread.cpu); 538 end_redir: 539 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) 540 free(rec->redir_err[i].cpu); 541 end_rx_cnt: 542 free(rec->rx_cnt.cpu); 543 end_rec: 544 free(rec); 545 return NULL; 546 } 547 548 static void free_stats_record(struct stats_record *r) 549 { 550 struct hlist_node *tmp; 551 struct map_entry *e; 552 int i; 553 554 for (i = 0; i < sample_n_cpus; i++) 555 free(r->enq[i].cpu); 556 hash_for_each_safe(r->xmit_map, i, tmp, e, node) { 557 hash_del(&e->node); 558 free(e->val.cpu); 559 free(e); 560 } 561 free(r->devmap_xmit.cpu); 562 for (i = 0; i < XDP_ACTION_MAX; i++) 563 free(r->exception[i].cpu); 564 free(r->kthread.cpu); 565 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) 566 free(r->redir_err[i].cpu); 567 free(r->rx_cnt.cpu); 568 free(r); 569 } 570 571 static double calc_period(struct record *r, struct record *p) 572 { 573 double period_ = 0; 574 __u64 period = 0; 575 576 period = r->timestamp - p->timestamp; 577 if (period > 0) 578 period_ = ((double)period / NANOSEC_PER_SEC); 579 580 return period_; 581 } 582 583 static double sample_round(double val) 584 { 585 if (val - floor(val) < 0.5) 586 return floor(val); 587 return ceil(val); 588 } 589 590 static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_) 591 { 592 __u64 packets = 0; 593 __u64 pps = 0; 594 595 if (period_ > 0) { 596 packets = r->processed - p->processed; 597 pps = sample_round(packets / period_); 598 } 599 return pps; 600 } 601 602 static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_) 603 { 604 __u64 packets = 0; 605 __u64 pps = 0; 606 607 if (period_ > 0) { 608 packets = r->dropped - p->dropped; 609 pps = sample_round(packets / period_); 610 } 611 return pps; 612 } 613 614 static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_) 615 { 616 __u64 packets = 0; 617 __u64 pps = 0; 618 619 if (period_ > 0) { 620 packets = r->issue - p->issue; 621 pps = sample_round(packets / period_); 622 } 623 return pps; 624 } 625 626 static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_) 627 { 628 __u64 packets = 0; 629 __u64 pps = 0; 630 631 if (period_ > 0) { 632 packets = r->info - p->info; 633 pps = sample_round(packets / period_); 634 } 635 return pps; 636 } 637 638 static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass, 639 double *xdp_drop, double *xdp_redirect, double period_) 640 { 641 *xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0; 642 if (period_ > 0) { 643 *xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_; 644 *xdp_pass = (r->xdp_pass - p->xdp_pass) / period_; 645 *xdp_drop = (r->xdp_drop - p->xdp_drop) / period_; 646 } 647 } 648 649 static void stats_get_rx_cnt(struct stats_record *stats_rec, 650 struct stats_record *stats_prev, 651 unsigned int nr_cpus, struct sample_output *out) 652 { 653 struct record *rec, *prev; 654 double t, pps, drop, err; 655 int i; 656 657 rec = &stats_rec->rx_cnt; 658 prev = &stats_prev->rx_cnt; 659 t = calc_period(rec, prev); 660 661 for (i = 0; i < nr_cpus; i++) { 662 struct datarec *r = &rec->cpu[i]; 663 struct datarec *p = &prev->cpu[i]; 664 char str[64]; 665 666 pps = calc_pps(r, p, t); 667 drop = calc_drop_pps(r, p, t); 668 err = calc_errs_pps(r, p, t); 669 if (!pps && !drop && !err) 670 continue; 671 672 snprintf(str, sizeof(str), "cpu:%d", i); 673 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 674 "\n", 675 str, PPS(pps), DROP(drop), ERR(err)); 676 } 677 678 if (out) { 679 pps = calc_pps(&rec->total, &prev->total, t); 680 drop = calc_drop_pps(&rec->total, &prev->total, t); 681 err = calc_errs_pps(&rec->total, &prev->total, t); 682 683 out->rx_cnt.pps = pps; 684 out->rx_cnt.drop = drop; 685 out->rx_cnt.err = err; 686 out->totals.rx += pps; 687 out->totals.drop += drop; 688 out->totals.err += err; 689 } 690 } 691 692 static void stats_get_cpumap_enqueue(struct stats_record *stats_rec, 693 struct stats_record *stats_prev, 694 unsigned int nr_cpus) 695 { 696 struct record *rec, *prev; 697 double t, pps, drop, err; 698 int i, to_cpu; 699 700 /* cpumap enqueue stats */ 701 for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) { 702 rec = &stats_rec->enq[to_cpu]; 703 prev = &stats_prev->enq[to_cpu]; 704 t = calc_period(rec, prev); 705 706 pps = calc_pps(&rec->total, &prev->total, t); 707 drop = calc_drop_pps(&rec->total, &prev->total, t); 708 err = calc_errs_pps(&rec->total, &prev->total, t); 709 710 if (pps > 0 || drop > 0) { 711 char str[64]; 712 713 snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu); 714 715 if (err > 0) 716 err = pps / err; /* calc average bulk size */ 717 718 print_err(drop, 719 " %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN( 720 ".2f") "\n", 721 str, PPS(pps), DROP(drop), err, "bulk-avg"); 722 } 723 724 for (i = 0; i < nr_cpus; i++) { 725 struct datarec *r = &rec->cpu[i]; 726 struct datarec *p = &prev->cpu[i]; 727 char str[64]; 728 729 pps = calc_pps(r, p, t); 730 drop = calc_drop_pps(r, p, t); 731 err = calc_errs_pps(r, p, t); 732 if (!pps && !drop && !err) 733 continue; 734 735 snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu); 736 if (err > 0) 737 err = pps / err; /* calc average bulk size */ 738 print_default( 739 " %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN( 740 ".2f") "\n", 741 str, PPS(pps), DROP(drop), err, "bulk-avg"); 742 } 743 } 744 } 745 746 static void stats_get_cpumap_remote(struct stats_record *stats_rec, 747 struct stats_record *stats_prev, 748 unsigned int nr_cpus) 749 { 750 double xdp_pass, xdp_drop, xdp_redirect; 751 struct record *rec, *prev; 752 double t; 753 int i; 754 755 rec = &stats_rec->kthread; 756 prev = &stats_prev->kthread; 757 t = calc_period(rec, prev); 758 759 calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop, 760 &xdp_redirect, t); 761 if (xdp_pass || xdp_drop || xdp_redirect) { 762 print_err(xdp_drop, 763 " %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n", 764 "xdp_stats", PASS(xdp_pass), DROP(xdp_drop), 765 REDIR(xdp_redirect)); 766 } 767 768 for (i = 0; i < nr_cpus; i++) { 769 struct datarec *r = &rec->cpu[i]; 770 struct datarec *p = &prev->cpu[i]; 771 char str[64]; 772 773 calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t); 774 if (!xdp_pass && !xdp_drop && !xdp_redirect) 775 continue; 776 777 snprintf(str, sizeof(str), "cpu:%d", i); 778 print_default(" %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 779 "\n", 780 str, PASS(xdp_pass), DROP(xdp_drop), 781 REDIR(xdp_redirect)); 782 } 783 } 784 785 static void stats_get_cpumap_kthread(struct stats_record *stats_rec, 786 struct stats_record *stats_prev, 787 unsigned int nr_cpus) 788 { 789 struct record *rec, *prev; 790 double t, pps, drop, err; 791 int i; 792 793 rec = &stats_rec->kthread; 794 prev = &stats_prev->kthread; 795 t = calc_period(rec, prev); 796 797 pps = calc_pps(&rec->total, &prev->total, t); 798 drop = calc_drop_pps(&rec->total, &prev->total, t); 799 err = calc_errs_pps(&rec->total, &prev->total, t); 800 801 print_err(drop, " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n", 802 pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err, 803 "sched"); 804 805 for (i = 0; i < nr_cpus; i++) { 806 struct datarec *r = &rec->cpu[i]; 807 struct datarec *p = &prev->cpu[i]; 808 char str[64]; 809 810 pps = calc_pps(r, p, t); 811 drop = calc_drop_pps(r, p, t); 812 err = calc_errs_pps(r, p, t); 813 if (!pps && !drop && !err) 814 continue; 815 816 snprintf(str, sizeof(str), "cpu:%d", i); 817 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 818 "\n", 819 str, PPS(pps), DROP(drop), err, "sched"); 820 } 821 } 822 823 static void stats_get_redirect_cnt(struct stats_record *stats_rec, 824 struct stats_record *stats_prev, 825 unsigned int nr_cpus, 826 struct sample_output *out) 827 { 828 struct record *rec, *prev; 829 double t, pps; 830 int i; 831 832 rec = &stats_rec->redir_err[0]; 833 prev = &stats_prev->redir_err[0]; 834 t = calc_period(rec, prev); 835 for (i = 0; i < nr_cpus; i++) { 836 struct datarec *r = &rec->cpu[i]; 837 struct datarec *p = &prev->cpu[i]; 838 char str[64]; 839 840 pps = calc_pps(r, p, t); 841 if (!pps) 842 continue; 843 844 snprintf(str, sizeof(str), "cpu:%d", i); 845 print_default(" %-18s " FMT_COLUMNf "\n", str, REDIR(pps)); 846 } 847 848 if (out) { 849 pps = calc_pps(&rec->total, &prev->total, t); 850 out->redir_cnt.suc = pps; 851 out->totals.redir += pps; 852 } 853 } 854 855 static void stats_get_redirect_err_cnt(struct stats_record *stats_rec, 856 struct stats_record *stats_prev, 857 unsigned int nr_cpus, 858 struct sample_output *out) 859 { 860 struct record *rec, *prev; 861 double t, drop, sum = 0; 862 int rec_i, i; 863 864 for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) { 865 char str[64]; 866 867 rec = &stats_rec->redir_err[rec_i]; 868 prev = &stats_prev->redir_err[rec_i]; 869 t = calc_period(rec, prev); 870 871 drop = calc_drop_pps(&rec->total, &prev->total, t); 872 if (drop > 0 && !out) { 873 snprintf(str, sizeof(str), 874 sample_log_level & LL_DEFAULT ? "%s total" : 875 "%s", 876 xdp_redirect_err_names[rec_i]); 877 print_err(drop, " %-18s " FMT_COLUMNf "\n", str, 878 ERR(drop)); 879 } 880 881 for (i = 0; i < nr_cpus; i++) { 882 struct datarec *r = &rec->cpu[i]; 883 struct datarec *p = &prev->cpu[i]; 884 double drop; 885 886 drop = calc_drop_pps(r, p, t); 887 if (!drop) 888 continue; 889 890 snprintf(str, sizeof(str), "cpu:%d", i); 891 print_default(" %-16s" FMT_COLUMNf "\n", str, 892 ERR(drop)); 893 } 894 895 sum += drop; 896 } 897 898 if (out) { 899 out->redir_cnt.err = sum; 900 out->totals.err += sum; 901 } 902 } 903 904 static void stats_get_exception_cnt(struct stats_record *stats_rec, 905 struct stats_record *stats_prev, 906 unsigned int nr_cpus, 907 struct sample_output *out) 908 { 909 double t, drop, sum = 0; 910 struct record *rec, *prev; 911 int rec_i, i; 912 913 for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) { 914 rec = &stats_rec->exception[rec_i]; 915 prev = &stats_prev->exception[rec_i]; 916 t = calc_period(rec, prev); 917 918 drop = calc_drop_pps(&rec->total, &prev->total, t); 919 /* Fold out errors after heading */ 920 sum += drop; 921 922 if (drop > 0 && !out) { 923 print_always(" %-18s " FMT_COLUMNf "\n", 924 action2str(rec_i), ERR(drop)); 925 926 for (i = 0; i < nr_cpus; i++) { 927 struct datarec *r = &rec->cpu[i]; 928 struct datarec *p = &prev->cpu[i]; 929 char str[64]; 930 double drop; 931 932 drop = calc_drop_pps(r, p, t); 933 if (!drop) 934 continue; 935 936 snprintf(str, sizeof(str), "cpu:%d", i); 937 print_default(" %-16s" FMT_COLUMNf "\n", 938 str, ERR(drop)); 939 } 940 } 941 } 942 943 if (out) { 944 out->except_cnt.hits = sum; 945 out->totals.err += sum; 946 } 947 } 948 949 static void stats_get_devmap_xmit(struct stats_record *stats_rec, 950 struct stats_record *stats_prev, 951 unsigned int nr_cpus, 952 struct sample_output *out) 953 { 954 double pps, drop, info, err; 955 struct record *rec, *prev; 956 double t; 957 int i; 958 959 rec = &stats_rec->devmap_xmit; 960 prev = &stats_prev->devmap_xmit; 961 t = calc_period(rec, prev); 962 for (i = 0; i < nr_cpus; i++) { 963 struct datarec *r = &rec->cpu[i]; 964 struct datarec *p = &prev->cpu[i]; 965 char str[64]; 966 967 pps = calc_pps(r, p, t); 968 drop = calc_drop_pps(r, p, t); 969 err = calc_errs_pps(r, p, t); 970 971 if (!pps && !drop && !err) 972 continue; 973 974 snprintf(str, sizeof(str), "cpu:%d", i); 975 info = calc_info_pps(r, p, t); 976 if (info > 0) 977 info = (pps + drop) / info; /* calc avg bulk */ 978 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 979 __COLUMN(".2f") "\n", 980 str, XMIT(pps), DROP(drop), err, "drv_err/s", 981 info, "bulk-avg"); 982 } 983 if (out) { 984 pps = calc_pps(&rec->total, &prev->total, t); 985 drop = calc_drop_pps(&rec->total, &prev->total, t); 986 info = calc_info_pps(&rec->total, &prev->total, t); 987 if (info > 0) 988 info = (pps + drop) / info; /* calc avg bulk */ 989 err = calc_errs_pps(&rec->total, &prev->total, t); 990 991 out->xmit_cnt.pps = pps; 992 out->xmit_cnt.drop = drop; 993 out->xmit_cnt.bavg = info; 994 out->xmit_cnt.err = err; 995 out->totals.xmit += pps; 996 out->totals.drop_xmit += drop; 997 out->totals.err += err; 998 } 999 } 1000 1001 static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec, 1002 struct stats_record *stats_prev, 1003 unsigned int nr_cpus, 1004 struct sample_output *out, 1005 bool xmit_total) 1006 { 1007 double pps, drop, info, err; 1008 struct map_entry *entry; 1009 struct record *r, *p; 1010 double t; 1011 int bkt; 1012 1013 hash_for_each(stats_rec->xmit_map, bkt, entry, node) { 1014 struct map_entry *e, *x = NULL; 1015 char ifname_from[IFNAMSIZ]; 1016 char ifname_to[IFNAMSIZ]; 1017 const char *fstr, *tstr; 1018 unsigned long prev_time; 1019 struct record beg = {}; 1020 __u32 from_idx, to_idx; 1021 char str[128]; 1022 __u64 pair; 1023 int i; 1024 1025 prev_time = sample_interval * NANOSEC_PER_SEC; 1026 1027 pair = entry->pair; 1028 from_idx = pair >> 32; 1029 to_idx = pair & 0xFFFFFFFF; 1030 1031 r = &entry->val; 1032 beg.timestamp = r->timestamp - prev_time; 1033 1034 /* Find matching entry from stats_prev map */ 1035 hash_for_each_possible(stats_prev->xmit_map, e, node, pair) { 1036 if (e->pair == pair) { 1037 x = e; 1038 break; 1039 } 1040 } 1041 if (x) 1042 p = &x->val; 1043 else 1044 p = &beg; 1045 t = calc_period(r, p); 1046 pps = calc_pps(&r->total, &p->total, t); 1047 drop = calc_drop_pps(&r->total, &p->total, t); 1048 info = calc_info_pps(&r->total, &p->total, t); 1049 if (info > 0) 1050 info = (pps + drop) / info; /* calc avg bulk */ 1051 err = calc_errs_pps(&r->total, &p->total, t); 1052 1053 if (out) { 1054 /* We are responsible for filling out totals */ 1055 out->totals.xmit += pps; 1056 out->totals.drop_xmit += drop; 1057 out->totals.err += err; 1058 continue; 1059 } 1060 1061 fstr = tstr = NULL; 1062 if (if_indextoname(from_idx, ifname_from)) 1063 fstr = ifname_from; 1064 if (if_indextoname(to_idx, ifname_to)) 1065 tstr = ifname_to; 1066 1067 snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?", 1068 tstr ?: "?"); 1069 /* Skip idle streams of redirection */ 1070 if (pps || drop || err) { 1071 print_err(drop, 1072 " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 1073 __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop), 1074 err, "drv_err/s", info, "bulk-avg"); 1075 } 1076 1077 for (i = 0; i < nr_cpus; i++) { 1078 struct datarec *rc = &r->cpu[i]; 1079 struct datarec *pc, p_beg = {}; 1080 char str[64]; 1081 1082 pc = p == &beg ? &p_beg : &p->cpu[i]; 1083 1084 pps = calc_pps(rc, pc, t); 1085 drop = calc_drop_pps(rc, pc, t); 1086 err = calc_errs_pps(rc, pc, t); 1087 1088 if (!pps && !drop && !err) 1089 continue; 1090 1091 snprintf(str, sizeof(str), "cpu:%d", i); 1092 info = calc_info_pps(rc, pc, t); 1093 if (info > 0) 1094 info = (pps + drop) / info; /* calc avg bulk */ 1095 1096 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 1097 __COLUMN(".2f") "\n", str, XMIT(pps), 1098 DROP(drop), err, "drv_err/s", info, "bulk-avg"); 1099 } 1100 } 1101 } 1102 1103 static void stats_print(const char *prefix, int mask, struct stats_record *r, 1104 struct stats_record *p, struct sample_output *out) 1105 { 1106 int nr_cpus = libbpf_num_possible_cpus(); 1107 const char *str; 1108 1109 print_always("%-23s", prefix ?: "Summary"); 1110 if (mask & SAMPLE_RX_CNT) 1111 print_always(FMT_COLUMNl, RX(out->totals.rx)); 1112 if (mask & SAMPLE_REDIRECT_CNT) 1113 print_always(FMT_COLUMNl, REDIR(out->totals.redir)); 1114 printf(FMT_COLUMNl, 1115 out->totals.err + out->totals.drop + out->totals.drop_xmit, 1116 "err,drop/s"); 1117 if (mask & SAMPLE_DEVMAP_XMIT_CNT || 1118 mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 1119 printf(FMT_COLUMNl, XMIT(out->totals.xmit)); 1120 printf("\n"); 1121 1122 if (mask & SAMPLE_RX_CNT) { 1123 str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ? 1124 "receive total" : 1125 "receive"; 1126 print_err((out->rx_cnt.err || out->rx_cnt.drop), 1127 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n", 1128 str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop), 1129 ERR(out->rx_cnt.err)); 1130 1131 stats_get_rx_cnt(r, p, nr_cpus, NULL); 1132 } 1133 1134 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) 1135 stats_get_cpumap_enqueue(r, p, nr_cpus); 1136 1137 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) { 1138 stats_get_cpumap_kthread(r, p, nr_cpus); 1139 stats_get_cpumap_remote(r, p, nr_cpus); 1140 } 1141 1142 if (mask & SAMPLE_REDIRECT_CNT) { 1143 str = out->redir_cnt.suc ? "redirect total" : "redirect"; 1144 print_default(" %-20s " FMT_COLUMNl "\n", str, 1145 REDIR(out->redir_cnt.suc)); 1146 1147 stats_get_redirect_cnt(r, p, nr_cpus, NULL); 1148 } 1149 1150 if (mask & SAMPLE_REDIRECT_ERR_CNT) { 1151 str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ? 1152 "redirect_err total" : 1153 "redirect_err"; 1154 print_err(out->redir_cnt.err, " %-20s " FMT_COLUMNl "\n", str, 1155 ERR(out->redir_cnt.err)); 1156 1157 stats_get_redirect_err_cnt(r, p, nr_cpus, NULL); 1158 } 1159 1160 if (mask & SAMPLE_EXCEPTION_CNT) { 1161 str = out->except_cnt.hits ? "xdp_exception total" : 1162 "xdp_exception"; 1163 1164 print_err(out->except_cnt.hits, " %-20s " FMT_COLUMNl "\n", str, 1165 HITS(out->except_cnt.hits)); 1166 1167 stats_get_exception_cnt(r, p, nr_cpus, NULL); 1168 } 1169 1170 if (mask & SAMPLE_DEVMAP_XMIT_CNT) { 1171 str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ? 1172 "devmap_xmit total" : 1173 "devmap_xmit"; 1174 1175 print_err(out->xmit_cnt.err || out->xmit_cnt.drop, 1176 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl 1177 __COLUMN(".2f") "\n", 1178 str, XMIT(out->xmit_cnt.pps), 1179 DROP(out->xmit_cnt.drop), out->xmit_cnt.err, 1180 "drv_err/s", out->xmit_cnt.bavg, "bulk-avg"); 1181 1182 stats_get_devmap_xmit(r, p, nr_cpus, NULL); 1183 } 1184 1185 if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 1186 stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL, 1187 mask & SAMPLE_DEVMAP_XMIT_CNT); 1188 1189 if (sample_log_level & LL_DEFAULT || 1190 ((sample_log_level & LL_SIMPLE) && sample_err_exp)) { 1191 sample_err_exp = false; 1192 printf("\n"); 1193 } 1194 } 1195 1196 int sample_setup_maps(struct bpf_map **maps) 1197 { 1198 sample_n_cpus = libbpf_num_possible_cpus(); 1199 1200 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) { 1201 sample_map[i] = maps[i]; 1202 1203 switch (i) { 1204 case MAP_RX: 1205 case MAP_CPUMAP_KTHREAD: 1206 case MAP_DEVMAP_XMIT: 1207 sample_map_count[i] = sample_n_cpus; 1208 break; 1209 case MAP_REDIRECT_ERR: 1210 sample_map_count[i] = 1211 XDP_REDIRECT_ERR_MAX * sample_n_cpus; 1212 break; 1213 case MAP_EXCEPTION: 1214 sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus; 1215 case MAP_CPUMAP_ENQUEUE: 1216 sample_map_count[i] = sample_n_cpus * sample_n_cpus; 1217 break; 1218 default: 1219 return -EINVAL; 1220 } 1221 if (bpf_map__resize(sample_map[i], sample_map_count[i]) < 0) 1222 return -errno; 1223 } 1224 sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI]; 1225 return 0; 1226 } 1227 1228 static int sample_setup_maps_mappings(void) 1229 { 1230 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) { 1231 size_t size = sample_map_count[i] * sizeof(struct datarec); 1232 1233 sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, 1234 MAP_SHARED, bpf_map__fd(sample_map[i]), 0); 1235 if (sample_mmap[i] == MAP_FAILED) 1236 return -errno; 1237 } 1238 return 0; 1239 } 1240 1241 int __sample_init(int mask) 1242 { 1243 sigset_t st; 1244 1245 sigemptyset(&st); 1246 sigaddset(&st, SIGQUIT); 1247 sigaddset(&st, SIGINT); 1248 sigaddset(&st, SIGTERM); 1249 1250 if (sigprocmask(SIG_BLOCK, &st, NULL) < 0) 1251 return -errno; 1252 1253 sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK); 1254 if (sample_sig_fd < 0) 1255 return -errno; 1256 1257 sample_mask = mask; 1258 1259 return sample_setup_maps_mappings(); 1260 } 1261 1262 static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags) 1263 { 1264 __u32 cur_prog_id = 0; 1265 int ret; 1266 1267 if (prog_id) { 1268 ret = bpf_get_link_xdp_id(ifindex, &cur_prog_id, xdp_flags); 1269 if (ret < 0) 1270 return -errno; 1271 1272 if (prog_id != cur_prog_id) { 1273 print_always( 1274 "Program on ifindex %d does not match installed " 1275 "program, skipping unload\n", 1276 ifindex); 1277 return -ENOENT; 1278 } 1279 } 1280 1281 return bpf_set_link_xdp_fd(ifindex, -1, xdp_flags); 1282 } 1283 1284 int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic, 1285 bool force) 1286 { 1287 int ret, xdp_flags = 0; 1288 __u32 prog_id = 0; 1289 1290 if (sample_xdp_cnt == 32) { 1291 fprintf(stderr, 1292 "Total limit for installed XDP programs in a sample reached\n"); 1293 return -ENOTSUP; 1294 } 1295 1296 xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0; 1297 xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; 1298 ret = bpf_set_link_xdp_fd(ifindex, bpf_program__fd(xdp_prog), 1299 xdp_flags); 1300 if (ret < 0) { 1301 ret = -errno; 1302 fprintf(stderr, 1303 "Failed to install program \"%s\" on ifindex %d, mode = %s, " 1304 "force = %s: %s\n", 1305 bpf_program__name(xdp_prog), ifindex, 1306 generic ? "skb" : "native", force ? "true" : "false", 1307 strerror(-ret)); 1308 return ret; 1309 } 1310 1311 ret = bpf_get_link_xdp_id(ifindex, &prog_id, xdp_flags); 1312 if (ret < 0) { 1313 ret = -errno; 1314 fprintf(stderr, 1315 "Failed to get XDP program id for ifindex %d, removing program: %s\n", 1316 ifindex, strerror(errno)); 1317 __sample_remove_xdp(ifindex, 0, xdp_flags); 1318 return ret; 1319 } 1320 sample_xdp_progs[sample_xdp_cnt++] = 1321 (struct xdp_desc){ ifindex, prog_id, xdp_flags }; 1322 1323 return 0; 1324 } 1325 1326 static void sample_summary_print(void) 1327 { 1328 double num = sample_out.rx_cnt.num; 1329 1330 if (sample_out.totals.rx) { 1331 double pkts = sample_out.totals.rx; 1332 1333 print_always(" Packets received : %'-10llu\n", 1334 sample_out.totals.rx); 1335 print_always(" Average packets/s : %'-10.0f\n", 1336 sample_round(pkts / num)); 1337 } 1338 if (sample_out.totals.redir) { 1339 double pkts = sample_out.totals.redir; 1340 1341 print_always(" Packets redirected : %'-10llu\n", 1342 sample_out.totals.redir); 1343 print_always(" Average redir/s : %'-10.0f\n", 1344 sample_round(pkts / num)); 1345 } 1346 if (sample_out.totals.drop) 1347 print_always(" Rx dropped : %'-10llu\n", 1348 sample_out.totals.drop); 1349 if (sample_out.totals.drop_xmit) 1350 print_always(" Tx dropped : %'-10llu\n", 1351 sample_out.totals.drop_xmit); 1352 if (sample_out.totals.err) 1353 print_always(" Errors recorded : %'-10llu\n", 1354 sample_out.totals.err); 1355 if (sample_out.totals.xmit) { 1356 double pkts = sample_out.totals.xmit; 1357 1358 print_always(" Packets transmitted : %'-10llu\n", 1359 sample_out.totals.xmit); 1360 print_always(" Average transmit/s : %'-10.0f\n", 1361 sample_round(pkts / num)); 1362 } 1363 } 1364 1365 void sample_exit(int status) 1366 { 1367 size_t size; 1368 1369 for (int i = 0; i < NUM_MAP; i++) { 1370 size = sample_map_count[i] * sizeof(**sample_mmap); 1371 munmap(sample_mmap[i], size); 1372 } 1373 while (sample_xdp_cnt--) { 1374 int i = sample_xdp_cnt, ifindex, xdp_flags; 1375 __u32 prog_id; 1376 1377 prog_id = sample_xdp_progs[i].prog_id; 1378 ifindex = sample_xdp_progs[i].ifindex; 1379 xdp_flags = sample_xdp_progs[i].flags; 1380 1381 __sample_remove_xdp(ifindex, prog_id, xdp_flags); 1382 } 1383 sample_summary_print(); 1384 close(sample_sig_fd); 1385 exit(status); 1386 } 1387 1388 static int sample_stats_collect(struct stats_record *rec) 1389 { 1390 int i; 1391 1392 if (sample_mask & SAMPLE_RX_CNT) 1393 map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt); 1394 1395 if (sample_mask & SAMPLE_REDIRECT_CNT) 1396 map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]); 1397 1398 if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) { 1399 for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++) 1400 map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus], 1401 &rec->redir_err[i]); 1402 } 1403 1404 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) 1405 for (i = 0; i < sample_n_cpus; i++) 1406 map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus], 1407 &rec->enq[i]); 1408 1409 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) 1410 map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD], 1411 &rec->kthread); 1412 1413 if (sample_mask & SAMPLE_EXCEPTION_CNT) 1414 for (i = 0; i < XDP_ACTION_MAX; i++) 1415 map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus], 1416 &rec->exception[i]); 1417 1418 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) 1419 map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit); 1420 1421 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) { 1422 if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0) 1423 return -EINVAL; 1424 } 1425 return 0; 1426 } 1427 1428 static void sample_summary_update(struct sample_output *out) 1429 { 1430 sample_out.totals.rx += out->totals.rx; 1431 sample_out.totals.redir += out->totals.redir; 1432 sample_out.totals.drop += out->totals.drop; 1433 sample_out.totals.drop_xmit += out->totals.drop_xmit; 1434 sample_out.totals.err += out->totals.err; 1435 sample_out.totals.xmit += out->totals.xmit; 1436 sample_out.rx_cnt.num++; 1437 } 1438 1439 static void sample_stats_print(int mask, struct stats_record *cur, 1440 struct stats_record *prev, char *prog_name) 1441 { 1442 struct sample_output out = {}; 1443 1444 if (mask & SAMPLE_RX_CNT) 1445 stats_get_rx_cnt(cur, prev, 0, &out); 1446 if (mask & SAMPLE_REDIRECT_CNT) 1447 stats_get_redirect_cnt(cur, prev, 0, &out); 1448 if (mask & SAMPLE_REDIRECT_ERR_CNT) 1449 stats_get_redirect_err_cnt(cur, prev, 0, &out); 1450 if (mask & SAMPLE_EXCEPTION_CNT) 1451 stats_get_exception_cnt(cur, prev, 0, &out); 1452 if (mask & SAMPLE_DEVMAP_XMIT_CNT) 1453 stats_get_devmap_xmit(cur, prev, 0, &out); 1454 else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 1455 stats_get_devmap_xmit_multi(cur, prev, 0, &out, 1456 mask & SAMPLE_DEVMAP_XMIT_CNT); 1457 sample_summary_update(&out); 1458 1459 stats_print(prog_name, mask, cur, prev, &out); 1460 } 1461 1462 void sample_switch_mode(void) 1463 { 1464 sample_log_level ^= LL_DEBUG - 1; 1465 } 1466 1467 static int sample_signal_cb(void) 1468 { 1469 struct signalfd_siginfo si; 1470 int r; 1471 1472 r = read(sample_sig_fd, &si, sizeof(si)); 1473 if (r < 0) 1474 return -errno; 1475 1476 switch (si.ssi_signo) { 1477 case SIGQUIT: 1478 sample_switch_mode(); 1479 printf("\n"); 1480 break; 1481 default: 1482 printf("\n"); 1483 return 1; 1484 } 1485 1486 return 0; 1487 } 1488 1489 /* Pointer swap trick */ 1490 static void swap(struct stats_record **a, struct stats_record **b) 1491 { 1492 struct stats_record *tmp; 1493 1494 tmp = *a; 1495 *a = *b; 1496 *b = tmp; 1497 } 1498 1499 static int sample_timer_cb(int timerfd, struct stats_record **rec, 1500 struct stats_record **prev) 1501 { 1502 char line[64] = "Summary"; 1503 int ret; 1504 __u64 t; 1505 1506 ret = read(timerfd, &t, sizeof(t)); 1507 if (ret < 0) 1508 return -errno; 1509 1510 swap(prev, rec); 1511 ret = sample_stats_collect(*rec); 1512 if (ret < 0) 1513 return ret; 1514 1515 if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) { 1516 char fi[IFNAMSIZ]; 1517 char to[IFNAMSIZ]; 1518 const char *f, *t; 1519 1520 f = t = NULL; 1521 if (if_indextoname(sample_xdp_progs[0].ifindex, fi)) 1522 f = fi; 1523 if (if_indextoname(sample_xdp_progs[1].ifindex, to)) 1524 t = to; 1525 1526 snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?"); 1527 } 1528 1529 sample_stats_print(sample_mask, *rec, *prev, line); 1530 return 0; 1531 } 1532 1533 int sample_run(int interval, void (*post_cb)(void *), void *ctx) 1534 { 1535 struct timespec ts = { interval, 0 }; 1536 struct itimerspec its = { ts, ts }; 1537 struct stats_record *rec, *prev; 1538 struct pollfd pfd[2] = {}; 1539 int timerfd, ret; 1540 1541 if (!interval) { 1542 fprintf(stderr, "Incorrect interval 0\n"); 1543 return -EINVAL; 1544 } 1545 sample_interval = interval; 1546 /* Pretty print numbers */ 1547 setlocale(LC_NUMERIC, "en_US.UTF-8"); 1548 1549 timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK); 1550 if (timerfd < 0) 1551 return -errno; 1552 timerfd_settime(timerfd, 0, &its, NULL); 1553 1554 pfd[0].fd = sample_sig_fd; 1555 pfd[0].events = POLLIN; 1556 1557 pfd[1].fd = timerfd; 1558 pfd[1].events = POLLIN; 1559 1560 ret = -ENOMEM; 1561 rec = alloc_stats_record(); 1562 if (!rec) 1563 goto end; 1564 prev = alloc_stats_record(); 1565 if (!prev) 1566 goto end_rec; 1567 1568 ret = sample_stats_collect(rec); 1569 if (ret < 0) 1570 goto end_rec_prev; 1571 1572 for (;;) { 1573 ret = poll(pfd, 2, -1); 1574 if (ret < 0) { 1575 if (errno == EINTR) 1576 continue; 1577 else 1578 break; 1579 } 1580 1581 if (pfd[0].revents & POLLIN) 1582 ret = sample_signal_cb(); 1583 else if (pfd[1].revents & POLLIN) 1584 ret = sample_timer_cb(timerfd, &rec, &prev); 1585 1586 if (ret) 1587 break; 1588 1589 if (post_cb) 1590 post_cb(ctx); 1591 } 1592 1593 end_rec_prev: 1594 free_stats_record(prev); 1595 end_rec: 1596 free_stats_record(rec); 1597 end: 1598 close(timerfd); 1599 1600 return ret; 1601 } 1602 1603 const char *get_driver_name(int ifindex) 1604 { 1605 struct ethtool_drvinfo drv = {}; 1606 char ifname[IF_NAMESIZE]; 1607 static char drvname[32]; 1608 struct ifreq ifr = {}; 1609 int fd, r = 0; 1610 1611 fd = socket(AF_INET, SOCK_DGRAM, 0); 1612 if (fd < 0) 1613 return "[error]"; 1614 1615 if (!if_indextoname(ifindex, ifname)) 1616 goto end; 1617 1618 drv.cmd = ETHTOOL_GDRVINFO; 1619 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name)); 1620 ifr.ifr_data = (void *)&drv; 1621 1622 r = ioctl(fd, SIOCETHTOOL, &ifr); 1623 if (r) 1624 goto end; 1625 1626 safe_strncpy(drvname, drv.driver, sizeof(drvname)); 1627 1628 close(fd); 1629 return drvname; 1630 1631 end: 1632 r = errno; 1633 close(fd); 1634 return r == EOPNOTSUPP ? "loopback" : "[error]"; 1635 } 1636 1637 int get_mac_addr(int ifindex, void *mac_addr) 1638 { 1639 char ifname[IF_NAMESIZE]; 1640 struct ifreq ifr = {}; 1641 int fd, r; 1642 1643 fd = socket(AF_INET, SOCK_DGRAM, 0); 1644 if (fd < 0) 1645 return -errno; 1646 1647 if (!if_indextoname(ifindex, ifname)) { 1648 r = -errno; 1649 goto end; 1650 } 1651 1652 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name)); 1653 1654 r = ioctl(fd, SIOCGIFHWADDR, &ifr); 1655 if (r) { 1656 r = -errno; 1657 goto end; 1658 } 1659 1660 memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char)); 1661 1662 end: 1663 close(fd); 1664 return r; 1665 } 1666 1667 __attribute__((constructor)) static void sample_ctor(void) 1668 { 1669 if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) { 1670 fprintf(stderr, "Failed to set libbpf strict mode: %s\n", 1671 strerror(errno)); 1672 /* Just exit, nothing to cleanup right now */ 1673 exit(EXIT_FAIL_BPF); 1674 } 1675 } 1676