1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/etherdevice.h> 8 #include <linux/filter.h> 9 #include <linux/sched/signal.h> 10 #include <net/bpf_sk_storage.h> 11 #include <net/sock.h> 12 #include <net/tcp.h> 13 #include <linux/error-injection.h> 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/bpf_test_run.h> 17 18 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 19 u32 *retval, u32 *time, bool xdp) 20 { 21 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 22 enum bpf_cgroup_storage_type stype; 23 u64 time_start, time_spent = 0; 24 int ret = 0; 25 u32 i; 26 27 for_each_cgroup_storage_type(stype) { 28 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 29 if (IS_ERR(storage[stype])) { 30 storage[stype] = NULL; 31 for_each_cgroup_storage_type(stype) 32 bpf_cgroup_storage_free(storage[stype]); 33 return -ENOMEM; 34 } 35 } 36 37 if (!repeat) 38 repeat = 1; 39 40 rcu_read_lock(); 41 migrate_disable(); 42 time_start = ktime_get_ns(); 43 for (i = 0; i < repeat; i++) { 44 bpf_cgroup_storage_set(storage); 45 46 if (xdp) 47 *retval = bpf_prog_run_xdp(prog, ctx); 48 else 49 *retval = BPF_PROG_RUN(prog, ctx); 50 51 if (signal_pending(current)) { 52 ret = -EINTR; 53 break; 54 } 55 56 if (need_resched()) { 57 time_spent += ktime_get_ns() - time_start; 58 migrate_enable(); 59 rcu_read_unlock(); 60 61 cond_resched(); 62 63 rcu_read_lock(); 64 migrate_disable(); 65 time_start = ktime_get_ns(); 66 } 67 } 68 time_spent += ktime_get_ns() - time_start; 69 migrate_enable(); 70 rcu_read_unlock(); 71 72 do_div(time_spent, repeat); 73 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 74 75 for_each_cgroup_storage_type(stype) 76 bpf_cgroup_storage_free(storage[stype]); 77 78 return ret; 79 } 80 81 static int bpf_test_finish(const union bpf_attr *kattr, 82 union bpf_attr __user *uattr, const void *data, 83 u32 size, u32 retval, u32 duration) 84 { 85 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 86 int err = -EFAULT; 87 u32 copy_size = size; 88 89 /* Clamp copy if the user has provided a size hint, but copy the full 90 * buffer if not to retain old behaviour. 91 */ 92 if (kattr->test.data_size_out && 93 copy_size > kattr->test.data_size_out) { 94 copy_size = kattr->test.data_size_out; 95 err = -ENOSPC; 96 } 97 98 if (data_out && copy_to_user(data_out, data, copy_size)) 99 goto out; 100 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 101 goto out; 102 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 103 goto out; 104 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 105 goto out; 106 if (err != -ENOSPC) 107 err = 0; 108 out: 109 trace_bpf_test_finish(&err); 110 return err; 111 } 112 113 /* Integer types of various sizes and pointer combinations cover variety of 114 * architecture dependent calling conventions. 7+ can be supported in the 115 * future. 116 */ 117 __diag_push(); 118 __diag_ignore(GCC, 8, "-Wmissing-prototypes", 119 "Global functions as their definitions will be in vmlinux BTF"); 120 int noinline bpf_fentry_test1(int a) 121 { 122 return a + 1; 123 } 124 125 int noinline bpf_fentry_test2(int a, u64 b) 126 { 127 return a + b; 128 } 129 130 int noinline bpf_fentry_test3(char a, int b, u64 c) 131 { 132 return a + b + c; 133 } 134 135 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 136 { 137 return (long)a + b + c + d; 138 } 139 140 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 141 { 142 return a + (long)b + c + d + e; 143 } 144 145 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 146 { 147 return a + (long)b + c + d + (long)e + f; 148 } 149 150 struct bpf_fentry_test_t { 151 struct bpf_fentry_test_t *a; 152 }; 153 154 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 155 { 156 return (long)arg; 157 } 158 159 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 160 { 161 return (long)arg->a; 162 } 163 164 int noinline bpf_modify_return_test(int a, int *b) 165 { 166 *b += 1; 167 return a + *b; 168 } 169 __diag_pop(); 170 171 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 172 173 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 174 u32 headroom, u32 tailroom) 175 { 176 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 177 u32 user_size = kattr->test.data_size_in; 178 void *data; 179 180 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 181 return ERR_PTR(-EINVAL); 182 183 if (user_size > size) 184 return ERR_PTR(-EMSGSIZE); 185 186 data = kzalloc(size + headroom + tailroom, GFP_USER); 187 if (!data) 188 return ERR_PTR(-ENOMEM); 189 190 if (copy_from_user(data + headroom, data_in, user_size)) { 191 kfree(data); 192 return ERR_PTR(-EFAULT); 193 } 194 195 return data; 196 } 197 198 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 199 const union bpf_attr *kattr, 200 union bpf_attr __user *uattr) 201 { 202 struct bpf_fentry_test_t arg = {}; 203 u16 side_effect = 0, ret = 0; 204 int b = 2, err = -EFAULT; 205 u32 retval = 0; 206 207 switch (prog->expected_attach_type) { 208 case BPF_TRACE_FENTRY: 209 case BPF_TRACE_FEXIT: 210 if (bpf_fentry_test1(1) != 2 || 211 bpf_fentry_test2(2, 3) != 5 || 212 bpf_fentry_test3(4, 5, 6) != 15 || 213 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 214 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 215 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 216 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 217 bpf_fentry_test8(&arg) != 0) 218 goto out; 219 break; 220 case BPF_MODIFY_RETURN: 221 ret = bpf_modify_return_test(1, &b); 222 if (b != 2) 223 side_effect = 1; 224 break; 225 default: 226 goto out; 227 } 228 229 retval = ((u32)side_effect << 16) | ret; 230 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 231 goto out; 232 233 err = 0; 234 out: 235 trace_bpf_test_finish(&err); 236 return err; 237 } 238 239 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 240 { 241 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 242 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 243 u32 size = kattr->test.ctx_size_in; 244 void *data; 245 int err; 246 247 if (!data_in && !data_out) 248 return NULL; 249 250 data = kzalloc(max_size, GFP_USER); 251 if (!data) 252 return ERR_PTR(-ENOMEM); 253 254 if (data_in) { 255 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 256 if (err) { 257 kfree(data); 258 return ERR_PTR(err); 259 } 260 261 size = min_t(u32, max_size, size); 262 if (copy_from_user(data, data_in, size)) { 263 kfree(data); 264 return ERR_PTR(-EFAULT); 265 } 266 } 267 return data; 268 } 269 270 static int bpf_ctx_finish(const union bpf_attr *kattr, 271 union bpf_attr __user *uattr, const void *data, 272 u32 size) 273 { 274 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 275 int err = -EFAULT; 276 u32 copy_size = size; 277 278 if (!data || !data_out) 279 return 0; 280 281 if (copy_size > kattr->test.ctx_size_out) { 282 copy_size = kattr->test.ctx_size_out; 283 err = -ENOSPC; 284 } 285 286 if (copy_to_user(data_out, data, copy_size)) 287 goto out; 288 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 289 goto out; 290 if (err != -ENOSPC) 291 err = 0; 292 out: 293 return err; 294 } 295 296 /** 297 * range_is_zero - test whether buffer is initialized 298 * @buf: buffer to check 299 * @from: check from this position 300 * @to: check up until (excluding) this position 301 * 302 * This function returns true if the there is a non-zero byte 303 * in the buf in the range [from,to). 304 */ 305 static inline bool range_is_zero(void *buf, size_t from, size_t to) 306 { 307 return !memchr_inv((u8 *)buf + from, 0, to - from); 308 } 309 310 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 311 { 312 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 313 314 if (!__skb) 315 return 0; 316 317 /* make sure the fields we don't use are zeroed */ 318 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 319 return -EINVAL; 320 321 /* mark is allowed */ 322 323 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 324 offsetof(struct __sk_buff, priority))) 325 return -EINVAL; 326 327 /* priority is allowed */ 328 329 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority), 330 offsetof(struct __sk_buff, ifindex))) 331 return -EINVAL; 332 333 /* ifindex is allowed */ 334 335 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 336 offsetof(struct __sk_buff, cb))) 337 return -EINVAL; 338 339 /* cb is allowed */ 340 341 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 342 offsetof(struct __sk_buff, tstamp))) 343 return -EINVAL; 344 345 /* tstamp is allowed */ 346 /* wire_len is allowed */ 347 /* gso_segs is allowed */ 348 349 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 350 offsetof(struct __sk_buff, gso_size))) 351 return -EINVAL; 352 353 /* gso_size is allowed */ 354 355 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 356 sizeof(struct __sk_buff))) 357 return -EINVAL; 358 359 skb->mark = __skb->mark; 360 skb->priority = __skb->priority; 361 skb->tstamp = __skb->tstamp; 362 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 363 364 if (__skb->wire_len == 0) { 365 cb->pkt_len = skb->len; 366 } else { 367 if (__skb->wire_len < skb->len || 368 __skb->wire_len > GSO_MAX_SIZE) 369 return -EINVAL; 370 cb->pkt_len = __skb->wire_len; 371 } 372 373 if (__skb->gso_segs > GSO_MAX_SEGS) 374 return -EINVAL; 375 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 376 skb_shinfo(skb)->gso_size = __skb->gso_size; 377 378 return 0; 379 } 380 381 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 382 { 383 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 384 385 if (!__skb) 386 return; 387 388 __skb->mark = skb->mark; 389 __skb->priority = skb->priority; 390 __skb->ifindex = skb->dev->ifindex; 391 __skb->tstamp = skb->tstamp; 392 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 393 __skb->wire_len = cb->pkt_len; 394 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 395 } 396 397 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 398 union bpf_attr __user *uattr) 399 { 400 bool is_l2 = false, is_direct_pkt_access = false; 401 struct net *net = current->nsproxy->net_ns; 402 struct net_device *dev = net->loopback_dev; 403 u32 size = kattr->test.data_size_in; 404 u32 repeat = kattr->test.repeat; 405 struct __sk_buff *ctx = NULL; 406 u32 retval, duration; 407 int hh_len = ETH_HLEN; 408 struct sk_buff *skb; 409 struct sock *sk; 410 void *data; 411 int ret; 412 413 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 414 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 415 if (IS_ERR(data)) 416 return PTR_ERR(data); 417 418 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 419 if (IS_ERR(ctx)) { 420 kfree(data); 421 return PTR_ERR(ctx); 422 } 423 424 switch (prog->type) { 425 case BPF_PROG_TYPE_SCHED_CLS: 426 case BPF_PROG_TYPE_SCHED_ACT: 427 is_l2 = true; 428 /* fall through */ 429 case BPF_PROG_TYPE_LWT_IN: 430 case BPF_PROG_TYPE_LWT_OUT: 431 case BPF_PROG_TYPE_LWT_XMIT: 432 is_direct_pkt_access = true; 433 break; 434 default: 435 break; 436 } 437 438 sk = kzalloc(sizeof(struct sock), GFP_USER); 439 if (!sk) { 440 kfree(data); 441 kfree(ctx); 442 return -ENOMEM; 443 } 444 sock_net_set(sk, net); 445 sock_init_data(NULL, sk); 446 447 skb = build_skb(data, 0); 448 if (!skb) { 449 kfree(data); 450 kfree(ctx); 451 kfree(sk); 452 return -ENOMEM; 453 } 454 skb->sk = sk; 455 456 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 457 __skb_put(skb, size); 458 if (ctx && ctx->ifindex > 1) { 459 dev = dev_get_by_index(net, ctx->ifindex); 460 if (!dev) { 461 ret = -ENODEV; 462 goto out; 463 } 464 } 465 skb->protocol = eth_type_trans(skb, dev); 466 skb_reset_network_header(skb); 467 468 switch (skb->protocol) { 469 case htons(ETH_P_IP): 470 sk->sk_family = AF_INET; 471 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 472 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 473 sk->sk_daddr = ip_hdr(skb)->daddr; 474 } 475 break; 476 #if IS_ENABLED(CONFIG_IPV6) 477 case htons(ETH_P_IPV6): 478 sk->sk_family = AF_INET6; 479 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 480 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 481 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 482 } 483 break; 484 #endif 485 default: 486 break; 487 } 488 489 if (is_l2) 490 __skb_push(skb, hh_len); 491 if (is_direct_pkt_access) 492 bpf_compute_data_pointers(skb); 493 ret = convert___skb_to_skb(skb, ctx); 494 if (ret) 495 goto out; 496 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 497 if (ret) 498 goto out; 499 if (!is_l2) { 500 if (skb_headroom(skb) < hh_len) { 501 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 502 503 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 504 ret = -ENOMEM; 505 goto out; 506 } 507 } 508 memset(__skb_push(skb, hh_len), 0, hh_len); 509 } 510 convert_skb_to___skb(skb, ctx); 511 512 size = skb->len; 513 /* bpf program can never convert linear skb to non-linear */ 514 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 515 size = skb_headlen(skb); 516 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 517 if (!ret) 518 ret = bpf_ctx_finish(kattr, uattr, ctx, 519 sizeof(struct __sk_buff)); 520 out: 521 if (dev && dev != net->loopback_dev) 522 dev_put(dev); 523 kfree_skb(skb); 524 bpf_sk_storage_free(sk); 525 kfree(sk); 526 kfree(ctx); 527 return ret; 528 } 529 530 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 531 union bpf_attr __user *uattr) 532 { 533 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 534 u32 headroom = XDP_PACKET_HEADROOM; 535 u32 size = kattr->test.data_size_in; 536 u32 repeat = kattr->test.repeat; 537 struct netdev_rx_queue *rxqueue; 538 struct xdp_buff xdp = {}; 539 u32 retval, duration; 540 u32 max_data_sz; 541 void *data; 542 int ret; 543 544 if (kattr->test.ctx_in || kattr->test.ctx_out) 545 return -EINVAL; 546 547 /* XDP have extra tailroom as (most) drivers use full page */ 548 max_data_sz = 4096 - headroom - tailroom; 549 550 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); 551 if (IS_ERR(data)) 552 return PTR_ERR(data); 553 554 xdp.data_hard_start = data; 555 xdp.data = data + headroom; 556 xdp.data_meta = xdp.data; 557 xdp.data_end = xdp.data + size; 558 xdp.frame_sz = headroom + max_data_sz + tailroom; 559 560 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 561 xdp.rxq = &rxqueue->xdp_rxq; 562 bpf_prog_change_xdp(NULL, prog); 563 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 564 if (ret) 565 goto out; 566 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) 567 size = xdp.data_end - xdp.data; 568 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 569 out: 570 bpf_prog_change_xdp(prog, NULL); 571 kfree(data); 572 return ret; 573 } 574 575 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 576 { 577 /* make sure the fields we don't use are zeroed */ 578 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 579 return -EINVAL; 580 581 /* flags is allowed */ 582 583 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 584 sizeof(struct bpf_flow_keys))) 585 return -EINVAL; 586 587 return 0; 588 } 589 590 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 591 const union bpf_attr *kattr, 592 union bpf_attr __user *uattr) 593 { 594 u32 size = kattr->test.data_size_in; 595 struct bpf_flow_dissector ctx = {}; 596 u32 repeat = kattr->test.repeat; 597 struct bpf_flow_keys *user_ctx; 598 struct bpf_flow_keys flow_keys; 599 u64 time_start, time_spent = 0; 600 const struct ethhdr *eth; 601 unsigned int flags = 0; 602 u32 retval, duration; 603 void *data; 604 int ret; 605 u32 i; 606 607 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 608 return -EINVAL; 609 610 if (size < ETH_HLEN) 611 return -EINVAL; 612 613 data = bpf_test_init(kattr, size, 0, 0); 614 if (IS_ERR(data)) 615 return PTR_ERR(data); 616 617 eth = (struct ethhdr *)data; 618 619 if (!repeat) 620 repeat = 1; 621 622 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 623 if (IS_ERR(user_ctx)) { 624 kfree(data); 625 return PTR_ERR(user_ctx); 626 } 627 if (user_ctx) { 628 ret = verify_user_bpf_flow_keys(user_ctx); 629 if (ret) 630 goto out; 631 flags = user_ctx->flags; 632 } 633 634 ctx.flow_keys = &flow_keys; 635 ctx.data = data; 636 ctx.data_end = (__u8 *)data + size; 637 638 rcu_read_lock(); 639 preempt_disable(); 640 time_start = ktime_get_ns(); 641 for (i = 0; i < repeat; i++) { 642 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 643 size, flags); 644 645 if (signal_pending(current)) { 646 preempt_enable(); 647 rcu_read_unlock(); 648 649 ret = -EINTR; 650 goto out; 651 } 652 653 if (need_resched()) { 654 time_spent += ktime_get_ns() - time_start; 655 preempt_enable(); 656 rcu_read_unlock(); 657 658 cond_resched(); 659 660 rcu_read_lock(); 661 preempt_disable(); 662 time_start = ktime_get_ns(); 663 } 664 } 665 time_spent += ktime_get_ns() - time_start; 666 preempt_enable(); 667 rcu_read_unlock(); 668 669 do_div(time_spent, repeat); 670 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 671 672 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 673 retval, duration); 674 if (!ret) 675 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 676 sizeof(struct bpf_flow_keys)); 677 678 out: 679 kfree(user_ctx); 680 kfree(data); 681 return ret; 682 } 683