1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/etherdevice.h> 8 #include <linux/filter.h> 9 #include <linux/sched/signal.h> 10 #include <net/bpf_sk_storage.h> 11 #include <net/sock.h> 12 #include <net/tcp.h> 13 #include <linux/error-injection.h> 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/bpf_test_run.h> 17 18 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 19 u32 *retval, u32 *time, bool xdp) 20 { 21 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 22 enum bpf_cgroup_storage_type stype; 23 u64 time_start, time_spent = 0; 24 int ret = 0; 25 u32 i; 26 27 for_each_cgroup_storage_type(stype) { 28 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 29 if (IS_ERR(storage[stype])) { 30 storage[stype] = NULL; 31 for_each_cgroup_storage_type(stype) 32 bpf_cgroup_storage_free(storage[stype]); 33 return -ENOMEM; 34 } 35 } 36 37 if (!repeat) 38 repeat = 1; 39 40 rcu_read_lock(); 41 migrate_disable(); 42 time_start = ktime_get_ns(); 43 for (i = 0; i < repeat; i++) { 44 bpf_cgroup_storage_set(storage); 45 46 if (xdp) 47 *retval = bpf_prog_run_xdp(prog, ctx); 48 else 49 *retval = BPF_PROG_RUN(prog, ctx); 50 51 if (signal_pending(current)) { 52 ret = -EINTR; 53 break; 54 } 55 56 if (need_resched()) { 57 time_spent += ktime_get_ns() - time_start; 58 migrate_enable(); 59 rcu_read_unlock(); 60 61 cond_resched(); 62 63 rcu_read_lock(); 64 migrate_disable(); 65 time_start = ktime_get_ns(); 66 } 67 } 68 time_spent += ktime_get_ns() - time_start; 69 migrate_enable(); 70 rcu_read_unlock(); 71 72 do_div(time_spent, repeat); 73 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 74 75 for_each_cgroup_storage_type(stype) 76 bpf_cgroup_storage_free(storage[stype]); 77 78 return ret; 79 } 80 81 static int bpf_test_finish(const union bpf_attr *kattr, 82 union bpf_attr __user *uattr, const void *data, 83 u32 size, u32 retval, u32 duration) 84 { 85 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 86 int err = -EFAULT; 87 u32 copy_size = size; 88 89 /* Clamp copy if the user has provided a size hint, but copy the full 90 * buffer if not to retain old behaviour. 91 */ 92 if (kattr->test.data_size_out && 93 copy_size > kattr->test.data_size_out) { 94 copy_size = kattr->test.data_size_out; 95 err = -ENOSPC; 96 } 97 98 if (data_out && copy_to_user(data_out, data, copy_size)) 99 goto out; 100 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 101 goto out; 102 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 103 goto out; 104 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 105 goto out; 106 if (err != -ENOSPC) 107 err = 0; 108 out: 109 trace_bpf_test_finish(&err); 110 return err; 111 } 112 113 /* Integer types of various sizes and pointer combinations cover variety of 114 * architecture dependent calling conventions. 7+ can be supported in the 115 * future. 116 */ 117 __diag_push(); 118 __diag_ignore(GCC, 8, "-Wmissing-prototypes", 119 "Global functions as their definitions will be in vmlinux BTF"); 120 int noinline bpf_fentry_test1(int a) 121 { 122 return a + 1; 123 } 124 125 int noinline bpf_fentry_test2(int a, u64 b) 126 { 127 return a + b; 128 } 129 130 int noinline bpf_fentry_test3(char a, int b, u64 c) 131 { 132 return a + b + c; 133 } 134 135 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 136 { 137 return (long)a + b + c + d; 138 } 139 140 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 141 { 142 return a + (long)b + c + d + e; 143 } 144 145 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 146 { 147 return a + (long)b + c + d + (long)e + f; 148 } 149 150 struct bpf_fentry_test_t { 151 struct bpf_fentry_test_t *a; 152 }; 153 154 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 155 { 156 return (long)arg; 157 } 158 159 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 160 { 161 return (long)arg->a; 162 } 163 164 int noinline bpf_modify_return_test(int a, int *b) 165 { 166 *b += 1; 167 return a + *b; 168 } 169 __diag_pop(); 170 171 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 172 173 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 174 u32 headroom, u32 tailroom) 175 { 176 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 177 u32 user_size = kattr->test.data_size_in; 178 void *data; 179 180 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 181 return ERR_PTR(-EINVAL); 182 183 if (user_size > size) 184 return ERR_PTR(-EMSGSIZE); 185 186 data = kzalloc(size + headroom + tailroom, GFP_USER); 187 if (!data) 188 return ERR_PTR(-ENOMEM); 189 190 if (copy_from_user(data + headroom, data_in, user_size)) { 191 kfree(data); 192 return ERR_PTR(-EFAULT); 193 } 194 195 return data; 196 } 197 198 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 199 const union bpf_attr *kattr, 200 union bpf_attr __user *uattr) 201 { 202 struct bpf_fentry_test_t arg = {}; 203 u16 side_effect = 0, ret = 0; 204 int b = 2, err = -EFAULT; 205 u32 retval = 0; 206 207 switch (prog->expected_attach_type) { 208 case BPF_TRACE_FENTRY: 209 case BPF_TRACE_FEXIT: 210 if (bpf_fentry_test1(1) != 2 || 211 bpf_fentry_test2(2, 3) != 5 || 212 bpf_fentry_test3(4, 5, 6) != 15 || 213 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 214 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 215 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 216 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 217 bpf_fentry_test8(&arg) != 0) 218 goto out; 219 break; 220 case BPF_MODIFY_RETURN: 221 ret = bpf_modify_return_test(1, &b); 222 if (b != 2) 223 side_effect = 1; 224 break; 225 default: 226 goto out; 227 } 228 229 retval = ((u32)side_effect << 16) | ret; 230 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 231 goto out; 232 233 err = 0; 234 out: 235 trace_bpf_test_finish(&err); 236 return err; 237 } 238 239 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 240 { 241 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 242 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 243 u32 size = kattr->test.ctx_size_in; 244 void *data; 245 int err; 246 247 if (!data_in && !data_out) 248 return NULL; 249 250 data = kzalloc(max_size, GFP_USER); 251 if (!data) 252 return ERR_PTR(-ENOMEM); 253 254 if (data_in) { 255 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 256 if (err) { 257 kfree(data); 258 return ERR_PTR(err); 259 } 260 261 size = min_t(u32, max_size, size); 262 if (copy_from_user(data, data_in, size)) { 263 kfree(data); 264 return ERR_PTR(-EFAULT); 265 } 266 } 267 return data; 268 } 269 270 static int bpf_ctx_finish(const union bpf_attr *kattr, 271 union bpf_attr __user *uattr, const void *data, 272 u32 size) 273 { 274 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 275 int err = -EFAULT; 276 u32 copy_size = size; 277 278 if (!data || !data_out) 279 return 0; 280 281 if (copy_size > kattr->test.ctx_size_out) { 282 copy_size = kattr->test.ctx_size_out; 283 err = -ENOSPC; 284 } 285 286 if (copy_to_user(data_out, data, copy_size)) 287 goto out; 288 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 289 goto out; 290 if (err != -ENOSPC) 291 err = 0; 292 out: 293 return err; 294 } 295 296 /** 297 * range_is_zero - test whether buffer is initialized 298 * @buf: buffer to check 299 * @from: check from this position 300 * @to: check up until (excluding) this position 301 * 302 * This function returns true if the there is a non-zero byte 303 * in the buf in the range [from,to). 304 */ 305 static inline bool range_is_zero(void *buf, size_t from, size_t to) 306 { 307 return !memchr_inv((u8 *)buf + from, 0, to - from); 308 } 309 310 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 311 { 312 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 313 314 if (!__skb) 315 return 0; 316 317 /* make sure the fields we don't use are zeroed */ 318 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 319 return -EINVAL; 320 321 /* mark is allowed */ 322 323 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 324 offsetof(struct __sk_buff, priority))) 325 return -EINVAL; 326 327 /* priority is allowed */ 328 329 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority), 330 offsetof(struct __sk_buff, cb))) 331 return -EINVAL; 332 333 /* cb is allowed */ 334 335 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 336 offsetof(struct __sk_buff, tstamp))) 337 return -EINVAL; 338 339 /* tstamp is allowed */ 340 /* wire_len is allowed */ 341 /* gso_segs is allowed */ 342 343 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 344 offsetof(struct __sk_buff, gso_size))) 345 return -EINVAL; 346 347 /* gso_size is allowed */ 348 349 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 350 sizeof(struct __sk_buff))) 351 return -EINVAL; 352 353 skb->mark = __skb->mark; 354 skb->priority = __skb->priority; 355 skb->tstamp = __skb->tstamp; 356 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 357 358 if (__skb->wire_len == 0) { 359 cb->pkt_len = skb->len; 360 } else { 361 if (__skb->wire_len < skb->len || 362 __skb->wire_len > GSO_MAX_SIZE) 363 return -EINVAL; 364 cb->pkt_len = __skb->wire_len; 365 } 366 367 if (__skb->gso_segs > GSO_MAX_SEGS) 368 return -EINVAL; 369 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 370 skb_shinfo(skb)->gso_size = __skb->gso_size; 371 372 return 0; 373 } 374 375 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 376 { 377 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 378 379 if (!__skb) 380 return; 381 382 __skb->mark = skb->mark; 383 __skb->priority = skb->priority; 384 __skb->tstamp = skb->tstamp; 385 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 386 __skb->wire_len = cb->pkt_len; 387 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 388 } 389 390 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 391 union bpf_attr __user *uattr) 392 { 393 bool is_l2 = false, is_direct_pkt_access = false; 394 u32 size = kattr->test.data_size_in; 395 u32 repeat = kattr->test.repeat; 396 struct __sk_buff *ctx = NULL; 397 u32 retval, duration; 398 int hh_len = ETH_HLEN; 399 struct sk_buff *skb; 400 struct sock *sk; 401 void *data; 402 int ret; 403 404 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 405 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 406 if (IS_ERR(data)) 407 return PTR_ERR(data); 408 409 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 410 if (IS_ERR(ctx)) { 411 kfree(data); 412 return PTR_ERR(ctx); 413 } 414 415 switch (prog->type) { 416 case BPF_PROG_TYPE_SCHED_CLS: 417 case BPF_PROG_TYPE_SCHED_ACT: 418 is_l2 = true; 419 /* fall through */ 420 case BPF_PROG_TYPE_LWT_IN: 421 case BPF_PROG_TYPE_LWT_OUT: 422 case BPF_PROG_TYPE_LWT_XMIT: 423 is_direct_pkt_access = true; 424 break; 425 default: 426 break; 427 } 428 429 sk = kzalloc(sizeof(struct sock), GFP_USER); 430 if (!sk) { 431 kfree(data); 432 kfree(ctx); 433 return -ENOMEM; 434 } 435 sock_net_set(sk, current->nsproxy->net_ns); 436 sock_init_data(NULL, sk); 437 438 skb = build_skb(data, 0); 439 if (!skb) { 440 kfree(data); 441 kfree(ctx); 442 kfree(sk); 443 return -ENOMEM; 444 } 445 skb->sk = sk; 446 447 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 448 __skb_put(skb, size); 449 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 450 skb_reset_network_header(skb); 451 452 if (is_l2) 453 __skb_push(skb, hh_len); 454 if (is_direct_pkt_access) 455 bpf_compute_data_pointers(skb); 456 ret = convert___skb_to_skb(skb, ctx); 457 if (ret) 458 goto out; 459 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 460 if (ret) 461 goto out; 462 if (!is_l2) { 463 if (skb_headroom(skb) < hh_len) { 464 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 465 466 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 467 ret = -ENOMEM; 468 goto out; 469 } 470 } 471 memset(__skb_push(skb, hh_len), 0, hh_len); 472 } 473 convert_skb_to___skb(skb, ctx); 474 475 size = skb->len; 476 /* bpf program can never convert linear skb to non-linear */ 477 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 478 size = skb_headlen(skb); 479 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 480 if (!ret) 481 ret = bpf_ctx_finish(kattr, uattr, ctx, 482 sizeof(struct __sk_buff)); 483 out: 484 kfree_skb(skb); 485 bpf_sk_storage_free(sk); 486 kfree(sk); 487 kfree(ctx); 488 return ret; 489 } 490 491 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 492 union bpf_attr __user *uattr) 493 { 494 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 495 u32 headroom = XDP_PACKET_HEADROOM; 496 u32 size = kattr->test.data_size_in; 497 u32 repeat = kattr->test.repeat; 498 struct netdev_rx_queue *rxqueue; 499 struct xdp_buff xdp = {}; 500 u32 retval, duration; 501 u32 max_data_sz; 502 void *data; 503 int ret; 504 505 if (kattr->test.ctx_in || kattr->test.ctx_out) 506 return -EINVAL; 507 508 /* XDP have extra tailroom as (most) drivers use full page */ 509 max_data_sz = 4096 - headroom - tailroom; 510 511 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); 512 if (IS_ERR(data)) 513 return PTR_ERR(data); 514 515 xdp.data_hard_start = data; 516 xdp.data = data + headroom; 517 xdp.data_meta = xdp.data; 518 xdp.data_end = xdp.data + size; 519 xdp.frame_sz = headroom + max_data_sz + tailroom; 520 521 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 522 xdp.rxq = &rxqueue->xdp_rxq; 523 bpf_prog_change_xdp(NULL, prog); 524 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 525 if (ret) 526 goto out; 527 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) 528 size = xdp.data_end - xdp.data; 529 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 530 out: 531 bpf_prog_change_xdp(prog, NULL); 532 kfree(data); 533 return ret; 534 } 535 536 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 537 { 538 /* make sure the fields we don't use are zeroed */ 539 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 540 return -EINVAL; 541 542 /* flags is allowed */ 543 544 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 545 sizeof(struct bpf_flow_keys))) 546 return -EINVAL; 547 548 return 0; 549 } 550 551 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 552 const union bpf_attr *kattr, 553 union bpf_attr __user *uattr) 554 { 555 u32 size = kattr->test.data_size_in; 556 struct bpf_flow_dissector ctx = {}; 557 u32 repeat = kattr->test.repeat; 558 struct bpf_flow_keys *user_ctx; 559 struct bpf_flow_keys flow_keys; 560 u64 time_start, time_spent = 0; 561 const struct ethhdr *eth; 562 unsigned int flags = 0; 563 u32 retval, duration; 564 void *data; 565 int ret; 566 u32 i; 567 568 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 569 return -EINVAL; 570 571 if (size < ETH_HLEN) 572 return -EINVAL; 573 574 data = bpf_test_init(kattr, size, 0, 0); 575 if (IS_ERR(data)) 576 return PTR_ERR(data); 577 578 eth = (struct ethhdr *)data; 579 580 if (!repeat) 581 repeat = 1; 582 583 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 584 if (IS_ERR(user_ctx)) { 585 kfree(data); 586 return PTR_ERR(user_ctx); 587 } 588 if (user_ctx) { 589 ret = verify_user_bpf_flow_keys(user_ctx); 590 if (ret) 591 goto out; 592 flags = user_ctx->flags; 593 } 594 595 ctx.flow_keys = &flow_keys; 596 ctx.data = data; 597 ctx.data_end = (__u8 *)data + size; 598 599 rcu_read_lock(); 600 preempt_disable(); 601 time_start = ktime_get_ns(); 602 for (i = 0; i < repeat; i++) { 603 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 604 size, flags); 605 606 if (signal_pending(current)) { 607 preempt_enable(); 608 rcu_read_unlock(); 609 610 ret = -EINTR; 611 goto out; 612 } 613 614 if (need_resched()) { 615 time_spent += ktime_get_ns() - time_start; 616 preempt_enable(); 617 rcu_read_unlock(); 618 619 cond_resched(); 620 621 rcu_read_lock(); 622 preempt_disable(); 623 time_start = ktime_get_ns(); 624 } 625 } 626 time_spent += ktime_get_ns() - time_start; 627 preempt_enable(); 628 rcu_read_unlock(); 629 630 do_div(time_spent, repeat); 631 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 632 633 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 634 retval, duration); 635 if (!ret) 636 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 637 sizeof(struct bpf_flow_keys)); 638 639 out: 640 kfree(user_ctx); 641 kfree(data); 642 return ret; 643 } 644