1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/etherdevice.h> 8 #include <linux/filter.h> 9 #include <linux/sched/signal.h> 10 #include <net/bpf_sk_storage.h> 11 #include <net/sock.h> 12 #include <net/tcp.h> 13 #include <linux/error-injection.h> 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/bpf_test_run.h> 17 18 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 19 u32 *retval, u32 *time, bool xdp) 20 { 21 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 22 enum bpf_cgroup_storage_type stype; 23 u64 time_start, time_spent = 0; 24 int ret = 0; 25 u32 i; 26 27 for_each_cgroup_storage_type(stype) { 28 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 29 if (IS_ERR(storage[stype])) { 30 storage[stype] = NULL; 31 for_each_cgroup_storage_type(stype) 32 bpf_cgroup_storage_free(storage[stype]); 33 return -ENOMEM; 34 } 35 } 36 37 if (!repeat) 38 repeat = 1; 39 40 rcu_read_lock(); 41 migrate_disable(); 42 time_start = ktime_get_ns(); 43 for (i = 0; i < repeat; i++) { 44 bpf_cgroup_storage_set(storage); 45 46 if (xdp) 47 *retval = bpf_prog_run_xdp(prog, ctx); 48 else 49 *retval = BPF_PROG_RUN(prog, ctx); 50 51 if (signal_pending(current)) { 52 ret = -EINTR; 53 break; 54 } 55 56 if (need_resched()) { 57 time_spent += ktime_get_ns() - time_start; 58 migrate_enable(); 59 rcu_read_unlock(); 60 61 cond_resched(); 62 63 rcu_read_lock(); 64 migrate_disable(); 65 time_start = ktime_get_ns(); 66 } 67 } 68 time_spent += ktime_get_ns() - time_start; 69 migrate_enable(); 70 rcu_read_unlock(); 71 72 do_div(time_spent, repeat); 73 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 74 75 for_each_cgroup_storage_type(stype) 76 bpf_cgroup_storage_free(storage[stype]); 77 78 return ret; 79 } 80 81 static int bpf_test_finish(const union bpf_attr *kattr, 82 union bpf_attr __user *uattr, const void *data, 83 u32 size, u32 retval, u32 duration) 84 { 85 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 86 int err = -EFAULT; 87 u32 copy_size = size; 88 89 /* Clamp copy if the user has provided a size hint, but copy the full 90 * buffer if not to retain old behaviour. 91 */ 92 if (kattr->test.data_size_out && 93 copy_size > kattr->test.data_size_out) { 94 copy_size = kattr->test.data_size_out; 95 err = -ENOSPC; 96 } 97 98 if (data_out && copy_to_user(data_out, data, copy_size)) 99 goto out; 100 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 101 goto out; 102 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 103 goto out; 104 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 105 goto out; 106 if (err != -ENOSPC) 107 err = 0; 108 out: 109 trace_bpf_test_finish(&err); 110 return err; 111 } 112 113 /* Integer types of various sizes and pointer combinations cover variety of 114 * architecture dependent calling conventions. 7+ can be supported in the 115 * future. 116 */ 117 __diag_push(); 118 __diag_ignore(GCC, 8, "-Wmissing-prototypes", 119 "Global functions as their definitions will be in vmlinux BTF"); 120 int noinline bpf_fentry_test1(int a) 121 { 122 return a + 1; 123 } 124 125 int noinline bpf_fentry_test2(int a, u64 b) 126 { 127 return a + b; 128 } 129 130 int noinline bpf_fentry_test3(char a, int b, u64 c) 131 { 132 return a + b + c; 133 } 134 135 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 136 { 137 return (long)a + b + c + d; 138 } 139 140 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 141 { 142 return a + (long)b + c + d + e; 143 } 144 145 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 146 { 147 return a + (long)b + c + d + (long)e + f; 148 } 149 150 int noinline bpf_modify_return_test(int a, int *b) 151 { 152 *b += 1; 153 return a + *b; 154 } 155 __diag_pop(); 156 157 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 158 159 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 160 u32 headroom, u32 tailroom) 161 { 162 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 163 u32 user_size = kattr->test.data_size_in; 164 void *data; 165 166 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 167 return ERR_PTR(-EINVAL); 168 169 if (user_size > size) 170 return ERR_PTR(-EMSGSIZE); 171 172 data = kzalloc(size + headroom + tailroom, GFP_USER); 173 if (!data) 174 return ERR_PTR(-ENOMEM); 175 176 if (copy_from_user(data + headroom, data_in, user_size)) { 177 kfree(data); 178 return ERR_PTR(-EFAULT); 179 } 180 181 return data; 182 } 183 184 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 185 const union bpf_attr *kattr, 186 union bpf_attr __user *uattr) 187 { 188 u16 side_effect = 0, ret = 0; 189 int b = 2, err = -EFAULT; 190 u32 retval = 0; 191 192 switch (prog->expected_attach_type) { 193 case BPF_TRACE_FENTRY: 194 case BPF_TRACE_FEXIT: 195 if (bpf_fentry_test1(1) != 2 || 196 bpf_fentry_test2(2, 3) != 5 || 197 bpf_fentry_test3(4, 5, 6) != 15 || 198 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 199 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 200 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111) 201 goto out; 202 break; 203 case BPF_MODIFY_RETURN: 204 ret = bpf_modify_return_test(1, &b); 205 if (b != 2) 206 side_effect = 1; 207 break; 208 default: 209 goto out; 210 } 211 212 retval = ((u32)side_effect << 16) | ret; 213 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 214 goto out; 215 216 err = 0; 217 out: 218 trace_bpf_test_finish(&err); 219 return err; 220 } 221 222 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 223 { 224 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 225 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 226 u32 size = kattr->test.ctx_size_in; 227 void *data; 228 int err; 229 230 if (!data_in && !data_out) 231 return NULL; 232 233 data = kzalloc(max_size, GFP_USER); 234 if (!data) 235 return ERR_PTR(-ENOMEM); 236 237 if (data_in) { 238 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 239 if (err) { 240 kfree(data); 241 return ERR_PTR(err); 242 } 243 244 size = min_t(u32, max_size, size); 245 if (copy_from_user(data, data_in, size)) { 246 kfree(data); 247 return ERR_PTR(-EFAULT); 248 } 249 } 250 return data; 251 } 252 253 static int bpf_ctx_finish(const union bpf_attr *kattr, 254 union bpf_attr __user *uattr, const void *data, 255 u32 size) 256 { 257 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 258 int err = -EFAULT; 259 u32 copy_size = size; 260 261 if (!data || !data_out) 262 return 0; 263 264 if (copy_size > kattr->test.ctx_size_out) { 265 copy_size = kattr->test.ctx_size_out; 266 err = -ENOSPC; 267 } 268 269 if (copy_to_user(data_out, data, copy_size)) 270 goto out; 271 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 272 goto out; 273 if (err != -ENOSPC) 274 err = 0; 275 out: 276 return err; 277 } 278 279 /** 280 * range_is_zero - test whether buffer is initialized 281 * @buf: buffer to check 282 * @from: check from this position 283 * @to: check up until (excluding) this position 284 * 285 * This function returns true if the there is a non-zero byte 286 * in the buf in the range [from,to). 287 */ 288 static inline bool range_is_zero(void *buf, size_t from, size_t to) 289 { 290 return !memchr_inv((u8 *)buf + from, 0, to - from); 291 } 292 293 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 294 { 295 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 296 297 if (!__skb) 298 return 0; 299 300 /* make sure the fields we don't use are zeroed */ 301 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 302 return -EINVAL; 303 304 /* mark is allowed */ 305 306 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 307 offsetof(struct __sk_buff, priority))) 308 return -EINVAL; 309 310 /* priority is allowed */ 311 312 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority), 313 offsetof(struct __sk_buff, cb))) 314 return -EINVAL; 315 316 /* cb is allowed */ 317 318 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 319 offsetof(struct __sk_buff, tstamp))) 320 return -EINVAL; 321 322 /* tstamp is allowed */ 323 /* wire_len is allowed */ 324 /* gso_segs is allowed */ 325 326 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 327 offsetof(struct __sk_buff, gso_size))) 328 return -EINVAL; 329 330 /* gso_size is allowed */ 331 332 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 333 sizeof(struct __sk_buff))) 334 return -EINVAL; 335 336 skb->mark = __skb->mark; 337 skb->priority = __skb->priority; 338 skb->tstamp = __skb->tstamp; 339 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 340 341 if (__skb->wire_len == 0) { 342 cb->pkt_len = skb->len; 343 } else { 344 if (__skb->wire_len < skb->len || 345 __skb->wire_len > GSO_MAX_SIZE) 346 return -EINVAL; 347 cb->pkt_len = __skb->wire_len; 348 } 349 350 if (__skb->gso_segs > GSO_MAX_SEGS) 351 return -EINVAL; 352 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 353 skb_shinfo(skb)->gso_size = __skb->gso_size; 354 355 return 0; 356 } 357 358 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 359 { 360 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 361 362 if (!__skb) 363 return; 364 365 __skb->mark = skb->mark; 366 __skb->priority = skb->priority; 367 __skb->tstamp = skb->tstamp; 368 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 369 __skb->wire_len = cb->pkt_len; 370 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 371 } 372 373 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 374 union bpf_attr __user *uattr) 375 { 376 bool is_l2 = false, is_direct_pkt_access = false; 377 u32 size = kattr->test.data_size_in; 378 u32 repeat = kattr->test.repeat; 379 struct __sk_buff *ctx = NULL; 380 u32 retval, duration; 381 int hh_len = ETH_HLEN; 382 struct sk_buff *skb; 383 struct sock *sk; 384 void *data; 385 int ret; 386 387 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 388 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 389 if (IS_ERR(data)) 390 return PTR_ERR(data); 391 392 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 393 if (IS_ERR(ctx)) { 394 kfree(data); 395 return PTR_ERR(ctx); 396 } 397 398 switch (prog->type) { 399 case BPF_PROG_TYPE_SCHED_CLS: 400 case BPF_PROG_TYPE_SCHED_ACT: 401 is_l2 = true; 402 /* fall through */ 403 case BPF_PROG_TYPE_LWT_IN: 404 case BPF_PROG_TYPE_LWT_OUT: 405 case BPF_PROG_TYPE_LWT_XMIT: 406 is_direct_pkt_access = true; 407 break; 408 default: 409 break; 410 } 411 412 sk = kzalloc(sizeof(struct sock), GFP_USER); 413 if (!sk) { 414 kfree(data); 415 kfree(ctx); 416 return -ENOMEM; 417 } 418 sock_net_set(sk, current->nsproxy->net_ns); 419 sock_init_data(NULL, sk); 420 421 skb = build_skb(data, 0); 422 if (!skb) { 423 kfree(data); 424 kfree(ctx); 425 kfree(sk); 426 return -ENOMEM; 427 } 428 skb->sk = sk; 429 430 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 431 __skb_put(skb, size); 432 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 433 skb_reset_network_header(skb); 434 435 if (is_l2) 436 __skb_push(skb, hh_len); 437 if (is_direct_pkt_access) 438 bpf_compute_data_pointers(skb); 439 ret = convert___skb_to_skb(skb, ctx); 440 if (ret) 441 goto out; 442 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 443 if (ret) 444 goto out; 445 if (!is_l2) { 446 if (skb_headroom(skb) < hh_len) { 447 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 448 449 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 450 ret = -ENOMEM; 451 goto out; 452 } 453 } 454 memset(__skb_push(skb, hh_len), 0, hh_len); 455 } 456 convert_skb_to___skb(skb, ctx); 457 458 size = skb->len; 459 /* bpf program can never convert linear skb to non-linear */ 460 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 461 size = skb_headlen(skb); 462 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 463 if (!ret) 464 ret = bpf_ctx_finish(kattr, uattr, ctx, 465 sizeof(struct __sk_buff)); 466 out: 467 kfree_skb(skb); 468 bpf_sk_storage_free(sk); 469 kfree(sk); 470 kfree(ctx); 471 return ret; 472 } 473 474 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 475 union bpf_attr __user *uattr) 476 { 477 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 478 u32 headroom = XDP_PACKET_HEADROOM; 479 u32 size = kattr->test.data_size_in; 480 u32 repeat = kattr->test.repeat; 481 struct netdev_rx_queue *rxqueue; 482 struct xdp_buff xdp = {}; 483 u32 retval, duration; 484 u32 max_data_sz; 485 void *data; 486 int ret; 487 488 if (kattr->test.ctx_in || kattr->test.ctx_out) 489 return -EINVAL; 490 491 /* XDP have extra tailroom as (most) drivers use full page */ 492 max_data_sz = 4096 - headroom - tailroom; 493 494 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); 495 if (IS_ERR(data)) 496 return PTR_ERR(data); 497 498 xdp.data_hard_start = data; 499 xdp.data = data + headroom; 500 xdp.data_meta = xdp.data; 501 xdp.data_end = xdp.data + size; 502 xdp.frame_sz = headroom + max_data_sz + tailroom; 503 504 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 505 xdp.rxq = &rxqueue->xdp_rxq; 506 bpf_prog_change_xdp(NULL, prog); 507 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 508 if (ret) 509 goto out; 510 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) 511 size = xdp.data_end - xdp.data; 512 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 513 out: 514 bpf_prog_change_xdp(prog, NULL); 515 kfree(data); 516 return ret; 517 } 518 519 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 520 { 521 /* make sure the fields we don't use are zeroed */ 522 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 523 return -EINVAL; 524 525 /* flags is allowed */ 526 527 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 528 sizeof(struct bpf_flow_keys))) 529 return -EINVAL; 530 531 return 0; 532 } 533 534 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 535 const union bpf_attr *kattr, 536 union bpf_attr __user *uattr) 537 { 538 u32 size = kattr->test.data_size_in; 539 struct bpf_flow_dissector ctx = {}; 540 u32 repeat = kattr->test.repeat; 541 struct bpf_flow_keys *user_ctx; 542 struct bpf_flow_keys flow_keys; 543 u64 time_start, time_spent = 0; 544 const struct ethhdr *eth; 545 unsigned int flags = 0; 546 u32 retval, duration; 547 void *data; 548 int ret; 549 u32 i; 550 551 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 552 return -EINVAL; 553 554 if (size < ETH_HLEN) 555 return -EINVAL; 556 557 data = bpf_test_init(kattr, size, 0, 0); 558 if (IS_ERR(data)) 559 return PTR_ERR(data); 560 561 eth = (struct ethhdr *)data; 562 563 if (!repeat) 564 repeat = 1; 565 566 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 567 if (IS_ERR(user_ctx)) { 568 kfree(data); 569 return PTR_ERR(user_ctx); 570 } 571 if (user_ctx) { 572 ret = verify_user_bpf_flow_keys(user_ctx); 573 if (ret) 574 goto out; 575 flags = user_ctx->flags; 576 } 577 578 ctx.flow_keys = &flow_keys; 579 ctx.data = data; 580 ctx.data_end = (__u8 *)data + size; 581 582 rcu_read_lock(); 583 preempt_disable(); 584 time_start = ktime_get_ns(); 585 for (i = 0; i < repeat; i++) { 586 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 587 size, flags); 588 589 if (signal_pending(current)) { 590 preempt_enable(); 591 rcu_read_unlock(); 592 593 ret = -EINTR; 594 goto out; 595 } 596 597 if (need_resched()) { 598 time_spent += ktime_get_ns() - time_start; 599 preempt_enable(); 600 rcu_read_unlock(); 601 602 cond_resched(); 603 604 rcu_read_lock(); 605 preempt_disable(); 606 time_start = ktime_get_ns(); 607 } 608 } 609 time_spent += ktime_get_ns() - time_start; 610 preempt_enable(); 611 rcu_read_unlock(); 612 613 do_div(time_spent, repeat); 614 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 615 616 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 617 retval, duration); 618 if (!ret) 619 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 620 sizeof(struct bpf_flow_keys)); 621 622 out: 623 kfree(user_ctx); 624 kfree(data); 625 return ret; 626 } 627