1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/btf.h> 6 #include <linux/btf_ids.h> 7 #include <linux/slab.h> 8 #include <linux/vmalloc.h> 9 #include <linux/etherdevice.h> 10 #include <linux/filter.h> 11 #include <linux/rcupdate_trace.h> 12 #include <linux/sched/signal.h> 13 #include <net/bpf_sk_storage.h> 14 #include <net/sock.h> 15 #include <net/tcp.h> 16 #include <net/net_namespace.h> 17 #include <linux/error-injection.h> 18 #include <linux/smp.h> 19 #include <linux/sock_diag.h> 20 #include <net/xdp.h> 21 22 #define CREATE_TRACE_POINTS 23 #include <trace/events/bpf_test_run.h> 24 25 struct bpf_test_timer { 26 enum { NO_PREEMPT, NO_MIGRATE } mode; 27 u32 i; 28 u64 time_start, time_spent; 29 }; 30 31 static void bpf_test_timer_enter(struct bpf_test_timer *t) 32 __acquires(rcu) 33 { 34 rcu_read_lock(); 35 if (t->mode == NO_PREEMPT) 36 preempt_disable(); 37 else 38 migrate_disable(); 39 40 t->time_start = ktime_get_ns(); 41 } 42 43 static void bpf_test_timer_leave(struct bpf_test_timer *t) 44 __releases(rcu) 45 { 46 t->time_start = 0; 47 48 if (t->mode == NO_PREEMPT) 49 preempt_enable(); 50 else 51 migrate_enable(); 52 rcu_read_unlock(); 53 } 54 55 static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration) 56 __must_hold(rcu) 57 { 58 t->i++; 59 if (t->i >= repeat) { 60 /* We're done. */ 61 t->time_spent += ktime_get_ns() - t->time_start; 62 do_div(t->time_spent, t->i); 63 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; 64 *err = 0; 65 goto reset; 66 } 67 68 if (signal_pending(current)) { 69 /* During iteration: we've been cancelled, abort. */ 70 *err = -EINTR; 71 goto reset; 72 } 73 74 if (need_resched()) { 75 /* During iteration: we need to reschedule between runs. */ 76 t->time_spent += ktime_get_ns() - t->time_start; 77 bpf_test_timer_leave(t); 78 cond_resched(); 79 bpf_test_timer_enter(t); 80 } 81 82 /* Do another round. */ 83 return true; 84 85 reset: 86 t->i = 0; 87 return false; 88 } 89 90 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 91 u32 *retval, u32 *time, bool xdp) 92 { 93 struct bpf_prog_array_item item = {.prog = prog}; 94 struct bpf_run_ctx *old_ctx; 95 struct bpf_cg_run_ctx run_ctx; 96 struct bpf_test_timer t = { NO_MIGRATE }; 97 enum bpf_cgroup_storage_type stype; 98 int ret; 99 100 for_each_cgroup_storage_type(stype) { 101 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 102 if (IS_ERR(item.cgroup_storage[stype])) { 103 item.cgroup_storage[stype] = NULL; 104 for_each_cgroup_storage_type(stype) 105 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 106 return -ENOMEM; 107 } 108 } 109 110 if (!repeat) 111 repeat = 1; 112 113 bpf_test_timer_enter(&t); 114 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 115 do { 116 run_ctx.prog_item = &item; 117 if (xdp) 118 *retval = bpf_prog_run_xdp(prog, ctx); 119 else 120 *retval = bpf_prog_run(prog, ctx); 121 } while (bpf_test_timer_continue(&t, repeat, &ret, time)); 122 bpf_reset_run_ctx(old_ctx); 123 bpf_test_timer_leave(&t); 124 125 for_each_cgroup_storage_type(stype) 126 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 127 128 return ret; 129 } 130 131 static int bpf_test_finish(const union bpf_attr *kattr, 132 union bpf_attr __user *uattr, const void *data, 133 u32 size, u32 retval, u32 duration) 134 { 135 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 136 int err = -EFAULT; 137 u32 copy_size = size; 138 139 /* Clamp copy if the user has provided a size hint, but copy the full 140 * buffer if not to retain old behaviour. 141 */ 142 if (kattr->test.data_size_out && 143 copy_size > kattr->test.data_size_out) { 144 copy_size = kattr->test.data_size_out; 145 err = -ENOSPC; 146 } 147 148 if (data_out && copy_to_user(data_out, data, copy_size)) 149 goto out; 150 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 151 goto out; 152 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 153 goto out; 154 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 155 goto out; 156 if (err != -ENOSPC) 157 err = 0; 158 out: 159 trace_bpf_test_finish(&err); 160 return err; 161 } 162 163 /* Integer types of various sizes and pointer combinations cover variety of 164 * architecture dependent calling conventions. 7+ can be supported in the 165 * future. 166 */ 167 __diag_push(); 168 __diag_ignore(GCC, 8, "-Wmissing-prototypes", 169 "Global functions as their definitions will be in vmlinux BTF"); 170 int noinline bpf_fentry_test1(int a) 171 { 172 return a + 1; 173 } 174 175 int noinline bpf_fentry_test2(int a, u64 b) 176 { 177 return a + b; 178 } 179 180 int noinline bpf_fentry_test3(char a, int b, u64 c) 181 { 182 return a + b + c; 183 } 184 185 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 186 { 187 return (long)a + b + c + d; 188 } 189 190 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 191 { 192 return a + (long)b + c + d + e; 193 } 194 195 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 196 { 197 return a + (long)b + c + d + (long)e + f; 198 } 199 200 struct bpf_fentry_test_t { 201 struct bpf_fentry_test_t *a; 202 }; 203 204 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 205 { 206 return (long)arg; 207 } 208 209 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 210 { 211 return (long)arg->a; 212 } 213 214 int noinline bpf_modify_return_test(int a, int *b) 215 { 216 *b += 1; 217 return a + *b; 218 } 219 220 u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d) 221 { 222 return a + b + c + d; 223 } 224 225 int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b) 226 { 227 return a + b; 228 } 229 230 struct sock * noinline bpf_kfunc_call_test3(struct sock *sk) 231 { 232 return sk; 233 } 234 235 __diag_pop(); 236 237 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 238 239 BTF_SET_START(test_sk_kfunc_ids) 240 BTF_ID(func, bpf_kfunc_call_test1) 241 BTF_ID(func, bpf_kfunc_call_test2) 242 BTF_ID(func, bpf_kfunc_call_test3) 243 BTF_SET_END(test_sk_kfunc_ids) 244 245 bool bpf_prog_test_check_kfunc_call(u32 kfunc_id, struct module *owner) 246 { 247 if (btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id)) 248 return true; 249 return bpf_check_mod_kfunc_call(&prog_test_kfunc_list, kfunc_id, owner); 250 } 251 252 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 253 u32 headroom, u32 tailroom) 254 { 255 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 256 u32 user_size = kattr->test.data_size_in; 257 void *data; 258 259 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 260 return ERR_PTR(-EINVAL); 261 262 if (user_size > size) 263 return ERR_PTR(-EMSGSIZE); 264 265 data = kzalloc(size + headroom + tailroom, GFP_USER); 266 if (!data) 267 return ERR_PTR(-ENOMEM); 268 269 if (copy_from_user(data + headroom, data_in, user_size)) { 270 kfree(data); 271 return ERR_PTR(-EFAULT); 272 } 273 274 return data; 275 } 276 277 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 278 const union bpf_attr *kattr, 279 union bpf_attr __user *uattr) 280 { 281 struct bpf_fentry_test_t arg = {}; 282 u16 side_effect = 0, ret = 0; 283 int b = 2, err = -EFAULT; 284 u32 retval = 0; 285 286 if (kattr->test.flags || kattr->test.cpu) 287 return -EINVAL; 288 289 switch (prog->expected_attach_type) { 290 case BPF_TRACE_FENTRY: 291 case BPF_TRACE_FEXIT: 292 if (bpf_fentry_test1(1) != 2 || 293 bpf_fentry_test2(2, 3) != 5 || 294 bpf_fentry_test3(4, 5, 6) != 15 || 295 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 296 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 297 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 298 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 299 bpf_fentry_test8(&arg) != 0) 300 goto out; 301 break; 302 case BPF_MODIFY_RETURN: 303 ret = bpf_modify_return_test(1, &b); 304 if (b != 2) 305 side_effect = 1; 306 break; 307 default: 308 goto out; 309 } 310 311 retval = ((u32)side_effect << 16) | ret; 312 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 313 goto out; 314 315 err = 0; 316 out: 317 trace_bpf_test_finish(&err); 318 return err; 319 } 320 321 struct bpf_raw_tp_test_run_info { 322 struct bpf_prog *prog; 323 void *ctx; 324 u32 retval; 325 }; 326 327 static void 328 __bpf_prog_test_run_raw_tp(void *data) 329 { 330 struct bpf_raw_tp_test_run_info *info = data; 331 332 rcu_read_lock(); 333 info->retval = bpf_prog_run(info->prog, info->ctx); 334 rcu_read_unlock(); 335 } 336 337 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 338 const union bpf_attr *kattr, 339 union bpf_attr __user *uattr) 340 { 341 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 342 __u32 ctx_size_in = kattr->test.ctx_size_in; 343 struct bpf_raw_tp_test_run_info info; 344 int cpu = kattr->test.cpu, err = 0; 345 int current_cpu; 346 347 /* doesn't support data_in/out, ctx_out, duration, or repeat */ 348 if (kattr->test.data_in || kattr->test.data_out || 349 kattr->test.ctx_out || kattr->test.duration || 350 kattr->test.repeat) 351 return -EINVAL; 352 353 if (ctx_size_in < prog->aux->max_ctx_offset || 354 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) 355 return -EINVAL; 356 357 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) 358 return -EINVAL; 359 360 if (ctx_size_in) { 361 info.ctx = memdup_user(ctx_in, ctx_size_in); 362 if (IS_ERR(info.ctx)) 363 return PTR_ERR(info.ctx); 364 } else { 365 info.ctx = NULL; 366 } 367 368 info.prog = prog; 369 370 current_cpu = get_cpu(); 371 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || 372 cpu == current_cpu) { 373 __bpf_prog_test_run_raw_tp(&info); 374 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { 375 /* smp_call_function_single() also checks cpu_online() 376 * after csd_lock(). However, since cpu is from user 377 * space, let's do an extra quick check to filter out 378 * invalid value before smp_call_function_single(). 379 */ 380 err = -ENXIO; 381 } else { 382 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, 383 &info, 1); 384 } 385 put_cpu(); 386 387 if (!err && 388 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) 389 err = -EFAULT; 390 391 kfree(info.ctx); 392 return err; 393 } 394 395 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 396 { 397 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 398 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 399 u32 size = kattr->test.ctx_size_in; 400 void *data; 401 int err; 402 403 if (!data_in && !data_out) 404 return NULL; 405 406 data = kzalloc(max_size, GFP_USER); 407 if (!data) 408 return ERR_PTR(-ENOMEM); 409 410 if (data_in) { 411 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size); 412 if (err) { 413 kfree(data); 414 return ERR_PTR(err); 415 } 416 417 size = min_t(u32, max_size, size); 418 if (copy_from_user(data, data_in, size)) { 419 kfree(data); 420 return ERR_PTR(-EFAULT); 421 } 422 } 423 return data; 424 } 425 426 static int bpf_ctx_finish(const union bpf_attr *kattr, 427 union bpf_attr __user *uattr, const void *data, 428 u32 size) 429 { 430 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 431 int err = -EFAULT; 432 u32 copy_size = size; 433 434 if (!data || !data_out) 435 return 0; 436 437 if (copy_size > kattr->test.ctx_size_out) { 438 copy_size = kattr->test.ctx_size_out; 439 err = -ENOSPC; 440 } 441 442 if (copy_to_user(data_out, data, copy_size)) 443 goto out; 444 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 445 goto out; 446 if (err != -ENOSPC) 447 err = 0; 448 out: 449 return err; 450 } 451 452 /** 453 * range_is_zero - test whether buffer is initialized 454 * @buf: buffer to check 455 * @from: check from this position 456 * @to: check up until (excluding) this position 457 * 458 * This function returns true if the there is a non-zero byte 459 * in the buf in the range [from,to). 460 */ 461 static inline bool range_is_zero(void *buf, size_t from, size_t to) 462 { 463 return !memchr_inv((u8 *)buf + from, 0, to - from); 464 } 465 466 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 467 { 468 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 469 470 if (!__skb) 471 return 0; 472 473 /* make sure the fields we don't use are zeroed */ 474 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 475 return -EINVAL; 476 477 /* mark is allowed */ 478 479 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 480 offsetof(struct __sk_buff, priority))) 481 return -EINVAL; 482 483 /* priority is allowed */ 484 /* ingress_ifindex is allowed */ 485 /* ifindex is allowed */ 486 487 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 488 offsetof(struct __sk_buff, cb))) 489 return -EINVAL; 490 491 /* cb is allowed */ 492 493 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 494 offsetof(struct __sk_buff, tstamp))) 495 return -EINVAL; 496 497 /* tstamp is allowed */ 498 /* wire_len is allowed */ 499 /* gso_segs is allowed */ 500 501 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 502 offsetof(struct __sk_buff, gso_size))) 503 return -EINVAL; 504 505 /* gso_size is allowed */ 506 507 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 508 offsetof(struct __sk_buff, hwtstamp))) 509 return -EINVAL; 510 511 /* hwtstamp is allowed */ 512 513 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp), 514 sizeof(struct __sk_buff))) 515 return -EINVAL; 516 517 skb->mark = __skb->mark; 518 skb->priority = __skb->priority; 519 skb->skb_iif = __skb->ingress_ifindex; 520 skb->tstamp = __skb->tstamp; 521 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 522 523 if (__skb->wire_len == 0) { 524 cb->pkt_len = skb->len; 525 } else { 526 if (__skb->wire_len < skb->len || 527 __skb->wire_len > GSO_MAX_SIZE) 528 return -EINVAL; 529 cb->pkt_len = __skb->wire_len; 530 } 531 532 if (__skb->gso_segs > GSO_MAX_SEGS) 533 return -EINVAL; 534 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 535 skb_shinfo(skb)->gso_size = __skb->gso_size; 536 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; 537 538 return 0; 539 } 540 541 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 542 { 543 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 544 545 if (!__skb) 546 return; 547 548 __skb->mark = skb->mark; 549 __skb->priority = skb->priority; 550 __skb->ingress_ifindex = skb->skb_iif; 551 __skb->ifindex = skb->dev->ifindex; 552 __skb->tstamp = skb->tstamp; 553 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 554 __skb->wire_len = cb->pkt_len; 555 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 556 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; 557 } 558 559 static struct proto bpf_dummy_proto = { 560 .name = "bpf_dummy", 561 .owner = THIS_MODULE, 562 .obj_size = sizeof(struct sock), 563 }; 564 565 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 566 union bpf_attr __user *uattr) 567 { 568 bool is_l2 = false, is_direct_pkt_access = false; 569 struct net *net = current->nsproxy->net_ns; 570 struct net_device *dev = net->loopback_dev; 571 u32 size = kattr->test.data_size_in; 572 u32 repeat = kattr->test.repeat; 573 struct __sk_buff *ctx = NULL; 574 u32 retval, duration; 575 int hh_len = ETH_HLEN; 576 struct sk_buff *skb; 577 struct sock *sk; 578 void *data; 579 int ret; 580 581 if (kattr->test.flags || kattr->test.cpu) 582 return -EINVAL; 583 584 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 585 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 586 if (IS_ERR(data)) 587 return PTR_ERR(data); 588 589 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 590 if (IS_ERR(ctx)) { 591 kfree(data); 592 return PTR_ERR(ctx); 593 } 594 595 switch (prog->type) { 596 case BPF_PROG_TYPE_SCHED_CLS: 597 case BPF_PROG_TYPE_SCHED_ACT: 598 is_l2 = true; 599 fallthrough; 600 case BPF_PROG_TYPE_LWT_IN: 601 case BPF_PROG_TYPE_LWT_OUT: 602 case BPF_PROG_TYPE_LWT_XMIT: 603 is_direct_pkt_access = true; 604 break; 605 default: 606 break; 607 } 608 609 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); 610 if (!sk) { 611 kfree(data); 612 kfree(ctx); 613 return -ENOMEM; 614 } 615 sock_init_data(NULL, sk); 616 617 skb = build_skb(data, 0); 618 if (!skb) { 619 kfree(data); 620 kfree(ctx); 621 sk_free(sk); 622 return -ENOMEM; 623 } 624 skb->sk = sk; 625 626 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 627 __skb_put(skb, size); 628 if (ctx && ctx->ifindex > 1) { 629 dev = dev_get_by_index(net, ctx->ifindex); 630 if (!dev) { 631 ret = -ENODEV; 632 goto out; 633 } 634 } 635 skb->protocol = eth_type_trans(skb, dev); 636 skb_reset_network_header(skb); 637 638 switch (skb->protocol) { 639 case htons(ETH_P_IP): 640 sk->sk_family = AF_INET; 641 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 642 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 643 sk->sk_daddr = ip_hdr(skb)->daddr; 644 } 645 break; 646 #if IS_ENABLED(CONFIG_IPV6) 647 case htons(ETH_P_IPV6): 648 sk->sk_family = AF_INET6; 649 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 650 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 651 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 652 } 653 break; 654 #endif 655 default: 656 break; 657 } 658 659 if (is_l2) 660 __skb_push(skb, hh_len); 661 if (is_direct_pkt_access) 662 bpf_compute_data_pointers(skb); 663 ret = convert___skb_to_skb(skb, ctx); 664 if (ret) 665 goto out; 666 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 667 if (ret) 668 goto out; 669 if (!is_l2) { 670 if (skb_headroom(skb) < hh_len) { 671 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 672 673 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 674 ret = -ENOMEM; 675 goto out; 676 } 677 } 678 memset(__skb_push(skb, hh_len), 0, hh_len); 679 } 680 convert_skb_to___skb(skb, ctx); 681 682 size = skb->len; 683 /* bpf program can never convert linear skb to non-linear */ 684 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 685 size = skb_headlen(skb); 686 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 687 if (!ret) 688 ret = bpf_ctx_finish(kattr, uattr, ctx, 689 sizeof(struct __sk_buff)); 690 out: 691 if (dev && dev != net->loopback_dev) 692 dev_put(dev); 693 kfree_skb(skb); 694 sk_free(sk); 695 kfree(ctx); 696 return ret; 697 } 698 699 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp) 700 { 701 unsigned int ingress_ifindex, rx_queue_index; 702 struct netdev_rx_queue *rxqueue; 703 struct net_device *device; 704 705 if (!xdp_md) 706 return 0; 707 708 if (xdp_md->egress_ifindex != 0) 709 return -EINVAL; 710 711 ingress_ifindex = xdp_md->ingress_ifindex; 712 rx_queue_index = xdp_md->rx_queue_index; 713 714 if (!ingress_ifindex && rx_queue_index) 715 return -EINVAL; 716 717 if (ingress_ifindex) { 718 device = dev_get_by_index(current->nsproxy->net_ns, 719 ingress_ifindex); 720 if (!device) 721 return -ENODEV; 722 723 if (rx_queue_index >= device->real_num_rx_queues) 724 goto free_dev; 725 726 rxqueue = __netif_get_rx_queue(device, rx_queue_index); 727 728 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) 729 goto free_dev; 730 731 xdp->rxq = &rxqueue->xdp_rxq; 732 /* The device is now tracked in the xdp->rxq for later 733 * dev_put() 734 */ 735 } 736 737 xdp->data = xdp->data_meta + xdp_md->data; 738 return 0; 739 740 free_dev: 741 dev_put(device); 742 return -EINVAL; 743 } 744 745 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) 746 { 747 if (!xdp_md) 748 return; 749 750 xdp_md->data = xdp->data - xdp->data_meta; 751 xdp_md->data_end = xdp->data_end - xdp->data_meta; 752 753 if (xdp_md->ingress_ifindex) 754 dev_put(xdp->rxq->dev); 755 } 756 757 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 758 union bpf_attr __user *uattr) 759 { 760 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 761 u32 headroom = XDP_PACKET_HEADROOM; 762 u32 size = kattr->test.data_size_in; 763 u32 repeat = kattr->test.repeat; 764 struct netdev_rx_queue *rxqueue; 765 struct xdp_buff xdp = {}; 766 u32 retval, duration; 767 struct xdp_md *ctx; 768 u32 max_data_sz; 769 void *data; 770 int ret = -EINVAL; 771 772 if (prog->expected_attach_type == BPF_XDP_DEVMAP || 773 prog->expected_attach_type == BPF_XDP_CPUMAP) 774 return -EINVAL; 775 776 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); 777 if (IS_ERR(ctx)) 778 return PTR_ERR(ctx); 779 780 if (ctx) { 781 /* There can't be user provided data before the meta data */ 782 if (ctx->data_meta || ctx->data_end != size || 783 ctx->data > ctx->data_end || 784 unlikely(xdp_metalen_invalid(ctx->data))) 785 goto free_ctx; 786 /* Meta data is allocated from the headroom */ 787 headroom -= ctx->data; 788 } 789 790 /* XDP have extra tailroom as (most) drivers use full page */ 791 max_data_sz = 4096 - headroom - tailroom; 792 793 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); 794 if (IS_ERR(data)) { 795 ret = PTR_ERR(data); 796 goto free_ctx; 797 } 798 799 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 800 xdp_init_buff(&xdp, headroom + max_data_sz + tailroom, 801 &rxqueue->xdp_rxq); 802 xdp_prepare_buff(&xdp, data, headroom, size, true); 803 804 ret = xdp_convert_md_to_buff(ctx, &xdp); 805 if (ret) 806 goto free_data; 807 808 if (repeat > 1) 809 bpf_prog_change_xdp(NULL, prog); 810 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 811 /* We convert the xdp_buff back to an xdp_md before checking the return 812 * code so the reference count of any held netdevice will be decremented 813 * even if the test run failed. 814 */ 815 xdp_convert_buff_to_md(&xdp, ctx); 816 if (ret) 817 goto out; 818 819 if (xdp.data_meta != data + headroom || 820 xdp.data_end != xdp.data_meta + size) 821 size = xdp.data_end - xdp.data_meta; 822 823 ret = bpf_test_finish(kattr, uattr, xdp.data_meta, size, retval, 824 duration); 825 if (!ret) 826 ret = bpf_ctx_finish(kattr, uattr, ctx, 827 sizeof(struct xdp_md)); 828 829 out: 830 if (repeat > 1) 831 bpf_prog_change_xdp(prog, NULL); 832 free_data: 833 kfree(data); 834 free_ctx: 835 kfree(ctx); 836 return ret; 837 } 838 839 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 840 { 841 /* make sure the fields we don't use are zeroed */ 842 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 843 return -EINVAL; 844 845 /* flags is allowed */ 846 847 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 848 sizeof(struct bpf_flow_keys))) 849 return -EINVAL; 850 851 return 0; 852 } 853 854 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 855 const union bpf_attr *kattr, 856 union bpf_attr __user *uattr) 857 { 858 struct bpf_test_timer t = { NO_PREEMPT }; 859 u32 size = kattr->test.data_size_in; 860 struct bpf_flow_dissector ctx = {}; 861 u32 repeat = kattr->test.repeat; 862 struct bpf_flow_keys *user_ctx; 863 struct bpf_flow_keys flow_keys; 864 const struct ethhdr *eth; 865 unsigned int flags = 0; 866 u32 retval, duration; 867 void *data; 868 int ret; 869 870 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 871 return -EINVAL; 872 873 if (kattr->test.flags || kattr->test.cpu) 874 return -EINVAL; 875 876 if (size < ETH_HLEN) 877 return -EINVAL; 878 879 data = bpf_test_init(kattr, size, 0, 0); 880 if (IS_ERR(data)) 881 return PTR_ERR(data); 882 883 eth = (struct ethhdr *)data; 884 885 if (!repeat) 886 repeat = 1; 887 888 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 889 if (IS_ERR(user_ctx)) { 890 kfree(data); 891 return PTR_ERR(user_ctx); 892 } 893 if (user_ctx) { 894 ret = verify_user_bpf_flow_keys(user_ctx); 895 if (ret) 896 goto out; 897 flags = user_ctx->flags; 898 } 899 900 ctx.flow_keys = &flow_keys; 901 ctx.data = data; 902 ctx.data_end = (__u8 *)data + size; 903 904 bpf_test_timer_enter(&t); 905 do { 906 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 907 size, flags); 908 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); 909 bpf_test_timer_leave(&t); 910 911 if (ret < 0) 912 goto out; 913 914 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 915 retval, duration); 916 if (!ret) 917 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 918 sizeof(struct bpf_flow_keys)); 919 920 out: 921 kfree(user_ctx); 922 kfree(data); 923 return ret; 924 } 925 926 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, 927 union bpf_attr __user *uattr) 928 { 929 struct bpf_test_timer t = { NO_PREEMPT }; 930 struct bpf_prog_array *progs = NULL; 931 struct bpf_sk_lookup_kern ctx = {}; 932 u32 repeat = kattr->test.repeat; 933 struct bpf_sk_lookup *user_ctx; 934 u32 retval, duration; 935 int ret = -EINVAL; 936 937 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) 938 return -EINVAL; 939 940 if (kattr->test.flags || kattr->test.cpu) 941 return -EINVAL; 942 943 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || 944 kattr->test.data_size_out) 945 return -EINVAL; 946 947 if (!repeat) 948 repeat = 1; 949 950 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); 951 if (IS_ERR(user_ctx)) 952 return PTR_ERR(user_ctx); 953 954 if (!user_ctx) 955 return -EINVAL; 956 957 if (user_ctx->sk) 958 goto out; 959 960 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) 961 goto out; 962 963 if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) { 964 ret = -ERANGE; 965 goto out; 966 } 967 968 ctx.family = (u16)user_ctx->family; 969 ctx.protocol = (u16)user_ctx->protocol; 970 ctx.dport = (u16)user_ctx->local_port; 971 ctx.sport = (__force __be16)user_ctx->remote_port; 972 973 switch (ctx.family) { 974 case AF_INET: 975 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; 976 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; 977 break; 978 979 #if IS_ENABLED(CONFIG_IPV6) 980 case AF_INET6: 981 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; 982 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; 983 break; 984 #endif 985 986 default: 987 ret = -EAFNOSUPPORT; 988 goto out; 989 } 990 991 progs = bpf_prog_array_alloc(1, GFP_KERNEL); 992 if (!progs) { 993 ret = -ENOMEM; 994 goto out; 995 } 996 997 progs->items[0].prog = prog; 998 999 bpf_test_timer_enter(&t); 1000 do { 1001 ctx.selected_sk = NULL; 1002 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); 1003 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); 1004 bpf_test_timer_leave(&t); 1005 1006 if (ret < 0) 1007 goto out; 1008 1009 user_ctx->cookie = 0; 1010 if (ctx.selected_sk) { 1011 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { 1012 ret = -EOPNOTSUPP; 1013 goto out; 1014 } 1015 1016 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); 1017 } 1018 1019 ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration); 1020 if (!ret) 1021 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); 1022 1023 out: 1024 bpf_prog_array_free(progs); 1025 kfree(user_ctx); 1026 return ret; 1027 } 1028 1029 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 1030 const union bpf_attr *kattr, 1031 union bpf_attr __user *uattr) 1032 { 1033 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 1034 __u32 ctx_size_in = kattr->test.ctx_size_in; 1035 void *ctx = NULL; 1036 u32 retval; 1037 int err = 0; 1038 1039 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ 1040 if (kattr->test.data_in || kattr->test.data_out || 1041 kattr->test.ctx_out || kattr->test.duration || 1042 kattr->test.repeat || kattr->test.flags) 1043 return -EINVAL; 1044 1045 if (ctx_size_in < prog->aux->max_ctx_offset || 1046 ctx_size_in > U16_MAX) 1047 return -EINVAL; 1048 1049 if (ctx_size_in) { 1050 ctx = memdup_user(ctx_in, ctx_size_in); 1051 if (IS_ERR(ctx)) 1052 return PTR_ERR(ctx); 1053 } 1054 1055 rcu_read_lock_trace(); 1056 retval = bpf_prog_run_pin_on_cpu(prog, ctx); 1057 rcu_read_unlock_trace(); 1058 1059 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { 1060 err = -EFAULT; 1061 goto out; 1062 } 1063 if (ctx_size_in) 1064 if (copy_to_user(ctx_in, ctx, ctx_size_in)) 1065 err = -EFAULT; 1066 out: 1067 kfree(ctx); 1068 return err; 1069 } 1070