1 /* Copyright (c) 2017 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #include <linux/bpf.h> 8 #include <linux/slab.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/sched/signal.h> 13 #include <net/sock.h> 14 #include <net/tcp.h> 15 16 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 17 u32 *retval, u32 *time) 18 { 19 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 20 enum bpf_cgroup_storage_type stype; 21 u64 time_start, time_spent = 0; 22 int ret = 0; 23 u32 i; 24 25 for_each_cgroup_storage_type(stype) { 26 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 27 if (IS_ERR(storage[stype])) { 28 storage[stype] = NULL; 29 for_each_cgroup_storage_type(stype) 30 bpf_cgroup_storage_free(storage[stype]); 31 return -ENOMEM; 32 } 33 } 34 35 if (!repeat) 36 repeat = 1; 37 38 rcu_read_lock(); 39 preempt_disable(); 40 time_start = ktime_get_ns(); 41 for (i = 0; i < repeat; i++) { 42 bpf_cgroup_storage_set(storage); 43 *retval = BPF_PROG_RUN(prog, ctx); 44 45 if (signal_pending(current)) { 46 ret = -EINTR; 47 break; 48 } 49 50 if (need_resched()) { 51 time_spent += ktime_get_ns() - time_start; 52 preempt_enable(); 53 rcu_read_unlock(); 54 55 cond_resched(); 56 57 rcu_read_lock(); 58 preempt_disable(); 59 time_start = ktime_get_ns(); 60 } 61 } 62 time_spent += ktime_get_ns() - time_start; 63 preempt_enable(); 64 rcu_read_unlock(); 65 66 do_div(time_spent, repeat); 67 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 68 69 for_each_cgroup_storage_type(stype) 70 bpf_cgroup_storage_free(storage[stype]); 71 72 return ret; 73 } 74 75 static int bpf_test_finish(const union bpf_attr *kattr, 76 union bpf_attr __user *uattr, const void *data, 77 u32 size, u32 retval, u32 duration) 78 { 79 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 80 int err = -EFAULT; 81 u32 copy_size = size; 82 83 /* Clamp copy if the user has provided a size hint, but copy the full 84 * buffer if not to retain old behaviour. 85 */ 86 if (kattr->test.data_size_out && 87 copy_size > kattr->test.data_size_out) { 88 copy_size = kattr->test.data_size_out; 89 err = -ENOSPC; 90 } 91 92 if (data_out && copy_to_user(data_out, data, copy_size)) 93 goto out; 94 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 95 goto out; 96 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 97 goto out; 98 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 99 goto out; 100 if (err != -ENOSPC) 101 err = 0; 102 out: 103 return err; 104 } 105 106 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 107 u32 headroom, u32 tailroom) 108 { 109 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 110 void *data; 111 112 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 113 return ERR_PTR(-EINVAL); 114 115 data = kzalloc(size + headroom + tailroom, GFP_USER); 116 if (!data) 117 return ERR_PTR(-ENOMEM); 118 119 if (copy_from_user(data + headroom, data_in, size)) { 120 kfree(data); 121 return ERR_PTR(-EFAULT); 122 } 123 return data; 124 } 125 126 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 127 { 128 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 129 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 130 u32 size = kattr->test.ctx_size_in; 131 void *data; 132 int err; 133 134 if (!data_in && !data_out) 135 return NULL; 136 137 data = kzalloc(max_size, GFP_USER); 138 if (!data) 139 return ERR_PTR(-ENOMEM); 140 141 if (data_in) { 142 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 143 if (err) { 144 kfree(data); 145 return ERR_PTR(err); 146 } 147 148 size = min_t(u32, max_size, size); 149 if (copy_from_user(data, data_in, size)) { 150 kfree(data); 151 return ERR_PTR(-EFAULT); 152 } 153 } 154 return data; 155 } 156 157 static int bpf_ctx_finish(const union bpf_attr *kattr, 158 union bpf_attr __user *uattr, const void *data, 159 u32 size) 160 { 161 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 162 int err = -EFAULT; 163 u32 copy_size = size; 164 165 if (!data || !data_out) 166 return 0; 167 168 if (copy_size > kattr->test.ctx_size_out) { 169 copy_size = kattr->test.ctx_size_out; 170 err = -ENOSPC; 171 } 172 173 if (copy_to_user(data_out, data, copy_size)) 174 goto out; 175 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 176 goto out; 177 if (err != -ENOSPC) 178 err = 0; 179 out: 180 return err; 181 } 182 183 /** 184 * range_is_zero - test whether buffer is initialized 185 * @buf: buffer to check 186 * @from: check from this position 187 * @to: check up until (excluding) this position 188 * 189 * This function returns true if the there is a non-zero byte 190 * in the buf in the range [from,to). 191 */ 192 static inline bool range_is_zero(void *buf, size_t from, size_t to) 193 { 194 return !memchr_inv((u8 *)buf + from, 0, to - from); 195 } 196 197 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 198 { 199 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 200 201 if (!__skb) 202 return 0; 203 204 /* make sure the fields we don't use are zeroed */ 205 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority))) 206 return -EINVAL; 207 208 /* priority is allowed */ 209 210 if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) + 211 FIELD_SIZEOF(struct __sk_buff, priority), 212 offsetof(struct __sk_buff, cb))) 213 return -EINVAL; 214 215 /* cb is allowed */ 216 217 if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) + 218 FIELD_SIZEOF(struct __sk_buff, cb), 219 sizeof(struct __sk_buff))) 220 return -EINVAL; 221 222 skb->priority = __skb->priority; 223 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 224 225 return 0; 226 } 227 228 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 229 { 230 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 231 232 if (!__skb) 233 return; 234 235 __skb->priority = skb->priority; 236 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 237 } 238 239 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 240 union bpf_attr __user *uattr) 241 { 242 bool is_l2 = false, is_direct_pkt_access = false; 243 u32 size = kattr->test.data_size_in; 244 u32 repeat = kattr->test.repeat; 245 struct __sk_buff *ctx = NULL; 246 u32 retval, duration; 247 int hh_len = ETH_HLEN; 248 struct sk_buff *skb; 249 struct sock *sk; 250 void *data; 251 int ret; 252 253 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 254 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 255 if (IS_ERR(data)) 256 return PTR_ERR(data); 257 258 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 259 if (IS_ERR(ctx)) { 260 kfree(data); 261 return PTR_ERR(ctx); 262 } 263 264 switch (prog->type) { 265 case BPF_PROG_TYPE_SCHED_CLS: 266 case BPF_PROG_TYPE_SCHED_ACT: 267 is_l2 = true; 268 /* fall through */ 269 case BPF_PROG_TYPE_LWT_IN: 270 case BPF_PROG_TYPE_LWT_OUT: 271 case BPF_PROG_TYPE_LWT_XMIT: 272 is_direct_pkt_access = true; 273 break; 274 default: 275 break; 276 } 277 278 sk = kzalloc(sizeof(struct sock), GFP_USER); 279 if (!sk) { 280 kfree(data); 281 kfree(ctx); 282 return -ENOMEM; 283 } 284 sock_net_set(sk, current->nsproxy->net_ns); 285 sock_init_data(NULL, sk); 286 287 skb = build_skb(data, 0); 288 if (!skb) { 289 kfree(data); 290 kfree(ctx); 291 kfree(sk); 292 return -ENOMEM; 293 } 294 skb->sk = sk; 295 296 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 297 __skb_put(skb, size); 298 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 299 skb_reset_network_header(skb); 300 301 if (is_l2) 302 __skb_push(skb, hh_len); 303 if (is_direct_pkt_access) 304 bpf_compute_data_pointers(skb); 305 ret = convert___skb_to_skb(skb, ctx); 306 if (ret) 307 goto out; 308 ret = bpf_test_run(prog, skb, repeat, &retval, &duration); 309 if (ret) 310 goto out; 311 if (!is_l2) { 312 if (skb_headroom(skb) < hh_len) { 313 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 314 315 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 316 ret = -ENOMEM; 317 goto out; 318 } 319 } 320 memset(__skb_push(skb, hh_len), 0, hh_len); 321 } 322 convert_skb_to___skb(skb, ctx); 323 324 size = skb->len; 325 /* bpf program can never convert linear skb to non-linear */ 326 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 327 size = skb_headlen(skb); 328 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 329 if (!ret) 330 ret = bpf_ctx_finish(kattr, uattr, ctx, 331 sizeof(struct __sk_buff)); 332 out: 333 kfree_skb(skb); 334 kfree(sk); 335 kfree(ctx); 336 return ret; 337 } 338 339 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 340 union bpf_attr __user *uattr) 341 { 342 u32 size = kattr->test.data_size_in; 343 u32 repeat = kattr->test.repeat; 344 struct netdev_rx_queue *rxqueue; 345 struct xdp_buff xdp = {}; 346 u32 retval, duration; 347 void *data; 348 int ret; 349 350 if (kattr->test.ctx_in || kattr->test.ctx_out) 351 return -EINVAL; 352 353 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); 354 if (IS_ERR(data)) 355 return PTR_ERR(data); 356 357 xdp.data_hard_start = data; 358 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; 359 xdp.data_meta = xdp.data; 360 xdp.data_end = xdp.data + size; 361 362 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 363 xdp.rxq = &rxqueue->xdp_rxq; 364 365 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration); 366 if (ret) 367 goto out; 368 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || 369 xdp.data_end != xdp.data + size) 370 size = xdp.data_end - xdp.data; 371 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 372 out: 373 kfree(data); 374 return ret; 375 } 376 377 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 378 const union bpf_attr *kattr, 379 union bpf_attr __user *uattr) 380 { 381 u32 size = kattr->test.data_size_in; 382 u32 repeat = kattr->test.repeat; 383 struct bpf_flow_keys flow_keys; 384 u64 time_start, time_spent = 0; 385 struct bpf_skb_data_end *cb; 386 u32 retval, duration; 387 struct sk_buff *skb; 388 struct sock *sk; 389 void *data; 390 int ret; 391 u32 i; 392 393 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 394 return -EINVAL; 395 396 if (kattr->test.ctx_in || kattr->test.ctx_out) 397 return -EINVAL; 398 399 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 400 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 401 if (IS_ERR(data)) 402 return PTR_ERR(data); 403 404 sk = kzalloc(sizeof(*sk), GFP_USER); 405 if (!sk) { 406 kfree(data); 407 return -ENOMEM; 408 } 409 sock_net_set(sk, current->nsproxy->net_ns); 410 sock_init_data(NULL, sk); 411 412 skb = build_skb(data, 0); 413 if (!skb) { 414 kfree(data); 415 kfree(sk); 416 return -ENOMEM; 417 } 418 skb->sk = sk; 419 420 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 421 __skb_put(skb, size); 422 skb->protocol = eth_type_trans(skb, 423 current->nsproxy->net_ns->loopback_dev); 424 skb_reset_network_header(skb); 425 426 cb = (struct bpf_skb_data_end *)skb->cb; 427 cb->qdisc_cb.flow_keys = &flow_keys; 428 429 if (!repeat) 430 repeat = 1; 431 432 rcu_read_lock(); 433 preempt_disable(); 434 time_start = ktime_get_ns(); 435 for (i = 0; i < repeat; i++) { 436 retval = __skb_flow_bpf_dissect(prog, skb, 437 &flow_keys_dissector, 438 &flow_keys); 439 440 if (signal_pending(current)) { 441 preempt_enable(); 442 rcu_read_unlock(); 443 444 ret = -EINTR; 445 goto out; 446 } 447 448 if (need_resched()) { 449 time_spent += ktime_get_ns() - time_start; 450 preempt_enable(); 451 rcu_read_unlock(); 452 453 cond_resched(); 454 455 rcu_read_lock(); 456 preempt_disable(); 457 time_start = ktime_get_ns(); 458 } 459 } 460 time_spent += ktime_get_ns() - time_start; 461 preempt_enable(); 462 rcu_read_unlock(); 463 464 do_div(time_spent, repeat); 465 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 466 467 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 468 retval, duration); 469 470 out: 471 kfree_skb(skb); 472 kfree(sk); 473 return ret; 474 } 475