1 /* Copyright (c) 2017 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #include <linux/bpf.h> 8 #include <linux/slab.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/sched/signal.h> 13 14 static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx) 15 { 16 u32 ret; 17 18 preempt_disable(); 19 rcu_read_lock(); 20 ret = BPF_PROG_RUN(prog, ctx); 21 rcu_read_unlock(); 22 preempt_enable(); 23 24 return ret; 25 } 26 27 static u32 bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *time) 28 { 29 u64 time_start, time_spent = 0; 30 u32 ret = 0, i; 31 32 if (!repeat) 33 repeat = 1; 34 time_start = ktime_get_ns(); 35 for (i = 0; i < repeat; i++) { 36 ret = bpf_test_run_one(prog, ctx); 37 if (need_resched()) { 38 if (signal_pending(current)) 39 break; 40 time_spent += ktime_get_ns() - time_start; 41 cond_resched(); 42 time_start = ktime_get_ns(); 43 } 44 } 45 time_spent += ktime_get_ns() - time_start; 46 do_div(time_spent, repeat); 47 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 48 49 return ret; 50 } 51 52 static int bpf_test_finish(const union bpf_attr *kattr, 53 union bpf_attr __user *uattr, const void *data, 54 u32 size, u32 retval, u32 duration) 55 { 56 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 57 int err = -EFAULT; 58 59 if (data_out && copy_to_user(data_out, data, size)) 60 goto out; 61 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 62 goto out; 63 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 64 goto out; 65 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 66 goto out; 67 err = 0; 68 out: 69 return err; 70 } 71 72 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 73 u32 headroom, u32 tailroom) 74 { 75 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 76 void *data; 77 78 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 79 return ERR_PTR(-EINVAL); 80 81 data = kzalloc(size + headroom + tailroom, GFP_USER); 82 if (!data) 83 return ERR_PTR(-ENOMEM); 84 85 if (copy_from_user(data + headroom, data_in, size)) { 86 kfree(data); 87 return ERR_PTR(-EFAULT); 88 } 89 return data; 90 } 91 92 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 93 union bpf_attr __user *uattr) 94 { 95 bool is_l2 = false, is_direct_pkt_access = false; 96 u32 size = kattr->test.data_size_in; 97 u32 repeat = kattr->test.repeat; 98 u32 retval, duration; 99 struct sk_buff *skb; 100 void *data; 101 int ret; 102 103 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 104 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 105 if (IS_ERR(data)) 106 return PTR_ERR(data); 107 108 switch (prog->type) { 109 case BPF_PROG_TYPE_SCHED_CLS: 110 case BPF_PROG_TYPE_SCHED_ACT: 111 is_l2 = true; 112 /* fall through */ 113 case BPF_PROG_TYPE_LWT_IN: 114 case BPF_PROG_TYPE_LWT_OUT: 115 case BPF_PROG_TYPE_LWT_XMIT: 116 is_direct_pkt_access = true; 117 break; 118 default: 119 break; 120 } 121 122 skb = build_skb(data, 0); 123 if (!skb) { 124 kfree(data); 125 return -ENOMEM; 126 } 127 128 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 129 __skb_put(skb, size); 130 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 131 skb_reset_network_header(skb); 132 133 if (is_l2) 134 __skb_push(skb, ETH_HLEN); 135 if (is_direct_pkt_access) 136 bpf_compute_data_pointers(skb); 137 retval = bpf_test_run(prog, skb, repeat, &duration); 138 if (!is_l2) 139 __skb_push(skb, ETH_HLEN); 140 size = skb->len; 141 /* bpf program can never convert linear skb to non-linear */ 142 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 143 size = skb_headlen(skb); 144 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 145 kfree_skb(skb); 146 return ret; 147 } 148 149 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 150 union bpf_attr __user *uattr) 151 { 152 u32 size = kattr->test.data_size_in; 153 u32 repeat = kattr->test.repeat; 154 struct netdev_rx_queue *rxqueue; 155 struct xdp_buff xdp = {}; 156 u32 retval, duration; 157 void *data; 158 int ret; 159 160 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); 161 if (IS_ERR(data)) 162 return PTR_ERR(data); 163 164 xdp.data_hard_start = data; 165 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; 166 xdp.data_meta = xdp.data; 167 xdp.data_end = xdp.data + size; 168 169 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 170 xdp.rxq = &rxqueue->xdp_rxq; 171 172 retval = bpf_test_run(prog, &xdp, repeat, &duration); 173 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || 174 xdp.data_end != xdp.data + size) 175 size = xdp.data_end - xdp.data; 176 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 177 kfree(data); 178 return ret; 179 } 180