1 /* Copyright (c) 2017 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #include <linux/bpf.h> 8 #include <linux/slab.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/sched/signal.h> 13 #include <net/sock.h> 14 #include <net/tcp.h> 15 16 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 17 u32 *retval, u32 *time) 18 { 19 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 20 enum bpf_cgroup_storage_type stype; 21 u64 time_start, time_spent = 0; 22 int ret = 0; 23 u32 i; 24 25 for_each_cgroup_storage_type(stype) { 26 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 27 if (IS_ERR(storage[stype])) { 28 storage[stype] = NULL; 29 for_each_cgroup_storage_type(stype) 30 bpf_cgroup_storage_free(storage[stype]); 31 return -ENOMEM; 32 } 33 } 34 35 if (!repeat) 36 repeat = 1; 37 38 rcu_read_lock(); 39 preempt_disable(); 40 time_start = ktime_get_ns(); 41 for (i = 0; i < repeat; i++) { 42 bpf_cgroup_storage_set(storage); 43 *retval = BPF_PROG_RUN(prog, ctx); 44 45 if (signal_pending(current)) { 46 ret = -EINTR; 47 break; 48 } 49 50 if (need_resched()) { 51 time_spent += ktime_get_ns() - time_start; 52 preempt_enable(); 53 rcu_read_unlock(); 54 55 cond_resched(); 56 57 rcu_read_lock(); 58 preempt_disable(); 59 time_start = ktime_get_ns(); 60 } 61 } 62 time_spent += ktime_get_ns() - time_start; 63 preempt_enable(); 64 rcu_read_unlock(); 65 66 do_div(time_spent, repeat); 67 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 68 69 for_each_cgroup_storage_type(stype) 70 bpf_cgroup_storage_free(storage[stype]); 71 72 return ret; 73 } 74 75 static int bpf_test_finish(const union bpf_attr *kattr, 76 union bpf_attr __user *uattr, const void *data, 77 u32 size, u32 retval, u32 duration) 78 { 79 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 80 int err = -EFAULT; 81 u32 copy_size = size; 82 83 /* Clamp copy if the user has provided a size hint, but copy the full 84 * buffer if not to retain old behaviour. 85 */ 86 if (kattr->test.data_size_out && 87 copy_size > kattr->test.data_size_out) { 88 copy_size = kattr->test.data_size_out; 89 err = -ENOSPC; 90 } 91 92 if (data_out && copy_to_user(data_out, data, copy_size)) 93 goto out; 94 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 95 goto out; 96 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 97 goto out; 98 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 99 goto out; 100 if (err != -ENOSPC) 101 err = 0; 102 out: 103 return err; 104 } 105 106 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 107 u32 headroom, u32 tailroom) 108 { 109 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 110 void *data; 111 112 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 113 return ERR_PTR(-EINVAL); 114 115 data = kzalloc(size + headroom + tailroom, GFP_USER); 116 if (!data) 117 return ERR_PTR(-ENOMEM); 118 119 if (copy_from_user(data + headroom, data_in, size)) { 120 kfree(data); 121 return ERR_PTR(-EFAULT); 122 } 123 return data; 124 } 125 126 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 127 union bpf_attr __user *uattr) 128 { 129 bool is_l2 = false, is_direct_pkt_access = false; 130 u32 size = kattr->test.data_size_in; 131 u32 repeat = kattr->test.repeat; 132 u32 retval, duration; 133 int hh_len = ETH_HLEN; 134 struct sk_buff *skb; 135 struct sock *sk; 136 void *data; 137 int ret; 138 139 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 140 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 141 if (IS_ERR(data)) 142 return PTR_ERR(data); 143 144 switch (prog->type) { 145 case BPF_PROG_TYPE_SCHED_CLS: 146 case BPF_PROG_TYPE_SCHED_ACT: 147 is_l2 = true; 148 /* fall through */ 149 case BPF_PROG_TYPE_LWT_IN: 150 case BPF_PROG_TYPE_LWT_OUT: 151 case BPF_PROG_TYPE_LWT_XMIT: 152 is_direct_pkt_access = true; 153 break; 154 default: 155 break; 156 } 157 158 sk = kzalloc(sizeof(struct sock), GFP_USER); 159 if (!sk) { 160 kfree(data); 161 return -ENOMEM; 162 } 163 sock_net_set(sk, current->nsproxy->net_ns); 164 sock_init_data(NULL, sk); 165 166 skb = build_skb(data, 0); 167 if (!skb) { 168 kfree(data); 169 kfree(sk); 170 return -ENOMEM; 171 } 172 skb->sk = sk; 173 174 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 175 __skb_put(skb, size); 176 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 177 skb_reset_network_header(skb); 178 179 if (is_l2) 180 __skb_push(skb, hh_len); 181 if (is_direct_pkt_access) 182 bpf_compute_data_pointers(skb); 183 ret = bpf_test_run(prog, skb, repeat, &retval, &duration); 184 if (ret) { 185 kfree_skb(skb); 186 kfree(sk); 187 return ret; 188 } 189 if (!is_l2) { 190 if (skb_headroom(skb) < hh_len) { 191 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 192 193 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 194 kfree_skb(skb); 195 kfree(sk); 196 return -ENOMEM; 197 } 198 } 199 memset(__skb_push(skb, hh_len), 0, hh_len); 200 } 201 202 size = skb->len; 203 /* bpf program can never convert linear skb to non-linear */ 204 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 205 size = skb_headlen(skb); 206 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 207 kfree_skb(skb); 208 kfree(sk); 209 return ret; 210 } 211 212 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 213 union bpf_attr __user *uattr) 214 { 215 u32 size = kattr->test.data_size_in; 216 u32 repeat = kattr->test.repeat; 217 struct netdev_rx_queue *rxqueue; 218 struct xdp_buff xdp = {}; 219 u32 retval, duration; 220 void *data; 221 int ret; 222 223 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); 224 if (IS_ERR(data)) 225 return PTR_ERR(data); 226 227 xdp.data_hard_start = data; 228 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; 229 xdp.data_meta = xdp.data; 230 xdp.data_end = xdp.data + size; 231 232 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 233 xdp.rxq = &rxqueue->xdp_rxq; 234 235 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration); 236 if (ret) 237 goto out; 238 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || 239 xdp.data_end != xdp.data + size) 240 size = xdp.data_end - xdp.data; 241 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 242 out: 243 kfree(data); 244 return ret; 245 } 246 247 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 248 const union bpf_attr *kattr, 249 union bpf_attr __user *uattr) 250 { 251 u32 size = kattr->test.data_size_in; 252 u32 repeat = kattr->test.repeat; 253 struct bpf_flow_keys flow_keys; 254 u64 time_start, time_spent = 0; 255 struct bpf_skb_data_end *cb; 256 u32 retval, duration; 257 struct sk_buff *skb; 258 struct sock *sk; 259 void *data; 260 int ret; 261 u32 i; 262 263 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 264 return -EINVAL; 265 266 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 267 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 268 if (IS_ERR(data)) 269 return PTR_ERR(data); 270 271 sk = kzalloc(sizeof(*sk), GFP_USER); 272 if (!sk) { 273 kfree(data); 274 return -ENOMEM; 275 } 276 sock_net_set(sk, current->nsproxy->net_ns); 277 sock_init_data(NULL, sk); 278 279 skb = build_skb(data, 0); 280 if (!skb) { 281 kfree(data); 282 kfree(sk); 283 return -ENOMEM; 284 } 285 skb->sk = sk; 286 287 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 288 __skb_put(skb, size); 289 skb->protocol = eth_type_trans(skb, 290 current->nsproxy->net_ns->loopback_dev); 291 skb_reset_network_header(skb); 292 293 cb = (struct bpf_skb_data_end *)skb->cb; 294 cb->qdisc_cb.flow_keys = &flow_keys; 295 296 if (!repeat) 297 repeat = 1; 298 299 rcu_read_lock(); 300 preempt_disable(); 301 time_start = ktime_get_ns(); 302 for (i = 0; i < repeat; i++) { 303 retval = __skb_flow_bpf_dissect(prog, skb, 304 &flow_keys_dissector, 305 &flow_keys); 306 307 if (signal_pending(current)) { 308 preempt_enable(); 309 rcu_read_unlock(); 310 311 ret = -EINTR; 312 goto out; 313 } 314 315 if (need_resched()) { 316 time_spent += ktime_get_ns() - time_start; 317 preempt_enable(); 318 rcu_read_unlock(); 319 320 cond_resched(); 321 322 rcu_read_lock(); 323 preempt_disable(); 324 time_start = ktime_get_ns(); 325 } 326 } 327 time_spent += ktime_get_ns() - time_start; 328 preempt_enable(); 329 rcu_read_unlock(); 330 331 do_div(time_spent, repeat); 332 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 333 334 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 335 retval, duration); 336 337 out: 338 kfree_skb(skb); 339 kfree(sk); 340 return ret; 341 } 342