1 /* 2 * Linux Socket Filter - Kernel level socket filtering 3 * 4 * Based on the design of the Berkeley Packet Filter. The new 5 * internal format has been designed by PLUMgrid: 6 * 7 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com 8 * 9 * Authors: 10 * 11 * Jay Schulist <jschlst@samba.org> 12 * Alexei Starovoitov <ast@plumgrid.com> 13 * Daniel Borkmann <dborkman@redhat.com> 14 * 15 * This program is free software; you can redistribute it and/or 16 * modify it under the terms of the GNU General Public License 17 * as published by the Free Software Foundation; either version 18 * 2 of the License, or (at your option) any later version. 19 * 20 * Andi Kleen - Fix a few bad bugs and races. 21 * Kris Katterjohn - Added many additional checks in bpf_check_classic() 22 */ 23 24 #include <linux/module.h> 25 #include <linux/types.h> 26 #include <linux/mm.h> 27 #include <linux/fcntl.h> 28 #include <linux/socket.h> 29 #include <linux/in.h> 30 #include <linux/inet.h> 31 #include <linux/netdevice.h> 32 #include <linux/if_packet.h> 33 #include <linux/gfp.h> 34 #include <net/ip.h> 35 #include <net/protocol.h> 36 #include <net/netlink.h> 37 #include <linux/skbuff.h> 38 #include <net/sock.h> 39 #include <net/flow_dissector.h> 40 #include <linux/errno.h> 41 #include <linux/timer.h> 42 #include <asm/uaccess.h> 43 #include <asm/unaligned.h> 44 #include <linux/filter.h> 45 #include <linux/ratelimit.h> 46 #include <linux/seccomp.h> 47 #include <linux/if_vlan.h> 48 #include <linux/bpf.h> 49 #include <net/sch_generic.h> 50 #include <net/cls_cgroup.h> 51 #include <net/dst_metadata.h> 52 #include <net/dst.h> 53 54 /** 55 * sk_filter - run a packet through a socket filter 56 * @sk: sock associated with &sk_buff 57 * @skb: buffer to filter 58 * 59 * Run the eBPF program and then cut skb->data to correct size returned by 60 * the program. If pkt_len is 0 we toss packet. If skb->len is smaller 61 * than pkt_len we keep whole skb->data. This is the socket level 62 * wrapper to BPF_PROG_RUN. It returns 0 if the packet should 63 * be accepted or -EPERM if the packet should be tossed. 64 * 65 */ 66 int sk_filter(struct sock *sk, struct sk_buff *skb) 67 { 68 int err; 69 struct sk_filter *filter; 70 71 /* 72 * If the skb was allocated from pfmemalloc reserves, only 73 * allow SOCK_MEMALLOC sockets to use it as this socket is 74 * helping free memory 75 */ 76 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC)) 77 return -ENOMEM; 78 79 err = security_sock_rcv_skb(sk, skb); 80 if (err) 81 return err; 82 83 rcu_read_lock(); 84 filter = rcu_dereference(sk->sk_filter); 85 if (filter) { 86 unsigned int pkt_len = bpf_prog_run_save_cb(filter->prog, skb); 87 88 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM; 89 } 90 rcu_read_unlock(); 91 92 return err; 93 } 94 EXPORT_SYMBOL(sk_filter); 95 96 static u64 __skb_get_pay_offset(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) 97 { 98 return skb_get_poff((struct sk_buff *)(unsigned long) ctx); 99 } 100 101 static u64 __skb_get_nlattr(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) 102 { 103 struct sk_buff *skb = (struct sk_buff *)(unsigned long) ctx; 104 struct nlattr *nla; 105 106 if (skb_is_nonlinear(skb)) 107 return 0; 108 109 if (skb->len < sizeof(struct nlattr)) 110 return 0; 111 112 if (a > skb->len - sizeof(struct nlattr)) 113 return 0; 114 115 nla = nla_find((struct nlattr *) &skb->data[a], skb->len - a, x); 116 if (nla) 117 return (void *) nla - (void *) skb->data; 118 119 return 0; 120 } 121 122 static u64 __skb_get_nlattr_nest(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) 123 { 124 struct sk_buff *skb = (struct sk_buff *)(unsigned long) ctx; 125 struct nlattr *nla; 126 127 if (skb_is_nonlinear(skb)) 128 return 0; 129 130 if (skb->len < sizeof(struct nlattr)) 131 return 0; 132 133 if (a > skb->len - sizeof(struct nlattr)) 134 return 0; 135 136 nla = (struct nlattr *) &skb->data[a]; 137 if (nla->nla_len > skb->len - a) 138 return 0; 139 140 nla = nla_find_nested(nla, x); 141 if (nla) 142 return (void *) nla - (void *) skb->data; 143 144 return 0; 145 } 146 147 static u64 __get_raw_cpu_id(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) 148 { 149 return raw_smp_processor_id(); 150 } 151 152 static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, 153 struct bpf_insn *insn_buf) 154 { 155 struct bpf_insn *insn = insn_buf; 156 157 switch (skb_field) { 158 case SKF_AD_MARK: 159 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 160 161 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, 162 offsetof(struct sk_buff, mark)); 163 break; 164 165 case SKF_AD_PKTTYPE: 166 *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET()); 167 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX); 168 #ifdef __BIG_ENDIAN_BITFIELD 169 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5); 170 #endif 171 break; 172 173 case SKF_AD_QUEUE: 174 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); 175 176 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, 177 offsetof(struct sk_buff, queue_mapping)); 178 break; 179 180 case SKF_AD_VLAN_TAG: 181 case SKF_AD_VLAN_TAG_PRESENT: 182 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); 183 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); 184 185 /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */ 186 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, 187 offsetof(struct sk_buff, vlan_tci)); 188 if (skb_field == SKF_AD_VLAN_TAG) { 189 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, 190 ~VLAN_TAG_PRESENT); 191 } else { 192 /* dst_reg >>= 12 */ 193 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 12); 194 /* dst_reg &= 1 */ 195 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, 1); 196 } 197 break; 198 } 199 200 return insn - insn_buf; 201 } 202 203 static bool convert_bpf_extensions(struct sock_filter *fp, 204 struct bpf_insn **insnp) 205 { 206 struct bpf_insn *insn = *insnp; 207 u32 cnt; 208 209 switch (fp->k) { 210 case SKF_AD_OFF + SKF_AD_PROTOCOL: 211 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 212 213 /* A = *(u16 *) (CTX + offsetof(protocol)) */ 214 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX, 215 offsetof(struct sk_buff, protocol)); 216 /* A = ntohs(A) [emitting a nop or swap16] */ 217 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16); 218 break; 219 220 case SKF_AD_OFF + SKF_AD_PKTTYPE: 221 cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn); 222 insn += cnt - 1; 223 break; 224 225 case SKF_AD_OFF + SKF_AD_IFINDEX: 226 case SKF_AD_OFF + SKF_AD_HATYPE: 227 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 228 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); 229 BUILD_BUG_ON(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)) < 0); 230 231 *insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)), 232 BPF_REG_TMP, BPF_REG_CTX, 233 offsetof(struct sk_buff, dev)); 234 /* if (tmp != 0) goto pc + 1 */ 235 *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1); 236 *insn++ = BPF_EXIT_INSN(); 237 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX) 238 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP, 239 offsetof(struct net_device, ifindex)); 240 else 241 *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP, 242 offsetof(struct net_device, type)); 243 break; 244 245 case SKF_AD_OFF + SKF_AD_MARK: 246 cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn); 247 insn += cnt - 1; 248 break; 249 250 case SKF_AD_OFF + SKF_AD_RXHASH: 251 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); 252 253 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, 254 offsetof(struct sk_buff, hash)); 255 break; 256 257 case SKF_AD_OFF + SKF_AD_QUEUE: 258 cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn); 259 insn += cnt - 1; 260 break; 261 262 case SKF_AD_OFF + SKF_AD_VLAN_TAG: 263 cnt = convert_skb_access(SKF_AD_VLAN_TAG, 264 BPF_REG_A, BPF_REG_CTX, insn); 265 insn += cnt - 1; 266 break; 267 268 case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT: 269 cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT, 270 BPF_REG_A, BPF_REG_CTX, insn); 271 insn += cnt - 1; 272 break; 273 274 case SKF_AD_OFF + SKF_AD_VLAN_TPID: 275 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_proto) != 2); 276 277 /* A = *(u16 *) (CTX + offsetof(vlan_proto)) */ 278 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX, 279 offsetof(struct sk_buff, vlan_proto)); 280 /* A = ntohs(A) [emitting a nop or swap16] */ 281 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16); 282 break; 283 284 case SKF_AD_OFF + SKF_AD_PAY_OFFSET: 285 case SKF_AD_OFF + SKF_AD_NLATTR: 286 case SKF_AD_OFF + SKF_AD_NLATTR_NEST: 287 case SKF_AD_OFF + SKF_AD_CPU: 288 case SKF_AD_OFF + SKF_AD_RANDOM: 289 /* arg1 = CTX */ 290 *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX); 291 /* arg2 = A */ 292 *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A); 293 /* arg3 = X */ 294 *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X); 295 /* Emit call(arg1=CTX, arg2=A, arg3=X) */ 296 switch (fp->k) { 297 case SKF_AD_OFF + SKF_AD_PAY_OFFSET: 298 *insn = BPF_EMIT_CALL(__skb_get_pay_offset); 299 break; 300 case SKF_AD_OFF + SKF_AD_NLATTR: 301 *insn = BPF_EMIT_CALL(__skb_get_nlattr); 302 break; 303 case SKF_AD_OFF + SKF_AD_NLATTR_NEST: 304 *insn = BPF_EMIT_CALL(__skb_get_nlattr_nest); 305 break; 306 case SKF_AD_OFF + SKF_AD_CPU: 307 *insn = BPF_EMIT_CALL(__get_raw_cpu_id); 308 break; 309 case SKF_AD_OFF + SKF_AD_RANDOM: 310 *insn = BPF_EMIT_CALL(bpf_user_rnd_u32); 311 bpf_user_rnd_init_once(); 312 break; 313 } 314 break; 315 316 case SKF_AD_OFF + SKF_AD_ALU_XOR_X: 317 /* A ^= X */ 318 *insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X); 319 break; 320 321 default: 322 /* This is just a dummy call to avoid letting the compiler 323 * evict __bpf_call_base() as an optimization. Placed here 324 * where no-one bothers. 325 */ 326 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0); 327 return false; 328 } 329 330 *insnp = insn; 331 return true; 332 } 333 334 /** 335 * bpf_convert_filter - convert filter program 336 * @prog: the user passed filter program 337 * @len: the length of the user passed filter program 338 * @new_prog: buffer where converted program will be stored 339 * @new_len: pointer to store length of converted program 340 * 341 * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style. 342 * Conversion workflow: 343 * 344 * 1) First pass for calculating the new program length: 345 * bpf_convert_filter(old_prog, old_len, NULL, &new_len) 346 * 347 * 2) 2nd pass to remap in two passes: 1st pass finds new 348 * jump offsets, 2nd pass remapping: 349 * new_prog = kmalloc(sizeof(struct bpf_insn) * new_len); 350 * bpf_convert_filter(old_prog, old_len, new_prog, &new_len); 351 * 352 * User BPF's register A is mapped to our BPF register 6, user BPF 353 * register X is mapped to BPF register 7; frame pointer is always 354 * register 10; Context 'void *ctx' is stored in register 1, that is, 355 * for socket filters: ctx == 'struct sk_buff *', for seccomp: 356 * ctx == 'struct seccomp_data *'. 357 */ 358 static int bpf_convert_filter(struct sock_filter *prog, int len, 359 struct bpf_insn *new_prog, int *new_len) 360 { 361 int new_flen = 0, pass = 0, target, i; 362 struct bpf_insn *new_insn; 363 struct sock_filter *fp; 364 int *addrs = NULL; 365 u8 bpf_src; 366 367 BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK); 368 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 369 370 if (len <= 0 || len > BPF_MAXINSNS) 371 return -EINVAL; 372 373 if (new_prog) { 374 addrs = kcalloc(len, sizeof(*addrs), 375 GFP_KERNEL | __GFP_NOWARN); 376 if (!addrs) 377 return -ENOMEM; 378 } 379 380 do_pass: 381 new_insn = new_prog; 382 fp = prog; 383 384 if (new_insn) 385 *new_insn = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1); 386 new_insn++; 387 388 for (i = 0; i < len; fp++, i++) { 389 struct bpf_insn tmp_insns[6] = { }; 390 struct bpf_insn *insn = tmp_insns; 391 392 if (addrs) 393 addrs[i] = new_insn - new_prog; 394 395 switch (fp->code) { 396 /* All arithmetic insns and skb loads map as-is. */ 397 case BPF_ALU | BPF_ADD | BPF_X: 398 case BPF_ALU | BPF_ADD | BPF_K: 399 case BPF_ALU | BPF_SUB | BPF_X: 400 case BPF_ALU | BPF_SUB | BPF_K: 401 case BPF_ALU | BPF_AND | BPF_X: 402 case BPF_ALU | BPF_AND | BPF_K: 403 case BPF_ALU | BPF_OR | BPF_X: 404 case BPF_ALU | BPF_OR | BPF_K: 405 case BPF_ALU | BPF_LSH | BPF_X: 406 case BPF_ALU | BPF_LSH | BPF_K: 407 case BPF_ALU | BPF_RSH | BPF_X: 408 case BPF_ALU | BPF_RSH | BPF_K: 409 case BPF_ALU | BPF_XOR | BPF_X: 410 case BPF_ALU | BPF_XOR | BPF_K: 411 case BPF_ALU | BPF_MUL | BPF_X: 412 case BPF_ALU | BPF_MUL | BPF_K: 413 case BPF_ALU | BPF_DIV | BPF_X: 414 case BPF_ALU | BPF_DIV | BPF_K: 415 case BPF_ALU | BPF_MOD | BPF_X: 416 case BPF_ALU | BPF_MOD | BPF_K: 417 case BPF_ALU | BPF_NEG: 418 case BPF_LD | BPF_ABS | BPF_W: 419 case BPF_LD | BPF_ABS | BPF_H: 420 case BPF_LD | BPF_ABS | BPF_B: 421 case BPF_LD | BPF_IND | BPF_W: 422 case BPF_LD | BPF_IND | BPF_H: 423 case BPF_LD | BPF_IND | BPF_B: 424 /* Check for overloaded BPF extension and 425 * directly convert it if found, otherwise 426 * just move on with mapping. 427 */ 428 if (BPF_CLASS(fp->code) == BPF_LD && 429 BPF_MODE(fp->code) == BPF_ABS && 430 convert_bpf_extensions(fp, &insn)) 431 break; 432 433 *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k); 434 break; 435 436 /* Jump transformation cannot use BPF block macros 437 * everywhere as offset calculation and target updates 438 * require a bit more work than the rest, i.e. jump 439 * opcodes map as-is, but offsets need adjustment. 440 */ 441 442 #define BPF_EMIT_JMP \ 443 do { \ 444 if (target >= len || target < 0) \ 445 goto err; \ 446 insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0; \ 447 /* Adjust pc relative offset for 2nd or 3rd insn. */ \ 448 insn->off -= insn - tmp_insns; \ 449 } while (0) 450 451 case BPF_JMP | BPF_JA: 452 target = i + fp->k + 1; 453 insn->code = fp->code; 454 BPF_EMIT_JMP; 455 break; 456 457 case BPF_JMP | BPF_JEQ | BPF_K: 458 case BPF_JMP | BPF_JEQ | BPF_X: 459 case BPF_JMP | BPF_JSET | BPF_K: 460 case BPF_JMP | BPF_JSET | BPF_X: 461 case BPF_JMP | BPF_JGT | BPF_K: 462 case BPF_JMP | BPF_JGT | BPF_X: 463 case BPF_JMP | BPF_JGE | BPF_K: 464 case BPF_JMP | BPF_JGE | BPF_X: 465 if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) { 466 /* BPF immediates are signed, zero extend 467 * immediate into tmp register and use it 468 * in compare insn. 469 */ 470 *insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k); 471 472 insn->dst_reg = BPF_REG_A; 473 insn->src_reg = BPF_REG_TMP; 474 bpf_src = BPF_X; 475 } else { 476 insn->dst_reg = BPF_REG_A; 477 insn->imm = fp->k; 478 bpf_src = BPF_SRC(fp->code); 479 insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0; 480 } 481 482 /* Common case where 'jump_false' is next insn. */ 483 if (fp->jf == 0) { 484 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src; 485 target = i + fp->jt + 1; 486 BPF_EMIT_JMP; 487 break; 488 } 489 490 /* Convert JEQ into JNE when 'jump_true' is next insn. */ 491 if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) { 492 insn->code = BPF_JMP | BPF_JNE | bpf_src; 493 target = i + fp->jf + 1; 494 BPF_EMIT_JMP; 495 break; 496 } 497 498 /* Other jumps are mapped into two insns: Jxx and JA. */ 499 target = i + fp->jt + 1; 500 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src; 501 BPF_EMIT_JMP; 502 insn++; 503 504 insn->code = BPF_JMP | BPF_JA; 505 target = i + fp->jf + 1; 506 BPF_EMIT_JMP; 507 break; 508 509 /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */ 510 case BPF_LDX | BPF_MSH | BPF_B: 511 /* tmp = A */ 512 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_A); 513 /* A = BPF_R0 = *(u8 *) (skb->data + K) */ 514 *insn++ = BPF_LD_ABS(BPF_B, fp->k); 515 /* A &= 0xf */ 516 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf); 517 /* A <<= 2 */ 518 *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2); 519 /* X = A */ 520 *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); 521 /* A = tmp */ 522 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP); 523 break; 524 525 /* RET_K, RET_A are remaped into 2 insns. */ 526 case BPF_RET | BPF_A: 527 case BPF_RET | BPF_K: 528 *insn++ = BPF_MOV32_RAW(BPF_RVAL(fp->code) == BPF_K ? 529 BPF_K : BPF_X, BPF_REG_0, 530 BPF_REG_A, fp->k); 531 *insn = BPF_EXIT_INSN(); 532 break; 533 534 /* Store to stack. */ 535 case BPF_ST: 536 case BPF_STX: 537 *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) == 538 BPF_ST ? BPF_REG_A : BPF_REG_X, 539 -(BPF_MEMWORDS - fp->k) * 4); 540 break; 541 542 /* Load from stack. */ 543 case BPF_LD | BPF_MEM: 544 case BPF_LDX | BPF_MEM: 545 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ? 546 BPF_REG_A : BPF_REG_X, BPF_REG_FP, 547 -(BPF_MEMWORDS - fp->k) * 4); 548 break; 549 550 /* A = K or X = K */ 551 case BPF_LD | BPF_IMM: 552 case BPF_LDX | BPF_IMM: 553 *insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ? 554 BPF_REG_A : BPF_REG_X, fp->k); 555 break; 556 557 /* X = A */ 558 case BPF_MISC | BPF_TAX: 559 *insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); 560 break; 561 562 /* A = X */ 563 case BPF_MISC | BPF_TXA: 564 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X); 565 break; 566 567 /* A = skb->len or X = skb->len */ 568 case BPF_LD | BPF_W | BPF_LEN: 569 case BPF_LDX | BPF_W | BPF_LEN: 570 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ? 571 BPF_REG_A : BPF_REG_X, BPF_REG_CTX, 572 offsetof(struct sk_buff, len)); 573 break; 574 575 /* Access seccomp_data fields. */ 576 case BPF_LDX | BPF_ABS | BPF_W: 577 /* A = *(u32 *) (ctx + K) */ 578 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k); 579 break; 580 581 /* Unknown instruction. */ 582 default: 583 goto err; 584 } 585 586 insn++; 587 if (new_prog) 588 memcpy(new_insn, tmp_insns, 589 sizeof(*insn) * (insn - tmp_insns)); 590 new_insn += insn - tmp_insns; 591 } 592 593 if (!new_prog) { 594 /* Only calculating new length. */ 595 *new_len = new_insn - new_prog; 596 return 0; 597 } 598 599 pass++; 600 if (new_flen != new_insn - new_prog) { 601 new_flen = new_insn - new_prog; 602 if (pass > 2) 603 goto err; 604 goto do_pass; 605 } 606 607 kfree(addrs); 608 BUG_ON(*new_len != new_flen); 609 return 0; 610 err: 611 kfree(addrs); 612 return -EINVAL; 613 } 614 615 /* Security: 616 * 617 * As we dont want to clear mem[] array for each packet going through 618 * __bpf_prog_run(), we check that filter loaded by user never try to read 619 * a cell if not previously written, and we check all branches to be sure 620 * a malicious user doesn't try to abuse us. 621 */ 622 static int check_load_and_stores(const struct sock_filter *filter, int flen) 623 { 624 u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */ 625 int pc, ret = 0; 626 627 BUILD_BUG_ON(BPF_MEMWORDS > 16); 628 629 masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL); 630 if (!masks) 631 return -ENOMEM; 632 633 memset(masks, 0xff, flen * sizeof(*masks)); 634 635 for (pc = 0; pc < flen; pc++) { 636 memvalid &= masks[pc]; 637 638 switch (filter[pc].code) { 639 case BPF_ST: 640 case BPF_STX: 641 memvalid |= (1 << filter[pc].k); 642 break; 643 case BPF_LD | BPF_MEM: 644 case BPF_LDX | BPF_MEM: 645 if (!(memvalid & (1 << filter[pc].k))) { 646 ret = -EINVAL; 647 goto error; 648 } 649 break; 650 case BPF_JMP | BPF_JA: 651 /* A jump must set masks on target */ 652 masks[pc + 1 + filter[pc].k] &= memvalid; 653 memvalid = ~0; 654 break; 655 case BPF_JMP | BPF_JEQ | BPF_K: 656 case BPF_JMP | BPF_JEQ | BPF_X: 657 case BPF_JMP | BPF_JGE | BPF_K: 658 case BPF_JMP | BPF_JGE | BPF_X: 659 case BPF_JMP | BPF_JGT | BPF_K: 660 case BPF_JMP | BPF_JGT | BPF_X: 661 case BPF_JMP | BPF_JSET | BPF_K: 662 case BPF_JMP | BPF_JSET | BPF_X: 663 /* A jump must set masks on targets */ 664 masks[pc + 1 + filter[pc].jt] &= memvalid; 665 masks[pc + 1 + filter[pc].jf] &= memvalid; 666 memvalid = ~0; 667 break; 668 } 669 } 670 error: 671 kfree(masks); 672 return ret; 673 } 674 675 static bool chk_code_allowed(u16 code_to_probe) 676 { 677 static const bool codes[] = { 678 /* 32 bit ALU operations */ 679 [BPF_ALU | BPF_ADD | BPF_K] = true, 680 [BPF_ALU | BPF_ADD | BPF_X] = true, 681 [BPF_ALU | BPF_SUB | BPF_K] = true, 682 [BPF_ALU | BPF_SUB | BPF_X] = true, 683 [BPF_ALU | BPF_MUL | BPF_K] = true, 684 [BPF_ALU | BPF_MUL | BPF_X] = true, 685 [BPF_ALU | BPF_DIV | BPF_K] = true, 686 [BPF_ALU | BPF_DIV | BPF_X] = true, 687 [BPF_ALU | BPF_MOD | BPF_K] = true, 688 [BPF_ALU | BPF_MOD | BPF_X] = true, 689 [BPF_ALU | BPF_AND | BPF_K] = true, 690 [BPF_ALU | BPF_AND | BPF_X] = true, 691 [BPF_ALU | BPF_OR | BPF_K] = true, 692 [BPF_ALU | BPF_OR | BPF_X] = true, 693 [BPF_ALU | BPF_XOR | BPF_K] = true, 694 [BPF_ALU | BPF_XOR | BPF_X] = true, 695 [BPF_ALU | BPF_LSH | BPF_K] = true, 696 [BPF_ALU | BPF_LSH | BPF_X] = true, 697 [BPF_ALU | BPF_RSH | BPF_K] = true, 698 [BPF_ALU | BPF_RSH | BPF_X] = true, 699 [BPF_ALU | BPF_NEG] = true, 700 /* Load instructions */ 701 [BPF_LD | BPF_W | BPF_ABS] = true, 702 [BPF_LD | BPF_H | BPF_ABS] = true, 703 [BPF_LD | BPF_B | BPF_ABS] = true, 704 [BPF_LD | BPF_W | BPF_LEN] = true, 705 [BPF_LD | BPF_W | BPF_IND] = true, 706 [BPF_LD | BPF_H | BPF_IND] = true, 707 [BPF_LD | BPF_B | BPF_IND] = true, 708 [BPF_LD | BPF_IMM] = true, 709 [BPF_LD | BPF_MEM] = true, 710 [BPF_LDX | BPF_W | BPF_LEN] = true, 711 [BPF_LDX | BPF_B | BPF_MSH] = true, 712 [BPF_LDX | BPF_IMM] = true, 713 [BPF_LDX | BPF_MEM] = true, 714 /* Store instructions */ 715 [BPF_ST] = true, 716 [BPF_STX] = true, 717 /* Misc instructions */ 718 [BPF_MISC | BPF_TAX] = true, 719 [BPF_MISC | BPF_TXA] = true, 720 /* Return instructions */ 721 [BPF_RET | BPF_K] = true, 722 [BPF_RET | BPF_A] = true, 723 /* Jump instructions */ 724 [BPF_JMP | BPF_JA] = true, 725 [BPF_JMP | BPF_JEQ | BPF_K] = true, 726 [BPF_JMP | BPF_JEQ | BPF_X] = true, 727 [BPF_JMP | BPF_JGE | BPF_K] = true, 728 [BPF_JMP | BPF_JGE | BPF_X] = true, 729 [BPF_JMP | BPF_JGT | BPF_K] = true, 730 [BPF_JMP | BPF_JGT | BPF_X] = true, 731 [BPF_JMP | BPF_JSET | BPF_K] = true, 732 [BPF_JMP | BPF_JSET | BPF_X] = true, 733 }; 734 735 if (code_to_probe >= ARRAY_SIZE(codes)) 736 return false; 737 738 return codes[code_to_probe]; 739 } 740 741 /** 742 * bpf_check_classic - verify socket filter code 743 * @filter: filter to verify 744 * @flen: length of filter 745 * 746 * Check the user's filter code. If we let some ugly 747 * filter code slip through kaboom! The filter must contain 748 * no references or jumps that are out of range, no illegal 749 * instructions, and must end with a RET instruction. 750 * 751 * All jumps are forward as they are not signed. 752 * 753 * Returns 0 if the rule set is legal or -EINVAL if not. 754 */ 755 static int bpf_check_classic(const struct sock_filter *filter, 756 unsigned int flen) 757 { 758 bool anc_found; 759 int pc; 760 761 if (flen == 0 || flen > BPF_MAXINSNS) 762 return -EINVAL; 763 764 /* Check the filter code now */ 765 for (pc = 0; pc < flen; pc++) { 766 const struct sock_filter *ftest = &filter[pc]; 767 768 /* May we actually operate on this code? */ 769 if (!chk_code_allowed(ftest->code)) 770 return -EINVAL; 771 772 /* Some instructions need special checks */ 773 switch (ftest->code) { 774 case BPF_ALU | BPF_DIV | BPF_K: 775 case BPF_ALU | BPF_MOD | BPF_K: 776 /* Check for division by zero */ 777 if (ftest->k == 0) 778 return -EINVAL; 779 break; 780 case BPF_LD | BPF_MEM: 781 case BPF_LDX | BPF_MEM: 782 case BPF_ST: 783 case BPF_STX: 784 /* Check for invalid memory addresses */ 785 if (ftest->k >= BPF_MEMWORDS) 786 return -EINVAL; 787 break; 788 case BPF_JMP | BPF_JA: 789 /* Note, the large ftest->k might cause loops. 790 * Compare this with conditional jumps below, 791 * where offsets are limited. --ANK (981016) 792 */ 793 if (ftest->k >= (unsigned int)(flen - pc - 1)) 794 return -EINVAL; 795 break; 796 case BPF_JMP | BPF_JEQ | BPF_K: 797 case BPF_JMP | BPF_JEQ | BPF_X: 798 case BPF_JMP | BPF_JGE | BPF_K: 799 case BPF_JMP | BPF_JGE | BPF_X: 800 case BPF_JMP | BPF_JGT | BPF_K: 801 case BPF_JMP | BPF_JGT | BPF_X: 802 case BPF_JMP | BPF_JSET | BPF_K: 803 case BPF_JMP | BPF_JSET | BPF_X: 804 /* Both conditionals must be safe */ 805 if (pc + ftest->jt + 1 >= flen || 806 pc + ftest->jf + 1 >= flen) 807 return -EINVAL; 808 break; 809 case BPF_LD | BPF_W | BPF_ABS: 810 case BPF_LD | BPF_H | BPF_ABS: 811 case BPF_LD | BPF_B | BPF_ABS: 812 anc_found = false; 813 if (bpf_anc_helper(ftest) & BPF_ANC) 814 anc_found = true; 815 /* Ancillary operation unknown or unsupported */ 816 if (anc_found == false && ftest->k >= SKF_AD_OFF) 817 return -EINVAL; 818 } 819 } 820 821 /* Last instruction must be a RET code */ 822 switch (filter[flen - 1].code) { 823 case BPF_RET | BPF_K: 824 case BPF_RET | BPF_A: 825 return check_load_and_stores(filter, flen); 826 } 827 828 return -EINVAL; 829 } 830 831 static int bpf_prog_store_orig_filter(struct bpf_prog *fp, 832 const struct sock_fprog *fprog) 833 { 834 unsigned int fsize = bpf_classic_proglen(fprog); 835 struct sock_fprog_kern *fkprog; 836 837 fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL); 838 if (!fp->orig_prog) 839 return -ENOMEM; 840 841 fkprog = fp->orig_prog; 842 fkprog->len = fprog->len; 843 844 fkprog->filter = kmemdup(fp->insns, fsize, 845 GFP_KERNEL | __GFP_NOWARN); 846 if (!fkprog->filter) { 847 kfree(fp->orig_prog); 848 return -ENOMEM; 849 } 850 851 return 0; 852 } 853 854 static void bpf_release_orig_filter(struct bpf_prog *fp) 855 { 856 struct sock_fprog_kern *fprog = fp->orig_prog; 857 858 if (fprog) { 859 kfree(fprog->filter); 860 kfree(fprog); 861 } 862 } 863 864 static void __bpf_prog_release(struct bpf_prog *prog) 865 { 866 if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) { 867 bpf_prog_put(prog); 868 } else { 869 bpf_release_orig_filter(prog); 870 bpf_prog_free(prog); 871 } 872 } 873 874 static void __sk_filter_release(struct sk_filter *fp) 875 { 876 __bpf_prog_release(fp->prog); 877 kfree(fp); 878 } 879 880 /** 881 * sk_filter_release_rcu - Release a socket filter by rcu_head 882 * @rcu: rcu_head that contains the sk_filter to free 883 */ 884 static void sk_filter_release_rcu(struct rcu_head *rcu) 885 { 886 struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu); 887 888 __sk_filter_release(fp); 889 } 890 891 /** 892 * sk_filter_release - release a socket filter 893 * @fp: filter to remove 894 * 895 * Remove a filter from a socket and release its resources. 896 */ 897 static void sk_filter_release(struct sk_filter *fp) 898 { 899 if (atomic_dec_and_test(&fp->refcnt)) 900 call_rcu(&fp->rcu, sk_filter_release_rcu); 901 } 902 903 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp) 904 { 905 u32 filter_size = bpf_prog_size(fp->prog->len); 906 907 atomic_sub(filter_size, &sk->sk_omem_alloc); 908 sk_filter_release(fp); 909 } 910 911 /* try to charge the socket memory if there is space available 912 * return true on success 913 */ 914 bool sk_filter_charge(struct sock *sk, struct sk_filter *fp) 915 { 916 u32 filter_size = bpf_prog_size(fp->prog->len); 917 918 /* same check as in sock_kmalloc() */ 919 if (filter_size <= sysctl_optmem_max && 920 atomic_read(&sk->sk_omem_alloc) + filter_size < sysctl_optmem_max) { 921 atomic_inc(&fp->refcnt); 922 atomic_add(filter_size, &sk->sk_omem_alloc); 923 return true; 924 } 925 return false; 926 } 927 928 static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp) 929 { 930 struct sock_filter *old_prog; 931 struct bpf_prog *old_fp; 932 int err, new_len, old_len = fp->len; 933 934 /* We are free to overwrite insns et al right here as it 935 * won't be used at this point in time anymore internally 936 * after the migration to the internal BPF instruction 937 * representation. 938 */ 939 BUILD_BUG_ON(sizeof(struct sock_filter) != 940 sizeof(struct bpf_insn)); 941 942 /* Conversion cannot happen on overlapping memory areas, 943 * so we need to keep the user BPF around until the 2nd 944 * pass. At this time, the user BPF is stored in fp->insns. 945 */ 946 old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter), 947 GFP_KERNEL | __GFP_NOWARN); 948 if (!old_prog) { 949 err = -ENOMEM; 950 goto out_err; 951 } 952 953 /* 1st pass: calculate the new program length. */ 954 err = bpf_convert_filter(old_prog, old_len, NULL, &new_len); 955 if (err) 956 goto out_err_free; 957 958 /* Expand fp for appending the new filter representation. */ 959 old_fp = fp; 960 fp = bpf_prog_realloc(old_fp, bpf_prog_size(new_len), 0); 961 if (!fp) { 962 /* The old_fp is still around in case we couldn't 963 * allocate new memory, so uncharge on that one. 964 */ 965 fp = old_fp; 966 err = -ENOMEM; 967 goto out_err_free; 968 } 969 970 fp->len = new_len; 971 972 /* 2nd pass: remap sock_filter insns into bpf_insn insns. */ 973 err = bpf_convert_filter(old_prog, old_len, fp->insnsi, &new_len); 974 if (err) 975 /* 2nd bpf_convert_filter() can fail only if it fails 976 * to allocate memory, remapping must succeed. Note, 977 * that at this time old_fp has already been released 978 * by krealloc(). 979 */ 980 goto out_err_free; 981 982 bpf_prog_select_runtime(fp); 983 984 kfree(old_prog); 985 return fp; 986 987 out_err_free: 988 kfree(old_prog); 989 out_err: 990 __bpf_prog_release(fp); 991 return ERR_PTR(err); 992 } 993 994 static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp, 995 bpf_aux_classic_check_t trans) 996 { 997 int err; 998 999 fp->bpf_func = NULL; 1000 fp->jited = 0; 1001 1002 err = bpf_check_classic(fp->insns, fp->len); 1003 if (err) { 1004 __bpf_prog_release(fp); 1005 return ERR_PTR(err); 1006 } 1007 1008 /* There might be additional checks and transformations 1009 * needed on classic filters, f.e. in case of seccomp. 1010 */ 1011 if (trans) { 1012 err = trans(fp->insns, fp->len); 1013 if (err) { 1014 __bpf_prog_release(fp); 1015 return ERR_PTR(err); 1016 } 1017 } 1018 1019 /* Probe if we can JIT compile the filter and if so, do 1020 * the compilation of the filter. 1021 */ 1022 bpf_jit_compile(fp); 1023 1024 /* JIT compiler couldn't process this filter, so do the 1025 * internal BPF translation for the optimized interpreter. 1026 */ 1027 if (!fp->jited) 1028 fp = bpf_migrate_filter(fp); 1029 1030 return fp; 1031 } 1032 1033 /** 1034 * bpf_prog_create - create an unattached filter 1035 * @pfp: the unattached filter that is created 1036 * @fprog: the filter program 1037 * 1038 * Create a filter independent of any socket. We first run some 1039 * sanity checks on it to make sure it does not explode on us later. 1040 * If an error occurs or there is insufficient memory for the filter 1041 * a negative errno code is returned. On success the return is zero. 1042 */ 1043 int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog) 1044 { 1045 unsigned int fsize = bpf_classic_proglen(fprog); 1046 struct bpf_prog *fp; 1047 1048 /* Make sure new filter is there and in the right amounts. */ 1049 if (fprog->filter == NULL) 1050 return -EINVAL; 1051 1052 fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); 1053 if (!fp) 1054 return -ENOMEM; 1055 1056 memcpy(fp->insns, fprog->filter, fsize); 1057 1058 fp->len = fprog->len; 1059 /* Since unattached filters are not copied back to user 1060 * space through sk_get_filter(), we do not need to hold 1061 * a copy here, and can spare us the work. 1062 */ 1063 fp->orig_prog = NULL; 1064 1065 /* bpf_prepare_filter() already takes care of freeing 1066 * memory in case something goes wrong. 1067 */ 1068 fp = bpf_prepare_filter(fp, NULL); 1069 if (IS_ERR(fp)) 1070 return PTR_ERR(fp); 1071 1072 *pfp = fp; 1073 return 0; 1074 } 1075 EXPORT_SYMBOL_GPL(bpf_prog_create); 1076 1077 /** 1078 * bpf_prog_create_from_user - create an unattached filter from user buffer 1079 * @pfp: the unattached filter that is created 1080 * @fprog: the filter program 1081 * @trans: post-classic verifier transformation handler 1082 * @save_orig: save classic BPF program 1083 * 1084 * This function effectively does the same as bpf_prog_create(), only 1085 * that it builds up its insns buffer from user space provided buffer. 1086 * It also allows for passing a bpf_aux_classic_check_t handler. 1087 */ 1088 int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog, 1089 bpf_aux_classic_check_t trans, bool save_orig) 1090 { 1091 unsigned int fsize = bpf_classic_proglen(fprog); 1092 struct bpf_prog *fp; 1093 int err; 1094 1095 /* Make sure new filter is there and in the right amounts. */ 1096 if (fprog->filter == NULL) 1097 return -EINVAL; 1098 1099 fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); 1100 if (!fp) 1101 return -ENOMEM; 1102 1103 if (copy_from_user(fp->insns, fprog->filter, fsize)) { 1104 __bpf_prog_free(fp); 1105 return -EFAULT; 1106 } 1107 1108 fp->len = fprog->len; 1109 fp->orig_prog = NULL; 1110 1111 if (save_orig) { 1112 err = bpf_prog_store_orig_filter(fp, fprog); 1113 if (err) { 1114 __bpf_prog_free(fp); 1115 return -ENOMEM; 1116 } 1117 } 1118 1119 /* bpf_prepare_filter() already takes care of freeing 1120 * memory in case something goes wrong. 1121 */ 1122 fp = bpf_prepare_filter(fp, trans); 1123 if (IS_ERR(fp)) 1124 return PTR_ERR(fp); 1125 1126 *pfp = fp; 1127 return 0; 1128 } 1129 EXPORT_SYMBOL_GPL(bpf_prog_create_from_user); 1130 1131 void bpf_prog_destroy(struct bpf_prog *fp) 1132 { 1133 __bpf_prog_release(fp); 1134 } 1135 EXPORT_SYMBOL_GPL(bpf_prog_destroy); 1136 1137 static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk) 1138 { 1139 struct sk_filter *fp, *old_fp; 1140 1141 fp = kmalloc(sizeof(*fp), GFP_KERNEL); 1142 if (!fp) 1143 return -ENOMEM; 1144 1145 fp->prog = prog; 1146 atomic_set(&fp->refcnt, 0); 1147 1148 if (!sk_filter_charge(sk, fp)) { 1149 kfree(fp); 1150 return -ENOMEM; 1151 } 1152 1153 old_fp = rcu_dereference_protected(sk->sk_filter, 1154 sock_owned_by_user(sk)); 1155 rcu_assign_pointer(sk->sk_filter, fp); 1156 1157 if (old_fp) 1158 sk_filter_uncharge(sk, old_fp); 1159 1160 return 0; 1161 } 1162 1163 /** 1164 * sk_attach_filter - attach a socket filter 1165 * @fprog: the filter program 1166 * @sk: the socket to use 1167 * 1168 * Attach the user's filter code. We first run some sanity checks on 1169 * it to make sure it does not explode on us later. If an error 1170 * occurs or there is insufficient memory for the filter a negative 1171 * errno code is returned. On success the return is zero. 1172 */ 1173 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk) 1174 { 1175 unsigned int fsize = bpf_classic_proglen(fprog); 1176 unsigned int bpf_fsize = bpf_prog_size(fprog->len); 1177 struct bpf_prog *prog; 1178 int err; 1179 1180 if (sock_flag(sk, SOCK_FILTER_LOCKED)) 1181 return -EPERM; 1182 1183 /* Make sure new filter is there and in the right amounts. */ 1184 if (fprog->filter == NULL) 1185 return -EINVAL; 1186 1187 prog = bpf_prog_alloc(bpf_fsize, 0); 1188 if (!prog) 1189 return -ENOMEM; 1190 1191 if (copy_from_user(prog->insns, fprog->filter, fsize)) { 1192 __bpf_prog_free(prog); 1193 return -EFAULT; 1194 } 1195 1196 prog->len = fprog->len; 1197 1198 err = bpf_prog_store_orig_filter(prog, fprog); 1199 if (err) { 1200 __bpf_prog_free(prog); 1201 return -ENOMEM; 1202 } 1203 1204 /* bpf_prepare_filter() already takes care of freeing 1205 * memory in case something goes wrong. 1206 */ 1207 prog = bpf_prepare_filter(prog, NULL); 1208 if (IS_ERR(prog)) 1209 return PTR_ERR(prog); 1210 1211 err = __sk_attach_prog(prog, sk); 1212 if (err < 0) { 1213 __bpf_prog_release(prog); 1214 return err; 1215 } 1216 1217 return 0; 1218 } 1219 EXPORT_SYMBOL_GPL(sk_attach_filter); 1220 1221 int sk_attach_bpf(u32 ufd, struct sock *sk) 1222 { 1223 struct bpf_prog *prog; 1224 int err; 1225 1226 if (sock_flag(sk, SOCK_FILTER_LOCKED)) 1227 return -EPERM; 1228 1229 prog = bpf_prog_get(ufd); 1230 if (IS_ERR(prog)) 1231 return PTR_ERR(prog); 1232 1233 if (prog->type != BPF_PROG_TYPE_SOCKET_FILTER) { 1234 bpf_prog_put(prog); 1235 return -EINVAL; 1236 } 1237 1238 err = __sk_attach_prog(prog, sk); 1239 if (err < 0) { 1240 bpf_prog_put(prog); 1241 return err; 1242 } 1243 1244 return 0; 1245 } 1246 1247 #define BPF_RECOMPUTE_CSUM(flags) ((flags) & 1) 1248 1249 static u64 bpf_skb_store_bytes(u64 r1, u64 r2, u64 r3, u64 r4, u64 flags) 1250 { 1251 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1252 int offset = (int) r2; 1253 void *from = (void *) (long) r3; 1254 unsigned int len = (unsigned int) r4; 1255 char buf[16]; 1256 void *ptr; 1257 1258 /* bpf verifier guarantees that: 1259 * 'from' pointer points to bpf program stack 1260 * 'len' bytes of it were initialized 1261 * 'len' > 0 1262 * 'skb' is a valid pointer to 'struct sk_buff' 1263 * 1264 * so check for invalid 'offset' and too large 'len' 1265 */ 1266 if (unlikely((u32) offset > 0xffff || len > sizeof(buf))) 1267 return -EFAULT; 1268 1269 if (unlikely(skb_cloned(skb) && 1270 !skb_clone_writable(skb, offset + len))) 1271 return -EFAULT; 1272 1273 ptr = skb_header_pointer(skb, offset, len, buf); 1274 if (unlikely(!ptr)) 1275 return -EFAULT; 1276 1277 if (BPF_RECOMPUTE_CSUM(flags)) 1278 skb_postpull_rcsum(skb, ptr, len); 1279 1280 memcpy(ptr, from, len); 1281 1282 if (ptr == buf) 1283 /* skb_store_bits cannot return -EFAULT here */ 1284 skb_store_bits(skb, offset, ptr, len); 1285 1286 if (BPF_RECOMPUTE_CSUM(flags) && skb->ip_summed == CHECKSUM_COMPLETE) 1287 skb->csum = csum_add(skb->csum, csum_partial(ptr, len, 0)); 1288 return 0; 1289 } 1290 1291 const struct bpf_func_proto bpf_skb_store_bytes_proto = { 1292 .func = bpf_skb_store_bytes, 1293 .gpl_only = false, 1294 .ret_type = RET_INTEGER, 1295 .arg1_type = ARG_PTR_TO_CTX, 1296 .arg2_type = ARG_ANYTHING, 1297 .arg3_type = ARG_PTR_TO_STACK, 1298 .arg4_type = ARG_CONST_STACK_SIZE, 1299 .arg5_type = ARG_ANYTHING, 1300 }; 1301 1302 #define BPF_HEADER_FIELD_SIZE(flags) ((flags) & 0x0f) 1303 #define BPF_IS_PSEUDO_HEADER(flags) ((flags) & 0x10) 1304 1305 static u64 bpf_l3_csum_replace(u64 r1, u64 r2, u64 from, u64 to, u64 flags) 1306 { 1307 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1308 int offset = (int) r2; 1309 __sum16 sum, *ptr; 1310 1311 if (unlikely((u32) offset > 0xffff)) 1312 return -EFAULT; 1313 1314 if (unlikely(skb_cloned(skb) && 1315 !skb_clone_writable(skb, offset + sizeof(sum)))) 1316 return -EFAULT; 1317 1318 ptr = skb_header_pointer(skb, offset, sizeof(sum), &sum); 1319 if (unlikely(!ptr)) 1320 return -EFAULT; 1321 1322 switch (BPF_HEADER_FIELD_SIZE(flags)) { 1323 case 2: 1324 csum_replace2(ptr, from, to); 1325 break; 1326 case 4: 1327 csum_replace4(ptr, from, to); 1328 break; 1329 default: 1330 return -EINVAL; 1331 } 1332 1333 if (ptr == &sum) 1334 /* skb_store_bits guaranteed to not return -EFAULT here */ 1335 skb_store_bits(skb, offset, ptr, sizeof(sum)); 1336 1337 return 0; 1338 } 1339 1340 const struct bpf_func_proto bpf_l3_csum_replace_proto = { 1341 .func = bpf_l3_csum_replace, 1342 .gpl_only = false, 1343 .ret_type = RET_INTEGER, 1344 .arg1_type = ARG_PTR_TO_CTX, 1345 .arg2_type = ARG_ANYTHING, 1346 .arg3_type = ARG_ANYTHING, 1347 .arg4_type = ARG_ANYTHING, 1348 .arg5_type = ARG_ANYTHING, 1349 }; 1350 1351 static u64 bpf_l4_csum_replace(u64 r1, u64 r2, u64 from, u64 to, u64 flags) 1352 { 1353 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1354 bool is_pseudo = !!BPF_IS_PSEUDO_HEADER(flags); 1355 int offset = (int) r2; 1356 __sum16 sum, *ptr; 1357 1358 if (unlikely((u32) offset > 0xffff)) 1359 return -EFAULT; 1360 1361 if (unlikely(skb_cloned(skb) && 1362 !skb_clone_writable(skb, offset + sizeof(sum)))) 1363 return -EFAULT; 1364 1365 ptr = skb_header_pointer(skb, offset, sizeof(sum), &sum); 1366 if (unlikely(!ptr)) 1367 return -EFAULT; 1368 1369 switch (BPF_HEADER_FIELD_SIZE(flags)) { 1370 case 2: 1371 inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo); 1372 break; 1373 case 4: 1374 inet_proto_csum_replace4(ptr, skb, from, to, is_pseudo); 1375 break; 1376 default: 1377 return -EINVAL; 1378 } 1379 1380 if (ptr == &sum) 1381 /* skb_store_bits guaranteed to not return -EFAULT here */ 1382 skb_store_bits(skb, offset, ptr, sizeof(sum)); 1383 1384 return 0; 1385 } 1386 1387 const struct bpf_func_proto bpf_l4_csum_replace_proto = { 1388 .func = bpf_l4_csum_replace, 1389 .gpl_only = false, 1390 .ret_type = RET_INTEGER, 1391 .arg1_type = ARG_PTR_TO_CTX, 1392 .arg2_type = ARG_ANYTHING, 1393 .arg3_type = ARG_ANYTHING, 1394 .arg4_type = ARG_ANYTHING, 1395 .arg5_type = ARG_ANYTHING, 1396 }; 1397 1398 #define BPF_IS_REDIRECT_INGRESS(flags) ((flags) & 1) 1399 1400 static u64 bpf_clone_redirect(u64 r1, u64 ifindex, u64 flags, u64 r4, u64 r5) 1401 { 1402 struct sk_buff *skb = (struct sk_buff *) (long) r1, *skb2; 1403 struct net_device *dev; 1404 1405 dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex); 1406 if (unlikely(!dev)) 1407 return -EINVAL; 1408 1409 skb2 = skb_clone(skb, GFP_ATOMIC); 1410 if (unlikely(!skb2)) 1411 return -ENOMEM; 1412 1413 if (BPF_IS_REDIRECT_INGRESS(flags)) 1414 return dev_forward_skb(dev, skb2); 1415 1416 skb2->dev = dev; 1417 skb_sender_cpu_clear(skb2); 1418 return dev_queue_xmit(skb2); 1419 } 1420 1421 const struct bpf_func_proto bpf_clone_redirect_proto = { 1422 .func = bpf_clone_redirect, 1423 .gpl_only = false, 1424 .ret_type = RET_INTEGER, 1425 .arg1_type = ARG_PTR_TO_CTX, 1426 .arg2_type = ARG_ANYTHING, 1427 .arg3_type = ARG_ANYTHING, 1428 }; 1429 1430 struct redirect_info { 1431 u32 ifindex; 1432 u32 flags; 1433 }; 1434 1435 static DEFINE_PER_CPU(struct redirect_info, redirect_info); 1436 static u64 bpf_redirect(u64 ifindex, u64 flags, u64 r3, u64 r4, u64 r5) 1437 { 1438 struct redirect_info *ri = this_cpu_ptr(&redirect_info); 1439 1440 ri->ifindex = ifindex; 1441 ri->flags = flags; 1442 return TC_ACT_REDIRECT; 1443 } 1444 1445 int skb_do_redirect(struct sk_buff *skb) 1446 { 1447 struct redirect_info *ri = this_cpu_ptr(&redirect_info); 1448 struct net_device *dev; 1449 1450 dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex); 1451 ri->ifindex = 0; 1452 if (unlikely(!dev)) { 1453 kfree_skb(skb); 1454 return -EINVAL; 1455 } 1456 1457 if (BPF_IS_REDIRECT_INGRESS(ri->flags)) 1458 return dev_forward_skb(dev, skb); 1459 1460 skb->dev = dev; 1461 skb_sender_cpu_clear(skb); 1462 return dev_queue_xmit(skb); 1463 } 1464 1465 const struct bpf_func_proto bpf_redirect_proto = { 1466 .func = bpf_redirect, 1467 .gpl_only = false, 1468 .ret_type = RET_INTEGER, 1469 .arg1_type = ARG_ANYTHING, 1470 .arg2_type = ARG_ANYTHING, 1471 }; 1472 1473 static u64 bpf_get_cgroup_classid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 1474 { 1475 return task_get_classid((struct sk_buff *) (unsigned long) r1); 1476 } 1477 1478 static const struct bpf_func_proto bpf_get_cgroup_classid_proto = { 1479 .func = bpf_get_cgroup_classid, 1480 .gpl_only = false, 1481 .ret_type = RET_INTEGER, 1482 .arg1_type = ARG_PTR_TO_CTX, 1483 }; 1484 1485 static u64 bpf_get_route_realm(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 1486 { 1487 #ifdef CONFIG_IP_ROUTE_CLASSID 1488 const struct dst_entry *dst; 1489 1490 dst = skb_dst((struct sk_buff *) (unsigned long) r1); 1491 if (dst) 1492 return dst->tclassid; 1493 #endif 1494 return 0; 1495 } 1496 1497 static const struct bpf_func_proto bpf_get_route_realm_proto = { 1498 .func = bpf_get_route_realm, 1499 .gpl_only = false, 1500 .ret_type = RET_INTEGER, 1501 .arg1_type = ARG_PTR_TO_CTX, 1502 }; 1503 1504 static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5) 1505 { 1506 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1507 __be16 vlan_proto = (__force __be16) r2; 1508 1509 if (unlikely(vlan_proto != htons(ETH_P_8021Q) && 1510 vlan_proto != htons(ETH_P_8021AD))) 1511 vlan_proto = htons(ETH_P_8021Q); 1512 1513 return skb_vlan_push(skb, vlan_proto, vlan_tci); 1514 } 1515 1516 const struct bpf_func_proto bpf_skb_vlan_push_proto = { 1517 .func = bpf_skb_vlan_push, 1518 .gpl_only = false, 1519 .ret_type = RET_INTEGER, 1520 .arg1_type = ARG_PTR_TO_CTX, 1521 .arg2_type = ARG_ANYTHING, 1522 .arg3_type = ARG_ANYTHING, 1523 }; 1524 EXPORT_SYMBOL_GPL(bpf_skb_vlan_push_proto); 1525 1526 static u64 bpf_skb_vlan_pop(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 1527 { 1528 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1529 1530 return skb_vlan_pop(skb); 1531 } 1532 1533 const struct bpf_func_proto bpf_skb_vlan_pop_proto = { 1534 .func = bpf_skb_vlan_pop, 1535 .gpl_only = false, 1536 .ret_type = RET_INTEGER, 1537 .arg1_type = ARG_PTR_TO_CTX, 1538 }; 1539 EXPORT_SYMBOL_GPL(bpf_skb_vlan_pop_proto); 1540 1541 bool bpf_helper_changes_skb_data(void *func) 1542 { 1543 if (func == bpf_skb_vlan_push) 1544 return true; 1545 if (func == bpf_skb_vlan_pop) 1546 return true; 1547 return false; 1548 } 1549 1550 static u64 bpf_skb_get_tunnel_key(u64 r1, u64 r2, u64 size, u64 flags, u64 r5) 1551 { 1552 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1553 struct bpf_tunnel_key *to = (struct bpf_tunnel_key *) (long) r2; 1554 struct ip_tunnel_info *info = skb_tunnel_info(skb); 1555 1556 if (unlikely(size != sizeof(struct bpf_tunnel_key) || flags || !info)) 1557 return -EINVAL; 1558 if (ip_tunnel_info_af(info) != AF_INET) 1559 return -EINVAL; 1560 1561 to->tunnel_id = be64_to_cpu(info->key.tun_id); 1562 to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src); 1563 1564 return 0; 1565 } 1566 1567 const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = { 1568 .func = bpf_skb_get_tunnel_key, 1569 .gpl_only = false, 1570 .ret_type = RET_INTEGER, 1571 .arg1_type = ARG_PTR_TO_CTX, 1572 .arg2_type = ARG_PTR_TO_STACK, 1573 .arg3_type = ARG_CONST_STACK_SIZE, 1574 .arg4_type = ARG_ANYTHING, 1575 }; 1576 1577 static struct metadata_dst __percpu *md_dst; 1578 1579 static u64 bpf_skb_set_tunnel_key(u64 r1, u64 r2, u64 size, u64 flags, u64 r5) 1580 { 1581 struct sk_buff *skb = (struct sk_buff *) (long) r1; 1582 struct bpf_tunnel_key *from = (struct bpf_tunnel_key *) (long) r2; 1583 struct metadata_dst *md = this_cpu_ptr(md_dst); 1584 struct ip_tunnel_info *info; 1585 1586 if (unlikely(size != sizeof(struct bpf_tunnel_key) || flags)) 1587 return -EINVAL; 1588 1589 skb_dst_drop(skb); 1590 dst_hold((struct dst_entry *) md); 1591 skb_dst_set(skb, (struct dst_entry *) md); 1592 1593 info = &md->u.tun_info; 1594 info->mode = IP_TUNNEL_INFO_TX; 1595 info->key.tun_flags = TUNNEL_KEY; 1596 info->key.tun_id = cpu_to_be64(from->tunnel_id); 1597 info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4); 1598 1599 return 0; 1600 } 1601 1602 const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { 1603 .func = bpf_skb_set_tunnel_key, 1604 .gpl_only = false, 1605 .ret_type = RET_INTEGER, 1606 .arg1_type = ARG_PTR_TO_CTX, 1607 .arg2_type = ARG_PTR_TO_STACK, 1608 .arg3_type = ARG_CONST_STACK_SIZE, 1609 .arg4_type = ARG_ANYTHING, 1610 }; 1611 1612 static const struct bpf_func_proto *bpf_get_skb_set_tunnel_key_proto(void) 1613 { 1614 if (!md_dst) { 1615 /* race is not possible, since it's called from 1616 * verifier that is holding verifier mutex 1617 */ 1618 md_dst = metadata_dst_alloc_percpu(0, GFP_KERNEL); 1619 if (!md_dst) 1620 return NULL; 1621 } 1622 return &bpf_skb_set_tunnel_key_proto; 1623 } 1624 1625 static const struct bpf_func_proto * 1626 sk_filter_func_proto(enum bpf_func_id func_id) 1627 { 1628 switch (func_id) { 1629 case BPF_FUNC_map_lookup_elem: 1630 return &bpf_map_lookup_elem_proto; 1631 case BPF_FUNC_map_update_elem: 1632 return &bpf_map_update_elem_proto; 1633 case BPF_FUNC_map_delete_elem: 1634 return &bpf_map_delete_elem_proto; 1635 case BPF_FUNC_get_prandom_u32: 1636 return &bpf_get_prandom_u32_proto; 1637 case BPF_FUNC_get_smp_processor_id: 1638 return &bpf_get_smp_processor_id_proto; 1639 case BPF_FUNC_tail_call: 1640 return &bpf_tail_call_proto; 1641 case BPF_FUNC_ktime_get_ns: 1642 return &bpf_ktime_get_ns_proto; 1643 case BPF_FUNC_trace_printk: 1644 if (capable(CAP_SYS_ADMIN)) 1645 return bpf_get_trace_printk_proto(); 1646 default: 1647 return NULL; 1648 } 1649 } 1650 1651 static const struct bpf_func_proto * 1652 tc_cls_act_func_proto(enum bpf_func_id func_id) 1653 { 1654 switch (func_id) { 1655 case BPF_FUNC_skb_store_bytes: 1656 return &bpf_skb_store_bytes_proto; 1657 case BPF_FUNC_l3_csum_replace: 1658 return &bpf_l3_csum_replace_proto; 1659 case BPF_FUNC_l4_csum_replace: 1660 return &bpf_l4_csum_replace_proto; 1661 case BPF_FUNC_clone_redirect: 1662 return &bpf_clone_redirect_proto; 1663 case BPF_FUNC_get_cgroup_classid: 1664 return &bpf_get_cgroup_classid_proto; 1665 case BPF_FUNC_skb_vlan_push: 1666 return &bpf_skb_vlan_push_proto; 1667 case BPF_FUNC_skb_vlan_pop: 1668 return &bpf_skb_vlan_pop_proto; 1669 case BPF_FUNC_skb_get_tunnel_key: 1670 return &bpf_skb_get_tunnel_key_proto; 1671 case BPF_FUNC_skb_set_tunnel_key: 1672 return bpf_get_skb_set_tunnel_key_proto(); 1673 case BPF_FUNC_redirect: 1674 return &bpf_redirect_proto; 1675 case BPF_FUNC_get_route_realm: 1676 return &bpf_get_route_realm_proto; 1677 default: 1678 return sk_filter_func_proto(func_id); 1679 } 1680 } 1681 1682 static bool __is_valid_access(int off, int size, enum bpf_access_type type) 1683 { 1684 /* check bounds */ 1685 if (off < 0 || off >= sizeof(struct __sk_buff)) 1686 return false; 1687 1688 /* disallow misaligned access */ 1689 if (off % size != 0) 1690 return false; 1691 1692 /* all __sk_buff fields are __u32 */ 1693 if (size != 4) 1694 return false; 1695 1696 return true; 1697 } 1698 1699 static bool sk_filter_is_valid_access(int off, int size, 1700 enum bpf_access_type type) 1701 { 1702 if (off == offsetof(struct __sk_buff, tc_classid)) 1703 return false; 1704 1705 if (type == BPF_WRITE) { 1706 switch (off) { 1707 case offsetof(struct __sk_buff, cb[0]) ... 1708 offsetof(struct __sk_buff, cb[4]): 1709 break; 1710 default: 1711 return false; 1712 } 1713 } 1714 1715 return __is_valid_access(off, size, type); 1716 } 1717 1718 static bool tc_cls_act_is_valid_access(int off, int size, 1719 enum bpf_access_type type) 1720 { 1721 if (off == offsetof(struct __sk_buff, tc_classid)) 1722 return type == BPF_WRITE ? true : false; 1723 1724 if (type == BPF_WRITE) { 1725 switch (off) { 1726 case offsetof(struct __sk_buff, mark): 1727 case offsetof(struct __sk_buff, tc_index): 1728 case offsetof(struct __sk_buff, priority): 1729 case offsetof(struct __sk_buff, cb[0]) ... 1730 offsetof(struct __sk_buff, cb[4]): 1731 break; 1732 default: 1733 return false; 1734 } 1735 } 1736 return __is_valid_access(off, size, type); 1737 } 1738 1739 static u32 bpf_net_convert_ctx_access(enum bpf_access_type type, int dst_reg, 1740 int src_reg, int ctx_off, 1741 struct bpf_insn *insn_buf, 1742 struct bpf_prog *prog) 1743 { 1744 struct bpf_insn *insn = insn_buf; 1745 1746 switch (ctx_off) { 1747 case offsetof(struct __sk_buff, len): 1748 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 1749 1750 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, 1751 offsetof(struct sk_buff, len)); 1752 break; 1753 1754 case offsetof(struct __sk_buff, protocol): 1755 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 1756 1757 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, 1758 offsetof(struct sk_buff, protocol)); 1759 break; 1760 1761 case offsetof(struct __sk_buff, vlan_proto): 1762 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_proto) != 2); 1763 1764 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, 1765 offsetof(struct sk_buff, vlan_proto)); 1766 break; 1767 1768 case offsetof(struct __sk_buff, priority): 1769 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, priority) != 4); 1770 1771 if (type == BPF_WRITE) 1772 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, 1773 offsetof(struct sk_buff, priority)); 1774 else 1775 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, 1776 offsetof(struct sk_buff, priority)); 1777 break; 1778 1779 case offsetof(struct __sk_buff, ingress_ifindex): 1780 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, skb_iif) != 4); 1781 1782 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, 1783 offsetof(struct sk_buff, skb_iif)); 1784 break; 1785 1786 case offsetof(struct __sk_buff, ifindex): 1787 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 1788 1789 *insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)), 1790 dst_reg, src_reg, 1791 offsetof(struct sk_buff, dev)); 1792 *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1); 1793 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, dst_reg, 1794 offsetof(struct net_device, ifindex)); 1795 break; 1796 1797 case offsetof(struct __sk_buff, hash): 1798 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); 1799 1800 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, 1801 offsetof(struct sk_buff, hash)); 1802 break; 1803 1804 case offsetof(struct __sk_buff, mark): 1805 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 1806 1807 if (type == BPF_WRITE) 1808 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, 1809 offsetof(struct sk_buff, mark)); 1810 else 1811 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, 1812 offsetof(struct sk_buff, mark)); 1813 break; 1814 1815 case offsetof(struct __sk_buff, pkt_type): 1816 return convert_skb_access(SKF_AD_PKTTYPE, dst_reg, src_reg, insn); 1817 1818 case offsetof(struct __sk_buff, queue_mapping): 1819 return convert_skb_access(SKF_AD_QUEUE, dst_reg, src_reg, insn); 1820 1821 case offsetof(struct __sk_buff, vlan_present): 1822 return convert_skb_access(SKF_AD_VLAN_TAG_PRESENT, 1823 dst_reg, src_reg, insn); 1824 1825 case offsetof(struct __sk_buff, vlan_tci): 1826 return convert_skb_access(SKF_AD_VLAN_TAG, 1827 dst_reg, src_reg, insn); 1828 1829 case offsetof(struct __sk_buff, cb[0]) ... 1830 offsetof(struct __sk_buff, cb[4]): 1831 BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, data) < 20); 1832 1833 prog->cb_access = 1; 1834 ctx_off -= offsetof(struct __sk_buff, cb[0]); 1835 ctx_off += offsetof(struct sk_buff, cb); 1836 ctx_off += offsetof(struct qdisc_skb_cb, data); 1837 if (type == BPF_WRITE) 1838 *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, ctx_off); 1839 else 1840 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, ctx_off); 1841 break; 1842 1843 case offsetof(struct __sk_buff, tc_classid): 1844 ctx_off -= offsetof(struct __sk_buff, tc_classid); 1845 ctx_off += offsetof(struct sk_buff, cb); 1846 ctx_off += offsetof(struct qdisc_skb_cb, tc_classid); 1847 WARN_ON(type != BPF_WRITE); 1848 *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, ctx_off); 1849 break; 1850 1851 case offsetof(struct __sk_buff, tc_index): 1852 #ifdef CONFIG_NET_SCHED 1853 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tc_index) != 2); 1854 1855 if (type == BPF_WRITE) 1856 *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, 1857 offsetof(struct sk_buff, tc_index)); 1858 else 1859 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, 1860 offsetof(struct sk_buff, tc_index)); 1861 break; 1862 #else 1863 if (type == BPF_WRITE) 1864 *insn++ = BPF_MOV64_REG(dst_reg, dst_reg); 1865 else 1866 *insn++ = BPF_MOV64_IMM(dst_reg, 0); 1867 break; 1868 #endif 1869 } 1870 1871 return insn - insn_buf; 1872 } 1873 1874 static const struct bpf_verifier_ops sk_filter_ops = { 1875 .get_func_proto = sk_filter_func_proto, 1876 .is_valid_access = sk_filter_is_valid_access, 1877 .convert_ctx_access = bpf_net_convert_ctx_access, 1878 }; 1879 1880 static const struct bpf_verifier_ops tc_cls_act_ops = { 1881 .get_func_proto = tc_cls_act_func_proto, 1882 .is_valid_access = tc_cls_act_is_valid_access, 1883 .convert_ctx_access = bpf_net_convert_ctx_access, 1884 }; 1885 1886 static struct bpf_prog_type_list sk_filter_type __read_mostly = { 1887 .ops = &sk_filter_ops, 1888 .type = BPF_PROG_TYPE_SOCKET_FILTER, 1889 }; 1890 1891 static struct bpf_prog_type_list sched_cls_type __read_mostly = { 1892 .ops = &tc_cls_act_ops, 1893 .type = BPF_PROG_TYPE_SCHED_CLS, 1894 }; 1895 1896 static struct bpf_prog_type_list sched_act_type __read_mostly = { 1897 .ops = &tc_cls_act_ops, 1898 .type = BPF_PROG_TYPE_SCHED_ACT, 1899 }; 1900 1901 static int __init register_sk_filter_ops(void) 1902 { 1903 bpf_register_prog_type(&sk_filter_type); 1904 bpf_register_prog_type(&sched_cls_type); 1905 bpf_register_prog_type(&sched_act_type); 1906 1907 return 0; 1908 } 1909 late_initcall(register_sk_filter_ops); 1910 1911 int sk_detach_filter(struct sock *sk) 1912 { 1913 int ret = -ENOENT; 1914 struct sk_filter *filter; 1915 1916 if (sock_flag(sk, SOCK_FILTER_LOCKED)) 1917 return -EPERM; 1918 1919 filter = rcu_dereference_protected(sk->sk_filter, 1920 sock_owned_by_user(sk)); 1921 if (filter) { 1922 RCU_INIT_POINTER(sk->sk_filter, NULL); 1923 sk_filter_uncharge(sk, filter); 1924 ret = 0; 1925 } 1926 1927 return ret; 1928 } 1929 EXPORT_SYMBOL_GPL(sk_detach_filter); 1930 1931 int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf, 1932 unsigned int len) 1933 { 1934 struct sock_fprog_kern *fprog; 1935 struct sk_filter *filter; 1936 int ret = 0; 1937 1938 lock_sock(sk); 1939 filter = rcu_dereference_protected(sk->sk_filter, 1940 sock_owned_by_user(sk)); 1941 if (!filter) 1942 goto out; 1943 1944 /* We're copying the filter that has been originally attached, 1945 * so no conversion/decode needed anymore. eBPF programs that 1946 * have no original program cannot be dumped through this. 1947 */ 1948 ret = -EACCES; 1949 fprog = filter->prog->orig_prog; 1950 if (!fprog) 1951 goto out; 1952 1953 ret = fprog->len; 1954 if (!len) 1955 /* User space only enquires number of filter blocks. */ 1956 goto out; 1957 1958 ret = -EINVAL; 1959 if (len < fprog->len) 1960 goto out; 1961 1962 ret = -EFAULT; 1963 if (copy_to_user(ubuf, fprog->filter, bpf_classic_proglen(fprog))) 1964 goto out; 1965 1966 /* Instead of bytes, the API requests to return the number 1967 * of filter blocks. 1968 */ 1969 ret = fprog->len; 1970 out: 1971 release_sock(sk); 1972 return ret; 1973 } 1974