1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/skbuff.h> 4 #include <linux/export.h> 5 #include <linux/ip.h> 6 #include <linux/ipv6.h> 7 #include <linux/if_vlan.h> 8 #include <net/dsa.h> 9 #include <net/dst_metadata.h> 10 #include <net/ip.h> 11 #include <net/ipv6.h> 12 #include <net/gre.h> 13 #include <net/pptp.h> 14 #include <net/tipc.h> 15 #include <linux/igmp.h> 16 #include <linux/icmp.h> 17 #include <linux/sctp.h> 18 #include <linux/dccp.h> 19 #include <linux/if_tunnel.h> 20 #include <linux/if_pppox.h> 21 #include <linux/ppp_defs.h> 22 #include <linux/stddef.h> 23 #include <linux/if_ether.h> 24 #include <linux/mpls.h> 25 #include <linux/tcp.h> 26 #include <net/flow_dissector.h> 27 #include <scsi/fc/fc_fcoe.h> 28 #include <uapi/linux/batadv_packet.h> 29 #include <linux/bpf.h> 30 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 31 #include <net/netfilter/nf_conntrack_core.h> 32 #include <net/netfilter/nf_conntrack_labels.h> 33 #endif 34 35 static DEFINE_MUTEX(flow_dissector_mutex); 36 37 static void dissector_set_key(struct flow_dissector *flow_dissector, 38 enum flow_dissector_key_id key_id) 39 { 40 flow_dissector->used_keys |= (1 << key_id); 41 } 42 43 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 44 const struct flow_dissector_key *key, 45 unsigned int key_count) 46 { 47 unsigned int i; 48 49 memset(flow_dissector, 0, sizeof(*flow_dissector)); 50 51 for (i = 0; i < key_count; i++, key++) { 52 /* User should make sure that every key target offset is withing 53 * boundaries of unsigned short. 54 */ 55 BUG_ON(key->offset > USHRT_MAX); 56 BUG_ON(dissector_uses_key(flow_dissector, 57 key->key_id)); 58 59 dissector_set_key(flow_dissector, key->key_id); 60 flow_dissector->offset[key->key_id] = key->offset; 61 } 62 63 /* Ensure that the dissector always includes control and basic key. 64 * That way we are able to avoid handling lack of these in fast path. 65 */ 66 BUG_ON(!dissector_uses_key(flow_dissector, 67 FLOW_DISSECTOR_KEY_CONTROL)); 68 BUG_ON(!dissector_uses_key(flow_dissector, 69 FLOW_DISSECTOR_KEY_BASIC)); 70 } 71 EXPORT_SYMBOL(skb_flow_dissector_init); 72 73 int skb_flow_dissector_prog_query(const union bpf_attr *attr, 74 union bpf_attr __user *uattr) 75 { 76 __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); 77 u32 prog_id, prog_cnt = 0, flags = 0; 78 struct bpf_prog *attached; 79 struct net *net; 80 81 if (attr->query.query_flags) 82 return -EINVAL; 83 84 net = get_net_ns_by_fd(attr->query.target_fd); 85 if (IS_ERR(net)) 86 return PTR_ERR(net); 87 88 rcu_read_lock(); 89 attached = rcu_dereference(net->flow_dissector_prog); 90 if (attached) { 91 prog_cnt = 1; 92 prog_id = attached->aux->id; 93 } 94 rcu_read_unlock(); 95 96 put_net(net); 97 98 if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) 99 return -EFAULT; 100 if (copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt))) 101 return -EFAULT; 102 103 if (!attr->query.prog_cnt || !prog_ids || !prog_cnt) 104 return 0; 105 106 if (copy_to_user(prog_ids, &prog_id, sizeof(u32))) 107 return -EFAULT; 108 109 return 0; 110 } 111 112 int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr, 113 struct bpf_prog *prog) 114 { 115 struct bpf_prog *attached; 116 struct net *net; 117 118 net = current->nsproxy->net_ns; 119 mutex_lock(&flow_dissector_mutex); 120 attached = rcu_dereference_protected(net->flow_dissector_prog, 121 lockdep_is_held(&flow_dissector_mutex)); 122 if (attached) { 123 /* Only one BPF program can be attached at a time */ 124 mutex_unlock(&flow_dissector_mutex); 125 return -EEXIST; 126 } 127 rcu_assign_pointer(net->flow_dissector_prog, prog); 128 mutex_unlock(&flow_dissector_mutex); 129 return 0; 130 } 131 132 int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr) 133 { 134 struct bpf_prog *attached; 135 struct net *net; 136 137 net = current->nsproxy->net_ns; 138 mutex_lock(&flow_dissector_mutex); 139 attached = rcu_dereference_protected(net->flow_dissector_prog, 140 lockdep_is_held(&flow_dissector_mutex)); 141 if (!attached) { 142 mutex_unlock(&flow_dissector_mutex); 143 return -ENOENT; 144 } 145 RCU_INIT_POINTER(net->flow_dissector_prog, NULL); 146 bpf_prog_put(attached); 147 mutex_unlock(&flow_dissector_mutex); 148 return 0; 149 } 150 /** 151 * skb_flow_get_be16 - extract be16 entity 152 * @skb: sk_buff to extract from 153 * @poff: offset to extract at 154 * @data: raw buffer pointer to the packet 155 * @hlen: packet header length 156 * 157 * The function will try to retrieve a be32 entity at 158 * offset poff 159 */ 160 static __be16 skb_flow_get_be16(const struct sk_buff *skb, int poff, 161 void *data, int hlen) 162 { 163 __be16 *u, _u; 164 165 u = __skb_header_pointer(skb, poff, sizeof(_u), data, hlen, &_u); 166 if (u) 167 return *u; 168 169 return 0; 170 } 171 172 /** 173 * __skb_flow_get_ports - extract the upper layer ports and return them 174 * @skb: sk_buff to extract the ports from 175 * @thoff: transport header offset 176 * @ip_proto: protocol for which to get port offset 177 * @data: raw buffer pointer to the packet, if NULL use skb->data 178 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 179 * 180 * The function will try to retrieve the ports at offset thoff + poff where poff 181 * is the protocol port offset returned from proto_ports_offset 182 */ 183 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 184 void *data, int hlen) 185 { 186 int poff = proto_ports_offset(ip_proto); 187 188 if (!data) { 189 data = skb->data; 190 hlen = skb_headlen(skb); 191 } 192 193 if (poff >= 0) { 194 __be32 *ports, _ports; 195 196 ports = __skb_header_pointer(skb, thoff + poff, 197 sizeof(_ports), data, hlen, &_ports); 198 if (ports) 199 return *ports; 200 } 201 202 return 0; 203 } 204 EXPORT_SYMBOL(__skb_flow_get_ports); 205 206 void skb_flow_dissect_meta(const struct sk_buff *skb, 207 struct flow_dissector *flow_dissector, 208 void *target_container) 209 { 210 struct flow_dissector_key_meta *meta; 211 212 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META)) 213 return; 214 215 meta = skb_flow_dissector_target(flow_dissector, 216 FLOW_DISSECTOR_KEY_META, 217 target_container); 218 meta->ingress_ifindex = skb->skb_iif; 219 } 220 EXPORT_SYMBOL(skb_flow_dissect_meta); 221 222 static void 223 skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type, 224 struct flow_dissector *flow_dissector, 225 void *target_container) 226 { 227 struct flow_dissector_key_control *ctrl; 228 229 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) 230 return; 231 232 ctrl = skb_flow_dissector_target(flow_dissector, 233 FLOW_DISSECTOR_KEY_ENC_CONTROL, 234 target_container); 235 ctrl->addr_type = type; 236 } 237 238 void 239 skb_flow_dissect_ct(const struct sk_buff *skb, 240 struct flow_dissector *flow_dissector, 241 void *target_container, 242 u16 *ctinfo_map, 243 size_t mapsize) 244 { 245 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 246 struct flow_dissector_key_ct *key; 247 enum ip_conntrack_info ctinfo; 248 struct nf_conn_labels *cl; 249 struct nf_conn *ct; 250 251 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT)) 252 return; 253 254 ct = nf_ct_get(skb, &ctinfo); 255 if (!ct) 256 return; 257 258 key = skb_flow_dissector_target(flow_dissector, 259 FLOW_DISSECTOR_KEY_CT, 260 target_container); 261 262 if (ctinfo < mapsize) 263 key->ct_state = ctinfo_map[ctinfo]; 264 #if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) 265 key->ct_zone = ct->zone.id; 266 #endif 267 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) 268 key->ct_mark = ct->mark; 269 #endif 270 271 cl = nf_ct_labels_find(ct); 272 if (cl) 273 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels)); 274 #endif /* CONFIG_NF_CONNTRACK */ 275 } 276 EXPORT_SYMBOL(skb_flow_dissect_ct); 277 278 void 279 skb_flow_dissect_tunnel_info(const struct sk_buff *skb, 280 struct flow_dissector *flow_dissector, 281 void *target_container) 282 { 283 struct ip_tunnel_info *info; 284 struct ip_tunnel_key *key; 285 286 /* A quick check to see if there might be something to do. */ 287 if (!dissector_uses_key(flow_dissector, 288 FLOW_DISSECTOR_KEY_ENC_KEYID) && 289 !dissector_uses_key(flow_dissector, 290 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) && 291 !dissector_uses_key(flow_dissector, 292 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) && 293 !dissector_uses_key(flow_dissector, 294 FLOW_DISSECTOR_KEY_ENC_CONTROL) && 295 !dissector_uses_key(flow_dissector, 296 FLOW_DISSECTOR_KEY_ENC_PORTS) && 297 !dissector_uses_key(flow_dissector, 298 FLOW_DISSECTOR_KEY_ENC_IP) && 299 !dissector_uses_key(flow_dissector, 300 FLOW_DISSECTOR_KEY_ENC_OPTS)) 301 return; 302 303 info = skb_tunnel_info(skb); 304 if (!info) 305 return; 306 307 key = &info->key; 308 309 switch (ip_tunnel_info_af(info)) { 310 case AF_INET: 311 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS, 312 flow_dissector, 313 target_container); 314 if (dissector_uses_key(flow_dissector, 315 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { 316 struct flow_dissector_key_ipv4_addrs *ipv4; 317 318 ipv4 = skb_flow_dissector_target(flow_dissector, 319 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 320 target_container); 321 ipv4->src = key->u.ipv4.src; 322 ipv4->dst = key->u.ipv4.dst; 323 } 324 break; 325 case AF_INET6: 326 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS, 327 flow_dissector, 328 target_container); 329 if (dissector_uses_key(flow_dissector, 330 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { 331 struct flow_dissector_key_ipv6_addrs *ipv6; 332 333 ipv6 = skb_flow_dissector_target(flow_dissector, 334 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, 335 target_container); 336 ipv6->src = key->u.ipv6.src; 337 ipv6->dst = key->u.ipv6.dst; 338 } 339 break; 340 } 341 342 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 343 struct flow_dissector_key_keyid *keyid; 344 345 keyid = skb_flow_dissector_target(flow_dissector, 346 FLOW_DISSECTOR_KEY_ENC_KEYID, 347 target_container); 348 keyid->keyid = tunnel_id_to_key32(key->tun_id); 349 } 350 351 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 352 struct flow_dissector_key_ports *tp; 353 354 tp = skb_flow_dissector_target(flow_dissector, 355 FLOW_DISSECTOR_KEY_ENC_PORTS, 356 target_container); 357 tp->src = key->tp_src; 358 tp->dst = key->tp_dst; 359 } 360 361 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) { 362 struct flow_dissector_key_ip *ip; 363 364 ip = skb_flow_dissector_target(flow_dissector, 365 FLOW_DISSECTOR_KEY_ENC_IP, 366 target_container); 367 ip->tos = key->tos; 368 ip->ttl = key->ttl; 369 } 370 371 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) { 372 struct flow_dissector_key_enc_opts *enc_opt; 373 374 enc_opt = skb_flow_dissector_target(flow_dissector, 375 FLOW_DISSECTOR_KEY_ENC_OPTS, 376 target_container); 377 378 if (info->options_len) { 379 enc_opt->len = info->options_len; 380 ip_tunnel_info_opts_get(enc_opt->data, info); 381 enc_opt->dst_opt_type = info->key.tun_flags & 382 TUNNEL_OPTIONS_PRESENT; 383 } 384 } 385 } 386 EXPORT_SYMBOL(skb_flow_dissect_tunnel_info); 387 388 static enum flow_dissect_ret 389 __skb_flow_dissect_mpls(const struct sk_buff *skb, 390 struct flow_dissector *flow_dissector, 391 void *target_container, void *data, int nhoff, int hlen) 392 { 393 struct flow_dissector_key_keyid *key_keyid; 394 struct mpls_label *hdr, _hdr[2]; 395 u32 entry, label; 396 397 if (!dissector_uses_key(flow_dissector, 398 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) && 399 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) 400 return FLOW_DISSECT_RET_OUT_GOOD; 401 402 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, 403 hlen, &_hdr); 404 if (!hdr) 405 return FLOW_DISSECT_RET_OUT_BAD; 406 407 entry = ntohl(hdr[0].entry); 408 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT; 409 410 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) { 411 struct flow_dissector_key_mpls *key_mpls; 412 413 key_mpls = skb_flow_dissector_target(flow_dissector, 414 FLOW_DISSECTOR_KEY_MPLS, 415 target_container); 416 key_mpls->mpls_label = label; 417 key_mpls->mpls_ttl = (entry & MPLS_LS_TTL_MASK) 418 >> MPLS_LS_TTL_SHIFT; 419 key_mpls->mpls_tc = (entry & MPLS_LS_TC_MASK) 420 >> MPLS_LS_TC_SHIFT; 421 key_mpls->mpls_bos = (entry & MPLS_LS_S_MASK) 422 >> MPLS_LS_S_SHIFT; 423 } 424 425 if (label == MPLS_LABEL_ENTROPY) { 426 key_keyid = skb_flow_dissector_target(flow_dissector, 427 FLOW_DISSECTOR_KEY_MPLS_ENTROPY, 428 target_container); 429 key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK); 430 } 431 return FLOW_DISSECT_RET_OUT_GOOD; 432 } 433 434 static enum flow_dissect_ret 435 __skb_flow_dissect_arp(const struct sk_buff *skb, 436 struct flow_dissector *flow_dissector, 437 void *target_container, void *data, int nhoff, int hlen) 438 { 439 struct flow_dissector_key_arp *key_arp; 440 struct { 441 unsigned char ar_sha[ETH_ALEN]; 442 unsigned char ar_sip[4]; 443 unsigned char ar_tha[ETH_ALEN]; 444 unsigned char ar_tip[4]; 445 } *arp_eth, _arp_eth; 446 const struct arphdr *arp; 447 struct arphdr _arp; 448 449 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP)) 450 return FLOW_DISSECT_RET_OUT_GOOD; 451 452 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data, 453 hlen, &_arp); 454 if (!arp) 455 return FLOW_DISSECT_RET_OUT_BAD; 456 457 if (arp->ar_hrd != htons(ARPHRD_ETHER) || 458 arp->ar_pro != htons(ETH_P_IP) || 459 arp->ar_hln != ETH_ALEN || 460 arp->ar_pln != 4 || 461 (arp->ar_op != htons(ARPOP_REPLY) && 462 arp->ar_op != htons(ARPOP_REQUEST))) 463 return FLOW_DISSECT_RET_OUT_BAD; 464 465 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp), 466 sizeof(_arp_eth), data, 467 hlen, &_arp_eth); 468 if (!arp_eth) 469 return FLOW_DISSECT_RET_OUT_BAD; 470 471 key_arp = skb_flow_dissector_target(flow_dissector, 472 FLOW_DISSECTOR_KEY_ARP, 473 target_container); 474 475 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip)); 476 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip)); 477 478 /* Only store the lower byte of the opcode; 479 * this covers ARPOP_REPLY and ARPOP_REQUEST. 480 */ 481 key_arp->op = ntohs(arp->ar_op) & 0xff; 482 483 ether_addr_copy(key_arp->sha, arp_eth->ar_sha); 484 ether_addr_copy(key_arp->tha, arp_eth->ar_tha); 485 486 return FLOW_DISSECT_RET_OUT_GOOD; 487 } 488 489 static enum flow_dissect_ret 490 __skb_flow_dissect_gre(const struct sk_buff *skb, 491 struct flow_dissector_key_control *key_control, 492 struct flow_dissector *flow_dissector, 493 void *target_container, void *data, 494 __be16 *p_proto, int *p_nhoff, int *p_hlen, 495 unsigned int flags) 496 { 497 struct flow_dissector_key_keyid *key_keyid; 498 struct gre_base_hdr *hdr, _hdr; 499 int offset = 0; 500 u16 gre_ver; 501 502 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), 503 data, *p_hlen, &_hdr); 504 if (!hdr) 505 return FLOW_DISSECT_RET_OUT_BAD; 506 507 /* Only look inside GRE without routing */ 508 if (hdr->flags & GRE_ROUTING) 509 return FLOW_DISSECT_RET_OUT_GOOD; 510 511 /* Only look inside GRE for version 0 and 1 */ 512 gre_ver = ntohs(hdr->flags & GRE_VERSION); 513 if (gre_ver > 1) 514 return FLOW_DISSECT_RET_OUT_GOOD; 515 516 *p_proto = hdr->protocol; 517 if (gre_ver) { 518 /* Version1 must be PPTP, and check the flags */ 519 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY))) 520 return FLOW_DISSECT_RET_OUT_GOOD; 521 } 522 523 offset += sizeof(struct gre_base_hdr); 524 525 if (hdr->flags & GRE_CSUM) 526 offset += FIELD_SIZEOF(struct gre_full_hdr, csum) + 527 FIELD_SIZEOF(struct gre_full_hdr, reserved1); 528 529 if (hdr->flags & GRE_KEY) { 530 const __be32 *keyid; 531 __be32 _keyid; 532 533 keyid = __skb_header_pointer(skb, *p_nhoff + offset, 534 sizeof(_keyid), 535 data, *p_hlen, &_keyid); 536 if (!keyid) 537 return FLOW_DISSECT_RET_OUT_BAD; 538 539 if (dissector_uses_key(flow_dissector, 540 FLOW_DISSECTOR_KEY_GRE_KEYID)) { 541 key_keyid = skb_flow_dissector_target(flow_dissector, 542 FLOW_DISSECTOR_KEY_GRE_KEYID, 543 target_container); 544 if (gre_ver == 0) 545 key_keyid->keyid = *keyid; 546 else 547 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK; 548 } 549 offset += FIELD_SIZEOF(struct gre_full_hdr, key); 550 } 551 552 if (hdr->flags & GRE_SEQ) 553 offset += FIELD_SIZEOF(struct pptp_gre_header, seq); 554 555 if (gre_ver == 0) { 556 if (*p_proto == htons(ETH_P_TEB)) { 557 const struct ethhdr *eth; 558 struct ethhdr _eth; 559 560 eth = __skb_header_pointer(skb, *p_nhoff + offset, 561 sizeof(_eth), 562 data, *p_hlen, &_eth); 563 if (!eth) 564 return FLOW_DISSECT_RET_OUT_BAD; 565 *p_proto = eth->h_proto; 566 offset += sizeof(*eth); 567 568 /* Cap headers that we access via pointers at the 569 * end of the Ethernet header as our maximum alignment 570 * at that point is only 2 bytes. 571 */ 572 if (NET_IP_ALIGN) 573 *p_hlen = *p_nhoff + offset; 574 } 575 } else { /* version 1, must be PPTP */ 576 u8 _ppp_hdr[PPP_HDRLEN]; 577 u8 *ppp_hdr; 578 579 if (hdr->flags & GRE_ACK) 580 offset += FIELD_SIZEOF(struct pptp_gre_header, ack); 581 582 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset, 583 sizeof(_ppp_hdr), 584 data, *p_hlen, _ppp_hdr); 585 if (!ppp_hdr) 586 return FLOW_DISSECT_RET_OUT_BAD; 587 588 switch (PPP_PROTOCOL(ppp_hdr)) { 589 case PPP_IP: 590 *p_proto = htons(ETH_P_IP); 591 break; 592 case PPP_IPV6: 593 *p_proto = htons(ETH_P_IPV6); 594 break; 595 default: 596 /* Could probably catch some more like MPLS */ 597 break; 598 } 599 600 offset += PPP_HDRLEN; 601 } 602 603 *p_nhoff += offset; 604 key_control->flags |= FLOW_DIS_ENCAPSULATION; 605 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 606 return FLOW_DISSECT_RET_OUT_GOOD; 607 608 return FLOW_DISSECT_RET_PROTO_AGAIN; 609 } 610 611 /** 612 * __skb_flow_dissect_batadv() - dissect batman-adv header 613 * @skb: sk_buff to with the batman-adv header 614 * @key_control: flow dissectors control key 615 * @data: raw buffer pointer to the packet, if NULL use skb->data 616 * @p_proto: pointer used to update the protocol to process next 617 * @p_nhoff: pointer used to update inner network header offset 618 * @hlen: packet header length 619 * @flags: any combination of FLOW_DISSECTOR_F_* 620 * 621 * ETH_P_BATMAN packets are tried to be dissected. Only 622 * &struct batadv_unicast packets are actually processed because they contain an 623 * inner ethernet header and are usually followed by actual network header. This 624 * allows the flow dissector to continue processing the packet. 625 * 626 * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found, 627 * FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation, 628 * otherwise FLOW_DISSECT_RET_OUT_BAD 629 */ 630 static enum flow_dissect_ret 631 __skb_flow_dissect_batadv(const struct sk_buff *skb, 632 struct flow_dissector_key_control *key_control, 633 void *data, __be16 *p_proto, int *p_nhoff, int hlen, 634 unsigned int flags) 635 { 636 struct { 637 struct batadv_unicast_packet batadv_unicast; 638 struct ethhdr eth; 639 } *hdr, _hdr; 640 641 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen, 642 &_hdr); 643 if (!hdr) 644 return FLOW_DISSECT_RET_OUT_BAD; 645 646 if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION) 647 return FLOW_DISSECT_RET_OUT_BAD; 648 649 if (hdr->batadv_unicast.packet_type != BATADV_UNICAST) 650 return FLOW_DISSECT_RET_OUT_BAD; 651 652 *p_proto = hdr->eth.h_proto; 653 *p_nhoff += sizeof(*hdr); 654 655 key_control->flags |= FLOW_DIS_ENCAPSULATION; 656 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 657 return FLOW_DISSECT_RET_OUT_GOOD; 658 659 return FLOW_DISSECT_RET_PROTO_AGAIN; 660 } 661 662 static void 663 __skb_flow_dissect_tcp(const struct sk_buff *skb, 664 struct flow_dissector *flow_dissector, 665 void *target_container, void *data, int thoff, int hlen) 666 { 667 struct flow_dissector_key_tcp *key_tcp; 668 struct tcphdr *th, _th; 669 670 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP)) 671 return; 672 673 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th); 674 if (!th) 675 return; 676 677 if (unlikely(__tcp_hdrlen(th) < sizeof(_th))) 678 return; 679 680 key_tcp = skb_flow_dissector_target(flow_dissector, 681 FLOW_DISSECTOR_KEY_TCP, 682 target_container); 683 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF)); 684 } 685 686 static void 687 __skb_flow_dissect_ipv4(const struct sk_buff *skb, 688 struct flow_dissector *flow_dissector, 689 void *target_container, void *data, const struct iphdr *iph) 690 { 691 struct flow_dissector_key_ip *key_ip; 692 693 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 694 return; 695 696 key_ip = skb_flow_dissector_target(flow_dissector, 697 FLOW_DISSECTOR_KEY_IP, 698 target_container); 699 key_ip->tos = iph->tos; 700 key_ip->ttl = iph->ttl; 701 } 702 703 static void 704 __skb_flow_dissect_ipv6(const struct sk_buff *skb, 705 struct flow_dissector *flow_dissector, 706 void *target_container, void *data, const struct ipv6hdr *iph) 707 { 708 struct flow_dissector_key_ip *key_ip; 709 710 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 711 return; 712 713 key_ip = skb_flow_dissector_target(flow_dissector, 714 FLOW_DISSECTOR_KEY_IP, 715 target_container); 716 key_ip->tos = ipv6_get_dsfield(iph); 717 key_ip->ttl = iph->hop_limit; 718 } 719 720 /* Maximum number of protocol headers that can be parsed in 721 * __skb_flow_dissect 722 */ 723 #define MAX_FLOW_DISSECT_HDRS 15 724 725 static bool skb_flow_dissect_allowed(int *num_hdrs) 726 { 727 ++*num_hdrs; 728 729 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS); 730 } 731 732 static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys, 733 struct flow_dissector *flow_dissector, 734 void *target_container) 735 { 736 struct flow_dissector_key_control *key_control; 737 struct flow_dissector_key_basic *key_basic; 738 struct flow_dissector_key_addrs *key_addrs; 739 struct flow_dissector_key_ports *key_ports; 740 struct flow_dissector_key_tags *key_tags; 741 742 key_control = skb_flow_dissector_target(flow_dissector, 743 FLOW_DISSECTOR_KEY_CONTROL, 744 target_container); 745 key_control->thoff = flow_keys->thoff; 746 if (flow_keys->is_frag) 747 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 748 if (flow_keys->is_first_frag) 749 key_control->flags |= FLOW_DIS_FIRST_FRAG; 750 if (flow_keys->is_encap) 751 key_control->flags |= FLOW_DIS_ENCAPSULATION; 752 753 key_basic = skb_flow_dissector_target(flow_dissector, 754 FLOW_DISSECTOR_KEY_BASIC, 755 target_container); 756 key_basic->n_proto = flow_keys->n_proto; 757 key_basic->ip_proto = flow_keys->ip_proto; 758 759 if (flow_keys->addr_proto == ETH_P_IP && 760 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 761 key_addrs = skb_flow_dissector_target(flow_dissector, 762 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 763 target_container); 764 key_addrs->v4addrs.src = flow_keys->ipv4_src; 765 key_addrs->v4addrs.dst = flow_keys->ipv4_dst; 766 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 767 } else if (flow_keys->addr_proto == ETH_P_IPV6 && 768 dissector_uses_key(flow_dissector, 769 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 770 key_addrs = skb_flow_dissector_target(flow_dissector, 771 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 772 target_container); 773 memcpy(&key_addrs->v6addrs, &flow_keys->ipv6_src, 774 sizeof(key_addrs->v6addrs)); 775 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 776 } 777 778 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS)) { 779 key_ports = skb_flow_dissector_target(flow_dissector, 780 FLOW_DISSECTOR_KEY_PORTS, 781 target_container); 782 key_ports->src = flow_keys->sport; 783 key_ports->dst = flow_keys->dport; 784 } 785 786 if (dissector_uses_key(flow_dissector, 787 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 788 key_tags = skb_flow_dissector_target(flow_dissector, 789 FLOW_DISSECTOR_KEY_FLOW_LABEL, 790 target_container); 791 key_tags->flow_label = ntohl(flow_keys->flow_label); 792 } 793 } 794 795 bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, 796 __be16 proto, int nhoff, int hlen, unsigned int flags) 797 { 798 struct bpf_flow_keys *flow_keys = ctx->flow_keys; 799 u32 result; 800 801 /* Pass parameters to the BPF program */ 802 memset(flow_keys, 0, sizeof(*flow_keys)); 803 flow_keys->n_proto = proto; 804 flow_keys->nhoff = nhoff; 805 flow_keys->thoff = flow_keys->nhoff; 806 807 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG != 808 (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG); 809 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL != 810 (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 811 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP != 812 (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP); 813 flow_keys->flags = flags; 814 815 preempt_disable(); 816 result = BPF_PROG_RUN(prog, ctx); 817 preempt_enable(); 818 819 flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen); 820 flow_keys->thoff = clamp_t(u16, flow_keys->thoff, 821 flow_keys->nhoff, hlen); 822 823 return result == BPF_OK; 824 } 825 826 /** 827 * __skb_flow_dissect - extract the flow_keys struct and return it 828 * @net: associated network namespace, derived from @skb if NULL 829 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 830 * @flow_dissector: list of keys to dissect 831 * @target_container: target structure to put dissected values into 832 * @data: raw buffer pointer to the packet, if NULL use skb->data 833 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 834 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 835 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 836 * @flags: flags that control the dissection process, e.g. 837 * FLOW_DISSECTOR_F_STOP_AT_ENCAP. 838 * 839 * The function will try to retrieve individual keys into target specified 840 * by flow_dissector from either the skbuff or a raw buffer specified by the 841 * rest parameters. 842 * 843 * Caller must take care of zeroing target container memory. 844 */ 845 bool __skb_flow_dissect(const struct net *net, 846 const struct sk_buff *skb, 847 struct flow_dissector *flow_dissector, 848 void *target_container, 849 void *data, __be16 proto, int nhoff, int hlen, 850 unsigned int flags) 851 { 852 struct flow_dissector_key_control *key_control; 853 struct flow_dissector_key_basic *key_basic; 854 struct flow_dissector_key_addrs *key_addrs; 855 struct flow_dissector_key_ports *key_ports; 856 struct flow_dissector_key_icmp *key_icmp; 857 struct flow_dissector_key_tags *key_tags; 858 struct flow_dissector_key_vlan *key_vlan; 859 struct bpf_prog *attached = NULL; 860 enum flow_dissect_ret fdret; 861 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX; 862 int num_hdrs = 0; 863 u8 ip_proto = 0; 864 bool ret; 865 866 if (!data) { 867 data = skb->data; 868 proto = skb_vlan_tag_present(skb) ? 869 skb->vlan_proto : skb->protocol; 870 nhoff = skb_network_offset(skb); 871 hlen = skb_headlen(skb); 872 #if IS_ENABLED(CONFIG_NET_DSA) 873 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev))) { 874 const struct dsa_device_ops *ops; 875 int offset; 876 877 ops = skb->dev->dsa_ptr->tag_ops; 878 if (ops->flow_dissect && 879 !ops->flow_dissect(skb, &proto, &offset)) { 880 hlen -= offset; 881 nhoff += offset; 882 } 883 } 884 #endif 885 } 886 887 /* It is ensured by skb_flow_dissector_init() that control key will 888 * be always present. 889 */ 890 key_control = skb_flow_dissector_target(flow_dissector, 891 FLOW_DISSECTOR_KEY_CONTROL, 892 target_container); 893 894 /* It is ensured by skb_flow_dissector_init() that basic key will 895 * be always present. 896 */ 897 key_basic = skb_flow_dissector_target(flow_dissector, 898 FLOW_DISSECTOR_KEY_BASIC, 899 target_container); 900 901 if (skb) { 902 if (!net) { 903 if (skb->dev) 904 net = dev_net(skb->dev); 905 else if (skb->sk) 906 net = sock_net(skb->sk); 907 } 908 } 909 910 WARN_ON_ONCE(!net); 911 if (net) { 912 rcu_read_lock(); 913 attached = rcu_dereference(net->flow_dissector_prog); 914 915 if (attached) { 916 struct bpf_flow_keys flow_keys; 917 struct bpf_flow_dissector ctx = { 918 .flow_keys = &flow_keys, 919 .data = data, 920 .data_end = data + hlen, 921 }; 922 __be16 n_proto = proto; 923 924 if (skb) { 925 ctx.skb = skb; 926 /* we can't use 'proto' in the skb case 927 * because it might be set to skb->vlan_proto 928 * which has been pulled from the data 929 */ 930 n_proto = skb->protocol; 931 } 932 933 ret = bpf_flow_dissect(attached, &ctx, n_proto, nhoff, 934 hlen, flags); 935 __skb_flow_bpf_to_target(&flow_keys, flow_dissector, 936 target_container); 937 rcu_read_unlock(); 938 return ret; 939 } 940 rcu_read_unlock(); 941 } 942 943 if (dissector_uses_key(flow_dissector, 944 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 945 struct ethhdr *eth = eth_hdr(skb); 946 struct flow_dissector_key_eth_addrs *key_eth_addrs; 947 948 key_eth_addrs = skb_flow_dissector_target(flow_dissector, 949 FLOW_DISSECTOR_KEY_ETH_ADDRS, 950 target_container); 951 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs)); 952 } 953 954 proto_again: 955 fdret = FLOW_DISSECT_RET_CONTINUE; 956 957 switch (proto) { 958 case htons(ETH_P_IP): { 959 const struct iphdr *iph; 960 struct iphdr _iph; 961 962 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 963 if (!iph || iph->ihl < 5) { 964 fdret = FLOW_DISSECT_RET_OUT_BAD; 965 break; 966 } 967 968 nhoff += iph->ihl * 4; 969 970 ip_proto = iph->protocol; 971 972 if (dissector_uses_key(flow_dissector, 973 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 974 key_addrs = skb_flow_dissector_target(flow_dissector, 975 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 976 target_container); 977 978 memcpy(&key_addrs->v4addrs, &iph->saddr, 979 sizeof(key_addrs->v4addrs)); 980 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 981 } 982 983 if (ip_is_fragment(iph)) { 984 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 985 986 if (iph->frag_off & htons(IP_OFFSET)) { 987 fdret = FLOW_DISSECT_RET_OUT_GOOD; 988 break; 989 } else { 990 key_control->flags |= FLOW_DIS_FIRST_FRAG; 991 if (!(flags & 992 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) { 993 fdret = FLOW_DISSECT_RET_OUT_GOOD; 994 break; 995 } 996 } 997 } 998 999 __skb_flow_dissect_ipv4(skb, flow_dissector, 1000 target_container, data, iph); 1001 1002 break; 1003 } 1004 case htons(ETH_P_IPV6): { 1005 const struct ipv6hdr *iph; 1006 struct ipv6hdr _iph; 1007 1008 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 1009 if (!iph) { 1010 fdret = FLOW_DISSECT_RET_OUT_BAD; 1011 break; 1012 } 1013 1014 ip_proto = iph->nexthdr; 1015 nhoff += sizeof(struct ipv6hdr); 1016 1017 if (dissector_uses_key(flow_dissector, 1018 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 1019 key_addrs = skb_flow_dissector_target(flow_dissector, 1020 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1021 target_container); 1022 1023 memcpy(&key_addrs->v6addrs, &iph->saddr, 1024 sizeof(key_addrs->v6addrs)); 1025 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1026 } 1027 1028 if ((dissector_uses_key(flow_dissector, 1029 FLOW_DISSECTOR_KEY_FLOW_LABEL) || 1030 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) && 1031 ip6_flowlabel(iph)) { 1032 __be32 flow_label = ip6_flowlabel(iph); 1033 1034 if (dissector_uses_key(flow_dissector, 1035 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 1036 key_tags = skb_flow_dissector_target(flow_dissector, 1037 FLOW_DISSECTOR_KEY_FLOW_LABEL, 1038 target_container); 1039 key_tags->flow_label = ntohl(flow_label); 1040 } 1041 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) { 1042 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1043 break; 1044 } 1045 } 1046 1047 __skb_flow_dissect_ipv6(skb, flow_dissector, 1048 target_container, data, iph); 1049 1050 break; 1051 } 1052 case htons(ETH_P_8021AD): 1053 case htons(ETH_P_8021Q): { 1054 const struct vlan_hdr *vlan = NULL; 1055 struct vlan_hdr _vlan; 1056 __be16 saved_vlan_tpid = proto; 1057 1058 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX && 1059 skb && skb_vlan_tag_present(skb)) { 1060 proto = skb->protocol; 1061 } else { 1062 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), 1063 data, hlen, &_vlan); 1064 if (!vlan) { 1065 fdret = FLOW_DISSECT_RET_OUT_BAD; 1066 break; 1067 } 1068 1069 proto = vlan->h_vlan_encapsulated_proto; 1070 nhoff += sizeof(*vlan); 1071 } 1072 1073 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) { 1074 dissector_vlan = FLOW_DISSECTOR_KEY_VLAN; 1075 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) { 1076 dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN; 1077 } else { 1078 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1079 break; 1080 } 1081 1082 if (dissector_uses_key(flow_dissector, dissector_vlan)) { 1083 key_vlan = skb_flow_dissector_target(flow_dissector, 1084 dissector_vlan, 1085 target_container); 1086 1087 if (!vlan) { 1088 key_vlan->vlan_id = skb_vlan_tag_get_id(skb); 1089 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb); 1090 } else { 1091 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) & 1092 VLAN_VID_MASK; 1093 key_vlan->vlan_priority = 1094 (ntohs(vlan->h_vlan_TCI) & 1095 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1096 } 1097 key_vlan->vlan_tpid = saved_vlan_tpid; 1098 } 1099 1100 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1101 break; 1102 } 1103 case htons(ETH_P_PPP_SES): { 1104 struct { 1105 struct pppoe_hdr hdr; 1106 __be16 proto; 1107 } *hdr, _hdr; 1108 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 1109 if (!hdr) { 1110 fdret = FLOW_DISSECT_RET_OUT_BAD; 1111 break; 1112 } 1113 1114 proto = hdr->proto; 1115 nhoff += PPPOE_SES_HLEN; 1116 switch (proto) { 1117 case htons(PPP_IP): 1118 proto = htons(ETH_P_IP); 1119 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1120 break; 1121 case htons(PPP_IPV6): 1122 proto = htons(ETH_P_IPV6); 1123 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1124 break; 1125 default: 1126 fdret = FLOW_DISSECT_RET_OUT_BAD; 1127 break; 1128 } 1129 break; 1130 } 1131 case htons(ETH_P_TIPC): { 1132 struct tipc_basic_hdr *hdr, _hdr; 1133 1134 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), 1135 data, hlen, &_hdr); 1136 if (!hdr) { 1137 fdret = FLOW_DISSECT_RET_OUT_BAD; 1138 break; 1139 } 1140 1141 if (dissector_uses_key(flow_dissector, 1142 FLOW_DISSECTOR_KEY_TIPC)) { 1143 key_addrs = skb_flow_dissector_target(flow_dissector, 1144 FLOW_DISSECTOR_KEY_TIPC, 1145 target_container); 1146 key_addrs->tipckey.key = tipc_hdr_rps_key(hdr); 1147 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC; 1148 } 1149 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1150 break; 1151 } 1152 1153 case htons(ETH_P_MPLS_UC): 1154 case htons(ETH_P_MPLS_MC): 1155 fdret = __skb_flow_dissect_mpls(skb, flow_dissector, 1156 target_container, data, 1157 nhoff, hlen); 1158 break; 1159 case htons(ETH_P_FCOE): 1160 if ((hlen - nhoff) < FCOE_HEADER_LEN) { 1161 fdret = FLOW_DISSECT_RET_OUT_BAD; 1162 break; 1163 } 1164 1165 nhoff += FCOE_HEADER_LEN; 1166 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1167 break; 1168 1169 case htons(ETH_P_ARP): 1170 case htons(ETH_P_RARP): 1171 fdret = __skb_flow_dissect_arp(skb, flow_dissector, 1172 target_container, data, 1173 nhoff, hlen); 1174 break; 1175 1176 case htons(ETH_P_BATMAN): 1177 fdret = __skb_flow_dissect_batadv(skb, key_control, data, 1178 &proto, &nhoff, hlen, flags); 1179 break; 1180 1181 default: 1182 fdret = FLOW_DISSECT_RET_OUT_BAD; 1183 break; 1184 } 1185 1186 /* Process result of proto processing */ 1187 switch (fdret) { 1188 case FLOW_DISSECT_RET_OUT_GOOD: 1189 goto out_good; 1190 case FLOW_DISSECT_RET_PROTO_AGAIN: 1191 if (skb_flow_dissect_allowed(&num_hdrs)) 1192 goto proto_again; 1193 goto out_good; 1194 case FLOW_DISSECT_RET_CONTINUE: 1195 case FLOW_DISSECT_RET_IPPROTO_AGAIN: 1196 break; 1197 case FLOW_DISSECT_RET_OUT_BAD: 1198 default: 1199 goto out_bad; 1200 } 1201 1202 ip_proto_again: 1203 fdret = FLOW_DISSECT_RET_CONTINUE; 1204 1205 switch (ip_proto) { 1206 case IPPROTO_GRE: 1207 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector, 1208 target_container, data, 1209 &proto, &nhoff, &hlen, flags); 1210 break; 1211 1212 case NEXTHDR_HOP: 1213 case NEXTHDR_ROUTING: 1214 case NEXTHDR_DEST: { 1215 u8 _opthdr[2], *opthdr; 1216 1217 if (proto != htons(ETH_P_IPV6)) 1218 break; 1219 1220 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr), 1221 data, hlen, &_opthdr); 1222 if (!opthdr) { 1223 fdret = FLOW_DISSECT_RET_OUT_BAD; 1224 break; 1225 } 1226 1227 ip_proto = opthdr[0]; 1228 nhoff += (opthdr[1] + 1) << 3; 1229 1230 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN; 1231 break; 1232 } 1233 case NEXTHDR_FRAGMENT: { 1234 struct frag_hdr _fh, *fh; 1235 1236 if (proto != htons(ETH_P_IPV6)) 1237 break; 1238 1239 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh), 1240 data, hlen, &_fh); 1241 1242 if (!fh) { 1243 fdret = FLOW_DISSECT_RET_OUT_BAD; 1244 break; 1245 } 1246 1247 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 1248 1249 nhoff += sizeof(_fh); 1250 ip_proto = fh->nexthdr; 1251 1252 if (!(fh->frag_off & htons(IP6_OFFSET))) { 1253 key_control->flags |= FLOW_DIS_FIRST_FRAG; 1254 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) { 1255 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN; 1256 break; 1257 } 1258 } 1259 1260 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1261 break; 1262 } 1263 case IPPROTO_IPIP: 1264 proto = htons(ETH_P_IP); 1265 1266 key_control->flags |= FLOW_DIS_ENCAPSULATION; 1267 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) { 1268 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1269 break; 1270 } 1271 1272 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1273 break; 1274 1275 case IPPROTO_IPV6: 1276 proto = htons(ETH_P_IPV6); 1277 1278 key_control->flags |= FLOW_DIS_ENCAPSULATION; 1279 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) { 1280 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1281 break; 1282 } 1283 1284 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1285 break; 1286 1287 1288 case IPPROTO_MPLS: 1289 proto = htons(ETH_P_MPLS_UC); 1290 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1291 break; 1292 1293 case IPPROTO_TCP: 1294 __skb_flow_dissect_tcp(skb, flow_dissector, target_container, 1295 data, nhoff, hlen); 1296 break; 1297 1298 default: 1299 break; 1300 } 1301 1302 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS) && 1303 !(key_control->flags & FLOW_DIS_IS_FRAGMENT)) { 1304 key_ports = skb_flow_dissector_target(flow_dissector, 1305 FLOW_DISSECTOR_KEY_PORTS, 1306 target_container); 1307 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 1308 data, hlen); 1309 } 1310 1311 if (dissector_uses_key(flow_dissector, 1312 FLOW_DISSECTOR_KEY_ICMP)) { 1313 key_icmp = skb_flow_dissector_target(flow_dissector, 1314 FLOW_DISSECTOR_KEY_ICMP, 1315 target_container); 1316 key_icmp->icmp = skb_flow_get_be16(skb, nhoff, data, hlen); 1317 } 1318 1319 /* Process result of IP proto processing */ 1320 switch (fdret) { 1321 case FLOW_DISSECT_RET_PROTO_AGAIN: 1322 if (skb_flow_dissect_allowed(&num_hdrs)) 1323 goto proto_again; 1324 break; 1325 case FLOW_DISSECT_RET_IPPROTO_AGAIN: 1326 if (skb_flow_dissect_allowed(&num_hdrs)) 1327 goto ip_proto_again; 1328 break; 1329 case FLOW_DISSECT_RET_OUT_GOOD: 1330 case FLOW_DISSECT_RET_CONTINUE: 1331 break; 1332 case FLOW_DISSECT_RET_OUT_BAD: 1333 default: 1334 goto out_bad; 1335 } 1336 1337 out_good: 1338 ret = true; 1339 1340 out: 1341 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); 1342 key_basic->n_proto = proto; 1343 key_basic->ip_proto = ip_proto; 1344 1345 return ret; 1346 1347 out_bad: 1348 ret = false; 1349 goto out; 1350 } 1351 EXPORT_SYMBOL(__skb_flow_dissect); 1352 1353 static u32 hashrnd __read_mostly; 1354 static __always_inline void __flow_hash_secret_init(void) 1355 { 1356 net_get_random_once(&hashrnd, sizeof(hashrnd)); 1357 } 1358 1359 static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, 1360 u32 keyval) 1361 { 1362 return jhash2(words, length, keyval); 1363 } 1364 1365 static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) 1366 { 1367 const void *p = flow; 1368 1369 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); 1370 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); 1371 } 1372 1373 static inline size_t flow_keys_hash_length(const struct flow_keys *flow) 1374 { 1375 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); 1376 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); 1377 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != 1378 sizeof(*flow) - sizeof(flow->addrs)); 1379 1380 switch (flow->control.addr_type) { 1381 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1382 diff -= sizeof(flow->addrs.v4addrs); 1383 break; 1384 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1385 diff -= sizeof(flow->addrs.v6addrs); 1386 break; 1387 case FLOW_DISSECTOR_KEY_TIPC: 1388 diff -= sizeof(flow->addrs.tipckey); 1389 break; 1390 } 1391 return (sizeof(*flow) - diff) / sizeof(u32); 1392 } 1393 1394 __be32 flow_get_u32_src(const struct flow_keys *flow) 1395 { 1396 switch (flow->control.addr_type) { 1397 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1398 return flow->addrs.v4addrs.src; 1399 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1400 return (__force __be32)ipv6_addr_hash( 1401 &flow->addrs.v6addrs.src); 1402 case FLOW_DISSECTOR_KEY_TIPC: 1403 return flow->addrs.tipckey.key; 1404 default: 1405 return 0; 1406 } 1407 } 1408 EXPORT_SYMBOL(flow_get_u32_src); 1409 1410 __be32 flow_get_u32_dst(const struct flow_keys *flow) 1411 { 1412 switch (flow->control.addr_type) { 1413 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1414 return flow->addrs.v4addrs.dst; 1415 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1416 return (__force __be32)ipv6_addr_hash( 1417 &flow->addrs.v6addrs.dst); 1418 default: 1419 return 0; 1420 } 1421 } 1422 EXPORT_SYMBOL(flow_get_u32_dst); 1423 1424 static inline void __flow_hash_consistentify(struct flow_keys *keys) 1425 { 1426 int addr_diff, i; 1427 1428 switch (keys->control.addr_type) { 1429 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1430 addr_diff = (__force u32)keys->addrs.v4addrs.dst - 1431 (__force u32)keys->addrs.v4addrs.src; 1432 if ((addr_diff < 0) || 1433 (addr_diff == 0 && 1434 ((__force u16)keys->ports.dst < 1435 (__force u16)keys->ports.src))) { 1436 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); 1437 swap(keys->ports.src, keys->ports.dst); 1438 } 1439 break; 1440 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1441 addr_diff = memcmp(&keys->addrs.v6addrs.dst, 1442 &keys->addrs.v6addrs.src, 1443 sizeof(keys->addrs.v6addrs.dst)); 1444 if ((addr_diff < 0) || 1445 (addr_diff == 0 && 1446 ((__force u16)keys->ports.dst < 1447 (__force u16)keys->ports.src))) { 1448 for (i = 0; i < 4; i++) 1449 swap(keys->addrs.v6addrs.src.s6_addr32[i], 1450 keys->addrs.v6addrs.dst.s6_addr32[i]); 1451 swap(keys->ports.src, keys->ports.dst); 1452 } 1453 break; 1454 } 1455 } 1456 1457 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) 1458 { 1459 u32 hash; 1460 1461 __flow_hash_consistentify(keys); 1462 1463 hash = __flow_hash_words(flow_keys_hash_start(keys), 1464 flow_keys_hash_length(keys), keyval); 1465 if (!hash) 1466 hash = 1; 1467 1468 return hash; 1469 } 1470 1471 u32 flow_hash_from_keys(struct flow_keys *keys) 1472 { 1473 __flow_hash_secret_init(); 1474 return __flow_hash_from_keys(keys, hashrnd); 1475 } 1476 EXPORT_SYMBOL(flow_hash_from_keys); 1477 1478 static inline u32 ___skb_get_hash(const struct sk_buff *skb, 1479 struct flow_keys *keys, u32 keyval) 1480 { 1481 skb_flow_dissect_flow_keys(skb, keys, 1482 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 1483 1484 return __flow_hash_from_keys(keys, keyval); 1485 } 1486 1487 struct _flow_keys_digest_data { 1488 __be16 n_proto; 1489 u8 ip_proto; 1490 u8 padding; 1491 __be32 ports; 1492 __be32 src; 1493 __be32 dst; 1494 }; 1495 1496 void make_flow_keys_digest(struct flow_keys_digest *digest, 1497 const struct flow_keys *flow) 1498 { 1499 struct _flow_keys_digest_data *data = 1500 (struct _flow_keys_digest_data *)digest; 1501 1502 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); 1503 1504 memset(digest, 0, sizeof(*digest)); 1505 1506 data->n_proto = flow->basic.n_proto; 1507 data->ip_proto = flow->basic.ip_proto; 1508 data->ports = flow->ports.ports; 1509 data->src = flow->addrs.v4addrs.src; 1510 data->dst = flow->addrs.v4addrs.dst; 1511 } 1512 EXPORT_SYMBOL(make_flow_keys_digest); 1513 1514 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly; 1515 1516 u32 __skb_get_hash_symmetric(const struct sk_buff *skb) 1517 { 1518 struct flow_keys keys; 1519 1520 __flow_hash_secret_init(); 1521 1522 memset(&keys, 0, sizeof(keys)); 1523 __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric, 1524 &keys, NULL, 0, 0, 0, 1525 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 1526 1527 return __flow_hash_from_keys(&keys, hashrnd); 1528 } 1529 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric); 1530 1531 /** 1532 * __skb_get_hash: calculate a flow hash 1533 * @skb: sk_buff to calculate flow hash from 1534 * 1535 * This function calculates a flow hash based on src/dst addresses 1536 * and src/dst port numbers. Sets hash in skb to non-zero hash value 1537 * on success, zero indicates no valid hash. Also, sets l4_hash in skb 1538 * if hash is a canonical 4-tuple hash over transport ports. 1539 */ 1540 void __skb_get_hash(struct sk_buff *skb) 1541 { 1542 struct flow_keys keys; 1543 u32 hash; 1544 1545 __flow_hash_secret_init(); 1546 1547 hash = ___skb_get_hash(skb, &keys, hashrnd); 1548 1549 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 1550 } 1551 EXPORT_SYMBOL(__skb_get_hash); 1552 1553 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) 1554 { 1555 struct flow_keys keys; 1556 1557 return ___skb_get_hash(skb, &keys, perturb); 1558 } 1559 EXPORT_SYMBOL(skb_get_hash_perturb); 1560 1561 u32 __skb_get_poff(const struct sk_buff *skb, void *data, 1562 const struct flow_keys_basic *keys, int hlen) 1563 { 1564 u32 poff = keys->control.thoff; 1565 1566 /* skip L4 headers for fragments after the first */ 1567 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) && 1568 !(keys->control.flags & FLOW_DIS_FIRST_FRAG)) 1569 return poff; 1570 1571 switch (keys->basic.ip_proto) { 1572 case IPPROTO_TCP: { 1573 /* access doff as u8 to avoid unaligned access */ 1574 const u8 *doff; 1575 u8 _doff; 1576 1577 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 1578 data, hlen, &_doff); 1579 if (!doff) 1580 return poff; 1581 1582 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 1583 break; 1584 } 1585 case IPPROTO_UDP: 1586 case IPPROTO_UDPLITE: 1587 poff += sizeof(struct udphdr); 1588 break; 1589 /* For the rest, we do not really care about header 1590 * extensions at this point for now. 1591 */ 1592 case IPPROTO_ICMP: 1593 poff += sizeof(struct icmphdr); 1594 break; 1595 case IPPROTO_ICMPV6: 1596 poff += sizeof(struct icmp6hdr); 1597 break; 1598 case IPPROTO_IGMP: 1599 poff += sizeof(struct igmphdr); 1600 break; 1601 case IPPROTO_DCCP: 1602 poff += sizeof(struct dccp_hdr); 1603 break; 1604 case IPPROTO_SCTP: 1605 poff += sizeof(struct sctphdr); 1606 break; 1607 } 1608 1609 return poff; 1610 } 1611 1612 /** 1613 * skb_get_poff - get the offset to the payload 1614 * @skb: sk_buff to get the payload offset from 1615 * 1616 * The function will get the offset to the payload as far as it could 1617 * be dissected. The main user is currently BPF, so that we can dynamically 1618 * truncate packets without needing to push actual payload to the user 1619 * space and can analyze headers only, instead. 1620 */ 1621 u32 skb_get_poff(const struct sk_buff *skb) 1622 { 1623 struct flow_keys_basic keys; 1624 1625 if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, 1626 NULL, 0, 0, 0, 0)) 1627 return 0; 1628 1629 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 1630 } 1631 1632 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) 1633 { 1634 memset(keys, 0, sizeof(*keys)); 1635 1636 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr, 1637 sizeof(keys->addrs.v6addrs.src)); 1638 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr, 1639 sizeof(keys->addrs.v6addrs.dst)); 1640 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1641 keys->ports.src = fl6->fl6_sport; 1642 keys->ports.dst = fl6->fl6_dport; 1643 keys->keyid.keyid = fl6->fl6_gre_key; 1644 keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 1645 keys->basic.ip_proto = fl6->flowi6_proto; 1646 1647 return flow_hash_from_keys(keys); 1648 } 1649 EXPORT_SYMBOL(__get_hash_from_flowi6); 1650 1651 static const struct flow_dissector_key flow_keys_dissector_keys[] = { 1652 { 1653 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1654 .offset = offsetof(struct flow_keys, control), 1655 }, 1656 { 1657 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1658 .offset = offsetof(struct flow_keys, basic), 1659 }, 1660 { 1661 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1662 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1663 }, 1664 { 1665 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1666 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1667 }, 1668 { 1669 .key_id = FLOW_DISSECTOR_KEY_TIPC, 1670 .offset = offsetof(struct flow_keys, addrs.tipckey), 1671 }, 1672 { 1673 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1674 .offset = offsetof(struct flow_keys, ports), 1675 }, 1676 { 1677 .key_id = FLOW_DISSECTOR_KEY_VLAN, 1678 .offset = offsetof(struct flow_keys, vlan), 1679 }, 1680 { 1681 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL, 1682 .offset = offsetof(struct flow_keys, tags), 1683 }, 1684 { 1685 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, 1686 .offset = offsetof(struct flow_keys, keyid), 1687 }, 1688 }; 1689 1690 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = { 1691 { 1692 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1693 .offset = offsetof(struct flow_keys, control), 1694 }, 1695 { 1696 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1697 .offset = offsetof(struct flow_keys, basic), 1698 }, 1699 { 1700 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1701 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1702 }, 1703 { 1704 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1705 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1706 }, 1707 { 1708 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1709 .offset = offsetof(struct flow_keys, ports), 1710 }, 1711 }; 1712 1713 static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = { 1714 { 1715 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1716 .offset = offsetof(struct flow_keys, control), 1717 }, 1718 { 1719 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1720 .offset = offsetof(struct flow_keys, basic), 1721 }, 1722 }; 1723 1724 struct flow_dissector flow_keys_dissector __read_mostly; 1725 EXPORT_SYMBOL(flow_keys_dissector); 1726 1727 struct flow_dissector flow_keys_basic_dissector __read_mostly; 1728 EXPORT_SYMBOL(flow_keys_basic_dissector); 1729 1730 static int __init init_default_flow_dissectors(void) 1731 { 1732 skb_flow_dissector_init(&flow_keys_dissector, 1733 flow_keys_dissector_keys, 1734 ARRAY_SIZE(flow_keys_dissector_keys)); 1735 skb_flow_dissector_init(&flow_keys_dissector_symmetric, 1736 flow_keys_dissector_symmetric_keys, 1737 ARRAY_SIZE(flow_keys_dissector_symmetric_keys)); 1738 skb_flow_dissector_init(&flow_keys_basic_dissector, 1739 flow_keys_basic_dissector_keys, 1740 ARRAY_SIZE(flow_keys_basic_dissector_keys)); 1741 return 0; 1742 } 1743 1744 core_initcall(init_default_flow_dissectors); 1745