1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/skbuff.h> 4 #include <linux/export.h> 5 #include <linux/ip.h> 6 #include <linux/ipv6.h> 7 #include <linux/if_vlan.h> 8 #include <net/dsa.h> 9 #include <net/dst_metadata.h> 10 #include <net/ip.h> 11 #include <net/ipv6.h> 12 #include <net/gre.h> 13 #include <net/pptp.h> 14 #include <net/tipc.h> 15 #include <linux/igmp.h> 16 #include <linux/icmp.h> 17 #include <linux/sctp.h> 18 #include <linux/dccp.h> 19 #include <linux/if_tunnel.h> 20 #include <linux/if_pppox.h> 21 #include <linux/ppp_defs.h> 22 #include <linux/stddef.h> 23 #include <linux/if_ether.h> 24 #include <linux/mpls.h> 25 #include <linux/tcp.h> 26 #include <net/flow_dissector.h> 27 #include <scsi/fc/fc_fcoe.h> 28 #include <uapi/linux/batadv_packet.h> 29 #include <linux/bpf.h> 30 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 31 #include <net/netfilter/nf_conntrack_core.h> 32 #include <net/netfilter/nf_conntrack_labels.h> 33 #endif 34 35 static DEFINE_MUTEX(flow_dissector_mutex); 36 37 static void dissector_set_key(struct flow_dissector *flow_dissector, 38 enum flow_dissector_key_id key_id) 39 { 40 flow_dissector->used_keys |= (1 << key_id); 41 } 42 43 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 44 const struct flow_dissector_key *key, 45 unsigned int key_count) 46 { 47 unsigned int i; 48 49 memset(flow_dissector, 0, sizeof(*flow_dissector)); 50 51 for (i = 0; i < key_count; i++, key++) { 52 /* User should make sure that every key target offset is withing 53 * boundaries of unsigned short. 54 */ 55 BUG_ON(key->offset > USHRT_MAX); 56 BUG_ON(dissector_uses_key(flow_dissector, 57 key->key_id)); 58 59 dissector_set_key(flow_dissector, key->key_id); 60 flow_dissector->offset[key->key_id] = key->offset; 61 } 62 63 /* Ensure that the dissector always includes control and basic key. 64 * That way we are able to avoid handling lack of these in fast path. 65 */ 66 BUG_ON(!dissector_uses_key(flow_dissector, 67 FLOW_DISSECTOR_KEY_CONTROL)); 68 BUG_ON(!dissector_uses_key(flow_dissector, 69 FLOW_DISSECTOR_KEY_BASIC)); 70 } 71 EXPORT_SYMBOL(skb_flow_dissector_init); 72 73 int skb_flow_dissector_prog_query(const union bpf_attr *attr, 74 union bpf_attr __user *uattr) 75 { 76 __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); 77 u32 prog_id, prog_cnt = 0, flags = 0; 78 struct bpf_prog *attached; 79 struct net *net; 80 81 if (attr->query.query_flags) 82 return -EINVAL; 83 84 net = get_net_ns_by_fd(attr->query.target_fd); 85 if (IS_ERR(net)) 86 return PTR_ERR(net); 87 88 rcu_read_lock(); 89 attached = rcu_dereference(net->flow_dissector_prog); 90 if (attached) { 91 prog_cnt = 1; 92 prog_id = attached->aux->id; 93 } 94 rcu_read_unlock(); 95 96 put_net(net); 97 98 if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) 99 return -EFAULT; 100 if (copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt))) 101 return -EFAULT; 102 103 if (!attr->query.prog_cnt || !prog_ids || !prog_cnt) 104 return 0; 105 106 if (copy_to_user(prog_ids, &prog_id, sizeof(u32))) 107 return -EFAULT; 108 109 return 0; 110 } 111 112 int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr, 113 struct bpf_prog *prog) 114 { 115 struct bpf_prog *attached; 116 struct net *net; 117 118 net = current->nsproxy->net_ns; 119 mutex_lock(&flow_dissector_mutex); 120 attached = rcu_dereference_protected(net->flow_dissector_prog, 121 lockdep_is_held(&flow_dissector_mutex)); 122 if (attached) { 123 /* Only one BPF program can be attached at a time */ 124 mutex_unlock(&flow_dissector_mutex); 125 return -EEXIST; 126 } 127 rcu_assign_pointer(net->flow_dissector_prog, prog); 128 mutex_unlock(&flow_dissector_mutex); 129 return 0; 130 } 131 132 int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr) 133 { 134 struct bpf_prog *attached; 135 struct net *net; 136 137 net = current->nsproxy->net_ns; 138 mutex_lock(&flow_dissector_mutex); 139 attached = rcu_dereference_protected(net->flow_dissector_prog, 140 lockdep_is_held(&flow_dissector_mutex)); 141 if (!attached) { 142 mutex_unlock(&flow_dissector_mutex); 143 return -ENOENT; 144 } 145 bpf_prog_put(attached); 146 RCU_INIT_POINTER(net->flow_dissector_prog, NULL); 147 mutex_unlock(&flow_dissector_mutex); 148 return 0; 149 } 150 /** 151 * skb_flow_get_be16 - extract be16 entity 152 * @skb: sk_buff to extract from 153 * @poff: offset to extract at 154 * @data: raw buffer pointer to the packet 155 * @hlen: packet header length 156 * 157 * The function will try to retrieve a be32 entity at 158 * offset poff 159 */ 160 static __be16 skb_flow_get_be16(const struct sk_buff *skb, int poff, 161 void *data, int hlen) 162 { 163 __be16 *u, _u; 164 165 u = __skb_header_pointer(skb, poff, sizeof(_u), data, hlen, &_u); 166 if (u) 167 return *u; 168 169 return 0; 170 } 171 172 /** 173 * __skb_flow_get_ports - extract the upper layer ports and return them 174 * @skb: sk_buff to extract the ports from 175 * @thoff: transport header offset 176 * @ip_proto: protocol for which to get port offset 177 * @data: raw buffer pointer to the packet, if NULL use skb->data 178 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 179 * 180 * The function will try to retrieve the ports at offset thoff + poff where poff 181 * is the protocol port offset returned from proto_ports_offset 182 */ 183 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 184 void *data, int hlen) 185 { 186 int poff = proto_ports_offset(ip_proto); 187 188 if (!data) { 189 data = skb->data; 190 hlen = skb_headlen(skb); 191 } 192 193 if (poff >= 0) { 194 __be32 *ports, _ports; 195 196 ports = __skb_header_pointer(skb, thoff + poff, 197 sizeof(_ports), data, hlen, &_ports); 198 if (ports) 199 return *ports; 200 } 201 202 return 0; 203 } 204 EXPORT_SYMBOL(__skb_flow_get_ports); 205 206 void skb_flow_dissect_meta(const struct sk_buff *skb, 207 struct flow_dissector *flow_dissector, 208 void *target_container) 209 { 210 struct flow_dissector_key_meta *meta; 211 212 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META)) 213 return; 214 215 meta = skb_flow_dissector_target(flow_dissector, 216 FLOW_DISSECTOR_KEY_META, 217 target_container); 218 meta->ingress_ifindex = skb->skb_iif; 219 } 220 EXPORT_SYMBOL(skb_flow_dissect_meta); 221 222 static void 223 skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type, 224 struct flow_dissector *flow_dissector, 225 void *target_container) 226 { 227 struct flow_dissector_key_control *ctrl; 228 229 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) 230 return; 231 232 ctrl = skb_flow_dissector_target(flow_dissector, 233 FLOW_DISSECTOR_KEY_ENC_CONTROL, 234 target_container); 235 ctrl->addr_type = type; 236 } 237 238 void 239 skb_flow_dissect_ct(const struct sk_buff *skb, 240 struct flow_dissector *flow_dissector, 241 void *target_container, 242 u16 *ctinfo_map, 243 size_t mapsize) 244 { 245 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 246 struct flow_dissector_key_ct *key; 247 enum ip_conntrack_info ctinfo; 248 struct nf_conn_labels *cl; 249 struct nf_conn *ct; 250 251 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT)) 252 return; 253 254 ct = nf_ct_get(skb, &ctinfo); 255 if (!ct) 256 return; 257 258 key = skb_flow_dissector_target(flow_dissector, 259 FLOW_DISSECTOR_KEY_CT, 260 target_container); 261 262 if (ctinfo < mapsize) 263 key->ct_state = ctinfo_map[ctinfo]; 264 #if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) 265 key->ct_zone = ct->zone.id; 266 #endif 267 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) 268 key->ct_mark = ct->mark; 269 #endif 270 271 cl = nf_ct_labels_find(ct); 272 if (cl) 273 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels)); 274 #endif /* CONFIG_NF_CONNTRACK */ 275 } 276 EXPORT_SYMBOL(skb_flow_dissect_ct); 277 278 void 279 skb_flow_dissect_tunnel_info(const struct sk_buff *skb, 280 struct flow_dissector *flow_dissector, 281 void *target_container) 282 { 283 struct ip_tunnel_info *info; 284 struct ip_tunnel_key *key; 285 286 /* A quick check to see if there might be something to do. */ 287 if (!dissector_uses_key(flow_dissector, 288 FLOW_DISSECTOR_KEY_ENC_KEYID) && 289 !dissector_uses_key(flow_dissector, 290 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) && 291 !dissector_uses_key(flow_dissector, 292 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) && 293 !dissector_uses_key(flow_dissector, 294 FLOW_DISSECTOR_KEY_ENC_CONTROL) && 295 !dissector_uses_key(flow_dissector, 296 FLOW_DISSECTOR_KEY_ENC_PORTS) && 297 !dissector_uses_key(flow_dissector, 298 FLOW_DISSECTOR_KEY_ENC_IP) && 299 !dissector_uses_key(flow_dissector, 300 FLOW_DISSECTOR_KEY_ENC_OPTS)) 301 return; 302 303 info = skb_tunnel_info(skb); 304 if (!info) 305 return; 306 307 key = &info->key; 308 309 switch (ip_tunnel_info_af(info)) { 310 case AF_INET: 311 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS, 312 flow_dissector, 313 target_container); 314 if (dissector_uses_key(flow_dissector, 315 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { 316 struct flow_dissector_key_ipv4_addrs *ipv4; 317 318 ipv4 = skb_flow_dissector_target(flow_dissector, 319 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 320 target_container); 321 ipv4->src = key->u.ipv4.src; 322 ipv4->dst = key->u.ipv4.dst; 323 } 324 break; 325 case AF_INET6: 326 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS, 327 flow_dissector, 328 target_container); 329 if (dissector_uses_key(flow_dissector, 330 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { 331 struct flow_dissector_key_ipv6_addrs *ipv6; 332 333 ipv6 = skb_flow_dissector_target(flow_dissector, 334 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, 335 target_container); 336 ipv6->src = key->u.ipv6.src; 337 ipv6->dst = key->u.ipv6.dst; 338 } 339 break; 340 } 341 342 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 343 struct flow_dissector_key_keyid *keyid; 344 345 keyid = skb_flow_dissector_target(flow_dissector, 346 FLOW_DISSECTOR_KEY_ENC_KEYID, 347 target_container); 348 keyid->keyid = tunnel_id_to_key32(key->tun_id); 349 } 350 351 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) { 352 struct flow_dissector_key_ports *tp; 353 354 tp = skb_flow_dissector_target(flow_dissector, 355 FLOW_DISSECTOR_KEY_ENC_PORTS, 356 target_container); 357 tp->src = key->tp_src; 358 tp->dst = key->tp_dst; 359 } 360 361 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) { 362 struct flow_dissector_key_ip *ip; 363 364 ip = skb_flow_dissector_target(flow_dissector, 365 FLOW_DISSECTOR_KEY_ENC_IP, 366 target_container); 367 ip->tos = key->tos; 368 ip->ttl = key->ttl; 369 } 370 371 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) { 372 struct flow_dissector_key_enc_opts *enc_opt; 373 374 enc_opt = skb_flow_dissector_target(flow_dissector, 375 FLOW_DISSECTOR_KEY_ENC_OPTS, 376 target_container); 377 378 if (info->options_len) { 379 enc_opt->len = info->options_len; 380 ip_tunnel_info_opts_get(enc_opt->data, info); 381 enc_opt->dst_opt_type = info->key.tun_flags & 382 TUNNEL_OPTIONS_PRESENT; 383 } 384 } 385 } 386 EXPORT_SYMBOL(skb_flow_dissect_tunnel_info); 387 388 static enum flow_dissect_ret 389 __skb_flow_dissect_mpls(const struct sk_buff *skb, 390 struct flow_dissector *flow_dissector, 391 void *target_container, void *data, int nhoff, int hlen) 392 { 393 struct flow_dissector_key_keyid *key_keyid; 394 struct mpls_label *hdr, _hdr[2]; 395 u32 entry, label; 396 397 if (!dissector_uses_key(flow_dissector, 398 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) && 399 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) 400 return FLOW_DISSECT_RET_OUT_GOOD; 401 402 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, 403 hlen, &_hdr); 404 if (!hdr) 405 return FLOW_DISSECT_RET_OUT_BAD; 406 407 entry = ntohl(hdr[0].entry); 408 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT; 409 410 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) { 411 struct flow_dissector_key_mpls *key_mpls; 412 413 key_mpls = skb_flow_dissector_target(flow_dissector, 414 FLOW_DISSECTOR_KEY_MPLS, 415 target_container); 416 key_mpls->mpls_label = label; 417 key_mpls->mpls_ttl = (entry & MPLS_LS_TTL_MASK) 418 >> MPLS_LS_TTL_SHIFT; 419 key_mpls->mpls_tc = (entry & MPLS_LS_TC_MASK) 420 >> MPLS_LS_TC_SHIFT; 421 key_mpls->mpls_bos = (entry & MPLS_LS_S_MASK) 422 >> MPLS_LS_S_SHIFT; 423 } 424 425 if (label == MPLS_LABEL_ENTROPY) { 426 key_keyid = skb_flow_dissector_target(flow_dissector, 427 FLOW_DISSECTOR_KEY_MPLS_ENTROPY, 428 target_container); 429 key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK); 430 } 431 return FLOW_DISSECT_RET_OUT_GOOD; 432 } 433 434 static enum flow_dissect_ret 435 __skb_flow_dissect_arp(const struct sk_buff *skb, 436 struct flow_dissector *flow_dissector, 437 void *target_container, void *data, int nhoff, int hlen) 438 { 439 struct flow_dissector_key_arp *key_arp; 440 struct { 441 unsigned char ar_sha[ETH_ALEN]; 442 unsigned char ar_sip[4]; 443 unsigned char ar_tha[ETH_ALEN]; 444 unsigned char ar_tip[4]; 445 } *arp_eth, _arp_eth; 446 const struct arphdr *arp; 447 struct arphdr _arp; 448 449 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP)) 450 return FLOW_DISSECT_RET_OUT_GOOD; 451 452 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data, 453 hlen, &_arp); 454 if (!arp) 455 return FLOW_DISSECT_RET_OUT_BAD; 456 457 if (arp->ar_hrd != htons(ARPHRD_ETHER) || 458 arp->ar_pro != htons(ETH_P_IP) || 459 arp->ar_hln != ETH_ALEN || 460 arp->ar_pln != 4 || 461 (arp->ar_op != htons(ARPOP_REPLY) && 462 arp->ar_op != htons(ARPOP_REQUEST))) 463 return FLOW_DISSECT_RET_OUT_BAD; 464 465 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp), 466 sizeof(_arp_eth), data, 467 hlen, &_arp_eth); 468 if (!arp_eth) 469 return FLOW_DISSECT_RET_OUT_BAD; 470 471 key_arp = skb_flow_dissector_target(flow_dissector, 472 FLOW_DISSECTOR_KEY_ARP, 473 target_container); 474 475 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip)); 476 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip)); 477 478 /* Only store the lower byte of the opcode; 479 * this covers ARPOP_REPLY and ARPOP_REQUEST. 480 */ 481 key_arp->op = ntohs(arp->ar_op) & 0xff; 482 483 ether_addr_copy(key_arp->sha, arp_eth->ar_sha); 484 ether_addr_copy(key_arp->tha, arp_eth->ar_tha); 485 486 return FLOW_DISSECT_RET_OUT_GOOD; 487 } 488 489 static enum flow_dissect_ret 490 __skb_flow_dissect_gre(const struct sk_buff *skb, 491 struct flow_dissector_key_control *key_control, 492 struct flow_dissector *flow_dissector, 493 void *target_container, void *data, 494 __be16 *p_proto, int *p_nhoff, int *p_hlen, 495 unsigned int flags) 496 { 497 struct flow_dissector_key_keyid *key_keyid; 498 struct gre_base_hdr *hdr, _hdr; 499 int offset = 0; 500 u16 gre_ver; 501 502 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), 503 data, *p_hlen, &_hdr); 504 if (!hdr) 505 return FLOW_DISSECT_RET_OUT_BAD; 506 507 /* Only look inside GRE without routing */ 508 if (hdr->flags & GRE_ROUTING) 509 return FLOW_DISSECT_RET_OUT_GOOD; 510 511 /* Only look inside GRE for version 0 and 1 */ 512 gre_ver = ntohs(hdr->flags & GRE_VERSION); 513 if (gre_ver > 1) 514 return FLOW_DISSECT_RET_OUT_GOOD; 515 516 *p_proto = hdr->protocol; 517 if (gre_ver) { 518 /* Version1 must be PPTP, and check the flags */ 519 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY))) 520 return FLOW_DISSECT_RET_OUT_GOOD; 521 } 522 523 offset += sizeof(struct gre_base_hdr); 524 525 if (hdr->flags & GRE_CSUM) 526 offset += FIELD_SIZEOF(struct gre_full_hdr, csum) + 527 FIELD_SIZEOF(struct gre_full_hdr, reserved1); 528 529 if (hdr->flags & GRE_KEY) { 530 const __be32 *keyid; 531 __be32 _keyid; 532 533 keyid = __skb_header_pointer(skb, *p_nhoff + offset, 534 sizeof(_keyid), 535 data, *p_hlen, &_keyid); 536 if (!keyid) 537 return FLOW_DISSECT_RET_OUT_BAD; 538 539 if (dissector_uses_key(flow_dissector, 540 FLOW_DISSECTOR_KEY_GRE_KEYID)) { 541 key_keyid = skb_flow_dissector_target(flow_dissector, 542 FLOW_DISSECTOR_KEY_GRE_KEYID, 543 target_container); 544 if (gre_ver == 0) 545 key_keyid->keyid = *keyid; 546 else 547 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK; 548 } 549 offset += FIELD_SIZEOF(struct gre_full_hdr, key); 550 } 551 552 if (hdr->flags & GRE_SEQ) 553 offset += FIELD_SIZEOF(struct pptp_gre_header, seq); 554 555 if (gre_ver == 0) { 556 if (*p_proto == htons(ETH_P_TEB)) { 557 const struct ethhdr *eth; 558 struct ethhdr _eth; 559 560 eth = __skb_header_pointer(skb, *p_nhoff + offset, 561 sizeof(_eth), 562 data, *p_hlen, &_eth); 563 if (!eth) 564 return FLOW_DISSECT_RET_OUT_BAD; 565 *p_proto = eth->h_proto; 566 offset += sizeof(*eth); 567 568 /* Cap headers that we access via pointers at the 569 * end of the Ethernet header as our maximum alignment 570 * at that point is only 2 bytes. 571 */ 572 if (NET_IP_ALIGN) 573 *p_hlen = *p_nhoff + offset; 574 } 575 } else { /* version 1, must be PPTP */ 576 u8 _ppp_hdr[PPP_HDRLEN]; 577 u8 *ppp_hdr; 578 579 if (hdr->flags & GRE_ACK) 580 offset += FIELD_SIZEOF(struct pptp_gre_header, ack); 581 582 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset, 583 sizeof(_ppp_hdr), 584 data, *p_hlen, _ppp_hdr); 585 if (!ppp_hdr) 586 return FLOW_DISSECT_RET_OUT_BAD; 587 588 switch (PPP_PROTOCOL(ppp_hdr)) { 589 case PPP_IP: 590 *p_proto = htons(ETH_P_IP); 591 break; 592 case PPP_IPV6: 593 *p_proto = htons(ETH_P_IPV6); 594 break; 595 default: 596 /* Could probably catch some more like MPLS */ 597 break; 598 } 599 600 offset += PPP_HDRLEN; 601 } 602 603 *p_nhoff += offset; 604 key_control->flags |= FLOW_DIS_ENCAPSULATION; 605 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 606 return FLOW_DISSECT_RET_OUT_GOOD; 607 608 return FLOW_DISSECT_RET_PROTO_AGAIN; 609 } 610 611 /** 612 * __skb_flow_dissect_batadv() - dissect batman-adv header 613 * @skb: sk_buff to with the batman-adv header 614 * @key_control: flow dissectors control key 615 * @data: raw buffer pointer to the packet, if NULL use skb->data 616 * @p_proto: pointer used to update the protocol to process next 617 * @p_nhoff: pointer used to update inner network header offset 618 * @hlen: packet header length 619 * @flags: any combination of FLOW_DISSECTOR_F_* 620 * 621 * ETH_P_BATMAN packets are tried to be dissected. Only 622 * &struct batadv_unicast packets are actually processed because they contain an 623 * inner ethernet header and are usually followed by actual network header. This 624 * allows the flow dissector to continue processing the packet. 625 * 626 * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found, 627 * FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation, 628 * otherwise FLOW_DISSECT_RET_OUT_BAD 629 */ 630 static enum flow_dissect_ret 631 __skb_flow_dissect_batadv(const struct sk_buff *skb, 632 struct flow_dissector_key_control *key_control, 633 void *data, __be16 *p_proto, int *p_nhoff, int hlen, 634 unsigned int flags) 635 { 636 struct { 637 struct batadv_unicast_packet batadv_unicast; 638 struct ethhdr eth; 639 } *hdr, _hdr; 640 641 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen, 642 &_hdr); 643 if (!hdr) 644 return FLOW_DISSECT_RET_OUT_BAD; 645 646 if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION) 647 return FLOW_DISSECT_RET_OUT_BAD; 648 649 if (hdr->batadv_unicast.packet_type != BATADV_UNICAST) 650 return FLOW_DISSECT_RET_OUT_BAD; 651 652 *p_proto = hdr->eth.h_proto; 653 *p_nhoff += sizeof(*hdr); 654 655 key_control->flags |= FLOW_DIS_ENCAPSULATION; 656 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 657 return FLOW_DISSECT_RET_OUT_GOOD; 658 659 return FLOW_DISSECT_RET_PROTO_AGAIN; 660 } 661 662 static void 663 __skb_flow_dissect_tcp(const struct sk_buff *skb, 664 struct flow_dissector *flow_dissector, 665 void *target_container, void *data, int thoff, int hlen) 666 { 667 struct flow_dissector_key_tcp *key_tcp; 668 struct tcphdr *th, _th; 669 670 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP)) 671 return; 672 673 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th); 674 if (!th) 675 return; 676 677 if (unlikely(__tcp_hdrlen(th) < sizeof(_th))) 678 return; 679 680 key_tcp = skb_flow_dissector_target(flow_dissector, 681 FLOW_DISSECTOR_KEY_TCP, 682 target_container); 683 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF)); 684 } 685 686 static void 687 __skb_flow_dissect_ipv4(const struct sk_buff *skb, 688 struct flow_dissector *flow_dissector, 689 void *target_container, void *data, const struct iphdr *iph) 690 { 691 struct flow_dissector_key_ip *key_ip; 692 693 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 694 return; 695 696 key_ip = skb_flow_dissector_target(flow_dissector, 697 FLOW_DISSECTOR_KEY_IP, 698 target_container); 699 key_ip->tos = iph->tos; 700 key_ip->ttl = iph->ttl; 701 } 702 703 static void 704 __skb_flow_dissect_ipv6(const struct sk_buff *skb, 705 struct flow_dissector *flow_dissector, 706 void *target_container, void *data, const struct ipv6hdr *iph) 707 { 708 struct flow_dissector_key_ip *key_ip; 709 710 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 711 return; 712 713 key_ip = skb_flow_dissector_target(flow_dissector, 714 FLOW_DISSECTOR_KEY_IP, 715 target_container); 716 key_ip->tos = ipv6_get_dsfield(iph); 717 key_ip->ttl = iph->hop_limit; 718 } 719 720 /* Maximum number of protocol headers that can be parsed in 721 * __skb_flow_dissect 722 */ 723 #define MAX_FLOW_DISSECT_HDRS 15 724 725 static bool skb_flow_dissect_allowed(int *num_hdrs) 726 { 727 ++*num_hdrs; 728 729 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS); 730 } 731 732 static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys, 733 struct flow_dissector *flow_dissector, 734 void *target_container) 735 { 736 struct flow_dissector_key_control *key_control; 737 struct flow_dissector_key_basic *key_basic; 738 struct flow_dissector_key_addrs *key_addrs; 739 struct flow_dissector_key_ports *key_ports; 740 741 key_control = skb_flow_dissector_target(flow_dissector, 742 FLOW_DISSECTOR_KEY_CONTROL, 743 target_container); 744 key_control->thoff = flow_keys->thoff; 745 if (flow_keys->is_frag) 746 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 747 if (flow_keys->is_first_frag) 748 key_control->flags |= FLOW_DIS_FIRST_FRAG; 749 if (flow_keys->is_encap) 750 key_control->flags |= FLOW_DIS_ENCAPSULATION; 751 752 key_basic = skb_flow_dissector_target(flow_dissector, 753 FLOW_DISSECTOR_KEY_BASIC, 754 target_container); 755 key_basic->n_proto = flow_keys->n_proto; 756 key_basic->ip_proto = flow_keys->ip_proto; 757 758 if (flow_keys->addr_proto == ETH_P_IP && 759 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 760 key_addrs = skb_flow_dissector_target(flow_dissector, 761 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 762 target_container); 763 key_addrs->v4addrs.src = flow_keys->ipv4_src; 764 key_addrs->v4addrs.dst = flow_keys->ipv4_dst; 765 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 766 } else if (flow_keys->addr_proto == ETH_P_IPV6 && 767 dissector_uses_key(flow_dissector, 768 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 769 key_addrs = skb_flow_dissector_target(flow_dissector, 770 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 771 target_container); 772 memcpy(&key_addrs->v6addrs, &flow_keys->ipv6_src, 773 sizeof(key_addrs->v6addrs)); 774 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 775 } 776 777 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS)) { 778 key_ports = skb_flow_dissector_target(flow_dissector, 779 FLOW_DISSECTOR_KEY_PORTS, 780 target_container); 781 key_ports->src = flow_keys->sport; 782 key_ports->dst = flow_keys->dport; 783 } 784 } 785 786 bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, 787 __be16 proto, int nhoff, int hlen) 788 { 789 struct bpf_flow_keys *flow_keys = ctx->flow_keys; 790 u32 result; 791 792 /* Pass parameters to the BPF program */ 793 memset(flow_keys, 0, sizeof(*flow_keys)); 794 flow_keys->n_proto = proto; 795 flow_keys->nhoff = nhoff; 796 flow_keys->thoff = flow_keys->nhoff; 797 798 preempt_disable(); 799 result = BPF_PROG_RUN(prog, ctx); 800 preempt_enable(); 801 802 flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen); 803 flow_keys->thoff = clamp_t(u16, flow_keys->thoff, 804 flow_keys->nhoff, hlen); 805 806 return result == BPF_OK; 807 } 808 809 /** 810 * __skb_flow_dissect - extract the flow_keys struct and return it 811 * @net: associated network namespace, derived from @skb if NULL 812 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 813 * @flow_dissector: list of keys to dissect 814 * @target_container: target structure to put dissected values into 815 * @data: raw buffer pointer to the packet, if NULL use skb->data 816 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 817 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 818 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 819 * @flags: flags that control the dissection process, e.g. 820 * FLOW_DISSECTOR_F_STOP_AT_ENCAP. 821 * 822 * The function will try to retrieve individual keys into target specified 823 * by flow_dissector from either the skbuff or a raw buffer specified by the 824 * rest parameters. 825 * 826 * Caller must take care of zeroing target container memory. 827 */ 828 bool __skb_flow_dissect(const struct net *net, 829 const struct sk_buff *skb, 830 struct flow_dissector *flow_dissector, 831 void *target_container, 832 void *data, __be16 proto, int nhoff, int hlen, 833 unsigned int flags) 834 { 835 struct flow_dissector_key_control *key_control; 836 struct flow_dissector_key_basic *key_basic; 837 struct flow_dissector_key_addrs *key_addrs; 838 struct flow_dissector_key_ports *key_ports; 839 struct flow_dissector_key_icmp *key_icmp; 840 struct flow_dissector_key_tags *key_tags; 841 struct flow_dissector_key_vlan *key_vlan; 842 struct bpf_prog *attached = NULL; 843 enum flow_dissect_ret fdret; 844 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX; 845 int num_hdrs = 0; 846 u8 ip_proto = 0; 847 bool ret; 848 849 if (!data) { 850 data = skb->data; 851 proto = skb_vlan_tag_present(skb) ? 852 skb->vlan_proto : skb->protocol; 853 nhoff = skb_network_offset(skb); 854 hlen = skb_headlen(skb); 855 #if IS_ENABLED(CONFIG_NET_DSA) 856 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev))) { 857 const struct dsa_device_ops *ops; 858 int offset; 859 860 ops = skb->dev->dsa_ptr->tag_ops; 861 if (ops->flow_dissect && 862 !ops->flow_dissect(skb, &proto, &offset)) { 863 hlen -= offset; 864 nhoff += offset; 865 } 866 } 867 #endif 868 } 869 870 /* It is ensured by skb_flow_dissector_init() that control key will 871 * be always present. 872 */ 873 key_control = skb_flow_dissector_target(flow_dissector, 874 FLOW_DISSECTOR_KEY_CONTROL, 875 target_container); 876 877 /* It is ensured by skb_flow_dissector_init() that basic key will 878 * be always present. 879 */ 880 key_basic = skb_flow_dissector_target(flow_dissector, 881 FLOW_DISSECTOR_KEY_BASIC, 882 target_container); 883 884 if (skb) { 885 if (!net) { 886 if (skb->dev) 887 net = dev_net(skb->dev); 888 else if (skb->sk) 889 net = sock_net(skb->sk); 890 } 891 } 892 893 WARN_ON_ONCE(!net); 894 if (net) { 895 rcu_read_lock(); 896 attached = rcu_dereference(net->flow_dissector_prog); 897 898 if (attached) { 899 struct bpf_flow_keys flow_keys; 900 struct bpf_flow_dissector ctx = { 901 .flow_keys = &flow_keys, 902 .data = data, 903 .data_end = data + hlen, 904 }; 905 __be16 n_proto = proto; 906 907 if (skb) { 908 ctx.skb = skb; 909 /* we can't use 'proto' in the skb case 910 * because it might be set to skb->vlan_proto 911 * which has been pulled from the data 912 */ 913 n_proto = skb->protocol; 914 } 915 916 ret = bpf_flow_dissect(attached, &ctx, n_proto, nhoff, 917 hlen); 918 __skb_flow_bpf_to_target(&flow_keys, flow_dissector, 919 target_container); 920 rcu_read_unlock(); 921 return ret; 922 } 923 rcu_read_unlock(); 924 } 925 926 if (dissector_uses_key(flow_dissector, 927 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 928 struct ethhdr *eth = eth_hdr(skb); 929 struct flow_dissector_key_eth_addrs *key_eth_addrs; 930 931 key_eth_addrs = skb_flow_dissector_target(flow_dissector, 932 FLOW_DISSECTOR_KEY_ETH_ADDRS, 933 target_container); 934 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs)); 935 } 936 937 proto_again: 938 fdret = FLOW_DISSECT_RET_CONTINUE; 939 940 switch (proto) { 941 case htons(ETH_P_IP): { 942 const struct iphdr *iph; 943 struct iphdr _iph; 944 945 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 946 if (!iph || iph->ihl < 5) { 947 fdret = FLOW_DISSECT_RET_OUT_BAD; 948 break; 949 } 950 951 nhoff += iph->ihl * 4; 952 953 ip_proto = iph->protocol; 954 955 if (dissector_uses_key(flow_dissector, 956 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 957 key_addrs = skb_flow_dissector_target(flow_dissector, 958 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 959 target_container); 960 961 memcpy(&key_addrs->v4addrs, &iph->saddr, 962 sizeof(key_addrs->v4addrs)); 963 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 964 } 965 966 if (ip_is_fragment(iph)) { 967 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 968 969 if (iph->frag_off & htons(IP_OFFSET)) { 970 fdret = FLOW_DISSECT_RET_OUT_GOOD; 971 break; 972 } else { 973 key_control->flags |= FLOW_DIS_FIRST_FRAG; 974 if (!(flags & 975 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) { 976 fdret = FLOW_DISSECT_RET_OUT_GOOD; 977 break; 978 } 979 } 980 } 981 982 __skb_flow_dissect_ipv4(skb, flow_dissector, 983 target_container, data, iph); 984 985 break; 986 } 987 case htons(ETH_P_IPV6): { 988 const struct ipv6hdr *iph; 989 struct ipv6hdr _iph; 990 991 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 992 if (!iph) { 993 fdret = FLOW_DISSECT_RET_OUT_BAD; 994 break; 995 } 996 997 ip_proto = iph->nexthdr; 998 nhoff += sizeof(struct ipv6hdr); 999 1000 if (dissector_uses_key(flow_dissector, 1001 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 1002 key_addrs = skb_flow_dissector_target(flow_dissector, 1003 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1004 target_container); 1005 1006 memcpy(&key_addrs->v6addrs, &iph->saddr, 1007 sizeof(key_addrs->v6addrs)); 1008 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1009 } 1010 1011 if ((dissector_uses_key(flow_dissector, 1012 FLOW_DISSECTOR_KEY_FLOW_LABEL) || 1013 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) && 1014 ip6_flowlabel(iph)) { 1015 __be32 flow_label = ip6_flowlabel(iph); 1016 1017 if (dissector_uses_key(flow_dissector, 1018 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 1019 key_tags = skb_flow_dissector_target(flow_dissector, 1020 FLOW_DISSECTOR_KEY_FLOW_LABEL, 1021 target_container); 1022 key_tags->flow_label = ntohl(flow_label); 1023 } 1024 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) { 1025 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1026 break; 1027 } 1028 } 1029 1030 __skb_flow_dissect_ipv6(skb, flow_dissector, 1031 target_container, data, iph); 1032 1033 break; 1034 } 1035 case htons(ETH_P_8021AD): 1036 case htons(ETH_P_8021Q): { 1037 const struct vlan_hdr *vlan = NULL; 1038 struct vlan_hdr _vlan; 1039 __be16 saved_vlan_tpid = proto; 1040 1041 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX && 1042 skb && skb_vlan_tag_present(skb)) { 1043 proto = skb->protocol; 1044 } else { 1045 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), 1046 data, hlen, &_vlan); 1047 if (!vlan) { 1048 fdret = FLOW_DISSECT_RET_OUT_BAD; 1049 break; 1050 } 1051 1052 proto = vlan->h_vlan_encapsulated_proto; 1053 nhoff += sizeof(*vlan); 1054 } 1055 1056 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) { 1057 dissector_vlan = FLOW_DISSECTOR_KEY_VLAN; 1058 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) { 1059 dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN; 1060 } else { 1061 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1062 break; 1063 } 1064 1065 if (dissector_uses_key(flow_dissector, dissector_vlan)) { 1066 key_vlan = skb_flow_dissector_target(flow_dissector, 1067 dissector_vlan, 1068 target_container); 1069 1070 if (!vlan) { 1071 key_vlan->vlan_id = skb_vlan_tag_get_id(skb); 1072 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb); 1073 } else { 1074 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) & 1075 VLAN_VID_MASK; 1076 key_vlan->vlan_priority = 1077 (ntohs(vlan->h_vlan_TCI) & 1078 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1079 } 1080 key_vlan->vlan_tpid = saved_vlan_tpid; 1081 } 1082 1083 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1084 break; 1085 } 1086 case htons(ETH_P_PPP_SES): { 1087 struct { 1088 struct pppoe_hdr hdr; 1089 __be16 proto; 1090 } *hdr, _hdr; 1091 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 1092 if (!hdr) { 1093 fdret = FLOW_DISSECT_RET_OUT_BAD; 1094 break; 1095 } 1096 1097 proto = hdr->proto; 1098 nhoff += PPPOE_SES_HLEN; 1099 switch (proto) { 1100 case htons(PPP_IP): 1101 proto = htons(ETH_P_IP); 1102 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1103 break; 1104 case htons(PPP_IPV6): 1105 proto = htons(ETH_P_IPV6); 1106 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1107 break; 1108 default: 1109 fdret = FLOW_DISSECT_RET_OUT_BAD; 1110 break; 1111 } 1112 break; 1113 } 1114 case htons(ETH_P_TIPC): { 1115 struct tipc_basic_hdr *hdr, _hdr; 1116 1117 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), 1118 data, hlen, &_hdr); 1119 if (!hdr) { 1120 fdret = FLOW_DISSECT_RET_OUT_BAD; 1121 break; 1122 } 1123 1124 if (dissector_uses_key(flow_dissector, 1125 FLOW_DISSECTOR_KEY_TIPC)) { 1126 key_addrs = skb_flow_dissector_target(flow_dissector, 1127 FLOW_DISSECTOR_KEY_TIPC, 1128 target_container); 1129 key_addrs->tipckey.key = tipc_hdr_rps_key(hdr); 1130 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC; 1131 } 1132 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1133 break; 1134 } 1135 1136 case htons(ETH_P_MPLS_UC): 1137 case htons(ETH_P_MPLS_MC): 1138 fdret = __skb_flow_dissect_mpls(skb, flow_dissector, 1139 target_container, data, 1140 nhoff, hlen); 1141 break; 1142 case htons(ETH_P_FCOE): 1143 if ((hlen - nhoff) < FCOE_HEADER_LEN) { 1144 fdret = FLOW_DISSECT_RET_OUT_BAD; 1145 break; 1146 } 1147 1148 nhoff += FCOE_HEADER_LEN; 1149 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1150 break; 1151 1152 case htons(ETH_P_ARP): 1153 case htons(ETH_P_RARP): 1154 fdret = __skb_flow_dissect_arp(skb, flow_dissector, 1155 target_container, data, 1156 nhoff, hlen); 1157 break; 1158 1159 case htons(ETH_P_BATMAN): 1160 fdret = __skb_flow_dissect_batadv(skb, key_control, data, 1161 &proto, &nhoff, hlen, flags); 1162 break; 1163 1164 default: 1165 fdret = FLOW_DISSECT_RET_OUT_BAD; 1166 break; 1167 } 1168 1169 /* Process result of proto processing */ 1170 switch (fdret) { 1171 case FLOW_DISSECT_RET_OUT_GOOD: 1172 goto out_good; 1173 case FLOW_DISSECT_RET_PROTO_AGAIN: 1174 if (skb_flow_dissect_allowed(&num_hdrs)) 1175 goto proto_again; 1176 goto out_good; 1177 case FLOW_DISSECT_RET_CONTINUE: 1178 case FLOW_DISSECT_RET_IPPROTO_AGAIN: 1179 break; 1180 case FLOW_DISSECT_RET_OUT_BAD: 1181 default: 1182 goto out_bad; 1183 } 1184 1185 ip_proto_again: 1186 fdret = FLOW_DISSECT_RET_CONTINUE; 1187 1188 switch (ip_proto) { 1189 case IPPROTO_GRE: 1190 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector, 1191 target_container, data, 1192 &proto, &nhoff, &hlen, flags); 1193 break; 1194 1195 case NEXTHDR_HOP: 1196 case NEXTHDR_ROUTING: 1197 case NEXTHDR_DEST: { 1198 u8 _opthdr[2], *opthdr; 1199 1200 if (proto != htons(ETH_P_IPV6)) 1201 break; 1202 1203 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr), 1204 data, hlen, &_opthdr); 1205 if (!opthdr) { 1206 fdret = FLOW_DISSECT_RET_OUT_BAD; 1207 break; 1208 } 1209 1210 ip_proto = opthdr[0]; 1211 nhoff += (opthdr[1] + 1) << 3; 1212 1213 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN; 1214 break; 1215 } 1216 case NEXTHDR_FRAGMENT: { 1217 struct frag_hdr _fh, *fh; 1218 1219 if (proto != htons(ETH_P_IPV6)) 1220 break; 1221 1222 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh), 1223 data, hlen, &_fh); 1224 1225 if (!fh) { 1226 fdret = FLOW_DISSECT_RET_OUT_BAD; 1227 break; 1228 } 1229 1230 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 1231 1232 nhoff += sizeof(_fh); 1233 ip_proto = fh->nexthdr; 1234 1235 if (!(fh->frag_off & htons(IP6_OFFSET))) { 1236 key_control->flags |= FLOW_DIS_FIRST_FRAG; 1237 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) { 1238 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN; 1239 break; 1240 } 1241 } 1242 1243 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1244 break; 1245 } 1246 case IPPROTO_IPIP: 1247 proto = htons(ETH_P_IP); 1248 1249 key_control->flags |= FLOW_DIS_ENCAPSULATION; 1250 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) { 1251 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1252 break; 1253 } 1254 1255 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1256 break; 1257 1258 case IPPROTO_IPV6: 1259 proto = htons(ETH_P_IPV6); 1260 1261 key_control->flags |= FLOW_DIS_ENCAPSULATION; 1262 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) { 1263 fdret = FLOW_DISSECT_RET_OUT_GOOD; 1264 break; 1265 } 1266 1267 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1268 break; 1269 1270 1271 case IPPROTO_MPLS: 1272 proto = htons(ETH_P_MPLS_UC); 1273 fdret = FLOW_DISSECT_RET_PROTO_AGAIN; 1274 break; 1275 1276 case IPPROTO_TCP: 1277 __skb_flow_dissect_tcp(skb, flow_dissector, target_container, 1278 data, nhoff, hlen); 1279 break; 1280 1281 default: 1282 break; 1283 } 1284 1285 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS) && 1286 !(key_control->flags & FLOW_DIS_IS_FRAGMENT)) { 1287 key_ports = skb_flow_dissector_target(flow_dissector, 1288 FLOW_DISSECTOR_KEY_PORTS, 1289 target_container); 1290 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 1291 data, hlen); 1292 } 1293 1294 if (dissector_uses_key(flow_dissector, 1295 FLOW_DISSECTOR_KEY_ICMP)) { 1296 key_icmp = skb_flow_dissector_target(flow_dissector, 1297 FLOW_DISSECTOR_KEY_ICMP, 1298 target_container); 1299 key_icmp->icmp = skb_flow_get_be16(skb, nhoff, data, hlen); 1300 } 1301 1302 /* Process result of IP proto processing */ 1303 switch (fdret) { 1304 case FLOW_DISSECT_RET_PROTO_AGAIN: 1305 if (skb_flow_dissect_allowed(&num_hdrs)) 1306 goto proto_again; 1307 break; 1308 case FLOW_DISSECT_RET_IPPROTO_AGAIN: 1309 if (skb_flow_dissect_allowed(&num_hdrs)) 1310 goto ip_proto_again; 1311 break; 1312 case FLOW_DISSECT_RET_OUT_GOOD: 1313 case FLOW_DISSECT_RET_CONTINUE: 1314 break; 1315 case FLOW_DISSECT_RET_OUT_BAD: 1316 default: 1317 goto out_bad; 1318 } 1319 1320 out_good: 1321 ret = true; 1322 1323 out: 1324 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); 1325 key_basic->n_proto = proto; 1326 key_basic->ip_proto = ip_proto; 1327 1328 return ret; 1329 1330 out_bad: 1331 ret = false; 1332 goto out; 1333 } 1334 EXPORT_SYMBOL(__skb_flow_dissect); 1335 1336 static u32 hashrnd __read_mostly; 1337 static __always_inline void __flow_hash_secret_init(void) 1338 { 1339 net_get_random_once(&hashrnd, sizeof(hashrnd)); 1340 } 1341 1342 static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, 1343 u32 keyval) 1344 { 1345 return jhash2(words, length, keyval); 1346 } 1347 1348 static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) 1349 { 1350 const void *p = flow; 1351 1352 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); 1353 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); 1354 } 1355 1356 static inline size_t flow_keys_hash_length(const struct flow_keys *flow) 1357 { 1358 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); 1359 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); 1360 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != 1361 sizeof(*flow) - sizeof(flow->addrs)); 1362 1363 switch (flow->control.addr_type) { 1364 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1365 diff -= sizeof(flow->addrs.v4addrs); 1366 break; 1367 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1368 diff -= sizeof(flow->addrs.v6addrs); 1369 break; 1370 case FLOW_DISSECTOR_KEY_TIPC: 1371 diff -= sizeof(flow->addrs.tipckey); 1372 break; 1373 } 1374 return (sizeof(*flow) - diff) / sizeof(u32); 1375 } 1376 1377 __be32 flow_get_u32_src(const struct flow_keys *flow) 1378 { 1379 switch (flow->control.addr_type) { 1380 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1381 return flow->addrs.v4addrs.src; 1382 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1383 return (__force __be32)ipv6_addr_hash( 1384 &flow->addrs.v6addrs.src); 1385 case FLOW_DISSECTOR_KEY_TIPC: 1386 return flow->addrs.tipckey.key; 1387 default: 1388 return 0; 1389 } 1390 } 1391 EXPORT_SYMBOL(flow_get_u32_src); 1392 1393 __be32 flow_get_u32_dst(const struct flow_keys *flow) 1394 { 1395 switch (flow->control.addr_type) { 1396 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1397 return flow->addrs.v4addrs.dst; 1398 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1399 return (__force __be32)ipv6_addr_hash( 1400 &flow->addrs.v6addrs.dst); 1401 default: 1402 return 0; 1403 } 1404 } 1405 EXPORT_SYMBOL(flow_get_u32_dst); 1406 1407 static inline void __flow_hash_consistentify(struct flow_keys *keys) 1408 { 1409 int addr_diff, i; 1410 1411 switch (keys->control.addr_type) { 1412 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 1413 addr_diff = (__force u32)keys->addrs.v4addrs.dst - 1414 (__force u32)keys->addrs.v4addrs.src; 1415 if ((addr_diff < 0) || 1416 (addr_diff == 0 && 1417 ((__force u16)keys->ports.dst < 1418 (__force u16)keys->ports.src))) { 1419 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); 1420 swap(keys->ports.src, keys->ports.dst); 1421 } 1422 break; 1423 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 1424 addr_diff = memcmp(&keys->addrs.v6addrs.dst, 1425 &keys->addrs.v6addrs.src, 1426 sizeof(keys->addrs.v6addrs.dst)); 1427 if ((addr_diff < 0) || 1428 (addr_diff == 0 && 1429 ((__force u16)keys->ports.dst < 1430 (__force u16)keys->ports.src))) { 1431 for (i = 0; i < 4; i++) 1432 swap(keys->addrs.v6addrs.src.s6_addr32[i], 1433 keys->addrs.v6addrs.dst.s6_addr32[i]); 1434 swap(keys->ports.src, keys->ports.dst); 1435 } 1436 break; 1437 } 1438 } 1439 1440 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) 1441 { 1442 u32 hash; 1443 1444 __flow_hash_consistentify(keys); 1445 1446 hash = __flow_hash_words(flow_keys_hash_start(keys), 1447 flow_keys_hash_length(keys), keyval); 1448 if (!hash) 1449 hash = 1; 1450 1451 return hash; 1452 } 1453 1454 u32 flow_hash_from_keys(struct flow_keys *keys) 1455 { 1456 __flow_hash_secret_init(); 1457 return __flow_hash_from_keys(keys, hashrnd); 1458 } 1459 EXPORT_SYMBOL(flow_hash_from_keys); 1460 1461 static inline u32 ___skb_get_hash(const struct sk_buff *skb, 1462 struct flow_keys *keys, u32 keyval) 1463 { 1464 skb_flow_dissect_flow_keys(skb, keys, 1465 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 1466 1467 return __flow_hash_from_keys(keys, keyval); 1468 } 1469 1470 struct _flow_keys_digest_data { 1471 __be16 n_proto; 1472 u8 ip_proto; 1473 u8 padding; 1474 __be32 ports; 1475 __be32 src; 1476 __be32 dst; 1477 }; 1478 1479 void make_flow_keys_digest(struct flow_keys_digest *digest, 1480 const struct flow_keys *flow) 1481 { 1482 struct _flow_keys_digest_data *data = 1483 (struct _flow_keys_digest_data *)digest; 1484 1485 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); 1486 1487 memset(digest, 0, sizeof(*digest)); 1488 1489 data->n_proto = flow->basic.n_proto; 1490 data->ip_proto = flow->basic.ip_proto; 1491 data->ports = flow->ports.ports; 1492 data->src = flow->addrs.v4addrs.src; 1493 data->dst = flow->addrs.v4addrs.dst; 1494 } 1495 EXPORT_SYMBOL(make_flow_keys_digest); 1496 1497 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly; 1498 1499 u32 __skb_get_hash_symmetric(const struct sk_buff *skb) 1500 { 1501 struct flow_keys keys; 1502 1503 __flow_hash_secret_init(); 1504 1505 memset(&keys, 0, sizeof(keys)); 1506 __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric, 1507 &keys, NULL, 0, 0, 0, 1508 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 1509 1510 return __flow_hash_from_keys(&keys, hashrnd); 1511 } 1512 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric); 1513 1514 /** 1515 * __skb_get_hash: calculate a flow hash 1516 * @skb: sk_buff to calculate flow hash from 1517 * 1518 * This function calculates a flow hash based on src/dst addresses 1519 * and src/dst port numbers. Sets hash in skb to non-zero hash value 1520 * on success, zero indicates no valid hash. Also, sets l4_hash in skb 1521 * if hash is a canonical 4-tuple hash over transport ports. 1522 */ 1523 void __skb_get_hash(struct sk_buff *skb) 1524 { 1525 struct flow_keys keys; 1526 u32 hash; 1527 1528 __flow_hash_secret_init(); 1529 1530 hash = ___skb_get_hash(skb, &keys, hashrnd); 1531 1532 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 1533 } 1534 EXPORT_SYMBOL(__skb_get_hash); 1535 1536 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) 1537 { 1538 struct flow_keys keys; 1539 1540 return ___skb_get_hash(skb, &keys, perturb); 1541 } 1542 EXPORT_SYMBOL(skb_get_hash_perturb); 1543 1544 u32 __skb_get_poff(const struct sk_buff *skb, void *data, 1545 const struct flow_keys_basic *keys, int hlen) 1546 { 1547 u32 poff = keys->control.thoff; 1548 1549 /* skip L4 headers for fragments after the first */ 1550 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) && 1551 !(keys->control.flags & FLOW_DIS_FIRST_FRAG)) 1552 return poff; 1553 1554 switch (keys->basic.ip_proto) { 1555 case IPPROTO_TCP: { 1556 /* access doff as u8 to avoid unaligned access */ 1557 const u8 *doff; 1558 u8 _doff; 1559 1560 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 1561 data, hlen, &_doff); 1562 if (!doff) 1563 return poff; 1564 1565 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 1566 break; 1567 } 1568 case IPPROTO_UDP: 1569 case IPPROTO_UDPLITE: 1570 poff += sizeof(struct udphdr); 1571 break; 1572 /* For the rest, we do not really care about header 1573 * extensions at this point for now. 1574 */ 1575 case IPPROTO_ICMP: 1576 poff += sizeof(struct icmphdr); 1577 break; 1578 case IPPROTO_ICMPV6: 1579 poff += sizeof(struct icmp6hdr); 1580 break; 1581 case IPPROTO_IGMP: 1582 poff += sizeof(struct igmphdr); 1583 break; 1584 case IPPROTO_DCCP: 1585 poff += sizeof(struct dccp_hdr); 1586 break; 1587 case IPPROTO_SCTP: 1588 poff += sizeof(struct sctphdr); 1589 break; 1590 } 1591 1592 return poff; 1593 } 1594 1595 /** 1596 * skb_get_poff - get the offset to the payload 1597 * @skb: sk_buff to get the payload offset from 1598 * 1599 * The function will get the offset to the payload as far as it could 1600 * be dissected. The main user is currently BPF, so that we can dynamically 1601 * truncate packets without needing to push actual payload to the user 1602 * space and can analyze headers only, instead. 1603 */ 1604 u32 skb_get_poff(const struct sk_buff *skb) 1605 { 1606 struct flow_keys_basic keys; 1607 1608 if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, 1609 NULL, 0, 0, 0, 0)) 1610 return 0; 1611 1612 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 1613 } 1614 1615 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) 1616 { 1617 memset(keys, 0, sizeof(*keys)); 1618 1619 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr, 1620 sizeof(keys->addrs.v6addrs.src)); 1621 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr, 1622 sizeof(keys->addrs.v6addrs.dst)); 1623 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1624 keys->ports.src = fl6->fl6_sport; 1625 keys->ports.dst = fl6->fl6_dport; 1626 keys->keyid.keyid = fl6->fl6_gre_key; 1627 keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 1628 keys->basic.ip_proto = fl6->flowi6_proto; 1629 1630 return flow_hash_from_keys(keys); 1631 } 1632 EXPORT_SYMBOL(__get_hash_from_flowi6); 1633 1634 static const struct flow_dissector_key flow_keys_dissector_keys[] = { 1635 { 1636 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1637 .offset = offsetof(struct flow_keys, control), 1638 }, 1639 { 1640 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1641 .offset = offsetof(struct flow_keys, basic), 1642 }, 1643 { 1644 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1645 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1646 }, 1647 { 1648 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1649 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1650 }, 1651 { 1652 .key_id = FLOW_DISSECTOR_KEY_TIPC, 1653 .offset = offsetof(struct flow_keys, addrs.tipckey), 1654 }, 1655 { 1656 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1657 .offset = offsetof(struct flow_keys, ports), 1658 }, 1659 { 1660 .key_id = FLOW_DISSECTOR_KEY_VLAN, 1661 .offset = offsetof(struct flow_keys, vlan), 1662 }, 1663 { 1664 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL, 1665 .offset = offsetof(struct flow_keys, tags), 1666 }, 1667 { 1668 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, 1669 .offset = offsetof(struct flow_keys, keyid), 1670 }, 1671 }; 1672 1673 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = { 1674 { 1675 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1676 .offset = offsetof(struct flow_keys, control), 1677 }, 1678 { 1679 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1680 .offset = offsetof(struct flow_keys, basic), 1681 }, 1682 { 1683 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1684 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1685 }, 1686 { 1687 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1688 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1689 }, 1690 { 1691 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1692 .offset = offsetof(struct flow_keys, ports), 1693 }, 1694 }; 1695 1696 static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = { 1697 { 1698 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1699 .offset = offsetof(struct flow_keys, control), 1700 }, 1701 { 1702 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1703 .offset = offsetof(struct flow_keys, basic), 1704 }, 1705 }; 1706 1707 struct flow_dissector flow_keys_dissector __read_mostly; 1708 EXPORT_SYMBOL(flow_keys_dissector); 1709 1710 struct flow_dissector flow_keys_basic_dissector __read_mostly; 1711 EXPORT_SYMBOL(flow_keys_basic_dissector); 1712 1713 static int __init init_default_flow_dissectors(void) 1714 { 1715 skb_flow_dissector_init(&flow_keys_dissector, 1716 flow_keys_dissector_keys, 1717 ARRAY_SIZE(flow_keys_dissector_keys)); 1718 skb_flow_dissector_init(&flow_keys_dissector_symmetric, 1719 flow_keys_dissector_symmetric_keys, 1720 ARRAY_SIZE(flow_keys_dissector_symmetric_keys)); 1721 skb_flow_dissector_init(&flow_keys_basic_dissector, 1722 flow_keys_basic_dissector_keys, 1723 ARRAY_SIZE(flow_keys_basic_dissector_keys)); 1724 return 0; 1725 } 1726 1727 core_initcall(init_default_flow_dissectors); 1728