1 #include <linux/kernel.h> 2 #include <linux/skbuff.h> 3 #include <linux/export.h> 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip.h> 8 #include <net/ipv6.h> 9 #include <net/gre.h> 10 #include <net/pptp.h> 11 #include <linux/igmp.h> 12 #include <linux/icmp.h> 13 #include <linux/sctp.h> 14 #include <linux/dccp.h> 15 #include <linux/if_tunnel.h> 16 #include <linux/if_pppox.h> 17 #include <linux/ppp_defs.h> 18 #include <linux/stddef.h> 19 #include <linux/if_ether.h> 20 #include <linux/mpls.h> 21 #include <linux/tcp.h> 22 #include <net/flow_dissector.h> 23 #include <scsi/fc/fc_fcoe.h> 24 25 static void dissector_set_key(struct flow_dissector *flow_dissector, 26 enum flow_dissector_key_id key_id) 27 { 28 flow_dissector->used_keys |= (1 << key_id); 29 } 30 31 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 32 const struct flow_dissector_key *key, 33 unsigned int key_count) 34 { 35 unsigned int i; 36 37 memset(flow_dissector, 0, sizeof(*flow_dissector)); 38 39 for (i = 0; i < key_count; i++, key++) { 40 /* User should make sure that every key target offset is withing 41 * boundaries of unsigned short. 42 */ 43 BUG_ON(key->offset > USHRT_MAX); 44 BUG_ON(dissector_uses_key(flow_dissector, 45 key->key_id)); 46 47 dissector_set_key(flow_dissector, key->key_id); 48 flow_dissector->offset[key->key_id] = key->offset; 49 } 50 51 /* Ensure that the dissector always includes control and basic key. 52 * That way we are able to avoid handling lack of these in fast path. 53 */ 54 BUG_ON(!dissector_uses_key(flow_dissector, 55 FLOW_DISSECTOR_KEY_CONTROL)); 56 BUG_ON(!dissector_uses_key(flow_dissector, 57 FLOW_DISSECTOR_KEY_BASIC)); 58 } 59 EXPORT_SYMBOL(skb_flow_dissector_init); 60 61 /** 62 * skb_flow_get_be16 - extract be16 entity 63 * @skb: sk_buff to extract from 64 * @poff: offset to extract at 65 * @data: raw buffer pointer to the packet 66 * @hlen: packet header length 67 * 68 * The function will try to retrieve a be32 entity at 69 * offset poff 70 */ 71 static __be16 skb_flow_get_be16(const struct sk_buff *skb, int poff, 72 void *data, int hlen) 73 { 74 __be16 *u, _u; 75 76 u = __skb_header_pointer(skb, poff, sizeof(_u), data, hlen, &_u); 77 if (u) 78 return *u; 79 80 return 0; 81 } 82 83 /** 84 * __skb_flow_get_ports - extract the upper layer ports and return them 85 * @skb: sk_buff to extract the ports from 86 * @thoff: transport header offset 87 * @ip_proto: protocol for which to get port offset 88 * @data: raw buffer pointer to the packet, if NULL use skb->data 89 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 90 * 91 * The function will try to retrieve the ports at offset thoff + poff where poff 92 * is the protocol port offset returned from proto_ports_offset 93 */ 94 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 95 void *data, int hlen) 96 { 97 int poff = proto_ports_offset(ip_proto); 98 99 if (!data) { 100 data = skb->data; 101 hlen = skb_headlen(skb); 102 } 103 104 if (poff >= 0) { 105 __be32 *ports, _ports; 106 107 ports = __skb_header_pointer(skb, thoff + poff, 108 sizeof(_ports), data, hlen, &_ports); 109 if (ports) 110 return *ports; 111 } 112 113 return 0; 114 } 115 EXPORT_SYMBOL(__skb_flow_get_ports); 116 117 enum flow_dissect_ret { 118 FLOW_DISSECT_RET_OUT_GOOD, 119 FLOW_DISSECT_RET_OUT_BAD, 120 FLOW_DISSECT_RET_OUT_PROTO_AGAIN, 121 }; 122 123 static enum flow_dissect_ret 124 __skb_flow_dissect_mpls(const struct sk_buff *skb, 125 struct flow_dissector *flow_dissector, 126 void *target_container, void *data, int nhoff, int hlen) 127 { 128 struct flow_dissector_key_keyid *key_keyid; 129 struct mpls_label *hdr, _hdr[2]; 130 u32 entry, label; 131 132 if (!dissector_uses_key(flow_dissector, 133 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) && 134 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) 135 return FLOW_DISSECT_RET_OUT_GOOD; 136 137 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, 138 hlen, &_hdr); 139 if (!hdr) 140 return FLOW_DISSECT_RET_OUT_BAD; 141 142 entry = ntohl(hdr[0].entry); 143 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT; 144 145 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) { 146 struct flow_dissector_key_mpls *key_mpls; 147 148 key_mpls = skb_flow_dissector_target(flow_dissector, 149 FLOW_DISSECTOR_KEY_MPLS, 150 target_container); 151 key_mpls->mpls_label = label; 152 key_mpls->mpls_ttl = (entry & MPLS_LS_TTL_MASK) 153 >> MPLS_LS_TTL_SHIFT; 154 key_mpls->mpls_tc = (entry & MPLS_LS_TC_MASK) 155 >> MPLS_LS_TC_SHIFT; 156 key_mpls->mpls_bos = (entry & MPLS_LS_S_MASK) 157 >> MPLS_LS_S_SHIFT; 158 } 159 160 if (label == MPLS_LABEL_ENTROPY) { 161 key_keyid = skb_flow_dissector_target(flow_dissector, 162 FLOW_DISSECTOR_KEY_MPLS_ENTROPY, 163 target_container); 164 key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK); 165 } 166 return FLOW_DISSECT_RET_OUT_GOOD; 167 } 168 169 static enum flow_dissect_ret 170 __skb_flow_dissect_arp(const struct sk_buff *skb, 171 struct flow_dissector *flow_dissector, 172 void *target_container, void *data, int nhoff, int hlen) 173 { 174 struct flow_dissector_key_arp *key_arp; 175 struct { 176 unsigned char ar_sha[ETH_ALEN]; 177 unsigned char ar_sip[4]; 178 unsigned char ar_tha[ETH_ALEN]; 179 unsigned char ar_tip[4]; 180 } *arp_eth, _arp_eth; 181 const struct arphdr *arp; 182 struct arphdr _arp; 183 184 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP)) 185 return FLOW_DISSECT_RET_OUT_GOOD; 186 187 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data, 188 hlen, &_arp); 189 if (!arp) 190 return FLOW_DISSECT_RET_OUT_BAD; 191 192 if (arp->ar_hrd != htons(ARPHRD_ETHER) || 193 arp->ar_pro != htons(ETH_P_IP) || 194 arp->ar_hln != ETH_ALEN || 195 arp->ar_pln != 4 || 196 (arp->ar_op != htons(ARPOP_REPLY) && 197 arp->ar_op != htons(ARPOP_REQUEST))) 198 return FLOW_DISSECT_RET_OUT_BAD; 199 200 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp), 201 sizeof(_arp_eth), data, 202 hlen, &_arp_eth); 203 if (!arp_eth) 204 return FLOW_DISSECT_RET_OUT_BAD; 205 206 key_arp = skb_flow_dissector_target(flow_dissector, 207 FLOW_DISSECTOR_KEY_ARP, 208 target_container); 209 210 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip)); 211 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip)); 212 213 /* Only store the lower byte of the opcode; 214 * this covers ARPOP_REPLY and ARPOP_REQUEST. 215 */ 216 key_arp->op = ntohs(arp->ar_op) & 0xff; 217 218 ether_addr_copy(key_arp->sha, arp_eth->ar_sha); 219 ether_addr_copy(key_arp->tha, arp_eth->ar_tha); 220 221 return FLOW_DISSECT_RET_OUT_GOOD; 222 } 223 224 static enum flow_dissect_ret 225 __skb_flow_dissect_gre(const struct sk_buff *skb, 226 struct flow_dissector_key_control *key_control, 227 struct flow_dissector *flow_dissector, 228 void *target_container, void *data, 229 __be16 *p_proto, int *p_nhoff, int *p_hlen, 230 unsigned int flags) 231 { 232 struct flow_dissector_key_keyid *key_keyid; 233 struct gre_base_hdr *hdr, _hdr; 234 int offset = 0; 235 u16 gre_ver; 236 237 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), 238 data, *p_hlen, &_hdr); 239 if (!hdr) 240 return FLOW_DISSECT_RET_OUT_BAD; 241 242 /* Only look inside GRE without routing */ 243 if (hdr->flags & GRE_ROUTING) 244 return FLOW_DISSECT_RET_OUT_GOOD; 245 246 /* Only look inside GRE for version 0 and 1 */ 247 gre_ver = ntohs(hdr->flags & GRE_VERSION); 248 if (gre_ver > 1) 249 return FLOW_DISSECT_RET_OUT_GOOD; 250 251 *p_proto = hdr->protocol; 252 if (gre_ver) { 253 /* Version1 must be PPTP, and check the flags */ 254 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY))) 255 return FLOW_DISSECT_RET_OUT_GOOD; 256 } 257 258 offset += sizeof(struct gre_base_hdr); 259 260 if (hdr->flags & GRE_CSUM) 261 offset += sizeof(((struct gre_full_hdr *) 0)->csum) + 262 sizeof(((struct gre_full_hdr *) 0)->reserved1); 263 264 if (hdr->flags & GRE_KEY) { 265 const __be32 *keyid; 266 __be32 _keyid; 267 268 keyid = __skb_header_pointer(skb, *p_nhoff + offset, 269 sizeof(_keyid), 270 data, *p_hlen, &_keyid); 271 if (!keyid) 272 return FLOW_DISSECT_RET_OUT_BAD; 273 274 if (dissector_uses_key(flow_dissector, 275 FLOW_DISSECTOR_KEY_GRE_KEYID)) { 276 key_keyid = skb_flow_dissector_target(flow_dissector, 277 FLOW_DISSECTOR_KEY_GRE_KEYID, 278 target_container); 279 if (gre_ver == 0) 280 key_keyid->keyid = *keyid; 281 else 282 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK; 283 } 284 offset += sizeof(((struct gre_full_hdr *) 0)->key); 285 } 286 287 if (hdr->flags & GRE_SEQ) 288 offset += sizeof(((struct pptp_gre_header *) 0)->seq); 289 290 if (gre_ver == 0) { 291 if (*p_proto == htons(ETH_P_TEB)) { 292 const struct ethhdr *eth; 293 struct ethhdr _eth; 294 295 eth = __skb_header_pointer(skb, *p_nhoff + offset, 296 sizeof(_eth), 297 data, *p_hlen, &_eth); 298 if (!eth) 299 return FLOW_DISSECT_RET_OUT_BAD; 300 *p_proto = eth->h_proto; 301 offset += sizeof(*eth); 302 303 /* Cap headers that we access via pointers at the 304 * end of the Ethernet header as our maximum alignment 305 * at that point is only 2 bytes. 306 */ 307 if (NET_IP_ALIGN) 308 *p_hlen = *p_nhoff + offset; 309 } 310 } else { /* version 1, must be PPTP */ 311 u8 _ppp_hdr[PPP_HDRLEN]; 312 u8 *ppp_hdr; 313 314 if (hdr->flags & GRE_ACK) 315 offset += sizeof(((struct pptp_gre_header *) 0)->ack); 316 317 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset, 318 sizeof(_ppp_hdr), 319 data, *p_hlen, _ppp_hdr); 320 if (!ppp_hdr) 321 return FLOW_DISSECT_RET_OUT_BAD; 322 323 switch (PPP_PROTOCOL(ppp_hdr)) { 324 case PPP_IP: 325 *p_proto = htons(ETH_P_IP); 326 break; 327 case PPP_IPV6: 328 *p_proto = htons(ETH_P_IPV6); 329 break; 330 default: 331 /* Could probably catch some more like MPLS */ 332 break; 333 } 334 335 offset += PPP_HDRLEN; 336 } 337 338 *p_nhoff += offset; 339 key_control->flags |= FLOW_DIS_ENCAPSULATION; 340 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 341 return FLOW_DISSECT_RET_OUT_GOOD; 342 343 return FLOW_DISSECT_RET_OUT_PROTO_AGAIN; 344 } 345 346 static void 347 __skb_flow_dissect_tcp(const struct sk_buff *skb, 348 struct flow_dissector *flow_dissector, 349 void *target_container, void *data, int thoff, int hlen) 350 { 351 struct flow_dissector_key_tcp *key_tcp; 352 struct tcphdr *th, _th; 353 354 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP)) 355 return; 356 357 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th); 358 if (!th) 359 return; 360 361 if (unlikely(__tcp_hdrlen(th) < sizeof(_th))) 362 return; 363 364 key_tcp = skb_flow_dissector_target(flow_dissector, 365 FLOW_DISSECTOR_KEY_TCP, 366 target_container); 367 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF)); 368 } 369 370 static void 371 __skb_flow_dissect_ipv4(const struct sk_buff *skb, 372 struct flow_dissector *flow_dissector, 373 void *target_container, void *data, const struct iphdr *iph) 374 { 375 struct flow_dissector_key_ip *key_ip; 376 377 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 378 return; 379 380 key_ip = skb_flow_dissector_target(flow_dissector, 381 FLOW_DISSECTOR_KEY_IP, 382 target_container); 383 key_ip->tos = iph->tos; 384 key_ip->ttl = iph->ttl; 385 } 386 387 static void 388 __skb_flow_dissect_ipv6(const struct sk_buff *skb, 389 struct flow_dissector *flow_dissector, 390 void *target_container, void *data, const struct ipv6hdr *iph) 391 { 392 struct flow_dissector_key_ip *key_ip; 393 394 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP)) 395 return; 396 397 key_ip = skb_flow_dissector_target(flow_dissector, 398 FLOW_DISSECTOR_KEY_IP, 399 target_container); 400 key_ip->tos = ipv6_get_dsfield(iph); 401 key_ip->ttl = iph->hop_limit; 402 } 403 404 /** 405 * __skb_flow_dissect - extract the flow_keys struct and return it 406 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 407 * @flow_dissector: list of keys to dissect 408 * @target_container: target structure to put dissected values into 409 * @data: raw buffer pointer to the packet, if NULL use skb->data 410 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 411 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 412 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 413 * 414 * The function will try to retrieve individual keys into target specified 415 * by flow_dissector from either the skbuff or a raw buffer specified by the 416 * rest parameters. 417 * 418 * Caller must take care of zeroing target container memory. 419 */ 420 bool __skb_flow_dissect(const struct sk_buff *skb, 421 struct flow_dissector *flow_dissector, 422 void *target_container, 423 void *data, __be16 proto, int nhoff, int hlen, 424 unsigned int flags) 425 { 426 struct flow_dissector_key_control *key_control; 427 struct flow_dissector_key_basic *key_basic; 428 struct flow_dissector_key_addrs *key_addrs; 429 struct flow_dissector_key_ports *key_ports; 430 struct flow_dissector_key_icmp *key_icmp; 431 struct flow_dissector_key_tags *key_tags; 432 struct flow_dissector_key_vlan *key_vlan; 433 bool skip_vlan = false; 434 u8 ip_proto = 0; 435 bool ret; 436 437 if (!data) { 438 data = skb->data; 439 proto = skb_vlan_tag_present(skb) ? 440 skb->vlan_proto : skb->protocol; 441 nhoff = skb_network_offset(skb); 442 hlen = skb_headlen(skb); 443 } 444 445 /* It is ensured by skb_flow_dissector_init() that control key will 446 * be always present. 447 */ 448 key_control = skb_flow_dissector_target(flow_dissector, 449 FLOW_DISSECTOR_KEY_CONTROL, 450 target_container); 451 452 /* It is ensured by skb_flow_dissector_init() that basic key will 453 * be always present. 454 */ 455 key_basic = skb_flow_dissector_target(flow_dissector, 456 FLOW_DISSECTOR_KEY_BASIC, 457 target_container); 458 459 if (dissector_uses_key(flow_dissector, 460 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 461 struct ethhdr *eth = eth_hdr(skb); 462 struct flow_dissector_key_eth_addrs *key_eth_addrs; 463 464 key_eth_addrs = skb_flow_dissector_target(flow_dissector, 465 FLOW_DISSECTOR_KEY_ETH_ADDRS, 466 target_container); 467 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs)); 468 } 469 470 proto_again: 471 switch (proto) { 472 case htons(ETH_P_IP): { 473 const struct iphdr *iph; 474 struct iphdr _iph; 475 ip: 476 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 477 if (!iph || iph->ihl < 5) 478 goto out_bad; 479 nhoff += iph->ihl * 4; 480 481 ip_proto = iph->protocol; 482 483 if (dissector_uses_key(flow_dissector, 484 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 485 key_addrs = skb_flow_dissector_target(flow_dissector, 486 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 487 target_container); 488 489 memcpy(&key_addrs->v4addrs, &iph->saddr, 490 sizeof(key_addrs->v4addrs)); 491 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 492 } 493 494 if (ip_is_fragment(iph)) { 495 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 496 497 if (iph->frag_off & htons(IP_OFFSET)) { 498 goto out_good; 499 } else { 500 key_control->flags |= FLOW_DIS_FIRST_FRAG; 501 if (!(flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) 502 goto out_good; 503 } 504 } 505 506 __skb_flow_dissect_ipv4(skb, flow_dissector, 507 target_container, data, iph); 508 509 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) 510 goto out_good; 511 512 break; 513 } 514 case htons(ETH_P_IPV6): { 515 const struct ipv6hdr *iph; 516 struct ipv6hdr _iph; 517 518 ipv6: 519 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 520 if (!iph) 521 goto out_bad; 522 523 ip_proto = iph->nexthdr; 524 nhoff += sizeof(struct ipv6hdr); 525 526 if (dissector_uses_key(flow_dissector, 527 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 528 key_addrs = skb_flow_dissector_target(flow_dissector, 529 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 530 target_container); 531 532 memcpy(&key_addrs->v6addrs, &iph->saddr, 533 sizeof(key_addrs->v6addrs)); 534 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 535 } 536 537 if ((dissector_uses_key(flow_dissector, 538 FLOW_DISSECTOR_KEY_FLOW_LABEL) || 539 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) && 540 ip6_flowlabel(iph)) { 541 __be32 flow_label = ip6_flowlabel(iph); 542 543 if (dissector_uses_key(flow_dissector, 544 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 545 key_tags = skb_flow_dissector_target(flow_dissector, 546 FLOW_DISSECTOR_KEY_FLOW_LABEL, 547 target_container); 548 key_tags->flow_label = ntohl(flow_label); 549 } 550 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) 551 goto out_good; 552 } 553 554 __skb_flow_dissect_ipv6(skb, flow_dissector, 555 target_container, data, iph); 556 557 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) 558 goto out_good; 559 560 break; 561 } 562 case htons(ETH_P_8021AD): 563 case htons(ETH_P_8021Q): { 564 const struct vlan_hdr *vlan; 565 struct vlan_hdr _vlan; 566 bool vlan_tag_present = skb && skb_vlan_tag_present(skb); 567 568 if (vlan_tag_present) 569 proto = skb->protocol; 570 571 if (!vlan_tag_present || eth_type_vlan(skb->protocol)) { 572 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), 573 data, hlen, &_vlan); 574 if (!vlan) 575 goto out_bad; 576 proto = vlan->h_vlan_encapsulated_proto; 577 nhoff += sizeof(*vlan); 578 if (skip_vlan) 579 goto proto_again; 580 } 581 582 skip_vlan = true; 583 if (dissector_uses_key(flow_dissector, 584 FLOW_DISSECTOR_KEY_VLAN)) { 585 key_vlan = skb_flow_dissector_target(flow_dissector, 586 FLOW_DISSECTOR_KEY_VLAN, 587 target_container); 588 589 if (vlan_tag_present) { 590 key_vlan->vlan_id = skb_vlan_tag_get_id(skb); 591 key_vlan->vlan_priority = 592 (skb_vlan_tag_get_prio(skb) >> VLAN_PRIO_SHIFT); 593 } else { 594 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) & 595 VLAN_VID_MASK; 596 key_vlan->vlan_priority = 597 (ntohs(vlan->h_vlan_TCI) & 598 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 599 } 600 } 601 602 goto proto_again; 603 } 604 case htons(ETH_P_PPP_SES): { 605 struct { 606 struct pppoe_hdr hdr; 607 __be16 proto; 608 } *hdr, _hdr; 609 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 610 if (!hdr) 611 goto out_bad; 612 proto = hdr->proto; 613 nhoff += PPPOE_SES_HLEN; 614 switch (proto) { 615 case htons(PPP_IP): 616 goto ip; 617 case htons(PPP_IPV6): 618 goto ipv6; 619 default: 620 goto out_bad; 621 } 622 } 623 case htons(ETH_P_TIPC): { 624 struct { 625 __be32 pre[3]; 626 __be32 srcnode; 627 } *hdr, _hdr; 628 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 629 if (!hdr) 630 goto out_bad; 631 632 if (dissector_uses_key(flow_dissector, 633 FLOW_DISSECTOR_KEY_TIPC_ADDRS)) { 634 key_addrs = skb_flow_dissector_target(flow_dissector, 635 FLOW_DISSECTOR_KEY_TIPC_ADDRS, 636 target_container); 637 key_addrs->tipcaddrs.srcnode = hdr->srcnode; 638 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC_ADDRS; 639 } 640 goto out_good; 641 } 642 643 case htons(ETH_P_MPLS_UC): 644 case htons(ETH_P_MPLS_MC): 645 mpls: 646 switch (__skb_flow_dissect_mpls(skb, flow_dissector, 647 target_container, data, 648 nhoff, hlen)) { 649 case FLOW_DISSECT_RET_OUT_GOOD: 650 goto out_good; 651 case FLOW_DISSECT_RET_OUT_BAD: 652 default: 653 goto out_bad; 654 } 655 case htons(ETH_P_FCOE): 656 if ((hlen - nhoff) < FCOE_HEADER_LEN) 657 goto out_bad; 658 659 nhoff += FCOE_HEADER_LEN; 660 goto out_good; 661 662 case htons(ETH_P_ARP): 663 case htons(ETH_P_RARP): 664 switch (__skb_flow_dissect_arp(skb, flow_dissector, 665 target_container, data, 666 nhoff, hlen)) { 667 case FLOW_DISSECT_RET_OUT_GOOD: 668 goto out_good; 669 case FLOW_DISSECT_RET_OUT_BAD: 670 default: 671 goto out_bad; 672 } 673 default: 674 goto out_bad; 675 } 676 677 ip_proto_again: 678 switch (ip_proto) { 679 case IPPROTO_GRE: 680 switch (__skb_flow_dissect_gre(skb, key_control, flow_dissector, 681 target_container, data, 682 &proto, &nhoff, &hlen, flags)) { 683 case FLOW_DISSECT_RET_OUT_GOOD: 684 goto out_good; 685 case FLOW_DISSECT_RET_OUT_BAD: 686 goto out_bad; 687 case FLOW_DISSECT_RET_OUT_PROTO_AGAIN: 688 goto proto_again; 689 } 690 case NEXTHDR_HOP: 691 case NEXTHDR_ROUTING: 692 case NEXTHDR_DEST: { 693 u8 _opthdr[2], *opthdr; 694 695 if (proto != htons(ETH_P_IPV6)) 696 break; 697 698 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr), 699 data, hlen, &_opthdr); 700 if (!opthdr) 701 goto out_bad; 702 703 ip_proto = opthdr[0]; 704 nhoff += (opthdr[1] + 1) << 3; 705 706 goto ip_proto_again; 707 } 708 case NEXTHDR_FRAGMENT: { 709 struct frag_hdr _fh, *fh; 710 711 if (proto != htons(ETH_P_IPV6)) 712 break; 713 714 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh), 715 data, hlen, &_fh); 716 717 if (!fh) 718 goto out_bad; 719 720 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 721 722 nhoff += sizeof(_fh); 723 ip_proto = fh->nexthdr; 724 725 if (!(fh->frag_off & htons(IP6_OFFSET))) { 726 key_control->flags |= FLOW_DIS_FIRST_FRAG; 727 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) 728 goto ip_proto_again; 729 } 730 goto out_good; 731 } 732 case IPPROTO_IPIP: 733 proto = htons(ETH_P_IP); 734 735 key_control->flags |= FLOW_DIS_ENCAPSULATION; 736 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 737 goto out_good; 738 739 goto ip; 740 case IPPROTO_IPV6: 741 proto = htons(ETH_P_IPV6); 742 743 key_control->flags |= FLOW_DIS_ENCAPSULATION; 744 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 745 goto out_good; 746 747 goto ipv6; 748 case IPPROTO_MPLS: 749 proto = htons(ETH_P_MPLS_UC); 750 goto mpls; 751 case IPPROTO_TCP: 752 __skb_flow_dissect_tcp(skb, flow_dissector, target_container, 753 data, nhoff, hlen); 754 break; 755 default: 756 break; 757 } 758 759 if (dissector_uses_key(flow_dissector, 760 FLOW_DISSECTOR_KEY_PORTS)) { 761 key_ports = skb_flow_dissector_target(flow_dissector, 762 FLOW_DISSECTOR_KEY_PORTS, 763 target_container); 764 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 765 data, hlen); 766 } 767 768 if (dissector_uses_key(flow_dissector, 769 FLOW_DISSECTOR_KEY_ICMP)) { 770 key_icmp = skb_flow_dissector_target(flow_dissector, 771 FLOW_DISSECTOR_KEY_ICMP, 772 target_container); 773 key_icmp->icmp = skb_flow_get_be16(skb, nhoff, data, hlen); 774 } 775 776 out_good: 777 ret = true; 778 779 key_control->thoff = (u16)nhoff; 780 out: 781 key_basic->n_proto = proto; 782 key_basic->ip_proto = ip_proto; 783 784 return ret; 785 786 out_bad: 787 ret = false; 788 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); 789 goto out; 790 } 791 EXPORT_SYMBOL(__skb_flow_dissect); 792 793 static u32 hashrnd __read_mostly; 794 static __always_inline void __flow_hash_secret_init(void) 795 { 796 net_get_random_once(&hashrnd, sizeof(hashrnd)); 797 } 798 799 static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, 800 u32 keyval) 801 { 802 return jhash2(words, length, keyval); 803 } 804 805 static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) 806 { 807 const void *p = flow; 808 809 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); 810 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); 811 } 812 813 static inline size_t flow_keys_hash_length(const struct flow_keys *flow) 814 { 815 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); 816 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); 817 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != 818 sizeof(*flow) - sizeof(flow->addrs)); 819 820 switch (flow->control.addr_type) { 821 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 822 diff -= sizeof(flow->addrs.v4addrs); 823 break; 824 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 825 diff -= sizeof(flow->addrs.v6addrs); 826 break; 827 case FLOW_DISSECTOR_KEY_TIPC_ADDRS: 828 diff -= sizeof(flow->addrs.tipcaddrs); 829 break; 830 } 831 return (sizeof(*flow) - diff) / sizeof(u32); 832 } 833 834 __be32 flow_get_u32_src(const struct flow_keys *flow) 835 { 836 switch (flow->control.addr_type) { 837 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 838 return flow->addrs.v4addrs.src; 839 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 840 return (__force __be32)ipv6_addr_hash( 841 &flow->addrs.v6addrs.src); 842 case FLOW_DISSECTOR_KEY_TIPC_ADDRS: 843 return flow->addrs.tipcaddrs.srcnode; 844 default: 845 return 0; 846 } 847 } 848 EXPORT_SYMBOL(flow_get_u32_src); 849 850 __be32 flow_get_u32_dst(const struct flow_keys *flow) 851 { 852 switch (flow->control.addr_type) { 853 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 854 return flow->addrs.v4addrs.dst; 855 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 856 return (__force __be32)ipv6_addr_hash( 857 &flow->addrs.v6addrs.dst); 858 default: 859 return 0; 860 } 861 } 862 EXPORT_SYMBOL(flow_get_u32_dst); 863 864 static inline void __flow_hash_consistentify(struct flow_keys *keys) 865 { 866 int addr_diff, i; 867 868 switch (keys->control.addr_type) { 869 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 870 addr_diff = (__force u32)keys->addrs.v4addrs.dst - 871 (__force u32)keys->addrs.v4addrs.src; 872 if ((addr_diff < 0) || 873 (addr_diff == 0 && 874 ((__force u16)keys->ports.dst < 875 (__force u16)keys->ports.src))) { 876 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); 877 swap(keys->ports.src, keys->ports.dst); 878 } 879 break; 880 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 881 addr_diff = memcmp(&keys->addrs.v6addrs.dst, 882 &keys->addrs.v6addrs.src, 883 sizeof(keys->addrs.v6addrs.dst)); 884 if ((addr_diff < 0) || 885 (addr_diff == 0 && 886 ((__force u16)keys->ports.dst < 887 (__force u16)keys->ports.src))) { 888 for (i = 0; i < 4; i++) 889 swap(keys->addrs.v6addrs.src.s6_addr32[i], 890 keys->addrs.v6addrs.dst.s6_addr32[i]); 891 swap(keys->ports.src, keys->ports.dst); 892 } 893 break; 894 } 895 } 896 897 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) 898 { 899 u32 hash; 900 901 __flow_hash_consistentify(keys); 902 903 hash = __flow_hash_words(flow_keys_hash_start(keys), 904 flow_keys_hash_length(keys), keyval); 905 if (!hash) 906 hash = 1; 907 908 return hash; 909 } 910 911 u32 flow_hash_from_keys(struct flow_keys *keys) 912 { 913 __flow_hash_secret_init(); 914 return __flow_hash_from_keys(keys, hashrnd); 915 } 916 EXPORT_SYMBOL(flow_hash_from_keys); 917 918 static inline u32 ___skb_get_hash(const struct sk_buff *skb, 919 struct flow_keys *keys, u32 keyval) 920 { 921 skb_flow_dissect_flow_keys(skb, keys, 922 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 923 924 return __flow_hash_from_keys(keys, keyval); 925 } 926 927 struct _flow_keys_digest_data { 928 __be16 n_proto; 929 u8 ip_proto; 930 u8 padding; 931 __be32 ports; 932 __be32 src; 933 __be32 dst; 934 }; 935 936 void make_flow_keys_digest(struct flow_keys_digest *digest, 937 const struct flow_keys *flow) 938 { 939 struct _flow_keys_digest_data *data = 940 (struct _flow_keys_digest_data *)digest; 941 942 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); 943 944 memset(digest, 0, sizeof(*digest)); 945 946 data->n_proto = flow->basic.n_proto; 947 data->ip_proto = flow->basic.ip_proto; 948 data->ports = flow->ports.ports; 949 data->src = flow->addrs.v4addrs.src; 950 data->dst = flow->addrs.v4addrs.dst; 951 } 952 EXPORT_SYMBOL(make_flow_keys_digest); 953 954 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly; 955 956 u32 __skb_get_hash_symmetric(const struct sk_buff *skb) 957 { 958 struct flow_keys keys; 959 960 __flow_hash_secret_init(); 961 962 memset(&keys, 0, sizeof(keys)); 963 __skb_flow_dissect(skb, &flow_keys_dissector_symmetric, &keys, 964 NULL, 0, 0, 0, 965 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 966 967 return __flow_hash_from_keys(&keys, hashrnd); 968 } 969 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric); 970 971 /** 972 * __skb_get_hash: calculate a flow hash 973 * @skb: sk_buff to calculate flow hash from 974 * 975 * This function calculates a flow hash based on src/dst addresses 976 * and src/dst port numbers. Sets hash in skb to non-zero hash value 977 * on success, zero indicates no valid hash. Also, sets l4_hash in skb 978 * if hash is a canonical 4-tuple hash over transport ports. 979 */ 980 void __skb_get_hash(struct sk_buff *skb) 981 { 982 struct flow_keys keys; 983 u32 hash; 984 985 __flow_hash_secret_init(); 986 987 hash = ___skb_get_hash(skb, &keys, hashrnd); 988 989 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 990 } 991 EXPORT_SYMBOL(__skb_get_hash); 992 993 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) 994 { 995 struct flow_keys keys; 996 997 return ___skb_get_hash(skb, &keys, perturb); 998 } 999 EXPORT_SYMBOL(skb_get_hash_perturb); 1000 1001 __u32 __skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) 1002 { 1003 struct flow_keys keys; 1004 1005 memset(&keys, 0, sizeof(keys)); 1006 1007 memcpy(&keys.addrs.v6addrs.src, &fl6->saddr, 1008 sizeof(keys.addrs.v6addrs.src)); 1009 memcpy(&keys.addrs.v6addrs.dst, &fl6->daddr, 1010 sizeof(keys.addrs.v6addrs.dst)); 1011 keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1012 keys.ports.src = fl6->fl6_sport; 1013 keys.ports.dst = fl6->fl6_dport; 1014 keys.keyid.keyid = fl6->fl6_gre_key; 1015 keys.tags.flow_label = (__force u32)fl6->flowlabel; 1016 keys.basic.ip_proto = fl6->flowi6_proto; 1017 1018 __skb_set_sw_hash(skb, flow_hash_from_keys(&keys), 1019 flow_keys_have_l4(&keys)); 1020 1021 return skb->hash; 1022 } 1023 EXPORT_SYMBOL(__skb_get_hash_flowi6); 1024 1025 __u32 __skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl4) 1026 { 1027 struct flow_keys keys; 1028 1029 memset(&keys, 0, sizeof(keys)); 1030 1031 keys.addrs.v4addrs.src = fl4->saddr; 1032 keys.addrs.v4addrs.dst = fl4->daddr; 1033 keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1034 keys.ports.src = fl4->fl4_sport; 1035 keys.ports.dst = fl4->fl4_dport; 1036 keys.keyid.keyid = fl4->fl4_gre_key; 1037 keys.basic.ip_proto = fl4->flowi4_proto; 1038 1039 __skb_set_sw_hash(skb, flow_hash_from_keys(&keys), 1040 flow_keys_have_l4(&keys)); 1041 1042 return skb->hash; 1043 } 1044 EXPORT_SYMBOL(__skb_get_hash_flowi4); 1045 1046 u32 __skb_get_poff(const struct sk_buff *skb, void *data, 1047 const struct flow_keys *keys, int hlen) 1048 { 1049 u32 poff = keys->control.thoff; 1050 1051 /* skip L4 headers for fragments after the first */ 1052 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) && 1053 !(keys->control.flags & FLOW_DIS_FIRST_FRAG)) 1054 return poff; 1055 1056 switch (keys->basic.ip_proto) { 1057 case IPPROTO_TCP: { 1058 /* access doff as u8 to avoid unaligned access */ 1059 const u8 *doff; 1060 u8 _doff; 1061 1062 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 1063 data, hlen, &_doff); 1064 if (!doff) 1065 return poff; 1066 1067 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 1068 break; 1069 } 1070 case IPPROTO_UDP: 1071 case IPPROTO_UDPLITE: 1072 poff += sizeof(struct udphdr); 1073 break; 1074 /* For the rest, we do not really care about header 1075 * extensions at this point for now. 1076 */ 1077 case IPPROTO_ICMP: 1078 poff += sizeof(struct icmphdr); 1079 break; 1080 case IPPROTO_ICMPV6: 1081 poff += sizeof(struct icmp6hdr); 1082 break; 1083 case IPPROTO_IGMP: 1084 poff += sizeof(struct igmphdr); 1085 break; 1086 case IPPROTO_DCCP: 1087 poff += sizeof(struct dccp_hdr); 1088 break; 1089 case IPPROTO_SCTP: 1090 poff += sizeof(struct sctphdr); 1091 break; 1092 } 1093 1094 return poff; 1095 } 1096 1097 /** 1098 * skb_get_poff - get the offset to the payload 1099 * @skb: sk_buff to get the payload offset from 1100 * 1101 * The function will get the offset to the payload as far as it could 1102 * be dissected. The main user is currently BPF, so that we can dynamically 1103 * truncate packets without needing to push actual payload to the user 1104 * space and can analyze headers only, instead. 1105 */ 1106 u32 skb_get_poff(const struct sk_buff *skb) 1107 { 1108 struct flow_keys keys; 1109 1110 if (!skb_flow_dissect_flow_keys(skb, &keys, 0)) 1111 return 0; 1112 1113 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 1114 } 1115 1116 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) 1117 { 1118 memset(keys, 0, sizeof(*keys)); 1119 1120 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr, 1121 sizeof(keys->addrs.v6addrs.src)); 1122 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr, 1123 sizeof(keys->addrs.v6addrs.dst)); 1124 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1125 keys->ports.src = fl6->fl6_sport; 1126 keys->ports.dst = fl6->fl6_dport; 1127 keys->keyid.keyid = fl6->fl6_gre_key; 1128 keys->tags.flow_label = (__force u32)fl6->flowlabel; 1129 keys->basic.ip_proto = fl6->flowi6_proto; 1130 1131 return flow_hash_from_keys(keys); 1132 } 1133 EXPORT_SYMBOL(__get_hash_from_flowi6); 1134 1135 __u32 __get_hash_from_flowi4(const struct flowi4 *fl4, struct flow_keys *keys) 1136 { 1137 memset(keys, 0, sizeof(*keys)); 1138 1139 keys->addrs.v4addrs.src = fl4->saddr; 1140 keys->addrs.v4addrs.dst = fl4->daddr; 1141 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1142 keys->ports.src = fl4->fl4_sport; 1143 keys->ports.dst = fl4->fl4_dport; 1144 keys->keyid.keyid = fl4->fl4_gre_key; 1145 keys->basic.ip_proto = fl4->flowi4_proto; 1146 1147 return flow_hash_from_keys(keys); 1148 } 1149 EXPORT_SYMBOL(__get_hash_from_flowi4); 1150 1151 static const struct flow_dissector_key flow_keys_dissector_keys[] = { 1152 { 1153 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1154 .offset = offsetof(struct flow_keys, control), 1155 }, 1156 { 1157 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1158 .offset = offsetof(struct flow_keys, basic), 1159 }, 1160 { 1161 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1162 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1163 }, 1164 { 1165 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1166 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1167 }, 1168 { 1169 .key_id = FLOW_DISSECTOR_KEY_TIPC_ADDRS, 1170 .offset = offsetof(struct flow_keys, addrs.tipcaddrs), 1171 }, 1172 { 1173 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1174 .offset = offsetof(struct flow_keys, ports), 1175 }, 1176 { 1177 .key_id = FLOW_DISSECTOR_KEY_VLAN, 1178 .offset = offsetof(struct flow_keys, vlan), 1179 }, 1180 { 1181 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL, 1182 .offset = offsetof(struct flow_keys, tags), 1183 }, 1184 { 1185 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, 1186 .offset = offsetof(struct flow_keys, keyid), 1187 }, 1188 }; 1189 1190 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = { 1191 { 1192 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1193 .offset = offsetof(struct flow_keys, control), 1194 }, 1195 { 1196 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1197 .offset = offsetof(struct flow_keys, basic), 1198 }, 1199 { 1200 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1201 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1202 }, 1203 { 1204 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1205 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1206 }, 1207 { 1208 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1209 .offset = offsetof(struct flow_keys, ports), 1210 }, 1211 }; 1212 1213 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = { 1214 { 1215 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1216 .offset = offsetof(struct flow_keys, control), 1217 }, 1218 { 1219 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1220 .offset = offsetof(struct flow_keys, basic), 1221 }, 1222 }; 1223 1224 struct flow_dissector flow_keys_dissector __read_mostly; 1225 EXPORT_SYMBOL(flow_keys_dissector); 1226 1227 struct flow_dissector flow_keys_buf_dissector __read_mostly; 1228 1229 static int __init init_default_flow_dissectors(void) 1230 { 1231 skb_flow_dissector_init(&flow_keys_dissector, 1232 flow_keys_dissector_keys, 1233 ARRAY_SIZE(flow_keys_dissector_keys)); 1234 skb_flow_dissector_init(&flow_keys_dissector_symmetric, 1235 flow_keys_dissector_symmetric_keys, 1236 ARRAY_SIZE(flow_keys_dissector_symmetric_keys)); 1237 skb_flow_dissector_init(&flow_keys_buf_dissector, 1238 flow_keys_buf_dissector_keys, 1239 ARRAY_SIZE(flow_keys_buf_dissector_keys)); 1240 return 0; 1241 } 1242 1243 core_initcall(init_default_flow_dissectors); 1244