1 #include <linux/kernel.h> 2 #include <linux/skbuff.h> 3 #include <linux/export.h> 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip.h> 8 #include <net/ipv6.h> 9 #include <net/gre.h> 10 #include <net/pptp.h> 11 #include <linux/igmp.h> 12 #include <linux/icmp.h> 13 #include <linux/sctp.h> 14 #include <linux/dccp.h> 15 #include <linux/if_tunnel.h> 16 #include <linux/if_pppox.h> 17 #include <linux/ppp_defs.h> 18 #include <linux/stddef.h> 19 #include <linux/if_ether.h> 20 #include <linux/mpls.h> 21 #include <net/flow_dissector.h> 22 #include <scsi/fc/fc_fcoe.h> 23 24 static void dissector_set_key(struct flow_dissector *flow_dissector, 25 enum flow_dissector_key_id key_id) 26 { 27 flow_dissector->used_keys |= (1 << key_id); 28 } 29 30 void skb_flow_dissector_init(struct flow_dissector *flow_dissector, 31 const struct flow_dissector_key *key, 32 unsigned int key_count) 33 { 34 unsigned int i; 35 36 memset(flow_dissector, 0, sizeof(*flow_dissector)); 37 38 for (i = 0; i < key_count; i++, key++) { 39 /* User should make sure that every key target offset is withing 40 * boundaries of unsigned short. 41 */ 42 BUG_ON(key->offset > USHRT_MAX); 43 BUG_ON(dissector_uses_key(flow_dissector, 44 key->key_id)); 45 46 dissector_set_key(flow_dissector, key->key_id); 47 flow_dissector->offset[key->key_id] = key->offset; 48 } 49 50 /* Ensure that the dissector always includes control and basic key. 51 * That way we are able to avoid handling lack of these in fast path. 52 */ 53 BUG_ON(!dissector_uses_key(flow_dissector, 54 FLOW_DISSECTOR_KEY_CONTROL)); 55 BUG_ON(!dissector_uses_key(flow_dissector, 56 FLOW_DISSECTOR_KEY_BASIC)); 57 } 58 EXPORT_SYMBOL(skb_flow_dissector_init); 59 60 /** 61 * skb_flow_get_be16 - extract be16 entity 62 * @skb: sk_buff to extract from 63 * @poff: offset to extract at 64 * @data: raw buffer pointer to the packet 65 * @hlen: packet header length 66 * 67 * The function will try to retrieve a be32 entity at 68 * offset poff 69 */ 70 static __be16 skb_flow_get_be16(const struct sk_buff *skb, int poff, 71 void *data, int hlen) 72 { 73 __be16 *u, _u; 74 75 u = __skb_header_pointer(skb, poff, sizeof(_u), data, hlen, &_u); 76 if (u) 77 return *u; 78 79 return 0; 80 } 81 82 /** 83 * __skb_flow_get_ports - extract the upper layer ports and return them 84 * @skb: sk_buff to extract the ports from 85 * @thoff: transport header offset 86 * @ip_proto: protocol for which to get port offset 87 * @data: raw buffer pointer to the packet, if NULL use skb->data 88 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 89 * 90 * The function will try to retrieve the ports at offset thoff + poff where poff 91 * is the protocol port offset returned from proto_ports_offset 92 */ 93 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 94 void *data, int hlen) 95 { 96 int poff = proto_ports_offset(ip_proto); 97 98 if (!data) { 99 data = skb->data; 100 hlen = skb_headlen(skb); 101 } 102 103 if (poff >= 0) { 104 __be32 *ports, _ports; 105 106 ports = __skb_header_pointer(skb, thoff + poff, 107 sizeof(_ports), data, hlen, &_ports); 108 if (ports) 109 return *ports; 110 } 111 112 return 0; 113 } 114 EXPORT_SYMBOL(__skb_flow_get_ports); 115 116 enum flow_dissect_ret { 117 FLOW_DISSECT_RET_OUT_GOOD, 118 FLOW_DISSECT_RET_OUT_BAD, 119 FLOW_DISSECT_RET_OUT_PROTO_AGAIN, 120 }; 121 122 static enum flow_dissect_ret 123 __skb_flow_dissect_mpls(const struct sk_buff *skb, 124 struct flow_dissector *flow_dissector, 125 void *target_container, void *data, int nhoff, int hlen) 126 { 127 struct flow_dissector_key_keyid *key_keyid; 128 struct mpls_label *hdr, _hdr[2]; 129 u32 entry, label; 130 131 if (!dissector_uses_key(flow_dissector, 132 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) && 133 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) 134 return FLOW_DISSECT_RET_OUT_GOOD; 135 136 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, 137 hlen, &_hdr); 138 if (!hdr) 139 return FLOW_DISSECT_RET_OUT_BAD; 140 141 entry = ntohl(hdr[0].entry); 142 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT; 143 144 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) { 145 struct flow_dissector_key_mpls *key_mpls; 146 147 key_mpls = skb_flow_dissector_target(flow_dissector, 148 FLOW_DISSECTOR_KEY_MPLS, 149 target_container); 150 key_mpls->mpls_label = label; 151 key_mpls->mpls_ttl = (entry & MPLS_LS_TTL_MASK) 152 >> MPLS_LS_TTL_SHIFT; 153 key_mpls->mpls_tc = (entry & MPLS_LS_TC_MASK) 154 >> MPLS_LS_TC_SHIFT; 155 key_mpls->mpls_bos = (entry & MPLS_LS_S_MASK) 156 >> MPLS_LS_S_SHIFT; 157 } 158 159 if (label == MPLS_LABEL_ENTROPY) { 160 key_keyid = skb_flow_dissector_target(flow_dissector, 161 FLOW_DISSECTOR_KEY_MPLS_ENTROPY, 162 target_container); 163 key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK); 164 } 165 return FLOW_DISSECT_RET_OUT_GOOD; 166 } 167 168 static enum flow_dissect_ret 169 __skb_flow_dissect_arp(const struct sk_buff *skb, 170 struct flow_dissector *flow_dissector, 171 void *target_container, void *data, int nhoff, int hlen) 172 { 173 struct flow_dissector_key_arp *key_arp; 174 struct { 175 unsigned char ar_sha[ETH_ALEN]; 176 unsigned char ar_sip[4]; 177 unsigned char ar_tha[ETH_ALEN]; 178 unsigned char ar_tip[4]; 179 } *arp_eth, _arp_eth; 180 const struct arphdr *arp; 181 struct arphdr _arp; 182 183 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP)) 184 return FLOW_DISSECT_RET_OUT_GOOD; 185 186 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data, 187 hlen, &_arp); 188 if (!arp) 189 return FLOW_DISSECT_RET_OUT_BAD; 190 191 if (arp->ar_hrd != htons(ARPHRD_ETHER) || 192 arp->ar_pro != htons(ETH_P_IP) || 193 arp->ar_hln != ETH_ALEN || 194 arp->ar_pln != 4 || 195 (arp->ar_op != htons(ARPOP_REPLY) && 196 arp->ar_op != htons(ARPOP_REQUEST))) 197 return FLOW_DISSECT_RET_OUT_BAD; 198 199 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp), 200 sizeof(_arp_eth), data, 201 hlen, &_arp_eth); 202 if (!arp_eth) 203 return FLOW_DISSECT_RET_OUT_BAD; 204 205 key_arp = skb_flow_dissector_target(flow_dissector, 206 FLOW_DISSECTOR_KEY_ARP, 207 target_container); 208 209 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip)); 210 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip)); 211 212 /* Only store the lower byte of the opcode; 213 * this covers ARPOP_REPLY and ARPOP_REQUEST. 214 */ 215 key_arp->op = ntohs(arp->ar_op) & 0xff; 216 217 ether_addr_copy(key_arp->sha, arp_eth->ar_sha); 218 ether_addr_copy(key_arp->tha, arp_eth->ar_tha); 219 220 return FLOW_DISSECT_RET_OUT_GOOD; 221 } 222 223 static enum flow_dissect_ret 224 __skb_flow_dissect_gre(const struct sk_buff *skb, 225 struct flow_dissector_key_control *key_control, 226 struct flow_dissector *flow_dissector, 227 void *target_container, void *data, 228 __be16 *p_proto, int *p_nhoff, int *p_hlen, 229 unsigned int flags) 230 { 231 struct flow_dissector_key_keyid *key_keyid; 232 struct gre_base_hdr *hdr, _hdr; 233 int offset = 0; 234 u16 gre_ver; 235 236 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), 237 data, *p_hlen, &_hdr); 238 if (!hdr) 239 return FLOW_DISSECT_RET_OUT_BAD; 240 241 /* Only look inside GRE without routing */ 242 if (hdr->flags & GRE_ROUTING) 243 return FLOW_DISSECT_RET_OUT_GOOD; 244 245 /* Only look inside GRE for version 0 and 1 */ 246 gre_ver = ntohs(hdr->flags & GRE_VERSION); 247 if (gre_ver > 1) 248 return FLOW_DISSECT_RET_OUT_GOOD; 249 250 *p_proto = hdr->protocol; 251 if (gre_ver) { 252 /* Version1 must be PPTP, and check the flags */ 253 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY))) 254 return FLOW_DISSECT_RET_OUT_GOOD; 255 } 256 257 offset += sizeof(struct gre_base_hdr); 258 259 if (hdr->flags & GRE_CSUM) 260 offset += sizeof(((struct gre_full_hdr *) 0)->csum) + 261 sizeof(((struct gre_full_hdr *) 0)->reserved1); 262 263 if (hdr->flags & GRE_KEY) { 264 const __be32 *keyid; 265 __be32 _keyid; 266 267 keyid = __skb_header_pointer(skb, *p_nhoff + offset, 268 sizeof(_keyid), 269 data, *p_hlen, &_keyid); 270 if (!keyid) 271 return FLOW_DISSECT_RET_OUT_BAD; 272 273 if (dissector_uses_key(flow_dissector, 274 FLOW_DISSECTOR_KEY_GRE_KEYID)) { 275 key_keyid = skb_flow_dissector_target(flow_dissector, 276 FLOW_DISSECTOR_KEY_GRE_KEYID, 277 target_container); 278 if (gre_ver == 0) 279 key_keyid->keyid = *keyid; 280 else 281 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK; 282 } 283 offset += sizeof(((struct gre_full_hdr *) 0)->key); 284 } 285 286 if (hdr->flags & GRE_SEQ) 287 offset += sizeof(((struct pptp_gre_header *) 0)->seq); 288 289 if (gre_ver == 0) { 290 if (*p_proto == htons(ETH_P_TEB)) { 291 const struct ethhdr *eth; 292 struct ethhdr _eth; 293 294 eth = __skb_header_pointer(skb, *p_nhoff + offset, 295 sizeof(_eth), 296 data, *p_hlen, &_eth); 297 if (!eth) 298 return FLOW_DISSECT_RET_OUT_BAD; 299 *p_proto = eth->h_proto; 300 offset += sizeof(*eth); 301 302 /* Cap headers that we access via pointers at the 303 * end of the Ethernet header as our maximum alignment 304 * at that point is only 2 bytes. 305 */ 306 if (NET_IP_ALIGN) 307 *p_hlen = *p_nhoff + offset; 308 } 309 } else { /* version 1, must be PPTP */ 310 u8 _ppp_hdr[PPP_HDRLEN]; 311 u8 *ppp_hdr; 312 313 if (hdr->flags & GRE_ACK) 314 offset += sizeof(((struct pptp_gre_header *) 0)->ack); 315 316 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset, 317 sizeof(_ppp_hdr), 318 data, *p_hlen, _ppp_hdr); 319 if (!ppp_hdr) 320 return FLOW_DISSECT_RET_OUT_BAD; 321 322 switch (PPP_PROTOCOL(ppp_hdr)) { 323 case PPP_IP: 324 *p_proto = htons(ETH_P_IP); 325 break; 326 case PPP_IPV6: 327 *p_proto = htons(ETH_P_IPV6); 328 break; 329 default: 330 /* Could probably catch some more like MPLS */ 331 break; 332 } 333 334 offset += PPP_HDRLEN; 335 } 336 337 *p_nhoff += offset; 338 key_control->flags |= FLOW_DIS_ENCAPSULATION; 339 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 340 return FLOW_DISSECT_RET_OUT_GOOD; 341 342 return FLOW_DISSECT_RET_OUT_PROTO_AGAIN; 343 } 344 345 /** 346 * __skb_flow_dissect - extract the flow_keys struct and return it 347 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 348 * @flow_dissector: list of keys to dissect 349 * @target_container: target structure to put dissected values into 350 * @data: raw buffer pointer to the packet, if NULL use skb->data 351 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 352 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 353 * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 354 * 355 * The function will try to retrieve individual keys into target specified 356 * by flow_dissector from either the skbuff or a raw buffer specified by the 357 * rest parameters. 358 * 359 * Caller must take care of zeroing target container memory. 360 */ 361 bool __skb_flow_dissect(const struct sk_buff *skb, 362 struct flow_dissector *flow_dissector, 363 void *target_container, 364 void *data, __be16 proto, int nhoff, int hlen, 365 unsigned int flags) 366 { 367 struct flow_dissector_key_control *key_control; 368 struct flow_dissector_key_basic *key_basic; 369 struct flow_dissector_key_addrs *key_addrs; 370 struct flow_dissector_key_ports *key_ports; 371 struct flow_dissector_key_icmp *key_icmp; 372 struct flow_dissector_key_tags *key_tags; 373 struct flow_dissector_key_vlan *key_vlan; 374 bool skip_vlan = false; 375 u8 ip_proto = 0; 376 bool ret; 377 378 if (!data) { 379 data = skb->data; 380 proto = skb_vlan_tag_present(skb) ? 381 skb->vlan_proto : skb->protocol; 382 nhoff = skb_network_offset(skb); 383 hlen = skb_headlen(skb); 384 } 385 386 /* It is ensured by skb_flow_dissector_init() that control key will 387 * be always present. 388 */ 389 key_control = skb_flow_dissector_target(flow_dissector, 390 FLOW_DISSECTOR_KEY_CONTROL, 391 target_container); 392 393 /* It is ensured by skb_flow_dissector_init() that basic key will 394 * be always present. 395 */ 396 key_basic = skb_flow_dissector_target(flow_dissector, 397 FLOW_DISSECTOR_KEY_BASIC, 398 target_container); 399 400 if (dissector_uses_key(flow_dissector, 401 FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 402 struct ethhdr *eth = eth_hdr(skb); 403 struct flow_dissector_key_eth_addrs *key_eth_addrs; 404 405 key_eth_addrs = skb_flow_dissector_target(flow_dissector, 406 FLOW_DISSECTOR_KEY_ETH_ADDRS, 407 target_container); 408 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs)); 409 } 410 411 proto_again: 412 switch (proto) { 413 case htons(ETH_P_IP): { 414 const struct iphdr *iph; 415 struct iphdr _iph; 416 ip: 417 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 418 if (!iph || iph->ihl < 5) 419 goto out_bad; 420 nhoff += iph->ihl * 4; 421 422 ip_proto = iph->protocol; 423 424 if (dissector_uses_key(flow_dissector, 425 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 426 key_addrs = skb_flow_dissector_target(flow_dissector, 427 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 428 target_container); 429 430 memcpy(&key_addrs->v4addrs, &iph->saddr, 431 sizeof(key_addrs->v4addrs)); 432 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 433 } 434 435 if (ip_is_fragment(iph)) { 436 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 437 438 if (iph->frag_off & htons(IP_OFFSET)) { 439 goto out_good; 440 } else { 441 key_control->flags |= FLOW_DIS_FIRST_FRAG; 442 if (!(flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) 443 goto out_good; 444 } 445 } 446 447 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) 448 goto out_good; 449 450 break; 451 } 452 case htons(ETH_P_IPV6): { 453 const struct ipv6hdr *iph; 454 struct ipv6hdr _iph; 455 456 ipv6: 457 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 458 if (!iph) 459 goto out_bad; 460 461 ip_proto = iph->nexthdr; 462 nhoff += sizeof(struct ipv6hdr); 463 464 if (dissector_uses_key(flow_dissector, 465 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 466 key_addrs = skb_flow_dissector_target(flow_dissector, 467 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 468 target_container); 469 470 memcpy(&key_addrs->v6addrs, &iph->saddr, 471 sizeof(key_addrs->v6addrs)); 472 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 473 } 474 475 if ((dissector_uses_key(flow_dissector, 476 FLOW_DISSECTOR_KEY_FLOW_LABEL) || 477 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) && 478 ip6_flowlabel(iph)) { 479 __be32 flow_label = ip6_flowlabel(iph); 480 481 if (dissector_uses_key(flow_dissector, 482 FLOW_DISSECTOR_KEY_FLOW_LABEL)) { 483 key_tags = skb_flow_dissector_target(flow_dissector, 484 FLOW_DISSECTOR_KEY_FLOW_LABEL, 485 target_container); 486 key_tags->flow_label = ntohl(flow_label); 487 } 488 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) 489 goto out_good; 490 } 491 492 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) 493 goto out_good; 494 495 break; 496 } 497 case htons(ETH_P_8021AD): 498 case htons(ETH_P_8021Q): { 499 const struct vlan_hdr *vlan; 500 struct vlan_hdr _vlan; 501 bool vlan_tag_present = skb && skb_vlan_tag_present(skb); 502 503 if (vlan_tag_present) 504 proto = skb->protocol; 505 506 if (!vlan_tag_present || eth_type_vlan(skb->protocol)) { 507 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), 508 data, hlen, &_vlan); 509 if (!vlan) 510 goto out_bad; 511 proto = vlan->h_vlan_encapsulated_proto; 512 nhoff += sizeof(*vlan); 513 if (skip_vlan) 514 goto proto_again; 515 } 516 517 skip_vlan = true; 518 if (dissector_uses_key(flow_dissector, 519 FLOW_DISSECTOR_KEY_VLAN)) { 520 key_vlan = skb_flow_dissector_target(flow_dissector, 521 FLOW_DISSECTOR_KEY_VLAN, 522 target_container); 523 524 if (vlan_tag_present) { 525 key_vlan->vlan_id = skb_vlan_tag_get_id(skb); 526 key_vlan->vlan_priority = 527 (skb_vlan_tag_get_prio(skb) >> VLAN_PRIO_SHIFT); 528 } else { 529 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) & 530 VLAN_VID_MASK; 531 key_vlan->vlan_priority = 532 (ntohs(vlan->h_vlan_TCI) & 533 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 534 } 535 } 536 537 goto proto_again; 538 } 539 case htons(ETH_P_PPP_SES): { 540 struct { 541 struct pppoe_hdr hdr; 542 __be16 proto; 543 } *hdr, _hdr; 544 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 545 if (!hdr) 546 goto out_bad; 547 proto = hdr->proto; 548 nhoff += PPPOE_SES_HLEN; 549 switch (proto) { 550 case htons(PPP_IP): 551 goto ip; 552 case htons(PPP_IPV6): 553 goto ipv6; 554 default: 555 goto out_bad; 556 } 557 } 558 case htons(ETH_P_TIPC): { 559 struct { 560 __be32 pre[3]; 561 __be32 srcnode; 562 } *hdr, _hdr; 563 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 564 if (!hdr) 565 goto out_bad; 566 567 if (dissector_uses_key(flow_dissector, 568 FLOW_DISSECTOR_KEY_TIPC_ADDRS)) { 569 key_addrs = skb_flow_dissector_target(flow_dissector, 570 FLOW_DISSECTOR_KEY_TIPC_ADDRS, 571 target_container); 572 key_addrs->tipcaddrs.srcnode = hdr->srcnode; 573 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC_ADDRS; 574 } 575 goto out_good; 576 } 577 578 case htons(ETH_P_MPLS_UC): 579 case htons(ETH_P_MPLS_MC): 580 mpls: 581 switch (__skb_flow_dissect_mpls(skb, flow_dissector, 582 target_container, data, 583 nhoff, hlen)) { 584 case FLOW_DISSECT_RET_OUT_GOOD: 585 goto out_good; 586 case FLOW_DISSECT_RET_OUT_BAD: 587 default: 588 goto out_bad; 589 } 590 case htons(ETH_P_FCOE): 591 if ((hlen - nhoff) < FCOE_HEADER_LEN) 592 goto out_bad; 593 594 nhoff += FCOE_HEADER_LEN; 595 goto out_good; 596 597 case htons(ETH_P_ARP): 598 case htons(ETH_P_RARP): 599 switch (__skb_flow_dissect_arp(skb, flow_dissector, 600 target_container, data, 601 nhoff, hlen)) { 602 case FLOW_DISSECT_RET_OUT_GOOD: 603 goto out_good; 604 case FLOW_DISSECT_RET_OUT_BAD: 605 default: 606 goto out_bad; 607 } 608 default: 609 goto out_bad; 610 } 611 612 ip_proto_again: 613 switch (ip_proto) { 614 case IPPROTO_GRE: 615 switch (__skb_flow_dissect_gre(skb, key_control, flow_dissector, 616 target_container, data, 617 &proto, &nhoff, &hlen, flags)) { 618 case FLOW_DISSECT_RET_OUT_GOOD: 619 goto out_good; 620 case FLOW_DISSECT_RET_OUT_BAD: 621 goto out_bad; 622 case FLOW_DISSECT_RET_OUT_PROTO_AGAIN: 623 goto proto_again; 624 } 625 case NEXTHDR_HOP: 626 case NEXTHDR_ROUTING: 627 case NEXTHDR_DEST: { 628 u8 _opthdr[2], *opthdr; 629 630 if (proto != htons(ETH_P_IPV6)) 631 break; 632 633 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr), 634 data, hlen, &_opthdr); 635 if (!opthdr) 636 goto out_bad; 637 638 ip_proto = opthdr[0]; 639 nhoff += (opthdr[1] + 1) << 3; 640 641 goto ip_proto_again; 642 } 643 case NEXTHDR_FRAGMENT: { 644 struct frag_hdr _fh, *fh; 645 646 if (proto != htons(ETH_P_IPV6)) 647 break; 648 649 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh), 650 data, hlen, &_fh); 651 652 if (!fh) 653 goto out_bad; 654 655 key_control->flags |= FLOW_DIS_IS_FRAGMENT; 656 657 nhoff += sizeof(_fh); 658 ip_proto = fh->nexthdr; 659 660 if (!(fh->frag_off & htons(IP6_OFFSET))) { 661 key_control->flags |= FLOW_DIS_FIRST_FRAG; 662 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) 663 goto ip_proto_again; 664 } 665 goto out_good; 666 } 667 case IPPROTO_IPIP: 668 proto = htons(ETH_P_IP); 669 670 key_control->flags |= FLOW_DIS_ENCAPSULATION; 671 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 672 goto out_good; 673 674 goto ip; 675 case IPPROTO_IPV6: 676 proto = htons(ETH_P_IPV6); 677 678 key_control->flags |= FLOW_DIS_ENCAPSULATION; 679 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) 680 goto out_good; 681 682 goto ipv6; 683 case IPPROTO_MPLS: 684 proto = htons(ETH_P_MPLS_UC); 685 goto mpls; 686 default: 687 break; 688 } 689 690 if (dissector_uses_key(flow_dissector, 691 FLOW_DISSECTOR_KEY_PORTS)) { 692 key_ports = skb_flow_dissector_target(flow_dissector, 693 FLOW_DISSECTOR_KEY_PORTS, 694 target_container); 695 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 696 data, hlen); 697 } 698 699 if (dissector_uses_key(flow_dissector, 700 FLOW_DISSECTOR_KEY_ICMP)) { 701 key_icmp = skb_flow_dissector_target(flow_dissector, 702 FLOW_DISSECTOR_KEY_ICMP, 703 target_container); 704 key_icmp->icmp = skb_flow_get_be16(skb, nhoff, data, hlen); 705 } 706 707 out_good: 708 ret = true; 709 710 key_control->thoff = (u16)nhoff; 711 out: 712 key_basic->n_proto = proto; 713 key_basic->ip_proto = ip_proto; 714 715 return ret; 716 717 out_bad: 718 ret = false; 719 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); 720 goto out; 721 } 722 EXPORT_SYMBOL(__skb_flow_dissect); 723 724 static u32 hashrnd __read_mostly; 725 static __always_inline void __flow_hash_secret_init(void) 726 { 727 net_get_random_once(&hashrnd, sizeof(hashrnd)); 728 } 729 730 static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, 731 u32 keyval) 732 { 733 return jhash2(words, length, keyval); 734 } 735 736 static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) 737 { 738 const void *p = flow; 739 740 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); 741 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); 742 } 743 744 static inline size_t flow_keys_hash_length(const struct flow_keys *flow) 745 { 746 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); 747 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); 748 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) != 749 sizeof(*flow) - sizeof(flow->addrs)); 750 751 switch (flow->control.addr_type) { 752 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 753 diff -= sizeof(flow->addrs.v4addrs); 754 break; 755 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 756 diff -= sizeof(flow->addrs.v6addrs); 757 break; 758 case FLOW_DISSECTOR_KEY_TIPC_ADDRS: 759 diff -= sizeof(flow->addrs.tipcaddrs); 760 break; 761 } 762 return (sizeof(*flow) - diff) / sizeof(u32); 763 } 764 765 __be32 flow_get_u32_src(const struct flow_keys *flow) 766 { 767 switch (flow->control.addr_type) { 768 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 769 return flow->addrs.v4addrs.src; 770 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 771 return (__force __be32)ipv6_addr_hash( 772 &flow->addrs.v6addrs.src); 773 case FLOW_DISSECTOR_KEY_TIPC_ADDRS: 774 return flow->addrs.tipcaddrs.srcnode; 775 default: 776 return 0; 777 } 778 } 779 EXPORT_SYMBOL(flow_get_u32_src); 780 781 __be32 flow_get_u32_dst(const struct flow_keys *flow) 782 { 783 switch (flow->control.addr_type) { 784 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 785 return flow->addrs.v4addrs.dst; 786 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 787 return (__force __be32)ipv6_addr_hash( 788 &flow->addrs.v6addrs.dst); 789 default: 790 return 0; 791 } 792 } 793 EXPORT_SYMBOL(flow_get_u32_dst); 794 795 static inline void __flow_hash_consistentify(struct flow_keys *keys) 796 { 797 int addr_diff, i; 798 799 switch (keys->control.addr_type) { 800 case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 801 addr_diff = (__force u32)keys->addrs.v4addrs.dst - 802 (__force u32)keys->addrs.v4addrs.src; 803 if ((addr_diff < 0) || 804 (addr_diff == 0 && 805 ((__force u16)keys->ports.dst < 806 (__force u16)keys->ports.src))) { 807 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); 808 swap(keys->ports.src, keys->ports.dst); 809 } 810 break; 811 case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 812 addr_diff = memcmp(&keys->addrs.v6addrs.dst, 813 &keys->addrs.v6addrs.src, 814 sizeof(keys->addrs.v6addrs.dst)); 815 if ((addr_diff < 0) || 816 (addr_diff == 0 && 817 ((__force u16)keys->ports.dst < 818 (__force u16)keys->ports.src))) { 819 for (i = 0; i < 4; i++) 820 swap(keys->addrs.v6addrs.src.s6_addr32[i], 821 keys->addrs.v6addrs.dst.s6_addr32[i]); 822 swap(keys->ports.src, keys->ports.dst); 823 } 824 break; 825 } 826 } 827 828 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) 829 { 830 u32 hash; 831 832 __flow_hash_consistentify(keys); 833 834 hash = __flow_hash_words(flow_keys_hash_start(keys), 835 flow_keys_hash_length(keys), keyval); 836 if (!hash) 837 hash = 1; 838 839 return hash; 840 } 841 842 u32 flow_hash_from_keys(struct flow_keys *keys) 843 { 844 __flow_hash_secret_init(); 845 return __flow_hash_from_keys(keys, hashrnd); 846 } 847 EXPORT_SYMBOL(flow_hash_from_keys); 848 849 static inline u32 ___skb_get_hash(const struct sk_buff *skb, 850 struct flow_keys *keys, u32 keyval) 851 { 852 skb_flow_dissect_flow_keys(skb, keys, 853 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 854 855 return __flow_hash_from_keys(keys, keyval); 856 } 857 858 struct _flow_keys_digest_data { 859 __be16 n_proto; 860 u8 ip_proto; 861 u8 padding; 862 __be32 ports; 863 __be32 src; 864 __be32 dst; 865 }; 866 867 void make_flow_keys_digest(struct flow_keys_digest *digest, 868 const struct flow_keys *flow) 869 { 870 struct _flow_keys_digest_data *data = 871 (struct _flow_keys_digest_data *)digest; 872 873 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); 874 875 memset(digest, 0, sizeof(*digest)); 876 877 data->n_proto = flow->basic.n_proto; 878 data->ip_proto = flow->basic.ip_proto; 879 data->ports = flow->ports.ports; 880 data->src = flow->addrs.v4addrs.src; 881 data->dst = flow->addrs.v4addrs.dst; 882 } 883 EXPORT_SYMBOL(make_flow_keys_digest); 884 885 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly; 886 887 u32 __skb_get_hash_symmetric(const struct sk_buff *skb) 888 { 889 struct flow_keys keys; 890 891 __flow_hash_secret_init(); 892 893 memset(&keys, 0, sizeof(keys)); 894 __skb_flow_dissect(skb, &flow_keys_dissector_symmetric, &keys, 895 NULL, 0, 0, 0, 896 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL); 897 898 return __flow_hash_from_keys(&keys, hashrnd); 899 } 900 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric); 901 902 /** 903 * __skb_get_hash: calculate a flow hash 904 * @skb: sk_buff to calculate flow hash from 905 * 906 * This function calculates a flow hash based on src/dst addresses 907 * and src/dst port numbers. Sets hash in skb to non-zero hash value 908 * on success, zero indicates no valid hash. Also, sets l4_hash in skb 909 * if hash is a canonical 4-tuple hash over transport ports. 910 */ 911 void __skb_get_hash(struct sk_buff *skb) 912 { 913 struct flow_keys keys; 914 u32 hash; 915 916 __flow_hash_secret_init(); 917 918 hash = ___skb_get_hash(skb, &keys, hashrnd); 919 920 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys)); 921 } 922 EXPORT_SYMBOL(__skb_get_hash); 923 924 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) 925 { 926 struct flow_keys keys; 927 928 return ___skb_get_hash(skb, &keys, perturb); 929 } 930 EXPORT_SYMBOL(skb_get_hash_perturb); 931 932 __u32 __skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) 933 { 934 struct flow_keys keys; 935 936 memset(&keys, 0, sizeof(keys)); 937 938 memcpy(&keys.addrs.v6addrs.src, &fl6->saddr, 939 sizeof(keys.addrs.v6addrs.src)); 940 memcpy(&keys.addrs.v6addrs.dst, &fl6->daddr, 941 sizeof(keys.addrs.v6addrs.dst)); 942 keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 943 keys.ports.src = fl6->fl6_sport; 944 keys.ports.dst = fl6->fl6_dport; 945 keys.keyid.keyid = fl6->fl6_gre_key; 946 keys.tags.flow_label = (__force u32)fl6->flowlabel; 947 keys.basic.ip_proto = fl6->flowi6_proto; 948 949 __skb_set_sw_hash(skb, flow_hash_from_keys(&keys), 950 flow_keys_have_l4(&keys)); 951 952 return skb->hash; 953 } 954 EXPORT_SYMBOL(__skb_get_hash_flowi6); 955 956 __u32 __skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl4) 957 { 958 struct flow_keys keys; 959 960 memset(&keys, 0, sizeof(keys)); 961 962 keys.addrs.v4addrs.src = fl4->saddr; 963 keys.addrs.v4addrs.dst = fl4->daddr; 964 keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 965 keys.ports.src = fl4->fl4_sport; 966 keys.ports.dst = fl4->fl4_dport; 967 keys.keyid.keyid = fl4->fl4_gre_key; 968 keys.basic.ip_proto = fl4->flowi4_proto; 969 970 __skb_set_sw_hash(skb, flow_hash_from_keys(&keys), 971 flow_keys_have_l4(&keys)); 972 973 return skb->hash; 974 } 975 EXPORT_SYMBOL(__skb_get_hash_flowi4); 976 977 u32 __skb_get_poff(const struct sk_buff *skb, void *data, 978 const struct flow_keys *keys, int hlen) 979 { 980 u32 poff = keys->control.thoff; 981 982 /* skip L4 headers for fragments after the first */ 983 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) && 984 !(keys->control.flags & FLOW_DIS_FIRST_FRAG)) 985 return poff; 986 987 switch (keys->basic.ip_proto) { 988 case IPPROTO_TCP: { 989 /* access doff as u8 to avoid unaligned access */ 990 const u8 *doff; 991 u8 _doff; 992 993 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 994 data, hlen, &_doff); 995 if (!doff) 996 return poff; 997 998 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 999 break; 1000 } 1001 case IPPROTO_UDP: 1002 case IPPROTO_UDPLITE: 1003 poff += sizeof(struct udphdr); 1004 break; 1005 /* For the rest, we do not really care about header 1006 * extensions at this point for now. 1007 */ 1008 case IPPROTO_ICMP: 1009 poff += sizeof(struct icmphdr); 1010 break; 1011 case IPPROTO_ICMPV6: 1012 poff += sizeof(struct icmp6hdr); 1013 break; 1014 case IPPROTO_IGMP: 1015 poff += sizeof(struct igmphdr); 1016 break; 1017 case IPPROTO_DCCP: 1018 poff += sizeof(struct dccp_hdr); 1019 break; 1020 case IPPROTO_SCTP: 1021 poff += sizeof(struct sctphdr); 1022 break; 1023 } 1024 1025 return poff; 1026 } 1027 1028 /** 1029 * skb_get_poff - get the offset to the payload 1030 * @skb: sk_buff to get the payload offset from 1031 * 1032 * The function will get the offset to the payload as far as it could 1033 * be dissected. The main user is currently BPF, so that we can dynamically 1034 * truncate packets without needing to push actual payload to the user 1035 * space and can analyze headers only, instead. 1036 */ 1037 u32 skb_get_poff(const struct sk_buff *skb) 1038 { 1039 struct flow_keys keys; 1040 1041 if (!skb_flow_dissect_flow_keys(skb, &keys, 0)) 1042 return 0; 1043 1044 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 1045 } 1046 1047 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) 1048 { 1049 memset(keys, 0, sizeof(*keys)); 1050 1051 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr, 1052 sizeof(keys->addrs.v6addrs.src)); 1053 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr, 1054 sizeof(keys->addrs.v6addrs.dst)); 1055 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1056 keys->ports.src = fl6->fl6_sport; 1057 keys->ports.dst = fl6->fl6_dport; 1058 keys->keyid.keyid = fl6->fl6_gre_key; 1059 keys->tags.flow_label = (__force u32)fl6->flowlabel; 1060 keys->basic.ip_proto = fl6->flowi6_proto; 1061 1062 return flow_hash_from_keys(keys); 1063 } 1064 EXPORT_SYMBOL(__get_hash_from_flowi6); 1065 1066 __u32 __get_hash_from_flowi4(const struct flowi4 *fl4, struct flow_keys *keys) 1067 { 1068 memset(keys, 0, sizeof(*keys)); 1069 1070 keys->addrs.v4addrs.src = fl4->saddr; 1071 keys->addrs.v4addrs.dst = fl4->daddr; 1072 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1073 keys->ports.src = fl4->fl4_sport; 1074 keys->ports.dst = fl4->fl4_dport; 1075 keys->keyid.keyid = fl4->fl4_gre_key; 1076 keys->basic.ip_proto = fl4->flowi4_proto; 1077 1078 return flow_hash_from_keys(keys); 1079 } 1080 EXPORT_SYMBOL(__get_hash_from_flowi4); 1081 1082 static const struct flow_dissector_key flow_keys_dissector_keys[] = { 1083 { 1084 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1085 .offset = offsetof(struct flow_keys, control), 1086 }, 1087 { 1088 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1089 .offset = offsetof(struct flow_keys, basic), 1090 }, 1091 { 1092 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1093 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1094 }, 1095 { 1096 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1097 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1098 }, 1099 { 1100 .key_id = FLOW_DISSECTOR_KEY_TIPC_ADDRS, 1101 .offset = offsetof(struct flow_keys, addrs.tipcaddrs), 1102 }, 1103 { 1104 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1105 .offset = offsetof(struct flow_keys, ports), 1106 }, 1107 { 1108 .key_id = FLOW_DISSECTOR_KEY_VLAN, 1109 .offset = offsetof(struct flow_keys, vlan), 1110 }, 1111 { 1112 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL, 1113 .offset = offsetof(struct flow_keys, tags), 1114 }, 1115 { 1116 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, 1117 .offset = offsetof(struct flow_keys, keyid), 1118 }, 1119 }; 1120 1121 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = { 1122 { 1123 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1124 .offset = offsetof(struct flow_keys, control), 1125 }, 1126 { 1127 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1128 .offset = offsetof(struct flow_keys, basic), 1129 }, 1130 { 1131 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1132 .offset = offsetof(struct flow_keys, addrs.v4addrs), 1133 }, 1134 { 1135 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1136 .offset = offsetof(struct flow_keys, addrs.v6addrs), 1137 }, 1138 { 1139 .key_id = FLOW_DISSECTOR_KEY_PORTS, 1140 .offset = offsetof(struct flow_keys, ports), 1141 }, 1142 }; 1143 1144 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = { 1145 { 1146 .key_id = FLOW_DISSECTOR_KEY_CONTROL, 1147 .offset = offsetof(struct flow_keys, control), 1148 }, 1149 { 1150 .key_id = FLOW_DISSECTOR_KEY_BASIC, 1151 .offset = offsetof(struct flow_keys, basic), 1152 }, 1153 }; 1154 1155 struct flow_dissector flow_keys_dissector __read_mostly; 1156 EXPORT_SYMBOL(flow_keys_dissector); 1157 1158 struct flow_dissector flow_keys_buf_dissector __read_mostly; 1159 1160 static int __init init_default_flow_dissectors(void) 1161 { 1162 skb_flow_dissector_init(&flow_keys_dissector, 1163 flow_keys_dissector_keys, 1164 ARRAY_SIZE(flow_keys_dissector_keys)); 1165 skb_flow_dissector_init(&flow_keys_dissector_symmetric, 1166 flow_keys_dissector_symmetric_keys, 1167 ARRAY_SIZE(flow_keys_dissector_symmetric_keys)); 1168 skb_flow_dissector_init(&flow_keys_buf_dissector, 1169 flow_keys_buf_dissector_keys, 1170 ARRAY_SIZE(flow_keys_buf_dissector_keys)); 1171 return 0; 1172 } 1173 1174 core_initcall(init_default_flow_dissectors); 1175