1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net> 4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org> 5 * 6 * Development of this code funded by Astaro AG (http://www.astaro.com/) 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/if_vlan.h> 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/netlink.h> 14 #include <linux/netfilter.h> 15 #include <linux/netfilter/nf_tables.h> 16 #include <net/netfilter/nf_tables_core.h> 17 #include <net/netfilter/nf_tables.h> 18 #include <net/netfilter/nf_tables_offload.h> 19 /* For layer 4 checksum field offset. */ 20 #include <linux/tcp.h> 21 #include <linux/udp.h> 22 #include <net/gre.h> 23 #include <linux/icmpv6.h> 24 #include <linux/ip.h> 25 #include <linux/ipv6.h> 26 #include <net/sctp/checksum.h> 27 28 static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off, 29 struct vlan_ethhdr *veth) 30 { 31 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN)) 32 return false; 33 34 veth->h_vlan_proto = skb->vlan_proto; 35 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 36 veth->h_vlan_encapsulated_proto = skb->protocol; 37 38 return true; 39 } 40 41 /* add vlan header into the user buffer for if tag was removed by offloads */ 42 static bool 43 nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len) 44 { 45 int mac_off = skb_mac_header(skb) - skb->data; 46 u8 *vlanh, *dst_u8 = (u8 *) d; 47 struct vlan_ethhdr veth; 48 u8 vlan_hlen = 0; 49 50 if ((skb->protocol == htons(ETH_P_8021AD) || 51 skb->protocol == htons(ETH_P_8021Q)) && 52 offset >= VLAN_ETH_HLEN && offset < VLAN_ETH_HLEN + VLAN_HLEN) 53 vlan_hlen += VLAN_HLEN; 54 55 vlanh = (u8 *) &veth; 56 if (offset < VLAN_ETH_HLEN + vlan_hlen) { 57 u8 ethlen = len; 58 59 if (vlan_hlen && 60 skb_copy_bits(skb, mac_off, &veth, VLAN_ETH_HLEN) < 0) 61 return false; 62 else if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth)) 63 return false; 64 65 if (offset + len > VLAN_ETH_HLEN + vlan_hlen) 66 ethlen -= offset + len - VLAN_ETH_HLEN - vlan_hlen; 67 68 memcpy(dst_u8, vlanh + offset - vlan_hlen, ethlen); 69 70 len -= ethlen; 71 if (len == 0) 72 return true; 73 74 dst_u8 += ethlen; 75 offset = ETH_HLEN + vlan_hlen; 76 } else { 77 offset -= VLAN_HLEN + vlan_hlen; 78 } 79 80 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0; 81 } 82 83 static int __nft_payload_inner_offset(struct nft_pktinfo *pkt) 84 { 85 unsigned int thoff = nft_thoff(pkt); 86 87 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 88 return -1; 89 90 switch (pkt->tprot) { 91 case IPPROTO_UDP: 92 pkt->inneroff = thoff + sizeof(struct udphdr); 93 break; 94 case IPPROTO_TCP: { 95 struct tcphdr *th, _tcph; 96 97 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph); 98 if (!th) 99 return -1; 100 101 pkt->inneroff = thoff + __tcp_hdrlen(th); 102 } 103 break; 104 case IPPROTO_GRE: { 105 u32 offset = sizeof(struct gre_base_hdr); 106 struct gre_base_hdr *gre, _gre; 107 __be16 version; 108 109 gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre); 110 if (!gre) 111 return -1; 112 113 version = gre->flags & GRE_VERSION; 114 switch (version) { 115 case GRE_VERSION_0: 116 if (gre->flags & GRE_ROUTING) 117 return -1; 118 119 if (gre->flags & GRE_CSUM) { 120 offset += sizeof_field(struct gre_full_hdr, csum) + 121 sizeof_field(struct gre_full_hdr, reserved1); 122 } 123 if (gre->flags & GRE_KEY) 124 offset += sizeof_field(struct gre_full_hdr, key); 125 126 if (gre->flags & GRE_SEQ) 127 offset += sizeof_field(struct gre_full_hdr, seq); 128 break; 129 default: 130 return -1; 131 } 132 133 pkt->inneroff = thoff + offset; 134 } 135 break; 136 case IPPROTO_IPIP: 137 pkt->inneroff = thoff; 138 break; 139 default: 140 return -1; 141 } 142 143 pkt->flags |= NFT_PKTINFO_INNER; 144 145 return 0; 146 } 147 148 int nft_payload_inner_offset(const struct nft_pktinfo *pkt) 149 { 150 if (!(pkt->flags & NFT_PKTINFO_INNER) && 151 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0) 152 return -1; 153 154 return pkt->inneroff; 155 } 156 157 void nft_payload_eval(const struct nft_expr *expr, 158 struct nft_regs *regs, 159 const struct nft_pktinfo *pkt) 160 { 161 const struct nft_payload *priv = nft_expr_priv(expr); 162 const struct sk_buff *skb = pkt->skb; 163 u32 *dest = ®s->data[priv->dreg]; 164 int offset; 165 166 if (priv->len % NFT_REG32_SIZE) 167 dest[priv->len / NFT_REG32_SIZE] = 0; 168 169 switch (priv->base) { 170 case NFT_PAYLOAD_LL_HEADER: 171 if (!skb_mac_header_was_set(skb)) 172 goto err; 173 174 if (skb_vlan_tag_present(skb) && 175 priv->offset >= offsetof(struct ethhdr, h_proto)) { 176 if (!nft_payload_copy_vlan(dest, skb, 177 priv->offset, priv->len)) 178 goto err; 179 return; 180 } 181 offset = skb_mac_header(skb) - skb->data; 182 break; 183 case NFT_PAYLOAD_NETWORK_HEADER: 184 offset = skb_network_offset(skb); 185 break; 186 case NFT_PAYLOAD_TRANSPORT_HEADER: 187 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 188 goto err; 189 offset = nft_thoff(pkt); 190 break; 191 case NFT_PAYLOAD_INNER_HEADER: 192 offset = nft_payload_inner_offset(pkt); 193 if (offset < 0) 194 goto err; 195 break; 196 default: 197 WARN_ON_ONCE(1); 198 goto err; 199 } 200 offset += priv->offset; 201 202 if (skb_copy_bits(skb, offset, dest, priv->len) < 0) 203 goto err; 204 return; 205 err: 206 regs->verdict.code = NFT_BREAK; 207 } 208 209 static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = { 210 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 }, 211 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 }, 212 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 }, 213 [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), 214 [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255), 215 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 }, 216 [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), 217 [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 }, 218 }; 219 220 static int nft_payload_init(const struct nft_ctx *ctx, 221 const struct nft_expr *expr, 222 const struct nlattr * const tb[]) 223 { 224 struct nft_payload *priv = nft_expr_priv(expr); 225 226 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 227 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); 228 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 229 230 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], 231 &priv->dreg, NULL, NFT_DATA_VALUE, 232 priv->len); 233 } 234 235 static int nft_payload_dump(struct sk_buff *skb, 236 const struct nft_expr *expr, bool reset) 237 { 238 const struct nft_payload *priv = nft_expr_priv(expr); 239 240 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) || 241 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) || 242 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) || 243 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len))) 244 goto nla_put_failure; 245 return 0; 246 247 nla_put_failure: 248 return -1; 249 } 250 251 static bool nft_payload_reduce(struct nft_regs_track *track, 252 const struct nft_expr *expr) 253 { 254 const struct nft_payload *priv = nft_expr_priv(expr); 255 const struct nft_payload *payload; 256 257 if (!nft_reg_track_cmp(track, expr, priv->dreg)) { 258 nft_reg_track_update(track, expr, priv->dreg, priv->len); 259 return false; 260 } 261 262 payload = nft_expr_priv(track->regs[priv->dreg].selector); 263 if (priv->base != payload->base || 264 priv->offset != payload->offset || 265 priv->len != payload->len) { 266 nft_reg_track_update(track, expr, priv->dreg, priv->len); 267 return false; 268 } 269 270 if (!track->regs[priv->dreg].bitwise) 271 return true; 272 273 return nft_expr_reduce_bitwise(track, expr); 274 } 275 276 static bool nft_payload_offload_mask(struct nft_offload_reg *reg, 277 u32 priv_len, u32 field_len) 278 { 279 unsigned int remainder, delta, k; 280 struct nft_data mask = {}; 281 __be32 remainder_mask; 282 283 if (priv_len == field_len) { 284 memset(®->mask, 0xff, priv_len); 285 return true; 286 } else if (priv_len > field_len) { 287 return false; 288 } 289 290 memset(&mask, 0xff, field_len); 291 remainder = priv_len % sizeof(u32); 292 if (remainder) { 293 k = priv_len / sizeof(u32); 294 delta = field_len - priv_len; 295 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1)); 296 mask.data[k] = (__force u32)remainder_mask; 297 } 298 299 memcpy(®->mask, &mask, field_len); 300 301 return true; 302 } 303 304 static int nft_payload_offload_ll(struct nft_offload_ctx *ctx, 305 struct nft_flow_rule *flow, 306 const struct nft_payload *priv) 307 { 308 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 309 310 switch (priv->offset) { 311 case offsetof(struct ethhdr, h_source): 312 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN)) 313 return -EOPNOTSUPP; 314 315 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs, 316 src, ETH_ALEN, reg); 317 break; 318 case offsetof(struct ethhdr, h_dest): 319 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN)) 320 return -EOPNOTSUPP; 321 322 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs, 323 dst, ETH_ALEN, reg); 324 break; 325 case offsetof(struct ethhdr, h_proto): 326 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 327 return -EOPNOTSUPP; 328 329 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, 330 n_proto, sizeof(__be16), reg); 331 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 332 break; 333 case offsetof(struct vlan_ethhdr, h_vlan_TCI): 334 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 335 return -EOPNOTSUPP; 336 337 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan, 338 vlan_tci, sizeof(__be16), reg, 339 NFT_OFFLOAD_F_NETWORK2HOST); 340 break; 341 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto): 342 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 343 return -EOPNOTSUPP; 344 345 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan, 346 vlan_tpid, sizeof(__be16), reg); 347 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 348 break; 349 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr): 350 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 351 return -EOPNOTSUPP; 352 353 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan, 354 vlan_tci, sizeof(__be16), reg, 355 NFT_OFFLOAD_F_NETWORK2HOST); 356 break; 357 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) + 358 sizeof(struct vlan_hdr): 359 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 360 return -EOPNOTSUPP; 361 362 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan, 363 vlan_tpid, sizeof(__be16), reg); 364 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 365 break; 366 default: 367 return -EOPNOTSUPP; 368 } 369 370 return 0; 371 } 372 373 static int nft_payload_offload_ip(struct nft_offload_ctx *ctx, 374 struct nft_flow_rule *flow, 375 const struct nft_payload *priv) 376 { 377 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 378 379 switch (priv->offset) { 380 case offsetof(struct iphdr, saddr): 381 if (!nft_payload_offload_mask(reg, priv->len, 382 sizeof(struct in_addr))) 383 return -EOPNOTSUPP; 384 385 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src, 386 sizeof(struct in_addr), reg); 387 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 388 break; 389 case offsetof(struct iphdr, daddr): 390 if (!nft_payload_offload_mask(reg, priv->len, 391 sizeof(struct in_addr))) 392 return -EOPNOTSUPP; 393 394 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst, 395 sizeof(struct in_addr), reg); 396 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 397 break; 398 case offsetof(struct iphdr, protocol): 399 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8))) 400 return -EOPNOTSUPP; 401 402 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto, 403 sizeof(__u8), reg); 404 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT); 405 break; 406 default: 407 return -EOPNOTSUPP; 408 } 409 410 return 0; 411 } 412 413 static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx, 414 struct nft_flow_rule *flow, 415 const struct nft_payload *priv) 416 { 417 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 418 419 switch (priv->offset) { 420 case offsetof(struct ipv6hdr, saddr): 421 if (!nft_payload_offload_mask(reg, priv->len, 422 sizeof(struct in6_addr))) 423 return -EOPNOTSUPP; 424 425 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src, 426 sizeof(struct in6_addr), reg); 427 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 428 break; 429 case offsetof(struct ipv6hdr, daddr): 430 if (!nft_payload_offload_mask(reg, priv->len, 431 sizeof(struct in6_addr))) 432 return -EOPNOTSUPP; 433 434 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst, 435 sizeof(struct in6_addr), reg); 436 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 437 break; 438 case offsetof(struct ipv6hdr, nexthdr): 439 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8))) 440 return -EOPNOTSUPP; 441 442 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto, 443 sizeof(__u8), reg); 444 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT); 445 break; 446 default: 447 return -EOPNOTSUPP; 448 } 449 450 return 0; 451 } 452 453 static int nft_payload_offload_nh(struct nft_offload_ctx *ctx, 454 struct nft_flow_rule *flow, 455 const struct nft_payload *priv) 456 { 457 int err; 458 459 switch (ctx->dep.l3num) { 460 case htons(ETH_P_IP): 461 err = nft_payload_offload_ip(ctx, flow, priv); 462 break; 463 case htons(ETH_P_IPV6): 464 err = nft_payload_offload_ip6(ctx, flow, priv); 465 break; 466 default: 467 return -EOPNOTSUPP; 468 } 469 470 return err; 471 } 472 473 static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx, 474 struct nft_flow_rule *flow, 475 const struct nft_payload *priv) 476 { 477 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 478 479 switch (priv->offset) { 480 case offsetof(struct tcphdr, source): 481 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 482 return -EOPNOTSUPP; 483 484 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src, 485 sizeof(__be16), reg); 486 break; 487 case offsetof(struct tcphdr, dest): 488 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 489 return -EOPNOTSUPP; 490 491 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst, 492 sizeof(__be16), reg); 493 break; 494 default: 495 return -EOPNOTSUPP; 496 } 497 498 return 0; 499 } 500 501 static int nft_payload_offload_udp(struct nft_offload_ctx *ctx, 502 struct nft_flow_rule *flow, 503 const struct nft_payload *priv) 504 { 505 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 506 507 switch (priv->offset) { 508 case offsetof(struct udphdr, source): 509 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 510 return -EOPNOTSUPP; 511 512 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src, 513 sizeof(__be16), reg); 514 break; 515 case offsetof(struct udphdr, dest): 516 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 517 return -EOPNOTSUPP; 518 519 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst, 520 sizeof(__be16), reg); 521 break; 522 default: 523 return -EOPNOTSUPP; 524 } 525 526 return 0; 527 } 528 529 static int nft_payload_offload_th(struct nft_offload_ctx *ctx, 530 struct nft_flow_rule *flow, 531 const struct nft_payload *priv) 532 { 533 int err; 534 535 switch (ctx->dep.protonum) { 536 case IPPROTO_TCP: 537 err = nft_payload_offload_tcp(ctx, flow, priv); 538 break; 539 case IPPROTO_UDP: 540 err = nft_payload_offload_udp(ctx, flow, priv); 541 break; 542 default: 543 return -EOPNOTSUPP; 544 } 545 546 return err; 547 } 548 549 static int nft_payload_offload(struct nft_offload_ctx *ctx, 550 struct nft_flow_rule *flow, 551 const struct nft_expr *expr) 552 { 553 const struct nft_payload *priv = nft_expr_priv(expr); 554 int err; 555 556 switch (priv->base) { 557 case NFT_PAYLOAD_LL_HEADER: 558 err = nft_payload_offload_ll(ctx, flow, priv); 559 break; 560 case NFT_PAYLOAD_NETWORK_HEADER: 561 err = nft_payload_offload_nh(ctx, flow, priv); 562 break; 563 case NFT_PAYLOAD_TRANSPORT_HEADER: 564 err = nft_payload_offload_th(ctx, flow, priv); 565 break; 566 default: 567 err = -EOPNOTSUPP; 568 break; 569 } 570 return err; 571 } 572 573 static const struct nft_expr_ops nft_payload_ops = { 574 .type = &nft_payload_type, 575 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 576 .eval = nft_payload_eval, 577 .init = nft_payload_init, 578 .dump = nft_payload_dump, 579 .reduce = nft_payload_reduce, 580 .offload = nft_payload_offload, 581 }; 582 583 const struct nft_expr_ops nft_payload_fast_ops = { 584 .type = &nft_payload_type, 585 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 586 .eval = nft_payload_eval, 587 .init = nft_payload_init, 588 .dump = nft_payload_dump, 589 .reduce = nft_payload_reduce, 590 .offload = nft_payload_offload, 591 }; 592 593 void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs, 594 const struct nft_pktinfo *pkt, 595 struct nft_inner_tun_ctx *tun_ctx) 596 { 597 const struct nft_payload *priv = nft_expr_priv(expr); 598 const struct sk_buff *skb = pkt->skb; 599 u32 *dest = ®s->data[priv->dreg]; 600 int offset; 601 602 if (priv->len % NFT_REG32_SIZE) 603 dest[priv->len / NFT_REG32_SIZE] = 0; 604 605 switch (priv->base) { 606 case NFT_PAYLOAD_TUN_HEADER: 607 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN)) 608 goto err; 609 610 offset = tun_ctx->inner_tunoff; 611 break; 612 case NFT_PAYLOAD_LL_HEADER: 613 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL)) 614 goto err; 615 616 offset = tun_ctx->inner_lloff; 617 break; 618 case NFT_PAYLOAD_NETWORK_HEADER: 619 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH)) 620 goto err; 621 622 offset = tun_ctx->inner_nhoff; 623 break; 624 case NFT_PAYLOAD_TRANSPORT_HEADER: 625 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH)) 626 goto err; 627 628 offset = tun_ctx->inner_thoff; 629 break; 630 default: 631 WARN_ON_ONCE(1); 632 goto err; 633 } 634 offset += priv->offset; 635 636 if (skb_copy_bits(skb, offset, dest, priv->len) < 0) 637 goto err; 638 639 return; 640 err: 641 regs->verdict.code = NFT_BREAK; 642 } 643 644 static int nft_payload_inner_init(const struct nft_ctx *ctx, 645 const struct nft_expr *expr, 646 const struct nlattr * const tb[]) 647 { 648 struct nft_payload *priv = nft_expr_priv(expr); 649 u32 base; 650 651 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 652 switch (base) { 653 case NFT_PAYLOAD_TUN_HEADER: 654 case NFT_PAYLOAD_LL_HEADER: 655 case NFT_PAYLOAD_NETWORK_HEADER: 656 case NFT_PAYLOAD_TRANSPORT_HEADER: 657 break; 658 default: 659 return -EOPNOTSUPP; 660 } 661 662 priv->base = base; 663 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); 664 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 665 666 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], 667 &priv->dreg, NULL, NFT_DATA_VALUE, 668 priv->len); 669 } 670 671 static const struct nft_expr_ops nft_payload_inner_ops = { 672 .type = &nft_payload_type, 673 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 674 .init = nft_payload_inner_init, 675 .dump = nft_payload_dump, 676 /* direct call to nft_payload_inner_eval(). */ 677 }; 678 679 static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum) 680 { 681 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum)); 682 if (*sum == 0) 683 *sum = CSUM_MANGLED_0; 684 } 685 686 static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff) 687 { 688 struct udphdr *uh, _uh; 689 690 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh); 691 if (!uh) 692 return false; 693 694 return (__force bool)uh->check; 695 } 696 697 static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt, 698 struct sk_buff *skb, 699 unsigned int *l4csum_offset) 700 { 701 if (pkt->fragoff) 702 return -1; 703 704 switch (pkt->tprot) { 705 case IPPROTO_TCP: 706 *l4csum_offset = offsetof(struct tcphdr, check); 707 break; 708 case IPPROTO_UDP: 709 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt))) 710 return -1; 711 fallthrough; 712 case IPPROTO_UDPLITE: 713 *l4csum_offset = offsetof(struct udphdr, check); 714 break; 715 case IPPROTO_ICMPV6: 716 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum); 717 break; 718 default: 719 return -1; 720 } 721 722 *l4csum_offset += nft_thoff(pkt); 723 return 0; 724 } 725 726 static int nft_payload_csum_sctp(struct sk_buff *skb, int offset) 727 { 728 struct sctphdr *sh; 729 730 if (skb_ensure_writable(skb, offset + sizeof(*sh))) 731 return -1; 732 733 sh = (struct sctphdr *)(skb->data + offset); 734 sh->checksum = sctp_compute_cksum(skb, offset); 735 skb->ip_summed = CHECKSUM_UNNECESSARY; 736 return 0; 737 } 738 739 static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt, 740 struct sk_buff *skb, 741 __wsum fsum, __wsum tsum) 742 { 743 int l4csum_offset; 744 __sum16 sum; 745 746 /* If we cannot determine layer 4 checksum offset or this packet doesn't 747 * require layer 4 checksum recalculation, skip this packet. 748 */ 749 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0) 750 return 0; 751 752 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0) 753 return -1; 754 755 /* Checksum mangling for an arbitrary amount of bytes, based on 756 * inet_proto_csum_replace*() functions. 757 */ 758 if (skb->ip_summed != CHECKSUM_PARTIAL) { 759 nft_csum_replace(&sum, fsum, tsum); 760 if (skb->ip_summed == CHECKSUM_COMPLETE) { 761 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum), 762 tsum); 763 } 764 } else { 765 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum), 766 tsum)); 767 } 768 769 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) || 770 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0) 771 return -1; 772 773 return 0; 774 } 775 776 static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src, 777 __wsum fsum, __wsum tsum, int csum_offset) 778 { 779 __sum16 sum; 780 781 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0) 782 return -1; 783 784 nft_csum_replace(&sum, fsum, tsum); 785 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) || 786 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0) 787 return -1; 788 789 return 0; 790 } 791 792 struct nft_payload_set { 793 enum nft_payload_bases base:8; 794 u8 offset; 795 u8 len; 796 u8 sreg; 797 u8 csum_type; 798 u8 csum_offset; 799 u8 csum_flags; 800 }; 801 802 static void nft_payload_set_eval(const struct nft_expr *expr, 803 struct nft_regs *regs, 804 const struct nft_pktinfo *pkt) 805 { 806 const struct nft_payload_set *priv = nft_expr_priv(expr); 807 struct sk_buff *skb = pkt->skb; 808 const u32 *src = ®s->data[priv->sreg]; 809 int offset, csum_offset; 810 __wsum fsum, tsum; 811 812 switch (priv->base) { 813 case NFT_PAYLOAD_LL_HEADER: 814 if (!skb_mac_header_was_set(skb)) 815 goto err; 816 offset = skb_mac_header(skb) - skb->data; 817 break; 818 case NFT_PAYLOAD_NETWORK_HEADER: 819 offset = skb_network_offset(skb); 820 break; 821 case NFT_PAYLOAD_TRANSPORT_HEADER: 822 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 823 goto err; 824 offset = nft_thoff(pkt); 825 break; 826 case NFT_PAYLOAD_INNER_HEADER: 827 offset = nft_payload_inner_offset(pkt); 828 if (offset < 0) 829 goto err; 830 break; 831 default: 832 WARN_ON_ONCE(1); 833 goto err; 834 } 835 836 csum_offset = offset + priv->csum_offset; 837 offset += priv->offset; 838 839 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) && 840 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER && 841 priv->base != NFT_PAYLOAD_INNER_HEADER) || 842 skb->ip_summed != CHECKSUM_PARTIAL)) { 843 fsum = skb_checksum(skb, offset, priv->len, 0); 844 tsum = csum_partial(src, priv->len, 0); 845 846 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET && 847 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset)) 848 goto err; 849 850 if (priv->csum_flags && 851 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0) 852 goto err; 853 } 854 855 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) || 856 skb_store_bits(skb, offset, src, priv->len) < 0) 857 goto err; 858 859 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP && 860 pkt->tprot == IPPROTO_SCTP && 861 skb->ip_summed != CHECKSUM_PARTIAL) { 862 if (pkt->fragoff == 0 && 863 nft_payload_csum_sctp(skb, nft_thoff(pkt))) 864 goto err; 865 } 866 867 return; 868 err: 869 regs->verdict.code = NFT_BREAK; 870 } 871 872 static int nft_payload_set_init(const struct nft_ctx *ctx, 873 const struct nft_expr *expr, 874 const struct nlattr * const tb[]) 875 { 876 struct nft_payload_set *priv = nft_expr_priv(expr); 877 u32 csum_offset, csum_type = NFT_PAYLOAD_CSUM_NONE; 878 int err; 879 880 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 881 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); 882 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 883 884 if (tb[NFTA_PAYLOAD_CSUM_TYPE]) 885 csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE])); 886 if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) { 887 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX, 888 &csum_offset); 889 if (err < 0) 890 return err; 891 892 priv->csum_offset = csum_offset; 893 } 894 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) { 895 u32 flags; 896 897 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS])); 898 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR) 899 return -EINVAL; 900 901 priv->csum_flags = flags; 902 } 903 904 switch (csum_type) { 905 case NFT_PAYLOAD_CSUM_NONE: 906 case NFT_PAYLOAD_CSUM_INET: 907 break; 908 case NFT_PAYLOAD_CSUM_SCTP: 909 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER) 910 return -EINVAL; 911 912 if (priv->csum_offset != offsetof(struct sctphdr, checksum)) 913 return -EINVAL; 914 break; 915 default: 916 return -EOPNOTSUPP; 917 } 918 priv->csum_type = csum_type; 919 920 return nft_parse_register_load(tb[NFTA_PAYLOAD_SREG], &priv->sreg, 921 priv->len); 922 } 923 924 static int nft_payload_set_dump(struct sk_buff *skb, 925 const struct nft_expr *expr, bool reset) 926 { 927 const struct nft_payload_set *priv = nft_expr_priv(expr); 928 929 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) || 930 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) || 931 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) || 932 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) || 933 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) || 934 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET, 935 htonl(priv->csum_offset)) || 936 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags))) 937 goto nla_put_failure; 938 return 0; 939 940 nla_put_failure: 941 return -1; 942 } 943 944 static bool nft_payload_set_reduce(struct nft_regs_track *track, 945 const struct nft_expr *expr) 946 { 947 int i; 948 949 for (i = 0; i < NFT_REG32_NUM; i++) { 950 if (!track->regs[i].selector) 951 continue; 952 953 if (track->regs[i].selector->ops != &nft_payload_ops && 954 track->regs[i].selector->ops != &nft_payload_fast_ops) 955 continue; 956 957 __nft_reg_track_cancel(track, i); 958 } 959 960 return false; 961 } 962 963 static const struct nft_expr_ops nft_payload_set_ops = { 964 .type = &nft_payload_type, 965 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)), 966 .eval = nft_payload_set_eval, 967 .init = nft_payload_set_init, 968 .dump = nft_payload_set_dump, 969 .reduce = nft_payload_set_reduce, 970 }; 971 972 static const struct nft_expr_ops * 973 nft_payload_select_ops(const struct nft_ctx *ctx, 974 const struct nlattr * const tb[]) 975 { 976 enum nft_payload_bases base; 977 unsigned int offset, len; 978 int err; 979 980 if (tb[NFTA_PAYLOAD_BASE] == NULL || 981 tb[NFTA_PAYLOAD_OFFSET] == NULL || 982 tb[NFTA_PAYLOAD_LEN] == NULL) 983 return ERR_PTR(-EINVAL); 984 985 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 986 switch (base) { 987 case NFT_PAYLOAD_LL_HEADER: 988 case NFT_PAYLOAD_NETWORK_HEADER: 989 case NFT_PAYLOAD_TRANSPORT_HEADER: 990 case NFT_PAYLOAD_INNER_HEADER: 991 break; 992 default: 993 return ERR_PTR(-EOPNOTSUPP); 994 } 995 996 if (tb[NFTA_PAYLOAD_SREG] != NULL) { 997 if (tb[NFTA_PAYLOAD_DREG] != NULL) 998 return ERR_PTR(-EINVAL); 999 return &nft_payload_set_ops; 1000 } 1001 1002 if (tb[NFTA_PAYLOAD_DREG] == NULL) 1003 return ERR_PTR(-EINVAL); 1004 1005 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U8_MAX, &offset); 1006 if (err < 0) 1007 return ERR_PTR(err); 1008 1009 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len); 1010 if (err < 0) 1011 return ERR_PTR(err); 1012 1013 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) && 1014 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER) 1015 return &nft_payload_fast_ops; 1016 else 1017 return &nft_payload_ops; 1018 } 1019 1020 struct nft_expr_type nft_payload_type __read_mostly = { 1021 .name = "payload", 1022 .select_ops = nft_payload_select_ops, 1023 .inner_ops = &nft_payload_inner_ops, 1024 .policy = nft_payload_policy, 1025 .maxattr = NFTA_PAYLOAD_MAX, 1026 .owner = THIS_MODULE, 1027 }; 1028