1 /* 2 * ip_vs_xmit.c: various packet transmitters for IPVS 3 * 4 * Authors: Wensong Zhang <wensong@linuxvirtualserver.org> 5 * Julian Anastasov <ja@ssi.bg> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 * 12 * Changes: 13 * 14 * Description of forwarding methods: 15 * - all transmitters are called from LOCAL_IN (remote clients) and 16 * LOCAL_OUT (local clients) but for ICMP can be called from FORWARD 17 * - not all connections have destination server, for example, 18 * connections in backup server when fwmark is used 19 * - bypass connections use daddr from packet 20 * - we can use dst without ref while sending in RCU section, we use 21 * ref when returning NF_ACCEPT for NAT-ed packet via loopback 22 * LOCAL_OUT rules: 23 * - skb->dev is NULL, skb->protocol is not set (both are set in POST_ROUTING) 24 * - skb->pkt_type is not set yet 25 * - the only place where we can see skb->sk != NULL 26 */ 27 28 #define KMSG_COMPONENT "IPVS" 29 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 30 31 #include <linux/kernel.h> 32 #include <linux/slab.h> 33 #include <linux/tcp.h> /* for tcphdr */ 34 #include <net/ip.h> 35 #include <net/tcp.h> /* for csum_tcpudp_magic */ 36 #include <net/udp.h> 37 #include <net/icmp.h> /* for icmp_send */ 38 #include <net/route.h> /* for ip_route_output */ 39 #include <net/ipv6.h> 40 #include <net/ip6_route.h> 41 #include <net/ip_tunnels.h> 42 #include <net/addrconf.h> 43 #include <linux/icmpv6.h> 44 #include <linux/netfilter.h> 45 #include <linux/netfilter_ipv4.h> 46 47 #include <net/ip_vs.h> 48 49 enum { 50 IP_VS_RT_MODE_LOCAL = 1, /* Allow local dest */ 51 IP_VS_RT_MODE_NON_LOCAL = 2, /* Allow non-local dest */ 52 IP_VS_RT_MODE_RDR = 4, /* Allow redirect from remote daddr to 53 * local 54 */ 55 IP_VS_RT_MODE_CONNECT = 8, /* Always bind route to saddr */ 56 IP_VS_RT_MODE_KNOWN_NH = 16,/* Route via remote addr */ 57 IP_VS_RT_MODE_TUNNEL = 32,/* Tunnel mode */ 58 }; 59 60 static inline struct ip_vs_dest_dst *ip_vs_dest_dst_alloc(void) 61 { 62 return kmalloc(sizeof(struct ip_vs_dest_dst), GFP_ATOMIC); 63 } 64 65 static inline void ip_vs_dest_dst_free(struct ip_vs_dest_dst *dest_dst) 66 { 67 kfree(dest_dst); 68 } 69 70 /* 71 * Destination cache to speed up outgoing route lookup 72 */ 73 static inline void 74 __ip_vs_dst_set(struct ip_vs_dest *dest, struct ip_vs_dest_dst *dest_dst, 75 struct dst_entry *dst, u32 dst_cookie) 76 { 77 struct ip_vs_dest_dst *old; 78 79 old = rcu_dereference_protected(dest->dest_dst, 80 lockdep_is_held(&dest->dst_lock)); 81 82 if (dest_dst) { 83 dest_dst->dst_cache = dst; 84 dest_dst->dst_cookie = dst_cookie; 85 } 86 rcu_assign_pointer(dest->dest_dst, dest_dst); 87 88 if (old) 89 call_rcu(&old->rcu_head, ip_vs_dest_dst_rcu_free); 90 } 91 92 static inline struct ip_vs_dest_dst * 93 __ip_vs_dst_check(struct ip_vs_dest *dest) 94 { 95 struct ip_vs_dest_dst *dest_dst = rcu_dereference(dest->dest_dst); 96 struct dst_entry *dst; 97 98 if (!dest_dst) 99 return NULL; 100 dst = dest_dst->dst_cache; 101 if (dst->obsolete && 102 dst->ops->check(dst, dest_dst->dst_cookie) == NULL) 103 return NULL; 104 return dest_dst; 105 } 106 107 static inline bool 108 __mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu) 109 { 110 if (IP6CB(skb)->frag_max_size) { 111 /* frag_max_size tell us that, this packet have been 112 * defragmented by netfilter IPv6 conntrack module. 113 */ 114 if (IP6CB(skb)->frag_max_size > mtu) 115 return true; /* largest fragment violate MTU */ 116 } 117 else if (skb->len > mtu && !skb_is_gso(skb)) { 118 return true; /* Packet size violate MTU size */ 119 } 120 return false; 121 } 122 123 /* Get route to daddr, update *saddr, optionally bind route to saddr */ 124 static struct rtable *do_output_route4(struct net *net, __be32 daddr, 125 int rt_mode, __be32 *saddr) 126 { 127 struct flowi4 fl4; 128 struct rtable *rt; 129 int loop = 0; 130 131 memset(&fl4, 0, sizeof(fl4)); 132 fl4.daddr = daddr; 133 fl4.flowi4_flags = (rt_mode & IP_VS_RT_MODE_KNOWN_NH) ? 134 FLOWI_FLAG_KNOWN_NH : 0; 135 136 retry: 137 rt = ip_route_output_key(net, &fl4); 138 if (IS_ERR(rt)) { 139 /* Invalid saddr ? */ 140 if (PTR_ERR(rt) == -EINVAL && *saddr && 141 rt_mode & IP_VS_RT_MODE_CONNECT && !loop) { 142 *saddr = 0; 143 flowi4_update_output(&fl4, 0, 0, daddr, 0); 144 goto retry; 145 } 146 IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n", &daddr); 147 return NULL; 148 } else if (!*saddr && rt_mode & IP_VS_RT_MODE_CONNECT && fl4.saddr) { 149 ip_rt_put(rt); 150 *saddr = fl4.saddr; 151 flowi4_update_output(&fl4, 0, 0, daddr, fl4.saddr); 152 loop++; 153 goto retry; 154 } 155 *saddr = fl4.saddr; 156 return rt; 157 } 158 159 #ifdef CONFIG_IP_VS_IPV6 160 static inline int __ip_vs_is_local_route6(struct rt6_info *rt) 161 { 162 return rt->dst.dev && rt->dst.dev->flags & IFF_LOOPBACK; 163 } 164 #endif 165 166 static inline bool crosses_local_route_boundary(int skb_af, struct sk_buff *skb, 167 int rt_mode, 168 bool new_rt_is_local) 169 { 170 bool rt_mode_allow_local = !!(rt_mode & IP_VS_RT_MODE_LOCAL); 171 bool rt_mode_allow_non_local = !!(rt_mode & IP_VS_RT_MODE_NON_LOCAL); 172 bool rt_mode_allow_redirect = !!(rt_mode & IP_VS_RT_MODE_RDR); 173 bool source_is_loopback; 174 bool old_rt_is_local; 175 176 #ifdef CONFIG_IP_VS_IPV6 177 if (skb_af == AF_INET6) { 178 int addr_type = ipv6_addr_type(&ipv6_hdr(skb)->saddr); 179 180 source_is_loopback = 181 (!skb->dev || skb->dev->flags & IFF_LOOPBACK) && 182 (addr_type & IPV6_ADDR_LOOPBACK); 183 old_rt_is_local = __ip_vs_is_local_route6( 184 (struct rt6_info *)skb_dst(skb)); 185 } else 186 #endif 187 { 188 source_is_loopback = ipv4_is_loopback(ip_hdr(skb)->saddr); 189 old_rt_is_local = skb_rtable(skb)->rt_flags & RTCF_LOCAL; 190 } 191 192 if (unlikely(new_rt_is_local)) { 193 if (!rt_mode_allow_local) 194 return true; 195 if (!rt_mode_allow_redirect && !old_rt_is_local) 196 return true; 197 } else { 198 if (!rt_mode_allow_non_local) 199 return true; 200 if (source_is_loopback) 201 return true; 202 } 203 return false; 204 } 205 206 static inline void maybe_update_pmtu(int skb_af, struct sk_buff *skb, int mtu) 207 { 208 struct sock *sk = skb->sk; 209 struct rtable *ort = skb_rtable(skb); 210 211 if (!skb->dev && sk && sk_fullsock(sk)) 212 ort->dst.ops->update_pmtu(&ort->dst, sk, NULL, mtu); 213 } 214 215 static inline bool ensure_mtu_is_adequate(struct netns_ipvs *ipvs, int skb_af, 216 int rt_mode, 217 struct ip_vs_iphdr *ipvsh, 218 struct sk_buff *skb, int mtu) 219 { 220 #ifdef CONFIG_IP_VS_IPV6 221 if (skb_af == AF_INET6) { 222 struct net *net = ipvs->net; 223 224 if (unlikely(__mtu_check_toobig_v6(skb, mtu))) { 225 if (!skb->dev) 226 skb->dev = net->loopback_dev; 227 /* only send ICMP too big on first fragment */ 228 if (!ipvsh->fragoffs && !ip_vs_iph_icmp(ipvsh)) 229 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 230 IP_VS_DBG(1, "frag needed for %pI6c\n", 231 &ipv6_hdr(skb)->saddr); 232 return false; 233 } 234 } else 235 #endif 236 { 237 /* If we're going to tunnel the packet and pmtu discovery 238 * is disabled, we'll just fragment it anyway 239 */ 240 if ((rt_mode & IP_VS_RT_MODE_TUNNEL) && !sysctl_pmtu_disc(ipvs)) 241 return true; 242 243 if (unlikely(ip_hdr(skb)->frag_off & htons(IP_DF) && 244 skb->len > mtu && !skb_is_gso(skb) && 245 !ip_vs_iph_icmp(ipvsh))) { 246 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, 247 htonl(mtu)); 248 IP_VS_DBG(1, "frag needed for %pI4\n", 249 &ip_hdr(skb)->saddr); 250 return false; 251 } 252 } 253 254 return true; 255 } 256 257 static inline bool decrement_ttl(struct netns_ipvs *ipvs, 258 int skb_af, 259 struct sk_buff *skb) 260 { 261 struct net *net = ipvs->net; 262 263 #ifdef CONFIG_IP_VS_IPV6 264 if (skb_af == AF_INET6) { 265 struct dst_entry *dst = skb_dst(skb); 266 267 /* check and decrement ttl */ 268 if (ipv6_hdr(skb)->hop_limit <= 1) { 269 struct inet6_dev *idev = __in6_dev_get_safely(skb->dev); 270 271 /* Force OUTPUT device used as source address */ 272 skb->dev = dst->dev; 273 icmpv6_send(skb, ICMPV6_TIME_EXCEED, 274 ICMPV6_EXC_HOPLIMIT, 0); 275 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); 276 277 return false; 278 } 279 280 /* don't propagate ttl change to cloned packets */ 281 if (!skb_make_writable(skb, sizeof(struct ipv6hdr))) 282 return false; 283 284 ipv6_hdr(skb)->hop_limit--; 285 } else 286 #endif 287 { 288 if (ip_hdr(skb)->ttl <= 1) { 289 /* Tell the sender its packet died... */ 290 __IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS); 291 icmp_send(skb, ICMP_TIME_EXCEEDED, ICMP_EXC_TTL, 0); 292 return false; 293 } 294 295 /* don't propagate ttl change to cloned packets */ 296 if (!skb_make_writable(skb, sizeof(struct iphdr))) 297 return false; 298 299 /* Decrease ttl */ 300 ip_decrease_ttl(ip_hdr(skb)); 301 } 302 303 return true; 304 } 305 306 /* Get route to destination or remote server */ 307 static int 308 __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, 309 struct ip_vs_dest *dest, 310 __be32 daddr, int rt_mode, __be32 *ret_saddr, 311 struct ip_vs_iphdr *ipvsh) 312 { 313 struct net *net = ipvs->net; 314 struct ip_vs_dest_dst *dest_dst; 315 struct rtable *rt; /* Route to the other host */ 316 int mtu; 317 int local, noref = 1; 318 319 if (dest) { 320 dest_dst = __ip_vs_dst_check(dest); 321 if (likely(dest_dst)) 322 rt = (struct rtable *) dest_dst->dst_cache; 323 else { 324 dest_dst = ip_vs_dest_dst_alloc(); 325 spin_lock_bh(&dest->dst_lock); 326 if (!dest_dst) { 327 __ip_vs_dst_set(dest, NULL, NULL, 0); 328 spin_unlock_bh(&dest->dst_lock); 329 goto err_unreach; 330 } 331 rt = do_output_route4(net, dest->addr.ip, rt_mode, 332 &dest_dst->dst_saddr.ip); 333 if (!rt) { 334 __ip_vs_dst_set(dest, NULL, NULL, 0); 335 spin_unlock_bh(&dest->dst_lock); 336 ip_vs_dest_dst_free(dest_dst); 337 goto err_unreach; 338 } 339 __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); 340 spin_unlock_bh(&dest->dst_lock); 341 IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n", 342 &dest->addr.ip, &dest_dst->dst_saddr.ip, 343 atomic_read(&rt->dst.__refcnt)); 344 } 345 if (ret_saddr) 346 *ret_saddr = dest_dst->dst_saddr.ip; 347 } else { 348 __be32 saddr = htonl(INADDR_ANY); 349 350 noref = 0; 351 352 /* For such unconfigured boxes avoid many route lookups 353 * for performance reasons because we do not remember saddr 354 */ 355 rt_mode &= ~IP_VS_RT_MODE_CONNECT; 356 rt = do_output_route4(net, daddr, rt_mode, &saddr); 357 if (!rt) 358 goto err_unreach; 359 if (ret_saddr) 360 *ret_saddr = saddr; 361 } 362 363 local = (rt->rt_flags & RTCF_LOCAL) ? 1 : 0; 364 if (unlikely(crosses_local_route_boundary(skb_af, skb, rt_mode, 365 local))) { 366 IP_VS_DBG_RL("We are crossing local and non-local addresses" 367 " daddr=%pI4\n", &daddr); 368 goto err_put; 369 } 370 371 if (unlikely(local)) { 372 /* skb to local stack, preserve old route */ 373 if (!noref) 374 ip_rt_put(rt); 375 return local; 376 } 377 378 if (!decrement_ttl(ipvs, skb_af, skb)) 379 goto err_put; 380 381 if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL))) { 382 mtu = dst_mtu(&rt->dst); 383 } else { 384 mtu = dst_mtu(&rt->dst) - sizeof(struct iphdr); 385 if (mtu < 68) { 386 IP_VS_DBG_RL("%s(): mtu less than 68\n", __func__); 387 goto err_put; 388 } 389 maybe_update_pmtu(skb_af, skb, mtu); 390 } 391 392 if (!ensure_mtu_is_adequate(ipvs, skb_af, rt_mode, ipvsh, skb, mtu)) 393 goto err_put; 394 395 skb_dst_drop(skb); 396 if (noref) { 397 if (!local) 398 skb_dst_set_noref(skb, &rt->dst); 399 else 400 skb_dst_set(skb, dst_clone(&rt->dst)); 401 } else 402 skb_dst_set(skb, &rt->dst); 403 404 return local; 405 406 err_put: 407 if (!noref) 408 ip_rt_put(rt); 409 return -1; 410 411 err_unreach: 412 dst_link_failure(skb); 413 return -1; 414 } 415 416 #ifdef CONFIG_IP_VS_IPV6 417 static struct dst_entry * 418 __ip_vs_route_output_v6(struct net *net, struct in6_addr *daddr, 419 struct in6_addr *ret_saddr, int do_xfrm, int rt_mode) 420 { 421 struct dst_entry *dst; 422 struct flowi6 fl6 = { 423 .daddr = *daddr, 424 }; 425 426 if (rt_mode & IP_VS_RT_MODE_KNOWN_NH) 427 fl6.flowi6_flags = FLOWI_FLAG_KNOWN_NH; 428 429 dst = ip6_route_output(net, NULL, &fl6); 430 if (dst->error) 431 goto out_err; 432 if (!ret_saddr) 433 return dst; 434 if (ipv6_addr_any(&fl6.saddr) && 435 ipv6_dev_get_saddr(net, ip6_dst_idev(dst)->dev, 436 &fl6.daddr, 0, &fl6.saddr) < 0) 437 goto out_err; 438 if (do_xfrm) { 439 dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), NULL, 0); 440 if (IS_ERR(dst)) { 441 dst = NULL; 442 goto out_err; 443 } 444 } 445 *ret_saddr = fl6.saddr; 446 return dst; 447 448 out_err: 449 dst_release(dst); 450 IP_VS_DBG_RL("ip6_route_output error, dest: %pI6\n", daddr); 451 return NULL; 452 } 453 454 /* 455 * Get route to destination or remote server 456 */ 457 static int 458 __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, 459 struct ip_vs_dest *dest, 460 struct in6_addr *daddr, struct in6_addr *ret_saddr, 461 struct ip_vs_iphdr *ipvsh, int do_xfrm, int rt_mode) 462 { 463 struct net *net = ipvs->net; 464 struct ip_vs_dest_dst *dest_dst; 465 struct rt6_info *rt; /* Route to the other host */ 466 struct dst_entry *dst; 467 int mtu; 468 int local, noref = 1; 469 470 if (dest) { 471 dest_dst = __ip_vs_dst_check(dest); 472 if (likely(dest_dst)) 473 rt = (struct rt6_info *) dest_dst->dst_cache; 474 else { 475 u32 cookie; 476 477 dest_dst = ip_vs_dest_dst_alloc(); 478 spin_lock_bh(&dest->dst_lock); 479 if (!dest_dst) { 480 __ip_vs_dst_set(dest, NULL, NULL, 0); 481 spin_unlock_bh(&dest->dst_lock); 482 goto err_unreach; 483 } 484 dst = __ip_vs_route_output_v6(net, &dest->addr.in6, 485 &dest_dst->dst_saddr.in6, 486 do_xfrm, rt_mode); 487 if (!dst) { 488 __ip_vs_dst_set(dest, NULL, NULL, 0); 489 spin_unlock_bh(&dest->dst_lock); 490 ip_vs_dest_dst_free(dest_dst); 491 goto err_unreach; 492 } 493 rt = (struct rt6_info *) dst; 494 cookie = rt6_get_cookie(rt); 495 __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); 496 spin_unlock_bh(&dest->dst_lock); 497 IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n", 498 &dest->addr.in6, &dest_dst->dst_saddr.in6, 499 atomic_read(&rt->dst.__refcnt)); 500 } 501 if (ret_saddr) 502 *ret_saddr = dest_dst->dst_saddr.in6; 503 } else { 504 noref = 0; 505 dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm, 506 rt_mode); 507 if (!dst) 508 goto err_unreach; 509 rt = (struct rt6_info *) dst; 510 } 511 512 local = __ip_vs_is_local_route6(rt); 513 514 if (unlikely(crosses_local_route_boundary(skb_af, skb, rt_mode, 515 local))) { 516 IP_VS_DBG_RL("We are crossing local and non-local addresses" 517 " daddr=%pI6\n", daddr); 518 goto err_put; 519 } 520 521 if (unlikely(local)) { 522 /* skb to local stack, preserve old route */ 523 if (!noref) 524 dst_release(&rt->dst); 525 return local; 526 } 527 528 if (!decrement_ttl(ipvs, skb_af, skb)) 529 goto err_put; 530 531 /* MTU checking */ 532 if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL))) 533 mtu = dst_mtu(&rt->dst); 534 else { 535 mtu = dst_mtu(&rt->dst) - sizeof(struct ipv6hdr); 536 if (mtu < IPV6_MIN_MTU) { 537 IP_VS_DBG_RL("%s(): mtu less than %d\n", __func__, 538 IPV6_MIN_MTU); 539 goto err_put; 540 } 541 maybe_update_pmtu(skb_af, skb, mtu); 542 } 543 544 if (!ensure_mtu_is_adequate(ipvs, skb_af, rt_mode, ipvsh, skb, mtu)) 545 goto err_put; 546 547 skb_dst_drop(skb); 548 if (noref) { 549 if (!local) 550 skb_dst_set_noref(skb, &rt->dst); 551 else 552 skb_dst_set(skb, dst_clone(&rt->dst)); 553 } else 554 skb_dst_set(skb, &rt->dst); 555 556 return local; 557 558 err_put: 559 if (!noref) 560 dst_release(&rt->dst); 561 return -1; 562 563 err_unreach: 564 /* The ip6_link_failure function requires the dev field to be set 565 * in order to get the net (further for the sake of fwmark 566 * reflection). 567 */ 568 if (!skb->dev) 569 skb->dev = skb_dst(skb)->dev; 570 571 dst_link_failure(skb); 572 return -1; 573 } 574 #endif 575 576 577 /* return NF_ACCEPT to allow forwarding or other NF_xxx on error */ 578 static inline int ip_vs_tunnel_xmit_prepare(struct sk_buff *skb, 579 struct ip_vs_conn *cp) 580 { 581 int ret = NF_ACCEPT; 582 583 skb->ipvs_property = 1; 584 if (unlikely(cp->flags & IP_VS_CONN_F_NFCT)) 585 ret = ip_vs_confirm_conntrack(skb); 586 if (ret == NF_ACCEPT) { 587 nf_reset(skb); 588 skb_forward_csum(skb); 589 } 590 return ret; 591 } 592 593 /* In the event of a remote destination, it's possible that we would have 594 * matches against an old socket (particularly a TIME-WAIT socket). This 595 * causes havoc down the line (ip_local_out et. al. expect regular sockets 596 * and invalid memory accesses will happen) so simply drop the association 597 * in this case. 598 */ 599 static inline void ip_vs_drop_early_demux_sk(struct sk_buff *skb) 600 { 601 /* If dev is set, the packet came from the LOCAL_IN callback and 602 * not from a local TCP socket. 603 */ 604 if (skb->dev) 605 skb_orphan(skb); 606 } 607 608 /* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */ 609 static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb, 610 struct ip_vs_conn *cp, int local) 611 { 612 int ret = NF_STOLEN; 613 614 skb->ipvs_property = 1; 615 if (likely(!(cp->flags & IP_VS_CONN_F_NFCT))) 616 ip_vs_notrack(skb); 617 else 618 ip_vs_update_conntrack(skb, cp, 1); 619 620 /* Remove the early_demux association unless it's bound for the 621 * exact same port and address on this host after translation. 622 */ 623 if (!local || cp->vport != cp->dport || 624 !ip_vs_addr_equal(cp->af, &cp->vaddr, &cp->daddr)) 625 ip_vs_drop_early_demux_sk(skb); 626 627 if (!local) { 628 skb_forward_csum(skb); 629 NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb, 630 NULL, skb_dst(skb)->dev, dst_output); 631 } else 632 ret = NF_ACCEPT; 633 634 return ret; 635 } 636 637 /* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */ 638 static inline int ip_vs_send_or_cont(int pf, struct sk_buff *skb, 639 struct ip_vs_conn *cp, int local) 640 { 641 int ret = NF_STOLEN; 642 643 skb->ipvs_property = 1; 644 if (likely(!(cp->flags & IP_VS_CONN_F_NFCT))) 645 ip_vs_notrack(skb); 646 if (!local) { 647 ip_vs_drop_early_demux_sk(skb); 648 skb_forward_csum(skb); 649 NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb, 650 NULL, skb_dst(skb)->dev, dst_output); 651 } else 652 ret = NF_ACCEPT; 653 return ret; 654 } 655 656 657 /* 658 * NULL transmitter (do nothing except return NF_ACCEPT) 659 */ 660 int 661 ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 662 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 663 { 664 /* we do not touch skb and do not need pskb ptr */ 665 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1); 666 } 667 668 669 /* 670 * Bypass transmitter 671 * Let packets bypass the destination when the destination is not 672 * available, it may be only used in transparent cache cluster. 673 */ 674 int 675 ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 676 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 677 { 678 struct iphdr *iph = ip_hdr(skb); 679 680 EnterFunction(10); 681 682 if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, iph->daddr, 683 IP_VS_RT_MODE_NON_LOCAL, NULL, ipvsh) < 0) 684 goto tx_error; 685 686 ip_send_check(iph); 687 688 /* Another hack: avoid icmp_send in ip_fragment */ 689 skb->ignore_df = 1; 690 691 ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0); 692 693 LeaveFunction(10); 694 return NF_STOLEN; 695 696 tx_error: 697 kfree_skb(skb); 698 LeaveFunction(10); 699 return NF_STOLEN; 700 } 701 702 #ifdef CONFIG_IP_VS_IPV6 703 int 704 ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 705 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 706 { 707 struct ipv6hdr *iph = ipv6_hdr(skb); 708 709 EnterFunction(10); 710 711 if (__ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, NULL, 712 &iph->daddr, NULL, 713 ipvsh, 0, IP_VS_RT_MODE_NON_LOCAL) < 0) 714 goto tx_error; 715 716 /* Another hack: avoid icmp_send in ip_fragment */ 717 skb->ignore_df = 1; 718 719 ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0); 720 721 LeaveFunction(10); 722 return NF_STOLEN; 723 724 tx_error: 725 kfree_skb(skb); 726 LeaveFunction(10); 727 return NF_STOLEN; 728 } 729 #endif 730 731 /* 732 * NAT transmitter (only for outside-to-inside nat forwarding) 733 * Not used for related ICMP 734 */ 735 int 736 ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 737 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 738 { 739 struct rtable *rt; /* Route to the other host */ 740 int local, rc, was_input; 741 742 EnterFunction(10); 743 744 /* check if it is a connection of no-client-port */ 745 if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) { 746 __be16 _pt, *p; 747 748 p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt); 749 if (p == NULL) 750 goto tx_error; 751 ip_vs_conn_fill_cport(cp, *p); 752 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p)); 753 } 754 755 was_input = rt_is_input_route(skb_rtable(skb)); 756 local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, 757 IP_VS_RT_MODE_LOCAL | 758 IP_VS_RT_MODE_NON_LOCAL | 759 IP_VS_RT_MODE_RDR, NULL, ipvsh); 760 if (local < 0) 761 goto tx_error; 762 rt = skb_rtable(skb); 763 /* 764 * Avoid duplicate tuple in reply direction for NAT traffic 765 * to local address when connection is sync-ed 766 */ 767 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 768 if (cp->flags & IP_VS_CONN_F_SYNC && local) { 769 enum ip_conntrack_info ctinfo; 770 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 771 772 if (ct) { 773 IP_VS_DBG_RL_PKT(10, AF_INET, pp, skb, ipvsh->off, 774 "ip_vs_nat_xmit(): " 775 "stopping DNAT to local address"); 776 goto tx_error; 777 } 778 } 779 #endif 780 781 /* From world but DNAT to loopback address? */ 782 if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) { 783 IP_VS_DBG_RL_PKT(1, AF_INET, pp, skb, ipvsh->off, 784 "ip_vs_nat_xmit(): stopping DNAT to loopback " 785 "address"); 786 goto tx_error; 787 } 788 789 /* copy-on-write the packet before mangling it */ 790 if (!skb_make_writable(skb, sizeof(struct iphdr))) 791 goto tx_error; 792 793 if (skb_cow(skb, rt->dst.dev->hard_header_len)) 794 goto tx_error; 795 796 /* mangle the packet */ 797 if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh)) 798 goto tx_error; 799 ip_hdr(skb)->daddr = cp->daddr.ip; 800 ip_send_check(ip_hdr(skb)); 801 802 IP_VS_DBG_PKT(10, AF_INET, pp, skb, ipvsh->off, "After DNAT"); 803 804 /* FIXME: when application helper enlarges the packet and the length 805 is larger than the MTU of outgoing device, there will be still 806 MTU problem. */ 807 808 /* Another hack: avoid icmp_send in ip_fragment */ 809 skb->ignore_df = 1; 810 811 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local); 812 813 LeaveFunction(10); 814 return rc; 815 816 tx_error: 817 kfree_skb(skb); 818 LeaveFunction(10); 819 return NF_STOLEN; 820 } 821 822 #ifdef CONFIG_IP_VS_IPV6 823 int 824 ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 825 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 826 { 827 struct rt6_info *rt; /* Route to the other host */ 828 int local, rc; 829 830 EnterFunction(10); 831 832 /* check if it is a connection of no-client-port */ 833 if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT && !ipvsh->fragoffs)) { 834 __be16 _pt, *p; 835 p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt); 836 if (p == NULL) 837 goto tx_error; 838 ip_vs_conn_fill_cport(cp, *p); 839 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p)); 840 } 841 842 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest, 843 &cp->daddr.in6, 844 NULL, ipvsh, 0, 845 IP_VS_RT_MODE_LOCAL | 846 IP_VS_RT_MODE_NON_LOCAL | 847 IP_VS_RT_MODE_RDR); 848 if (local < 0) 849 goto tx_error; 850 rt = (struct rt6_info *) skb_dst(skb); 851 /* 852 * Avoid duplicate tuple in reply direction for NAT traffic 853 * to local address when connection is sync-ed 854 */ 855 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 856 if (cp->flags & IP_VS_CONN_F_SYNC && local) { 857 enum ip_conntrack_info ctinfo; 858 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 859 860 if (ct) { 861 IP_VS_DBG_RL_PKT(10, AF_INET6, pp, skb, ipvsh->off, 862 "ip_vs_nat_xmit_v6(): " 863 "stopping DNAT to local address"); 864 goto tx_error; 865 } 866 } 867 #endif 868 869 /* From world but DNAT to loopback address? */ 870 if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) && 871 ipv6_addr_type(&cp->daddr.in6) & IPV6_ADDR_LOOPBACK) { 872 IP_VS_DBG_RL_PKT(1, AF_INET6, pp, skb, ipvsh->off, 873 "ip_vs_nat_xmit_v6(): " 874 "stopping DNAT to loopback address"); 875 goto tx_error; 876 } 877 878 /* copy-on-write the packet before mangling it */ 879 if (!skb_make_writable(skb, sizeof(struct ipv6hdr))) 880 goto tx_error; 881 882 if (skb_cow(skb, rt->dst.dev->hard_header_len)) 883 goto tx_error; 884 885 /* mangle the packet */ 886 if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh)) 887 goto tx_error; 888 ipv6_hdr(skb)->daddr = cp->daddr.in6; 889 890 IP_VS_DBG_PKT(10, AF_INET6, pp, skb, ipvsh->off, "After DNAT"); 891 892 /* FIXME: when application helper enlarges the packet and the length 893 is larger than the MTU of outgoing device, there will be still 894 MTU problem. */ 895 896 /* Another hack: avoid icmp_send in ip_fragment */ 897 skb->ignore_df = 1; 898 899 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local); 900 901 LeaveFunction(10); 902 return rc; 903 904 tx_error: 905 LeaveFunction(10); 906 kfree_skb(skb); 907 return NF_STOLEN; 908 } 909 #endif 910 911 /* When forwarding a packet, we must ensure that we've got enough headroom 912 * for the encapsulation packet in the skb. This also gives us an 913 * opportunity to figure out what the payload_len, dsfield, ttl, and df 914 * values should be, so that we won't need to look at the old ip header 915 * again 916 */ 917 static struct sk_buff * 918 ip_vs_prepare_tunneled_skb(struct sk_buff *skb, int skb_af, 919 unsigned int max_headroom, __u8 *next_protocol, 920 __u32 *payload_len, __u8 *dsfield, __u8 *ttl, 921 __be16 *df) 922 { 923 struct sk_buff *new_skb = NULL; 924 struct iphdr *old_iph = NULL; 925 __u8 old_dsfield; 926 #ifdef CONFIG_IP_VS_IPV6 927 struct ipv6hdr *old_ipv6h = NULL; 928 #endif 929 930 ip_vs_drop_early_demux_sk(skb); 931 932 if (skb_headroom(skb) < max_headroom || skb_cloned(skb)) { 933 new_skb = skb_realloc_headroom(skb, max_headroom); 934 if (!new_skb) 935 goto error; 936 if (skb->sk) 937 skb_set_owner_w(new_skb, skb->sk); 938 consume_skb(skb); 939 skb = new_skb; 940 } 941 942 #ifdef CONFIG_IP_VS_IPV6 943 if (skb_af == AF_INET6) { 944 old_ipv6h = ipv6_hdr(skb); 945 *next_protocol = IPPROTO_IPV6; 946 if (payload_len) 947 *payload_len = 948 ntohs(old_ipv6h->payload_len) + 949 sizeof(*old_ipv6h); 950 old_dsfield = ipv6_get_dsfield(old_ipv6h); 951 *ttl = old_ipv6h->hop_limit; 952 if (df) 953 *df = 0; 954 } else 955 #endif 956 { 957 old_iph = ip_hdr(skb); 958 /* Copy DF, reset fragment offset and MF */ 959 if (df) 960 *df = (old_iph->frag_off & htons(IP_DF)); 961 *next_protocol = IPPROTO_IPIP; 962 963 /* fix old IP header checksum */ 964 ip_send_check(old_iph); 965 old_dsfield = ipv4_get_dsfield(old_iph); 966 *ttl = old_iph->ttl; 967 if (payload_len) 968 *payload_len = ntohs(old_iph->tot_len); 969 } 970 971 /* Implement full-functionality option for ECN encapsulation */ 972 *dsfield = INET_ECN_encapsulate(old_dsfield, old_dsfield); 973 974 return skb; 975 error: 976 kfree_skb(skb); 977 return ERR_PTR(-ENOMEM); 978 } 979 980 static inline int __tun_gso_type_mask(int encaps_af, int orig_af) 981 { 982 switch (encaps_af) { 983 case AF_INET: 984 return SKB_GSO_IPXIP4; 985 case AF_INET6: 986 return SKB_GSO_IPXIP6; 987 default: 988 return 0; 989 } 990 } 991 992 /* 993 * IP Tunneling transmitter 994 * 995 * This function encapsulates the packet in a new IP packet, its 996 * destination will be set to cp->daddr. Most code of this function 997 * is taken from ipip.c. 998 * 999 * It is used in VS/TUN cluster. The load balancer selects a real 1000 * server from a cluster based on a scheduling algorithm, 1001 * encapsulates the request packet and forwards it to the selected 1002 * server. For example, all real servers are configured with 1003 * "ifconfig tunl0 <Virtual IP Address> up". When the server receives 1004 * the encapsulated packet, it will decapsulate the packet, processe 1005 * the request and return the response packets directly to the client 1006 * without passing the load balancer. This can greatly increase the 1007 * scalability of virtual server. 1008 * 1009 * Used for ANY protocol 1010 */ 1011 int 1012 ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1013 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 1014 { 1015 struct netns_ipvs *ipvs = cp->ipvs; 1016 struct net *net = ipvs->net; 1017 struct rtable *rt; /* Route to the other host */ 1018 __be32 saddr; /* Source for tunnel */ 1019 struct net_device *tdev; /* Device to other host */ 1020 __u8 next_protocol = 0; 1021 __u8 dsfield = 0; 1022 __u8 ttl = 0; 1023 __be16 df = 0; 1024 __be16 *dfp = NULL; 1025 struct iphdr *iph; /* Our new IP header */ 1026 unsigned int max_headroom; /* The extra header space needed */ 1027 int ret, local; 1028 1029 EnterFunction(10); 1030 1031 local = __ip_vs_get_out_rt(ipvs, cp->af, skb, cp->dest, cp->daddr.ip, 1032 IP_VS_RT_MODE_LOCAL | 1033 IP_VS_RT_MODE_NON_LOCAL | 1034 IP_VS_RT_MODE_CONNECT | 1035 IP_VS_RT_MODE_TUNNEL, &saddr, ipvsh); 1036 if (local < 0) 1037 goto tx_error; 1038 if (local) 1039 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1); 1040 1041 rt = skb_rtable(skb); 1042 tdev = rt->dst.dev; 1043 1044 /* 1045 * Okay, now see if we can stuff it in the buffer as-is. 1046 */ 1047 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr); 1048 1049 /* We only care about the df field if sysctl_pmtu_disc(ipvs) is set */ 1050 dfp = sysctl_pmtu_disc(ipvs) ? &df : NULL; 1051 skb = ip_vs_prepare_tunneled_skb(skb, cp->af, max_headroom, 1052 &next_protocol, NULL, &dsfield, 1053 &ttl, dfp); 1054 if (IS_ERR(skb)) 1055 goto tx_error; 1056 1057 if (iptunnel_handle_offloads(skb, __tun_gso_type_mask(AF_INET, cp->af))) 1058 goto tx_error; 1059 1060 skb->transport_header = skb->network_header; 1061 1062 skb_push(skb, sizeof(struct iphdr)); 1063 skb_reset_network_header(skb); 1064 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); 1065 1066 /* 1067 * Push down and install the IPIP header. 1068 */ 1069 iph = ip_hdr(skb); 1070 iph->version = 4; 1071 iph->ihl = sizeof(struct iphdr)>>2; 1072 iph->frag_off = df; 1073 iph->protocol = next_protocol; 1074 iph->tos = dsfield; 1075 iph->daddr = cp->daddr.ip; 1076 iph->saddr = saddr; 1077 iph->ttl = ttl; 1078 ip_select_ident(net, skb, NULL); 1079 1080 /* Another hack: avoid icmp_send in ip_fragment */ 1081 skb->ignore_df = 1; 1082 1083 ret = ip_vs_tunnel_xmit_prepare(skb, cp); 1084 if (ret == NF_ACCEPT) 1085 ip_local_out(net, skb->sk, skb); 1086 else if (ret == NF_DROP) 1087 kfree_skb(skb); 1088 1089 LeaveFunction(10); 1090 1091 return NF_STOLEN; 1092 1093 tx_error: 1094 if (!IS_ERR(skb)) 1095 kfree_skb(skb); 1096 LeaveFunction(10); 1097 return NF_STOLEN; 1098 } 1099 1100 #ifdef CONFIG_IP_VS_IPV6 1101 int 1102 ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1103 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 1104 { 1105 struct rt6_info *rt; /* Route to the other host */ 1106 struct in6_addr saddr; /* Source for tunnel */ 1107 struct net_device *tdev; /* Device to other host */ 1108 __u8 next_protocol = 0; 1109 __u32 payload_len = 0; 1110 __u8 dsfield = 0; 1111 __u8 ttl = 0; 1112 struct ipv6hdr *iph; /* Our new IP header */ 1113 unsigned int max_headroom; /* The extra header space needed */ 1114 int ret, local; 1115 1116 EnterFunction(10); 1117 1118 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest, 1119 &cp->daddr.in6, 1120 &saddr, ipvsh, 1, 1121 IP_VS_RT_MODE_LOCAL | 1122 IP_VS_RT_MODE_NON_LOCAL | 1123 IP_VS_RT_MODE_TUNNEL); 1124 if (local < 0) 1125 goto tx_error; 1126 if (local) 1127 return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1); 1128 1129 rt = (struct rt6_info *) skb_dst(skb); 1130 tdev = rt->dst.dev; 1131 1132 /* 1133 * Okay, now see if we can stuff it in the buffer as-is. 1134 */ 1135 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct ipv6hdr); 1136 1137 skb = ip_vs_prepare_tunneled_skb(skb, cp->af, max_headroom, 1138 &next_protocol, &payload_len, 1139 &dsfield, &ttl, NULL); 1140 if (IS_ERR(skb)) 1141 goto tx_error; 1142 1143 if (iptunnel_handle_offloads(skb, __tun_gso_type_mask(AF_INET6, cp->af))) 1144 goto tx_error; 1145 1146 skb->transport_header = skb->network_header; 1147 1148 skb_push(skb, sizeof(struct ipv6hdr)); 1149 skb_reset_network_header(skb); 1150 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); 1151 1152 /* 1153 * Push down and install the IPIP header. 1154 */ 1155 iph = ipv6_hdr(skb); 1156 iph->version = 6; 1157 iph->nexthdr = next_protocol; 1158 iph->payload_len = htons(payload_len); 1159 memset(&iph->flow_lbl, 0, sizeof(iph->flow_lbl)); 1160 ipv6_change_dsfield(iph, 0, dsfield); 1161 iph->daddr = cp->daddr.in6; 1162 iph->saddr = saddr; 1163 iph->hop_limit = ttl; 1164 1165 /* Another hack: avoid icmp_send in ip_fragment */ 1166 skb->ignore_df = 1; 1167 1168 ret = ip_vs_tunnel_xmit_prepare(skb, cp); 1169 if (ret == NF_ACCEPT) 1170 ip6_local_out(cp->ipvs->net, skb->sk, skb); 1171 else if (ret == NF_DROP) 1172 kfree_skb(skb); 1173 1174 LeaveFunction(10); 1175 1176 return NF_STOLEN; 1177 1178 tx_error: 1179 if (!IS_ERR(skb)) 1180 kfree_skb(skb); 1181 LeaveFunction(10); 1182 return NF_STOLEN; 1183 } 1184 #endif 1185 1186 1187 /* 1188 * Direct Routing transmitter 1189 * Used for ANY protocol 1190 */ 1191 int 1192 ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1193 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 1194 { 1195 int local; 1196 1197 EnterFunction(10); 1198 1199 local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, 1200 IP_VS_RT_MODE_LOCAL | 1201 IP_VS_RT_MODE_NON_LOCAL | 1202 IP_VS_RT_MODE_KNOWN_NH, NULL, ipvsh); 1203 if (local < 0) 1204 goto tx_error; 1205 if (local) 1206 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1); 1207 1208 ip_send_check(ip_hdr(skb)); 1209 1210 /* Another hack: avoid icmp_send in ip_fragment */ 1211 skb->ignore_df = 1; 1212 1213 ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0); 1214 1215 LeaveFunction(10); 1216 return NF_STOLEN; 1217 1218 tx_error: 1219 kfree_skb(skb); 1220 LeaveFunction(10); 1221 return NF_STOLEN; 1222 } 1223 1224 #ifdef CONFIG_IP_VS_IPV6 1225 int 1226 ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1227 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) 1228 { 1229 int local; 1230 1231 EnterFunction(10); 1232 1233 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest, 1234 &cp->daddr.in6, 1235 NULL, ipvsh, 0, 1236 IP_VS_RT_MODE_LOCAL | 1237 IP_VS_RT_MODE_NON_LOCAL | 1238 IP_VS_RT_MODE_KNOWN_NH); 1239 if (local < 0) 1240 goto tx_error; 1241 if (local) 1242 return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1); 1243 1244 /* Another hack: avoid icmp_send in ip_fragment */ 1245 skb->ignore_df = 1; 1246 1247 ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0); 1248 1249 LeaveFunction(10); 1250 return NF_STOLEN; 1251 1252 tx_error: 1253 kfree_skb(skb); 1254 LeaveFunction(10); 1255 return NF_STOLEN; 1256 } 1257 #endif 1258 1259 1260 /* 1261 * ICMP packet transmitter 1262 * called by the ip_vs_in_icmp 1263 */ 1264 int 1265 ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1266 struct ip_vs_protocol *pp, int offset, unsigned int hooknum, 1267 struct ip_vs_iphdr *iph) 1268 { 1269 struct rtable *rt; /* Route to the other host */ 1270 int rc; 1271 int local; 1272 int rt_mode, was_input; 1273 1274 EnterFunction(10); 1275 1276 /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be 1277 forwarded directly here, because there is no need to 1278 translate address/port back */ 1279 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { 1280 if (cp->packet_xmit) 1281 rc = cp->packet_xmit(skb, cp, pp, iph); 1282 else 1283 rc = NF_ACCEPT; 1284 /* do not touch skb anymore */ 1285 atomic_inc(&cp->in_pkts); 1286 goto out; 1287 } 1288 1289 /* 1290 * mangle and send the packet here (only for VS/NAT) 1291 */ 1292 was_input = rt_is_input_route(skb_rtable(skb)); 1293 1294 /* LOCALNODE from FORWARD hook is not supported */ 1295 rt_mode = (hooknum != NF_INET_FORWARD) ? 1296 IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | 1297 IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; 1298 local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, rt_mode, 1299 NULL, iph); 1300 if (local < 0) 1301 goto tx_error; 1302 rt = skb_rtable(skb); 1303 1304 /* 1305 * Avoid duplicate tuple in reply direction for NAT traffic 1306 * to local address when connection is sync-ed 1307 */ 1308 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 1309 if (cp->flags & IP_VS_CONN_F_SYNC && local) { 1310 enum ip_conntrack_info ctinfo; 1311 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 1312 1313 if (ct) { 1314 IP_VS_DBG(10, "%s(): " 1315 "stopping DNAT to local address %pI4\n", 1316 __func__, &cp->daddr.ip); 1317 goto tx_error; 1318 } 1319 } 1320 #endif 1321 1322 /* From world but DNAT to loopback address? */ 1323 if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) { 1324 IP_VS_DBG(1, "%s(): " 1325 "stopping DNAT to loopback %pI4\n", 1326 __func__, &cp->daddr.ip); 1327 goto tx_error; 1328 } 1329 1330 /* copy-on-write the packet before mangling it */ 1331 if (!skb_make_writable(skb, offset)) 1332 goto tx_error; 1333 1334 if (skb_cow(skb, rt->dst.dev->hard_header_len)) 1335 goto tx_error; 1336 1337 ip_vs_nat_icmp(skb, pp, cp, 0); 1338 1339 /* Another hack: avoid icmp_send in ip_fragment */ 1340 skb->ignore_df = 1; 1341 1342 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local); 1343 goto out; 1344 1345 tx_error: 1346 kfree_skb(skb); 1347 rc = NF_STOLEN; 1348 out: 1349 LeaveFunction(10); 1350 return rc; 1351 } 1352 1353 #ifdef CONFIG_IP_VS_IPV6 1354 int 1355 ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1356 struct ip_vs_protocol *pp, int offset, unsigned int hooknum, 1357 struct ip_vs_iphdr *ipvsh) 1358 { 1359 struct rt6_info *rt; /* Route to the other host */ 1360 int rc; 1361 int local; 1362 int rt_mode; 1363 1364 EnterFunction(10); 1365 1366 /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be 1367 forwarded directly here, because there is no need to 1368 translate address/port back */ 1369 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { 1370 if (cp->packet_xmit) 1371 rc = cp->packet_xmit(skb, cp, pp, ipvsh); 1372 else 1373 rc = NF_ACCEPT; 1374 /* do not touch skb anymore */ 1375 atomic_inc(&cp->in_pkts); 1376 goto out; 1377 } 1378 1379 /* 1380 * mangle and send the packet here (only for VS/NAT) 1381 */ 1382 1383 /* LOCALNODE from FORWARD hook is not supported */ 1384 rt_mode = (hooknum != NF_INET_FORWARD) ? 1385 IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | 1386 IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; 1387 local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest, 1388 &cp->daddr.in6, NULL, ipvsh, 0, rt_mode); 1389 if (local < 0) 1390 goto tx_error; 1391 rt = (struct rt6_info *) skb_dst(skb); 1392 /* 1393 * Avoid duplicate tuple in reply direction for NAT traffic 1394 * to local address when connection is sync-ed 1395 */ 1396 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 1397 if (cp->flags & IP_VS_CONN_F_SYNC && local) { 1398 enum ip_conntrack_info ctinfo; 1399 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 1400 1401 if (ct) { 1402 IP_VS_DBG(10, "%s(): " 1403 "stopping DNAT to local address %pI6\n", 1404 __func__, &cp->daddr.in6); 1405 goto tx_error; 1406 } 1407 } 1408 #endif 1409 1410 /* From world but DNAT to loopback address? */ 1411 if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) && 1412 ipv6_addr_type(&cp->daddr.in6) & IPV6_ADDR_LOOPBACK) { 1413 IP_VS_DBG(1, "%s(): " 1414 "stopping DNAT to loopback %pI6\n", 1415 __func__, &cp->daddr.in6); 1416 goto tx_error; 1417 } 1418 1419 /* copy-on-write the packet before mangling it */ 1420 if (!skb_make_writable(skb, offset)) 1421 goto tx_error; 1422 1423 if (skb_cow(skb, rt->dst.dev->hard_header_len)) 1424 goto tx_error; 1425 1426 ip_vs_nat_icmp_v6(skb, pp, cp, 0); 1427 1428 /* Another hack: avoid icmp_send in ip_fragment */ 1429 skb->ignore_df = 1; 1430 1431 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local); 1432 goto out; 1433 1434 tx_error: 1435 kfree_skb(skb); 1436 rc = NF_STOLEN; 1437 out: 1438 LeaveFunction(10); 1439 return rc; 1440 } 1441 #endif 1442