1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * IPv6 output functions 4 * Linux INET6 implementation 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 * 9 * Based on linux/net/ipv4/ip_output.c 10 * 11 * Changes: 12 * A.N.Kuznetsov : airthmetics in fragmentation. 13 * extension headers are implemented. 14 * route changes now work. 15 * ip6_forward does not confuse sniffers. 16 * etc. 17 * 18 * H. von Brand : Added missing #include <linux/string.h> 19 * Imran Patel : frag id should be in NBO 20 * Kazunori MIYAZAWA @USAGI 21 * : add ip6_append_data and related functions 22 * for datagram xmit 23 */ 24 25 #include <linux/errno.h> 26 #include <linux/kernel.h> 27 #include <linux/string.h> 28 #include <linux/socket.h> 29 #include <linux/net.h> 30 #include <linux/netdevice.h> 31 #include <linux/if_arp.h> 32 #include <linux/in6.h> 33 #include <linux/tcp.h> 34 #include <linux/route.h> 35 #include <linux/module.h> 36 #include <linux/slab.h> 37 38 #include <linux/bpf-cgroup.h> 39 #include <linux/netfilter.h> 40 #include <linux/netfilter_ipv6.h> 41 42 #include <net/sock.h> 43 #include <net/snmp.h> 44 45 #include <net/ipv6.h> 46 #include <net/ndisc.h> 47 #include <net/protocol.h> 48 #include <net/ip6_route.h> 49 #include <net/addrconf.h> 50 #include <net/rawv6.h> 51 #include <net/icmp.h> 52 #include <net/xfrm.h> 53 #include <net/checksum.h> 54 #include <linux/mroute6.h> 55 #include <net/l3mdev.h> 56 #include <net/lwtunnel.h> 57 #include <net/ip_tunnels.h> 58 59 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 60 { 61 struct dst_entry *dst = skb_dst(skb); 62 struct net_device *dev = dst->dev; 63 struct inet6_dev *idev = ip6_dst_idev(dst); 64 unsigned int hh_len = LL_RESERVED_SPACE(dev); 65 const struct in6_addr *daddr, *nexthop; 66 struct ipv6hdr *hdr; 67 struct neighbour *neigh; 68 int ret; 69 70 /* Be paranoid, rather than too clever. */ 71 if (unlikely(hh_len > skb_headroom(skb)) && dev->header_ops) { 72 skb = skb_expand_head(skb, hh_len); 73 if (!skb) { 74 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 75 return -ENOMEM; 76 } 77 } 78 79 hdr = ipv6_hdr(skb); 80 daddr = &hdr->daddr; 81 if (ipv6_addr_is_multicast(daddr)) { 82 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 83 ((mroute6_is_socket(net, skb) && 84 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || 85 ipv6_chk_mcast_addr(dev, daddr, &hdr->saddr))) { 86 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 87 88 /* Do not check for IFF_ALLMULTI; multicast routing 89 is not supported in any case. 90 */ 91 if (newskb) 92 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING, 93 net, sk, newskb, NULL, newskb->dev, 94 dev_loopback_xmit); 95 96 if (hdr->hop_limit == 0) { 97 IP6_INC_STATS(net, idev, 98 IPSTATS_MIB_OUTDISCARDS); 99 kfree_skb(skb); 100 return 0; 101 } 102 } 103 104 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len); 105 if (IPV6_ADDR_MC_SCOPE(daddr) <= IPV6_ADDR_SCOPE_NODELOCAL && 106 !(dev->flags & IFF_LOOPBACK)) { 107 kfree_skb(skb); 108 return 0; 109 } 110 } 111 112 if (lwtunnel_xmit_redirect(dst->lwtstate)) { 113 int res = lwtunnel_xmit(skb); 114 115 if (res < 0 || res == LWTUNNEL_XMIT_DONE) 116 return res; 117 } 118 119 rcu_read_lock_bh(); 120 nexthop = rt6_nexthop((struct rt6_info *)dst, daddr); 121 neigh = __ipv6_neigh_lookup_noref(dev, nexthop); 122 123 if (unlikely(IS_ERR_OR_NULL(neigh))) { 124 if (unlikely(!neigh)) 125 neigh = __neigh_create(&nd_tbl, nexthop, dev, false); 126 if (IS_ERR(neigh)) { 127 rcu_read_unlock_bh(); 128 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES); 129 kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL); 130 return -EINVAL; 131 } 132 } 133 sock_confirm_neigh(skb, neigh); 134 ret = neigh_output(neigh, skb, false); 135 rcu_read_unlock_bh(); 136 return ret; 137 } 138 139 static int 140 ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk, 141 struct sk_buff *skb, unsigned int mtu) 142 { 143 struct sk_buff *segs, *nskb; 144 netdev_features_t features; 145 int ret = 0; 146 147 /* Please see corresponding comment in ip_finish_output_gso 148 * describing the cases where GSO segment length exceeds the 149 * egress MTU. 150 */ 151 features = netif_skb_features(skb); 152 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 153 if (IS_ERR_OR_NULL(segs)) { 154 kfree_skb(skb); 155 return -ENOMEM; 156 } 157 158 consume_skb(skb); 159 160 skb_list_walk_safe(segs, segs, nskb) { 161 int err; 162 163 skb_mark_not_on_list(segs); 164 err = ip6_fragment(net, sk, segs, ip6_finish_output2); 165 if (err && ret == 0) 166 ret = err; 167 } 168 169 return ret; 170 } 171 172 static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 173 { 174 unsigned int mtu; 175 176 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) 177 /* Policy lookup after SNAT yielded a new policy */ 178 if (skb_dst(skb)->xfrm) { 179 IP6CB(skb)->flags |= IP6SKB_REROUTED; 180 return dst_output(net, sk, skb); 181 } 182 #endif 183 184 mtu = ip6_skb_dst_mtu(skb); 185 if (skb_is_gso(skb) && 186 !(IP6CB(skb)->flags & IP6SKB_FAKEJUMBO) && 187 !skb_gso_validate_network_len(skb, mtu)) 188 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu); 189 190 if ((skb->len > mtu && !skb_is_gso(skb)) || 191 dst_allfrag(skb_dst(skb)) || 192 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)) 193 return ip6_fragment(net, sk, skb, ip6_finish_output2); 194 else 195 return ip6_finish_output2(net, sk, skb); 196 } 197 198 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 199 { 200 int ret; 201 202 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 203 switch (ret) { 204 case NET_XMIT_SUCCESS: 205 case NET_XMIT_CN: 206 return __ip6_finish_output(net, sk, skb) ? : ret; 207 default: 208 kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS); 209 return ret; 210 } 211 } 212 213 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb) 214 { 215 struct net_device *dev = skb_dst(skb)->dev, *indev = skb->dev; 216 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 217 218 skb->protocol = htons(ETH_P_IPV6); 219 skb->dev = dev; 220 221 if (unlikely(idev->cnf.disable_ipv6)) { 222 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 223 kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED); 224 return 0; 225 } 226 227 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, 228 net, sk, skb, indev, dev, 229 ip6_finish_output, 230 !(IP6CB(skb)->flags & IP6SKB_REROUTED)); 231 } 232 EXPORT_SYMBOL(ip6_output); 233 234 bool ip6_autoflowlabel(struct net *net, const struct ipv6_pinfo *np) 235 { 236 if (!np->autoflowlabel_set) 237 return ip6_default_np_autolabel(net); 238 else 239 return np->autoflowlabel; 240 } 241 242 /* 243 * xmit an sk_buff (used by TCP, SCTP and DCCP) 244 * Note : socket lock is not held for SYNACK packets, but might be modified 245 * by calls to skb_set_owner_w() and ipv6_local_error(), 246 * which are using proper atomic operations or spinlocks. 247 */ 248 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, 249 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority) 250 { 251 struct net *net = sock_net(sk); 252 const struct ipv6_pinfo *np = inet6_sk(sk); 253 struct in6_addr *first_hop = &fl6->daddr; 254 struct dst_entry *dst = skb_dst(skb); 255 struct net_device *dev = dst->dev; 256 struct inet6_dev *idev = ip6_dst_idev(dst); 257 struct hop_jumbo_hdr *hop_jumbo; 258 int hoplen = sizeof(*hop_jumbo); 259 unsigned int head_room; 260 struct ipv6hdr *hdr; 261 u8 proto = fl6->flowi6_proto; 262 int seg_len = skb->len; 263 int hlimit = -1; 264 u32 mtu; 265 266 head_room = sizeof(struct ipv6hdr) + hoplen + LL_RESERVED_SPACE(dev); 267 if (opt) 268 head_room += opt->opt_nflen + opt->opt_flen; 269 270 if (unlikely(head_room > skb_headroom(skb))) { 271 skb = skb_expand_head(skb, head_room); 272 if (!skb) { 273 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 274 return -ENOBUFS; 275 } 276 } 277 278 if (opt) { 279 seg_len += opt->opt_nflen + opt->opt_flen; 280 281 if (opt->opt_flen) 282 ipv6_push_frag_opts(skb, opt, &proto); 283 284 if (opt->opt_nflen) 285 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop, 286 &fl6->saddr); 287 } 288 289 if (unlikely(seg_len > IPV6_MAXPLEN)) { 290 hop_jumbo = skb_push(skb, hoplen); 291 292 hop_jumbo->nexthdr = proto; 293 hop_jumbo->hdrlen = 0; 294 hop_jumbo->tlv_type = IPV6_TLV_JUMBO; 295 hop_jumbo->tlv_len = 4; 296 hop_jumbo->jumbo_payload_len = htonl(seg_len + hoplen); 297 298 proto = IPPROTO_HOPOPTS; 299 seg_len = 0; 300 IP6CB(skb)->flags |= IP6SKB_FAKEJUMBO; 301 } 302 303 skb_push(skb, sizeof(struct ipv6hdr)); 304 skb_reset_network_header(skb); 305 hdr = ipv6_hdr(skb); 306 307 /* 308 * Fill in the IPv6 header 309 */ 310 if (np) 311 hlimit = np->hop_limit; 312 if (hlimit < 0) 313 hlimit = ip6_dst_hoplimit(dst); 314 315 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel, 316 ip6_autoflowlabel(net, np), fl6)); 317 318 hdr->payload_len = htons(seg_len); 319 hdr->nexthdr = proto; 320 hdr->hop_limit = hlimit; 321 322 hdr->saddr = fl6->saddr; 323 hdr->daddr = *first_hop; 324 325 skb->protocol = htons(ETH_P_IPV6); 326 skb->priority = priority; 327 skb->mark = mark; 328 329 mtu = dst_mtu(dst); 330 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) { 331 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len); 332 333 /* if egress device is enslaved to an L3 master device pass the 334 * skb to its handler for processing 335 */ 336 skb = l3mdev_ip6_out((struct sock *)sk, skb); 337 if (unlikely(!skb)) 338 return 0; 339 340 /* hooks should never assume socket lock is held. 341 * we promote our socket to non const 342 */ 343 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, 344 net, (struct sock *)sk, skb, NULL, dev, 345 dst_output); 346 } 347 348 skb->dev = dev; 349 /* ipv6_local_error() does not require socket lock, 350 * we promote our socket to non const 351 */ 352 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 353 354 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS); 355 kfree_skb(skb); 356 return -EMSGSIZE; 357 } 358 EXPORT_SYMBOL(ip6_xmit); 359 360 static int ip6_call_ra_chain(struct sk_buff *skb, int sel) 361 { 362 struct ip6_ra_chain *ra; 363 struct sock *last = NULL; 364 365 read_lock(&ip6_ra_lock); 366 for (ra = ip6_ra_chain; ra; ra = ra->next) { 367 struct sock *sk = ra->sk; 368 if (sk && ra->sel == sel && 369 (!sk->sk_bound_dev_if || 370 sk->sk_bound_dev_if == skb->dev->ifindex)) { 371 struct ipv6_pinfo *np = inet6_sk(sk); 372 373 if (np && np->rtalert_isolate && 374 !net_eq(sock_net(sk), dev_net(skb->dev))) { 375 continue; 376 } 377 if (last) { 378 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 379 if (skb2) 380 rawv6_rcv(last, skb2); 381 } 382 last = sk; 383 } 384 } 385 386 if (last) { 387 rawv6_rcv(last, skb); 388 read_unlock(&ip6_ra_lock); 389 return 1; 390 } 391 read_unlock(&ip6_ra_lock); 392 return 0; 393 } 394 395 static int ip6_forward_proxy_check(struct sk_buff *skb) 396 { 397 struct ipv6hdr *hdr = ipv6_hdr(skb); 398 u8 nexthdr = hdr->nexthdr; 399 __be16 frag_off; 400 int offset; 401 402 if (ipv6_ext_hdr(nexthdr)) { 403 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off); 404 if (offset < 0) 405 return 0; 406 } else 407 offset = sizeof(struct ipv6hdr); 408 409 if (nexthdr == IPPROTO_ICMPV6) { 410 struct icmp6hdr *icmp6; 411 412 if (!pskb_may_pull(skb, (skb_network_header(skb) + 413 offset + 1 - skb->data))) 414 return 0; 415 416 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset); 417 418 switch (icmp6->icmp6_type) { 419 case NDISC_ROUTER_SOLICITATION: 420 case NDISC_ROUTER_ADVERTISEMENT: 421 case NDISC_NEIGHBOUR_SOLICITATION: 422 case NDISC_NEIGHBOUR_ADVERTISEMENT: 423 case NDISC_REDIRECT: 424 /* For reaction involving unicast neighbor discovery 425 * message destined to the proxied address, pass it to 426 * input function. 427 */ 428 return 1; 429 default: 430 break; 431 } 432 } 433 434 /* 435 * The proxying router can't forward traffic sent to a link-local 436 * address, so signal the sender and discard the packet. This 437 * behavior is clarified by the MIPv6 specification. 438 */ 439 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) { 440 dst_link_failure(skb); 441 return -1; 442 } 443 444 return 0; 445 } 446 447 static inline int ip6_forward_finish(struct net *net, struct sock *sk, 448 struct sk_buff *skb) 449 { 450 struct dst_entry *dst = skb_dst(skb); 451 452 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS); 453 __IP6_ADD_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len); 454 455 #ifdef CONFIG_NET_SWITCHDEV 456 if (skb->offload_l3_fwd_mark) { 457 consume_skb(skb); 458 return 0; 459 } 460 #endif 461 462 skb_clear_tstamp(skb); 463 return dst_output(net, sk, skb); 464 } 465 466 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) 467 { 468 if (skb->len <= mtu) 469 return false; 470 471 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */ 472 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu) 473 return true; 474 475 if (skb->ignore_df) 476 return false; 477 478 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 479 return false; 480 481 return true; 482 } 483 484 int ip6_forward(struct sk_buff *skb) 485 { 486 struct dst_entry *dst = skb_dst(skb); 487 struct ipv6hdr *hdr = ipv6_hdr(skb); 488 struct inet6_skb_parm *opt = IP6CB(skb); 489 struct net *net = dev_net(dst->dev); 490 struct inet6_dev *idev; 491 SKB_DR(reason); 492 u32 mtu; 493 494 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 495 if (net->ipv6.devconf_all->forwarding == 0) 496 goto error; 497 498 if (skb->pkt_type != PACKET_HOST) 499 goto drop; 500 501 if (unlikely(skb->sk)) 502 goto drop; 503 504 if (skb_warn_if_lro(skb)) 505 goto drop; 506 507 if (!net->ipv6.devconf_all->disable_policy && 508 (!idev || !idev->cnf.disable_policy) && 509 !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) { 510 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 511 goto drop; 512 } 513 514 skb_forward_csum(skb); 515 516 /* 517 * We DO NOT make any processing on 518 * RA packets, pushing them to user level AS IS 519 * without ane WARRANTY that application will be able 520 * to interpret them. The reason is that we 521 * cannot make anything clever here. 522 * 523 * We are not end-node, so that if packet contains 524 * AH/ESP, we cannot make anything. 525 * Defragmentation also would be mistake, RA packets 526 * cannot be fragmented, because there is no warranty 527 * that different fragments will go along one path. --ANK 528 */ 529 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) { 530 if (ip6_call_ra_chain(skb, ntohs(opt->ra))) 531 return 0; 532 } 533 534 /* 535 * check and decrement ttl 536 */ 537 if (hdr->hop_limit <= 1) { 538 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); 539 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); 540 541 kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR); 542 return -ETIMEDOUT; 543 } 544 545 /* XXX: idev->cnf.proxy_ndp? */ 546 if (net->ipv6.devconf_all->proxy_ndp && 547 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) { 548 int proxied = ip6_forward_proxy_check(skb); 549 if (proxied > 0) { 550 hdr->hop_limit--; 551 return ip6_input(skb); 552 } else if (proxied < 0) { 553 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 554 goto drop; 555 } 556 } 557 558 if (!xfrm6_route_forward(skb)) { 559 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 560 SKB_DR_SET(reason, XFRM_POLICY); 561 goto drop; 562 } 563 dst = skb_dst(skb); 564 565 /* IPv6 specs say nothing about it, but it is clear that we cannot 566 send redirects to source routed frames. 567 We don't send redirects to frames decapsulated from IPsec. 568 */ 569 if (IP6CB(skb)->iif == dst->dev->ifindex && 570 opt->srcrt == 0 && !skb_sec_path(skb)) { 571 struct in6_addr *target = NULL; 572 struct inet_peer *peer; 573 struct rt6_info *rt; 574 575 /* 576 * incoming and outgoing devices are the same 577 * send a redirect. 578 */ 579 580 rt = (struct rt6_info *) dst; 581 if (rt->rt6i_flags & RTF_GATEWAY) 582 target = &rt->rt6i_gateway; 583 else 584 target = &hdr->daddr; 585 586 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1); 587 588 /* Limit redirects both by destination (here) 589 and by source (inside ndisc_send_redirect) 590 */ 591 if (inet_peer_xrlim_allow(peer, 1*HZ)) 592 ndisc_send_redirect(skb, target); 593 if (peer) 594 inet_putpeer(peer); 595 } else { 596 int addrtype = ipv6_addr_type(&hdr->saddr); 597 598 /* This check is security critical. */ 599 if (addrtype == IPV6_ADDR_ANY || 600 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK)) 601 goto error; 602 if (addrtype & IPV6_ADDR_LINKLOCAL) { 603 icmpv6_send(skb, ICMPV6_DEST_UNREACH, 604 ICMPV6_NOT_NEIGHBOUR, 0); 605 goto error; 606 } 607 } 608 609 mtu = ip6_dst_mtu_maybe_forward(dst, true); 610 if (mtu < IPV6_MIN_MTU) 611 mtu = IPV6_MIN_MTU; 612 613 if (ip6_pkt_too_big(skb, mtu)) { 614 /* Again, force OUTPUT device used as source address */ 615 skb->dev = dst->dev; 616 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 617 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS); 618 __IP6_INC_STATS(net, ip6_dst_idev(dst), 619 IPSTATS_MIB_FRAGFAILS); 620 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 621 return -EMSGSIZE; 622 } 623 624 if (skb_cow(skb, dst->dev->hard_header_len)) { 625 __IP6_INC_STATS(net, ip6_dst_idev(dst), 626 IPSTATS_MIB_OUTDISCARDS); 627 goto drop; 628 } 629 630 hdr = ipv6_hdr(skb); 631 632 /* Mangling hops number delayed to point after skb COW */ 633 634 hdr->hop_limit--; 635 636 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, 637 net, NULL, skb, skb->dev, dst->dev, 638 ip6_forward_finish); 639 640 error: 641 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 642 SKB_DR_SET(reason, IP_INADDRERRORS); 643 drop: 644 kfree_skb_reason(skb, reason); 645 return -EINVAL; 646 } 647 648 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from) 649 { 650 to->pkt_type = from->pkt_type; 651 to->priority = from->priority; 652 to->protocol = from->protocol; 653 skb_dst_drop(to); 654 skb_dst_set(to, dst_clone(skb_dst(from))); 655 to->dev = from->dev; 656 to->mark = from->mark; 657 658 skb_copy_hash(to, from); 659 660 #ifdef CONFIG_NET_SCHED 661 to->tc_index = from->tc_index; 662 #endif 663 nf_copy(to, from); 664 skb_ext_copy(to, from); 665 skb_copy_secmark(to, from); 666 } 667 668 int ip6_fraglist_init(struct sk_buff *skb, unsigned int hlen, u8 *prevhdr, 669 u8 nexthdr, __be32 frag_id, 670 struct ip6_fraglist_iter *iter) 671 { 672 unsigned int first_len; 673 struct frag_hdr *fh; 674 675 /* BUILD HEADER */ 676 *prevhdr = NEXTHDR_FRAGMENT; 677 iter->tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); 678 if (!iter->tmp_hdr) 679 return -ENOMEM; 680 681 iter->frag = skb_shinfo(skb)->frag_list; 682 skb_frag_list_init(skb); 683 684 iter->offset = 0; 685 iter->hlen = hlen; 686 iter->frag_id = frag_id; 687 iter->nexthdr = nexthdr; 688 689 __skb_pull(skb, hlen); 690 fh = __skb_push(skb, sizeof(struct frag_hdr)); 691 __skb_push(skb, hlen); 692 skb_reset_network_header(skb); 693 memcpy(skb_network_header(skb), iter->tmp_hdr, hlen); 694 695 fh->nexthdr = nexthdr; 696 fh->reserved = 0; 697 fh->frag_off = htons(IP6_MF); 698 fh->identification = frag_id; 699 700 first_len = skb_pagelen(skb); 701 skb->data_len = first_len - skb_headlen(skb); 702 skb->len = first_len; 703 ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr)); 704 705 return 0; 706 } 707 EXPORT_SYMBOL(ip6_fraglist_init); 708 709 void ip6_fraglist_prepare(struct sk_buff *skb, 710 struct ip6_fraglist_iter *iter) 711 { 712 struct sk_buff *frag = iter->frag; 713 unsigned int hlen = iter->hlen; 714 struct frag_hdr *fh; 715 716 frag->ip_summed = CHECKSUM_NONE; 717 skb_reset_transport_header(frag); 718 fh = __skb_push(frag, sizeof(struct frag_hdr)); 719 __skb_push(frag, hlen); 720 skb_reset_network_header(frag); 721 memcpy(skb_network_header(frag), iter->tmp_hdr, hlen); 722 iter->offset += skb->len - hlen - sizeof(struct frag_hdr); 723 fh->nexthdr = iter->nexthdr; 724 fh->reserved = 0; 725 fh->frag_off = htons(iter->offset); 726 if (frag->next) 727 fh->frag_off |= htons(IP6_MF); 728 fh->identification = iter->frag_id; 729 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); 730 ip6_copy_metadata(frag, skb); 731 } 732 EXPORT_SYMBOL(ip6_fraglist_prepare); 733 734 void ip6_frag_init(struct sk_buff *skb, unsigned int hlen, unsigned int mtu, 735 unsigned short needed_tailroom, int hdr_room, u8 *prevhdr, 736 u8 nexthdr, __be32 frag_id, struct ip6_frag_state *state) 737 { 738 state->prevhdr = prevhdr; 739 state->nexthdr = nexthdr; 740 state->frag_id = frag_id; 741 742 state->hlen = hlen; 743 state->mtu = mtu; 744 745 state->left = skb->len - hlen; /* Space per frame */ 746 state->ptr = hlen; /* Where to start from */ 747 748 state->hroom = hdr_room; 749 state->troom = needed_tailroom; 750 751 state->offset = 0; 752 } 753 EXPORT_SYMBOL(ip6_frag_init); 754 755 struct sk_buff *ip6_frag_next(struct sk_buff *skb, struct ip6_frag_state *state) 756 { 757 u8 *prevhdr = state->prevhdr, *fragnexthdr_offset; 758 struct sk_buff *frag; 759 struct frag_hdr *fh; 760 unsigned int len; 761 762 len = state->left; 763 /* IF: it doesn't fit, use 'mtu' - the data space left */ 764 if (len > state->mtu) 765 len = state->mtu; 766 /* IF: we are not sending up to and including the packet end 767 then align the next start on an eight byte boundary */ 768 if (len < state->left) 769 len &= ~7; 770 771 /* Allocate buffer */ 772 frag = alloc_skb(len + state->hlen + sizeof(struct frag_hdr) + 773 state->hroom + state->troom, GFP_ATOMIC); 774 if (!frag) 775 return ERR_PTR(-ENOMEM); 776 777 /* 778 * Set up data on packet 779 */ 780 781 ip6_copy_metadata(frag, skb); 782 skb_reserve(frag, state->hroom); 783 skb_put(frag, len + state->hlen + sizeof(struct frag_hdr)); 784 skb_reset_network_header(frag); 785 fh = (struct frag_hdr *)(skb_network_header(frag) + state->hlen); 786 frag->transport_header = (frag->network_header + state->hlen + 787 sizeof(struct frag_hdr)); 788 789 /* 790 * Charge the memory for the fragment to any owner 791 * it might possess 792 */ 793 if (skb->sk) 794 skb_set_owner_w(frag, skb->sk); 795 796 /* 797 * Copy the packet header into the new buffer. 798 */ 799 skb_copy_from_linear_data(skb, skb_network_header(frag), state->hlen); 800 801 fragnexthdr_offset = skb_network_header(frag); 802 fragnexthdr_offset += prevhdr - skb_network_header(skb); 803 *fragnexthdr_offset = NEXTHDR_FRAGMENT; 804 805 /* 806 * Build fragment header. 807 */ 808 fh->nexthdr = state->nexthdr; 809 fh->reserved = 0; 810 fh->identification = state->frag_id; 811 812 /* 813 * Copy a block of the IP datagram. 814 */ 815 BUG_ON(skb_copy_bits(skb, state->ptr, skb_transport_header(frag), 816 len)); 817 state->left -= len; 818 819 fh->frag_off = htons(state->offset); 820 if (state->left > 0) 821 fh->frag_off |= htons(IP6_MF); 822 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr)); 823 824 state->ptr += len; 825 state->offset += len; 826 827 return frag; 828 } 829 EXPORT_SYMBOL(ip6_frag_next); 830 831 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 832 int (*output)(struct net *, struct sock *, struct sk_buff *)) 833 { 834 struct sk_buff *frag; 835 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 836 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? 837 inet6_sk(skb->sk) : NULL; 838 bool mono_delivery_time = skb->mono_delivery_time; 839 struct ip6_frag_state state; 840 unsigned int mtu, hlen, nexthdr_offset; 841 ktime_t tstamp = skb->tstamp; 842 int hroom, err = 0; 843 __be32 frag_id; 844 u8 *prevhdr, nexthdr = 0; 845 846 err = ip6_find_1stfragopt(skb, &prevhdr); 847 if (err < 0) 848 goto fail; 849 hlen = err; 850 nexthdr = *prevhdr; 851 nexthdr_offset = prevhdr - skb_network_header(skb); 852 853 mtu = ip6_skb_dst_mtu(skb); 854 855 /* We must not fragment if the socket is set to force MTU discovery 856 * or if the skb it not generated by a local socket. 857 */ 858 if (unlikely(!skb->ignore_df && skb->len > mtu)) 859 goto fail_toobig; 860 861 if (IP6CB(skb)->frag_max_size) { 862 if (IP6CB(skb)->frag_max_size > mtu) 863 goto fail_toobig; 864 865 /* don't send fragments larger than what we received */ 866 mtu = IP6CB(skb)->frag_max_size; 867 if (mtu < IPV6_MIN_MTU) 868 mtu = IPV6_MIN_MTU; 869 } 870 871 if (np && np->frag_size < mtu) { 872 if (np->frag_size) 873 mtu = np->frag_size; 874 } 875 if (mtu < hlen + sizeof(struct frag_hdr) + 8) 876 goto fail_toobig; 877 mtu -= hlen + sizeof(struct frag_hdr); 878 879 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr, 880 &ipv6_hdr(skb)->saddr); 881 882 if (skb->ip_summed == CHECKSUM_PARTIAL && 883 (err = skb_checksum_help(skb))) 884 goto fail; 885 886 prevhdr = skb_network_header(skb) + nexthdr_offset; 887 hroom = LL_RESERVED_SPACE(rt->dst.dev); 888 if (skb_has_frag_list(skb)) { 889 unsigned int first_len = skb_pagelen(skb); 890 struct ip6_fraglist_iter iter; 891 struct sk_buff *frag2; 892 893 if (first_len - hlen > mtu || 894 ((first_len - hlen) & 7) || 895 skb_cloned(skb) || 896 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) 897 goto slow_path; 898 899 skb_walk_frags(skb, frag) { 900 /* Correct geometry. */ 901 if (frag->len > mtu || 902 ((frag->len & 7) && frag->next) || 903 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr))) 904 goto slow_path_clean; 905 906 /* Partially cloned skb? */ 907 if (skb_shared(frag)) 908 goto slow_path_clean; 909 910 BUG_ON(frag->sk); 911 if (skb->sk) { 912 frag->sk = skb->sk; 913 frag->destructor = sock_wfree; 914 } 915 skb->truesize -= frag->truesize; 916 } 917 918 err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id, 919 &iter); 920 if (err < 0) 921 goto fail; 922 923 /* We prevent @rt from being freed. */ 924 rcu_read_lock(); 925 926 for (;;) { 927 /* Prepare header of the next frame, 928 * before previous one went down. */ 929 if (iter.frag) 930 ip6_fraglist_prepare(skb, &iter); 931 932 skb_set_delivery_time(skb, tstamp, mono_delivery_time); 933 err = output(net, sk, skb); 934 if (!err) 935 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 936 IPSTATS_MIB_FRAGCREATES); 937 938 if (err || !iter.frag) 939 break; 940 941 skb = ip6_fraglist_next(&iter); 942 } 943 944 kfree(iter.tmp_hdr); 945 946 if (err == 0) { 947 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 948 IPSTATS_MIB_FRAGOKS); 949 rcu_read_unlock(); 950 return 0; 951 } 952 953 kfree_skb_list(iter.frag); 954 955 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 956 IPSTATS_MIB_FRAGFAILS); 957 rcu_read_unlock(); 958 return err; 959 960 slow_path_clean: 961 skb_walk_frags(skb, frag2) { 962 if (frag2 == frag) 963 break; 964 frag2->sk = NULL; 965 frag2->destructor = NULL; 966 skb->truesize += frag2->truesize; 967 } 968 } 969 970 slow_path: 971 /* 972 * Fragment the datagram. 973 */ 974 975 ip6_frag_init(skb, hlen, mtu, rt->dst.dev->needed_tailroom, 976 LL_RESERVED_SPACE(rt->dst.dev), prevhdr, nexthdr, frag_id, 977 &state); 978 979 /* 980 * Keep copying data until we run out. 981 */ 982 983 while (state.left > 0) { 984 frag = ip6_frag_next(skb, &state); 985 if (IS_ERR(frag)) { 986 err = PTR_ERR(frag); 987 goto fail; 988 } 989 990 /* 991 * Put this fragment into the sending queue. 992 */ 993 skb_set_delivery_time(frag, tstamp, mono_delivery_time); 994 err = output(net, sk, frag); 995 if (err) 996 goto fail; 997 998 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 999 IPSTATS_MIB_FRAGCREATES); 1000 } 1001 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1002 IPSTATS_MIB_FRAGOKS); 1003 consume_skb(skb); 1004 return err; 1005 1006 fail_toobig: 1007 if (skb->sk && dst_allfrag(skb_dst(skb))) 1008 sk_gso_disable(skb->sk); 1009 1010 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 1011 err = -EMSGSIZE; 1012 1013 fail: 1014 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 1015 IPSTATS_MIB_FRAGFAILS); 1016 kfree_skb(skb); 1017 return err; 1018 } 1019 1020 static inline int ip6_rt_check(const struct rt6key *rt_key, 1021 const struct in6_addr *fl_addr, 1022 const struct in6_addr *addr_cache) 1023 { 1024 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) && 1025 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache)); 1026 } 1027 1028 static struct dst_entry *ip6_sk_dst_check(struct sock *sk, 1029 struct dst_entry *dst, 1030 const struct flowi6 *fl6) 1031 { 1032 struct ipv6_pinfo *np = inet6_sk(sk); 1033 struct rt6_info *rt; 1034 1035 if (!dst) 1036 goto out; 1037 1038 if (dst->ops->family != AF_INET6) { 1039 dst_release(dst); 1040 return NULL; 1041 } 1042 1043 rt = (struct rt6_info *)dst; 1044 /* Yes, checking route validity in not connected 1045 * case is not very simple. Take into account, 1046 * that we do not support routing by source, TOS, 1047 * and MSG_DONTROUTE --ANK (980726) 1048 * 1049 * 1. ip6_rt_check(): If route was host route, 1050 * check that cached destination is current. 1051 * If it is network route, we still may 1052 * check its validity using saved pointer 1053 * to the last used address: daddr_cache. 1054 * We do not want to save whole address now, 1055 * (because main consumer of this service 1056 * is tcp, which has not this problem), 1057 * so that the last trick works only on connected 1058 * sockets. 1059 * 2. oif also should be the same. 1060 */ 1061 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) || 1062 #ifdef CONFIG_IPV6_SUBTREES 1063 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) || 1064 #endif 1065 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex)) { 1066 dst_release(dst); 1067 dst = NULL; 1068 } 1069 1070 out: 1071 return dst; 1072 } 1073 1074 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk, 1075 struct dst_entry **dst, struct flowi6 *fl6) 1076 { 1077 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1078 struct neighbour *n; 1079 struct rt6_info *rt; 1080 #endif 1081 int err; 1082 int flags = 0; 1083 1084 /* The correct way to handle this would be to do 1085 * ip6_route_get_saddr, and then ip6_route_output; however, 1086 * the route-specific preferred source forces the 1087 * ip6_route_output call _before_ ip6_route_get_saddr. 1088 * 1089 * In source specific routing (no src=any default route), 1090 * ip6_route_output will fail given src=any saddr, though, so 1091 * that's why we try it again later. 1092 */ 1093 if (ipv6_addr_any(&fl6->saddr)) { 1094 struct fib6_info *from; 1095 struct rt6_info *rt; 1096 1097 *dst = ip6_route_output(net, sk, fl6); 1098 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst; 1099 1100 rcu_read_lock(); 1101 from = rt ? rcu_dereference(rt->from) : NULL; 1102 err = ip6_route_get_saddr(net, from, &fl6->daddr, 1103 sk ? inet6_sk(sk)->srcprefs : 0, 1104 &fl6->saddr); 1105 rcu_read_unlock(); 1106 1107 if (err) 1108 goto out_err_release; 1109 1110 /* If we had an erroneous initial result, pretend it 1111 * never existed and let the SA-enabled version take 1112 * over. 1113 */ 1114 if ((*dst)->error) { 1115 dst_release(*dst); 1116 *dst = NULL; 1117 } 1118 1119 if (fl6->flowi6_oif) 1120 flags |= RT6_LOOKUP_F_IFACE; 1121 } 1122 1123 if (!*dst) 1124 *dst = ip6_route_output_flags(net, sk, fl6, flags); 1125 1126 err = (*dst)->error; 1127 if (err) 1128 goto out_err_release; 1129 1130 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1131 /* 1132 * Here if the dst entry we've looked up 1133 * has a neighbour entry that is in the INCOMPLETE 1134 * state and the src address from the flow is 1135 * marked as OPTIMISTIC, we release the found 1136 * dst entry and replace it instead with the 1137 * dst entry of the nexthop router 1138 */ 1139 rt = (struct rt6_info *) *dst; 1140 rcu_read_lock_bh(); 1141 n = __ipv6_neigh_lookup_noref(rt->dst.dev, 1142 rt6_nexthop(rt, &fl6->daddr)); 1143 err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0; 1144 rcu_read_unlock_bh(); 1145 1146 if (err) { 1147 struct inet6_ifaddr *ifp; 1148 struct flowi6 fl_gw6; 1149 int redirect; 1150 1151 ifp = ipv6_get_ifaddr(net, &fl6->saddr, 1152 (*dst)->dev, 1); 1153 1154 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC); 1155 if (ifp) 1156 in6_ifa_put(ifp); 1157 1158 if (redirect) { 1159 /* 1160 * We need to get the dst entry for the 1161 * default router instead 1162 */ 1163 dst_release(*dst); 1164 memcpy(&fl_gw6, fl6, sizeof(struct flowi6)); 1165 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr)); 1166 *dst = ip6_route_output(net, sk, &fl_gw6); 1167 err = (*dst)->error; 1168 if (err) 1169 goto out_err_release; 1170 } 1171 } 1172 #endif 1173 if (ipv6_addr_v4mapped(&fl6->saddr) && 1174 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) { 1175 err = -EAFNOSUPPORT; 1176 goto out_err_release; 1177 } 1178 1179 return 0; 1180 1181 out_err_release: 1182 dst_release(*dst); 1183 *dst = NULL; 1184 1185 if (err == -ENETUNREACH) 1186 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); 1187 return err; 1188 } 1189 1190 /** 1191 * ip6_dst_lookup - perform route lookup on flow 1192 * @net: Network namespace to perform lookup in 1193 * @sk: socket which provides route info 1194 * @dst: pointer to dst_entry * for result 1195 * @fl6: flow to lookup 1196 * 1197 * This function performs a route lookup on the given flow. 1198 * 1199 * It returns zero on success, or a standard errno code on error. 1200 */ 1201 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, 1202 struct flowi6 *fl6) 1203 { 1204 *dst = NULL; 1205 return ip6_dst_lookup_tail(net, sk, dst, fl6); 1206 } 1207 EXPORT_SYMBOL_GPL(ip6_dst_lookup); 1208 1209 /** 1210 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec 1211 * @net: Network namespace to perform lookup in 1212 * @sk: socket which provides route info 1213 * @fl6: flow to lookup 1214 * @final_dst: final destination address for ipsec lookup 1215 * 1216 * This function performs a route lookup on the given flow. 1217 * 1218 * It returns a valid dst pointer on success, or a pointer encoded 1219 * error code. 1220 */ 1221 struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6, 1222 const struct in6_addr *final_dst) 1223 { 1224 struct dst_entry *dst = NULL; 1225 int err; 1226 1227 err = ip6_dst_lookup_tail(net, sk, &dst, fl6); 1228 if (err) 1229 return ERR_PTR(err); 1230 if (final_dst) 1231 fl6->daddr = *final_dst; 1232 1233 return xfrm_lookup_route(net, dst, flowi6_to_flowi(fl6), sk, 0); 1234 } 1235 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow); 1236 1237 /** 1238 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow 1239 * @sk: socket which provides the dst cache and route info 1240 * @fl6: flow to lookup 1241 * @final_dst: final destination address for ipsec lookup 1242 * @connected: whether @sk is connected or not 1243 * 1244 * This function performs a route lookup on the given flow with the 1245 * possibility of using the cached route in the socket if it is valid. 1246 * It will take the socket dst lock when operating on the dst cache. 1247 * As a result, this function can only be used in process context. 1248 * 1249 * In addition, for a connected socket, cache the dst in the socket 1250 * if the current cache is not valid. 1251 * 1252 * It returns a valid dst pointer on success, or a pointer encoded 1253 * error code. 1254 */ 1255 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, 1256 const struct in6_addr *final_dst, 1257 bool connected) 1258 { 1259 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie); 1260 1261 dst = ip6_sk_dst_check(sk, dst, fl6); 1262 if (dst) 1263 return dst; 1264 1265 dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_dst); 1266 if (connected && !IS_ERR(dst)) 1267 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6); 1268 1269 return dst; 1270 } 1271 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow); 1272 1273 /** 1274 * ip6_dst_lookup_tunnel - perform route lookup on tunnel 1275 * @skb: Packet for which lookup is done 1276 * @dev: Tunnel device 1277 * @net: Network namespace of tunnel device 1278 * @sock: Socket which provides route info 1279 * @saddr: Memory to store the src ip address 1280 * @info: Tunnel information 1281 * @protocol: IP protocol 1282 * @use_cache: Flag to enable cache usage 1283 * This function performs a route lookup on a tunnel 1284 * 1285 * It returns a valid dst pointer and stores src address to be used in 1286 * tunnel in param saddr on success, else a pointer encoded error code. 1287 */ 1288 1289 struct dst_entry *ip6_dst_lookup_tunnel(struct sk_buff *skb, 1290 struct net_device *dev, 1291 struct net *net, 1292 struct socket *sock, 1293 struct in6_addr *saddr, 1294 const struct ip_tunnel_info *info, 1295 u8 protocol, 1296 bool use_cache) 1297 { 1298 struct dst_entry *dst = NULL; 1299 #ifdef CONFIG_DST_CACHE 1300 struct dst_cache *dst_cache; 1301 #endif 1302 struct flowi6 fl6; 1303 __u8 prio; 1304 1305 #ifdef CONFIG_DST_CACHE 1306 dst_cache = (struct dst_cache *)&info->dst_cache; 1307 if (use_cache) { 1308 dst = dst_cache_get_ip6(dst_cache, saddr); 1309 if (dst) 1310 return dst; 1311 } 1312 #endif 1313 memset(&fl6, 0, sizeof(fl6)); 1314 fl6.flowi6_mark = skb->mark; 1315 fl6.flowi6_proto = protocol; 1316 fl6.daddr = info->key.u.ipv6.dst; 1317 fl6.saddr = info->key.u.ipv6.src; 1318 prio = info->key.tos; 1319 fl6.flowlabel = ip6_make_flowinfo(prio, info->key.label); 1320 1321 dst = ipv6_stub->ipv6_dst_lookup_flow(net, sock->sk, &fl6, 1322 NULL); 1323 if (IS_ERR(dst)) { 1324 netdev_dbg(dev, "no route to %pI6\n", &fl6.daddr); 1325 return ERR_PTR(-ENETUNREACH); 1326 } 1327 if (dst->dev == dev) { /* is this necessary? */ 1328 netdev_dbg(dev, "circular route to %pI6\n", &fl6.daddr); 1329 dst_release(dst); 1330 return ERR_PTR(-ELOOP); 1331 } 1332 #ifdef CONFIG_DST_CACHE 1333 if (use_cache) 1334 dst_cache_set_ip6(dst_cache, dst, &fl6.saddr); 1335 #endif 1336 *saddr = fl6.saddr; 1337 return dst; 1338 } 1339 EXPORT_SYMBOL_GPL(ip6_dst_lookup_tunnel); 1340 1341 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src, 1342 gfp_t gfp) 1343 { 1344 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1345 } 1346 1347 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src, 1348 gfp_t gfp) 1349 { 1350 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1351 } 1352 1353 static void ip6_append_data_mtu(unsigned int *mtu, 1354 int *maxfraglen, 1355 unsigned int fragheaderlen, 1356 struct sk_buff *skb, 1357 struct rt6_info *rt, 1358 unsigned int orig_mtu) 1359 { 1360 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) { 1361 if (!skb) { 1362 /* first fragment, reserve header_len */ 1363 *mtu = orig_mtu - rt->dst.header_len; 1364 1365 } else { 1366 /* 1367 * this fragment is not first, the headers 1368 * space is regarded as data space. 1369 */ 1370 *mtu = orig_mtu; 1371 } 1372 *maxfraglen = ((*mtu - fragheaderlen) & ~7) 1373 + fragheaderlen - sizeof(struct frag_hdr); 1374 } 1375 } 1376 1377 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1378 struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6, 1379 struct rt6_info *rt) 1380 { 1381 struct ipv6_pinfo *np = inet6_sk(sk); 1382 unsigned int mtu; 1383 struct ipv6_txoptions *nopt, *opt = ipc6->opt; 1384 1385 /* callers pass dst together with a reference, set it first so 1386 * ip6_cork_release() can put it down even in case of an error. 1387 */ 1388 cork->base.dst = &rt->dst; 1389 1390 /* 1391 * setup for corking 1392 */ 1393 if (opt) { 1394 if (WARN_ON(v6_cork->opt)) 1395 return -EINVAL; 1396 1397 nopt = v6_cork->opt = kzalloc(sizeof(*opt), sk->sk_allocation); 1398 if (unlikely(!nopt)) 1399 return -ENOBUFS; 1400 1401 nopt->tot_len = sizeof(*opt); 1402 nopt->opt_flen = opt->opt_flen; 1403 nopt->opt_nflen = opt->opt_nflen; 1404 1405 nopt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation); 1406 if (opt->dst0opt && !nopt->dst0opt) 1407 return -ENOBUFS; 1408 1409 nopt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation); 1410 if (opt->dst1opt && !nopt->dst1opt) 1411 return -ENOBUFS; 1412 1413 nopt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation); 1414 if (opt->hopopt && !nopt->hopopt) 1415 return -ENOBUFS; 1416 1417 nopt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation); 1418 if (opt->srcrt && !nopt->srcrt) 1419 return -ENOBUFS; 1420 1421 /* need source address above miyazawa*/ 1422 } 1423 v6_cork->hop_limit = ipc6->hlimit; 1424 v6_cork->tclass = ipc6->tclass; 1425 if (rt->dst.flags & DST_XFRM_TUNNEL) 1426 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1427 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst); 1428 else 1429 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1430 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(xfrm_dst_path(&rt->dst)); 1431 if (np->frag_size < mtu) { 1432 if (np->frag_size) 1433 mtu = np->frag_size; 1434 } 1435 cork->base.fragsize = mtu; 1436 cork->base.gso_size = ipc6->gso_size; 1437 cork->base.tx_flags = 0; 1438 cork->base.mark = ipc6->sockc.mark; 1439 sock_tx_timestamp(sk, ipc6->sockc.tsflags, &cork->base.tx_flags); 1440 1441 if (dst_allfrag(xfrm_dst_path(&rt->dst))) 1442 cork->base.flags |= IPCORK_ALLFRAG; 1443 cork->base.length = 0; 1444 1445 cork->base.transmit_time = ipc6->sockc.transmit_time; 1446 1447 return 0; 1448 } 1449 1450 static int __ip6_append_data(struct sock *sk, 1451 struct sk_buff_head *queue, 1452 struct inet_cork_full *cork_full, 1453 struct inet6_cork *v6_cork, 1454 struct page_frag *pfrag, 1455 int getfrag(void *from, char *to, int offset, 1456 int len, int odd, struct sk_buff *skb), 1457 void *from, size_t length, int transhdrlen, 1458 unsigned int flags, struct ipcm6_cookie *ipc6) 1459 { 1460 struct sk_buff *skb, *skb_prev = NULL; 1461 struct inet_cork *cork = &cork_full->base; 1462 struct flowi6 *fl6 = &cork_full->fl.u.ip6; 1463 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu; 1464 struct ubuf_info *uarg = NULL; 1465 int exthdrlen = 0; 1466 int dst_exthdrlen = 0; 1467 int hh_len; 1468 int copy; 1469 int err; 1470 int offset = 0; 1471 bool zc = false; 1472 u32 tskey = 0; 1473 struct rt6_info *rt = (struct rt6_info *)cork->dst; 1474 struct ipv6_txoptions *opt = v6_cork->opt; 1475 int csummode = CHECKSUM_NONE; 1476 unsigned int maxnonfragsize, headersize; 1477 unsigned int wmem_alloc_delta = 0; 1478 bool paged, extra_uref = false; 1479 1480 skb = skb_peek_tail(queue); 1481 if (!skb) { 1482 exthdrlen = opt ? opt->opt_flen : 0; 1483 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; 1484 } 1485 1486 paged = !!cork->gso_size; 1487 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize; 1488 orig_mtu = mtu; 1489 1490 if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP && 1491 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) 1492 tskey = atomic_inc_return(&sk->sk_tskey) - 1; 1493 1494 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1495 1496 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + 1497 (opt ? opt->opt_nflen : 0); 1498 1499 headersize = sizeof(struct ipv6hdr) + 1500 (opt ? opt->opt_flen + opt->opt_nflen : 0) + 1501 (dst_allfrag(&rt->dst) ? 1502 sizeof(struct frag_hdr) : 0) + 1503 rt->rt6i_nfheader_len; 1504 1505 if (mtu <= fragheaderlen || 1506 ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr)) 1507 goto emsgsize; 1508 1509 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - 1510 sizeof(struct frag_hdr); 1511 1512 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit 1513 * the first fragment 1514 */ 1515 if (headersize + transhdrlen > mtu) 1516 goto emsgsize; 1517 1518 if (cork->length + length > mtu - headersize && ipc6->dontfrag && 1519 (sk->sk_protocol == IPPROTO_UDP || 1520 sk->sk_protocol == IPPROTO_ICMPV6 || 1521 sk->sk_protocol == IPPROTO_RAW)) { 1522 ipv6_local_rxpmtu(sk, fl6, mtu - headersize + 1523 sizeof(struct ipv6hdr)); 1524 goto emsgsize; 1525 } 1526 1527 if (ip6_sk_ignore_df(sk)) 1528 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; 1529 else 1530 maxnonfragsize = mtu; 1531 1532 if (cork->length + length > maxnonfragsize - headersize) { 1533 emsgsize: 1534 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0); 1535 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu); 1536 return -EMSGSIZE; 1537 } 1538 1539 /* CHECKSUM_PARTIAL only with no extension headers and when 1540 * we are not going to fragment 1541 */ 1542 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && 1543 headersize == sizeof(struct ipv6hdr) && 1544 length <= mtu - headersize && 1545 (!(flags & MSG_MORE) || cork->gso_size) && 1546 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 1547 csummode = CHECKSUM_PARTIAL; 1548 1549 if ((flags & MSG_ZEROCOPY) && length) { 1550 struct msghdr *msg = from; 1551 1552 if (getfrag == ip_generic_getfrag && msg->msg_ubuf) { 1553 if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb)) 1554 return -EINVAL; 1555 1556 /* Leave uarg NULL if can't zerocopy, callers should 1557 * be able to handle it. 1558 */ 1559 if ((rt->dst.dev->features & NETIF_F_SG) && 1560 csummode == CHECKSUM_PARTIAL) { 1561 paged = true; 1562 zc = true; 1563 uarg = msg->msg_ubuf; 1564 } 1565 } else if (sock_flag(sk, SOCK_ZEROCOPY)) { 1566 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); 1567 if (!uarg) 1568 return -ENOBUFS; 1569 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ 1570 if (rt->dst.dev->features & NETIF_F_SG && 1571 csummode == CHECKSUM_PARTIAL) { 1572 paged = true; 1573 zc = true; 1574 } else { 1575 uarg_to_msgzc(uarg)->zerocopy = 0; 1576 skb_zcopy_set(skb, uarg, &extra_uref); 1577 } 1578 } 1579 } 1580 1581 /* 1582 * Let's try using as much space as possible. 1583 * Use MTU if total length of the message fits into the MTU. 1584 * Otherwise, we need to reserve fragment header and 1585 * fragment alignment (= 8-15 octects, in total). 1586 * 1587 * Note that we may need to "move" the data from the tail 1588 * of the buffer to the new fragment when we split 1589 * the message. 1590 * 1591 * FIXME: It may be fragmented into multiple chunks 1592 * at once if non-fragmentable extension headers 1593 * are too large. 1594 * --yoshfuji 1595 */ 1596 1597 cork->length += length; 1598 if (!skb) 1599 goto alloc_new_skb; 1600 1601 while (length > 0) { 1602 /* Check if the remaining data fits into current packet. */ 1603 copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; 1604 if (copy < length) 1605 copy = maxfraglen - skb->len; 1606 1607 if (copy <= 0) { 1608 char *data; 1609 unsigned int datalen; 1610 unsigned int fraglen; 1611 unsigned int fraggap; 1612 unsigned int alloclen, alloc_extra; 1613 unsigned int pagedlen; 1614 alloc_new_skb: 1615 /* There's no room in the current skb */ 1616 if (skb) 1617 fraggap = skb->len - maxfraglen; 1618 else 1619 fraggap = 0; 1620 /* update mtu and maxfraglen if necessary */ 1621 if (!skb || !skb_prev) 1622 ip6_append_data_mtu(&mtu, &maxfraglen, 1623 fragheaderlen, skb, rt, 1624 orig_mtu); 1625 1626 skb_prev = skb; 1627 1628 /* 1629 * If remaining data exceeds the mtu, 1630 * we know we need more fragment(s). 1631 */ 1632 datalen = length + fraggap; 1633 1634 if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) 1635 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; 1636 fraglen = datalen + fragheaderlen; 1637 pagedlen = 0; 1638 1639 alloc_extra = hh_len; 1640 alloc_extra += dst_exthdrlen; 1641 alloc_extra += rt->dst.trailer_len; 1642 1643 /* We just reserve space for fragment header. 1644 * Note: this may be overallocation if the message 1645 * (without MSG_MORE) fits into the MTU. 1646 */ 1647 alloc_extra += sizeof(struct frag_hdr); 1648 1649 if ((flags & MSG_MORE) && 1650 !(rt->dst.dev->features&NETIF_F_SG)) 1651 alloclen = mtu; 1652 else if (!paged && 1653 (fraglen + alloc_extra < SKB_MAX_ALLOC || 1654 !(rt->dst.dev->features & NETIF_F_SG))) 1655 alloclen = fraglen; 1656 else { 1657 alloclen = fragheaderlen + transhdrlen; 1658 pagedlen = datalen - transhdrlen; 1659 } 1660 alloclen += alloc_extra; 1661 1662 if (datalen != length + fraggap) { 1663 /* 1664 * this is not the last fragment, the trailer 1665 * space is regarded as data space. 1666 */ 1667 datalen += rt->dst.trailer_len; 1668 } 1669 1670 fraglen = datalen + fragheaderlen; 1671 1672 copy = datalen - transhdrlen - fraggap - pagedlen; 1673 if (copy < 0) { 1674 err = -EINVAL; 1675 goto error; 1676 } 1677 if (transhdrlen) { 1678 skb = sock_alloc_send_skb(sk, alloclen, 1679 (flags & MSG_DONTWAIT), &err); 1680 } else { 1681 skb = NULL; 1682 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 1683 2 * sk->sk_sndbuf) 1684 skb = alloc_skb(alloclen, 1685 sk->sk_allocation); 1686 if (unlikely(!skb)) 1687 err = -ENOBUFS; 1688 } 1689 if (!skb) 1690 goto error; 1691 /* 1692 * Fill in the control structures 1693 */ 1694 skb->protocol = htons(ETH_P_IPV6); 1695 skb->ip_summed = csummode; 1696 skb->csum = 0; 1697 /* reserve for fragmentation and ipsec header */ 1698 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + 1699 dst_exthdrlen); 1700 1701 /* 1702 * Find where to start putting bytes 1703 */ 1704 data = skb_put(skb, fraglen - pagedlen); 1705 skb_set_network_header(skb, exthdrlen); 1706 data += fragheaderlen; 1707 skb->transport_header = (skb->network_header + 1708 fragheaderlen); 1709 if (fraggap) { 1710 skb->csum = skb_copy_and_csum_bits( 1711 skb_prev, maxfraglen, 1712 data + transhdrlen, fraggap); 1713 skb_prev->csum = csum_sub(skb_prev->csum, 1714 skb->csum); 1715 data += fraggap; 1716 pskb_trim_unique(skb_prev, maxfraglen); 1717 } 1718 if (copy > 0 && 1719 getfrag(from, data + transhdrlen, offset, 1720 copy, fraggap, skb) < 0) { 1721 err = -EFAULT; 1722 kfree_skb(skb); 1723 goto error; 1724 } 1725 1726 offset += copy; 1727 length -= copy + transhdrlen; 1728 transhdrlen = 0; 1729 exthdrlen = 0; 1730 dst_exthdrlen = 0; 1731 1732 /* Only the initial fragment is time stamped */ 1733 skb_shinfo(skb)->tx_flags = cork->tx_flags; 1734 cork->tx_flags = 0; 1735 skb_shinfo(skb)->tskey = tskey; 1736 tskey = 0; 1737 skb_zcopy_set(skb, uarg, &extra_uref); 1738 1739 if ((flags & MSG_CONFIRM) && !skb_prev) 1740 skb_set_dst_pending_confirm(skb, 1); 1741 1742 /* 1743 * Put the packet on the pending queue 1744 */ 1745 if (!skb->destructor) { 1746 skb->destructor = sock_wfree; 1747 skb->sk = sk; 1748 wmem_alloc_delta += skb->truesize; 1749 } 1750 __skb_queue_tail(queue, skb); 1751 continue; 1752 } 1753 1754 if (copy > length) 1755 copy = length; 1756 1757 if (!(rt->dst.dev->features&NETIF_F_SG) && 1758 skb_tailroom(skb) >= copy) { 1759 unsigned int off; 1760 1761 off = skb->len; 1762 if (getfrag(from, skb_put(skb, copy), 1763 offset, copy, off, skb) < 0) { 1764 __skb_trim(skb, off); 1765 err = -EFAULT; 1766 goto error; 1767 } 1768 } else if (!zc) { 1769 int i = skb_shinfo(skb)->nr_frags; 1770 1771 err = -ENOMEM; 1772 if (!sk_page_frag_refill(sk, pfrag)) 1773 goto error; 1774 1775 skb_zcopy_downgrade_managed(skb); 1776 if (!skb_can_coalesce(skb, i, pfrag->page, 1777 pfrag->offset)) { 1778 err = -EMSGSIZE; 1779 if (i == MAX_SKB_FRAGS) 1780 goto error; 1781 1782 __skb_fill_page_desc(skb, i, pfrag->page, 1783 pfrag->offset, 0); 1784 skb_shinfo(skb)->nr_frags = ++i; 1785 get_page(pfrag->page); 1786 } 1787 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1788 if (getfrag(from, 1789 page_address(pfrag->page) + pfrag->offset, 1790 offset, copy, skb->len, skb) < 0) 1791 goto error_efault; 1792 1793 pfrag->offset += copy; 1794 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1795 skb->len += copy; 1796 skb->data_len += copy; 1797 skb->truesize += copy; 1798 wmem_alloc_delta += copy; 1799 } else { 1800 err = skb_zerocopy_iter_dgram(skb, from, copy); 1801 if (err < 0) 1802 goto error; 1803 } 1804 offset += copy; 1805 length -= copy; 1806 } 1807 1808 if (wmem_alloc_delta) 1809 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1810 return 0; 1811 1812 error_efault: 1813 err = -EFAULT; 1814 error: 1815 net_zcopy_put_abort(uarg, extra_uref); 1816 cork->length -= length; 1817 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); 1818 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1819 return err; 1820 } 1821 1822 int ip6_append_data(struct sock *sk, 1823 int getfrag(void *from, char *to, int offset, int len, 1824 int odd, struct sk_buff *skb), 1825 void *from, size_t length, int transhdrlen, 1826 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1827 struct rt6_info *rt, unsigned int flags) 1828 { 1829 struct inet_sock *inet = inet_sk(sk); 1830 struct ipv6_pinfo *np = inet6_sk(sk); 1831 int exthdrlen; 1832 int err; 1833 1834 if (flags&MSG_PROBE) 1835 return 0; 1836 if (skb_queue_empty(&sk->sk_write_queue)) { 1837 /* 1838 * setup for corking 1839 */ 1840 dst_hold(&rt->dst); 1841 err = ip6_setup_cork(sk, &inet->cork, &np->cork, 1842 ipc6, rt); 1843 if (err) 1844 return err; 1845 1846 inet->cork.fl.u.ip6 = *fl6; 1847 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1848 length += exthdrlen; 1849 transhdrlen += exthdrlen; 1850 } else { 1851 transhdrlen = 0; 1852 } 1853 1854 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork, 1855 &np->cork, sk_page_frag(sk), getfrag, 1856 from, length, transhdrlen, flags, ipc6); 1857 } 1858 EXPORT_SYMBOL_GPL(ip6_append_data); 1859 1860 static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork) 1861 { 1862 struct dst_entry *dst = cork->base.dst; 1863 1864 cork->base.dst = NULL; 1865 cork->base.flags &= ~IPCORK_ALLFRAG; 1866 skb_dst_set(skb, dst); 1867 } 1868 1869 static void ip6_cork_release(struct inet_cork_full *cork, 1870 struct inet6_cork *v6_cork) 1871 { 1872 if (v6_cork->opt) { 1873 struct ipv6_txoptions *opt = v6_cork->opt; 1874 1875 kfree(opt->dst0opt); 1876 kfree(opt->dst1opt); 1877 kfree(opt->hopopt); 1878 kfree(opt->srcrt); 1879 kfree(opt); 1880 v6_cork->opt = NULL; 1881 } 1882 1883 if (cork->base.dst) { 1884 dst_release(cork->base.dst); 1885 cork->base.dst = NULL; 1886 cork->base.flags &= ~IPCORK_ALLFRAG; 1887 } 1888 } 1889 1890 struct sk_buff *__ip6_make_skb(struct sock *sk, 1891 struct sk_buff_head *queue, 1892 struct inet_cork_full *cork, 1893 struct inet6_cork *v6_cork) 1894 { 1895 struct sk_buff *skb, *tmp_skb; 1896 struct sk_buff **tail_skb; 1897 struct in6_addr *final_dst; 1898 struct ipv6_pinfo *np = inet6_sk(sk); 1899 struct net *net = sock_net(sk); 1900 struct ipv6hdr *hdr; 1901 struct ipv6_txoptions *opt = v6_cork->opt; 1902 struct rt6_info *rt = (struct rt6_info *)cork->base.dst; 1903 struct flowi6 *fl6 = &cork->fl.u.ip6; 1904 unsigned char proto = fl6->flowi6_proto; 1905 1906 skb = __skb_dequeue(queue); 1907 if (!skb) 1908 goto out; 1909 tail_skb = &(skb_shinfo(skb)->frag_list); 1910 1911 /* move skb->data to ip header from ext header */ 1912 if (skb->data < skb_network_header(skb)) 1913 __skb_pull(skb, skb_network_offset(skb)); 1914 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1915 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1916 *tail_skb = tmp_skb; 1917 tail_skb = &(tmp_skb->next); 1918 skb->len += tmp_skb->len; 1919 skb->data_len += tmp_skb->len; 1920 skb->truesize += tmp_skb->truesize; 1921 tmp_skb->destructor = NULL; 1922 tmp_skb->sk = NULL; 1923 } 1924 1925 /* Allow local fragmentation. */ 1926 skb->ignore_df = ip6_sk_ignore_df(sk); 1927 __skb_pull(skb, skb_network_header_len(skb)); 1928 1929 final_dst = &fl6->daddr; 1930 if (opt && opt->opt_flen) 1931 ipv6_push_frag_opts(skb, opt, &proto); 1932 if (opt && opt->opt_nflen) 1933 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr); 1934 1935 skb_push(skb, sizeof(struct ipv6hdr)); 1936 skb_reset_network_header(skb); 1937 hdr = ipv6_hdr(skb); 1938 1939 ip6_flow_hdr(hdr, v6_cork->tclass, 1940 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1941 ip6_autoflowlabel(net, np), fl6)); 1942 hdr->hop_limit = v6_cork->hop_limit; 1943 hdr->nexthdr = proto; 1944 hdr->saddr = fl6->saddr; 1945 hdr->daddr = *final_dst; 1946 1947 skb->priority = sk->sk_priority; 1948 skb->mark = cork->base.mark; 1949 skb->tstamp = cork->base.transmit_time; 1950 1951 ip6_cork_steal_dst(skb, cork); 1952 IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len); 1953 if (proto == IPPROTO_ICMPV6) { 1954 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1955 1956 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type); 1957 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 1958 } 1959 1960 ip6_cork_release(cork, v6_cork); 1961 out: 1962 return skb; 1963 } 1964 1965 int ip6_send_skb(struct sk_buff *skb) 1966 { 1967 struct net *net = sock_net(skb->sk); 1968 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 1969 int err; 1970 1971 err = ip6_local_out(net, skb->sk, skb); 1972 if (err) { 1973 if (err > 0) 1974 err = net_xmit_errno(err); 1975 if (err) 1976 IP6_INC_STATS(net, rt->rt6i_idev, 1977 IPSTATS_MIB_OUTDISCARDS); 1978 } 1979 1980 return err; 1981 } 1982 1983 int ip6_push_pending_frames(struct sock *sk) 1984 { 1985 struct sk_buff *skb; 1986 1987 skb = ip6_finish_skb(sk); 1988 if (!skb) 1989 return 0; 1990 1991 return ip6_send_skb(skb); 1992 } 1993 EXPORT_SYMBOL_GPL(ip6_push_pending_frames); 1994 1995 static void __ip6_flush_pending_frames(struct sock *sk, 1996 struct sk_buff_head *queue, 1997 struct inet_cork_full *cork, 1998 struct inet6_cork *v6_cork) 1999 { 2000 struct sk_buff *skb; 2001 2002 while ((skb = __skb_dequeue_tail(queue)) != NULL) { 2003 if (skb_dst(skb)) 2004 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)), 2005 IPSTATS_MIB_OUTDISCARDS); 2006 kfree_skb(skb); 2007 } 2008 2009 ip6_cork_release(cork, v6_cork); 2010 } 2011 2012 void ip6_flush_pending_frames(struct sock *sk) 2013 { 2014 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 2015 &inet_sk(sk)->cork, &inet6_sk(sk)->cork); 2016 } 2017 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 2018 2019 struct sk_buff *ip6_make_skb(struct sock *sk, 2020 int getfrag(void *from, char *to, int offset, 2021 int len, int odd, struct sk_buff *skb), 2022 void *from, size_t length, int transhdrlen, 2023 struct ipcm6_cookie *ipc6, struct rt6_info *rt, 2024 unsigned int flags, struct inet_cork_full *cork) 2025 { 2026 struct inet6_cork v6_cork; 2027 struct sk_buff_head queue; 2028 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 2029 int err; 2030 2031 if (flags & MSG_PROBE) { 2032 dst_release(&rt->dst); 2033 return NULL; 2034 } 2035 2036 __skb_queue_head_init(&queue); 2037 2038 cork->base.flags = 0; 2039 cork->base.addr = 0; 2040 cork->base.opt = NULL; 2041 v6_cork.opt = NULL; 2042 err = ip6_setup_cork(sk, cork, &v6_cork, ipc6, rt); 2043 if (err) { 2044 ip6_cork_release(cork, &v6_cork); 2045 return ERR_PTR(err); 2046 } 2047 if (ipc6->dontfrag < 0) 2048 ipc6->dontfrag = inet6_sk(sk)->dontfrag; 2049 2050 err = __ip6_append_data(sk, &queue, cork, &v6_cork, 2051 ¤t->task_frag, getfrag, from, 2052 length + exthdrlen, transhdrlen + exthdrlen, 2053 flags, ipc6); 2054 if (err) { 2055 __ip6_flush_pending_frames(sk, &queue, cork, &v6_cork); 2056 return ERR_PTR(err); 2057 } 2058 2059 return __ip6_make_skb(sk, &queue, cork, &v6_cork); 2060 } 2061