1 /* 2 * IPv6 output functions 3 * Linux INET6 implementation 4 * 5 * Authors: 6 * Pedro Roque <roque@di.fc.ul.pt> 7 * 8 * Based on linux/net/ipv4/ip_output.c 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 * Changes: 16 * A.N.Kuznetsov : airthmetics in fragmentation. 17 * extension headers are implemented. 18 * route changes now work. 19 * ip6_forward does not confuse sniffers. 20 * etc. 21 * 22 * H. von Brand : Added missing #include <linux/string.h> 23 * Imran Patel : frag id should be in NBO 24 * Kazunori MIYAZAWA @USAGI 25 * : add ip6_append_data and related functions 26 * for datagram xmit 27 */ 28 29 #include <linux/errno.h> 30 #include <linux/kernel.h> 31 #include <linux/string.h> 32 #include <linux/socket.h> 33 #include <linux/net.h> 34 #include <linux/netdevice.h> 35 #include <linux/if_arp.h> 36 #include <linux/in6.h> 37 #include <linux/tcp.h> 38 #include <linux/route.h> 39 #include <linux/module.h> 40 #include <linux/slab.h> 41 42 #include <linux/bpf-cgroup.h> 43 #include <linux/netfilter.h> 44 #include <linux/netfilter_ipv6.h> 45 46 #include <net/sock.h> 47 #include <net/snmp.h> 48 49 #include <net/ipv6.h> 50 #include <net/ndisc.h> 51 #include <net/protocol.h> 52 #include <net/ip6_route.h> 53 #include <net/addrconf.h> 54 #include <net/rawv6.h> 55 #include <net/icmp.h> 56 #include <net/xfrm.h> 57 #include <net/checksum.h> 58 #include <linux/mroute6.h> 59 #include <net/l3mdev.h> 60 #include <net/lwtunnel.h> 61 62 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb) 63 { 64 struct dst_entry *dst = skb_dst(skb); 65 struct net_device *dev = dst->dev; 66 struct neighbour *neigh; 67 struct in6_addr *nexthop; 68 int ret; 69 70 if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) { 71 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 72 73 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && 74 ((mroute6_is_socket(net, skb) && 75 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || 76 ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr, 77 &ipv6_hdr(skb)->saddr))) { 78 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 79 80 /* Do not check for IFF_ALLMULTI; multicast routing 81 is not supported in any case. 82 */ 83 if (newskb) 84 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING, 85 net, sk, newskb, NULL, newskb->dev, 86 dev_loopback_xmit); 87 88 if (ipv6_hdr(skb)->hop_limit == 0) { 89 IP6_INC_STATS(net, idev, 90 IPSTATS_MIB_OUTDISCARDS); 91 kfree_skb(skb); 92 return 0; 93 } 94 } 95 96 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len); 97 98 if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <= 99 IPV6_ADDR_SCOPE_NODELOCAL && 100 !(dev->flags & IFF_LOOPBACK)) { 101 kfree_skb(skb); 102 return 0; 103 } 104 } 105 106 if (lwtunnel_xmit_redirect(dst->lwtstate)) { 107 int res = lwtunnel_xmit(skb); 108 109 if (res < 0 || res == LWTUNNEL_XMIT_DONE) 110 return res; 111 } 112 113 rcu_read_lock_bh(); 114 nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr); 115 neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop); 116 if (unlikely(!neigh)) 117 neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false); 118 if (!IS_ERR(neigh)) { 119 sock_confirm_neigh(skb, neigh); 120 ret = neigh_output(neigh, skb, false); 121 rcu_read_unlock_bh(); 122 return ret; 123 } 124 rcu_read_unlock_bh(); 125 126 IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES); 127 kfree_skb(skb); 128 return -EINVAL; 129 } 130 131 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb) 132 { 133 int ret; 134 135 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb); 136 if (ret) { 137 kfree_skb(skb); 138 return ret; 139 } 140 141 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) 142 /* Policy lookup after SNAT yielded a new policy */ 143 if (skb_dst(skb)->xfrm) { 144 IPCB(skb)->flags |= IPSKB_REROUTED; 145 return dst_output(net, sk, skb); 146 } 147 #endif 148 149 if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) || 150 dst_allfrag(skb_dst(skb)) || 151 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)) 152 return ip6_fragment(net, sk, skb, ip6_finish_output2); 153 else 154 return ip6_finish_output2(net, sk, skb); 155 } 156 157 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb) 158 { 159 struct net_device *dev = skb_dst(skb)->dev; 160 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 161 162 skb->protocol = htons(ETH_P_IPV6); 163 skb->dev = dev; 164 165 if (unlikely(idev->cnf.disable_ipv6)) { 166 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS); 167 kfree_skb(skb); 168 return 0; 169 } 170 171 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, 172 net, sk, skb, NULL, dev, 173 ip6_finish_output, 174 !(IP6CB(skb)->flags & IP6SKB_REROUTED)); 175 } 176 177 bool ip6_autoflowlabel(struct net *net, const struct ipv6_pinfo *np) 178 { 179 if (!np->autoflowlabel_set) 180 return ip6_default_np_autolabel(net); 181 else 182 return np->autoflowlabel; 183 } 184 185 /* 186 * xmit an sk_buff (used by TCP, SCTP and DCCP) 187 * Note : socket lock is not held for SYNACK packets, but might be modified 188 * by calls to skb_set_owner_w() and ipv6_local_error(), 189 * which are using proper atomic operations or spinlocks. 190 */ 191 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, 192 __u32 mark, struct ipv6_txoptions *opt, int tclass) 193 { 194 struct net *net = sock_net(sk); 195 const struct ipv6_pinfo *np = inet6_sk(sk); 196 struct in6_addr *first_hop = &fl6->daddr; 197 struct dst_entry *dst = skb_dst(skb); 198 unsigned int head_room; 199 struct ipv6hdr *hdr; 200 u8 proto = fl6->flowi6_proto; 201 int seg_len = skb->len; 202 int hlimit = -1; 203 u32 mtu; 204 205 head_room = sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev); 206 if (opt) 207 head_room += opt->opt_nflen + opt->opt_flen; 208 209 if (unlikely(skb_headroom(skb) < head_room)) { 210 struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room); 211 if (!skb2) { 212 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 213 IPSTATS_MIB_OUTDISCARDS); 214 kfree_skb(skb); 215 return -ENOBUFS; 216 } 217 if (skb->sk) 218 skb_set_owner_w(skb2, skb->sk); 219 consume_skb(skb); 220 skb = skb2; 221 } 222 223 if (opt) { 224 seg_len += opt->opt_nflen + opt->opt_flen; 225 226 if (opt->opt_flen) 227 ipv6_push_frag_opts(skb, opt, &proto); 228 229 if (opt->opt_nflen) 230 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop, 231 &fl6->saddr); 232 } 233 234 skb_push(skb, sizeof(struct ipv6hdr)); 235 skb_reset_network_header(skb); 236 hdr = ipv6_hdr(skb); 237 238 /* 239 * Fill in the IPv6 header 240 */ 241 if (np) 242 hlimit = np->hop_limit; 243 if (hlimit < 0) 244 hlimit = ip6_dst_hoplimit(dst); 245 246 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel, 247 ip6_autoflowlabel(net, np), fl6)); 248 249 hdr->payload_len = htons(seg_len); 250 hdr->nexthdr = proto; 251 hdr->hop_limit = hlimit; 252 253 hdr->saddr = fl6->saddr; 254 hdr->daddr = *first_hop; 255 256 skb->protocol = htons(ETH_P_IPV6); 257 skb->priority = sk->sk_priority; 258 skb->mark = mark; 259 260 mtu = dst_mtu(dst); 261 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) { 262 IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)), 263 IPSTATS_MIB_OUT, skb->len); 264 265 /* if egress device is enslaved to an L3 master device pass the 266 * skb to its handler for processing 267 */ 268 skb = l3mdev_ip6_out((struct sock *)sk, skb); 269 if (unlikely(!skb)) 270 return 0; 271 272 /* hooks should never assume socket lock is held. 273 * we promote our socket to non const 274 */ 275 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, 276 net, (struct sock *)sk, skb, NULL, dst->dev, 277 dst_output); 278 } 279 280 skb->dev = dst->dev; 281 /* ipv6_local_error() does not require socket lock, 282 * we promote our socket to non const 283 */ 284 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu); 285 286 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS); 287 kfree_skb(skb); 288 return -EMSGSIZE; 289 } 290 EXPORT_SYMBOL(ip6_xmit); 291 292 static int ip6_call_ra_chain(struct sk_buff *skb, int sel) 293 { 294 struct ip6_ra_chain *ra; 295 struct sock *last = NULL; 296 297 read_lock(&ip6_ra_lock); 298 for (ra = ip6_ra_chain; ra; ra = ra->next) { 299 struct sock *sk = ra->sk; 300 if (sk && ra->sel == sel && 301 (!sk->sk_bound_dev_if || 302 sk->sk_bound_dev_if == skb->dev->ifindex)) { 303 struct ipv6_pinfo *np = inet6_sk(sk); 304 305 if (np && np->rtalert_isolate && 306 !net_eq(sock_net(sk), dev_net(skb->dev))) { 307 continue; 308 } 309 if (last) { 310 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 311 if (skb2) 312 rawv6_rcv(last, skb2); 313 } 314 last = sk; 315 } 316 } 317 318 if (last) { 319 rawv6_rcv(last, skb); 320 read_unlock(&ip6_ra_lock); 321 return 1; 322 } 323 read_unlock(&ip6_ra_lock); 324 return 0; 325 } 326 327 static int ip6_forward_proxy_check(struct sk_buff *skb) 328 { 329 struct ipv6hdr *hdr = ipv6_hdr(skb); 330 u8 nexthdr = hdr->nexthdr; 331 __be16 frag_off; 332 int offset; 333 334 if (ipv6_ext_hdr(nexthdr)) { 335 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off); 336 if (offset < 0) 337 return 0; 338 } else 339 offset = sizeof(struct ipv6hdr); 340 341 if (nexthdr == IPPROTO_ICMPV6) { 342 struct icmp6hdr *icmp6; 343 344 if (!pskb_may_pull(skb, (skb_network_header(skb) + 345 offset + 1 - skb->data))) 346 return 0; 347 348 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset); 349 350 switch (icmp6->icmp6_type) { 351 case NDISC_ROUTER_SOLICITATION: 352 case NDISC_ROUTER_ADVERTISEMENT: 353 case NDISC_NEIGHBOUR_SOLICITATION: 354 case NDISC_NEIGHBOUR_ADVERTISEMENT: 355 case NDISC_REDIRECT: 356 /* For reaction involving unicast neighbor discovery 357 * message destined to the proxied address, pass it to 358 * input function. 359 */ 360 return 1; 361 default: 362 break; 363 } 364 } 365 366 /* 367 * The proxying router can't forward traffic sent to a link-local 368 * address, so signal the sender and discard the packet. This 369 * behavior is clarified by the MIPv6 specification. 370 */ 371 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) { 372 dst_link_failure(skb); 373 return -1; 374 } 375 376 return 0; 377 } 378 379 static inline int ip6_forward_finish(struct net *net, struct sock *sk, 380 struct sk_buff *skb) 381 { 382 struct dst_entry *dst = skb_dst(skb); 383 384 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS); 385 __IP6_ADD_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len); 386 387 #ifdef CONFIG_NET_SWITCHDEV 388 if (skb->offload_l3_fwd_mark) { 389 consume_skb(skb); 390 return 0; 391 } 392 #endif 393 394 skb->tstamp = 0; 395 return dst_output(net, sk, skb); 396 } 397 398 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) 399 { 400 if (skb->len <= mtu) 401 return false; 402 403 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */ 404 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu) 405 return true; 406 407 if (skb->ignore_df) 408 return false; 409 410 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 411 return false; 412 413 return true; 414 } 415 416 int ip6_forward(struct sk_buff *skb) 417 { 418 struct inet6_dev *idev = __in6_dev_get_safely(skb->dev); 419 struct dst_entry *dst = skb_dst(skb); 420 struct ipv6hdr *hdr = ipv6_hdr(skb); 421 struct inet6_skb_parm *opt = IP6CB(skb); 422 struct net *net = dev_net(dst->dev); 423 u32 mtu; 424 425 if (net->ipv6.devconf_all->forwarding == 0) 426 goto error; 427 428 if (skb->pkt_type != PACKET_HOST) 429 goto drop; 430 431 if (unlikely(skb->sk)) 432 goto drop; 433 434 if (skb_warn_if_lro(skb)) 435 goto drop; 436 437 if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) { 438 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 439 goto drop; 440 } 441 442 skb_forward_csum(skb); 443 444 /* 445 * We DO NOT make any processing on 446 * RA packets, pushing them to user level AS IS 447 * without ane WARRANTY that application will be able 448 * to interpret them. The reason is that we 449 * cannot make anything clever here. 450 * 451 * We are not end-node, so that if packet contains 452 * AH/ESP, we cannot make anything. 453 * Defragmentation also would be mistake, RA packets 454 * cannot be fragmented, because there is no warranty 455 * that different fragments will go along one path. --ANK 456 */ 457 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) { 458 if (ip6_call_ra_chain(skb, ntohs(opt->ra))) 459 return 0; 460 } 461 462 /* 463 * check and decrement ttl 464 */ 465 if (hdr->hop_limit <= 1) { 466 /* Force OUTPUT device used as source address */ 467 skb->dev = dst->dev; 468 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); 469 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); 470 471 kfree_skb(skb); 472 return -ETIMEDOUT; 473 } 474 475 /* XXX: idev->cnf.proxy_ndp? */ 476 if (net->ipv6.devconf_all->proxy_ndp && 477 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) { 478 int proxied = ip6_forward_proxy_check(skb); 479 if (proxied > 0) 480 return ip6_input(skb); 481 else if (proxied < 0) { 482 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 483 goto drop; 484 } 485 } 486 487 if (!xfrm6_route_forward(skb)) { 488 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS); 489 goto drop; 490 } 491 dst = skb_dst(skb); 492 493 /* IPv6 specs say nothing about it, but it is clear that we cannot 494 send redirects to source routed frames. 495 We don't send redirects to frames decapsulated from IPsec. 496 */ 497 if (IP6CB(skb)->iif == dst->dev->ifindex && 498 opt->srcrt == 0 && !skb_sec_path(skb)) { 499 struct in6_addr *target = NULL; 500 struct inet_peer *peer; 501 struct rt6_info *rt; 502 503 /* 504 * incoming and outgoing devices are the same 505 * send a redirect. 506 */ 507 508 rt = (struct rt6_info *) dst; 509 if (rt->rt6i_flags & RTF_GATEWAY) 510 target = &rt->rt6i_gateway; 511 else 512 target = &hdr->daddr; 513 514 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1); 515 516 /* Limit redirects both by destination (here) 517 and by source (inside ndisc_send_redirect) 518 */ 519 if (inet_peer_xrlim_allow(peer, 1*HZ)) 520 ndisc_send_redirect(skb, target); 521 if (peer) 522 inet_putpeer(peer); 523 } else { 524 int addrtype = ipv6_addr_type(&hdr->saddr); 525 526 /* This check is security critical. */ 527 if (addrtype == IPV6_ADDR_ANY || 528 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK)) 529 goto error; 530 if (addrtype & IPV6_ADDR_LINKLOCAL) { 531 icmpv6_send(skb, ICMPV6_DEST_UNREACH, 532 ICMPV6_NOT_NEIGHBOUR, 0); 533 goto error; 534 } 535 } 536 537 mtu = ip6_dst_mtu_forward(dst); 538 if (mtu < IPV6_MIN_MTU) 539 mtu = IPV6_MIN_MTU; 540 541 if (ip6_pkt_too_big(skb, mtu)) { 542 /* Again, force OUTPUT device used as source address */ 543 skb->dev = dst->dev; 544 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 545 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS); 546 __IP6_INC_STATS(net, ip6_dst_idev(dst), 547 IPSTATS_MIB_FRAGFAILS); 548 kfree_skb(skb); 549 return -EMSGSIZE; 550 } 551 552 if (skb_cow(skb, dst->dev->hard_header_len)) { 553 __IP6_INC_STATS(net, ip6_dst_idev(dst), 554 IPSTATS_MIB_OUTDISCARDS); 555 goto drop; 556 } 557 558 hdr = ipv6_hdr(skb); 559 560 /* Mangling hops number delayed to point after skb COW */ 561 562 hdr->hop_limit--; 563 564 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, 565 net, NULL, skb, skb->dev, dst->dev, 566 ip6_forward_finish); 567 568 error: 569 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 570 drop: 571 kfree_skb(skb); 572 return -EINVAL; 573 } 574 575 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from) 576 { 577 to->pkt_type = from->pkt_type; 578 to->priority = from->priority; 579 to->protocol = from->protocol; 580 skb_dst_drop(to); 581 skb_dst_set(to, dst_clone(skb_dst(from))); 582 to->dev = from->dev; 583 to->mark = from->mark; 584 585 skb_copy_hash(to, from); 586 587 #ifdef CONFIG_NET_SCHED 588 to->tc_index = from->tc_index; 589 #endif 590 nf_copy(to, from); 591 skb_ext_copy(to, from); 592 skb_copy_secmark(to, from); 593 } 594 595 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, 596 int (*output)(struct net *, struct sock *, struct sk_buff *)) 597 { 598 struct sk_buff *frag; 599 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 600 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? 601 inet6_sk(skb->sk) : NULL; 602 struct ipv6hdr *tmp_hdr; 603 struct frag_hdr *fh; 604 unsigned int mtu, hlen, left, len, nexthdr_offset; 605 int hroom, troom; 606 __be32 frag_id; 607 int ptr, offset = 0, err = 0; 608 u8 *prevhdr, nexthdr = 0; 609 610 err = ip6_find_1stfragopt(skb, &prevhdr); 611 if (err < 0) 612 goto fail; 613 hlen = err; 614 nexthdr = *prevhdr; 615 nexthdr_offset = prevhdr - skb_network_header(skb); 616 617 mtu = ip6_skb_dst_mtu(skb); 618 619 /* We must not fragment if the socket is set to force MTU discovery 620 * or if the skb it not generated by a local socket. 621 */ 622 if (unlikely(!skb->ignore_df && skb->len > mtu)) 623 goto fail_toobig; 624 625 if (IP6CB(skb)->frag_max_size) { 626 if (IP6CB(skb)->frag_max_size > mtu) 627 goto fail_toobig; 628 629 /* don't send fragments larger than what we received */ 630 mtu = IP6CB(skb)->frag_max_size; 631 if (mtu < IPV6_MIN_MTU) 632 mtu = IPV6_MIN_MTU; 633 } 634 635 if (np && np->frag_size < mtu) { 636 if (np->frag_size) 637 mtu = np->frag_size; 638 } 639 if (mtu < hlen + sizeof(struct frag_hdr) + 8) 640 goto fail_toobig; 641 mtu -= hlen + sizeof(struct frag_hdr); 642 643 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr, 644 &ipv6_hdr(skb)->saddr); 645 646 if (skb->ip_summed == CHECKSUM_PARTIAL && 647 (err = skb_checksum_help(skb))) 648 goto fail; 649 650 prevhdr = skb_network_header(skb) + nexthdr_offset; 651 hroom = LL_RESERVED_SPACE(rt->dst.dev); 652 if (skb_has_frag_list(skb)) { 653 unsigned int first_len = skb_pagelen(skb); 654 struct sk_buff *frag2; 655 656 if (first_len - hlen > mtu || 657 ((first_len - hlen) & 7) || 658 skb_cloned(skb) || 659 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr))) 660 goto slow_path; 661 662 skb_walk_frags(skb, frag) { 663 /* Correct geometry. */ 664 if (frag->len > mtu || 665 ((frag->len & 7) && frag->next) || 666 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr))) 667 goto slow_path_clean; 668 669 /* Partially cloned skb? */ 670 if (skb_shared(frag)) 671 goto slow_path_clean; 672 673 BUG_ON(frag->sk); 674 if (skb->sk) { 675 frag->sk = skb->sk; 676 frag->destructor = sock_wfree; 677 } 678 skb->truesize -= frag->truesize; 679 } 680 681 err = 0; 682 offset = 0; 683 /* BUILD HEADER */ 684 685 *prevhdr = NEXTHDR_FRAGMENT; 686 tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC); 687 if (!tmp_hdr) { 688 err = -ENOMEM; 689 goto fail; 690 } 691 frag = skb_shinfo(skb)->frag_list; 692 skb_frag_list_init(skb); 693 694 __skb_pull(skb, hlen); 695 fh = __skb_push(skb, sizeof(struct frag_hdr)); 696 __skb_push(skb, hlen); 697 skb_reset_network_header(skb); 698 memcpy(skb_network_header(skb), tmp_hdr, hlen); 699 700 fh->nexthdr = nexthdr; 701 fh->reserved = 0; 702 fh->frag_off = htons(IP6_MF); 703 fh->identification = frag_id; 704 705 first_len = skb_pagelen(skb); 706 skb->data_len = first_len - skb_headlen(skb); 707 skb->len = first_len; 708 ipv6_hdr(skb)->payload_len = htons(first_len - 709 sizeof(struct ipv6hdr)); 710 711 for (;;) { 712 /* Prepare header of the next frame, 713 * before previous one went down. */ 714 if (frag) { 715 frag->ip_summed = CHECKSUM_NONE; 716 skb_reset_transport_header(frag); 717 fh = __skb_push(frag, sizeof(struct frag_hdr)); 718 __skb_push(frag, hlen); 719 skb_reset_network_header(frag); 720 memcpy(skb_network_header(frag), tmp_hdr, 721 hlen); 722 offset += skb->len - hlen - sizeof(struct frag_hdr); 723 fh->nexthdr = nexthdr; 724 fh->reserved = 0; 725 fh->frag_off = htons(offset); 726 if (frag->next) 727 fh->frag_off |= htons(IP6_MF); 728 fh->identification = frag_id; 729 ipv6_hdr(frag)->payload_len = 730 htons(frag->len - 731 sizeof(struct ipv6hdr)); 732 ip6_copy_metadata(frag, skb); 733 } 734 735 err = output(net, sk, skb); 736 if (!err) 737 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 738 IPSTATS_MIB_FRAGCREATES); 739 740 if (err || !frag) 741 break; 742 743 skb = frag; 744 frag = skb->next; 745 skb_mark_not_on_list(skb); 746 } 747 748 kfree(tmp_hdr); 749 750 if (err == 0) { 751 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 752 IPSTATS_MIB_FRAGOKS); 753 return 0; 754 } 755 756 kfree_skb_list(frag); 757 758 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst), 759 IPSTATS_MIB_FRAGFAILS); 760 return err; 761 762 slow_path_clean: 763 skb_walk_frags(skb, frag2) { 764 if (frag2 == frag) 765 break; 766 frag2->sk = NULL; 767 frag2->destructor = NULL; 768 skb->truesize += frag2->truesize; 769 } 770 } 771 772 slow_path: 773 left = skb->len - hlen; /* Space per frame */ 774 ptr = hlen; /* Where to start from */ 775 776 /* 777 * Fragment the datagram. 778 */ 779 780 troom = rt->dst.dev->needed_tailroom; 781 782 /* 783 * Keep copying data until we run out. 784 */ 785 while (left > 0) { 786 u8 *fragnexthdr_offset; 787 788 len = left; 789 /* IF: it doesn't fit, use 'mtu' - the data space left */ 790 if (len > mtu) 791 len = mtu; 792 /* IF: we are not sending up to and including the packet end 793 then align the next start on an eight byte boundary */ 794 if (len < left) { 795 len &= ~7; 796 } 797 798 /* Allocate buffer */ 799 frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) + 800 hroom + troom, GFP_ATOMIC); 801 if (!frag) { 802 err = -ENOMEM; 803 goto fail; 804 } 805 806 /* 807 * Set up data on packet 808 */ 809 810 ip6_copy_metadata(frag, skb); 811 skb_reserve(frag, hroom); 812 skb_put(frag, len + hlen + sizeof(struct frag_hdr)); 813 skb_reset_network_header(frag); 814 fh = (struct frag_hdr *)(skb_network_header(frag) + hlen); 815 frag->transport_header = (frag->network_header + hlen + 816 sizeof(struct frag_hdr)); 817 818 /* 819 * Charge the memory for the fragment to any owner 820 * it might possess 821 */ 822 if (skb->sk) 823 skb_set_owner_w(frag, skb->sk); 824 825 /* 826 * Copy the packet header into the new buffer. 827 */ 828 skb_copy_from_linear_data(skb, skb_network_header(frag), hlen); 829 830 fragnexthdr_offset = skb_network_header(frag); 831 fragnexthdr_offset += prevhdr - skb_network_header(skb); 832 *fragnexthdr_offset = NEXTHDR_FRAGMENT; 833 834 /* 835 * Build fragment header. 836 */ 837 fh->nexthdr = nexthdr; 838 fh->reserved = 0; 839 fh->identification = frag_id; 840 841 /* 842 * Copy a block of the IP datagram. 843 */ 844 BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag), 845 len)); 846 left -= len; 847 848 fh->frag_off = htons(offset); 849 if (left > 0) 850 fh->frag_off |= htons(IP6_MF); 851 ipv6_hdr(frag)->payload_len = htons(frag->len - 852 sizeof(struct ipv6hdr)); 853 854 ptr += len; 855 offset += len; 856 857 /* 858 * Put this fragment into the sending queue. 859 */ 860 err = output(net, sk, frag); 861 if (err) 862 goto fail; 863 864 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 865 IPSTATS_MIB_FRAGCREATES); 866 } 867 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 868 IPSTATS_MIB_FRAGOKS); 869 consume_skb(skb); 870 return err; 871 872 fail_toobig: 873 if (skb->sk && dst_allfrag(skb_dst(skb))) 874 sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK); 875 876 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); 877 err = -EMSGSIZE; 878 879 fail: 880 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), 881 IPSTATS_MIB_FRAGFAILS); 882 kfree_skb(skb); 883 return err; 884 } 885 886 static inline int ip6_rt_check(const struct rt6key *rt_key, 887 const struct in6_addr *fl_addr, 888 const struct in6_addr *addr_cache) 889 { 890 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) && 891 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache)); 892 } 893 894 static struct dst_entry *ip6_sk_dst_check(struct sock *sk, 895 struct dst_entry *dst, 896 const struct flowi6 *fl6) 897 { 898 struct ipv6_pinfo *np = inet6_sk(sk); 899 struct rt6_info *rt; 900 901 if (!dst) 902 goto out; 903 904 if (dst->ops->family != AF_INET6) { 905 dst_release(dst); 906 return NULL; 907 } 908 909 rt = (struct rt6_info *)dst; 910 /* Yes, checking route validity in not connected 911 * case is not very simple. Take into account, 912 * that we do not support routing by source, TOS, 913 * and MSG_DONTROUTE --ANK (980726) 914 * 915 * 1. ip6_rt_check(): If route was host route, 916 * check that cached destination is current. 917 * If it is network route, we still may 918 * check its validity using saved pointer 919 * to the last used address: daddr_cache. 920 * We do not want to save whole address now, 921 * (because main consumer of this service 922 * is tcp, which has not this problem), 923 * so that the last trick works only on connected 924 * sockets. 925 * 2. oif also should be the same. 926 */ 927 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) || 928 #ifdef CONFIG_IPV6_SUBTREES 929 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) || 930 #endif 931 (!(fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) && 932 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex))) { 933 dst_release(dst); 934 dst = NULL; 935 } 936 937 out: 938 return dst; 939 } 940 941 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk, 942 struct dst_entry **dst, struct flowi6 *fl6) 943 { 944 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 945 struct neighbour *n; 946 struct rt6_info *rt; 947 #endif 948 int err; 949 int flags = 0; 950 951 /* The correct way to handle this would be to do 952 * ip6_route_get_saddr, and then ip6_route_output; however, 953 * the route-specific preferred source forces the 954 * ip6_route_output call _before_ ip6_route_get_saddr. 955 * 956 * In source specific routing (no src=any default route), 957 * ip6_route_output will fail given src=any saddr, though, so 958 * that's why we try it again later. 959 */ 960 if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) { 961 struct fib6_info *from; 962 struct rt6_info *rt; 963 bool had_dst = *dst != NULL; 964 965 if (!had_dst) 966 *dst = ip6_route_output(net, sk, fl6); 967 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst; 968 969 rcu_read_lock(); 970 from = rt ? rcu_dereference(rt->from) : NULL; 971 err = ip6_route_get_saddr(net, from, &fl6->daddr, 972 sk ? inet6_sk(sk)->srcprefs : 0, 973 &fl6->saddr); 974 rcu_read_unlock(); 975 976 if (err) 977 goto out_err_release; 978 979 /* If we had an erroneous initial result, pretend it 980 * never existed and let the SA-enabled version take 981 * over. 982 */ 983 if (!had_dst && (*dst)->error) { 984 dst_release(*dst); 985 *dst = NULL; 986 } 987 988 if (fl6->flowi6_oif) 989 flags |= RT6_LOOKUP_F_IFACE; 990 } 991 992 if (!*dst) 993 *dst = ip6_route_output_flags(net, sk, fl6, flags); 994 995 err = (*dst)->error; 996 if (err) 997 goto out_err_release; 998 999 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD 1000 /* 1001 * Here if the dst entry we've looked up 1002 * has a neighbour entry that is in the INCOMPLETE 1003 * state and the src address from the flow is 1004 * marked as OPTIMISTIC, we release the found 1005 * dst entry and replace it instead with the 1006 * dst entry of the nexthop router 1007 */ 1008 rt = (struct rt6_info *) *dst; 1009 rcu_read_lock_bh(); 1010 n = __ipv6_neigh_lookup_noref(rt->dst.dev, 1011 rt6_nexthop(rt, &fl6->daddr)); 1012 err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0; 1013 rcu_read_unlock_bh(); 1014 1015 if (err) { 1016 struct inet6_ifaddr *ifp; 1017 struct flowi6 fl_gw6; 1018 int redirect; 1019 1020 ifp = ipv6_get_ifaddr(net, &fl6->saddr, 1021 (*dst)->dev, 1); 1022 1023 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC); 1024 if (ifp) 1025 in6_ifa_put(ifp); 1026 1027 if (redirect) { 1028 /* 1029 * We need to get the dst entry for the 1030 * default router instead 1031 */ 1032 dst_release(*dst); 1033 memcpy(&fl_gw6, fl6, sizeof(struct flowi6)); 1034 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr)); 1035 *dst = ip6_route_output(net, sk, &fl_gw6); 1036 err = (*dst)->error; 1037 if (err) 1038 goto out_err_release; 1039 } 1040 } 1041 #endif 1042 if (ipv6_addr_v4mapped(&fl6->saddr) && 1043 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) { 1044 err = -EAFNOSUPPORT; 1045 goto out_err_release; 1046 } 1047 1048 return 0; 1049 1050 out_err_release: 1051 dst_release(*dst); 1052 *dst = NULL; 1053 1054 if (err == -ENETUNREACH) 1055 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); 1056 return err; 1057 } 1058 1059 /** 1060 * ip6_dst_lookup - perform route lookup on flow 1061 * @sk: socket which provides route info 1062 * @dst: pointer to dst_entry * for result 1063 * @fl6: flow to lookup 1064 * 1065 * This function performs a route lookup on the given flow. 1066 * 1067 * It returns zero on success, or a standard errno code on error. 1068 */ 1069 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, 1070 struct flowi6 *fl6) 1071 { 1072 *dst = NULL; 1073 return ip6_dst_lookup_tail(net, sk, dst, fl6); 1074 } 1075 EXPORT_SYMBOL_GPL(ip6_dst_lookup); 1076 1077 /** 1078 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec 1079 * @sk: socket which provides route info 1080 * @fl6: flow to lookup 1081 * @final_dst: final destination address for ipsec lookup 1082 * 1083 * This function performs a route lookup on the given flow. 1084 * 1085 * It returns a valid dst pointer on success, or a pointer encoded 1086 * error code. 1087 */ 1088 struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6, 1089 const struct in6_addr *final_dst) 1090 { 1091 struct dst_entry *dst = NULL; 1092 int err; 1093 1094 err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6); 1095 if (err) 1096 return ERR_PTR(err); 1097 if (final_dst) 1098 fl6->daddr = *final_dst; 1099 1100 return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0); 1101 } 1102 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow); 1103 1104 /** 1105 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow 1106 * @sk: socket which provides the dst cache and route info 1107 * @fl6: flow to lookup 1108 * @final_dst: final destination address for ipsec lookup 1109 * @connected: whether @sk is connected or not 1110 * 1111 * This function performs a route lookup on the given flow with the 1112 * possibility of using the cached route in the socket if it is valid. 1113 * It will take the socket dst lock when operating on the dst cache. 1114 * As a result, this function can only be used in process context. 1115 * 1116 * In addition, for a connected socket, cache the dst in the socket 1117 * if the current cache is not valid. 1118 * 1119 * It returns a valid dst pointer on success, or a pointer encoded 1120 * error code. 1121 */ 1122 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, 1123 const struct in6_addr *final_dst, 1124 bool connected) 1125 { 1126 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie); 1127 1128 dst = ip6_sk_dst_check(sk, dst, fl6); 1129 if (dst) 1130 return dst; 1131 1132 dst = ip6_dst_lookup_flow(sk, fl6, final_dst); 1133 if (connected && !IS_ERR(dst)) 1134 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6); 1135 1136 return dst; 1137 } 1138 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow); 1139 1140 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src, 1141 gfp_t gfp) 1142 { 1143 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1144 } 1145 1146 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src, 1147 gfp_t gfp) 1148 { 1149 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL; 1150 } 1151 1152 static void ip6_append_data_mtu(unsigned int *mtu, 1153 int *maxfraglen, 1154 unsigned int fragheaderlen, 1155 struct sk_buff *skb, 1156 struct rt6_info *rt, 1157 unsigned int orig_mtu) 1158 { 1159 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) { 1160 if (!skb) { 1161 /* first fragment, reserve header_len */ 1162 *mtu = orig_mtu - rt->dst.header_len; 1163 1164 } else { 1165 /* 1166 * this fragment is not first, the headers 1167 * space is regarded as data space. 1168 */ 1169 *mtu = orig_mtu; 1170 } 1171 *maxfraglen = ((*mtu - fragheaderlen) & ~7) 1172 + fragheaderlen - sizeof(struct frag_hdr); 1173 } 1174 } 1175 1176 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, 1177 struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6, 1178 struct rt6_info *rt, struct flowi6 *fl6) 1179 { 1180 struct ipv6_pinfo *np = inet6_sk(sk); 1181 unsigned int mtu; 1182 struct ipv6_txoptions *opt = ipc6->opt; 1183 1184 /* 1185 * setup for corking 1186 */ 1187 if (opt) { 1188 if (WARN_ON(v6_cork->opt)) 1189 return -EINVAL; 1190 1191 v6_cork->opt = kzalloc(sizeof(*opt), sk->sk_allocation); 1192 if (unlikely(!v6_cork->opt)) 1193 return -ENOBUFS; 1194 1195 v6_cork->opt->tot_len = sizeof(*opt); 1196 v6_cork->opt->opt_flen = opt->opt_flen; 1197 v6_cork->opt->opt_nflen = opt->opt_nflen; 1198 1199 v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt, 1200 sk->sk_allocation); 1201 if (opt->dst0opt && !v6_cork->opt->dst0opt) 1202 return -ENOBUFS; 1203 1204 v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt, 1205 sk->sk_allocation); 1206 if (opt->dst1opt && !v6_cork->opt->dst1opt) 1207 return -ENOBUFS; 1208 1209 v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt, 1210 sk->sk_allocation); 1211 if (opt->hopopt && !v6_cork->opt->hopopt) 1212 return -ENOBUFS; 1213 1214 v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt, 1215 sk->sk_allocation); 1216 if (opt->srcrt && !v6_cork->opt->srcrt) 1217 return -ENOBUFS; 1218 1219 /* need source address above miyazawa*/ 1220 } 1221 dst_hold(&rt->dst); 1222 cork->base.dst = &rt->dst; 1223 cork->fl.u.ip6 = *fl6; 1224 v6_cork->hop_limit = ipc6->hlimit; 1225 v6_cork->tclass = ipc6->tclass; 1226 if (rt->dst.flags & DST_XFRM_TUNNEL) 1227 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1228 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst); 1229 else 1230 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ? 1231 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(xfrm_dst_path(&rt->dst)); 1232 if (np->frag_size < mtu) { 1233 if (np->frag_size) 1234 mtu = np->frag_size; 1235 } 1236 if (mtu < IPV6_MIN_MTU) 1237 return -EINVAL; 1238 cork->base.fragsize = mtu; 1239 cork->base.gso_size = ipc6->gso_size; 1240 cork->base.tx_flags = 0; 1241 sock_tx_timestamp(sk, ipc6->sockc.tsflags, &cork->base.tx_flags); 1242 1243 if (dst_allfrag(xfrm_dst_path(&rt->dst))) 1244 cork->base.flags |= IPCORK_ALLFRAG; 1245 cork->base.length = 0; 1246 1247 cork->base.transmit_time = ipc6->sockc.transmit_time; 1248 1249 return 0; 1250 } 1251 1252 static int __ip6_append_data(struct sock *sk, 1253 struct flowi6 *fl6, 1254 struct sk_buff_head *queue, 1255 struct inet_cork *cork, 1256 struct inet6_cork *v6_cork, 1257 struct page_frag *pfrag, 1258 int getfrag(void *from, char *to, int offset, 1259 int len, int odd, struct sk_buff *skb), 1260 void *from, int length, int transhdrlen, 1261 unsigned int flags, struct ipcm6_cookie *ipc6) 1262 { 1263 struct sk_buff *skb, *skb_prev = NULL; 1264 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu; 1265 struct ubuf_info *uarg = NULL; 1266 int exthdrlen = 0; 1267 int dst_exthdrlen = 0; 1268 int hh_len; 1269 int copy; 1270 int err; 1271 int offset = 0; 1272 u32 tskey = 0; 1273 struct rt6_info *rt = (struct rt6_info *)cork->dst; 1274 struct ipv6_txoptions *opt = v6_cork->opt; 1275 int csummode = CHECKSUM_NONE; 1276 unsigned int maxnonfragsize, headersize; 1277 unsigned int wmem_alloc_delta = 0; 1278 bool paged, extra_uref; 1279 1280 skb = skb_peek_tail(queue); 1281 if (!skb) { 1282 exthdrlen = opt ? opt->opt_flen : 0; 1283 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; 1284 } 1285 1286 paged = !!cork->gso_size; 1287 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize; 1288 orig_mtu = mtu; 1289 1290 if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP && 1291 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) 1292 tskey = sk->sk_tskey++; 1293 1294 hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1295 1296 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len + 1297 (opt ? opt->opt_nflen : 0); 1298 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - 1299 sizeof(struct frag_hdr); 1300 1301 headersize = sizeof(struct ipv6hdr) + 1302 (opt ? opt->opt_flen + opt->opt_nflen : 0) + 1303 (dst_allfrag(&rt->dst) ? 1304 sizeof(struct frag_hdr) : 0) + 1305 rt->rt6i_nfheader_len; 1306 1307 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit 1308 * the first fragment 1309 */ 1310 if (headersize + transhdrlen > mtu) 1311 goto emsgsize; 1312 1313 if (cork->length + length > mtu - headersize && ipc6->dontfrag && 1314 (sk->sk_protocol == IPPROTO_UDP || 1315 sk->sk_protocol == IPPROTO_RAW)) { 1316 ipv6_local_rxpmtu(sk, fl6, mtu - headersize + 1317 sizeof(struct ipv6hdr)); 1318 goto emsgsize; 1319 } 1320 1321 if (ip6_sk_ignore_df(sk)) 1322 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN; 1323 else 1324 maxnonfragsize = mtu; 1325 1326 if (cork->length + length > maxnonfragsize - headersize) { 1327 emsgsize: 1328 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0); 1329 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu); 1330 return -EMSGSIZE; 1331 } 1332 1333 /* CHECKSUM_PARTIAL only with no extension headers and when 1334 * we are not going to fragment 1335 */ 1336 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP && 1337 headersize == sizeof(struct ipv6hdr) && 1338 length <= mtu - headersize && 1339 (!(flags & MSG_MORE) || cork->gso_size) && 1340 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 1341 csummode = CHECKSUM_PARTIAL; 1342 1343 if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) { 1344 uarg = sock_zerocopy_realloc(sk, length, skb_zcopy(skb)); 1345 if (!uarg) 1346 return -ENOBUFS; 1347 extra_uref = true; 1348 if (rt->dst.dev->features & NETIF_F_SG && 1349 csummode == CHECKSUM_PARTIAL) { 1350 paged = true; 1351 } else { 1352 uarg->zerocopy = 0; 1353 skb_zcopy_set(skb, uarg, &extra_uref); 1354 } 1355 } 1356 1357 /* 1358 * Let's try using as much space as possible. 1359 * Use MTU if total length of the message fits into the MTU. 1360 * Otherwise, we need to reserve fragment header and 1361 * fragment alignment (= 8-15 octects, in total). 1362 * 1363 * Note that we may need to "move" the data from the tail of 1364 * of the buffer to the new fragment when we split 1365 * the message. 1366 * 1367 * FIXME: It may be fragmented into multiple chunks 1368 * at once if non-fragmentable extension headers 1369 * are too large. 1370 * --yoshfuji 1371 */ 1372 1373 cork->length += length; 1374 if (!skb) 1375 goto alloc_new_skb; 1376 1377 while (length > 0) { 1378 /* Check if the remaining data fits into current packet. */ 1379 copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len; 1380 if (copy < length) 1381 copy = maxfraglen - skb->len; 1382 1383 if (copy <= 0) { 1384 char *data; 1385 unsigned int datalen; 1386 unsigned int fraglen; 1387 unsigned int fraggap; 1388 unsigned int alloclen; 1389 unsigned int pagedlen; 1390 alloc_new_skb: 1391 /* There's no room in the current skb */ 1392 if (skb) 1393 fraggap = skb->len - maxfraglen; 1394 else 1395 fraggap = 0; 1396 /* update mtu and maxfraglen if necessary */ 1397 if (!skb || !skb_prev) 1398 ip6_append_data_mtu(&mtu, &maxfraglen, 1399 fragheaderlen, skb, rt, 1400 orig_mtu); 1401 1402 skb_prev = skb; 1403 1404 /* 1405 * If remaining data exceeds the mtu, 1406 * we know we need more fragment(s). 1407 */ 1408 datalen = length + fraggap; 1409 1410 if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) 1411 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; 1412 fraglen = datalen + fragheaderlen; 1413 pagedlen = 0; 1414 1415 if ((flags & MSG_MORE) && 1416 !(rt->dst.dev->features&NETIF_F_SG)) 1417 alloclen = mtu; 1418 else if (!paged) 1419 alloclen = fraglen; 1420 else { 1421 alloclen = min_t(int, fraglen, MAX_HEADER); 1422 pagedlen = fraglen - alloclen; 1423 } 1424 1425 alloclen += dst_exthdrlen; 1426 1427 if (datalen != length + fraggap) { 1428 /* 1429 * this is not the last fragment, the trailer 1430 * space is regarded as data space. 1431 */ 1432 datalen += rt->dst.trailer_len; 1433 } 1434 1435 alloclen += rt->dst.trailer_len; 1436 fraglen = datalen + fragheaderlen; 1437 1438 /* 1439 * We just reserve space for fragment header. 1440 * Note: this may be overallocation if the message 1441 * (without MSG_MORE) fits into the MTU. 1442 */ 1443 alloclen += sizeof(struct frag_hdr); 1444 1445 copy = datalen - transhdrlen - fraggap - pagedlen; 1446 if (copy < 0) { 1447 err = -EINVAL; 1448 goto error; 1449 } 1450 if (transhdrlen) { 1451 skb = sock_alloc_send_skb(sk, 1452 alloclen + hh_len, 1453 (flags & MSG_DONTWAIT), &err); 1454 } else { 1455 skb = NULL; 1456 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <= 1457 2 * sk->sk_sndbuf) 1458 skb = alloc_skb(alloclen + hh_len, 1459 sk->sk_allocation); 1460 if (unlikely(!skb)) 1461 err = -ENOBUFS; 1462 } 1463 if (!skb) 1464 goto error; 1465 /* 1466 * Fill in the control structures 1467 */ 1468 skb->protocol = htons(ETH_P_IPV6); 1469 skb->ip_summed = csummode; 1470 skb->csum = 0; 1471 /* reserve for fragmentation and ipsec header */ 1472 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) + 1473 dst_exthdrlen); 1474 1475 /* 1476 * Find where to start putting bytes 1477 */ 1478 data = skb_put(skb, fraglen - pagedlen); 1479 skb_set_network_header(skb, exthdrlen); 1480 data += fragheaderlen; 1481 skb->transport_header = (skb->network_header + 1482 fragheaderlen); 1483 if (fraggap) { 1484 skb->csum = skb_copy_and_csum_bits( 1485 skb_prev, maxfraglen, 1486 data + transhdrlen, fraggap, 0); 1487 skb_prev->csum = csum_sub(skb_prev->csum, 1488 skb->csum); 1489 data += fraggap; 1490 pskb_trim_unique(skb_prev, maxfraglen); 1491 } 1492 if (copy > 0 && 1493 getfrag(from, data + transhdrlen, offset, 1494 copy, fraggap, skb) < 0) { 1495 err = -EFAULT; 1496 kfree_skb(skb); 1497 goto error; 1498 } 1499 1500 offset += copy; 1501 length -= copy + transhdrlen; 1502 transhdrlen = 0; 1503 exthdrlen = 0; 1504 dst_exthdrlen = 0; 1505 1506 /* Only the initial fragment is time stamped */ 1507 skb_shinfo(skb)->tx_flags = cork->tx_flags; 1508 cork->tx_flags = 0; 1509 skb_shinfo(skb)->tskey = tskey; 1510 tskey = 0; 1511 skb_zcopy_set(skb, uarg, &extra_uref); 1512 1513 if ((flags & MSG_CONFIRM) && !skb_prev) 1514 skb_set_dst_pending_confirm(skb, 1); 1515 1516 /* 1517 * Put the packet on the pending queue 1518 */ 1519 if (!skb->destructor) { 1520 skb->destructor = sock_wfree; 1521 skb->sk = sk; 1522 wmem_alloc_delta += skb->truesize; 1523 } 1524 __skb_queue_tail(queue, skb); 1525 continue; 1526 } 1527 1528 if (copy > length) 1529 copy = length; 1530 1531 if (!(rt->dst.dev->features&NETIF_F_SG) && 1532 skb_tailroom(skb) >= copy) { 1533 unsigned int off; 1534 1535 off = skb->len; 1536 if (getfrag(from, skb_put(skb, copy), 1537 offset, copy, off, skb) < 0) { 1538 __skb_trim(skb, off); 1539 err = -EFAULT; 1540 goto error; 1541 } 1542 } else if (!uarg || !uarg->zerocopy) { 1543 int i = skb_shinfo(skb)->nr_frags; 1544 1545 err = -ENOMEM; 1546 if (!sk_page_frag_refill(sk, pfrag)) 1547 goto error; 1548 1549 if (!skb_can_coalesce(skb, i, pfrag->page, 1550 pfrag->offset)) { 1551 err = -EMSGSIZE; 1552 if (i == MAX_SKB_FRAGS) 1553 goto error; 1554 1555 __skb_fill_page_desc(skb, i, pfrag->page, 1556 pfrag->offset, 0); 1557 skb_shinfo(skb)->nr_frags = ++i; 1558 get_page(pfrag->page); 1559 } 1560 copy = min_t(int, copy, pfrag->size - pfrag->offset); 1561 if (getfrag(from, 1562 page_address(pfrag->page) + pfrag->offset, 1563 offset, copy, skb->len, skb) < 0) 1564 goto error_efault; 1565 1566 pfrag->offset += copy; 1567 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1568 skb->len += copy; 1569 skb->data_len += copy; 1570 skb->truesize += copy; 1571 wmem_alloc_delta += copy; 1572 } else { 1573 err = skb_zerocopy_iter_dgram(skb, from, copy); 1574 if (err < 0) 1575 goto error; 1576 } 1577 offset += copy; 1578 length -= copy; 1579 } 1580 1581 if (wmem_alloc_delta) 1582 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1583 return 0; 1584 1585 error_efault: 1586 err = -EFAULT; 1587 error: 1588 if (uarg) 1589 sock_zerocopy_put_abort(uarg, extra_uref); 1590 cork->length -= length; 1591 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); 1592 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); 1593 return err; 1594 } 1595 1596 int ip6_append_data(struct sock *sk, 1597 int getfrag(void *from, char *to, int offset, int len, 1598 int odd, struct sk_buff *skb), 1599 void *from, int length, int transhdrlen, 1600 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1601 struct rt6_info *rt, unsigned int flags) 1602 { 1603 struct inet_sock *inet = inet_sk(sk); 1604 struct ipv6_pinfo *np = inet6_sk(sk); 1605 int exthdrlen; 1606 int err; 1607 1608 if (flags&MSG_PROBE) 1609 return 0; 1610 if (skb_queue_empty(&sk->sk_write_queue)) { 1611 /* 1612 * setup for corking 1613 */ 1614 err = ip6_setup_cork(sk, &inet->cork, &np->cork, 1615 ipc6, rt, fl6); 1616 if (err) 1617 return err; 1618 1619 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1620 length += exthdrlen; 1621 transhdrlen += exthdrlen; 1622 } else { 1623 fl6 = &inet->cork.fl.u.ip6; 1624 transhdrlen = 0; 1625 } 1626 1627 return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base, 1628 &np->cork, sk_page_frag(sk), getfrag, 1629 from, length, transhdrlen, flags, ipc6); 1630 } 1631 EXPORT_SYMBOL_GPL(ip6_append_data); 1632 1633 static void ip6_cork_release(struct inet_cork_full *cork, 1634 struct inet6_cork *v6_cork) 1635 { 1636 if (v6_cork->opt) { 1637 kfree(v6_cork->opt->dst0opt); 1638 kfree(v6_cork->opt->dst1opt); 1639 kfree(v6_cork->opt->hopopt); 1640 kfree(v6_cork->opt->srcrt); 1641 kfree(v6_cork->opt); 1642 v6_cork->opt = NULL; 1643 } 1644 1645 if (cork->base.dst) { 1646 dst_release(cork->base.dst); 1647 cork->base.dst = NULL; 1648 cork->base.flags &= ~IPCORK_ALLFRAG; 1649 } 1650 memset(&cork->fl, 0, sizeof(cork->fl)); 1651 } 1652 1653 struct sk_buff *__ip6_make_skb(struct sock *sk, 1654 struct sk_buff_head *queue, 1655 struct inet_cork_full *cork, 1656 struct inet6_cork *v6_cork) 1657 { 1658 struct sk_buff *skb, *tmp_skb; 1659 struct sk_buff **tail_skb; 1660 struct in6_addr final_dst_buf, *final_dst = &final_dst_buf; 1661 struct ipv6_pinfo *np = inet6_sk(sk); 1662 struct net *net = sock_net(sk); 1663 struct ipv6hdr *hdr; 1664 struct ipv6_txoptions *opt = v6_cork->opt; 1665 struct rt6_info *rt = (struct rt6_info *)cork->base.dst; 1666 struct flowi6 *fl6 = &cork->fl.u.ip6; 1667 unsigned char proto = fl6->flowi6_proto; 1668 1669 skb = __skb_dequeue(queue); 1670 if (!skb) 1671 goto out; 1672 tail_skb = &(skb_shinfo(skb)->frag_list); 1673 1674 /* move skb->data to ip header from ext header */ 1675 if (skb->data < skb_network_header(skb)) 1676 __skb_pull(skb, skb_network_offset(skb)); 1677 while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1678 __skb_pull(tmp_skb, skb_network_header_len(skb)); 1679 *tail_skb = tmp_skb; 1680 tail_skb = &(tmp_skb->next); 1681 skb->len += tmp_skb->len; 1682 skb->data_len += tmp_skb->len; 1683 skb->truesize += tmp_skb->truesize; 1684 tmp_skb->destructor = NULL; 1685 tmp_skb->sk = NULL; 1686 } 1687 1688 /* Allow local fragmentation. */ 1689 skb->ignore_df = ip6_sk_ignore_df(sk); 1690 1691 *final_dst = fl6->daddr; 1692 __skb_pull(skb, skb_network_header_len(skb)); 1693 if (opt && opt->opt_flen) 1694 ipv6_push_frag_opts(skb, opt, &proto); 1695 if (opt && opt->opt_nflen) 1696 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr); 1697 1698 skb_push(skb, sizeof(struct ipv6hdr)); 1699 skb_reset_network_header(skb); 1700 hdr = ipv6_hdr(skb); 1701 1702 ip6_flow_hdr(hdr, v6_cork->tclass, 1703 ip6_make_flowlabel(net, skb, fl6->flowlabel, 1704 ip6_autoflowlabel(net, np), fl6)); 1705 hdr->hop_limit = v6_cork->hop_limit; 1706 hdr->nexthdr = proto; 1707 hdr->saddr = fl6->saddr; 1708 hdr->daddr = *final_dst; 1709 1710 skb->priority = sk->sk_priority; 1711 skb->mark = sk->sk_mark; 1712 1713 skb->tstamp = cork->base.transmit_time; 1714 1715 skb_dst_set(skb, dst_clone(&rt->dst)); 1716 IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len); 1717 if (proto == IPPROTO_ICMPV6) { 1718 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); 1719 1720 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type); 1721 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS); 1722 } 1723 1724 ip6_cork_release(cork, v6_cork); 1725 out: 1726 return skb; 1727 } 1728 1729 int ip6_send_skb(struct sk_buff *skb) 1730 { 1731 struct net *net = sock_net(skb->sk); 1732 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb); 1733 int err; 1734 1735 err = ip6_local_out(net, skb->sk, skb); 1736 if (err) { 1737 if (err > 0) 1738 err = net_xmit_errno(err); 1739 if (err) 1740 IP6_INC_STATS(net, rt->rt6i_idev, 1741 IPSTATS_MIB_OUTDISCARDS); 1742 } 1743 1744 return err; 1745 } 1746 1747 int ip6_push_pending_frames(struct sock *sk) 1748 { 1749 struct sk_buff *skb; 1750 1751 skb = ip6_finish_skb(sk); 1752 if (!skb) 1753 return 0; 1754 1755 return ip6_send_skb(skb); 1756 } 1757 EXPORT_SYMBOL_GPL(ip6_push_pending_frames); 1758 1759 static void __ip6_flush_pending_frames(struct sock *sk, 1760 struct sk_buff_head *queue, 1761 struct inet_cork_full *cork, 1762 struct inet6_cork *v6_cork) 1763 { 1764 struct sk_buff *skb; 1765 1766 while ((skb = __skb_dequeue_tail(queue)) != NULL) { 1767 if (skb_dst(skb)) 1768 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)), 1769 IPSTATS_MIB_OUTDISCARDS); 1770 kfree_skb(skb); 1771 } 1772 1773 ip6_cork_release(cork, v6_cork); 1774 } 1775 1776 void ip6_flush_pending_frames(struct sock *sk) 1777 { 1778 __ip6_flush_pending_frames(sk, &sk->sk_write_queue, 1779 &inet_sk(sk)->cork, &inet6_sk(sk)->cork); 1780 } 1781 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames); 1782 1783 struct sk_buff *ip6_make_skb(struct sock *sk, 1784 int getfrag(void *from, char *to, int offset, 1785 int len, int odd, struct sk_buff *skb), 1786 void *from, int length, int transhdrlen, 1787 struct ipcm6_cookie *ipc6, struct flowi6 *fl6, 1788 struct rt6_info *rt, unsigned int flags, 1789 struct inet_cork_full *cork) 1790 { 1791 struct inet6_cork v6_cork; 1792 struct sk_buff_head queue; 1793 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0); 1794 int err; 1795 1796 if (flags & MSG_PROBE) 1797 return NULL; 1798 1799 __skb_queue_head_init(&queue); 1800 1801 cork->base.flags = 0; 1802 cork->base.addr = 0; 1803 cork->base.opt = NULL; 1804 cork->base.dst = NULL; 1805 v6_cork.opt = NULL; 1806 err = ip6_setup_cork(sk, cork, &v6_cork, ipc6, rt, fl6); 1807 if (err) { 1808 ip6_cork_release(cork, &v6_cork); 1809 return ERR_PTR(err); 1810 } 1811 if (ipc6->dontfrag < 0) 1812 ipc6->dontfrag = inet6_sk(sk)->dontfrag; 1813 1814 err = __ip6_append_data(sk, fl6, &queue, &cork->base, &v6_cork, 1815 ¤t->task_frag, getfrag, from, 1816 length + exthdrlen, transhdrlen + exthdrlen, 1817 flags, ipc6); 1818 if (err) { 1819 __ip6_flush_pending_frames(sk, &queue, cork, &v6_cork); 1820 return ERR_PTR(err); 1821 } 1822 1823 return __ip6_make_skb(sk, &queue, cork, &v6_cork); 1824 } 1825