1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * The Internet Protocol (IP) output module. 7 * 8 * Version: $Id: ip_output.c,v 1.100 2002/02/01 22:01:03 davem Exp $ 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Donald Becker, <becker@super.org> 13 * Alan Cox, <Alan.Cox@linux.org> 14 * Richard Underwood 15 * Stefan Becker, <stefanb@yello.ping.de> 16 * Jorge Cwik, <jorge@laser.satlink.net> 17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 18 * Hirokazu Takahashi, <taka@valinux.co.jp> 19 * 20 * See ip_input.c for original log 21 * 22 * Fixes: 23 * Alan Cox : Missing nonblock feature in ip_build_xmit. 24 * Mike Kilburn : htons() missing in ip_build_xmit. 25 * Bradford Johnson: Fix faulty handling of some frames when 26 * no route is found. 27 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit 28 * (in case if packet not accepted by 29 * output firewall rules) 30 * Mike McLagan : Routing by source 31 * Alexey Kuznetsov: use new route cache 32 * Andi Kleen: Fix broken PMTU recovery and remove 33 * some redundant tests. 34 * Vitaly E. Lavrov : Transparent proxy revived after year coma. 35 * Andi Kleen : Replace ip_reply with ip_send_reply. 36 * Andi Kleen : Split fast and slow ip_build_xmit path 37 * for decreased register pressure on x86 38 * and more readibility. 39 * Marc Boucher : When call_out_firewall returns FW_QUEUE, 40 * silently drop skb instead of failing with -EPERM. 41 * Detlev Wengorz : Copy protocol for fragments. 42 * Hirokazu Takahashi: HW checksumming for outgoing UDP 43 * datagrams. 44 * Hirokazu Takahashi: sendfile() on UDP works now. 45 */ 46 47 #include <asm/uaccess.h> 48 #include <asm/system.h> 49 #include <linux/module.h> 50 #include <linux/types.h> 51 #include <linux/kernel.h> 52 #include <linux/sched.h> 53 #include <linux/mm.h> 54 #include <linux/string.h> 55 #include <linux/errno.h> 56 #include <linux/config.h> 57 58 #include <linux/socket.h> 59 #include <linux/sockios.h> 60 #include <linux/in.h> 61 #include <linux/inet.h> 62 #include <linux/netdevice.h> 63 #include <linux/etherdevice.h> 64 #include <linux/proc_fs.h> 65 #include <linux/stat.h> 66 #include <linux/init.h> 67 68 #include <net/snmp.h> 69 #include <net/ip.h> 70 #include <net/protocol.h> 71 #include <net/route.h> 72 #include <net/tcp.h> 73 #include <net/udp.h> 74 #include <linux/skbuff.h> 75 #include <net/sock.h> 76 #include <net/arp.h> 77 #include <net/icmp.h> 78 #include <net/raw.h> 79 #include <net/checksum.h> 80 #include <net/inetpeer.h> 81 #include <net/checksum.h> 82 #include <linux/igmp.h> 83 #include <linux/netfilter_ipv4.h> 84 #include <linux/netfilter_bridge.h> 85 #include <linux/mroute.h> 86 #include <linux/netlink.h> 87 88 /* 89 * Shall we try to damage output packets if routing dev changes? 90 */ 91 92 int sysctl_ip_dynaddr; 93 int sysctl_ip_default_ttl = IPDEFTTL; 94 95 /* Generate a checksum for an outgoing IP datagram. */ 96 __inline__ void ip_send_check(struct iphdr *iph) 97 { 98 iph->check = 0; 99 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 100 } 101 102 /* dev_loopback_xmit for use with netfilter. */ 103 static int ip_dev_loopback_xmit(struct sk_buff *newskb) 104 { 105 newskb->mac.raw = newskb->data; 106 __skb_pull(newskb, newskb->nh.raw - newskb->data); 107 newskb->pkt_type = PACKET_LOOPBACK; 108 newskb->ip_summed = CHECKSUM_UNNECESSARY; 109 BUG_TRAP(newskb->dst); 110 111 #ifdef CONFIG_NETFILTER_DEBUG 112 nf_debug_ip_loopback_xmit(newskb); 113 #endif 114 nf_reset(newskb); 115 netif_rx(newskb); 116 return 0; 117 } 118 119 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst) 120 { 121 int ttl = inet->uc_ttl; 122 123 if (ttl < 0) 124 ttl = dst_metric(dst, RTAX_HOPLIMIT); 125 return ttl; 126 } 127 128 /* 129 * Add an ip header to a skbuff and send it out. 130 * 131 */ 132 int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk, 133 u32 saddr, u32 daddr, struct ip_options *opt) 134 { 135 struct inet_sock *inet = inet_sk(sk); 136 struct rtable *rt = (struct rtable *)skb->dst; 137 struct iphdr *iph; 138 139 /* Build the IP header. */ 140 if (opt) 141 iph=(struct iphdr *)skb_push(skb,sizeof(struct iphdr) + opt->optlen); 142 else 143 iph=(struct iphdr *)skb_push(skb,sizeof(struct iphdr)); 144 145 iph->version = 4; 146 iph->ihl = 5; 147 iph->tos = inet->tos; 148 if (ip_dont_fragment(sk, &rt->u.dst)) 149 iph->frag_off = htons(IP_DF); 150 else 151 iph->frag_off = 0; 152 iph->ttl = ip_select_ttl(inet, &rt->u.dst); 153 iph->daddr = rt->rt_dst; 154 iph->saddr = rt->rt_src; 155 iph->protocol = sk->sk_protocol; 156 iph->tot_len = htons(skb->len); 157 ip_select_ident(iph, &rt->u.dst, sk); 158 skb->nh.iph = iph; 159 160 if (opt && opt->optlen) { 161 iph->ihl += opt->optlen>>2; 162 ip_options_build(skb, opt, daddr, rt, 0); 163 } 164 ip_send_check(iph); 165 166 skb->priority = sk->sk_priority; 167 168 /* Send it out. */ 169 return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev, 170 dst_output); 171 } 172 173 static inline int ip_finish_output2(struct sk_buff *skb) 174 { 175 struct dst_entry *dst = skb->dst; 176 struct hh_cache *hh = dst->hh; 177 struct net_device *dev = dst->dev; 178 int hh_len = LL_RESERVED_SPACE(dev); 179 180 /* Be paranoid, rather than too clever. */ 181 if (unlikely(skb_headroom(skb) < hh_len && dev->hard_header)) { 182 struct sk_buff *skb2; 183 184 skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev)); 185 if (skb2 == NULL) { 186 kfree_skb(skb); 187 return -ENOMEM; 188 } 189 if (skb->sk) 190 skb_set_owner_w(skb2, skb->sk); 191 kfree_skb(skb); 192 skb = skb2; 193 } 194 195 #ifdef CONFIG_NETFILTER_DEBUG 196 nf_debug_ip_finish_output2(skb); 197 #endif /*CONFIG_NETFILTER_DEBUG*/ 198 199 nf_reset(skb); 200 201 if (hh) { 202 int hh_alen; 203 204 read_lock_bh(&hh->hh_lock); 205 hh_alen = HH_DATA_ALIGN(hh->hh_len); 206 memcpy(skb->data - hh_alen, hh->hh_data, hh_alen); 207 read_unlock_bh(&hh->hh_lock); 208 skb_push(skb, hh->hh_len); 209 return hh->hh_output(skb); 210 } else if (dst->neighbour) 211 return dst->neighbour->output(skb); 212 213 if (net_ratelimit()) 214 printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n"); 215 kfree_skb(skb); 216 return -EINVAL; 217 } 218 219 int ip_finish_output(struct sk_buff *skb) 220 { 221 struct net_device *dev = skb->dst->dev; 222 223 skb->dev = dev; 224 skb->protocol = htons(ETH_P_IP); 225 226 return NF_HOOK(PF_INET, NF_IP_POST_ROUTING, skb, NULL, dev, 227 ip_finish_output2); 228 } 229 230 int ip_mc_output(struct sk_buff *skb) 231 { 232 struct sock *sk = skb->sk; 233 struct rtable *rt = (struct rtable*)skb->dst; 234 struct net_device *dev = rt->u.dst.dev; 235 236 /* 237 * If the indicated interface is up and running, send the packet. 238 */ 239 IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS); 240 241 skb->dev = dev; 242 skb->protocol = htons(ETH_P_IP); 243 244 /* 245 * Multicasts are looped back for other local users 246 */ 247 248 if (rt->rt_flags&RTCF_MULTICAST) { 249 if ((!sk || inet_sk(sk)->mc_loop) 250 #ifdef CONFIG_IP_MROUTE 251 /* Small optimization: do not loopback not local frames, 252 which returned after forwarding; they will be dropped 253 by ip_mr_input in any case. 254 Note, that local frames are looped back to be delivered 255 to local recipients. 256 257 This check is duplicated in ip_mr_input at the moment. 258 */ 259 && ((rt->rt_flags&RTCF_LOCAL) || !(IPCB(skb)->flags&IPSKB_FORWARDED)) 260 #endif 261 ) { 262 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 263 if (newskb) 264 NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL, 265 newskb->dev, 266 ip_dev_loopback_xmit); 267 } 268 269 /* Multicasts with ttl 0 must not go beyond the host */ 270 271 if (skb->nh.iph->ttl == 0) { 272 kfree_skb(skb); 273 return 0; 274 } 275 } 276 277 if (rt->rt_flags&RTCF_BROADCAST) { 278 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 279 if (newskb) 280 NF_HOOK(PF_INET, NF_IP_POST_ROUTING, newskb, NULL, 281 newskb->dev, ip_dev_loopback_xmit); 282 } 283 284 if (skb->len > dst_mtu(&rt->u.dst)) 285 return ip_fragment(skb, ip_finish_output); 286 else 287 return ip_finish_output(skb); 288 } 289 290 int ip_output(struct sk_buff *skb) 291 { 292 IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS); 293 294 if (skb->len > dst_mtu(skb->dst) && !skb_shinfo(skb)->tso_size) 295 return ip_fragment(skb, ip_finish_output); 296 else 297 return ip_finish_output(skb); 298 } 299 300 int ip_queue_xmit(struct sk_buff *skb, int ipfragok) 301 { 302 struct sock *sk = skb->sk; 303 struct inet_sock *inet = inet_sk(sk); 304 struct ip_options *opt = inet->opt; 305 struct rtable *rt; 306 struct iphdr *iph; 307 308 /* Skip all of this if the packet is already routed, 309 * f.e. by something like SCTP. 310 */ 311 rt = (struct rtable *) skb->dst; 312 if (rt != NULL) 313 goto packet_routed; 314 315 /* Make sure we can route this packet. */ 316 rt = (struct rtable *)__sk_dst_check(sk, 0); 317 if (rt == NULL) { 318 u32 daddr; 319 320 /* Use correct destination address if we have options. */ 321 daddr = inet->daddr; 322 if(opt && opt->srr) 323 daddr = opt->faddr; 324 325 { 326 struct flowi fl = { .oif = sk->sk_bound_dev_if, 327 .nl_u = { .ip4_u = 328 { .daddr = daddr, 329 .saddr = inet->saddr, 330 .tos = RT_CONN_FLAGS(sk) } }, 331 .proto = sk->sk_protocol, 332 .uli_u = { .ports = 333 { .sport = inet->sport, 334 .dport = inet->dport } } }; 335 336 /* If this fails, retransmit mechanism of transport layer will 337 * keep trying until route appears or the connection times 338 * itself out. 339 */ 340 if (ip_route_output_flow(&rt, &fl, sk, 0)) 341 goto no_route; 342 } 343 __sk_dst_set(sk, &rt->u.dst); 344 tcp_v4_setup_caps(sk, &rt->u.dst); 345 } 346 skb->dst = dst_clone(&rt->u.dst); 347 348 packet_routed: 349 if (opt && opt->is_strictroute && rt->rt_dst != rt->rt_gateway) 350 goto no_route; 351 352 /* OK, we know where to send it, allocate and build IP header. */ 353 iph = (struct iphdr *) skb_push(skb, sizeof(struct iphdr) + (opt ? opt->optlen : 0)); 354 *((__u16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff)); 355 iph->tot_len = htons(skb->len); 356 if (ip_dont_fragment(sk, &rt->u.dst) && !ipfragok) 357 iph->frag_off = htons(IP_DF); 358 else 359 iph->frag_off = 0; 360 iph->ttl = ip_select_ttl(inet, &rt->u.dst); 361 iph->protocol = sk->sk_protocol; 362 iph->saddr = rt->rt_src; 363 iph->daddr = rt->rt_dst; 364 skb->nh.iph = iph; 365 /* Transport layer set skb->h.foo itself. */ 366 367 if (opt && opt->optlen) { 368 iph->ihl += opt->optlen >> 2; 369 ip_options_build(skb, opt, inet->daddr, rt, 0); 370 } 371 372 ip_select_ident_more(iph, &rt->u.dst, sk, skb_shinfo(skb)->tso_segs); 373 374 /* Add an IP checksum. */ 375 ip_send_check(iph); 376 377 skb->priority = sk->sk_priority; 378 379 return NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, rt->u.dst.dev, 380 dst_output); 381 382 no_route: 383 IP_INC_STATS(IPSTATS_MIB_OUTNOROUTES); 384 kfree_skb(skb); 385 return -EHOSTUNREACH; 386 } 387 388 389 static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from) 390 { 391 to->pkt_type = from->pkt_type; 392 to->priority = from->priority; 393 to->protocol = from->protocol; 394 to->security = from->security; 395 dst_release(to->dst); 396 to->dst = dst_clone(from->dst); 397 to->dev = from->dev; 398 399 /* Copy the flags to each fragment. */ 400 IPCB(to)->flags = IPCB(from)->flags; 401 402 #ifdef CONFIG_NET_SCHED 403 to->tc_index = from->tc_index; 404 #endif 405 #ifdef CONFIG_NETFILTER 406 to->nfmark = from->nfmark; 407 to->nfcache = from->nfcache; 408 /* Connection association is same as pre-frag packet */ 409 nf_conntrack_put(to->nfct); 410 to->nfct = from->nfct; 411 nf_conntrack_get(to->nfct); 412 to->nfctinfo = from->nfctinfo; 413 #ifdef CONFIG_BRIDGE_NETFILTER 414 nf_bridge_put(to->nf_bridge); 415 to->nf_bridge = from->nf_bridge; 416 nf_bridge_get(to->nf_bridge); 417 #endif 418 #ifdef CONFIG_NETFILTER_DEBUG 419 to->nf_debug = from->nf_debug; 420 #endif 421 #endif 422 } 423 424 /* 425 * This IP datagram is too large to be sent in one piece. Break it up into 426 * smaller pieces (each of size equal to IP header plus 427 * a block of the data of the original IP data part) that will yet fit in a 428 * single device frame, and queue such a frame for sending. 429 */ 430 431 int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff*)) 432 { 433 struct iphdr *iph; 434 int raw = 0; 435 int ptr; 436 struct net_device *dev; 437 struct sk_buff *skb2; 438 unsigned int mtu, hlen, left, len, ll_rs; 439 int offset; 440 int not_last_frag; 441 struct rtable *rt = (struct rtable*)skb->dst; 442 int err = 0; 443 444 dev = rt->u.dst.dev; 445 446 /* 447 * Point into the IP datagram header. 448 */ 449 450 iph = skb->nh.iph; 451 452 if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) { 453 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, 454 htonl(dst_mtu(&rt->u.dst))); 455 kfree_skb(skb); 456 return -EMSGSIZE; 457 } 458 459 /* 460 * Setup starting values. 461 */ 462 463 hlen = iph->ihl * 4; 464 mtu = dst_mtu(&rt->u.dst) - hlen; /* Size of data space */ 465 466 /* When frag_list is given, use it. First, check its validity: 467 * some transformers could create wrong frag_list or break existing 468 * one, it is not prohibited. In this case fall back to copying. 469 * 470 * LATER: this step can be merged to real generation of fragments, 471 * we can switch to copy when see the first bad fragment. 472 */ 473 if (skb_shinfo(skb)->frag_list) { 474 struct sk_buff *frag; 475 int first_len = skb_pagelen(skb); 476 477 if (first_len - hlen > mtu || 478 ((first_len - hlen) & 7) || 479 (iph->frag_off & htons(IP_MF|IP_OFFSET)) || 480 skb_cloned(skb)) 481 goto slow_path; 482 483 for (frag = skb_shinfo(skb)->frag_list; frag; frag = frag->next) { 484 /* Correct geometry. */ 485 if (frag->len > mtu || 486 ((frag->len & 7) && frag->next) || 487 skb_headroom(frag) < hlen) 488 goto slow_path; 489 490 /* Partially cloned skb? */ 491 if (skb_shared(frag)) 492 goto slow_path; 493 } 494 495 /* Everything is OK. Generate! */ 496 497 err = 0; 498 offset = 0; 499 frag = skb_shinfo(skb)->frag_list; 500 skb_shinfo(skb)->frag_list = NULL; 501 skb->data_len = first_len - skb_headlen(skb); 502 skb->len = first_len; 503 iph->tot_len = htons(first_len); 504 iph->frag_off = htons(IP_MF); 505 ip_send_check(iph); 506 507 for (;;) { 508 /* Prepare header of the next frame, 509 * before previous one went down. */ 510 if (frag) { 511 frag->ip_summed = CHECKSUM_NONE; 512 frag->h.raw = frag->data; 513 frag->nh.raw = __skb_push(frag, hlen); 514 memcpy(frag->nh.raw, iph, hlen); 515 iph = frag->nh.iph; 516 iph->tot_len = htons(frag->len); 517 ip_copy_metadata(frag, skb); 518 if (offset == 0) 519 ip_options_fragment(frag); 520 offset += skb->len - hlen; 521 iph->frag_off = htons(offset>>3); 522 if (frag->next != NULL) 523 iph->frag_off |= htons(IP_MF); 524 /* Ready, complete checksum */ 525 ip_send_check(iph); 526 } 527 528 err = output(skb); 529 530 if (err || !frag) 531 break; 532 533 skb = frag; 534 frag = skb->next; 535 skb->next = NULL; 536 } 537 538 if (err == 0) { 539 IP_INC_STATS(IPSTATS_MIB_FRAGOKS); 540 return 0; 541 } 542 543 while (frag) { 544 skb = frag->next; 545 kfree_skb(frag); 546 frag = skb; 547 } 548 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS); 549 return err; 550 } 551 552 slow_path: 553 left = skb->len - hlen; /* Space per frame */ 554 ptr = raw + hlen; /* Where to start from */ 555 556 #ifdef CONFIG_BRIDGE_NETFILTER 557 /* for bridged IP traffic encapsulated inside f.e. a vlan header, 558 * we need to make room for the encapsulating header */ 559 ll_rs = LL_RESERVED_SPACE_EXTRA(rt->u.dst.dev, nf_bridge_pad(skb)); 560 mtu -= nf_bridge_pad(skb); 561 #else 562 ll_rs = LL_RESERVED_SPACE(rt->u.dst.dev); 563 #endif 564 /* 565 * Fragment the datagram. 566 */ 567 568 offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3; 569 not_last_frag = iph->frag_off & htons(IP_MF); 570 571 /* 572 * Keep copying data until we run out. 573 */ 574 575 while(left > 0) { 576 len = left; 577 /* IF: it doesn't fit, use 'mtu' - the data space left */ 578 if (len > mtu) 579 len = mtu; 580 /* IF: we are not sending upto and including the packet end 581 then align the next start on an eight byte boundary */ 582 if (len < left) { 583 len &= ~7; 584 } 585 /* 586 * Allocate buffer. 587 */ 588 589 if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) { 590 NETDEBUG(printk(KERN_INFO "IP: frag: no memory for new fragment!\n")); 591 err = -ENOMEM; 592 goto fail; 593 } 594 595 /* 596 * Set up data on packet 597 */ 598 599 ip_copy_metadata(skb2, skb); 600 skb_reserve(skb2, ll_rs); 601 skb_put(skb2, len + hlen); 602 skb2->nh.raw = skb2->data; 603 skb2->h.raw = skb2->data + hlen; 604 605 /* 606 * Charge the memory for the fragment to any owner 607 * it might possess 608 */ 609 610 if (skb->sk) 611 skb_set_owner_w(skb2, skb->sk); 612 613 /* 614 * Copy the packet header into the new buffer. 615 */ 616 617 memcpy(skb2->nh.raw, skb->data, hlen); 618 619 /* 620 * Copy a block of the IP datagram. 621 */ 622 if (skb_copy_bits(skb, ptr, skb2->h.raw, len)) 623 BUG(); 624 left -= len; 625 626 /* 627 * Fill in the new header fields. 628 */ 629 iph = skb2->nh.iph; 630 iph->frag_off = htons((offset >> 3)); 631 632 /* ANK: dirty, but effective trick. Upgrade options only if 633 * the segment to be fragmented was THE FIRST (otherwise, 634 * options are already fixed) and make it ONCE 635 * on the initial skb, so that all the following fragments 636 * will inherit fixed options. 637 */ 638 if (offset == 0) 639 ip_options_fragment(skb); 640 641 /* 642 * Added AC : If we are fragmenting a fragment that's not the 643 * last fragment then keep MF on each bit 644 */ 645 if (left > 0 || not_last_frag) 646 iph->frag_off |= htons(IP_MF); 647 ptr += len; 648 offset += len; 649 650 /* 651 * Put this fragment into the sending queue. 652 */ 653 654 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES); 655 656 iph->tot_len = htons(len + hlen); 657 658 ip_send_check(iph); 659 660 err = output(skb2); 661 if (err) 662 goto fail; 663 } 664 kfree_skb(skb); 665 IP_INC_STATS(IPSTATS_MIB_FRAGOKS); 666 return err; 667 668 fail: 669 kfree_skb(skb); 670 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS); 671 return err; 672 } 673 674 int 675 ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb) 676 { 677 struct iovec *iov = from; 678 679 if (skb->ip_summed == CHECKSUM_HW) { 680 if (memcpy_fromiovecend(to, iov, offset, len) < 0) 681 return -EFAULT; 682 } else { 683 unsigned int csum = 0; 684 if (csum_partial_copy_fromiovecend(to, iov, offset, len, &csum) < 0) 685 return -EFAULT; 686 skb->csum = csum_block_add(skb->csum, csum, odd); 687 } 688 return 0; 689 } 690 691 static inline unsigned int 692 csum_page(struct page *page, int offset, int copy) 693 { 694 char *kaddr; 695 unsigned int csum; 696 kaddr = kmap(page); 697 csum = csum_partial(kaddr + offset, copy, 0); 698 kunmap(page); 699 return csum; 700 } 701 702 /* 703 * ip_append_data() and ip_append_page() can make one large IP datagram 704 * from many pieces of data. Each pieces will be holded on the socket 705 * until ip_push_pending_frames() is called. Each piece can be a page 706 * or non-page data. 707 * 708 * Not only UDP, other transport protocols - e.g. raw sockets - can use 709 * this interface potentially. 710 * 711 * LATER: length must be adjusted by pad at tail, when it is required. 712 */ 713 int ip_append_data(struct sock *sk, 714 int getfrag(void *from, char *to, int offset, int len, 715 int odd, struct sk_buff *skb), 716 void *from, int length, int transhdrlen, 717 struct ipcm_cookie *ipc, struct rtable *rt, 718 unsigned int flags) 719 { 720 struct inet_sock *inet = inet_sk(sk); 721 struct sk_buff *skb; 722 723 struct ip_options *opt = NULL; 724 int hh_len; 725 int exthdrlen; 726 int mtu; 727 int copy; 728 int err; 729 int offset = 0; 730 unsigned int maxfraglen, fragheaderlen; 731 int csummode = CHECKSUM_NONE; 732 733 if (flags&MSG_PROBE) 734 return 0; 735 736 if (skb_queue_empty(&sk->sk_write_queue)) { 737 /* 738 * setup for corking. 739 */ 740 opt = ipc->opt; 741 if (opt) { 742 if (inet->cork.opt == NULL) { 743 inet->cork.opt = kmalloc(sizeof(struct ip_options) + 40, sk->sk_allocation); 744 if (unlikely(inet->cork.opt == NULL)) 745 return -ENOBUFS; 746 } 747 memcpy(inet->cork.opt, opt, sizeof(struct ip_options)+opt->optlen); 748 inet->cork.flags |= IPCORK_OPT; 749 inet->cork.addr = ipc->addr; 750 } 751 dst_hold(&rt->u.dst); 752 inet->cork.fragsize = mtu = dst_mtu(rt->u.dst.path); 753 inet->cork.rt = rt; 754 inet->cork.length = 0; 755 sk->sk_sndmsg_page = NULL; 756 sk->sk_sndmsg_off = 0; 757 if ((exthdrlen = rt->u.dst.header_len) != 0) { 758 length += exthdrlen; 759 transhdrlen += exthdrlen; 760 } 761 } else { 762 rt = inet->cork.rt; 763 if (inet->cork.flags & IPCORK_OPT) 764 opt = inet->cork.opt; 765 766 transhdrlen = 0; 767 exthdrlen = 0; 768 mtu = inet->cork.fragsize; 769 } 770 hh_len = LL_RESERVED_SPACE(rt->u.dst.dev); 771 772 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0); 773 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen; 774 775 if (inet->cork.length + length > 0xFFFF - fragheaderlen) { 776 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu-exthdrlen); 777 return -EMSGSIZE; 778 } 779 780 /* 781 * transhdrlen > 0 means that this is the first fragment and we wish 782 * it won't be fragmented in the future. 783 */ 784 if (transhdrlen && 785 length + fragheaderlen <= mtu && 786 rt->u.dst.dev->features&(NETIF_F_IP_CSUM|NETIF_F_NO_CSUM|NETIF_F_HW_CSUM) && 787 !exthdrlen) 788 csummode = CHECKSUM_HW; 789 790 inet->cork.length += length; 791 792 /* So, what's going on in the loop below? 793 * 794 * We use calculated fragment length to generate chained skb, 795 * each of segments is IP fragment ready for sending to network after 796 * adding appropriate IP header. 797 */ 798 799 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) 800 goto alloc_new_skb; 801 802 while (length > 0) { 803 /* Check if the remaining data fits into current packet. */ 804 copy = mtu - skb->len; 805 if (copy < length) 806 copy = maxfraglen - skb->len; 807 if (copy <= 0) { 808 char *data; 809 unsigned int datalen; 810 unsigned int fraglen; 811 unsigned int fraggap; 812 unsigned int alloclen; 813 struct sk_buff *skb_prev; 814 alloc_new_skb: 815 skb_prev = skb; 816 if (skb_prev) 817 fraggap = skb_prev->len - maxfraglen; 818 else 819 fraggap = 0; 820 821 /* 822 * If remaining data exceeds the mtu, 823 * we know we need more fragment(s). 824 */ 825 datalen = length + fraggap; 826 if (datalen > mtu - fragheaderlen) 827 datalen = maxfraglen - fragheaderlen; 828 fraglen = datalen + fragheaderlen; 829 830 if ((flags & MSG_MORE) && 831 !(rt->u.dst.dev->features&NETIF_F_SG)) 832 alloclen = mtu; 833 else 834 alloclen = datalen + fragheaderlen; 835 836 /* The last fragment gets additional space at tail. 837 * Note, with MSG_MORE we overallocate on fragments, 838 * because we have no idea what fragment will be 839 * the last. 840 */ 841 if (datalen == length) 842 alloclen += rt->u.dst.trailer_len; 843 844 if (transhdrlen) { 845 skb = sock_alloc_send_skb(sk, 846 alloclen + hh_len + 15, 847 (flags & MSG_DONTWAIT), &err); 848 } else { 849 skb = NULL; 850 if (atomic_read(&sk->sk_wmem_alloc) <= 851 2 * sk->sk_sndbuf) 852 skb = sock_wmalloc(sk, 853 alloclen + hh_len + 15, 1, 854 sk->sk_allocation); 855 if (unlikely(skb == NULL)) 856 err = -ENOBUFS; 857 } 858 if (skb == NULL) 859 goto error; 860 861 /* 862 * Fill in the control structures 863 */ 864 skb->ip_summed = csummode; 865 skb->csum = 0; 866 skb_reserve(skb, hh_len); 867 868 /* 869 * Find where to start putting bytes. 870 */ 871 data = skb_put(skb, fraglen); 872 skb->nh.raw = data + exthdrlen; 873 data += fragheaderlen; 874 skb->h.raw = data + exthdrlen; 875 876 if (fraggap) { 877 skb->csum = skb_copy_and_csum_bits( 878 skb_prev, maxfraglen, 879 data + transhdrlen, fraggap, 0); 880 skb_prev->csum = csum_sub(skb_prev->csum, 881 skb->csum); 882 data += fraggap; 883 skb_trim(skb_prev, maxfraglen); 884 } 885 886 copy = datalen - transhdrlen - fraggap; 887 if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { 888 err = -EFAULT; 889 kfree_skb(skb); 890 goto error; 891 } 892 893 offset += copy; 894 length -= datalen - fraggap; 895 transhdrlen = 0; 896 exthdrlen = 0; 897 csummode = CHECKSUM_NONE; 898 899 /* 900 * Put the packet on the pending queue. 901 */ 902 __skb_queue_tail(&sk->sk_write_queue, skb); 903 continue; 904 } 905 906 if (copy > length) 907 copy = length; 908 909 if (!(rt->u.dst.dev->features&NETIF_F_SG)) { 910 unsigned int off; 911 912 off = skb->len; 913 if (getfrag(from, skb_put(skb, copy), 914 offset, copy, off, skb) < 0) { 915 __skb_trim(skb, off); 916 err = -EFAULT; 917 goto error; 918 } 919 } else { 920 int i = skb_shinfo(skb)->nr_frags; 921 skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1]; 922 struct page *page = sk->sk_sndmsg_page; 923 int off = sk->sk_sndmsg_off; 924 unsigned int left; 925 926 if (page && (left = PAGE_SIZE - off) > 0) { 927 if (copy >= left) 928 copy = left; 929 if (page != frag->page) { 930 if (i == MAX_SKB_FRAGS) { 931 err = -EMSGSIZE; 932 goto error; 933 } 934 get_page(page); 935 skb_fill_page_desc(skb, i, page, sk->sk_sndmsg_off, 0); 936 frag = &skb_shinfo(skb)->frags[i]; 937 } 938 } else if (i < MAX_SKB_FRAGS) { 939 if (copy > PAGE_SIZE) 940 copy = PAGE_SIZE; 941 page = alloc_pages(sk->sk_allocation, 0); 942 if (page == NULL) { 943 err = -ENOMEM; 944 goto error; 945 } 946 sk->sk_sndmsg_page = page; 947 sk->sk_sndmsg_off = 0; 948 949 skb_fill_page_desc(skb, i, page, 0, 0); 950 frag = &skb_shinfo(skb)->frags[i]; 951 skb->truesize += PAGE_SIZE; 952 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc); 953 } else { 954 err = -EMSGSIZE; 955 goto error; 956 } 957 if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) { 958 err = -EFAULT; 959 goto error; 960 } 961 sk->sk_sndmsg_off += copy; 962 frag->size += copy; 963 skb->len += copy; 964 skb->data_len += copy; 965 } 966 offset += copy; 967 length -= copy; 968 } 969 970 return 0; 971 972 error: 973 inet->cork.length -= length; 974 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS); 975 return err; 976 } 977 978 ssize_t ip_append_page(struct sock *sk, struct page *page, 979 int offset, size_t size, int flags) 980 { 981 struct inet_sock *inet = inet_sk(sk); 982 struct sk_buff *skb; 983 struct rtable *rt; 984 struct ip_options *opt = NULL; 985 int hh_len; 986 int mtu; 987 int len; 988 int err; 989 unsigned int maxfraglen, fragheaderlen, fraggap; 990 991 if (inet->hdrincl) 992 return -EPERM; 993 994 if (flags&MSG_PROBE) 995 return 0; 996 997 if (skb_queue_empty(&sk->sk_write_queue)) 998 return -EINVAL; 999 1000 rt = inet->cork.rt; 1001 if (inet->cork.flags & IPCORK_OPT) 1002 opt = inet->cork.opt; 1003 1004 if (!(rt->u.dst.dev->features&NETIF_F_SG)) 1005 return -EOPNOTSUPP; 1006 1007 hh_len = LL_RESERVED_SPACE(rt->u.dst.dev); 1008 mtu = inet->cork.fragsize; 1009 1010 fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0); 1011 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen; 1012 1013 if (inet->cork.length + size > 0xFFFF - fragheaderlen) { 1014 ip_local_error(sk, EMSGSIZE, rt->rt_dst, inet->dport, mtu); 1015 return -EMSGSIZE; 1016 } 1017 1018 if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) 1019 return -EINVAL; 1020 1021 inet->cork.length += size; 1022 1023 while (size > 0) { 1024 int i; 1025 1026 /* Check if the remaining data fits into current packet. */ 1027 len = mtu - skb->len; 1028 if (len < size) 1029 len = maxfraglen - skb->len; 1030 if (len <= 0) { 1031 struct sk_buff *skb_prev; 1032 char *data; 1033 struct iphdr *iph; 1034 int alloclen; 1035 1036 skb_prev = skb; 1037 if (skb_prev) 1038 fraggap = skb_prev->len - maxfraglen; 1039 else 1040 fraggap = 0; 1041 1042 alloclen = fragheaderlen + hh_len + fraggap + 15; 1043 skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation); 1044 if (unlikely(!skb)) { 1045 err = -ENOBUFS; 1046 goto error; 1047 } 1048 1049 /* 1050 * Fill in the control structures 1051 */ 1052 skb->ip_summed = CHECKSUM_NONE; 1053 skb->csum = 0; 1054 skb_reserve(skb, hh_len); 1055 1056 /* 1057 * Find where to start putting bytes. 1058 */ 1059 data = skb_put(skb, fragheaderlen + fraggap); 1060 skb->nh.iph = iph = (struct iphdr *)data; 1061 data += fragheaderlen; 1062 skb->h.raw = data; 1063 1064 if (fraggap) { 1065 skb->csum = skb_copy_and_csum_bits( 1066 skb_prev, maxfraglen, 1067 data, fraggap, 0); 1068 skb_prev->csum = csum_sub(skb_prev->csum, 1069 skb->csum); 1070 skb_trim(skb_prev, maxfraglen); 1071 } 1072 1073 /* 1074 * Put the packet on the pending queue. 1075 */ 1076 __skb_queue_tail(&sk->sk_write_queue, skb); 1077 continue; 1078 } 1079 1080 i = skb_shinfo(skb)->nr_frags; 1081 if (len > size) 1082 len = size; 1083 if (skb_can_coalesce(skb, i, page, offset)) { 1084 skb_shinfo(skb)->frags[i-1].size += len; 1085 } else if (i < MAX_SKB_FRAGS) { 1086 get_page(page); 1087 skb_fill_page_desc(skb, i, page, offset, len); 1088 } else { 1089 err = -EMSGSIZE; 1090 goto error; 1091 } 1092 1093 if (skb->ip_summed == CHECKSUM_NONE) { 1094 unsigned int csum; 1095 csum = csum_page(page, offset, len); 1096 skb->csum = csum_block_add(skb->csum, csum, skb->len); 1097 } 1098 1099 skb->len += len; 1100 skb->data_len += len; 1101 offset += len; 1102 size -= len; 1103 } 1104 return 0; 1105 1106 error: 1107 inet->cork.length -= size; 1108 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS); 1109 return err; 1110 } 1111 1112 /* 1113 * Combined all pending IP fragments on the socket as one IP datagram 1114 * and push them out. 1115 */ 1116 int ip_push_pending_frames(struct sock *sk) 1117 { 1118 struct sk_buff *skb, *tmp_skb; 1119 struct sk_buff **tail_skb; 1120 struct inet_sock *inet = inet_sk(sk); 1121 struct ip_options *opt = NULL; 1122 struct rtable *rt = inet->cork.rt; 1123 struct iphdr *iph; 1124 int df = 0; 1125 __u8 ttl; 1126 int err = 0; 1127 1128 if ((skb = __skb_dequeue(&sk->sk_write_queue)) == NULL) 1129 goto out; 1130 tail_skb = &(skb_shinfo(skb)->frag_list); 1131 1132 /* move skb->data to ip header from ext header */ 1133 if (skb->data < skb->nh.raw) 1134 __skb_pull(skb, skb->nh.raw - skb->data); 1135 while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { 1136 __skb_pull(tmp_skb, skb->h.raw - skb->nh.raw); 1137 *tail_skb = tmp_skb; 1138 tail_skb = &(tmp_skb->next); 1139 skb->len += tmp_skb->len; 1140 skb->data_len += tmp_skb->len; 1141 skb->truesize += tmp_skb->truesize; 1142 __sock_put(tmp_skb->sk); 1143 tmp_skb->destructor = NULL; 1144 tmp_skb->sk = NULL; 1145 } 1146 1147 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow 1148 * to fragment the frame generated here. No matter, what transforms 1149 * how transforms change size of the packet, it will come out. 1150 */ 1151 if (inet->pmtudisc != IP_PMTUDISC_DO) 1152 skb->local_df = 1; 1153 1154 /* DF bit is set when we want to see DF on outgoing frames. 1155 * If local_df is set too, we still allow to fragment this frame 1156 * locally. */ 1157 if (inet->pmtudisc == IP_PMTUDISC_DO || 1158 (skb->len <= dst_mtu(&rt->u.dst) && 1159 ip_dont_fragment(sk, &rt->u.dst))) 1160 df = htons(IP_DF); 1161 1162 if (inet->cork.flags & IPCORK_OPT) 1163 opt = inet->cork.opt; 1164 1165 if (rt->rt_type == RTN_MULTICAST) 1166 ttl = inet->mc_ttl; 1167 else 1168 ttl = ip_select_ttl(inet, &rt->u.dst); 1169 1170 iph = (struct iphdr *)skb->data; 1171 iph->version = 4; 1172 iph->ihl = 5; 1173 if (opt) { 1174 iph->ihl += opt->optlen>>2; 1175 ip_options_build(skb, opt, inet->cork.addr, rt, 0); 1176 } 1177 iph->tos = inet->tos; 1178 iph->tot_len = htons(skb->len); 1179 iph->frag_off = df; 1180 if (!df) { 1181 __ip_select_ident(iph, &rt->u.dst, 0); 1182 } else { 1183 iph->id = htons(inet->id++); 1184 } 1185 iph->ttl = ttl; 1186 iph->protocol = sk->sk_protocol; 1187 iph->saddr = rt->rt_src; 1188 iph->daddr = rt->rt_dst; 1189 ip_send_check(iph); 1190 1191 skb->priority = sk->sk_priority; 1192 skb->dst = dst_clone(&rt->u.dst); 1193 1194 /* Netfilter gets whole the not fragmented skb. */ 1195 err = NF_HOOK(PF_INET, NF_IP_LOCAL_OUT, skb, NULL, 1196 skb->dst->dev, dst_output); 1197 if (err) { 1198 if (err > 0) 1199 err = inet->recverr ? net_xmit_errno(err) : 0; 1200 if (err) 1201 goto error; 1202 } 1203 1204 out: 1205 inet->cork.flags &= ~IPCORK_OPT; 1206 if (inet->cork.opt) { 1207 kfree(inet->cork.opt); 1208 inet->cork.opt = NULL; 1209 } 1210 if (inet->cork.rt) { 1211 ip_rt_put(inet->cork.rt); 1212 inet->cork.rt = NULL; 1213 } 1214 return err; 1215 1216 error: 1217 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS); 1218 goto out; 1219 } 1220 1221 /* 1222 * Throw away all pending data on the socket. 1223 */ 1224 void ip_flush_pending_frames(struct sock *sk) 1225 { 1226 struct inet_sock *inet = inet_sk(sk); 1227 struct sk_buff *skb; 1228 1229 while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL) 1230 kfree_skb(skb); 1231 1232 inet->cork.flags &= ~IPCORK_OPT; 1233 if (inet->cork.opt) { 1234 kfree(inet->cork.opt); 1235 inet->cork.opt = NULL; 1236 } 1237 if (inet->cork.rt) { 1238 ip_rt_put(inet->cork.rt); 1239 inet->cork.rt = NULL; 1240 } 1241 } 1242 1243 1244 /* 1245 * Fetch data from kernel space and fill in checksum if needed. 1246 */ 1247 static int ip_reply_glue_bits(void *dptr, char *to, int offset, 1248 int len, int odd, struct sk_buff *skb) 1249 { 1250 unsigned int csum; 1251 1252 csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0); 1253 skb->csum = csum_block_add(skb->csum, csum, odd); 1254 return 0; 1255 } 1256 1257 /* 1258 * Generic function to send a packet as reply to another packet. 1259 * Used to send TCP resets so far. ICMP should use this function too. 1260 * 1261 * Should run single threaded per socket because it uses the sock 1262 * structure to pass arguments. 1263 * 1264 * LATER: switch from ip_build_xmit to ip_append_* 1265 */ 1266 void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *arg, 1267 unsigned int len) 1268 { 1269 struct inet_sock *inet = inet_sk(sk); 1270 struct { 1271 struct ip_options opt; 1272 char data[40]; 1273 } replyopts; 1274 struct ipcm_cookie ipc; 1275 u32 daddr; 1276 struct rtable *rt = (struct rtable*)skb->dst; 1277 1278 if (ip_options_echo(&replyopts.opt, skb)) 1279 return; 1280 1281 daddr = ipc.addr = rt->rt_src; 1282 ipc.opt = NULL; 1283 1284 if (replyopts.opt.optlen) { 1285 ipc.opt = &replyopts.opt; 1286 1287 if (ipc.opt->srr) 1288 daddr = replyopts.opt.faddr; 1289 } 1290 1291 { 1292 struct flowi fl = { .nl_u = { .ip4_u = 1293 { .daddr = daddr, 1294 .saddr = rt->rt_spec_dst, 1295 .tos = RT_TOS(skb->nh.iph->tos) } }, 1296 /* Not quite clean, but right. */ 1297 .uli_u = { .ports = 1298 { .sport = skb->h.th->dest, 1299 .dport = skb->h.th->source } }, 1300 .proto = sk->sk_protocol }; 1301 if (ip_route_output_key(&rt, &fl)) 1302 return; 1303 } 1304 1305 /* And let IP do all the hard work. 1306 1307 This chunk is not reenterable, hence spinlock. 1308 Note that it uses the fact, that this function is called 1309 with locally disabled BH and that sk cannot be already spinlocked. 1310 */ 1311 bh_lock_sock(sk); 1312 inet->tos = skb->nh.iph->tos; 1313 sk->sk_priority = skb->priority; 1314 sk->sk_protocol = skb->nh.iph->protocol; 1315 ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0, 1316 &ipc, rt, MSG_DONTWAIT); 1317 if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) { 1318 if (arg->csumoffset >= 0) 1319 *((u16 *)skb->h.raw + arg->csumoffset) = csum_fold(csum_add(skb->csum, arg->csum)); 1320 skb->ip_summed = CHECKSUM_NONE; 1321 ip_push_pending_frames(sk); 1322 } 1323 1324 bh_unlock_sock(sk); 1325 1326 ip_rt_put(rt); 1327 } 1328 1329 /* 1330 * IP protocol layer initialiser 1331 */ 1332 1333 static struct packet_type ip_packet_type = { 1334 .type = __constant_htons(ETH_P_IP), 1335 .func = ip_rcv, 1336 }; 1337 1338 /* 1339 * IP registers the packet type and then calls the subprotocol initialisers 1340 */ 1341 1342 void __init ip_init(void) 1343 { 1344 dev_add_pack(&ip_packet_type); 1345 1346 ip_rt_init(); 1347 inet_initpeers(); 1348 1349 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS) 1350 igmp_mc_proc_init(); 1351 #endif 1352 } 1353 1354 EXPORT_SYMBOL(ip_finish_output); 1355 EXPORT_SYMBOL(ip_fragment); 1356 EXPORT_SYMBOL(ip_generic_getfrag); 1357 EXPORT_SYMBOL(ip_queue_xmit); 1358 EXPORT_SYMBOL(ip_send_check); 1359 1360 #ifdef CONFIG_SYSCTL 1361 EXPORT_SYMBOL(sysctl_ip_default_ttl); 1362 #endif 1363