11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 31da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 41da177e4SLinus Torvalds * interface as the means of communication with the user level. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * The Internet Protocol (IP) output module. 71da177e4SLinus Torvalds * 802c30a84SJesper Juhl * Authors: Ross Biro 91da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 101da177e4SLinus Torvalds * Donald Becker, <becker@super.org> 111da177e4SLinus Torvalds * Alan Cox, <Alan.Cox@linux.org> 121da177e4SLinus Torvalds * Richard Underwood 131da177e4SLinus Torvalds * Stefan Becker, <stefanb@yello.ping.de> 141da177e4SLinus Torvalds * Jorge Cwik, <jorge@laser.satlink.net> 151da177e4SLinus Torvalds * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 161da177e4SLinus Torvalds * Hirokazu Takahashi, <taka@valinux.co.jp> 171da177e4SLinus Torvalds * 181da177e4SLinus Torvalds * See ip_input.c for original log 191da177e4SLinus Torvalds * 201da177e4SLinus Torvalds * Fixes: 211da177e4SLinus Torvalds * Alan Cox : Missing nonblock feature in ip_build_xmit. 221da177e4SLinus Torvalds * Mike Kilburn : htons() missing in ip_build_xmit. 231da177e4SLinus Torvalds * Bradford Johnson: Fix faulty handling of some frames when 241da177e4SLinus Torvalds * no route is found. 251da177e4SLinus Torvalds * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit 261da177e4SLinus Torvalds * (in case if packet not accepted by 271da177e4SLinus Torvalds * output firewall rules) 281da177e4SLinus Torvalds * Mike McLagan : Routing by source 291da177e4SLinus Torvalds * Alexey Kuznetsov: use new route cache 301da177e4SLinus Torvalds * Andi Kleen: Fix broken PMTU recovery and remove 311da177e4SLinus Torvalds * some redundant tests. 321da177e4SLinus Torvalds * Vitaly E. Lavrov : Transparent proxy revived after year coma. 331da177e4SLinus Torvalds * Andi Kleen : Replace ip_reply with ip_send_reply. 341da177e4SLinus Torvalds * Andi Kleen : Split fast and slow ip_build_xmit path 351da177e4SLinus Torvalds * for decreased register pressure on x86 361da177e4SLinus Torvalds * and more readibility. 371da177e4SLinus Torvalds * Marc Boucher : When call_out_firewall returns FW_QUEUE, 381da177e4SLinus Torvalds * silently drop skb instead of failing with -EPERM. 391da177e4SLinus Torvalds * Detlev Wengorz : Copy protocol for fragments. 401da177e4SLinus Torvalds * Hirokazu Takahashi: HW checksumming for outgoing UDP 411da177e4SLinus Torvalds * datagrams. 421da177e4SLinus Torvalds * Hirokazu Takahashi: sendfile() on UDP works now. 431da177e4SLinus Torvalds */ 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds #include <asm/uaccess.h> 461da177e4SLinus Torvalds #include <linux/module.h> 471da177e4SLinus Torvalds #include <linux/types.h> 481da177e4SLinus Torvalds #include <linux/kernel.h> 491da177e4SLinus Torvalds #include <linux/mm.h> 501da177e4SLinus Torvalds #include <linux/string.h> 511da177e4SLinus Torvalds #include <linux/errno.h> 52a1f8e7f7SAl Viro #include <linux/highmem.h> 535a0e3ad6STejun Heo #include <linux/slab.h> 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds #include <linux/socket.h> 561da177e4SLinus Torvalds #include <linux/sockios.h> 571da177e4SLinus Torvalds #include <linux/in.h> 581da177e4SLinus Torvalds #include <linux/inet.h> 591da177e4SLinus Torvalds #include <linux/netdevice.h> 601da177e4SLinus Torvalds #include <linux/etherdevice.h> 611da177e4SLinus Torvalds #include <linux/proc_fs.h> 621da177e4SLinus Torvalds #include <linux/stat.h> 631da177e4SLinus Torvalds #include <linux/init.h> 641da177e4SLinus Torvalds 651da177e4SLinus Torvalds #include <net/snmp.h> 661da177e4SLinus Torvalds #include <net/ip.h> 671da177e4SLinus Torvalds #include <net/protocol.h> 681da177e4SLinus Torvalds #include <net/route.h> 69cfacb057SPatrick McHardy #include <net/xfrm.h> 701da177e4SLinus Torvalds #include <linux/skbuff.h> 711da177e4SLinus Torvalds #include <net/sock.h> 721da177e4SLinus Torvalds #include <net/arp.h> 731da177e4SLinus Torvalds #include <net/icmp.h> 741da177e4SLinus Torvalds #include <net/checksum.h> 751da177e4SLinus Torvalds #include <net/inetpeer.h> 761da177e4SLinus Torvalds #include <linux/igmp.h> 771da177e4SLinus Torvalds #include <linux/netfilter_ipv4.h> 781da177e4SLinus Torvalds #include <linux/netfilter_bridge.h> 791da177e4SLinus Torvalds #include <linux/mroute.h> 801da177e4SLinus Torvalds #include <linux/netlink.h> 816cbb0df7SArnaldo Carvalho de Melo #include <linux/tcp.h> 821da177e4SLinus Torvalds 83ab32ea5dSBrian Haley int sysctl_ip_default_ttl __read_mostly = IPDEFTTL; 84323e126fSDavid S. Miller EXPORT_SYMBOL(sysctl_ip_default_ttl); 851da177e4SLinus Torvalds 861da177e4SLinus Torvalds /* Generate a checksum for an outgoing IP datagram. */ 872fbd9679SDenis Efremov void ip_send_check(struct iphdr *iph) 881da177e4SLinus Torvalds { 891da177e4SLinus Torvalds iph->check = 0; 901da177e4SLinus Torvalds iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 911da177e4SLinus Torvalds } 924bc2f18bSEric Dumazet EXPORT_SYMBOL(ip_send_check); 931da177e4SLinus Torvalds 94c439cb2eSHerbert Xu int __ip_local_out(struct sk_buff *skb) 95c439cb2eSHerbert Xu { 96c439cb2eSHerbert Xu struct iphdr *iph = ip_hdr(skb); 97c439cb2eSHerbert Xu 98c439cb2eSHerbert Xu iph->tot_len = htons(skb->len); 99c439cb2eSHerbert Xu ip_send_check(iph); 1009bbc768aSJan Engelhardt return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, skb, NULL, 1019bbc768aSJan Engelhardt skb_dst(skb)->dev, dst_output); 102c439cb2eSHerbert Xu } 103c439cb2eSHerbert Xu 104aad88724SEric Dumazet int ip_local_out_sk(struct sock *sk, struct sk_buff *skb) 105c439cb2eSHerbert Xu { 106c439cb2eSHerbert Xu int err; 107c439cb2eSHerbert Xu 108c439cb2eSHerbert Xu err = __ip_local_out(skb); 109c439cb2eSHerbert Xu if (likely(err == 1)) 110aad88724SEric Dumazet err = dst_output_sk(sk, skb); 111c439cb2eSHerbert Xu 112c439cb2eSHerbert Xu return err; 113c439cb2eSHerbert Xu } 114aad88724SEric Dumazet EXPORT_SYMBOL_GPL(ip_local_out_sk); 115c439cb2eSHerbert Xu 1161da177e4SLinus Torvalds static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst) 1171da177e4SLinus Torvalds { 1181da177e4SLinus Torvalds int ttl = inet->uc_ttl; 1191da177e4SLinus Torvalds 1201da177e4SLinus Torvalds if (ttl < 0) 121323e126fSDavid S. Miller ttl = ip4_dst_hoplimit(dst); 1221da177e4SLinus Torvalds return ttl; 1231da177e4SLinus Torvalds } 1241da177e4SLinus Torvalds 1251da177e4SLinus Torvalds /* 1261da177e4SLinus Torvalds * Add an ip header to a skbuff and send it out. 1271da177e4SLinus Torvalds * 1281da177e4SLinus Torvalds */ 1291da177e4SLinus Torvalds int ip_build_and_send_pkt(struct sk_buff *skb, struct sock *sk, 130f6d8bd05SEric Dumazet __be32 saddr, __be32 daddr, struct ip_options_rcu *opt) 1311da177e4SLinus Torvalds { 1321da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 133511c3f92SEric Dumazet struct rtable *rt = skb_rtable(skb); 1341da177e4SLinus Torvalds struct iphdr *iph; 1351da177e4SLinus Torvalds 1361da177e4SLinus Torvalds /* Build the IP header. */ 137f6d8bd05SEric Dumazet skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0)); 1388856dfa3SArnaldo Carvalho de Melo skb_reset_network_header(skb); 139eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(skb); 1401da177e4SLinus Torvalds iph->version = 4; 1411da177e4SLinus Torvalds iph->ihl = 5; 1421da177e4SLinus Torvalds iph->tos = inet->tos; 143d8d1f30bSChangli Gao if (ip_dont_fragment(sk, &rt->dst)) 1441da177e4SLinus Torvalds iph->frag_off = htons(IP_DF); 1451da177e4SLinus Torvalds else 1461da177e4SLinus Torvalds iph->frag_off = 0; 147d8d1f30bSChangli Gao iph->ttl = ip_select_ttl(inet, &rt->dst); 148dd927a26SDavid S. Miller iph->daddr = (opt && opt->opt.srr ? opt->opt.faddr : daddr); 149dd927a26SDavid S. Miller iph->saddr = saddr; 1501da177e4SLinus Torvalds iph->protocol = sk->sk_protocol; 151*b6a7719aSHannes Frederic Sowa ip_select_ident(sock_net(sk), skb, sk); 1521da177e4SLinus Torvalds 153f6d8bd05SEric Dumazet if (opt && opt->opt.optlen) { 154f6d8bd05SEric Dumazet iph->ihl += opt->opt.optlen>>2; 155f6d8bd05SEric Dumazet ip_options_build(skb, &opt->opt, daddr, rt, 0); 1561da177e4SLinus Torvalds } 1571da177e4SLinus Torvalds 1581da177e4SLinus Torvalds skb->priority = sk->sk_priority; 1594a19ec58SLaszlo Attila Toth skb->mark = sk->sk_mark; 1601da177e4SLinus Torvalds 1611da177e4SLinus Torvalds /* Send it out. */ 162c439cb2eSHerbert Xu return ip_local_out(skb); 1631da177e4SLinus Torvalds } 164d8c97a94SArnaldo Carvalho de Melo EXPORT_SYMBOL_GPL(ip_build_and_send_pkt); 165d8c97a94SArnaldo Carvalho de Melo 1661da177e4SLinus Torvalds static inline int ip_finish_output2(struct sk_buff *skb) 1671da177e4SLinus Torvalds { 168adf30907SEric Dumazet struct dst_entry *dst = skb_dst(skb); 16980787ebcSMitsuru Chinen struct rtable *rt = (struct rtable *)dst; 1701da177e4SLinus Torvalds struct net_device *dev = dst->dev; 171c2636b4dSChuck Lever unsigned int hh_len = LL_RESERVED_SPACE(dev); 172f6b72b62SDavid S. Miller struct neighbour *neigh; 173a263b309SDavid S. Miller u32 nexthop; 1741da177e4SLinus Torvalds 175edf391ffSNeil Horman if (rt->rt_type == RTN_MULTICAST) { 176edf391ffSNeil Horman IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len); 177edf391ffSNeil Horman } else if (rt->rt_type == RTN_BROADCAST) 178edf391ffSNeil Horman IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTBCAST, skb->len); 17980787ebcSMitsuru Chinen 1801da177e4SLinus Torvalds /* Be paranoid, rather than too clever. */ 1813b04dddeSStephen Hemminger if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { 1821da177e4SLinus Torvalds struct sk_buff *skb2; 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev)); 1851da177e4SLinus Torvalds if (skb2 == NULL) { 1861da177e4SLinus Torvalds kfree_skb(skb); 1871da177e4SLinus Torvalds return -ENOMEM; 1881da177e4SLinus Torvalds } 1891da177e4SLinus Torvalds if (skb->sk) 1901da177e4SLinus Torvalds skb_set_owner_w(skb2, skb->sk); 1915d0ba55bSEric Dumazet consume_skb(skb); 1921da177e4SLinus Torvalds skb = skb2; 1931da177e4SLinus Torvalds } 1941da177e4SLinus Torvalds 195a263b309SDavid S. Miller rcu_read_lock_bh(); 196155e8336SJulian Anastasov nexthop = (__force u32) rt_nexthop(rt, ip_hdr(skb)->daddr); 197a263b309SDavid S. Miller neigh = __ipv4_neigh_lookup_noref(dev, nexthop); 198a263b309SDavid S. Miller if (unlikely(!neigh)) 199a263b309SDavid S. Miller neigh = __neigh_create(&arp_tbl, &nexthop, dev, false); 2009871f1adSVasiliy Kulikov if (!IS_ERR(neigh)) { 2015110effeSDavid S. Miller int res = dst_neigh_output(dst, neigh, skb); 202f2c31e32SEric Dumazet 203a263b309SDavid S. Miller rcu_read_unlock_bh(); 204f2c31e32SEric Dumazet return res; 205f2c31e32SEric Dumazet } 206a263b309SDavid S. Miller rcu_read_unlock_bh(); 20705e3aa09SDavid S. Miller 208e87cc472SJoe Perches net_dbg_ratelimited("%s: No header cache and no neighbour!\n", 209e87cc472SJoe Perches __func__); 2101da177e4SLinus Torvalds kfree_skb(skb); 2111da177e4SLinus Torvalds return -EINVAL; 2121da177e4SLinus Torvalds } 2131da177e4SLinus Torvalds 214c7ba65d7SFlorian Westphal static int ip_finish_output_gso(struct sk_buff *skb) 215c7ba65d7SFlorian Westphal { 216c7ba65d7SFlorian Westphal netdev_features_t features; 217c7ba65d7SFlorian Westphal struct sk_buff *segs; 218c7ba65d7SFlorian Westphal int ret = 0; 219c7ba65d7SFlorian Westphal 220c7ba65d7SFlorian Westphal /* common case: locally created skb or seglen is <= mtu */ 221c7ba65d7SFlorian Westphal if (((IPCB(skb)->flags & IPSKB_FORWARDED) == 0) || 222c7ba65d7SFlorian Westphal skb_gso_network_seglen(skb) <= ip_skb_dst_mtu(skb)) 223c7ba65d7SFlorian Westphal return ip_finish_output2(skb); 224c7ba65d7SFlorian Westphal 225c7ba65d7SFlorian Westphal /* Slowpath - GSO segment length is exceeding the dst MTU. 226c7ba65d7SFlorian Westphal * 227c7ba65d7SFlorian Westphal * This can happen in two cases: 228c7ba65d7SFlorian Westphal * 1) TCP GRO packet, DF bit not set 229c7ba65d7SFlorian Westphal * 2) skb arrived via virtio-net, we thus get TSO/GSO skbs directly 230c7ba65d7SFlorian Westphal * from host network stack. 231c7ba65d7SFlorian Westphal */ 232c7ba65d7SFlorian Westphal features = netif_skb_features(skb); 233c7ba65d7SFlorian Westphal segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 234330966e5SFlorian Westphal if (IS_ERR_OR_NULL(segs)) { 235c7ba65d7SFlorian Westphal kfree_skb(skb); 236c7ba65d7SFlorian Westphal return -ENOMEM; 237c7ba65d7SFlorian Westphal } 238c7ba65d7SFlorian Westphal 239c7ba65d7SFlorian Westphal consume_skb(skb); 240c7ba65d7SFlorian Westphal 241c7ba65d7SFlorian Westphal do { 242c7ba65d7SFlorian Westphal struct sk_buff *nskb = segs->next; 243c7ba65d7SFlorian Westphal int err; 244c7ba65d7SFlorian Westphal 245c7ba65d7SFlorian Westphal segs->next = NULL; 246c7ba65d7SFlorian Westphal err = ip_fragment(segs, ip_finish_output2); 247c7ba65d7SFlorian Westphal 248c7ba65d7SFlorian Westphal if (err && ret == 0) 249c7ba65d7SFlorian Westphal ret = err; 250c7ba65d7SFlorian Westphal segs = nskb; 251c7ba65d7SFlorian Westphal } while (segs); 252c7ba65d7SFlorian Westphal 253c7ba65d7SFlorian Westphal return ret; 254c7ba65d7SFlorian Westphal } 255c7ba65d7SFlorian Westphal 256861d0486SPatrick McHardy static int ip_finish_output(struct sk_buff *skb) 2571da177e4SLinus Torvalds { 2585c901daaSPatrick McHardy #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM) 2595c901daaSPatrick McHardy /* Policy lookup after SNAT yielded a new policy */ 260adf30907SEric Dumazet if (skb_dst(skb)->xfrm != NULL) { 26148d5cad8SPatrick McHardy IPCB(skb)->flags |= IPSKB_REROUTED; 26248d5cad8SPatrick McHardy return dst_output(skb); 26348d5cad8SPatrick McHardy } 2645c901daaSPatrick McHardy #endif 265c7ba65d7SFlorian Westphal if (skb_is_gso(skb)) 266c7ba65d7SFlorian Westphal return ip_finish_output_gso(skb); 267c7ba65d7SFlorian Westphal 268c7ba65d7SFlorian Westphal if (skb->len > ip_skb_dst_mtu(skb)) 2691bd9bef6SPatrick McHardy return ip_fragment(skb, ip_finish_output2); 270c7ba65d7SFlorian Westphal 2711bd9bef6SPatrick McHardy return ip_finish_output2(skb); 2721da177e4SLinus Torvalds } 2731da177e4SLinus Torvalds 274aad88724SEric Dumazet int ip_mc_output(struct sock *sk, struct sk_buff *skb) 2751da177e4SLinus Torvalds { 276511c3f92SEric Dumazet struct rtable *rt = skb_rtable(skb); 277d8d1f30bSChangli Gao struct net_device *dev = rt->dst.dev; 2781da177e4SLinus Torvalds 2791da177e4SLinus Torvalds /* 2801da177e4SLinus Torvalds * If the indicated interface is up and running, send the packet. 2811da177e4SLinus Torvalds */ 282edf391ffSNeil Horman IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len); 2831da177e4SLinus Torvalds 2841da177e4SLinus Torvalds skb->dev = dev; 2851da177e4SLinus Torvalds skb->protocol = htons(ETH_P_IP); 2861da177e4SLinus Torvalds 2871da177e4SLinus Torvalds /* 2881da177e4SLinus Torvalds * Multicasts are looped back for other local users 2891da177e4SLinus Torvalds */ 2901da177e4SLinus Torvalds 2911da177e4SLinus Torvalds if (rt->rt_flags&RTCF_MULTICAST) { 2927ad6848cSOctavian Purdila if (sk_mc_loop(sk) 2931da177e4SLinus Torvalds #ifdef CONFIG_IP_MROUTE 2941da177e4SLinus Torvalds /* Small optimization: do not loopback not local frames, 2951da177e4SLinus Torvalds which returned after forwarding; they will be dropped 2961da177e4SLinus Torvalds by ip_mr_input in any case. 2971da177e4SLinus Torvalds Note, that local frames are looped back to be delivered 2981da177e4SLinus Torvalds to local recipients. 2991da177e4SLinus Torvalds 3001da177e4SLinus Torvalds This check is duplicated in ip_mr_input at the moment. 3011da177e4SLinus Torvalds */ 3029d4fb27dSJoe Perches && 3039d4fb27dSJoe Perches ((rt->rt_flags & RTCF_LOCAL) || 3049d4fb27dSJoe Perches !(IPCB(skb)->flags & IPSKB_FORWARDED)) 3051da177e4SLinus Torvalds #endif 3061da177e4SLinus Torvalds ) { 3071da177e4SLinus Torvalds struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 3081da177e4SLinus Torvalds if (newskb) 3099bbc768aSJan Engelhardt NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, 3109bbc768aSJan Engelhardt newskb, NULL, newskb->dev, 31195603e22SMichel Machado dev_loopback_xmit); 3121da177e4SLinus Torvalds } 3131da177e4SLinus Torvalds 3141da177e4SLinus Torvalds /* Multicasts with ttl 0 must not go beyond the host */ 3151da177e4SLinus Torvalds 316eddc9ec5SArnaldo Carvalho de Melo if (ip_hdr(skb)->ttl == 0) { 3171da177e4SLinus Torvalds kfree_skb(skb); 3181da177e4SLinus Torvalds return 0; 3191da177e4SLinus Torvalds } 3201da177e4SLinus Torvalds } 3211da177e4SLinus Torvalds 3221da177e4SLinus Torvalds if (rt->rt_flags&RTCF_BROADCAST) { 3231da177e4SLinus Torvalds struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC); 3241da177e4SLinus Torvalds if (newskb) 3259bbc768aSJan Engelhardt NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, newskb, 32695603e22SMichel Machado NULL, newskb->dev, dev_loopback_xmit); 3271da177e4SLinus Torvalds } 3281da177e4SLinus Torvalds 3299bbc768aSJan Engelhardt return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL, 3309bbc768aSJan Engelhardt skb->dev, ip_finish_output, 33148d5cad8SPatrick McHardy !(IPCB(skb)->flags & IPSKB_REROUTED)); 3321da177e4SLinus Torvalds } 3331da177e4SLinus Torvalds 334aad88724SEric Dumazet int ip_output(struct sock *sk, struct sk_buff *skb) 3351da177e4SLinus Torvalds { 336adf30907SEric Dumazet struct net_device *dev = skb_dst(skb)->dev; 3371bd9bef6SPatrick McHardy 338edf391ffSNeil Horman IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len); 3391da177e4SLinus Torvalds 3401bd9bef6SPatrick McHardy skb->dev = dev; 3411bd9bef6SPatrick McHardy skb->protocol = htons(ETH_P_IP); 3421bd9bef6SPatrick McHardy 3439bbc768aSJan Engelhardt return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL, dev, 34448d5cad8SPatrick McHardy ip_finish_output, 34548d5cad8SPatrick McHardy !(IPCB(skb)->flags & IPSKB_REROUTED)); 3461da177e4SLinus Torvalds } 3471da177e4SLinus Torvalds 34884f9307cSEric Dumazet /* 34984f9307cSEric Dumazet * copy saddr and daddr, possibly using 64bit load/stores 35084f9307cSEric Dumazet * Equivalent to : 35184f9307cSEric Dumazet * iph->saddr = fl4->saddr; 35284f9307cSEric Dumazet * iph->daddr = fl4->daddr; 35384f9307cSEric Dumazet */ 35484f9307cSEric Dumazet static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4) 35584f9307cSEric Dumazet { 35684f9307cSEric Dumazet BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) != 35784f9307cSEric Dumazet offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr)); 35884f9307cSEric Dumazet memcpy(&iph->saddr, &fl4->saddr, 35984f9307cSEric Dumazet sizeof(fl4->saddr) + sizeof(fl4->daddr)); 36084f9307cSEric Dumazet } 36184f9307cSEric Dumazet 362b0270e91SEric Dumazet /* Note: skb->sk can be different from sk, in case of tunnels */ 363b0270e91SEric Dumazet int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl) 3641da177e4SLinus Torvalds { 3651da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 366f6d8bd05SEric Dumazet struct ip_options_rcu *inet_opt; 367b57ae01aSDavid S. Miller struct flowi4 *fl4; 3681da177e4SLinus Torvalds struct rtable *rt; 3691da177e4SLinus Torvalds struct iphdr *iph; 370ab6e3febSEric Dumazet int res; 3711da177e4SLinus Torvalds 3721da177e4SLinus Torvalds /* Skip all of this if the packet is already routed, 3731da177e4SLinus Torvalds * f.e. by something like SCTP. 3741da177e4SLinus Torvalds */ 375ab6e3febSEric Dumazet rcu_read_lock(); 376f6d8bd05SEric Dumazet inet_opt = rcu_dereference(inet->inet_opt); 377ea4fc0d6SDavid S. Miller fl4 = &fl->u.ip4; 378511c3f92SEric Dumazet rt = skb_rtable(skb); 3791da177e4SLinus Torvalds if (rt != NULL) 3801da177e4SLinus Torvalds goto packet_routed; 3811da177e4SLinus Torvalds 3821da177e4SLinus Torvalds /* Make sure we can route this packet. */ 3831da177e4SLinus Torvalds rt = (struct rtable *)__sk_dst_check(sk, 0); 3841da177e4SLinus Torvalds if (rt == NULL) { 3853ca3c68eSAl Viro __be32 daddr; 3861da177e4SLinus Torvalds 3871da177e4SLinus Torvalds /* Use correct destination address if we have options. */ 388c720c7e8SEric Dumazet daddr = inet->inet_daddr; 389f6d8bd05SEric Dumazet if (inet_opt && inet_opt->opt.srr) 390f6d8bd05SEric Dumazet daddr = inet_opt->opt.faddr; 3911da177e4SLinus Torvalds 3921da177e4SLinus Torvalds /* If this fails, retransmit mechanism of transport layer will 3931da177e4SLinus Torvalds * keep trying until route appears or the connection times 3941da177e4SLinus Torvalds * itself out. 3951da177e4SLinus Torvalds */ 396b57ae01aSDavid S. Miller rt = ip_route_output_ports(sock_net(sk), fl4, sk, 39778fbfd8aSDavid S. Miller daddr, inet->inet_saddr, 39878fbfd8aSDavid S. Miller inet->inet_dport, 39978fbfd8aSDavid S. Miller inet->inet_sport, 40078fbfd8aSDavid S. Miller sk->sk_protocol, 40178fbfd8aSDavid S. Miller RT_CONN_FLAGS(sk), 40278fbfd8aSDavid S. Miller sk->sk_bound_dev_if); 403b23dd4feSDavid S. Miller if (IS_ERR(rt)) 4041da177e4SLinus Torvalds goto no_route; 405d8d1f30bSChangli Gao sk_setup_caps(sk, &rt->dst); 4061da177e4SLinus Torvalds } 407d8d1f30bSChangli Gao skb_dst_set_noref(skb, &rt->dst); 4081da177e4SLinus Torvalds 4091da177e4SLinus Torvalds packet_routed: 410155e8336SJulian Anastasov if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway) 4111da177e4SLinus Torvalds goto no_route; 4121da177e4SLinus Torvalds 4131da177e4SLinus Torvalds /* OK, we know where to send it, allocate and build IP header. */ 414f6d8bd05SEric Dumazet skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0)); 4158856dfa3SArnaldo Carvalho de Melo skb_reset_network_header(skb); 416eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(skb); 417714e85beSAl Viro *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff)); 41860ff7467SWANG Cong if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df) 4191da177e4SLinus Torvalds iph->frag_off = htons(IP_DF); 4201da177e4SLinus Torvalds else 4211da177e4SLinus Torvalds iph->frag_off = 0; 422d8d1f30bSChangli Gao iph->ttl = ip_select_ttl(inet, &rt->dst); 4231da177e4SLinus Torvalds iph->protocol = sk->sk_protocol; 42484f9307cSEric Dumazet ip_copy_addrs(iph, fl4); 42584f9307cSEric Dumazet 4261da177e4SLinus Torvalds /* Transport layer set skb->h.foo itself. */ 4271da177e4SLinus Torvalds 428f6d8bd05SEric Dumazet if (inet_opt && inet_opt->opt.optlen) { 429f6d8bd05SEric Dumazet iph->ihl += inet_opt->opt.optlen >> 2; 430f6d8bd05SEric Dumazet ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0); 4311da177e4SLinus Torvalds } 4321da177e4SLinus Torvalds 433*b6a7719aSHannes Frederic Sowa ip_select_ident_segs(sock_net(sk), skb, sk, 434*b6a7719aSHannes Frederic Sowa skb_shinfo(skb)->gso_segs ?: 1); 4351da177e4SLinus Torvalds 436b0270e91SEric Dumazet /* TODO : should we use skb->sk here instead of sk ? */ 4371da177e4SLinus Torvalds skb->priority = sk->sk_priority; 4384a19ec58SLaszlo Attila Toth skb->mark = sk->sk_mark; 4391da177e4SLinus Torvalds 440ab6e3febSEric Dumazet res = ip_local_out(skb); 441ab6e3febSEric Dumazet rcu_read_unlock(); 442ab6e3febSEric Dumazet return res; 4431da177e4SLinus Torvalds 4441da177e4SLinus Torvalds no_route: 445ab6e3febSEric Dumazet rcu_read_unlock(); 4465e38e270SPavel Emelyanov IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES); 4471da177e4SLinus Torvalds kfree_skb(skb); 4481da177e4SLinus Torvalds return -EHOSTUNREACH; 4491da177e4SLinus Torvalds } 4504bc2f18bSEric Dumazet EXPORT_SYMBOL(ip_queue_xmit); 4511da177e4SLinus Torvalds 4521da177e4SLinus Torvalds 4531da177e4SLinus Torvalds static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from) 4541da177e4SLinus Torvalds { 4551da177e4SLinus Torvalds to->pkt_type = from->pkt_type; 4561da177e4SLinus Torvalds to->priority = from->priority; 4571da177e4SLinus Torvalds to->protocol = from->protocol; 458adf30907SEric Dumazet skb_dst_drop(to); 459fe76cda3SEric Dumazet skb_dst_copy(to, from); 4601da177e4SLinus Torvalds to->dev = from->dev; 46182e91ffeSThomas Graf to->mark = from->mark; 4621da177e4SLinus Torvalds 4631da177e4SLinus Torvalds /* Copy the flags to each fragment. */ 4641da177e4SLinus Torvalds IPCB(to)->flags = IPCB(from)->flags; 4651da177e4SLinus Torvalds 4661da177e4SLinus Torvalds #ifdef CONFIG_NET_SCHED 4671da177e4SLinus Torvalds to->tc_index = from->tc_index; 4681da177e4SLinus Torvalds #endif 469e7ac05f3SYasuyuki Kozakai nf_copy(to, from); 470c98d80edSJulian Anastasov #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE) 471c98d80edSJulian Anastasov to->ipvs_property = from->ipvs_property; 472c98d80edSJulian Anastasov #endif 473984bc16cSJames Morris skb_copy_secmark(to, from); 4741da177e4SLinus Torvalds } 4751da177e4SLinus Torvalds 4761da177e4SLinus Torvalds /* 4771da177e4SLinus Torvalds * This IP datagram is too large to be sent in one piece. Break it up into 4781da177e4SLinus Torvalds * smaller pieces (each of size equal to IP header plus 4791da177e4SLinus Torvalds * a block of the data of the original IP data part) that will yet fit in a 4801da177e4SLinus Torvalds * single device frame, and queue such a frame for sending. 4811da177e4SLinus Torvalds */ 4821da177e4SLinus Torvalds 4832e2f7aefSPatrick McHardy int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *)) 4841da177e4SLinus Torvalds { 4851da177e4SLinus Torvalds struct iphdr *iph; 4861da177e4SLinus Torvalds int ptr; 4871da177e4SLinus Torvalds struct net_device *dev; 4881da177e4SLinus Torvalds struct sk_buff *skb2; 489c893b806SChangli Gao unsigned int mtu, hlen, left, len, ll_rs; 4901da177e4SLinus Torvalds int offset; 49176ab608dSAlexey Dobriyan __be16 not_last_frag; 492511c3f92SEric Dumazet struct rtable *rt = skb_rtable(skb); 4931da177e4SLinus Torvalds int err = 0; 4941da177e4SLinus Torvalds 495d8d1f30bSChangli Gao dev = rt->dst.dev; 4961da177e4SLinus Torvalds 4971da177e4SLinus Torvalds /* 4981da177e4SLinus Torvalds * Point into the IP datagram header. 4991da177e4SLinus Torvalds */ 5001da177e4SLinus Torvalds 501eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(skb); 5021da177e4SLinus Torvalds 50369647ce4SHannes Frederic Sowa mtu = ip_skb_dst_mtu(skb); 50460ff7467SWANG Cong if (unlikely(((iph->frag_off & htons(IP_DF)) && !skb->ignore_df) || 5055f2d04f1SPatrick McHardy (IPCB(skb)->frag_max_size && 506f87c10a8SHannes Frederic Sowa IPCB(skb)->frag_max_size > mtu))) { 5075e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS); 5081da177e4SLinus Torvalds icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, 509f87c10a8SHannes Frederic Sowa htonl(mtu)); 5101da177e4SLinus Torvalds kfree_skb(skb); 5111da177e4SLinus Torvalds return -EMSGSIZE; 5121da177e4SLinus Torvalds } 5131da177e4SLinus Torvalds 5141da177e4SLinus Torvalds /* 5151da177e4SLinus Torvalds * Setup starting values. 5161da177e4SLinus Torvalds */ 5171da177e4SLinus Torvalds 5181da177e4SLinus Torvalds hlen = iph->ihl * 4; 519f87c10a8SHannes Frederic Sowa mtu = mtu - hlen; /* Size of data space */ 5201109a90cSPablo Neira Ayuso #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 5216c79bf0fSBart De Schuymer if (skb->nf_bridge) 5226c79bf0fSBart De Schuymer mtu -= nf_bridge_mtu_reduction(skb); 5236c79bf0fSBart De Schuymer #endif 52489cee8b1SHerbert Xu IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE; 5251da177e4SLinus Torvalds 5261da177e4SLinus Torvalds /* When frag_list is given, use it. First, check its validity: 5271da177e4SLinus Torvalds * some transformers could create wrong frag_list or break existing 5281da177e4SLinus Torvalds * one, it is not prohibited. In this case fall back to copying. 5291da177e4SLinus Torvalds * 5301da177e4SLinus Torvalds * LATER: this step can be merged to real generation of fragments, 5311da177e4SLinus Torvalds * we can switch to copy when see the first bad fragment. 5321da177e4SLinus Torvalds */ 53321dc3301SDavid S. Miller if (skb_has_frag_list(skb)) { 5343d13008eSEric Dumazet struct sk_buff *frag, *frag2; 5351da177e4SLinus Torvalds int first_len = skb_pagelen(skb); 5361da177e4SLinus Torvalds 5371da177e4SLinus Torvalds if (first_len - hlen > mtu || 5381da177e4SLinus Torvalds ((first_len - hlen) & 7) || 53956f8a75cSPaul Gortmaker ip_is_fragment(iph) || 5401da177e4SLinus Torvalds skb_cloned(skb)) 5411da177e4SLinus Torvalds goto slow_path; 5421da177e4SLinus Torvalds 543d7fcf1a5SDavid S. Miller skb_walk_frags(skb, frag) { 5441da177e4SLinus Torvalds /* Correct geometry. */ 5451da177e4SLinus Torvalds if (frag->len > mtu || 5461da177e4SLinus Torvalds ((frag->len & 7) && frag->next) || 5471da177e4SLinus Torvalds skb_headroom(frag) < hlen) 5483d13008eSEric Dumazet goto slow_path_clean; 5491da177e4SLinus Torvalds 5501da177e4SLinus Torvalds /* Partially cloned skb? */ 5511da177e4SLinus Torvalds if (skb_shared(frag)) 5523d13008eSEric Dumazet goto slow_path_clean; 5532fdba6b0SHerbert Xu 5542fdba6b0SHerbert Xu BUG_ON(frag->sk); 5552fdba6b0SHerbert Xu if (skb->sk) { 5562fdba6b0SHerbert Xu frag->sk = skb->sk; 5572fdba6b0SHerbert Xu frag->destructor = sock_wfree; 5582fdba6b0SHerbert Xu } 5593d13008eSEric Dumazet skb->truesize -= frag->truesize; 5601da177e4SLinus Torvalds } 5611da177e4SLinus Torvalds 5621da177e4SLinus Torvalds /* Everything is OK. Generate! */ 5631da177e4SLinus Torvalds 5641da177e4SLinus Torvalds err = 0; 5651da177e4SLinus Torvalds offset = 0; 5661da177e4SLinus Torvalds frag = skb_shinfo(skb)->frag_list; 567d7fcf1a5SDavid S. Miller skb_frag_list_init(skb); 5681da177e4SLinus Torvalds skb->data_len = first_len - skb_headlen(skb); 5691da177e4SLinus Torvalds skb->len = first_len; 5701da177e4SLinus Torvalds iph->tot_len = htons(first_len); 5711da177e4SLinus Torvalds iph->frag_off = htons(IP_MF); 5721da177e4SLinus Torvalds ip_send_check(iph); 5731da177e4SLinus Torvalds 5741da177e4SLinus Torvalds for (;;) { 5751da177e4SLinus Torvalds /* Prepare header of the next frame, 5761da177e4SLinus Torvalds * before previous one went down. */ 5771da177e4SLinus Torvalds if (frag) { 5781da177e4SLinus Torvalds frag->ip_summed = CHECKSUM_NONE; 579badff6d0SArnaldo Carvalho de Melo skb_reset_transport_header(frag); 580e2d1bca7SArnaldo Carvalho de Melo __skb_push(frag, hlen); 581e2d1bca7SArnaldo Carvalho de Melo skb_reset_network_header(frag); 582d56f90a7SArnaldo Carvalho de Melo memcpy(skb_network_header(frag), iph, hlen); 583eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(frag); 5841da177e4SLinus Torvalds iph->tot_len = htons(frag->len); 5851da177e4SLinus Torvalds ip_copy_metadata(frag, skb); 5861da177e4SLinus Torvalds if (offset == 0) 5871da177e4SLinus Torvalds ip_options_fragment(frag); 5881da177e4SLinus Torvalds offset += skb->len - hlen; 5891da177e4SLinus Torvalds iph->frag_off = htons(offset>>3); 5901da177e4SLinus Torvalds if (frag->next != NULL) 5911da177e4SLinus Torvalds iph->frag_off |= htons(IP_MF); 5921da177e4SLinus Torvalds /* Ready, complete checksum */ 5931da177e4SLinus Torvalds ip_send_check(iph); 5941da177e4SLinus Torvalds } 5951da177e4SLinus Torvalds 5961da177e4SLinus Torvalds err = output(skb); 5971da177e4SLinus Torvalds 598dafee490SWei Dong if (!err) 5995e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES); 6001da177e4SLinus Torvalds if (err || !frag) 6011da177e4SLinus Torvalds break; 6021da177e4SLinus Torvalds 6031da177e4SLinus Torvalds skb = frag; 6041da177e4SLinus Torvalds frag = skb->next; 6051da177e4SLinus Torvalds skb->next = NULL; 6061da177e4SLinus Torvalds } 6071da177e4SLinus Torvalds 6081da177e4SLinus Torvalds if (err == 0) { 6095e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS); 6101da177e4SLinus Torvalds return 0; 6111da177e4SLinus Torvalds } 6121da177e4SLinus Torvalds 6131da177e4SLinus Torvalds while (frag) { 6141da177e4SLinus Torvalds skb = frag->next; 6151da177e4SLinus Torvalds kfree_skb(frag); 6161da177e4SLinus Torvalds frag = skb; 6171da177e4SLinus Torvalds } 6185e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS); 6191da177e4SLinus Torvalds return err; 6203d13008eSEric Dumazet 6213d13008eSEric Dumazet slow_path_clean: 6223d13008eSEric Dumazet skb_walk_frags(skb, frag2) { 6233d13008eSEric Dumazet if (frag2 == frag) 6243d13008eSEric Dumazet break; 6253d13008eSEric Dumazet frag2->sk = NULL; 6263d13008eSEric Dumazet frag2->destructor = NULL; 6273d13008eSEric Dumazet skb->truesize += frag2->truesize; 6283d13008eSEric Dumazet } 6291da177e4SLinus Torvalds } 6301da177e4SLinus Torvalds 6311da177e4SLinus Torvalds slow_path: 632fc70fb64SAlexander Duyck /* for offloaded checksums cleanup checksum before fragmentation */ 633fc70fb64SAlexander Duyck if ((skb->ip_summed == CHECKSUM_PARTIAL) && skb_checksum_help(skb)) 634fc70fb64SAlexander Duyck goto fail; 635c9af6db4SPravin B Shelar iph = ip_hdr(skb); 636fc70fb64SAlexander Duyck 6371da177e4SLinus Torvalds left = skb->len - hlen; /* Space per frame */ 63849085bd7SGeorge Kadianakis ptr = hlen; /* Where to start from */ 6391da177e4SLinus Torvalds 6408d045163SFlorian Westphal ll_rs = LL_RESERVED_SPACE(rt->dst.dev); 6419bcfcaf5SStephen Hemminger 6421da177e4SLinus Torvalds /* 6431da177e4SLinus Torvalds * Fragment the datagram. 6441da177e4SLinus Torvalds */ 6451da177e4SLinus Torvalds 6461da177e4SLinus Torvalds offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3; 6471da177e4SLinus Torvalds not_last_frag = iph->frag_off & htons(IP_MF); 6481da177e4SLinus Torvalds 6491da177e4SLinus Torvalds /* 6501da177e4SLinus Torvalds * Keep copying data until we run out. 6511da177e4SLinus Torvalds */ 6521da177e4SLinus Torvalds 6531da177e4SLinus Torvalds while (left > 0) { 6541da177e4SLinus Torvalds len = left; 6551da177e4SLinus Torvalds /* IF: it doesn't fit, use 'mtu' - the data space left */ 6561da177e4SLinus Torvalds if (len > mtu) 6571da177e4SLinus Torvalds len = mtu; 6581da177e4SLinus Torvalds /* IF: we are not sending up to and including the packet end 6591da177e4SLinus Torvalds then align the next start on an eight byte boundary */ 6601da177e4SLinus Torvalds if (len < left) { 6611da177e4SLinus Torvalds len &= ~7; 6621da177e4SLinus Torvalds } 6631da177e4SLinus Torvalds 664cbffccc9SJoe Perches /* Allocate buffer */ 665cbffccc9SJoe Perches skb2 = alloc_skb(len + hlen + ll_rs, GFP_ATOMIC); 666cbffccc9SJoe Perches if (!skb2) { 6671da177e4SLinus Torvalds err = -ENOMEM; 6681da177e4SLinus Torvalds goto fail; 6691da177e4SLinus Torvalds } 6701da177e4SLinus Torvalds 6711da177e4SLinus Torvalds /* 6721da177e4SLinus Torvalds * Set up data on packet 6731da177e4SLinus Torvalds */ 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds ip_copy_metadata(skb2, skb); 6761da177e4SLinus Torvalds skb_reserve(skb2, ll_rs); 6771da177e4SLinus Torvalds skb_put(skb2, len + hlen); 678c1d2bbe1SArnaldo Carvalho de Melo skb_reset_network_header(skb2); 679b0e380b1SArnaldo Carvalho de Melo skb2->transport_header = skb2->network_header + hlen; 6801da177e4SLinus Torvalds 6811da177e4SLinus Torvalds /* 6821da177e4SLinus Torvalds * Charge the memory for the fragment to any owner 6831da177e4SLinus Torvalds * it might possess 6841da177e4SLinus Torvalds */ 6851da177e4SLinus Torvalds 6861da177e4SLinus Torvalds if (skb->sk) 6871da177e4SLinus Torvalds skb_set_owner_w(skb2, skb->sk); 6881da177e4SLinus Torvalds 6891da177e4SLinus Torvalds /* 6901da177e4SLinus Torvalds * Copy the packet header into the new buffer. 6911da177e4SLinus Torvalds */ 6921da177e4SLinus Torvalds 693d626f62bSArnaldo Carvalho de Melo skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen); 6941da177e4SLinus Torvalds 6951da177e4SLinus Torvalds /* 6961da177e4SLinus Torvalds * Copy a block of the IP datagram. 6971da177e4SLinus Torvalds */ 698bff9b61cSArnaldo Carvalho de Melo if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len)) 6991da177e4SLinus Torvalds BUG(); 7001da177e4SLinus Torvalds left -= len; 7011da177e4SLinus Torvalds 7021da177e4SLinus Torvalds /* 7031da177e4SLinus Torvalds * Fill in the new header fields. 7041da177e4SLinus Torvalds */ 705eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(skb2); 7061da177e4SLinus Torvalds iph->frag_off = htons((offset >> 3)); 7071da177e4SLinus Torvalds 7081da177e4SLinus Torvalds /* ANK: dirty, but effective trick. Upgrade options only if 7091da177e4SLinus Torvalds * the segment to be fragmented was THE FIRST (otherwise, 7101da177e4SLinus Torvalds * options are already fixed) and make it ONCE 7111da177e4SLinus Torvalds * on the initial skb, so that all the following fragments 7121da177e4SLinus Torvalds * will inherit fixed options. 7131da177e4SLinus Torvalds */ 7141da177e4SLinus Torvalds if (offset == 0) 7151da177e4SLinus Torvalds ip_options_fragment(skb); 7161da177e4SLinus Torvalds 7171da177e4SLinus Torvalds /* 7181da177e4SLinus Torvalds * Added AC : If we are fragmenting a fragment that's not the 7191da177e4SLinus Torvalds * last fragment then keep MF on each bit 7201da177e4SLinus Torvalds */ 7211da177e4SLinus Torvalds if (left > 0 || not_last_frag) 7221da177e4SLinus Torvalds iph->frag_off |= htons(IP_MF); 7231da177e4SLinus Torvalds ptr += len; 7241da177e4SLinus Torvalds offset += len; 7251da177e4SLinus Torvalds 7261da177e4SLinus Torvalds /* 7271da177e4SLinus Torvalds * Put this fragment into the sending queue. 7281da177e4SLinus Torvalds */ 7291da177e4SLinus Torvalds iph->tot_len = htons(len + hlen); 7301da177e4SLinus Torvalds 7311da177e4SLinus Torvalds ip_send_check(iph); 7321da177e4SLinus Torvalds 7331da177e4SLinus Torvalds err = output(skb2); 7341da177e4SLinus Torvalds if (err) 7351da177e4SLinus Torvalds goto fail; 736dafee490SWei Dong 7375e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES); 7381da177e4SLinus Torvalds } 7395d0ba55bSEric Dumazet consume_skb(skb); 7405e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS); 7411da177e4SLinus Torvalds return err; 7421da177e4SLinus Torvalds 7431da177e4SLinus Torvalds fail: 7441da177e4SLinus Torvalds kfree_skb(skb); 7455e38e270SPavel Emelyanov IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS); 7461da177e4SLinus Torvalds return err; 7471da177e4SLinus Torvalds } 7482e2f7aefSPatrick McHardy EXPORT_SYMBOL(ip_fragment); 7492e2f7aefSPatrick McHardy 7501da177e4SLinus Torvalds int 7511da177e4SLinus Torvalds ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb) 7521da177e4SLinus Torvalds { 753f69e6d13SAl Viro struct msghdr *msg = from; 7541da177e4SLinus Torvalds 75584fa7933SPatrick McHardy if (skb->ip_summed == CHECKSUM_PARTIAL) { 75621226abbSAl Viro if (copy_from_iter(to, len, &msg->msg_iter) != len) 7571da177e4SLinus Torvalds return -EFAULT; 7581da177e4SLinus Torvalds } else { 75944bb9363SAl Viro __wsum csum = 0; 76021226abbSAl Viro if (csum_and_copy_from_iter(to, len, &csum, &msg->msg_iter) != len) 7611da177e4SLinus Torvalds return -EFAULT; 7621da177e4SLinus Torvalds skb->csum = csum_block_add(skb->csum, csum, odd); 7631da177e4SLinus Torvalds } 7641da177e4SLinus Torvalds return 0; 7651da177e4SLinus Torvalds } 7664bc2f18bSEric Dumazet EXPORT_SYMBOL(ip_generic_getfrag); 7671da177e4SLinus Torvalds 76844bb9363SAl Viro static inline __wsum 7691da177e4SLinus Torvalds csum_page(struct page *page, int offset, int copy) 7701da177e4SLinus Torvalds { 7711da177e4SLinus Torvalds char *kaddr; 77244bb9363SAl Viro __wsum csum; 7731da177e4SLinus Torvalds kaddr = kmap(page); 7741da177e4SLinus Torvalds csum = csum_partial(kaddr + offset, copy, 0); 7751da177e4SLinus Torvalds kunmap(page); 7761da177e4SLinus Torvalds return csum; 7771da177e4SLinus Torvalds } 7781da177e4SLinus Torvalds 7794b30b1c6SAdrian Bunk static inline int ip_ufo_append_data(struct sock *sk, 7801470ddf7SHerbert Xu struct sk_buff_head *queue, 781e89e9cf5SAnanda Raju int getfrag(void *from, char *to, int offset, int len, 782e89e9cf5SAnanda Raju int odd, struct sk_buff *skb), 783e89e9cf5SAnanda Raju void *from, int length, int hh_len, int fragheaderlen, 784d9be4f7aSBill Sommerfeld int transhdrlen, int maxfraglen, unsigned int flags) 785e89e9cf5SAnanda Raju { 786e89e9cf5SAnanda Raju struct sk_buff *skb; 787e89e9cf5SAnanda Raju int err; 788e89e9cf5SAnanda Raju 789e89e9cf5SAnanda Raju /* There is support for UDP fragmentation offload by network 790e89e9cf5SAnanda Raju * device, so create one single skb packet containing complete 791e89e9cf5SAnanda Raju * udp datagram 792e89e9cf5SAnanda Raju */ 7931470ddf7SHerbert Xu if ((skb = skb_peek_tail(queue)) == NULL) { 794e89e9cf5SAnanda Raju skb = sock_alloc_send_skb(sk, 795e89e9cf5SAnanda Raju hh_len + fragheaderlen + transhdrlen + 20, 796e89e9cf5SAnanda Raju (flags & MSG_DONTWAIT), &err); 797e89e9cf5SAnanda Raju 798e89e9cf5SAnanda Raju if (skb == NULL) 799e89e9cf5SAnanda Raju return err; 800e89e9cf5SAnanda Raju 801e89e9cf5SAnanda Raju /* reserve space for Hardware header */ 802e89e9cf5SAnanda Raju skb_reserve(skb, hh_len); 803e89e9cf5SAnanda Raju 804e89e9cf5SAnanda Raju /* create space for UDP/IP header */ 805e89e9cf5SAnanda Raju skb_put(skb, fragheaderlen + transhdrlen); 806e89e9cf5SAnanda Raju 807e89e9cf5SAnanda Raju /* initialize network header pointer */ 808c1d2bbe1SArnaldo Carvalho de Melo skb_reset_network_header(skb); 809e89e9cf5SAnanda Raju 810e89e9cf5SAnanda Raju /* initialize protocol header pointer */ 811b0e380b1SArnaldo Carvalho de Melo skb->transport_header = skb->network_header + fragheaderlen; 812e89e9cf5SAnanda Raju 813e89e9cf5SAnanda Raju skb->csum = 0; 814e89e9cf5SAnanda Raju 815e93b7d74SJiri Pirko 816e93b7d74SJiri Pirko __skb_queue_tail(queue, skb); 817e93b7d74SJiri Pirko } else if (skb_is_gso(skb)) { 818e93b7d74SJiri Pirko goto append; 819e93b7d74SJiri Pirko } 820e93b7d74SJiri Pirko 821e93b7d74SJiri Pirko skb->ip_summed = CHECKSUM_PARTIAL; 822e89e9cf5SAnanda Raju /* specify the length of each IP datagram fragment */ 823d9be4f7aSBill Sommerfeld skb_shinfo(skb)->gso_size = maxfraglen - fragheaderlen; 824f83ef8c0SHerbert Xu skb_shinfo(skb)->gso_type = SKB_GSO_UDP; 825be9164e7SKostya B 826e93b7d74SJiri Pirko append: 827be9164e7SKostya B return skb_append_datato_frags(sk, skb, getfrag, from, 828be9164e7SKostya B (length - transhdrlen)); 829e89e9cf5SAnanda Raju } 830e89e9cf5SAnanda Raju 831f5fca608SDavid S. Miller static int __ip_append_data(struct sock *sk, 832f5fca608SDavid S. Miller struct flowi4 *fl4, 833f5fca608SDavid S. Miller struct sk_buff_head *queue, 8341470ddf7SHerbert Xu struct inet_cork *cork, 8355640f768SEric Dumazet struct page_frag *pfrag, 8361470ddf7SHerbert Xu int getfrag(void *from, char *to, int offset, 8371470ddf7SHerbert Xu int len, int odd, struct sk_buff *skb), 8381da177e4SLinus Torvalds void *from, int length, int transhdrlen, 8391da177e4SLinus Torvalds unsigned int flags) 8401da177e4SLinus Torvalds { 8411da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 8421da177e4SLinus Torvalds struct sk_buff *skb; 8431da177e4SLinus Torvalds 84407df5294SHerbert Xu struct ip_options *opt = cork->opt; 8451da177e4SLinus Torvalds int hh_len; 8461da177e4SLinus Torvalds int exthdrlen; 8471da177e4SLinus Torvalds int mtu; 8481da177e4SLinus Torvalds int copy; 8491da177e4SLinus Torvalds int err; 8501da177e4SLinus Torvalds int offset = 0; 851daba287bSHannes Frederic Sowa unsigned int maxfraglen, fragheaderlen, maxnonfragsize; 8521da177e4SLinus Torvalds int csummode = CHECKSUM_NONE; 8531470ddf7SHerbert Xu struct rtable *rt = (struct rtable *)cork->dst; 85409c2d251SWillem de Bruijn u32 tskey = 0; 8551da177e4SLinus Torvalds 85696d7303eSSteffen Klassert skb = skb_peek_tail(queue); 85796d7303eSSteffen Klassert 85896d7303eSSteffen Klassert exthdrlen = !skb ? rt->dst.header_len : 0; 85907df5294SHerbert Xu mtu = cork->fragsize; 86009c2d251SWillem de Bruijn if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP && 86109c2d251SWillem de Bruijn sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) 86209c2d251SWillem de Bruijn tskey = sk->sk_tskey++; 8631470ddf7SHerbert Xu 864d8d1f30bSChangli Gao hh_len = LL_RESERVED_SPACE(rt->dst.dev); 8651da177e4SLinus Torvalds 8661da177e4SLinus Torvalds fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0); 8671da177e4SLinus Torvalds maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen; 86860ff7467SWANG Cong maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu; 8691da177e4SLinus Torvalds 870daba287bSHannes Frederic Sowa if (cork->length + length > maxnonfragsize - fragheaderlen) { 871f5fca608SDavid S. Miller ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, 87261e7f09dSHannes Frederic Sowa mtu - (opt ? opt->optlen : 0)); 8731da177e4SLinus Torvalds return -EMSGSIZE; 8741da177e4SLinus Torvalds } 8751da177e4SLinus Torvalds 8761da177e4SLinus Torvalds /* 8771da177e4SLinus Torvalds * transhdrlen > 0 means that this is the first fragment and we wish 8781da177e4SLinus Torvalds * it won't be fragmented in the future. 8791da177e4SLinus Torvalds */ 8801da177e4SLinus Torvalds if (transhdrlen && 8811da177e4SLinus Torvalds length + fragheaderlen <= mtu && 882d8d1f30bSChangli Gao rt->dst.dev->features & NETIF_F_V4_CSUM && 8831da177e4SLinus Torvalds !exthdrlen) 88484fa7933SPatrick McHardy csummode = CHECKSUM_PARTIAL; 8851da177e4SLinus Torvalds 8861470ddf7SHerbert Xu cork->length += length; 88726cde9f7SHerbert Xu if (((length > mtu) || (skb && skb_is_gso(skb))) && 888be9164e7SKostya B (sk->sk_protocol == IPPROTO_UDP) && 889acf8dd0aSMichal Kubeček (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len && 890acf8dd0aSMichal Kubeček (sk->sk_type == SOCK_DGRAM)) { 8911470ddf7SHerbert Xu err = ip_ufo_append_data(sk, queue, getfrag, from, length, 8921470ddf7SHerbert Xu hh_len, fragheaderlen, transhdrlen, 893d9be4f7aSBill Sommerfeld maxfraglen, flags); 894baa829d8SPatrick McHardy if (err) 895e89e9cf5SAnanda Raju goto error; 896e89e9cf5SAnanda Raju return 0; 897e89e9cf5SAnanda Raju } 8981da177e4SLinus Torvalds 8991da177e4SLinus Torvalds /* So, what's going on in the loop below? 9001da177e4SLinus Torvalds * 9011da177e4SLinus Torvalds * We use calculated fragment length to generate chained skb, 9021da177e4SLinus Torvalds * each of segments is IP fragment ready for sending to network after 9031da177e4SLinus Torvalds * adding appropriate IP header. 9041da177e4SLinus Torvalds */ 9051da177e4SLinus Torvalds 90626cde9f7SHerbert Xu if (!skb) 9071da177e4SLinus Torvalds goto alloc_new_skb; 9081da177e4SLinus Torvalds 9091da177e4SLinus Torvalds while (length > 0) { 9101da177e4SLinus Torvalds /* Check if the remaining data fits into current packet. */ 9111da177e4SLinus Torvalds copy = mtu - skb->len; 9121da177e4SLinus Torvalds if (copy < length) 9131da177e4SLinus Torvalds copy = maxfraglen - skb->len; 9141da177e4SLinus Torvalds if (copy <= 0) { 9151da177e4SLinus Torvalds char *data; 9161da177e4SLinus Torvalds unsigned int datalen; 9171da177e4SLinus Torvalds unsigned int fraglen; 9181da177e4SLinus Torvalds unsigned int fraggap; 9191da177e4SLinus Torvalds unsigned int alloclen; 9201da177e4SLinus Torvalds struct sk_buff *skb_prev; 9211da177e4SLinus Torvalds alloc_new_skb: 9221da177e4SLinus Torvalds skb_prev = skb; 9231da177e4SLinus Torvalds if (skb_prev) 9241da177e4SLinus Torvalds fraggap = skb_prev->len - maxfraglen; 9251da177e4SLinus Torvalds else 9261da177e4SLinus Torvalds fraggap = 0; 9271da177e4SLinus Torvalds 9281da177e4SLinus Torvalds /* 9291da177e4SLinus Torvalds * If remaining data exceeds the mtu, 9301da177e4SLinus Torvalds * we know we need more fragment(s). 9311da177e4SLinus Torvalds */ 9321da177e4SLinus Torvalds datalen = length + fraggap; 9331da177e4SLinus Torvalds if (datalen > mtu - fragheaderlen) 9341da177e4SLinus Torvalds datalen = maxfraglen - fragheaderlen; 9351da177e4SLinus Torvalds fraglen = datalen + fragheaderlen; 9361da177e4SLinus Torvalds 9371da177e4SLinus Torvalds if ((flags & MSG_MORE) && 938d8d1f30bSChangli Gao !(rt->dst.dev->features&NETIF_F_SG)) 9391da177e4SLinus Torvalds alloclen = mtu; 9401da177e4SLinus Torvalds else 94159104f06SEric Dumazet alloclen = fraglen; 9421da177e4SLinus Torvalds 943353e5c9aSSteffen Klassert alloclen += exthdrlen; 944353e5c9aSSteffen Klassert 9451da177e4SLinus Torvalds /* The last fragment gets additional space at tail. 9461da177e4SLinus Torvalds * Note, with MSG_MORE we overallocate on fragments, 9471da177e4SLinus Torvalds * because we have no idea what fragment will be 9481da177e4SLinus Torvalds * the last. 9491da177e4SLinus Torvalds */ 95033f99dc7SSteffen Klassert if (datalen == length + fraggap) 951d8d1f30bSChangli Gao alloclen += rt->dst.trailer_len; 95233f99dc7SSteffen Klassert 9531da177e4SLinus Torvalds if (transhdrlen) { 9541da177e4SLinus Torvalds skb = sock_alloc_send_skb(sk, 9551da177e4SLinus Torvalds alloclen + hh_len + 15, 9561da177e4SLinus Torvalds (flags & MSG_DONTWAIT), &err); 9571da177e4SLinus Torvalds } else { 9581da177e4SLinus Torvalds skb = NULL; 9591da177e4SLinus Torvalds if (atomic_read(&sk->sk_wmem_alloc) <= 9601da177e4SLinus Torvalds 2 * sk->sk_sndbuf) 9611da177e4SLinus Torvalds skb = sock_wmalloc(sk, 9621da177e4SLinus Torvalds alloclen + hh_len + 15, 1, 9631da177e4SLinus Torvalds sk->sk_allocation); 9641da177e4SLinus Torvalds if (unlikely(skb == NULL)) 9651da177e4SLinus Torvalds err = -ENOBUFS; 9661da177e4SLinus Torvalds } 9671da177e4SLinus Torvalds if (skb == NULL) 9681da177e4SLinus Torvalds goto error; 9691da177e4SLinus Torvalds 9701da177e4SLinus Torvalds /* 9711da177e4SLinus Torvalds * Fill in the control structures 9721da177e4SLinus Torvalds */ 9731da177e4SLinus Torvalds skb->ip_summed = csummode; 9741da177e4SLinus Torvalds skb->csum = 0; 9751da177e4SLinus Torvalds skb_reserve(skb, hh_len); 97611878b40SWillem de Bruijn 97711878b40SWillem de Bruijn /* only the initial fragment is time stamped */ 9781470ddf7SHerbert Xu skb_shinfo(skb)->tx_flags = cork->tx_flags; 97911878b40SWillem de Bruijn cork->tx_flags = 0; 98009c2d251SWillem de Bruijn skb_shinfo(skb)->tskey = tskey; 98109c2d251SWillem de Bruijn tskey = 0; 9821da177e4SLinus Torvalds 9831da177e4SLinus Torvalds /* 9841da177e4SLinus Torvalds * Find where to start putting bytes. 9851da177e4SLinus Torvalds */ 986353e5c9aSSteffen Klassert data = skb_put(skb, fraglen + exthdrlen); 987c14d2450SArnaldo Carvalho de Melo skb_set_network_header(skb, exthdrlen); 988b0e380b1SArnaldo Carvalho de Melo skb->transport_header = (skb->network_header + 989b0e380b1SArnaldo Carvalho de Melo fragheaderlen); 990353e5c9aSSteffen Klassert data += fragheaderlen + exthdrlen; 9911da177e4SLinus Torvalds 9921da177e4SLinus Torvalds if (fraggap) { 9931da177e4SLinus Torvalds skb->csum = skb_copy_and_csum_bits( 9941da177e4SLinus Torvalds skb_prev, maxfraglen, 9951da177e4SLinus Torvalds data + transhdrlen, fraggap, 0); 9961da177e4SLinus Torvalds skb_prev->csum = csum_sub(skb_prev->csum, 9971da177e4SLinus Torvalds skb->csum); 9981da177e4SLinus Torvalds data += fraggap; 999e9fa4f7bSHerbert Xu pskb_trim_unique(skb_prev, maxfraglen); 10001da177e4SLinus Torvalds } 10011da177e4SLinus Torvalds 10021da177e4SLinus Torvalds copy = datalen - transhdrlen - fraggap; 10031da177e4SLinus Torvalds if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { 10041da177e4SLinus Torvalds err = -EFAULT; 10051da177e4SLinus Torvalds kfree_skb(skb); 10061da177e4SLinus Torvalds goto error; 10071da177e4SLinus Torvalds } 10081da177e4SLinus Torvalds 10091da177e4SLinus Torvalds offset += copy; 10101da177e4SLinus Torvalds length -= datalen - fraggap; 10111da177e4SLinus Torvalds transhdrlen = 0; 10121da177e4SLinus Torvalds exthdrlen = 0; 10131da177e4SLinus Torvalds csummode = CHECKSUM_NONE; 10141da177e4SLinus Torvalds 10151da177e4SLinus Torvalds /* 10161da177e4SLinus Torvalds * Put the packet on the pending queue. 10171da177e4SLinus Torvalds */ 10181470ddf7SHerbert Xu __skb_queue_tail(queue, skb); 10191da177e4SLinus Torvalds continue; 10201da177e4SLinus Torvalds } 10211da177e4SLinus Torvalds 10221da177e4SLinus Torvalds if (copy > length) 10231da177e4SLinus Torvalds copy = length; 10241da177e4SLinus Torvalds 1025d8d1f30bSChangli Gao if (!(rt->dst.dev->features&NETIF_F_SG)) { 10261da177e4SLinus Torvalds unsigned int off; 10271da177e4SLinus Torvalds 10281da177e4SLinus Torvalds off = skb->len; 10291da177e4SLinus Torvalds if (getfrag(from, skb_put(skb, copy), 10301da177e4SLinus Torvalds offset, copy, off, skb) < 0) { 10311da177e4SLinus Torvalds __skb_trim(skb, off); 10321da177e4SLinus Torvalds err = -EFAULT; 10331da177e4SLinus Torvalds goto error; 10341da177e4SLinus Torvalds } 10351da177e4SLinus Torvalds } else { 10361da177e4SLinus Torvalds int i = skb_shinfo(skb)->nr_frags; 10371da177e4SLinus Torvalds 10381da177e4SLinus Torvalds err = -ENOMEM; 10395640f768SEric Dumazet if (!sk_page_frag_refill(sk, pfrag)) 10401da177e4SLinus Torvalds goto error; 10411da177e4SLinus Torvalds 10425640f768SEric Dumazet if (!skb_can_coalesce(skb, i, pfrag->page, 10435640f768SEric Dumazet pfrag->offset)) { 10441da177e4SLinus Torvalds err = -EMSGSIZE; 10455640f768SEric Dumazet if (i == MAX_SKB_FRAGS) 10461da177e4SLinus Torvalds goto error; 10475640f768SEric Dumazet 10485640f768SEric Dumazet __skb_fill_page_desc(skb, i, pfrag->page, 10495640f768SEric Dumazet pfrag->offset, 0); 10505640f768SEric Dumazet skb_shinfo(skb)->nr_frags = ++i; 10515640f768SEric Dumazet get_page(pfrag->page); 10521da177e4SLinus Torvalds } 10535640f768SEric Dumazet copy = min_t(int, copy, pfrag->size - pfrag->offset); 10545640f768SEric Dumazet if (getfrag(from, 10555640f768SEric Dumazet page_address(pfrag->page) + pfrag->offset, 10565640f768SEric Dumazet offset, copy, skb->len, skb) < 0) 10575640f768SEric Dumazet goto error_efault; 10585640f768SEric Dumazet 10595640f768SEric Dumazet pfrag->offset += copy; 10605640f768SEric Dumazet skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 10611da177e4SLinus Torvalds skb->len += copy; 10621da177e4SLinus Torvalds skb->data_len += copy; 1063f945fa7aSHerbert Xu skb->truesize += copy; 1064f945fa7aSHerbert Xu atomic_add(copy, &sk->sk_wmem_alloc); 10651da177e4SLinus Torvalds } 10661da177e4SLinus Torvalds offset += copy; 10671da177e4SLinus Torvalds length -= copy; 10681da177e4SLinus Torvalds } 10691da177e4SLinus Torvalds 10701da177e4SLinus Torvalds return 0; 10711da177e4SLinus Torvalds 10725640f768SEric Dumazet error_efault: 10735640f768SEric Dumazet err = -EFAULT; 10741da177e4SLinus Torvalds error: 10751470ddf7SHerbert Xu cork->length -= length; 10765e38e270SPavel Emelyanov IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); 10771da177e4SLinus Torvalds return err; 10781da177e4SLinus Torvalds } 10791da177e4SLinus Torvalds 10801470ddf7SHerbert Xu static int ip_setup_cork(struct sock *sk, struct inet_cork *cork, 10811470ddf7SHerbert Xu struct ipcm_cookie *ipc, struct rtable **rtp) 10821470ddf7SHerbert Xu { 1083f6d8bd05SEric Dumazet struct ip_options_rcu *opt; 10841470ddf7SHerbert Xu struct rtable *rt; 10851470ddf7SHerbert Xu 10861470ddf7SHerbert Xu /* 10871470ddf7SHerbert Xu * setup for corking. 10881470ddf7SHerbert Xu */ 10891470ddf7SHerbert Xu opt = ipc->opt; 10901470ddf7SHerbert Xu if (opt) { 10911470ddf7SHerbert Xu if (cork->opt == NULL) { 10921470ddf7SHerbert Xu cork->opt = kmalloc(sizeof(struct ip_options) + 40, 10931470ddf7SHerbert Xu sk->sk_allocation); 10941470ddf7SHerbert Xu if (unlikely(cork->opt == NULL)) 10951470ddf7SHerbert Xu return -ENOBUFS; 10961470ddf7SHerbert Xu } 1097f6d8bd05SEric Dumazet memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen); 10981470ddf7SHerbert Xu cork->flags |= IPCORK_OPT; 10991470ddf7SHerbert Xu cork->addr = ipc->addr; 11001470ddf7SHerbert Xu } 11011470ddf7SHerbert Xu rt = *rtp; 11021470ddf7SHerbert Xu if (unlikely(!rt)) 11031470ddf7SHerbert Xu return -EFAULT; 11041470ddf7SHerbert Xu /* 11051470ddf7SHerbert Xu * We steal reference to this route, caller should not release it 11061470ddf7SHerbert Xu */ 11071470ddf7SHerbert Xu *rtp = NULL; 1108482fc609SHannes Frederic Sowa cork->fragsize = ip_sk_use_pmtu(sk) ? 1109482fc609SHannes Frederic Sowa dst_mtu(&rt->dst) : rt->dst.dev->mtu; 11101470ddf7SHerbert Xu cork->dst = &rt->dst; 11111470ddf7SHerbert Xu cork->length = 0; 1112aa661581SFrancesco Fusco cork->ttl = ipc->ttl; 1113aa661581SFrancesco Fusco cork->tos = ipc->tos; 1114aa661581SFrancesco Fusco cork->priority = ipc->priority; 11151470ddf7SHerbert Xu cork->tx_flags = ipc->tx_flags; 11161470ddf7SHerbert Xu 11171470ddf7SHerbert Xu return 0; 11181470ddf7SHerbert Xu } 11191470ddf7SHerbert Xu 11201470ddf7SHerbert Xu /* 11211470ddf7SHerbert Xu * ip_append_data() and ip_append_page() can make one large IP datagram 11221470ddf7SHerbert Xu * from many pieces of data. Each pieces will be holded on the socket 11231470ddf7SHerbert Xu * until ip_push_pending_frames() is called. Each piece can be a page 11241470ddf7SHerbert Xu * or non-page data. 11251470ddf7SHerbert Xu * 11261470ddf7SHerbert Xu * Not only UDP, other transport protocols - e.g. raw sockets - can use 11271470ddf7SHerbert Xu * this interface potentially. 11281470ddf7SHerbert Xu * 11291470ddf7SHerbert Xu * LATER: length must be adjusted by pad at tail, when it is required. 11301470ddf7SHerbert Xu */ 1131f5fca608SDavid S. Miller int ip_append_data(struct sock *sk, struct flowi4 *fl4, 11321470ddf7SHerbert Xu int getfrag(void *from, char *to, int offset, int len, 11331470ddf7SHerbert Xu int odd, struct sk_buff *skb), 11341470ddf7SHerbert Xu void *from, int length, int transhdrlen, 11351470ddf7SHerbert Xu struct ipcm_cookie *ipc, struct rtable **rtp, 11361470ddf7SHerbert Xu unsigned int flags) 11371470ddf7SHerbert Xu { 11381470ddf7SHerbert Xu struct inet_sock *inet = inet_sk(sk); 11391470ddf7SHerbert Xu int err; 11401470ddf7SHerbert Xu 11411470ddf7SHerbert Xu if (flags&MSG_PROBE) 11421470ddf7SHerbert Xu return 0; 11431470ddf7SHerbert Xu 11441470ddf7SHerbert Xu if (skb_queue_empty(&sk->sk_write_queue)) { 1145bdc712b4SDavid S. Miller err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp); 11461470ddf7SHerbert Xu if (err) 11471470ddf7SHerbert Xu return err; 11481470ddf7SHerbert Xu } else { 11491470ddf7SHerbert Xu transhdrlen = 0; 11501470ddf7SHerbert Xu } 11511470ddf7SHerbert Xu 11525640f768SEric Dumazet return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, 11535640f768SEric Dumazet sk_page_frag(sk), getfrag, 11541470ddf7SHerbert Xu from, length, transhdrlen, flags); 11551470ddf7SHerbert Xu } 11561470ddf7SHerbert Xu 1157f5fca608SDavid S. Miller ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page, 11581da177e4SLinus Torvalds int offset, size_t size, int flags) 11591da177e4SLinus Torvalds { 11601da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 11611da177e4SLinus Torvalds struct sk_buff *skb; 11621da177e4SLinus Torvalds struct rtable *rt; 11631da177e4SLinus Torvalds struct ip_options *opt = NULL; 1164bdc712b4SDavid S. Miller struct inet_cork *cork; 11651da177e4SLinus Torvalds int hh_len; 11661da177e4SLinus Torvalds int mtu; 11671da177e4SLinus Torvalds int len; 11681da177e4SLinus Torvalds int err; 1169daba287bSHannes Frederic Sowa unsigned int maxfraglen, fragheaderlen, fraggap, maxnonfragsize; 11701da177e4SLinus Torvalds 11711da177e4SLinus Torvalds if (inet->hdrincl) 11721da177e4SLinus Torvalds return -EPERM; 11731da177e4SLinus Torvalds 11741da177e4SLinus Torvalds if (flags&MSG_PROBE) 11751da177e4SLinus Torvalds return 0; 11761da177e4SLinus Torvalds 11771da177e4SLinus Torvalds if (skb_queue_empty(&sk->sk_write_queue)) 11781da177e4SLinus Torvalds return -EINVAL; 11791da177e4SLinus Torvalds 1180bdc712b4SDavid S. Miller cork = &inet->cork.base; 1181bdc712b4SDavid S. Miller rt = (struct rtable *)cork->dst; 1182bdc712b4SDavid S. Miller if (cork->flags & IPCORK_OPT) 1183bdc712b4SDavid S. Miller opt = cork->opt; 11841da177e4SLinus Torvalds 1185d8d1f30bSChangli Gao if (!(rt->dst.dev->features&NETIF_F_SG)) 11861da177e4SLinus Torvalds return -EOPNOTSUPP; 11871da177e4SLinus Torvalds 1188d8d1f30bSChangli Gao hh_len = LL_RESERVED_SPACE(rt->dst.dev); 1189bdc712b4SDavid S. Miller mtu = cork->fragsize; 11901da177e4SLinus Torvalds 11911da177e4SLinus Torvalds fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0); 11921da177e4SLinus Torvalds maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen; 119360ff7467SWANG Cong maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu; 11941da177e4SLinus Torvalds 1195daba287bSHannes Frederic Sowa if (cork->length + size > maxnonfragsize - fragheaderlen) { 119661e7f09dSHannes Frederic Sowa ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, 119761e7f09dSHannes Frederic Sowa mtu - (opt ? opt->optlen : 0)); 11981da177e4SLinus Torvalds return -EMSGSIZE; 11991da177e4SLinus Torvalds } 12001da177e4SLinus Torvalds 12011da177e4SLinus Torvalds if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) 12021da177e4SLinus Torvalds return -EINVAL; 12031da177e4SLinus Torvalds 1204bdc712b4SDavid S. Miller cork->length += size; 120526cde9f7SHerbert Xu if ((size + skb->len > mtu) && 120626cde9f7SHerbert Xu (sk->sk_protocol == IPPROTO_UDP) && 1207d8d1f30bSChangli Gao (rt->dst.dev->features & NETIF_F_UFO)) { 12087967168cSHerbert Xu skb_shinfo(skb)->gso_size = mtu - fragheaderlen; 1209f83ef8c0SHerbert Xu skb_shinfo(skb)->gso_type = SKB_GSO_UDP; 12107967168cSHerbert Xu } 1211e89e9cf5SAnanda Raju 12121da177e4SLinus Torvalds 12131da177e4SLinus Torvalds while (size > 0) { 12141da177e4SLinus Torvalds int i; 12151da177e4SLinus Torvalds 121689114afdSHerbert Xu if (skb_is_gso(skb)) 1217e89e9cf5SAnanda Raju len = size; 1218e89e9cf5SAnanda Raju else { 1219e89e9cf5SAnanda Raju 12201da177e4SLinus Torvalds /* Check if the remaining data fits into current packet. */ 12211da177e4SLinus Torvalds len = mtu - skb->len; 12221da177e4SLinus Torvalds if (len < size) 12231da177e4SLinus Torvalds len = maxfraglen - skb->len; 1224e89e9cf5SAnanda Raju } 12251da177e4SLinus Torvalds if (len <= 0) { 12261da177e4SLinus Torvalds struct sk_buff *skb_prev; 12271da177e4SLinus Torvalds int alloclen; 12281da177e4SLinus Torvalds 12291da177e4SLinus Torvalds skb_prev = skb; 12301da177e4SLinus Torvalds fraggap = skb_prev->len - maxfraglen; 12311da177e4SLinus Torvalds 12321da177e4SLinus Torvalds alloclen = fragheaderlen + hh_len + fraggap + 15; 12331da177e4SLinus Torvalds skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation); 12341da177e4SLinus Torvalds if (unlikely(!skb)) { 12351da177e4SLinus Torvalds err = -ENOBUFS; 12361da177e4SLinus Torvalds goto error; 12371da177e4SLinus Torvalds } 12381da177e4SLinus Torvalds 12391da177e4SLinus Torvalds /* 12401da177e4SLinus Torvalds * Fill in the control structures 12411da177e4SLinus Torvalds */ 12421da177e4SLinus Torvalds skb->ip_summed = CHECKSUM_NONE; 12431da177e4SLinus Torvalds skb->csum = 0; 12441da177e4SLinus Torvalds skb_reserve(skb, hh_len); 12451da177e4SLinus Torvalds 12461da177e4SLinus Torvalds /* 12471da177e4SLinus Torvalds * Find where to start putting bytes. 12481da177e4SLinus Torvalds */ 1249967b05f6SArnaldo Carvalho de Melo skb_put(skb, fragheaderlen + fraggap); 12502ca9e6f2SArnaldo Carvalho de Melo skb_reset_network_header(skb); 1251b0e380b1SArnaldo Carvalho de Melo skb->transport_header = (skb->network_header + 1252b0e380b1SArnaldo Carvalho de Melo fragheaderlen); 12531da177e4SLinus Torvalds if (fraggap) { 1254967b05f6SArnaldo Carvalho de Melo skb->csum = skb_copy_and_csum_bits(skb_prev, 1255967b05f6SArnaldo Carvalho de Melo maxfraglen, 12569c70220bSArnaldo Carvalho de Melo skb_transport_header(skb), 1257967b05f6SArnaldo Carvalho de Melo fraggap, 0); 12581da177e4SLinus Torvalds skb_prev->csum = csum_sub(skb_prev->csum, 12591da177e4SLinus Torvalds skb->csum); 1260e9fa4f7bSHerbert Xu pskb_trim_unique(skb_prev, maxfraglen); 12611da177e4SLinus Torvalds } 12621da177e4SLinus Torvalds 12631da177e4SLinus Torvalds /* 12641da177e4SLinus Torvalds * Put the packet on the pending queue. 12651da177e4SLinus Torvalds */ 12661da177e4SLinus Torvalds __skb_queue_tail(&sk->sk_write_queue, skb); 12671da177e4SLinus Torvalds continue; 12681da177e4SLinus Torvalds } 12691da177e4SLinus Torvalds 12701da177e4SLinus Torvalds i = skb_shinfo(skb)->nr_frags; 12711da177e4SLinus Torvalds if (len > size) 12721da177e4SLinus Torvalds len = size; 12731da177e4SLinus Torvalds if (skb_can_coalesce(skb, i, page, offset)) { 12749e903e08SEric Dumazet skb_frag_size_add(&skb_shinfo(skb)->frags[i-1], len); 12751da177e4SLinus Torvalds } else if (i < MAX_SKB_FRAGS) { 12761da177e4SLinus Torvalds get_page(page); 12771da177e4SLinus Torvalds skb_fill_page_desc(skb, i, page, offset, len); 12781da177e4SLinus Torvalds } else { 12791da177e4SLinus Torvalds err = -EMSGSIZE; 12801da177e4SLinus Torvalds goto error; 12811da177e4SLinus Torvalds } 12821da177e4SLinus Torvalds 12831da177e4SLinus Torvalds if (skb->ip_summed == CHECKSUM_NONE) { 128444bb9363SAl Viro __wsum csum; 12851da177e4SLinus Torvalds csum = csum_page(page, offset, len); 12861da177e4SLinus Torvalds skb->csum = csum_block_add(skb->csum, csum, skb->len); 12871da177e4SLinus Torvalds } 12881da177e4SLinus Torvalds 12891da177e4SLinus Torvalds skb->len += len; 12901da177e4SLinus Torvalds skb->data_len += len; 12911e34a11dSDavid S. Miller skb->truesize += len; 12921e34a11dSDavid S. Miller atomic_add(len, &sk->sk_wmem_alloc); 12931da177e4SLinus Torvalds offset += len; 12941da177e4SLinus Torvalds size -= len; 12951da177e4SLinus Torvalds } 12961da177e4SLinus Torvalds return 0; 12971da177e4SLinus Torvalds 12981da177e4SLinus Torvalds error: 1299bdc712b4SDavid S. Miller cork->length -= size; 13005e38e270SPavel Emelyanov IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); 13011da177e4SLinus Torvalds return err; 13021da177e4SLinus Torvalds } 13031da177e4SLinus Torvalds 13041470ddf7SHerbert Xu static void ip_cork_release(struct inet_cork *cork) 1305429f08e9SPavel Emelyanov { 13061470ddf7SHerbert Xu cork->flags &= ~IPCORK_OPT; 13071470ddf7SHerbert Xu kfree(cork->opt); 13081470ddf7SHerbert Xu cork->opt = NULL; 13091470ddf7SHerbert Xu dst_release(cork->dst); 13101470ddf7SHerbert Xu cork->dst = NULL; 1311429f08e9SPavel Emelyanov } 1312429f08e9SPavel Emelyanov 13131da177e4SLinus Torvalds /* 13141da177e4SLinus Torvalds * Combined all pending IP fragments on the socket as one IP datagram 13151da177e4SLinus Torvalds * and push them out. 13161da177e4SLinus Torvalds */ 13171c32c5adSHerbert Xu struct sk_buff *__ip_make_skb(struct sock *sk, 131877968b78SDavid S. Miller struct flowi4 *fl4, 13191470ddf7SHerbert Xu struct sk_buff_head *queue, 13201470ddf7SHerbert Xu struct inet_cork *cork) 13211da177e4SLinus Torvalds { 13221da177e4SLinus Torvalds struct sk_buff *skb, *tmp_skb; 13231da177e4SLinus Torvalds struct sk_buff **tail_skb; 13241da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 13250388b004SPavel Emelyanov struct net *net = sock_net(sk); 13261da177e4SLinus Torvalds struct ip_options *opt = NULL; 13271470ddf7SHerbert Xu struct rtable *rt = (struct rtable *)cork->dst; 13281da177e4SLinus Torvalds struct iphdr *iph; 132976ab608dSAlexey Dobriyan __be16 df = 0; 13301da177e4SLinus Torvalds __u8 ttl; 13311da177e4SLinus Torvalds 13321470ddf7SHerbert Xu if ((skb = __skb_dequeue(queue)) == NULL) 13331da177e4SLinus Torvalds goto out; 13341da177e4SLinus Torvalds tail_skb = &(skb_shinfo(skb)->frag_list); 13351da177e4SLinus Torvalds 13361da177e4SLinus Torvalds /* move skb->data to ip header from ext header */ 1337d56f90a7SArnaldo Carvalho de Melo if (skb->data < skb_network_header(skb)) 1338bbe735e4SArnaldo Carvalho de Melo __skb_pull(skb, skb_network_offset(skb)); 13391470ddf7SHerbert Xu while ((tmp_skb = __skb_dequeue(queue)) != NULL) { 1340cfe1fc77SArnaldo Carvalho de Melo __skb_pull(tmp_skb, skb_network_header_len(skb)); 13411da177e4SLinus Torvalds *tail_skb = tmp_skb; 13421da177e4SLinus Torvalds tail_skb = &(tmp_skb->next); 13431da177e4SLinus Torvalds skb->len += tmp_skb->len; 13441da177e4SLinus Torvalds skb->data_len += tmp_skb->len; 13451da177e4SLinus Torvalds skb->truesize += tmp_skb->truesize; 13461da177e4SLinus Torvalds tmp_skb->destructor = NULL; 13471da177e4SLinus Torvalds tmp_skb->sk = NULL; 13481da177e4SLinus Torvalds } 13491da177e4SLinus Torvalds 13501da177e4SLinus Torvalds /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow 13511da177e4SLinus Torvalds * to fragment the frame generated here. No matter, what transforms 13521da177e4SLinus Torvalds * how transforms change size of the packet, it will come out. 13531da177e4SLinus Torvalds */ 135460ff7467SWANG Cong skb->ignore_df = ip_sk_ignore_df(sk); 13551da177e4SLinus Torvalds 13561da177e4SLinus Torvalds /* DF bit is set when we want to see DF on outgoing frames. 135760ff7467SWANG Cong * If ignore_df is set too, we still allow to fragment this frame 13581da177e4SLinus Torvalds * locally. */ 1359482fc609SHannes Frederic Sowa if (inet->pmtudisc == IP_PMTUDISC_DO || 1360482fc609SHannes Frederic Sowa inet->pmtudisc == IP_PMTUDISC_PROBE || 1361d8d1f30bSChangli Gao (skb->len <= dst_mtu(&rt->dst) && 1362d8d1f30bSChangli Gao ip_dont_fragment(sk, &rt->dst))) 13631da177e4SLinus Torvalds df = htons(IP_DF); 13641da177e4SLinus Torvalds 13651470ddf7SHerbert Xu if (cork->flags & IPCORK_OPT) 13661470ddf7SHerbert Xu opt = cork->opt; 13671da177e4SLinus Torvalds 1368aa661581SFrancesco Fusco if (cork->ttl != 0) 1369aa661581SFrancesco Fusco ttl = cork->ttl; 1370aa661581SFrancesco Fusco else if (rt->rt_type == RTN_MULTICAST) 13711da177e4SLinus Torvalds ttl = inet->mc_ttl; 13721da177e4SLinus Torvalds else 1373d8d1f30bSChangli Gao ttl = ip_select_ttl(inet, &rt->dst); 13741da177e4SLinus Torvalds 1375749154aaSAnsis Atteka iph = ip_hdr(skb); 13761da177e4SLinus Torvalds iph->version = 4; 13771da177e4SLinus Torvalds iph->ihl = 5; 1378aa661581SFrancesco Fusco iph->tos = (cork->tos != -1) ? cork->tos : inet->tos; 13791da177e4SLinus Torvalds iph->frag_off = df; 13801da177e4SLinus Torvalds iph->ttl = ttl; 13811da177e4SLinus Torvalds iph->protocol = sk->sk_protocol; 138284f9307cSEric Dumazet ip_copy_addrs(iph, fl4); 1383*b6a7719aSHannes Frederic Sowa ip_select_ident(net, skb, sk); 13841da177e4SLinus Torvalds 138522f728f8SDavid S. Miller if (opt) { 138622f728f8SDavid S. Miller iph->ihl += opt->optlen>>2; 138722f728f8SDavid S. Miller ip_options_build(skb, opt, cork->addr, rt, 0); 138822f728f8SDavid S. Miller } 138922f728f8SDavid S. Miller 1390aa661581SFrancesco Fusco skb->priority = (cork->tos != -1) ? cork->priority: sk->sk_priority; 13914a19ec58SLaszlo Attila Toth skb->mark = sk->sk_mark; 1392a21bba94SEric Dumazet /* 1393a21bba94SEric Dumazet * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec 1394a21bba94SEric Dumazet * on dst refcount 1395a21bba94SEric Dumazet */ 13961470ddf7SHerbert Xu cork->dst = NULL; 1397d8d1f30bSChangli Gao skb_dst_set(skb, &rt->dst); 13981da177e4SLinus Torvalds 139996793b48SDavid L Stevens if (iph->protocol == IPPROTO_ICMP) 14000388b004SPavel Emelyanov icmp_out_count(net, ((struct icmphdr *) 140196793b48SDavid L Stevens skb_transport_header(skb))->type); 140296793b48SDavid L Stevens 14031c32c5adSHerbert Xu ip_cork_release(cork); 14041c32c5adSHerbert Xu out: 14051c32c5adSHerbert Xu return skb; 14061c32c5adSHerbert Xu } 14071c32c5adSHerbert Xu 1408b5ec8eeaSEric Dumazet int ip_send_skb(struct net *net, struct sk_buff *skb) 14091c32c5adSHerbert Xu { 14101c32c5adSHerbert Xu int err; 14111c32c5adSHerbert Xu 1412c439cb2eSHerbert Xu err = ip_local_out(skb); 14131da177e4SLinus Torvalds if (err) { 14141da177e4SLinus Torvalds if (err > 0) 14156ce9e7b5SEric Dumazet err = net_xmit_errno(err); 14161da177e4SLinus Torvalds if (err) 14171c32c5adSHerbert Xu IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS); 14181da177e4SLinus Torvalds } 14191da177e4SLinus Torvalds 14201da177e4SLinus Torvalds return err; 14211da177e4SLinus Torvalds } 14221da177e4SLinus Torvalds 142377968b78SDavid S. Miller int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4) 14241470ddf7SHerbert Xu { 14251c32c5adSHerbert Xu struct sk_buff *skb; 14261c32c5adSHerbert Xu 142777968b78SDavid S. Miller skb = ip_finish_skb(sk, fl4); 14281c32c5adSHerbert Xu if (!skb) 14291c32c5adSHerbert Xu return 0; 14301c32c5adSHerbert Xu 14311c32c5adSHerbert Xu /* Netfilter gets whole the not fragmented skb. */ 1432b5ec8eeaSEric Dumazet return ip_send_skb(sock_net(sk), skb); 14331470ddf7SHerbert Xu } 14341470ddf7SHerbert Xu 14351da177e4SLinus Torvalds /* 14361da177e4SLinus Torvalds * Throw away all pending data on the socket. 14371da177e4SLinus Torvalds */ 14381470ddf7SHerbert Xu static void __ip_flush_pending_frames(struct sock *sk, 14391470ddf7SHerbert Xu struct sk_buff_head *queue, 14401470ddf7SHerbert Xu struct inet_cork *cork) 14411da177e4SLinus Torvalds { 14421da177e4SLinus Torvalds struct sk_buff *skb; 14431da177e4SLinus Torvalds 14441470ddf7SHerbert Xu while ((skb = __skb_dequeue_tail(queue)) != NULL) 14451da177e4SLinus Torvalds kfree_skb(skb); 14461da177e4SLinus Torvalds 14471470ddf7SHerbert Xu ip_cork_release(cork); 14481470ddf7SHerbert Xu } 14491470ddf7SHerbert Xu 14501470ddf7SHerbert Xu void ip_flush_pending_frames(struct sock *sk) 14511470ddf7SHerbert Xu { 1452bdc712b4SDavid S. Miller __ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base); 14531da177e4SLinus Torvalds } 14541da177e4SLinus Torvalds 14551c32c5adSHerbert Xu struct sk_buff *ip_make_skb(struct sock *sk, 145677968b78SDavid S. Miller struct flowi4 *fl4, 14571c32c5adSHerbert Xu int getfrag(void *from, char *to, int offset, 14581c32c5adSHerbert Xu int len, int odd, struct sk_buff *skb), 14591c32c5adSHerbert Xu void *from, int length, int transhdrlen, 14601c32c5adSHerbert Xu struct ipcm_cookie *ipc, struct rtable **rtp, 14611c32c5adSHerbert Xu unsigned int flags) 14621c32c5adSHerbert Xu { 1463b80d7226SDavid S. Miller struct inet_cork cork; 14641c32c5adSHerbert Xu struct sk_buff_head queue; 14651c32c5adSHerbert Xu int err; 14661c32c5adSHerbert Xu 14671c32c5adSHerbert Xu if (flags & MSG_PROBE) 14681c32c5adSHerbert Xu return NULL; 14691c32c5adSHerbert Xu 14701c32c5adSHerbert Xu __skb_queue_head_init(&queue); 14711c32c5adSHerbert Xu 1472b80d7226SDavid S. Miller cork.flags = 0; 1473b80d7226SDavid S. Miller cork.addr = 0; 147470652728SDavid S. Miller cork.opt = NULL; 14751c32c5adSHerbert Xu err = ip_setup_cork(sk, &cork, ipc, rtp); 14761c32c5adSHerbert Xu if (err) 14771c32c5adSHerbert Xu return ERR_PTR(err); 14781c32c5adSHerbert Xu 14795640f768SEric Dumazet err = __ip_append_data(sk, fl4, &queue, &cork, 14805640f768SEric Dumazet ¤t->task_frag, getfrag, 14811c32c5adSHerbert Xu from, length, transhdrlen, flags); 14821c32c5adSHerbert Xu if (err) { 14831c32c5adSHerbert Xu __ip_flush_pending_frames(sk, &queue, &cork); 14841c32c5adSHerbert Xu return ERR_PTR(err); 14851c32c5adSHerbert Xu } 14861c32c5adSHerbert Xu 148777968b78SDavid S. Miller return __ip_make_skb(sk, fl4, &queue, &cork); 14881c32c5adSHerbert Xu } 14891da177e4SLinus Torvalds 14901da177e4SLinus Torvalds /* 14911da177e4SLinus Torvalds * Fetch data from kernel space and fill in checksum if needed. 14921da177e4SLinus Torvalds */ 14931da177e4SLinus Torvalds static int ip_reply_glue_bits(void *dptr, char *to, int offset, 14941da177e4SLinus Torvalds int len, int odd, struct sk_buff *skb) 14951da177e4SLinus Torvalds { 14965084205fSAl Viro __wsum csum; 14971da177e4SLinus Torvalds 14981da177e4SLinus Torvalds csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0); 14991da177e4SLinus Torvalds skb->csum = csum_block_add(skb->csum, csum, odd); 15001da177e4SLinus Torvalds return 0; 15011da177e4SLinus Torvalds } 15021da177e4SLinus Torvalds 15031da177e4SLinus Torvalds /* 15041da177e4SLinus Torvalds * Generic function to send a packet as reply to another packet. 1505be9f4a44SEric Dumazet * Used to send some TCP resets/acks so far. 15061da177e4SLinus Torvalds */ 1507bdbbb852SEric Dumazet void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb, 150824a2d43dSEric Dumazet const struct ip_options *sopt, 150924a2d43dSEric Dumazet __be32 daddr, __be32 saddr, 151024a2d43dSEric Dumazet const struct ip_reply_arg *arg, 151170e73416SDavid S. Miller unsigned int len) 15121da177e4SLinus Torvalds { 1513f6d8bd05SEric Dumazet struct ip_options_data replyopts; 15141da177e4SLinus Torvalds struct ipcm_cookie ipc; 151577968b78SDavid S. Miller struct flowi4 fl4; 1516511c3f92SEric Dumazet struct rtable *rt = skb_rtable(skb); 1517bdbbb852SEric Dumazet struct net *net = sock_net(sk); 1518be9f4a44SEric Dumazet struct sk_buff *nskb; 15194062090eSVasily Averin int err; 15201da177e4SLinus Torvalds 152124a2d43dSEric Dumazet if (__ip_options_echo(&replyopts.opt.opt, skb, sopt)) 15221da177e4SLinus Torvalds return; 15231da177e4SLinus Torvalds 15240a5ebb80SDavid S. Miller ipc.addr = daddr; 15251da177e4SLinus Torvalds ipc.opt = NULL; 15262244d07bSOliver Hartkopp ipc.tx_flags = 0; 1527aa661581SFrancesco Fusco ipc.ttl = 0; 1528aa661581SFrancesco Fusco ipc.tos = -1; 15291da177e4SLinus Torvalds 1530f6d8bd05SEric Dumazet if (replyopts.opt.opt.optlen) { 15311da177e4SLinus Torvalds ipc.opt = &replyopts.opt; 15321da177e4SLinus Torvalds 1533f6d8bd05SEric Dumazet if (replyopts.opt.opt.srr) 1534f6d8bd05SEric Dumazet daddr = replyopts.opt.opt.faddr; 15351da177e4SLinus Torvalds } 15361da177e4SLinus Torvalds 1537e110861fSLorenzo Colitti flowi4_init_output(&fl4, arg->bound_dev_if, 1538e110861fSLorenzo Colitti IP4_REPLY_MARK(net, skb->mark), 153966b13d99SEric Dumazet RT_TOS(arg->tos), 1540be9f4a44SEric Dumazet RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol, 1541538de0e0SDavid S. Miller ip_reply_arg_flowi_flags(arg), 154270e73416SDavid S. Miller daddr, saddr, 1543538de0e0SDavid S. Miller tcp_hdr(skb)->source, tcp_hdr(skb)->dest); 15449d6ec938SDavid S. Miller security_skb_classify_flow(skb, flowi4_to_flowi(&fl4)); 1545be9f4a44SEric Dumazet rt = ip_route_output_key(net, &fl4); 1546b23dd4feSDavid S. Miller if (IS_ERR(rt)) 15471da177e4SLinus Torvalds return; 15481da177e4SLinus Torvalds 1549bdbbb852SEric Dumazet inet_sk(sk)->tos = arg->tos; 15501da177e4SLinus Torvalds 15511da177e4SLinus Torvalds sk->sk_priority = skb->priority; 1552eddc9ec5SArnaldo Carvalho de Melo sk->sk_protocol = ip_hdr(skb)->protocol; 1553f0e48dbfSPatrick McHardy sk->sk_bound_dev_if = arg->bound_dev_if; 1554be9f4a44SEric Dumazet sk->sk_sndbuf = sysctl_wmem_default; 15554062090eSVasily Averin err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, 15564062090eSVasily Averin len, 0, &ipc, &rt, MSG_DONTWAIT); 15574062090eSVasily Averin if (unlikely(err)) { 15584062090eSVasily Averin ip_flush_pending_frames(sk); 15594062090eSVasily Averin goto out; 15604062090eSVasily Averin } 15614062090eSVasily Averin 1562be9f4a44SEric Dumazet nskb = skb_peek(&sk->sk_write_queue); 1563be9f4a44SEric Dumazet if (nskb) { 15641da177e4SLinus Torvalds if (arg->csumoffset >= 0) 1565be9f4a44SEric Dumazet *((__sum16 *)skb_transport_header(nskb) + 1566be9f4a44SEric Dumazet arg->csumoffset) = csum_fold(csum_add(nskb->csum, 15679c70220bSArnaldo Carvalho de Melo arg->csum)); 1568be9f4a44SEric Dumazet nskb->ip_summed = CHECKSUM_NONE; 1569be9f4a44SEric Dumazet skb_set_queue_mapping(nskb, skb_get_queue_mapping(skb)); 157077968b78SDavid S. Miller ip_push_pending_frames(sk, &fl4); 15711da177e4SLinus Torvalds } 15724062090eSVasily Averin out: 15731da177e4SLinus Torvalds ip_rt_put(rt); 15741da177e4SLinus Torvalds } 15751da177e4SLinus Torvalds 15761da177e4SLinus Torvalds void __init ip_init(void) 15771da177e4SLinus Torvalds { 15781da177e4SLinus Torvalds ip_rt_init(); 15791da177e4SLinus Torvalds inet_initpeers(); 15801da177e4SLinus Torvalds 158172c1d3bdSWANG Cong #if defined(CONFIG_IP_MULTICAST) 158272c1d3bdSWANG Cong igmp_mc_init(); 15831da177e4SLinus Torvalds #endif 15841da177e4SLinus Torvalds } 1585