1 /* 2 * IPV4 GSO/GRO offload support 3 * Linux INET implementation 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 8 * 2 of the License, or (at your option) any later version. 9 * 10 * UDPv4 GSO support 11 */ 12 13 #include <linux/skbuff.h> 14 #include <net/udp.h> 15 #include <net/protocol.h> 16 17 static DEFINE_SPINLOCK(udp_offload_lock); 18 static struct udp_offload_priv __rcu *udp_offload_base __read_mostly; 19 20 #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock)) 21 22 struct udp_offload_priv { 23 struct udp_offload *offload; 24 struct rcu_head rcu; 25 struct udp_offload_priv __rcu *next; 26 }; 27 28 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, 29 netdev_features_t features, 30 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, 31 netdev_features_t features), 32 __be16 new_protocol, bool is_ipv6) 33 { 34 struct sk_buff *segs = ERR_PTR(-EINVAL); 35 u16 mac_offset = skb->mac_header; 36 int mac_len = skb->mac_len; 37 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); 38 __be16 protocol = skb->protocol; 39 netdev_features_t enc_features; 40 int udp_offset, outer_hlen; 41 unsigned int oldlen; 42 bool need_csum = !!(skb_shinfo(skb)->gso_type & 43 SKB_GSO_UDP_TUNNEL_CSUM); 44 bool offload_csum = false, dont_encap = need_csum; 45 46 oldlen = (u16)~skb->len; 47 48 if (unlikely(!pskb_may_pull(skb, tnl_hlen))) 49 goto out; 50 51 skb->encapsulation = 0; 52 __skb_pull(skb, tnl_hlen); 53 skb_reset_mac_header(skb); 54 skb_set_network_header(skb, skb_inner_network_offset(skb)); 55 skb->mac_len = skb_inner_network_offset(skb); 56 skb->protocol = new_protocol; 57 skb->encap_hdr_csum = need_csum; 58 59 /* Try to offload checksum if possible */ 60 offload_csum = !!(need_csum && 61 (skb->dev->features & 62 (is_ipv6 ? NETIF_F_V6_CSUM : NETIF_F_V4_CSUM))); 63 64 /* segment inner packet. */ 65 enc_features = skb->dev->hw_enc_features & features; 66 segs = gso_inner_segment(skb, enc_features); 67 if (IS_ERR_OR_NULL(segs)) { 68 skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, 69 mac_len); 70 goto out; 71 } 72 73 outer_hlen = skb_tnl_header_len(skb); 74 udp_offset = outer_hlen - tnl_hlen; 75 skb = segs; 76 do { 77 struct udphdr *uh; 78 int len; 79 __be32 delta; 80 81 if (dont_encap) { 82 skb->encapsulation = 0; 83 skb->ip_summed = CHECKSUM_NONE; 84 } else { 85 /* Only set up inner headers if we might be offloading 86 * inner checksum. 87 */ 88 skb_reset_inner_headers(skb); 89 skb->encapsulation = 1; 90 } 91 92 skb->mac_len = mac_len; 93 skb->protocol = protocol; 94 95 skb_push(skb, outer_hlen); 96 skb_reset_mac_header(skb); 97 skb_set_network_header(skb, mac_len); 98 skb_set_transport_header(skb, udp_offset); 99 len = skb->len - udp_offset; 100 uh = udp_hdr(skb); 101 uh->len = htons(len); 102 103 if (!need_csum) 104 continue; 105 106 delta = htonl(oldlen + len); 107 108 uh->check = ~csum_fold((__force __wsum) 109 ((__force u32)uh->check + 110 (__force u32)delta)); 111 112 if (offload_csum) { 113 skb->ip_summed = CHECKSUM_PARTIAL; 114 skb->csum_start = skb_transport_header(skb) - skb->head; 115 skb->csum_offset = offsetof(struct udphdr, check); 116 } else { 117 uh->check = gso_make_checksum(skb, ~uh->check); 118 119 if (uh->check == 0) 120 uh->check = CSUM_MANGLED_0; 121 } 122 } while ((skb = skb->next)); 123 out: 124 return segs; 125 } 126 127 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, 128 netdev_features_t features, 129 bool is_ipv6) 130 { 131 __be16 protocol = skb->protocol; 132 const struct net_offload **offloads; 133 const struct net_offload *ops; 134 struct sk_buff *segs = ERR_PTR(-EINVAL); 135 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, 136 netdev_features_t features); 137 138 rcu_read_lock(); 139 140 switch (skb->inner_protocol_type) { 141 case ENCAP_TYPE_ETHER: 142 protocol = skb->inner_protocol; 143 gso_inner_segment = skb_mac_gso_segment; 144 break; 145 case ENCAP_TYPE_IPPROTO: 146 offloads = is_ipv6 ? inet6_offloads : inet_offloads; 147 ops = rcu_dereference(offloads[skb->inner_ipproto]); 148 if (!ops || !ops->callbacks.gso_segment) 149 goto out_unlock; 150 gso_inner_segment = ops->callbacks.gso_segment; 151 break; 152 default: 153 goto out_unlock; 154 } 155 156 segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, 157 protocol, is_ipv6); 158 159 out_unlock: 160 rcu_read_unlock(); 161 162 return segs; 163 } 164 165 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, 166 netdev_features_t features) 167 { 168 struct sk_buff *segs = ERR_PTR(-EINVAL); 169 unsigned int mss; 170 __wsum csum; 171 struct udphdr *uh; 172 struct iphdr *iph; 173 174 if (skb->encapsulation && 175 (skb_shinfo(skb)->gso_type & 176 (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { 177 segs = skb_udp_tunnel_segment(skb, features, false); 178 goto out; 179 } 180 181 if (!pskb_may_pull(skb, sizeof(struct udphdr))) 182 goto out; 183 184 mss = skb_shinfo(skb)->gso_size; 185 if (unlikely(skb->len <= mss)) 186 goto out; 187 188 if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { 189 /* Packet is from an untrusted source, reset gso_segs. */ 190 int type = skb_shinfo(skb)->gso_type; 191 192 if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY | 193 SKB_GSO_UDP_TUNNEL | 194 SKB_GSO_UDP_TUNNEL_CSUM | 195 SKB_GSO_IPIP | 196 SKB_GSO_GRE | SKB_GSO_GRE_CSUM | 197 SKB_GSO_MPLS) || 198 !(type & (SKB_GSO_UDP)))) 199 goto out; 200 201 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); 202 203 segs = NULL; 204 goto out; 205 } 206 207 /* Do software UFO. Complete and fill in the UDP checksum as 208 * HW cannot do checksum of UDP packets sent as multiple 209 * IP fragments. 210 */ 211 212 uh = udp_hdr(skb); 213 iph = ip_hdr(skb); 214 215 uh->check = 0; 216 csum = skb_checksum(skb, 0, skb->len, 0); 217 uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); 218 if (uh->check == 0) 219 uh->check = CSUM_MANGLED_0; 220 221 skb->ip_summed = CHECKSUM_NONE; 222 223 /* Fragment the skb. IP headers of the fragments are updated in 224 * inet_gso_segment() 225 */ 226 segs = skb_segment(skb, features); 227 out: 228 return segs; 229 } 230 231 int udp_add_offload(struct udp_offload *uo) 232 { 233 struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC); 234 235 if (!new_offload) 236 return -ENOMEM; 237 238 new_offload->offload = uo; 239 240 spin_lock(&udp_offload_lock); 241 new_offload->next = udp_offload_base; 242 rcu_assign_pointer(udp_offload_base, new_offload); 243 spin_unlock(&udp_offload_lock); 244 245 return 0; 246 } 247 EXPORT_SYMBOL(udp_add_offload); 248 249 static void udp_offload_free_routine(struct rcu_head *head) 250 { 251 struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu); 252 kfree(ou_priv); 253 } 254 255 void udp_del_offload(struct udp_offload *uo) 256 { 257 struct udp_offload_priv __rcu **head = &udp_offload_base; 258 struct udp_offload_priv *uo_priv; 259 260 spin_lock(&udp_offload_lock); 261 262 uo_priv = udp_deref_protected(*head); 263 for (; uo_priv != NULL; 264 uo_priv = udp_deref_protected(*head)) { 265 if (uo_priv->offload == uo) { 266 rcu_assign_pointer(*head, 267 udp_deref_protected(uo_priv->next)); 268 goto unlock; 269 } 270 head = &uo_priv->next; 271 } 272 pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port)); 273 unlock: 274 spin_unlock(&udp_offload_lock); 275 if (uo_priv != NULL) 276 call_rcu(&uo_priv->rcu, udp_offload_free_routine); 277 } 278 EXPORT_SYMBOL(udp_del_offload); 279 280 struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, 281 struct udphdr *uh) 282 { 283 struct udp_offload_priv *uo_priv; 284 struct sk_buff *p, **pp = NULL; 285 struct udphdr *uh2; 286 unsigned int off = skb_gro_offset(skb); 287 int flush = 1; 288 289 if (NAPI_GRO_CB(skb)->udp_mark || 290 (skb->ip_summed != CHECKSUM_PARTIAL && 291 NAPI_GRO_CB(skb)->csum_cnt == 0 && 292 !NAPI_GRO_CB(skb)->csum_valid)) 293 goto out; 294 295 /* mark that this skb passed once through the udp gro layer */ 296 NAPI_GRO_CB(skb)->udp_mark = 1; 297 298 rcu_read_lock(); 299 uo_priv = rcu_dereference(udp_offload_base); 300 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { 301 if (uo_priv->offload->port == uh->dest && 302 uo_priv->offload->callbacks.gro_receive) 303 goto unflush; 304 } 305 goto out_unlock; 306 307 unflush: 308 flush = 0; 309 310 for (p = *head; p; p = p->next) { 311 if (!NAPI_GRO_CB(p)->same_flow) 312 continue; 313 314 uh2 = (struct udphdr *)(p->data + off); 315 316 /* Match ports and either checksums are either both zero 317 * or nonzero. 318 */ 319 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || 320 (!uh->check ^ !uh2->check)) { 321 NAPI_GRO_CB(p)->same_flow = 0; 322 continue; 323 } 324 } 325 326 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ 327 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); 328 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; 329 pp = uo_priv->offload->callbacks.gro_receive(head, skb); 330 331 out_unlock: 332 rcu_read_unlock(); 333 out: 334 NAPI_GRO_CB(skb)->flush |= flush; 335 return pp; 336 } 337 338 static struct sk_buff **udp4_gro_receive(struct sk_buff **head, 339 struct sk_buff *skb) 340 { 341 struct udphdr *uh = udp_gro_udphdr(skb); 342 343 if (unlikely(!uh)) 344 goto flush; 345 346 /* Don't bother verifying checksum if we're going to flush anyway. */ 347 if (NAPI_GRO_CB(skb)->flush) 348 goto skip; 349 350 if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, 351 inet_gro_compute_pseudo)) 352 goto flush; 353 else if (uh->check) 354 skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, 355 inet_gro_compute_pseudo); 356 skip: 357 NAPI_GRO_CB(skb)->is_ipv6 = 0; 358 return udp_gro_receive(head, skb, uh); 359 360 flush: 361 NAPI_GRO_CB(skb)->flush = 1; 362 return NULL; 363 } 364 365 int udp_gro_complete(struct sk_buff *skb, int nhoff) 366 { 367 struct udp_offload_priv *uo_priv; 368 __be16 newlen = htons(skb->len - nhoff); 369 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); 370 int err = -ENOSYS; 371 372 uh->len = newlen; 373 374 rcu_read_lock(); 375 376 uo_priv = rcu_dereference(udp_offload_base); 377 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { 378 if (uo_priv->offload->port == uh->dest && 379 uo_priv->offload->callbacks.gro_complete) 380 break; 381 } 382 383 if (uo_priv != NULL) { 384 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; 385 err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr)); 386 } 387 388 rcu_read_unlock(); 389 return err; 390 } 391 392 static int udp4_gro_complete(struct sk_buff *skb, int nhoff) 393 { 394 const struct iphdr *iph = ip_hdr(skb); 395 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); 396 397 if (uh->check) 398 uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, 399 iph->daddr, 0); 400 401 return udp_gro_complete(skb, nhoff); 402 } 403 404 static const struct net_offload udpv4_offload = { 405 .callbacks = { 406 .gso_segment = udp4_ufo_fragment, 407 .gro_receive = udp4_gro_receive, 408 .gro_complete = udp4_gro_complete, 409 }, 410 }; 411 412 int __init udpv4_offload_init(void) 413 { 414 return inet_add_offload(&udpv4_offload, IPPROTO_UDP); 415 } 416