1 /* 2 * IPV4 GSO/GRO offload support 3 * Linux INET implementation 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 8 * 2 of the License, or (at your option) any later version. 9 * 10 * UDPv4 GSO support 11 */ 12 13 #include <linux/skbuff.h> 14 #include <net/udp.h> 15 #include <net/protocol.h> 16 17 static DEFINE_SPINLOCK(udp_offload_lock); 18 static struct udp_offload_priv __rcu *udp_offload_base __read_mostly; 19 20 #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock)) 21 22 struct udp_offload_priv { 23 struct udp_offload *offload; 24 possible_net_t net; 25 struct rcu_head rcu; 26 struct udp_offload_priv __rcu *next; 27 }; 28 29 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, 30 netdev_features_t features, 31 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, 32 netdev_features_t features), 33 __be16 new_protocol, bool is_ipv6) 34 { 35 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); 36 bool remcsum, need_csum, offload_csum, ufo; 37 struct sk_buff *segs = ERR_PTR(-EINVAL); 38 struct udphdr *uh = udp_hdr(skb); 39 u16 mac_offset = skb->mac_header; 40 __be16 protocol = skb->protocol; 41 u16 mac_len = skb->mac_len; 42 int udp_offset, outer_hlen; 43 __wsum partial; 44 45 if (unlikely(!pskb_may_pull(skb, tnl_hlen))) 46 goto out; 47 48 /* Adjust partial header checksum to negate old length. 49 * We cannot rely on the value contained in uh->len as it is 50 * possible that the actual value exceeds the boundaries of the 51 * 16 bit length field due to the header being added outside of an 52 * IP or IPv6 frame that was already limited to 64K - 1. 53 */ 54 partial = csum_sub(csum_unfold(uh->check), 55 (__force __wsum)htonl(skb->len)); 56 57 /* setup inner skb. */ 58 skb->encapsulation = 0; 59 SKB_GSO_CB(skb)->encap_level = 0; 60 __skb_pull(skb, tnl_hlen); 61 skb_reset_mac_header(skb); 62 skb_set_network_header(skb, skb_inner_network_offset(skb)); 63 skb->mac_len = skb_inner_network_offset(skb); 64 skb->protocol = new_protocol; 65 66 need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); 67 skb->encap_hdr_csum = need_csum; 68 69 remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM); 70 skb->remcsum_offload = remcsum; 71 72 ufo = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP); 73 74 /* Try to offload checksum if possible */ 75 offload_csum = !!(need_csum && 76 (skb->dev->features & 77 (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) : 78 (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)))); 79 80 features &= skb->dev->hw_enc_features; 81 82 /* The only checksum offload we care about from here on out is the 83 * outer one so strip the existing checksum feature flags and 84 * instead set the flag based on our outer checksum offload value. 85 */ 86 if (remcsum || ufo) { 87 features &= ~NETIF_F_CSUM_MASK; 88 if (!need_csum || offload_csum) 89 features |= NETIF_F_HW_CSUM; 90 } 91 92 /* segment inner packet. */ 93 segs = gso_inner_segment(skb, features); 94 if (IS_ERR_OR_NULL(segs)) { 95 skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, 96 mac_len); 97 goto out; 98 } 99 100 outer_hlen = skb_tnl_header_len(skb); 101 udp_offset = outer_hlen - tnl_hlen; 102 skb = segs; 103 do { 104 __be16 len; 105 106 if (remcsum) 107 skb->ip_summed = CHECKSUM_NONE; 108 109 /* Set up inner headers if we are offloading inner checksum */ 110 if (skb->ip_summed == CHECKSUM_PARTIAL) { 111 skb_reset_inner_headers(skb); 112 skb->encapsulation = 1; 113 } 114 115 skb->mac_len = mac_len; 116 skb->protocol = protocol; 117 118 __skb_push(skb, outer_hlen); 119 skb_reset_mac_header(skb); 120 skb_set_network_header(skb, mac_len); 121 skb_set_transport_header(skb, udp_offset); 122 len = htons(skb->len - udp_offset); 123 uh = udp_hdr(skb); 124 uh->len = len; 125 126 if (!need_csum) 127 continue; 128 129 uh->check = ~csum_fold(csum_add(partial, (__force __wsum)len)); 130 131 if (skb->encapsulation || !offload_csum) { 132 uh->check = gso_make_checksum(skb, ~uh->check); 133 if (uh->check == 0) 134 uh->check = CSUM_MANGLED_0; 135 } else { 136 skb->ip_summed = CHECKSUM_PARTIAL; 137 skb->csum_start = skb_transport_header(skb) - skb->head; 138 skb->csum_offset = offsetof(struct udphdr, check); 139 } 140 } while ((skb = skb->next)); 141 out: 142 return segs; 143 } 144 145 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, 146 netdev_features_t features, 147 bool is_ipv6) 148 { 149 __be16 protocol = skb->protocol; 150 const struct net_offload **offloads; 151 const struct net_offload *ops; 152 struct sk_buff *segs = ERR_PTR(-EINVAL); 153 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, 154 netdev_features_t features); 155 156 rcu_read_lock(); 157 158 switch (skb->inner_protocol_type) { 159 case ENCAP_TYPE_ETHER: 160 protocol = skb->inner_protocol; 161 gso_inner_segment = skb_mac_gso_segment; 162 break; 163 case ENCAP_TYPE_IPPROTO: 164 offloads = is_ipv6 ? inet6_offloads : inet_offloads; 165 ops = rcu_dereference(offloads[skb->inner_ipproto]); 166 if (!ops || !ops->callbacks.gso_segment) 167 goto out_unlock; 168 gso_inner_segment = ops->callbacks.gso_segment; 169 break; 170 default: 171 goto out_unlock; 172 } 173 174 segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, 175 protocol, is_ipv6); 176 177 out_unlock: 178 rcu_read_unlock(); 179 180 return segs; 181 } 182 183 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, 184 netdev_features_t features) 185 { 186 struct sk_buff *segs = ERR_PTR(-EINVAL); 187 unsigned int mss; 188 __wsum csum; 189 struct udphdr *uh; 190 struct iphdr *iph; 191 192 if (skb->encapsulation && 193 (skb_shinfo(skb)->gso_type & 194 (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { 195 segs = skb_udp_tunnel_segment(skb, features, false); 196 goto out; 197 } 198 199 if (!pskb_may_pull(skb, sizeof(struct udphdr))) 200 goto out; 201 202 mss = skb_shinfo(skb)->gso_size; 203 if (unlikely(skb->len <= mss)) 204 goto out; 205 206 if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { 207 /* Packet is from an untrusted source, reset gso_segs. */ 208 int type = skb_shinfo(skb)->gso_type; 209 210 if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY | 211 SKB_GSO_UDP_TUNNEL | 212 SKB_GSO_UDP_TUNNEL_CSUM | 213 SKB_GSO_TUNNEL_REMCSUM | 214 SKB_GSO_IPIP | 215 SKB_GSO_GRE | SKB_GSO_GRE_CSUM) || 216 !(type & (SKB_GSO_UDP)))) 217 goto out; 218 219 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); 220 221 segs = NULL; 222 goto out; 223 } 224 225 /* Do software UFO. Complete and fill in the UDP checksum as 226 * HW cannot do checksum of UDP packets sent as multiple 227 * IP fragments. 228 */ 229 230 uh = udp_hdr(skb); 231 iph = ip_hdr(skb); 232 233 uh->check = 0; 234 csum = skb_checksum(skb, 0, skb->len, 0); 235 uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); 236 if (uh->check == 0) 237 uh->check = CSUM_MANGLED_0; 238 239 skb->ip_summed = CHECKSUM_NONE; 240 241 /* If there is no outer header we can fake a checksum offload 242 * due to the fact that we have already done the checksum in 243 * software prior to segmenting the frame. 244 */ 245 if (!skb->encap_hdr_csum) 246 features |= NETIF_F_HW_CSUM; 247 248 /* Fragment the skb. IP headers of the fragments are updated in 249 * inet_gso_segment() 250 */ 251 segs = skb_segment(skb, features); 252 out: 253 return segs; 254 } 255 256 int udp_add_offload(struct net *net, struct udp_offload *uo) 257 { 258 struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC); 259 260 if (!new_offload) 261 return -ENOMEM; 262 263 write_pnet(&new_offload->net, net); 264 new_offload->offload = uo; 265 266 spin_lock(&udp_offload_lock); 267 new_offload->next = udp_offload_base; 268 rcu_assign_pointer(udp_offload_base, new_offload); 269 spin_unlock(&udp_offload_lock); 270 271 return 0; 272 } 273 EXPORT_SYMBOL(udp_add_offload); 274 275 static void udp_offload_free_routine(struct rcu_head *head) 276 { 277 struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu); 278 kfree(ou_priv); 279 } 280 281 void udp_del_offload(struct udp_offload *uo) 282 { 283 struct udp_offload_priv __rcu **head = &udp_offload_base; 284 struct udp_offload_priv *uo_priv; 285 286 spin_lock(&udp_offload_lock); 287 288 uo_priv = udp_deref_protected(*head); 289 for (; uo_priv != NULL; 290 uo_priv = udp_deref_protected(*head)) { 291 if (uo_priv->offload == uo) { 292 rcu_assign_pointer(*head, 293 udp_deref_protected(uo_priv->next)); 294 goto unlock; 295 } 296 head = &uo_priv->next; 297 } 298 pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port)); 299 unlock: 300 spin_unlock(&udp_offload_lock); 301 if (uo_priv) 302 call_rcu(&uo_priv->rcu, udp_offload_free_routine); 303 } 304 EXPORT_SYMBOL(udp_del_offload); 305 306 struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, 307 struct udphdr *uh) 308 { 309 struct udp_offload_priv *uo_priv; 310 struct sk_buff *p, **pp = NULL; 311 struct udphdr *uh2; 312 unsigned int off = skb_gro_offset(skb); 313 int flush = 1; 314 315 if (NAPI_GRO_CB(skb)->encap_mark || 316 (skb->ip_summed != CHECKSUM_PARTIAL && 317 NAPI_GRO_CB(skb)->csum_cnt == 0 && 318 !NAPI_GRO_CB(skb)->csum_valid)) 319 goto out; 320 321 /* mark that this skb passed once through the tunnel gro layer */ 322 NAPI_GRO_CB(skb)->encap_mark = 1; 323 324 rcu_read_lock(); 325 uo_priv = rcu_dereference(udp_offload_base); 326 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { 327 if (net_eq(read_pnet(&uo_priv->net), dev_net(skb->dev)) && 328 uo_priv->offload->port == uh->dest && 329 uo_priv->offload->callbacks.gro_receive) 330 goto unflush; 331 } 332 goto out_unlock; 333 334 unflush: 335 flush = 0; 336 337 for (p = *head; p; p = p->next) { 338 if (!NAPI_GRO_CB(p)->same_flow) 339 continue; 340 341 uh2 = (struct udphdr *)(p->data + off); 342 343 /* Match ports and either checksums are either both zero 344 * or nonzero. 345 */ 346 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || 347 (!uh->check ^ !uh2->check)) { 348 NAPI_GRO_CB(p)->same_flow = 0; 349 continue; 350 } 351 } 352 353 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ 354 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); 355 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; 356 pp = uo_priv->offload->callbacks.gro_receive(head, skb, 357 uo_priv->offload); 358 359 out_unlock: 360 rcu_read_unlock(); 361 out: 362 NAPI_GRO_CB(skb)->flush |= flush; 363 return pp; 364 } 365 366 static struct sk_buff **udp4_gro_receive(struct sk_buff **head, 367 struct sk_buff *skb) 368 { 369 struct udphdr *uh = udp_gro_udphdr(skb); 370 371 if (unlikely(!uh)) 372 goto flush; 373 374 /* Don't bother verifying checksum if we're going to flush anyway. */ 375 if (NAPI_GRO_CB(skb)->flush) 376 goto skip; 377 378 if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, 379 inet_gro_compute_pseudo)) 380 goto flush; 381 else if (uh->check) 382 skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, 383 inet_gro_compute_pseudo); 384 skip: 385 NAPI_GRO_CB(skb)->is_ipv6 = 0; 386 return udp_gro_receive(head, skb, uh); 387 388 flush: 389 NAPI_GRO_CB(skb)->flush = 1; 390 return NULL; 391 } 392 393 int udp_gro_complete(struct sk_buff *skb, int nhoff) 394 { 395 struct udp_offload_priv *uo_priv; 396 __be16 newlen = htons(skb->len - nhoff); 397 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); 398 int err = -ENOSYS; 399 400 uh->len = newlen; 401 402 /* Set encapsulation before calling into inner gro_complete() functions 403 * to make them set up the inner offsets. 404 */ 405 skb->encapsulation = 1; 406 407 rcu_read_lock(); 408 409 uo_priv = rcu_dereference(udp_offload_base); 410 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { 411 if (net_eq(read_pnet(&uo_priv->net), dev_net(skb->dev)) && 412 uo_priv->offload->port == uh->dest && 413 uo_priv->offload->callbacks.gro_complete) 414 break; 415 } 416 417 if (uo_priv) { 418 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; 419 err = uo_priv->offload->callbacks.gro_complete(skb, 420 nhoff + sizeof(struct udphdr), 421 uo_priv->offload); 422 } 423 424 rcu_read_unlock(); 425 426 if (skb->remcsum_offload) 427 skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM; 428 429 return err; 430 } 431 432 static int udp4_gro_complete(struct sk_buff *skb, int nhoff) 433 { 434 const struct iphdr *iph = ip_hdr(skb); 435 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); 436 437 if (uh->check) { 438 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 439 uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, 440 iph->daddr, 0); 441 } else { 442 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 443 } 444 445 return udp_gro_complete(skb, nhoff); 446 } 447 448 static const struct net_offload udpv4_offload = { 449 .callbacks = { 450 .gso_segment = udp4_ufo_fragment, 451 .gro_receive = udp4_gro_receive, 452 .gro_complete = udp4_gro_complete, 453 }, 454 }; 455 456 int __init udpv4_offload_init(void) 457 { 458 return inet_add_offload(&udpv4_offload, IPPROTO_UDP); 459 } 460