1 /* 2 * IPV4 GSO/GRO offload support 3 * Linux INET implementation 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 8 * 2 of the License, or (at your option) any later version. 9 * 10 * UDPv4 GSO support 11 */ 12 13 #include <linux/skbuff.h> 14 #include <net/udp.h> 15 #include <net/protocol.h> 16 17 static DEFINE_SPINLOCK(udp_offload_lock); 18 static struct udp_offload_priv __rcu *udp_offload_base __read_mostly; 19 20 #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock)) 21 22 struct udp_offload_priv { 23 struct udp_offload *offload; 24 struct rcu_head rcu; 25 struct udp_offload_priv __rcu *next; 26 }; 27 28 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, 29 netdev_features_t features, 30 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, 31 netdev_features_t features), 32 __be16 new_protocol) 33 { 34 struct sk_buff *segs = ERR_PTR(-EINVAL); 35 u16 mac_offset = skb->mac_header; 36 int mac_len = skb->mac_len; 37 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); 38 __be16 protocol = skb->protocol; 39 netdev_features_t enc_features; 40 int udp_offset, outer_hlen; 41 unsigned int oldlen; 42 bool need_csum; 43 44 oldlen = (u16)~skb->len; 45 46 if (unlikely(!pskb_may_pull(skb, tnl_hlen))) 47 goto out; 48 49 skb->encapsulation = 0; 50 __skb_pull(skb, tnl_hlen); 51 skb_reset_mac_header(skb); 52 skb_set_network_header(skb, skb_inner_network_offset(skb)); 53 skb->mac_len = skb_inner_network_offset(skb); 54 skb->protocol = new_protocol; 55 56 need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); 57 if (need_csum) 58 skb->encap_hdr_csum = 1; 59 60 /* segment inner packet. */ 61 enc_features = skb->dev->hw_enc_features & features; 62 segs = gso_inner_segment(skb, enc_features); 63 if (IS_ERR_OR_NULL(segs)) { 64 skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, 65 mac_len); 66 goto out; 67 } 68 69 outer_hlen = skb_tnl_header_len(skb); 70 udp_offset = outer_hlen - tnl_hlen; 71 skb = segs; 72 do { 73 struct udphdr *uh; 74 int len; 75 76 skb_reset_inner_headers(skb); 77 skb->encapsulation = 1; 78 79 skb->mac_len = mac_len; 80 81 skb_push(skb, outer_hlen); 82 skb_reset_mac_header(skb); 83 skb_set_network_header(skb, mac_len); 84 skb_set_transport_header(skb, udp_offset); 85 len = skb->len - udp_offset; 86 uh = udp_hdr(skb); 87 uh->len = htons(len); 88 89 if (need_csum) { 90 __be32 delta = htonl(oldlen + len); 91 92 uh->check = ~csum_fold((__force __wsum) 93 ((__force u32)uh->check + 94 (__force u32)delta)); 95 uh->check = gso_make_checksum(skb, ~uh->check); 96 97 if (uh->check == 0) 98 uh->check = CSUM_MANGLED_0; 99 } 100 101 skb->protocol = protocol; 102 } while ((skb = skb->next)); 103 out: 104 return segs; 105 } 106 107 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, 108 netdev_features_t features, 109 bool is_ipv6) 110 { 111 __be16 protocol = skb->protocol; 112 const struct net_offload **offloads; 113 const struct net_offload *ops; 114 struct sk_buff *segs = ERR_PTR(-EINVAL); 115 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, 116 netdev_features_t features); 117 118 rcu_read_lock(); 119 120 switch (skb->inner_protocol_type) { 121 case ENCAP_TYPE_ETHER: 122 protocol = skb->inner_protocol; 123 gso_inner_segment = skb_mac_gso_segment; 124 break; 125 case ENCAP_TYPE_IPPROTO: 126 offloads = is_ipv6 ? inet6_offloads : inet_offloads; 127 ops = rcu_dereference(offloads[skb->inner_ipproto]); 128 if (!ops || !ops->callbacks.gso_segment) 129 goto out_unlock; 130 gso_inner_segment = ops->callbacks.gso_segment; 131 break; 132 default: 133 goto out_unlock; 134 } 135 136 segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, 137 protocol); 138 139 out_unlock: 140 rcu_read_unlock(); 141 142 return segs; 143 } 144 145 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, 146 netdev_features_t features) 147 { 148 struct sk_buff *segs = ERR_PTR(-EINVAL); 149 unsigned int mss; 150 __wsum csum; 151 struct udphdr *uh; 152 struct iphdr *iph; 153 154 if (skb->encapsulation && 155 (skb_shinfo(skb)->gso_type & 156 (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { 157 segs = skb_udp_tunnel_segment(skb, features, false); 158 goto out; 159 } 160 161 if (!pskb_may_pull(skb, sizeof(struct udphdr))) 162 goto out; 163 164 mss = skb_shinfo(skb)->gso_size; 165 if (unlikely(skb->len <= mss)) 166 goto out; 167 168 if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { 169 /* Packet is from an untrusted source, reset gso_segs. */ 170 int type = skb_shinfo(skb)->gso_type; 171 172 if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY | 173 SKB_GSO_UDP_TUNNEL | 174 SKB_GSO_UDP_TUNNEL_CSUM | 175 SKB_GSO_IPIP | 176 SKB_GSO_GRE | SKB_GSO_GRE_CSUM | 177 SKB_GSO_MPLS) || 178 !(type & (SKB_GSO_UDP)))) 179 goto out; 180 181 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); 182 183 segs = NULL; 184 goto out; 185 } 186 187 /* Do software UFO. Complete and fill in the UDP checksum as 188 * HW cannot do checksum of UDP packets sent as multiple 189 * IP fragments. 190 */ 191 192 uh = udp_hdr(skb); 193 iph = ip_hdr(skb); 194 195 uh->check = 0; 196 csum = skb_checksum(skb, 0, skb->len, 0); 197 uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); 198 if (uh->check == 0) 199 uh->check = CSUM_MANGLED_0; 200 201 skb->ip_summed = CHECKSUM_NONE; 202 203 /* Fragment the skb. IP headers of the fragments are updated in 204 * inet_gso_segment() 205 */ 206 segs = skb_segment(skb, features); 207 out: 208 return segs; 209 } 210 211 int udp_add_offload(struct udp_offload *uo) 212 { 213 struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC); 214 215 if (!new_offload) 216 return -ENOMEM; 217 218 new_offload->offload = uo; 219 220 spin_lock(&udp_offload_lock); 221 new_offload->next = udp_offload_base; 222 rcu_assign_pointer(udp_offload_base, new_offload); 223 spin_unlock(&udp_offload_lock); 224 225 return 0; 226 } 227 EXPORT_SYMBOL(udp_add_offload); 228 229 static void udp_offload_free_routine(struct rcu_head *head) 230 { 231 struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu); 232 kfree(ou_priv); 233 } 234 235 void udp_del_offload(struct udp_offload *uo) 236 { 237 struct udp_offload_priv __rcu **head = &udp_offload_base; 238 struct udp_offload_priv *uo_priv; 239 240 spin_lock(&udp_offload_lock); 241 242 uo_priv = udp_deref_protected(*head); 243 for (; uo_priv != NULL; 244 uo_priv = udp_deref_protected(*head)) { 245 if (uo_priv->offload == uo) { 246 rcu_assign_pointer(*head, 247 udp_deref_protected(uo_priv->next)); 248 goto unlock; 249 } 250 head = &uo_priv->next; 251 } 252 pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port)); 253 unlock: 254 spin_unlock(&udp_offload_lock); 255 if (uo_priv != NULL) 256 call_rcu(&uo_priv->rcu, udp_offload_free_routine); 257 } 258 EXPORT_SYMBOL(udp_del_offload); 259 260 struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, 261 struct udphdr *uh) 262 { 263 struct udp_offload_priv *uo_priv; 264 struct sk_buff *p, **pp = NULL; 265 struct udphdr *uh2; 266 unsigned int off = skb_gro_offset(skb); 267 int flush = 1; 268 269 if (NAPI_GRO_CB(skb)->udp_mark || 270 (skb->ip_summed != CHECKSUM_PARTIAL && 271 NAPI_GRO_CB(skb)->csum_cnt == 0 && 272 !NAPI_GRO_CB(skb)->csum_valid)) 273 goto out; 274 275 /* mark that this skb passed once through the udp gro layer */ 276 NAPI_GRO_CB(skb)->udp_mark = 1; 277 278 rcu_read_lock(); 279 uo_priv = rcu_dereference(udp_offload_base); 280 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { 281 if (uo_priv->offload->port == uh->dest && 282 uo_priv->offload->callbacks.gro_receive) 283 goto unflush; 284 } 285 goto out_unlock; 286 287 unflush: 288 flush = 0; 289 290 for (p = *head; p; p = p->next) { 291 if (!NAPI_GRO_CB(p)->same_flow) 292 continue; 293 294 uh2 = (struct udphdr *)(p->data + off); 295 296 /* Match ports and either checksums are either both zero 297 * or nonzero. 298 */ 299 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || 300 (!uh->check ^ !uh2->check)) { 301 NAPI_GRO_CB(p)->same_flow = 0; 302 continue; 303 } 304 } 305 306 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ 307 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); 308 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; 309 pp = uo_priv->offload->callbacks.gro_receive(head, skb); 310 311 out_unlock: 312 rcu_read_unlock(); 313 out: 314 NAPI_GRO_CB(skb)->flush |= flush; 315 return pp; 316 } 317 318 static struct sk_buff **udp4_gro_receive(struct sk_buff **head, 319 struct sk_buff *skb) 320 { 321 struct udphdr *uh = udp_gro_udphdr(skb); 322 323 if (unlikely(!uh)) 324 goto flush; 325 326 /* Don't bother verifying checksum if we're going to flush anyway. */ 327 if (NAPI_GRO_CB(skb)->flush) 328 goto skip; 329 330 if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, 331 inet_gro_compute_pseudo)) 332 goto flush; 333 else if (uh->check) 334 skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, 335 inet_gro_compute_pseudo); 336 skip: 337 NAPI_GRO_CB(skb)->is_ipv6 = 0; 338 return udp_gro_receive(head, skb, uh); 339 340 flush: 341 NAPI_GRO_CB(skb)->flush = 1; 342 return NULL; 343 } 344 345 int udp_gro_complete(struct sk_buff *skb, int nhoff) 346 { 347 struct udp_offload_priv *uo_priv; 348 __be16 newlen = htons(skb->len - nhoff); 349 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); 350 int err = -ENOSYS; 351 352 uh->len = newlen; 353 354 rcu_read_lock(); 355 356 uo_priv = rcu_dereference(udp_offload_base); 357 for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { 358 if (uo_priv->offload->port == uh->dest && 359 uo_priv->offload->callbacks.gro_complete) 360 break; 361 } 362 363 if (uo_priv != NULL) { 364 NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; 365 err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr)); 366 } 367 368 rcu_read_unlock(); 369 return err; 370 } 371 372 static int udp4_gro_complete(struct sk_buff *skb, int nhoff) 373 { 374 const struct iphdr *iph = ip_hdr(skb); 375 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); 376 377 if (uh->check) 378 uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, 379 iph->daddr, 0); 380 381 return udp_gro_complete(skb, nhoff); 382 } 383 384 static const struct net_offload udpv4_offload = { 385 .callbacks = { 386 .gso_segment = udp4_ufo_fragment, 387 .gro_receive = udp4_gro_receive, 388 .gro_complete = udp4_gro_complete, 389 }, 390 }; 391 392 int __init udpv4_offload_init(void) 393 { 394 return inet_add_offload(&udpv4_offload, IPPROTO_UDP); 395 } 396