1 /* 2 * IPV4 GSO/GRO offload support 3 * Linux INET implementation 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License 7 * as published by the Free Software Foundation; either version 8 * 2 of the License, or (at your option) any later version. 9 * 10 * GRE GSO support 11 */ 12 13 #include <linux/skbuff.h> 14 #include <linux/init.h> 15 #include <net/protocol.h> 16 #include <net/gre.h> 17 18 static int gre_gso_send_check(struct sk_buff *skb) 19 { 20 if (!skb->encapsulation) 21 return -EINVAL; 22 return 0; 23 } 24 25 static struct sk_buff *gre_gso_segment(struct sk_buff *skb, 26 netdev_features_t features) 27 { 28 struct sk_buff *segs = ERR_PTR(-EINVAL); 29 netdev_features_t enc_features; 30 int ghl; 31 struct gre_base_hdr *greh; 32 u16 mac_offset = skb->mac_header; 33 int mac_len = skb->mac_len; 34 __be16 protocol = skb->protocol; 35 int tnl_hlen; 36 bool csum; 37 38 if (unlikely(skb_shinfo(skb)->gso_type & 39 ~(SKB_GSO_TCPV4 | 40 SKB_GSO_TCPV6 | 41 SKB_GSO_UDP | 42 SKB_GSO_DODGY | 43 SKB_GSO_TCP_ECN | 44 SKB_GSO_GRE | 45 SKB_GSO_GRE_CSUM | 46 SKB_GSO_IPIP))) 47 goto out; 48 49 if (unlikely(!pskb_may_pull(skb, sizeof(*greh)))) 50 goto out; 51 52 greh = (struct gre_base_hdr *)skb_transport_header(skb); 53 54 ghl = skb_inner_network_header(skb) - skb_transport_header(skb); 55 if (unlikely(ghl < sizeof(*greh))) 56 goto out; 57 58 csum = !!(greh->flags & GRE_CSUM); 59 if (csum) 60 skb->encap_hdr_csum = 1; 61 62 if (unlikely(!pskb_may_pull(skb, ghl))) 63 goto out; 64 65 /* setup inner skb. */ 66 skb->protocol = greh->protocol; 67 skb->encapsulation = 0; 68 69 __skb_pull(skb, ghl); 70 skb_reset_mac_header(skb); 71 skb_set_network_header(skb, skb_inner_network_offset(skb)); 72 skb->mac_len = skb_inner_network_offset(skb); 73 74 /* segment inner packet. */ 75 enc_features = skb->dev->hw_enc_features & netif_skb_features(skb); 76 segs = skb_mac_gso_segment(skb, enc_features); 77 if (!segs || IS_ERR(segs)) { 78 skb_gso_error_unwind(skb, protocol, ghl, mac_offset, mac_len); 79 goto out; 80 } 81 82 skb = segs; 83 tnl_hlen = skb_tnl_header_len(skb); 84 do { 85 __skb_push(skb, ghl); 86 if (csum) { 87 __be32 *pcsum; 88 89 if (skb_has_shared_frag(skb)) { 90 int err; 91 92 err = __skb_linearize(skb); 93 if (err) { 94 kfree_skb_list(segs); 95 segs = ERR_PTR(err); 96 goto out; 97 } 98 } 99 100 skb_reset_transport_header(skb); 101 102 greh = (struct gre_base_hdr *) 103 skb_transport_header(skb); 104 pcsum = (__be32 *)(greh + 1); 105 *pcsum = 0; 106 *(__sum16 *)pcsum = gso_make_checksum(skb, 0); 107 } 108 __skb_push(skb, tnl_hlen - ghl); 109 110 skb_reset_inner_headers(skb); 111 skb->encapsulation = 1; 112 113 skb_reset_mac_header(skb); 114 skb_set_network_header(skb, mac_len); 115 skb->mac_len = mac_len; 116 skb->protocol = protocol; 117 } while ((skb = skb->next)); 118 out: 119 return segs; 120 } 121 122 /* Compute the whole skb csum in s/w and store it, then verify GRO csum 123 * starting from gro_offset. 124 */ 125 static __sum16 gro_skb_checksum(struct sk_buff *skb) 126 { 127 __sum16 sum; 128 129 skb->csum = skb_checksum(skb, 0, skb->len, 0); 130 NAPI_GRO_CB(skb)->csum = csum_sub(skb->csum, 131 csum_partial(skb->data, skb_gro_offset(skb), 0)); 132 sum = csum_fold(NAPI_GRO_CB(skb)->csum); 133 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE)) { 134 if (unlikely(!sum) && !skb->csum_complete_sw) 135 netdev_rx_csum_fault(skb->dev); 136 } else { 137 skb->ip_summed = CHECKSUM_COMPLETE; 138 skb->csum_complete_sw = 1; 139 } 140 141 return sum; 142 } 143 144 static struct sk_buff **gre_gro_receive(struct sk_buff **head, 145 struct sk_buff *skb) 146 { 147 struct sk_buff **pp = NULL; 148 struct sk_buff *p; 149 const struct gre_base_hdr *greh; 150 unsigned int hlen, grehlen; 151 unsigned int off; 152 int flush = 1; 153 struct packet_offload *ptype; 154 __be16 type; 155 156 off = skb_gro_offset(skb); 157 hlen = off + sizeof(*greh); 158 greh = skb_gro_header_fast(skb, off); 159 if (skb_gro_header_hard(skb, hlen)) { 160 greh = skb_gro_header_slow(skb, hlen, off); 161 if (unlikely(!greh)) 162 goto out; 163 } 164 165 /* Only support version 0 and K (key), C (csum) flags. Note that 166 * although the support for the S (seq#) flag can be added easily 167 * for GRO, this is problematic for GSO hence can not be enabled 168 * here because a GRO pkt may end up in the forwarding path, thus 169 * requiring GSO support to break it up correctly. 170 */ 171 if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0) 172 goto out; 173 174 type = greh->protocol; 175 176 rcu_read_lock(); 177 ptype = gro_find_receive_by_type(type); 178 if (ptype == NULL) 179 goto out_unlock; 180 181 grehlen = GRE_HEADER_SECTION; 182 183 if (greh->flags & GRE_KEY) 184 grehlen += GRE_HEADER_SECTION; 185 186 if (greh->flags & GRE_CSUM) 187 grehlen += GRE_HEADER_SECTION; 188 189 hlen = off + grehlen; 190 if (skb_gro_header_hard(skb, hlen)) { 191 greh = skb_gro_header_slow(skb, hlen, off); 192 if (unlikely(!greh)) 193 goto out_unlock; 194 } 195 if (greh->flags & GRE_CSUM) { /* Need to verify GRE csum first */ 196 __sum16 csum = 0; 197 198 if (skb->ip_summed == CHECKSUM_COMPLETE) 199 csum = csum_fold(NAPI_GRO_CB(skb)->csum); 200 /* Don't trust csum error calculated/reported by h/w */ 201 if (skb->ip_summed == CHECKSUM_NONE || csum != 0) 202 csum = gro_skb_checksum(skb); 203 204 /* GRE CSUM is the 1's complement of the 1's complement sum 205 * of the GRE hdr plus payload so it should add up to 0xffff 206 * (and 0 after csum_fold()) just like the IPv4 hdr csum. 207 */ 208 if (csum) 209 goto out_unlock; 210 } 211 flush = 0; 212 213 for (p = *head; p; p = p->next) { 214 const struct gre_base_hdr *greh2; 215 216 if (!NAPI_GRO_CB(p)->same_flow) 217 continue; 218 219 /* The following checks are needed to ensure only pkts 220 * from the same tunnel are considered for aggregation. 221 * The criteria for "the same tunnel" includes: 222 * 1) same version (we only support version 0 here) 223 * 2) same protocol (we only support ETH_P_IP for now) 224 * 3) same set of flags 225 * 4) same key if the key field is present. 226 */ 227 greh2 = (struct gre_base_hdr *)(p->data + off); 228 229 if (greh2->flags != greh->flags || 230 greh2->protocol != greh->protocol) { 231 NAPI_GRO_CB(p)->same_flow = 0; 232 continue; 233 } 234 if (greh->flags & GRE_KEY) { 235 /* compare keys */ 236 if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) { 237 NAPI_GRO_CB(p)->same_flow = 0; 238 continue; 239 } 240 } 241 } 242 243 skb_gro_pull(skb, grehlen); 244 245 /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/ 246 skb_gro_postpull_rcsum(skb, greh, grehlen); 247 248 pp = ptype->callbacks.gro_receive(head, skb); 249 250 out_unlock: 251 rcu_read_unlock(); 252 out: 253 NAPI_GRO_CB(skb)->flush |= flush; 254 255 return pp; 256 } 257 258 static int gre_gro_complete(struct sk_buff *skb, int nhoff) 259 { 260 struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff); 261 struct packet_offload *ptype; 262 unsigned int grehlen = sizeof(*greh); 263 int err = -ENOENT; 264 __be16 type; 265 266 type = greh->protocol; 267 if (greh->flags & GRE_KEY) 268 grehlen += GRE_HEADER_SECTION; 269 270 if (greh->flags & GRE_CSUM) 271 grehlen += GRE_HEADER_SECTION; 272 273 rcu_read_lock(); 274 ptype = gro_find_complete_by_type(type); 275 if (ptype != NULL) 276 err = ptype->callbacks.gro_complete(skb, nhoff + grehlen); 277 278 rcu_read_unlock(); 279 return err; 280 } 281 282 static const struct net_offload gre_offload = { 283 .callbacks = { 284 .gso_send_check = gre_gso_send_check, 285 .gso_segment = gre_gso_segment, 286 .gro_receive = gre_gro_receive, 287 .gro_complete = gre_gro_complete, 288 }, 289 }; 290 291 static int __init gre_offload_init(void) 292 { 293 return inet_add_offload(&gre_offload, IPPROTO_GRE); 294 } 295 device_initcall(gre_offload_init); 296