1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * IPV4 GSO/GRO offload support 4 * Linux INET implementation 5 * 6 * GRE GSO support 7 */ 8 9 #include <linux/skbuff.h> 10 #include <linux/init.h> 11 #include <net/protocol.h> 12 #include <net/gre.h> 13 14 static struct sk_buff *gre_gso_segment(struct sk_buff *skb, 15 netdev_features_t features) 16 { 17 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); 18 struct sk_buff *segs = ERR_PTR(-EINVAL); 19 u16 mac_offset = skb->mac_header; 20 __be16 protocol = skb->protocol; 21 bool need_csum, gso_partial; 22 u16 mac_len = skb->mac_len; 23 int gre_offset, outer_hlen; 24 25 if (!skb->encapsulation) 26 goto out; 27 28 if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr))) 29 goto out; 30 31 if (unlikely(!pskb_may_pull(skb, tnl_hlen))) 32 goto out; 33 34 /* setup inner skb. */ 35 skb->encapsulation = 0; 36 SKB_GSO_CB(skb)->encap_level = 0; 37 __skb_pull(skb, tnl_hlen); 38 skb_reset_mac_header(skb); 39 skb_set_network_header(skb, skb_inner_network_offset(skb)); 40 skb->mac_len = skb_inner_network_offset(skb); 41 skb->protocol = skb->inner_protocol; 42 43 need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM); 44 skb->encap_hdr_csum = need_csum; 45 46 features &= skb->dev->hw_enc_features; 47 if (need_csum) 48 features &= ~NETIF_F_SCTP_CRC; 49 50 /* segment inner packet. */ 51 segs = skb_mac_gso_segment(skb, features); 52 if (IS_ERR_OR_NULL(segs)) { 53 skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, 54 mac_len); 55 goto out; 56 } 57 58 gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL); 59 60 outer_hlen = skb_tnl_header_len(skb); 61 gre_offset = outer_hlen - tnl_hlen; 62 skb = segs; 63 do { 64 struct gre_base_hdr *greh; 65 __sum16 *pcsum; 66 67 /* Set up inner headers if we are offloading inner checksum */ 68 if (skb->ip_summed == CHECKSUM_PARTIAL) { 69 skb_reset_inner_headers(skb); 70 skb->encapsulation = 1; 71 } 72 73 skb->mac_len = mac_len; 74 skb->protocol = protocol; 75 76 __skb_push(skb, outer_hlen); 77 skb_reset_mac_header(skb); 78 skb_set_network_header(skb, mac_len); 79 skb_set_transport_header(skb, gre_offset); 80 81 if (!need_csum) 82 continue; 83 84 greh = (struct gre_base_hdr *)skb_transport_header(skb); 85 pcsum = (__sum16 *)(greh + 1); 86 87 if (gso_partial && skb_is_gso(skb)) { 88 unsigned int partial_adj; 89 90 /* Adjust checksum to account for the fact that 91 * the partial checksum is based on actual size 92 * whereas headers should be based on MSS size. 93 */ 94 partial_adj = skb->len + skb_headroom(skb) - 95 SKB_GSO_CB(skb)->data_offset - 96 skb_shinfo(skb)->gso_size; 97 *pcsum = ~csum_fold((__force __wsum)htonl(partial_adj)); 98 } else { 99 *pcsum = 0; 100 } 101 102 *(pcsum + 1) = 0; 103 *pcsum = gso_make_checksum(skb, 0); 104 } while ((skb = skb->next)); 105 out: 106 return segs; 107 } 108 109 static struct sk_buff *gre_gro_receive(struct list_head *head, 110 struct sk_buff *skb) 111 { 112 struct sk_buff *pp = NULL; 113 struct sk_buff *p; 114 const struct gre_base_hdr *greh; 115 unsigned int hlen, grehlen; 116 unsigned int off; 117 int flush = 1; 118 struct packet_offload *ptype; 119 __be16 type; 120 121 if (NAPI_GRO_CB(skb)->encap_mark) 122 goto out; 123 124 NAPI_GRO_CB(skb)->encap_mark = 1; 125 126 off = skb_gro_offset(skb); 127 hlen = off + sizeof(*greh); 128 greh = skb_gro_header_fast(skb, off); 129 if (skb_gro_header_hard(skb, hlen)) { 130 greh = skb_gro_header_slow(skb, hlen, off); 131 if (unlikely(!greh)) 132 goto out; 133 } 134 135 /* Only support version 0 and K (key), C (csum) flags. Note that 136 * although the support for the S (seq#) flag can be added easily 137 * for GRO, this is problematic for GSO hence can not be enabled 138 * here because a GRO pkt may end up in the forwarding path, thus 139 * requiring GSO support to break it up correctly. 140 */ 141 if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0) 142 goto out; 143 144 /* We can only support GRE_CSUM if we can track the location of 145 * the GRE header. In the case of FOU/GUE we cannot because the 146 * outer UDP header displaces the GRE header leaving us in a state 147 * of limbo. 148 */ 149 if ((greh->flags & GRE_CSUM) && NAPI_GRO_CB(skb)->is_fou) 150 goto out; 151 152 type = greh->protocol; 153 154 rcu_read_lock(); 155 ptype = gro_find_receive_by_type(type); 156 if (!ptype) 157 goto out_unlock; 158 159 grehlen = GRE_HEADER_SECTION; 160 161 if (greh->flags & GRE_KEY) 162 grehlen += GRE_HEADER_SECTION; 163 164 if (greh->flags & GRE_CSUM) 165 grehlen += GRE_HEADER_SECTION; 166 167 hlen = off + grehlen; 168 if (skb_gro_header_hard(skb, hlen)) { 169 greh = skb_gro_header_slow(skb, hlen, off); 170 if (unlikely(!greh)) 171 goto out_unlock; 172 } 173 174 /* Don't bother verifying checksum if we're going to flush anyway. */ 175 if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) { 176 if (skb_gro_checksum_simple_validate(skb)) 177 goto out_unlock; 178 179 skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 180 null_compute_pseudo); 181 } 182 183 list_for_each_entry(p, head, list) { 184 const struct gre_base_hdr *greh2; 185 186 if (!NAPI_GRO_CB(p)->same_flow) 187 continue; 188 189 /* The following checks are needed to ensure only pkts 190 * from the same tunnel are considered for aggregation. 191 * The criteria for "the same tunnel" includes: 192 * 1) same version (we only support version 0 here) 193 * 2) same protocol (we only support ETH_P_IP for now) 194 * 3) same set of flags 195 * 4) same key if the key field is present. 196 */ 197 greh2 = (struct gre_base_hdr *)(p->data + off); 198 199 if (greh2->flags != greh->flags || 200 greh2->protocol != greh->protocol) { 201 NAPI_GRO_CB(p)->same_flow = 0; 202 continue; 203 } 204 if (greh->flags & GRE_KEY) { 205 /* compare keys */ 206 if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) { 207 NAPI_GRO_CB(p)->same_flow = 0; 208 continue; 209 } 210 } 211 } 212 213 skb_gro_pull(skb, grehlen); 214 215 /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/ 216 skb_gro_postpull_rcsum(skb, greh, grehlen); 217 218 pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); 219 flush = 0; 220 221 out_unlock: 222 rcu_read_unlock(); 223 out: 224 skb_gro_flush_final(skb, pp, flush); 225 226 return pp; 227 } 228 229 static int gre_gro_complete(struct sk_buff *skb, int nhoff) 230 { 231 struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff); 232 struct packet_offload *ptype; 233 unsigned int grehlen = sizeof(*greh); 234 int err = -ENOENT; 235 __be16 type; 236 237 skb->encapsulation = 1; 238 skb_shinfo(skb)->gso_type = SKB_GSO_GRE; 239 240 type = greh->protocol; 241 if (greh->flags & GRE_KEY) 242 grehlen += GRE_HEADER_SECTION; 243 244 if (greh->flags & GRE_CSUM) 245 grehlen += GRE_HEADER_SECTION; 246 247 rcu_read_lock(); 248 ptype = gro_find_complete_by_type(type); 249 if (ptype) 250 err = ptype->callbacks.gro_complete(skb, nhoff + grehlen); 251 252 rcu_read_unlock(); 253 254 skb_set_inner_mac_header(skb, nhoff + grehlen); 255 256 return err; 257 } 258 259 static const struct net_offload gre_offload = { 260 .callbacks = { 261 .gso_segment = gre_gso_segment, 262 .gro_receive = gre_gro_receive, 263 .gro_complete = gre_gro_complete, 264 }, 265 }; 266 267 static int __init gre_offload_init(void) 268 { 269 int err; 270 271 err = inet_add_offload(&gre_offload, IPPROTO_GRE); 272 #if IS_ENABLED(CONFIG_IPV6) 273 if (err) 274 return err; 275 276 err = inet6_add_offload(&gre_offload, IPPROTO_GRE); 277 if (err) 278 inet_del_offload(&gre_offload, IPPROTO_GRE); 279 #endif 280 281 return err; 282 } 283 device_initcall(gre_offload_init); 284