12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2c50cd357SDaniel Borkmann /*
3c50cd357SDaniel Borkmann * IPV4 GSO/GRO offload support
4c50cd357SDaniel Borkmann * Linux INET implementation
5c50cd357SDaniel Borkmann *
6c50cd357SDaniel Borkmann * GRE GSO support
7c50cd357SDaniel Borkmann */
8c50cd357SDaniel Borkmann
9c50cd357SDaniel Borkmann #include <linux/skbuff.h>
10cf172283SPaul Gortmaker #include <linux/init.h>
11c50cd357SDaniel Borkmann #include <net/protocol.h>
12c50cd357SDaniel Borkmann #include <net/gre.h>
134721031cSEric Dumazet #include <net/gro.h>
14*d457a0e3SEric Dumazet #include <net/gso.h>
15c50cd357SDaniel Borkmann
gre_gso_segment(struct sk_buff * skb,netdev_features_t features)16c50cd357SDaniel Borkmann static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
17c50cd357SDaniel Borkmann netdev_features_t features)
18c50cd357SDaniel Borkmann {
192e598af7SAlexander Duyck int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
20efa1a65cSXin Long bool need_csum, offload_csum, gso_partial, need_ipsec;
21c50cd357SDaniel Borkmann struct sk_buff *segs = ERR_PTR(-EINVAL);
227a7ffbabSWei-Chun Chao u16 mac_offset = skb->mac_header;
23c50cd357SDaniel Borkmann __be16 protocol = skb->protocol;
242e598af7SAlexander Duyck u16 mac_len = skb->mac_len;
252e598af7SAlexander Duyck int gre_offset, outer_hlen;
26c50cd357SDaniel Borkmann
2753e50398STom Herbert if (!skb->encapsulation)
2853e50398STom Herbert goto out;
2953e50398STom Herbert
302e598af7SAlexander Duyck if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr)))
312e598af7SAlexander Duyck goto out;
322e598af7SAlexander Duyck
332e598af7SAlexander Duyck if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
34c50cd357SDaniel Borkmann goto out;
35c50cd357SDaniel Borkmann
36c50cd357SDaniel Borkmann /* setup inner skb. */
37c50cd357SDaniel Borkmann skb->encapsulation = 0;
385197f349SAlexander Duyck SKB_GSO_CB(skb)->encap_level = 0;
392e598af7SAlexander Duyck __skb_pull(skb, tnl_hlen);
40c50cd357SDaniel Borkmann skb_reset_mac_header(skb);
41c50cd357SDaniel Borkmann skb_set_network_header(skb, skb_inner_network_offset(skb));
42c50cd357SDaniel Borkmann skb->mac_len = skb_inner_network_offset(skb);
4338720352SAlexander Duyck skb->protocol = skb->inner_protocol;
44c50cd357SDaniel Borkmann
452e598af7SAlexander Duyck need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM);
462e598af7SAlexander Duyck skb->encap_hdr_csum = need_csum;
472e598af7SAlexander Duyck
48bef3c6c9SAlexander Duyck features &= skb->dev->hw_enc_features;
491a236766SXin Long if (need_csum)
501a236766SXin Long features &= ~NETIF_F_SCTP_CRC;
51bef3c6c9SAlexander Duyck
52efa1a65cSXin Long need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));
53efa1a65cSXin Long /* Try to offload checksum if possible */
54efa1a65cSXin Long offload_csum = !!(need_csum && !need_ipsec &&
55efa1a65cSXin Long (skb->dev->features & NETIF_F_HW_CSUM));
56efa1a65cSXin Long
57c50cd357SDaniel Borkmann /* segment inner packet. */
58bef3c6c9SAlexander Duyck segs = skb_mac_gso_segment(skb, features);
595a8dbf03SHimangi Saraogi if (IS_ERR_OR_NULL(segs)) {
602e598af7SAlexander Duyck skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
612e598af7SAlexander Duyck mac_len);
62c50cd357SDaniel Borkmann goto out;
637a7ffbabSWei-Chun Chao }
64c50cd357SDaniel Borkmann
6507b26c94SSteffen Klassert gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
6607b26c94SSteffen Klassert
672e598af7SAlexander Duyck outer_hlen = skb_tnl_header_len(skb);
682e598af7SAlexander Duyck gre_offset = outer_hlen - tnl_hlen;
69c50cd357SDaniel Borkmann skb = segs;
70c50cd357SDaniel Borkmann do {
7138720352SAlexander Duyck struct gre_base_hdr *greh;
72802ab55aSAlexander Duyck __sum16 *pcsum;
73c50cd357SDaniel Borkmann
7422463876SAlexander Duyck /* Set up inner headers if we are offloading inner checksum */
7522463876SAlexander Duyck if (skb->ip_summed == CHECKSUM_PARTIAL) {
76cdbaa0bbSAlexander Duyck skb_reset_inner_headers(skb);
77cdbaa0bbSAlexander Duyck skb->encapsulation = 1;
7822463876SAlexander Duyck }
79cdbaa0bbSAlexander Duyck
80c50cd357SDaniel Borkmann skb->mac_len = mac_len;
81c50cd357SDaniel Borkmann skb->protocol = protocol;
822e598af7SAlexander Duyck
832e598af7SAlexander Duyck __skb_push(skb, outer_hlen);
842e598af7SAlexander Duyck skb_reset_mac_header(skb);
852e598af7SAlexander Duyck skb_set_network_header(skb, mac_len);
862e598af7SAlexander Duyck skb_set_transport_header(skb, gre_offset);
872e598af7SAlexander Duyck
882e598af7SAlexander Duyck if (!need_csum)
892e598af7SAlexander Duyck continue;
902e598af7SAlexander Duyck
912e598af7SAlexander Duyck greh = (struct gre_base_hdr *)skb_transport_header(skb);
92802ab55aSAlexander Duyck pcsum = (__sum16 *)(greh + 1);
932e598af7SAlexander Duyck
943d0241d5SAlexey Kodanev if (gso_partial && skb_is_gso(skb)) {
95802ab55aSAlexander Duyck unsigned int partial_adj;
96802ab55aSAlexander Duyck
97802ab55aSAlexander Duyck /* Adjust checksum to account for the fact that
98802ab55aSAlexander Duyck * the partial checksum is based on actual size
99802ab55aSAlexander Duyck * whereas headers should be based on MSS size.
100802ab55aSAlexander Duyck */
101802ab55aSAlexander Duyck partial_adj = skb->len + skb_headroom(skb) -
102802ab55aSAlexander Duyck SKB_GSO_CB(skb)->data_offset -
103802ab55aSAlexander Duyck skb_shinfo(skb)->gso_size;
104802ab55aSAlexander Duyck *pcsum = ~csum_fold((__force __wsum)htonl(partial_adj));
105802ab55aSAlexander Duyck } else {
1062e598af7SAlexander Duyck *pcsum = 0;
107802ab55aSAlexander Duyck }
108802ab55aSAlexander Duyck
109802ab55aSAlexander Duyck *(pcsum + 1) = 0;
110efa1a65cSXin Long if (skb->encapsulation || !offload_csum) {
111802ab55aSAlexander Duyck *pcsum = gso_make_checksum(skb, 0);
112efa1a65cSXin Long } else {
113efa1a65cSXin Long skb->ip_summed = CHECKSUM_PARTIAL;
114efa1a65cSXin Long skb->csum_start = skb_transport_header(skb) - skb->head;
115efa1a65cSXin Long skb->csum_offset = sizeof(*greh);
116efa1a65cSXin Long }
117c50cd357SDaniel Borkmann } while ((skb = skb->next));
118c50cd357SDaniel Borkmann out:
119c50cd357SDaniel Borkmann return segs;
120c50cd357SDaniel Borkmann }
121c50cd357SDaniel Borkmann
gre_gro_receive(struct list_head * head,struct sk_buff * skb)122d4546c25SDavid Miller static struct sk_buff *gre_gro_receive(struct list_head *head,
123bf5a755fSJerry Chu struct sk_buff *skb)
124bf5a755fSJerry Chu {
125d4546c25SDavid Miller struct sk_buff *pp = NULL;
126bf5a755fSJerry Chu struct sk_buff *p;
127bf5a755fSJerry Chu const struct gre_base_hdr *greh;
128bf5a755fSJerry Chu unsigned int hlen, grehlen;
129bf5a755fSJerry Chu unsigned int off;
130bf5a755fSJerry Chu int flush = 1;
131bf5a755fSJerry Chu struct packet_offload *ptype;
132bf5a755fSJerry Chu __be16 type;
133bf5a755fSJerry Chu
134fac8e0f5SJesse Gross if (NAPI_GRO_CB(skb)->encap_mark)
135fac8e0f5SJesse Gross goto out;
136fac8e0f5SJesse Gross
137fac8e0f5SJesse Gross NAPI_GRO_CB(skb)->encap_mark = 1;
138fac8e0f5SJesse Gross
139bf5a755fSJerry Chu off = skb_gro_offset(skb);
140bf5a755fSJerry Chu hlen = off + sizeof(*greh);
14135ffb665SRichard Gobert greh = skb_gro_header(skb, hlen, off);
142bf5a755fSJerry Chu if (unlikely(!greh))
143bf5a755fSJerry Chu goto out;
144bf5a755fSJerry Chu
145bf5a755fSJerry Chu /* Only support version 0 and K (key), C (csum) flags. Note that
146bf5a755fSJerry Chu * although the support for the S (seq#) flag can be added easily
147bf5a755fSJerry Chu * for GRO, this is problematic for GSO hence can not be enabled
148bf5a755fSJerry Chu * here because a GRO pkt may end up in the forwarding path, thus
149bf5a755fSJerry Chu * requiring GSO support to break it up correctly.
150bf5a755fSJerry Chu */
151bf5a755fSJerry Chu if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
152bf5a755fSJerry Chu goto out;
153bf5a755fSJerry Chu
154a0ca153fSAlexander Duyck /* We can only support GRE_CSUM if we can track the location of
155a0ca153fSAlexander Duyck * the GRE header. In the case of FOU/GUE we cannot because the
156a0ca153fSAlexander Duyck * outer UDP header displaces the GRE header leaving us in a state
157a0ca153fSAlexander Duyck * of limbo.
158a0ca153fSAlexander Duyck */
159a0ca153fSAlexander Duyck if ((greh->flags & GRE_CSUM) && NAPI_GRO_CB(skb)->is_fou)
160a0ca153fSAlexander Duyck goto out;
161a0ca153fSAlexander Duyck
162bf5a755fSJerry Chu type = greh->protocol;
163bf5a755fSJerry Chu
164bf5a755fSJerry Chu ptype = gro_find_receive_by_type(type);
16551456b29SIan Morris if (!ptype)
166fc1ca334SEric Dumazet goto out;
167bf5a755fSJerry Chu
168bf5a755fSJerry Chu grehlen = GRE_HEADER_SECTION;
169bf5a755fSJerry Chu
170bf5a755fSJerry Chu if (greh->flags & GRE_KEY)
171bf5a755fSJerry Chu grehlen += GRE_HEADER_SECTION;
172bf5a755fSJerry Chu
173bf5a755fSJerry Chu if (greh->flags & GRE_CSUM)
174bf5a755fSJerry Chu grehlen += GRE_HEADER_SECTION;
175bf5a755fSJerry Chu
176bf5a755fSJerry Chu hlen = off + grehlen;
177bf5a755fSJerry Chu if (skb_gro_header_hard(skb, hlen)) {
178bf5a755fSJerry Chu greh = skb_gro_header_slow(skb, hlen, off);
179bf5a755fSJerry Chu if (unlikely(!greh))
180fc1ca334SEric Dumazet goto out;
181bf5a755fSJerry Chu }
182bf5a755fSJerry Chu
183758f75d1STom Herbert /* Don't bother verifying checksum if we're going to flush anyway. */
184884d338cSTom Herbert if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
185884d338cSTom Herbert if (skb_gro_checksum_simple_validate(skb))
186fc1ca334SEric Dumazet goto out;
187758f75d1STom Herbert
188b39c78b2SLi RongQing skb_gro_checksum_try_convert(skb, IPPROTO_GRE,
189884d338cSTom Herbert null_compute_pseudo);
190884d338cSTom Herbert }
191884d338cSTom Herbert
192d4546c25SDavid Miller list_for_each_entry(p, head, list) {
193bf5a755fSJerry Chu const struct gre_base_hdr *greh2;
194bf5a755fSJerry Chu
195bf5a755fSJerry Chu if (!NAPI_GRO_CB(p)->same_flow)
196bf5a755fSJerry Chu continue;
197bf5a755fSJerry Chu
198bf5a755fSJerry Chu /* The following checks are needed to ensure only pkts
199bf5a755fSJerry Chu * from the same tunnel are considered for aggregation.
200bf5a755fSJerry Chu * The criteria for "the same tunnel" includes:
201bf5a755fSJerry Chu * 1) same version (we only support version 0 here)
202bf5a755fSJerry Chu * 2) same protocol (we only support ETH_P_IP for now)
203bf5a755fSJerry Chu * 3) same set of flags
204bf5a755fSJerry Chu * 4) same key if the key field is present.
205bf5a755fSJerry Chu */
206bf5a755fSJerry Chu greh2 = (struct gre_base_hdr *)(p->data + off);
207bf5a755fSJerry Chu
208bf5a755fSJerry Chu if (greh2->flags != greh->flags ||
209bf5a755fSJerry Chu greh2->protocol != greh->protocol) {
210bf5a755fSJerry Chu NAPI_GRO_CB(p)->same_flow = 0;
211bf5a755fSJerry Chu continue;
212bf5a755fSJerry Chu }
213bf5a755fSJerry Chu if (greh->flags & GRE_KEY) {
214bf5a755fSJerry Chu /* compare keys */
215bf5a755fSJerry Chu if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
216bf5a755fSJerry Chu NAPI_GRO_CB(p)->same_flow = 0;
217bf5a755fSJerry Chu continue;
218bf5a755fSJerry Chu }
219bf5a755fSJerry Chu }
220bf5a755fSJerry Chu }
221bf5a755fSJerry Chu
222bf5a755fSJerry Chu skb_gro_pull(skb, grehlen);
223bf5a755fSJerry Chu
224bf5a755fSJerry Chu /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
225bf5a755fSJerry Chu skb_gro_postpull_rcsum(skb, greh, grehlen);
226bf5a755fSJerry Chu
227fcd91dd4SSabrina Dubroca pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
228c194cf93SAlexander Duyck flush = 0;
229bf5a755fSJerry Chu
230bf5a755fSJerry Chu out:
231603d4cf8SSabrina Dubroca skb_gro_flush_final(skb, pp, flush);
232bf5a755fSJerry Chu
233bf5a755fSJerry Chu return pp;
234bf5a755fSJerry Chu }
235bf5a755fSJerry Chu
gre_gro_complete(struct sk_buff * skb,int nhoff)236d10dbad2SWei Yongjun static int gre_gro_complete(struct sk_buff *skb, int nhoff)
237bf5a755fSJerry Chu {
238bf5a755fSJerry Chu struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
239bf5a755fSJerry Chu struct packet_offload *ptype;
240bf5a755fSJerry Chu unsigned int grehlen = sizeof(*greh);
241bf5a755fSJerry Chu int err = -ENOENT;
242bf5a755fSJerry Chu __be16 type;
243bf5a755fSJerry Chu
244c3caf119SJerry Chu skb->encapsulation = 1;
245c3caf119SJerry Chu skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
246c3caf119SJerry Chu
247bf5a755fSJerry Chu type = greh->protocol;
248bf5a755fSJerry Chu if (greh->flags & GRE_KEY)
249bf5a755fSJerry Chu grehlen += GRE_HEADER_SECTION;
250bf5a755fSJerry Chu
251bf5a755fSJerry Chu if (greh->flags & GRE_CSUM)
252bf5a755fSJerry Chu grehlen += GRE_HEADER_SECTION;
253bf5a755fSJerry Chu
254bf5a755fSJerry Chu ptype = gro_find_complete_by_type(type);
25500db4124SIan Morris if (ptype)
256bf5a755fSJerry Chu err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
257bf5a755fSJerry Chu
2586fb2a756STom Herbert skb_set_inner_mac_header(skb, nhoff + grehlen);
2596fb2a756STom Herbert
260bf5a755fSJerry Chu return err;
261bf5a755fSJerry Chu }
262bf5a755fSJerry Chu
263c50cd357SDaniel Borkmann static const struct net_offload gre_offload = {
264c50cd357SDaniel Borkmann .callbacks = {
265c50cd357SDaniel Borkmann .gso_segment = gre_gso_segment,
266bf5a755fSJerry Chu .gro_receive = gre_gro_receive,
267bf5a755fSJerry Chu .gro_complete = gre_gro_complete,
268c50cd357SDaniel Borkmann },
269c50cd357SDaniel Borkmann };
270c50cd357SDaniel Borkmann
gre_offload_init(void)271438e38faSEric Dumazet static int __init gre_offload_init(void)
272c50cd357SDaniel Borkmann {
273e0c20967SAlexander Duyck int err;
274e0c20967SAlexander Duyck
275e0c20967SAlexander Duyck err = inet_add_offload(&gre_offload, IPPROTO_GRE);
276e0c20967SAlexander Duyck #if IS_ENABLED(CONFIG_IPV6)
277e0c20967SAlexander Duyck if (err)
278e0c20967SAlexander Duyck return err;
279e0c20967SAlexander Duyck
280e0c20967SAlexander Duyck err = inet6_add_offload(&gre_offload, IPPROTO_GRE);
281e0c20967SAlexander Duyck if (err)
282e0c20967SAlexander Duyck inet_del_offload(&gre_offload, IPPROTO_GRE);
283e0c20967SAlexander Duyck #endif
284e0c20967SAlexander Duyck
285e0c20967SAlexander Duyck return err;
286c50cd357SDaniel Borkmann }
287cf172283SPaul Gortmaker device_initcall(gre_offload_init);
288