xref: /openbmc/linux/net/ipv4/udp_offload.c (revision a2fb4d78)
1 /*
2  *	IPV4 GSO/GRO offload support
3  *	Linux INET implementation
4  *
5  *	This program is free software; you can redistribute it and/or
6  *	modify it under the terms of the GNU General Public License
7  *	as published by the Free Software Foundation; either version
8  *	2 of the License, or (at your option) any later version.
9  *
10  *	UDPv4 GSO support
11  */
12 
13 #include <linux/skbuff.h>
14 #include <net/udp.h>
15 #include <net/protocol.h>
16 
17 static DEFINE_SPINLOCK(udp_offload_lock);
18 static struct udp_offload_priv __rcu *udp_offload_base __read_mostly;
19 
20 #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock))
21 
22 struct udp_offload_priv {
23 	struct udp_offload	*offload;
24 	struct rcu_head		rcu;
25 	struct udp_offload_priv __rcu *next;
26 };
27 
28 static int udp4_ufo_send_check(struct sk_buff *skb)
29 {
30 	if (!pskb_may_pull(skb, sizeof(struct udphdr)))
31 		return -EINVAL;
32 
33 	if (likely(!skb->encapsulation)) {
34 		const struct iphdr *iph;
35 		struct udphdr *uh;
36 
37 		iph = ip_hdr(skb);
38 		uh = udp_hdr(skb);
39 
40 		uh->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, skb->len,
41 				IPPROTO_UDP, 0);
42 		skb->csum_start = skb_transport_header(skb) - skb->head;
43 		skb->csum_offset = offsetof(struct udphdr, check);
44 		skb->ip_summed = CHECKSUM_PARTIAL;
45 	}
46 
47 	return 0;
48 }
49 
50 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
51 					 netdev_features_t features)
52 {
53 	struct sk_buff *segs = ERR_PTR(-EINVAL);
54 	unsigned int mss;
55 	int offset;
56 	__wsum csum;
57 
58 	if (skb->encapsulation &&
59 	    skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL) {
60 		segs = skb_udp_tunnel_segment(skb, features);
61 		goto out;
62 	}
63 
64 	mss = skb_shinfo(skb)->gso_size;
65 	if (unlikely(skb->len <= mss))
66 		goto out;
67 
68 	if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
69 		/* Packet is from an untrusted source, reset gso_segs. */
70 		int type = skb_shinfo(skb)->gso_type;
71 
72 		if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY |
73 				      SKB_GSO_UDP_TUNNEL |
74 				      SKB_GSO_IPIP |
75 				      SKB_GSO_GRE | SKB_GSO_MPLS) ||
76 			     !(type & (SKB_GSO_UDP))))
77 			goto out;
78 
79 		skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);
80 
81 		segs = NULL;
82 		goto out;
83 	}
84 
85 	/* Do software UFO. Complete and fill in the UDP checksum as
86 	 * HW cannot do checksum of UDP packets sent as multiple
87 	 * IP fragments.
88 	 */
89 	offset = skb_checksum_start_offset(skb);
90 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
91 	offset += skb->csum_offset;
92 	*(__sum16 *)(skb->data + offset) = csum_fold(csum);
93 	skb->ip_summed = CHECKSUM_NONE;
94 
95 	/* Fragment the skb. IP headers of the fragments are updated in
96 	 * inet_gso_segment()
97 	 */
98 	segs = skb_segment(skb, features);
99 out:
100 	return segs;
101 }
102 
103 int udp_add_offload(struct udp_offload *uo)
104 {
105 	struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC);
106 
107 	if (!new_offload)
108 		return -ENOMEM;
109 
110 	new_offload->offload = uo;
111 
112 	spin_lock(&udp_offload_lock);
113 	new_offload->next = udp_offload_base;
114 	rcu_assign_pointer(udp_offload_base, new_offload);
115 	spin_unlock(&udp_offload_lock);
116 
117 	return 0;
118 }
119 EXPORT_SYMBOL(udp_add_offload);
120 
121 static void udp_offload_free_routine(struct rcu_head *head)
122 {
123 	struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu);
124 	kfree(ou_priv);
125 }
126 
127 void udp_del_offload(struct udp_offload *uo)
128 {
129 	struct udp_offload_priv __rcu **head = &udp_offload_base;
130 	struct udp_offload_priv *uo_priv;
131 
132 	spin_lock(&udp_offload_lock);
133 
134 	uo_priv = udp_deref_protected(*head);
135 	for (; uo_priv != NULL;
136 	     uo_priv = udp_deref_protected(*head)) {
137 		if (uo_priv->offload == uo) {
138 			rcu_assign_pointer(*head,
139 					   udp_deref_protected(uo_priv->next));
140 			goto unlock;
141 		}
142 		head = &uo_priv->next;
143 	}
144 	pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port));
145 unlock:
146 	spin_unlock(&udp_offload_lock);
147 	if (uo_priv != NULL)
148 		call_rcu(&uo_priv->rcu, udp_offload_free_routine);
149 }
150 EXPORT_SYMBOL(udp_del_offload);
151 
152 static struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
153 {
154 	struct udp_offload_priv *uo_priv;
155 	struct sk_buff *p, **pp = NULL;
156 	struct udphdr *uh, *uh2;
157 	unsigned int hlen, off;
158 	int flush = 1;
159 
160 	if (NAPI_GRO_CB(skb)->udp_mark ||
161 	    (!skb->encapsulation && skb->ip_summed != CHECKSUM_COMPLETE))
162 		goto out;
163 
164 	/* mark that this skb passed once through the udp gro layer */
165 	NAPI_GRO_CB(skb)->udp_mark = 1;
166 
167 	off  = skb_gro_offset(skb);
168 	hlen = off + sizeof(*uh);
169 	uh   = skb_gro_header_fast(skb, off);
170 	if (skb_gro_header_hard(skb, hlen)) {
171 		uh = skb_gro_header_slow(skb, hlen, off);
172 		if (unlikely(!uh))
173 			goto out;
174 	}
175 
176 	rcu_read_lock();
177 	uo_priv = rcu_dereference(udp_offload_base);
178 	for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
179 		if (uo_priv->offload->port == uh->dest &&
180 		    uo_priv->offload->callbacks.gro_receive)
181 			goto unflush;
182 	}
183 	goto out_unlock;
184 
185 unflush:
186 	flush = 0;
187 
188 	for (p = *head; p; p = p->next) {
189 		if (!NAPI_GRO_CB(p)->same_flow)
190 			continue;
191 
192 		uh2 = (struct udphdr   *)(p->data + off);
193 		if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) {
194 			NAPI_GRO_CB(p)->same_flow = 0;
195 			continue;
196 		}
197 	}
198 
199 	skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
200 	pp = uo_priv->offload->callbacks.gro_receive(head, skb);
201 
202 out_unlock:
203 	rcu_read_unlock();
204 out:
205 	NAPI_GRO_CB(skb)->flush |= flush;
206 	return pp;
207 }
208 
209 static int udp_gro_complete(struct sk_buff *skb, int nhoff)
210 {
211 	struct udp_offload_priv *uo_priv;
212 	__be16 newlen = htons(skb->len - nhoff);
213 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
214 	int err = -ENOSYS;
215 
216 	uh->len = newlen;
217 
218 	rcu_read_lock();
219 
220 	uo_priv = rcu_dereference(udp_offload_base);
221 	for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
222 		if (uo_priv->offload->port == uh->dest &&
223 		    uo_priv->offload->callbacks.gro_complete)
224 			break;
225 	}
226 
227 	if (uo_priv != NULL)
228 		err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr));
229 
230 	rcu_read_unlock();
231 	return err;
232 }
233 
234 static const struct net_offload udpv4_offload = {
235 	.callbacks = {
236 		.gso_send_check = udp4_ufo_send_check,
237 		.gso_segment = udp4_ufo_fragment,
238 		.gro_receive  =	udp_gro_receive,
239 		.gro_complete =	udp_gro_complete,
240 	},
241 };
242 
243 int __init udpv4_offload_init(void)
244 {
245 	return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
246 }
247