33 #define indirect_call_gro_receive_l4(f2, f1, cb, head, skb)	\  argument
35 	unlikely(gro_recursion_inc_test(skb)) ?			\
36 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
37 		INDIRECT_CALL_L4(cb, f2, f1, head, skb);	\
40 static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto)  in ipv6_gro_pull_exthdrs()  argument
56 		opth = skb_gro_header(skb, off + sizeof(*opth), off);  in ipv6_gro_pull_exthdrs()
62 		opth = skb_gro_header(skb, off + len, off);  in ipv6_gro_pull_exthdrs()
70 	skb_gro_pull(skb, off - skb_network_offset(skb));  in ipv6_gro_pull_exthdrs()
74 static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)  in ipv6_gso_pull_exthdrs()  argument
92 		if (unlikely(!pskb_may_pull(skb, 8)))  in ipv6_gso_pull_exthdrs()
95 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
98 		if (unlikely(!pskb_may_pull(skb, len)))  in ipv6_gso_pull_exthdrs()
101 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
103 		__skb_pull(skb, len);  in ipv6_gso_pull_exthdrs()
109 static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,  in ipv6_gso_segment()  argument
124 	skb_reset_network_header(skb);  in ipv6_gso_segment()
125 	err = ipv6_hopopt_jumbo_remove(skb);  in ipv6_gso_segment()
128 	nhoff = skb_network_header(skb) - skb_mac_header(skb);  in ipv6_gso_segment()
129 	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))  in ipv6_gso_segment()
132 	encap = SKB_GSO_CB(skb)->encap_level > 0;  in ipv6_gso_segment()
134 		features &= skb->dev->hw_enc_features;  in ipv6_gso_segment()
135 	SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);  in ipv6_gso_segment()
137 	ipv6h = ipv6_hdr(skb);  in ipv6_gso_segment()
138 	__skb_pull(skb, sizeof(*ipv6h));  in ipv6_gso_segment()
141 	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);  in ipv6_gso_segment()
143 	if (skb->encapsulation &&  in ipv6_gso_segment()
144 	    skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))  in ipv6_gso_segment()
146 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
148 		udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&  in ipv6_gso_segment()
149 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
153 		skb_reset_transport_header(skb);  in ipv6_gso_segment()
154 		segs = ops->callbacks.gso_segment(skb, features);  in ipv6_gso_segment()
156 			skb->network_header = skb_mac_header(skb) + nhoff - skb->head;  in ipv6_gso_segment()
164 	for (skb = segs; skb; skb = skb->next) {  in ipv6_gso_segment()
165 		ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);  in ipv6_gso_segment()
166 		if (gso_partial && skb_is_gso(skb))  in ipv6_gso_segment()
167 			payload_len = skb_shinfo(skb)->gso_size +  in ipv6_gso_segment()
168 				      SKB_GSO_CB(skb)->data_offset +  in ipv6_gso_segment()
169 				      skb->head - (unsigned char *)(ipv6h + 1);  in ipv6_gso_segment()
171 			payload_len = skb->len - nhoff - sizeof(*ipv6h);  in ipv6_gso_segment()
173 		skb->network_header = (u8 *)ipv6h - skb->head;  in ipv6_gso_segment()
174 		skb_reset_mac_len(skb);  in ipv6_gso_segment()
177 			int err = ip6_find_1stfragopt(skb, &prevhdr);  in ipv6_gso_segment()
184 			if (skb->next)  in ipv6_gso_segment()
190 			skb_reset_inner_headers(skb);  in ipv6_gso_segment()
224 							 struct sk_buff *skb)  in ipv6_gro_receive()  argument
236 	off = skb_gro_offset(skb);  in ipv6_gro_receive()
238 	iph = skb_gro_header(skb, hlen, off);  in ipv6_gro_receive()
242 	skb_set_network_header(skb, off);  in ipv6_gro_receive()
243 	NAPI_GRO_CB(skb)->inner_network_offset = off;  in ipv6_gro_receive()
245 	flush += ntohs(iph->payload_len) != skb->len - hlen;  in ipv6_gro_receive()
250 		proto = ipv6_gro_pull_exthdrs(skb, hlen, proto);  in ipv6_gro_receive()
256 		iph = skb_gro_network_header(skb);  in ipv6_gro_receive()
258 		skb_gro_pull(skb, sizeof(*iph));  in ipv6_gro_receive()
261 	skb_set_transport_header(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
263 	NAPI_GRO_CB(skb)->proto = proto;  in ipv6_gro_receive()
266 	nlen = skb_network_header_len(skb);  in ipv6_gro_receive()
305 		if (NAPI_GRO_CB(skb)->is_atomic)  in ipv6_gro_receive()
309 	NAPI_GRO_CB(skb)->is_atomic = true;  in ipv6_gro_receive()
310 	NAPI_GRO_CB(skb)->flush |= flush;  in ipv6_gro_receive()
312 	skb_gro_postpull_rcsum(skb, iph, nlen);  in ipv6_gro_receive()
315 					 ops->callbacks.gro_receive, head, skb);  in ipv6_gro_receive()
318 	skb_gro_flush_final(skb, pp, flush);  in ipv6_gro_receive()
324 					      struct sk_buff *skb)  in sit_ip6ip6_gro_receive()  argument
328 	if (NAPI_GRO_CB(skb)->encap_mark) {  in sit_ip6ip6_gro_receive()
329 		NAPI_GRO_CB(skb)->flush = 1;  in sit_ip6ip6_gro_receive()
333 	NAPI_GRO_CB(skb)->encap_mark = 1;  in sit_ip6ip6_gro_receive()
335 	return ipv6_gro_receive(head, skb);  in sit_ip6ip6_gro_receive()
339 					  struct sk_buff *skb)  in ip4ip6_gro_receive()  argument
343 	if (NAPI_GRO_CB(skb)->encap_mark) {  in ip4ip6_gro_receive()
344 		NAPI_GRO_CB(skb)->flush = 1;  in ip4ip6_gro_receive()
348 	NAPI_GRO_CB(skb)->encap_mark = 1;  in ip4ip6_gro_receive()
350 	return inet_gro_receive(head, skb);  in ip4ip6_gro_receive()
353 INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)  in ipv6_gro_complete()  argument
360 	if (skb->encapsulation) {  in ipv6_gro_complete()
361 		skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));  in ipv6_gro_complete()
362 		skb_set_inner_network_header(skb, nhoff);  in ipv6_gro_complete()
365 	payload_len = skb->len - nhoff - sizeof(*iph);  in ipv6_gro_complete()
371 		memmove(skb_mac_header(skb) - hoplen, skb_mac_header(skb),  in ipv6_gro_complete()
372 			skb->transport_header - skb->mac_header);  in ipv6_gro_complete()
373 		skb->data -= hoplen;  in ipv6_gro_complete()
374 		skb->len += hoplen;  in ipv6_gro_complete()
375 		skb->mac_header -= hoplen;  in ipv6_gro_complete()
376 		skb->network_header -= hoplen;  in ipv6_gro_complete()
377 		iph = (struct ipv6hdr *)(skb->data + nhoff);  in ipv6_gro_complete()
390 		iph = (struct ipv6hdr *)(skb->data + nhoff);  in ipv6_gro_complete()
399 			       udp6_gro_complete, skb, nhoff);  in ipv6_gro_complete()
405 static int sit_gro_complete(struct sk_buff *skb, int nhoff)  in sit_gro_complete()  argument
407 	skb->encapsulation = 1;  in sit_gro_complete()
408 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;  in sit_gro_complete()
409 	return ipv6_gro_complete(skb, nhoff);  in sit_gro_complete()
412 static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip6ip6_gro_complete()  argument
414 	skb->encapsulation = 1;  in ip6ip6_gro_complete()
415 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip6ip6_gro_complete()
416 	return ipv6_gro_complete(skb, nhoff);  in ip6ip6_gro_complete()
419 static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip4ip6_gro_complete()  argument
421 	skb->encapsulation = 1;  in ip4ip6_gro_complete()
422 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip4ip6_gro_complete()
423 	return inet_gro_complete(skb, nhoff);  in ip4ip6_gro_complete()
435 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,  in sit_gso_segment()  argument
438 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))  in sit_gso_segment()
441 	return ipv6_gso_segment(skb, features);  in sit_gso_segment()
444 static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,  in ip4ip6_gso_segment()  argument
447 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip4ip6_gso_segment()
450 	return inet_gso_segment(skb, features);  in ip4ip6_gso_segment()
453 static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,  in ip6ip6_gso_segment()  argument
456 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip6ip6_gso_segment()
459 	return ipv6_gso_segment(skb, features);  in ip6ip6_gso_segment()