16 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
18 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
22 	int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);  in __skb_udp_tunnel_segment()
25 	struct udphdr *uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
26 	u16 mac_offset = skb->mac_header;  in __skb_udp_tunnel_segment()
27 	__be16 protocol = skb->protocol;  in __skb_udp_tunnel_segment()
28 	u16 mac_len = skb->mac_len;  in __skb_udp_tunnel_segment()
33 	if (unlikely(!pskb_may_pull(skb, tnl_hlen)))  in __skb_udp_tunnel_segment()
42 	if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)  in __skb_udp_tunnel_segment()
45 		partial = (__force __wsum)htonl(skb->len);  in __skb_udp_tunnel_segment()
49 	skb->encapsulation = 0;  in __skb_udp_tunnel_segment()
50 	SKB_GSO_CB(skb)->encap_level = 0;  in __skb_udp_tunnel_segment()
51 	__skb_pull(skb, tnl_hlen);  in __skb_udp_tunnel_segment()
52 	skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
53 	skb_set_network_header(skb, skb_inner_network_offset(skb));  in __skb_udp_tunnel_segment()
54 	skb_set_transport_header(skb, skb_inner_transport_offset(skb));  in __skb_udp_tunnel_segment()
55 	skb->mac_len = skb_inner_network_offset(skb);  in __skb_udp_tunnel_segment()
56 	skb->protocol = new_protocol;  in __skb_udp_tunnel_segment()
58 	need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);  in __skb_udp_tunnel_segment()
59 	skb->encap_hdr_csum = need_csum;  in __skb_udp_tunnel_segment()
61 	remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);  in __skb_udp_tunnel_segment()
62 	skb->remcsum_offload = remcsum;  in __skb_udp_tunnel_segment()
64 	need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));  in __skb_udp_tunnel_segment()
68 			  (skb->dev->features &  in __skb_udp_tunnel_segment()
72 	features &= skb->dev->hw_enc_features;  in __skb_udp_tunnel_segment()
87 	segs = gso_inner_segment(skb, features);  in __skb_udp_tunnel_segment()
89 		skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,  in __skb_udp_tunnel_segment()
96 	outer_hlen = skb_tnl_header_len(skb);  in __skb_udp_tunnel_segment()
98 	skb = segs;  in __skb_udp_tunnel_segment()
103 			skb->ip_summed = CHECKSUM_NONE;  in __skb_udp_tunnel_segment()
106 		if (skb->ip_summed == CHECKSUM_PARTIAL) {  in __skb_udp_tunnel_segment()
107 			skb_reset_inner_headers(skb);  in __skb_udp_tunnel_segment()
108 			skb->encapsulation = 1;  in __skb_udp_tunnel_segment()
111 		skb->mac_len = mac_len;  in __skb_udp_tunnel_segment()
112 		skb->protocol = protocol;  in __skb_udp_tunnel_segment()
114 		__skb_push(skb, outer_hlen);  in __skb_udp_tunnel_segment()
115 		skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
116 		skb_set_network_header(skb, mac_len);  in __skb_udp_tunnel_segment()
117 		skb_set_transport_header(skb, udp_offset);  in __skb_udp_tunnel_segment()
118 		len = skb->len - udp_offset;  in __skb_udp_tunnel_segment()
119 		uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
125 		if (gso_partial && skb_is_gso(skb)) {  in __skb_udp_tunnel_segment()
126 			uh->len = htons(skb_shinfo(skb)->gso_size +  in __skb_udp_tunnel_segment()
127 					SKB_GSO_CB(skb)->data_offset +  in __skb_udp_tunnel_segment()
128 					skb->head - (unsigned char *)uh);  in __skb_udp_tunnel_segment()
139 		if (skb->encapsulation || !offload_csum) {  in __skb_udp_tunnel_segment()
140 			uh->check = gso_make_checksum(skb, ~uh->check);  in __skb_udp_tunnel_segment()
144 			skb->ip_summed = CHECKSUM_PARTIAL;  in __skb_udp_tunnel_segment()
145 			skb->csum_start = skb_transport_header(skb) - skb->head;  in __skb_udp_tunnel_segment()
146 			skb->csum_offset = offsetof(struct udphdr, check);  in __skb_udp_tunnel_segment()
148 	} while ((skb = skb->next));  in __skb_udp_tunnel_segment()
153 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,  in skb_udp_tunnel_segment()  argument
158 	__be16 protocol = skb->protocol;  in skb_udp_tunnel_segment()
161 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in skb_udp_tunnel_segment()
166 	switch (skb->inner_protocol_type) {  in skb_udp_tunnel_segment()
168 		protocol = skb->inner_protocol;  in skb_udp_tunnel_segment()
173 		ops = rcu_dereference(offloads[skb->inner_ipproto]);  in skb_udp_tunnel_segment()
182 	segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,  in skb_udp_tunnel_segment()
250 static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,  in __udp_gso_segment_list()  argument
254 	unsigned int mss = skb_shinfo(skb)->gso_size;  in __udp_gso_segment_list()
256 	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));  in __udp_gso_segment_list()
257 	if (IS_ERR(skb))  in __udp_gso_segment_list()
258 		return skb;  in __udp_gso_segment_list()
260 	udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);  in __udp_gso_segment_list()
262 	return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);  in __udp_gso_segment_list()
399 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,  in udp4_ufo_fragment()  argument
408 	if (skb->encapsulation &&  in udp4_ufo_fragment()
409 	    (skb_shinfo(skb)->gso_type &  in udp4_ufo_fragment()
411 		segs = skb_udp_tunnel_segment(skb, features, false);  in udp4_ufo_fragment()
415 	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))  in udp4_ufo_fragment()
418 	if (!pskb_may_pull(skb, sizeof(struct udphdr)))  in udp4_ufo_fragment()
421 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)  in udp4_ufo_fragment()
422 		return __udp_gso_segment(skb, features, false);  in udp4_ufo_fragment()
424 	mss = skb_shinfo(skb)->gso_size;  in udp4_ufo_fragment()
425 	if (unlikely(skb->len <= mss))  in udp4_ufo_fragment()
433 	uh = udp_hdr(skb);  in udp4_ufo_fragment()
434 	iph = ip_hdr(skb);  in udp4_ufo_fragment()
437 	csum = skb_checksum(skb, 0, skb->len, 0);  in udp4_ufo_fragment()
438 	uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);  in udp4_ufo_fragment()
442 	skb->ip_summed = CHECKSUM_UNNECESSARY;  in udp4_ufo_fragment()
448 	if (!skb->encap_hdr_csum)  in udp4_ufo_fragment()
454 	segs = skb_segment(skb, features);  in udp4_ufo_fragment()
459 static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)  in skb_gro_receive_list()  argument
461 	if (unlikely(p->len + skb->len >= 65536))  in skb_gro_receive_list()
465 		skb_shinfo(p)->frag_list = skb;  in skb_gro_receive_list()
467 		NAPI_GRO_CB(p)->last->next = skb;  in skb_gro_receive_list()
469 	skb_pull(skb, skb_gro_offset(skb));  in skb_gro_receive_list()
471 	NAPI_GRO_CB(p)->last = skb;  in skb_gro_receive_list()
473 	p->data_len += skb->len;  in skb_gro_receive_list()
476 	skb->destructor = NULL;  in skb_gro_receive_list()
477 	skb->sk = NULL;  in skb_gro_receive_list()
478 	p->truesize += skb->truesize;  in skb_gro_receive_list()
479 	p->len += skb->len;  in skb_gro_receive_list()
481 	NAPI_GRO_CB(skb)->same_flow = 1;  in skb_gro_receive_list()
489 					       struct sk_buff *skb)  in udp_gro_receive_segment()  argument
491 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp_gro_receive_segment()
501 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
507 	if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {  in udp_gro_receive_segment()
508 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
512 	skb_gro_pull(skb, sizeof(struct udphdr));  in udp_gro_receive_segment()
526 		if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {  in udp_gro_receive_segment()
527 			NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
549 			if (NAPI_GRO_CB(skb)->is_flist) {  in udp_gro_receive_segment()
550 				if (!pskb_may_pull(skb, skb_gro_offset(skb))) {  in udp_gro_receive_segment()
551 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
554 				if ((skb->ip_summed != p->ip_summed) ||  in udp_gro_receive_segment()
555 				    (skb->csum_level != p->csum_level)) {  in udp_gro_receive_segment()
556 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
559 				ret = skb_gro_receive_list(p, skb);  in udp_gro_receive_segment()
561 				skb_gro_postpull_rcsum(skb, uh,  in udp_gro_receive_segment()
564 				ret = skb_gro_receive(p, skb);  in udp_gro_receive_segment()
579 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,  in udp_gro_receive()  argument
585 	unsigned int off = skb_gro_offset(skb);  in udp_gro_receive()
593 	NAPI_GRO_CB(skb)->is_flist = 0;  in udp_gro_receive()
598 		if (skb->encapsulation)  in udp_gro_receive()
601 		if (skb->dev->features & NETIF_F_GRO_FRAGLIST)  in udp_gro_receive()
602 			NAPI_GRO_CB(skb)->is_flist = sk ? !udp_test_bit(GRO_ENABLED, sk) : 1;  in udp_gro_receive()
604 		if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||  in udp_gro_receive()
605 		    (sk && udp_test_bit(GRO_ENABLED, sk)) || NAPI_GRO_CB(skb)->is_flist)  in udp_gro_receive()
606 			return call_gro_receive(udp_gro_receive_segment, head, skb);  in udp_gro_receive()
612 	if (NAPI_GRO_CB(skb)->encap_mark ||  in udp_gro_receive()
613 	    (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&  in udp_gro_receive()
614 	     NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in udp_gro_receive()
615 	     !NAPI_GRO_CB(skb)->csum_valid))  in udp_gro_receive()
619 	NAPI_GRO_CB(skb)->encap_mark = 1;  in udp_gro_receive()
639 	skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */  in udp_gro_receive()
640 	skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));  in udp_gro_receive()
641 	pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);  in udp_gro_receive()
644 	skb_gro_flush_final(skb, pp, flush);  in udp_gro_receive()
649 static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,  in udp4_gro_lookup_skb()  argument
652 	const struct iphdr *iph = skb_gro_network_header(skb);  in udp4_gro_lookup_skb()
653 	struct net *net = dev_net(skb->dev);  in udp4_gro_lookup_skb()
656 	inet_get_iif_sdif(skb, &iif, &sdif);  in udp4_gro_lookup_skb()
664 struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)  in udp4_gro_receive()  argument
666 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp4_gro_receive()
674 	if (NAPI_GRO_CB(skb)->flush)  in udp4_gro_receive()
677 	if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,  in udp4_gro_receive()
681 		skb_gro_checksum_try_convert(skb, IPPROTO_UDP,  in udp4_gro_receive()
684 	NAPI_GRO_CB(skb)->is_ipv6 = 0;  in udp4_gro_receive()
687 		sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);  in udp4_gro_receive()
689 	pp = udp_gro_receive(head, skb, uh, sk);  in udp4_gro_receive()
693 	NAPI_GRO_CB(skb)->flush = 1;  in udp4_gro_receive()
697 static int udp_gro_complete_segment(struct sk_buff *skb)  in udp_gro_complete_segment()  argument
699 	struct udphdr *uh = udp_hdr(skb);  in udp_gro_complete_segment()
701 	skb->csum_start = (unsigned char *)uh - skb->head;  in udp_gro_complete_segment()
702 	skb->csum_offset = offsetof(struct udphdr, check);  in udp_gro_complete_segment()
703 	skb->ip_summed = CHECKSUM_PARTIAL;  in udp_gro_complete_segment()
705 	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp_gro_complete_segment()
706 	skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;  in udp_gro_complete_segment()
708 	if (skb->encapsulation)  in udp_gro_complete_segment()
709 		skb->inner_transport_header = skb->transport_header;  in udp_gro_complete_segment()
714 int udp_gro_complete(struct sk_buff *skb, int nhoff,  in udp_gro_complete()  argument
717 	__be16 newlen = htons(skb->len - nhoff);  in udp_gro_complete()
718 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp_gro_complete()
725 				udp4_lib_lookup_skb, skb, uh->source, uh->dest);  in udp_gro_complete()
727 		skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM  in udp_gro_complete()
733 		NAPI_GRO_CB(skb)->encap_mark = 0;  in udp_gro_complete()
738 		skb->encapsulation = 1;  in udp_gro_complete()
739 		err = udp_sk(sk)->gro_complete(sk, skb,  in udp_gro_complete()
742 		err = udp_gro_complete_segment(skb);  in udp_gro_complete()
745 	if (skb->remcsum_offload)  in udp_gro_complete()
746 		skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;  in udp_gro_complete()
752 INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)  in udp4_gro_complete()  argument
754 	const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];  in udp4_gro_complete()
755 	const struct iphdr *iph = (struct iphdr *)(skb->data + offset);  in udp4_gro_complete()
756 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp4_gro_complete()
759 	if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {  in udp4_gro_complete()
760 		uh->len = htons(skb->len - nhoff);  in udp4_gro_complete()
762 		skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);  in udp4_gro_complete()
763 		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp4_gro_complete()
765 		__skb_incr_checksum_unnecessary(skb);  in udp4_gro_complete()
771 		uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,  in udp4_gro_complete()
774 	return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);  in udp4_gro_complete()