Lines Matching refs:skb
96 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) in skb_gro_receive() argument
98 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); in skb_gro_receive()
99 unsigned int offset = skb_gro_offset(skb); in skb_gro_receive()
100 unsigned int headlen = skb_headlen(skb); in skb_gro_receive()
101 unsigned int len = skb_gro_len(skb); in skb_gro_receive()
114 if (p->pp_recycle != skb->pp_recycle) in skb_gro_receive()
122 if (unlikely(p->len + len >= gro_max_size || NAPI_GRO_CB(skb)->flush)) in skb_gro_receive()
126 if (NAPI_GRO_CB(skb)->proto != IPPROTO_TCP || in skb_gro_receive()
133 segs = NAPI_GRO_CB(skb)->count; in skb_gro_receive()
160 new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); in skb_gro_receive()
161 delta_truesize = skb->truesize - new_truesize; in skb_gro_receive()
163 skb->truesize = new_truesize; in skb_gro_receive()
164 skb->len -= skb->data_len; in skb_gro_receive()
165 skb->data_len = 0; in skb_gro_receive()
167 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; in skb_gro_receive()
169 } else if (skb->head_frag) { in skb_gro_receive()
172 struct page *page = virt_to_head_page(skb->head); in skb_gro_receive()
179 first_offset = skb->data - in skb_gro_receive()
191 delta_truesize = skb->truesize - new_truesize; in skb_gro_receive()
192 skb->truesize = new_truesize; in skb_gro_receive()
193 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; in skb_gro_receive()
199 skb->destructor = NULL; in skb_gro_receive()
200 skb->sk = NULL; in skb_gro_receive()
201 delta_truesize = skb->truesize; in skb_gro_receive()
207 skb->data_len -= eat; in skb_gro_receive()
208 skb->len -= eat; in skb_gro_receive()
212 __skb_pull(skb, offset); in skb_gro_receive()
215 skb_shinfo(p)->frag_list = skb; in skb_gro_receive()
217 NAPI_GRO_CB(p)->last->next = skb; in skb_gro_receive()
218 NAPI_GRO_CB(p)->last = skb; in skb_gro_receive()
219 __skb_header_release(skb); in skb_gro_receive()
232 NAPI_GRO_CB(skb)->same_flow = 1; in skb_gro_receive()
237 static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) in napi_gro_complete() argument
240 __be16 type = skb->protocol; in napi_gro_complete()
244 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); in napi_gro_complete()
246 if (NAPI_GRO_CB(skb)->count == 1) { in napi_gro_complete()
247 skb_shinfo(skb)->gso_size = 0; in napi_gro_complete()
258 skb, 0); in napi_gro_complete()
265 kfree_skb(skb); in napi_gro_complete()
270 gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count); in napi_gro_complete()
277 struct sk_buff *skb, *p; in __napi_gro_flush_chain() local
279 list_for_each_entry_safe_reverse(skb, p, head, list) { in __napi_gro_flush_chain()
280 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) in __napi_gro_flush_chain()
282 skb_list_del_init(skb); in __napi_gro_flush_chain()
283 napi_gro_complete(napi, skb); in __napi_gro_flush_chain()
308 static unsigned long gro_list_prepare_tc_ext(const struct sk_buff *skb, in gro_list_prepare_tc_ext() argument
316 skb_ext = skb_ext_find(skb, TC_SKB_EXT); in gro_list_prepare_tc_ext()
327 const struct sk_buff *skb) in gro_list_prepare() argument
329 unsigned int maclen = skb->dev->hard_header_len; in gro_list_prepare()
330 u32 hash = skb_get_hash_raw(skb); in gro_list_prepare()
343 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; in gro_list_prepare()
344 diffs |= p->vlan_all ^ skb->vlan_all; in gro_list_prepare()
345 diffs |= skb_metadata_differs(p, skb); in gro_list_prepare()
348 skb_mac_header(skb)); in gro_list_prepare()
351 skb_mac_header(skb), in gro_list_prepare()
359 if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) { in gro_list_prepare()
360 diffs |= p->sk != skb->sk; in gro_list_prepare()
361 diffs |= skb_metadata_dst_cmp(p, skb); in gro_list_prepare()
362 diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); in gro_list_prepare()
364 diffs |= gro_list_prepare_tc_ext(skb, p, diffs); in gro_list_prepare()
371 static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) in skb_gro_reset_offset() argument
373 const struct skb_shared_info *pinfo = skb_shinfo(skb); in skb_gro_reset_offset()
376 NAPI_GRO_CB(skb)->network_offset = 0; in skb_gro_reset_offset()
377 NAPI_GRO_CB(skb)->data_offset = 0; in skb_gro_reset_offset()
378 NAPI_GRO_CB(skb)->frag0 = NULL; in skb_gro_reset_offset()
379 NAPI_GRO_CB(skb)->frag0_len = 0; in skb_gro_reset_offset()
381 if (!skb_headlen(skb) && pinfo->nr_frags && in skb_gro_reset_offset()
384 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); in skb_gro_reset_offset()
385 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, in skb_gro_reset_offset()
387 skb->end - skb->tail); in skb_gro_reset_offset()
391 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) in gro_pull_from_frag0() argument
393 struct skb_shared_info *pinfo = skb_shinfo(skb); in gro_pull_from_frag0()
395 BUG_ON(skb->end - skb->tail < grow); in gro_pull_from_frag0()
397 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); in gro_pull_from_frag0()
399 skb->data_len -= grow; in gro_pull_from_frag0()
400 skb->tail += grow; in gro_pull_from_frag0()
406 skb_frag_unref(skb, 0); in gro_pull_from_frag0()
412 static void gro_try_pull_from_frag0(struct sk_buff *skb) in gro_try_pull_from_frag0() argument
414 int grow = skb_gro_offset(skb) - skb_headlen(skb); in gro_try_pull_from_frag0()
417 gro_pull_from_frag0(skb, grow); in gro_try_pull_from_frag0()
439 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) in dev_gro_receive() argument
441 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); in dev_gro_receive()
445 __be16 type = skb->protocol; in dev_gro_receive()
450 if (netif_elide_gro(skb->dev)) in dev_gro_receive()
453 gro_list_prepare(&gro_list->list, skb); in dev_gro_receive()
464 skb_set_network_header(skb, skb_gro_offset(skb)); in dev_gro_receive()
465 skb_reset_mac_len(skb); in dev_gro_receive()
469 *(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0; in dev_gro_receive()
470 NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb); in dev_gro_receive()
471 NAPI_GRO_CB(skb)->is_atomic = 1; in dev_gro_receive()
472 NAPI_GRO_CB(skb)->count = 1; in dev_gro_receive()
473 if (unlikely(skb_is_gso(skb))) { in dev_gro_receive()
474 NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs; in dev_gro_receive()
476 if (!skb_is_gso_tcp(skb) || in dev_gro_receive()
477 (skb_shinfo(skb)->gso_type & SKB_GSO_DODGY)) in dev_gro_receive()
478 NAPI_GRO_CB(skb)->flush = 1; in dev_gro_receive()
482 switch (skb->ip_summed) { in dev_gro_receive()
484 NAPI_GRO_CB(skb)->csum = skb->csum; in dev_gro_receive()
485 NAPI_GRO_CB(skb)->csum_valid = 1; in dev_gro_receive()
488 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; in dev_gro_receive()
494 &gro_list->list, skb); in dev_gro_receive()
503 same_flow = NAPI_GRO_CB(skb)->same_flow; in dev_gro_receive()
504 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; in dev_gro_receive()
515 if (NAPI_GRO_CB(skb)->flush) in dev_gro_receive()
524 gro_try_pull_from_frag0(skb); in dev_gro_receive()
525 NAPI_GRO_CB(skb)->age = jiffies; in dev_gro_receive()
526 NAPI_GRO_CB(skb)->last = skb; in dev_gro_receive()
527 if (!skb_is_gso(skb)) in dev_gro_receive()
528 skb_shinfo(skb)->gso_size = skb_gro_len(skb); in dev_gro_receive()
529 list_add(&skb->list, &gro_list->list); in dev_gro_receive()
543 gro_try_pull_from_frag0(skb); in dev_gro_receive()
576 struct sk_buff *skb, in napi_skb_finish() argument
581 gro_normal_one(napi, skb, 1); in napi_skb_finish()
585 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) in napi_skb_finish()
586 napi_skb_free_stolen_head(skb); in napi_skb_finish()
587 else if (skb->fclone != SKB_FCLONE_UNAVAILABLE) in napi_skb_finish()
588 __kfree_skb(skb); in napi_skb_finish()
590 __napi_kfree_skb(skb, SKB_CONSUMED); in napi_skb_finish()
602 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) in napi_gro_receive() argument
606 skb_mark_napi_id(skb, napi); in napi_gro_receive()
607 trace_napi_gro_receive_entry(skb); in napi_gro_receive()
609 skb_gro_reset_offset(skb, 0); in napi_gro_receive()
611 ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); in napi_gro_receive()
618 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) in napi_reuse_skb() argument
620 if (unlikely(skb->pfmemalloc)) { in napi_reuse_skb()
621 consume_skb(skb); in napi_reuse_skb()
624 __skb_pull(skb, skb_headlen(skb)); in napi_reuse_skb()
626 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); in napi_reuse_skb()
627 __vlan_hwaccel_clear_tag(skb); in napi_reuse_skb()
628 skb->dev = napi->dev; in napi_reuse_skb()
629 skb->skb_iif = 0; in napi_reuse_skb()
632 skb->pkt_type = PACKET_HOST; in napi_reuse_skb()
634 skb->encapsulation = 0; in napi_reuse_skb()
635 skb_shinfo(skb)->gso_type = 0; in napi_reuse_skb()
636 skb_shinfo(skb)->gso_size = 0; in napi_reuse_skb()
637 if (unlikely(skb->slow_gro)) { in napi_reuse_skb()
638 skb_orphan(skb); in napi_reuse_skb()
639 skb_ext_reset(skb); in napi_reuse_skb()
640 nf_reset_ct(skb); in napi_reuse_skb()
641 skb->slow_gro = 0; in napi_reuse_skb()
644 napi->skb = skb; in napi_reuse_skb()
649 struct sk_buff *skb = napi->skb; in napi_get_frags() local
651 if (!skb) { in napi_get_frags()
652 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); in napi_get_frags()
653 if (skb) { in napi_get_frags()
654 napi->skb = skb; in napi_get_frags()
655 skb_mark_napi_id(skb, napi); in napi_get_frags()
658 return skb; in napi_get_frags()
663 struct sk_buff *skb, in napi_frags_finish() argument
669 __skb_push(skb, ETH_HLEN); in napi_frags_finish()
670 skb->protocol = eth_type_trans(skb, skb->dev); in napi_frags_finish()
672 gro_normal_one(napi, skb, 1); in napi_frags_finish()
676 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) in napi_frags_finish()
677 napi_skb_free_stolen_head(skb); in napi_frags_finish()
679 napi_reuse_skb(napi, skb); in napi_frags_finish()
696 struct sk_buff *skb = napi->skb; in napi_frags_skb() local
700 napi->skb = NULL; in napi_frags_skb()
702 skb_reset_mac_header(skb); in napi_frags_skb()
703 skb_gro_reset_offset(skb, hlen); in napi_frags_skb()
705 if (unlikely(skb_gro_header_hard(skb, hlen))) { in napi_frags_skb()
706 eth = skb_gro_header_slow(skb, hlen, 0); in napi_frags_skb()
710 napi_reuse_skb(napi, skb); in napi_frags_skb()
714 eth = (const struct ethhdr *)skb->data; in napi_frags_skb()
715 gro_pull_from_frag0(skb, hlen); in napi_frags_skb()
716 NAPI_GRO_CB(skb)->frag0 += hlen; in napi_frags_skb()
717 NAPI_GRO_CB(skb)->frag0_len -= hlen; in napi_frags_skb()
719 __skb_pull(skb, hlen); in napi_frags_skb()
726 skb->protocol = eth->h_proto; in napi_frags_skb()
728 return skb; in napi_frags_skb()
734 struct sk_buff *skb = napi_frags_skb(napi); in napi_gro_frags() local
736 trace_napi_gro_frags_entry(skb); in napi_gro_frags()
738 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); in napi_gro_frags()
748 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) in __skb_gro_checksum_complete() argument
753 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); in __skb_gro_checksum_complete()
756 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); in __skb_gro_checksum_complete()
759 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && in __skb_gro_checksum_complete()
760 !skb->csum_complete_sw) in __skb_gro_checksum_complete()
761 netdev_rx_csum_fault(skb->dev, skb); in __skb_gro_checksum_complete()
764 NAPI_GRO_CB(skb)->csum = wsum; in __skb_gro_checksum_complete()
765 NAPI_GRO_CB(skb)->csum_valid = 1; in __skb_gro_checksum_complete()