Lines Matching refs:buff
23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, in aq_get_rxpages_xdp() argument
38 page_ref_inc(buff->rxdata.page); in aq_get_rxpages_xdp()
292 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; in aq_ring_tx_clean() local
294 if (likely(buff->is_mapped)) { in aq_ring_tx_clean()
295 if (unlikely(buff->is_sop)) { in aq_ring_tx_clean()
296 if (!buff->is_eop && in aq_ring_tx_clean()
297 buff->eop_index != 0xffffU && in aq_ring_tx_clean()
299 buff->eop_index, in aq_ring_tx_clean()
303 dma_unmap_single(dev, buff->pa, buff->len, in aq_ring_tx_clean()
306 dma_unmap_page(dev, buff->pa, buff->len, in aq_ring_tx_clean()
311 if (likely(!buff->is_eop)) in aq_ring_tx_clean()
314 if (buff->skb) { in aq_ring_tx_clean()
317 self->stats.tx.bytes += buff->skb->len; in aq_ring_tx_clean()
319 dev_kfree_skb_any(buff->skb); in aq_ring_tx_clean()
320 } else if (buff->xdpf) { in aq_ring_tx_clean()
323 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); in aq_ring_tx_clean()
325 xdp_return_frame_rx_napi(buff->xdpf); in aq_ring_tx_clean()
329 buff->skb = NULL; in aq_ring_tx_clean()
330 buff->xdpf = NULL; in aq_ring_tx_clean()
331 buff->pa = 0U; in aq_ring_tx_clean()
332 buff->eop_index = 0xffffU; in aq_ring_tx_clean()
340 struct aq_ring_buff_s *buff, in aq_rx_checksum() argument
346 if (unlikely(buff->is_cso_err)) { in aq_rx_checksum()
353 if (buff->is_ip_cso) { in aq_rx_checksum()
359 if (buff->is_udp_cso || buff->is_tcp_cso) in aq_rx_checksum()
388 struct aq_ring_buff_s *buff) in aq_xdp_build_skb() argument
401 aq_get_rxpages_xdp(buff, xdp); in aq_xdp_build_skb()
408 struct aq_ring_buff_s *buff) in aq_xdp_run_prog() argument
424 return aq_xdp_build_skb(xdp, aq_nic->ndev, buff); in aq_xdp_run_prog()
435 skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff); in aq_xdp_run_prog()
453 aq_get_rxpages_xdp(buff, xdp); in aq_xdp_run_prog()
462 aq_get_rxpages_xdp(buff, xdp); in aq_xdp_run_prog()
486 struct aq_ring_buff_s *buff, in aq_add_rx_fragment() argument
490 struct aq_ring_buff_s *buff_ = buff; in aq_add_rx_fragment()
513 buff->is_ip_cso &= buff_->is_ip_cso; in aq_add_rx_fragment()
514 buff->is_udp_cso &= buff_->is_udp_cso; in aq_add_rx_fragment()
515 buff->is_tcp_cso &= buff_->is_tcp_cso; in aq_add_rx_fragment()
516 buff->is_cso_err |= buff_->is_cso_err; in aq_add_rx_fragment()
537 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; in __aq_ring_rx_clean() local
545 if (buff->is_cleaned) in __aq_ring_rx_clean()
548 if (!buff->is_eop) { in __aq_ring_rx_clean()
550 buff_ = buff; in __aq_ring_rx_clean()
573 buff->is_error |= buff_->is_error; in __aq_ring_rx_clean()
574 buff->is_cso_err |= buff_->is_cso_err; in __aq_ring_rx_clean()
578 if (buff->is_error || in __aq_ring_rx_clean()
579 (buff->is_lro && buff->is_cso_err)) { in __aq_ring_rx_clean()
580 buff_ = buff; in __aq_ring_rx_clean()
599 if (buff->is_error) { in __aq_ring_rx_clean()
607 buff->rxdata.daddr, in __aq_ring_rx_clean()
608 buff->rxdata.pg_off, in __aq_ring_rx_clean()
609 buff->len, DMA_FROM_DEVICE); in __aq_ring_rx_clean()
620 buff->len -= in __aq_ring_rx_clean()
622 aq_buf_vaddr(&buff->rxdata), in __aq_ring_rx_clean()
623 buff->len); in __aq_ring_rx_clean()
625 hdr_len = buff->len; in __aq_ring_rx_clean()
628 aq_buf_vaddr(&buff->rxdata), in __aq_ring_rx_clean()
631 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), in __aq_ring_rx_clean()
634 if (buff->len - hdr_len > 0) { in __aq_ring_rx_clean()
635 skb_add_rx_frag(skb, i++, buff->rxdata.page, in __aq_ring_rx_clean()
636 buff->rxdata.pg_off + hdr_len, in __aq_ring_rx_clean()
637 buff->len - hdr_len, in __aq_ring_rx_clean()
639 page_ref_inc(buff->rxdata.page); in __aq_ring_rx_clean()
642 if (!buff->is_eop) { in __aq_ring_rx_clean()
643 buff_ = buff; in __aq_ring_rx_clean()
661 buff->is_ip_cso &= buff_->is_ip_cso; in __aq_ring_rx_clean()
662 buff->is_udp_cso &= buff_->is_udp_cso; in __aq_ring_rx_clean()
663 buff->is_tcp_cso &= buff_->is_tcp_cso; in __aq_ring_rx_clean()
664 buff->is_cso_err |= buff_->is_cso_err; in __aq_ring_rx_clean()
669 if (buff->is_vlan) in __aq_ring_rx_clean()
671 buff->vlan_rx_tag); in __aq_ring_rx_clean()
675 aq_rx_checksum(self, buff, skb); in __aq_ring_rx_clean()
677 skb_set_hash(skb, buff->rss_hash, in __aq_ring_rx_clean()
678 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : in __aq_ring_rx_clean()
713 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; in __aq_ring_xdp_clean() local
723 if (buff->is_cleaned) in __aq_ring_xdp_clean()
726 if (!buff->is_eop) { in __aq_ring_xdp_clean()
727 buff_ = buff; in __aq_ring_xdp_clean()
743 buff->is_error |= buff_->is_error; in __aq_ring_xdp_clean()
744 buff->is_cso_err |= buff_->is_cso_err; in __aq_ring_xdp_clean()
751 if (buff->is_error || in __aq_ring_xdp_clean()
752 (buff->is_lro && buff->is_cso_err)) { in __aq_ring_xdp_clean()
753 buff_ = buff; in __aq_ring_xdp_clean()
772 if (buff->is_error) { in __aq_ring_xdp_clean()
780 buff->rxdata.daddr, in __aq_ring_xdp_clean()
781 buff->rxdata.pg_off, in __aq_ring_xdp_clean()
782 buff->len, DMA_FROM_DEVICE); in __aq_ring_xdp_clean()
783 hard_start = page_address(buff->rxdata.page) + in __aq_ring_xdp_clean()
784 buff->rxdata.pg_off - rx_ring->page_offset; in __aq_ring_xdp_clean()
788 aq_buf_vaddr(&buff->rxdata), in __aq_ring_xdp_clean()
789 buff->len); in __aq_ring_xdp_clean()
790 buff->len -= ptp_hwtstamp_len; in __aq_ring_xdp_clean()
795 buff->len, false); in __aq_ring_xdp_clean()
796 if (!buff->is_eop) { in __aq_ring_xdp_clean()
797 if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) { in __aq_ring_xdp_clean()
807 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff); in __aq_ring_xdp_clean()
814 if (buff->is_vlan) in __aq_ring_xdp_clean()
816 buff->vlan_rx_tag); in __aq_ring_xdp_clean()
818 aq_rx_checksum(rx_ring, buff, skb); in __aq_ring_xdp_clean()
820 skb_set_hash(skb, buff->rss_hash, in __aq_ring_xdp_clean()
821 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : in __aq_ring_xdp_clean()
866 struct aq_ring_buff_s *buff = NULL; in aq_ring_rx_fill() local
876 buff = &self->buff_ring[self->sw_tail]; in aq_ring_rx_fill()
878 buff->flags = 0U; in aq_ring_rx_fill()
879 buff->len = self->frame_max; in aq_ring_rx_fill()
881 err = aq_get_rxpages(self, buff); in aq_ring_rx_fill()
885 buff->pa = aq_buf_daddr(&buff->rxdata); in aq_ring_rx_fill()
886 buff = NULL; in aq_ring_rx_fill()
900 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; in aq_ring_rx_deinit() local
902 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); in aq_ring_rx_deinit()