1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 9 10 #include "aq_nic.h" 11 #include "aq_hw.h" 12 #include "aq_hw_utils.h" 13 #include "aq_ptp.h" 14 #include "aq_vec.h" 15 #include "aq_main.h" 16 17 #include <net/xdp.h> 18 #include <linux/filter.h> 19 #include <linux/bpf_trace.h> 20 #include <linux/netdevice.h> 21 #include <linux/etherdevice.h> 22 23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, 24 struct xdp_buff *xdp) 25 { 26 struct skb_shared_info *sinfo; 27 int i; 28 29 if (xdp_buff_has_frags(xdp)) { 30 sinfo = xdp_get_shared_info_from_buff(xdp); 31 32 for (i = 0; i < sinfo->nr_frags; i++) { 33 skb_frag_t *frag = &sinfo->frags[i]; 34 35 page_ref_inc(skb_frag_page(frag)); 36 } 37 } 38 page_ref_inc(buff->rxdata.page); 39 } 40 41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 42 { 43 unsigned int len = PAGE_SIZE << rxpage->order; 44 45 dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); 46 47 /* Drop the ref for being in the ring. */ 48 __free_pages(rxpage->page, rxpage->order); 49 rxpage->page = NULL; 50 } 51 52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring) 53 { 54 struct device *dev = aq_nic_get_dev(rx_ring->aq_nic); 55 unsigned int order = rx_ring->page_order; 56 struct page *page; 57 int ret = -ENOMEM; 58 dma_addr_t daddr; 59 60 page = dev_alloc_pages(order); 61 if (unlikely(!page)) 62 goto err_exit; 63 64 daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, 65 DMA_FROM_DEVICE); 66 67 if (unlikely(dma_mapping_error(dev, daddr))) 68 goto free_page; 69 70 rxpage->page = page; 71 rxpage->daddr = daddr; 72 rxpage->order = order; 73 rxpage->pg_off = rx_ring->page_offset; 74 75 return 0; 76 77 free_page: 78 __free_pages(page, order); 79 80 err_exit: 81 return ret; 82 } 83 84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf) 85 { 86 unsigned int order = self->page_order; 87 u16 page_offset = self->page_offset; 88 u16 frame_max = self->frame_max; 89 u16 tail_size = self->tail_size; 90 int ret; 91 92 if (rxbuf->rxdata.page) { 93 /* One means ring is the only user and can reuse */ 94 if (page_ref_count(rxbuf->rxdata.page) > 1) { 95 /* Try reuse buffer */ 96 rxbuf->rxdata.pg_off += frame_max + page_offset + 97 tail_size; 98 if (rxbuf->rxdata.pg_off + frame_max + tail_size <= 99 (PAGE_SIZE << order)) { 100 u64_stats_update_begin(&self->stats.rx.syncp); 101 self->stats.rx.pg_flips++; 102 u64_stats_update_end(&self->stats.rx.syncp); 103 104 } else { 105 /* Buffer exhausted. We have other users and 106 * should release this page and realloc 107 */ 108 aq_free_rxpage(&rxbuf->rxdata, 109 aq_nic_get_dev(self->aq_nic)); 110 u64_stats_update_begin(&self->stats.rx.syncp); 111 self->stats.rx.pg_losts++; 112 u64_stats_update_end(&self->stats.rx.syncp); 113 } 114 } else { 115 rxbuf->rxdata.pg_off = page_offset; 116 u64_stats_update_begin(&self->stats.rx.syncp); 117 self->stats.rx.pg_reuses++; 118 u64_stats_update_end(&self->stats.rx.syncp); 119 } 120 } 121 122 if (!rxbuf->rxdata.page) { 123 ret = aq_alloc_rxpages(&rxbuf->rxdata, self); 124 if (ret) { 125 u64_stats_update_begin(&self->stats.rx.syncp); 126 self->stats.rx.alloc_fails++; 127 u64_stats_update_end(&self->stats.rx.syncp); 128 } 129 return ret; 130 } 131 132 return 0; 133 } 134 135 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 136 struct aq_nic_s *aq_nic) 137 { 138 int err = 0; 139 140 self->buff_ring = 141 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 142 143 if (!self->buff_ring) { 144 err = -ENOMEM; 145 goto err_exit; 146 } 147 148 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 149 self->size * self->dx_size, 150 &self->dx_ring_pa, GFP_KERNEL); 151 if (!self->dx_ring) { 152 err = -ENOMEM; 153 goto err_exit; 154 } 155 156 err_exit: 157 if (err < 0) { 158 aq_ring_free(self); 159 self = NULL; 160 } 161 162 return self; 163 } 164 165 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 166 struct aq_nic_s *aq_nic, 167 unsigned int idx, 168 struct aq_nic_cfg_s *aq_nic_cfg) 169 { 170 int err = 0; 171 172 self->aq_nic = aq_nic; 173 self->idx = idx; 174 self->size = aq_nic_cfg->txds; 175 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 176 177 self = aq_ring_alloc(self, aq_nic); 178 if (!self) { 179 err = -ENOMEM; 180 goto err_exit; 181 } 182 183 err_exit: 184 if (err < 0) { 185 aq_ring_free(self); 186 self = NULL; 187 } 188 189 return self; 190 } 191 192 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 193 struct aq_nic_s *aq_nic, 194 unsigned int idx, 195 struct aq_nic_cfg_s *aq_nic_cfg) 196 { 197 int err = 0; 198 199 self->aq_nic = aq_nic; 200 self->idx = idx; 201 self->size = aq_nic_cfg->rxds; 202 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 203 self->xdp_prog = aq_nic->xdp_prog; 204 self->frame_max = AQ_CFG_RX_FRAME_MAX; 205 206 /* Only order-2 is allowed if XDP is enabled */ 207 if (READ_ONCE(self->xdp_prog)) { 208 self->page_offset = AQ_XDP_HEADROOM; 209 self->page_order = AQ_CFG_XDP_PAGEORDER; 210 self->tail_size = AQ_XDP_TAILROOM; 211 } else { 212 self->page_offset = 0; 213 self->page_order = fls(self->frame_max / PAGE_SIZE + 214 (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1; 215 if (aq_nic_cfg->rxpageorder > self->page_order) 216 self->page_order = aq_nic_cfg->rxpageorder; 217 self->tail_size = 0; 218 } 219 220 self = aq_ring_alloc(self, aq_nic); 221 if (!self) { 222 err = -ENOMEM; 223 goto err_exit; 224 } 225 226 err_exit: 227 if (err < 0) { 228 aq_ring_free(self); 229 self = NULL; 230 } 231 232 return self; 233 } 234 235 struct aq_ring_s * 236 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, 237 unsigned int idx, unsigned int size, unsigned int dx_size) 238 { 239 struct device *dev = aq_nic_get_dev(aq_nic); 240 size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; 241 242 memset(self, 0, sizeof(*self)); 243 244 self->aq_nic = aq_nic; 245 self->idx = idx; 246 self->size = size; 247 self->dx_size = dx_size; 248 249 self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa, 250 GFP_KERNEL); 251 if (!self->dx_ring) { 252 aq_ring_free(self); 253 return NULL; 254 } 255 256 return self; 257 } 258 259 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type) 260 { 261 self->hw_head = 0; 262 self->sw_head = 0; 263 self->sw_tail = 0; 264 self->ring_type = ring_type; 265 266 if (self->ring_type == ATL_RING_RX) 267 u64_stats_init(&self->stats.rx.syncp); 268 else 269 u64_stats_init(&self->stats.tx.syncp); 270 271 return 0; 272 } 273 274 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 275 unsigned int t) 276 { 277 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 278 } 279 280 void aq_ring_update_queue_state(struct aq_ring_s *ring) 281 { 282 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 283 aq_ring_queue_stop(ring); 284 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 285 aq_ring_queue_wake(ring); 286 } 287 288 void aq_ring_queue_wake(struct aq_ring_s *ring) 289 { 290 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 291 292 if (__netif_subqueue_stopped(ndev, 293 AQ_NIC_RING2QMAP(ring->aq_nic, 294 ring->idx))) { 295 netif_wake_subqueue(ndev, 296 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 297 u64_stats_update_begin(&ring->stats.tx.syncp); 298 ring->stats.tx.queue_restarts++; 299 u64_stats_update_end(&ring->stats.tx.syncp); 300 } 301 } 302 303 void aq_ring_queue_stop(struct aq_ring_s *ring) 304 { 305 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 306 307 if (!__netif_subqueue_stopped(ndev, 308 AQ_NIC_RING2QMAP(ring->aq_nic, 309 ring->idx))) 310 netif_stop_subqueue(ndev, 311 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 312 } 313 314 bool aq_ring_tx_clean(struct aq_ring_s *self) 315 { 316 struct device *dev = aq_nic_get_dev(self->aq_nic); 317 unsigned int budget; 318 319 for (budget = AQ_CFG_TX_CLEAN_BUDGET; 320 budget && self->sw_head != self->hw_head; budget--) { 321 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 322 323 if (likely(buff->is_mapped)) { 324 if (unlikely(buff->is_sop)) { 325 if (!buff->is_eop && 326 buff->eop_index != 0xffffU && 327 (!aq_ring_dx_in_range(self->sw_head, 328 buff->eop_index, 329 self->hw_head))) 330 break; 331 332 dma_unmap_single(dev, buff->pa, buff->len, 333 DMA_TO_DEVICE); 334 } else { 335 dma_unmap_page(dev, buff->pa, buff->len, 336 DMA_TO_DEVICE); 337 } 338 } 339 340 if (likely(!buff->is_eop)) 341 goto out; 342 343 if (buff->skb) { 344 u64_stats_update_begin(&self->stats.tx.syncp); 345 ++self->stats.tx.packets; 346 self->stats.tx.bytes += buff->skb->len; 347 u64_stats_update_end(&self->stats.tx.syncp); 348 dev_kfree_skb_any(buff->skb); 349 } else if (buff->xdpf) { 350 u64_stats_update_begin(&self->stats.tx.syncp); 351 ++self->stats.tx.packets; 352 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); 353 u64_stats_update_end(&self->stats.tx.syncp); 354 xdp_return_frame_rx_napi(buff->xdpf); 355 } 356 357 out: 358 buff->skb = NULL; 359 buff->xdpf = NULL; 360 buff->pa = 0U; 361 buff->eop_index = 0xffffU; 362 self->sw_head = aq_ring_next_dx(self, self->sw_head); 363 } 364 365 return !!budget; 366 } 367 368 static void aq_rx_checksum(struct aq_ring_s *self, 369 struct aq_ring_buff_s *buff, 370 struct sk_buff *skb) 371 { 372 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 373 return; 374 375 if (unlikely(buff->is_cso_err)) { 376 u64_stats_update_begin(&self->stats.rx.syncp); 377 ++self->stats.rx.errors; 378 u64_stats_update_end(&self->stats.rx.syncp); 379 skb->ip_summed = CHECKSUM_NONE; 380 return; 381 } 382 if (buff->is_ip_cso) { 383 __skb_incr_checksum_unnecessary(skb); 384 } else { 385 skb->ip_summed = CHECKSUM_NONE; 386 } 387 388 if (buff->is_udp_cso || buff->is_tcp_cso) 389 __skb_incr_checksum_unnecessary(skb); 390 } 391 392 int aq_xdp_xmit(struct net_device *dev, int num_frames, 393 struct xdp_frame **frames, u32 flags) 394 { 395 struct aq_nic_s *aq_nic = netdev_priv(dev); 396 unsigned int vec, i, drop = 0; 397 int cpu = smp_processor_id(); 398 struct aq_nic_cfg_s *aq_cfg; 399 struct aq_ring_s *ring; 400 401 aq_cfg = aq_nic_get_cfg(aq_nic); 402 vec = cpu % aq_cfg->vecs; 403 ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)]; 404 405 for (i = 0; i < num_frames; i++) { 406 struct xdp_frame *xdpf = frames[i]; 407 408 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY) 409 drop++; 410 } 411 412 return num_frames - drop; 413 } 414 415 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic, 416 struct xdp_buff *xdp, 417 struct aq_ring_s *rx_ring, 418 struct aq_ring_buff_s *buff) 419 { 420 int result = NETDEV_TX_BUSY; 421 struct aq_ring_s *tx_ring; 422 struct xdp_frame *xdpf; 423 struct bpf_prog *prog; 424 u32 act = XDP_ABORTED; 425 struct sk_buff *skb; 426 427 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 428 ++rx_ring->stats.rx.packets; 429 rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp); 430 u64_stats_update_end(&rx_ring->stats.rx.syncp); 431 432 prog = READ_ONCE(rx_ring->xdp_prog); 433 if (!prog) 434 goto pass; 435 436 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 437 438 /* single buffer XDP program, but packet is multi buffer, aborted */ 439 if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags) 440 goto out_aborted; 441 442 act = bpf_prog_run_xdp(prog, xdp); 443 switch (act) { 444 case XDP_PASS: 445 pass: 446 xdpf = xdp_convert_buff_to_frame(xdp); 447 if (unlikely(!xdpf)) 448 goto out_aborted; 449 skb = xdp_build_skb_from_frame(xdpf, aq_nic->ndev); 450 if (!skb) 451 goto out_aborted; 452 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 453 ++rx_ring->stats.rx.xdp_pass; 454 u64_stats_update_end(&rx_ring->stats.rx.syncp); 455 aq_get_rxpages_xdp(buff, xdp); 456 return skb; 457 case XDP_TX: 458 xdpf = xdp_convert_buff_to_frame(xdp); 459 if (unlikely(!xdpf)) 460 goto out_aborted; 461 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx]; 462 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf); 463 if (result == NETDEV_TX_BUSY) 464 goto out_aborted; 465 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 466 ++rx_ring->stats.rx.xdp_tx; 467 u64_stats_update_end(&rx_ring->stats.rx.syncp); 468 aq_get_rxpages_xdp(buff, xdp); 469 break; 470 case XDP_REDIRECT: 471 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0) 472 goto out_aborted; 473 xdp_do_flush(); 474 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 475 ++rx_ring->stats.rx.xdp_redirect; 476 u64_stats_update_end(&rx_ring->stats.rx.syncp); 477 aq_get_rxpages_xdp(buff, xdp); 478 break; 479 default: 480 fallthrough; 481 case XDP_ABORTED: 482 out_aborted: 483 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 484 ++rx_ring->stats.rx.xdp_aborted; 485 u64_stats_update_end(&rx_ring->stats.rx.syncp); 486 trace_xdp_exception(aq_nic->ndev, prog, act); 487 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act); 488 break; 489 case XDP_DROP: 490 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 491 ++rx_ring->stats.rx.xdp_drop; 492 u64_stats_update_end(&rx_ring->stats.rx.syncp); 493 break; 494 } 495 496 return ERR_PTR(-result); 497 } 498 499 static bool aq_add_rx_fragment(struct device *dev, 500 struct aq_ring_s *ring, 501 struct aq_ring_buff_s *buff, 502 struct xdp_buff *xdp) 503 { 504 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 505 struct aq_ring_buff_s *buff_ = buff; 506 507 memset(sinfo, 0, sizeof(*sinfo)); 508 do { 509 skb_frag_t *frag; 510 511 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) 512 return true; 513 514 frag = &sinfo->frags[sinfo->nr_frags++]; 515 buff_ = &ring->buff_ring[buff_->next]; 516 dma_sync_single_range_for_cpu(dev, 517 buff_->rxdata.daddr, 518 buff_->rxdata.pg_off, 519 buff_->len, 520 DMA_FROM_DEVICE); 521 skb_frag_off_set(frag, buff_->rxdata.pg_off); 522 skb_frag_size_set(frag, buff_->len); 523 sinfo->xdp_frags_size += buff_->len; 524 __skb_frag_set_page(frag, buff_->rxdata.page); 525 526 buff_->is_cleaned = 1; 527 528 buff->is_ip_cso &= buff_->is_ip_cso; 529 buff->is_udp_cso &= buff_->is_udp_cso; 530 buff->is_tcp_cso &= buff_->is_tcp_cso; 531 buff->is_cso_err |= buff_->is_cso_err; 532 533 if (page_is_pfmemalloc(buff_->rxdata.page)) 534 xdp_buff_set_frag_pfmemalloc(xdp); 535 536 } while (!buff_->is_eop); 537 538 xdp_buff_set_frags_flag(xdp); 539 540 return false; 541 } 542 543 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi, 544 int *work_done, int budget) 545 { 546 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 547 bool is_rsc_completed = true; 548 int err = 0; 549 550 for (; (self->sw_head != self->hw_head) && budget; 551 self->sw_head = aq_ring_next_dx(self, self->sw_head), 552 --budget, ++(*work_done)) { 553 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 554 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self); 555 struct aq_ring_buff_s *buff_ = NULL; 556 struct sk_buff *skb = NULL; 557 unsigned int next_ = 0U; 558 unsigned int i = 0U; 559 u16 hdr_len; 560 561 if (buff->is_cleaned) 562 continue; 563 564 if (!buff->is_eop) { 565 buff_ = buff; 566 do { 567 if (buff_->next >= self->size) { 568 err = -EIO; 569 goto err_exit; 570 } 571 next_ = buff_->next, 572 buff_ = &self->buff_ring[next_]; 573 is_rsc_completed = 574 aq_ring_dx_in_range(self->sw_head, 575 next_, 576 self->hw_head); 577 578 if (unlikely(!is_rsc_completed)) 579 break; 580 581 buff->is_error |= buff_->is_error; 582 buff->is_cso_err |= buff_->is_cso_err; 583 584 } while (!buff_->is_eop); 585 586 if (!is_rsc_completed) { 587 err = 0; 588 goto err_exit; 589 } 590 if (buff->is_error || 591 (buff->is_lro && buff->is_cso_err)) { 592 buff_ = buff; 593 do { 594 if (buff_->next >= self->size) { 595 err = -EIO; 596 goto err_exit; 597 } 598 next_ = buff_->next, 599 buff_ = &self->buff_ring[next_]; 600 601 buff_->is_cleaned = true; 602 } while (!buff_->is_eop); 603 604 u64_stats_update_begin(&self->stats.rx.syncp); 605 ++self->stats.rx.errors; 606 u64_stats_update_end(&self->stats.rx.syncp); 607 continue; 608 } 609 } 610 611 if (buff->is_error) { 612 u64_stats_update_begin(&self->stats.rx.syncp); 613 ++self->stats.rx.errors; 614 u64_stats_update_end(&self->stats.rx.syncp); 615 continue; 616 } 617 618 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 619 buff->rxdata.daddr, 620 buff->rxdata.pg_off, 621 buff->len, DMA_FROM_DEVICE); 622 623 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); 624 if (unlikely(!skb)) { 625 u64_stats_update_begin(&self->stats.rx.syncp); 626 self->stats.rx.skb_alloc_fails++; 627 u64_stats_update_end(&self->stats.rx.syncp); 628 err = -ENOMEM; 629 goto err_exit; 630 } 631 if (is_ptp_ring) 632 buff->len -= 633 aq_ptp_extract_ts(self->aq_nic, skb, 634 aq_buf_vaddr(&buff->rxdata), 635 buff->len); 636 637 hdr_len = buff->len; 638 if (hdr_len > AQ_CFG_RX_HDR_SIZE) 639 hdr_len = eth_get_headlen(skb->dev, 640 aq_buf_vaddr(&buff->rxdata), 641 AQ_CFG_RX_HDR_SIZE); 642 643 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), 644 ALIGN(hdr_len, sizeof(long))); 645 646 if (buff->len - hdr_len > 0) { 647 skb_add_rx_frag(skb, 0, buff->rxdata.page, 648 buff->rxdata.pg_off + hdr_len, 649 buff->len - hdr_len, 650 self->frame_max); 651 page_ref_inc(buff->rxdata.page); 652 } 653 654 if (!buff->is_eop) { 655 buff_ = buff; 656 i = 1U; 657 do { 658 next_ = buff_->next; 659 buff_ = &self->buff_ring[next_]; 660 661 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 662 buff_->rxdata.daddr, 663 buff_->rxdata.pg_off, 664 buff_->len, 665 DMA_FROM_DEVICE); 666 skb_add_rx_frag(skb, i++, 667 buff_->rxdata.page, 668 buff_->rxdata.pg_off, 669 buff_->len, 670 self->frame_max); 671 page_ref_inc(buff_->rxdata.page); 672 buff_->is_cleaned = 1; 673 674 buff->is_ip_cso &= buff_->is_ip_cso; 675 buff->is_udp_cso &= buff_->is_udp_cso; 676 buff->is_tcp_cso &= buff_->is_tcp_cso; 677 buff->is_cso_err |= buff_->is_cso_err; 678 679 } while (!buff_->is_eop); 680 } 681 682 if (buff->is_vlan) 683 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 684 buff->vlan_rx_tag); 685 686 skb->protocol = eth_type_trans(skb, ndev); 687 688 aq_rx_checksum(self, buff, skb); 689 690 skb_set_hash(skb, buff->rss_hash, 691 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 692 PKT_HASH_TYPE_NONE); 693 /* Send all PTP traffic to 0 queue */ 694 skb_record_rx_queue(skb, 695 is_ptp_ring ? 0 696 : AQ_NIC_RING2QMAP(self->aq_nic, 697 self->idx)); 698 699 u64_stats_update_begin(&self->stats.rx.syncp); 700 ++self->stats.rx.packets; 701 self->stats.rx.bytes += skb->len; 702 u64_stats_update_end(&self->stats.rx.syncp); 703 704 napi_gro_receive(napi, skb); 705 } 706 707 err_exit: 708 return err; 709 } 710 711 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring, 712 struct napi_struct *napi, int *work_done, 713 int budget) 714 { 715 int frame_sz = rx_ring->page_offset + rx_ring->frame_max + 716 rx_ring->tail_size; 717 struct aq_nic_s *aq_nic = rx_ring->aq_nic; 718 bool is_rsc_completed = true; 719 struct device *dev; 720 int err = 0; 721 722 dev = aq_nic_get_dev(aq_nic); 723 for (; (rx_ring->sw_head != rx_ring->hw_head) && budget; 724 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head), 725 --budget, ++(*work_done)) { 726 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; 727 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring); 728 struct aq_ring_buff_s *buff_ = NULL; 729 struct sk_buff *skb = NULL; 730 unsigned int next_ = 0U; 731 struct xdp_buff xdp; 732 void *hard_start; 733 734 if (buff->is_cleaned) 735 continue; 736 737 if (!buff->is_eop) { 738 buff_ = buff; 739 do { 740 if (buff_->next >= rx_ring->size) { 741 err = -EIO; 742 goto err_exit; 743 } 744 next_ = buff_->next; 745 buff_ = &rx_ring->buff_ring[next_]; 746 is_rsc_completed = 747 aq_ring_dx_in_range(rx_ring->sw_head, 748 next_, 749 rx_ring->hw_head); 750 751 if (unlikely(!is_rsc_completed)) 752 break; 753 754 buff->is_error |= buff_->is_error; 755 buff->is_cso_err |= buff_->is_cso_err; 756 } while (!buff_->is_eop); 757 758 if (!is_rsc_completed) { 759 err = 0; 760 goto err_exit; 761 } 762 if (buff->is_error || 763 (buff->is_lro && buff->is_cso_err)) { 764 buff_ = buff; 765 do { 766 if (buff_->next >= rx_ring->size) { 767 err = -EIO; 768 goto err_exit; 769 } 770 next_ = buff_->next; 771 buff_ = &rx_ring->buff_ring[next_]; 772 773 buff_->is_cleaned = true; 774 } while (!buff_->is_eop); 775 776 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 777 ++rx_ring->stats.rx.errors; 778 u64_stats_update_end(&rx_ring->stats.rx.syncp); 779 continue; 780 } 781 } 782 783 if (buff->is_error) { 784 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 785 ++rx_ring->stats.rx.errors; 786 u64_stats_update_end(&rx_ring->stats.rx.syncp); 787 continue; 788 } 789 790 dma_sync_single_range_for_cpu(dev, 791 buff->rxdata.daddr, 792 buff->rxdata.pg_off, 793 buff->len, DMA_FROM_DEVICE); 794 hard_start = page_address(buff->rxdata.page) + 795 buff->rxdata.pg_off - rx_ring->page_offset; 796 797 if (is_ptp_ring) 798 buff->len -= 799 aq_ptp_extract_ts(rx_ring->aq_nic, skb, 800 aq_buf_vaddr(&buff->rxdata), 801 buff->len); 802 803 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); 804 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset, 805 buff->len, false); 806 if (!buff->is_eop) { 807 if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) { 808 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 809 ++rx_ring->stats.rx.packets; 810 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp); 811 ++rx_ring->stats.rx.xdp_aborted; 812 u64_stats_update_end(&rx_ring->stats.rx.syncp); 813 continue; 814 } 815 } 816 817 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff); 818 if (IS_ERR(skb) || !skb) 819 continue; 820 821 if (buff->is_vlan) 822 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 823 buff->vlan_rx_tag); 824 825 aq_rx_checksum(rx_ring, buff, skb); 826 827 skb_set_hash(skb, buff->rss_hash, 828 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 829 PKT_HASH_TYPE_NONE); 830 /* Send all PTP traffic to 0 queue */ 831 skb_record_rx_queue(skb, 832 is_ptp_ring ? 0 833 : AQ_NIC_RING2QMAP(rx_ring->aq_nic, 834 rx_ring->idx)); 835 836 napi_gro_receive(napi, skb); 837 } 838 839 err_exit: 840 return err; 841 } 842 843 int aq_ring_rx_clean(struct aq_ring_s *self, 844 struct napi_struct *napi, 845 int *work_done, 846 int budget) 847 { 848 if (static_branch_unlikely(&aq_xdp_locking_key)) 849 return __aq_ring_xdp_clean(self, napi, work_done, budget); 850 else 851 return __aq_ring_rx_clean(self, napi, work_done, budget); 852 } 853 854 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 855 { 856 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK) 857 while (self->sw_head != self->hw_head) { 858 u64 ns; 859 860 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, 861 self->dx_ring + 862 (self->sw_head * self->dx_size), 863 self->dx_size, &ns); 864 aq_ptp_tx_hwtstamp(aq_nic, ns); 865 866 self->sw_head = aq_ring_next_dx(self, self->sw_head); 867 } 868 #endif 869 } 870 871 int aq_ring_rx_fill(struct aq_ring_s *self) 872 { 873 struct aq_ring_buff_s *buff = NULL; 874 int err = 0; 875 int i = 0; 876 877 if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, 878 self->size / 2)) 879 return err; 880 881 for (i = aq_ring_avail_dx(self); i--; 882 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 883 buff = &self->buff_ring[self->sw_tail]; 884 885 buff->flags = 0U; 886 buff->len = self->frame_max; 887 888 err = aq_get_rxpages(self, buff); 889 if (err) 890 goto err_exit; 891 892 buff->pa = aq_buf_daddr(&buff->rxdata); 893 buff = NULL; 894 } 895 896 err_exit: 897 return err; 898 } 899 900 void aq_ring_rx_deinit(struct aq_ring_s *self) 901 { 902 if (!self) 903 return; 904 905 for (; self->sw_head != self->sw_tail; 906 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 907 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 908 909 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); 910 } 911 } 912 913 void aq_ring_free(struct aq_ring_s *self) 914 { 915 if (!self) 916 return; 917 918 kfree(self->buff_ring); 919 920 if (self->dx_ring) 921 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 922 self->size * self->dx_size, self->dx_ring, 923 self->dx_ring_pa); 924 } 925 926 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) 927 { 928 unsigned int count; 929 unsigned int start; 930 931 if (self->ring_type == ATL_RING_RX) { 932 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ 933 do { 934 count = 0; 935 start = u64_stats_fetch_begin_irq(&self->stats.rx.syncp); 936 data[count] = self->stats.rx.packets; 937 data[++count] = self->stats.rx.jumbo_packets; 938 data[++count] = self->stats.rx.lro_packets; 939 data[++count] = self->stats.rx.errors; 940 data[++count] = self->stats.rx.alloc_fails; 941 data[++count] = self->stats.rx.skb_alloc_fails; 942 data[++count] = self->stats.rx.polls; 943 data[++count] = self->stats.rx.pg_flips; 944 data[++count] = self->stats.rx.pg_reuses; 945 data[++count] = self->stats.rx.pg_losts; 946 data[++count] = self->stats.rx.xdp_aborted; 947 data[++count] = self->stats.rx.xdp_drop; 948 data[++count] = self->stats.rx.xdp_pass; 949 data[++count] = self->stats.rx.xdp_tx; 950 data[++count] = self->stats.rx.xdp_invalid; 951 data[++count] = self->stats.rx.xdp_redirect; 952 } while (u64_stats_fetch_retry_irq(&self->stats.rx.syncp, start)); 953 } else { 954 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ 955 do { 956 count = 0; 957 start = u64_stats_fetch_begin_irq(&self->stats.tx.syncp); 958 data[count] = self->stats.tx.packets; 959 data[++count] = self->stats.tx.queue_restarts; 960 } while (u64_stats_fetch_retry_irq(&self->stats.tx.syncp, start)); 961 } 962 963 return ++count; 964 } 965