Lines Matching +full:rx +full:- +full:tx
1 // SPDX-License-Identifier: GPL-2.0+
14 static struct page *lan966x_fdma_rx_alloc_page(struct lan966x_rx *rx, in lan966x_fdma_rx_alloc_page() argument
19 page = page_pool_dev_alloc_pages(rx->page_pool); in lan966x_fdma_rx_alloc_page()
23 db->dataptr = page_pool_get_dma_addr(page) + XDP_PACKET_HEADROOM; in lan966x_fdma_rx_alloc_page()
28 static void lan966x_fdma_rx_free_pages(struct lan966x_rx *rx) in lan966x_fdma_rx_free_pages() argument
34 page_pool_put_full_page(rx->page_pool, in lan966x_fdma_rx_free_pages()
35 rx->page[i][j], false); in lan966x_fdma_rx_free_pages()
39 static void lan966x_fdma_rx_free_page(struct lan966x_rx *rx) in lan966x_fdma_rx_free_page() argument
43 page = rx->page[rx->dcb_index][rx->db_index]; in lan966x_fdma_rx_free_page()
47 page_pool_recycle_direct(rx->page_pool, page); in lan966x_fdma_rx_free_page()
50 static void lan966x_fdma_rx_add_dcb(struct lan966x_rx *rx, in lan966x_fdma_rx_add_dcb() argument
58 db = &dcb->db[i]; in lan966x_fdma_rx_add_dcb()
59 db->status = FDMA_DCB_STATUS_INTR; in lan966x_fdma_rx_add_dcb()
62 dcb->nextptr = FDMA_DCB_INVALID_DATA; in lan966x_fdma_rx_add_dcb()
63 dcb->info = FDMA_DCB_INFO_DATAL(PAGE_SIZE << rx->page_order); in lan966x_fdma_rx_add_dcb()
65 rx->last_entry->nextptr = nextptr; in lan966x_fdma_rx_add_dcb()
66 rx->last_entry = dcb; in lan966x_fdma_rx_add_dcb()
69 static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx) in lan966x_fdma_rx_alloc_page_pool() argument
71 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_alloc_page_pool()
73 .order = rx->page_order, in lan966x_fdma_rx_alloc_page_pool()
77 .dev = lan966x->dev, in lan966x_fdma_rx_alloc_page_pool()
80 .max_len = rx->max_mtu - in lan966x_fdma_rx_alloc_page_pool()
87 rx->page_pool = page_pool_create(&pp_params); in lan966x_fdma_rx_alloc_page_pool()
89 for (int i = 0; i < lan966x->num_phys_ports; ++i) { in lan966x_fdma_rx_alloc_page_pool()
92 if (!lan966x->ports[i]) in lan966x_fdma_rx_alloc_page_pool()
95 port = lan966x->ports[i]; in lan966x_fdma_rx_alloc_page_pool()
96 xdp_rxq_info_unreg_mem_model(&port->xdp_rxq); in lan966x_fdma_rx_alloc_page_pool()
97 xdp_rxq_info_reg_mem_model(&port->xdp_rxq, MEM_TYPE_PAGE_POOL, in lan966x_fdma_rx_alloc_page_pool()
98 rx->page_pool); in lan966x_fdma_rx_alloc_page_pool()
101 return PTR_ERR_OR_ZERO(rx->page_pool); in lan966x_fdma_rx_alloc_page_pool()
104 static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) in lan966x_fdma_rx_alloc() argument
106 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_alloc()
113 if (lan966x_fdma_rx_alloc_page_pool(rx)) in lan966x_fdma_rx_alloc()
114 return PTR_ERR(rx->page_pool); in lan966x_fdma_rx_alloc()
120 rx->dcbs = dma_alloc_coherent(lan966x->dev, size, &rx->dma, GFP_KERNEL); in lan966x_fdma_rx_alloc()
121 if (!rx->dcbs) in lan966x_fdma_rx_alloc()
122 return -ENOMEM; in lan966x_fdma_rx_alloc()
124 rx->last_entry = rx->dcbs; in lan966x_fdma_rx_alloc()
125 rx->db_index = 0; in lan966x_fdma_rx_alloc()
126 rx->dcb_index = 0; in lan966x_fdma_rx_alloc()
130 dcb = &rx->dcbs[i]; in lan966x_fdma_rx_alloc()
131 dcb->info = 0; in lan966x_fdma_rx_alloc()
135 db = &dcb->db[j]; in lan966x_fdma_rx_alloc()
136 page = lan966x_fdma_rx_alloc_page(rx, db); in lan966x_fdma_rx_alloc()
138 return -ENOMEM; in lan966x_fdma_rx_alloc()
140 db->status = 0; in lan966x_fdma_rx_alloc()
141 rx->page[i][j] = page; in lan966x_fdma_rx_alloc()
144 lan966x_fdma_rx_add_dcb(rx, dcb, rx->dma + sizeof(*dcb) * i); in lan966x_fdma_rx_alloc()
150 static void lan966x_fdma_rx_advance_dcb(struct lan966x_rx *rx) in lan966x_fdma_rx_advance_dcb() argument
152 rx->dcb_index++; in lan966x_fdma_rx_advance_dcb()
153 rx->dcb_index &= FDMA_DCB_MAX - 1; in lan966x_fdma_rx_advance_dcb()
156 static void lan966x_fdma_rx_free(struct lan966x_rx *rx) in lan966x_fdma_rx_free() argument
158 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_free()
164 dma_free_coherent(lan966x->dev, size, rx->dcbs, rx->dma); in lan966x_fdma_rx_free()
167 static void lan966x_fdma_rx_start(struct lan966x_rx *rx) in lan966x_fdma_rx_start() argument
169 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_start()
175 lan_wr(lower_32_bits((u64)rx->dma), lan966x, in lan966x_fdma_rx_start()
176 FDMA_DCB_LLP(rx->channel_id)); in lan966x_fdma_rx_start()
177 lan_wr(upper_32_bits((u64)rx->dma), lan966x, in lan966x_fdma_rx_start()
178 FDMA_DCB_LLP1(rx->channel_id)); in lan966x_fdma_rx_start()
184 lan966x, FDMA_CH_CFG(rx->channel_id)); in lan966x_fdma_rx_start()
194 mask |= BIT(rx->channel_id); in lan966x_fdma_rx_start()
200 lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(rx->channel_id)), in lan966x_fdma_rx_start()
205 static void lan966x_fdma_rx_disable(struct lan966x_rx *rx) in lan966x_fdma_rx_disable() argument
207 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_disable()
211 lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(rx->channel_id)), in lan966x_fdma_rx_disable()
216 val, !(val & BIT(rx->channel_id)), in lan966x_fdma_rx_disable()
219 lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(rx->channel_id)), in lan966x_fdma_rx_disable()
224 static void lan966x_fdma_rx_reload(struct lan966x_rx *rx) in lan966x_fdma_rx_reload() argument
226 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_reload()
228 lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(rx->channel_id)), in lan966x_fdma_rx_reload()
233 static void lan966x_fdma_tx_add_dcb(struct lan966x_tx *tx, in lan966x_fdma_tx_add_dcb() argument
236 dcb->nextptr = FDMA_DCB_INVALID_DATA; in lan966x_fdma_tx_add_dcb()
237 dcb->info = 0; in lan966x_fdma_tx_add_dcb()
240 static int lan966x_fdma_tx_alloc(struct lan966x_tx *tx) in lan966x_fdma_tx_alloc() argument
242 struct lan966x *lan966x = tx->lan966x; in lan966x_fdma_tx_alloc()
248 tx->dcbs_buf = kcalloc(FDMA_DCB_MAX, sizeof(struct lan966x_tx_dcb_buf), in lan966x_fdma_tx_alloc()
250 if (!tx->dcbs_buf) in lan966x_fdma_tx_alloc()
251 return -ENOMEM; in lan966x_fdma_tx_alloc()
256 tx->dcbs = dma_alloc_coherent(lan966x->dev, size, &tx->dma, GFP_KERNEL); in lan966x_fdma_tx_alloc()
257 if (!tx->dcbs) in lan966x_fdma_tx_alloc()
262 dcb = &tx->dcbs[i]; in lan966x_fdma_tx_alloc()
265 db = &dcb->db[j]; in lan966x_fdma_tx_alloc()
266 db->dataptr = 0; in lan966x_fdma_tx_alloc()
267 db->status = 0; in lan966x_fdma_tx_alloc()
270 lan966x_fdma_tx_add_dcb(tx, dcb); in lan966x_fdma_tx_alloc()
276 kfree(tx->dcbs_buf); in lan966x_fdma_tx_alloc()
277 return -ENOMEM; in lan966x_fdma_tx_alloc()
280 static void lan966x_fdma_tx_free(struct lan966x_tx *tx) in lan966x_fdma_tx_free() argument
282 struct lan966x *lan966x = tx->lan966x; in lan966x_fdma_tx_free()
285 kfree(tx->dcbs_buf); in lan966x_fdma_tx_free()
289 dma_free_coherent(lan966x->dev, size, tx->dcbs, tx->dma); in lan966x_fdma_tx_free()
292 static void lan966x_fdma_tx_activate(struct lan966x_tx *tx) in lan966x_fdma_tx_activate() argument
294 struct lan966x *lan966x = tx->lan966x; in lan966x_fdma_tx_activate()
300 lan_wr(lower_32_bits((u64)tx->dma), lan966x, in lan966x_fdma_tx_activate()
301 FDMA_DCB_LLP(tx->channel_id)); in lan966x_fdma_tx_activate()
302 lan_wr(upper_32_bits((u64)tx->dma), lan966x, in lan966x_fdma_tx_activate()
303 FDMA_DCB_LLP1(tx->channel_id)); in lan966x_fdma_tx_activate()
309 lan966x, FDMA_CH_CFG(tx->channel_id)); in lan966x_fdma_tx_activate()
319 mask |= BIT(tx->channel_id); in lan966x_fdma_tx_activate()
325 lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(tx->channel_id)), in lan966x_fdma_tx_activate()
330 static void lan966x_fdma_tx_disable(struct lan966x_tx *tx) in lan966x_fdma_tx_disable() argument
332 struct lan966x *lan966x = tx->lan966x; in lan966x_fdma_tx_disable()
336 lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(tx->channel_id)), in lan966x_fdma_tx_disable()
341 val, !(val & BIT(tx->channel_id)), in lan966x_fdma_tx_disable()
344 lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(tx->channel_id)), in lan966x_fdma_tx_disable()
348 tx->activated = false; in lan966x_fdma_tx_disable()
349 tx->last_in_use = -1; in lan966x_fdma_tx_disable()
352 static void lan966x_fdma_tx_reload(struct lan966x_tx *tx) in lan966x_fdma_tx_reload() argument
354 struct lan966x *lan966x = tx->lan966x; in lan966x_fdma_tx_reload()
357 lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(tx->channel_id)), in lan966x_fdma_tx_reload()
367 for (i = 0; i < lan966x->num_phys_ports; ++i) { in lan966x_fdma_wakeup_netdev()
368 port = lan966x->ports[i]; in lan966x_fdma_wakeup_netdev()
372 if (netif_queue_stopped(port->dev)) in lan966x_fdma_wakeup_netdev()
373 netif_wake_queue(port->dev); in lan966x_fdma_wakeup_netdev()
382 for (i = 0; i < lan966x->num_phys_ports; ++i) { in lan966x_fdma_stop_netdev()
383 port = lan966x->ports[i]; in lan966x_fdma_stop_netdev()
387 netif_stop_queue(port->dev); in lan966x_fdma_stop_netdev()
393 struct lan966x_tx *tx = &lan966x->tx; in lan966x_fdma_tx_clear_buf() local
394 struct lan966x_rx *rx = &lan966x->rx; in lan966x_fdma_tx_clear_buf() local
404 spin_lock_irqsave(&lan966x->tx_lock, flags); in lan966x_fdma_tx_clear_buf()
406 dcb_buf = &tx->dcbs_buf[i]; in lan966x_fdma_tx_clear_buf()
408 if (!dcb_buf->used) in lan966x_fdma_tx_clear_buf()
411 db = &tx->dcbs[i].db[0]; in lan966x_fdma_tx_clear_buf()
412 if (!(db->status & FDMA_DCB_STATUS_DONE)) in lan966x_fdma_tx_clear_buf()
415 dcb_buf->dev->stats.tx_packets++; in lan966x_fdma_tx_clear_buf()
416 dcb_buf->dev->stats.tx_bytes += dcb_buf->len; in lan966x_fdma_tx_clear_buf()
418 dcb_buf->used = false; in lan966x_fdma_tx_clear_buf()
419 if (dcb_buf->use_skb) { in lan966x_fdma_tx_clear_buf()
420 dma_unmap_single(lan966x->dev, in lan966x_fdma_tx_clear_buf()
421 dcb_buf->dma_addr, in lan966x_fdma_tx_clear_buf()
422 dcb_buf->len, in lan966x_fdma_tx_clear_buf()
425 if (!dcb_buf->ptp) in lan966x_fdma_tx_clear_buf()
426 napi_consume_skb(dcb_buf->data.skb, weight); in lan966x_fdma_tx_clear_buf()
428 if (dcb_buf->xdp_ndo) in lan966x_fdma_tx_clear_buf()
429 dma_unmap_single(lan966x->dev, in lan966x_fdma_tx_clear_buf()
430 dcb_buf->dma_addr, in lan966x_fdma_tx_clear_buf()
431 dcb_buf->len, in lan966x_fdma_tx_clear_buf()
434 if (dcb_buf->xdp_ndo) in lan966x_fdma_tx_clear_buf()
435 xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq); in lan966x_fdma_tx_clear_buf()
437 page_pool_recycle_direct(rx->page_pool, in lan966x_fdma_tx_clear_buf()
438 dcb_buf->data.page); in lan966x_fdma_tx_clear_buf()
449 spin_unlock_irqrestore(&lan966x->tx_lock, flags); in lan966x_fdma_tx_clear_buf()
452 static bool lan966x_fdma_rx_more_frames(struct lan966x_rx *rx) in lan966x_fdma_rx_more_frames() argument
457 db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; in lan966x_fdma_rx_more_frames()
458 if (unlikely(!(db->status & FDMA_DCB_STATUS_DONE))) in lan966x_fdma_rx_more_frames()
464 static int lan966x_fdma_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) in lan966x_fdma_rx_check_frame() argument
466 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_check_frame()
471 db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; in lan966x_fdma_rx_check_frame()
472 page = rx->page[rx->dcb_index][rx->db_index]; in lan966x_fdma_rx_check_frame()
476 dma_sync_single_for_cpu(lan966x->dev, in lan966x_fdma_rx_check_frame()
477 (dma_addr_t)db->dataptr + XDP_PACKET_HEADROOM, in lan966x_fdma_rx_check_frame()
478 FDMA_DCB_STATUS_BLOCKL(db->status), in lan966x_fdma_rx_check_frame()
483 if (WARN_ON(*src_port >= lan966x->num_phys_ports)) in lan966x_fdma_rx_check_frame()
486 port = lan966x->ports[*src_port]; in lan966x_fdma_rx_check_frame()
490 return lan966x_xdp_run(port, page, FDMA_DCB_STATUS_BLOCKL(db->status)); in lan966x_fdma_rx_check_frame()
493 static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx, in lan966x_fdma_rx_get_frame() argument
496 struct lan966x *lan966x = rx->lan966x; in lan966x_fdma_rx_get_frame()
503 db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; in lan966x_fdma_rx_get_frame()
504 page = rx->page[rx->dcb_index][rx->db_index]; in lan966x_fdma_rx_get_frame()
506 skb = build_skb(page_address(page), PAGE_SIZE << rx->page_order); in lan966x_fdma_rx_get_frame()
513 skb_put(skb, FDMA_DCB_STATUS_BLOCKL(db->status)); in lan966x_fdma_rx_get_frame()
515 lan966x_ifh_get_timestamp(skb->data, ×tamp); in lan966x_fdma_rx_get_frame()
517 skb->dev = lan966x->ports[src_port]->dev; in lan966x_fdma_rx_get_frame()
520 if (likely(!(skb->dev->features & NETIF_F_RXFCS))) in lan966x_fdma_rx_get_frame()
521 skb_trim(skb, skb->len - ETH_FCS_LEN); in lan966x_fdma_rx_get_frame()
524 skb->protocol = eth_type_trans(skb, skb->dev); in lan966x_fdma_rx_get_frame()
526 if (lan966x->bridge_mask & BIT(src_port)) { in lan966x_fdma_rx_get_frame()
527 skb->offload_fwd_mark = 1; in lan966x_fdma_rx_get_frame()
531 skb->offload_fwd_mark = 0; in lan966x_fdma_rx_get_frame()
534 skb->dev->stats.rx_bytes += skb->len; in lan966x_fdma_rx_get_frame()
535 skb->dev->stats.rx_packets++; in lan966x_fdma_rx_get_frame()
540 page_pool_recycle_direct(rx->page_pool, page); in lan966x_fdma_rx_get_frame()
548 struct lan966x_rx *rx = &lan966x->rx; in lan966x_fdma_napi_poll() local
549 int dcb_reload = rx->dcb_index; in lan966x_fdma_napi_poll()
563 if (!lan966x_fdma_rx_more_frames(rx)) in lan966x_fdma_napi_poll()
568 switch (lan966x_fdma_rx_check_frame(rx, &src_port)) { in lan966x_fdma_napi_poll()
572 lan966x_fdma_rx_free_page(rx); in lan966x_fdma_napi_poll()
573 lan966x_fdma_rx_advance_dcb(rx); in lan966x_fdma_napi_poll()
579 lan966x_fdma_rx_advance_dcb(rx); in lan966x_fdma_napi_poll()
582 lan966x_fdma_rx_free_page(rx); in lan966x_fdma_napi_poll()
583 lan966x_fdma_rx_advance_dcb(rx); in lan966x_fdma_napi_poll()
587 skb = lan966x_fdma_rx_get_frame(rx, src_port); in lan966x_fdma_napi_poll()
588 lan966x_fdma_rx_advance_dcb(rx); in lan966x_fdma_napi_poll()
592 napi_gro_receive(&lan966x->napi, skb); in lan966x_fdma_napi_poll()
597 while (dcb_reload != rx->dcb_index) { in lan966x_fdma_napi_poll()
598 db = &rx->dcbs[dcb_reload].db[rx->db_index]; in lan966x_fdma_napi_poll()
599 page = lan966x_fdma_rx_alloc_page(rx, db); in lan966x_fdma_napi_poll()
602 rx->page[dcb_reload][rx->db_index] = page; in lan966x_fdma_napi_poll()
604 old_dcb = &rx->dcbs[dcb_reload]; in lan966x_fdma_napi_poll()
606 dcb_reload &= FDMA_DCB_MAX - 1; in lan966x_fdma_napi_poll()
608 nextptr = rx->dma + ((unsigned long)old_dcb - in lan966x_fdma_napi_poll()
609 (unsigned long)rx->dcbs); in lan966x_fdma_napi_poll()
610 lan966x_fdma_rx_add_dcb(rx, old_dcb, nextptr); in lan966x_fdma_napi_poll()
611 lan966x_fdma_rx_reload(rx); in lan966x_fdma_napi_poll()
635 napi_schedule(&lan966x->napi); in lan966x_fdma_irq_handler()
650 static int lan966x_fdma_get_next_dcb(struct lan966x_tx *tx) in lan966x_fdma_get_next_dcb() argument
656 dcb_buf = &tx->dcbs_buf[i]; in lan966x_fdma_get_next_dcb()
657 if (!dcb_buf->used && i != tx->last_in_use) in lan966x_fdma_get_next_dcb()
661 return -1; in lan966x_fdma_get_next_dcb()
664 static void lan966x_fdma_tx_setup_dcb(struct lan966x_tx *tx, in lan966x_fdma_tx_setup_dcb() argument
671 next_dcb = &tx->dcbs[next_to_use]; in lan966x_fdma_tx_setup_dcb()
672 next_dcb->nextptr = FDMA_DCB_INVALID_DATA; in lan966x_fdma_tx_setup_dcb()
674 next_db = &next_dcb->db[0]; in lan966x_fdma_tx_setup_dcb()
675 next_db->dataptr = dma_addr; in lan966x_fdma_tx_setup_dcb()
676 next_db->status = FDMA_DCB_STATUS_SOF | in lan966x_fdma_tx_setup_dcb()
683 static void lan966x_fdma_tx_start(struct lan966x_tx *tx, int next_to_use) in lan966x_fdma_tx_start() argument
685 struct lan966x *lan966x = tx->lan966x; in lan966x_fdma_tx_start()
688 if (likely(lan966x->tx.activated)) { in lan966x_fdma_tx_start()
690 dcb = &tx->dcbs[tx->last_in_use]; in lan966x_fdma_tx_start()
691 dcb->nextptr = tx->dma + (next_to_use * in lan966x_fdma_tx_start()
694 lan966x_fdma_tx_reload(tx); in lan966x_fdma_tx_start()
697 lan966x->tx.activated = true; in lan966x_fdma_tx_start()
698 lan966x_fdma_tx_activate(tx); in lan966x_fdma_tx_start()
702 tx->last_in_use = next_to_use; in lan966x_fdma_tx_start()
707 struct lan966x *lan966x = port->lan966x; in lan966x_fdma_xmit_xdpf()
709 struct lan966x_tx *tx = &lan966x->tx; in lan966x_fdma_xmit_xdpf() local
717 spin_lock(&lan966x->tx_lock); in lan966x_fdma_xmit_xdpf()
720 next_to_use = lan966x_fdma_get_next_dcb(tx); in lan966x_fdma_xmit_xdpf()
722 netif_stop_queue(port->dev); in lan966x_fdma_xmit_xdpf()
728 next_dcb_buf = &tx->dcbs_buf[next_to_use]; in lan966x_fdma_xmit_xdpf()
734 if (xdpf->headroom < IFH_LEN_BYTES) { in lan966x_fdma_xmit_xdpf()
739 ifh = xdpf->data - IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf()
742 lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); in lan966x_fdma_xmit_xdpf()
744 dma_addr = dma_map_single(lan966x->dev, in lan966x_fdma_xmit_xdpf()
745 xdpf->data - IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
746 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
748 if (dma_mapping_error(lan966x->dev, dma_addr)) { in lan966x_fdma_xmit_xdpf()
753 next_dcb_buf->data.xdpf = xdpf; in lan966x_fdma_xmit_xdpf()
754 next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf()
757 lan966x_fdma_tx_setup_dcb(tx, next_to_use, in lan966x_fdma_xmit_xdpf()
758 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
766 lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); in lan966x_fdma_xmit_xdpf()
769 dma_sync_single_for_device(lan966x->dev, in lan966x_fdma_xmit_xdpf()
774 next_dcb_buf->data.page = page; in lan966x_fdma_xmit_xdpf()
775 next_dcb_buf->len = len + IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf()
778 lan966x_fdma_tx_setup_dcb(tx, next_to_use, in lan966x_fdma_xmit_xdpf()
784 next_dcb_buf->use_skb = false; in lan966x_fdma_xmit_xdpf()
785 next_dcb_buf->xdp_ndo = !len; in lan966x_fdma_xmit_xdpf()
786 next_dcb_buf->dma_addr = dma_addr; in lan966x_fdma_xmit_xdpf()
787 next_dcb_buf->used = true; in lan966x_fdma_xmit_xdpf()
788 next_dcb_buf->ptp = false; in lan966x_fdma_xmit_xdpf()
789 next_dcb_buf->dev = port->dev; in lan966x_fdma_xmit_xdpf()
792 lan966x_fdma_tx_start(tx, next_to_use); in lan966x_fdma_xmit_xdpf()
795 spin_unlock(&lan966x->tx_lock); in lan966x_fdma_xmit_xdpf()
803 struct lan966x *lan966x = port->lan966x; in lan966x_fdma_xmit()
805 struct lan966x_tx *tx = &lan966x->tx; in lan966x_fdma_xmit() local
813 next_to_use = lan966x_fdma_get_next_dcb(tx); in lan966x_fdma_xmit()
820 dev->stats.tx_dropped++; in lan966x_fdma_xmit()
825 needed_headroom = max_t(int, IFH_LEN_BYTES - skb_headroom(skb), 0); in lan966x_fdma_xmit()
826 needed_tailroom = max_t(int, ETH_FCS_LEN - skb_tailroom(skb), 0); in lan966x_fdma_xmit()
831 dev->stats.tx_dropped++; in lan966x_fdma_xmit()
839 memcpy(skb->data, ifh, IFH_LEN_BYTES); in lan966x_fdma_xmit()
842 dma_addr = dma_map_single(lan966x->dev, skb->data, skb->len, in lan966x_fdma_xmit()
844 if (dma_mapping_error(lan966x->dev, dma_addr)) { in lan966x_fdma_xmit()
845 dev->stats.tx_dropped++; in lan966x_fdma_xmit()
851 lan966x_fdma_tx_setup_dcb(tx, next_to_use, skb->len, dma_addr); in lan966x_fdma_xmit()
854 next_dcb_buf = &tx->dcbs_buf[next_to_use]; in lan966x_fdma_xmit()
855 next_dcb_buf->use_skb = true; in lan966x_fdma_xmit()
856 next_dcb_buf->data.skb = skb; in lan966x_fdma_xmit()
857 next_dcb_buf->xdp_ndo = false; in lan966x_fdma_xmit()
858 next_dcb_buf->len = skb->len; in lan966x_fdma_xmit()
859 next_dcb_buf->dma_addr = dma_addr; in lan966x_fdma_xmit()
860 next_dcb_buf->used = true; in lan966x_fdma_xmit()
861 next_dcb_buf->ptp = false; in lan966x_fdma_xmit()
862 next_dcb_buf->dev = dev; in lan966x_fdma_xmit()
864 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && in lan966x_fdma_xmit()
865 LAN966X_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) in lan966x_fdma_xmit()
866 next_dcb_buf->ptp = true; in lan966x_fdma_xmit()
869 lan966x_fdma_tx_start(tx, next_to_use); in lan966x_fdma_xmit()
874 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && in lan966x_fdma_xmit()
875 LAN966X_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) in lan966x_fdma_xmit()
887 for (i = 0; i < lan966x->num_phys_ports; ++i) { in lan966x_fdma_get_max_mtu()
891 port = lan966x->ports[i]; in lan966x_fdma_get_max_mtu()
895 mtu = lan_rd(lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); in lan966x_fdma_get_max_mtu()
917 rx_dma = lan966x->rx.dma; in lan966x_fdma_reload()
918 rx_dcbs = lan966x->rx.dcbs; in lan966x_fdma_reload()
919 page_pool = lan966x->rx.page_pool; in lan966x_fdma_reload()
921 napi_synchronize(&lan966x->napi); in lan966x_fdma_reload()
922 napi_disable(&lan966x->napi); in lan966x_fdma_reload()
925 lan966x_fdma_rx_disable(&lan966x->rx); in lan966x_fdma_reload()
926 lan966x_fdma_rx_free_pages(&lan966x->rx); in lan966x_fdma_reload()
927 lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1; in lan966x_fdma_reload()
928 lan966x->rx.max_mtu = new_mtu; in lan966x_fdma_reload()
929 err = lan966x_fdma_rx_alloc(&lan966x->rx); in lan966x_fdma_reload()
932 lan966x_fdma_rx_start(&lan966x->rx); in lan966x_fdma_reload()
936 dma_free_coherent(lan966x->dev, size, rx_dcbs, rx_dma); in lan966x_fdma_reload()
941 napi_enable(&lan966x->napi); in lan966x_fdma_reload()
945 lan966x->rx.page_pool = page_pool; in lan966x_fdma_reload()
946 lan966x->rx.dma = rx_dma; in lan966x_fdma_reload()
947 lan966x->rx.dcbs = rx_dcbs; in lan966x_fdma_reload()
948 lan966x_fdma_rx_start(&lan966x->rx); in lan966x_fdma_reload()
997 if (max_mtu == lan966x->rx.max_mtu) in lan966x_fdma_change_mtu()
1013 if (lan966x->fdma_ndev) in lan966x_fdma_netdev_init()
1016 lan966x->fdma_ndev = dev; in lan966x_fdma_netdev_init()
1017 netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll); in lan966x_fdma_netdev_init()
1018 napi_enable(&lan966x->napi); in lan966x_fdma_netdev_init()
1023 if (lan966x->fdma_ndev == dev) { in lan966x_fdma_netdev_deinit()
1024 netif_napi_del(&lan966x->napi); in lan966x_fdma_netdev_deinit()
1025 lan966x->fdma_ndev = NULL; in lan966x_fdma_netdev_deinit()
1033 if (!lan966x->fdma) in lan966x_fdma_init()
1036 lan966x->rx.lan966x = lan966x; in lan966x_fdma_init()
1037 lan966x->rx.channel_id = FDMA_XTR_CHANNEL; in lan966x_fdma_init()
1038 lan966x->rx.max_mtu = lan966x_fdma_get_max_frame(lan966x); in lan966x_fdma_init()
1039 lan966x->tx.lan966x = lan966x; in lan966x_fdma_init()
1040 lan966x->tx.channel_id = FDMA_INJ_CHANNEL; in lan966x_fdma_init()
1041 lan966x->tx.last_in_use = -1; in lan966x_fdma_init()
1043 err = lan966x_fdma_rx_alloc(&lan966x->rx); in lan966x_fdma_init()
1047 err = lan966x_fdma_tx_alloc(&lan966x->tx); in lan966x_fdma_init()
1049 lan966x_fdma_rx_free(&lan966x->rx); in lan966x_fdma_init()
1053 lan966x_fdma_rx_start(&lan966x->rx); in lan966x_fdma_init()
1060 if (!lan966x->fdma) in lan966x_fdma_deinit()
1063 lan966x_fdma_rx_disable(&lan966x->rx); in lan966x_fdma_deinit()
1064 lan966x_fdma_tx_disable(&lan966x->tx); in lan966x_fdma_deinit()
1066 napi_synchronize(&lan966x->napi); in lan966x_fdma_deinit()
1067 napi_disable(&lan966x->napi); in lan966x_fdma_deinit()
1069 lan966x_fdma_rx_free_pages(&lan966x->rx); in lan966x_fdma_deinit()
1070 lan966x_fdma_rx_free(&lan966x->rx); in lan966x_fdma_deinit()
1071 page_pool_destroy(lan966x->rx.page_pool); in lan966x_fdma_deinit()
1072 lan966x_fdma_tx_free(&lan966x->tx); in lan966x_fdma_deinit()