56 static void *txq_end(const struct funeth_txq *q)  in txq_end()  argument
58 	return (void *)q->hw_wb;  in txq_end()
64 static unsigned int txq_to_end(const struct funeth_txq *q, void *p)  in txq_to_end()  argument
66 	return txq_end(q) - p;  in txq_to_end()
78 static struct fun_dataop_gl *fun_write_gl(const struct funeth_txq *q,  in fun_write_gl()  argument
90 	     i < ngle && txq_to_end(q, gle); i++, gle++)  in fun_write_gl()
93 	if (txq_to_end(q, gle) == 0) {  in fun_write_gl()
94 		gle = (struct fun_dataop_gl *)q->desc;  in fun_write_gl()
107 static struct sk_buff *fun_tls_tx(struct sk_buff *skb, struct funeth_txq *q,  in fun_tls_tx()  argument
132 	FUN_QSTAT_INC(q, tx_tls_fallback);  in fun_tls_tx()
135 		FUN_QSTAT_INC(q, tx_tls_drops);  in fun_tls_tx()
149 static unsigned int write_pkt_desc(struct sk_buff *skb, struct funeth_txq *q,  in write_pkt_desc()  argument
153 	unsigned int idx = q->prod_cnt & q->mask;  in write_pkt_desc()
165 	if (unlikely(fun_map_pkt(q->dma_dev, shinfo, skb->data,  in write_pkt_desc()
167 		FUN_QSTAT_INC(q, tx_map_err);  in write_pkt_desc()
171 	req = fun_tx_desc_addr(q, idx);  in write_pkt_desc()
218 			FUN_QSTAT_INC(q, tx_encap_tso);  in write_pkt_desc()
236 			FUN_QSTAT_INC(q, tx_uso);  in write_pkt_desc()
253 			FUN_QSTAT_INC(q, tx_tso);  in write_pkt_desc()
256 		u64_stats_update_begin(&q->syncp);  in write_pkt_desc()
257 		q->stats.tx_cso += shinfo->gso_segs;  in write_pkt_desc()
258 		u64_stats_update_end(&q->syncp);  in write_pkt_desc()
269 		FUN_QSTAT_INC(q, tx_cso);  in write_pkt_desc()
277 	gle = fun_write_gl(q, req, addrs, lens, ngle);  in write_pkt_desc()
290 		u64_stats_update_begin(&q->syncp);  in write_pkt_desc()
291 		q->stats.tx_tls_bytes += tls_len;  in write_pkt_desc()
292 		q->stats.tx_tls_pkts += 1 + extra_pkts;  in write_pkt_desc()
293 		u64_stats_update_end(&q->syncp);  in write_pkt_desc()
296 	u64_stats_update_begin(&q->syncp);  in write_pkt_desc()
297 	q->stats.tx_bytes += skb->len + extra_bytes;  in write_pkt_desc()
298 	q->stats.tx_pkts += 1 + extra_pkts;  in write_pkt_desc()
299 	u64_stats_update_end(&q->syncp);  in write_pkt_desc()
301 	q->info[idx].skb = skb;  in write_pkt_desc()
303 	trace_funeth_tx(q, skb->len, idx, req->dataop.ngather);  in write_pkt_desc()
311 static unsigned int fun_txq_avail(const struct funeth_txq *q)  in fun_txq_avail()  argument
313 	return q->mask - q->prod_cnt + q->cons_cnt;  in fun_txq_avail()
317 static void fun_tx_check_stop(struct funeth_txq *q)  in fun_tx_check_stop()  argument
319 	if (likely(fun_txq_avail(q) >= FUNETH_MAX_PKT_DESC))  in fun_tx_check_stop()
322 	netif_tx_stop_queue(q->ndq);  in fun_tx_check_stop()
329 	if (likely(fun_txq_avail(q) < FUNETH_MAX_PKT_DESC))  in fun_tx_check_stop()
330 		FUN_QSTAT_INC(q, tx_nstops);  in fun_tx_check_stop()
332 		netif_tx_start_queue(q->ndq);  in fun_tx_check_stop()
338 static bool fun_txq_may_restart(struct funeth_txq *q)  in fun_txq_may_restart()  argument
340 	return fun_txq_avail(q) >= q->mask / 4;  in fun_txq_may_restart()
347 	struct funeth_txq *q = fp->txqs[qid];  in fun_start_xmit()  local
352 		skb = fun_tls_tx(skb, q, &tls_len);  in fun_start_xmit()
357 	ndesc = write_pkt_desc(skb, q, tls_len);  in fun_start_xmit()
363 	q->prod_cnt += ndesc;  in fun_start_xmit()
364 	fun_tx_check_stop(q);  in fun_start_xmit()
368 	if (__netdev_tx_sent_queue(q->ndq, skb->len, netdev_xmit_more()))  in fun_start_xmit()
369 		fun_txq_wr_db(q);  in fun_start_xmit()
371 		FUN_QSTAT_INC(q, tx_more);  in fun_start_xmit()
380 		fun_txq_wr_db(q);  in fun_start_xmit()
385 static u16 txq_hw_head(const struct funeth_txq *q)  in txq_hw_head()  argument
387 	return (u16)be64_to_cpu(*q->hw_wb);  in txq_hw_head()
393 static unsigned int fun_unmap_pkt(const struct funeth_txq *q, unsigned int idx)  in fun_unmap_pkt()  argument
395 	const struct fun_eth_tx_req *req = fun_tx_desc_addr(q, idx);  in fun_unmap_pkt()
401 		dma_unmap_single(q->dma_dev, be64_to_cpu(gle->sgl_data),  in fun_unmap_pkt()
404 		for (gle++; --ngle && txq_to_end(q, gle); gle++)  in fun_unmap_pkt()
405 			dma_unmap_page(q->dma_dev, be64_to_cpu(gle->sgl_data),  in fun_unmap_pkt()
409 		for (gle = (struct fun_dataop_gl *)q->desc; ngle; ngle--, gle++)  in fun_unmap_pkt()
410 			dma_unmap_page(q->dma_dev, be64_to_cpu(gle->sgl_data),  in fun_unmap_pkt()
423 static bool fun_txq_reclaim(struct funeth_txq *q, int budget)  in fun_txq_reclaim()  argument
431 	for (head = txq_hw_head(q), reclaim_idx = q->cons_cnt & q->mask;  in fun_txq_reclaim()
432 	     head != reclaim_idx && npkts < limit; head = txq_hw_head(q)) {  in fun_txq_reclaim()
441 			unsigned int pkt_desc = fun_unmap_pkt(q, reclaim_idx);  in fun_txq_reclaim()
442 			struct sk_buff *skb = q->info[reclaim_idx].skb;  in fun_txq_reclaim()
444 			trace_funeth_tx_free(q, reclaim_idx, pkt_desc, head);  in fun_txq_reclaim()
449 			reclaim_idx = (reclaim_idx + pkt_desc) & q->mask;  in fun_txq_reclaim()
454 	q->cons_cnt += ndesc;  in fun_txq_reclaim()
455 	netdev_tx_completed_queue(q->ndq, npkts, nbytes);  in fun_txq_reclaim()
458 	if (unlikely(netif_tx_queue_stopped(q->ndq) &&  in fun_txq_reclaim()
459 		     fun_txq_may_restart(q))) {  in fun_txq_reclaim()
460 		netif_tx_wake_queue(q->ndq);  in fun_txq_reclaim()
461 		FUN_QSTAT_INC(q, tx_nrestarts);  in fun_txq_reclaim()
471 	struct funeth_txq *q = irq->txq;  in fun_txq_napi_poll()  local
474 	if (fun_txq_reclaim(q, budget))  in fun_txq_napi_poll()
478 	db_val = READ_ONCE(q->irq_db_val) | (q->cons_cnt & q->mask);  in fun_txq_napi_poll()
479 	writel(db_val, q->db);  in fun_txq_napi_poll()
484 static unsigned int fun_xdpq_clean(struct funeth_txq *q, unsigned int budget)  in fun_xdpq_clean()  argument
488 	for (head = txq_hw_head(q), reclaim_idx = q->cons_cnt & q->mask;  in fun_xdpq_clean()
489 	     head != reclaim_idx && npkts < budget; head = txq_hw_head(q)) {  in fun_xdpq_clean()
498 			unsigned int pkt_desc = fun_unmap_pkt(q, reclaim_idx);  in fun_xdpq_clean()
500 			xdp_return_frame(q->info[reclaim_idx].xdpf);  in fun_xdpq_clean()
502 			trace_funeth_tx_free(q, reclaim_idx, pkt_desc, head);  in fun_xdpq_clean()
504 			reclaim_idx = (reclaim_idx + pkt_desc) & q->mask;  in fun_xdpq_clean()
510 	q->cons_cnt += ndesc;  in fun_xdpq_clean()
514 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf)  in fun_xdp_tx()  argument
522 	if (fun_txq_avail(q) < FUN_XDP_CLEAN_THRES)  in fun_xdp_tx()
523 		fun_xdpq_clean(q, FUN_XDP_CLEAN_BATCH);  in fun_xdp_tx()
534 	if (unlikely(fun_txq_avail(q) < ndesc)) {  in fun_xdp_tx()
535 		FUN_QSTAT_INC(q, tx_xdp_full);  in fun_xdp_tx()
539 	if (unlikely(fun_map_pkt(q->dma_dev, si, xdpf->data, xdpf->len, dma,  in fun_xdp_tx()
541 		FUN_QSTAT_INC(q, tx_map_err);  in fun_xdp_tx()
545 	idx = q->prod_cnt & q->mask;  in fun_xdp_tx()
546 	req = fun_tx_desc_addr(q, idx);  in fun_xdp_tx()
556 	fun_write_gl(q, req, dma, lens, nfrags);  in fun_xdp_tx()
558 	q->info[idx].xdpf = xdpf;  in fun_xdp_tx()
560 	u64_stats_update_begin(&q->syncp);  in fun_xdp_tx()
561 	q->stats.tx_bytes += tot_len;  in fun_xdp_tx()
562 	q->stats.tx_pkts++;  in fun_xdp_tx()
563 	u64_stats_update_end(&q->syncp);  in fun_xdp_tx()
565 	trace_funeth_tx(q, tot_len, idx, nfrags);  in fun_xdp_tx()
566 	q->prod_cnt += ndesc;  in fun_xdp_tx()
575 	struct funeth_txq *q, **xdpqs;  in fun_xdp_xmit_frames()  local
589 	for (q = xdpqs[q_idx], i = 0; i < n; i++)  in fun_xdp_xmit_frames()
590 		if (!fun_xdp_tx(q, frames[i]))  in fun_xdp_xmit_frames()
594 		fun_txq_wr_db(q);  in fun_xdp_xmit_frames()
601 static void fun_txq_purge(struct funeth_txq *q)  in fun_txq_purge()  argument
603 	while (q->cons_cnt != q->prod_cnt) {  in fun_txq_purge()
604 		unsigned int idx = q->cons_cnt & q->mask;  in fun_txq_purge()
606 		q->cons_cnt += fun_unmap_pkt(q, idx);  in fun_txq_purge()
607 		dev_kfree_skb_any(q->info[idx].skb);  in fun_txq_purge()
609 	netdev_tx_reset_queue(q->ndq);  in fun_txq_purge()
612 static void fun_xdpq_purge(struct funeth_txq *q)  in fun_xdpq_purge()  argument
614 	while (q->cons_cnt != q->prod_cnt) {  in fun_xdpq_purge()
615 		unsigned int idx = q->cons_cnt & q->mask;  in fun_xdpq_purge()
617 		q->cons_cnt += fun_unmap_pkt(q, idx);  in fun_xdpq_purge()
618 		xdp_return_frame(q->info[idx].xdpf);  in fun_xdpq_purge()
629 	struct funeth_txq *q;  in fun_txq_create_sw()  local
637 	q = kzalloc_node(sizeof(*q), GFP_KERNEL, numa_node);  in fun_txq_create_sw()
638 	if (!q)  in fun_txq_create_sw()
641 	q->dma_dev = &fp->pdev->dev;  in fun_txq_create_sw()
642 	q->desc = fun_alloc_ring_mem(q->dma_dev, ndesc, FUNETH_SQE_SIZE,  in fun_txq_create_sw()
643 				     sizeof(*q->info), true, numa_node,  in fun_txq_create_sw()
644 				     &q->dma_addr, (void **)&q->info,  in fun_txq_create_sw()
645 				     &q->hw_wb);  in fun_txq_create_sw()
646 	if (!q->desc)  in fun_txq_create_sw()
649 	q->netdev = dev;  in fun_txq_create_sw()
650 	q->mask = ndesc - 1;  in fun_txq_create_sw()
651 	q->qidx = qidx;  in fun_txq_create_sw()
652 	q->numa_node = numa_node;  in fun_txq_create_sw()
653 	u64_stats_init(&q->syncp);  in fun_txq_create_sw()
654 	q->init_state = FUN_QSTATE_INIT_SW;  in fun_txq_create_sw()
655 	return q;  in fun_txq_create_sw()
658 	kfree(q);  in fun_txq_create_sw()
665 static void fun_txq_free_sw(struct funeth_txq *q)  in fun_txq_free_sw()  argument
667 	struct funeth_priv *fp = netdev_priv(q->netdev);  in fun_txq_free_sw()
669 	fun_free_ring_mem(q->dma_dev, q->mask + 1, FUNETH_SQE_SIZE, true,  in fun_txq_free_sw()
670 			  q->desc, q->dma_addr, q->info);  in fun_txq_free_sw()
672 	fp->tx_packets += q->stats.tx_pkts;  in fun_txq_free_sw()
673 	fp->tx_bytes   += q->stats.tx_bytes;  in fun_txq_free_sw()
674 	fp->tx_dropped += q->stats.tx_map_err;  in fun_txq_free_sw()
676 	kfree(q);  in fun_txq_free_sw()
680 int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq)  in fun_txq_create_dev()  argument
682 	struct funeth_priv *fp = netdev_priv(q->netdev);  in fun_txq_create_dev()
683 	unsigned int irq_idx, ndesc = q->mask + 1;  in fun_txq_create_dev()
686 	q->irq = irq;  in fun_txq_create_dev()
687 	*q->hw_wb = 0;  in fun_txq_create_dev()
688 	q->prod_cnt = 0;  in fun_txq_create_dev()
689 	q->cons_cnt = 0;  in fun_txq_create_dev()
696 			    q->dma_addr, fp->tx_coal_count, fp->tx_coal_usec,  in fun_txq_create_dev()
698 			    &q->hw_qid, &q->db);  in fun_txq_create_dev()
702 	err = fun_create_and_bind_tx(fp, q->hw_qid);  in fun_txq_create_dev()
705 	q->ethid = err;  in fun_txq_create_dev()
708 		irq->txq = q;  in fun_txq_create_dev()
709 		q->ndq = netdev_get_tx_queue(q->netdev, q->qidx);  in fun_txq_create_dev()
710 		q->irq_db_val = FUN_IRQ_SQ_DB(fp->tx_coal_usec,  in fun_txq_create_dev()
712 		writel(q->irq_db_val, q->db);  in fun_txq_create_dev()
715 	q->init_state = FUN_QSTATE_INIT_FULL;  in fun_txq_create_dev()
716 	netif_info(fp, ifup, q->netdev,  in fun_txq_create_dev()
718 		   irq ? "Tx" : "XDP", q->qidx, ndesc, q->hw_qid, irq_idx,  in fun_txq_create_dev()
719 		   q->ethid, q->numa_node);  in fun_txq_create_dev()
723 	fun_destroy_sq(fp->fdev, q->hw_qid);  in fun_txq_create_dev()
725 	netdev_err(q->netdev,  in fun_txq_create_dev()
727 		   irq ? "Tx" : "XDP", q->qidx, err);  in fun_txq_create_dev()
731 static void fun_txq_free_dev(struct funeth_txq *q)  in fun_txq_free_dev()  argument
733 	struct funeth_priv *fp = netdev_priv(q->netdev);  in fun_txq_free_dev()
735 	if (q->init_state < FUN_QSTATE_INIT_FULL)  in fun_txq_free_dev()
738 	netif_info(fp, ifdown, q->netdev,  in fun_txq_free_dev()
740 		   q->irq ? "Tx" : "XDP", q->qidx, q->hw_qid,  in fun_txq_free_dev()
741 		   q->irq ? q->irq->irq_idx : 0, q->ethid);  in fun_txq_free_dev()
743 	fun_destroy_sq(fp->fdev, q->hw_qid);  in fun_txq_free_dev()
744 	fun_res_destroy(fp->fdev, FUN_ADMIN_OP_ETH, 0, q->ethid);  in fun_txq_free_dev()
746 	if (q->irq) {  in fun_txq_free_dev()
747 		q->irq->txq = NULL;  in fun_txq_free_dev()
748 		fun_txq_purge(q);  in fun_txq_free_dev()
750 		fun_xdpq_purge(q);  in fun_txq_free_dev()
753 	q->init_state = FUN_QSTATE_INIT_SW;  in fun_txq_free_dev()
763 	struct funeth_txq *q = *qp;  in funeth_txq_create()  local
766 	if (!q)  in funeth_txq_create()
767 		q = fun_txq_create_sw(dev, qidx, ndesc, irq);  in funeth_txq_create()
768 	if (!q)  in funeth_txq_create()
771 	if (q->init_state >= state)  in funeth_txq_create()
774 	err = fun_txq_create_dev(q, irq);  in funeth_txq_create()
777 			fun_txq_free_sw(q);  in funeth_txq_create()
782 	*qp = q;  in funeth_txq_create()
789 struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state)  in funeth_txq_free()  argument
792 		fun_txq_free_dev(q);  in funeth_txq_free()
795 		fun_txq_free_sw(q);  in funeth_txq_free()
796 		q = NULL;  in funeth_txq_free()
799 	return q;  in funeth_txq_free()