13 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,  in ionic_txq_post()  argument
16 	ionic_q_post(q, ring_dbell, cb_func, cb_arg);  in ionic_txq_post()
19 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,  in ionic_rxq_post()  argument
22 	ionic_q_post(q, ring_dbell, cb_func, cb_arg);  in ionic_rxq_post()
25 bool ionic_txq_poke_doorbell(struct ionic_queue *q)  in ionic_txq_poke_doorbell()  argument
31 	netdev = q->lif->netdev;  in ionic_txq_poke_doorbell()
32 	netdev_txq = netdev_get_tx_queue(netdev, q->index);  in ionic_txq_poke_doorbell()
36 	if (q->tail_idx == q->head_idx) {  in ionic_txq_poke_doorbell()
42 	then = q->dbell_jiffies;  in ionic_txq_poke_doorbell()
45 	if (dif > q->dbell_deadline) {  in ionic_txq_poke_doorbell()
46 		ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,  in ionic_txq_poke_doorbell()
47 				 q->dbval | q->head_idx);  in ionic_txq_poke_doorbell()
49 		q->dbell_jiffies = now;  in ionic_txq_poke_doorbell()
57 bool ionic_rxq_poke_doorbell(struct ionic_queue *q)  in ionic_rxq_poke_doorbell()  argument
63 	if (q->tail_idx == q->head_idx)  in ionic_rxq_poke_doorbell()
67 	then = q->dbell_jiffies;  in ionic_rxq_poke_doorbell()
70 	if (dif > q->dbell_deadline) {  in ionic_rxq_poke_doorbell()
71 		ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,  in ionic_rxq_poke_doorbell()
72 				 q->dbval | q->head_idx);  in ionic_rxq_poke_doorbell()
74 		q->dbell_jiffies = now;  in ionic_rxq_poke_doorbell()
76 		dif = 2 * q->dbell_deadline;  in ionic_rxq_poke_doorbell()
80 		q->dbell_deadline = dif;  in ionic_rxq_poke_doorbell()
86 static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)  in q_to_ndq()  argument
88 	return netdev_get_tx_queue(q->lif->netdev, q->index);  in q_to_ndq()
91 static int ionic_rx_page_alloc(struct ionic_queue *q,  in ionic_rx_page_alloc()  argument
94 	struct net_device *netdev = q->lif->netdev;  in ionic_rx_page_alloc()
99 	dev = q->dev;  in ionic_rx_page_alloc()
100 	stats = q_to_rx_stats(q);  in ionic_rx_page_alloc()
104 				    netdev->name, q->name);  in ionic_rx_page_alloc()
111 				    netdev->name, q->name);  in ionic_rx_page_alloc()
121 				    netdev->name, q->name);  in ionic_rx_page_alloc()
132 static void ionic_rx_page_free(struct ionic_queue *q,  in ionic_rx_page_free()  argument
135 	struct net_device *netdev = q->lif->netdev;  in ionic_rx_page_free()
136 	struct device *dev = q->dev;  in ionic_rx_page_free()
140 				    netdev->name, q->name);  in ionic_rx_page_free()
152 static bool ionic_rx_buf_recycle(struct ionic_queue *q,  in ionic_rx_buf_recycle()  argument
175 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,  in ionic_rx_frags()  argument
179 	struct net_device *netdev = q->lif->netdev;  in ionic_rx_frags()
182 	struct device *dev = q->dev;  in ionic_rx_frags()
188 	stats = q_to_rx_stats(q);  in ionic_rx_frags()
195 	skb = napi_get_frags(&q_to_qcq(q)->napi);  in ionic_rx_frags()
198 				     netdev->name, q->name);  in ionic_rx_frags()
222 		if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {  in ionic_rx_frags()
236 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,  in ionic_rx_copybreak()  argument
240 	struct net_device *netdev = q->lif->netdev;  in ionic_rx_copybreak()
243 	struct device *dev = q->dev;  in ionic_rx_copybreak()
247 	stats = q_to_rx_stats(q);  in ionic_rx_copybreak()
252 	skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);  in ionic_rx_copybreak()
255 				     netdev->name, q->name);  in ionic_rx_copybreak()
272 	skb->protocol = eth_type_trans(skb, q->lif->netdev);  in ionic_rx_copybreak()
277 static void ionic_rx_clean(struct ionic_queue *q,  in ionic_rx_clean()  argument
282 	struct net_device *netdev = q->lif->netdev;  in ionic_rx_clean()
283 	struct ionic_qcq *qcq = q_to_qcq(q);  in ionic_rx_clean()
290 	stats = q_to_rx_stats(q);  in ionic_rx_clean()
300 	if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)  in ionic_rx_clean()
301 		skb = ionic_rx_copybreak(q, desc_info, comp);  in ionic_rx_clean()
303 		skb = ionic_rx_frags(q, desc_info, comp);  in ionic_rx_clean()
310 	skb_record_rx_queue(skb, q->index);  in ionic_rx_clean()
350 	if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {  in ionic_rx_clean()
363 			skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);  in ionic_rx_clean()
370 	if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)  in ionic_rx_clean()
378 	struct ionic_queue *q = cq->bound_q;  in ionic_rx_service()  local
388 	if (q->tail_idx == q->head_idx)  in ionic_rx_service()
391 	if (q->tail_idx != le16_to_cpu(comp->comp_index))  in ionic_rx_service()
394 	desc_info = &q->info[q->tail_idx];  in ionic_rx_service()
395 	q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);  in ionic_rx_service()
398 	ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);  in ionic_rx_service()
406 static inline void ionic_write_cmb_desc(struct ionic_queue *q,  in ionic_write_cmb_desc()  argument
410 	if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)  in ionic_write_cmb_desc()
411 		memcpy_toio(cmb_desc, desc, q->desc_size);  in ionic_write_cmb_desc()
414 void ionic_rx_fill(struct ionic_queue *q)  in ionic_rx_fill()  argument
416 	struct net_device *netdev = q->lif->netdev;  in ionic_rx_fill()
430 	n_fill = ionic_q_space_avail(q);  in ionic_rx_fill()
433 			       q->num_descs / IONIC_RX_FILL_DIV);  in ionic_rx_fill()
442 		desc_info = &q->info[q->head_idx];  in ionic_rx_fill()
447 			if (unlikely(ionic_rx_page_alloc(q, buf_info))) {  in ionic_rx_fill()
465 		for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {  in ionic_rx_fill()
468 				if (unlikely(ionic_rx_page_alloc(q, buf_info))) {  in ionic_rx_fill()
486 		if (j < q->max_sg_elems) {  in ionic_rx_fill()
495 		ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);  in ionic_rx_fill()
497 		ionic_rxq_post(q, false, ionic_rx_clean, NULL);  in ionic_rx_fill()
500 	ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,  in ionic_rx_fill()
501 			 q->dbval | q->head_idx);  in ionic_rx_fill()
503 	q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;  in ionic_rx_fill()
504 	q->dbell_jiffies = jiffies;  in ionic_rx_fill()
506 	mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,  in ionic_rx_fill()
510 void ionic_rx_empty(struct ionic_queue *q)  in ionic_rx_empty()  argument
516 	for (i = 0; i < q->num_descs; i++) {  in ionic_rx_empty()
517 		desc_info = &q->info[i];  in ionic_rx_empty()
521 				ionic_rx_page_free(q, buf_info);  in ionic_rx_empty()
529 	q->head_idx = 0;  in ionic_rx_empty()
530 	q->tail_idx = 0;  in ionic_rx_empty()
543 	lif = qcq->q.lif;  in ionic_dim_update()
595 	if (!work_done && ionic_txq_poke_doorbell(&qcq->q))  in ionic_tx_napi()
631 	if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))  in ionic_rx_napi()
676 	if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))  in ionic_txrx_napi()
678 	if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))  in ionic_txrx_napi()
686 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,  in ionic_tx_map_single()  argument
689 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_map_single()
690 	struct device *dev = q->dev;  in ionic_tx_map_single()
696 				     q->lif->netdev->name, q->name);  in ionic_tx_map_single()
703 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,  in ionic_tx_map_frag()  argument
707 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_map_frag()
708 	struct device *dev = q->dev;  in ionic_tx_map_frag()
714 				     q->lif->netdev->name, q->name);  in ionic_tx_map_frag()
720 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,  in ionic_tx_map_skb()  argument
724 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_map_skb()
725 	struct device *dev = q->dev;  in ionic_tx_map_skb()
731 	dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));  in ionic_tx_map_skb()
743 		dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));  in ionic_tx_map_skb()
769 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,  in ionic_tx_desc_unmap_bufs()  argument
773 	struct device *dev = q->dev;  in ionic_tx_desc_unmap_bufs()
789 static void ionic_tx_clean(struct ionic_queue *q,  in ionic_tx_clean()  argument
794 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_clean()
795 	struct ionic_qcq *qcq = q_to_qcq(q);  in ionic_tx_clean()
799 	ionic_tx_desc_unmap_bufs(q, desc_info);  in ionic_tx_clean()
806 	if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) {  in ionic_tx_clean()
821 				hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);  in ionic_tx_clean()
832 	} else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) {  in ionic_tx_clean()
833 		netif_wake_subqueue(q->lif->netdev, qi);  in ionic_tx_clean()
844 	struct ionic_queue *q = cq->bound_q;  in ionic_tx_service()  local
860 		desc_info = &q->info[q->tail_idx];  in ionic_tx_service()
862 		index = q->tail_idx;  in ionic_tx_service()
863 		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);  in ionic_tx_service()
864 		ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);  in ionic_tx_service()
873 	if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))  in ionic_tx_service()
874 		netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);  in ionic_tx_service()
891 void ionic_tx_empty(struct ionic_queue *q)  in ionic_tx_empty()  argument
898 	while (q->head_idx != q->tail_idx) {  in ionic_tx_empty()
899 		desc_info = &q->info[q->tail_idx];  in ionic_tx_empty()
901 		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);  in ionic_tx_empty()
902 		ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);  in ionic_tx_empty()
911 	if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))  in ionic_tx_empty()
912 		netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);  in ionic_tx_empty()
960 static void ionic_tx_tso_post(struct ionic_queue *q,  in ionic_tx_tso_post()  argument
985 	ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);  in ionic_tx_tso_post()
989 		if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))  in ionic_tx_tso_post()
990 			netdev_tx_sent_queue(q_to_ndq(q), skb->len);  in ionic_tx_tso_post()
991 		ionic_txq_post(q, false, ionic_tx_clean, skb);  in ionic_tx_tso_post()
993 		ionic_txq_post(q, done, NULL, NULL);  in ionic_tx_tso_post()
997 static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)  in ionic_tx_tso()  argument
999 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_tso()
1022 	desc_info = &q->info[q->head_idx];  in ionic_tx_tso()
1025 	if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))  in ionic_tx_tso()
1051 		ionic_tx_desc_unmap_bufs(q, desc_info);  in ionic_tx_tso()
1105 		ionic_tx_tso_post(q, desc_info, skb,  in ionic_tx_tso()
1111 		desc_info = &q->info[q->head_idx];  in ionic_tx_tso()
1123 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,  in ionic_tx_calc_csum()  argument
1128 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_calc_csum()
1154 	ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);  in ionic_tx_calc_csum()
1162 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,  in ionic_tx_calc_no_csum()  argument
1167 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_calc_no_csum()
1193 	ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);  in ionic_tx_calc_no_csum()
1198 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,  in ionic_tx_skb_frags()  argument
1204 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_skb_frags()
1215 static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)  in ionic_tx()  argument
1217 	struct ionic_desc_info *desc_info = &q->info[q->head_idx];  in ionic_tx()
1218 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx()
1220 	if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))  in ionic_tx()
1225 		ionic_tx_calc_csum(q, skb, desc_info);  in ionic_tx()
1227 		ionic_tx_calc_no_csum(q, skb, desc_info);  in ionic_tx()
1230 	ionic_tx_skb_frags(q, skb, desc_info);  in ionic_tx()
1236 	if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))  in ionic_tx()
1237 		netdev_tx_sent_queue(q_to_ndq(q), skb->len);  in ionic_tx()
1238 	ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);  in ionic_tx()
1243 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)  in ionic_tx_descs_needed()  argument
1245 	struct ionic_tx_stats *stats = q_to_tx_stats(q);  in ionic_tx_descs_needed()
1256 	if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems)  in ionic_tx_descs_needed()
1269 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)  in ionic_maybe_stop_tx()  argument
1273 	if (unlikely(!ionic_q_has_space(q, ndescs))) {  in ionic_maybe_stop_tx()
1274 		netif_stop_subqueue(q->lif->netdev, q->index);  in ionic_maybe_stop_tx()
1279 		if (ionic_q_has_space(q, ndescs)) {  in ionic_maybe_stop_tx()
1280 			netif_wake_subqueue(q->lif->netdev, q->index);  in ionic_maybe_stop_tx()
1292 	struct ionic_queue *q = &lif->hwstamp_txq->q;  in ionic_start_hwstamp_xmit()  local
1300 	ndescs = ionic_tx_descs_needed(q, skb);  in ionic_start_hwstamp_xmit()
1304 	if (unlikely(!ionic_q_has_space(q, ndescs)))  in ionic_start_hwstamp_xmit()
1309 		err = ionic_tx_tso(q, skb);  in ionic_start_hwstamp_xmit()
1311 		err = ionic_tx(q, skb);  in ionic_start_hwstamp_xmit()
1319 	q->drop++;  in ionic_start_hwstamp_xmit()
1328 	struct ionic_queue *q;  in ionic_start_xmit()  local
1343 	q = &lif->txqcqs[queue_index]->q;  in ionic_start_xmit()
1345 	ndescs = ionic_tx_descs_needed(q, skb);  in ionic_start_xmit()
1349 	if (unlikely(ionic_maybe_stop_tx(q, ndescs)))  in ionic_start_xmit()
1353 		err = ionic_tx_tso(q, skb);  in ionic_start_xmit()
1355 		err = ionic_tx(q, skb);  in ionic_start_xmit()
1364 	ionic_maybe_stop_tx(q, 4);  in ionic_start_xmit()
1369 	q->drop++;  in ionic_start_xmit()