1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3  * Copyright(c) 2020 Intel Corporation.
4  *
5  */
6 
7 /*
8  * This file contains HFI1 support for IPOIB SDMA functionality
9  */
10 
11 #include <linux/log2.h>
12 #include <linux/circ_buf.h>
13 
14 #include "sdma.h"
15 #include "verbs.h"
16 #include "trace_ibhdrs.h"
17 #include "ipoib.h"
18 
19 /* Add a convenience helper */
20 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
21 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
22 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
23 
24 /**
25  * struct ipoib_txreq - IPOIB transmit descriptor
26  * @txreq: sdma transmit request
27  * @sdma_hdr: 9b ib headers
28  * @sdma_status: status returned by sdma engine
29  * @priv: ipoib netdev private data
30  * @txq: txq on which skb was output
31  * @skb: skb to send
32  */
33 struct ipoib_txreq {
34 	struct sdma_txreq           txreq;
35 	struct hfi1_sdma_header     sdma_hdr;
36 	int                         sdma_status;
37 	struct hfi1_ipoib_dev_priv *priv;
38 	struct hfi1_ipoib_txq      *txq;
39 	struct sk_buff             *skb;
40 };
41 
42 struct ipoib_txparms {
43 	struct hfi1_devdata        *dd;
44 	struct rdma_ah_attr        *ah_attr;
45 	struct hfi1_ibport         *ibp;
46 	struct hfi1_ipoib_txq      *txq;
47 	union hfi1_ipoib_flow       flow;
48 	u32                         dqpn;
49 	u8                          hdr_dwords;
50 	u8                          entropy;
51 };
52 
53 static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
54 {
55 	return sent - completed;
56 }
57 
58 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
59 {
60 	return hfi1_ipoib_txreqs(txq->sent_txreqs,
61 				 atomic64_read(&txq->complete_txreqs));
62 }
63 
64 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
65 {
66 	if (atomic_inc_return(&txq->stops) == 1)
67 		netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
68 }
69 
70 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
71 {
72 	if (atomic_dec_and_test(&txq->stops))
73 		netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
74 }
75 
76 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)
77 {
78 	return min_t(uint, txq->priv->netdev->tx_queue_len,
79 		     txq->tx_ring.max_items - 1);
80 }
81 
82 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)
83 {
84 	return min_t(uint, txq->priv->netdev->tx_queue_len,
85 		     txq->tx_ring.max_items) >> 1;
86 }
87 
88 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
89 {
90 	++txq->sent_txreqs;
91 	if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
92 	    !atomic_xchg(&txq->ring_full, 1))
93 		hfi1_ipoib_stop_txq(txq);
94 }
95 
96 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
97 {
98 	struct net_device *dev = txq->priv->netdev;
99 
100 	/* If shutting down just return as queue state is irrelevant */
101 	if (unlikely(dev->reg_state != NETREG_REGISTERED))
102 		return;
103 
104 	/*
105 	 * When the queue has been drained to less than half full it will be
106 	 * restarted.
107 	 * The size of the txreq ring is fixed at initialization.
108 	 * The tx queue len can be adjusted upward while the interface is
109 	 * running.
110 	 * The tx queue len can be large enough to overflow the txreq_ring.
111 	 * Use the minimum of the current tx_queue_len or the rings max txreqs
112 	 * to protect against ring overflow.
113 	 */
114 	if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
115 	    atomic_xchg(&txq->ring_full, 0))
116 		hfi1_ipoib_wake_txq(txq);
117 }
118 
119 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
120 {
121 	struct hfi1_ipoib_dev_priv *priv = tx->priv;
122 
123 	if (likely(!tx->sdma_status)) {
124 		hfi1_ipoib_update_tx_netstats(priv, 1, tx->skb->len);
125 	} else {
126 		++priv->netdev->stats.tx_errors;
127 		dd_dev_warn(priv->dd,
128 			    "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
129 			    __func__, tx->sdma_status,
130 			    le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx,
131 			    tx->txq->sde->this_idx);
132 	}
133 
134 	napi_consume_skb(tx->skb, budget);
135 	sdma_txclean(priv->dd, &tx->txreq);
136 	kmem_cache_free(priv->txreq_cache, tx);
137 }
138 
139 static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget)
140 {
141 	struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
142 	unsigned long head;
143 	unsigned long tail;
144 	unsigned int max_tx;
145 	int work_done;
146 	int tx_count;
147 
148 	spin_lock_bh(&tx_ring->consumer_lock);
149 
150 	/* Read index before reading contents at that index. */
151 	head = smp_load_acquire(&tx_ring->head);
152 	tail = tx_ring->tail;
153 	max_tx = tx_ring->max_items;
154 
155 	work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget);
156 
157 	for (tx_count = work_done; tx_count; tx_count--) {
158 		hfi1_ipoib_free_tx(tx_ring->items[tail], budget);
159 		tail = CIRC_NEXT(tail, max_tx);
160 	}
161 
162 	atomic64_add(work_done, &txq->complete_txreqs);
163 
164 	/* Finished freeing tx items so store the tail value. */
165 	smp_store_release(&tx_ring->tail, tail);
166 
167 	spin_unlock_bh(&tx_ring->consumer_lock);
168 
169 	hfi1_ipoib_check_queue_stopped(txq);
170 
171 	return work_done;
172 }
173 
174 static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget)
175 {
176 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev);
177 	struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis];
178 
179 	int work_done = hfi1_ipoib_drain_tx_ring(txq, budget);
180 
181 	if (work_done < budget)
182 		napi_complete_done(napi, work_done);
183 
184 	return work_done;
185 }
186 
187 static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx)
188 {
189 	struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring;
190 	unsigned long head;
191 	unsigned long tail;
192 	size_t max_tx;
193 
194 	spin_lock(&tx_ring->producer_lock);
195 
196 	head = tx_ring->head;
197 	tail = READ_ONCE(tx_ring->tail);
198 	max_tx = tx_ring->max_items;
199 
200 	if (likely(CIRC_SPACE(head, tail, max_tx))) {
201 		tx_ring->items[head] = tx;
202 
203 		/* Finish storing txreq before incrementing head. */
204 		smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx));
205 		napi_schedule(tx->txq->napi);
206 	} else {
207 		struct hfi1_ipoib_txq *txq = tx->txq;
208 		struct hfi1_ipoib_dev_priv *priv = tx->priv;
209 
210 		/* Ring was full */
211 		hfi1_ipoib_free_tx(tx, 0);
212 		atomic64_inc(&txq->complete_txreqs);
213 		dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx);
214 	}
215 
216 	spin_unlock(&tx_ring->producer_lock);
217 }
218 
219 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
220 {
221 	struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
222 
223 	tx->sdma_status = status;
224 
225 	hfi1_ipoib_add_tx(tx);
226 }
227 
228 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
229 					struct ipoib_txparms *txp)
230 {
231 	struct hfi1_devdata *dd = txp->dd;
232 	struct sdma_txreq *txreq = &tx->txreq;
233 	struct sk_buff *skb = tx->skb;
234 	int ret = 0;
235 	int i;
236 
237 	if (skb_headlen(skb)) {
238 		ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
239 		if (unlikely(ret))
240 			return ret;
241 	}
242 
243 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
244 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
245 
246 		ret = sdma_txadd_page(dd,
247 				      txreq,
248 				      skb_frag_page(frag),
249 				      frag->bv_offset,
250 				      skb_frag_size(frag));
251 		if (unlikely(ret))
252 			break;
253 	}
254 
255 	return ret;
256 }
257 
258 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
259 				    struct ipoib_txparms *txp)
260 {
261 	struct hfi1_devdata *dd = txp->dd;
262 	struct sdma_txreq *txreq = &tx->txreq;
263 	struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
264 	u16 pkt_bytes =
265 		sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
266 	int ret;
267 
268 	ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
269 	if (unlikely(ret))
270 		return ret;
271 
272 	/* add pbc + headers */
273 	ret = sdma_txadd_kvaddr(dd,
274 				txreq,
275 				sdma_hdr,
276 				sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
277 	if (unlikely(ret))
278 		return ret;
279 
280 	/* add the ulp payload */
281 	return hfi1_ipoib_build_ulp_payload(tx, txp);
282 }
283 
284 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
285 					   struct ipoib_txparms *txp)
286 {
287 	struct hfi1_ipoib_dev_priv *priv = tx->priv;
288 	struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
289 	struct sk_buff *skb = tx->skb;
290 	struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
291 	struct rdma_ah_attr *ah_attr = txp->ah_attr;
292 	struct ib_other_headers *ohdr;
293 	struct ib_grh *grh;
294 	u16 dwords;
295 	u16 slid;
296 	u16 dlid;
297 	u16 lrh0;
298 	u32 bth0;
299 	u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
300 			 priv->netdev->dev_addr[2] << 8 |
301 			 priv->netdev->dev_addr[3]);
302 	u16 payload_dwords;
303 	u8 pad_cnt;
304 
305 	pad_cnt = -skb->len & 3;
306 
307 	/* Includes ICRC */
308 	payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
309 
310 	/* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
311 	txp->hdr_dwords = 7;
312 
313 	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
314 		grh = &sdma_hdr->hdr.ibh.u.l.grh;
315 		txp->hdr_dwords +=
316 			hfi1_make_grh(txp->ibp,
317 				      grh,
318 				      rdma_ah_read_grh(ah_attr),
319 				      txp->hdr_dwords - LRH_9B_DWORDS,
320 				      payload_dwords);
321 		lrh0 = HFI1_LRH_GRH;
322 		ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
323 	} else {
324 		lrh0 = HFI1_LRH_BTH;
325 		ohdr = &sdma_hdr->hdr.ibh.u.oth;
326 	}
327 
328 	lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
329 	lrh0 |= (txp->flow.sc5 & 0xf) << 12;
330 
331 	dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
332 	if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
333 		slid = be16_to_cpu(IB_LID_PERMISSIVE);
334 	} else {
335 		u16 lid = (u16)ppd->lid;
336 
337 		if (lid) {
338 			lid |= rdma_ah_get_path_bits(ah_attr) &
339 				((1 << ppd->lmc) - 1);
340 			slid = lid;
341 		} else {
342 			slid = be16_to_cpu(IB_LID_PERMISSIVE);
343 		}
344 	}
345 
346 	/* Includes ICRC */
347 	dwords = txp->hdr_dwords + payload_dwords;
348 
349 	/* Build the lrh */
350 	sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
351 	hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
352 
353 	/* Build the bth */
354 	bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
355 
356 	ohdr->bth[0] = cpu_to_be32(bth0);
357 	ohdr->bth[1] = cpu_to_be32(txp->dqpn);
358 	ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs));
359 
360 	/* Build the deth */
361 	ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
362 	ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
363 					  HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
364 
365 	/* Construct the pbc. */
366 	sdma_hdr->pbc =
367 		cpu_to_le64(create_pbc(ppd,
368 				       ib_is_sc5(txp->flow.sc5) <<
369 							      PBC_DC_INFO_SHIFT,
370 				       0,
371 				       sc_to_vlt(priv->dd, txp->flow.sc5),
372 				       dwords - SIZE_OF_CRC +
373 						(sizeof(sdma_hdr->pbc) >> 2)));
374 }
375 
376 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
377 						      struct sk_buff *skb,
378 						      struct ipoib_txparms *txp)
379 {
380 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
381 	struct ipoib_txreq *tx;
382 	int ret;
383 
384 	tx = kmem_cache_alloc_node(priv->txreq_cache,
385 				   GFP_ATOMIC,
386 				   priv->dd->node);
387 	if (unlikely(!tx))
388 		return ERR_PTR(-ENOMEM);
389 
390 	/* so that we can test if the sdma descriptors are there */
391 	tx->txreq.num_desc = 0;
392 	tx->priv = priv;
393 	tx->txq = txp->txq;
394 	tx->skb = skb;
395 	INIT_LIST_HEAD(&tx->txreq.list);
396 
397 	hfi1_ipoib_build_ib_tx_headers(tx, txp);
398 
399 	ret = hfi1_ipoib_build_tx_desc(tx, txp);
400 	if (likely(!ret)) {
401 		if (txp->txq->flow.as_int != txp->flow.as_int) {
402 			txp->txq->flow.tx_queue = txp->flow.tx_queue;
403 			txp->txq->flow.sc5 = txp->flow.sc5;
404 			txp->txq->sde =
405 				sdma_select_engine_sc(priv->dd,
406 						      txp->flow.tx_queue,
407 						      txp->flow.sc5);
408 		}
409 
410 		return tx;
411 	}
412 
413 	sdma_txclean(priv->dd, &tx->txreq);
414 	kmem_cache_free(priv->txreq_cache, tx);
415 
416 	return ERR_PTR(ret);
417 }
418 
419 static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
420 				     struct hfi1_ipoib_txq *txq)
421 {
422 	int ret;
423 	u16 count_out;
424 
425 	ret = sdma_send_txlist(txq->sde,
426 			       iowait_get_ib_work(&txq->wait),
427 			       &txq->tx_list,
428 			       &count_out);
429 	if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
430 		return ret;
431 
432 	dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
433 
434 	return ret;
435 }
436 
437 static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
438 				    struct hfi1_ipoib_txq *txq)
439 {
440 	int ret = 0;
441 
442 	if (!list_empty(&txq->tx_list)) {
443 		/* Flush the current list */
444 		ret = hfi1_ipoib_submit_tx_list(dev, txq);
445 
446 		if (unlikely(ret))
447 			if (ret != -EBUSY)
448 				++dev->stats.tx_carrier_errors;
449 	}
450 
451 	return ret;
452 }
453 
454 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
455 				struct ipoib_txreq *tx)
456 {
457 	int ret;
458 
459 	ret = sdma_send_txreq(txq->sde,
460 			      iowait_get_ib_work(&txq->wait),
461 			      &tx->txreq,
462 			      txq->pkts_sent);
463 	if (likely(!ret)) {
464 		txq->pkts_sent = true;
465 		iowait_starve_clear(txq->pkts_sent, &txq->wait);
466 	}
467 
468 	return ret;
469 }
470 
471 static int hfi1_ipoib_send_dma_single(struct net_device *dev,
472 				      struct sk_buff *skb,
473 				      struct ipoib_txparms *txp)
474 {
475 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
476 	struct hfi1_ipoib_txq *txq = txp->txq;
477 	struct ipoib_txreq *tx;
478 	int ret;
479 
480 	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
481 	if (IS_ERR(tx)) {
482 		int ret = PTR_ERR(tx);
483 
484 		dev_kfree_skb_any(skb);
485 
486 		if (ret == -ENOMEM)
487 			++dev->stats.tx_errors;
488 		else
489 			++dev->stats.tx_carrier_errors;
490 
491 		return NETDEV_TX_OK;
492 	}
493 
494 	ret = hfi1_ipoib_submit_tx(txq, tx);
495 	if (likely(!ret)) {
496 tx_ok:
497 		trace_sdma_output_ibhdr(tx->priv->dd,
498 					&tx->sdma_hdr.hdr,
499 					ib_is_sc5(txp->flow.sc5));
500 		hfi1_ipoib_check_queue_depth(txq);
501 		return NETDEV_TX_OK;
502 	}
503 
504 	txq->pkts_sent = false;
505 
506 	if (ret == -EBUSY || ret == -ECOMM)
507 		goto tx_ok;
508 
509 	sdma_txclean(priv->dd, &tx->txreq);
510 	dev_kfree_skb_any(skb);
511 	kmem_cache_free(priv->txreq_cache, tx);
512 	++dev->stats.tx_carrier_errors;
513 
514 	return NETDEV_TX_OK;
515 }
516 
517 static int hfi1_ipoib_send_dma_list(struct net_device *dev,
518 				    struct sk_buff *skb,
519 				    struct ipoib_txparms *txp)
520 {
521 	struct hfi1_ipoib_txq *txq = txp->txq;
522 	struct ipoib_txreq *tx;
523 
524 	/* Has the flow change ? */
525 	if (txq->flow.as_int != txp->flow.as_int) {
526 		int ret;
527 
528 		ret = hfi1_ipoib_flush_tx_list(dev, txq);
529 		if (unlikely(ret)) {
530 			if (ret == -EBUSY)
531 				++dev->stats.tx_dropped;
532 			dev_kfree_skb_any(skb);
533 			return NETDEV_TX_OK;
534 		}
535 	}
536 	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
537 	if (IS_ERR(tx)) {
538 		int ret = PTR_ERR(tx);
539 
540 		dev_kfree_skb_any(skb);
541 
542 		if (ret == -ENOMEM)
543 			++dev->stats.tx_errors;
544 		else
545 			++dev->stats.tx_carrier_errors;
546 
547 		return NETDEV_TX_OK;
548 	}
549 
550 	list_add_tail(&tx->txreq.list, &txq->tx_list);
551 
552 	hfi1_ipoib_check_queue_depth(txq);
553 
554 	trace_sdma_output_ibhdr(tx->priv->dd,
555 				&tx->sdma_hdr.hdr,
556 				ib_is_sc5(txp->flow.sc5));
557 
558 	if (!netdev_xmit_more())
559 		(void)hfi1_ipoib_flush_tx_list(dev, txq);
560 
561 	return NETDEV_TX_OK;
562 }
563 
564 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
565 {
566 	if (skb_transport_header_was_set(skb)) {
567 		u8 *hdr = (u8 *)skb_transport_header(skb);
568 
569 		return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
570 	}
571 
572 	return (u8)skb_get_queue_mapping(skb);
573 }
574 
575 int hfi1_ipoib_send_dma(struct net_device *dev,
576 			struct sk_buff *skb,
577 			struct ib_ah *address,
578 			u32 dqpn)
579 {
580 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
581 	struct ipoib_txparms txp;
582 	struct rdma_netdev *rn = netdev_priv(dev);
583 
584 	if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
585 		dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
586 			    skb->len,
587 			    rn->mtu + HFI1_IPOIB_ENCAP_LEN);
588 		++dev->stats.tx_dropped;
589 		++dev->stats.tx_errors;
590 		dev_kfree_skb_any(skb);
591 		return NETDEV_TX_OK;
592 	}
593 
594 	txp.dd = priv->dd;
595 	txp.ah_attr = &ibah_to_rvtah(address)->attr;
596 	txp.ibp = to_iport(priv->device, priv->port_num);
597 	txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
598 	txp.dqpn = dqpn;
599 	txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
600 	txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
601 	txp.entropy = hfi1_ipoib_calc_entropy(skb);
602 
603 	if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
604 		return hfi1_ipoib_send_dma_list(dev, skb, &txp);
605 
606 	return hfi1_ipoib_send_dma_single(dev, skb,  &txp);
607 }
608 
609 /*
610  * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
611  *
612  * This function gets called from sdma_send_txreq() when there are not enough
613  * sdma descriptors available to send the packet. It adds Tx queue's wait
614  * structure to sdma engine's dmawait list to be woken up when descriptors
615  * become available.
616  */
617 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
618 				 struct iowait_work *wait,
619 				 struct sdma_txreq *txreq,
620 				 uint seq,
621 				 bool pkts_sent)
622 {
623 	struct hfi1_ipoib_txq *txq =
624 		container_of(wait->iow, struct hfi1_ipoib_txq, wait);
625 
626 	write_seqlock(&sde->waitlock);
627 
628 	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
629 		if (sdma_progress(sde, seq, txreq)) {
630 			write_sequnlock(&sde->waitlock);
631 			return -EAGAIN;
632 		}
633 
634 		if (list_empty(&txreq->list))
635 			/* came from non-list submit */
636 			list_add_tail(&txreq->list, &txq->tx_list);
637 		if (list_empty(&txq->wait.list)) {
638 			if (!atomic_xchg(&txq->no_desc, 1))
639 				hfi1_ipoib_stop_txq(txq);
640 			iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
641 		}
642 
643 		write_sequnlock(&sde->waitlock);
644 		return -EBUSY;
645 	}
646 
647 	write_sequnlock(&sde->waitlock);
648 	return -EINVAL;
649 }
650 
651 /*
652  * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
653  *
654  * This function gets called when SDMA descriptors becomes available and Tx
655  * queue's wait structure was previously added to sdma engine's dmawait list.
656  */
657 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
658 {
659 	struct hfi1_ipoib_txq *txq =
660 		container_of(wait, struct hfi1_ipoib_txq, wait);
661 
662 	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
663 		iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
664 }
665 
666 static void hfi1_ipoib_flush_txq(struct work_struct *work)
667 {
668 	struct iowait_work *ioww =
669 		container_of(work, struct iowait_work, iowork);
670 	struct iowait *wait = iowait_ioww_to_iow(ioww);
671 	struct hfi1_ipoib_txq *txq =
672 		container_of(wait, struct hfi1_ipoib_txq, wait);
673 	struct net_device *dev = txq->priv->netdev;
674 
675 	if (likely(dev->reg_state == NETREG_REGISTERED) &&
676 	    likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
677 		if (atomic_xchg(&txq->no_desc, 0))
678 			hfi1_ipoib_wake_txq(txq);
679 }
680 
681 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
682 {
683 	struct net_device *dev = priv->netdev;
684 	char buf[HFI1_IPOIB_TXREQ_NAME_LEN];
685 	unsigned long tx_ring_size;
686 	int i;
687 
688 	/*
689 	 * Ring holds 1 less than tx_ring_size
690 	 * Round up to next power of 2 in order to hold at least tx_queue_len
691 	 */
692 	tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1);
693 
694 	snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit);
695 	priv->txreq_cache = kmem_cache_create(buf,
696 					      sizeof(struct ipoib_txreq),
697 					      0,
698 					      0,
699 					      NULL);
700 	if (!priv->txreq_cache)
701 		return -ENOMEM;
702 
703 	priv->tx_napis = kcalloc_node(dev->num_tx_queues,
704 				      sizeof(struct napi_struct),
705 				      GFP_ATOMIC,
706 				      priv->dd->node);
707 	if (!priv->tx_napis)
708 		goto free_txreq_cache;
709 
710 	priv->txqs = kcalloc_node(dev->num_tx_queues,
711 				  sizeof(struct hfi1_ipoib_txq),
712 				  GFP_ATOMIC,
713 				  priv->dd->node);
714 	if (!priv->txqs)
715 		goto free_tx_napis;
716 
717 	for (i = 0; i < dev->num_tx_queues; i++) {
718 		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
719 
720 		iowait_init(&txq->wait,
721 			    0,
722 			    hfi1_ipoib_flush_txq,
723 			    NULL,
724 			    hfi1_ipoib_sdma_sleep,
725 			    hfi1_ipoib_sdma_wakeup,
726 			    NULL,
727 			    NULL);
728 		txq->priv = priv;
729 		txq->sde = NULL;
730 		INIT_LIST_HEAD(&txq->tx_list);
731 		atomic64_set(&txq->complete_txreqs, 0);
732 		atomic_set(&txq->stops, 0);
733 		atomic_set(&txq->ring_full, 0);
734 		atomic_set(&txq->no_desc, 0);
735 		txq->q_idx = i;
736 		txq->flow.tx_queue = 0xff;
737 		txq->flow.sc5 = 0xff;
738 		txq->pkts_sent = false;
739 
740 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
741 					     priv->dd->node);
742 
743 		txq->tx_ring.items =
744 			vzalloc_node(array_size(tx_ring_size,
745 						sizeof(struct ipoib_txreq)),
746 				     priv->dd->node);
747 		if (!txq->tx_ring.items)
748 			goto free_txqs;
749 
750 		spin_lock_init(&txq->tx_ring.producer_lock);
751 		spin_lock_init(&txq->tx_ring.consumer_lock);
752 		txq->tx_ring.max_items = tx_ring_size;
753 
754 		txq->napi = &priv->tx_napis[i];
755 		netif_tx_napi_add(dev, txq->napi,
756 				  hfi1_ipoib_process_tx_ring,
757 				  NAPI_POLL_WEIGHT);
758 	}
759 
760 	return 0;
761 
762 free_txqs:
763 	for (i--; i >= 0; i--) {
764 		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
765 
766 		netif_napi_del(txq->napi);
767 		vfree(txq->tx_ring.items);
768 	}
769 
770 	kfree(priv->txqs);
771 	priv->txqs = NULL;
772 
773 free_tx_napis:
774 	kfree(priv->tx_napis);
775 	priv->tx_napis = NULL;
776 
777 free_txreq_cache:
778 	kmem_cache_destroy(priv->txreq_cache);
779 	priv->txreq_cache = NULL;
780 	return -ENOMEM;
781 }
782 
783 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
784 {
785 	struct sdma_txreq *txreq;
786 	struct sdma_txreq *txreq_tmp;
787 	atomic64_t *complete_txreqs = &txq->complete_txreqs;
788 
789 	list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
790 		struct ipoib_txreq *tx =
791 			container_of(txreq, struct ipoib_txreq, txreq);
792 
793 		list_del(&txreq->list);
794 		sdma_txclean(txq->priv->dd, &tx->txreq);
795 		dev_kfree_skb_any(tx->skb);
796 		kmem_cache_free(txq->priv->txreq_cache, tx);
797 		atomic64_inc(complete_txreqs);
798 	}
799 
800 	if (hfi1_ipoib_used(txq))
801 		dd_dev_warn(txq->priv->dd,
802 			    "txq %d not empty found %llu requests\n",
803 			    txq->q_idx,
804 			    hfi1_ipoib_txreqs(txq->sent_txreqs,
805 					      atomic64_read(complete_txreqs)));
806 }
807 
808 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
809 {
810 	int i;
811 
812 	for (i = 0; i < priv->netdev->num_tx_queues; i++) {
813 		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
814 
815 		iowait_cancel_work(&txq->wait);
816 		iowait_sdma_drain(&txq->wait);
817 		hfi1_ipoib_drain_tx_list(txq);
818 		netif_napi_del(txq->napi);
819 		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
820 		vfree(txq->tx_ring.items);
821 	}
822 
823 	kfree(priv->txqs);
824 	priv->txqs = NULL;
825 
826 	kfree(priv->tx_napis);
827 	priv->tx_napis = NULL;
828 
829 	kmem_cache_destroy(priv->txreq_cache);
830 	priv->txreq_cache = NULL;
831 }
832 
833 void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
834 {
835 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
836 	int i;
837 
838 	for (i = 0; i < dev->num_tx_queues; i++) {
839 		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
840 
841 		napi_enable(txq->napi);
842 	}
843 }
844 
845 void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
846 {
847 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
848 	int i;
849 
850 	for (i = 0; i < dev->num_tx_queues; i++) {
851 		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
852 
853 		napi_disable(txq->napi);
854 		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
855 	}
856 }
857