1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2020 Intel Corporation. 4 * 5 */ 6 7 /* 8 * This file contains HFI1 support for IPOIB SDMA functionality 9 */ 10 11 #include <linux/log2.h> 12 #include <linux/circ_buf.h> 13 14 #include "sdma.h" 15 #include "verbs.h" 16 #include "trace_ibhdrs.h" 17 #include "ipoib.h" 18 19 /* Add a convenience helper */ 20 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 21 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 22 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 23 24 /** 25 * struct ipoib_txreq - IPOIB transmit descriptor 26 * @txreq: sdma transmit request 27 * @sdma_hdr: 9b ib headers 28 * @sdma_status: status returned by sdma engine 29 * @priv: ipoib netdev private data 30 * @txq: txq on which skb was output 31 * @skb: skb to send 32 */ 33 struct ipoib_txreq { 34 struct sdma_txreq txreq; 35 struct hfi1_sdma_header sdma_hdr; 36 int sdma_status; 37 struct hfi1_ipoib_dev_priv *priv; 38 struct hfi1_ipoib_txq *txq; 39 struct sk_buff *skb; 40 }; 41 42 struct ipoib_txparms { 43 struct hfi1_devdata *dd; 44 struct rdma_ah_attr *ah_attr; 45 struct hfi1_ibport *ibp; 46 struct hfi1_ipoib_txq *txq; 47 union hfi1_ipoib_flow flow; 48 u32 dqpn; 49 u8 hdr_dwords; 50 u8 entropy; 51 }; 52 53 static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed) 54 { 55 return sent - completed; 56 } 57 58 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq) 59 { 60 if (unlikely(hfi1_ipoib_txreqs(++txq->sent_txreqs, 61 atomic64_read(&txq->complete_txreqs)) >= 62 min_t(unsigned int, txq->priv->netdev->tx_queue_len, 63 txq->tx_ring.max_items - 1))) 64 netif_stop_subqueue(txq->priv->netdev, txq->q_idx); 65 } 66 67 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) 68 { 69 struct net_device *dev = txq->priv->netdev; 70 71 /* If the queue is already running just return */ 72 if (likely(!__netif_subqueue_stopped(dev, txq->q_idx))) 73 return; 74 75 /* If shutting down just return as queue state is irrelevant */ 76 if (unlikely(dev->reg_state != NETREG_REGISTERED)) 77 return; 78 79 /* 80 * When the queue has been drained to less than half full it will be 81 * restarted. 82 * The size of the txreq ring is fixed at initialization. 83 * The tx queue len can be adjusted upward while the interface is 84 * running. 85 * The tx queue len can be large enough to overflow the txreq_ring. 86 * Use the minimum of the current tx_queue_len or the rings max txreqs 87 * to protect against ring overflow. 88 */ 89 if (hfi1_ipoib_txreqs(txq->sent_txreqs, 90 atomic64_read(&txq->complete_txreqs)) 91 < min_t(unsigned int, dev->tx_queue_len, 92 txq->tx_ring.max_items) >> 1) 93 netif_wake_subqueue(dev, txq->q_idx); 94 } 95 96 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget) 97 { 98 struct hfi1_ipoib_dev_priv *priv = tx->priv; 99 100 if (likely(!tx->sdma_status)) { 101 hfi1_ipoib_update_tx_netstats(priv, 1, tx->skb->len); 102 } else { 103 ++priv->netdev->stats.tx_errors; 104 dd_dev_warn(priv->dd, 105 "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n", 106 __func__, tx->sdma_status, 107 le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx, 108 tx->txq->sde->this_idx); 109 } 110 111 napi_consume_skb(tx->skb, budget); 112 sdma_txclean(priv->dd, &tx->txreq); 113 kmem_cache_free(priv->txreq_cache, tx); 114 } 115 116 static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget) 117 { 118 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; 119 unsigned long head; 120 unsigned long tail; 121 unsigned int max_tx; 122 int work_done; 123 int tx_count; 124 125 spin_lock_bh(&tx_ring->consumer_lock); 126 127 /* Read index before reading contents at that index. */ 128 head = smp_load_acquire(&tx_ring->head); 129 tail = tx_ring->tail; 130 max_tx = tx_ring->max_items; 131 132 work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget); 133 134 for (tx_count = work_done; tx_count; tx_count--) { 135 hfi1_ipoib_free_tx(tx_ring->items[tail], budget); 136 tail = CIRC_NEXT(tail, max_tx); 137 } 138 139 atomic64_add(work_done, &txq->complete_txreqs); 140 141 /* Finished freeing tx items so store the tail value. */ 142 smp_store_release(&tx_ring->tail, tail); 143 144 spin_unlock_bh(&tx_ring->consumer_lock); 145 146 hfi1_ipoib_check_queue_stopped(txq); 147 148 return work_done; 149 } 150 151 static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget) 152 { 153 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev); 154 struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis]; 155 156 int work_done = hfi1_ipoib_drain_tx_ring(txq, budget); 157 158 if (work_done < budget) 159 napi_complete_done(napi, work_done); 160 161 return work_done; 162 } 163 164 static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx) 165 { 166 struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring; 167 unsigned long head; 168 unsigned long tail; 169 size_t max_tx; 170 171 spin_lock(&tx_ring->producer_lock); 172 173 head = tx_ring->head; 174 tail = READ_ONCE(tx_ring->tail); 175 max_tx = tx_ring->max_items; 176 177 if (likely(CIRC_SPACE(head, tail, max_tx))) { 178 tx_ring->items[head] = tx; 179 180 /* Finish storing txreq before incrementing head. */ 181 smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx)); 182 napi_schedule(tx->txq->napi); 183 } else { 184 struct hfi1_ipoib_txq *txq = tx->txq; 185 struct hfi1_ipoib_dev_priv *priv = tx->priv; 186 187 /* Ring was full */ 188 hfi1_ipoib_free_tx(tx, 0); 189 atomic64_inc(&txq->complete_txreqs); 190 dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx); 191 } 192 193 spin_unlock(&tx_ring->producer_lock); 194 } 195 196 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status) 197 { 198 struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq); 199 200 tx->sdma_status = status; 201 202 hfi1_ipoib_add_tx(tx); 203 } 204 205 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx, 206 struct ipoib_txparms *txp) 207 { 208 struct hfi1_devdata *dd = txp->dd; 209 struct sdma_txreq *txreq = &tx->txreq; 210 struct sk_buff *skb = tx->skb; 211 int ret = 0; 212 int i; 213 214 if (skb_headlen(skb)) { 215 ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb)); 216 if (unlikely(ret)) 217 return ret; 218 } 219 220 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 221 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 222 223 ret = sdma_txadd_page(dd, 224 txreq, 225 skb_frag_page(frag), 226 frag->bv_offset, 227 skb_frag_size(frag)); 228 if (unlikely(ret)) 229 break; 230 } 231 232 return ret; 233 } 234 235 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx, 236 struct ipoib_txparms *txp) 237 { 238 struct hfi1_devdata *dd = txp->dd; 239 struct sdma_txreq *txreq = &tx->txreq; 240 struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr; 241 u16 pkt_bytes = 242 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len; 243 int ret; 244 245 ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete); 246 if (unlikely(ret)) 247 return ret; 248 249 /* add pbc + headers */ 250 ret = sdma_txadd_kvaddr(dd, 251 txreq, 252 sdma_hdr, 253 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2)); 254 if (unlikely(ret)) 255 return ret; 256 257 /* add the ulp payload */ 258 return hfi1_ipoib_build_ulp_payload(tx, txp); 259 } 260 261 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx, 262 struct ipoib_txparms *txp) 263 { 264 struct hfi1_ipoib_dev_priv *priv = tx->priv; 265 struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr; 266 struct sk_buff *skb = tx->skb; 267 struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp); 268 struct rdma_ah_attr *ah_attr = txp->ah_attr; 269 struct ib_other_headers *ohdr; 270 struct ib_grh *grh; 271 u16 dwords; 272 u16 slid; 273 u16 dlid; 274 u16 lrh0; 275 u32 bth0; 276 u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 | 277 priv->netdev->dev_addr[2] << 8 | 278 priv->netdev->dev_addr[3]); 279 u16 payload_dwords; 280 u8 pad_cnt; 281 282 pad_cnt = -skb->len & 3; 283 284 /* Includes ICRC */ 285 payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC; 286 287 /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */ 288 txp->hdr_dwords = 7; 289 290 if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { 291 grh = &sdma_hdr->hdr.ibh.u.l.grh; 292 txp->hdr_dwords += 293 hfi1_make_grh(txp->ibp, 294 grh, 295 rdma_ah_read_grh(ah_attr), 296 txp->hdr_dwords - LRH_9B_DWORDS, 297 payload_dwords); 298 lrh0 = HFI1_LRH_GRH; 299 ohdr = &sdma_hdr->hdr.ibh.u.l.oth; 300 } else { 301 lrh0 = HFI1_LRH_BTH; 302 ohdr = &sdma_hdr->hdr.ibh.u.oth; 303 } 304 305 lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4; 306 lrh0 |= (txp->flow.sc5 & 0xf) << 12; 307 308 dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B); 309 if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) { 310 slid = be16_to_cpu(IB_LID_PERMISSIVE); 311 } else { 312 u16 lid = (u16)ppd->lid; 313 314 if (lid) { 315 lid |= rdma_ah_get_path_bits(ah_attr) & 316 ((1 << ppd->lmc) - 1); 317 slid = lid; 318 } else { 319 slid = be16_to_cpu(IB_LID_PERMISSIVE); 320 } 321 } 322 323 /* Includes ICRC */ 324 dwords = txp->hdr_dwords + payload_dwords; 325 326 /* Build the lrh */ 327 sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B; 328 hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid); 329 330 /* Build the bth */ 331 bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey; 332 333 ohdr->bth[0] = cpu_to_be32(bth0); 334 ohdr->bth[1] = cpu_to_be32(txp->dqpn); 335 ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs)); 336 337 /* Build the deth */ 338 ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey); 339 ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy << 340 HFI1_IPOIB_ENTROPY_SHIFT) | sqpn); 341 342 /* Construct the pbc. */ 343 sdma_hdr->pbc = 344 cpu_to_le64(create_pbc(ppd, 345 ib_is_sc5(txp->flow.sc5) << 346 PBC_DC_INFO_SHIFT, 347 0, 348 sc_to_vlt(priv->dd, txp->flow.sc5), 349 dwords - SIZE_OF_CRC + 350 (sizeof(sdma_hdr->pbc) >> 2))); 351 } 352 353 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev, 354 struct sk_buff *skb, 355 struct ipoib_txparms *txp) 356 { 357 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 358 struct ipoib_txreq *tx; 359 int ret; 360 361 tx = kmem_cache_alloc_node(priv->txreq_cache, 362 GFP_ATOMIC, 363 priv->dd->node); 364 if (unlikely(!tx)) 365 return ERR_PTR(-ENOMEM); 366 367 /* so that we can test if the sdma decriptors are there */ 368 tx->txreq.num_desc = 0; 369 tx->priv = priv; 370 tx->txq = txp->txq; 371 tx->skb = skb; 372 373 hfi1_ipoib_build_ib_tx_headers(tx, txp); 374 375 ret = hfi1_ipoib_build_tx_desc(tx, txp); 376 if (likely(!ret)) { 377 if (txp->txq->flow.as_int != txp->flow.as_int) { 378 txp->txq->flow.tx_queue = txp->flow.tx_queue; 379 txp->txq->flow.sc5 = txp->flow.sc5; 380 txp->txq->sde = 381 sdma_select_engine_sc(priv->dd, 382 txp->flow.tx_queue, 383 txp->flow.sc5); 384 } 385 386 return tx; 387 } 388 389 sdma_txclean(priv->dd, &tx->txreq); 390 kmem_cache_free(priv->txreq_cache, tx); 391 392 return ERR_PTR(ret); 393 } 394 395 static int hfi1_ipoib_submit_tx_list(struct net_device *dev, 396 struct hfi1_ipoib_txq *txq) 397 { 398 int ret; 399 u16 count_out; 400 401 ret = sdma_send_txlist(txq->sde, 402 iowait_get_ib_work(&txq->wait), 403 &txq->tx_list, 404 &count_out); 405 if (likely(!ret) || ret == -EBUSY || ret == -ECOMM) 406 return ret; 407 408 dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret); 409 410 return ret; 411 } 412 413 static int hfi1_ipoib_flush_tx_list(struct net_device *dev, 414 struct hfi1_ipoib_txq *txq) 415 { 416 int ret = 0; 417 418 if (!list_empty(&txq->tx_list)) { 419 /* Flush the current list */ 420 ret = hfi1_ipoib_submit_tx_list(dev, txq); 421 422 if (unlikely(ret)) 423 if (ret != -EBUSY) 424 ++dev->stats.tx_carrier_errors; 425 } 426 427 return ret; 428 } 429 430 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq, 431 struct ipoib_txreq *tx) 432 { 433 int ret; 434 435 ret = sdma_send_txreq(txq->sde, 436 iowait_get_ib_work(&txq->wait), 437 &tx->txreq, 438 txq->pkts_sent); 439 if (likely(!ret)) { 440 txq->pkts_sent = true; 441 iowait_starve_clear(txq->pkts_sent, &txq->wait); 442 } 443 444 return ret; 445 } 446 447 static int hfi1_ipoib_send_dma_single(struct net_device *dev, 448 struct sk_buff *skb, 449 struct ipoib_txparms *txp) 450 { 451 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 452 struct hfi1_ipoib_txq *txq = txp->txq; 453 struct ipoib_txreq *tx; 454 int ret; 455 456 tx = hfi1_ipoib_send_dma_common(dev, skb, txp); 457 if (IS_ERR(tx)) { 458 int ret = PTR_ERR(tx); 459 460 dev_kfree_skb_any(skb); 461 462 if (ret == -ENOMEM) 463 ++dev->stats.tx_errors; 464 else 465 ++dev->stats.tx_carrier_errors; 466 467 return NETDEV_TX_OK; 468 } 469 470 ret = hfi1_ipoib_submit_tx(txq, tx); 471 if (likely(!ret)) { 472 trace_sdma_output_ibhdr(tx->priv->dd, 473 &tx->sdma_hdr.hdr, 474 ib_is_sc5(txp->flow.sc5)); 475 hfi1_ipoib_check_queue_depth(txq); 476 return NETDEV_TX_OK; 477 } 478 479 txq->pkts_sent = false; 480 481 if (ret == -EBUSY) { 482 list_add_tail(&tx->txreq.list, &txq->tx_list); 483 484 trace_sdma_output_ibhdr(tx->priv->dd, 485 &tx->sdma_hdr.hdr, 486 ib_is_sc5(txp->flow.sc5)); 487 hfi1_ipoib_check_queue_depth(txq); 488 return NETDEV_TX_OK; 489 } 490 491 if (ret == -ECOMM) { 492 hfi1_ipoib_check_queue_depth(txq); 493 return NETDEV_TX_OK; 494 } 495 496 sdma_txclean(priv->dd, &tx->txreq); 497 dev_kfree_skb_any(skb); 498 kmem_cache_free(priv->txreq_cache, tx); 499 ++dev->stats.tx_carrier_errors; 500 501 return NETDEV_TX_OK; 502 } 503 504 static int hfi1_ipoib_send_dma_list(struct net_device *dev, 505 struct sk_buff *skb, 506 struct ipoib_txparms *txp) 507 { 508 struct hfi1_ipoib_txq *txq = txp->txq; 509 struct ipoib_txreq *tx; 510 511 /* Has the flow change ? */ 512 if (txq->flow.as_int != txp->flow.as_int) 513 (void)hfi1_ipoib_flush_tx_list(dev, txq); 514 515 tx = hfi1_ipoib_send_dma_common(dev, skb, txp); 516 if (IS_ERR(tx)) { 517 int ret = PTR_ERR(tx); 518 519 dev_kfree_skb_any(skb); 520 521 if (ret == -ENOMEM) 522 ++dev->stats.tx_errors; 523 else 524 ++dev->stats.tx_carrier_errors; 525 526 return NETDEV_TX_OK; 527 } 528 529 list_add_tail(&tx->txreq.list, &txq->tx_list); 530 531 hfi1_ipoib_check_queue_depth(txq); 532 533 trace_sdma_output_ibhdr(tx->priv->dd, 534 &tx->sdma_hdr.hdr, 535 ib_is_sc5(txp->flow.sc5)); 536 537 if (!netdev_xmit_more()) 538 (void)hfi1_ipoib_flush_tx_list(dev, txq); 539 540 return NETDEV_TX_OK; 541 } 542 543 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb) 544 { 545 if (skb_transport_header_was_set(skb)) { 546 u8 *hdr = (u8 *)skb_transport_header(skb); 547 548 return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]); 549 } 550 551 return (u8)skb_get_queue_mapping(skb); 552 } 553 554 int hfi1_ipoib_send_dma(struct net_device *dev, 555 struct sk_buff *skb, 556 struct ib_ah *address, 557 u32 dqpn) 558 { 559 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 560 struct ipoib_txparms txp; 561 struct rdma_netdev *rn = netdev_priv(dev); 562 563 if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) { 564 dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n", 565 skb->len, 566 rn->mtu + HFI1_IPOIB_ENCAP_LEN); 567 ++dev->stats.tx_dropped; 568 ++dev->stats.tx_errors; 569 dev_kfree_skb_any(skb); 570 return NETDEV_TX_OK; 571 } 572 573 txp.dd = priv->dd; 574 txp.ah_attr = &ibah_to_rvtah(address)->attr; 575 txp.ibp = to_iport(priv->device, priv->port_num); 576 txp.txq = &priv->txqs[skb_get_queue_mapping(skb)]; 577 txp.dqpn = dqpn; 578 txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)]; 579 txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb); 580 txp.entropy = hfi1_ipoib_calc_entropy(skb); 581 582 if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list)) 583 return hfi1_ipoib_send_dma_list(dev, skb, &txp); 584 585 return hfi1_ipoib_send_dma_single(dev, skb, &txp); 586 } 587 588 /* 589 * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function 590 * 591 * This function gets called from sdma_send_txreq() when there are not enough 592 * sdma descriptors available to send the packet. It adds Tx queue's wait 593 * structure to sdma engine's dmawait list to be woken up when descriptors 594 * become available. 595 */ 596 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde, 597 struct iowait_work *wait, 598 struct sdma_txreq *txreq, 599 uint seq, 600 bool pkts_sent) 601 { 602 struct hfi1_ipoib_txq *txq = 603 container_of(wait->iow, struct hfi1_ipoib_txq, wait); 604 605 write_seqlock(&sde->waitlock); 606 607 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) { 608 if (sdma_progress(sde, seq, txreq)) { 609 write_sequnlock(&sde->waitlock); 610 return -EAGAIN; 611 } 612 613 netif_stop_subqueue(txq->priv->netdev, txq->q_idx); 614 615 if (list_empty(&txq->wait.list)) 616 iowait_queue(pkts_sent, wait->iow, &sde->dmawait); 617 618 write_sequnlock(&sde->waitlock); 619 return -EBUSY; 620 } 621 622 write_sequnlock(&sde->waitlock); 623 return -EINVAL; 624 } 625 626 /* 627 * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function 628 * 629 * This function gets called when SDMA descriptors becomes available and Tx 630 * queue's wait structure was previously added to sdma engine's dmawait list. 631 */ 632 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason) 633 { 634 struct hfi1_ipoib_txq *txq = 635 container_of(wait, struct hfi1_ipoib_txq, wait); 636 637 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) 638 iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND); 639 } 640 641 static void hfi1_ipoib_flush_txq(struct work_struct *work) 642 { 643 struct iowait_work *ioww = 644 container_of(work, struct iowait_work, iowork); 645 struct iowait *wait = iowait_ioww_to_iow(ioww); 646 struct hfi1_ipoib_txq *txq = 647 container_of(wait, struct hfi1_ipoib_txq, wait); 648 struct net_device *dev = txq->priv->netdev; 649 650 if (likely(dev->reg_state == NETREG_REGISTERED) && 651 likely(__netif_subqueue_stopped(dev, txq->q_idx)) && 652 likely(!hfi1_ipoib_flush_tx_list(dev, txq))) 653 netif_wake_subqueue(dev, txq->q_idx); 654 } 655 656 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) 657 { 658 struct net_device *dev = priv->netdev; 659 char buf[HFI1_IPOIB_TXREQ_NAME_LEN]; 660 unsigned long tx_ring_size; 661 int i; 662 663 /* 664 * Ring holds 1 less than tx_ring_size 665 * Round up to next power of 2 in order to hold at least tx_queue_len 666 */ 667 tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1); 668 669 snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit); 670 priv->txreq_cache = kmem_cache_create(buf, 671 sizeof(struct ipoib_txreq), 672 0, 673 0, 674 NULL); 675 if (!priv->txreq_cache) 676 return -ENOMEM; 677 678 priv->tx_napis = kcalloc_node(dev->num_tx_queues, 679 sizeof(struct napi_struct), 680 GFP_ATOMIC, 681 priv->dd->node); 682 if (!priv->tx_napis) 683 goto free_txreq_cache; 684 685 priv->txqs = kcalloc_node(dev->num_tx_queues, 686 sizeof(struct hfi1_ipoib_txq), 687 GFP_ATOMIC, 688 priv->dd->node); 689 if (!priv->txqs) 690 goto free_tx_napis; 691 692 for (i = 0; i < dev->num_tx_queues; i++) { 693 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 694 695 iowait_init(&txq->wait, 696 0, 697 hfi1_ipoib_flush_txq, 698 NULL, 699 hfi1_ipoib_sdma_sleep, 700 hfi1_ipoib_sdma_wakeup, 701 NULL, 702 NULL); 703 txq->priv = priv; 704 txq->sde = NULL; 705 INIT_LIST_HEAD(&txq->tx_list); 706 atomic64_set(&txq->complete_txreqs, 0); 707 txq->q_idx = i; 708 txq->flow.tx_queue = 0xff; 709 txq->flow.sc5 = 0xff; 710 txq->pkts_sent = false; 711 712 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 713 priv->dd->node); 714 715 txq->tx_ring.items = 716 vzalloc_node(array_size(tx_ring_size, 717 sizeof(struct ipoib_txreq)), 718 priv->dd->node); 719 if (!txq->tx_ring.items) 720 goto free_txqs; 721 722 spin_lock_init(&txq->tx_ring.producer_lock); 723 spin_lock_init(&txq->tx_ring.consumer_lock); 724 txq->tx_ring.max_items = tx_ring_size; 725 726 txq->napi = &priv->tx_napis[i]; 727 netif_tx_napi_add(dev, txq->napi, 728 hfi1_ipoib_process_tx_ring, 729 NAPI_POLL_WEIGHT); 730 } 731 732 return 0; 733 734 free_txqs: 735 for (i--; i >= 0; i--) { 736 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 737 738 netif_napi_del(txq->napi); 739 vfree(txq->tx_ring.items); 740 } 741 742 kfree(priv->txqs); 743 priv->txqs = NULL; 744 745 free_tx_napis: 746 kfree(priv->tx_napis); 747 priv->tx_napis = NULL; 748 749 free_txreq_cache: 750 kmem_cache_destroy(priv->txreq_cache); 751 priv->txreq_cache = NULL; 752 return -ENOMEM; 753 } 754 755 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq) 756 { 757 struct sdma_txreq *txreq; 758 struct sdma_txreq *txreq_tmp; 759 atomic64_t *complete_txreqs = &txq->complete_txreqs; 760 761 list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) { 762 struct ipoib_txreq *tx = 763 container_of(txreq, struct ipoib_txreq, txreq); 764 765 list_del(&txreq->list); 766 sdma_txclean(txq->priv->dd, &tx->txreq); 767 dev_kfree_skb_any(tx->skb); 768 kmem_cache_free(txq->priv->txreq_cache, tx); 769 atomic64_inc(complete_txreqs); 770 } 771 772 if (hfi1_ipoib_txreqs(txq->sent_txreqs, atomic64_read(complete_txreqs))) 773 dd_dev_warn(txq->priv->dd, 774 "txq %d not empty found %llu requests\n", 775 txq->q_idx, 776 hfi1_ipoib_txreqs(txq->sent_txreqs, 777 atomic64_read(complete_txreqs))); 778 } 779 780 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv) 781 { 782 int i; 783 784 for (i = 0; i < priv->netdev->num_tx_queues; i++) { 785 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 786 787 iowait_cancel_work(&txq->wait); 788 iowait_sdma_drain(&txq->wait); 789 hfi1_ipoib_drain_tx_list(txq); 790 netif_napi_del(txq->napi); 791 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); 792 vfree(txq->tx_ring.items); 793 } 794 795 kfree(priv->txqs); 796 priv->txqs = NULL; 797 798 kfree(priv->tx_napis); 799 priv->tx_napis = NULL; 800 801 kmem_cache_destroy(priv->txreq_cache); 802 priv->txreq_cache = NULL; 803 } 804 805 void hfi1_ipoib_napi_tx_enable(struct net_device *dev) 806 { 807 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 808 int i; 809 810 for (i = 0; i < dev->num_tx_queues; i++) { 811 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 812 813 napi_enable(txq->napi); 814 } 815 } 816 817 void hfi1_ipoib_napi_tx_disable(struct net_device *dev) 818 { 819 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 820 int i; 821 822 for (i = 0; i < dev->num_tx_queues; i++) { 823 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 824 825 napi_disable(txq->napi); 826 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); 827 } 828 } 829