1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 9 #include "ionic.h" 10 #include "ionic_lif.h" 11 #include "ionic_txrx.h" 12 13 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 14 ionic_desc_cb cb_func, void *cb_arg) 15 { 16 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 17 } 18 19 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 20 ionic_desc_cb cb_func, void *cb_arg) 21 { 22 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 23 } 24 25 bool ionic_txq_poke_doorbell(struct ionic_queue *q) 26 { 27 unsigned long now, then, dif; 28 struct netdev_queue *netdev_txq; 29 struct net_device *netdev; 30 31 netdev = q->lif->netdev; 32 netdev_txq = netdev_get_tx_queue(netdev, q->index); 33 34 HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id()); 35 36 if (q->tail_idx == q->head_idx) { 37 HARD_TX_UNLOCK(netdev, netdev_txq); 38 return false; 39 } 40 41 now = READ_ONCE(jiffies); 42 then = q->dbell_jiffies; 43 dif = now - then; 44 45 if (dif > q->dbell_deadline) { 46 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 47 q->dbval | q->head_idx); 48 49 q->dbell_jiffies = now; 50 } 51 52 HARD_TX_UNLOCK(netdev, netdev_txq); 53 54 return true; 55 } 56 57 bool ionic_rxq_poke_doorbell(struct ionic_queue *q) 58 { 59 unsigned long now, then, dif; 60 61 /* no lock, called from rx napi or txrx napi, nothing else can fill */ 62 63 if (q->tail_idx == q->head_idx) 64 return false; 65 66 now = READ_ONCE(jiffies); 67 then = q->dbell_jiffies; 68 dif = now - then; 69 70 if (dif > q->dbell_deadline) { 71 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 72 q->dbval | q->head_idx); 73 74 q->dbell_jiffies = now; 75 76 dif = 2 * q->dbell_deadline; 77 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE) 78 dif = IONIC_RX_MAX_DOORBELL_DEADLINE; 79 80 q->dbell_deadline = dif; 81 } 82 83 return true; 84 } 85 86 static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 87 { 88 return netdev_get_tx_queue(q->lif->netdev, q->index); 89 } 90 91 static int ionic_rx_page_alloc(struct ionic_queue *q, 92 struct ionic_buf_info *buf_info) 93 { 94 struct net_device *netdev = q->lif->netdev; 95 struct ionic_rx_stats *stats; 96 struct device *dev; 97 struct page *page; 98 99 dev = q->dev; 100 stats = q_to_rx_stats(q); 101 102 if (unlikely(!buf_info)) { 103 net_err_ratelimited("%s: %s invalid buf_info in alloc\n", 104 netdev->name, q->name); 105 return -EINVAL; 106 } 107 108 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 109 if (unlikely(!page)) { 110 net_err_ratelimited("%s: %s page alloc failed\n", 111 netdev->name, q->name); 112 stats->alloc_err++; 113 return -ENOMEM; 114 } 115 116 buf_info->dma_addr = dma_map_page(dev, page, 0, 117 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 118 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { 119 __free_pages(page, 0); 120 net_err_ratelimited("%s: %s dma map failed\n", 121 netdev->name, q->name); 122 stats->dma_map_err++; 123 return -EIO; 124 } 125 126 buf_info->page = page; 127 buf_info->page_offset = 0; 128 129 return 0; 130 } 131 132 static void ionic_rx_page_free(struct ionic_queue *q, 133 struct ionic_buf_info *buf_info) 134 { 135 struct net_device *netdev = q->lif->netdev; 136 struct device *dev = q->dev; 137 138 if (unlikely(!buf_info)) { 139 net_err_ratelimited("%s: %s invalid buf_info in free\n", 140 netdev->name, q->name); 141 return; 142 } 143 144 if (!buf_info->page) 145 return; 146 147 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 148 __free_pages(buf_info->page, 0); 149 buf_info->page = NULL; 150 } 151 152 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 153 struct ionic_buf_info *buf_info, u32 used) 154 { 155 u32 size; 156 157 /* don't re-use pages allocated in low-mem condition */ 158 if (page_is_pfmemalloc(buf_info->page)) 159 return false; 160 161 /* don't re-use buffers from non-local numa nodes */ 162 if (page_to_nid(buf_info->page) != numa_mem_id()) 163 return false; 164 165 size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); 166 buf_info->page_offset += size; 167 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 168 return false; 169 170 get_page(buf_info->page); 171 172 return true; 173 } 174 175 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 176 struct ionic_desc_info *desc_info, 177 struct ionic_rxq_comp *comp) 178 { 179 struct net_device *netdev = q->lif->netdev; 180 struct ionic_buf_info *buf_info; 181 struct ionic_rx_stats *stats; 182 struct device *dev = q->dev; 183 struct sk_buff *skb; 184 unsigned int i; 185 u16 frag_len; 186 u16 len; 187 188 stats = q_to_rx_stats(q); 189 190 buf_info = &desc_info->bufs[0]; 191 len = le16_to_cpu(comp->len); 192 193 prefetchw(buf_info->page); 194 195 skb = napi_get_frags(&q_to_qcq(q)->napi); 196 if (unlikely(!skb)) { 197 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 198 netdev->name, q->name); 199 stats->alloc_err++; 200 return NULL; 201 } 202 203 i = comp->num_sg_elems + 1; 204 do { 205 if (unlikely(!buf_info->page)) { 206 dev_kfree_skb(skb); 207 return NULL; 208 } 209 210 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 211 len -= frag_len; 212 213 dma_sync_single_for_cpu(dev, 214 buf_info->dma_addr + buf_info->page_offset, 215 frag_len, DMA_FROM_DEVICE); 216 217 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 218 buf_info->page, buf_info->page_offset, frag_len, 219 IONIC_PAGE_SIZE); 220 221 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 222 dma_unmap_page(dev, buf_info->dma_addr, 223 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 224 buf_info->page = NULL; 225 } 226 227 buf_info++; 228 229 i--; 230 } while (i > 0); 231 232 return skb; 233 } 234 235 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 236 struct ionic_desc_info *desc_info, 237 struct ionic_rxq_comp *comp) 238 { 239 struct net_device *netdev = q->lif->netdev; 240 struct ionic_buf_info *buf_info; 241 struct ionic_rx_stats *stats; 242 struct device *dev = q->dev; 243 struct sk_buff *skb; 244 u16 len; 245 246 stats = q_to_rx_stats(q); 247 248 buf_info = &desc_info->bufs[0]; 249 len = le16_to_cpu(comp->len); 250 251 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 252 if (unlikely(!skb)) { 253 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 254 netdev->name, q->name); 255 stats->alloc_err++; 256 return NULL; 257 } 258 259 if (unlikely(!buf_info->page)) { 260 dev_kfree_skb(skb); 261 return NULL; 262 } 263 264 dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, 265 len, DMA_FROM_DEVICE); 266 skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); 267 dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, 268 len, DMA_FROM_DEVICE); 269 270 skb_put(skb, len); 271 skb->protocol = eth_type_trans(skb, q->lif->netdev); 272 273 return skb; 274 } 275 276 static void ionic_rx_clean(struct ionic_queue *q, 277 struct ionic_desc_info *desc_info, 278 struct ionic_cq_info *cq_info, 279 void *cb_arg) 280 { 281 struct net_device *netdev = q->lif->netdev; 282 struct ionic_qcq *qcq = q_to_qcq(q); 283 struct ionic_rx_stats *stats; 284 struct ionic_rxq_comp *comp; 285 struct sk_buff *skb; 286 287 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); 288 289 stats = q_to_rx_stats(q); 290 291 if (comp->status) { 292 stats->dropped++; 293 return; 294 } 295 296 stats->pkts++; 297 stats->bytes += le16_to_cpu(comp->len); 298 299 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 300 skb = ionic_rx_copybreak(q, desc_info, comp); 301 else 302 skb = ionic_rx_frags(q, desc_info, comp); 303 304 if (unlikely(!skb)) { 305 stats->dropped++; 306 return; 307 } 308 309 skb_record_rx_queue(skb, q->index); 310 311 if (likely(netdev->features & NETIF_F_RXHASH)) { 312 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 313 case IONIC_PKT_TYPE_IPV4: 314 case IONIC_PKT_TYPE_IPV6: 315 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 316 PKT_HASH_TYPE_L3); 317 break; 318 case IONIC_PKT_TYPE_IPV4_TCP: 319 case IONIC_PKT_TYPE_IPV6_TCP: 320 case IONIC_PKT_TYPE_IPV4_UDP: 321 case IONIC_PKT_TYPE_IPV6_UDP: 322 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 323 PKT_HASH_TYPE_L4); 324 break; 325 } 326 } 327 328 if (likely(netdev->features & NETIF_F_RXCSUM) && 329 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 330 skb->ip_summed = CHECKSUM_COMPLETE; 331 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 332 stats->csum_complete++; 333 } else { 334 stats->csum_none++; 335 } 336 337 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 338 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 339 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 340 stats->csum_error++; 341 342 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 343 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 344 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 345 le16_to_cpu(comp->vlan_tci)); 346 stats->vlan_stripped++; 347 } 348 349 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 350 __le64 *cq_desc_hwstamp; 351 u64 hwstamp; 352 353 cq_desc_hwstamp = 354 cq_info->cq_desc + 355 qcq->cq.desc_size - 356 sizeof(struct ionic_rxq_comp) - 357 IONIC_HWSTAMP_CQ_NEGOFFSET; 358 359 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 360 361 if (hwstamp != IONIC_HWSTAMP_INVALID) { 362 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 363 stats->hwstamp_valid++; 364 } else { 365 stats->hwstamp_invalid++; 366 } 367 } 368 369 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 370 napi_gro_receive(&qcq->napi, skb); 371 else 372 napi_gro_frags(&qcq->napi); 373 } 374 375 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 376 { 377 struct ionic_queue *q = cq->bound_q; 378 struct ionic_desc_info *desc_info; 379 struct ionic_rxq_comp *comp; 380 381 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 382 383 if (!color_match(comp->pkt_type_color, cq->done_color)) 384 return false; 385 386 /* check for empty queue */ 387 if (q->tail_idx == q->head_idx) 388 return false; 389 390 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 391 return false; 392 393 desc_info = &q->info[q->tail_idx]; 394 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 395 396 /* clean the related q entry, only one per qc completion */ 397 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 398 399 desc_info->cb = NULL; 400 desc_info->cb_arg = NULL; 401 402 return true; 403 } 404 405 static inline void ionic_write_cmb_desc(struct ionic_queue *q, 406 void __iomem *cmb_desc, 407 void *desc) 408 { 409 if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS) 410 memcpy_toio(cmb_desc, desc, q->desc_size); 411 } 412 413 void ionic_rx_fill(struct ionic_queue *q) 414 { 415 struct net_device *netdev = q->lif->netdev; 416 struct ionic_desc_info *desc_info; 417 struct ionic_rxq_sg_desc *sg_desc; 418 struct ionic_rxq_sg_elem *sg_elem; 419 struct ionic_buf_info *buf_info; 420 unsigned int fill_threshold; 421 struct ionic_rxq_desc *desc; 422 unsigned int remain_len; 423 unsigned int frag_len; 424 unsigned int nfrags; 425 unsigned int n_fill; 426 unsigned int i, j; 427 unsigned int len; 428 429 n_fill = ionic_q_space_avail(q); 430 431 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD, 432 q->num_descs / IONIC_RX_FILL_DIV); 433 if (n_fill < fill_threshold) 434 return; 435 436 len = netdev->mtu + ETH_HLEN + VLAN_HLEN; 437 438 for (i = n_fill; i; i--) { 439 nfrags = 0; 440 remain_len = len; 441 desc_info = &q->info[q->head_idx]; 442 desc = desc_info->desc; 443 buf_info = &desc_info->bufs[0]; 444 445 if (!buf_info->page) { /* alloc a new buffer? */ 446 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 447 desc->addr = 0; 448 desc->len = 0; 449 return; 450 } 451 } 452 453 /* fill main descriptor - buf[0] */ 454 desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 455 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 456 desc->len = cpu_to_le16(frag_len); 457 remain_len -= frag_len; 458 buf_info++; 459 nfrags++; 460 461 /* fill sg descriptors - buf[1..n] */ 462 sg_desc = desc_info->sg_desc; 463 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { 464 sg_elem = &sg_desc->elems[j]; 465 if (!buf_info->page) { /* alloc a new sg buffer? */ 466 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 467 sg_elem->addr = 0; 468 sg_elem->len = 0; 469 return; 470 } 471 } 472 473 sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 474 frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset); 475 sg_elem->len = cpu_to_le16(frag_len); 476 remain_len -= frag_len; 477 buf_info++; 478 nfrags++; 479 } 480 481 /* clear end sg element as a sentinel */ 482 if (j < q->max_sg_elems) { 483 sg_elem = &sg_desc->elems[j]; 484 memset(sg_elem, 0, sizeof(*sg_elem)); 485 } 486 487 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 488 IONIC_RXQ_DESC_OPCODE_SIMPLE; 489 desc_info->nbufs = nfrags; 490 491 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 492 493 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 494 } 495 496 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 497 q->dbval | q->head_idx); 498 499 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE; 500 q->dbell_jiffies = jiffies; 501 502 mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline, 503 jiffies + IONIC_NAPI_DEADLINE); 504 } 505 506 void ionic_rx_empty(struct ionic_queue *q) 507 { 508 struct ionic_desc_info *desc_info; 509 struct ionic_buf_info *buf_info; 510 unsigned int i, j; 511 512 for (i = 0; i < q->num_descs; i++) { 513 desc_info = &q->info[i]; 514 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 515 buf_info = &desc_info->bufs[j]; 516 if (buf_info->page) 517 ionic_rx_page_free(q, buf_info); 518 } 519 520 desc_info->nbufs = 0; 521 desc_info->cb = NULL; 522 desc_info->cb_arg = NULL; 523 } 524 525 q->head_idx = 0; 526 q->tail_idx = 0; 527 } 528 529 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 530 { 531 struct dim_sample dim_sample; 532 struct ionic_lif *lif; 533 unsigned int qi; 534 u64 pkts, bytes; 535 536 if (!qcq->intr.dim_coal_hw) 537 return; 538 539 lif = qcq->q.lif; 540 qi = qcq->cq.bound_q->index; 541 542 switch (napi_mode) { 543 case IONIC_LIF_F_TX_DIM_INTR: 544 pkts = lif->txqstats[qi].pkts; 545 bytes = lif->txqstats[qi].bytes; 546 break; 547 case IONIC_LIF_F_RX_DIM_INTR: 548 pkts = lif->rxqstats[qi].pkts; 549 bytes = lif->rxqstats[qi].bytes; 550 break; 551 default: 552 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 553 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 554 break; 555 } 556 557 dim_update_sample(qcq->cq.bound_intr->rearm_count, 558 pkts, bytes, &dim_sample); 559 560 net_dim(&qcq->dim, dim_sample); 561 } 562 563 int ionic_tx_napi(struct napi_struct *napi, int budget) 564 { 565 struct ionic_qcq *qcq = napi_to_qcq(napi); 566 struct ionic_cq *cq = napi_to_cq(napi); 567 struct ionic_dev *idev; 568 struct ionic_lif *lif; 569 u32 work_done = 0; 570 u32 flags = 0; 571 572 lif = cq->bound_q->lif; 573 idev = &lif->ionic->idev; 574 575 work_done = ionic_cq_service(cq, budget, 576 ionic_tx_service, NULL, NULL); 577 578 if (work_done < budget && napi_complete_done(napi, work_done)) { 579 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 580 flags |= IONIC_INTR_CRED_UNMASK; 581 cq->bound_intr->rearm_count++; 582 } 583 584 if (work_done || flags) { 585 flags |= IONIC_INTR_CRED_RESET_COALESCE; 586 ionic_intr_credits(idev->intr_ctrl, 587 cq->bound_intr->index, 588 work_done, flags); 589 } 590 591 if (!work_done && ionic_txq_poke_doorbell(&qcq->q)) 592 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 593 594 return work_done; 595 } 596 597 int ionic_rx_napi(struct napi_struct *napi, int budget) 598 { 599 struct ionic_qcq *qcq = napi_to_qcq(napi); 600 struct ionic_cq *cq = napi_to_cq(napi); 601 struct ionic_dev *idev; 602 struct ionic_lif *lif; 603 u32 work_done = 0; 604 u32 flags = 0; 605 606 lif = cq->bound_q->lif; 607 idev = &lif->ionic->idev; 608 609 work_done = ionic_cq_service(cq, budget, 610 ionic_rx_service, NULL, NULL); 611 612 ionic_rx_fill(cq->bound_q); 613 614 if (work_done < budget && napi_complete_done(napi, work_done)) { 615 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 616 flags |= IONIC_INTR_CRED_UNMASK; 617 cq->bound_intr->rearm_count++; 618 } 619 620 if (work_done || flags) { 621 flags |= IONIC_INTR_CRED_RESET_COALESCE; 622 ionic_intr_credits(idev->intr_ctrl, 623 cq->bound_intr->index, 624 work_done, flags); 625 } 626 627 if (!work_done && ionic_rxq_poke_doorbell(&qcq->q)) 628 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 629 630 return work_done; 631 } 632 633 int ionic_txrx_napi(struct napi_struct *napi, int budget) 634 { 635 struct ionic_qcq *rxqcq = napi_to_qcq(napi); 636 struct ionic_cq *rxcq = napi_to_cq(napi); 637 unsigned int qi = rxcq->bound_q->index; 638 struct ionic_qcq *txqcq; 639 struct ionic_dev *idev; 640 struct ionic_lif *lif; 641 struct ionic_cq *txcq; 642 bool resched = false; 643 u32 rx_work_done = 0; 644 u32 tx_work_done = 0; 645 u32 flags = 0; 646 647 lif = rxcq->bound_q->lif; 648 idev = &lif->ionic->idev; 649 txqcq = lif->txqcqs[qi]; 650 txcq = &lif->txqcqs[qi]->cq; 651 652 tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, 653 ionic_tx_service, NULL, NULL); 654 655 rx_work_done = ionic_cq_service(rxcq, budget, 656 ionic_rx_service, NULL, NULL); 657 658 ionic_rx_fill(rxcq->bound_q); 659 660 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 661 ionic_dim_update(rxqcq, 0); 662 flags |= IONIC_INTR_CRED_UNMASK; 663 rxcq->bound_intr->rearm_count++; 664 } 665 666 if (rx_work_done || flags) { 667 flags |= IONIC_INTR_CRED_RESET_COALESCE; 668 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 669 tx_work_done + rx_work_done, flags); 670 } 671 672 if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q)) 673 resched = true; 674 if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q)) 675 resched = true; 676 if (resched) 677 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 678 679 return rx_work_done; 680 } 681 682 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 683 void *data, size_t len) 684 { 685 struct ionic_tx_stats *stats = q_to_tx_stats(q); 686 struct device *dev = q->dev; 687 dma_addr_t dma_addr; 688 689 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 690 if (dma_mapping_error(dev, dma_addr)) { 691 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 692 q->lif->netdev->name, q->name); 693 stats->dma_map_err++; 694 return 0; 695 } 696 return dma_addr; 697 } 698 699 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 700 const skb_frag_t *frag, 701 size_t offset, size_t len) 702 { 703 struct ionic_tx_stats *stats = q_to_tx_stats(q); 704 struct device *dev = q->dev; 705 dma_addr_t dma_addr; 706 707 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 708 if (dma_mapping_error(dev, dma_addr)) { 709 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 710 q->lif->netdev->name, q->name); 711 stats->dma_map_err++; 712 } 713 return dma_addr; 714 } 715 716 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 717 struct ionic_desc_info *desc_info) 718 { 719 struct ionic_buf_info *buf_info = desc_info->bufs; 720 struct ionic_tx_stats *stats = q_to_tx_stats(q); 721 struct device *dev = q->dev; 722 dma_addr_t dma_addr; 723 unsigned int nfrags; 724 skb_frag_t *frag; 725 int frag_idx; 726 727 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 728 if (dma_mapping_error(dev, dma_addr)) { 729 stats->dma_map_err++; 730 return -EIO; 731 } 732 buf_info->dma_addr = dma_addr; 733 buf_info->len = skb_headlen(skb); 734 buf_info++; 735 736 frag = skb_shinfo(skb)->frags; 737 nfrags = skb_shinfo(skb)->nr_frags; 738 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 739 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 740 if (dma_mapping_error(dev, dma_addr)) { 741 stats->dma_map_err++; 742 goto dma_fail; 743 } 744 buf_info->dma_addr = dma_addr; 745 buf_info->len = skb_frag_size(frag); 746 buf_info++; 747 } 748 749 desc_info->nbufs = 1 + nfrags; 750 751 return 0; 752 753 dma_fail: 754 /* unwind the frag mappings and the head mapping */ 755 while (frag_idx > 0) { 756 frag_idx--; 757 buf_info--; 758 dma_unmap_page(dev, buf_info->dma_addr, 759 buf_info->len, DMA_TO_DEVICE); 760 } 761 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); 762 return -EIO; 763 } 764 765 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 766 struct ionic_desc_info *desc_info) 767 { 768 struct ionic_buf_info *buf_info = desc_info->bufs; 769 struct device *dev = q->dev; 770 unsigned int i; 771 772 if (!desc_info->nbufs) 773 return; 774 775 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, 776 buf_info->len, DMA_TO_DEVICE); 777 buf_info++; 778 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 779 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, 780 buf_info->len, DMA_TO_DEVICE); 781 782 desc_info->nbufs = 0; 783 } 784 785 static void ionic_tx_clean(struct ionic_queue *q, 786 struct ionic_desc_info *desc_info, 787 struct ionic_cq_info *cq_info, 788 void *cb_arg) 789 { 790 struct ionic_tx_stats *stats = q_to_tx_stats(q); 791 struct ionic_qcq *qcq = q_to_qcq(q); 792 struct sk_buff *skb = cb_arg; 793 u16 qi; 794 795 ionic_tx_desc_unmap_bufs(q, desc_info); 796 797 if (!skb) 798 return; 799 800 qi = skb_get_queue_mapping(skb); 801 802 if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) { 803 if (cq_info) { 804 struct skb_shared_hwtstamps hwts = {}; 805 __le64 *cq_desc_hwstamp; 806 u64 hwstamp; 807 808 cq_desc_hwstamp = 809 cq_info->cq_desc + 810 qcq->cq.desc_size - 811 sizeof(struct ionic_txq_comp) - 812 IONIC_HWSTAMP_CQ_NEGOFFSET; 813 814 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 815 816 if (hwstamp != IONIC_HWSTAMP_INVALID) { 817 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 818 819 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 820 skb_tstamp_tx(skb, &hwts); 821 822 stats->hwstamp_valid++; 823 } else { 824 stats->hwstamp_invalid++; 825 } 826 } 827 828 } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) { 829 netif_wake_subqueue(q->lif->netdev, qi); 830 } 831 832 desc_info->bytes = skb->len; 833 stats->clean++; 834 835 dev_consume_skb_any(skb); 836 } 837 838 bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 839 { 840 struct ionic_queue *q = cq->bound_q; 841 struct ionic_desc_info *desc_info; 842 struct ionic_txq_comp *comp; 843 int bytes = 0; 844 int pkts = 0; 845 u16 index; 846 847 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 848 849 if (!color_match(comp->color, cq->done_color)) 850 return false; 851 852 /* clean the related q entries, there could be 853 * several q entries completed for each cq completion 854 */ 855 do { 856 desc_info = &q->info[q->tail_idx]; 857 desc_info->bytes = 0; 858 index = q->tail_idx; 859 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 860 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 861 if (desc_info->cb_arg) { 862 pkts++; 863 bytes += desc_info->bytes; 864 } 865 desc_info->cb = NULL; 866 desc_info->cb_arg = NULL; 867 } while (index != le16_to_cpu(comp->comp_index)); 868 869 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 870 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 871 872 return true; 873 } 874 875 void ionic_tx_flush(struct ionic_cq *cq) 876 { 877 struct ionic_dev *idev = &cq->lif->ionic->idev; 878 u32 work_done; 879 880 work_done = ionic_cq_service(cq, cq->num_descs, 881 ionic_tx_service, NULL, NULL); 882 if (work_done) 883 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 884 work_done, IONIC_INTR_CRED_RESET_COALESCE); 885 } 886 887 void ionic_tx_empty(struct ionic_queue *q) 888 { 889 struct ionic_desc_info *desc_info; 890 int bytes = 0; 891 int pkts = 0; 892 893 /* walk the not completed tx entries, if any */ 894 while (q->head_idx != q->tail_idx) { 895 desc_info = &q->info[q->tail_idx]; 896 desc_info->bytes = 0; 897 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 898 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 899 if (desc_info->cb_arg) { 900 pkts++; 901 bytes += desc_info->bytes; 902 } 903 desc_info->cb = NULL; 904 desc_info->cb_arg = NULL; 905 } 906 907 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 908 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 909 } 910 911 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 912 { 913 int err; 914 915 err = skb_cow_head(skb, 0); 916 if (err) 917 return err; 918 919 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 920 inner_ip_hdr(skb)->check = 0; 921 inner_tcp_hdr(skb)->check = 922 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 923 inner_ip_hdr(skb)->daddr, 924 0, IPPROTO_TCP, 0); 925 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 926 inner_tcp_hdr(skb)->check = 927 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 928 &inner_ipv6_hdr(skb)->daddr, 929 0, IPPROTO_TCP, 0); 930 } 931 932 return 0; 933 } 934 935 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 936 { 937 int err; 938 939 err = skb_cow_head(skb, 0); 940 if (err) 941 return err; 942 943 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 944 ip_hdr(skb)->check = 0; 945 tcp_hdr(skb)->check = 946 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 947 ip_hdr(skb)->daddr, 948 0, IPPROTO_TCP, 0); 949 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 950 tcp_v6_gso_csum_prep(skb); 951 } 952 953 return 0; 954 } 955 956 static void ionic_tx_tso_post(struct ionic_queue *q, 957 struct ionic_desc_info *desc_info, 958 struct sk_buff *skb, 959 dma_addr_t addr, u8 nsge, u16 len, 960 unsigned int hdrlen, unsigned int mss, 961 bool outer_csum, 962 u16 vlan_tci, bool has_vlan, 963 bool start, bool done) 964 { 965 struct ionic_txq_desc *desc = desc_info->desc; 966 u8 flags = 0; 967 u64 cmd; 968 969 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 970 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 971 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 972 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 973 974 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 975 desc->cmd = cpu_to_le64(cmd); 976 desc->len = cpu_to_le16(len); 977 desc->vlan_tci = cpu_to_le16(vlan_tci); 978 desc->hdr_len = cpu_to_le16(hdrlen); 979 desc->mss = cpu_to_le16(mss); 980 981 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 982 983 if (start) { 984 skb_tx_timestamp(skb); 985 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 986 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 987 ionic_txq_post(q, false, ionic_tx_clean, skb); 988 } else { 989 ionic_txq_post(q, done, NULL, NULL); 990 } 991 } 992 993 static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 994 { 995 struct ionic_tx_stats *stats = q_to_tx_stats(q); 996 struct ionic_desc_info *desc_info; 997 struct ionic_buf_info *buf_info; 998 struct ionic_txq_sg_elem *elem; 999 struct ionic_txq_desc *desc; 1000 unsigned int chunk_len; 1001 unsigned int frag_rem; 1002 unsigned int tso_rem; 1003 unsigned int seg_rem; 1004 dma_addr_t desc_addr; 1005 dma_addr_t frag_addr; 1006 unsigned int hdrlen; 1007 unsigned int len; 1008 unsigned int mss; 1009 bool start, done; 1010 bool outer_csum; 1011 bool has_vlan; 1012 u16 desc_len; 1013 u8 desc_nsge; 1014 u16 vlan_tci; 1015 bool encap; 1016 int err; 1017 1018 desc_info = &q->info[q->head_idx]; 1019 buf_info = desc_info->bufs; 1020 1021 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1022 return -EIO; 1023 1024 len = skb->len; 1025 mss = skb_shinfo(skb)->gso_size; 1026 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 1027 SKB_GSO_GRE_CSUM | 1028 SKB_GSO_IPXIP4 | 1029 SKB_GSO_IPXIP6 | 1030 SKB_GSO_UDP_TUNNEL | 1031 SKB_GSO_UDP_TUNNEL_CSUM)); 1032 has_vlan = !!skb_vlan_tag_present(skb); 1033 vlan_tci = skb_vlan_tag_get(skb); 1034 encap = skb->encapsulation; 1035 1036 /* Preload inner-most TCP csum field with IP pseudo hdr 1037 * calculated with IP length set to zero. HW will later 1038 * add in length to each TCP segment resulting from the TSO. 1039 */ 1040 1041 if (encap) 1042 err = ionic_tx_tcp_inner_pseudo_csum(skb); 1043 else 1044 err = ionic_tx_tcp_pseudo_csum(skb); 1045 if (err) { 1046 /* clean up mapping from ionic_tx_map_skb */ 1047 ionic_tx_desc_unmap_bufs(q, desc_info); 1048 return err; 1049 } 1050 1051 if (encap) 1052 hdrlen = skb_inner_tcp_all_headers(skb); 1053 else 1054 hdrlen = skb_tcp_all_headers(skb); 1055 1056 tso_rem = len; 1057 seg_rem = min(tso_rem, hdrlen + mss); 1058 1059 frag_addr = 0; 1060 frag_rem = 0; 1061 1062 start = true; 1063 1064 while (tso_rem > 0) { 1065 desc = NULL; 1066 elem = NULL; 1067 desc_addr = 0; 1068 desc_len = 0; 1069 desc_nsge = 0; 1070 /* use fragments until we have enough to post a single descriptor */ 1071 while (seg_rem > 0) { 1072 /* if the fragment is exhausted then move to the next one */ 1073 if (frag_rem == 0) { 1074 /* grab the next fragment */ 1075 frag_addr = buf_info->dma_addr; 1076 frag_rem = buf_info->len; 1077 buf_info++; 1078 } 1079 chunk_len = min(frag_rem, seg_rem); 1080 if (!desc) { 1081 /* fill main descriptor */ 1082 desc = desc_info->txq_desc; 1083 elem = desc_info->txq_sg_desc->elems; 1084 desc_addr = frag_addr; 1085 desc_len = chunk_len; 1086 } else { 1087 /* fill sg descriptor */ 1088 elem->addr = cpu_to_le64(frag_addr); 1089 elem->len = cpu_to_le16(chunk_len); 1090 elem++; 1091 desc_nsge++; 1092 } 1093 frag_addr += chunk_len; 1094 frag_rem -= chunk_len; 1095 tso_rem -= chunk_len; 1096 seg_rem -= chunk_len; 1097 } 1098 seg_rem = min(tso_rem, mss); 1099 done = (tso_rem == 0); 1100 /* post descriptor */ 1101 ionic_tx_tso_post(q, desc_info, skb, 1102 desc_addr, desc_nsge, desc_len, 1103 hdrlen, mss, outer_csum, vlan_tci, has_vlan, 1104 start, done); 1105 start = false; 1106 /* Buffer information is stored with the first tso descriptor */ 1107 desc_info = &q->info[q->head_idx]; 1108 desc_info->nbufs = 0; 1109 } 1110 1111 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1112 stats->bytes += len; 1113 stats->tso++; 1114 stats->tso_bytes = len; 1115 1116 return 0; 1117 } 1118 1119 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1120 struct ionic_desc_info *desc_info) 1121 { 1122 struct ionic_txq_desc *desc = desc_info->txq_desc; 1123 struct ionic_buf_info *buf_info = desc_info->bufs; 1124 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1125 bool has_vlan; 1126 u8 flags = 0; 1127 bool encap; 1128 u64 cmd; 1129 1130 has_vlan = !!skb_vlan_tag_present(skb); 1131 encap = skb->encapsulation; 1132 1133 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1134 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1135 1136 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1137 flags, skb_shinfo(skb)->nr_frags, 1138 buf_info->dma_addr); 1139 desc->cmd = cpu_to_le64(cmd); 1140 desc->len = cpu_to_le16(buf_info->len); 1141 if (has_vlan) { 1142 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1143 stats->vlan_inserted++; 1144 } else { 1145 desc->vlan_tci = 0; 1146 } 1147 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1148 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1149 1150 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1151 1152 if (skb_csum_is_sctp(skb)) 1153 stats->crc32_csum++; 1154 else 1155 stats->csum++; 1156 } 1157 1158 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1159 struct ionic_desc_info *desc_info) 1160 { 1161 struct ionic_txq_desc *desc = desc_info->txq_desc; 1162 struct ionic_buf_info *buf_info = desc_info->bufs; 1163 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1164 bool has_vlan; 1165 u8 flags = 0; 1166 bool encap; 1167 u64 cmd; 1168 1169 has_vlan = !!skb_vlan_tag_present(skb); 1170 encap = skb->encapsulation; 1171 1172 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1173 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1174 1175 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1176 flags, skb_shinfo(skb)->nr_frags, 1177 buf_info->dma_addr); 1178 desc->cmd = cpu_to_le64(cmd); 1179 desc->len = cpu_to_le16(buf_info->len); 1180 if (has_vlan) { 1181 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1182 stats->vlan_inserted++; 1183 } else { 1184 desc->vlan_tci = 0; 1185 } 1186 desc->csum_start = 0; 1187 desc->csum_offset = 0; 1188 1189 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1190 1191 stats->csum_none++; 1192 } 1193 1194 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1195 struct ionic_desc_info *desc_info) 1196 { 1197 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; 1198 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1199 struct ionic_txq_sg_elem *elem = sg_desc->elems; 1200 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1201 unsigned int i; 1202 1203 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1204 elem->addr = cpu_to_le64(buf_info->dma_addr); 1205 elem->len = cpu_to_le16(buf_info->len); 1206 } 1207 1208 stats->frags += skb_shinfo(skb)->nr_frags; 1209 } 1210 1211 static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 1212 { 1213 struct ionic_desc_info *desc_info = &q->info[q->head_idx]; 1214 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1215 1216 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1217 return -EIO; 1218 1219 /* set up the initial descriptor */ 1220 if (skb->ip_summed == CHECKSUM_PARTIAL) 1221 ionic_tx_calc_csum(q, skb, desc_info); 1222 else 1223 ionic_tx_calc_no_csum(q, skb, desc_info); 1224 1225 /* add frags */ 1226 ionic_tx_skb_frags(q, skb, desc_info); 1227 1228 skb_tx_timestamp(skb); 1229 stats->pkts++; 1230 stats->bytes += skb->len; 1231 1232 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 1233 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 1234 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 1235 1236 return 0; 1237 } 1238 1239 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1240 { 1241 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1242 int ndescs; 1243 int err; 1244 1245 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1246 if (skb_is_gso(skb)) 1247 ndescs = skb_shinfo(skb)->gso_segs; 1248 else 1249 ndescs = 1; 1250 1251 /* If non-TSO, just need 1 desc and nr_frags sg elems */ 1252 if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems) 1253 return ndescs; 1254 1255 /* Too many frags, so linearize */ 1256 err = skb_linearize(skb); 1257 if (err) 1258 return err; 1259 1260 stats->linearize++; 1261 1262 return ndescs; 1263 } 1264 1265 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 1266 { 1267 int stopped = 0; 1268 1269 if (unlikely(!ionic_q_has_space(q, ndescs))) { 1270 netif_stop_subqueue(q->lif->netdev, q->index); 1271 stopped = 1; 1272 1273 /* Might race with ionic_tx_clean, check again */ 1274 smp_rmb(); 1275 if (ionic_q_has_space(q, ndescs)) { 1276 netif_wake_subqueue(q->lif->netdev, q->index); 1277 stopped = 0; 1278 } 1279 } 1280 1281 return stopped; 1282 } 1283 1284 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1285 struct net_device *netdev) 1286 { 1287 struct ionic_lif *lif = netdev_priv(netdev); 1288 struct ionic_queue *q = &lif->hwstamp_txq->q; 1289 int err, ndescs; 1290 1291 /* Does not stop/start txq, because we post to a separate tx queue 1292 * for timestamping, and if a packet can't be posted immediately to 1293 * the timestamping queue, it is dropped. 1294 */ 1295 1296 ndescs = ionic_tx_descs_needed(q, skb); 1297 if (unlikely(ndescs < 0)) 1298 goto err_out_drop; 1299 1300 if (unlikely(!ionic_q_has_space(q, ndescs))) 1301 goto err_out_drop; 1302 1303 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1304 if (skb_is_gso(skb)) 1305 err = ionic_tx_tso(q, skb); 1306 else 1307 err = ionic_tx(q, skb); 1308 1309 if (err) 1310 goto err_out_drop; 1311 1312 return NETDEV_TX_OK; 1313 1314 err_out_drop: 1315 q->drop++; 1316 dev_kfree_skb(skb); 1317 return NETDEV_TX_OK; 1318 } 1319 1320 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1321 { 1322 u16 queue_index = skb_get_queue_mapping(skb); 1323 struct ionic_lif *lif = netdev_priv(netdev); 1324 struct ionic_queue *q; 1325 int ndescs; 1326 int err; 1327 1328 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1329 dev_kfree_skb(skb); 1330 return NETDEV_TX_OK; 1331 } 1332 1333 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1334 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1335 return ionic_start_hwstamp_xmit(skb, netdev); 1336 1337 if (unlikely(queue_index >= lif->nxqs)) 1338 queue_index = 0; 1339 q = &lif->txqcqs[queue_index]->q; 1340 1341 ndescs = ionic_tx_descs_needed(q, skb); 1342 if (ndescs < 0) 1343 goto err_out_drop; 1344 1345 if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 1346 return NETDEV_TX_BUSY; 1347 1348 if (skb_is_gso(skb)) 1349 err = ionic_tx_tso(q, skb); 1350 else 1351 err = ionic_tx(q, skb); 1352 1353 if (err) 1354 goto err_out_drop; 1355 1356 /* Stop the queue if there aren't descriptors for the next packet. 1357 * Since our SG lists per descriptor take care of most of the possible 1358 * fragmentation, we don't need to have many descriptors available. 1359 */ 1360 ionic_maybe_stop_tx(q, 4); 1361 1362 return NETDEV_TX_OK; 1363 1364 err_out_drop: 1365 q->drop++; 1366 dev_kfree_skb(skb); 1367 return NETDEV_TX_OK; 1368 } 1369