1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 9 #include "ionic.h" 10 #include "ionic_lif.h" 11 #include "ionic_txrx.h" 12 13 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 14 ionic_desc_cb cb_func, void *cb_arg) 15 { 16 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 17 } 18 19 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 20 ionic_desc_cb cb_func, void *cb_arg) 21 { 22 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 23 } 24 25 static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 26 { 27 return netdev_get_tx_queue(q->lif->netdev, q->index); 28 } 29 30 static int ionic_rx_page_alloc(struct ionic_queue *q, 31 struct ionic_buf_info *buf_info) 32 { 33 struct net_device *netdev = q->lif->netdev; 34 struct ionic_rx_stats *stats; 35 struct device *dev; 36 struct page *page; 37 38 dev = q->dev; 39 stats = q_to_rx_stats(q); 40 41 if (unlikely(!buf_info)) { 42 net_err_ratelimited("%s: %s invalid buf_info in alloc\n", 43 netdev->name, q->name); 44 return -EINVAL; 45 } 46 47 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 48 if (unlikely(!page)) { 49 net_err_ratelimited("%s: %s page alloc failed\n", 50 netdev->name, q->name); 51 stats->alloc_err++; 52 return -ENOMEM; 53 } 54 55 buf_info->dma_addr = dma_map_page(dev, page, 0, 56 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 57 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { 58 __free_pages(page, 0); 59 net_err_ratelimited("%s: %s dma map failed\n", 60 netdev->name, q->name); 61 stats->dma_map_err++; 62 return -EIO; 63 } 64 65 buf_info->page = page; 66 buf_info->page_offset = 0; 67 68 return 0; 69 } 70 71 static void ionic_rx_page_free(struct ionic_queue *q, 72 struct ionic_buf_info *buf_info) 73 { 74 struct net_device *netdev = q->lif->netdev; 75 struct device *dev = q->dev; 76 77 if (unlikely(!buf_info)) { 78 net_err_ratelimited("%s: %s invalid buf_info in free\n", 79 netdev->name, q->name); 80 return; 81 } 82 83 if (!buf_info->page) 84 return; 85 86 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 87 __free_pages(buf_info->page, 0); 88 buf_info->page = NULL; 89 } 90 91 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 92 struct ionic_buf_info *buf_info, u32 used) 93 { 94 u32 size; 95 96 /* don't re-use pages allocated in low-mem condition */ 97 if (page_is_pfmemalloc(buf_info->page)) 98 return false; 99 100 /* don't re-use buffers from non-local numa nodes */ 101 if (page_to_nid(buf_info->page) != numa_mem_id()) 102 return false; 103 104 size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); 105 buf_info->page_offset += size; 106 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 107 return false; 108 109 get_page(buf_info->page); 110 111 return true; 112 } 113 114 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 115 struct ionic_desc_info *desc_info, 116 struct ionic_rxq_comp *comp) 117 { 118 struct net_device *netdev = q->lif->netdev; 119 struct ionic_buf_info *buf_info; 120 struct ionic_rx_stats *stats; 121 struct device *dev = q->dev; 122 struct sk_buff *skb; 123 unsigned int i; 124 u16 frag_len; 125 u16 len; 126 127 stats = q_to_rx_stats(q); 128 129 buf_info = &desc_info->bufs[0]; 130 len = le16_to_cpu(comp->len); 131 132 prefetchw(buf_info->page); 133 134 skb = napi_get_frags(&q_to_qcq(q)->napi); 135 if (unlikely(!skb)) { 136 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 137 netdev->name, q->name); 138 stats->alloc_err++; 139 return NULL; 140 } 141 142 i = comp->num_sg_elems + 1; 143 do { 144 if (unlikely(!buf_info->page)) { 145 dev_kfree_skb(skb); 146 return NULL; 147 } 148 149 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 150 len -= frag_len; 151 152 dma_sync_single_for_cpu(dev, 153 buf_info->dma_addr + buf_info->page_offset, 154 frag_len, DMA_FROM_DEVICE); 155 156 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 157 buf_info->page, buf_info->page_offset, frag_len, 158 IONIC_PAGE_SIZE); 159 160 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 161 dma_unmap_page(dev, buf_info->dma_addr, 162 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 163 buf_info->page = NULL; 164 } 165 166 buf_info++; 167 168 i--; 169 } while (i > 0); 170 171 return skb; 172 } 173 174 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 175 struct ionic_desc_info *desc_info, 176 struct ionic_rxq_comp *comp) 177 { 178 struct net_device *netdev = q->lif->netdev; 179 struct ionic_buf_info *buf_info; 180 struct ionic_rx_stats *stats; 181 struct device *dev = q->dev; 182 struct sk_buff *skb; 183 u16 len; 184 185 stats = q_to_rx_stats(q); 186 187 buf_info = &desc_info->bufs[0]; 188 len = le16_to_cpu(comp->len); 189 190 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 191 if (unlikely(!skb)) { 192 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 193 netdev->name, q->name); 194 stats->alloc_err++; 195 return NULL; 196 } 197 198 if (unlikely(!buf_info->page)) { 199 dev_kfree_skb(skb); 200 return NULL; 201 } 202 203 dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, 204 len, DMA_FROM_DEVICE); 205 skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); 206 dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, 207 len, DMA_FROM_DEVICE); 208 209 skb_put(skb, len); 210 skb->protocol = eth_type_trans(skb, q->lif->netdev); 211 212 return skb; 213 } 214 215 static void ionic_rx_clean(struct ionic_queue *q, 216 struct ionic_desc_info *desc_info, 217 struct ionic_cq_info *cq_info, 218 void *cb_arg) 219 { 220 struct net_device *netdev = q->lif->netdev; 221 struct ionic_qcq *qcq = q_to_qcq(q); 222 struct ionic_rx_stats *stats; 223 struct ionic_rxq_comp *comp; 224 struct sk_buff *skb; 225 226 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); 227 228 stats = q_to_rx_stats(q); 229 230 if (comp->status) { 231 stats->dropped++; 232 return; 233 } 234 235 stats->pkts++; 236 stats->bytes += le16_to_cpu(comp->len); 237 238 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 239 skb = ionic_rx_copybreak(q, desc_info, comp); 240 else 241 skb = ionic_rx_frags(q, desc_info, comp); 242 243 if (unlikely(!skb)) { 244 stats->dropped++; 245 return; 246 } 247 248 skb_record_rx_queue(skb, q->index); 249 250 if (likely(netdev->features & NETIF_F_RXHASH)) { 251 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 252 case IONIC_PKT_TYPE_IPV4: 253 case IONIC_PKT_TYPE_IPV6: 254 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 255 PKT_HASH_TYPE_L3); 256 break; 257 case IONIC_PKT_TYPE_IPV4_TCP: 258 case IONIC_PKT_TYPE_IPV6_TCP: 259 case IONIC_PKT_TYPE_IPV4_UDP: 260 case IONIC_PKT_TYPE_IPV6_UDP: 261 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 262 PKT_HASH_TYPE_L4); 263 break; 264 } 265 } 266 267 if (likely(netdev->features & NETIF_F_RXCSUM) && 268 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 269 skb->ip_summed = CHECKSUM_COMPLETE; 270 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 271 stats->csum_complete++; 272 } else { 273 stats->csum_none++; 274 } 275 276 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 277 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 278 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 279 stats->csum_error++; 280 281 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 282 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 283 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 284 le16_to_cpu(comp->vlan_tci)); 285 stats->vlan_stripped++; 286 } 287 288 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 289 __le64 *cq_desc_hwstamp; 290 u64 hwstamp; 291 292 cq_desc_hwstamp = 293 cq_info->cq_desc + 294 qcq->cq.desc_size - 295 sizeof(struct ionic_rxq_comp) - 296 IONIC_HWSTAMP_CQ_NEGOFFSET; 297 298 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 299 300 if (hwstamp != IONIC_HWSTAMP_INVALID) { 301 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 302 stats->hwstamp_valid++; 303 } else { 304 stats->hwstamp_invalid++; 305 } 306 } 307 308 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 309 napi_gro_receive(&qcq->napi, skb); 310 else 311 napi_gro_frags(&qcq->napi); 312 } 313 314 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 315 { 316 struct ionic_queue *q = cq->bound_q; 317 struct ionic_desc_info *desc_info; 318 struct ionic_rxq_comp *comp; 319 320 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 321 322 if (!color_match(comp->pkt_type_color, cq->done_color)) 323 return false; 324 325 /* check for empty queue */ 326 if (q->tail_idx == q->head_idx) 327 return false; 328 329 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 330 return false; 331 332 desc_info = &q->info[q->tail_idx]; 333 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 334 335 /* clean the related q entry, only one per qc completion */ 336 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 337 338 desc_info->cb = NULL; 339 desc_info->cb_arg = NULL; 340 341 return true; 342 } 343 344 void ionic_rx_fill(struct ionic_queue *q) 345 { 346 struct net_device *netdev = q->lif->netdev; 347 struct ionic_desc_info *desc_info; 348 struct ionic_rxq_sg_desc *sg_desc; 349 struct ionic_rxq_sg_elem *sg_elem; 350 struct ionic_buf_info *buf_info; 351 unsigned int fill_threshold; 352 struct ionic_rxq_desc *desc; 353 unsigned int remain_len; 354 unsigned int frag_len; 355 unsigned int nfrags; 356 unsigned int n_fill; 357 unsigned int i, j; 358 unsigned int len; 359 360 n_fill = ionic_q_space_avail(q); 361 362 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD, 363 q->num_descs / IONIC_RX_FILL_DIV); 364 if (n_fill < fill_threshold) 365 return; 366 367 len = netdev->mtu + ETH_HLEN + VLAN_HLEN; 368 369 for (i = n_fill; i; i--) { 370 nfrags = 0; 371 remain_len = len; 372 desc_info = &q->info[q->head_idx]; 373 desc = desc_info->desc; 374 buf_info = &desc_info->bufs[0]; 375 376 if (!buf_info->page) { /* alloc a new buffer? */ 377 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 378 desc->addr = 0; 379 desc->len = 0; 380 return; 381 } 382 } 383 384 /* fill main descriptor - buf[0] */ 385 desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 386 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 387 desc->len = cpu_to_le16(frag_len); 388 remain_len -= frag_len; 389 buf_info++; 390 nfrags++; 391 392 /* fill sg descriptors - buf[1..n] */ 393 sg_desc = desc_info->sg_desc; 394 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { 395 sg_elem = &sg_desc->elems[j]; 396 if (!buf_info->page) { /* alloc a new sg buffer? */ 397 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 398 sg_elem->addr = 0; 399 sg_elem->len = 0; 400 return; 401 } 402 } 403 404 sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 405 frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset); 406 sg_elem->len = cpu_to_le16(frag_len); 407 remain_len -= frag_len; 408 buf_info++; 409 nfrags++; 410 } 411 412 /* clear end sg element as a sentinel */ 413 if (j < q->max_sg_elems) { 414 sg_elem = &sg_desc->elems[j]; 415 memset(sg_elem, 0, sizeof(*sg_elem)); 416 } 417 418 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 419 IONIC_RXQ_DESC_OPCODE_SIMPLE; 420 desc_info->nbufs = nfrags; 421 422 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 423 } 424 425 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 426 q->dbval | q->head_idx); 427 } 428 429 void ionic_rx_empty(struct ionic_queue *q) 430 { 431 struct ionic_desc_info *desc_info; 432 struct ionic_buf_info *buf_info; 433 unsigned int i, j; 434 435 for (i = 0; i < q->num_descs; i++) { 436 desc_info = &q->info[i]; 437 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 438 buf_info = &desc_info->bufs[j]; 439 if (buf_info->page) 440 ionic_rx_page_free(q, buf_info); 441 } 442 443 desc_info->nbufs = 0; 444 desc_info->cb = NULL; 445 desc_info->cb_arg = NULL; 446 } 447 448 q->head_idx = 0; 449 q->tail_idx = 0; 450 } 451 452 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 453 { 454 struct dim_sample dim_sample; 455 struct ionic_lif *lif; 456 unsigned int qi; 457 u64 pkts, bytes; 458 459 if (!qcq->intr.dim_coal_hw) 460 return; 461 462 lif = qcq->q.lif; 463 qi = qcq->cq.bound_q->index; 464 465 switch (napi_mode) { 466 case IONIC_LIF_F_TX_DIM_INTR: 467 pkts = lif->txqstats[qi].pkts; 468 bytes = lif->txqstats[qi].bytes; 469 break; 470 case IONIC_LIF_F_RX_DIM_INTR: 471 pkts = lif->rxqstats[qi].pkts; 472 bytes = lif->rxqstats[qi].bytes; 473 break; 474 default: 475 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 476 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 477 break; 478 } 479 480 dim_update_sample(qcq->cq.bound_intr->rearm_count, 481 pkts, bytes, &dim_sample); 482 483 net_dim(&qcq->dim, dim_sample); 484 } 485 486 int ionic_tx_napi(struct napi_struct *napi, int budget) 487 { 488 struct ionic_qcq *qcq = napi_to_qcq(napi); 489 struct ionic_cq *cq = napi_to_cq(napi); 490 struct ionic_dev *idev; 491 struct ionic_lif *lif; 492 u32 work_done = 0; 493 u32 flags = 0; 494 495 lif = cq->bound_q->lif; 496 idev = &lif->ionic->idev; 497 498 work_done = ionic_cq_service(cq, budget, 499 ionic_tx_service, NULL, NULL); 500 501 if (work_done < budget && napi_complete_done(napi, work_done)) { 502 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 503 flags |= IONIC_INTR_CRED_UNMASK; 504 cq->bound_intr->rearm_count++; 505 } 506 507 if (work_done || flags) { 508 flags |= IONIC_INTR_CRED_RESET_COALESCE; 509 ionic_intr_credits(idev->intr_ctrl, 510 cq->bound_intr->index, 511 work_done, flags); 512 } 513 514 return work_done; 515 } 516 517 int ionic_rx_napi(struct napi_struct *napi, int budget) 518 { 519 struct ionic_qcq *qcq = napi_to_qcq(napi); 520 struct ionic_cq *cq = napi_to_cq(napi); 521 struct ionic_dev *idev; 522 struct ionic_lif *lif; 523 u32 work_done = 0; 524 u32 flags = 0; 525 526 lif = cq->bound_q->lif; 527 idev = &lif->ionic->idev; 528 529 work_done = ionic_cq_service(cq, budget, 530 ionic_rx_service, NULL, NULL); 531 532 ionic_rx_fill(cq->bound_q); 533 534 if (work_done < budget && napi_complete_done(napi, work_done)) { 535 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 536 flags |= IONIC_INTR_CRED_UNMASK; 537 cq->bound_intr->rearm_count++; 538 } 539 540 if (work_done || flags) { 541 flags |= IONIC_INTR_CRED_RESET_COALESCE; 542 ionic_intr_credits(idev->intr_ctrl, 543 cq->bound_intr->index, 544 work_done, flags); 545 } 546 547 return work_done; 548 } 549 550 int ionic_txrx_napi(struct napi_struct *napi, int budget) 551 { 552 struct ionic_qcq *qcq = napi_to_qcq(napi); 553 struct ionic_cq *rxcq = napi_to_cq(napi); 554 unsigned int qi = rxcq->bound_q->index; 555 struct ionic_dev *idev; 556 struct ionic_lif *lif; 557 struct ionic_cq *txcq; 558 u32 rx_work_done = 0; 559 u32 tx_work_done = 0; 560 u32 flags = 0; 561 562 lif = rxcq->bound_q->lif; 563 idev = &lif->ionic->idev; 564 txcq = &lif->txqcqs[qi]->cq; 565 566 tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, 567 ionic_tx_service, NULL, NULL); 568 569 rx_work_done = ionic_cq_service(rxcq, budget, 570 ionic_rx_service, NULL, NULL); 571 572 ionic_rx_fill(rxcq->bound_q); 573 574 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 575 ionic_dim_update(qcq, 0); 576 flags |= IONIC_INTR_CRED_UNMASK; 577 rxcq->bound_intr->rearm_count++; 578 } 579 580 if (rx_work_done || flags) { 581 flags |= IONIC_INTR_CRED_RESET_COALESCE; 582 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 583 tx_work_done + rx_work_done, flags); 584 } 585 586 return rx_work_done; 587 } 588 589 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 590 void *data, size_t len) 591 { 592 struct ionic_tx_stats *stats = q_to_tx_stats(q); 593 struct device *dev = q->dev; 594 dma_addr_t dma_addr; 595 596 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 597 if (dma_mapping_error(dev, dma_addr)) { 598 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 599 q->lif->netdev->name, q->name); 600 stats->dma_map_err++; 601 return 0; 602 } 603 return dma_addr; 604 } 605 606 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 607 const skb_frag_t *frag, 608 size_t offset, size_t len) 609 { 610 struct ionic_tx_stats *stats = q_to_tx_stats(q); 611 struct device *dev = q->dev; 612 dma_addr_t dma_addr; 613 614 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 615 if (dma_mapping_error(dev, dma_addr)) { 616 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 617 q->lif->netdev->name, q->name); 618 stats->dma_map_err++; 619 } 620 return dma_addr; 621 } 622 623 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 624 struct ionic_desc_info *desc_info) 625 { 626 struct ionic_buf_info *buf_info = desc_info->bufs; 627 struct ionic_tx_stats *stats = q_to_tx_stats(q); 628 struct device *dev = q->dev; 629 dma_addr_t dma_addr; 630 unsigned int nfrags; 631 skb_frag_t *frag; 632 int frag_idx; 633 634 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 635 if (dma_mapping_error(dev, dma_addr)) { 636 stats->dma_map_err++; 637 return -EIO; 638 } 639 buf_info->dma_addr = dma_addr; 640 buf_info->len = skb_headlen(skb); 641 buf_info++; 642 643 frag = skb_shinfo(skb)->frags; 644 nfrags = skb_shinfo(skb)->nr_frags; 645 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 646 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 647 if (dma_mapping_error(dev, dma_addr)) { 648 stats->dma_map_err++; 649 goto dma_fail; 650 } 651 buf_info->dma_addr = dma_addr; 652 buf_info->len = skb_frag_size(frag); 653 buf_info++; 654 } 655 656 desc_info->nbufs = 1 + nfrags; 657 658 return 0; 659 660 dma_fail: 661 /* unwind the frag mappings and the head mapping */ 662 while (frag_idx > 0) { 663 frag_idx--; 664 buf_info--; 665 dma_unmap_page(dev, buf_info->dma_addr, 666 buf_info->len, DMA_TO_DEVICE); 667 } 668 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); 669 return -EIO; 670 } 671 672 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 673 struct ionic_desc_info *desc_info) 674 { 675 struct ionic_buf_info *buf_info = desc_info->bufs; 676 struct device *dev = q->dev; 677 unsigned int i; 678 679 if (!desc_info->nbufs) 680 return; 681 682 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, 683 buf_info->len, DMA_TO_DEVICE); 684 buf_info++; 685 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 686 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, 687 buf_info->len, DMA_TO_DEVICE); 688 689 desc_info->nbufs = 0; 690 } 691 692 static void ionic_tx_clean(struct ionic_queue *q, 693 struct ionic_desc_info *desc_info, 694 struct ionic_cq_info *cq_info, 695 void *cb_arg) 696 { 697 struct ionic_tx_stats *stats = q_to_tx_stats(q); 698 struct ionic_qcq *qcq = q_to_qcq(q); 699 struct sk_buff *skb = cb_arg; 700 u16 qi; 701 702 ionic_tx_desc_unmap_bufs(q, desc_info); 703 704 if (!skb) 705 return; 706 707 qi = skb_get_queue_mapping(skb); 708 709 if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) { 710 if (cq_info) { 711 struct skb_shared_hwtstamps hwts = {}; 712 __le64 *cq_desc_hwstamp; 713 u64 hwstamp; 714 715 cq_desc_hwstamp = 716 cq_info->cq_desc + 717 qcq->cq.desc_size - 718 sizeof(struct ionic_txq_comp) - 719 IONIC_HWSTAMP_CQ_NEGOFFSET; 720 721 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 722 723 if (hwstamp != IONIC_HWSTAMP_INVALID) { 724 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 725 726 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 727 skb_tstamp_tx(skb, &hwts); 728 729 stats->hwstamp_valid++; 730 } else { 731 stats->hwstamp_invalid++; 732 } 733 } 734 735 } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) { 736 netif_wake_subqueue(q->lif->netdev, qi); 737 } 738 739 desc_info->bytes = skb->len; 740 stats->clean++; 741 742 dev_consume_skb_any(skb); 743 } 744 745 bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 746 { 747 struct ionic_queue *q = cq->bound_q; 748 struct ionic_desc_info *desc_info; 749 struct ionic_txq_comp *comp; 750 int bytes = 0; 751 int pkts = 0; 752 u16 index; 753 754 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 755 756 if (!color_match(comp->color, cq->done_color)) 757 return false; 758 759 /* clean the related q entries, there could be 760 * several q entries completed for each cq completion 761 */ 762 do { 763 desc_info = &q->info[q->tail_idx]; 764 desc_info->bytes = 0; 765 index = q->tail_idx; 766 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 767 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 768 if (desc_info->cb_arg) { 769 pkts++; 770 bytes += desc_info->bytes; 771 } 772 desc_info->cb = NULL; 773 desc_info->cb_arg = NULL; 774 } while (index != le16_to_cpu(comp->comp_index)); 775 776 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 777 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 778 779 return true; 780 } 781 782 void ionic_tx_flush(struct ionic_cq *cq) 783 { 784 struct ionic_dev *idev = &cq->lif->ionic->idev; 785 u32 work_done; 786 787 work_done = ionic_cq_service(cq, cq->num_descs, 788 ionic_tx_service, NULL, NULL); 789 if (work_done) 790 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 791 work_done, IONIC_INTR_CRED_RESET_COALESCE); 792 } 793 794 void ionic_tx_empty(struct ionic_queue *q) 795 { 796 struct ionic_desc_info *desc_info; 797 int bytes = 0; 798 int pkts = 0; 799 800 /* walk the not completed tx entries, if any */ 801 while (q->head_idx != q->tail_idx) { 802 desc_info = &q->info[q->tail_idx]; 803 desc_info->bytes = 0; 804 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 805 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 806 if (desc_info->cb_arg) { 807 pkts++; 808 bytes += desc_info->bytes; 809 } 810 desc_info->cb = NULL; 811 desc_info->cb_arg = NULL; 812 } 813 814 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 815 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 816 } 817 818 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 819 { 820 int err; 821 822 err = skb_cow_head(skb, 0); 823 if (err) 824 return err; 825 826 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 827 inner_ip_hdr(skb)->check = 0; 828 inner_tcp_hdr(skb)->check = 829 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 830 inner_ip_hdr(skb)->daddr, 831 0, IPPROTO_TCP, 0); 832 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 833 inner_tcp_hdr(skb)->check = 834 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 835 &inner_ipv6_hdr(skb)->daddr, 836 0, IPPROTO_TCP, 0); 837 } 838 839 return 0; 840 } 841 842 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 843 { 844 int err; 845 846 err = skb_cow_head(skb, 0); 847 if (err) 848 return err; 849 850 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 851 ip_hdr(skb)->check = 0; 852 tcp_hdr(skb)->check = 853 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 854 ip_hdr(skb)->daddr, 855 0, IPPROTO_TCP, 0); 856 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 857 tcp_v6_gso_csum_prep(skb); 858 } 859 860 return 0; 861 } 862 863 static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 864 struct sk_buff *skb, 865 dma_addr_t addr, u8 nsge, u16 len, 866 unsigned int hdrlen, unsigned int mss, 867 bool outer_csum, 868 u16 vlan_tci, bool has_vlan, 869 bool start, bool done) 870 { 871 u8 flags = 0; 872 u64 cmd; 873 874 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 875 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 876 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 877 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 878 879 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 880 desc->cmd = cpu_to_le64(cmd); 881 desc->len = cpu_to_le16(len); 882 desc->vlan_tci = cpu_to_le16(vlan_tci); 883 desc->hdr_len = cpu_to_le16(hdrlen); 884 desc->mss = cpu_to_le16(mss); 885 886 if (start) { 887 skb_tx_timestamp(skb); 888 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 889 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 890 ionic_txq_post(q, false, ionic_tx_clean, skb); 891 } else { 892 ionic_txq_post(q, done, NULL, NULL); 893 } 894 } 895 896 static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 897 { 898 struct ionic_tx_stats *stats = q_to_tx_stats(q); 899 struct ionic_desc_info *desc_info; 900 struct ionic_buf_info *buf_info; 901 struct ionic_txq_sg_elem *elem; 902 struct ionic_txq_desc *desc; 903 unsigned int chunk_len; 904 unsigned int frag_rem; 905 unsigned int tso_rem; 906 unsigned int seg_rem; 907 dma_addr_t desc_addr; 908 dma_addr_t frag_addr; 909 unsigned int hdrlen; 910 unsigned int len; 911 unsigned int mss; 912 bool start, done; 913 bool outer_csum; 914 bool has_vlan; 915 u16 desc_len; 916 u8 desc_nsge; 917 u16 vlan_tci; 918 bool encap; 919 int err; 920 921 desc_info = &q->info[q->head_idx]; 922 buf_info = desc_info->bufs; 923 924 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 925 return -EIO; 926 927 len = skb->len; 928 mss = skb_shinfo(skb)->gso_size; 929 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 930 SKB_GSO_GRE_CSUM | 931 SKB_GSO_IPXIP4 | 932 SKB_GSO_IPXIP6 | 933 SKB_GSO_UDP_TUNNEL | 934 SKB_GSO_UDP_TUNNEL_CSUM)); 935 has_vlan = !!skb_vlan_tag_present(skb); 936 vlan_tci = skb_vlan_tag_get(skb); 937 encap = skb->encapsulation; 938 939 /* Preload inner-most TCP csum field with IP pseudo hdr 940 * calculated with IP length set to zero. HW will later 941 * add in length to each TCP segment resulting from the TSO. 942 */ 943 944 if (encap) 945 err = ionic_tx_tcp_inner_pseudo_csum(skb); 946 else 947 err = ionic_tx_tcp_pseudo_csum(skb); 948 if (err) { 949 /* clean up mapping from ionic_tx_map_skb */ 950 ionic_tx_desc_unmap_bufs(q, desc_info); 951 return err; 952 } 953 954 if (encap) 955 hdrlen = skb_inner_tcp_all_headers(skb); 956 else 957 hdrlen = skb_tcp_all_headers(skb); 958 959 tso_rem = len; 960 seg_rem = min(tso_rem, hdrlen + mss); 961 962 frag_addr = 0; 963 frag_rem = 0; 964 965 start = true; 966 967 while (tso_rem > 0) { 968 desc = NULL; 969 elem = NULL; 970 desc_addr = 0; 971 desc_len = 0; 972 desc_nsge = 0; 973 /* use fragments until we have enough to post a single descriptor */ 974 while (seg_rem > 0) { 975 /* if the fragment is exhausted then move to the next one */ 976 if (frag_rem == 0) { 977 /* grab the next fragment */ 978 frag_addr = buf_info->dma_addr; 979 frag_rem = buf_info->len; 980 buf_info++; 981 } 982 chunk_len = min(frag_rem, seg_rem); 983 if (!desc) { 984 /* fill main descriptor */ 985 desc = desc_info->txq_desc; 986 elem = desc_info->txq_sg_desc->elems; 987 desc_addr = frag_addr; 988 desc_len = chunk_len; 989 } else { 990 /* fill sg descriptor */ 991 elem->addr = cpu_to_le64(frag_addr); 992 elem->len = cpu_to_le16(chunk_len); 993 elem++; 994 desc_nsge++; 995 } 996 frag_addr += chunk_len; 997 frag_rem -= chunk_len; 998 tso_rem -= chunk_len; 999 seg_rem -= chunk_len; 1000 } 1001 seg_rem = min(tso_rem, mss); 1002 done = (tso_rem == 0); 1003 /* post descriptor */ 1004 ionic_tx_tso_post(q, desc, skb, 1005 desc_addr, desc_nsge, desc_len, 1006 hdrlen, mss, outer_csum, vlan_tci, has_vlan, 1007 start, done); 1008 start = false; 1009 /* Buffer information is stored with the first tso descriptor */ 1010 desc_info = &q->info[q->head_idx]; 1011 desc_info->nbufs = 0; 1012 } 1013 1014 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1015 stats->bytes += len; 1016 stats->tso++; 1017 stats->tso_bytes = len; 1018 1019 return 0; 1020 } 1021 1022 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1023 struct ionic_desc_info *desc_info) 1024 { 1025 struct ionic_txq_desc *desc = desc_info->txq_desc; 1026 struct ionic_buf_info *buf_info = desc_info->bufs; 1027 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1028 bool has_vlan; 1029 u8 flags = 0; 1030 bool encap; 1031 u64 cmd; 1032 1033 has_vlan = !!skb_vlan_tag_present(skb); 1034 encap = skb->encapsulation; 1035 1036 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1037 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1038 1039 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1040 flags, skb_shinfo(skb)->nr_frags, 1041 buf_info->dma_addr); 1042 desc->cmd = cpu_to_le64(cmd); 1043 desc->len = cpu_to_le16(buf_info->len); 1044 if (has_vlan) { 1045 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1046 stats->vlan_inserted++; 1047 } else { 1048 desc->vlan_tci = 0; 1049 } 1050 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1051 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1052 1053 if (skb_csum_is_sctp(skb)) 1054 stats->crc32_csum++; 1055 else 1056 stats->csum++; 1057 } 1058 1059 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1060 struct ionic_desc_info *desc_info) 1061 { 1062 struct ionic_txq_desc *desc = desc_info->txq_desc; 1063 struct ionic_buf_info *buf_info = desc_info->bufs; 1064 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1065 bool has_vlan; 1066 u8 flags = 0; 1067 bool encap; 1068 u64 cmd; 1069 1070 has_vlan = !!skb_vlan_tag_present(skb); 1071 encap = skb->encapsulation; 1072 1073 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1074 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1075 1076 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1077 flags, skb_shinfo(skb)->nr_frags, 1078 buf_info->dma_addr); 1079 desc->cmd = cpu_to_le64(cmd); 1080 desc->len = cpu_to_le16(buf_info->len); 1081 if (has_vlan) { 1082 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1083 stats->vlan_inserted++; 1084 } else { 1085 desc->vlan_tci = 0; 1086 } 1087 desc->csum_start = 0; 1088 desc->csum_offset = 0; 1089 1090 stats->csum_none++; 1091 } 1092 1093 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1094 struct ionic_desc_info *desc_info) 1095 { 1096 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; 1097 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1098 struct ionic_txq_sg_elem *elem = sg_desc->elems; 1099 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1100 unsigned int i; 1101 1102 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1103 elem->addr = cpu_to_le64(buf_info->dma_addr); 1104 elem->len = cpu_to_le16(buf_info->len); 1105 } 1106 1107 stats->frags += skb_shinfo(skb)->nr_frags; 1108 } 1109 1110 static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 1111 { 1112 struct ionic_desc_info *desc_info = &q->info[q->head_idx]; 1113 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1114 1115 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1116 return -EIO; 1117 1118 /* set up the initial descriptor */ 1119 if (skb->ip_summed == CHECKSUM_PARTIAL) 1120 ionic_tx_calc_csum(q, skb, desc_info); 1121 else 1122 ionic_tx_calc_no_csum(q, skb, desc_info); 1123 1124 /* add frags */ 1125 ionic_tx_skb_frags(q, skb, desc_info); 1126 1127 skb_tx_timestamp(skb); 1128 stats->pkts++; 1129 stats->bytes += skb->len; 1130 1131 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 1132 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 1133 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 1134 1135 return 0; 1136 } 1137 1138 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1139 { 1140 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1141 int ndescs; 1142 int err; 1143 1144 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1145 if (skb_is_gso(skb)) 1146 ndescs = skb_shinfo(skb)->gso_segs; 1147 else 1148 ndescs = 1; 1149 1150 /* If non-TSO, just need 1 desc and nr_frags sg elems */ 1151 if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems) 1152 return ndescs; 1153 1154 /* Too many frags, so linearize */ 1155 err = skb_linearize(skb); 1156 if (err) 1157 return err; 1158 1159 stats->linearize++; 1160 1161 return ndescs; 1162 } 1163 1164 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 1165 { 1166 int stopped = 0; 1167 1168 if (unlikely(!ionic_q_has_space(q, ndescs))) { 1169 netif_stop_subqueue(q->lif->netdev, q->index); 1170 stopped = 1; 1171 1172 /* Might race with ionic_tx_clean, check again */ 1173 smp_rmb(); 1174 if (ionic_q_has_space(q, ndescs)) { 1175 netif_wake_subqueue(q->lif->netdev, q->index); 1176 stopped = 0; 1177 } 1178 } 1179 1180 return stopped; 1181 } 1182 1183 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1184 struct net_device *netdev) 1185 { 1186 struct ionic_lif *lif = netdev_priv(netdev); 1187 struct ionic_queue *q = &lif->hwstamp_txq->q; 1188 int err, ndescs; 1189 1190 /* Does not stop/start txq, because we post to a separate tx queue 1191 * for timestamping, and if a packet can't be posted immediately to 1192 * the timestamping queue, it is dropped. 1193 */ 1194 1195 ndescs = ionic_tx_descs_needed(q, skb); 1196 if (unlikely(ndescs < 0)) 1197 goto err_out_drop; 1198 1199 if (unlikely(!ionic_q_has_space(q, ndescs))) 1200 goto err_out_drop; 1201 1202 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1203 if (skb_is_gso(skb)) 1204 err = ionic_tx_tso(q, skb); 1205 else 1206 err = ionic_tx(q, skb); 1207 1208 if (err) 1209 goto err_out_drop; 1210 1211 return NETDEV_TX_OK; 1212 1213 err_out_drop: 1214 q->drop++; 1215 dev_kfree_skb(skb); 1216 return NETDEV_TX_OK; 1217 } 1218 1219 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1220 { 1221 u16 queue_index = skb_get_queue_mapping(skb); 1222 struct ionic_lif *lif = netdev_priv(netdev); 1223 struct ionic_queue *q; 1224 int ndescs; 1225 int err; 1226 1227 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1228 dev_kfree_skb(skb); 1229 return NETDEV_TX_OK; 1230 } 1231 1232 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1233 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1234 return ionic_start_hwstamp_xmit(skb, netdev); 1235 1236 if (unlikely(queue_index >= lif->nxqs)) 1237 queue_index = 0; 1238 q = &lif->txqcqs[queue_index]->q; 1239 1240 ndescs = ionic_tx_descs_needed(q, skb); 1241 if (ndescs < 0) 1242 goto err_out_drop; 1243 1244 if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 1245 return NETDEV_TX_BUSY; 1246 1247 if (skb_is_gso(skb)) 1248 err = ionic_tx_tso(q, skb); 1249 else 1250 err = ionic_tx(q, skb); 1251 1252 if (err) 1253 goto err_out_drop; 1254 1255 /* Stop the queue if there aren't descriptors for the next packet. 1256 * Since our SG lists per descriptor take care of most of the possible 1257 * fragmentation, we don't need to have many descriptors available. 1258 */ 1259 ionic_maybe_stop_tx(q, 4); 1260 1261 return NETDEV_TX_OK; 1262 1263 err_out_drop: 1264 q->drop++; 1265 dev_kfree_skb(skb); 1266 return NETDEV_TX_OK; 1267 } 1268