1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 9 #include "ionic.h" 10 #include "ionic_lif.h" 11 #include "ionic_txrx.h" 12 13 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 14 ionic_desc_cb cb_func, void *cb_arg) 15 { 16 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 17 } 18 19 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 20 ionic_desc_cb cb_func, void *cb_arg) 21 { 22 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 23 } 24 25 static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 26 { 27 return netdev_get_tx_queue(q->lif->netdev, q->index); 28 } 29 30 static int ionic_rx_page_alloc(struct ionic_queue *q, 31 struct ionic_buf_info *buf_info) 32 { 33 struct net_device *netdev = q->lif->netdev; 34 struct ionic_rx_stats *stats; 35 struct device *dev; 36 struct page *page; 37 38 dev = q->dev; 39 stats = q_to_rx_stats(q); 40 41 if (unlikely(!buf_info)) { 42 net_err_ratelimited("%s: %s invalid buf_info in alloc\n", 43 netdev->name, q->name); 44 return -EINVAL; 45 } 46 47 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 48 if (unlikely(!page)) { 49 net_err_ratelimited("%s: %s page alloc failed\n", 50 netdev->name, q->name); 51 stats->alloc_err++; 52 return -ENOMEM; 53 } 54 55 buf_info->dma_addr = dma_map_page(dev, page, 0, 56 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 57 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { 58 __free_pages(page, 0); 59 net_err_ratelimited("%s: %s dma map failed\n", 60 netdev->name, q->name); 61 stats->dma_map_err++; 62 return -EIO; 63 } 64 65 buf_info->page = page; 66 buf_info->page_offset = 0; 67 68 return 0; 69 } 70 71 static void ionic_rx_page_free(struct ionic_queue *q, 72 struct ionic_buf_info *buf_info) 73 { 74 struct net_device *netdev = q->lif->netdev; 75 struct device *dev = q->dev; 76 77 if (unlikely(!buf_info)) { 78 net_err_ratelimited("%s: %s invalid buf_info in free\n", 79 netdev->name, q->name); 80 return; 81 } 82 83 if (!buf_info->page) 84 return; 85 86 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 87 __free_pages(buf_info->page, 0); 88 buf_info->page = NULL; 89 } 90 91 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 92 struct ionic_buf_info *buf_info, u32 used) 93 { 94 u32 size; 95 96 /* don't re-use pages allocated in low-mem condition */ 97 if (page_is_pfmemalloc(buf_info->page)) 98 return false; 99 100 /* don't re-use buffers from non-local numa nodes */ 101 if (page_to_nid(buf_info->page) != numa_mem_id()) 102 return false; 103 104 size = ALIGN(used, IONIC_PAGE_SPLIT_SZ); 105 buf_info->page_offset += size; 106 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 107 return false; 108 109 get_page(buf_info->page); 110 111 return true; 112 } 113 114 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 115 struct ionic_desc_info *desc_info, 116 struct ionic_rxq_comp *comp) 117 { 118 struct net_device *netdev = q->lif->netdev; 119 struct ionic_buf_info *buf_info; 120 struct ionic_rx_stats *stats; 121 struct device *dev = q->dev; 122 struct sk_buff *skb; 123 unsigned int i; 124 u16 frag_len; 125 u16 len; 126 127 stats = q_to_rx_stats(q); 128 129 buf_info = &desc_info->bufs[0]; 130 len = le16_to_cpu(comp->len); 131 132 prefetchw(buf_info->page); 133 134 skb = napi_get_frags(&q_to_qcq(q)->napi); 135 if (unlikely(!skb)) { 136 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 137 netdev->name, q->name); 138 stats->alloc_err++; 139 return NULL; 140 } 141 142 i = comp->num_sg_elems + 1; 143 do { 144 if (unlikely(!buf_info->page)) { 145 dev_kfree_skb(skb); 146 return NULL; 147 } 148 149 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 150 len -= frag_len; 151 152 dma_sync_single_for_cpu(dev, 153 buf_info->dma_addr + buf_info->page_offset, 154 frag_len, DMA_FROM_DEVICE); 155 156 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 157 buf_info->page, buf_info->page_offset, frag_len, 158 IONIC_PAGE_SIZE); 159 160 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 161 dma_unmap_page(dev, buf_info->dma_addr, 162 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 163 buf_info->page = NULL; 164 } 165 166 buf_info++; 167 168 i--; 169 } while (i > 0); 170 171 return skb; 172 } 173 174 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 175 struct ionic_desc_info *desc_info, 176 struct ionic_rxq_comp *comp) 177 { 178 struct net_device *netdev = q->lif->netdev; 179 struct ionic_buf_info *buf_info; 180 struct ionic_rx_stats *stats; 181 struct device *dev = q->dev; 182 struct sk_buff *skb; 183 u16 len; 184 185 stats = q_to_rx_stats(q); 186 187 buf_info = &desc_info->bufs[0]; 188 len = le16_to_cpu(comp->len); 189 190 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 191 if (unlikely(!skb)) { 192 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 193 netdev->name, q->name); 194 stats->alloc_err++; 195 return NULL; 196 } 197 198 if (unlikely(!buf_info->page)) { 199 dev_kfree_skb(skb); 200 return NULL; 201 } 202 203 dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset, 204 len, DMA_FROM_DEVICE); 205 skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len); 206 dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset, 207 len, DMA_FROM_DEVICE); 208 209 skb_put(skb, len); 210 skb->protocol = eth_type_trans(skb, q->lif->netdev); 211 212 return skb; 213 } 214 215 static void ionic_rx_clean(struct ionic_queue *q, 216 struct ionic_desc_info *desc_info, 217 struct ionic_cq_info *cq_info, 218 void *cb_arg) 219 { 220 struct net_device *netdev = q->lif->netdev; 221 struct ionic_qcq *qcq = q_to_qcq(q); 222 struct ionic_rx_stats *stats; 223 struct ionic_rxq_comp *comp; 224 struct sk_buff *skb; 225 226 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); 227 228 stats = q_to_rx_stats(q); 229 230 if (comp->status) { 231 stats->dropped++; 232 return; 233 } 234 235 stats->pkts++; 236 stats->bytes += le16_to_cpu(comp->len); 237 238 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 239 skb = ionic_rx_copybreak(q, desc_info, comp); 240 else 241 skb = ionic_rx_frags(q, desc_info, comp); 242 243 if (unlikely(!skb)) { 244 stats->dropped++; 245 return; 246 } 247 248 skb_record_rx_queue(skb, q->index); 249 250 if (likely(netdev->features & NETIF_F_RXHASH)) { 251 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 252 case IONIC_PKT_TYPE_IPV4: 253 case IONIC_PKT_TYPE_IPV6: 254 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 255 PKT_HASH_TYPE_L3); 256 break; 257 case IONIC_PKT_TYPE_IPV4_TCP: 258 case IONIC_PKT_TYPE_IPV6_TCP: 259 case IONIC_PKT_TYPE_IPV4_UDP: 260 case IONIC_PKT_TYPE_IPV6_UDP: 261 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 262 PKT_HASH_TYPE_L4); 263 break; 264 } 265 } 266 267 if (likely(netdev->features & NETIF_F_RXCSUM) && 268 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 269 skb->ip_summed = CHECKSUM_COMPLETE; 270 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 271 stats->csum_complete++; 272 } else { 273 stats->csum_none++; 274 } 275 276 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 277 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 278 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 279 stats->csum_error++; 280 281 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 282 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 283 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 284 le16_to_cpu(comp->vlan_tci)); 285 stats->vlan_stripped++; 286 } 287 288 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 289 __le64 *cq_desc_hwstamp; 290 u64 hwstamp; 291 292 cq_desc_hwstamp = 293 cq_info->cq_desc + 294 qcq->cq.desc_size - 295 sizeof(struct ionic_rxq_comp) - 296 IONIC_HWSTAMP_CQ_NEGOFFSET; 297 298 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 299 300 if (hwstamp != IONIC_HWSTAMP_INVALID) { 301 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 302 stats->hwstamp_valid++; 303 } else { 304 stats->hwstamp_invalid++; 305 } 306 } 307 308 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak) 309 napi_gro_receive(&qcq->napi, skb); 310 else 311 napi_gro_frags(&qcq->napi); 312 } 313 314 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 315 { 316 struct ionic_queue *q = cq->bound_q; 317 struct ionic_desc_info *desc_info; 318 struct ionic_rxq_comp *comp; 319 320 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 321 322 if (!color_match(comp->pkt_type_color, cq->done_color)) 323 return false; 324 325 /* check for empty queue */ 326 if (q->tail_idx == q->head_idx) 327 return false; 328 329 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 330 return false; 331 332 desc_info = &q->info[q->tail_idx]; 333 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 334 335 /* clean the related q entry, only one per qc completion */ 336 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 337 338 desc_info->cb = NULL; 339 desc_info->cb_arg = NULL; 340 341 return true; 342 } 343 344 void ionic_rx_fill(struct ionic_queue *q) 345 { 346 struct net_device *netdev = q->lif->netdev; 347 struct ionic_desc_info *desc_info; 348 struct ionic_rxq_sg_desc *sg_desc; 349 struct ionic_rxq_sg_elem *sg_elem; 350 struct ionic_buf_info *buf_info; 351 struct ionic_rxq_desc *desc; 352 unsigned int remain_len; 353 unsigned int frag_len; 354 unsigned int nfrags; 355 unsigned int i, j; 356 unsigned int len; 357 358 len = netdev->mtu + ETH_HLEN + VLAN_HLEN; 359 360 for (i = ionic_q_space_avail(q); i; i--) { 361 nfrags = 0; 362 remain_len = len; 363 desc_info = &q->info[q->head_idx]; 364 desc = desc_info->desc; 365 buf_info = &desc_info->bufs[0]; 366 367 if (!buf_info->page) { /* alloc a new buffer? */ 368 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 369 desc->addr = 0; 370 desc->len = 0; 371 return; 372 } 373 } 374 375 /* fill main descriptor - buf[0] */ 376 desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 377 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset); 378 desc->len = cpu_to_le16(frag_len); 379 remain_len -= frag_len; 380 buf_info++; 381 nfrags++; 382 383 /* fill sg descriptors - buf[1..n] */ 384 sg_desc = desc_info->sg_desc; 385 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { 386 sg_elem = &sg_desc->elems[j]; 387 if (!buf_info->page) { /* alloc a new sg buffer? */ 388 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 389 sg_elem->addr = 0; 390 sg_elem->len = 0; 391 return; 392 } 393 } 394 395 sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset); 396 frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset); 397 sg_elem->len = cpu_to_le16(frag_len); 398 remain_len -= frag_len; 399 buf_info++; 400 nfrags++; 401 } 402 403 /* clear end sg element as a sentinel */ 404 if (j < q->max_sg_elems) { 405 sg_elem = &sg_desc->elems[j]; 406 memset(sg_elem, 0, sizeof(*sg_elem)); 407 } 408 409 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 410 IONIC_RXQ_DESC_OPCODE_SIMPLE; 411 desc_info->nbufs = nfrags; 412 413 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 414 } 415 416 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 417 q->dbval | q->head_idx); 418 } 419 420 void ionic_rx_empty(struct ionic_queue *q) 421 { 422 struct ionic_desc_info *desc_info; 423 struct ionic_buf_info *buf_info; 424 unsigned int i, j; 425 426 for (i = 0; i < q->num_descs; i++) { 427 desc_info = &q->info[i]; 428 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 429 buf_info = &desc_info->bufs[j]; 430 if (buf_info->page) 431 ionic_rx_page_free(q, buf_info); 432 } 433 434 desc_info->nbufs = 0; 435 desc_info->cb = NULL; 436 desc_info->cb_arg = NULL; 437 } 438 439 q->head_idx = 0; 440 q->tail_idx = 0; 441 } 442 443 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 444 { 445 struct dim_sample dim_sample; 446 struct ionic_lif *lif; 447 unsigned int qi; 448 u64 pkts, bytes; 449 450 if (!qcq->intr.dim_coal_hw) 451 return; 452 453 lif = qcq->q.lif; 454 qi = qcq->cq.bound_q->index; 455 456 switch (napi_mode) { 457 case IONIC_LIF_F_TX_DIM_INTR: 458 pkts = lif->txqstats[qi].pkts; 459 bytes = lif->txqstats[qi].bytes; 460 break; 461 case IONIC_LIF_F_RX_DIM_INTR: 462 pkts = lif->rxqstats[qi].pkts; 463 bytes = lif->rxqstats[qi].bytes; 464 break; 465 default: 466 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 467 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 468 break; 469 } 470 471 dim_update_sample(qcq->cq.bound_intr->rearm_count, 472 pkts, bytes, &dim_sample); 473 474 net_dim(&qcq->dim, dim_sample); 475 } 476 477 int ionic_tx_napi(struct napi_struct *napi, int budget) 478 { 479 struct ionic_qcq *qcq = napi_to_qcq(napi); 480 struct ionic_cq *cq = napi_to_cq(napi); 481 struct ionic_dev *idev; 482 struct ionic_lif *lif; 483 u32 work_done = 0; 484 u32 flags = 0; 485 486 lif = cq->bound_q->lif; 487 idev = &lif->ionic->idev; 488 489 work_done = ionic_cq_service(cq, budget, 490 ionic_tx_service, NULL, NULL); 491 492 if (work_done < budget && napi_complete_done(napi, work_done)) { 493 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 494 flags |= IONIC_INTR_CRED_UNMASK; 495 cq->bound_intr->rearm_count++; 496 } 497 498 if (work_done || flags) { 499 flags |= IONIC_INTR_CRED_RESET_COALESCE; 500 ionic_intr_credits(idev->intr_ctrl, 501 cq->bound_intr->index, 502 work_done, flags); 503 } 504 505 return work_done; 506 } 507 508 int ionic_rx_napi(struct napi_struct *napi, int budget) 509 { 510 struct ionic_qcq *qcq = napi_to_qcq(napi); 511 struct ionic_cq *cq = napi_to_cq(napi); 512 struct ionic_dev *idev; 513 struct ionic_lif *lif; 514 u16 rx_fill_threshold; 515 u32 work_done = 0; 516 u32 flags = 0; 517 518 lif = cq->bound_q->lif; 519 idev = &lif->ionic->idev; 520 521 work_done = ionic_cq_service(cq, budget, 522 ionic_rx_service, NULL, NULL); 523 524 rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD, 525 cq->num_descs / IONIC_RX_FILL_DIV); 526 if (work_done && ionic_q_space_avail(cq->bound_q) >= rx_fill_threshold) 527 ionic_rx_fill(cq->bound_q); 528 529 if (work_done < budget && napi_complete_done(napi, work_done)) { 530 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 531 flags |= IONIC_INTR_CRED_UNMASK; 532 cq->bound_intr->rearm_count++; 533 } 534 535 if (work_done || flags) { 536 flags |= IONIC_INTR_CRED_RESET_COALESCE; 537 ionic_intr_credits(idev->intr_ctrl, 538 cq->bound_intr->index, 539 work_done, flags); 540 } 541 542 return work_done; 543 } 544 545 int ionic_txrx_napi(struct napi_struct *napi, int budget) 546 { 547 struct ionic_qcq *qcq = napi_to_qcq(napi); 548 struct ionic_cq *rxcq = napi_to_cq(napi); 549 unsigned int qi = rxcq->bound_q->index; 550 struct ionic_dev *idev; 551 struct ionic_lif *lif; 552 struct ionic_cq *txcq; 553 u16 rx_fill_threshold; 554 u32 rx_work_done = 0; 555 u32 tx_work_done = 0; 556 u32 flags = 0; 557 558 lif = rxcq->bound_q->lif; 559 idev = &lif->ionic->idev; 560 txcq = &lif->txqcqs[qi]->cq; 561 562 tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, 563 ionic_tx_service, NULL, NULL); 564 565 rx_work_done = ionic_cq_service(rxcq, budget, 566 ionic_rx_service, NULL, NULL); 567 568 rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD, 569 rxcq->num_descs / IONIC_RX_FILL_DIV); 570 if (rx_work_done && ionic_q_space_avail(rxcq->bound_q) >= rx_fill_threshold) 571 ionic_rx_fill(rxcq->bound_q); 572 573 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 574 ionic_dim_update(qcq, 0); 575 flags |= IONIC_INTR_CRED_UNMASK; 576 rxcq->bound_intr->rearm_count++; 577 } 578 579 if (rx_work_done || flags) { 580 flags |= IONIC_INTR_CRED_RESET_COALESCE; 581 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 582 tx_work_done + rx_work_done, flags); 583 } 584 585 return rx_work_done; 586 } 587 588 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 589 void *data, size_t len) 590 { 591 struct ionic_tx_stats *stats = q_to_tx_stats(q); 592 struct device *dev = q->dev; 593 dma_addr_t dma_addr; 594 595 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 596 if (dma_mapping_error(dev, dma_addr)) { 597 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 598 q->lif->netdev->name, q->name); 599 stats->dma_map_err++; 600 return 0; 601 } 602 return dma_addr; 603 } 604 605 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 606 const skb_frag_t *frag, 607 size_t offset, size_t len) 608 { 609 struct ionic_tx_stats *stats = q_to_tx_stats(q); 610 struct device *dev = q->dev; 611 dma_addr_t dma_addr; 612 613 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 614 if (dma_mapping_error(dev, dma_addr)) { 615 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 616 q->lif->netdev->name, q->name); 617 stats->dma_map_err++; 618 } 619 return dma_addr; 620 } 621 622 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 623 struct ionic_desc_info *desc_info) 624 { 625 struct ionic_buf_info *buf_info = desc_info->bufs; 626 struct ionic_tx_stats *stats = q_to_tx_stats(q); 627 struct device *dev = q->dev; 628 dma_addr_t dma_addr; 629 unsigned int nfrags; 630 skb_frag_t *frag; 631 int frag_idx; 632 633 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 634 if (dma_mapping_error(dev, dma_addr)) { 635 stats->dma_map_err++; 636 return -EIO; 637 } 638 buf_info->dma_addr = dma_addr; 639 buf_info->len = skb_headlen(skb); 640 buf_info++; 641 642 frag = skb_shinfo(skb)->frags; 643 nfrags = skb_shinfo(skb)->nr_frags; 644 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 645 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 646 if (dma_mapping_error(dev, dma_addr)) { 647 stats->dma_map_err++; 648 goto dma_fail; 649 } 650 buf_info->dma_addr = dma_addr; 651 buf_info->len = skb_frag_size(frag); 652 buf_info++; 653 } 654 655 desc_info->nbufs = 1 + nfrags; 656 657 return 0; 658 659 dma_fail: 660 /* unwind the frag mappings and the head mapping */ 661 while (frag_idx > 0) { 662 frag_idx--; 663 buf_info--; 664 dma_unmap_page(dev, buf_info->dma_addr, 665 buf_info->len, DMA_TO_DEVICE); 666 } 667 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); 668 return -EIO; 669 } 670 671 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 672 struct ionic_desc_info *desc_info) 673 { 674 struct ionic_buf_info *buf_info = desc_info->bufs; 675 struct device *dev = q->dev; 676 unsigned int i; 677 678 if (!desc_info->nbufs) 679 return; 680 681 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, 682 buf_info->len, DMA_TO_DEVICE); 683 buf_info++; 684 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 685 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, 686 buf_info->len, DMA_TO_DEVICE); 687 688 desc_info->nbufs = 0; 689 } 690 691 static void ionic_tx_clean(struct ionic_queue *q, 692 struct ionic_desc_info *desc_info, 693 struct ionic_cq_info *cq_info, 694 void *cb_arg) 695 { 696 struct ionic_tx_stats *stats = q_to_tx_stats(q); 697 struct ionic_qcq *qcq = q_to_qcq(q); 698 struct sk_buff *skb = cb_arg; 699 u16 qi; 700 701 ionic_tx_desc_unmap_bufs(q, desc_info); 702 703 if (!skb) 704 return; 705 706 qi = skb_get_queue_mapping(skb); 707 708 if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) { 709 if (cq_info) { 710 struct skb_shared_hwtstamps hwts = {}; 711 __le64 *cq_desc_hwstamp; 712 u64 hwstamp; 713 714 cq_desc_hwstamp = 715 cq_info->cq_desc + 716 qcq->cq.desc_size - 717 sizeof(struct ionic_txq_comp) - 718 IONIC_HWSTAMP_CQ_NEGOFFSET; 719 720 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 721 722 if (hwstamp != IONIC_HWSTAMP_INVALID) { 723 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 724 725 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 726 skb_tstamp_tx(skb, &hwts); 727 728 stats->hwstamp_valid++; 729 } else { 730 stats->hwstamp_invalid++; 731 } 732 } 733 734 } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) { 735 netif_wake_subqueue(q->lif->netdev, qi); 736 } 737 738 desc_info->bytes = skb->len; 739 stats->clean++; 740 741 dev_consume_skb_any(skb); 742 } 743 744 bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 745 { 746 struct ionic_queue *q = cq->bound_q; 747 struct ionic_desc_info *desc_info; 748 struct ionic_txq_comp *comp; 749 int bytes = 0; 750 int pkts = 0; 751 u16 index; 752 753 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 754 755 if (!color_match(comp->color, cq->done_color)) 756 return false; 757 758 /* clean the related q entries, there could be 759 * several q entries completed for each cq completion 760 */ 761 do { 762 desc_info = &q->info[q->tail_idx]; 763 desc_info->bytes = 0; 764 index = q->tail_idx; 765 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 766 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 767 if (desc_info->cb_arg) { 768 pkts++; 769 bytes += desc_info->bytes; 770 } 771 desc_info->cb = NULL; 772 desc_info->cb_arg = NULL; 773 } while (index != le16_to_cpu(comp->comp_index)); 774 775 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 776 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 777 778 return true; 779 } 780 781 void ionic_tx_flush(struct ionic_cq *cq) 782 { 783 struct ionic_dev *idev = &cq->lif->ionic->idev; 784 u32 work_done; 785 786 work_done = ionic_cq_service(cq, cq->num_descs, 787 ionic_tx_service, NULL, NULL); 788 if (work_done) 789 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 790 work_done, IONIC_INTR_CRED_RESET_COALESCE); 791 } 792 793 void ionic_tx_empty(struct ionic_queue *q) 794 { 795 struct ionic_desc_info *desc_info; 796 int bytes = 0; 797 int pkts = 0; 798 799 /* walk the not completed tx entries, if any */ 800 while (q->head_idx != q->tail_idx) { 801 desc_info = &q->info[q->tail_idx]; 802 desc_info->bytes = 0; 803 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 804 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 805 if (desc_info->cb_arg) { 806 pkts++; 807 bytes += desc_info->bytes; 808 } 809 desc_info->cb = NULL; 810 desc_info->cb_arg = NULL; 811 } 812 813 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 814 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 815 } 816 817 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 818 { 819 int err; 820 821 err = skb_cow_head(skb, 0); 822 if (err) 823 return err; 824 825 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 826 inner_ip_hdr(skb)->check = 0; 827 inner_tcp_hdr(skb)->check = 828 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 829 inner_ip_hdr(skb)->daddr, 830 0, IPPROTO_TCP, 0); 831 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 832 inner_tcp_hdr(skb)->check = 833 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 834 &inner_ipv6_hdr(skb)->daddr, 835 0, IPPROTO_TCP, 0); 836 } 837 838 return 0; 839 } 840 841 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 842 { 843 int err; 844 845 err = skb_cow_head(skb, 0); 846 if (err) 847 return err; 848 849 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 850 ip_hdr(skb)->check = 0; 851 tcp_hdr(skb)->check = 852 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 853 ip_hdr(skb)->daddr, 854 0, IPPROTO_TCP, 0); 855 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 856 tcp_v6_gso_csum_prep(skb); 857 } 858 859 return 0; 860 } 861 862 static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 863 struct sk_buff *skb, 864 dma_addr_t addr, u8 nsge, u16 len, 865 unsigned int hdrlen, unsigned int mss, 866 bool outer_csum, 867 u16 vlan_tci, bool has_vlan, 868 bool start, bool done) 869 { 870 u8 flags = 0; 871 u64 cmd; 872 873 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 874 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 875 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 876 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 877 878 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 879 desc->cmd = cpu_to_le64(cmd); 880 desc->len = cpu_to_le16(len); 881 desc->vlan_tci = cpu_to_le16(vlan_tci); 882 desc->hdr_len = cpu_to_le16(hdrlen); 883 desc->mss = cpu_to_le16(mss); 884 885 if (start) { 886 skb_tx_timestamp(skb); 887 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 888 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 889 ionic_txq_post(q, false, ionic_tx_clean, skb); 890 } else { 891 ionic_txq_post(q, done, NULL, NULL); 892 } 893 } 894 895 static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 896 { 897 struct ionic_tx_stats *stats = q_to_tx_stats(q); 898 struct ionic_desc_info *desc_info; 899 struct ionic_buf_info *buf_info; 900 struct ionic_txq_sg_elem *elem; 901 struct ionic_txq_desc *desc; 902 unsigned int chunk_len; 903 unsigned int frag_rem; 904 unsigned int tso_rem; 905 unsigned int seg_rem; 906 dma_addr_t desc_addr; 907 dma_addr_t frag_addr; 908 unsigned int hdrlen; 909 unsigned int len; 910 unsigned int mss; 911 bool start, done; 912 bool outer_csum; 913 bool has_vlan; 914 u16 desc_len; 915 u8 desc_nsge; 916 u16 vlan_tci; 917 bool encap; 918 int err; 919 920 desc_info = &q->info[q->head_idx]; 921 buf_info = desc_info->bufs; 922 923 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 924 return -EIO; 925 926 len = skb->len; 927 mss = skb_shinfo(skb)->gso_size; 928 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) || 929 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); 930 has_vlan = !!skb_vlan_tag_present(skb); 931 vlan_tci = skb_vlan_tag_get(skb); 932 encap = skb->encapsulation; 933 934 /* Preload inner-most TCP csum field with IP pseudo hdr 935 * calculated with IP length set to zero. HW will later 936 * add in length to each TCP segment resulting from the TSO. 937 */ 938 939 if (encap) 940 err = ionic_tx_tcp_inner_pseudo_csum(skb); 941 else 942 err = ionic_tx_tcp_pseudo_csum(skb); 943 if (err) { 944 /* clean up mapping from ionic_tx_map_skb */ 945 ionic_tx_desc_unmap_bufs(q, desc_info); 946 return err; 947 } 948 949 if (encap) 950 hdrlen = skb_inner_tcp_all_headers(skb); 951 else 952 hdrlen = skb_tcp_all_headers(skb); 953 954 tso_rem = len; 955 seg_rem = min(tso_rem, hdrlen + mss); 956 957 frag_addr = 0; 958 frag_rem = 0; 959 960 start = true; 961 962 while (tso_rem > 0) { 963 desc = NULL; 964 elem = NULL; 965 desc_addr = 0; 966 desc_len = 0; 967 desc_nsge = 0; 968 /* use fragments until we have enough to post a single descriptor */ 969 while (seg_rem > 0) { 970 /* if the fragment is exhausted then move to the next one */ 971 if (frag_rem == 0) { 972 /* grab the next fragment */ 973 frag_addr = buf_info->dma_addr; 974 frag_rem = buf_info->len; 975 buf_info++; 976 } 977 chunk_len = min(frag_rem, seg_rem); 978 if (!desc) { 979 /* fill main descriptor */ 980 desc = desc_info->txq_desc; 981 elem = desc_info->txq_sg_desc->elems; 982 desc_addr = frag_addr; 983 desc_len = chunk_len; 984 } else { 985 /* fill sg descriptor */ 986 elem->addr = cpu_to_le64(frag_addr); 987 elem->len = cpu_to_le16(chunk_len); 988 elem++; 989 desc_nsge++; 990 } 991 frag_addr += chunk_len; 992 frag_rem -= chunk_len; 993 tso_rem -= chunk_len; 994 seg_rem -= chunk_len; 995 } 996 seg_rem = min(tso_rem, mss); 997 done = (tso_rem == 0); 998 /* post descriptor */ 999 ionic_tx_tso_post(q, desc, skb, 1000 desc_addr, desc_nsge, desc_len, 1001 hdrlen, mss, outer_csum, vlan_tci, has_vlan, 1002 start, done); 1003 start = false; 1004 /* Buffer information is stored with the first tso descriptor */ 1005 desc_info = &q->info[q->head_idx]; 1006 desc_info->nbufs = 0; 1007 } 1008 1009 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1010 stats->bytes += len; 1011 stats->tso++; 1012 stats->tso_bytes = len; 1013 1014 return 0; 1015 } 1016 1017 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1018 struct ionic_desc_info *desc_info) 1019 { 1020 struct ionic_txq_desc *desc = desc_info->txq_desc; 1021 struct ionic_buf_info *buf_info = desc_info->bufs; 1022 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1023 bool has_vlan; 1024 u8 flags = 0; 1025 bool encap; 1026 u64 cmd; 1027 1028 has_vlan = !!skb_vlan_tag_present(skb); 1029 encap = skb->encapsulation; 1030 1031 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1032 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1033 1034 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1035 flags, skb_shinfo(skb)->nr_frags, 1036 buf_info->dma_addr); 1037 desc->cmd = cpu_to_le64(cmd); 1038 desc->len = cpu_to_le16(buf_info->len); 1039 if (has_vlan) { 1040 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1041 stats->vlan_inserted++; 1042 } else { 1043 desc->vlan_tci = 0; 1044 } 1045 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1046 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1047 1048 if (skb_csum_is_sctp(skb)) 1049 stats->crc32_csum++; 1050 else 1051 stats->csum++; 1052 } 1053 1054 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1055 struct ionic_desc_info *desc_info) 1056 { 1057 struct ionic_txq_desc *desc = desc_info->txq_desc; 1058 struct ionic_buf_info *buf_info = desc_info->bufs; 1059 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1060 bool has_vlan; 1061 u8 flags = 0; 1062 bool encap; 1063 u64 cmd; 1064 1065 has_vlan = !!skb_vlan_tag_present(skb); 1066 encap = skb->encapsulation; 1067 1068 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1069 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1070 1071 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1072 flags, skb_shinfo(skb)->nr_frags, 1073 buf_info->dma_addr); 1074 desc->cmd = cpu_to_le64(cmd); 1075 desc->len = cpu_to_le16(buf_info->len); 1076 if (has_vlan) { 1077 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1078 stats->vlan_inserted++; 1079 } else { 1080 desc->vlan_tci = 0; 1081 } 1082 desc->csum_start = 0; 1083 desc->csum_offset = 0; 1084 1085 stats->csum_none++; 1086 } 1087 1088 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1089 struct ionic_desc_info *desc_info) 1090 { 1091 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; 1092 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1093 struct ionic_txq_sg_elem *elem = sg_desc->elems; 1094 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1095 unsigned int i; 1096 1097 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1098 elem->addr = cpu_to_le64(buf_info->dma_addr); 1099 elem->len = cpu_to_le16(buf_info->len); 1100 } 1101 1102 stats->frags += skb_shinfo(skb)->nr_frags; 1103 } 1104 1105 static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 1106 { 1107 struct ionic_desc_info *desc_info = &q->info[q->head_idx]; 1108 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1109 1110 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1111 return -EIO; 1112 1113 /* set up the initial descriptor */ 1114 if (skb->ip_summed == CHECKSUM_PARTIAL) 1115 ionic_tx_calc_csum(q, skb, desc_info); 1116 else 1117 ionic_tx_calc_no_csum(q, skb, desc_info); 1118 1119 /* add frags */ 1120 ionic_tx_skb_frags(q, skb, desc_info); 1121 1122 skb_tx_timestamp(skb); 1123 stats->pkts++; 1124 stats->bytes += skb->len; 1125 1126 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) 1127 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 1128 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 1129 1130 return 0; 1131 } 1132 1133 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1134 { 1135 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1136 int ndescs; 1137 int err; 1138 1139 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1140 if (skb_is_gso(skb)) 1141 ndescs = skb_shinfo(skb)->gso_segs; 1142 else 1143 ndescs = 1; 1144 1145 /* If non-TSO, just need 1 desc and nr_frags sg elems */ 1146 if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems) 1147 return ndescs; 1148 1149 /* Too many frags, so linearize */ 1150 err = skb_linearize(skb); 1151 if (err) 1152 return err; 1153 1154 stats->linearize++; 1155 1156 return ndescs; 1157 } 1158 1159 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 1160 { 1161 int stopped = 0; 1162 1163 if (unlikely(!ionic_q_has_space(q, ndescs))) { 1164 netif_stop_subqueue(q->lif->netdev, q->index); 1165 stopped = 1; 1166 1167 /* Might race with ionic_tx_clean, check again */ 1168 smp_rmb(); 1169 if (ionic_q_has_space(q, ndescs)) { 1170 netif_wake_subqueue(q->lif->netdev, q->index); 1171 stopped = 0; 1172 } 1173 } 1174 1175 return stopped; 1176 } 1177 1178 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1179 struct net_device *netdev) 1180 { 1181 struct ionic_lif *lif = netdev_priv(netdev); 1182 struct ionic_queue *q = &lif->hwstamp_txq->q; 1183 int err, ndescs; 1184 1185 /* Does not stop/start txq, because we post to a separate tx queue 1186 * for timestamping, and if a packet can't be posted immediately to 1187 * the timestamping queue, it is dropped. 1188 */ 1189 1190 ndescs = ionic_tx_descs_needed(q, skb); 1191 if (unlikely(ndescs < 0)) 1192 goto err_out_drop; 1193 1194 if (unlikely(!ionic_q_has_space(q, ndescs))) 1195 goto err_out_drop; 1196 1197 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1198 if (skb_is_gso(skb)) 1199 err = ionic_tx_tso(q, skb); 1200 else 1201 err = ionic_tx(q, skb); 1202 1203 if (err) 1204 goto err_out_drop; 1205 1206 return NETDEV_TX_OK; 1207 1208 err_out_drop: 1209 q->drop++; 1210 dev_kfree_skb(skb); 1211 return NETDEV_TX_OK; 1212 } 1213 1214 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1215 { 1216 u16 queue_index = skb_get_queue_mapping(skb); 1217 struct ionic_lif *lif = netdev_priv(netdev); 1218 struct ionic_queue *q; 1219 int ndescs; 1220 int err; 1221 1222 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1223 dev_kfree_skb(skb); 1224 return NETDEV_TX_OK; 1225 } 1226 1227 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1228 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1229 return ionic_start_hwstamp_xmit(skb, netdev); 1230 1231 if (unlikely(queue_index >= lif->nxqs)) 1232 queue_index = 0; 1233 q = &lif->txqcqs[queue_index]->q; 1234 1235 ndescs = ionic_tx_descs_needed(q, skb); 1236 if (ndescs < 0) 1237 goto err_out_drop; 1238 1239 if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 1240 return NETDEV_TX_BUSY; 1241 1242 if (skb_is_gso(skb)) 1243 err = ionic_tx_tso(q, skb); 1244 else 1245 err = ionic_tx(q, skb); 1246 1247 if (err) 1248 goto err_out_drop; 1249 1250 /* Stop the queue if there aren't descriptors for the next packet. 1251 * Since our SG lists per descriptor take care of most of the possible 1252 * fragmentation, we don't need to have many descriptors available. 1253 */ 1254 ionic_maybe_stop_tx(q, 4); 1255 1256 return NETDEV_TX_OK; 1257 1258 err_out_drop: 1259 q->drop++; 1260 dev_kfree_skb(skb); 1261 return NETDEV_TX_OK; 1262 } 1263