1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/of_mdio.h> 8 #include <linux/vmalloc.h> 9 10 /* ENETC overhead: optional extension BD + 1 BD gap */ 11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 12 /* max # of chained Tx BDs is 15, including head and extension BD */ 13 #define ENETC_MAX_SKB_FRAGS 13 14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 15 16 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 17 int active_offloads); 18 19 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 20 { 21 struct enetc_ndev_priv *priv = netdev_priv(ndev); 22 struct enetc_bdr *tx_ring; 23 int count; 24 25 tx_ring = priv->tx_ring[skb->queue_mapping]; 26 27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 28 if (unlikely(skb_linearize(skb))) 29 goto drop_packet_err; 30 31 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 33 netif_stop_subqueue(ndev, tx_ring->index); 34 return NETDEV_TX_BUSY; 35 } 36 37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 38 if (unlikely(!count)) 39 goto drop_packet_err; 40 41 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 42 netif_stop_subqueue(ndev, tx_ring->index); 43 44 return NETDEV_TX_OK; 45 46 drop_packet_err: 47 dev_kfree_skb_any(skb); 48 return NETDEV_TX_OK; 49 } 50 51 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd) 52 { 53 int l3_start, l3_hsize; 54 u16 l3_flags, l4_flags; 55 56 if (skb->ip_summed != CHECKSUM_PARTIAL) 57 return false; 58 59 switch (skb->csum_offset) { 60 case offsetof(struct tcphdr, check): 61 l4_flags = ENETC_TXBD_L4_TCP; 62 break; 63 case offsetof(struct udphdr, check): 64 l4_flags = ENETC_TXBD_L4_UDP; 65 break; 66 default: 67 skb_checksum_help(skb); 68 return false; 69 } 70 71 l3_start = skb_network_offset(skb); 72 l3_hsize = skb_network_header_len(skb); 73 74 l3_flags = 0; 75 if (skb->protocol == htons(ETH_P_IPV6)) 76 l3_flags = ENETC_TXBD_L3_IPV6; 77 78 /* write BD fields */ 79 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags); 80 txbd->l4_csoff = l4_flags; 81 82 return true; 83 } 84 85 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 86 struct enetc_tx_swbd *tx_swbd) 87 { 88 if (tx_swbd->is_dma_page) 89 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 90 tx_swbd->len, DMA_TO_DEVICE); 91 else 92 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 93 tx_swbd->len, DMA_TO_DEVICE); 94 tx_swbd->dma = 0; 95 } 96 97 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 98 struct enetc_tx_swbd *tx_swbd) 99 { 100 if (tx_swbd->dma) 101 enetc_unmap_tx_buff(tx_ring, tx_swbd); 102 103 if (tx_swbd->skb) { 104 dev_kfree_skb_any(tx_swbd->skb); 105 tx_swbd->skb = NULL; 106 } 107 } 108 109 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 110 int active_offloads) 111 { 112 struct enetc_tx_swbd *tx_swbd; 113 struct skb_frag_struct *frag; 114 int len = skb_headlen(skb); 115 union enetc_tx_bd temp_bd; 116 union enetc_tx_bd *txbd; 117 bool do_vlan, do_tstamp; 118 int i, count = 0; 119 unsigned int f; 120 dma_addr_t dma; 121 u8 flags = 0; 122 123 i = tx_ring->next_to_use; 124 txbd = ENETC_TXBD(*tx_ring, i); 125 prefetchw(txbd); 126 127 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 128 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 129 goto dma_err; 130 131 temp_bd.addr = cpu_to_le64(dma); 132 temp_bd.buf_len = cpu_to_le16(len); 133 temp_bd.lstatus = 0; 134 135 tx_swbd = &tx_ring->tx_swbd[i]; 136 tx_swbd->dma = dma; 137 tx_swbd->len = len; 138 tx_swbd->is_dma_page = 0; 139 count++; 140 141 do_vlan = skb_vlan_tag_present(skb); 142 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 143 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 144 tx_swbd->do_tstamp = do_tstamp; 145 tx_swbd->check_wb = tx_swbd->do_tstamp; 146 147 if (do_vlan || do_tstamp) 148 flags |= ENETC_TXBD_FLAGS_EX; 149 150 if (enetc_tx_csum(skb, &temp_bd)) 151 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS; 152 153 /* first BD needs frm_len and offload flags set */ 154 temp_bd.frm_len = cpu_to_le16(skb->len); 155 temp_bd.flags = flags; 156 157 if (flags & ENETC_TXBD_FLAGS_EX) { 158 u8 e_flags = 0; 159 *txbd = temp_bd; 160 enetc_clear_tx_bd(&temp_bd); 161 162 /* add extension BD for VLAN and/or timestamping */ 163 flags = 0; 164 tx_swbd++; 165 txbd++; 166 i++; 167 if (unlikely(i == tx_ring->bd_count)) { 168 i = 0; 169 tx_swbd = tx_ring->tx_swbd; 170 txbd = ENETC_TXBD(*tx_ring, 0); 171 } 172 prefetchw(txbd); 173 174 if (do_vlan) { 175 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 176 temp_bd.ext.tpid = 0; /* < C-TAG */ 177 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 178 } 179 180 if (do_tstamp) { 181 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 182 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 183 } 184 185 temp_bd.ext.e_flags = e_flags; 186 count++; 187 } 188 189 frag = &skb_shinfo(skb)->frags[0]; 190 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 191 len = skb_frag_size(frag); 192 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 193 DMA_TO_DEVICE); 194 if (dma_mapping_error(tx_ring->dev, dma)) 195 goto dma_err; 196 197 *txbd = temp_bd; 198 enetc_clear_tx_bd(&temp_bd); 199 200 flags = 0; 201 tx_swbd++; 202 txbd++; 203 i++; 204 if (unlikely(i == tx_ring->bd_count)) { 205 i = 0; 206 tx_swbd = tx_ring->tx_swbd; 207 txbd = ENETC_TXBD(*tx_ring, 0); 208 } 209 prefetchw(txbd); 210 211 temp_bd.addr = cpu_to_le64(dma); 212 temp_bd.buf_len = cpu_to_le16(len); 213 214 tx_swbd->dma = dma; 215 tx_swbd->len = len; 216 tx_swbd->is_dma_page = 1; 217 count++; 218 } 219 220 /* last BD needs 'F' bit set */ 221 flags |= ENETC_TXBD_FLAGS_F; 222 temp_bd.flags = flags; 223 *txbd = temp_bd; 224 225 tx_ring->tx_swbd[i].skb = skb; 226 227 enetc_bdr_idx_inc(tx_ring, &i); 228 tx_ring->next_to_use = i; 229 230 /* let H/W know BD ring has been updated */ 231 enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */ 232 233 return count; 234 235 dma_err: 236 dev_err(tx_ring->dev, "DMA map error"); 237 238 do { 239 tx_swbd = &tx_ring->tx_swbd[i]; 240 enetc_free_tx_skb(tx_ring, tx_swbd); 241 if (i == 0) 242 i = tx_ring->bd_count; 243 i--; 244 } while (count--); 245 246 return 0; 247 } 248 249 static irqreturn_t enetc_msix(int irq, void *data) 250 { 251 struct enetc_int_vector *v = data; 252 int i; 253 254 /* disable interrupts */ 255 enetc_wr_reg(v->rbier, 0); 256 257 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings) 258 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0); 259 260 napi_schedule_irqoff(&v->napi); 261 262 return IRQ_HANDLED; 263 } 264 265 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 266 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 267 struct napi_struct *napi, int work_limit); 268 269 static int enetc_poll(struct napi_struct *napi, int budget) 270 { 271 struct enetc_int_vector 272 *v = container_of(napi, struct enetc_int_vector, napi); 273 bool complete = true; 274 int work_done; 275 int i; 276 277 for (i = 0; i < v->count_tx_rings; i++) 278 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 279 complete = false; 280 281 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 282 if (work_done == budget) 283 complete = false; 284 285 if (!complete) 286 return budget; 287 288 napi_complete_done(napi, work_done); 289 290 /* enable interrupts */ 291 enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE); 292 293 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings) 294 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 295 ENETC_TBIER_TXTIE); 296 297 return work_done; 298 } 299 300 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 301 { 302 int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 303 304 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 305 } 306 307 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 308 u64 *tstamp) 309 { 310 u32 lo, hi, tstamp_lo; 311 312 lo = enetc_rd(hw, ENETC_SICTR0); 313 hi = enetc_rd(hw, ENETC_SICTR1); 314 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 315 if (lo <= tstamp_lo) 316 hi -= 1; 317 *tstamp = (u64)hi << 32 | tstamp_lo; 318 } 319 320 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 321 { 322 struct skb_shared_hwtstamps shhwtstamps; 323 324 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 325 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 326 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 327 skb_tstamp_tx(skb, &shhwtstamps); 328 } 329 } 330 331 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 332 { 333 struct net_device *ndev = tx_ring->ndev; 334 int tx_frm_cnt = 0, tx_byte_cnt = 0; 335 struct enetc_tx_swbd *tx_swbd; 336 int i, bds_to_clean; 337 bool do_tstamp; 338 u64 tstamp = 0; 339 340 i = tx_ring->next_to_clean; 341 tx_swbd = &tx_ring->tx_swbd[i]; 342 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 343 344 do_tstamp = false; 345 346 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 347 bool is_eof = !!tx_swbd->skb; 348 349 if (unlikely(tx_swbd->check_wb)) { 350 struct enetc_ndev_priv *priv = netdev_priv(ndev); 351 union enetc_tx_bd *txbd; 352 353 txbd = ENETC_TXBD(*tx_ring, i); 354 355 if (txbd->flags & ENETC_TXBD_FLAGS_W && 356 tx_swbd->do_tstamp) { 357 enetc_get_tx_tstamp(&priv->si->hw, txbd, 358 &tstamp); 359 do_tstamp = true; 360 } 361 } 362 363 if (likely(tx_swbd->dma)) 364 enetc_unmap_tx_buff(tx_ring, tx_swbd); 365 366 if (is_eof) { 367 if (unlikely(do_tstamp)) { 368 enetc_tstamp_tx(tx_swbd->skb, tstamp); 369 do_tstamp = false; 370 } 371 napi_consume_skb(tx_swbd->skb, napi_budget); 372 tx_swbd->skb = NULL; 373 } 374 375 tx_byte_cnt += tx_swbd->len; 376 377 bds_to_clean--; 378 tx_swbd++; 379 i++; 380 if (unlikely(i == tx_ring->bd_count)) { 381 i = 0; 382 tx_swbd = tx_ring->tx_swbd; 383 } 384 385 /* BD iteration loop end */ 386 if (is_eof) { 387 tx_frm_cnt++; 388 /* re-arm interrupt source */ 389 enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) | 390 BIT(16 + tx_ring->index)); 391 } 392 393 if (unlikely(!bds_to_clean)) 394 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 395 } 396 397 tx_ring->next_to_clean = i; 398 tx_ring->stats.packets += tx_frm_cnt; 399 tx_ring->stats.bytes += tx_byte_cnt; 400 401 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 402 __netif_subqueue_stopped(ndev, tx_ring->index) && 403 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 404 netif_wake_subqueue(ndev, tx_ring->index); 405 } 406 407 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 408 } 409 410 static bool enetc_new_page(struct enetc_bdr *rx_ring, 411 struct enetc_rx_swbd *rx_swbd) 412 { 413 struct page *page; 414 dma_addr_t addr; 415 416 page = dev_alloc_page(); 417 if (unlikely(!page)) 418 return false; 419 420 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 421 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 422 __free_page(page); 423 424 return false; 425 } 426 427 rx_swbd->dma = addr; 428 rx_swbd->page = page; 429 rx_swbd->page_offset = ENETC_RXB_PAD; 430 431 return true; 432 } 433 434 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 435 { 436 struct enetc_rx_swbd *rx_swbd; 437 union enetc_rx_bd *rxbd; 438 int i, j; 439 440 i = rx_ring->next_to_use; 441 rx_swbd = &rx_ring->rx_swbd[i]; 442 rxbd = ENETC_RXBD(*rx_ring, i); 443 444 for (j = 0; j < buff_cnt; j++) { 445 /* try reuse page */ 446 if (unlikely(!rx_swbd->page)) { 447 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 448 rx_ring->stats.rx_alloc_errs++; 449 break; 450 } 451 } 452 453 /* update RxBD */ 454 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 455 rx_swbd->page_offset); 456 /* clear 'R" as well */ 457 rxbd->r.lstatus = 0; 458 459 rx_swbd++; 460 rxbd++; 461 i++; 462 if (unlikely(i == rx_ring->bd_count)) { 463 i = 0; 464 rx_swbd = rx_ring->rx_swbd; 465 rxbd = ENETC_RXBD(*rx_ring, 0); 466 } 467 } 468 469 if (likely(j)) { 470 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 471 rx_ring->next_to_use = i; 472 /* update ENETC's consumer index */ 473 enetc_wr_reg(rx_ring->rcir, i); 474 } 475 476 return j; 477 } 478 479 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 480 static void enetc_get_rx_tstamp(struct net_device *ndev, 481 union enetc_rx_bd *rxbd, 482 struct sk_buff *skb) 483 { 484 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 485 struct enetc_ndev_priv *priv = netdev_priv(ndev); 486 struct enetc_hw *hw = &priv->si->hw; 487 u32 lo, hi, tstamp_lo; 488 u64 tstamp; 489 490 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 491 lo = enetc_rd(hw, ENETC_SICTR0); 492 hi = enetc_rd(hw, ENETC_SICTR1); 493 tstamp_lo = le32_to_cpu(rxbd->r.tstamp); 494 if (lo <= tstamp_lo) 495 hi -= 1; 496 497 tstamp = (u64)hi << 32 | tstamp_lo; 498 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 499 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 500 } 501 } 502 #endif 503 504 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 505 union enetc_rx_bd *rxbd, struct sk_buff *skb) 506 { 507 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 508 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 509 #endif 510 /* TODO: hashing */ 511 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 512 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 513 514 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 515 skb->ip_summed = CHECKSUM_COMPLETE; 516 } 517 518 /* copy VLAN to skb, if one is extracted, for now we assume it's a 519 * standard TPID, but HW also supports custom values 520 */ 521 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) 522 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 523 le16_to_cpu(rxbd->r.vlan_opt)); 524 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 525 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 526 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 527 #endif 528 } 529 530 static void enetc_process_skb(struct enetc_bdr *rx_ring, 531 struct sk_buff *skb) 532 { 533 skb_record_rx_queue(skb, rx_ring->index); 534 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 535 } 536 537 static bool enetc_page_reusable(struct page *page) 538 { 539 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 540 } 541 542 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 543 struct enetc_rx_swbd *old) 544 { 545 struct enetc_rx_swbd *new; 546 547 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 548 549 /* next buf that may reuse a page */ 550 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 551 552 /* copy page reference */ 553 *new = *old; 554 } 555 556 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 557 int i, u16 size) 558 { 559 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 560 561 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 562 rx_swbd->page_offset, 563 size, DMA_FROM_DEVICE); 564 return rx_swbd; 565 } 566 567 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 568 struct enetc_rx_swbd *rx_swbd) 569 { 570 if (likely(enetc_page_reusable(rx_swbd->page))) { 571 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 572 page_ref_inc(rx_swbd->page); 573 574 enetc_reuse_page(rx_ring, rx_swbd); 575 576 /* sync for use by the device */ 577 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 578 rx_swbd->page_offset, 579 ENETC_RXB_DMA_SIZE, 580 DMA_FROM_DEVICE); 581 } else { 582 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 583 PAGE_SIZE, DMA_FROM_DEVICE); 584 } 585 586 rx_swbd->page = NULL; 587 } 588 589 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 590 int i, u16 size) 591 { 592 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 593 struct sk_buff *skb; 594 void *ba; 595 596 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 597 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 598 if (unlikely(!skb)) { 599 rx_ring->stats.rx_alloc_errs++; 600 return NULL; 601 } 602 603 skb_reserve(skb, ENETC_RXB_PAD); 604 __skb_put(skb, size); 605 606 enetc_put_rx_buff(rx_ring, rx_swbd); 607 608 return skb; 609 } 610 611 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 612 u16 size, struct sk_buff *skb) 613 { 614 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 615 616 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 617 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 618 619 enetc_put_rx_buff(rx_ring, rx_swbd); 620 } 621 622 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 623 624 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 625 struct napi_struct *napi, int work_limit) 626 { 627 int rx_frm_cnt = 0, rx_byte_cnt = 0; 628 int cleaned_cnt, i; 629 630 cleaned_cnt = enetc_bd_unused(rx_ring); 631 /* next descriptor to process */ 632 i = rx_ring->next_to_clean; 633 634 while (likely(rx_frm_cnt < work_limit)) { 635 union enetc_rx_bd *rxbd; 636 struct sk_buff *skb; 637 u32 bd_status; 638 u16 size; 639 640 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 641 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 642 643 cleaned_cnt -= count; 644 } 645 646 rxbd = ENETC_RXBD(*rx_ring, i); 647 bd_status = le32_to_cpu(rxbd->r.lstatus); 648 if (!bd_status) 649 break; 650 651 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index)); 652 dma_rmb(); /* for reading other rxbd fields */ 653 size = le16_to_cpu(rxbd->r.buf_len); 654 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 655 if (!skb) 656 break; 657 658 enetc_get_offloads(rx_ring, rxbd, skb); 659 660 cleaned_cnt++; 661 rxbd++; 662 i++; 663 if (unlikely(i == rx_ring->bd_count)) { 664 i = 0; 665 rxbd = ENETC_RXBD(*rx_ring, 0); 666 } 667 668 if (unlikely(bd_status & 669 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 670 dev_kfree_skb(skb); 671 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 672 dma_rmb(); 673 bd_status = le32_to_cpu(rxbd->r.lstatus); 674 rxbd++; 675 i++; 676 if (unlikely(i == rx_ring->bd_count)) { 677 i = 0; 678 rxbd = ENETC_RXBD(*rx_ring, 0); 679 } 680 } 681 682 rx_ring->ndev->stats.rx_dropped++; 683 rx_ring->ndev->stats.rx_errors++; 684 685 break; 686 } 687 688 /* not last BD in frame? */ 689 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 690 bd_status = le32_to_cpu(rxbd->r.lstatus); 691 size = ENETC_RXB_DMA_SIZE; 692 693 if (bd_status & ENETC_RXBD_LSTATUS_F) { 694 dma_rmb(); 695 size = le16_to_cpu(rxbd->r.buf_len); 696 } 697 698 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 699 700 cleaned_cnt++; 701 rxbd++; 702 i++; 703 if (unlikely(i == rx_ring->bd_count)) { 704 i = 0; 705 rxbd = ENETC_RXBD(*rx_ring, 0); 706 } 707 } 708 709 rx_byte_cnt += skb->len; 710 711 enetc_process_skb(rx_ring, skb); 712 713 napi_gro_receive(napi, skb); 714 715 rx_frm_cnt++; 716 } 717 718 rx_ring->next_to_clean = i; 719 720 rx_ring->stats.packets += rx_frm_cnt; 721 rx_ring->stats.bytes += rx_byte_cnt; 722 723 return rx_frm_cnt; 724 } 725 726 /* Probing and Init */ 727 #define ENETC_MAX_RFS_SIZE 64 728 void enetc_get_si_caps(struct enetc_si *si) 729 { 730 struct enetc_hw *hw = &si->hw; 731 u32 val; 732 733 /* find out how many of various resources we have to work with */ 734 val = enetc_rd(hw, ENETC_SICAPR0); 735 si->num_rx_rings = (val >> 16) & 0xff; 736 si->num_tx_rings = val & 0xff; 737 738 val = enetc_rd(hw, ENETC_SIRFSCAPR); 739 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 740 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 741 742 si->num_rss = 0; 743 val = enetc_rd(hw, ENETC_SIPCAPR0); 744 if (val & ENETC_SIPCAPR0_RSS) { 745 val = enetc_rd(hw, ENETC_SIRSSCAPR); 746 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(val); 747 } 748 } 749 750 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 751 { 752 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 753 &r->bd_dma_base, GFP_KERNEL); 754 if (!r->bd_base) 755 return -ENOMEM; 756 757 /* h/w requires 128B alignment */ 758 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 759 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 760 r->bd_dma_base); 761 return -EINVAL; 762 } 763 764 return 0; 765 } 766 767 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 768 { 769 int err; 770 771 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 772 if (!txr->tx_swbd) 773 return -ENOMEM; 774 775 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 776 if (err) { 777 vfree(txr->tx_swbd); 778 return err; 779 } 780 781 txr->next_to_clean = 0; 782 txr->next_to_use = 0; 783 784 return 0; 785 } 786 787 static void enetc_free_txbdr(struct enetc_bdr *txr) 788 { 789 int size, i; 790 791 for (i = 0; i < txr->bd_count; i++) 792 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 793 794 size = txr->bd_count * sizeof(union enetc_tx_bd); 795 796 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 797 txr->bd_base = NULL; 798 799 vfree(txr->tx_swbd); 800 txr->tx_swbd = NULL; 801 } 802 803 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 804 { 805 int i, err; 806 807 for (i = 0; i < priv->num_tx_rings; i++) { 808 err = enetc_alloc_txbdr(priv->tx_ring[i]); 809 810 if (err) 811 goto fail; 812 } 813 814 return 0; 815 816 fail: 817 while (i-- > 0) 818 enetc_free_txbdr(priv->tx_ring[i]); 819 820 return err; 821 } 822 823 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 824 { 825 int i; 826 827 for (i = 0; i < priv->num_tx_rings; i++) 828 enetc_free_txbdr(priv->tx_ring[i]); 829 } 830 831 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr) 832 { 833 int err; 834 835 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 836 if (!rxr->rx_swbd) 837 return -ENOMEM; 838 839 err = enetc_dma_alloc_bdr(rxr, sizeof(union enetc_rx_bd)); 840 if (err) { 841 vfree(rxr->rx_swbd); 842 return err; 843 } 844 845 rxr->next_to_clean = 0; 846 rxr->next_to_use = 0; 847 rxr->next_to_alloc = 0; 848 849 return 0; 850 } 851 852 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 853 { 854 int size; 855 856 size = rxr->bd_count * sizeof(union enetc_rx_bd); 857 858 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 859 rxr->bd_base = NULL; 860 861 vfree(rxr->rx_swbd); 862 rxr->rx_swbd = NULL; 863 } 864 865 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 866 { 867 int i, err; 868 869 for (i = 0; i < priv->num_rx_rings; i++) { 870 err = enetc_alloc_rxbdr(priv->rx_ring[i]); 871 872 if (err) 873 goto fail; 874 } 875 876 return 0; 877 878 fail: 879 while (i-- > 0) 880 enetc_free_rxbdr(priv->rx_ring[i]); 881 882 return err; 883 } 884 885 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 886 { 887 int i; 888 889 for (i = 0; i < priv->num_rx_rings; i++) 890 enetc_free_rxbdr(priv->rx_ring[i]); 891 } 892 893 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 894 { 895 int i; 896 897 if (!tx_ring->tx_swbd) 898 return; 899 900 for (i = 0; i < tx_ring->bd_count; i++) { 901 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 902 903 enetc_free_tx_skb(tx_ring, tx_swbd); 904 } 905 906 tx_ring->next_to_clean = 0; 907 tx_ring->next_to_use = 0; 908 } 909 910 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 911 { 912 int i; 913 914 if (!rx_ring->rx_swbd) 915 return; 916 917 for (i = 0; i < rx_ring->bd_count; i++) { 918 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 919 920 if (!rx_swbd->page) 921 continue; 922 923 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 924 PAGE_SIZE, DMA_FROM_DEVICE); 925 __free_page(rx_swbd->page); 926 rx_swbd->page = NULL; 927 } 928 929 rx_ring->next_to_clean = 0; 930 rx_ring->next_to_use = 0; 931 rx_ring->next_to_alloc = 0; 932 } 933 934 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 935 { 936 int i; 937 938 for (i = 0; i < priv->num_rx_rings; i++) 939 enetc_free_rx_ring(priv->rx_ring[i]); 940 941 for (i = 0; i < priv->num_tx_rings; i++) 942 enetc_free_tx_ring(priv->tx_ring[i]); 943 } 944 945 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 946 { 947 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 948 949 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 950 GFP_KERNEL); 951 if (!cbdr->bd_base) 952 return -ENOMEM; 953 954 /* h/w requires 128B alignment */ 955 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 956 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 957 return -EINVAL; 958 } 959 960 cbdr->next_to_clean = 0; 961 cbdr->next_to_use = 0; 962 963 return 0; 964 } 965 966 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 967 { 968 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 969 970 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 971 cbdr->bd_base = NULL; 972 } 973 974 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 975 { 976 /* set CBDR cache attributes */ 977 enetc_wr(hw, ENETC_SICAR2, 978 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 979 980 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 981 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 982 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 983 984 enetc_wr(hw, ENETC_SICBDRPIR, 0); 985 enetc_wr(hw, ENETC_SICBDRCIR, 0); 986 987 /* enable ring */ 988 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 989 990 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 991 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 992 } 993 994 static void enetc_clear_cbdr(struct enetc_hw *hw) 995 { 996 enetc_wr(hw, ENETC_SICBDRMR, 0); 997 } 998 999 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1000 { 1001 int *rss_table; 1002 int i; 1003 1004 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1005 if (!rss_table) 1006 return -ENOMEM; 1007 1008 /* Set up RSS table defaults */ 1009 for (i = 0; i < si->num_rss; i++) 1010 rss_table[i] = i % num_groups; 1011 1012 enetc_set_rss_table(si, rss_table, si->num_rss); 1013 1014 kfree(rss_table); 1015 1016 return 0; 1017 } 1018 1019 static int enetc_configure_si(struct enetc_ndev_priv *priv) 1020 { 1021 struct enetc_si *si = priv->si; 1022 struct enetc_hw *hw = &si->hw; 1023 int err; 1024 1025 enetc_setup_cbdr(hw, &si->cbd_ring); 1026 /* set SI cache attributes */ 1027 enetc_wr(hw, ENETC_SICAR0, 1028 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1029 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1030 /* enable SI */ 1031 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1032 1033 if (si->num_rss) { 1034 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1035 if (err) 1036 return err; 1037 } 1038 1039 return 0; 1040 } 1041 1042 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1043 { 1044 struct enetc_si *si = priv->si; 1045 int cpus = num_online_cpus(); 1046 1047 priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE; 1048 priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE; 1049 1050 /* Enable all available TX rings in order to configure as many 1051 * priorities as possible, when needed. 1052 * TODO: Make # of TX rings run-time configurable 1053 */ 1054 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1055 priv->num_tx_rings = si->num_tx_rings; 1056 priv->bdr_int_num = cpus; 1057 1058 /* SI specific */ 1059 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1060 } 1061 1062 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1063 { 1064 struct enetc_si *si = priv->si; 1065 int err; 1066 1067 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1068 if (err) 1069 return err; 1070 1071 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1072 GFP_KERNEL); 1073 if (!priv->cls_rules) { 1074 err = -ENOMEM; 1075 goto err_alloc_cls; 1076 } 1077 1078 err = enetc_configure_si(priv); 1079 if (err) 1080 goto err_config_si; 1081 1082 return 0; 1083 1084 err_config_si: 1085 kfree(priv->cls_rules); 1086 err_alloc_cls: 1087 enetc_clear_cbdr(&si->hw); 1088 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1089 1090 return err; 1091 } 1092 1093 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1094 { 1095 struct enetc_si *si = priv->si; 1096 1097 enetc_clear_cbdr(&si->hw); 1098 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1099 1100 kfree(priv->cls_rules); 1101 } 1102 1103 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1104 { 1105 int idx = tx_ring->index; 1106 u32 tbmr; 1107 1108 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1109 lower_32_bits(tx_ring->bd_dma_base)); 1110 1111 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1112 upper_32_bits(tx_ring->bd_dma_base)); 1113 1114 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1115 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1116 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1117 1118 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1119 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1120 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1121 1122 /* enable Tx ints by setting pkt thr to 1 */ 1123 enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1); 1124 1125 tbmr = ENETC_TBMR_EN; 1126 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1127 tbmr |= ENETC_TBMR_VIH; 1128 1129 /* enable ring */ 1130 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1131 1132 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1133 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1134 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1135 } 1136 1137 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1138 { 1139 int idx = rx_ring->index; 1140 u32 rbmr; 1141 1142 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1143 lower_32_bits(rx_ring->bd_dma_base)); 1144 1145 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1146 upper_32_bits(rx_ring->bd_dma_base)); 1147 1148 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1149 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1150 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1151 1152 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1153 1154 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1155 1156 /* enable Rx ints by setting pkt thr to 1 */ 1157 enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1); 1158 1159 rbmr = ENETC_RBMR_EN; 1160 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1161 rbmr |= ENETC_RBMR_BDS; 1162 #endif 1163 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1164 rbmr |= ENETC_RBMR_VTE; 1165 1166 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1167 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1168 1169 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1170 1171 /* enable ring */ 1172 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1173 } 1174 1175 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1176 { 1177 int i; 1178 1179 for (i = 0; i < priv->num_tx_rings; i++) 1180 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1181 1182 for (i = 0; i < priv->num_rx_rings; i++) 1183 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1184 } 1185 1186 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1187 { 1188 int idx = rx_ring->index; 1189 1190 /* disable EN bit on ring */ 1191 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1192 } 1193 1194 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1195 { 1196 int delay = 8, timeout = 100; 1197 int idx = tx_ring->index; 1198 1199 /* disable EN bit on ring */ 1200 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1201 1202 /* wait for busy to clear */ 1203 while (delay < timeout && 1204 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1205 msleep(delay); 1206 delay *= 2; 1207 } 1208 1209 if (delay >= timeout) 1210 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1211 idx); 1212 } 1213 1214 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1215 { 1216 int i; 1217 1218 for (i = 0; i < priv->num_tx_rings; i++) 1219 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1220 1221 for (i = 0; i < priv->num_rx_rings; i++) 1222 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1223 1224 udelay(1); 1225 } 1226 1227 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1228 { 1229 struct pci_dev *pdev = priv->si->pdev; 1230 cpumask_t cpu_mask; 1231 int i, j, err; 1232 1233 for (i = 0; i < priv->bdr_int_num; i++) { 1234 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1235 struct enetc_int_vector *v = priv->int_vector[i]; 1236 int entry = ENETC_BDR_INT_BASE_IDX + i; 1237 struct enetc_hw *hw = &priv->si->hw; 1238 1239 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1240 priv->ndev->name, i); 1241 err = request_irq(irq, enetc_msix, 0, v->name, v); 1242 if (err) { 1243 dev_err(priv->dev, "request_irq() failed!\n"); 1244 goto irq_err; 1245 } 1246 1247 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1248 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1249 1250 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1251 1252 for (j = 0; j < v->count_tx_rings; j++) { 1253 int idx = v->tx_ring[j].index; 1254 1255 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1256 } 1257 cpumask_clear(&cpu_mask); 1258 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1259 irq_set_affinity_hint(irq, &cpu_mask); 1260 } 1261 1262 return 0; 1263 1264 irq_err: 1265 while (i--) { 1266 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1267 1268 irq_set_affinity_hint(irq, NULL); 1269 free_irq(irq, priv->int_vector[i]); 1270 } 1271 1272 return err; 1273 } 1274 1275 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1276 { 1277 struct pci_dev *pdev = priv->si->pdev; 1278 int i; 1279 1280 for (i = 0; i < priv->bdr_int_num; i++) { 1281 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1282 1283 irq_set_affinity_hint(irq, NULL); 1284 free_irq(irq, priv->int_vector[i]); 1285 } 1286 } 1287 1288 static void enetc_enable_interrupts(struct enetc_ndev_priv *priv) 1289 { 1290 int i; 1291 1292 /* enable Tx & Rx event indication */ 1293 for (i = 0; i < priv->num_rx_rings; i++) { 1294 enetc_rxbdr_wr(&priv->si->hw, i, 1295 ENETC_RBIER, ENETC_RBIER_RXTIE); 1296 } 1297 1298 for (i = 0; i < priv->num_tx_rings; i++) { 1299 enetc_txbdr_wr(&priv->si->hw, i, 1300 ENETC_TBIER, ENETC_TBIER_TXTIE); 1301 } 1302 } 1303 1304 static void enetc_disable_interrupts(struct enetc_ndev_priv *priv) 1305 { 1306 int i; 1307 1308 for (i = 0; i < priv->num_tx_rings; i++) 1309 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1310 1311 for (i = 0; i < priv->num_rx_rings; i++) 1312 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1313 } 1314 1315 static void adjust_link(struct net_device *ndev) 1316 { 1317 struct phy_device *phydev = ndev->phydev; 1318 1319 phy_print_status(phydev); 1320 } 1321 1322 static int enetc_phy_connect(struct net_device *ndev) 1323 { 1324 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1325 struct phy_device *phydev; 1326 1327 if (!priv->phy_node) 1328 return 0; /* phy-less mode */ 1329 1330 phydev = of_phy_connect(ndev, priv->phy_node, &adjust_link, 1331 0, priv->if_mode); 1332 if (!phydev) { 1333 dev_err(&ndev->dev, "could not attach to PHY\n"); 1334 return -ENODEV; 1335 } 1336 1337 phy_attached_info(phydev); 1338 1339 return 0; 1340 } 1341 1342 int enetc_open(struct net_device *ndev) 1343 { 1344 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1345 int i, err; 1346 1347 err = enetc_setup_irqs(priv); 1348 if (err) 1349 return err; 1350 1351 err = enetc_phy_connect(ndev); 1352 if (err) 1353 goto err_phy_connect; 1354 1355 err = enetc_alloc_tx_resources(priv); 1356 if (err) 1357 goto err_alloc_tx; 1358 1359 err = enetc_alloc_rx_resources(priv); 1360 if (err) 1361 goto err_alloc_rx; 1362 1363 enetc_setup_bdrs(priv); 1364 1365 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1366 if (err) 1367 goto err_set_queues; 1368 1369 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1370 if (err) 1371 goto err_set_queues; 1372 1373 for (i = 0; i < priv->bdr_int_num; i++) 1374 napi_enable(&priv->int_vector[i]->napi); 1375 1376 enetc_enable_interrupts(priv); 1377 1378 if (ndev->phydev) 1379 phy_start(ndev->phydev); 1380 else 1381 netif_carrier_on(ndev); 1382 1383 netif_tx_start_all_queues(ndev); 1384 1385 return 0; 1386 1387 err_set_queues: 1388 enetc_free_rx_resources(priv); 1389 err_alloc_rx: 1390 enetc_free_tx_resources(priv); 1391 err_alloc_tx: 1392 if (ndev->phydev) 1393 phy_disconnect(ndev->phydev); 1394 err_phy_connect: 1395 enetc_free_irqs(priv); 1396 1397 return err; 1398 } 1399 1400 int enetc_close(struct net_device *ndev) 1401 { 1402 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1403 int i; 1404 1405 netif_tx_stop_all_queues(ndev); 1406 1407 if (ndev->phydev) { 1408 phy_stop(ndev->phydev); 1409 phy_disconnect(ndev->phydev); 1410 } else { 1411 netif_carrier_off(ndev); 1412 } 1413 1414 for (i = 0; i < priv->bdr_int_num; i++) { 1415 napi_synchronize(&priv->int_vector[i]->napi); 1416 napi_disable(&priv->int_vector[i]->napi); 1417 } 1418 1419 enetc_disable_interrupts(priv); 1420 enetc_clear_bdrs(priv); 1421 1422 enetc_free_rxtx_rings(priv); 1423 enetc_free_rx_resources(priv); 1424 enetc_free_tx_resources(priv); 1425 enetc_free_irqs(priv); 1426 1427 return 0; 1428 } 1429 1430 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1431 void *type_data) 1432 { 1433 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1434 struct tc_mqprio_qopt *mqprio = type_data; 1435 struct enetc_bdr *tx_ring; 1436 u8 num_tc; 1437 int i; 1438 1439 if (type != TC_SETUP_QDISC_MQPRIO) 1440 return -EOPNOTSUPP; 1441 1442 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1443 num_tc = mqprio->num_tc; 1444 1445 if (!num_tc) { 1446 netdev_reset_tc(ndev); 1447 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1448 1449 /* Reset all ring priorities to 0 */ 1450 for (i = 0; i < priv->num_tx_rings; i++) { 1451 tx_ring = priv->tx_ring[i]; 1452 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1453 } 1454 1455 return 0; 1456 } 1457 1458 /* Check if we have enough BD rings available to accommodate all TCs */ 1459 if (num_tc > priv->num_tx_rings) { 1460 netdev_err(ndev, "Max %d traffic classes supported\n", 1461 priv->num_tx_rings); 1462 return -EINVAL; 1463 } 1464 1465 /* For the moment, we use only one BD ring per TC. 1466 * 1467 * Configure num_tc BD rings with increasing priorities. 1468 */ 1469 for (i = 0; i < num_tc; i++) { 1470 tx_ring = priv->tx_ring[i]; 1471 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1472 } 1473 1474 /* Reset the number of netdev queues based on the TC count */ 1475 netif_set_real_num_tx_queues(ndev, num_tc); 1476 1477 netdev_set_num_tc(ndev, num_tc); 1478 1479 /* Each TC is associated with one netdev queue */ 1480 for (i = 0; i < num_tc; i++) 1481 netdev_set_tc_queue(ndev, i, 1, i); 1482 1483 return 0; 1484 } 1485 1486 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1487 { 1488 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1489 struct net_device_stats *stats = &ndev->stats; 1490 unsigned long packets = 0, bytes = 0; 1491 int i; 1492 1493 for (i = 0; i < priv->num_rx_rings; i++) { 1494 packets += priv->rx_ring[i]->stats.packets; 1495 bytes += priv->rx_ring[i]->stats.bytes; 1496 } 1497 1498 stats->rx_packets = packets; 1499 stats->rx_bytes = bytes; 1500 bytes = 0; 1501 packets = 0; 1502 1503 for (i = 0; i < priv->num_tx_rings; i++) { 1504 packets += priv->tx_ring[i]->stats.packets; 1505 bytes += priv->tx_ring[i]->stats.bytes; 1506 } 1507 1508 stats->tx_packets = packets; 1509 stats->tx_bytes = bytes; 1510 1511 return stats; 1512 } 1513 1514 static int enetc_set_rss(struct net_device *ndev, int en) 1515 { 1516 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1517 struct enetc_hw *hw = &priv->si->hw; 1518 u32 reg; 1519 1520 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1521 1522 reg = enetc_rd(hw, ENETC_SIMR); 1523 reg &= ~ENETC_SIMR_RSSE; 1524 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1525 enetc_wr(hw, ENETC_SIMR, reg); 1526 1527 return 0; 1528 } 1529 1530 int enetc_set_features(struct net_device *ndev, 1531 netdev_features_t features) 1532 { 1533 netdev_features_t changed = ndev->features ^ features; 1534 1535 if (changed & NETIF_F_RXHASH) 1536 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1537 1538 return 0; 1539 } 1540 1541 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1542 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1543 { 1544 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1545 struct hwtstamp_config config; 1546 1547 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1548 return -EFAULT; 1549 1550 switch (config.tx_type) { 1551 case HWTSTAMP_TX_OFF: 1552 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1553 break; 1554 case HWTSTAMP_TX_ON: 1555 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1556 break; 1557 default: 1558 return -ERANGE; 1559 } 1560 1561 switch (config.rx_filter) { 1562 case HWTSTAMP_FILTER_NONE: 1563 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1564 break; 1565 default: 1566 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1567 config.rx_filter = HWTSTAMP_FILTER_ALL; 1568 } 1569 1570 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1571 -EFAULT : 0; 1572 } 1573 1574 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1575 { 1576 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1577 struct hwtstamp_config config; 1578 1579 config.flags = 0; 1580 1581 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1582 config.tx_type = HWTSTAMP_TX_ON; 1583 else 1584 config.tx_type = HWTSTAMP_TX_OFF; 1585 1586 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1587 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1588 1589 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1590 -EFAULT : 0; 1591 } 1592 #endif 1593 1594 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1595 { 1596 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1597 if (cmd == SIOCSHWTSTAMP) 1598 return enetc_hwtstamp_set(ndev, rq); 1599 if (cmd == SIOCGHWTSTAMP) 1600 return enetc_hwtstamp_get(ndev, rq); 1601 #endif 1602 return -EINVAL; 1603 } 1604 1605 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1606 { 1607 struct pci_dev *pdev = priv->si->pdev; 1608 int size, v_tx_rings; 1609 int i, n, err, nvec; 1610 1611 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1612 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1613 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1614 1615 if (n < 0) 1616 return n; 1617 1618 if (n != nvec) 1619 return -EPERM; 1620 1621 /* # of tx rings per int vector */ 1622 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1623 size = sizeof(struct enetc_int_vector) + 1624 sizeof(struct enetc_bdr) * v_tx_rings; 1625 1626 for (i = 0; i < priv->bdr_int_num; i++) { 1627 struct enetc_int_vector *v; 1628 struct enetc_bdr *bdr; 1629 int j; 1630 1631 v = kzalloc(size, GFP_KERNEL); 1632 if (!v) { 1633 err = -ENOMEM; 1634 goto fail; 1635 } 1636 1637 priv->int_vector[i] = v; 1638 1639 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1640 NAPI_POLL_WEIGHT); 1641 v->count_tx_rings = v_tx_rings; 1642 1643 for (j = 0; j < v_tx_rings; j++) { 1644 int idx; 1645 1646 /* default tx ring mapping policy */ 1647 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1648 idx = 2 * j + i; /* 2 CPUs */ 1649 else 1650 idx = j + i * v_tx_rings; /* default */ 1651 1652 __set_bit(idx, &v->tx_rings_map); 1653 bdr = &v->tx_ring[j]; 1654 bdr->index = idx; 1655 bdr->ndev = priv->ndev; 1656 bdr->dev = priv->dev; 1657 bdr->bd_count = priv->tx_bd_count; 1658 priv->tx_ring[idx] = bdr; 1659 } 1660 1661 bdr = &v->rx_ring; 1662 bdr->index = i; 1663 bdr->ndev = priv->ndev; 1664 bdr->dev = priv->dev; 1665 bdr->bd_count = priv->rx_bd_count; 1666 priv->rx_ring[i] = bdr; 1667 } 1668 1669 return 0; 1670 1671 fail: 1672 while (i--) { 1673 netif_napi_del(&priv->int_vector[i]->napi); 1674 kfree(priv->int_vector[i]); 1675 } 1676 1677 pci_free_irq_vectors(pdev); 1678 1679 return err; 1680 } 1681 1682 void enetc_free_msix(struct enetc_ndev_priv *priv) 1683 { 1684 int i; 1685 1686 for (i = 0; i < priv->bdr_int_num; i++) { 1687 struct enetc_int_vector *v = priv->int_vector[i]; 1688 1689 netif_napi_del(&v->napi); 1690 } 1691 1692 for (i = 0; i < priv->num_rx_rings; i++) 1693 priv->rx_ring[i] = NULL; 1694 1695 for (i = 0; i < priv->num_tx_rings; i++) 1696 priv->tx_ring[i] = NULL; 1697 1698 for (i = 0; i < priv->bdr_int_num; i++) { 1699 kfree(priv->int_vector[i]); 1700 priv->int_vector[i] = NULL; 1701 } 1702 1703 /* disable all MSIX for this device */ 1704 pci_free_irq_vectors(priv->si->pdev); 1705 } 1706 1707 static void enetc_kfree_si(struct enetc_si *si) 1708 { 1709 char *p = (char *)si - si->pad; 1710 1711 kfree(p); 1712 } 1713 1714 static void enetc_detect_errata(struct enetc_si *si) 1715 { 1716 if (si->pdev->revision == ENETC_REV1) 1717 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL | 1718 ENETC_ERR_UCMCSWP; 1719 } 1720 1721 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1722 { 1723 struct enetc_si *si, *p; 1724 struct enetc_hw *hw; 1725 size_t alloc_size; 1726 int err, len; 1727 1728 pcie_flr(pdev); 1729 err = pci_enable_device_mem(pdev); 1730 if (err) { 1731 dev_err(&pdev->dev, "device enable failed\n"); 1732 return err; 1733 } 1734 1735 /* set up for high or low dma */ 1736 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1737 if (err) { 1738 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1739 if (err) { 1740 dev_err(&pdev->dev, 1741 "DMA configuration failed: 0x%x\n", err); 1742 goto err_dma; 1743 } 1744 } 1745 1746 err = pci_request_mem_regions(pdev, name); 1747 if (err) { 1748 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1749 goto err_pci_mem_reg; 1750 } 1751 1752 pci_set_master(pdev); 1753 1754 alloc_size = sizeof(struct enetc_si); 1755 if (sizeof_priv) { 1756 /* align priv to 32B */ 1757 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1758 alloc_size += sizeof_priv; 1759 } 1760 /* force 32B alignment for enetc_si */ 1761 alloc_size += ENETC_SI_ALIGN - 1; 1762 1763 p = kzalloc(alloc_size, GFP_KERNEL); 1764 if (!p) { 1765 err = -ENOMEM; 1766 goto err_alloc_si; 1767 } 1768 1769 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1770 si->pad = (char *)si - (char *)p; 1771 1772 pci_set_drvdata(pdev, si); 1773 si->pdev = pdev; 1774 hw = &si->hw; 1775 1776 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1777 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1778 if (!hw->reg) { 1779 err = -ENXIO; 1780 dev_err(&pdev->dev, "ioremap() failed\n"); 1781 goto err_ioremap; 1782 } 1783 if (len > ENETC_PORT_BASE) 1784 hw->port = hw->reg + ENETC_PORT_BASE; 1785 if (len > ENETC_GLOBAL_BASE) 1786 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1787 1788 enetc_detect_errata(si); 1789 1790 return 0; 1791 1792 err_ioremap: 1793 enetc_kfree_si(si); 1794 err_alloc_si: 1795 pci_release_mem_regions(pdev); 1796 err_pci_mem_reg: 1797 err_dma: 1798 pci_disable_device(pdev); 1799 1800 return err; 1801 } 1802 1803 void enetc_pci_remove(struct pci_dev *pdev) 1804 { 1805 struct enetc_si *si = pci_get_drvdata(pdev); 1806 struct enetc_hw *hw = &si->hw; 1807 1808 iounmap(hw->reg); 1809 enetc_kfree_si(si); 1810 pci_release_mem_regions(pdev); 1811 pci_disable_device(pdev); 1812 } 1813