1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/of_mdio.h> 8 #include <linux/vmalloc.h> 9 10 /* ENETC overhead: optional extension BD + 1 BD gap */ 11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 12 /* max # of chained Tx BDs is 15, including head and extension BD */ 13 #define ENETC_MAX_SKB_FRAGS 13 14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 15 16 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 17 int active_offloads); 18 19 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 20 { 21 struct enetc_ndev_priv *priv = netdev_priv(ndev); 22 struct enetc_bdr *tx_ring; 23 int count; 24 25 tx_ring = priv->tx_ring[skb->queue_mapping]; 26 27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 28 if (unlikely(skb_linearize(skb))) 29 goto drop_packet_err; 30 31 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 33 netif_stop_subqueue(ndev, tx_ring->index); 34 return NETDEV_TX_BUSY; 35 } 36 37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 38 if (unlikely(!count)) 39 goto drop_packet_err; 40 41 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 42 netif_stop_subqueue(ndev, tx_ring->index); 43 44 return NETDEV_TX_OK; 45 46 drop_packet_err: 47 dev_kfree_skb_any(skb); 48 return NETDEV_TX_OK; 49 } 50 51 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd) 52 { 53 int l3_start, l3_hsize; 54 u16 l3_flags, l4_flags; 55 56 if (skb->ip_summed != CHECKSUM_PARTIAL) 57 return false; 58 59 switch (skb->csum_offset) { 60 case offsetof(struct tcphdr, check): 61 l4_flags = ENETC_TXBD_L4_TCP; 62 break; 63 case offsetof(struct udphdr, check): 64 l4_flags = ENETC_TXBD_L4_UDP; 65 break; 66 default: 67 skb_checksum_help(skb); 68 return false; 69 } 70 71 l3_start = skb_network_offset(skb); 72 l3_hsize = skb_network_header_len(skb); 73 74 l3_flags = 0; 75 if (skb->protocol == htons(ETH_P_IPV6)) 76 l3_flags = ENETC_TXBD_L3_IPV6; 77 78 /* write BD fields */ 79 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags); 80 txbd->l4_csoff = l4_flags; 81 82 return true; 83 } 84 85 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 86 struct enetc_tx_swbd *tx_swbd) 87 { 88 if (tx_swbd->is_dma_page) 89 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 90 tx_swbd->len, DMA_TO_DEVICE); 91 else 92 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 93 tx_swbd->len, DMA_TO_DEVICE); 94 tx_swbd->dma = 0; 95 } 96 97 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 98 struct enetc_tx_swbd *tx_swbd) 99 { 100 if (tx_swbd->dma) 101 enetc_unmap_tx_buff(tx_ring, tx_swbd); 102 103 if (tx_swbd->skb) { 104 dev_kfree_skb_any(tx_swbd->skb); 105 tx_swbd->skb = NULL; 106 } 107 } 108 109 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 110 int active_offloads) 111 { 112 struct enetc_tx_swbd *tx_swbd; 113 skb_frag_t *frag; 114 int len = skb_headlen(skb); 115 union enetc_tx_bd temp_bd; 116 union enetc_tx_bd *txbd; 117 bool do_vlan, do_tstamp; 118 int i, count = 0; 119 unsigned int f; 120 dma_addr_t dma; 121 u8 flags = 0; 122 123 i = tx_ring->next_to_use; 124 txbd = ENETC_TXBD(*tx_ring, i); 125 prefetchw(txbd); 126 127 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 128 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 129 goto dma_err; 130 131 temp_bd.addr = cpu_to_le64(dma); 132 temp_bd.buf_len = cpu_to_le16(len); 133 temp_bd.lstatus = 0; 134 135 tx_swbd = &tx_ring->tx_swbd[i]; 136 tx_swbd->dma = dma; 137 tx_swbd->len = len; 138 tx_swbd->is_dma_page = 0; 139 count++; 140 141 do_vlan = skb_vlan_tag_present(skb); 142 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 143 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 144 tx_swbd->do_tstamp = do_tstamp; 145 tx_swbd->check_wb = tx_swbd->do_tstamp; 146 147 if (do_vlan || do_tstamp) 148 flags |= ENETC_TXBD_FLAGS_EX; 149 150 if (enetc_tx_csum(skb, &temp_bd)) 151 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS; 152 153 /* first BD needs frm_len and offload flags set */ 154 temp_bd.frm_len = cpu_to_le16(skb->len); 155 temp_bd.flags = flags; 156 157 if (flags & ENETC_TXBD_FLAGS_EX) { 158 u8 e_flags = 0; 159 *txbd = temp_bd; 160 enetc_clear_tx_bd(&temp_bd); 161 162 /* add extension BD for VLAN and/or timestamping */ 163 flags = 0; 164 tx_swbd++; 165 txbd++; 166 i++; 167 if (unlikely(i == tx_ring->bd_count)) { 168 i = 0; 169 tx_swbd = tx_ring->tx_swbd; 170 txbd = ENETC_TXBD(*tx_ring, 0); 171 } 172 prefetchw(txbd); 173 174 if (do_vlan) { 175 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 176 temp_bd.ext.tpid = 0; /* < C-TAG */ 177 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 178 } 179 180 if (do_tstamp) { 181 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 182 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 183 } 184 185 temp_bd.ext.e_flags = e_flags; 186 count++; 187 } 188 189 frag = &skb_shinfo(skb)->frags[0]; 190 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 191 len = skb_frag_size(frag); 192 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 193 DMA_TO_DEVICE); 194 if (dma_mapping_error(tx_ring->dev, dma)) 195 goto dma_err; 196 197 *txbd = temp_bd; 198 enetc_clear_tx_bd(&temp_bd); 199 200 flags = 0; 201 tx_swbd++; 202 txbd++; 203 i++; 204 if (unlikely(i == tx_ring->bd_count)) { 205 i = 0; 206 tx_swbd = tx_ring->tx_swbd; 207 txbd = ENETC_TXBD(*tx_ring, 0); 208 } 209 prefetchw(txbd); 210 211 temp_bd.addr = cpu_to_le64(dma); 212 temp_bd.buf_len = cpu_to_le16(len); 213 214 tx_swbd->dma = dma; 215 tx_swbd->len = len; 216 tx_swbd->is_dma_page = 1; 217 count++; 218 } 219 220 /* last BD needs 'F' bit set */ 221 flags |= ENETC_TXBD_FLAGS_F; 222 temp_bd.flags = flags; 223 *txbd = temp_bd; 224 225 tx_ring->tx_swbd[i].skb = skb; 226 227 enetc_bdr_idx_inc(tx_ring, &i); 228 tx_ring->next_to_use = i; 229 230 /* let H/W know BD ring has been updated */ 231 enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */ 232 233 return count; 234 235 dma_err: 236 dev_err(tx_ring->dev, "DMA map error"); 237 238 do { 239 tx_swbd = &tx_ring->tx_swbd[i]; 240 enetc_free_tx_skb(tx_ring, tx_swbd); 241 if (i == 0) 242 i = tx_ring->bd_count; 243 i--; 244 } while (count--); 245 246 return 0; 247 } 248 249 static irqreturn_t enetc_msix(int irq, void *data) 250 { 251 struct enetc_int_vector *v = data; 252 int i; 253 254 /* disable interrupts */ 255 enetc_wr_reg(v->rbier, 0); 256 257 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings) 258 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0); 259 260 napi_schedule_irqoff(&v->napi); 261 262 return IRQ_HANDLED; 263 } 264 265 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 266 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 267 struct napi_struct *napi, int work_limit); 268 269 static int enetc_poll(struct napi_struct *napi, int budget) 270 { 271 struct enetc_int_vector 272 *v = container_of(napi, struct enetc_int_vector, napi); 273 bool complete = true; 274 int work_done; 275 int i; 276 277 for (i = 0; i < v->count_tx_rings; i++) 278 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 279 complete = false; 280 281 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 282 if (work_done == budget) 283 complete = false; 284 285 if (!complete) 286 return budget; 287 288 napi_complete_done(napi, work_done); 289 290 /* enable interrupts */ 291 enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE); 292 293 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings) 294 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 295 ENETC_TBIER_TXTIE); 296 297 return work_done; 298 } 299 300 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 301 { 302 int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 303 304 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 305 } 306 307 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 308 u64 *tstamp) 309 { 310 u32 lo, hi, tstamp_lo; 311 312 lo = enetc_rd(hw, ENETC_SICTR0); 313 hi = enetc_rd(hw, ENETC_SICTR1); 314 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 315 if (lo <= tstamp_lo) 316 hi -= 1; 317 *tstamp = (u64)hi << 32 | tstamp_lo; 318 } 319 320 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 321 { 322 struct skb_shared_hwtstamps shhwtstamps; 323 324 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 325 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 326 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 327 skb_tstamp_tx(skb, &shhwtstamps); 328 } 329 } 330 331 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 332 { 333 struct net_device *ndev = tx_ring->ndev; 334 int tx_frm_cnt = 0, tx_byte_cnt = 0; 335 struct enetc_tx_swbd *tx_swbd; 336 int i, bds_to_clean; 337 bool do_tstamp; 338 u64 tstamp = 0; 339 340 i = tx_ring->next_to_clean; 341 tx_swbd = &tx_ring->tx_swbd[i]; 342 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 343 344 do_tstamp = false; 345 346 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 347 bool is_eof = !!tx_swbd->skb; 348 349 if (unlikely(tx_swbd->check_wb)) { 350 struct enetc_ndev_priv *priv = netdev_priv(ndev); 351 union enetc_tx_bd *txbd; 352 353 txbd = ENETC_TXBD(*tx_ring, i); 354 355 if (txbd->flags & ENETC_TXBD_FLAGS_W && 356 tx_swbd->do_tstamp) { 357 enetc_get_tx_tstamp(&priv->si->hw, txbd, 358 &tstamp); 359 do_tstamp = true; 360 } 361 } 362 363 if (likely(tx_swbd->dma)) 364 enetc_unmap_tx_buff(tx_ring, tx_swbd); 365 366 if (is_eof) { 367 if (unlikely(do_tstamp)) { 368 enetc_tstamp_tx(tx_swbd->skb, tstamp); 369 do_tstamp = false; 370 } 371 napi_consume_skb(tx_swbd->skb, napi_budget); 372 tx_swbd->skb = NULL; 373 } 374 375 tx_byte_cnt += tx_swbd->len; 376 377 bds_to_clean--; 378 tx_swbd++; 379 i++; 380 if (unlikely(i == tx_ring->bd_count)) { 381 i = 0; 382 tx_swbd = tx_ring->tx_swbd; 383 } 384 385 /* BD iteration loop end */ 386 if (is_eof) { 387 tx_frm_cnt++; 388 /* re-arm interrupt source */ 389 enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) | 390 BIT(16 + tx_ring->index)); 391 } 392 393 if (unlikely(!bds_to_clean)) 394 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 395 } 396 397 tx_ring->next_to_clean = i; 398 tx_ring->stats.packets += tx_frm_cnt; 399 tx_ring->stats.bytes += tx_byte_cnt; 400 401 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 402 __netif_subqueue_stopped(ndev, tx_ring->index) && 403 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 404 netif_wake_subqueue(ndev, tx_ring->index); 405 } 406 407 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 408 } 409 410 static bool enetc_new_page(struct enetc_bdr *rx_ring, 411 struct enetc_rx_swbd *rx_swbd) 412 { 413 struct page *page; 414 dma_addr_t addr; 415 416 page = dev_alloc_page(); 417 if (unlikely(!page)) 418 return false; 419 420 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 421 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 422 __free_page(page); 423 424 return false; 425 } 426 427 rx_swbd->dma = addr; 428 rx_swbd->page = page; 429 rx_swbd->page_offset = ENETC_RXB_PAD; 430 431 return true; 432 } 433 434 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 435 { 436 struct enetc_rx_swbd *rx_swbd; 437 union enetc_rx_bd *rxbd; 438 int i, j; 439 440 i = rx_ring->next_to_use; 441 rx_swbd = &rx_ring->rx_swbd[i]; 442 rxbd = ENETC_RXBD(*rx_ring, i); 443 444 for (j = 0; j < buff_cnt; j++) { 445 /* try reuse page */ 446 if (unlikely(!rx_swbd->page)) { 447 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 448 rx_ring->stats.rx_alloc_errs++; 449 break; 450 } 451 } 452 453 /* update RxBD */ 454 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 455 rx_swbd->page_offset); 456 /* clear 'R" as well */ 457 rxbd->r.lstatus = 0; 458 459 rx_swbd++; 460 rxbd++; 461 i++; 462 if (unlikely(i == rx_ring->bd_count)) { 463 i = 0; 464 rx_swbd = rx_ring->rx_swbd; 465 rxbd = ENETC_RXBD(*rx_ring, 0); 466 } 467 } 468 469 if (likely(j)) { 470 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 471 rx_ring->next_to_use = i; 472 /* update ENETC's consumer index */ 473 enetc_wr_reg(rx_ring->rcir, i); 474 } 475 476 return j; 477 } 478 479 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 480 static void enetc_get_rx_tstamp(struct net_device *ndev, 481 union enetc_rx_bd *rxbd, 482 struct sk_buff *skb) 483 { 484 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 485 struct enetc_ndev_priv *priv = netdev_priv(ndev); 486 struct enetc_hw *hw = &priv->si->hw; 487 u32 lo, hi, tstamp_lo; 488 u64 tstamp; 489 490 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 491 lo = enetc_rd(hw, ENETC_SICTR0); 492 hi = enetc_rd(hw, ENETC_SICTR1); 493 tstamp_lo = le32_to_cpu(rxbd->r.tstamp); 494 if (lo <= tstamp_lo) 495 hi -= 1; 496 497 tstamp = (u64)hi << 32 | tstamp_lo; 498 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 499 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 500 } 501 } 502 #endif 503 504 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 505 union enetc_rx_bd *rxbd, struct sk_buff *skb) 506 { 507 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 508 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 509 #endif 510 /* TODO: hashing */ 511 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 512 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 513 514 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 515 skb->ip_summed = CHECKSUM_COMPLETE; 516 } 517 518 /* copy VLAN to skb, if one is extracted, for now we assume it's a 519 * standard TPID, but HW also supports custom values 520 */ 521 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) 522 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 523 le16_to_cpu(rxbd->r.vlan_opt)); 524 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 525 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 526 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 527 #endif 528 } 529 530 static void enetc_process_skb(struct enetc_bdr *rx_ring, 531 struct sk_buff *skb) 532 { 533 skb_record_rx_queue(skb, rx_ring->index); 534 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 535 } 536 537 static bool enetc_page_reusable(struct page *page) 538 { 539 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 540 } 541 542 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 543 struct enetc_rx_swbd *old) 544 { 545 struct enetc_rx_swbd *new; 546 547 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 548 549 /* next buf that may reuse a page */ 550 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 551 552 /* copy page reference */ 553 *new = *old; 554 } 555 556 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 557 int i, u16 size) 558 { 559 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 560 561 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 562 rx_swbd->page_offset, 563 size, DMA_FROM_DEVICE); 564 return rx_swbd; 565 } 566 567 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 568 struct enetc_rx_swbd *rx_swbd) 569 { 570 if (likely(enetc_page_reusable(rx_swbd->page))) { 571 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 572 page_ref_inc(rx_swbd->page); 573 574 enetc_reuse_page(rx_ring, rx_swbd); 575 576 /* sync for use by the device */ 577 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 578 rx_swbd->page_offset, 579 ENETC_RXB_DMA_SIZE, 580 DMA_FROM_DEVICE); 581 } else { 582 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 583 PAGE_SIZE, DMA_FROM_DEVICE); 584 } 585 586 rx_swbd->page = NULL; 587 } 588 589 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 590 int i, u16 size) 591 { 592 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 593 struct sk_buff *skb; 594 void *ba; 595 596 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 597 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 598 if (unlikely(!skb)) { 599 rx_ring->stats.rx_alloc_errs++; 600 return NULL; 601 } 602 603 skb_reserve(skb, ENETC_RXB_PAD); 604 __skb_put(skb, size); 605 606 enetc_put_rx_buff(rx_ring, rx_swbd); 607 608 return skb; 609 } 610 611 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 612 u16 size, struct sk_buff *skb) 613 { 614 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 615 616 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 617 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 618 619 enetc_put_rx_buff(rx_ring, rx_swbd); 620 } 621 622 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 623 624 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 625 struct napi_struct *napi, int work_limit) 626 { 627 int rx_frm_cnt = 0, rx_byte_cnt = 0; 628 int cleaned_cnt, i; 629 630 cleaned_cnt = enetc_bd_unused(rx_ring); 631 /* next descriptor to process */ 632 i = rx_ring->next_to_clean; 633 634 while (likely(rx_frm_cnt < work_limit)) { 635 union enetc_rx_bd *rxbd; 636 struct sk_buff *skb; 637 u32 bd_status; 638 u16 size; 639 640 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 641 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 642 643 cleaned_cnt -= count; 644 } 645 646 rxbd = ENETC_RXBD(*rx_ring, i); 647 bd_status = le32_to_cpu(rxbd->r.lstatus); 648 if (!bd_status) 649 break; 650 651 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index)); 652 dma_rmb(); /* for reading other rxbd fields */ 653 size = le16_to_cpu(rxbd->r.buf_len); 654 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 655 if (!skb) 656 break; 657 658 enetc_get_offloads(rx_ring, rxbd, skb); 659 660 cleaned_cnt++; 661 rxbd++; 662 i++; 663 if (unlikely(i == rx_ring->bd_count)) { 664 i = 0; 665 rxbd = ENETC_RXBD(*rx_ring, 0); 666 } 667 668 if (unlikely(bd_status & 669 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 670 dev_kfree_skb(skb); 671 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 672 dma_rmb(); 673 bd_status = le32_to_cpu(rxbd->r.lstatus); 674 rxbd++; 675 i++; 676 if (unlikely(i == rx_ring->bd_count)) { 677 i = 0; 678 rxbd = ENETC_RXBD(*rx_ring, 0); 679 } 680 } 681 682 rx_ring->ndev->stats.rx_dropped++; 683 rx_ring->ndev->stats.rx_errors++; 684 685 break; 686 } 687 688 /* not last BD in frame? */ 689 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 690 bd_status = le32_to_cpu(rxbd->r.lstatus); 691 size = ENETC_RXB_DMA_SIZE; 692 693 if (bd_status & ENETC_RXBD_LSTATUS_F) { 694 dma_rmb(); 695 size = le16_to_cpu(rxbd->r.buf_len); 696 } 697 698 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 699 700 cleaned_cnt++; 701 rxbd++; 702 i++; 703 if (unlikely(i == rx_ring->bd_count)) { 704 i = 0; 705 rxbd = ENETC_RXBD(*rx_ring, 0); 706 } 707 } 708 709 rx_byte_cnt += skb->len; 710 711 enetc_process_skb(rx_ring, skb); 712 713 napi_gro_receive(napi, skb); 714 715 rx_frm_cnt++; 716 } 717 718 rx_ring->next_to_clean = i; 719 720 rx_ring->stats.packets += rx_frm_cnt; 721 rx_ring->stats.bytes += rx_byte_cnt; 722 723 return rx_frm_cnt; 724 } 725 726 /* Probing and Init */ 727 #define ENETC_MAX_RFS_SIZE 64 728 void enetc_get_si_caps(struct enetc_si *si) 729 { 730 struct enetc_hw *hw = &si->hw; 731 u32 val; 732 733 /* find out how many of various resources we have to work with */ 734 val = enetc_rd(hw, ENETC_SICAPR0); 735 si->num_rx_rings = (val >> 16) & 0xff; 736 si->num_tx_rings = val & 0xff; 737 738 val = enetc_rd(hw, ENETC_SIRFSCAPR); 739 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 740 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 741 742 si->num_rss = 0; 743 val = enetc_rd(hw, ENETC_SIPCAPR0); 744 if (val & ENETC_SIPCAPR0_RSS) { 745 u32 rss; 746 747 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 748 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 749 } 750 751 if (val & ENETC_SIPCAPR0_QBV) 752 si->hw_features |= ENETC_SI_F_QBV; 753 } 754 755 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 756 { 757 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 758 &r->bd_dma_base, GFP_KERNEL); 759 if (!r->bd_base) 760 return -ENOMEM; 761 762 /* h/w requires 128B alignment */ 763 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 764 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 765 r->bd_dma_base); 766 return -EINVAL; 767 } 768 769 return 0; 770 } 771 772 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 773 { 774 int err; 775 776 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 777 if (!txr->tx_swbd) 778 return -ENOMEM; 779 780 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 781 if (err) { 782 vfree(txr->tx_swbd); 783 return err; 784 } 785 786 txr->next_to_clean = 0; 787 txr->next_to_use = 0; 788 789 return 0; 790 } 791 792 static void enetc_free_txbdr(struct enetc_bdr *txr) 793 { 794 int size, i; 795 796 for (i = 0; i < txr->bd_count; i++) 797 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 798 799 size = txr->bd_count * sizeof(union enetc_tx_bd); 800 801 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 802 txr->bd_base = NULL; 803 804 vfree(txr->tx_swbd); 805 txr->tx_swbd = NULL; 806 } 807 808 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 809 { 810 int i, err; 811 812 for (i = 0; i < priv->num_tx_rings; i++) { 813 err = enetc_alloc_txbdr(priv->tx_ring[i]); 814 815 if (err) 816 goto fail; 817 } 818 819 return 0; 820 821 fail: 822 while (i-- > 0) 823 enetc_free_txbdr(priv->tx_ring[i]); 824 825 return err; 826 } 827 828 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 829 { 830 int i; 831 832 for (i = 0; i < priv->num_tx_rings; i++) 833 enetc_free_txbdr(priv->tx_ring[i]); 834 } 835 836 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr) 837 { 838 int err; 839 840 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 841 if (!rxr->rx_swbd) 842 return -ENOMEM; 843 844 err = enetc_dma_alloc_bdr(rxr, sizeof(union enetc_rx_bd)); 845 if (err) { 846 vfree(rxr->rx_swbd); 847 return err; 848 } 849 850 rxr->next_to_clean = 0; 851 rxr->next_to_use = 0; 852 rxr->next_to_alloc = 0; 853 854 return 0; 855 } 856 857 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 858 { 859 int size; 860 861 size = rxr->bd_count * sizeof(union enetc_rx_bd); 862 863 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 864 rxr->bd_base = NULL; 865 866 vfree(rxr->rx_swbd); 867 rxr->rx_swbd = NULL; 868 } 869 870 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 871 { 872 int i, err; 873 874 for (i = 0; i < priv->num_rx_rings; i++) { 875 err = enetc_alloc_rxbdr(priv->rx_ring[i]); 876 877 if (err) 878 goto fail; 879 } 880 881 return 0; 882 883 fail: 884 while (i-- > 0) 885 enetc_free_rxbdr(priv->rx_ring[i]); 886 887 return err; 888 } 889 890 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 891 { 892 int i; 893 894 for (i = 0; i < priv->num_rx_rings; i++) 895 enetc_free_rxbdr(priv->rx_ring[i]); 896 } 897 898 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 899 { 900 int i; 901 902 if (!tx_ring->tx_swbd) 903 return; 904 905 for (i = 0; i < tx_ring->bd_count; i++) { 906 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 907 908 enetc_free_tx_skb(tx_ring, tx_swbd); 909 } 910 911 tx_ring->next_to_clean = 0; 912 tx_ring->next_to_use = 0; 913 } 914 915 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 916 { 917 int i; 918 919 if (!rx_ring->rx_swbd) 920 return; 921 922 for (i = 0; i < rx_ring->bd_count; i++) { 923 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 924 925 if (!rx_swbd->page) 926 continue; 927 928 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 929 PAGE_SIZE, DMA_FROM_DEVICE); 930 __free_page(rx_swbd->page); 931 rx_swbd->page = NULL; 932 } 933 934 rx_ring->next_to_clean = 0; 935 rx_ring->next_to_use = 0; 936 rx_ring->next_to_alloc = 0; 937 } 938 939 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 940 { 941 int i; 942 943 for (i = 0; i < priv->num_rx_rings; i++) 944 enetc_free_rx_ring(priv->rx_ring[i]); 945 946 for (i = 0; i < priv->num_tx_rings; i++) 947 enetc_free_tx_ring(priv->tx_ring[i]); 948 } 949 950 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 951 { 952 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 953 954 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 955 GFP_KERNEL); 956 if (!cbdr->bd_base) 957 return -ENOMEM; 958 959 /* h/w requires 128B alignment */ 960 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 961 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 962 return -EINVAL; 963 } 964 965 cbdr->next_to_clean = 0; 966 cbdr->next_to_use = 0; 967 968 return 0; 969 } 970 971 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 972 { 973 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 974 975 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 976 cbdr->bd_base = NULL; 977 } 978 979 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 980 { 981 /* set CBDR cache attributes */ 982 enetc_wr(hw, ENETC_SICAR2, 983 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 984 985 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 986 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 987 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 988 989 enetc_wr(hw, ENETC_SICBDRPIR, 0); 990 enetc_wr(hw, ENETC_SICBDRCIR, 0); 991 992 /* enable ring */ 993 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 994 995 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 996 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 997 } 998 999 static void enetc_clear_cbdr(struct enetc_hw *hw) 1000 { 1001 enetc_wr(hw, ENETC_SICBDRMR, 0); 1002 } 1003 1004 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1005 { 1006 int *rss_table; 1007 int i; 1008 1009 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1010 if (!rss_table) 1011 return -ENOMEM; 1012 1013 /* Set up RSS table defaults */ 1014 for (i = 0; i < si->num_rss; i++) 1015 rss_table[i] = i % num_groups; 1016 1017 enetc_set_rss_table(si, rss_table, si->num_rss); 1018 1019 kfree(rss_table); 1020 1021 return 0; 1022 } 1023 1024 static int enetc_configure_si(struct enetc_ndev_priv *priv) 1025 { 1026 struct enetc_si *si = priv->si; 1027 struct enetc_hw *hw = &si->hw; 1028 int err; 1029 1030 enetc_setup_cbdr(hw, &si->cbd_ring); 1031 /* set SI cache attributes */ 1032 enetc_wr(hw, ENETC_SICAR0, 1033 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1034 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1035 /* enable SI */ 1036 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1037 1038 if (si->num_rss) { 1039 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1040 if (err) 1041 return err; 1042 } 1043 1044 return 0; 1045 } 1046 1047 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1048 { 1049 struct enetc_si *si = priv->si; 1050 int cpus = num_online_cpus(); 1051 1052 priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE; 1053 priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE; 1054 1055 /* Enable all available TX rings in order to configure as many 1056 * priorities as possible, when needed. 1057 * TODO: Make # of TX rings run-time configurable 1058 */ 1059 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1060 priv->num_tx_rings = si->num_tx_rings; 1061 priv->bdr_int_num = cpus; 1062 1063 /* SI specific */ 1064 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1065 } 1066 1067 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1068 { 1069 struct enetc_si *si = priv->si; 1070 int err; 1071 1072 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1073 if (err) 1074 return err; 1075 1076 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1077 GFP_KERNEL); 1078 if (!priv->cls_rules) { 1079 err = -ENOMEM; 1080 goto err_alloc_cls; 1081 } 1082 1083 err = enetc_configure_si(priv); 1084 if (err) 1085 goto err_config_si; 1086 1087 return 0; 1088 1089 err_config_si: 1090 kfree(priv->cls_rules); 1091 err_alloc_cls: 1092 enetc_clear_cbdr(&si->hw); 1093 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1094 1095 return err; 1096 } 1097 1098 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1099 { 1100 struct enetc_si *si = priv->si; 1101 1102 enetc_clear_cbdr(&si->hw); 1103 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1104 1105 kfree(priv->cls_rules); 1106 } 1107 1108 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1109 { 1110 int idx = tx_ring->index; 1111 u32 tbmr; 1112 1113 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1114 lower_32_bits(tx_ring->bd_dma_base)); 1115 1116 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1117 upper_32_bits(tx_ring->bd_dma_base)); 1118 1119 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1120 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1121 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1122 1123 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1124 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1125 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1126 1127 /* enable Tx ints by setting pkt thr to 1 */ 1128 enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1); 1129 1130 tbmr = ENETC_TBMR_EN; 1131 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1132 tbmr |= ENETC_TBMR_VIH; 1133 1134 /* enable ring */ 1135 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1136 1137 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1138 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1139 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1140 } 1141 1142 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1143 { 1144 int idx = rx_ring->index; 1145 u32 rbmr; 1146 1147 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1148 lower_32_bits(rx_ring->bd_dma_base)); 1149 1150 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1151 upper_32_bits(rx_ring->bd_dma_base)); 1152 1153 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1154 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1155 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1156 1157 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1158 1159 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1160 1161 /* enable Rx ints by setting pkt thr to 1 */ 1162 enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1); 1163 1164 rbmr = ENETC_RBMR_EN; 1165 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1166 rbmr |= ENETC_RBMR_BDS; 1167 #endif 1168 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1169 rbmr |= ENETC_RBMR_VTE; 1170 1171 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1172 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1173 1174 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1175 1176 /* enable ring */ 1177 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1178 } 1179 1180 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1181 { 1182 int i; 1183 1184 for (i = 0; i < priv->num_tx_rings; i++) 1185 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1186 1187 for (i = 0; i < priv->num_rx_rings; i++) 1188 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1189 } 1190 1191 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1192 { 1193 int idx = rx_ring->index; 1194 1195 /* disable EN bit on ring */ 1196 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1197 } 1198 1199 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1200 { 1201 int delay = 8, timeout = 100; 1202 int idx = tx_ring->index; 1203 1204 /* disable EN bit on ring */ 1205 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1206 1207 /* wait for busy to clear */ 1208 while (delay < timeout && 1209 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1210 msleep(delay); 1211 delay *= 2; 1212 } 1213 1214 if (delay >= timeout) 1215 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1216 idx); 1217 } 1218 1219 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1220 { 1221 int i; 1222 1223 for (i = 0; i < priv->num_tx_rings; i++) 1224 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1225 1226 for (i = 0; i < priv->num_rx_rings; i++) 1227 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1228 1229 udelay(1); 1230 } 1231 1232 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1233 { 1234 struct pci_dev *pdev = priv->si->pdev; 1235 cpumask_t cpu_mask; 1236 int i, j, err; 1237 1238 for (i = 0; i < priv->bdr_int_num; i++) { 1239 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1240 struct enetc_int_vector *v = priv->int_vector[i]; 1241 int entry = ENETC_BDR_INT_BASE_IDX + i; 1242 struct enetc_hw *hw = &priv->si->hw; 1243 1244 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1245 priv->ndev->name, i); 1246 err = request_irq(irq, enetc_msix, 0, v->name, v); 1247 if (err) { 1248 dev_err(priv->dev, "request_irq() failed!\n"); 1249 goto irq_err; 1250 } 1251 1252 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1253 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1254 1255 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1256 1257 for (j = 0; j < v->count_tx_rings; j++) { 1258 int idx = v->tx_ring[j].index; 1259 1260 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1261 } 1262 cpumask_clear(&cpu_mask); 1263 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1264 irq_set_affinity_hint(irq, &cpu_mask); 1265 } 1266 1267 return 0; 1268 1269 irq_err: 1270 while (i--) { 1271 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1272 1273 irq_set_affinity_hint(irq, NULL); 1274 free_irq(irq, priv->int_vector[i]); 1275 } 1276 1277 return err; 1278 } 1279 1280 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1281 { 1282 struct pci_dev *pdev = priv->si->pdev; 1283 int i; 1284 1285 for (i = 0; i < priv->bdr_int_num; i++) { 1286 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1287 1288 irq_set_affinity_hint(irq, NULL); 1289 free_irq(irq, priv->int_vector[i]); 1290 } 1291 } 1292 1293 static void enetc_enable_interrupts(struct enetc_ndev_priv *priv) 1294 { 1295 int i; 1296 1297 /* enable Tx & Rx event indication */ 1298 for (i = 0; i < priv->num_rx_rings; i++) { 1299 enetc_rxbdr_wr(&priv->si->hw, i, 1300 ENETC_RBIER, ENETC_RBIER_RXTIE); 1301 } 1302 1303 for (i = 0; i < priv->num_tx_rings; i++) { 1304 enetc_txbdr_wr(&priv->si->hw, i, 1305 ENETC_TBIER, ENETC_TBIER_TXTIE); 1306 } 1307 } 1308 1309 static void enetc_disable_interrupts(struct enetc_ndev_priv *priv) 1310 { 1311 int i; 1312 1313 for (i = 0; i < priv->num_tx_rings; i++) 1314 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1315 1316 for (i = 0; i < priv->num_rx_rings; i++) 1317 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1318 } 1319 1320 static void adjust_link(struct net_device *ndev) 1321 { 1322 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1323 struct phy_device *phydev = ndev->phydev; 1324 1325 if (priv->active_offloads & ENETC_F_QBV) 1326 enetc_sched_speed_set(ndev); 1327 1328 phy_print_status(phydev); 1329 } 1330 1331 static int enetc_phy_connect(struct net_device *ndev) 1332 { 1333 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1334 struct phy_device *phydev; 1335 struct ethtool_eee edata; 1336 1337 if (!priv->phy_node) 1338 return 0; /* phy-less mode */ 1339 1340 phydev = of_phy_connect(ndev, priv->phy_node, &adjust_link, 1341 0, priv->if_mode); 1342 if (!phydev) { 1343 dev_err(&ndev->dev, "could not attach to PHY\n"); 1344 return -ENODEV; 1345 } 1346 1347 phy_attached_info(phydev); 1348 1349 /* disable EEE autoneg, until ENETC driver supports it */ 1350 memset(&edata, 0, sizeof(struct ethtool_eee)); 1351 phy_ethtool_set_eee(phydev, &edata); 1352 1353 return 0; 1354 } 1355 1356 int enetc_open(struct net_device *ndev) 1357 { 1358 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1359 int i, err; 1360 1361 err = enetc_setup_irqs(priv); 1362 if (err) 1363 return err; 1364 1365 err = enetc_phy_connect(ndev); 1366 if (err) 1367 goto err_phy_connect; 1368 1369 err = enetc_alloc_tx_resources(priv); 1370 if (err) 1371 goto err_alloc_tx; 1372 1373 err = enetc_alloc_rx_resources(priv); 1374 if (err) 1375 goto err_alloc_rx; 1376 1377 enetc_setup_bdrs(priv); 1378 1379 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1380 if (err) 1381 goto err_set_queues; 1382 1383 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1384 if (err) 1385 goto err_set_queues; 1386 1387 for (i = 0; i < priv->bdr_int_num; i++) 1388 napi_enable(&priv->int_vector[i]->napi); 1389 1390 enetc_enable_interrupts(priv); 1391 1392 if (ndev->phydev) 1393 phy_start(ndev->phydev); 1394 else 1395 netif_carrier_on(ndev); 1396 1397 netif_tx_start_all_queues(ndev); 1398 1399 return 0; 1400 1401 err_set_queues: 1402 enetc_free_rx_resources(priv); 1403 err_alloc_rx: 1404 enetc_free_tx_resources(priv); 1405 err_alloc_tx: 1406 if (ndev->phydev) 1407 phy_disconnect(ndev->phydev); 1408 err_phy_connect: 1409 enetc_free_irqs(priv); 1410 1411 return err; 1412 } 1413 1414 int enetc_close(struct net_device *ndev) 1415 { 1416 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1417 int i; 1418 1419 netif_tx_stop_all_queues(ndev); 1420 1421 if (ndev->phydev) { 1422 phy_stop(ndev->phydev); 1423 phy_disconnect(ndev->phydev); 1424 } else { 1425 netif_carrier_off(ndev); 1426 } 1427 1428 for (i = 0; i < priv->bdr_int_num; i++) { 1429 napi_synchronize(&priv->int_vector[i]->napi); 1430 napi_disable(&priv->int_vector[i]->napi); 1431 } 1432 1433 enetc_disable_interrupts(priv); 1434 enetc_clear_bdrs(priv); 1435 1436 enetc_free_rxtx_rings(priv); 1437 enetc_free_rx_resources(priv); 1438 enetc_free_tx_resources(priv); 1439 enetc_free_irqs(priv); 1440 1441 return 0; 1442 } 1443 1444 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1445 { 1446 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1447 struct tc_mqprio_qopt *mqprio = type_data; 1448 struct enetc_bdr *tx_ring; 1449 u8 num_tc; 1450 int i; 1451 1452 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1453 num_tc = mqprio->num_tc; 1454 1455 if (!num_tc) { 1456 netdev_reset_tc(ndev); 1457 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1458 1459 /* Reset all ring priorities to 0 */ 1460 for (i = 0; i < priv->num_tx_rings; i++) { 1461 tx_ring = priv->tx_ring[i]; 1462 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1463 } 1464 1465 return 0; 1466 } 1467 1468 /* Check if we have enough BD rings available to accommodate all TCs */ 1469 if (num_tc > priv->num_tx_rings) { 1470 netdev_err(ndev, "Max %d traffic classes supported\n", 1471 priv->num_tx_rings); 1472 return -EINVAL; 1473 } 1474 1475 /* For the moment, we use only one BD ring per TC. 1476 * 1477 * Configure num_tc BD rings with increasing priorities. 1478 */ 1479 for (i = 0; i < num_tc; i++) { 1480 tx_ring = priv->tx_ring[i]; 1481 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1482 } 1483 1484 /* Reset the number of netdev queues based on the TC count */ 1485 netif_set_real_num_tx_queues(ndev, num_tc); 1486 1487 netdev_set_num_tc(ndev, num_tc); 1488 1489 /* Each TC is associated with one netdev queue */ 1490 for (i = 0; i < num_tc; i++) 1491 netdev_set_tc_queue(ndev, i, 1, i); 1492 1493 return 0; 1494 } 1495 1496 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1497 void *type_data) 1498 { 1499 switch (type) { 1500 case TC_SETUP_QDISC_MQPRIO: 1501 return enetc_setup_tc_mqprio(ndev, type_data); 1502 case TC_SETUP_QDISC_TAPRIO: 1503 return enetc_setup_tc_taprio(ndev, type_data); 1504 case TC_SETUP_QDISC_CBS: 1505 return enetc_setup_tc_cbs(ndev, type_data); 1506 default: 1507 return -EOPNOTSUPP; 1508 } 1509 } 1510 1511 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1512 { 1513 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1514 struct net_device_stats *stats = &ndev->stats; 1515 unsigned long packets = 0, bytes = 0; 1516 int i; 1517 1518 for (i = 0; i < priv->num_rx_rings; i++) { 1519 packets += priv->rx_ring[i]->stats.packets; 1520 bytes += priv->rx_ring[i]->stats.bytes; 1521 } 1522 1523 stats->rx_packets = packets; 1524 stats->rx_bytes = bytes; 1525 bytes = 0; 1526 packets = 0; 1527 1528 for (i = 0; i < priv->num_tx_rings; i++) { 1529 packets += priv->tx_ring[i]->stats.packets; 1530 bytes += priv->tx_ring[i]->stats.bytes; 1531 } 1532 1533 stats->tx_packets = packets; 1534 stats->tx_bytes = bytes; 1535 1536 return stats; 1537 } 1538 1539 static int enetc_set_rss(struct net_device *ndev, int en) 1540 { 1541 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1542 struct enetc_hw *hw = &priv->si->hw; 1543 u32 reg; 1544 1545 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1546 1547 reg = enetc_rd(hw, ENETC_SIMR); 1548 reg &= ~ENETC_SIMR_RSSE; 1549 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1550 enetc_wr(hw, ENETC_SIMR, reg); 1551 1552 return 0; 1553 } 1554 1555 int enetc_set_features(struct net_device *ndev, 1556 netdev_features_t features) 1557 { 1558 netdev_features_t changed = ndev->features ^ features; 1559 1560 if (changed & NETIF_F_RXHASH) 1561 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1562 1563 return 0; 1564 } 1565 1566 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1567 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1568 { 1569 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1570 struct hwtstamp_config config; 1571 1572 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1573 return -EFAULT; 1574 1575 switch (config.tx_type) { 1576 case HWTSTAMP_TX_OFF: 1577 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1578 break; 1579 case HWTSTAMP_TX_ON: 1580 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1581 break; 1582 default: 1583 return -ERANGE; 1584 } 1585 1586 switch (config.rx_filter) { 1587 case HWTSTAMP_FILTER_NONE: 1588 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1589 break; 1590 default: 1591 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1592 config.rx_filter = HWTSTAMP_FILTER_ALL; 1593 } 1594 1595 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1596 -EFAULT : 0; 1597 } 1598 1599 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1600 { 1601 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1602 struct hwtstamp_config config; 1603 1604 config.flags = 0; 1605 1606 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1607 config.tx_type = HWTSTAMP_TX_ON; 1608 else 1609 config.tx_type = HWTSTAMP_TX_OFF; 1610 1611 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1612 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1613 1614 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1615 -EFAULT : 0; 1616 } 1617 #endif 1618 1619 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1620 { 1621 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1622 if (cmd == SIOCSHWTSTAMP) 1623 return enetc_hwtstamp_set(ndev, rq); 1624 if (cmd == SIOCGHWTSTAMP) 1625 return enetc_hwtstamp_get(ndev, rq); 1626 #endif 1627 1628 if (!ndev->phydev) 1629 return -EOPNOTSUPP; 1630 return phy_mii_ioctl(ndev->phydev, rq, cmd); 1631 } 1632 1633 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1634 { 1635 struct pci_dev *pdev = priv->si->pdev; 1636 int size, v_tx_rings; 1637 int i, n, err, nvec; 1638 1639 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1640 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1641 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1642 1643 if (n < 0) 1644 return n; 1645 1646 if (n != nvec) 1647 return -EPERM; 1648 1649 /* # of tx rings per int vector */ 1650 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1651 size = sizeof(struct enetc_int_vector) + 1652 sizeof(struct enetc_bdr) * v_tx_rings; 1653 1654 for (i = 0; i < priv->bdr_int_num; i++) { 1655 struct enetc_int_vector *v; 1656 struct enetc_bdr *bdr; 1657 int j; 1658 1659 v = kzalloc(size, GFP_KERNEL); 1660 if (!v) { 1661 err = -ENOMEM; 1662 goto fail; 1663 } 1664 1665 priv->int_vector[i] = v; 1666 1667 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1668 NAPI_POLL_WEIGHT); 1669 v->count_tx_rings = v_tx_rings; 1670 1671 for (j = 0; j < v_tx_rings; j++) { 1672 int idx; 1673 1674 /* default tx ring mapping policy */ 1675 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1676 idx = 2 * j + i; /* 2 CPUs */ 1677 else 1678 idx = j + i * v_tx_rings; /* default */ 1679 1680 __set_bit(idx, &v->tx_rings_map); 1681 bdr = &v->tx_ring[j]; 1682 bdr->index = idx; 1683 bdr->ndev = priv->ndev; 1684 bdr->dev = priv->dev; 1685 bdr->bd_count = priv->tx_bd_count; 1686 priv->tx_ring[idx] = bdr; 1687 } 1688 1689 bdr = &v->rx_ring; 1690 bdr->index = i; 1691 bdr->ndev = priv->ndev; 1692 bdr->dev = priv->dev; 1693 bdr->bd_count = priv->rx_bd_count; 1694 priv->rx_ring[i] = bdr; 1695 } 1696 1697 return 0; 1698 1699 fail: 1700 while (i--) { 1701 netif_napi_del(&priv->int_vector[i]->napi); 1702 kfree(priv->int_vector[i]); 1703 } 1704 1705 pci_free_irq_vectors(pdev); 1706 1707 return err; 1708 } 1709 1710 void enetc_free_msix(struct enetc_ndev_priv *priv) 1711 { 1712 int i; 1713 1714 for (i = 0; i < priv->bdr_int_num; i++) { 1715 struct enetc_int_vector *v = priv->int_vector[i]; 1716 1717 netif_napi_del(&v->napi); 1718 } 1719 1720 for (i = 0; i < priv->num_rx_rings; i++) 1721 priv->rx_ring[i] = NULL; 1722 1723 for (i = 0; i < priv->num_tx_rings; i++) 1724 priv->tx_ring[i] = NULL; 1725 1726 for (i = 0; i < priv->bdr_int_num; i++) { 1727 kfree(priv->int_vector[i]); 1728 priv->int_vector[i] = NULL; 1729 } 1730 1731 /* disable all MSIX for this device */ 1732 pci_free_irq_vectors(priv->si->pdev); 1733 } 1734 1735 static void enetc_kfree_si(struct enetc_si *si) 1736 { 1737 char *p = (char *)si - si->pad; 1738 1739 kfree(p); 1740 } 1741 1742 static void enetc_detect_errata(struct enetc_si *si) 1743 { 1744 if (si->pdev->revision == ENETC_REV1) 1745 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL | 1746 ENETC_ERR_UCMCSWP; 1747 } 1748 1749 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1750 { 1751 struct enetc_si *si, *p; 1752 struct enetc_hw *hw; 1753 size_t alloc_size; 1754 int err, len; 1755 1756 pcie_flr(pdev); 1757 err = pci_enable_device_mem(pdev); 1758 if (err) { 1759 dev_err(&pdev->dev, "device enable failed\n"); 1760 return err; 1761 } 1762 1763 /* set up for high or low dma */ 1764 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1765 if (err) { 1766 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1767 if (err) { 1768 dev_err(&pdev->dev, 1769 "DMA configuration failed: 0x%x\n", err); 1770 goto err_dma; 1771 } 1772 } 1773 1774 err = pci_request_mem_regions(pdev, name); 1775 if (err) { 1776 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1777 goto err_pci_mem_reg; 1778 } 1779 1780 pci_set_master(pdev); 1781 1782 alloc_size = sizeof(struct enetc_si); 1783 if (sizeof_priv) { 1784 /* align priv to 32B */ 1785 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1786 alloc_size += sizeof_priv; 1787 } 1788 /* force 32B alignment for enetc_si */ 1789 alloc_size += ENETC_SI_ALIGN - 1; 1790 1791 p = kzalloc(alloc_size, GFP_KERNEL); 1792 if (!p) { 1793 err = -ENOMEM; 1794 goto err_alloc_si; 1795 } 1796 1797 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1798 si->pad = (char *)si - (char *)p; 1799 1800 pci_set_drvdata(pdev, si); 1801 si->pdev = pdev; 1802 hw = &si->hw; 1803 1804 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1805 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1806 if (!hw->reg) { 1807 err = -ENXIO; 1808 dev_err(&pdev->dev, "ioremap() failed\n"); 1809 goto err_ioremap; 1810 } 1811 if (len > ENETC_PORT_BASE) 1812 hw->port = hw->reg + ENETC_PORT_BASE; 1813 if (len > ENETC_GLOBAL_BASE) 1814 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1815 1816 enetc_detect_errata(si); 1817 1818 return 0; 1819 1820 err_ioremap: 1821 enetc_kfree_si(si); 1822 err_alloc_si: 1823 pci_release_mem_regions(pdev); 1824 err_pci_mem_reg: 1825 err_dma: 1826 pci_disable_device(pdev); 1827 1828 return err; 1829 } 1830 1831 void enetc_pci_remove(struct pci_dev *pdev) 1832 { 1833 struct enetc_si *si = pci_get_drvdata(pdev); 1834 struct enetc_hw *hw = &si->hw; 1835 1836 iounmap(hw->reg); 1837 enetc_kfree_si(si); 1838 pci_release_mem_regions(pdev); 1839 pci_disable_device(pdev); 1840 } 1841