1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/vmalloc.h> 8 #include <net/pkt_sched.h> 9 10 /* ENETC overhead: optional extension BD + 1 BD gap */ 11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 12 /* max # of chained Tx BDs is 15, including head and extension BD */ 13 #define ENETC_MAX_SKB_FRAGS 13 14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 15 16 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 17 struct enetc_tx_swbd *tx_swbd) 18 { 19 if (tx_swbd->is_dma_page) 20 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 21 tx_swbd->len, DMA_TO_DEVICE); 22 else 23 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 24 tx_swbd->len, DMA_TO_DEVICE); 25 tx_swbd->dma = 0; 26 } 27 28 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 29 struct enetc_tx_swbd *tx_swbd) 30 { 31 if (tx_swbd->dma) 32 enetc_unmap_tx_buff(tx_ring, tx_swbd); 33 34 if (tx_swbd->skb) { 35 dev_kfree_skb_any(tx_swbd->skb); 36 tx_swbd->skb = NULL; 37 } 38 } 39 40 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 41 int active_offloads) 42 { 43 struct enetc_tx_swbd *tx_swbd; 44 skb_frag_t *frag; 45 int len = skb_headlen(skb); 46 union enetc_tx_bd temp_bd; 47 union enetc_tx_bd *txbd; 48 bool do_vlan, do_tstamp; 49 int i, count = 0; 50 unsigned int f; 51 dma_addr_t dma; 52 u8 flags = 0; 53 54 i = tx_ring->next_to_use; 55 txbd = ENETC_TXBD(*tx_ring, i); 56 prefetchw(txbd); 57 58 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 59 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 60 goto dma_err; 61 62 temp_bd.addr = cpu_to_le64(dma); 63 temp_bd.buf_len = cpu_to_le16(len); 64 temp_bd.lstatus = 0; 65 66 tx_swbd = &tx_ring->tx_swbd[i]; 67 tx_swbd->dma = dma; 68 tx_swbd->len = len; 69 tx_swbd->is_dma_page = 0; 70 count++; 71 72 do_vlan = skb_vlan_tag_present(skb); 73 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 74 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 75 tx_swbd->do_tstamp = do_tstamp; 76 tx_swbd->check_wb = tx_swbd->do_tstamp; 77 78 if (do_vlan || do_tstamp) 79 flags |= ENETC_TXBD_FLAGS_EX; 80 81 if (tx_ring->tsd_enable) 82 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 83 84 /* first BD needs frm_len and offload flags set */ 85 temp_bd.frm_len = cpu_to_le16(skb->len); 86 temp_bd.flags = flags; 87 88 if (flags & ENETC_TXBD_FLAGS_TSE) 89 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 90 flags); 91 92 if (flags & ENETC_TXBD_FLAGS_EX) { 93 u8 e_flags = 0; 94 *txbd = temp_bd; 95 enetc_clear_tx_bd(&temp_bd); 96 97 /* add extension BD for VLAN and/or timestamping */ 98 flags = 0; 99 tx_swbd++; 100 txbd++; 101 i++; 102 if (unlikely(i == tx_ring->bd_count)) { 103 i = 0; 104 tx_swbd = tx_ring->tx_swbd; 105 txbd = ENETC_TXBD(*tx_ring, 0); 106 } 107 prefetchw(txbd); 108 109 if (do_vlan) { 110 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 111 temp_bd.ext.tpid = 0; /* < C-TAG */ 112 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 113 } 114 115 if (do_tstamp) { 116 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 117 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 118 } 119 120 temp_bd.ext.e_flags = e_flags; 121 count++; 122 } 123 124 frag = &skb_shinfo(skb)->frags[0]; 125 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 126 len = skb_frag_size(frag); 127 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 128 DMA_TO_DEVICE); 129 if (dma_mapping_error(tx_ring->dev, dma)) 130 goto dma_err; 131 132 *txbd = temp_bd; 133 enetc_clear_tx_bd(&temp_bd); 134 135 flags = 0; 136 tx_swbd++; 137 txbd++; 138 i++; 139 if (unlikely(i == tx_ring->bd_count)) { 140 i = 0; 141 tx_swbd = tx_ring->tx_swbd; 142 txbd = ENETC_TXBD(*tx_ring, 0); 143 } 144 prefetchw(txbd); 145 146 temp_bd.addr = cpu_to_le64(dma); 147 temp_bd.buf_len = cpu_to_le16(len); 148 149 tx_swbd->dma = dma; 150 tx_swbd->len = len; 151 tx_swbd->is_dma_page = 1; 152 count++; 153 } 154 155 /* last BD needs 'F' bit set */ 156 flags |= ENETC_TXBD_FLAGS_F; 157 temp_bd.flags = flags; 158 *txbd = temp_bd; 159 160 tx_ring->tx_swbd[i].skb = skb; 161 162 enetc_bdr_idx_inc(tx_ring, &i); 163 tx_ring->next_to_use = i; 164 165 skb_tx_timestamp(skb); 166 167 /* let H/W know BD ring has been updated */ 168 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */ 169 170 return count; 171 172 dma_err: 173 dev_err(tx_ring->dev, "DMA map error"); 174 175 do { 176 tx_swbd = &tx_ring->tx_swbd[i]; 177 enetc_free_tx_skb(tx_ring, tx_swbd); 178 if (i == 0) 179 i = tx_ring->bd_count; 180 i--; 181 } while (count--); 182 183 return 0; 184 } 185 186 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 187 { 188 struct enetc_ndev_priv *priv = netdev_priv(ndev); 189 struct enetc_bdr *tx_ring; 190 int count; 191 192 tx_ring = priv->tx_ring[skb->queue_mapping]; 193 194 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 195 if (unlikely(skb_linearize(skb))) 196 goto drop_packet_err; 197 198 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 199 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 200 netif_stop_subqueue(ndev, tx_ring->index); 201 return NETDEV_TX_BUSY; 202 } 203 204 enetc_lock_mdio(); 205 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 206 enetc_unlock_mdio(); 207 208 if (unlikely(!count)) 209 goto drop_packet_err; 210 211 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 212 netif_stop_subqueue(ndev, tx_ring->index); 213 214 return NETDEV_TX_OK; 215 216 drop_packet_err: 217 dev_kfree_skb_any(skb); 218 return NETDEV_TX_OK; 219 } 220 221 static irqreturn_t enetc_msix(int irq, void *data) 222 { 223 struct enetc_int_vector *v = data; 224 int i; 225 226 enetc_lock_mdio(); 227 228 /* disable interrupts */ 229 enetc_wr_reg_hot(v->rbier, 0); 230 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 231 232 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 233 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 234 235 enetc_unlock_mdio(); 236 237 napi_schedule(&v->napi); 238 239 return IRQ_HANDLED; 240 } 241 242 static void enetc_rx_dim_work(struct work_struct *w) 243 { 244 struct dim *dim = container_of(w, struct dim, work); 245 struct dim_cq_moder moder = 246 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 247 struct enetc_int_vector *v = 248 container_of(dim, struct enetc_int_vector, rx_dim); 249 250 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 251 dim->state = DIM_START_MEASURE; 252 } 253 254 static void enetc_rx_net_dim(struct enetc_int_vector *v) 255 { 256 struct dim_sample dim_sample; 257 258 v->comp_cnt++; 259 260 if (!v->rx_napi_work) 261 return; 262 263 dim_update_sample(v->comp_cnt, 264 v->rx_ring.stats.packets, 265 v->rx_ring.stats.bytes, 266 &dim_sample); 267 net_dim(&v->rx_dim, dim_sample); 268 } 269 270 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 271 { 272 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 273 274 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 275 } 276 277 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 278 u64 *tstamp) 279 { 280 u32 lo, hi, tstamp_lo; 281 282 lo = enetc_rd_hot(hw, ENETC_SICTR0); 283 hi = enetc_rd_hot(hw, ENETC_SICTR1); 284 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 285 if (lo <= tstamp_lo) 286 hi -= 1; 287 *tstamp = (u64)hi << 32 | tstamp_lo; 288 } 289 290 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 291 { 292 struct skb_shared_hwtstamps shhwtstamps; 293 294 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 295 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 296 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 297 skb_txtime_consumed(skb); 298 skb_tstamp_tx(skb, &shhwtstamps); 299 } 300 } 301 302 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 303 { 304 struct net_device *ndev = tx_ring->ndev; 305 int tx_frm_cnt = 0, tx_byte_cnt = 0; 306 struct enetc_tx_swbd *tx_swbd; 307 int i, bds_to_clean; 308 bool do_tstamp; 309 u64 tstamp = 0; 310 311 i = tx_ring->next_to_clean; 312 tx_swbd = &tx_ring->tx_swbd[i]; 313 314 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 315 316 do_tstamp = false; 317 318 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 319 bool is_eof = !!tx_swbd->skb; 320 321 if (unlikely(tx_swbd->check_wb)) { 322 struct enetc_ndev_priv *priv = netdev_priv(ndev); 323 union enetc_tx_bd *txbd; 324 325 txbd = ENETC_TXBD(*tx_ring, i); 326 327 if (txbd->flags & ENETC_TXBD_FLAGS_W && 328 tx_swbd->do_tstamp) { 329 enetc_get_tx_tstamp(&priv->si->hw, txbd, 330 &tstamp); 331 do_tstamp = true; 332 } 333 } 334 335 if (likely(tx_swbd->dma)) 336 enetc_unmap_tx_buff(tx_ring, tx_swbd); 337 338 if (is_eof) { 339 if (unlikely(do_tstamp)) { 340 enetc_tstamp_tx(tx_swbd->skb, tstamp); 341 do_tstamp = false; 342 } 343 napi_consume_skb(tx_swbd->skb, napi_budget); 344 tx_swbd->skb = NULL; 345 } 346 347 tx_byte_cnt += tx_swbd->len; 348 349 bds_to_clean--; 350 tx_swbd++; 351 i++; 352 if (unlikely(i == tx_ring->bd_count)) { 353 i = 0; 354 tx_swbd = tx_ring->tx_swbd; 355 } 356 357 /* BD iteration loop end */ 358 if (is_eof) { 359 tx_frm_cnt++; 360 /* re-arm interrupt source */ 361 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 362 BIT(16 + tx_ring->index)); 363 } 364 365 if (unlikely(!bds_to_clean)) 366 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 367 } 368 369 tx_ring->next_to_clean = i; 370 tx_ring->stats.packets += tx_frm_cnt; 371 tx_ring->stats.bytes += tx_byte_cnt; 372 373 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 374 __netif_subqueue_stopped(ndev, tx_ring->index) && 375 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 376 netif_wake_subqueue(ndev, tx_ring->index); 377 } 378 379 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 380 } 381 382 static bool enetc_new_page(struct enetc_bdr *rx_ring, 383 struct enetc_rx_swbd *rx_swbd) 384 { 385 struct page *page; 386 dma_addr_t addr; 387 388 page = dev_alloc_page(); 389 if (unlikely(!page)) 390 return false; 391 392 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 393 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 394 __free_page(page); 395 396 return false; 397 } 398 399 rx_swbd->dma = addr; 400 rx_swbd->page = page; 401 rx_swbd->page_offset = ENETC_RXB_PAD; 402 403 return true; 404 } 405 406 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 407 { 408 struct enetc_rx_swbd *rx_swbd; 409 union enetc_rx_bd *rxbd; 410 int i, j; 411 412 i = rx_ring->next_to_use; 413 rx_swbd = &rx_ring->rx_swbd[i]; 414 rxbd = enetc_rxbd(rx_ring, i); 415 416 for (j = 0; j < buff_cnt; j++) { 417 /* try reuse page */ 418 if (unlikely(!rx_swbd->page)) { 419 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 420 rx_ring->stats.rx_alloc_errs++; 421 break; 422 } 423 } 424 425 /* update RxBD */ 426 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 427 rx_swbd->page_offset); 428 /* clear 'R" as well */ 429 rxbd->r.lstatus = 0; 430 431 enetc_rxbd_next(rx_ring, &rxbd, &i); 432 rx_swbd = &rx_ring->rx_swbd[i]; 433 } 434 435 if (likely(j)) { 436 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 437 rx_ring->next_to_use = i; 438 439 /* update ENETC's consumer index */ 440 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 441 } 442 443 return j; 444 } 445 446 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 447 static void enetc_get_rx_tstamp(struct net_device *ndev, 448 union enetc_rx_bd *rxbd, 449 struct sk_buff *skb) 450 { 451 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 452 struct enetc_ndev_priv *priv = netdev_priv(ndev); 453 struct enetc_hw *hw = &priv->si->hw; 454 u32 lo, hi, tstamp_lo; 455 u64 tstamp; 456 457 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 458 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 459 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 460 rxbd = enetc_rxbd_ext(rxbd); 461 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 462 if (lo <= tstamp_lo) 463 hi -= 1; 464 465 tstamp = (u64)hi << 32 | tstamp_lo; 466 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 467 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 468 } 469 } 470 #endif 471 472 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 473 union enetc_rx_bd *rxbd, struct sk_buff *skb) 474 { 475 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 476 477 /* TODO: hashing */ 478 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 479 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 480 481 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 482 skb->ip_summed = CHECKSUM_COMPLETE; 483 } 484 485 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 486 __be16 tpid = 0; 487 488 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 489 case 0: 490 tpid = htons(ETH_P_8021Q); 491 break; 492 case 1: 493 tpid = htons(ETH_P_8021AD); 494 break; 495 case 2: 496 tpid = htons(enetc_port_rd(&priv->si->hw, 497 ENETC_PCVLANR1)); 498 break; 499 case 3: 500 tpid = htons(enetc_port_rd(&priv->si->hw, 501 ENETC_PCVLANR2)); 502 break; 503 default: 504 break; 505 } 506 507 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 508 } 509 510 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 511 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 512 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 513 #endif 514 } 515 516 static void enetc_process_skb(struct enetc_bdr *rx_ring, 517 struct sk_buff *skb) 518 { 519 skb_record_rx_queue(skb, rx_ring->index); 520 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 521 } 522 523 static bool enetc_page_reusable(struct page *page) 524 { 525 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 526 } 527 528 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 529 struct enetc_rx_swbd *old) 530 { 531 struct enetc_rx_swbd *new; 532 533 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 534 535 /* next buf that may reuse a page */ 536 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 537 538 /* copy page reference */ 539 *new = *old; 540 } 541 542 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 543 int i, u16 size) 544 { 545 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 546 547 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 548 rx_swbd->page_offset, 549 size, DMA_FROM_DEVICE); 550 return rx_swbd; 551 } 552 553 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 554 struct enetc_rx_swbd *rx_swbd) 555 { 556 if (likely(enetc_page_reusable(rx_swbd->page))) { 557 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 558 page_ref_inc(rx_swbd->page); 559 560 enetc_reuse_page(rx_ring, rx_swbd); 561 562 /* sync for use by the device */ 563 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 564 rx_swbd->page_offset, 565 ENETC_RXB_DMA_SIZE, 566 DMA_FROM_DEVICE); 567 } else { 568 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 569 PAGE_SIZE, DMA_FROM_DEVICE); 570 } 571 572 rx_swbd->page = NULL; 573 } 574 575 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 576 int i, u16 size) 577 { 578 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 579 struct sk_buff *skb; 580 void *ba; 581 582 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 583 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 584 if (unlikely(!skb)) { 585 rx_ring->stats.rx_alloc_errs++; 586 return NULL; 587 } 588 589 skb_reserve(skb, ENETC_RXB_PAD); 590 __skb_put(skb, size); 591 592 enetc_put_rx_buff(rx_ring, rx_swbd); 593 594 return skb; 595 } 596 597 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 598 u16 size, struct sk_buff *skb) 599 { 600 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 601 602 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 603 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 604 605 enetc_put_rx_buff(rx_ring, rx_swbd); 606 } 607 608 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 609 610 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 611 struct napi_struct *napi, int work_limit) 612 { 613 int rx_frm_cnt = 0, rx_byte_cnt = 0; 614 int cleaned_cnt, i; 615 616 cleaned_cnt = enetc_bd_unused(rx_ring); 617 /* next descriptor to process */ 618 i = rx_ring->next_to_clean; 619 620 while (likely(rx_frm_cnt < work_limit)) { 621 union enetc_rx_bd *rxbd; 622 struct sk_buff *skb; 623 u32 bd_status; 624 u16 size; 625 626 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 627 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 628 cleaned_cnt); 629 630 rxbd = enetc_rxbd(rx_ring, i); 631 bd_status = le32_to_cpu(rxbd->r.lstatus); 632 if (!bd_status) 633 break; 634 635 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 636 dma_rmb(); /* for reading other rxbd fields */ 637 size = le16_to_cpu(rxbd->r.buf_len); 638 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 639 if (!skb) 640 break; 641 642 enetc_get_offloads(rx_ring, rxbd, skb); 643 644 cleaned_cnt++; 645 646 enetc_rxbd_next(rx_ring, &rxbd, &i); 647 648 if (unlikely(bd_status & 649 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 650 dev_kfree_skb(skb); 651 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 652 dma_rmb(); 653 bd_status = le32_to_cpu(rxbd->r.lstatus); 654 655 enetc_rxbd_next(rx_ring, &rxbd, &i); 656 } 657 658 rx_ring->ndev->stats.rx_dropped++; 659 rx_ring->ndev->stats.rx_errors++; 660 661 break; 662 } 663 664 /* not last BD in frame? */ 665 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 666 bd_status = le32_to_cpu(rxbd->r.lstatus); 667 size = ENETC_RXB_DMA_SIZE; 668 669 if (bd_status & ENETC_RXBD_LSTATUS_F) { 670 dma_rmb(); 671 size = le16_to_cpu(rxbd->r.buf_len); 672 } 673 674 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 675 676 cleaned_cnt++; 677 678 enetc_rxbd_next(rx_ring, &rxbd, &i); 679 } 680 681 rx_byte_cnt += skb->len; 682 683 enetc_process_skb(rx_ring, skb); 684 685 napi_gro_receive(napi, skb); 686 687 rx_frm_cnt++; 688 } 689 690 rx_ring->next_to_clean = i; 691 692 rx_ring->stats.packets += rx_frm_cnt; 693 rx_ring->stats.bytes += rx_byte_cnt; 694 695 return rx_frm_cnt; 696 } 697 698 static int enetc_poll(struct napi_struct *napi, int budget) 699 { 700 struct enetc_int_vector 701 *v = container_of(napi, struct enetc_int_vector, napi); 702 bool complete = true; 703 int work_done; 704 int i; 705 706 enetc_lock_mdio(); 707 708 for (i = 0; i < v->count_tx_rings; i++) 709 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 710 complete = false; 711 712 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 713 if (work_done == budget) 714 complete = false; 715 if (work_done) 716 v->rx_napi_work = true; 717 718 if (!complete) { 719 enetc_unlock_mdio(); 720 return budget; 721 } 722 723 napi_complete_done(napi, work_done); 724 725 if (likely(v->rx_dim_en)) 726 enetc_rx_net_dim(v); 727 728 v->rx_napi_work = false; 729 730 /* enable interrupts */ 731 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 732 733 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 734 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 735 ENETC_TBIER_TXTIE); 736 737 enetc_unlock_mdio(); 738 739 return work_done; 740 } 741 742 /* Probing and Init */ 743 #define ENETC_MAX_RFS_SIZE 64 744 void enetc_get_si_caps(struct enetc_si *si) 745 { 746 struct enetc_hw *hw = &si->hw; 747 u32 val; 748 749 /* find out how many of various resources we have to work with */ 750 val = enetc_rd(hw, ENETC_SICAPR0); 751 si->num_rx_rings = (val >> 16) & 0xff; 752 si->num_tx_rings = val & 0xff; 753 754 val = enetc_rd(hw, ENETC_SIRFSCAPR); 755 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 756 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 757 758 si->num_rss = 0; 759 val = enetc_rd(hw, ENETC_SIPCAPR0); 760 if (val & ENETC_SIPCAPR0_RSS) { 761 u32 rss; 762 763 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 764 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 765 } 766 767 if (val & ENETC_SIPCAPR0_QBV) 768 si->hw_features |= ENETC_SI_F_QBV; 769 770 if (val & ENETC_SIPCAPR0_PSFP) 771 si->hw_features |= ENETC_SI_F_PSFP; 772 } 773 774 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 775 { 776 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 777 &r->bd_dma_base, GFP_KERNEL); 778 if (!r->bd_base) 779 return -ENOMEM; 780 781 /* h/w requires 128B alignment */ 782 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 783 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 784 r->bd_dma_base); 785 return -EINVAL; 786 } 787 788 return 0; 789 } 790 791 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 792 { 793 int err; 794 795 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 796 if (!txr->tx_swbd) 797 return -ENOMEM; 798 799 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 800 if (err) { 801 vfree(txr->tx_swbd); 802 return err; 803 } 804 805 txr->next_to_clean = 0; 806 txr->next_to_use = 0; 807 808 return 0; 809 } 810 811 static void enetc_free_txbdr(struct enetc_bdr *txr) 812 { 813 int size, i; 814 815 for (i = 0; i < txr->bd_count; i++) 816 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 817 818 size = txr->bd_count * sizeof(union enetc_tx_bd); 819 820 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 821 txr->bd_base = NULL; 822 823 vfree(txr->tx_swbd); 824 txr->tx_swbd = NULL; 825 } 826 827 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 828 { 829 int i, err; 830 831 for (i = 0; i < priv->num_tx_rings; i++) { 832 err = enetc_alloc_txbdr(priv->tx_ring[i]); 833 834 if (err) 835 goto fail; 836 } 837 838 return 0; 839 840 fail: 841 while (i-- > 0) 842 enetc_free_txbdr(priv->tx_ring[i]); 843 844 return err; 845 } 846 847 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 848 { 849 int i; 850 851 for (i = 0; i < priv->num_tx_rings; i++) 852 enetc_free_txbdr(priv->tx_ring[i]); 853 } 854 855 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 856 { 857 size_t size = sizeof(union enetc_rx_bd); 858 int err; 859 860 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 861 if (!rxr->rx_swbd) 862 return -ENOMEM; 863 864 if (extended) 865 size *= 2; 866 867 err = enetc_dma_alloc_bdr(rxr, size); 868 if (err) { 869 vfree(rxr->rx_swbd); 870 return err; 871 } 872 873 rxr->next_to_clean = 0; 874 rxr->next_to_use = 0; 875 rxr->next_to_alloc = 0; 876 rxr->ext_en = extended; 877 878 return 0; 879 } 880 881 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 882 { 883 int size; 884 885 size = rxr->bd_count * sizeof(union enetc_rx_bd); 886 887 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 888 rxr->bd_base = NULL; 889 890 vfree(rxr->rx_swbd); 891 rxr->rx_swbd = NULL; 892 } 893 894 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 895 { 896 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 897 int i, err; 898 899 for (i = 0; i < priv->num_rx_rings; i++) { 900 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 901 902 if (err) 903 goto fail; 904 } 905 906 return 0; 907 908 fail: 909 while (i-- > 0) 910 enetc_free_rxbdr(priv->rx_ring[i]); 911 912 return err; 913 } 914 915 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 916 { 917 int i; 918 919 for (i = 0; i < priv->num_rx_rings; i++) 920 enetc_free_rxbdr(priv->rx_ring[i]); 921 } 922 923 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 924 { 925 int i; 926 927 if (!tx_ring->tx_swbd) 928 return; 929 930 for (i = 0; i < tx_ring->bd_count; i++) { 931 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 932 933 enetc_free_tx_skb(tx_ring, tx_swbd); 934 } 935 936 tx_ring->next_to_clean = 0; 937 tx_ring->next_to_use = 0; 938 } 939 940 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 941 { 942 int i; 943 944 if (!rx_ring->rx_swbd) 945 return; 946 947 for (i = 0; i < rx_ring->bd_count; i++) { 948 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 949 950 if (!rx_swbd->page) 951 continue; 952 953 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 954 PAGE_SIZE, DMA_FROM_DEVICE); 955 __free_page(rx_swbd->page); 956 rx_swbd->page = NULL; 957 } 958 959 rx_ring->next_to_clean = 0; 960 rx_ring->next_to_use = 0; 961 rx_ring->next_to_alloc = 0; 962 } 963 964 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 965 { 966 int i; 967 968 for (i = 0; i < priv->num_rx_rings; i++) 969 enetc_free_rx_ring(priv->rx_ring[i]); 970 971 for (i = 0; i < priv->num_tx_rings; i++) 972 enetc_free_tx_ring(priv->tx_ring[i]); 973 } 974 975 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 976 { 977 int *rss_table; 978 int i; 979 980 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 981 if (!rss_table) 982 return -ENOMEM; 983 984 /* Set up RSS table defaults */ 985 for (i = 0; i < si->num_rss; i++) 986 rss_table[i] = i % num_groups; 987 988 enetc_set_rss_table(si, rss_table, si->num_rss); 989 990 kfree(rss_table); 991 992 return 0; 993 } 994 995 int enetc_configure_si(struct enetc_ndev_priv *priv) 996 { 997 struct enetc_si *si = priv->si; 998 struct enetc_hw *hw = &si->hw; 999 int err; 1000 1001 /* set SI cache attributes */ 1002 enetc_wr(hw, ENETC_SICAR0, 1003 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1004 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1005 /* enable SI */ 1006 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1007 1008 if (si->num_rss) { 1009 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1010 if (err) 1011 return err; 1012 } 1013 1014 return 0; 1015 } 1016 1017 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1018 { 1019 struct enetc_si *si = priv->si; 1020 int cpus = num_online_cpus(); 1021 1022 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 1023 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 1024 1025 /* Enable all available TX rings in order to configure as many 1026 * priorities as possible, when needed. 1027 * TODO: Make # of TX rings run-time configurable 1028 */ 1029 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1030 priv->num_tx_rings = si->num_tx_rings; 1031 priv->bdr_int_num = cpus; 1032 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 1033 priv->tx_ictt = ENETC_TXIC_TIMETHR; 1034 } 1035 1036 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1037 { 1038 struct enetc_si *si = priv->si; 1039 1040 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1041 GFP_KERNEL); 1042 if (!priv->cls_rules) 1043 return -ENOMEM; 1044 1045 return 0; 1046 } 1047 1048 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1049 { 1050 kfree(priv->cls_rules); 1051 } 1052 1053 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1054 { 1055 int idx = tx_ring->index; 1056 u32 tbmr; 1057 1058 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1059 lower_32_bits(tx_ring->bd_dma_base)); 1060 1061 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1062 upper_32_bits(tx_ring->bd_dma_base)); 1063 1064 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1065 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1066 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1067 1068 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1069 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1070 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1071 1072 /* enable Tx ints by setting pkt thr to 1 */ 1073 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 1074 1075 tbmr = ENETC_TBMR_EN; 1076 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1077 tbmr |= ENETC_TBMR_VIH; 1078 1079 /* enable ring */ 1080 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1081 1082 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1083 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1084 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1085 } 1086 1087 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1088 { 1089 int idx = rx_ring->index; 1090 u32 rbmr; 1091 1092 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1093 lower_32_bits(rx_ring->bd_dma_base)); 1094 1095 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1096 upper_32_bits(rx_ring->bd_dma_base)); 1097 1098 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1099 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1100 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1101 1102 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1103 1104 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1105 1106 /* enable Rx ints by setting pkt thr to 1 */ 1107 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 1108 1109 rbmr = ENETC_RBMR_EN; 1110 1111 if (rx_ring->ext_en) 1112 rbmr |= ENETC_RBMR_BDS; 1113 1114 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1115 rbmr |= ENETC_RBMR_VTE; 1116 1117 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1118 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1119 1120 enetc_lock_mdio(); 1121 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1122 enetc_unlock_mdio(); 1123 1124 /* enable ring */ 1125 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1126 } 1127 1128 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1129 { 1130 int i; 1131 1132 for (i = 0; i < priv->num_tx_rings; i++) 1133 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1134 1135 for (i = 0; i < priv->num_rx_rings; i++) 1136 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1137 } 1138 1139 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1140 { 1141 int idx = rx_ring->index; 1142 1143 /* disable EN bit on ring */ 1144 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1145 } 1146 1147 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1148 { 1149 int delay = 8, timeout = 100; 1150 int idx = tx_ring->index; 1151 1152 /* disable EN bit on ring */ 1153 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1154 1155 /* wait for busy to clear */ 1156 while (delay < timeout && 1157 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1158 msleep(delay); 1159 delay *= 2; 1160 } 1161 1162 if (delay >= timeout) 1163 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1164 idx); 1165 } 1166 1167 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1168 { 1169 int i; 1170 1171 for (i = 0; i < priv->num_tx_rings; i++) 1172 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1173 1174 for (i = 0; i < priv->num_rx_rings; i++) 1175 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1176 1177 udelay(1); 1178 } 1179 1180 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1181 { 1182 struct pci_dev *pdev = priv->si->pdev; 1183 cpumask_t cpu_mask; 1184 int i, j, err; 1185 1186 for (i = 0; i < priv->bdr_int_num; i++) { 1187 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1188 struct enetc_int_vector *v = priv->int_vector[i]; 1189 int entry = ENETC_BDR_INT_BASE_IDX + i; 1190 struct enetc_hw *hw = &priv->si->hw; 1191 1192 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1193 priv->ndev->name, i); 1194 err = request_irq(irq, enetc_msix, 0, v->name, v); 1195 if (err) { 1196 dev_err(priv->dev, "request_irq() failed!\n"); 1197 goto irq_err; 1198 } 1199 disable_irq(irq); 1200 1201 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1202 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1203 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 1204 1205 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1206 1207 for (j = 0; j < v->count_tx_rings; j++) { 1208 int idx = v->tx_ring[j].index; 1209 1210 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1211 } 1212 cpumask_clear(&cpu_mask); 1213 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1214 irq_set_affinity_hint(irq, &cpu_mask); 1215 } 1216 1217 return 0; 1218 1219 irq_err: 1220 while (i--) { 1221 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1222 1223 irq_set_affinity_hint(irq, NULL); 1224 free_irq(irq, priv->int_vector[i]); 1225 } 1226 1227 return err; 1228 } 1229 1230 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1231 { 1232 struct pci_dev *pdev = priv->si->pdev; 1233 int i; 1234 1235 for (i = 0; i < priv->bdr_int_num; i++) { 1236 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1237 1238 irq_set_affinity_hint(irq, NULL); 1239 free_irq(irq, priv->int_vector[i]); 1240 } 1241 } 1242 1243 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 1244 { 1245 struct enetc_hw *hw = &priv->si->hw; 1246 u32 icpt, ictt; 1247 int i; 1248 1249 /* enable Tx & Rx event indication */ 1250 if (priv->ic_mode & 1251 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 1252 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 1253 /* init to non-0 minimum, will be adjusted later */ 1254 ictt = 0x1; 1255 } else { 1256 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 1257 ictt = 0; 1258 } 1259 1260 for (i = 0; i < priv->num_rx_rings; i++) { 1261 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 1262 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 1263 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 1264 } 1265 1266 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 1267 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 1268 else 1269 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 1270 1271 for (i = 0; i < priv->num_tx_rings; i++) { 1272 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 1273 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 1274 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 1275 } 1276 } 1277 1278 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 1279 { 1280 int i; 1281 1282 for (i = 0; i < priv->num_tx_rings; i++) 1283 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1284 1285 for (i = 0; i < priv->num_rx_rings; i++) 1286 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1287 } 1288 1289 static int enetc_phylink_connect(struct net_device *ndev) 1290 { 1291 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1292 struct ethtool_eee edata; 1293 int err; 1294 1295 if (!priv->phylink) 1296 return 0; /* phy-less mode */ 1297 1298 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 1299 if (err) { 1300 dev_err(&ndev->dev, "could not attach to PHY\n"); 1301 return err; 1302 } 1303 1304 /* disable EEE autoneg, until ENETC driver supports it */ 1305 memset(&edata, 0, sizeof(struct ethtool_eee)); 1306 phylink_ethtool_set_eee(priv->phylink, &edata); 1307 1308 return 0; 1309 } 1310 1311 void enetc_start(struct net_device *ndev) 1312 { 1313 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1314 int i; 1315 1316 enetc_setup_interrupts(priv); 1317 1318 for (i = 0; i < priv->bdr_int_num; i++) { 1319 int irq = pci_irq_vector(priv->si->pdev, 1320 ENETC_BDR_INT_BASE_IDX + i); 1321 1322 napi_enable(&priv->int_vector[i]->napi); 1323 enable_irq(irq); 1324 } 1325 1326 if (priv->phylink) 1327 phylink_start(priv->phylink); 1328 else 1329 netif_carrier_on(ndev); 1330 1331 netif_tx_start_all_queues(ndev); 1332 } 1333 1334 int enetc_open(struct net_device *ndev) 1335 { 1336 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1337 int err; 1338 1339 err = enetc_setup_irqs(priv); 1340 if (err) 1341 return err; 1342 1343 err = enetc_phylink_connect(ndev); 1344 if (err) 1345 goto err_phy_connect; 1346 1347 err = enetc_alloc_tx_resources(priv); 1348 if (err) 1349 goto err_alloc_tx; 1350 1351 err = enetc_alloc_rx_resources(priv); 1352 if (err) 1353 goto err_alloc_rx; 1354 1355 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1356 if (err) 1357 goto err_set_queues; 1358 1359 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1360 if (err) 1361 goto err_set_queues; 1362 1363 enetc_setup_bdrs(priv); 1364 enetc_start(ndev); 1365 1366 return 0; 1367 1368 err_set_queues: 1369 enetc_free_rx_resources(priv); 1370 err_alloc_rx: 1371 enetc_free_tx_resources(priv); 1372 err_alloc_tx: 1373 if (priv->phylink) 1374 phylink_disconnect_phy(priv->phylink); 1375 err_phy_connect: 1376 enetc_free_irqs(priv); 1377 1378 return err; 1379 } 1380 1381 void enetc_stop(struct net_device *ndev) 1382 { 1383 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1384 int i; 1385 1386 netif_tx_stop_all_queues(ndev); 1387 1388 for (i = 0; i < priv->bdr_int_num; i++) { 1389 int irq = pci_irq_vector(priv->si->pdev, 1390 ENETC_BDR_INT_BASE_IDX + i); 1391 1392 disable_irq(irq); 1393 napi_synchronize(&priv->int_vector[i]->napi); 1394 napi_disable(&priv->int_vector[i]->napi); 1395 } 1396 1397 if (priv->phylink) 1398 phylink_stop(priv->phylink); 1399 else 1400 netif_carrier_off(ndev); 1401 1402 enetc_clear_interrupts(priv); 1403 } 1404 1405 int enetc_close(struct net_device *ndev) 1406 { 1407 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1408 1409 enetc_stop(ndev); 1410 enetc_clear_bdrs(priv); 1411 1412 if (priv->phylink) 1413 phylink_disconnect_phy(priv->phylink); 1414 enetc_free_rxtx_rings(priv); 1415 enetc_free_rx_resources(priv); 1416 enetc_free_tx_resources(priv); 1417 enetc_free_irqs(priv); 1418 1419 return 0; 1420 } 1421 1422 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1423 { 1424 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1425 struct tc_mqprio_qopt *mqprio = type_data; 1426 struct enetc_bdr *tx_ring; 1427 u8 num_tc; 1428 int i; 1429 1430 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1431 num_tc = mqprio->num_tc; 1432 1433 if (!num_tc) { 1434 netdev_reset_tc(ndev); 1435 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1436 1437 /* Reset all ring priorities to 0 */ 1438 for (i = 0; i < priv->num_tx_rings; i++) { 1439 tx_ring = priv->tx_ring[i]; 1440 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1441 } 1442 1443 return 0; 1444 } 1445 1446 /* Check if we have enough BD rings available to accommodate all TCs */ 1447 if (num_tc > priv->num_tx_rings) { 1448 netdev_err(ndev, "Max %d traffic classes supported\n", 1449 priv->num_tx_rings); 1450 return -EINVAL; 1451 } 1452 1453 /* For the moment, we use only one BD ring per TC. 1454 * 1455 * Configure num_tc BD rings with increasing priorities. 1456 */ 1457 for (i = 0; i < num_tc; i++) { 1458 tx_ring = priv->tx_ring[i]; 1459 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1460 } 1461 1462 /* Reset the number of netdev queues based on the TC count */ 1463 netif_set_real_num_tx_queues(ndev, num_tc); 1464 1465 netdev_set_num_tc(ndev, num_tc); 1466 1467 /* Each TC is associated with one netdev queue */ 1468 for (i = 0; i < num_tc; i++) 1469 netdev_set_tc_queue(ndev, i, 1, i); 1470 1471 return 0; 1472 } 1473 1474 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1475 void *type_data) 1476 { 1477 switch (type) { 1478 case TC_SETUP_QDISC_MQPRIO: 1479 return enetc_setup_tc_mqprio(ndev, type_data); 1480 case TC_SETUP_QDISC_TAPRIO: 1481 return enetc_setup_tc_taprio(ndev, type_data); 1482 case TC_SETUP_QDISC_CBS: 1483 return enetc_setup_tc_cbs(ndev, type_data); 1484 case TC_SETUP_QDISC_ETF: 1485 return enetc_setup_tc_txtime(ndev, type_data); 1486 case TC_SETUP_BLOCK: 1487 return enetc_setup_tc_psfp(ndev, type_data); 1488 default: 1489 return -EOPNOTSUPP; 1490 } 1491 } 1492 1493 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1494 { 1495 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1496 struct net_device_stats *stats = &ndev->stats; 1497 unsigned long packets = 0, bytes = 0; 1498 int i; 1499 1500 for (i = 0; i < priv->num_rx_rings; i++) { 1501 packets += priv->rx_ring[i]->stats.packets; 1502 bytes += priv->rx_ring[i]->stats.bytes; 1503 } 1504 1505 stats->rx_packets = packets; 1506 stats->rx_bytes = bytes; 1507 bytes = 0; 1508 packets = 0; 1509 1510 for (i = 0; i < priv->num_tx_rings; i++) { 1511 packets += priv->tx_ring[i]->stats.packets; 1512 bytes += priv->tx_ring[i]->stats.bytes; 1513 } 1514 1515 stats->tx_packets = packets; 1516 stats->tx_bytes = bytes; 1517 1518 return stats; 1519 } 1520 1521 static int enetc_set_rss(struct net_device *ndev, int en) 1522 { 1523 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1524 struct enetc_hw *hw = &priv->si->hw; 1525 u32 reg; 1526 1527 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1528 1529 reg = enetc_rd(hw, ENETC_SIMR); 1530 reg &= ~ENETC_SIMR_RSSE; 1531 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1532 enetc_wr(hw, ENETC_SIMR, reg); 1533 1534 return 0; 1535 } 1536 1537 static int enetc_set_psfp(struct net_device *ndev, int en) 1538 { 1539 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1540 int err; 1541 1542 if (en) { 1543 err = enetc_psfp_enable(priv); 1544 if (err) 1545 return err; 1546 1547 priv->active_offloads |= ENETC_F_QCI; 1548 return 0; 1549 } 1550 1551 err = enetc_psfp_disable(priv); 1552 if (err) 1553 return err; 1554 1555 priv->active_offloads &= ~ENETC_F_QCI; 1556 1557 return 0; 1558 } 1559 1560 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 1561 { 1562 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1563 int i; 1564 1565 for (i = 0; i < priv->num_rx_rings; i++) 1566 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en); 1567 } 1568 1569 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 1570 { 1571 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1572 int i; 1573 1574 for (i = 0; i < priv->num_tx_rings; i++) 1575 enetc_bdr_enable_txvlan(&priv->si->hw, i, en); 1576 } 1577 1578 int enetc_set_features(struct net_device *ndev, 1579 netdev_features_t features) 1580 { 1581 netdev_features_t changed = ndev->features ^ features; 1582 int err = 0; 1583 1584 if (changed & NETIF_F_RXHASH) 1585 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1586 1587 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 1588 enetc_enable_rxvlan(ndev, 1589 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 1590 1591 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 1592 enetc_enable_txvlan(ndev, 1593 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 1594 1595 if (changed & NETIF_F_HW_TC) 1596 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); 1597 1598 return err; 1599 } 1600 1601 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1602 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1603 { 1604 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1605 struct hwtstamp_config config; 1606 int ao; 1607 1608 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1609 return -EFAULT; 1610 1611 switch (config.tx_type) { 1612 case HWTSTAMP_TX_OFF: 1613 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1614 break; 1615 case HWTSTAMP_TX_ON: 1616 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1617 break; 1618 default: 1619 return -ERANGE; 1620 } 1621 1622 ao = priv->active_offloads; 1623 switch (config.rx_filter) { 1624 case HWTSTAMP_FILTER_NONE: 1625 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1626 break; 1627 default: 1628 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1629 config.rx_filter = HWTSTAMP_FILTER_ALL; 1630 } 1631 1632 if (netif_running(ndev) && ao != priv->active_offloads) { 1633 enetc_close(ndev); 1634 enetc_open(ndev); 1635 } 1636 1637 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1638 -EFAULT : 0; 1639 } 1640 1641 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1642 { 1643 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1644 struct hwtstamp_config config; 1645 1646 config.flags = 0; 1647 1648 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1649 config.tx_type = HWTSTAMP_TX_ON; 1650 else 1651 config.tx_type = HWTSTAMP_TX_OFF; 1652 1653 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1654 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1655 1656 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1657 -EFAULT : 0; 1658 } 1659 #endif 1660 1661 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1662 { 1663 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1664 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1665 if (cmd == SIOCSHWTSTAMP) 1666 return enetc_hwtstamp_set(ndev, rq); 1667 if (cmd == SIOCGHWTSTAMP) 1668 return enetc_hwtstamp_get(ndev, rq); 1669 #endif 1670 1671 if (!priv->phylink) 1672 return -EOPNOTSUPP; 1673 1674 return phylink_mii_ioctl(priv->phylink, rq, cmd); 1675 } 1676 1677 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1678 { 1679 struct pci_dev *pdev = priv->si->pdev; 1680 int v_tx_rings; 1681 int i, n, err, nvec; 1682 1683 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1684 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1685 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1686 1687 if (n < 0) 1688 return n; 1689 1690 if (n != nvec) 1691 return -EPERM; 1692 1693 /* # of tx rings per int vector */ 1694 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1695 1696 for (i = 0; i < priv->bdr_int_num; i++) { 1697 struct enetc_int_vector *v; 1698 struct enetc_bdr *bdr; 1699 int j; 1700 1701 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 1702 if (!v) { 1703 err = -ENOMEM; 1704 goto fail; 1705 } 1706 1707 priv->int_vector[i] = v; 1708 1709 /* init defaults for adaptive IC */ 1710 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 1711 v->rx_ictt = 0x1; 1712 v->rx_dim_en = true; 1713 } 1714 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 1715 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1716 NAPI_POLL_WEIGHT); 1717 v->count_tx_rings = v_tx_rings; 1718 1719 for (j = 0; j < v_tx_rings; j++) { 1720 int idx; 1721 1722 /* default tx ring mapping policy */ 1723 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1724 idx = 2 * j + i; /* 2 CPUs */ 1725 else 1726 idx = j + i * v_tx_rings; /* default */ 1727 1728 __set_bit(idx, &v->tx_rings_map); 1729 bdr = &v->tx_ring[j]; 1730 bdr->index = idx; 1731 bdr->ndev = priv->ndev; 1732 bdr->dev = priv->dev; 1733 bdr->bd_count = priv->tx_bd_count; 1734 priv->tx_ring[idx] = bdr; 1735 } 1736 1737 bdr = &v->rx_ring; 1738 bdr->index = i; 1739 bdr->ndev = priv->ndev; 1740 bdr->dev = priv->dev; 1741 bdr->bd_count = priv->rx_bd_count; 1742 priv->rx_ring[i] = bdr; 1743 } 1744 1745 return 0; 1746 1747 fail: 1748 while (i--) { 1749 netif_napi_del(&priv->int_vector[i]->napi); 1750 cancel_work_sync(&priv->int_vector[i]->rx_dim.work); 1751 kfree(priv->int_vector[i]); 1752 } 1753 1754 pci_free_irq_vectors(pdev); 1755 1756 return err; 1757 } 1758 1759 void enetc_free_msix(struct enetc_ndev_priv *priv) 1760 { 1761 int i; 1762 1763 for (i = 0; i < priv->bdr_int_num; i++) { 1764 struct enetc_int_vector *v = priv->int_vector[i]; 1765 1766 netif_napi_del(&v->napi); 1767 cancel_work_sync(&v->rx_dim.work); 1768 } 1769 1770 for (i = 0; i < priv->num_rx_rings; i++) 1771 priv->rx_ring[i] = NULL; 1772 1773 for (i = 0; i < priv->num_tx_rings; i++) 1774 priv->tx_ring[i] = NULL; 1775 1776 for (i = 0; i < priv->bdr_int_num; i++) { 1777 kfree(priv->int_vector[i]); 1778 priv->int_vector[i] = NULL; 1779 } 1780 1781 /* disable all MSIX for this device */ 1782 pci_free_irq_vectors(priv->si->pdev); 1783 } 1784 1785 static void enetc_kfree_si(struct enetc_si *si) 1786 { 1787 char *p = (char *)si - si->pad; 1788 1789 kfree(p); 1790 } 1791 1792 static void enetc_detect_errata(struct enetc_si *si) 1793 { 1794 if (si->pdev->revision == ENETC_REV1) 1795 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 1796 } 1797 1798 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1799 { 1800 struct enetc_si *si, *p; 1801 struct enetc_hw *hw; 1802 size_t alloc_size; 1803 int err, len; 1804 1805 pcie_flr(pdev); 1806 err = pci_enable_device_mem(pdev); 1807 if (err) { 1808 dev_err(&pdev->dev, "device enable failed\n"); 1809 return err; 1810 } 1811 1812 /* set up for high or low dma */ 1813 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1814 if (err) { 1815 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1816 if (err) { 1817 dev_err(&pdev->dev, 1818 "DMA configuration failed: 0x%x\n", err); 1819 goto err_dma; 1820 } 1821 } 1822 1823 err = pci_request_mem_regions(pdev, name); 1824 if (err) { 1825 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1826 goto err_pci_mem_reg; 1827 } 1828 1829 pci_set_master(pdev); 1830 1831 alloc_size = sizeof(struct enetc_si); 1832 if (sizeof_priv) { 1833 /* align priv to 32B */ 1834 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1835 alloc_size += sizeof_priv; 1836 } 1837 /* force 32B alignment for enetc_si */ 1838 alloc_size += ENETC_SI_ALIGN - 1; 1839 1840 p = kzalloc(alloc_size, GFP_KERNEL); 1841 if (!p) { 1842 err = -ENOMEM; 1843 goto err_alloc_si; 1844 } 1845 1846 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1847 si->pad = (char *)si - (char *)p; 1848 1849 pci_set_drvdata(pdev, si); 1850 si->pdev = pdev; 1851 hw = &si->hw; 1852 1853 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1854 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1855 if (!hw->reg) { 1856 err = -ENXIO; 1857 dev_err(&pdev->dev, "ioremap() failed\n"); 1858 goto err_ioremap; 1859 } 1860 if (len > ENETC_PORT_BASE) 1861 hw->port = hw->reg + ENETC_PORT_BASE; 1862 if (len > ENETC_GLOBAL_BASE) 1863 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1864 1865 enetc_detect_errata(si); 1866 1867 return 0; 1868 1869 err_ioremap: 1870 enetc_kfree_si(si); 1871 err_alloc_si: 1872 pci_release_mem_regions(pdev); 1873 err_pci_mem_reg: 1874 err_dma: 1875 pci_disable_device(pdev); 1876 1877 return err; 1878 } 1879 1880 void enetc_pci_remove(struct pci_dev *pdev) 1881 { 1882 struct enetc_si *si = pci_get_drvdata(pdev); 1883 struct enetc_hw *hw = &si->hw; 1884 1885 iounmap(hw->reg); 1886 enetc_kfree_si(si); 1887 pci_release_mem_regions(pdev); 1888 pci_disable_device(pdev); 1889 } 1890