1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/vmalloc.h> 8 9 /* ENETC overhead: optional extension BD + 1 BD gap */ 10 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 11 /* max # of chained Tx BDs is 15, including head and extension BD */ 12 #define ENETC_MAX_SKB_FRAGS 13 13 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 14 15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 16 int active_offloads); 17 18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 19 { 20 struct enetc_ndev_priv *priv = netdev_priv(ndev); 21 struct enetc_bdr *tx_ring; 22 int count; 23 24 tx_ring = priv->tx_ring[skb->queue_mapping]; 25 26 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 27 if (unlikely(skb_linearize(skb))) 28 goto drop_packet_err; 29 30 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 31 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 32 netif_stop_subqueue(ndev, tx_ring->index); 33 return NETDEV_TX_BUSY; 34 } 35 36 enetc_lock_mdio(); 37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 38 enetc_unlock_mdio(); 39 40 if (unlikely(!count)) 41 goto drop_packet_err; 42 43 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 44 netif_stop_subqueue(ndev, tx_ring->index); 45 46 return NETDEV_TX_OK; 47 48 drop_packet_err: 49 dev_kfree_skb_any(skb); 50 return NETDEV_TX_OK; 51 } 52 53 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 54 struct enetc_tx_swbd *tx_swbd) 55 { 56 if (tx_swbd->is_dma_page) 57 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 58 tx_swbd->len, DMA_TO_DEVICE); 59 else 60 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 61 tx_swbd->len, DMA_TO_DEVICE); 62 tx_swbd->dma = 0; 63 } 64 65 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 66 struct enetc_tx_swbd *tx_swbd) 67 { 68 if (tx_swbd->dma) 69 enetc_unmap_tx_buff(tx_ring, tx_swbd); 70 71 if (tx_swbd->skb) { 72 dev_kfree_skb_any(tx_swbd->skb); 73 tx_swbd->skb = NULL; 74 } 75 } 76 77 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 78 int active_offloads) 79 { 80 struct enetc_tx_swbd *tx_swbd; 81 skb_frag_t *frag; 82 int len = skb_headlen(skb); 83 union enetc_tx_bd temp_bd; 84 union enetc_tx_bd *txbd; 85 bool do_vlan, do_tstamp; 86 int i, count = 0; 87 unsigned int f; 88 dma_addr_t dma; 89 u8 flags = 0; 90 91 i = tx_ring->next_to_use; 92 txbd = ENETC_TXBD(*tx_ring, i); 93 prefetchw(txbd); 94 95 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 96 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 97 goto dma_err; 98 99 temp_bd.addr = cpu_to_le64(dma); 100 temp_bd.buf_len = cpu_to_le16(len); 101 temp_bd.lstatus = 0; 102 103 tx_swbd = &tx_ring->tx_swbd[i]; 104 tx_swbd->dma = dma; 105 tx_swbd->len = len; 106 tx_swbd->is_dma_page = 0; 107 count++; 108 109 do_vlan = skb_vlan_tag_present(skb); 110 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 111 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 112 tx_swbd->do_tstamp = do_tstamp; 113 tx_swbd->check_wb = tx_swbd->do_tstamp; 114 115 if (do_vlan || do_tstamp) 116 flags |= ENETC_TXBD_FLAGS_EX; 117 118 if (tx_ring->tsd_enable) 119 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 120 121 /* first BD needs frm_len and offload flags set */ 122 temp_bd.frm_len = cpu_to_le16(skb->len); 123 temp_bd.flags = flags; 124 125 if (flags & ENETC_TXBD_FLAGS_TSE) 126 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 127 flags); 128 129 if (flags & ENETC_TXBD_FLAGS_EX) { 130 u8 e_flags = 0; 131 *txbd = temp_bd; 132 enetc_clear_tx_bd(&temp_bd); 133 134 /* add extension BD for VLAN and/or timestamping */ 135 flags = 0; 136 tx_swbd++; 137 txbd++; 138 i++; 139 if (unlikely(i == tx_ring->bd_count)) { 140 i = 0; 141 tx_swbd = tx_ring->tx_swbd; 142 txbd = ENETC_TXBD(*tx_ring, 0); 143 } 144 prefetchw(txbd); 145 146 if (do_vlan) { 147 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 148 temp_bd.ext.tpid = 0; /* < C-TAG */ 149 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 150 } 151 152 if (do_tstamp) { 153 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 154 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 155 } 156 157 temp_bd.ext.e_flags = e_flags; 158 count++; 159 } 160 161 frag = &skb_shinfo(skb)->frags[0]; 162 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 163 len = skb_frag_size(frag); 164 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 165 DMA_TO_DEVICE); 166 if (dma_mapping_error(tx_ring->dev, dma)) 167 goto dma_err; 168 169 *txbd = temp_bd; 170 enetc_clear_tx_bd(&temp_bd); 171 172 flags = 0; 173 tx_swbd++; 174 txbd++; 175 i++; 176 if (unlikely(i == tx_ring->bd_count)) { 177 i = 0; 178 tx_swbd = tx_ring->tx_swbd; 179 txbd = ENETC_TXBD(*tx_ring, 0); 180 } 181 prefetchw(txbd); 182 183 temp_bd.addr = cpu_to_le64(dma); 184 temp_bd.buf_len = cpu_to_le16(len); 185 186 tx_swbd->dma = dma; 187 tx_swbd->len = len; 188 tx_swbd->is_dma_page = 1; 189 count++; 190 } 191 192 /* last BD needs 'F' bit set */ 193 flags |= ENETC_TXBD_FLAGS_F; 194 temp_bd.flags = flags; 195 *txbd = temp_bd; 196 197 tx_ring->tx_swbd[i].skb = skb; 198 199 enetc_bdr_idx_inc(tx_ring, &i); 200 tx_ring->next_to_use = i; 201 202 skb_tx_timestamp(skb); 203 204 /* let H/W know BD ring has been updated */ 205 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */ 206 207 return count; 208 209 dma_err: 210 dev_err(tx_ring->dev, "DMA map error"); 211 212 do { 213 tx_swbd = &tx_ring->tx_swbd[i]; 214 enetc_free_tx_skb(tx_ring, tx_swbd); 215 if (i == 0) 216 i = tx_ring->bd_count; 217 i--; 218 } while (count--); 219 220 return 0; 221 } 222 223 static irqreturn_t enetc_msix(int irq, void *data) 224 { 225 struct enetc_int_vector *v = data; 226 int i; 227 228 enetc_lock_mdio(); 229 230 /* disable interrupts */ 231 enetc_wr_reg_hot(v->rbier, 0); 232 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 233 234 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 235 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 236 237 enetc_unlock_mdio(); 238 239 napi_schedule(&v->napi); 240 241 return IRQ_HANDLED; 242 } 243 244 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 245 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 246 struct napi_struct *napi, int work_limit); 247 248 static void enetc_rx_dim_work(struct work_struct *w) 249 { 250 struct dim *dim = container_of(w, struct dim, work); 251 struct dim_cq_moder moder = 252 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 253 struct enetc_int_vector *v = 254 container_of(dim, struct enetc_int_vector, rx_dim); 255 256 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 257 dim->state = DIM_START_MEASURE; 258 } 259 260 static void enetc_rx_net_dim(struct enetc_int_vector *v) 261 { 262 struct dim_sample dim_sample; 263 264 v->comp_cnt++; 265 266 if (!v->rx_napi_work) 267 return; 268 269 dim_update_sample(v->comp_cnt, 270 v->rx_ring.stats.packets, 271 v->rx_ring.stats.bytes, 272 &dim_sample); 273 net_dim(&v->rx_dim, dim_sample); 274 } 275 276 static int enetc_poll(struct napi_struct *napi, int budget) 277 { 278 struct enetc_int_vector 279 *v = container_of(napi, struct enetc_int_vector, napi); 280 bool complete = true; 281 int work_done; 282 int i; 283 284 for (i = 0; i < v->count_tx_rings; i++) 285 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 286 complete = false; 287 288 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 289 if (work_done == budget) 290 complete = false; 291 if (work_done) 292 v->rx_napi_work = true; 293 294 if (!complete) 295 return budget; 296 297 napi_complete_done(napi, work_done); 298 299 if (likely(v->rx_dim_en)) 300 enetc_rx_net_dim(v); 301 302 v->rx_napi_work = false; 303 304 enetc_lock_mdio(); 305 306 /* enable interrupts */ 307 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 308 309 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 310 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 311 ENETC_TBIER_TXTIE); 312 313 enetc_unlock_mdio(); 314 315 return work_done; 316 } 317 318 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 319 { 320 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 321 322 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 323 } 324 325 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 326 u64 *tstamp) 327 { 328 u32 lo, hi, tstamp_lo; 329 330 lo = enetc_rd(hw, ENETC_SICTR0); 331 hi = enetc_rd(hw, ENETC_SICTR1); 332 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 333 if (lo <= tstamp_lo) 334 hi -= 1; 335 *tstamp = (u64)hi << 32 | tstamp_lo; 336 } 337 338 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 339 { 340 struct skb_shared_hwtstamps shhwtstamps; 341 342 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 343 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 344 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 345 skb_tstamp_tx(skb, &shhwtstamps); 346 } 347 } 348 349 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 350 { 351 struct net_device *ndev = tx_ring->ndev; 352 int tx_frm_cnt = 0, tx_byte_cnt = 0; 353 struct enetc_tx_swbd *tx_swbd; 354 int i, bds_to_clean; 355 bool do_tstamp; 356 u64 tstamp = 0; 357 358 i = tx_ring->next_to_clean; 359 tx_swbd = &tx_ring->tx_swbd[i]; 360 361 enetc_lock_mdio(); 362 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 363 enetc_unlock_mdio(); 364 365 do_tstamp = false; 366 367 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 368 bool is_eof = !!tx_swbd->skb; 369 370 if (unlikely(tx_swbd->check_wb)) { 371 struct enetc_ndev_priv *priv = netdev_priv(ndev); 372 union enetc_tx_bd *txbd; 373 374 txbd = ENETC_TXBD(*tx_ring, i); 375 376 if (txbd->flags & ENETC_TXBD_FLAGS_W && 377 tx_swbd->do_tstamp) { 378 enetc_get_tx_tstamp(&priv->si->hw, txbd, 379 &tstamp); 380 do_tstamp = true; 381 } 382 } 383 384 if (likely(tx_swbd->dma)) 385 enetc_unmap_tx_buff(tx_ring, tx_swbd); 386 387 if (is_eof) { 388 if (unlikely(do_tstamp)) { 389 enetc_tstamp_tx(tx_swbd->skb, tstamp); 390 do_tstamp = false; 391 } 392 napi_consume_skb(tx_swbd->skb, napi_budget); 393 tx_swbd->skb = NULL; 394 } 395 396 tx_byte_cnt += tx_swbd->len; 397 398 bds_to_clean--; 399 tx_swbd++; 400 i++; 401 if (unlikely(i == tx_ring->bd_count)) { 402 i = 0; 403 tx_swbd = tx_ring->tx_swbd; 404 } 405 406 enetc_lock_mdio(); 407 408 /* BD iteration loop end */ 409 if (is_eof) { 410 tx_frm_cnt++; 411 /* re-arm interrupt source */ 412 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 413 BIT(16 + tx_ring->index)); 414 } 415 416 if (unlikely(!bds_to_clean)) 417 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 418 419 enetc_unlock_mdio(); 420 } 421 422 tx_ring->next_to_clean = i; 423 tx_ring->stats.packets += tx_frm_cnt; 424 tx_ring->stats.bytes += tx_byte_cnt; 425 426 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 427 __netif_subqueue_stopped(ndev, tx_ring->index) && 428 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 429 netif_wake_subqueue(ndev, tx_ring->index); 430 } 431 432 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 433 } 434 435 static bool enetc_new_page(struct enetc_bdr *rx_ring, 436 struct enetc_rx_swbd *rx_swbd) 437 { 438 struct page *page; 439 dma_addr_t addr; 440 441 page = dev_alloc_page(); 442 if (unlikely(!page)) 443 return false; 444 445 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 446 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 447 __free_page(page); 448 449 return false; 450 } 451 452 rx_swbd->dma = addr; 453 rx_swbd->page = page; 454 rx_swbd->page_offset = ENETC_RXB_PAD; 455 456 return true; 457 } 458 459 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 460 { 461 struct enetc_rx_swbd *rx_swbd; 462 union enetc_rx_bd *rxbd; 463 int i, j; 464 465 i = rx_ring->next_to_use; 466 rx_swbd = &rx_ring->rx_swbd[i]; 467 rxbd = enetc_rxbd(rx_ring, i); 468 469 for (j = 0; j < buff_cnt; j++) { 470 /* try reuse page */ 471 if (unlikely(!rx_swbd->page)) { 472 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 473 rx_ring->stats.rx_alloc_errs++; 474 break; 475 } 476 } 477 478 /* update RxBD */ 479 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 480 rx_swbd->page_offset); 481 /* clear 'R" as well */ 482 rxbd->r.lstatus = 0; 483 484 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 485 rx_swbd++; 486 i++; 487 if (unlikely(i == rx_ring->bd_count)) { 488 i = 0; 489 rx_swbd = rx_ring->rx_swbd; 490 } 491 } 492 493 if (likely(j)) { 494 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 495 rx_ring->next_to_use = i; 496 } 497 498 return j; 499 } 500 501 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 502 static void enetc_get_rx_tstamp(struct net_device *ndev, 503 union enetc_rx_bd *rxbd, 504 struct sk_buff *skb) 505 { 506 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 507 struct enetc_ndev_priv *priv = netdev_priv(ndev); 508 struct enetc_hw *hw = &priv->si->hw; 509 u32 lo, hi, tstamp_lo; 510 u64 tstamp; 511 512 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 513 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 514 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 515 rxbd = enetc_rxbd_ext(rxbd); 516 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 517 if (lo <= tstamp_lo) 518 hi -= 1; 519 520 tstamp = (u64)hi << 32 | tstamp_lo; 521 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 522 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 523 } 524 } 525 #endif 526 527 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 528 union enetc_rx_bd *rxbd, struct sk_buff *skb) 529 { 530 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 531 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 532 #endif 533 /* TODO: hashing */ 534 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 535 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 536 537 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 538 skb->ip_summed = CHECKSUM_COMPLETE; 539 } 540 541 /* copy VLAN to skb, if one is extracted, for now we assume it's a 542 * standard TPID, but HW also supports custom values 543 */ 544 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) 545 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 546 le16_to_cpu(rxbd->r.vlan_opt)); 547 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 548 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 549 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 550 #endif 551 } 552 553 static void enetc_process_skb(struct enetc_bdr *rx_ring, 554 struct sk_buff *skb) 555 { 556 skb_record_rx_queue(skb, rx_ring->index); 557 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 558 } 559 560 static bool enetc_page_reusable(struct page *page) 561 { 562 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 563 } 564 565 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 566 struct enetc_rx_swbd *old) 567 { 568 struct enetc_rx_swbd *new; 569 570 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 571 572 /* next buf that may reuse a page */ 573 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 574 575 /* copy page reference */ 576 *new = *old; 577 } 578 579 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 580 int i, u16 size) 581 { 582 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 583 584 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 585 rx_swbd->page_offset, 586 size, DMA_FROM_DEVICE); 587 return rx_swbd; 588 } 589 590 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 591 struct enetc_rx_swbd *rx_swbd) 592 { 593 if (likely(enetc_page_reusable(rx_swbd->page))) { 594 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 595 page_ref_inc(rx_swbd->page); 596 597 enetc_reuse_page(rx_ring, rx_swbd); 598 599 /* sync for use by the device */ 600 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 601 rx_swbd->page_offset, 602 ENETC_RXB_DMA_SIZE, 603 DMA_FROM_DEVICE); 604 } else { 605 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 606 PAGE_SIZE, DMA_FROM_DEVICE); 607 } 608 609 rx_swbd->page = NULL; 610 } 611 612 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 613 int i, u16 size) 614 { 615 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 616 struct sk_buff *skb; 617 void *ba; 618 619 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 620 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 621 if (unlikely(!skb)) { 622 rx_ring->stats.rx_alloc_errs++; 623 return NULL; 624 } 625 626 skb_reserve(skb, ENETC_RXB_PAD); 627 __skb_put(skb, size); 628 629 enetc_put_rx_buff(rx_ring, rx_swbd); 630 631 return skb; 632 } 633 634 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 635 u16 size, struct sk_buff *skb) 636 { 637 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 638 639 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 640 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 641 642 enetc_put_rx_buff(rx_ring, rx_swbd); 643 } 644 645 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 646 647 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 648 struct napi_struct *napi, int work_limit) 649 { 650 int rx_frm_cnt = 0, rx_byte_cnt = 0; 651 int cleaned_cnt, i; 652 653 cleaned_cnt = enetc_bd_unused(rx_ring); 654 /* next descriptor to process */ 655 i = rx_ring->next_to_clean; 656 657 while (likely(rx_frm_cnt < work_limit)) { 658 union enetc_rx_bd *rxbd; 659 struct sk_buff *skb; 660 u32 bd_status; 661 u16 size; 662 663 enetc_lock_mdio(); 664 665 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 666 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 667 668 /* update ENETC's consumer index */ 669 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 670 cleaned_cnt -= count; 671 } 672 673 rxbd = enetc_rxbd(rx_ring, i); 674 bd_status = le32_to_cpu(rxbd->r.lstatus); 675 if (!bd_status) { 676 enetc_unlock_mdio(); 677 break; 678 } 679 680 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 681 dma_rmb(); /* for reading other rxbd fields */ 682 size = le16_to_cpu(rxbd->r.buf_len); 683 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 684 if (!skb) { 685 enetc_unlock_mdio(); 686 break; 687 } 688 689 enetc_get_offloads(rx_ring, rxbd, skb); 690 691 cleaned_cnt++; 692 693 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 694 if (unlikely(++i == rx_ring->bd_count)) 695 i = 0; 696 697 if (unlikely(bd_status & 698 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 699 enetc_unlock_mdio(); 700 dev_kfree_skb(skb); 701 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 702 dma_rmb(); 703 bd_status = le32_to_cpu(rxbd->r.lstatus); 704 705 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 706 if (unlikely(++i == rx_ring->bd_count)) 707 i = 0; 708 } 709 710 rx_ring->ndev->stats.rx_dropped++; 711 rx_ring->ndev->stats.rx_errors++; 712 713 break; 714 } 715 716 /* not last BD in frame? */ 717 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 718 bd_status = le32_to_cpu(rxbd->r.lstatus); 719 size = ENETC_RXB_DMA_SIZE; 720 721 if (bd_status & ENETC_RXBD_LSTATUS_F) { 722 dma_rmb(); 723 size = le16_to_cpu(rxbd->r.buf_len); 724 } 725 726 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 727 728 cleaned_cnt++; 729 730 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 731 if (unlikely(++i == rx_ring->bd_count)) 732 i = 0; 733 } 734 735 rx_byte_cnt += skb->len; 736 737 enetc_process_skb(rx_ring, skb); 738 739 enetc_unlock_mdio(); 740 741 napi_gro_receive(napi, skb); 742 743 rx_frm_cnt++; 744 } 745 746 rx_ring->next_to_clean = i; 747 748 rx_ring->stats.packets += rx_frm_cnt; 749 rx_ring->stats.bytes += rx_byte_cnt; 750 751 return rx_frm_cnt; 752 } 753 754 /* Probing and Init */ 755 #define ENETC_MAX_RFS_SIZE 64 756 void enetc_get_si_caps(struct enetc_si *si) 757 { 758 struct enetc_hw *hw = &si->hw; 759 u32 val; 760 761 /* find out how many of various resources we have to work with */ 762 val = enetc_rd(hw, ENETC_SICAPR0); 763 si->num_rx_rings = (val >> 16) & 0xff; 764 si->num_tx_rings = val & 0xff; 765 766 val = enetc_rd(hw, ENETC_SIRFSCAPR); 767 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 768 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 769 770 si->num_rss = 0; 771 val = enetc_rd(hw, ENETC_SIPCAPR0); 772 if (val & ENETC_SIPCAPR0_RSS) { 773 u32 rss; 774 775 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 776 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 777 } 778 779 if (val & ENETC_SIPCAPR0_QBV) 780 si->hw_features |= ENETC_SI_F_QBV; 781 782 if (val & ENETC_SIPCAPR0_PSFP) 783 si->hw_features |= ENETC_SI_F_PSFP; 784 } 785 786 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 787 { 788 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 789 &r->bd_dma_base, GFP_KERNEL); 790 if (!r->bd_base) 791 return -ENOMEM; 792 793 /* h/w requires 128B alignment */ 794 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 795 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 796 r->bd_dma_base); 797 return -EINVAL; 798 } 799 800 return 0; 801 } 802 803 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 804 { 805 int err; 806 807 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 808 if (!txr->tx_swbd) 809 return -ENOMEM; 810 811 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 812 if (err) { 813 vfree(txr->tx_swbd); 814 return err; 815 } 816 817 txr->next_to_clean = 0; 818 txr->next_to_use = 0; 819 820 return 0; 821 } 822 823 static void enetc_free_txbdr(struct enetc_bdr *txr) 824 { 825 int size, i; 826 827 for (i = 0; i < txr->bd_count; i++) 828 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 829 830 size = txr->bd_count * sizeof(union enetc_tx_bd); 831 832 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 833 txr->bd_base = NULL; 834 835 vfree(txr->tx_swbd); 836 txr->tx_swbd = NULL; 837 } 838 839 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 840 { 841 int i, err; 842 843 for (i = 0; i < priv->num_tx_rings; i++) { 844 err = enetc_alloc_txbdr(priv->tx_ring[i]); 845 846 if (err) 847 goto fail; 848 } 849 850 return 0; 851 852 fail: 853 while (i-- > 0) 854 enetc_free_txbdr(priv->tx_ring[i]); 855 856 return err; 857 } 858 859 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 860 { 861 int i; 862 863 for (i = 0; i < priv->num_tx_rings; i++) 864 enetc_free_txbdr(priv->tx_ring[i]); 865 } 866 867 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 868 { 869 size_t size = sizeof(union enetc_rx_bd); 870 int err; 871 872 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 873 if (!rxr->rx_swbd) 874 return -ENOMEM; 875 876 if (extended) 877 size *= 2; 878 879 err = enetc_dma_alloc_bdr(rxr, size); 880 if (err) { 881 vfree(rxr->rx_swbd); 882 return err; 883 } 884 885 rxr->next_to_clean = 0; 886 rxr->next_to_use = 0; 887 rxr->next_to_alloc = 0; 888 rxr->ext_en = extended; 889 890 return 0; 891 } 892 893 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 894 { 895 int size; 896 897 size = rxr->bd_count * sizeof(union enetc_rx_bd); 898 899 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 900 rxr->bd_base = NULL; 901 902 vfree(rxr->rx_swbd); 903 rxr->rx_swbd = NULL; 904 } 905 906 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 907 { 908 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 909 int i, err; 910 911 for (i = 0; i < priv->num_rx_rings; i++) { 912 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 913 914 if (err) 915 goto fail; 916 } 917 918 return 0; 919 920 fail: 921 while (i-- > 0) 922 enetc_free_rxbdr(priv->rx_ring[i]); 923 924 return err; 925 } 926 927 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 928 { 929 int i; 930 931 for (i = 0; i < priv->num_rx_rings; i++) 932 enetc_free_rxbdr(priv->rx_ring[i]); 933 } 934 935 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 936 { 937 int i; 938 939 if (!tx_ring->tx_swbd) 940 return; 941 942 for (i = 0; i < tx_ring->bd_count; i++) { 943 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 944 945 enetc_free_tx_skb(tx_ring, tx_swbd); 946 } 947 948 tx_ring->next_to_clean = 0; 949 tx_ring->next_to_use = 0; 950 } 951 952 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 953 { 954 int i; 955 956 if (!rx_ring->rx_swbd) 957 return; 958 959 for (i = 0; i < rx_ring->bd_count; i++) { 960 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 961 962 if (!rx_swbd->page) 963 continue; 964 965 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 966 PAGE_SIZE, DMA_FROM_DEVICE); 967 __free_page(rx_swbd->page); 968 rx_swbd->page = NULL; 969 } 970 971 rx_ring->next_to_clean = 0; 972 rx_ring->next_to_use = 0; 973 rx_ring->next_to_alloc = 0; 974 } 975 976 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 977 { 978 int i; 979 980 for (i = 0; i < priv->num_rx_rings; i++) 981 enetc_free_rx_ring(priv->rx_ring[i]); 982 983 for (i = 0; i < priv->num_tx_rings; i++) 984 enetc_free_tx_ring(priv->tx_ring[i]); 985 } 986 987 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 988 { 989 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 990 991 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 992 GFP_KERNEL); 993 if (!cbdr->bd_base) 994 return -ENOMEM; 995 996 /* h/w requires 128B alignment */ 997 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 998 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 999 return -EINVAL; 1000 } 1001 1002 cbdr->next_to_clean = 0; 1003 cbdr->next_to_use = 0; 1004 1005 return 0; 1006 } 1007 1008 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1009 { 1010 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1011 1012 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1013 cbdr->bd_base = NULL; 1014 } 1015 1016 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 1017 { 1018 /* set CBDR cache attributes */ 1019 enetc_wr(hw, ENETC_SICAR2, 1020 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1021 1022 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 1023 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 1024 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 1025 1026 enetc_wr(hw, ENETC_SICBDRPIR, 0); 1027 enetc_wr(hw, ENETC_SICBDRCIR, 0); 1028 1029 /* enable ring */ 1030 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 1031 1032 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 1033 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 1034 } 1035 1036 static void enetc_clear_cbdr(struct enetc_hw *hw) 1037 { 1038 enetc_wr(hw, ENETC_SICBDRMR, 0); 1039 } 1040 1041 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1042 { 1043 int *rss_table; 1044 int i; 1045 1046 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1047 if (!rss_table) 1048 return -ENOMEM; 1049 1050 /* Set up RSS table defaults */ 1051 for (i = 0; i < si->num_rss; i++) 1052 rss_table[i] = i % num_groups; 1053 1054 enetc_set_rss_table(si, rss_table, si->num_rss); 1055 1056 kfree(rss_table); 1057 1058 return 0; 1059 } 1060 1061 static int enetc_configure_si(struct enetc_ndev_priv *priv) 1062 { 1063 struct enetc_si *si = priv->si; 1064 struct enetc_hw *hw = &si->hw; 1065 int err; 1066 1067 enetc_setup_cbdr(hw, &si->cbd_ring); 1068 /* set SI cache attributes */ 1069 enetc_wr(hw, ENETC_SICAR0, 1070 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1071 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1072 /* enable SI */ 1073 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1074 1075 if (si->num_rss) { 1076 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1077 if (err) 1078 return err; 1079 } 1080 1081 return 0; 1082 } 1083 1084 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1085 { 1086 struct enetc_si *si = priv->si; 1087 int cpus = num_online_cpus(); 1088 1089 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 1090 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 1091 1092 /* Enable all available TX rings in order to configure as many 1093 * priorities as possible, when needed. 1094 * TODO: Make # of TX rings run-time configurable 1095 */ 1096 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1097 priv->num_tx_rings = si->num_tx_rings; 1098 priv->bdr_int_num = cpus; 1099 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 1100 priv->tx_ictt = ENETC_TXIC_TIMETHR; 1101 1102 /* SI specific */ 1103 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1104 } 1105 1106 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1107 { 1108 struct enetc_si *si = priv->si; 1109 int err; 1110 1111 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1112 if (err) 1113 return err; 1114 1115 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1116 GFP_KERNEL); 1117 if (!priv->cls_rules) { 1118 err = -ENOMEM; 1119 goto err_alloc_cls; 1120 } 1121 1122 err = enetc_configure_si(priv); 1123 if (err) 1124 goto err_config_si; 1125 1126 return 0; 1127 1128 err_config_si: 1129 kfree(priv->cls_rules); 1130 err_alloc_cls: 1131 enetc_clear_cbdr(&si->hw); 1132 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1133 1134 return err; 1135 } 1136 1137 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1138 { 1139 struct enetc_si *si = priv->si; 1140 1141 enetc_clear_cbdr(&si->hw); 1142 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1143 1144 kfree(priv->cls_rules); 1145 } 1146 1147 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1148 { 1149 int idx = tx_ring->index; 1150 u32 tbmr; 1151 1152 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1153 lower_32_bits(tx_ring->bd_dma_base)); 1154 1155 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1156 upper_32_bits(tx_ring->bd_dma_base)); 1157 1158 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1159 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1160 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1161 1162 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1163 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1164 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1165 1166 /* enable Tx ints by setting pkt thr to 1 */ 1167 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 1168 1169 tbmr = ENETC_TBMR_EN; 1170 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1171 tbmr |= ENETC_TBMR_VIH; 1172 1173 /* enable ring */ 1174 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1175 1176 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1177 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1178 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1179 } 1180 1181 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1182 { 1183 int idx = rx_ring->index; 1184 u32 rbmr; 1185 1186 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1187 lower_32_bits(rx_ring->bd_dma_base)); 1188 1189 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1190 upper_32_bits(rx_ring->bd_dma_base)); 1191 1192 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1193 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1194 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1195 1196 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1197 1198 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1199 1200 /* enable Rx ints by setting pkt thr to 1 */ 1201 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 1202 1203 rbmr = ENETC_RBMR_EN; 1204 1205 if (rx_ring->ext_en) 1206 rbmr |= ENETC_RBMR_BDS; 1207 1208 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1209 rbmr |= ENETC_RBMR_VTE; 1210 1211 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1212 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1213 1214 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1215 enetc_wr(hw, ENETC_SIRXIDR, rx_ring->next_to_use); 1216 1217 /* enable ring */ 1218 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1219 } 1220 1221 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1222 { 1223 int i; 1224 1225 for (i = 0; i < priv->num_tx_rings; i++) 1226 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1227 1228 for (i = 0; i < priv->num_rx_rings; i++) 1229 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1230 } 1231 1232 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1233 { 1234 int idx = rx_ring->index; 1235 1236 /* disable EN bit on ring */ 1237 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1238 } 1239 1240 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1241 { 1242 int delay = 8, timeout = 100; 1243 int idx = tx_ring->index; 1244 1245 /* disable EN bit on ring */ 1246 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1247 1248 /* wait for busy to clear */ 1249 while (delay < timeout && 1250 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1251 msleep(delay); 1252 delay *= 2; 1253 } 1254 1255 if (delay >= timeout) 1256 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1257 idx); 1258 } 1259 1260 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1261 { 1262 int i; 1263 1264 for (i = 0; i < priv->num_tx_rings; i++) 1265 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1266 1267 for (i = 0; i < priv->num_rx_rings; i++) 1268 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1269 1270 udelay(1); 1271 } 1272 1273 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1274 { 1275 struct pci_dev *pdev = priv->si->pdev; 1276 cpumask_t cpu_mask; 1277 int i, j, err; 1278 1279 for (i = 0; i < priv->bdr_int_num; i++) { 1280 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1281 struct enetc_int_vector *v = priv->int_vector[i]; 1282 int entry = ENETC_BDR_INT_BASE_IDX + i; 1283 struct enetc_hw *hw = &priv->si->hw; 1284 1285 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1286 priv->ndev->name, i); 1287 err = request_irq(irq, enetc_msix, 0, v->name, v); 1288 if (err) { 1289 dev_err(priv->dev, "request_irq() failed!\n"); 1290 goto irq_err; 1291 } 1292 disable_irq(irq); 1293 1294 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1295 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1296 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 1297 1298 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1299 1300 for (j = 0; j < v->count_tx_rings; j++) { 1301 int idx = v->tx_ring[j].index; 1302 1303 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1304 } 1305 cpumask_clear(&cpu_mask); 1306 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1307 irq_set_affinity_hint(irq, &cpu_mask); 1308 } 1309 1310 return 0; 1311 1312 irq_err: 1313 while (i--) { 1314 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1315 1316 irq_set_affinity_hint(irq, NULL); 1317 free_irq(irq, priv->int_vector[i]); 1318 } 1319 1320 return err; 1321 } 1322 1323 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1324 { 1325 struct pci_dev *pdev = priv->si->pdev; 1326 int i; 1327 1328 for (i = 0; i < priv->bdr_int_num; i++) { 1329 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1330 1331 irq_set_affinity_hint(irq, NULL); 1332 free_irq(irq, priv->int_vector[i]); 1333 } 1334 } 1335 1336 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 1337 { 1338 struct enetc_hw *hw = &priv->si->hw; 1339 u32 icpt, ictt; 1340 int i; 1341 1342 /* enable Tx & Rx event indication */ 1343 if (priv->ic_mode & 1344 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 1345 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 1346 /* init to non-0 minimum, will be adjusted later */ 1347 ictt = 0x1; 1348 } else { 1349 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 1350 ictt = 0; 1351 } 1352 1353 for (i = 0; i < priv->num_rx_rings; i++) { 1354 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 1355 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 1356 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 1357 } 1358 1359 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 1360 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 1361 else 1362 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 1363 1364 for (i = 0; i < priv->num_tx_rings; i++) { 1365 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 1366 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 1367 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 1368 } 1369 } 1370 1371 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 1372 { 1373 int i; 1374 1375 for (i = 0; i < priv->num_tx_rings; i++) 1376 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1377 1378 for (i = 0; i < priv->num_rx_rings; i++) 1379 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1380 } 1381 1382 static int enetc_phylink_connect(struct net_device *ndev) 1383 { 1384 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1385 struct ethtool_eee edata; 1386 int err; 1387 1388 if (!priv->phylink) 1389 return 0; /* phy-less mode */ 1390 1391 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 1392 if (err) { 1393 dev_err(&ndev->dev, "could not attach to PHY\n"); 1394 return err; 1395 } 1396 1397 /* disable EEE autoneg, until ENETC driver supports it */ 1398 memset(&edata, 0, sizeof(struct ethtool_eee)); 1399 phylink_ethtool_set_eee(priv->phylink, &edata); 1400 1401 return 0; 1402 } 1403 1404 void enetc_start(struct net_device *ndev) 1405 { 1406 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1407 int i; 1408 1409 enetc_setup_interrupts(priv); 1410 1411 for (i = 0; i < priv->bdr_int_num; i++) { 1412 int irq = pci_irq_vector(priv->si->pdev, 1413 ENETC_BDR_INT_BASE_IDX + i); 1414 1415 napi_enable(&priv->int_vector[i]->napi); 1416 enable_irq(irq); 1417 } 1418 1419 if (priv->phylink) 1420 phylink_start(priv->phylink); 1421 else 1422 netif_carrier_on(ndev); 1423 1424 netif_tx_start_all_queues(ndev); 1425 } 1426 1427 int enetc_open(struct net_device *ndev) 1428 { 1429 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1430 int err; 1431 1432 err = enetc_setup_irqs(priv); 1433 if (err) 1434 return err; 1435 1436 err = enetc_phylink_connect(ndev); 1437 if (err) 1438 goto err_phy_connect; 1439 1440 err = enetc_alloc_tx_resources(priv); 1441 if (err) 1442 goto err_alloc_tx; 1443 1444 err = enetc_alloc_rx_resources(priv); 1445 if (err) 1446 goto err_alloc_rx; 1447 1448 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1449 if (err) 1450 goto err_set_queues; 1451 1452 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1453 if (err) 1454 goto err_set_queues; 1455 1456 enetc_setup_bdrs(priv); 1457 enetc_start(ndev); 1458 1459 return 0; 1460 1461 err_set_queues: 1462 enetc_free_rx_resources(priv); 1463 err_alloc_rx: 1464 enetc_free_tx_resources(priv); 1465 err_alloc_tx: 1466 if (priv->phylink) 1467 phylink_disconnect_phy(priv->phylink); 1468 err_phy_connect: 1469 enetc_free_irqs(priv); 1470 1471 return err; 1472 } 1473 1474 void enetc_stop(struct net_device *ndev) 1475 { 1476 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1477 int i; 1478 1479 netif_tx_stop_all_queues(ndev); 1480 1481 for (i = 0; i < priv->bdr_int_num; i++) { 1482 int irq = pci_irq_vector(priv->si->pdev, 1483 ENETC_BDR_INT_BASE_IDX + i); 1484 1485 disable_irq(irq); 1486 napi_synchronize(&priv->int_vector[i]->napi); 1487 napi_disable(&priv->int_vector[i]->napi); 1488 } 1489 1490 if (priv->phylink) 1491 phylink_stop(priv->phylink); 1492 else 1493 netif_carrier_off(ndev); 1494 1495 enetc_clear_interrupts(priv); 1496 } 1497 1498 int enetc_close(struct net_device *ndev) 1499 { 1500 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1501 1502 enetc_stop(ndev); 1503 enetc_clear_bdrs(priv); 1504 1505 if (priv->phylink) 1506 phylink_disconnect_phy(priv->phylink); 1507 enetc_free_rxtx_rings(priv); 1508 enetc_free_rx_resources(priv); 1509 enetc_free_tx_resources(priv); 1510 enetc_free_irqs(priv); 1511 1512 return 0; 1513 } 1514 1515 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1516 { 1517 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1518 struct tc_mqprio_qopt *mqprio = type_data; 1519 struct enetc_bdr *tx_ring; 1520 u8 num_tc; 1521 int i; 1522 1523 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1524 num_tc = mqprio->num_tc; 1525 1526 if (!num_tc) { 1527 netdev_reset_tc(ndev); 1528 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1529 1530 /* Reset all ring priorities to 0 */ 1531 for (i = 0; i < priv->num_tx_rings; i++) { 1532 tx_ring = priv->tx_ring[i]; 1533 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1534 } 1535 1536 return 0; 1537 } 1538 1539 /* Check if we have enough BD rings available to accommodate all TCs */ 1540 if (num_tc > priv->num_tx_rings) { 1541 netdev_err(ndev, "Max %d traffic classes supported\n", 1542 priv->num_tx_rings); 1543 return -EINVAL; 1544 } 1545 1546 /* For the moment, we use only one BD ring per TC. 1547 * 1548 * Configure num_tc BD rings with increasing priorities. 1549 */ 1550 for (i = 0; i < num_tc; i++) { 1551 tx_ring = priv->tx_ring[i]; 1552 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1553 } 1554 1555 /* Reset the number of netdev queues based on the TC count */ 1556 netif_set_real_num_tx_queues(ndev, num_tc); 1557 1558 netdev_set_num_tc(ndev, num_tc); 1559 1560 /* Each TC is associated with one netdev queue */ 1561 for (i = 0; i < num_tc; i++) 1562 netdev_set_tc_queue(ndev, i, 1, i); 1563 1564 return 0; 1565 } 1566 1567 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1568 void *type_data) 1569 { 1570 switch (type) { 1571 case TC_SETUP_QDISC_MQPRIO: 1572 return enetc_setup_tc_mqprio(ndev, type_data); 1573 case TC_SETUP_QDISC_TAPRIO: 1574 return enetc_setup_tc_taprio(ndev, type_data); 1575 case TC_SETUP_QDISC_CBS: 1576 return enetc_setup_tc_cbs(ndev, type_data); 1577 case TC_SETUP_QDISC_ETF: 1578 return enetc_setup_tc_txtime(ndev, type_data); 1579 case TC_SETUP_BLOCK: 1580 return enetc_setup_tc_psfp(ndev, type_data); 1581 default: 1582 return -EOPNOTSUPP; 1583 } 1584 } 1585 1586 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1587 { 1588 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1589 struct net_device_stats *stats = &ndev->stats; 1590 unsigned long packets = 0, bytes = 0; 1591 int i; 1592 1593 for (i = 0; i < priv->num_rx_rings; i++) { 1594 packets += priv->rx_ring[i]->stats.packets; 1595 bytes += priv->rx_ring[i]->stats.bytes; 1596 } 1597 1598 stats->rx_packets = packets; 1599 stats->rx_bytes = bytes; 1600 bytes = 0; 1601 packets = 0; 1602 1603 for (i = 0; i < priv->num_tx_rings; i++) { 1604 packets += priv->tx_ring[i]->stats.packets; 1605 bytes += priv->tx_ring[i]->stats.bytes; 1606 } 1607 1608 stats->tx_packets = packets; 1609 stats->tx_bytes = bytes; 1610 1611 return stats; 1612 } 1613 1614 static int enetc_set_rss(struct net_device *ndev, int en) 1615 { 1616 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1617 struct enetc_hw *hw = &priv->si->hw; 1618 u32 reg; 1619 1620 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1621 1622 reg = enetc_rd(hw, ENETC_SIMR); 1623 reg &= ~ENETC_SIMR_RSSE; 1624 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1625 enetc_wr(hw, ENETC_SIMR, reg); 1626 1627 return 0; 1628 } 1629 1630 static int enetc_set_psfp(struct net_device *ndev, int en) 1631 { 1632 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1633 int err; 1634 1635 if (en) { 1636 err = enetc_psfp_enable(priv); 1637 if (err) 1638 return err; 1639 1640 priv->active_offloads |= ENETC_F_QCI; 1641 return 0; 1642 } 1643 1644 err = enetc_psfp_disable(priv); 1645 if (err) 1646 return err; 1647 1648 priv->active_offloads &= ~ENETC_F_QCI; 1649 1650 return 0; 1651 } 1652 1653 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 1654 { 1655 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1656 int i; 1657 1658 for (i = 0; i < priv->num_rx_rings; i++) 1659 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en); 1660 } 1661 1662 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 1663 { 1664 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1665 int i; 1666 1667 for (i = 0; i < priv->num_tx_rings; i++) 1668 enetc_bdr_enable_txvlan(&priv->si->hw, i, en); 1669 } 1670 1671 int enetc_set_features(struct net_device *ndev, 1672 netdev_features_t features) 1673 { 1674 netdev_features_t changed = ndev->features ^ features; 1675 int err = 0; 1676 1677 if (changed & NETIF_F_RXHASH) 1678 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1679 1680 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 1681 enetc_enable_rxvlan(ndev, 1682 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 1683 1684 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 1685 enetc_enable_txvlan(ndev, 1686 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 1687 1688 if (changed & NETIF_F_HW_TC) 1689 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); 1690 1691 return err; 1692 } 1693 1694 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1695 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1696 { 1697 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1698 struct hwtstamp_config config; 1699 int ao; 1700 1701 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1702 return -EFAULT; 1703 1704 switch (config.tx_type) { 1705 case HWTSTAMP_TX_OFF: 1706 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1707 break; 1708 case HWTSTAMP_TX_ON: 1709 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1710 break; 1711 default: 1712 return -ERANGE; 1713 } 1714 1715 ao = priv->active_offloads; 1716 switch (config.rx_filter) { 1717 case HWTSTAMP_FILTER_NONE: 1718 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1719 break; 1720 default: 1721 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1722 config.rx_filter = HWTSTAMP_FILTER_ALL; 1723 } 1724 1725 if (netif_running(ndev) && ao != priv->active_offloads) { 1726 enetc_close(ndev); 1727 enetc_open(ndev); 1728 } 1729 1730 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1731 -EFAULT : 0; 1732 } 1733 1734 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1735 { 1736 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1737 struct hwtstamp_config config; 1738 1739 config.flags = 0; 1740 1741 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1742 config.tx_type = HWTSTAMP_TX_ON; 1743 else 1744 config.tx_type = HWTSTAMP_TX_OFF; 1745 1746 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1747 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1748 1749 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1750 -EFAULT : 0; 1751 } 1752 #endif 1753 1754 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1755 { 1756 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1757 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1758 if (cmd == SIOCSHWTSTAMP) 1759 return enetc_hwtstamp_set(ndev, rq); 1760 if (cmd == SIOCGHWTSTAMP) 1761 return enetc_hwtstamp_get(ndev, rq); 1762 #endif 1763 1764 if (!priv->phylink) 1765 return -EOPNOTSUPP; 1766 1767 return phylink_mii_ioctl(priv->phylink, rq, cmd); 1768 } 1769 1770 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1771 { 1772 struct pci_dev *pdev = priv->si->pdev; 1773 int v_tx_rings; 1774 int i, n, err, nvec; 1775 1776 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1777 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1778 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1779 1780 if (n < 0) 1781 return n; 1782 1783 if (n != nvec) 1784 return -EPERM; 1785 1786 /* # of tx rings per int vector */ 1787 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1788 1789 for (i = 0; i < priv->bdr_int_num; i++) { 1790 struct enetc_int_vector *v; 1791 struct enetc_bdr *bdr; 1792 int j; 1793 1794 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 1795 if (!v) { 1796 err = -ENOMEM; 1797 goto fail; 1798 } 1799 1800 priv->int_vector[i] = v; 1801 1802 /* init defaults for adaptive IC */ 1803 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 1804 v->rx_ictt = 0x1; 1805 v->rx_dim_en = true; 1806 } 1807 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 1808 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1809 NAPI_POLL_WEIGHT); 1810 v->count_tx_rings = v_tx_rings; 1811 1812 for (j = 0; j < v_tx_rings; j++) { 1813 int idx; 1814 1815 /* default tx ring mapping policy */ 1816 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1817 idx = 2 * j + i; /* 2 CPUs */ 1818 else 1819 idx = j + i * v_tx_rings; /* default */ 1820 1821 __set_bit(idx, &v->tx_rings_map); 1822 bdr = &v->tx_ring[j]; 1823 bdr->index = idx; 1824 bdr->ndev = priv->ndev; 1825 bdr->dev = priv->dev; 1826 bdr->bd_count = priv->tx_bd_count; 1827 priv->tx_ring[idx] = bdr; 1828 } 1829 1830 bdr = &v->rx_ring; 1831 bdr->index = i; 1832 bdr->ndev = priv->ndev; 1833 bdr->dev = priv->dev; 1834 bdr->bd_count = priv->rx_bd_count; 1835 priv->rx_ring[i] = bdr; 1836 } 1837 1838 return 0; 1839 1840 fail: 1841 while (i--) { 1842 netif_napi_del(&priv->int_vector[i]->napi); 1843 cancel_work_sync(&priv->int_vector[i]->rx_dim.work); 1844 kfree(priv->int_vector[i]); 1845 } 1846 1847 pci_free_irq_vectors(pdev); 1848 1849 return err; 1850 } 1851 1852 void enetc_free_msix(struct enetc_ndev_priv *priv) 1853 { 1854 int i; 1855 1856 for (i = 0; i < priv->bdr_int_num; i++) { 1857 struct enetc_int_vector *v = priv->int_vector[i]; 1858 1859 netif_napi_del(&v->napi); 1860 cancel_work_sync(&v->rx_dim.work); 1861 } 1862 1863 for (i = 0; i < priv->num_rx_rings; i++) 1864 priv->rx_ring[i] = NULL; 1865 1866 for (i = 0; i < priv->num_tx_rings; i++) 1867 priv->tx_ring[i] = NULL; 1868 1869 for (i = 0; i < priv->bdr_int_num; i++) { 1870 kfree(priv->int_vector[i]); 1871 priv->int_vector[i] = NULL; 1872 } 1873 1874 /* disable all MSIX for this device */ 1875 pci_free_irq_vectors(priv->si->pdev); 1876 } 1877 1878 static void enetc_kfree_si(struct enetc_si *si) 1879 { 1880 char *p = (char *)si - si->pad; 1881 1882 kfree(p); 1883 } 1884 1885 static void enetc_detect_errata(struct enetc_si *si) 1886 { 1887 if (si->pdev->revision == ENETC_REV1) 1888 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 1889 } 1890 1891 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1892 { 1893 struct enetc_si *si, *p; 1894 struct enetc_hw *hw; 1895 size_t alloc_size; 1896 int err, len; 1897 1898 pcie_flr(pdev); 1899 err = pci_enable_device_mem(pdev); 1900 if (err) { 1901 dev_err(&pdev->dev, "device enable failed\n"); 1902 return err; 1903 } 1904 1905 /* set up for high or low dma */ 1906 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1907 if (err) { 1908 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1909 if (err) { 1910 dev_err(&pdev->dev, 1911 "DMA configuration failed: 0x%x\n", err); 1912 goto err_dma; 1913 } 1914 } 1915 1916 err = pci_request_mem_regions(pdev, name); 1917 if (err) { 1918 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1919 goto err_pci_mem_reg; 1920 } 1921 1922 pci_set_master(pdev); 1923 1924 alloc_size = sizeof(struct enetc_si); 1925 if (sizeof_priv) { 1926 /* align priv to 32B */ 1927 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1928 alloc_size += sizeof_priv; 1929 } 1930 /* force 32B alignment for enetc_si */ 1931 alloc_size += ENETC_SI_ALIGN - 1; 1932 1933 p = kzalloc(alloc_size, GFP_KERNEL); 1934 if (!p) { 1935 err = -ENOMEM; 1936 goto err_alloc_si; 1937 } 1938 1939 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1940 si->pad = (char *)si - (char *)p; 1941 1942 pci_set_drvdata(pdev, si); 1943 si->pdev = pdev; 1944 hw = &si->hw; 1945 1946 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1947 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1948 if (!hw->reg) { 1949 err = -ENXIO; 1950 dev_err(&pdev->dev, "ioremap() failed\n"); 1951 goto err_ioremap; 1952 } 1953 if (len > ENETC_PORT_BASE) 1954 hw->port = hw->reg + ENETC_PORT_BASE; 1955 if (len > ENETC_GLOBAL_BASE) 1956 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1957 1958 enetc_detect_errata(si); 1959 1960 return 0; 1961 1962 err_ioremap: 1963 enetc_kfree_si(si); 1964 err_alloc_si: 1965 pci_release_mem_regions(pdev); 1966 err_pci_mem_reg: 1967 err_dma: 1968 pci_disable_device(pdev); 1969 1970 return err; 1971 } 1972 1973 void enetc_pci_remove(struct pci_dev *pdev) 1974 { 1975 struct enetc_si *si = pci_get_drvdata(pdev); 1976 struct enetc_hw *hw = &si->hw; 1977 1978 iounmap(hw->reg); 1979 enetc_kfree_si(si); 1980 pci_release_mem_regions(pdev); 1981 pci_disable_device(pdev); 1982 } 1983