1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/vmalloc.h> 8 9 /* ENETC overhead: optional extension BD + 1 BD gap */ 10 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 11 /* max # of chained Tx BDs is 15, including head and extension BD */ 12 #define ENETC_MAX_SKB_FRAGS 13 13 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 14 15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 16 int active_offloads); 17 18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 19 { 20 struct enetc_ndev_priv *priv = netdev_priv(ndev); 21 struct enetc_bdr *tx_ring; 22 int count; 23 24 tx_ring = priv->tx_ring[skb->queue_mapping]; 25 26 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 27 if (unlikely(skb_linearize(skb))) 28 goto drop_packet_err; 29 30 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 31 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 32 netif_stop_subqueue(ndev, tx_ring->index); 33 return NETDEV_TX_BUSY; 34 } 35 36 enetc_lock_mdio(); 37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 38 enetc_unlock_mdio(); 39 40 if (unlikely(!count)) 41 goto drop_packet_err; 42 43 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 44 netif_stop_subqueue(ndev, tx_ring->index); 45 46 return NETDEV_TX_OK; 47 48 drop_packet_err: 49 dev_kfree_skb_any(skb); 50 return NETDEV_TX_OK; 51 } 52 53 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd) 54 { 55 int l3_start, l3_hsize; 56 u16 l3_flags, l4_flags; 57 58 if (skb->ip_summed != CHECKSUM_PARTIAL) 59 return false; 60 61 switch (skb->csum_offset) { 62 case offsetof(struct tcphdr, check): 63 l4_flags = ENETC_TXBD_L4_TCP; 64 break; 65 case offsetof(struct udphdr, check): 66 l4_flags = ENETC_TXBD_L4_UDP; 67 break; 68 default: 69 skb_checksum_help(skb); 70 return false; 71 } 72 73 l3_start = skb_network_offset(skb); 74 l3_hsize = skb_network_header_len(skb); 75 76 l3_flags = 0; 77 if (skb->protocol == htons(ETH_P_IPV6)) 78 l3_flags = ENETC_TXBD_L3_IPV6; 79 80 /* write BD fields */ 81 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags); 82 txbd->l4_csoff = l4_flags; 83 84 return true; 85 } 86 87 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 88 struct enetc_tx_swbd *tx_swbd) 89 { 90 if (tx_swbd->is_dma_page) 91 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 92 tx_swbd->len, DMA_TO_DEVICE); 93 else 94 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 95 tx_swbd->len, DMA_TO_DEVICE); 96 tx_swbd->dma = 0; 97 } 98 99 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 100 struct enetc_tx_swbd *tx_swbd) 101 { 102 if (tx_swbd->dma) 103 enetc_unmap_tx_buff(tx_ring, tx_swbd); 104 105 if (tx_swbd->skb) { 106 dev_kfree_skb_any(tx_swbd->skb); 107 tx_swbd->skb = NULL; 108 } 109 } 110 111 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 112 int active_offloads) 113 { 114 struct enetc_tx_swbd *tx_swbd; 115 skb_frag_t *frag; 116 int len = skb_headlen(skb); 117 union enetc_tx_bd temp_bd; 118 union enetc_tx_bd *txbd; 119 bool do_vlan, do_tstamp; 120 int i, count = 0; 121 unsigned int f; 122 dma_addr_t dma; 123 u8 flags = 0; 124 125 i = tx_ring->next_to_use; 126 txbd = ENETC_TXBD(*tx_ring, i); 127 prefetchw(txbd); 128 129 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 130 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 131 goto dma_err; 132 133 temp_bd.addr = cpu_to_le64(dma); 134 temp_bd.buf_len = cpu_to_le16(len); 135 temp_bd.lstatus = 0; 136 137 tx_swbd = &tx_ring->tx_swbd[i]; 138 tx_swbd->dma = dma; 139 tx_swbd->len = len; 140 tx_swbd->is_dma_page = 0; 141 count++; 142 143 do_vlan = skb_vlan_tag_present(skb); 144 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 145 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 146 tx_swbd->do_tstamp = do_tstamp; 147 tx_swbd->check_wb = tx_swbd->do_tstamp; 148 149 if (do_vlan || do_tstamp) 150 flags |= ENETC_TXBD_FLAGS_EX; 151 152 if (enetc_tx_csum(skb, &temp_bd)) 153 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS; 154 else if (tx_ring->tsd_enable) 155 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 156 157 /* first BD needs frm_len and offload flags set */ 158 temp_bd.frm_len = cpu_to_le16(skb->len); 159 temp_bd.flags = flags; 160 161 if (flags & ENETC_TXBD_FLAGS_TSE) { 162 u32 temp; 163 164 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK) 165 | (flags << ENETC_TXBD_FLAGS_OFFSET); 166 temp_bd.txstart = cpu_to_le32(temp); 167 } 168 169 if (flags & ENETC_TXBD_FLAGS_EX) { 170 u8 e_flags = 0; 171 *txbd = temp_bd; 172 enetc_clear_tx_bd(&temp_bd); 173 174 /* add extension BD for VLAN and/or timestamping */ 175 flags = 0; 176 tx_swbd++; 177 txbd++; 178 i++; 179 if (unlikely(i == tx_ring->bd_count)) { 180 i = 0; 181 tx_swbd = tx_ring->tx_swbd; 182 txbd = ENETC_TXBD(*tx_ring, 0); 183 } 184 prefetchw(txbd); 185 186 if (do_vlan) { 187 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 188 temp_bd.ext.tpid = 0; /* < C-TAG */ 189 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 190 } 191 192 if (do_tstamp) { 193 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 194 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 195 } 196 197 temp_bd.ext.e_flags = e_flags; 198 count++; 199 } 200 201 frag = &skb_shinfo(skb)->frags[0]; 202 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 203 len = skb_frag_size(frag); 204 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 205 DMA_TO_DEVICE); 206 if (dma_mapping_error(tx_ring->dev, dma)) 207 goto dma_err; 208 209 *txbd = temp_bd; 210 enetc_clear_tx_bd(&temp_bd); 211 212 flags = 0; 213 tx_swbd++; 214 txbd++; 215 i++; 216 if (unlikely(i == tx_ring->bd_count)) { 217 i = 0; 218 tx_swbd = tx_ring->tx_swbd; 219 txbd = ENETC_TXBD(*tx_ring, 0); 220 } 221 prefetchw(txbd); 222 223 temp_bd.addr = cpu_to_le64(dma); 224 temp_bd.buf_len = cpu_to_le16(len); 225 226 tx_swbd->dma = dma; 227 tx_swbd->len = len; 228 tx_swbd->is_dma_page = 1; 229 count++; 230 } 231 232 /* last BD needs 'F' bit set */ 233 flags |= ENETC_TXBD_FLAGS_F; 234 temp_bd.flags = flags; 235 *txbd = temp_bd; 236 237 tx_ring->tx_swbd[i].skb = skb; 238 239 enetc_bdr_idx_inc(tx_ring, &i); 240 tx_ring->next_to_use = i; 241 242 skb_tx_timestamp(skb); 243 244 /* let H/W know BD ring has been updated */ 245 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */ 246 247 return count; 248 249 dma_err: 250 dev_err(tx_ring->dev, "DMA map error"); 251 252 do { 253 tx_swbd = &tx_ring->tx_swbd[i]; 254 enetc_free_tx_skb(tx_ring, tx_swbd); 255 if (i == 0) 256 i = tx_ring->bd_count; 257 i--; 258 } while (count--); 259 260 return 0; 261 } 262 263 static irqreturn_t enetc_msix(int irq, void *data) 264 { 265 struct enetc_int_vector *v = data; 266 int i; 267 268 enetc_lock_mdio(); 269 270 /* disable interrupts */ 271 enetc_wr_reg_hot(v->rbier, 0); 272 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 273 274 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 275 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 276 277 enetc_unlock_mdio(); 278 279 napi_schedule(&v->napi); 280 281 return IRQ_HANDLED; 282 } 283 284 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 285 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 286 struct napi_struct *napi, int work_limit); 287 288 static void enetc_rx_dim_work(struct work_struct *w) 289 { 290 struct dim *dim = container_of(w, struct dim, work); 291 struct dim_cq_moder moder = 292 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 293 struct enetc_int_vector *v = 294 container_of(dim, struct enetc_int_vector, rx_dim); 295 296 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 297 dim->state = DIM_START_MEASURE; 298 } 299 300 static void enetc_rx_net_dim(struct enetc_int_vector *v) 301 { 302 struct dim_sample dim_sample; 303 304 v->comp_cnt++; 305 306 if (!v->rx_napi_work) 307 return; 308 309 dim_update_sample(v->comp_cnt, 310 v->rx_ring.stats.packets, 311 v->rx_ring.stats.bytes, 312 &dim_sample); 313 net_dim(&v->rx_dim, dim_sample); 314 } 315 316 static int enetc_poll(struct napi_struct *napi, int budget) 317 { 318 struct enetc_int_vector 319 *v = container_of(napi, struct enetc_int_vector, napi); 320 bool complete = true; 321 int work_done; 322 int i; 323 324 for (i = 0; i < v->count_tx_rings; i++) 325 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 326 complete = false; 327 328 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 329 if (work_done == budget) 330 complete = false; 331 if (work_done) 332 v->rx_napi_work = true; 333 334 if (!complete) 335 return budget; 336 337 napi_complete_done(napi, work_done); 338 339 if (likely(v->rx_dim_en)) 340 enetc_rx_net_dim(v); 341 342 v->rx_napi_work = false; 343 344 enetc_lock_mdio(); 345 346 /* enable interrupts */ 347 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 348 349 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 350 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 351 ENETC_TBIER_TXTIE); 352 353 enetc_unlock_mdio(); 354 355 return work_done; 356 } 357 358 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 359 { 360 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 361 362 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 363 } 364 365 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 366 u64 *tstamp) 367 { 368 u32 lo, hi, tstamp_lo; 369 370 lo = enetc_rd(hw, ENETC_SICTR0); 371 hi = enetc_rd(hw, ENETC_SICTR1); 372 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 373 if (lo <= tstamp_lo) 374 hi -= 1; 375 *tstamp = (u64)hi << 32 | tstamp_lo; 376 } 377 378 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 379 { 380 struct skb_shared_hwtstamps shhwtstamps; 381 382 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 383 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 384 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 385 skb_tstamp_tx(skb, &shhwtstamps); 386 } 387 } 388 389 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 390 { 391 struct net_device *ndev = tx_ring->ndev; 392 int tx_frm_cnt = 0, tx_byte_cnt = 0; 393 struct enetc_tx_swbd *tx_swbd; 394 int i, bds_to_clean; 395 bool do_tstamp; 396 u64 tstamp = 0; 397 398 i = tx_ring->next_to_clean; 399 tx_swbd = &tx_ring->tx_swbd[i]; 400 401 enetc_lock_mdio(); 402 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 403 enetc_unlock_mdio(); 404 405 do_tstamp = false; 406 407 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 408 bool is_eof = !!tx_swbd->skb; 409 410 if (unlikely(tx_swbd->check_wb)) { 411 struct enetc_ndev_priv *priv = netdev_priv(ndev); 412 union enetc_tx_bd *txbd; 413 414 txbd = ENETC_TXBD(*tx_ring, i); 415 416 if (txbd->flags & ENETC_TXBD_FLAGS_W && 417 tx_swbd->do_tstamp) { 418 enetc_get_tx_tstamp(&priv->si->hw, txbd, 419 &tstamp); 420 do_tstamp = true; 421 } 422 } 423 424 if (likely(tx_swbd->dma)) 425 enetc_unmap_tx_buff(tx_ring, tx_swbd); 426 427 if (is_eof) { 428 if (unlikely(do_tstamp)) { 429 enetc_tstamp_tx(tx_swbd->skb, tstamp); 430 do_tstamp = false; 431 } 432 napi_consume_skb(tx_swbd->skb, napi_budget); 433 tx_swbd->skb = NULL; 434 } 435 436 tx_byte_cnt += tx_swbd->len; 437 438 bds_to_clean--; 439 tx_swbd++; 440 i++; 441 if (unlikely(i == tx_ring->bd_count)) { 442 i = 0; 443 tx_swbd = tx_ring->tx_swbd; 444 } 445 446 enetc_lock_mdio(); 447 448 /* BD iteration loop end */ 449 if (is_eof) { 450 tx_frm_cnt++; 451 /* re-arm interrupt source */ 452 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 453 BIT(16 + tx_ring->index)); 454 } 455 456 if (unlikely(!bds_to_clean)) 457 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 458 459 enetc_unlock_mdio(); 460 } 461 462 tx_ring->next_to_clean = i; 463 tx_ring->stats.packets += tx_frm_cnt; 464 tx_ring->stats.bytes += tx_byte_cnt; 465 466 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 467 __netif_subqueue_stopped(ndev, tx_ring->index) && 468 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 469 netif_wake_subqueue(ndev, tx_ring->index); 470 } 471 472 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 473 } 474 475 static bool enetc_new_page(struct enetc_bdr *rx_ring, 476 struct enetc_rx_swbd *rx_swbd) 477 { 478 struct page *page; 479 dma_addr_t addr; 480 481 page = dev_alloc_page(); 482 if (unlikely(!page)) 483 return false; 484 485 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 486 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 487 __free_page(page); 488 489 return false; 490 } 491 492 rx_swbd->dma = addr; 493 rx_swbd->page = page; 494 rx_swbd->page_offset = ENETC_RXB_PAD; 495 496 return true; 497 } 498 499 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 500 { 501 struct enetc_rx_swbd *rx_swbd; 502 union enetc_rx_bd *rxbd; 503 int i, j; 504 505 i = rx_ring->next_to_use; 506 rx_swbd = &rx_ring->rx_swbd[i]; 507 rxbd = enetc_rxbd(rx_ring, i); 508 509 for (j = 0; j < buff_cnt; j++) { 510 /* try reuse page */ 511 if (unlikely(!rx_swbd->page)) { 512 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 513 rx_ring->stats.rx_alloc_errs++; 514 break; 515 } 516 } 517 518 /* update RxBD */ 519 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 520 rx_swbd->page_offset); 521 /* clear 'R" as well */ 522 rxbd->r.lstatus = 0; 523 524 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 525 rx_swbd++; 526 i++; 527 if (unlikely(i == rx_ring->bd_count)) { 528 i = 0; 529 rx_swbd = rx_ring->rx_swbd; 530 } 531 } 532 533 if (likely(j)) { 534 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 535 rx_ring->next_to_use = i; 536 } 537 538 return j; 539 } 540 541 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 542 static void enetc_get_rx_tstamp(struct net_device *ndev, 543 union enetc_rx_bd *rxbd, 544 struct sk_buff *skb) 545 { 546 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 547 struct enetc_ndev_priv *priv = netdev_priv(ndev); 548 struct enetc_hw *hw = &priv->si->hw; 549 u32 lo, hi, tstamp_lo; 550 u64 tstamp; 551 552 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 553 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 554 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 555 rxbd = enetc_rxbd_ext(rxbd); 556 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 557 if (lo <= tstamp_lo) 558 hi -= 1; 559 560 tstamp = (u64)hi << 32 | tstamp_lo; 561 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 562 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 563 } 564 } 565 #endif 566 567 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 568 union enetc_rx_bd *rxbd, struct sk_buff *skb) 569 { 570 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 571 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 572 #endif 573 /* TODO: hashing */ 574 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 575 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 576 577 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 578 skb->ip_summed = CHECKSUM_COMPLETE; 579 } 580 581 /* copy VLAN to skb, if one is extracted, for now we assume it's a 582 * standard TPID, but HW also supports custom values 583 */ 584 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) 585 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 586 le16_to_cpu(rxbd->r.vlan_opt)); 587 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 588 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 589 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 590 #endif 591 } 592 593 static void enetc_process_skb(struct enetc_bdr *rx_ring, 594 struct sk_buff *skb) 595 { 596 skb_record_rx_queue(skb, rx_ring->index); 597 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 598 } 599 600 static bool enetc_page_reusable(struct page *page) 601 { 602 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 603 } 604 605 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 606 struct enetc_rx_swbd *old) 607 { 608 struct enetc_rx_swbd *new; 609 610 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 611 612 /* next buf that may reuse a page */ 613 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 614 615 /* copy page reference */ 616 *new = *old; 617 } 618 619 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 620 int i, u16 size) 621 { 622 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 623 624 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 625 rx_swbd->page_offset, 626 size, DMA_FROM_DEVICE); 627 return rx_swbd; 628 } 629 630 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 631 struct enetc_rx_swbd *rx_swbd) 632 { 633 if (likely(enetc_page_reusable(rx_swbd->page))) { 634 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 635 page_ref_inc(rx_swbd->page); 636 637 enetc_reuse_page(rx_ring, rx_swbd); 638 639 /* sync for use by the device */ 640 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 641 rx_swbd->page_offset, 642 ENETC_RXB_DMA_SIZE, 643 DMA_FROM_DEVICE); 644 } else { 645 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 646 PAGE_SIZE, DMA_FROM_DEVICE); 647 } 648 649 rx_swbd->page = NULL; 650 } 651 652 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 653 int i, u16 size) 654 { 655 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 656 struct sk_buff *skb; 657 void *ba; 658 659 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 660 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 661 if (unlikely(!skb)) { 662 rx_ring->stats.rx_alloc_errs++; 663 return NULL; 664 } 665 666 skb_reserve(skb, ENETC_RXB_PAD); 667 __skb_put(skb, size); 668 669 enetc_put_rx_buff(rx_ring, rx_swbd); 670 671 return skb; 672 } 673 674 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 675 u16 size, struct sk_buff *skb) 676 { 677 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 678 679 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 680 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 681 682 enetc_put_rx_buff(rx_ring, rx_swbd); 683 } 684 685 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 686 687 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 688 struct napi_struct *napi, int work_limit) 689 { 690 int rx_frm_cnt = 0, rx_byte_cnt = 0; 691 int cleaned_cnt, i; 692 693 cleaned_cnt = enetc_bd_unused(rx_ring); 694 /* next descriptor to process */ 695 i = rx_ring->next_to_clean; 696 697 while (likely(rx_frm_cnt < work_limit)) { 698 union enetc_rx_bd *rxbd; 699 struct sk_buff *skb; 700 u32 bd_status; 701 u16 size; 702 703 enetc_lock_mdio(); 704 705 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 706 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 707 708 /* update ENETC's consumer index */ 709 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 710 cleaned_cnt -= count; 711 } 712 713 rxbd = enetc_rxbd(rx_ring, i); 714 bd_status = le32_to_cpu(rxbd->r.lstatus); 715 if (!bd_status) { 716 enetc_unlock_mdio(); 717 break; 718 } 719 720 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 721 dma_rmb(); /* for reading other rxbd fields */ 722 size = le16_to_cpu(rxbd->r.buf_len); 723 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 724 if (!skb) { 725 enetc_unlock_mdio(); 726 break; 727 } 728 729 enetc_get_offloads(rx_ring, rxbd, skb); 730 731 cleaned_cnt++; 732 733 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 734 if (unlikely(++i == rx_ring->bd_count)) 735 i = 0; 736 737 if (unlikely(bd_status & 738 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 739 enetc_unlock_mdio(); 740 dev_kfree_skb(skb); 741 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 742 dma_rmb(); 743 bd_status = le32_to_cpu(rxbd->r.lstatus); 744 745 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 746 if (unlikely(++i == rx_ring->bd_count)) 747 i = 0; 748 } 749 750 rx_ring->ndev->stats.rx_dropped++; 751 rx_ring->ndev->stats.rx_errors++; 752 753 break; 754 } 755 756 /* not last BD in frame? */ 757 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 758 bd_status = le32_to_cpu(rxbd->r.lstatus); 759 size = ENETC_RXB_DMA_SIZE; 760 761 if (bd_status & ENETC_RXBD_LSTATUS_F) { 762 dma_rmb(); 763 size = le16_to_cpu(rxbd->r.buf_len); 764 } 765 766 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 767 768 cleaned_cnt++; 769 770 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 771 if (unlikely(++i == rx_ring->bd_count)) 772 i = 0; 773 } 774 775 rx_byte_cnt += skb->len; 776 777 enetc_process_skb(rx_ring, skb); 778 779 enetc_unlock_mdio(); 780 781 napi_gro_receive(napi, skb); 782 783 rx_frm_cnt++; 784 } 785 786 rx_ring->next_to_clean = i; 787 788 rx_ring->stats.packets += rx_frm_cnt; 789 rx_ring->stats.bytes += rx_byte_cnt; 790 791 return rx_frm_cnt; 792 } 793 794 /* Probing and Init */ 795 #define ENETC_MAX_RFS_SIZE 64 796 void enetc_get_si_caps(struct enetc_si *si) 797 { 798 struct enetc_hw *hw = &si->hw; 799 u32 val; 800 801 /* find out how many of various resources we have to work with */ 802 val = enetc_rd(hw, ENETC_SICAPR0); 803 si->num_rx_rings = (val >> 16) & 0xff; 804 si->num_tx_rings = val & 0xff; 805 806 val = enetc_rd(hw, ENETC_SIRFSCAPR); 807 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 808 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 809 810 si->num_rss = 0; 811 val = enetc_rd(hw, ENETC_SIPCAPR0); 812 if (val & ENETC_SIPCAPR0_RSS) { 813 u32 rss; 814 815 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 816 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 817 } 818 819 if (val & ENETC_SIPCAPR0_QBV) 820 si->hw_features |= ENETC_SI_F_QBV; 821 822 if (val & ENETC_SIPCAPR0_PSFP) 823 si->hw_features |= ENETC_SI_F_PSFP; 824 } 825 826 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 827 { 828 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 829 &r->bd_dma_base, GFP_KERNEL); 830 if (!r->bd_base) 831 return -ENOMEM; 832 833 /* h/w requires 128B alignment */ 834 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 835 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 836 r->bd_dma_base); 837 return -EINVAL; 838 } 839 840 return 0; 841 } 842 843 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 844 { 845 int err; 846 847 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 848 if (!txr->tx_swbd) 849 return -ENOMEM; 850 851 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 852 if (err) { 853 vfree(txr->tx_swbd); 854 return err; 855 } 856 857 txr->next_to_clean = 0; 858 txr->next_to_use = 0; 859 860 return 0; 861 } 862 863 static void enetc_free_txbdr(struct enetc_bdr *txr) 864 { 865 int size, i; 866 867 for (i = 0; i < txr->bd_count; i++) 868 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 869 870 size = txr->bd_count * sizeof(union enetc_tx_bd); 871 872 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 873 txr->bd_base = NULL; 874 875 vfree(txr->tx_swbd); 876 txr->tx_swbd = NULL; 877 } 878 879 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 880 { 881 int i, err; 882 883 for (i = 0; i < priv->num_tx_rings; i++) { 884 err = enetc_alloc_txbdr(priv->tx_ring[i]); 885 886 if (err) 887 goto fail; 888 } 889 890 return 0; 891 892 fail: 893 while (i-- > 0) 894 enetc_free_txbdr(priv->tx_ring[i]); 895 896 return err; 897 } 898 899 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 900 { 901 int i; 902 903 for (i = 0; i < priv->num_tx_rings; i++) 904 enetc_free_txbdr(priv->tx_ring[i]); 905 } 906 907 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 908 { 909 size_t size = sizeof(union enetc_rx_bd); 910 int err; 911 912 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 913 if (!rxr->rx_swbd) 914 return -ENOMEM; 915 916 if (extended) 917 size *= 2; 918 919 err = enetc_dma_alloc_bdr(rxr, size); 920 if (err) { 921 vfree(rxr->rx_swbd); 922 return err; 923 } 924 925 rxr->next_to_clean = 0; 926 rxr->next_to_use = 0; 927 rxr->next_to_alloc = 0; 928 rxr->ext_en = extended; 929 930 return 0; 931 } 932 933 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 934 { 935 int size; 936 937 size = rxr->bd_count * sizeof(union enetc_rx_bd); 938 939 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 940 rxr->bd_base = NULL; 941 942 vfree(rxr->rx_swbd); 943 rxr->rx_swbd = NULL; 944 } 945 946 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 947 { 948 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 949 int i, err; 950 951 for (i = 0; i < priv->num_rx_rings; i++) { 952 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 953 954 if (err) 955 goto fail; 956 } 957 958 return 0; 959 960 fail: 961 while (i-- > 0) 962 enetc_free_rxbdr(priv->rx_ring[i]); 963 964 return err; 965 } 966 967 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 968 { 969 int i; 970 971 for (i = 0; i < priv->num_rx_rings; i++) 972 enetc_free_rxbdr(priv->rx_ring[i]); 973 } 974 975 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 976 { 977 int i; 978 979 if (!tx_ring->tx_swbd) 980 return; 981 982 for (i = 0; i < tx_ring->bd_count; i++) { 983 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 984 985 enetc_free_tx_skb(tx_ring, tx_swbd); 986 } 987 988 tx_ring->next_to_clean = 0; 989 tx_ring->next_to_use = 0; 990 } 991 992 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 993 { 994 int i; 995 996 if (!rx_ring->rx_swbd) 997 return; 998 999 for (i = 0; i < rx_ring->bd_count; i++) { 1000 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1001 1002 if (!rx_swbd->page) 1003 continue; 1004 1005 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 1006 PAGE_SIZE, DMA_FROM_DEVICE); 1007 __free_page(rx_swbd->page); 1008 rx_swbd->page = NULL; 1009 } 1010 1011 rx_ring->next_to_clean = 0; 1012 rx_ring->next_to_use = 0; 1013 rx_ring->next_to_alloc = 0; 1014 } 1015 1016 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 1017 { 1018 int i; 1019 1020 for (i = 0; i < priv->num_rx_rings; i++) 1021 enetc_free_rx_ring(priv->rx_ring[i]); 1022 1023 for (i = 0; i < priv->num_tx_rings; i++) 1024 enetc_free_tx_ring(priv->tx_ring[i]); 1025 } 1026 1027 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1028 { 1029 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1030 1031 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 1032 GFP_KERNEL); 1033 if (!cbdr->bd_base) 1034 return -ENOMEM; 1035 1036 /* h/w requires 128B alignment */ 1037 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 1038 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1039 return -EINVAL; 1040 } 1041 1042 cbdr->next_to_clean = 0; 1043 cbdr->next_to_use = 0; 1044 1045 return 0; 1046 } 1047 1048 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1049 { 1050 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1051 1052 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1053 cbdr->bd_base = NULL; 1054 } 1055 1056 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 1057 { 1058 /* set CBDR cache attributes */ 1059 enetc_wr(hw, ENETC_SICAR2, 1060 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1061 1062 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 1063 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 1064 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 1065 1066 enetc_wr(hw, ENETC_SICBDRPIR, 0); 1067 enetc_wr(hw, ENETC_SICBDRCIR, 0); 1068 1069 /* enable ring */ 1070 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 1071 1072 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 1073 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 1074 } 1075 1076 static void enetc_clear_cbdr(struct enetc_hw *hw) 1077 { 1078 enetc_wr(hw, ENETC_SICBDRMR, 0); 1079 } 1080 1081 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1082 { 1083 int *rss_table; 1084 int i; 1085 1086 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1087 if (!rss_table) 1088 return -ENOMEM; 1089 1090 /* Set up RSS table defaults */ 1091 for (i = 0; i < si->num_rss; i++) 1092 rss_table[i] = i % num_groups; 1093 1094 enetc_set_rss_table(si, rss_table, si->num_rss); 1095 1096 kfree(rss_table); 1097 1098 return 0; 1099 } 1100 1101 static int enetc_configure_si(struct enetc_ndev_priv *priv) 1102 { 1103 struct enetc_si *si = priv->si; 1104 struct enetc_hw *hw = &si->hw; 1105 int err; 1106 1107 enetc_setup_cbdr(hw, &si->cbd_ring); 1108 /* set SI cache attributes */ 1109 enetc_wr(hw, ENETC_SICAR0, 1110 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1111 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1112 /* enable SI */ 1113 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1114 1115 if (si->num_rss) { 1116 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1117 if (err) 1118 return err; 1119 } 1120 1121 return 0; 1122 } 1123 1124 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1125 { 1126 struct enetc_si *si = priv->si; 1127 int cpus = num_online_cpus(); 1128 1129 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 1130 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 1131 1132 /* Enable all available TX rings in order to configure as many 1133 * priorities as possible, when needed. 1134 * TODO: Make # of TX rings run-time configurable 1135 */ 1136 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1137 priv->num_tx_rings = si->num_tx_rings; 1138 priv->bdr_int_num = cpus; 1139 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 1140 priv->tx_ictt = ENETC_TXIC_TIMETHR; 1141 1142 /* SI specific */ 1143 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1144 } 1145 1146 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1147 { 1148 struct enetc_si *si = priv->si; 1149 int err; 1150 1151 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1152 if (err) 1153 return err; 1154 1155 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1156 GFP_KERNEL); 1157 if (!priv->cls_rules) { 1158 err = -ENOMEM; 1159 goto err_alloc_cls; 1160 } 1161 1162 err = enetc_configure_si(priv); 1163 if (err) 1164 goto err_config_si; 1165 1166 return 0; 1167 1168 err_config_si: 1169 kfree(priv->cls_rules); 1170 err_alloc_cls: 1171 enetc_clear_cbdr(&si->hw); 1172 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1173 1174 return err; 1175 } 1176 1177 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1178 { 1179 struct enetc_si *si = priv->si; 1180 1181 enetc_clear_cbdr(&si->hw); 1182 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1183 1184 kfree(priv->cls_rules); 1185 } 1186 1187 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1188 { 1189 int idx = tx_ring->index; 1190 u32 tbmr; 1191 1192 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1193 lower_32_bits(tx_ring->bd_dma_base)); 1194 1195 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1196 upper_32_bits(tx_ring->bd_dma_base)); 1197 1198 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1199 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1200 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1201 1202 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1203 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1204 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1205 1206 /* enable Tx ints by setting pkt thr to 1 */ 1207 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 1208 1209 tbmr = ENETC_TBMR_EN; 1210 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1211 tbmr |= ENETC_TBMR_VIH; 1212 1213 /* enable ring */ 1214 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1215 1216 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1217 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1218 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1219 } 1220 1221 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1222 { 1223 int idx = rx_ring->index; 1224 u32 rbmr; 1225 1226 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1227 lower_32_bits(rx_ring->bd_dma_base)); 1228 1229 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1230 upper_32_bits(rx_ring->bd_dma_base)); 1231 1232 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1233 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1234 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1235 1236 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1237 1238 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1239 1240 /* enable Rx ints by setting pkt thr to 1 */ 1241 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 1242 1243 rbmr = ENETC_RBMR_EN; 1244 1245 if (rx_ring->ext_en) 1246 rbmr |= ENETC_RBMR_BDS; 1247 1248 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1249 rbmr |= ENETC_RBMR_VTE; 1250 1251 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1252 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1253 1254 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1255 enetc_wr(hw, ENETC_SIRXIDR, rx_ring->next_to_use); 1256 1257 /* enable ring */ 1258 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1259 } 1260 1261 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1262 { 1263 int i; 1264 1265 for (i = 0; i < priv->num_tx_rings; i++) 1266 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1267 1268 for (i = 0; i < priv->num_rx_rings; i++) 1269 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1270 } 1271 1272 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1273 { 1274 int idx = rx_ring->index; 1275 1276 /* disable EN bit on ring */ 1277 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1278 } 1279 1280 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1281 { 1282 int delay = 8, timeout = 100; 1283 int idx = tx_ring->index; 1284 1285 /* disable EN bit on ring */ 1286 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1287 1288 /* wait for busy to clear */ 1289 while (delay < timeout && 1290 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1291 msleep(delay); 1292 delay *= 2; 1293 } 1294 1295 if (delay >= timeout) 1296 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1297 idx); 1298 } 1299 1300 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1301 { 1302 int i; 1303 1304 for (i = 0; i < priv->num_tx_rings; i++) 1305 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1306 1307 for (i = 0; i < priv->num_rx_rings; i++) 1308 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1309 1310 udelay(1); 1311 } 1312 1313 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1314 { 1315 struct pci_dev *pdev = priv->si->pdev; 1316 cpumask_t cpu_mask; 1317 int i, j, err; 1318 1319 for (i = 0; i < priv->bdr_int_num; i++) { 1320 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1321 struct enetc_int_vector *v = priv->int_vector[i]; 1322 int entry = ENETC_BDR_INT_BASE_IDX + i; 1323 struct enetc_hw *hw = &priv->si->hw; 1324 1325 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1326 priv->ndev->name, i); 1327 err = request_irq(irq, enetc_msix, 0, v->name, v); 1328 if (err) { 1329 dev_err(priv->dev, "request_irq() failed!\n"); 1330 goto irq_err; 1331 } 1332 disable_irq(irq); 1333 1334 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1335 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1336 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 1337 1338 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1339 1340 for (j = 0; j < v->count_tx_rings; j++) { 1341 int idx = v->tx_ring[j].index; 1342 1343 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1344 } 1345 cpumask_clear(&cpu_mask); 1346 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1347 irq_set_affinity_hint(irq, &cpu_mask); 1348 } 1349 1350 return 0; 1351 1352 irq_err: 1353 while (i--) { 1354 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1355 1356 irq_set_affinity_hint(irq, NULL); 1357 free_irq(irq, priv->int_vector[i]); 1358 } 1359 1360 return err; 1361 } 1362 1363 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1364 { 1365 struct pci_dev *pdev = priv->si->pdev; 1366 int i; 1367 1368 for (i = 0; i < priv->bdr_int_num; i++) { 1369 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1370 1371 irq_set_affinity_hint(irq, NULL); 1372 free_irq(irq, priv->int_vector[i]); 1373 } 1374 } 1375 1376 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 1377 { 1378 struct enetc_hw *hw = &priv->si->hw; 1379 u32 icpt, ictt; 1380 int i; 1381 1382 /* enable Tx & Rx event indication */ 1383 if (priv->ic_mode & 1384 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 1385 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 1386 /* init to non-0 minimum, will be adjusted later */ 1387 ictt = 0x1; 1388 } else { 1389 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 1390 ictt = 0; 1391 } 1392 1393 for (i = 0; i < priv->num_rx_rings; i++) { 1394 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 1395 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 1396 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 1397 } 1398 1399 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 1400 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 1401 else 1402 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 1403 1404 for (i = 0; i < priv->num_tx_rings; i++) { 1405 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 1406 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 1407 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 1408 } 1409 } 1410 1411 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 1412 { 1413 int i; 1414 1415 for (i = 0; i < priv->num_tx_rings; i++) 1416 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1417 1418 for (i = 0; i < priv->num_rx_rings; i++) 1419 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1420 } 1421 1422 static int enetc_phylink_connect(struct net_device *ndev) 1423 { 1424 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1425 struct ethtool_eee edata; 1426 int err; 1427 1428 if (!priv->phylink) 1429 return 0; /* phy-less mode */ 1430 1431 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 1432 if (err) { 1433 dev_err(&ndev->dev, "could not attach to PHY\n"); 1434 return err; 1435 } 1436 1437 /* disable EEE autoneg, until ENETC driver supports it */ 1438 memset(&edata, 0, sizeof(struct ethtool_eee)); 1439 phylink_ethtool_set_eee(priv->phylink, &edata); 1440 1441 return 0; 1442 } 1443 1444 void enetc_start(struct net_device *ndev) 1445 { 1446 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1447 int i; 1448 1449 enetc_setup_interrupts(priv); 1450 1451 for (i = 0; i < priv->bdr_int_num; i++) { 1452 int irq = pci_irq_vector(priv->si->pdev, 1453 ENETC_BDR_INT_BASE_IDX + i); 1454 1455 napi_enable(&priv->int_vector[i]->napi); 1456 enable_irq(irq); 1457 } 1458 1459 if (priv->phylink) 1460 phylink_start(priv->phylink); 1461 else 1462 netif_carrier_on(ndev); 1463 1464 netif_tx_start_all_queues(ndev); 1465 } 1466 1467 int enetc_open(struct net_device *ndev) 1468 { 1469 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1470 int err; 1471 1472 err = enetc_setup_irqs(priv); 1473 if (err) 1474 return err; 1475 1476 err = enetc_phylink_connect(ndev); 1477 if (err) 1478 goto err_phy_connect; 1479 1480 err = enetc_alloc_tx_resources(priv); 1481 if (err) 1482 goto err_alloc_tx; 1483 1484 err = enetc_alloc_rx_resources(priv); 1485 if (err) 1486 goto err_alloc_rx; 1487 1488 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1489 if (err) 1490 goto err_set_queues; 1491 1492 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1493 if (err) 1494 goto err_set_queues; 1495 1496 enetc_setup_bdrs(priv); 1497 enetc_start(ndev); 1498 1499 return 0; 1500 1501 err_set_queues: 1502 enetc_free_rx_resources(priv); 1503 err_alloc_rx: 1504 enetc_free_tx_resources(priv); 1505 err_alloc_tx: 1506 if (priv->phylink) 1507 phylink_disconnect_phy(priv->phylink); 1508 err_phy_connect: 1509 enetc_free_irqs(priv); 1510 1511 return err; 1512 } 1513 1514 void enetc_stop(struct net_device *ndev) 1515 { 1516 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1517 int i; 1518 1519 netif_tx_stop_all_queues(ndev); 1520 1521 for (i = 0; i < priv->bdr_int_num; i++) { 1522 int irq = pci_irq_vector(priv->si->pdev, 1523 ENETC_BDR_INT_BASE_IDX + i); 1524 1525 disable_irq(irq); 1526 napi_synchronize(&priv->int_vector[i]->napi); 1527 napi_disable(&priv->int_vector[i]->napi); 1528 } 1529 1530 if (priv->phylink) 1531 phylink_stop(priv->phylink); 1532 else 1533 netif_carrier_off(ndev); 1534 1535 enetc_clear_interrupts(priv); 1536 } 1537 1538 int enetc_close(struct net_device *ndev) 1539 { 1540 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1541 1542 enetc_stop(ndev); 1543 enetc_clear_bdrs(priv); 1544 1545 if (priv->phylink) 1546 phylink_disconnect_phy(priv->phylink); 1547 enetc_free_rxtx_rings(priv); 1548 enetc_free_rx_resources(priv); 1549 enetc_free_tx_resources(priv); 1550 enetc_free_irqs(priv); 1551 1552 return 0; 1553 } 1554 1555 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1556 { 1557 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1558 struct tc_mqprio_qopt *mqprio = type_data; 1559 struct enetc_bdr *tx_ring; 1560 u8 num_tc; 1561 int i; 1562 1563 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1564 num_tc = mqprio->num_tc; 1565 1566 if (!num_tc) { 1567 netdev_reset_tc(ndev); 1568 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1569 1570 /* Reset all ring priorities to 0 */ 1571 for (i = 0; i < priv->num_tx_rings; i++) { 1572 tx_ring = priv->tx_ring[i]; 1573 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1574 } 1575 1576 return 0; 1577 } 1578 1579 /* Check if we have enough BD rings available to accommodate all TCs */ 1580 if (num_tc > priv->num_tx_rings) { 1581 netdev_err(ndev, "Max %d traffic classes supported\n", 1582 priv->num_tx_rings); 1583 return -EINVAL; 1584 } 1585 1586 /* For the moment, we use only one BD ring per TC. 1587 * 1588 * Configure num_tc BD rings with increasing priorities. 1589 */ 1590 for (i = 0; i < num_tc; i++) { 1591 tx_ring = priv->tx_ring[i]; 1592 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1593 } 1594 1595 /* Reset the number of netdev queues based on the TC count */ 1596 netif_set_real_num_tx_queues(ndev, num_tc); 1597 1598 netdev_set_num_tc(ndev, num_tc); 1599 1600 /* Each TC is associated with one netdev queue */ 1601 for (i = 0; i < num_tc; i++) 1602 netdev_set_tc_queue(ndev, i, 1, i); 1603 1604 return 0; 1605 } 1606 1607 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1608 void *type_data) 1609 { 1610 switch (type) { 1611 case TC_SETUP_QDISC_MQPRIO: 1612 return enetc_setup_tc_mqprio(ndev, type_data); 1613 case TC_SETUP_QDISC_TAPRIO: 1614 return enetc_setup_tc_taprio(ndev, type_data); 1615 case TC_SETUP_QDISC_CBS: 1616 return enetc_setup_tc_cbs(ndev, type_data); 1617 case TC_SETUP_QDISC_ETF: 1618 return enetc_setup_tc_txtime(ndev, type_data); 1619 case TC_SETUP_BLOCK: 1620 return enetc_setup_tc_psfp(ndev, type_data); 1621 default: 1622 return -EOPNOTSUPP; 1623 } 1624 } 1625 1626 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1627 { 1628 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1629 struct net_device_stats *stats = &ndev->stats; 1630 unsigned long packets = 0, bytes = 0; 1631 int i; 1632 1633 for (i = 0; i < priv->num_rx_rings; i++) { 1634 packets += priv->rx_ring[i]->stats.packets; 1635 bytes += priv->rx_ring[i]->stats.bytes; 1636 } 1637 1638 stats->rx_packets = packets; 1639 stats->rx_bytes = bytes; 1640 bytes = 0; 1641 packets = 0; 1642 1643 for (i = 0; i < priv->num_tx_rings; i++) { 1644 packets += priv->tx_ring[i]->stats.packets; 1645 bytes += priv->tx_ring[i]->stats.bytes; 1646 } 1647 1648 stats->tx_packets = packets; 1649 stats->tx_bytes = bytes; 1650 1651 return stats; 1652 } 1653 1654 static int enetc_set_rss(struct net_device *ndev, int en) 1655 { 1656 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1657 struct enetc_hw *hw = &priv->si->hw; 1658 u32 reg; 1659 1660 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1661 1662 reg = enetc_rd(hw, ENETC_SIMR); 1663 reg &= ~ENETC_SIMR_RSSE; 1664 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1665 enetc_wr(hw, ENETC_SIMR, reg); 1666 1667 return 0; 1668 } 1669 1670 static int enetc_set_psfp(struct net_device *ndev, int en) 1671 { 1672 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1673 int err; 1674 1675 if (en) { 1676 err = enetc_psfp_enable(priv); 1677 if (err) 1678 return err; 1679 1680 priv->active_offloads |= ENETC_F_QCI; 1681 return 0; 1682 } 1683 1684 err = enetc_psfp_disable(priv); 1685 if (err) 1686 return err; 1687 1688 priv->active_offloads &= ~ENETC_F_QCI; 1689 1690 return 0; 1691 } 1692 1693 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 1694 { 1695 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1696 int i; 1697 1698 for (i = 0; i < priv->num_rx_rings; i++) 1699 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en); 1700 } 1701 1702 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 1703 { 1704 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1705 int i; 1706 1707 for (i = 0; i < priv->num_tx_rings; i++) 1708 enetc_bdr_enable_txvlan(&priv->si->hw, i, en); 1709 } 1710 1711 int enetc_set_features(struct net_device *ndev, 1712 netdev_features_t features) 1713 { 1714 netdev_features_t changed = ndev->features ^ features; 1715 int err = 0; 1716 1717 if (changed & NETIF_F_RXHASH) 1718 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1719 1720 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 1721 enetc_enable_rxvlan(ndev, 1722 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 1723 1724 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 1725 enetc_enable_txvlan(ndev, 1726 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 1727 1728 if (changed & NETIF_F_HW_TC) 1729 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); 1730 1731 return err; 1732 } 1733 1734 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1735 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1736 { 1737 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1738 struct hwtstamp_config config; 1739 int ao; 1740 1741 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1742 return -EFAULT; 1743 1744 switch (config.tx_type) { 1745 case HWTSTAMP_TX_OFF: 1746 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1747 break; 1748 case HWTSTAMP_TX_ON: 1749 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1750 break; 1751 default: 1752 return -ERANGE; 1753 } 1754 1755 ao = priv->active_offloads; 1756 switch (config.rx_filter) { 1757 case HWTSTAMP_FILTER_NONE: 1758 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1759 break; 1760 default: 1761 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1762 config.rx_filter = HWTSTAMP_FILTER_ALL; 1763 } 1764 1765 if (netif_running(ndev) && ao != priv->active_offloads) { 1766 enetc_close(ndev); 1767 enetc_open(ndev); 1768 } 1769 1770 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1771 -EFAULT : 0; 1772 } 1773 1774 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1775 { 1776 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1777 struct hwtstamp_config config; 1778 1779 config.flags = 0; 1780 1781 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1782 config.tx_type = HWTSTAMP_TX_ON; 1783 else 1784 config.tx_type = HWTSTAMP_TX_OFF; 1785 1786 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1787 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1788 1789 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1790 -EFAULT : 0; 1791 } 1792 #endif 1793 1794 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1795 { 1796 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1797 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1798 if (cmd == SIOCSHWTSTAMP) 1799 return enetc_hwtstamp_set(ndev, rq); 1800 if (cmd == SIOCGHWTSTAMP) 1801 return enetc_hwtstamp_get(ndev, rq); 1802 #endif 1803 1804 if (!priv->phylink) 1805 return -EOPNOTSUPP; 1806 1807 return phylink_mii_ioctl(priv->phylink, rq, cmd); 1808 } 1809 1810 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1811 { 1812 struct pci_dev *pdev = priv->si->pdev; 1813 int v_tx_rings; 1814 int i, n, err, nvec; 1815 1816 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1817 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1818 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1819 1820 if (n < 0) 1821 return n; 1822 1823 if (n != nvec) 1824 return -EPERM; 1825 1826 /* # of tx rings per int vector */ 1827 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1828 1829 for (i = 0; i < priv->bdr_int_num; i++) { 1830 struct enetc_int_vector *v; 1831 struct enetc_bdr *bdr; 1832 int j; 1833 1834 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 1835 if (!v) { 1836 err = -ENOMEM; 1837 goto fail; 1838 } 1839 1840 priv->int_vector[i] = v; 1841 1842 /* init defaults for adaptive IC */ 1843 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 1844 v->rx_ictt = 0x1; 1845 v->rx_dim_en = true; 1846 } 1847 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 1848 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1849 NAPI_POLL_WEIGHT); 1850 v->count_tx_rings = v_tx_rings; 1851 1852 for (j = 0; j < v_tx_rings; j++) { 1853 int idx; 1854 1855 /* default tx ring mapping policy */ 1856 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1857 idx = 2 * j + i; /* 2 CPUs */ 1858 else 1859 idx = j + i * v_tx_rings; /* default */ 1860 1861 __set_bit(idx, &v->tx_rings_map); 1862 bdr = &v->tx_ring[j]; 1863 bdr->index = idx; 1864 bdr->ndev = priv->ndev; 1865 bdr->dev = priv->dev; 1866 bdr->bd_count = priv->tx_bd_count; 1867 priv->tx_ring[idx] = bdr; 1868 } 1869 1870 bdr = &v->rx_ring; 1871 bdr->index = i; 1872 bdr->ndev = priv->ndev; 1873 bdr->dev = priv->dev; 1874 bdr->bd_count = priv->rx_bd_count; 1875 priv->rx_ring[i] = bdr; 1876 } 1877 1878 return 0; 1879 1880 fail: 1881 while (i--) { 1882 netif_napi_del(&priv->int_vector[i]->napi); 1883 cancel_work_sync(&priv->int_vector[i]->rx_dim.work); 1884 kfree(priv->int_vector[i]); 1885 } 1886 1887 pci_free_irq_vectors(pdev); 1888 1889 return err; 1890 } 1891 1892 void enetc_free_msix(struct enetc_ndev_priv *priv) 1893 { 1894 int i; 1895 1896 for (i = 0; i < priv->bdr_int_num; i++) { 1897 struct enetc_int_vector *v = priv->int_vector[i]; 1898 1899 netif_napi_del(&v->napi); 1900 cancel_work_sync(&v->rx_dim.work); 1901 } 1902 1903 for (i = 0; i < priv->num_rx_rings; i++) 1904 priv->rx_ring[i] = NULL; 1905 1906 for (i = 0; i < priv->num_tx_rings; i++) 1907 priv->tx_ring[i] = NULL; 1908 1909 for (i = 0; i < priv->bdr_int_num; i++) { 1910 kfree(priv->int_vector[i]); 1911 priv->int_vector[i] = NULL; 1912 } 1913 1914 /* disable all MSIX for this device */ 1915 pci_free_irq_vectors(priv->si->pdev); 1916 } 1917 1918 static void enetc_kfree_si(struct enetc_si *si) 1919 { 1920 char *p = (char *)si - si->pad; 1921 1922 kfree(p); 1923 } 1924 1925 static void enetc_detect_errata(struct enetc_si *si) 1926 { 1927 if (si->pdev->revision == ENETC_REV1) 1928 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL | 1929 ENETC_ERR_UCMCSWP; 1930 } 1931 1932 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1933 { 1934 struct enetc_si *si, *p; 1935 struct enetc_hw *hw; 1936 size_t alloc_size; 1937 int err, len; 1938 1939 pcie_flr(pdev); 1940 err = pci_enable_device_mem(pdev); 1941 if (err) { 1942 dev_err(&pdev->dev, "device enable failed\n"); 1943 return err; 1944 } 1945 1946 /* set up for high or low dma */ 1947 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1948 if (err) { 1949 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1950 if (err) { 1951 dev_err(&pdev->dev, 1952 "DMA configuration failed: 0x%x\n", err); 1953 goto err_dma; 1954 } 1955 } 1956 1957 err = pci_request_mem_regions(pdev, name); 1958 if (err) { 1959 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1960 goto err_pci_mem_reg; 1961 } 1962 1963 pci_set_master(pdev); 1964 1965 alloc_size = sizeof(struct enetc_si); 1966 if (sizeof_priv) { 1967 /* align priv to 32B */ 1968 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1969 alloc_size += sizeof_priv; 1970 } 1971 /* force 32B alignment for enetc_si */ 1972 alloc_size += ENETC_SI_ALIGN - 1; 1973 1974 p = kzalloc(alloc_size, GFP_KERNEL); 1975 if (!p) { 1976 err = -ENOMEM; 1977 goto err_alloc_si; 1978 } 1979 1980 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1981 si->pad = (char *)si - (char *)p; 1982 1983 pci_set_drvdata(pdev, si); 1984 si->pdev = pdev; 1985 hw = &si->hw; 1986 1987 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1988 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1989 if (!hw->reg) { 1990 err = -ENXIO; 1991 dev_err(&pdev->dev, "ioremap() failed\n"); 1992 goto err_ioremap; 1993 } 1994 if (len > ENETC_PORT_BASE) 1995 hw->port = hw->reg + ENETC_PORT_BASE; 1996 if (len > ENETC_GLOBAL_BASE) 1997 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1998 1999 enetc_detect_errata(si); 2000 2001 return 0; 2002 2003 err_ioremap: 2004 enetc_kfree_si(si); 2005 err_alloc_si: 2006 pci_release_mem_regions(pdev); 2007 err_pci_mem_reg: 2008 err_dma: 2009 pci_disable_device(pdev); 2010 2011 return err; 2012 } 2013 2014 void enetc_pci_remove(struct pci_dev *pdev) 2015 { 2016 struct enetc_si *si = pci_get_drvdata(pdev); 2017 struct enetc_hw *hw = &si->hw; 2018 2019 iounmap(hw->reg); 2020 enetc_kfree_si(si); 2021 pci_release_mem_regions(pdev); 2022 pci_disable_device(pdev); 2023 } 2024