1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/vmalloc.h> 8 9 /* ENETC overhead: optional extension BD + 1 BD gap */ 10 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 11 /* max # of chained Tx BDs is 15, including head and extension BD */ 12 #define ENETC_MAX_SKB_FRAGS 13 13 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 14 15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 16 int active_offloads); 17 18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 19 { 20 struct enetc_ndev_priv *priv = netdev_priv(ndev); 21 struct enetc_bdr *tx_ring; 22 int count; 23 24 tx_ring = priv->tx_ring[skb->queue_mapping]; 25 26 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 27 if (unlikely(skb_linearize(skb))) 28 goto drop_packet_err; 29 30 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 31 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 32 netif_stop_subqueue(ndev, tx_ring->index); 33 return NETDEV_TX_BUSY; 34 } 35 36 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 37 if (unlikely(!count)) 38 goto drop_packet_err; 39 40 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 41 netif_stop_subqueue(ndev, tx_ring->index); 42 43 return NETDEV_TX_OK; 44 45 drop_packet_err: 46 dev_kfree_skb_any(skb); 47 return NETDEV_TX_OK; 48 } 49 50 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd) 51 { 52 int l3_start, l3_hsize; 53 u16 l3_flags, l4_flags; 54 55 if (skb->ip_summed != CHECKSUM_PARTIAL) 56 return false; 57 58 switch (skb->csum_offset) { 59 case offsetof(struct tcphdr, check): 60 l4_flags = ENETC_TXBD_L4_TCP; 61 break; 62 case offsetof(struct udphdr, check): 63 l4_flags = ENETC_TXBD_L4_UDP; 64 break; 65 default: 66 skb_checksum_help(skb); 67 return false; 68 } 69 70 l3_start = skb_network_offset(skb); 71 l3_hsize = skb_network_header_len(skb); 72 73 l3_flags = 0; 74 if (skb->protocol == htons(ETH_P_IPV6)) 75 l3_flags = ENETC_TXBD_L3_IPV6; 76 77 /* write BD fields */ 78 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags); 79 txbd->l4_csoff = l4_flags; 80 81 return true; 82 } 83 84 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 85 struct enetc_tx_swbd *tx_swbd) 86 { 87 if (tx_swbd->is_dma_page) 88 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 89 tx_swbd->len, DMA_TO_DEVICE); 90 else 91 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 92 tx_swbd->len, DMA_TO_DEVICE); 93 tx_swbd->dma = 0; 94 } 95 96 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 97 struct enetc_tx_swbd *tx_swbd) 98 { 99 if (tx_swbd->dma) 100 enetc_unmap_tx_buff(tx_ring, tx_swbd); 101 102 if (tx_swbd->skb) { 103 dev_kfree_skb_any(tx_swbd->skb); 104 tx_swbd->skb = NULL; 105 } 106 } 107 108 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 109 int active_offloads) 110 { 111 struct enetc_tx_swbd *tx_swbd; 112 skb_frag_t *frag; 113 int len = skb_headlen(skb); 114 union enetc_tx_bd temp_bd; 115 union enetc_tx_bd *txbd; 116 bool do_vlan, do_tstamp; 117 int i, count = 0; 118 unsigned int f; 119 dma_addr_t dma; 120 u8 flags = 0; 121 122 i = tx_ring->next_to_use; 123 txbd = ENETC_TXBD(*tx_ring, i); 124 prefetchw(txbd); 125 126 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 127 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 128 goto dma_err; 129 130 temp_bd.addr = cpu_to_le64(dma); 131 temp_bd.buf_len = cpu_to_le16(len); 132 temp_bd.lstatus = 0; 133 134 tx_swbd = &tx_ring->tx_swbd[i]; 135 tx_swbd->dma = dma; 136 tx_swbd->len = len; 137 tx_swbd->is_dma_page = 0; 138 count++; 139 140 do_vlan = skb_vlan_tag_present(skb); 141 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 142 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 143 tx_swbd->do_tstamp = do_tstamp; 144 tx_swbd->check_wb = tx_swbd->do_tstamp; 145 146 if (do_vlan || do_tstamp) 147 flags |= ENETC_TXBD_FLAGS_EX; 148 149 if (enetc_tx_csum(skb, &temp_bd)) 150 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS; 151 else if (tx_ring->tsd_enable) 152 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 153 154 /* first BD needs frm_len and offload flags set */ 155 temp_bd.frm_len = cpu_to_le16(skb->len); 156 temp_bd.flags = flags; 157 158 if (flags & ENETC_TXBD_FLAGS_TSE) { 159 u32 temp; 160 161 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK) 162 | (flags << ENETC_TXBD_FLAGS_OFFSET); 163 temp_bd.txstart = cpu_to_le32(temp); 164 } 165 166 if (flags & ENETC_TXBD_FLAGS_EX) { 167 u8 e_flags = 0; 168 *txbd = temp_bd; 169 enetc_clear_tx_bd(&temp_bd); 170 171 /* add extension BD for VLAN and/or timestamping */ 172 flags = 0; 173 tx_swbd++; 174 txbd++; 175 i++; 176 if (unlikely(i == tx_ring->bd_count)) { 177 i = 0; 178 tx_swbd = tx_ring->tx_swbd; 179 txbd = ENETC_TXBD(*tx_ring, 0); 180 } 181 prefetchw(txbd); 182 183 if (do_vlan) { 184 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 185 temp_bd.ext.tpid = 0; /* < C-TAG */ 186 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 187 } 188 189 if (do_tstamp) { 190 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 191 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 192 } 193 194 temp_bd.ext.e_flags = e_flags; 195 count++; 196 } 197 198 frag = &skb_shinfo(skb)->frags[0]; 199 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 200 len = skb_frag_size(frag); 201 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 202 DMA_TO_DEVICE); 203 if (dma_mapping_error(tx_ring->dev, dma)) 204 goto dma_err; 205 206 *txbd = temp_bd; 207 enetc_clear_tx_bd(&temp_bd); 208 209 flags = 0; 210 tx_swbd++; 211 txbd++; 212 i++; 213 if (unlikely(i == tx_ring->bd_count)) { 214 i = 0; 215 tx_swbd = tx_ring->tx_swbd; 216 txbd = ENETC_TXBD(*tx_ring, 0); 217 } 218 prefetchw(txbd); 219 220 temp_bd.addr = cpu_to_le64(dma); 221 temp_bd.buf_len = cpu_to_le16(len); 222 223 tx_swbd->dma = dma; 224 tx_swbd->len = len; 225 tx_swbd->is_dma_page = 1; 226 count++; 227 } 228 229 /* last BD needs 'F' bit set */ 230 flags |= ENETC_TXBD_FLAGS_F; 231 temp_bd.flags = flags; 232 *txbd = temp_bd; 233 234 tx_ring->tx_swbd[i].skb = skb; 235 236 enetc_bdr_idx_inc(tx_ring, &i); 237 tx_ring->next_to_use = i; 238 239 skb_tx_timestamp(skb); 240 241 /* let H/W know BD ring has been updated */ 242 enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */ 243 244 return count; 245 246 dma_err: 247 dev_err(tx_ring->dev, "DMA map error"); 248 249 do { 250 tx_swbd = &tx_ring->tx_swbd[i]; 251 enetc_free_tx_skb(tx_ring, tx_swbd); 252 if (i == 0) 253 i = tx_ring->bd_count; 254 i--; 255 } while (count--); 256 257 return 0; 258 } 259 260 static irqreturn_t enetc_msix(int irq, void *data) 261 { 262 struct enetc_int_vector *v = data; 263 int i; 264 265 /* disable interrupts */ 266 enetc_wr_reg(v->rbier, 0); 267 enetc_wr_reg(v->ricr1, v->rx_ictt); 268 269 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 270 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0); 271 272 napi_schedule(&v->napi); 273 274 return IRQ_HANDLED; 275 } 276 277 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 278 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 279 struct napi_struct *napi, int work_limit); 280 281 static void enetc_rx_dim_work(struct work_struct *w) 282 { 283 struct dim *dim = container_of(w, struct dim, work); 284 struct dim_cq_moder moder = 285 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 286 struct enetc_int_vector *v = 287 container_of(dim, struct enetc_int_vector, rx_dim); 288 289 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 290 dim->state = DIM_START_MEASURE; 291 } 292 293 static void enetc_rx_net_dim(struct enetc_int_vector *v) 294 { 295 struct dim_sample dim_sample; 296 297 v->comp_cnt++; 298 299 if (!v->rx_napi_work) 300 return; 301 302 dim_update_sample(v->comp_cnt, 303 v->rx_ring.stats.packets, 304 v->rx_ring.stats.bytes, 305 &dim_sample); 306 net_dim(&v->rx_dim, dim_sample); 307 } 308 309 static int enetc_poll(struct napi_struct *napi, int budget) 310 { 311 struct enetc_int_vector 312 *v = container_of(napi, struct enetc_int_vector, napi); 313 bool complete = true; 314 int work_done; 315 int i; 316 317 for (i = 0; i < v->count_tx_rings; i++) 318 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 319 complete = false; 320 321 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 322 if (work_done == budget) 323 complete = false; 324 if (work_done) 325 v->rx_napi_work = true; 326 327 if (!complete) 328 return budget; 329 330 napi_complete_done(napi, work_done); 331 332 if (likely(v->rx_dim_en)) 333 enetc_rx_net_dim(v); 334 335 v->rx_napi_work = false; 336 337 /* enable interrupts */ 338 enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE); 339 340 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 341 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 342 ENETC_TBIER_TXTIE); 343 344 return work_done; 345 } 346 347 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 348 { 349 int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 350 351 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 352 } 353 354 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 355 u64 *tstamp) 356 { 357 u32 lo, hi, tstamp_lo; 358 359 lo = enetc_rd(hw, ENETC_SICTR0); 360 hi = enetc_rd(hw, ENETC_SICTR1); 361 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 362 if (lo <= tstamp_lo) 363 hi -= 1; 364 *tstamp = (u64)hi << 32 | tstamp_lo; 365 } 366 367 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 368 { 369 struct skb_shared_hwtstamps shhwtstamps; 370 371 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 372 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 373 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 374 skb_tstamp_tx(skb, &shhwtstamps); 375 } 376 } 377 378 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 379 { 380 struct net_device *ndev = tx_ring->ndev; 381 int tx_frm_cnt = 0, tx_byte_cnt = 0; 382 struct enetc_tx_swbd *tx_swbd; 383 int i, bds_to_clean; 384 bool do_tstamp; 385 u64 tstamp = 0; 386 387 i = tx_ring->next_to_clean; 388 tx_swbd = &tx_ring->tx_swbd[i]; 389 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 390 391 do_tstamp = false; 392 393 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 394 bool is_eof = !!tx_swbd->skb; 395 396 if (unlikely(tx_swbd->check_wb)) { 397 struct enetc_ndev_priv *priv = netdev_priv(ndev); 398 union enetc_tx_bd *txbd; 399 400 txbd = ENETC_TXBD(*tx_ring, i); 401 402 if (txbd->flags & ENETC_TXBD_FLAGS_W && 403 tx_swbd->do_tstamp) { 404 enetc_get_tx_tstamp(&priv->si->hw, txbd, 405 &tstamp); 406 do_tstamp = true; 407 } 408 } 409 410 if (likely(tx_swbd->dma)) 411 enetc_unmap_tx_buff(tx_ring, tx_swbd); 412 413 if (is_eof) { 414 if (unlikely(do_tstamp)) { 415 enetc_tstamp_tx(tx_swbd->skb, tstamp); 416 do_tstamp = false; 417 } 418 napi_consume_skb(tx_swbd->skb, napi_budget); 419 tx_swbd->skb = NULL; 420 } 421 422 tx_byte_cnt += tx_swbd->len; 423 424 bds_to_clean--; 425 tx_swbd++; 426 i++; 427 if (unlikely(i == tx_ring->bd_count)) { 428 i = 0; 429 tx_swbd = tx_ring->tx_swbd; 430 } 431 432 /* BD iteration loop end */ 433 if (is_eof) { 434 tx_frm_cnt++; 435 /* re-arm interrupt source */ 436 enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) | 437 BIT(16 + tx_ring->index)); 438 } 439 440 if (unlikely(!bds_to_clean)) 441 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 442 } 443 444 tx_ring->next_to_clean = i; 445 tx_ring->stats.packets += tx_frm_cnt; 446 tx_ring->stats.bytes += tx_byte_cnt; 447 448 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 449 __netif_subqueue_stopped(ndev, tx_ring->index) && 450 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 451 netif_wake_subqueue(ndev, tx_ring->index); 452 } 453 454 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 455 } 456 457 static bool enetc_new_page(struct enetc_bdr *rx_ring, 458 struct enetc_rx_swbd *rx_swbd) 459 { 460 struct page *page; 461 dma_addr_t addr; 462 463 page = dev_alloc_page(); 464 if (unlikely(!page)) 465 return false; 466 467 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 468 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 469 __free_page(page); 470 471 return false; 472 } 473 474 rx_swbd->dma = addr; 475 rx_swbd->page = page; 476 rx_swbd->page_offset = ENETC_RXB_PAD; 477 478 return true; 479 } 480 481 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 482 { 483 struct enetc_rx_swbd *rx_swbd; 484 union enetc_rx_bd *rxbd; 485 int i, j; 486 487 i = rx_ring->next_to_use; 488 rx_swbd = &rx_ring->rx_swbd[i]; 489 rxbd = enetc_rxbd(rx_ring, i); 490 491 for (j = 0; j < buff_cnt; j++) { 492 /* try reuse page */ 493 if (unlikely(!rx_swbd->page)) { 494 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 495 rx_ring->stats.rx_alloc_errs++; 496 break; 497 } 498 } 499 500 /* update RxBD */ 501 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 502 rx_swbd->page_offset); 503 /* clear 'R" as well */ 504 rxbd->r.lstatus = 0; 505 506 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 507 rx_swbd++; 508 i++; 509 if (unlikely(i == rx_ring->bd_count)) { 510 i = 0; 511 rx_swbd = rx_ring->rx_swbd; 512 } 513 } 514 515 if (likely(j)) { 516 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 517 rx_ring->next_to_use = i; 518 /* update ENETC's consumer index */ 519 enetc_wr_reg(rx_ring->rcir, i); 520 } 521 522 return j; 523 } 524 525 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 526 static void enetc_get_rx_tstamp(struct net_device *ndev, 527 union enetc_rx_bd *rxbd, 528 struct sk_buff *skb) 529 { 530 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 531 struct enetc_ndev_priv *priv = netdev_priv(ndev); 532 struct enetc_hw *hw = &priv->si->hw; 533 u32 lo, hi, tstamp_lo; 534 u64 tstamp; 535 536 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 537 lo = enetc_rd(hw, ENETC_SICTR0); 538 hi = enetc_rd(hw, ENETC_SICTR1); 539 rxbd = enetc_rxbd_ext(rxbd); 540 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 541 if (lo <= tstamp_lo) 542 hi -= 1; 543 544 tstamp = (u64)hi << 32 | tstamp_lo; 545 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 546 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 547 } 548 } 549 #endif 550 551 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 552 union enetc_rx_bd *rxbd, struct sk_buff *skb) 553 { 554 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 555 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 556 #endif 557 /* TODO: hashing */ 558 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 559 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 560 561 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 562 skb->ip_summed = CHECKSUM_COMPLETE; 563 } 564 565 /* copy VLAN to skb, if one is extracted, for now we assume it's a 566 * standard TPID, but HW also supports custom values 567 */ 568 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) 569 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 570 le16_to_cpu(rxbd->r.vlan_opt)); 571 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 572 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 573 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 574 #endif 575 } 576 577 static void enetc_process_skb(struct enetc_bdr *rx_ring, 578 struct sk_buff *skb) 579 { 580 skb_record_rx_queue(skb, rx_ring->index); 581 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 582 } 583 584 static bool enetc_page_reusable(struct page *page) 585 { 586 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 587 } 588 589 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 590 struct enetc_rx_swbd *old) 591 { 592 struct enetc_rx_swbd *new; 593 594 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 595 596 /* next buf that may reuse a page */ 597 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 598 599 /* copy page reference */ 600 *new = *old; 601 } 602 603 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 604 int i, u16 size) 605 { 606 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 607 608 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 609 rx_swbd->page_offset, 610 size, DMA_FROM_DEVICE); 611 return rx_swbd; 612 } 613 614 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 615 struct enetc_rx_swbd *rx_swbd) 616 { 617 if (likely(enetc_page_reusable(rx_swbd->page))) { 618 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 619 page_ref_inc(rx_swbd->page); 620 621 enetc_reuse_page(rx_ring, rx_swbd); 622 623 /* sync for use by the device */ 624 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 625 rx_swbd->page_offset, 626 ENETC_RXB_DMA_SIZE, 627 DMA_FROM_DEVICE); 628 } else { 629 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 630 PAGE_SIZE, DMA_FROM_DEVICE); 631 } 632 633 rx_swbd->page = NULL; 634 } 635 636 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 637 int i, u16 size) 638 { 639 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 640 struct sk_buff *skb; 641 void *ba; 642 643 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 644 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 645 if (unlikely(!skb)) { 646 rx_ring->stats.rx_alloc_errs++; 647 return NULL; 648 } 649 650 skb_reserve(skb, ENETC_RXB_PAD); 651 __skb_put(skb, size); 652 653 enetc_put_rx_buff(rx_ring, rx_swbd); 654 655 return skb; 656 } 657 658 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 659 u16 size, struct sk_buff *skb) 660 { 661 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 662 663 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 664 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 665 666 enetc_put_rx_buff(rx_ring, rx_swbd); 667 } 668 669 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 670 671 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 672 struct napi_struct *napi, int work_limit) 673 { 674 int rx_frm_cnt = 0, rx_byte_cnt = 0; 675 int cleaned_cnt, i; 676 677 cleaned_cnt = enetc_bd_unused(rx_ring); 678 /* next descriptor to process */ 679 i = rx_ring->next_to_clean; 680 681 while (likely(rx_frm_cnt < work_limit)) { 682 union enetc_rx_bd *rxbd; 683 struct sk_buff *skb; 684 u32 bd_status; 685 u16 size; 686 687 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 688 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 689 690 cleaned_cnt -= count; 691 } 692 693 rxbd = enetc_rxbd(rx_ring, i); 694 bd_status = le32_to_cpu(rxbd->r.lstatus); 695 if (!bd_status) 696 break; 697 698 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index)); 699 dma_rmb(); /* for reading other rxbd fields */ 700 size = le16_to_cpu(rxbd->r.buf_len); 701 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 702 if (!skb) 703 break; 704 705 enetc_get_offloads(rx_ring, rxbd, skb); 706 707 cleaned_cnt++; 708 709 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 710 if (unlikely(++i == rx_ring->bd_count)) 711 i = 0; 712 713 if (unlikely(bd_status & 714 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 715 dev_kfree_skb(skb); 716 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 717 dma_rmb(); 718 bd_status = le32_to_cpu(rxbd->r.lstatus); 719 720 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 721 if (unlikely(++i == rx_ring->bd_count)) 722 i = 0; 723 } 724 725 rx_ring->ndev->stats.rx_dropped++; 726 rx_ring->ndev->stats.rx_errors++; 727 728 break; 729 } 730 731 /* not last BD in frame? */ 732 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 733 bd_status = le32_to_cpu(rxbd->r.lstatus); 734 size = ENETC_RXB_DMA_SIZE; 735 736 if (bd_status & ENETC_RXBD_LSTATUS_F) { 737 dma_rmb(); 738 size = le16_to_cpu(rxbd->r.buf_len); 739 } 740 741 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 742 743 cleaned_cnt++; 744 745 rxbd = enetc_rxbd_next(rx_ring, rxbd, i); 746 if (unlikely(++i == rx_ring->bd_count)) 747 i = 0; 748 } 749 750 rx_byte_cnt += skb->len; 751 752 enetc_process_skb(rx_ring, skb); 753 754 napi_gro_receive(napi, skb); 755 756 rx_frm_cnt++; 757 } 758 759 rx_ring->next_to_clean = i; 760 761 rx_ring->stats.packets += rx_frm_cnt; 762 rx_ring->stats.bytes += rx_byte_cnt; 763 764 return rx_frm_cnt; 765 } 766 767 /* Probing and Init */ 768 #define ENETC_MAX_RFS_SIZE 64 769 void enetc_get_si_caps(struct enetc_si *si) 770 { 771 struct enetc_hw *hw = &si->hw; 772 u32 val; 773 774 /* find out how many of various resources we have to work with */ 775 val = enetc_rd(hw, ENETC_SICAPR0); 776 si->num_rx_rings = (val >> 16) & 0xff; 777 si->num_tx_rings = val & 0xff; 778 779 val = enetc_rd(hw, ENETC_SIRFSCAPR); 780 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 781 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 782 783 si->num_rss = 0; 784 val = enetc_rd(hw, ENETC_SIPCAPR0); 785 if (val & ENETC_SIPCAPR0_RSS) { 786 u32 rss; 787 788 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 789 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 790 } 791 792 if (val & ENETC_SIPCAPR0_QBV) 793 si->hw_features |= ENETC_SI_F_QBV; 794 795 if (val & ENETC_SIPCAPR0_PSFP) 796 si->hw_features |= ENETC_SI_F_PSFP; 797 } 798 799 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 800 { 801 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 802 &r->bd_dma_base, GFP_KERNEL); 803 if (!r->bd_base) 804 return -ENOMEM; 805 806 /* h/w requires 128B alignment */ 807 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 808 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 809 r->bd_dma_base); 810 return -EINVAL; 811 } 812 813 return 0; 814 } 815 816 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 817 { 818 int err; 819 820 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 821 if (!txr->tx_swbd) 822 return -ENOMEM; 823 824 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 825 if (err) { 826 vfree(txr->tx_swbd); 827 return err; 828 } 829 830 txr->next_to_clean = 0; 831 txr->next_to_use = 0; 832 833 return 0; 834 } 835 836 static void enetc_free_txbdr(struct enetc_bdr *txr) 837 { 838 int size, i; 839 840 for (i = 0; i < txr->bd_count; i++) 841 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 842 843 size = txr->bd_count * sizeof(union enetc_tx_bd); 844 845 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 846 txr->bd_base = NULL; 847 848 vfree(txr->tx_swbd); 849 txr->tx_swbd = NULL; 850 } 851 852 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 853 { 854 int i, err; 855 856 for (i = 0; i < priv->num_tx_rings; i++) { 857 err = enetc_alloc_txbdr(priv->tx_ring[i]); 858 859 if (err) 860 goto fail; 861 } 862 863 return 0; 864 865 fail: 866 while (i-- > 0) 867 enetc_free_txbdr(priv->tx_ring[i]); 868 869 return err; 870 } 871 872 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 873 { 874 int i; 875 876 for (i = 0; i < priv->num_tx_rings; i++) 877 enetc_free_txbdr(priv->tx_ring[i]); 878 } 879 880 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 881 { 882 size_t size = sizeof(union enetc_rx_bd); 883 int err; 884 885 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 886 if (!rxr->rx_swbd) 887 return -ENOMEM; 888 889 if (extended) 890 size *= 2; 891 892 err = enetc_dma_alloc_bdr(rxr, size); 893 if (err) { 894 vfree(rxr->rx_swbd); 895 return err; 896 } 897 898 rxr->next_to_clean = 0; 899 rxr->next_to_use = 0; 900 rxr->next_to_alloc = 0; 901 rxr->ext_en = extended; 902 903 return 0; 904 } 905 906 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 907 { 908 int size; 909 910 size = rxr->bd_count * sizeof(union enetc_rx_bd); 911 912 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 913 rxr->bd_base = NULL; 914 915 vfree(rxr->rx_swbd); 916 rxr->rx_swbd = NULL; 917 } 918 919 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 920 { 921 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 922 int i, err; 923 924 for (i = 0; i < priv->num_rx_rings; i++) { 925 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 926 927 if (err) 928 goto fail; 929 } 930 931 return 0; 932 933 fail: 934 while (i-- > 0) 935 enetc_free_rxbdr(priv->rx_ring[i]); 936 937 return err; 938 } 939 940 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 941 { 942 int i; 943 944 for (i = 0; i < priv->num_rx_rings; i++) 945 enetc_free_rxbdr(priv->rx_ring[i]); 946 } 947 948 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 949 { 950 int i; 951 952 if (!tx_ring->tx_swbd) 953 return; 954 955 for (i = 0; i < tx_ring->bd_count; i++) { 956 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 957 958 enetc_free_tx_skb(tx_ring, tx_swbd); 959 } 960 961 tx_ring->next_to_clean = 0; 962 tx_ring->next_to_use = 0; 963 } 964 965 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 966 { 967 int i; 968 969 if (!rx_ring->rx_swbd) 970 return; 971 972 for (i = 0; i < rx_ring->bd_count; i++) { 973 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 974 975 if (!rx_swbd->page) 976 continue; 977 978 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 979 PAGE_SIZE, DMA_FROM_DEVICE); 980 __free_page(rx_swbd->page); 981 rx_swbd->page = NULL; 982 } 983 984 rx_ring->next_to_clean = 0; 985 rx_ring->next_to_use = 0; 986 rx_ring->next_to_alloc = 0; 987 } 988 989 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 990 { 991 int i; 992 993 for (i = 0; i < priv->num_rx_rings; i++) 994 enetc_free_rx_ring(priv->rx_ring[i]); 995 996 for (i = 0; i < priv->num_tx_rings; i++) 997 enetc_free_tx_ring(priv->tx_ring[i]); 998 } 999 1000 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1001 { 1002 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1003 1004 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 1005 GFP_KERNEL); 1006 if (!cbdr->bd_base) 1007 return -ENOMEM; 1008 1009 /* h/w requires 128B alignment */ 1010 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 1011 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1012 return -EINVAL; 1013 } 1014 1015 cbdr->next_to_clean = 0; 1016 cbdr->next_to_use = 0; 1017 1018 return 0; 1019 } 1020 1021 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 1022 { 1023 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 1024 1025 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 1026 cbdr->bd_base = NULL; 1027 } 1028 1029 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 1030 { 1031 /* set CBDR cache attributes */ 1032 enetc_wr(hw, ENETC_SICAR2, 1033 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1034 1035 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 1036 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 1037 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 1038 1039 enetc_wr(hw, ENETC_SICBDRPIR, 0); 1040 enetc_wr(hw, ENETC_SICBDRCIR, 0); 1041 1042 /* enable ring */ 1043 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 1044 1045 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 1046 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 1047 } 1048 1049 static void enetc_clear_cbdr(struct enetc_hw *hw) 1050 { 1051 enetc_wr(hw, ENETC_SICBDRMR, 0); 1052 } 1053 1054 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1055 { 1056 int *rss_table; 1057 int i; 1058 1059 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1060 if (!rss_table) 1061 return -ENOMEM; 1062 1063 /* Set up RSS table defaults */ 1064 for (i = 0; i < si->num_rss; i++) 1065 rss_table[i] = i % num_groups; 1066 1067 enetc_set_rss_table(si, rss_table, si->num_rss); 1068 1069 kfree(rss_table); 1070 1071 return 0; 1072 } 1073 1074 static int enetc_configure_si(struct enetc_ndev_priv *priv) 1075 { 1076 struct enetc_si *si = priv->si; 1077 struct enetc_hw *hw = &si->hw; 1078 int err; 1079 1080 enetc_setup_cbdr(hw, &si->cbd_ring); 1081 /* set SI cache attributes */ 1082 enetc_wr(hw, ENETC_SICAR0, 1083 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1084 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1085 /* enable SI */ 1086 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1087 1088 if (si->num_rss) { 1089 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1090 if (err) 1091 return err; 1092 } 1093 1094 return 0; 1095 } 1096 1097 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1098 { 1099 struct enetc_si *si = priv->si; 1100 int cpus = num_online_cpus(); 1101 1102 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 1103 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 1104 1105 /* Enable all available TX rings in order to configure as many 1106 * priorities as possible, when needed. 1107 * TODO: Make # of TX rings run-time configurable 1108 */ 1109 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1110 priv->num_tx_rings = si->num_tx_rings; 1111 priv->bdr_int_num = cpus; 1112 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 1113 priv->tx_ictt = ENETC_TXIC_TIMETHR; 1114 1115 /* SI specific */ 1116 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1117 } 1118 1119 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1120 { 1121 struct enetc_si *si = priv->si; 1122 int err; 1123 1124 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1125 if (err) 1126 return err; 1127 1128 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1129 GFP_KERNEL); 1130 if (!priv->cls_rules) { 1131 err = -ENOMEM; 1132 goto err_alloc_cls; 1133 } 1134 1135 err = enetc_configure_si(priv); 1136 if (err) 1137 goto err_config_si; 1138 1139 return 0; 1140 1141 err_config_si: 1142 kfree(priv->cls_rules); 1143 err_alloc_cls: 1144 enetc_clear_cbdr(&si->hw); 1145 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1146 1147 return err; 1148 } 1149 1150 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1151 { 1152 struct enetc_si *si = priv->si; 1153 1154 enetc_clear_cbdr(&si->hw); 1155 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1156 1157 kfree(priv->cls_rules); 1158 } 1159 1160 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1161 { 1162 int idx = tx_ring->index; 1163 u32 tbmr; 1164 1165 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1166 lower_32_bits(tx_ring->bd_dma_base)); 1167 1168 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1169 upper_32_bits(tx_ring->bd_dma_base)); 1170 1171 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1172 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1173 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1174 1175 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1176 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1177 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1178 1179 /* enable Tx ints by setting pkt thr to 1 */ 1180 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 1181 1182 tbmr = ENETC_TBMR_EN; 1183 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1184 tbmr |= ENETC_TBMR_VIH; 1185 1186 /* enable ring */ 1187 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1188 1189 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1190 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1191 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1192 } 1193 1194 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1195 { 1196 int idx = rx_ring->index; 1197 u32 rbmr; 1198 1199 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1200 lower_32_bits(rx_ring->bd_dma_base)); 1201 1202 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1203 upper_32_bits(rx_ring->bd_dma_base)); 1204 1205 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1206 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1207 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1208 1209 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1210 1211 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1212 1213 /* enable Rx ints by setting pkt thr to 1 */ 1214 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 1215 1216 rbmr = ENETC_RBMR_EN; 1217 1218 if (rx_ring->ext_en) 1219 rbmr |= ENETC_RBMR_BDS; 1220 1221 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1222 rbmr |= ENETC_RBMR_VTE; 1223 1224 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1225 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1226 1227 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1228 1229 /* enable ring */ 1230 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1231 } 1232 1233 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1234 { 1235 int i; 1236 1237 for (i = 0; i < priv->num_tx_rings; i++) 1238 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1239 1240 for (i = 0; i < priv->num_rx_rings; i++) 1241 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1242 } 1243 1244 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1245 { 1246 int idx = rx_ring->index; 1247 1248 /* disable EN bit on ring */ 1249 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1250 } 1251 1252 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1253 { 1254 int delay = 8, timeout = 100; 1255 int idx = tx_ring->index; 1256 1257 /* disable EN bit on ring */ 1258 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1259 1260 /* wait for busy to clear */ 1261 while (delay < timeout && 1262 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1263 msleep(delay); 1264 delay *= 2; 1265 } 1266 1267 if (delay >= timeout) 1268 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1269 idx); 1270 } 1271 1272 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1273 { 1274 int i; 1275 1276 for (i = 0; i < priv->num_tx_rings; i++) 1277 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1278 1279 for (i = 0; i < priv->num_rx_rings; i++) 1280 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1281 1282 udelay(1); 1283 } 1284 1285 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1286 { 1287 struct pci_dev *pdev = priv->si->pdev; 1288 cpumask_t cpu_mask; 1289 int i, j, err; 1290 1291 for (i = 0; i < priv->bdr_int_num; i++) { 1292 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1293 struct enetc_int_vector *v = priv->int_vector[i]; 1294 int entry = ENETC_BDR_INT_BASE_IDX + i; 1295 struct enetc_hw *hw = &priv->si->hw; 1296 1297 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1298 priv->ndev->name, i); 1299 err = request_irq(irq, enetc_msix, 0, v->name, v); 1300 if (err) { 1301 dev_err(priv->dev, "request_irq() failed!\n"); 1302 goto irq_err; 1303 } 1304 disable_irq(irq); 1305 1306 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1307 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1308 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 1309 1310 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1311 1312 for (j = 0; j < v->count_tx_rings; j++) { 1313 int idx = v->tx_ring[j].index; 1314 1315 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1316 } 1317 cpumask_clear(&cpu_mask); 1318 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1319 irq_set_affinity_hint(irq, &cpu_mask); 1320 } 1321 1322 return 0; 1323 1324 irq_err: 1325 while (i--) { 1326 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1327 1328 irq_set_affinity_hint(irq, NULL); 1329 free_irq(irq, priv->int_vector[i]); 1330 } 1331 1332 return err; 1333 } 1334 1335 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1336 { 1337 struct pci_dev *pdev = priv->si->pdev; 1338 int i; 1339 1340 for (i = 0; i < priv->bdr_int_num; i++) { 1341 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1342 1343 irq_set_affinity_hint(irq, NULL); 1344 free_irq(irq, priv->int_vector[i]); 1345 } 1346 } 1347 1348 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 1349 { 1350 struct enetc_hw *hw = &priv->si->hw; 1351 u32 icpt, ictt; 1352 int i; 1353 1354 /* enable Tx & Rx event indication */ 1355 if (priv->ic_mode & 1356 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 1357 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 1358 /* init to non-0 minimum, will be adjusted later */ 1359 ictt = 0x1; 1360 } else { 1361 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 1362 ictt = 0; 1363 } 1364 1365 for (i = 0; i < priv->num_rx_rings; i++) { 1366 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 1367 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 1368 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 1369 } 1370 1371 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 1372 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 1373 else 1374 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 1375 1376 for (i = 0; i < priv->num_tx_rings; i++) { 1377 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 1378 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 1379 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 1380 } 1381 } 1382 1383 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 1384 { 1385 int i; 1386 1387 for (i = 0; i < priv->num_tx_rings; i++) 1388 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1389 1390 for (i = 0; i < priv->num_rx_rings; i++) 1391 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1392 } 1393 1394 static int enetc_phylink_connect(struct net_device *ndev) 1395 { 1396 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1397 struct ethtool_eee edata; 1398 int err; 1399 1400 if (!priv->phylink) 1401 return 0; /* phy-less mode */ 1402 1403 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 1404 if (err) { 1405 dev_err(&ndev->dev, "could not attach to PHY\n"); 1406 return err; 1407 } 1408 1409 /* disable EEE autoneg, until ENETC driver supports it */ 1410 memset(&edata, 0, sizeof(struct ethtool_eee)); 1411 phylink_ethtool_set_eee(priv->phylink, &edata); 1412 1413 return 0; 1414 } 1415 1416 void enetc_start(struct net_device *ndev) 1417 { 1418 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1419 int i; 1420 1421 enetc_setup_interrupts(priv); 1422 1423 for (i = 0; i < priv->bdr_int_num; i++) { 1424 int irq = pci_irq_vector(priv->si->pdev, 1425 ENETC_BDR_INT_BASE_IDX + i); 1426 1427 napi_enable(&priv->int_vector[i]->napi); 1428 enable_irq(irq); 1429 } 1430 1431 if (priv->phylink) 1432 phylink_start(priv->phylink); 1433 else 1434 netif_carrier_on(ndev); 1435 1436 netif_tx_start_all_queues(ndev); 1437 } 1438 1439 int enetc_open(struct net_device *ndev) 1440 { 1441 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1442 int err; 1443 1444 err = enetc_setup_irqs(priv); 1445 if (err) 1446 return err; 1447 1448 err = enetc_phylink_connect(ndev); 1449 if (err) 1450 goto err_phy_connect; 1451 1452 err = enetc_alloc_tx_resources(priv); 1453 if (err) 1454 goto err_alloc_tx; 1455 1456 err = enetc_alloc_rx_resources(priv); 1457 if (err) 1458 goto err_alloc_rx; 1459 1460 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1461 if (err) 1462 goto err_set_queues; 1463 1464 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1465 if (err) 1466 goto err_set_queues; 1467 1468 enetc_setup_bdrs(priv); 1469 enetc_start(ndev); 1470 1471 return 0; 1472 1473 err_set_queues: 1474 enetc_free_rx_resources(priv); 1475 err_alloc_rx: 1476 enetc_free_tx_resources(priv); 1477 err_alloc_tx: 1478 if (priv->phylink) 1479 phylink_disconnect_phy(priv->phylink); 1480 err_phy_connect: 1481 enetc_free_irqs(priv); 1482 1483 return err; 1484 } 1485 1486 void enetc_stop(struct net_device *ndev) 1487 { 1488 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1489 int i; 1490 1491 netif_tx_stop_all_queues(ndev); 1492 1493 for (i = 0; i < priv->bdr_int_num; i++) { 1494 int irq = pci_irq_vector(priv->si->pdev, 1495 ENETC_BDR_INT_BASE_IDX + i); 1496 1497 disable_irq(irq); 1498 napi_synchronize(&priv->int_vector[i]->napi); 1499 napi_disable(&priv->int_vector[i]->napi); 1500 } 1501 1502 if (priv->phylink) 1503 phylink_stop(priv->phylink); 1504 else 1505 netif_carrier_off(ndev); 1506 1507 enetc_clear_interrupts(priv); 1508 } 1509 1510 int enetc_close(struct net_device *ndev) 1511 { 1512 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1513 1514 enetc_stop(ndev); 1515 enetc_clear_bdrs(priv); 1516 1517 if (priv->phylink) 1518 phylink_disconnect_phy(priv->phylink); 1519 enetc_free_rxtx_rings(priv); 1520 enetc_free_rx_resources(priv); 1521 enetc_free_tx_resources(priv); 1522 enetc_free_irqs(priv); 1523 1524 return 0; 1525 } 1526 1527 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1528 { 1529 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1530 struct tc_mqprio_qopt *mqprio = type_data; 1531 struct enetc_bdr *tx_ring; 1532 u8 num_tc; 1533 int i; 1534 1535 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1536 num_tc = mqprio->num_tc; 1537 1538 if (!num_tc) { 1539 netdev_reset_tc(ndev); 1540 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1541 1542 /* Reset all ring priorities to 0 */ 1543 for (i = 0; i < priv->num_tx_rings; i++) { 1544 tx_ring = priv->tx_ring[i]; 1545 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1546 } 1547 1548 return 0; 1549 } 1550 1551 /* Check if we have enough BD rings available to accommodate all TCs */ 1552 if (num_tc > priv->num_tx_rings) { 1553 netdev_err(ndev, "Max %d traffic classes supported\n", 1554 priv->num_tx_rings); 1555 return -EINVAL; 1556 } 1557 1558 /* For the moment, we use only one BD ring per TC. 1559 * 1560 * Configure num_tc BD rings with increasing priorities. 1561 */ 1562 for (i = 0; i < num_tc; i++) { 1563 tx_ring = priv->tx_ring[i]; 1564 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1565 } 1566 1567 /* Reset the number of netdev queues based on the TC count */ 1568 netif_set_real_num_tx_queues(ndev, num_tc); 1569 1570 netdev_set_num_tc(ndev, num_tc); 1571 1572 /* Each TC is associated with one netdev queue */ 1573 for (i = 0; i < num_tc; i++) 1574 netdev_set_tc_queue(ndev, i, 1, i); 1575 1576 return 0; 1577 } 1578 1579 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1580 void *type_data) 1581 { 1582 switch (type) { 1583 case TC_SETUP_QDISC_MQPRIO: 1584 return enetc_setup_tc_mqprio(ndev, type_data); 1585 case TC_SETUP_QDISC_TAPRIO: 1586 return enetc_setup_tc_taprio(ndev, type_data); 1587 case TC_SETUP_QDISC_CBS: 1588 return enetc_setup_tc_cbs(ndev, type_data); 1589 case TC_SETUP_QDISC_ETF: 1590 return enetc_setup_tc_txtime(ndev, type_data); 1591 case TC_SETUP_BLOCK: 1592 return enetc_setup_tc_psfp(ndev, type_data); 1593 default: 1594 return -EOPNOTSUPP; 1595 } 1596 } 1597 1598 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1599 { 1600 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1601 struct net_device_stats *stats = &ndev->stats; 1602 unsigned long packets = 0, bytes = 0; 1603 int i; 1604 1605 for (i = 0; i < priv->num_rx_rings; i++) { 1606 packets += priv->rx_ring[i]->stats.packets; 1607 bytes += priv->rx_ring[i]->stats.bytes; 1608 } 1609 1610 stats->rx_packets = packets; 1611 stats->rx_bytes = bytes; 1612 bytes = 0; 1613 packets = 0; 1614 1615 for (i = 0; i < priv->num_tx_rings; i++) { 1616 packets += priv->tx_ring[i]->stats.packets; 1617 bytes += priv->tx_ring[i]->stats.bytes; 1618 } 1619 1620 stats->tx_packets = packets; 1621 stats->tx_bytes = bytes; 1622 1623 return stats; 1624 } 1625 1626 static int enetc_set_rss(struct net_device *ndev, int en) 1627 { 1628 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1629 struct enetc_hw *hw = &priv->si->hw; 1630 u32 reg; 1631 1632 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1633 1634 reg = enetc_rd(hw, ENETC_SIMR); 1635 reg &= ~ENETC_SIMR_RSSE; 1636 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1637 enetc_wr(hw, ENETC_SIMR, reg); 1638 1639 return 0; 1640 } 1641 1642 static int enetc_set_psfp(struct net_device *ndev, int en) 1643 { 1644 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1645 int err; 1646 1647 if (en) { 1648 err = enetc_psfp_enable(priv); 1649 if (err) 1650 return err; 1651 1652 priv->active_offloads |= ENETC_F_QCI; 1653 return 0; 1654 } 1655 1656 err = enetc_psfp_disable(priv); 1657 if (err) 1658 return err; 1659 1660 priv->active_offloads &= ~ENETC_F_QCI; 1661 1662 return 0; 1663 } 1664 1665 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 1666 { 1667 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1668 int i; 1669 1670 for (i = 0; i < priv->num_rx_rings; i++) 1671 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en); 1672 } 1673 1674 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 1675 { 1676 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1677 int i; 1678 1679 for (i = 0; i < priv->num_tx_rings; i++) 1680 enetc_bdr_enable_txvlan(&priv->si->hw, i, en); 1681 } 1682 1683 int enetc_set_features(struct net_device *ndev, 1684 netdev_features_t features) 1685 { 1686 netdev_features_t changed = ndev->features ^ features; 1687 int err = 0; 1688 1689 if (changed & NETIF_F_RXHASH) 1690 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1691 1692 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 1693 enetc_enable_rxvlan(ndev, 1694 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 1695 1696 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 1697 enetc_enable_txvlan(ndev, 1698 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 1699 1700 if (changed & NETIF_F_HW_TC) 1701 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); 1702 1703 return err; 1704 } 1705 1706 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1707 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1708 { 1709 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1710 struct hwtstamp_config config; 1711 int ao; 1712 1713 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1714 return -EFAULT; 1715 1716 switch (config.tx_type) { 1717 case HWTSTAMP_TX_OFF: 1718 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1719 break; 1720 case HWTSTAMP_TX_ON: 1721 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1722 break; 1723 default: 1724 return -ERANGE; 1725 } 1726 1727 ao = priv->active_offloads; 1728 switch (config.rx_filter) { 1729 case HWTSTAMP_FILTER_NONE: 1730 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1731 break; 1732 default: 1733 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1734 config.rx_filter = HWTSTAMP_FILTER_ALL; 1735 } 1736 1737 if (netif_running(ndev) && ao != priv->active_offloads) { 1738 enetc_close(ndev); 1739 enetc_open(ndev); 1740 } 1741 1742 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1743 -EFAULT : 0; 1744 } 1745 1746 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1747 { 1748 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1749 struct hwtstamp_config config; 1750 1751 config.flags = 0; 1752 1753 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1754 config.tx_type = HWTSTAMP_TX_ON; 1755 else 1756 config.tx_type = HWTSTAMP_TX_OFF; 1757 1758 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1759 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1760 1761 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1762 -EFAULT : 0; 1763 } 1764 #endif 1765 1766 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1767 { 1768 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1769 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1770 if (cmd == SIOCSHWTSTAMP) 1771 return enetc_hwtstamp_set(ndev, rq); 1772 if (cmd == SIOCGHWTSTAMP) 1773 return enetc_hwtstamp_get(ndev, rq); 1774 #endif 1775 1776 if (!priv->phylink) 1777 return -EOPNOTSUPP; 1778 1779 return phylink_mii_ioctl(priv->phylink, rq, cmd); 1780 } 1781 1782 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1783 { 1784 struct pci_dev *pdev = priv->si->pdev; 1785 int v_tx_rings; 1786 int i, n, err, nvec; 1787 1788 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1789 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1790 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1791 1792 if (n < 0) 1793 return n; 1794 1795 if (n != nvec) 1796 return -EPERM; 1797 1798 /* # of tx rings per int vector */ 1799 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1800 1801 for (i = 0; i < priv->bdr_int_num; i++) { 1802 struct enetc_int_vector *v; 1803 struct enetc_bdr *bdr; 1804 int j; 1805 1806 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 1807 if (!v) { 1808 err = -ENOMEM; 1809 goto fail; 1810 } 1811 1812 priv->int_vector[i] = v; 1813 1814 /* init defaults for adaptive IC */ 1815 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 1816 v->rx_ictt = 0x1; 1817 v->rx_dim_en = true; 1818 } 1819 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 1820 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1821 NAPI_POLL_WEIGHT); 1822 v->count_tx_rings = v_tx_rings; 1823 1824 for (j = 0; j < v_tx_rings; j++) { 1825 int idx; 1826 1827 /* default tx ring mapping policy */ 1828 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1829 idx = 2 * j + i; /* 2 CPUs */ 1830 else 1831 idx = j + i * v_tx_rings; /* default */ 1832 1833 __set_bit(idx, &v->tx_rings_map); 1834 bdr = &v->tx_ring[j]; 1835 bdr->index = idx; 1836 bdr->ndev = priv->ndev; 1837 bdr->dev = priv->dev; 1838 bdr->bd_count = priv->tx_bd_count; 1839 priv->tx_ring[idx] = bdr; 1840 } 1841 1842 bdr = &v->rx_ring; 1843 bdr->index = i; 1844 bdr->ndev = priv->ndev; 1845 bdr->dev = priv->dev; 1846 bdr->bd_count = priv->rx_bd_count; 1847 priv->rx_ring[i] = bdr; 1848 } 1849 1850 return 0; 1851 1852 fail: 1853 while (i--) { 1854 netif_napi_del(&priv->int_vector[i]->napi); 1855 cancel_work_sync(&priv->int_vector[i]->rx_dim.work); 1856 kfree(priv->int_vector[i]); 1857 } 1858 1859 pci_free_irq_vectors(pdev); 1860 1861 return err; 1862 } 1863 1864 void enetc_free_msix(struct enetc_ndev_priv *priv) 1865 { 1866 int i; 1867 1868 for (i = 0; i < priv->bdr_int_num; i++) { 1869 struct enetc_int_vector *v = priv->int_vector[i]; 1870 1871 netif_napi_del(&v->napi); 1872 cancel_work_sync(&v->rx_dim.work); 1873 } 1874 1875 for (i = 0; i < priv->num_rx_rings; i++) 1876 priv->rx_ring[i] = NULL; 1877 1878 for (i = 0; i < priv->num_tx_rings; i++) 1879 priv->tx_ring[i] = NULL; 1880 1881 for (i = 0; i < priv->bdr_int_num; i++) { 1882 kfree(priv->int_vector[i]); 1883 priv->int_vector[i] = NULL; 1884 } 1885 1886 /* disable all MSIX for this device */ 1887 pci_free_irq_vectors(priv->si->pdev); 1888 } 1889 1890 static void enetc_kfree_si(struct enetc_si *si) 1891 { 1892 char *p = (char *)si - si->pad; 1893 1894 kfree(p); 1895 } 1896 1897 static void enetc_detect_errata(struct enetc_si *si) 1898 { 1899 if (si->pdev->revision == ENETC_REV1) 1900 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL | 1901 ENETC_ERR_UCMCSWP; 1902 } 1903 1904 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1905 { 1906 struct enetc_si *si, *p; 1907 struct enetc_hw *hw; 1908 size_t alloc_size; 1909 int err, len; 1910 1911 pcie_flr(pdev); 1912 err = pci_enable_device_mem(pdev); 1913 if (err) { 1914 dev_err(&pdev->dev, "device enable failed\n"); 1915 return err; 1916 } 1917 1918 /* set up for high or low dma */ 1919 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1920 if (err) { 1921 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1922 if (err) { 1923 dev_err(&pdev->dev, 1924 "DMA configuration failed: 0x%x\n", err); 1925 goto err_dma; 1926 } 1927 } 1928 1929 err = pci_request_mem_regions(pdev, name); 1930 if (err) { 1931 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1932 goto err_pci_mem_reg; 1933 } 1934 1935 pci_set_master(pdev); 1936 1937 alloc_size = sizeof(struct enetc_si); 1938 if (sizeof_priv) { 1939 /* align priv to 32B */ 1940 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1941 alloc_size += sizeof_priv; 1942 } 1943 /* force 32B alignment for enetc_si */ 1944 alloc_size += ENETC_SI_ALIGN - 1; 1945 1946 p = kzalloc(alloc_size, GFP_KERNEL); 1947 if (!p) { 1948 err = -ENOMEM; 1949 goto err_alloc_si; 1950 } 1951 1952 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1953 si->pad = (char *)si - (char *)p; 1954 1955 pci_set_drvdata(pdev, si); 1956 si->pdev = pdev; 1957 hw = &si->hw; 1958 1959 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1960 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1961 if (!hw->reg) { 1962 err = -ENXIO; 1963 dev_err(&pdev->dev, "ioremap() failed\n"); 1964 goto err_ioremap; 1965 } 1966 if (len > ENETC_PORT_BASE) 1967 hw->port = hw->reg + ENETC_PORT_BASE; 1968 if (len > ENETC_GLOBAL_BASE) 1969 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1970 1971 enetc_detect_errata(si); 1972 1973 return 0; 1974 1975 err_ioremap: 1976 enetc_kfree_si(si); 1977 err_alloc_si: 1978 pci_release_mem_regions(pdev); 1979 err_pci_mem_reg: 1980 err_dma: 1981 pci_disable_device(pdev); 1982 1983 return err; 1984 } 1985 1986 void enetc_pci_remove(struct pci_dev *pdev) 1987 { 1988 struct enetc_si *si = pci_get_drvdata(pdev); 1989 struct enetc_hw *hw = &si->hw; 1990 1991 iounmap(hw->reg); 1992 enetc_kfree_si(si); 1993 pci_release_mem_regions(pdev); 1994 pci_disable_device(pdev); 1995 } 1996