1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/tcp.h> 7 #include <linux/udp.h> 8 #include <linux/vmalloc.h> 9 #include <linux/ptp_classify.h> 10 #include <net/ip6_checksum.h> 11 #include <net/pkt_sched.h> 12 #include <net/tso.h> 13 14 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 15 { 16 return enetc_port_rd(&si->hw, reg); 17 } 18 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 19 20 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 21 { 22 enetc_port_wr(&si->hw, reg, val); 23 if (si->hw_features & ENETC_SI_F_QBU) 24 enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val); 25 } 26 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 27 28 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 29 u8 preemptible_tcs) 30 { 31 if (!(priv->si->hw_features & ENETC_SI_F_QBU)) 32 return; 33 34 priv->preemptible_tcs = preemptible_tcs; 35 enetc_mm_commit_preemptible_tcs(priv); 36 } 37 38 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 39 { 40 int num_tx_rings = priv->num_tx_rings; 41 42 if (priv->xdp_prog) 43 return num_tx_rings - num_possible_cpus(); 44 45 return num_tx_rings; 46 } 47 48 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 49 struct enetc_bdr *tx_ring) 50 { 51 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 52 53 return priv->rx_ring[index]; 54 } 55 56 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 57 { 58 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 59 return NULL; 60 61 return tx_swbd->skb; 62 } 63 64 static struct xdp_frame * 65 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 66 { 67 if (tx_swbd->is_xdp_redirect) 68 return tx_swbd->xdp_frame; 69 70 return NULL; 71 } 72 73 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 74 struct enetc_tx_swbd *tx_swbd) 75 { 76 /* For XDP_TX, pages come from RX, whereas for the other contexts where 77 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 78 * to match the DMA mapping length, so we need to differentiate those. 79 */ 80 if (tx_swbd->is_dma_page) 81 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 82 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 83 tx_swbd->dir); 84 else 85 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 86 tx_swbd->len, tx_swbd->dir); 87 tx_swbd->dma = 0; 88 } 89 90 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 91 struct enetc_tx_swbd *tx_swbd) 92 { 93 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 94 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 95 96 if (tx_swbd->dma) 97 enetc_unmap_tx_buff(tx_ring, tx_swbd); 98 99 if (xdp_frame) { 100 xdp_return_frame(tx_swbd->xdp_frame); 101 tx_swbd->xdp_frame = NULL; 102 } else if (skb) { 103 dev_kfree_skb_any(skb); 104 tx_swbd->skb = NULL; 105 } 106 } 107 108 /* Let H/W know BD ring has been updated */ 109 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 110 { 111 /* includes wmb() */ 112 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 113 } 114 115 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 116 u8 *msgtype, u8 *twostep, 117 u16 *correction_offset, u16 *body_offset) 118 { 119 unsigned int ptp_class; 120 struct ptp_header *hdr; 121 unsigned int type; 122 u8 *base; 123 124 ptp_class = ptp_classify_raw(skb); 125 if (ptp_class == PTP_CLASS_NONE) 126 return -EINVAL; 127 128 hdr = ptp_parse_header(skb, ptp_class); 129 if (!hdr) 130 return -EINVAL; 131 132 type = ptp_class & PTP_CLASS_PMASK; 133 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 134 *udp = 1; 135 else 136 *udp = 0; 137 138 *msgtype = ptp_get_msgtype(hdr, ptp_class); 139 *twostep = hdr->flag_field[0] & 0x2; 140 141 base = skb_mac_header(skb); 142 *correction_offset = (u8 *)&hdr->correction - base; 143 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 144 145 return 0; 146 } 147 148 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 149 { 150 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 151 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 152 struct enetc_hw *hw = &priv->si->hw; 153 struct enetc_tx_swbd *tx_swbd; 154 int len = skb_headlen(skb); 155 union enetc_tx_bd temp_bd; 156 u8 msgtype, twostep, udp; 157 union enetc_tx_bd *txbd; 158 u16 offset1, offset2; 159 int i, count = 0; 160 skb_frag_t *frag; 161 unsigned int f; 162 dma_addr_t dma; 163 u8 flags = 0; 164 165 i = tx_ring->next_to_use; 166 txbd = ENETC_TXBD(*tx_ring, i); 167 prefetchw(txbd); 168 169 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 170 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 171 goto dma_err; 172 173 temp_bd.addr = cpu_to_le64(dma); 174 temp_bd.buf_len = cpu_to_le16(len); 175 temp_bd.lstatus = 0; 176 177 tx_swbd = &tx_ring->tx_swbd[i]; 178 tx_swbd->dma = dma; 179 tx_swbd->len = len; 180 tx_swbd->is_dma_page = 0; 181 tx_swbd->dir = DMA_TO_DEVICE; 182 count++; 183 184 do_vlan = skb_vlan_tag_present(skb); 185 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 186 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 187 &offset2) || 188 msgtype != PTP_MSGTYPE_SYNC || twostep) 189 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 190 else 191 do_onestep_tstamp = true; 192 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 193 do_twostep_tstamp = true; 194 } 195 196 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 197 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 198 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 199 200 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 201 flags |= ENETC_TXBD_FLAGS_EX; 202 203 if (tx_ring->tsd_enable) 204 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 205 206 /* first BD needs frm_len and offload flags set */ 207 temp_bd.frm_len = cpu_to_le16(skb->len); 208 temp_bd.flags = flags; 209 210 if (flags & ENETC_TXBD_FLAGS_TSE) 211 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 212 flags); 213 214 if (flags & ENETC_TXBD_FLAGS_EX) { 215 u8 e_flags = 0; 216 *txbd = temp_bd; 217 enetc_clear_tx_bd(&temp_bd); 218 219 /* add extension BD for VLAN and/or timestamping */ 220 flags = 0; 221 tx_swbd++; 222 txbd++; 223 i++; 224 if (unlikely(i == tx_ring->bd_count)) { 225 i = 0; 226 tx_swbd = tx_ring->tx_swbd; 227 txbd = ENETC_TXBD(*tx_ring, 0); 228 } 229 prefetchw(txbd); 230 231 if (do_vlan) { 232 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 233 temp_bd.ext.tpid = 0; /* < C-TAG */ 234 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 235 } 236 237 if (do_onestep_tstamp) { 238 u32 lo, hi, val; 239 u64 sec, nsec; 240 u8 *data; 241 242 lo = enetc_rd_hot(hw, ENETC_SICTR0); 243 hi = enetc_rd_hot(hw, ENETC_SICTR1); 244 sec = (u64)hi << 32 | lo; 245 nsec = do_div(sec, 1000000000); 246 247 /* Configure extension BD */ 248 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 249 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 250 251 /* Update originTimestamp field of Sync packet 252 * - 48 bits seconds field 253 * - 32 bits nanseconds field 254 */ 255 data = skb_mac_header(skb); 256 *(__be16 *)(data + offset2) = 257 htons((sec >> 32) & 0xffff); 258 *(__be32 *)(data + offset2 + 2) = 259 htonl(sec & 0xffffffff); 260 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 261 262 /* Configure single-step register */ 263 val = ENETC_PM0_SINGLE_STEP_EN; 264 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 265 if (udp) 266 val |= ENETC_PM0_SINGLE_STEP_CH; 267 268 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP, 269 val); 270 } else if (do_twostep_tstamp) { 271 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 272 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 273 } 274 275 temp_bd.ext.e_flags = e_flags; 276 count++; 277 } 278 279 frag = &skb_shinfo(skb)->frags[0]; 280 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 281 len = skb_frag_size(frag); 282 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 283 DMA_TO_DEVICE); 284 if (dma_mapping_error(tx_ring->dev, dma)) 285 goto dma_err; 286 287 *txbd = temp_bd; 288 enetc_clear_tx_bd(&temp_bd); 289 290 flags = 0; 291 tx_swbd++; 292 txbd++; 293 i++; 294 if (unlikely(i == tx_ring->bd_count)) { 295 i = 0; 296 tx_swbd = tx_ring->tx_swbd; 297 txbd = ENETC_TXBD(*tx_ring, 0); 298 } 299 prefetchw(txbd); 300 301 temp_bd.addr = cpu_to_le64(dma); 302 temp_bd.buf_len = cpu_to_le16(len); 303 304 tx_swbd->dma = dma; 305 tx_swbd->len = len; 306 tx_swbd->is_dma_page = 1; 307 tx_swbd->dir = DMA_TO_DEVICE; 308 count++; 309 } 310 311 /* last BD needs 'F' bit set */ 312 flags |= ENETC_TXBD_FLAGS_F; 313 temp_bd.flags = flags; 314 *txbd = temp_bd; 315 316 tx_ring->tx_swbd[i].is_eof = true; 317 tx_ring->tx_swbd[i].skb = skb; 318 319 enetc_bdr_idx_inc(tx_ring, &i); 320 tx_ring->next_to_use = i; 321 322 skb_tx_timestamp(skb); 323 324 enetc_update_tx_ring_tail(tx_ring); 325 326 return count; 327 328 dma_err: 329 dev_err(tx_ring->dev, "DMA map error"); 330 331 do { 332 tx_swbd = &tx_ring->tx_swbd[i]; 333 enetc_free_tx_frame(tx_ring, tx_swbd); 334 if (i == 0) 335 i = tx_ring->bd_count; 336 i--; 337 } while (count--); 338 339 return 0; 340 } 341 342 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 343 struct enetc_tx_swbd *tx_swbd, 344 union enetc_tx_bd *txbd, int *i, int hdr_len, 345 int data_len) 346 { 347 union enetc_tx_bd txbd_tmp; 348 u8 flags = 0, e_flags = 0; 349 dma_addr_t addr; 350 351 enetc_clear_tx_bd(&txbd_tmp); 352 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 353 354 if (skb_vlan_tag_present(skb)) 355 flags |= ENETC_TXBD_FLAGS_EX; 356 357 txbd_tmp.addr = cpu_to_le64(addr); 358 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 359 360 /* first BD needs frm_len and offload flags set */ 361 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 362 txbd_tmp.flags = flags; 363 364 /* For the TSO header we do not set the dma address since we do not 365 * want it unmapped when we do cleanup. We still set len so that we 366 * count the bytes sent. 367 */ 368 tx_swbd->len = hdr_len; 369 tx_swbd->do_twostep_tstamp = false; 370 tx_swbd->check_wb = false; 371 372 /* Actually write the header in the BD */ 373 *txbd = txbd_tmp; 374 375 /* Add extension BD for VLAN */ 376 if (flags & ENETC_TXBD_FLAGS_EX) { 377 /* Get the next BD */ 378 enetc_bdr_idx_inc(tx_ring, i); 379 txbd = ENETC_TXBD(*tx_ring, *i); 380 tx_swbd = &tx_ring->tx_swbd[*i]; 381 prefetchw(txbd); 382 383 /* Setup the VLAN fields */ 384 enetc_clear_tx_bd(&txbd_tmp); 385 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 386 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 387 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 388 389 /* Write the BD */ 390 txbd_tmp.ext.e_flags = e_flags; 391 *txbd = txbd_tmp; 392 } 393 } 394 395 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 396 struct enetc_tx_swbd *tx_swbd, 397 union enetc_tx_bd *txbd, char *data, 398 int size, bool last_bd) 399 { 400 union enetc_tx_bd txbd_tmp; 401 dma_addr_t addr; 402 u8 flags = 0; 403 404 enetc_clear_tx_bd(&txbd_tmp); 405 406 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 407 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 408 netdev_err(tx_ring->ndev, "DMA map error\n"); 409 return -ENOMEM; 410 } 411 412 if (last_bd) { 413 flags |= ENETC_TXBD_FLAGS_F; 414 tx_swbd->is_eof = 1; 415 } 416 417 txbd_tmp.addr = cpu_to_le64(addr); 418 txbd_tmp.buf_len = cpu_to_le16(size); 419 txbd_tmp.flags = flags; 420 421 tx_swbd->dma = addr; 422 tx_swbd->len = size; 423 tx_swbd->dir = DMA_TO_DEVICE; 424 425 *txbd = txbd_tmp; 426 427 return 0; 428 } 429 430 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 431 char *hdr, int hdr_len, int *l4_hdr_len) 432 { 433 char *l4_hdr = hdr + skb_transport_offset(skb); 434 int mac_hdr_len = skb_network_offset(skb); 435 436 if (tso->tlen != sizeof(struct udphdr)) { 437 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 438 439 tcph->check = 0; 440 } else { 441 struct udphdr *udph = (struct udphdr *)(l4_hdr); 442 443 udph->check = 0; 444 } 445 446 /* Compute the IP checksum. This is necessary since tso_build_hdr() 447 * already incremented the IP ID field. 448 */ 449 if (!tso->ipv6) { 450 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 451 452 iph->check = 0; 453 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 454 } 455 456 /* Compute the checksum over the L4 header. */ 457 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 458 return csum_partial(l4_hdr, *l4_hdr_len, 0); 459 } 460 461 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 462 struct sk_buff *skb, char *hdr, int len, 463 __wsum sum) 464 { 465 char *l4_hdr = hdr + skb_transport_offset(skb); 466 __sum16 csum_final; 467 468 /* Complete the L4 checksum by appending the pseudo-header to the 469 * already computed checksum. 470 */ 471 if (!tso->ipv6) 472 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 473 ip_hdr(skb)->daddr, 474 len, ip_hdr(skb)->protocol, sum); 475 else 476 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 477 &ipv6_hdr(skb)->daddr, 478 len, ipv6_hdr(skb)->nexthdr, sum); 479 480 if (tso->tlen != sizeof(struct udphdr)) { 481 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 482 483 tcph->check = csum_final; 484 } else { 485 struct udphdr *udph = (struct udphdr *)(l4_hdr); 486 487 udph->check = csum_final; 488 } 489 } 490 491 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 492 { 493 int hdr_len, total_len, data_len; 494 struct enetc_tx_swbd *tx_swbd; 495 union enetc_tx_bd *txbd; 496 struct tso_t tso; 497 __wsum csum, csum2; 498 int count = 0, pos; 499 int err, i, bd_data_num; 500 501 /* Initialize the TSO handler, and prepare the first payload */ 502 hdr_len = tso_start(skb, &tso); 503 total_len = skb->len - hdr_len; 504 i = tx_ring->next_to_use; 505 506 while (total_len > 0) { 507 char *hdr; 508 509 /* Get the BD */ 510 txbd = ENETC_TXBD(*tx_ring, i); 511 tx_swbd = &tx_ring->tx_swbd[i]; 512 prefetchw(txbd); 513 514 /* Determine the length of this packet */ 515 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 516 total_len -= data_len; 517 518 /* prepare packet headers: MAC + IP + TCP */ 519 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 520 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 521 522 /* compute the csum over the L4 header */ 523 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 524 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 525 bd_data_num = 0; 526 count++; 527 528 while (data_len > 0) { 529 int size; 530 531 size = min_t(int, tso.size, data_len); 532 533 /* Advance the index in the BDR */ 534 enetc_bdr_idx_inc(tx_ring, &i); 535 txbd = ENETC_TXBD(*tx_ring, i); 536 tx_swbd = &tx_ring->tx_swbd[i]; 537 prefetchw(txbd); 538 539 /* Compute the checksum over this segment of data and 540 * add it to the csum already computed (over the L4 541 * header and possible other data segments). 542 */ 543 csum2 = csum_partial(tso.data, size, 0); 544 csum = csum_block_add(csum, csum2, pos); 545 pos += size; 546 547 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 548 tso.data, size, 549 size == data_len); 550 if (err) 551 goto err_map_data; 552 553 data_len -= size; 554 count++; 555 bd_data_num++; 556 tso_build_data(skb, &tso, size); 557 558 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 559 goto err_chained_bd; 560 } 561 562 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 563 564 if (total_len == 0) 565 tx_swbd->skb = skb; 566 567 /* Go to the next BD */ 568 enetc_bdr_idx_inc(tx_ring, &i); 569 } 570 571 tx_ring->next_to_use = i; 572 enetc_update_tx_ring_tail(tx_ring); 573 574 return count; 575 576 err_map_data: 577 dev_err(tx_ring->dev, "DMA map error"); 578 579 err_chained_bd: 580 do { 581 tx_swbd = &tx_ring->tx_swbd[i]; 582 enetc_free_tx_frame(tx_ring, tx_swbd); 583 if (i == 0) 584 i = tx_ring->bd_count; 585 i--; 586 } while (count--); 587 588 return 0; 589 } 590 591 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 592 struct net_device *ndev) 593 { 594 struct enetc_ndev_priv *priv = netdev_priv(ndev); 595 struct enetc_bdr *tx_ring; 596 int count, err; 597 598 /* Queue one-step Sync packet if already locked */ 599 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 600 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 601 &priv->flags)) { 602 skb_queue_tail(&priv->tx_skbs, skb); 603 return NETDEV_TX_OK; 604 } 605 } 606 607 tx_ring = priv->tx_ring[skb->queue_mapping]; 608 609 if (skb_is_gso(skb)) { 610 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 611 netif_stop_subqueue(ndev, tx_ring->index); 612 return NETDEV_TX_BUSY; 613 } 614 615 enetc_lock_mdio(); 616 count = enetc_map_tx_tso_buffs(tx_ring, skb); 617 enetc_unlock_mdio(); 618 } else { 619 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 620 if (unlikely(skb_linearize(skb))) 621 goto drop_packet_err; 622 623 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 624 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 625 netif_stop_subqueue(ndev, tx_ring->index); 626 return NETDEV_TX_BUSY; 627 } 628 629 if (skb->ip_summed == CHECKSUM_PARTIAL) { 630 err = skb_checksum_help(skb); 631 if (err) 632 goto drop_packet_err; 633 } 634 enetc_lock_mdio(); 635 count = enetc_map_tx_buffs(tx_ring, skb); 636 enetc_unlock_mdio(); 637 } 638 639 if (unlikely(!count)) 640 goto drop_packet_err; 641 642 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 643 netif_stop_subqueue(ndev, tx_ring->index); 644 645 return NETDEV_TX_OK; 646 647 drop_packet_err: 648 dev_kfree_skb_any(skb); 649 return NETDEV_TX_OK; 650 } 651 652 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 653 { 654 struct enetc_ndev_priv *priv = netdev_priv(ndev); 655 u8 udp, msgtype, twostep; 656 u16 offset1, offset2; 657 658 /* Mark tx timestamp type on skb->cb[0] if requires */ 659 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 660 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 661 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 662 } else { 663 skb->cb[0] = 0; 664 } 665 666 /* Fall back to two-step timestamp if not one-step Sync packet */ 667 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 668 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 669 &offset1, &offset2) || 670 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 671 skb->cb[0] = ENETC_F_TX_TSTAMP; 672 } 673 674 return enetc_start_xmit(skb, ndev); 675 } 676 EXPORT_SYMBOL_GPL(enetc_xmit); 677 678 static irqreturn_t enetc_msix(int irq, void *data) 679 { 680 struct enetc_int_vector *v = data; 681 int i; 682 683 enetc_lock_mdio(); 684 685 /* disable interrupts */ 686 enetc_wr_reg_hot(v->rbier, 0); 687 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 688 689 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 690 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 691 692 enetc_unlock_mdio(); 693 694 napi_schedule(&v->napi); 695 696 return IRQ_HANDLED; 697 } 698 699 static void enetc_rx_dim_work(struct work_struct *w) 700 { 701 struct dim *dim = container_of(w, struct dim, work); 702 struct dim_cq_moder moder = 703 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 704 struct enetc_int_vector *v = 705 container_of(dim, struct enetc_int_vector, rx_dim); 706 707 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 708 dim->state = DIM_START_MEASURE; 709 } 710 711 static void enetc_rx_net_dim(struct enetc_int_vector *v) 712 { 713 struct dim_sample dim_sample = {}; 714 715 v->comp_cnt++; 716 717 if (!v->rx_napi_work) 718 return; 719 720 dim_update_sample(v->comp_cnt, 721 v->rx_ring.stats.packets, 722 v->rx_ring.stats.bytes, 723 &dim_sample); 724 net_dim(&v->rx_dim, dim_sample); 725 } 726 727 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 728 { 729 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 730 731 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 732 } 733 734 static bool enetc_page_reusable(struct page *page) 735 { 736 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 737 } 738 739 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 740 struct enetc_rx_swbd *old) 741 { 742 struct enetc_rx_swbd *new; 743 744 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 745 746 /* next buf that may reuse a page */ 747 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 748 749 /* copy page reference */ 750 *new = *old; 751 } 752 753 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 754 u64 *tstamp) 755 { 756 u32 lo, hi, tstamp_lo; 757 758 lo = enetc_rd_hot(hw, ENETC_SICTR0); 759 hi = enetc_rd_hot(hw, ENETC_SICTR1); 760 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 761 if (lo <= tstamp_lo) 762 hi -= 1; 763 *tstamp = (u64)hi << 32 | tstamp_lo; 764 } 765 766 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 767 { 768 struct skb_shared_hwtstamps shhwtstamps; 769 770 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 771 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 772 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 773 skb_txtime_consumed(skb); 774 skb_tstamp_tx(skb, &shhwtstamps); 775 } 776 } 777 778 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 779 struct enetc_tx_swbd *tx_swbd) 780 { 781 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 782 struct enetc_rx_swbd rx_swbd = { 783 .dma = tx_swbd->dma, 784 .page = tx_swbd->page, 785 .page_offset = tx_swbd->page_offset, 786 .dir = tx_swbd->dir, 787 .len = tx_swbd->len, 788 }; 789 struct enetc_bdr *rx_ring; 790 791 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 792 793 if (likely(enetc_swbd_unused(rx_ring))) { 794 enetc_reuse_page(rx_ring, &rx_swbd); 795 796 /* sync for use by the device */ 797 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 798 rx_swbd.page_offset, 799 ENETC_RXB_DMA_SIZE_XDP, 800 rx_swbd.dir); 801 802 rx_ring->stats.recycles++; 803 } else { 804 /* RX ring is already full, we need to unmap and free the 805 * page, since there's nothing useful we can do with it. 806 */ 807 rx_ring->stats.recycle_failures++; 808 809 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 810 rx_swbd.dir); 811 __free_page(rx_swbd.page); 812 } 813 814 rx_ring->xdp.xdp_tx_in_flight--; 815 } 816 817 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 818 { 819 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 820 struct net_device *ndev = tx_ring->ndev; 821 struct enetc_ndev_priv *priv = netdev_priv(ndev); 822 struct enetc_tx_swbd *tx_swbd; 823 int i, bds_to_clean; 824 bool do_twostep_tstamp; 825 u64 tstamp = 0; 826 827 i = tx_ring->next_to_clean; 828 tx_swbd = &tx_ring->tx_swbd[i]; 829 830 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 831 832 do_twostep_tstamp = false; 833 834 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 835 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 836 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 837 bool is_eof = tx_swbd->is_eof; 838 839 if (unlikely(tx_swbd->check_wb)) { 840 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 841 842 if (txbd->flags & ENETC_TXBD_FLAGS_W && 843 tx_swbd->do_twostep_tstamp) { 844 enetc_get_tx_tstamp(&priv->si->hw, txbd, 845 &tstamp); 846 do_twostep_tstamp = true; 847 } 848 849 if (tx_swbd->qbv_en && 850 txbd->wb.status & ENETC_TXBD_STATS_WIN) 851 tx_win_drop++; 852 } 853 854 if (tx_swbd->is_xdp_tx) 855 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 856 else if (likely(tx_swbd->dma)) 857 enetc_unmap_tx_buff(tx_ring, tx_swbd); 858 859 if (xdp_frame) { 860 xdp_return_frame(xdp_frame); 861 } else if (skb) { 862 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 863 /* Start work to release lock for next one-step 864 * timestamping packet. And send one skb in 865 * tx_skbs queue if has. 866 */ 867 schedule_work(&priv->tx_onestep_tstamp); 868 } else if (unlikely(do_twostep_tstamp)) { 869 enetc_tstamp_tx(skb, tstamp); 870 do_twostep_tstamp = false; 871 } 872 napi_consume_skb(skb, napi_budget); 873 } 874 875 tx_byte_cnt += tx_swbd->len; 876 /* Scrub the swbd here so we don't have to do that 877 * when we reuse it during xmit 878 */ 879 memset(tx_swbd, 0, sizeof(*tx_swbd)); 880 881 bds_to_clean--; 882 tx_swbd++; 883 i++; 884 if (unlikely(i == tx_ring->bd_count)) { 885 i = 0; 886 tx_swbd = tx_ring->tx_swbd; 887 } 888 889 /* BD iteration loop end */ 890 if (is_eof) { 891 tx_frm_cnt++; 892 /* re-arm interrupt source */ 893 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 894 BIT(16 + tx_ring->index)); 895 } 896 897 if (unlikely(!bds_to_clean)) 898 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 899 } 900 901 tx_ring->next_to_clean = i; 902 tx_ring->stats.packets += tx_frm_cnt; 903 tx_ring->stats.bytes += tx_byte_cnt; 904 tx_ring->stats.win_drop += tx_win_drop; 905 906 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 907 __netif_subqueue_stopped(ndev, tx_ring->index) && 908 !test_bit(ENETC_TX_DOWN, &priv->flags) && 909 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 910 netif_wake_subqueue(ndev, tx_ring->index); 911 } 912 913 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 914 } 915 916 static bool enetc_new_page(struct enetc_bdr *rx_ring, 917 struct enetc_rx_swbd *rx_swbd) 918 { 919 bool xdp = !!(rx_ring->xdp.prog); 920 struct page *page; 921 dma_addr_t addr; 922 923 page = dev_alloc_page(); 924 if (unlikely(!page)) 925 return false; 926 927 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 928 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 929 930 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 931 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 932 __free_page(page); 933 934 return false; 935 } 936 937 rx_swbd->dma = addr; 938 rx_swbd->page = page; 939 rx_swbd->page_offset = rx_ring->buffer_offset; 940 941 return true; 942 } 943 944 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 945 { 946 struct enetc_rx_swbd *rx_swbd; 947 union enetc_rx_bd *rxbd; 948 int i, j; 949 950 i = rx_ring->next_to_use; 951 rx_swbd = &rx_ring->rx_swbd[i]; 952 rxbd = enetc_rxbd(rx_ring, i); 953 954 for (j = 0; j < buff_cnt; j++) { 955 /* try reuse page */ 956 if (unlikely(!rx_swbd->page)) { 957 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 958 rx_ring->stats.rx_alloc_errs++; 959 break; 960 } 961 } 962 963 /* update RxBD */ 964 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 965 rx_swbd->page_offset); 966 /* clear 'R" as well */ 967 rxbd->r.lstatus = 0; 968 969 enetc_rxbd_next(rx_ring, &rxbd, &i); 970 rx_swbd = &rx_ring->rx_swbd[i]; 971 } 972 973 if (likely(j)) { 974 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 975 rx_ring->next_to_use = i; 976 977 /* update ENETC's consumer index */ 978 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 979 } 980 981 return j; 982 } 983 984 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 985 static void enetc_get_rx_tstamp(struct net_device *ndev, 986 union enetc_rx_bd *rxbd, 987 struct sk_buff *skb) 988 { 989 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 990 struct enetc_ndev_priv *priv = netdev_priv(ndev); 991 struct enetc_hw *hw = &priv->si->hw; 992 u32 lo, hi, tstamp_lo; 993 u64 tstamp; 994 995 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 996 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 997 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 998 rxbd = enetc_rxbd_ext(rxbd); 999 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 1000 if (lo <= tstamp_lo) 1001 hi -= 1; 1002 1003 tstamp = (u64)hi << 32 | tstamp_lo; 1004 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1005 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1006 } 1007 } 1008 #endif 1009 1010 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1011 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1012 { 1013 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1014 1015 /* TODO: hashing */ 1016 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1017 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1018 1019 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1020 skb->ip_summed = CHECKSUM_COMPLETE; 1021 } 1022 1023 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1024 __be16 tpid = 0; 1025 1026 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1027 case 0: 1028 tpid = htons(ETH_P_8021Q); 1029 break; 1030 case 1: 1031 tpid = htons(ETH_P_8021AD); 1032 break; 1033 case 2: 1034 tpid = htons(enetc_port_rd(&priv->si->hw, 1035 ENETC_PCVLANR1)); 1036 break; 1037 case 3: 1038 tpid = htons(enetc_port_rd(&priv->si->hw, 1039 ENETC_PCVLANR2)); 1040 break; 1041 default: 1042 break; 1043 } 1044 1045 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1046 } 1047 1048 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1049 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1050 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1051 #endif 1052 } 1053 1054 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1055 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1056 * mapped buffers. 1057 */ 1058 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1059 int i, u16 size) 1060 { 1061 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1062 1063 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1064 rx_swbd->page_offset, 1065 size, rx_swbd->dir); 1066 return rx_swbd; 1067 } 1068 1069 /* Reuse the current page without performing half-page buffer flipping */ 1070 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1071 struct enetc_rx_swbd *rx_swbd) 1072 { 1073 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1074 1075 enetc_reuse_page(rx_ring, rx_swbd); 1076 1077 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1078 rx_swbd->page_offset, 1079 buffer_size, rx_swbd->dir); 1080 1081 rx_swbd->page = NULL; 1082 } 1083 1084 /* Reuse the current page by performing half-page buffer flipping */ 1085 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1086 struct enetc_rx_swbd *rx_swbd) 1087 { 1088 if (likely(enetc_page_reusable(rx_swbd->page))) { 1089 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1090 page_ref_inc(rx_swbd->page); 1091 1092 enetc_put_rx_buff(rx_ring, rx_swbd); 1093 } else { 1094 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1095 rx_swbd->dir); 1096 rx_swbd->page = NULL; 1097 } 1098 } 1099 1100 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1101 int i, u16 size) 1102 { 1103 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1104 struct sk_buff *skb; 1105 void *ba; 1106 1107 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1108 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1109 if (unlikely(!skb)) { 1110 rx_ring->stats.rx_alloc_errs++; 1111 return NULL; 1112 } 1113 1114 skb_reserve(skb, rx_ring->buffer_offset); 1115 __skb_put(skb, size); 1116 1117 enetc_flip_rx_buff(rx_ring, rx_swbd); 1118 1119 return skb; 1120 } 1121 1122 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1123 u16 size, struct sk_buff *skb) 1124 { 1125 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1126 1127 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1128 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1129 1130 enetc_flip_rx_buff(rx_ring, rx_swbd); 1131 } 1132 1133 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1134 u32 bd_status, 1135 union enetc_rx_bd **rxbd, int *i) 1136 { 1137 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1138 return false; 1139 1140 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1141 enetc_rxbd_next(rx_ring, rxbd, i); 1142 1143 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1144 dma_rmb(); 1145 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1146 1147 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1148 enetc_rxbd_next(rx_ring, rxbd, i); 1149 } 1150 1151 rx_ring->ndev->stats.rx_dropped++; 1152 rx_ring->ndev->stats.rx_errors++; 1153 1154 return true; 1155 } 1156 1157 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1158 u32 bd_status, union enetc_rx_bd **rxbd, 1159 int *i, int *cleaned_cnt, int buffer_size) 1160 { 1161 struct sk_buff *skb; 1162 u16 size; 1163 1164 size = le16_to_cpu((*rxbd)->r.buf_len); 1165 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1166 if (!skb) 1167 return NULL; 1168 1169 enetc_get_offloads(rx_ring, *rxbd, skb); 1170 1171 (*cleaned_cnt)++; 1172 1173 enetc_rxbd_next(rx_ring, rxbd, i); 1174 1175 /* not last BD in frame? */ 1176 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1177 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1178 size = buffer_size; 1179 1180 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1181 dma_rmb(); 1182 size = le16_to_cpu((*rxbd)->r.buf_len); 1183 } 1184 1185 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1186 1187 (*cleaned_cnt)++; 1188 1189 enetc_rxbd_next(rx_ring, rxbd, i); 1190 } 1191 1192 skb_record_rx_queue(skb, rx_ring->index); 1193 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1194 1195 return skb; 1196 } 1197 1198 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1199 1200 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1201 struct napi_struct *napi, int work_limit) 1202 { 1203 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1204 int cleaned_cnt, i; 1205 1206 cleaned_cnt = enetc_bd_unused(rx_ring); 1207 /* next descriptor to process */ 1208 i = rx_ring->next_to_clean; 1209 1210 while (likely(rx_frm_cnt < work_limit)) { 1211 union enetc_rx_bd *rxbd; 1212 struct sk_buff *skb; 1213 u32 bd_status; 1214 1215 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1216 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1217 cleaned_cnt); 1218 1219 rxbd = enetc_rxbd(rx_ring, i); 1220 bd_status = le32_to_cpu(rxbd->r.lstatus); 1221 if (!bd_status) 1222 break; 1223 1224 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1225 dma_rmb(); /* for reading other rxbd fields */ 1226 1227 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1228 &rxbd, &i)) 1229 break; 1230 1231 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1232 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1233 if (!skb) 1234 break; 1235 1236 /* When set, the outer VLAN header is extracted and reported 1237 * in the receive buffer descriptor. So rx_byte_cnt should 1238 * add the length of the extracted VLAN header. 1239 */ 1240 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1241 rx_byte_cnt += VLAN_HLEN; 1242 rx_byte_cnt += skb->len + ETH_HLEN; 1243 rx_frm_cnt++; 1244 1245 napi_gro_receive(napi, skb); 1246 } 1247 1248 rx_ring->next_to_clean = i; 1249 1250 rx_ring->stats.packets += rx_frm_cnt; 1251 rx_ring->stats.bytes += rx_byte_cnt; 1252 1253 return rx_frm_cnt; 1254 } 1255 1256 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1257 struct enetc_tx_swbd *tx_swbd, 1258 int frm_len) 1259 { 1260 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1261 1262 prefetchw(txbd); 1263 1264 enetc_clear_tx_bd(txbd); 1265 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1266 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1267 txbd->frm_len = cpu_to_le16(frm_len); 1268 1269 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1270 } 1271 1272 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1273 * descriptors. 1274 */ 1275 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1276 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1277 { 1278 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1279 int i, k, frm_len = tmp_tx_swbd->len; 1280 1281 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1282 return false; 1283 1284 while (unlikely(!tmp_tx_swbd->is_eof)) { 1285 tmp_tx_swbd++; 1286 frm_len += tmp_tx_swbd->len; 1287 } 1288 1289 i = tx_ring->next_to_use; 1290 1291 for (k = 0; k < num_tx_swbd; k++) { 1292 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1293 1294 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1295 1296 /* last BD needs 'F' bit set */ 1297 if (xdp_tx_swbd->is_eof) { 1298 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1299 1300 txbd->flags = ENETC_TXBD_FLAGS_F; 1301 } 1302 1303 enetc_bdr_idx_inc(tx_ring, &i); 1304 } 1305 1306 tx_ring->next_to_use = i; 1307 1308 return true; 1309 } 1310 1311 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1312 struct enetc_tx_swbd *xdp_tx_arr, 1313 struct xdp_frame *xdp_frame) 1314 { 1315 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1316 struct skb_shared_info *shinfo; 1317 void *data = xdp_frame->data; 1318 int len = xdp_frame->len; 1319 skb_frag_t *frag; 1320 dma_addr_t dma; 1321 unsigned int f; 1322 int n = 0; 1323 1324 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1325 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1326 netdev_err(tx_ring->ndev, "DMA map error\n"); 1327 return -1; 1328 } 1329 1330 xdp_tx_swbd->dma = dma; 1331 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1332 xdp_tx_swbd->len = len; 1333 xdp_tx_swbd->is_xdp_redirect = true; 1334 xdp_tx_swbd->is_eof = false; 1335 xdp_tx_swbd->xdp_frame = NULL; 1336 1337 n++; 1338 1339 if (!xdp_frame_has_frags(xdp_frame)) 1340 goto out; 1341 1342 xdp_tx_swbd = &xdp_tx_arr[n]; 1343 1344 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1345 1346 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1347 f++, frag++) { 1348 data = skb_frag_address(frag); 1349 len = skb_frag_size(frag); 1350 1351 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1352 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1353 /* Undo the DMA mapping for all fragments */ 1354 while (--n >= 0) 1355 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1356 1357 netdev_err(tx_ring->ndev, "DMA map error\n"); 1358 return -1; 1359 } 1360 1361 xdp_tx_swbd->dma = dma; 1362 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1363 xdp_tx_swbd->len = len; 1364 xdp_tx_swbd->is_xdp_redirect = true; 1365 xdp_tx_swbd->is_eof = false; 1366 xdp_tx_swbd->xdp_frame = NULL; 1367 1368 n++; 1369 xdp_tx_swbd = &xdp_tx_arr[n]; 1370 } 1371 out: 1372 xdp_tx_arr[n - 1].is_eof = true; 1373 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1374 1375 return n; 1376 } 1377 1378 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1379 struct xdp_frame **frames, u32 flags) 1380 { 1381 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1382 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1383 struct enetc_bdr *tx_ring; 1384 int xdp_tx_bd_cnt, i, k; 1385 int xdp_tx_frm_cnt = 0; 1386 1387 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) 1388 return -ENETDOWN; 1389 1390 enetc_lock_mdio(); 1391 1392 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1393 1394 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1395 1396 for (k = 0; k < num_frames; k++) { 1397 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1398 xdp_redirect_arr, 1399 frames[k]); 1400 if (unlikely(xdp_tx_bd_cnt < 0)) 1401 break; 1402 1403 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1404 xdp_tx_bd_cnt))) { 1405 for (i = 0; i < xdp_tx_bd_cnt; i++) 1406 enetc_unmap_tx_buff(tx_ring, 1407 &xdp_redirect_arr[i]); 1408 tx_ring->stats.xdp_tx_drops++; 1409 break; 1410 } 1411 1412 xdp_tx_frm_cnt++; 1413 } 1414 1415 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1416 enetc_update_tx_ring_tail(tx_ring); 1417 1418 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1419 1420 enetc_unlock_mdio(); 1421 1422 return xdp_tx_frm_cnt; 1423 } 1424 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1425 1426 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1427 struct xdp_buff *xdp_buff, u16 size) 1428 { 1429 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1430 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1431 1432 /* To be used for XDP_TX */ 1433 rx_swbd->len = size; 1434 1435 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1436 rx_ring->buffer_offset, size, false); 1437 } 1438 1439 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1440 u16 size, struct xdp_buff *xdp_buff) 1441 { 1442 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1443 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1444 skb_frag_t *frag; 1445 1446 /* To be used for XDP_TX */ 1447 rx_swbd->len = size; 1448 1449 if (!xdp_buff_has_frags(xdp_buff)) { 1450 xdp_buff_set_frags_flag(xdp_buff); 1451 shinfo->xdp_frags_size = size; 1452 shinfo->nr_frags = 0; 1453 } else { 1454 shinfo->xdp_frags_size += size; 1455 } 1456 1457 if (page_is_pfmemalloc(rx_swbd->page)) 1458 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1459 1460 frag = &shinfo->frags[shinfo->nr_frags]; 1461 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1462 size); 1463 1464 shinfo->nr_frags++; 1465 } 1466 1467 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1468 union enetc_rx_bd **rxbd, int *i, 1469 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1470 { 1471 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1472 1473 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1474 1475 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1476 (*cleaned_cnt)++; 1477 enetc_rxbd_next(rx_ring, rxbd, i); 1478 1479 /* not last BD in frame? */ 1480 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1481 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1482 size = ENETC_RXB_DMA_SIZE_XDP; 1483 1484 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1485 dma_rmb(); 1486 size = le16_to_cpu((*rxbd)->r.buf_len); 1487 } 1488 1489 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1490 (*cleaned_cnt)++; 1491 enetc_rxbd_next(rx_ring, rxbd, i); 1492 } 1493 } 1494 1495 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1496 * recycled back into the RX ring in enetc_clean_tx_ring. 1497 */ 1498 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1499 struct enetc_bdr *rx_ring, 1500 int rx_ring_first, int rx_ring_last) 1501 { 1502 int n = 0; 1503 1504 for (; rx_ring_first != rx_ring_last; 1505 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1506 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1507 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1508 1509 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1510 tx_swbd->dma = rx_swbd->dma; 1511 tx_swbd->dir = rx_swbd->dir; 1512 tx_swbd->page = rx_swbd->page; 1513 tx_swbd->page_offset = rx_swbd->page_offset; 1514 tx_swbd->len = rx_swbd->len; 1515 tx_swbd->is_dma_page = true; 1516 tx_swbd->is_xdp_tx = true; 1517 tx_swbd->is_eof = false; 1518 } 1519 1520 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1521 xdp_tx_arr[n - 1].is_eof = true; 1522 1523 return n; 1524 } 1525 1526 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1527 int rx_ring_last) 1528 { 1529 while (rx_ring_first != rx_ring_last) { 1530 enetc_put_rx_buff(rx_ring, 1531 &rx_ring->rx_swbd[rx_ring_first]); 1532 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1533 } 1534 } 1535 1536 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1537 struct napi_struct *napi, int work_limit, 1538 struct bpf_prog *prog) 1539 { 1540 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1541 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1542 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1543 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1544 struct enetc_bdr *tx_ring; 1545 int cleaned_cnt, i; 1546 u32 xdp_act; 1547 1548 cleaned_cnt = enetc_bd_unused(rx_ring); 1549 /* next descriptor to process */ 1550 i = rx_ring->next_to_clean; 1551 1552 while (likely(rx_frm_cnt < work_limit)) { 1553 union enetc_rx_bd *rxbd, *orig_rxbd; 1554 int orig_i, orig_cleaned_cnt; 1555 struct xdp_buff xdp_buff; 1556 struct sk_buff *skb; 1557 u32 bd_status; 1558 int err; 1559 1560 rxbd = enetc_rxbd(rx_ring, i); 1561 bd_status = le32_to_cpu(rxbd->r.lstatus); 1562 if (!bd_status) 1563 break; 1564 1565 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1566 dma_rmb(); /* for reading other rxbd fields */ 1567 1568 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1569 &rxbd, &i)) 1570 break; 1571 1572 orig_rxbd = rxbd; 1573 orig_cleaned_cnt = cleaned_cnt; 1574 orig_i = i; 1575 1576 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1577 &cleaned_cnt, &xdp_buff); 1578 1579 /* When set, the outer VLAN header is extracted and reported 1580 * in the receive buffer descriptor. So rx_byte_cnt should 1581 * add the length of the extracted VLAN header. 1582 */ 1583 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1584 rx_byte_cnt += VLAN_HLEN; 1585 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 1586 1587 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1588 1589 switch (xdp_act) { 1590 default: 1591 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1592 fallthrough; 1593 case XDP_ABORTED: 1594 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1595 fallthrough; 1596 case XDP_DROP: 1597 enetc_xdp_drop(rx_ring, orig_i, i); 1598 rx_ring->stats.xdp_drops++; 1599 break; 1600 case XDP_PASS: 1601 rxbd = orig_rxbd; 1602 cleaned_cnt = orig_cleaned_cnt; 1603 i = orig_i; 1604 1605 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1606 &i, &cleaned_cnt, 1607 ENETC_RXB_DMA_SIZE_XDP); 1608 if (unlikely(!skb)) 1609 goto out; 1610 1611 napi_gro_receive(napi, skb); 1612 break; 1613 case XDP_TX: 1614 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1615 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 1616 enetc_xdp_drop(rx_ring, orig_i, i); 1617 tx_ring->stats.xdp_tx_drops++; 1618 break; 1619 } 1620 1621 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1622 rx_ring, 1623 orig_i, i); 1624 1625 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1626 enetc_xdp_drop(rx_ring, orig_i, i); 1627 tx_ring->stats.xdp_tx_drops++; 1628 } else { 1629 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1630 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1631 xdp_tx_frm_cnt++; 1632 /* The XDP_TX enqueue was successful, so we 1633 * need to scrub the RX software BDs because 1634 * the ownership of the buffers no longer 1635 * belongs to the RX ring, and we must prevent 1636 * enetc_refill_rx_ring() from reusing 1637 * rx_swbd->page. 1638 */ 1639 while (orig_i != i) { 1640 rx_ring->rx_swbd[orig_i].page = NULL; 1641 enetc_bdr_idx_inc(rx_ring, &orig_i); 1642 } 1643 } 1644 break; 1645 case XDP_REDIRECT: 1646 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1647 if (unlikely(err)) { 1648 enetc_xdp_drop(rx_ring, orig_i, i); 1649 rx_ring->stats.xdp_redirect_failures++; 1650 } else { 1651 while (orig_i != i) { 1652 enetc_flip_rx_buff(rx_ring, 1653 &rx_ring->rx_swbd[orig_i]); 1654 enetc_bdr_idx_inc(rx_ring, &orig_i); 1655 } 1656 xdp_redirect_frm_cnt++; 1657 rx_ring->stats.xdp_redirect++; 1658 } 1659 } 1660 1661 rx_frm_cnt++; 1662 } 1663 1664 out: 1665 rx_ring->next_to_clean = i; 1666 1667 rx_ring->stats.packets += rx_frm_cnt; 1668 rx_ring->stats.bytes += rx_byte_cnt; 1669 1670 if (xdp_redirect_frm_cnt) 1671 xdp_do_flush_map(); 1672 1673 if (xdp_tx_frm_cnt) 1674 enetc_update_tx_ring_tail(tx_ring); 1675 1676 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1677 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1678 rx_ring->xdp.xdp_tx_in_flight); 1679 1680 return rx_frm_cnt; 1681 } 1682 1683 static int enetc_poll(struct napi_struct *napi, int budget) 1684 { 1685 struct enetc_int_vector 1686 *v = container_of(napi, struct enetc_int_vector, napi); 1687 struct enetc_bdr *rx_ring = &v->rx_ring; 1688 struct bpf_prog *prog; 1689 bool complete = true; 1690 int work_done; 1691 int i; 1692 1693 enetc_lock_mdio(); 1694 1695 for (i = 0; i < v->count_tx_rings; i++) 1696 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1697 complete = false; 1698 1699 prog = rx_ring->xdp.prog; 1700 if (prog) 1701 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1702 else 1703 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1704 if (work_done == budget) 1705 complete = false; 1706 if (work_done) 1707 v->rx_napi_work = true; 1708 1709 if (!complete) { 1710 enetc_unlock_mdio(); 1711 return budget; 1712 } 1713 1714 napi_complete_done(napi, work_done); 1715 1716 if (likely(v->rx_dim_en)) 1717 enetc_rx_net_dim(v); 1718 1719 v->rx_napi_work = false; 1720 1721 /* enable interrupts */ 1722 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1723 1724 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1725 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1726 ENETC_TBIER_TXTIE); 1727 1728 enetc_unlock_mdio(); 1729 1730 return work_done; 1731 } 1732 1733 /* Probing and Init */ 1734 #define ENETC_MAX_RFS_SIZE 64 1735 void enetc_get_si_caps(struct enetc_si *si) 1736 { 1737 struct enetc_hw *hw = &si->hw; 1738 u32 val; 1739 1740 /* find out how many of various resources we have to work with */ 1741 val = enetc_rd(hw, ENETC_SICAPR0); 1742 si->num_rx_rings = (val >> 16) & 0xff; 1743 si->num_tx_rings = val & 0xff; 1744 1745 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1746 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1747 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1748 1749 si->num_rss = 0; 1750 val = enetc_rd(hw, ENETC_SIPCAPR0); 1751 if (val & ENETC_SIPCAPR0_RSS) { 1752 u32 rss; 1753 1754 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1755 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1756 } 1757 1758 if (val & ENETC_SIPCAPR0_QBV) 1759 si->hw_features |= ENETC_SI_F_QBV; 1760 1761 if (val & ENETC_SIPCAPR0_QBU) 1762 si->hw_features |= ENETC_SI_F_QBU; 1763 1764 if (val & ENETC_SIPCAPR0_PSFP) 1765 si->hw_features |= ENETC_SI_F_PSFP; 1766 } 1767 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 1768 1769 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 1770 { 1771 size_t bd_base_size = res->bd_count * res->bd_size; 1772 1773 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 1774 &res->bd_dma_base, GFP_KERNEL); 1775 if (!res->bd_base) 1776 return -ENOMEM; 1777 1778 /* h/w requires 128B alignment */ 1779 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 1780 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1781 res->bd_dma_base); 1782 return -EINVAL; 1783 } 1784 1785 return 0; 1786 } 1787 1788 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 1789 { 1790 size_t bd_base_size = res->bd_count * res->bd_size; 1791 1792 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1793 res->bd_dma_base); 1794 } 1795 1796 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 1797 struct device *dev, size_t bd_count) 1798 { 1799 int err; 1800 1801 res->dev = dev; 1802 res->bd_count = bd_count; 1803 res->bd_size = sizeof(union enetc_tx_bd); 1804 1805 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 1806 if (!res->tx_swbd) 1807 return -ENOMEM; 1808 1809 err = enetc_dma_alloc_bdr(res); 1810 if (err) 1811 goto err_alloc_bdr; 1812 1813 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 1814 &res->tso_headers_dma, 1815 GFP_KERNEL); 1816 if (!res->tso_headers) { 1817 err = -ENOMEM; 1818 goto err_alloc_tso; 1819 } 1820 1821 return 0; 1822 1823 err_alloc_tso: 1824 enetc_dma_free_bdr(res); 1825 err_alloc_bdr: 1826 vfree(res->tx_swbd); 1827 res->tx_swbd = NULL; 1828 1829 return err; 1830 } 1831 1832 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 1833 { 1834 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 1835 res->tso_headers, res->tso_headers_dma); 1836 enetc_dma_free_bdr(res); 1837 vfree(res->tx_swbd); 1838 } 1839 1840 static struct enetc_bdr_resource * 1841 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1842 { 1843 struct enetc_bdr_resource *tx_res; 1844 int i, err; 1845 1846 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 1847 if (!tx_res) 1848 return ERR_PTR(-ENOMEM); 1849 1850 for (i = 0; i < priv->num_tx_rings; i++) { 1851 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 1852 1853 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 1854 tx_ring->bd_count); 1855 if (err) 1856 goto fail; 1857 } 1858 1859 return tx_res; 1860 1861 fail: 1862 while (i-- > 0) 1863 enetc_free_tx_resource(&tx_res[i]); 1864 1865 kfree(tx_res); 1866 1867 return ERR_PTR(err); 1868 } 1869 1870 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 1871 size_t num_resources) 1872 { 1873 size_t i; 1874 1875 for (i = 0; i < num_resources; i++) 1876 enetc_free_tx_resource(&tx_res[i]); 1877 1878 kfree(tx_res); 1879 } 1880 1881 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 1882 struct device *dev, size_t bd_count, 1883 bool extended) 1884 { 1885 int err; 1886 1887 res->dev = dev; 1888 res->bd_count = bd_count; 1889 res->bd_size = sizeof(union enetc_rx_bd); 1890 if (extended) 1891 res->bd_size *= 2; 1892 1893 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 1894 if (!res->rx_swbd) 1895 return -ENOMEM; 1896 1897 err = enetc_dma_alloc_bdr(res); 1898 if (err) { 1899 vfree(res->rx_swbd); 1900 return err; 1901 } 1902 1903 return 0; 1904 } 1905 1906 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 1907 { 1908 enetc_dma_free_bdr(res); 1909 vfree(res->rx_swbd); 1910 } 1911 1912 static struct enetc_bdr_resource * 1913 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 1914 { 1915 struct enetc_bdr_resource *rx_res; 1916 int i, err; 1917 1918 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 1919 if (!rx_res) 1920 return ERR_PTR(-ENOMEM); 1921 1922 for (i = 0; i < priv->num_rx_rings; i++) { 1923 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 1924 1925 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 1926 rx_ring->bd_count, extended); 1927 if (err) 1928 goto fail; 1929 } 1930 1931 return rx_res; 1932 1933 fail: 1934 while (i-- > 0) 1935 enetc_free_rx_resource(&rx_res[i]); 1936 1937 kfree(rx_res); 1938 1939 return ERR_PTR(err); 1940 } 1941 1942 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 1943 size_t num_resources) 1944 { 1945 size_t i; 1946 1947 for (i = 0; i < num_resources; i++) 1948 enetc_free_rx_resource(&rx_res[i]); 1949 1950 kfree(rx_res); 1951 } 1952 1953 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 1954 const struct enetc_bdr_resource *res) 1955 { 1956 tx_ring->bd_base = res ? res->bd_base : NULL; 1957 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1958 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 1959 tx_ring->tso_headers = res ? res->tso_headers : NULL; 1960 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 1961 } 1962 1963 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 1964 const struct enetc_bdr_resource *res) 1965 { 1966 rx_ring->bd_base = res ? res->bd_base : NULL; 1967 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1968 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 1969 } 1970 1971 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 1972 const struct enetc_bdr_resource *res) 1973 { 1974 int i; 1975 1976 if (priv->tx_res) 1977 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 1978 1979 for (i = 0; i < priv->num_tx_rings; i++) { 1980 enetc_assign_tx_resource(priv->tx_ring[i], 1981 res ? &res[i] : NULL); 1982 } 1983 1984 priv->tx_res = res; 1985 } 1986 1987 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 1988 const struct enetc_bdr_resource *res) 1989 { 1990 int i; 1991 1992 if (priv->rx_res) 1993 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 1994 1995 for (i = 0; i < priv->num_rx_rings; i++) { 1996 enetc_assign_rx_resource(priv->rx_ring[i], 1997 res ? &res[i] : NULL); 1998 } 1999 2000 priv->rx_res = res; 2001 } 2002 2003 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2004 { 2005 int i; 2006 2007 for (i = 0; i < tx_ring->bd_count; i++) { 2008 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2009 2010 enetc_free_tx_frame(tx_ring, tx_swbd); 2011 } 2012 } 2013 2014 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2015 { 2016 int i; 2017 2018 for (i = 0; i < rx_ring->bd_count; i++) { 2019 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2020 2021 if (!rx_swbd->page) 2022 continue; 2023 2024 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2025 rx_swbd->dir); 2026 __free_page(rx_swbd->page); 2027 rx_swbd->page = NULL; 2028 } 2029 } 2030 2031 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2032 { 2033 int i; 2034 2035 for (i = 0; i < priv->num_rx_rings; i++) 2036 enetc_free_rx_ring(priv->rx_ring[i]); 2037 2038 for (i = 0; i < priv->num_tx_rings; i++) 2039 enetc_free_tx_ring(priv->tx_ring[i]); 2040 } 2041 2042 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2043 { 2044 int *rss_table; 2045 int i; 2046 2047 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2048 if (!rss_table) 2049 return -ENOMEM; 2050 2051 /* Set up RSS table defaults */ 2052 for (i = 0; i < si->num_rss; i++) 2053 rss_table[i] = i % num_groups; 2054 2055 enetc_set_rss_table(si, rss_table, si->num_rss); 2056 2057 kfree(rss_table); 2058 2059 return 0; 2060 } 2061 2062 int enetc_configure_si(struct enetc_ndev_priv *priv) 2063 { 2064 struct enetc_si *si = priv->si; 2065 struct enetc_hw *hw = &si->hw; 2066 int err; 2067 2068 /* set SI cache attributes */ 2069 enetc_wr(hw, ENETC_SICAR0, 2070 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2071 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2072 /* enable SI */ 2073 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2074 2075 if (si->num_rss) { 2076 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2077 if (err) 2078 return err; 2079 } 2080 2081 return 0; 2082 } 2083 EXPORT_SYMBOL_GPL(enetc_configure_si); 2084 2085 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2086 { 2087 struct enetc_si *si = priv->si; 2088 int cpus = num_online_cpus(); 2089 2090 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2091 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2092 2093 /* Enable all available TX rings in order to configure as many 2094 * priorities as possible, when needed. 2095 * TODO: Make # of TX rings run-time configurable 2096 */ 2097 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2098 priv->num_tx_rings = si->num_tx_rings; 2099 priv->bdr_int_num = cpus; 2100 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2101 priv->tx_ictt = ENETC_TXIC_TIMETHR; 2102 } 2103 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2104 2105 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2106 { 2107 struct enetc_si *si = priv->si; 2108 2109 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2110 GFP_KERNEL); 2111 if (!priv->cls_rules) 2112 return -ENOMEM; 2113 2114 return 0; 2115 } 2116 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2117 2118 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2119 { 2120 kfree(priv->cls_rules); 2121 } 2122 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2123 2124 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2125 { 2126 int idx = tx_ring->index; 2127 u32 tbmr; 2128 2129 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2130 lower_32_bits(tx_ring->bd_dma_base)); 2131 2132 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2133 upper_32_bits(tx_ring->bd_dma_base)); 2134 2135 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2136 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2137 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2138 2139 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2140 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2141 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2142 2143 /* enable Tx ints by setting pkt thr to 1 */ 2144 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2145 2146 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2147 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2148 tbmr |= ENETC_TBMR_VIH; 2149 2150 /* enable ring */ 2151 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2152 2153 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2154 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2155 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2156 } 2157 2158 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2159 bool extended) 2160 { 2161 int idx = rx_ring->index; 2162 u32 rbmr = 0; 2163 2164 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2165 lower_32_bits(rx_ring->bd_dma_base)); 2166 2167 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2168 upper_32_bits(rx_ring->bd_dma_base)); 2169 2170 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2171 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2172 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2173 2174 if (rx_ring->xdp.prog) 2175 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2176 else 2177 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2178 2179 /* Also prepare the consumer index in case page allocation never 2180 * succeeds. In that case, hardware will never advance producer index 2181 * to match consumer index, and will drop all frames. 2182 */ 2183 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2184 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2185 2186 /* enable Rx ints by setting pkt thr to 1 */ 2187 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2188 2189 rx_ring->ext_en = extended; 2190 if (rx_ring->ext_en) 2191 rbmr |= ENETC_RBMR_BDS; 2192 2193 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2194 rbmr |= ENETC_RBMR_VTE; 2195 2196 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2197 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2198 2199 rx_ring->next_to_clean = 0; 2200 rx_ring->next_to_use = 0; 2201 rx_ring->next_to_alloc = 0; 2202 2203 enetc_lock_mdio(); 2204 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2205 enetc_unlock_mdio(); 2206 2207 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2208 } 2209 2210 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2211 { 2212 struct enetc_hw *hw = &priv->si->hw; 2213 int i; 2214 2215 for (i = 0; i < priv->num_tx_rings; i++) 2216 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2217 2218 for (i = 0; i < priv->num_rx_rings; i++) 2219 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2220 } 2221 2222 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2223 { 2224 int idx = tx_ring->index; 2225 u32 tbmr; 2226 2227 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2228 tbmr |= ENETC_TBMR_EN; 2229 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2230 } 2231 2232 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2233 { 2234 int idx = rx_ring->index; 2235 u32 rbmr; 2236 2237 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2238 rbmr |= ENETC_RBMR_EN; 2239 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2240 } 2241 2242 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2243 { 2244 struct enetc_hw *hw = &priv->si->hw; 2245 int i; 2246 2247 for (i = 0; i < priv->num_rx_rings; i++) 2248 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2249 } 2250 2251 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2252 { 2253 struct enetc_hw *hw = &priv->si->hw; 2254 int i; 2255 2256 for (i = 0; i < priv->num_tx_rings; i++) 2257 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2258 } 2259 2260 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2261 { 2262 int idx = rx_ring->index; 2263 2264 /* disable EN bit on ring */ 2265 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2266 } 2267 2268 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2269 { 2270 int idx = rx_ring->index; 2271 2272 /* disable EN bit on ring */ 2273 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2274 } 2275 2276 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2277 { 2278 struct enetc_hw *hw = &priv->si->hw; 2279 int i; 2280 2281 for (i = 0; i < priv->num_rx_rings; i++) 2282 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2283 } 2284 2285 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2286 { 2287 struct enetc_hw *hw = &priv->si->hw; 2288 int i; 2289 2290 for (i = 0; i < priv->num_tx_rings; i++) 2291 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2292 } 2293 2294 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2295 { 2296 int delay = 8, timeout = 100; 2297 int idx = tx_ring->index; 2298 2299 /* wait for busy to clear */ 2300 while (delay < timeout && 2301 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2302 msleep(delay); 2303 delay *= 2; 2304 } 2305 2306 if (delay >= timeout) 2307 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2308 idx); 2309 } 2310 2311 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2312 { 2313 struct enetc_hw *hw = &priv->si->hw; 2314 int i; 2315 2316 for (i = 0; i < priv->num_tx_rings; i++) 2317 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2318 } 2319 2320 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2321 { 2322 struct pci_dev *pdev = priv->si->pdev; 2323 struct enetc_hw *hw = &priv->si->hw; 2324 int i, j, err; 2325 2326 for (i = 0; i < priv->bdr_int_num; i++) { 2327 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2328 struct enetc_int_vector *v = priv->int_vector[i]; 2329 int entry = ENETC_BDR_INT_BASE_IDX + i; 2330 2331 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2332 priv->ndev->name, i); 2333 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2334 if (err) { 2335 dev_err(priv->dev, "request_irq() failed!\n"); 2336 goto irq_err; 2337 } 2338 2339 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2340 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2341 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2342 2343 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2344 2345 for (j = 0; j < v->count_tx_rings; j++) { 2346 int idx = v->tx_ring[j].index; 2347 2348 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2349 } 2350 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2351 } 2352 2353 return 0; 2354 2355 irq_err: 2356 while (i--) { 2357 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2358 2359 irq_set_affinity_hint(irq, NULL); 2360 free_irq(irq, priv->int_vector[i]); 2361 } 2362 2363 return err; 2364 } 2365 2366 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2367 { 2368 struct pci_dev *pdev = priv->si->pdev; 2369 int i; 2370 2371 for (i = 0; i < priv->bdr_int_num; i++) { 2372 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2373 2374 irq_set_affinity_hint(irq, NULL); 2375 free_irq(irq, priv->int_vector[i]); 2376 } 2377 } 2378 2379 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2380 { 2381 struct enetc_hw *hw = &priv->si->hw; 2382 u32 icpt, ictt; 2383 int i; 2384 2385 /* enable Tx & Rx event indication */ 2386 if (priv->ic_mode & 2387 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2388 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2389 /* init to non-0 minimum, will be adjusted later */ 2390 ictt = 0x1; 2391 } else { 2392 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2393 ictt = 0; 2394 } 2395 2396 for (i = 0; i < priv->num_rx_rings; i++) { 2397 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2398 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2399 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2400 } 2401 2402 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2403 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2404 else 2405 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2406 2407 for (i = 0; i < priv->num_tx_rings; i++) { 2408 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2409 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2410 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2411 } 2412 } 2413 2414 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2415 { 2416 struct enetc_hw *hw = &priv->si->hw; 2417 int i; 2418 2419 for (i = 0; i < priv->num_tx_rings; i++) 2420 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2421 2422 for (i = 0; i < priv->num_rx_rings; i++) 2423 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2424 } 2425 2426 static int enetc_phylink_connect(struct net_device *ndev) 2427 { 2428 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2429 struct ethtool_eee edata; 2430 int err; 2431 2432 if (!priv->phylink) { 2433 /* phy-less mode */ 2434 netif_carrier_on(ndev); 2435 return 0; 2436 } 2437 2438 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2439 if (err) { 2440 dev_err(&ndev->dev, "could not attach to PHY\n"); 2441 return err; 2442 } 2443 2444 /* disable EEE autoneg, until ENETC driver supports it */ 2445 memset(&edata, 0, sizeof(struct ethtool_eee)); 2446 phylink_ethtool_set_eee(priv->phylink, &edata); 2447 2448 phylink_start(priv->phylink); 2449 2450 return 0; 2451 } 2452 2453 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2454 { 2455 struct enetc_ndev_priv *priv; 2456 struct sk_buff *skb; 2457 2458 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2459 2460 netif_tx_lock_bh(priv->ndev); 2461 2462 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2463 skb = skb_dequeue(&priv->tx_skbs); 2464 if (skb) 2465 enetc_start_xmit(skb, priv->ndev); 2466 2467 netif_tx_unlock_bh(priv->ndev); 2468 } 2469 2470 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2471 { 2472 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2473 skb_queue_head_init(&priv->tx_skbs); 2474 } 2475 2476 void enetc_start(struct net_device *ndev) 2477 { 2478 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2479 int i; 2480 2481 enetc_setup_interrupts(priv); 2482 2483 for (i = 0; i < priv->bdr_int_num; i++) { 2484 int irq = pci_irq_vector(priv->si->pdev, 2485 ENETC_BDR_INT_BASE_IDX + i); 2486 2487 napi_enable(&priv->int_vector[i]->napi); 2488 enable_irq(irq); 2489 } 2490 2491 enetc_enable_tx_bdrs(priv); 2492 2493 enetc_enable_rx_bdrs(priv); 2494 2495 netif_tx_start_all_queues(ndev); 2496 2497 clear_bit(ENETC_TX_DOWN, &priv->flags); 2498 } 2499 EXPORT_SYMBOL_GPL(enetc_start); 2500 2501 int enetc_open(struct net_device *ndev) 2502 { 2503 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2504 struct enetc_bdr_resource *tx_res, *rx_res; 2505 bool extended; 2506 int err; 2507 2508 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2509 2510 err = enetc_setup_irqs(priv); 2511 if (err) 2512 return err; 2513 2514 err = enetc_phylink_connect(ndev); 2515 if (err) 2516 goto err_phy_connect; 2517 2518 tx_res = enetc_alloc_tx_resources(priv); 2519 if (IS_ERR(tx_res)) { 2520 err = PTR_ERR(tx_res); 2521 goto err_alloc_tx; 2522 } 2523 2524 rx_res = enetc_alloc_rx_resources(priv, extended); 2525 if (IS_ERR(rx_res)) { 2526 err = PTR_ERR(rx_res); 2527 goto err_alloc_rx; 2528 } 2529 2530 enetc_tx_onestep_tstamp_init(priv); 2531 enetc_assign_tx_resources(priv, tx_res); 2532 enetc_assign_rx_resources(priv, rx_res); 2533 enetc_setup_bdrs(priv, extended); 2534 enetc_start(ndev); 2535 2536 return 0; 2537 2538 err_alloc_rx: 2539 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2540 err_alloc_tx: 2541 if (priv->phylink) 2542 phylink_disconnect_phy(priv->phylink); 2543 err_phy_connect: 2544 enetc_free_irqs(priv); 2545 2546 return err; 2547 } 2548 EXPORT_SYMBOL_GPL(enetc_open); 2549 2550 void enetc_stop(struct net_device *ndev) 2551 { 2552 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2553 int i; 2554 2555 set_bit(ENETC_TX_DOWN, &priv->flags); 2556 2557 netif_tx_stop_all_queues(ndev); 2558 2559 enetc_disable_rx_bdrs(priv); 2560 2561 enetc_wait_bdrs(priv); 2562 2563 enetc_disable_tx_bdrs(priv); 2564 2565 for (i = 0; i < priv->bdr_int_num; i++) { 2566 int irq = pci_irq_vector(priv->si->pdev, 2567 ENETC_BDR_INT_BASE_IDX + i); 2568 2569 disable_irq(irq); 2570 napi_synchronize(&priv->int_vector[i]->napi); 2571 napi_disable(&priv->int_vector[i]->napi); 2572 } 2573 2574 enetc_clear_interrupts(priv); 2575 } 2576 EXPORT_SYMBOL_GPL(enetc_stop); 2577 2578 int enetc_close(struct net_device *ndev) 2579 { 2580 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2581 2582 enetc_stop(ndev); 2583 2584 if (priv->phylink) { 2585 phylink_stop(priv->phylink); 2586 phylink_disconnect_phy(priv->phylink); 2587 } else { 2588 netif_carrier_off(ndev); 2589 } 2590 2591 enetc_free_rxtx_rings(priv); 2592 2593 /* Avoids dangling pointers and also frees old resources */ 2594 enetc_assign_rx_resources(priv, NULL); 2595 enetc_assign_tx_resources(priv, NULL); 2596 2597 enetc_free_irqs(priv); 2598 2599 return 0; 2600 } 2601 EXPORT_SYMBOL_GPL(enetc_close); 2602 2603 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 2604 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 2605 void *ctx) 2606 { 2607 struct enetc_bdr_resource *tx_res, *rx_res; 2608 int err; 2609 2610 ASSERT_RTNL(); 2611 2612 /* If the interface is down, run the callback right away, 2613 * without reconfiguration. 2614 */ 2615 if (!netif_running(priv->ndev)) { 2616 if (cb) { 2617 err = cb(priv, ctx); 2618 if (err) 2619 return err; 2620 } 2621 2622 return 0; 2623 } 2624 2625 tx_res = enetc_alloc_tx_resources(priv); 2626 if (IS_ERR(tx_res)) { 2627 err = PTR_ERR(tx_res); 2628 goto out; 2629 } 2630 2631 rx_res = enetc_alloc_rx_resources(priv, extended); 2632 if (IS_ERR(rx_res)) { 2633 err = PTR_ERR(rx_res); 2634 goto out_free_tx_res; 2635 } 2636 2637 enetc_stop(priv->ndev); 2638 enetc_free_rxtx_rings(priv); 2639 2640 /* Interface is down, run optional callback now */ 2641 if (cb) { 2642 err = cb(priv, ctx); 2643 if (err) 2644 goto out_restart; 2645 } 2646 2647 enetc_assign_tx_resources(priv, tx_res); 2648 enetc_assign_rx_resources(priv, rx_res); 2649 enetc_setup_bdrs(priv, extended); 2650 enetc_start(priv->ndev); 2651 2652 return 0; 2653 2654 out_restart: 2655 enetc_setup_bdrs(priv, extended); 2656 enetc_start(priv->ndev); 2657 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 2658 out_free_tx_res: 2659 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2660 out: 2661 return err; 2662 } 2663 2664 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 2665 { 2666 int i; 2667 2668 for (i = 0; i < priv->num_tx_rings; i++) 2669 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 2670 priv->tx_ring[i]->prio); 2671 } 2672 2673 void enetc_reset_tc_mqprio(struct net_device *ndev) 2674 { 2675 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2676 struct enetc_hw *hw = &priv->si->hw; 2677 struct enetc_bdr *tx_ring; 2678 int num_stack_tx_queues; 2679 int i; 2680 2681 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2682 2683 netdev_reset_tc(ndev); 2684 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2685 priv->min_num_stack_tx_queues = num_possible_cpus(); 2686 2687 /* Reset all ring priorities to 0 */ 2688 for (i = 0; i < priv->num_tx_rings; i++) { 2689 tx_ring = priv->tx_ring[i]; 2690 tx_ring->prio = 0; 2691 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2692 } 2693 2694 enetc_debug_tx_ring_prios(priv); 2695 2696 enetc_change_preemptible_tcs(priv, 0); 2697 } 2698 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 2699 2700 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2701 { 2702 struct tc_mqprio_qopt_offload *mqprio = type_data; 2703 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2704 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 2705 struct enetc_hw *hw = &priv->si->hw; 2706 int num_stack_tx_queues = 0; 2707 struct enetc_bdr *tx_ring; 2708 u8 num_tc = qopt->num_tc; 2709 int offset, count; 2710 int err, tc, q; 2711 2712 if (!num_tc) { 2713 enetc_reset_tc_mqprio(ndev); 2714 return 0; 2715 } 2716 2717 err = netdev_set_num_tc(ndev, num_tc); 2718 if (err) 2719 return err; 2720 2721 for (tc = 0; tc < num_tc; tc++) { 2722 offset = qopt->offset[tc]; 2723 count = qopt->count[tc]; 2724 num_stack_tx_queues += count; 2725 2726 err = netdev_set_tc_queue(ndev, tc, count, offset); 2727 if (err) 2728 goto err_reset_tc; 2729 2730 for (q = offset; q < offset + count; q++) { 2731 tx_ring = priv->tx_ring[q]; 2732 /* The prio_tc_map is skb_tx_hash()'s way of selecting 2733 * between TX queues based on skb->priority. As such, 2734 * there's nothing to offload based on it. 2735 * Make the mqprio "traffic class" be the priority of 2736 * this ring group, and leave the Tx IPV to traffic 2737 * class mapping as its default mapping value of 1:1. 2738 */ 2739 tx_ring->prio = tc; 2740 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2741 } 2742 } 2743 2744 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2745 if (err) 2746 goto err_reset_tc; 2747 2748 priv->min_num_stack_tx_queues = num_stack_tx_queues; 2749 2750 enetc_debug_tx_ring_prios(priv); 2751 2752 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 2753 2754 return 0; 2755 2756 err_reset_tc: 2757 enetc_reset_tc_mqprio(ndev); 2758 return err; 2759 } 2760 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 2761 2762 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 2763 { 2764 struct bpf_prog *old_prog, *prog = ctx; 2765 int num_stack_tx_queues; 2766 int err, i; 2767 2768 old_prog = xchg(&priv->xdp_prog, prog); 2769 2770 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2771 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 2772 if (err) { 2773 xchg(&priv->xdp_prog, old_prog); 2774 return err; 2775 } 2776 2777 if (old_prog) 2778 bpf_prog_put(old_prog); 2779 2780 for (i = 0; i < priv->num_rx_rings; i++) { 2781 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2782 2783 rx_ring->xdp.prog = prog; 2784 2785 if (prog) 2786 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2787 else 2788 rx_ring->buffer_offset = ENETC_RXB_PAD; 2789 } 2790 2791 return 0; 2792 } 2793 2794 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 2795 struct netlink_ext_ack *extack) 2796 { 2797 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 2798 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2799 bool extended; 2800 2801 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 2802 priv->num_tx_rings) { 2803 NL_SET_ERR_MSG_FMT_MOD(extack, 2804 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 2805 num_xdp_tx_queues, 2806 priv->min_num_stack_tx_queues, 2807 priv->num_tx_rings); 2808 return -EBUSY; 2809 } 2810 2811 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2812 2813 /* The buffer layout is changing, so we need to drain the old 2814 * RX buffers and seed new ones. 2815 */ 2816 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 2817 } 2818 2819 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 2820 { 2821 switch (bpf->command) { 2822 case XDP_SETUP_PROG: 2823 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 2824 default: 2825 return -EINVAL; 2826 } 2827 2828 return 0; 2829 } 2830 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 2831 2832 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2833 { 2834 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2835 struct net_device_stats *stats = &ndev->stats; 2836 unsigned long packets = 0, bytes = 0; 2837 unsigned long tx_dropped = 0; 2838 int i; 2839 2840 for (i = 0; i < priv->num_rx_rings; i++) { 2841 packets += priv->rx_ring[i]->stats.packets; 2842 bytes += priv->rx_ring[i]->stats.bytes; 2843 } 2844 2845 stats->rx_packets = packets; 2846 stats->rx_bytes = bytes; 2847 bytes = 0; 2848 packets = 0; 2849 2850 for (i = 0; i < priv->num_tx_rings; i++) { 2851 packets += priv->tx_ring[i]->stats.packets; 2852 bytes += priv->tx_ring[i]->stats.bytes; 2853 tx_dropped += priv->tx_ring[i]->stats.win_drop; 2854 } 2855 2856 stats->tx_packets = packets; 2857 stats->tx_bytes = bytes; 2858 stats->tx_dropped = tx_dropped; 2859 2860 return stats; 2861 } 2862 EXPORT_SYMBOL_GPL(enetc_get_stats); 2863 2864 static int enetc_set_rss(struct net_device *ndev, int en) 2865 { 2866 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2867 struct enetc_hw *hw = &priv->si->hw; 2868 u32 reg; 2869 2870 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2871 2872 reg = enetc_rd(hw, ENETC_SIMR); 2873 reg &= ~ENETC_SIMR_RSSE; 2874 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2875 enetc_wr(hw, ENETC_SIMR, reg); 2876 2877 return 0; 2878 } 2879 2880 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2881 { 2882 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2883 struct enetc_hw *hw = &priv->si->hw; 2884 int i; 2885 2886 for (i = 0; i < priv->num_rx_rings; i++) 2887 enetc_bdr_enable_rxvlan(hw, i, en); 2888 } 2889 2890 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2891 { 2892 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2893 struct enetc_hw *hw = &priv->si->hw; 2894 int i; 2895 2896 for (i = 0; i < priv->num_tx_rings; i++) 2897 enetc_bdr_enable_txvlan(hw, i, en); 2898 } 2899 2900 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 2901 { 2902 netdev_features_t changed = ndev->features ^ features; 2903 2904 if (changed & NETIF_F_RXHASH) 2905 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2906 2907 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2908 enetc_enable_rxvlan(ndev, 2909 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2910 2911 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2912 enetc_enable_txvlan(ndev, 2913 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2914 } 2915 EXPORT_SYMBOL_GPL(enetc_set_features); 2916 2917 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2918 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2919 { 2920 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2921 int err, new_offloads = priv->active_offloads; 2922 struct hwtstamp_config config; 2923 2924 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2925 return -EFAULT; 2926 2927 switch (config.tx_type) { 2928 case HWTSTAMP_TX_OFF: 2929 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2930 break; 2931 case HWTSTAMP_TX_ON: 2932 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2933 new_offloads |= ENETC_F_TX_TSTAMP; 2934 break; 2935 case HWTSTAMP_TX_ONESTEP_SYNC: 2936 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2937 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2938 break; 2939 default: 2940 return -ERANGE; 2941 } 2942 2943 switch (config.rx_filter) { 2944 case HWTSTAMP_FILTER_NONE: 2945 new_offloads &= ~ENETC_F_RX_TSTAMP; 2946 break; 2947 default: 2948 new_offloads |= ENETC_F_RX_TSTAMP; 2949 config.rx_filter = HWTSTAMP_FILTER_ALL; 2950 } 2951 2952 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 2953 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 2954 2955 err = enetc_reconfigure(priv, extended, NULL, NULL); 2956 if (err) 2957 return err; 2958 } 2959 2960 priv->active_offloads = new_offloads; 2961 2962 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2963 -EFAULT : 0; 2964 } 2965 2966 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2967 { 2968 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2969 struct hwtstamp_config config; 2970 2971 config.flags = 0; 2972 2973 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2974 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2975 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2976 config.tx_type = HWTSTAMP_TX_ON; 2977 else 2978 config.tx_type = HWTSTAMP_TX_OFF; 2979 2980 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2981 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2982 2983 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2984 -EFAULT : 0; 2985 } 2986 #endif 2987 2988 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2989 { 2990 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2991 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2992 if (cmd == SIOCSHWTSTAMP) 2993 return enetc_hwtstamp_set(ndev, rq); 2994 if (cmd == SIOCGHWTSTAMP) 2995 return enetc_hwtstamp_get(ndev, rq); 2996 #endif 2997 2998 if (!priv->phylink) 2999 return -EOPNOTSUPP; 3000 3001 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3002 } 3003 EXPORT_SYMBOL_GPL(enetc_ioctl); 3004 3005 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3006 { 3007 struct pci_dev *pdev = priv->si->pdev; 3008 int num_stack_tx_queues; 3009 int first_xdp_tx_ring; 3010 int i, n, err, nvec; 3011 int v_tx_rings; 3012 3013 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3014 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3015 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3016 3017 if (n < 0) 3018 return n; 3019 3020 if (n != nvec) 3021 return -EPERM; 3022 3023 /* # of tx rings per int vector */ 3024 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3025 3026 for (i = 0; i < priv->bdr_int_num; i++) { 3027 struct enetc_int_vector *v; 3028 struct enetc_bdr *bdr; 3029 int j; 3030 3031 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3032 if (!v) { 3033 err = -ENOMEM; 3034 goto fail; 3035 } 3036 3037 priv->int_vector[i] = v; 3038 3039 bdr = &v->rx_ring; 3040 bdr->index = i; 3041 bdr->ndev = priv->ndev; 3042 bdr->dev = priv->dev; 3043 bdr->bd_count = priv->rx_bd_count; 3044 bdr->buffer_offset = ENETC_RXB_PAD; 3045 priv->rx_ring[i] = bdr; 3046 3047 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 3048 if (err) { 3049 kfree(v); 3050 goto fail; 3051 } 3052 3053 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, 3054 MEM_TYPE_PAGE_SHARED, NULL); 3055 if (err) { 3056 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3057 kfree(v); 3058 goto fail; 3059 } 3060 3061 /* init defaults for adaptive IC */ 3062 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3063 v->rx_ictt = 0x1; 3064 v->rx_dim_en = true; 3065 } 3066 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3067 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3068 v->count_tx_rings = v_tx_rings; 3069 3070 for (j = 0; j < v_tx_rings; j++) { 3071 int idx; 3072 3073 /* default tx ring mapping policy */ 3074 idx = priv->bdr_int_num * j + i; 3075 __set_bit(idx, &v->tx_rings_map); 3076 bdr = &v->tx_ring[j]; 3077 bdr->index = idx; 3078 bdr->ndev = priv->ndev; 3079 bdr->dev = priv->dev; 3080 bdr->bd_count = priv->tx_bd_count; 3081 priv->tx_ring[idx] = bdr; 3082 } 3083 } 3084 3085 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3086 3087 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3088 if (err) 3089 goto fail; 3090 3091 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3092 if (err) 3093 goto fail; 3094 3095 priv->min_num_stack_tx_queues = num_possible_cpus(); 3096 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3097 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3098 3099 return 0; 3100 3101 fail: 3102 while (i--) { 3103 struct enetc_int_vector *v = priv->int_vector[i]; 3104 struct enetc_bdr *rx_ring = &v->rx_ring; 3105 3106 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3107 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3108 netif_napi_del(&v->napi); 3109 cancel_work_sync(&v->rx_dim.work); 3110 kfree(v); 3111 } 3112 3113 pci_free_irq_vectors(pdev); 3114 3115 return err; 3116 } 3117 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3118 3119 void enetc_free_msix(struct enetc_ndev_priv *priv) 3120 { 3121 int i; 3122 3123 for (i = 0; i < priv->bdr_int_num; i++) { 3124 struct enetc_int_vector *v = priv->int_vector[i]; 3125 struct enetc_bdr *rx_ring = &v->rx_ring; 3126 3127 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3128 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3129 netif_napi_del(&v->napi); 3130 cancel_work_sync(&v->rx_dim.work); 3131 } 3132 3133 for (i = 0; i < priv->num_rx_rings; i++) 3134 priv->rx_ring[i] = NULL; 3135 3136 for (i = 0; i < priv->num_tx_rings; i++) 3137 priv->tx_ring[i] = NULL; 3138 3139 for (i = 0; i < priv->bdr_int_num; i++) { 3140 kfree(priv->int_vector[i]); 3141 priv->int_vector[i] = NULL; 3142 } 3143 3144 /* disable all MSIX for this device */ 3145 pci_free_irq_vectors(priv->si->pdev); 3146 } 3147 EXPORT_SYMBOL_GPL(enetc_free_msix); 3148 3149 static void enetc_kfree_si(struct enetc_si *si) 3150 { 3151 char *p = (char *)si - si->pad; 3152 3153 kfree(p); 3154 } 3155 3156 static void enetc_detect_errata(struct enetc_si *si) 3157 { 3158 if (si->pdev->revision == ENETC_REV1) 3159 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3160 } 3161 3162 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3163 { 3164 struct enetc_si *si, *p; 3165 struct enetc_hw *hw; 3166 size_t alloc_size; 3167 int err, len; 3168 3169 pcie_flr(pdev); 3170 err = pci_enable_device_mem(pdev); 3171 if (err) 3172 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3173 3174 /* set up for high or low dma */ 3175 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3176 if (err) { 3177 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3178 goto err_dma; 3179 } 3180 3181 err = pci_request_mem_regions(pdev, name); 3182 if (err) { 3183 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3184 goto err_pci_mem_reg; 3185 } 3186 3187 pci_set_master(pdev); 3188 3189 alloc_size = sizeof(struct enetc_si); 3190 if (sizeof_priv) { 3191 /* align priv to 32B */ 3192 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3193 alloc_size += sizeof_priv; 3194 } 3195 /* force 32B alignment for enetc_si */ 3196 alloc_size += ENETC_SI_ALIGN - 1; 3197 3198 p = kzalloc(alloc_size, GFP_KERNEL); 3199 if (!p) { 3200 err = -ENOMEM; 3201 goto err_alloc_si; 3202 } 3203 3204 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3205 si->pad = (char *)si - (char *)p; 3206 3207 pci_set_drvdata(pdev, si); 3208 si->pdev = pdev; 3209 hw = &si->hw; 3210 3211 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3212 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3213 if (!hw->reg) { 3214 err = -ENXIO; 3215 dev_err(&pdev->dev, "ioremap() failed\n"); 3216 goto err_ioremap; 3217 } 3218 if (len > ENETC_PORT_BASE) 3219 hw->port = hw->reg + ENETC_PORT_BASE; 3220 if (len > ENETC_GLOBAL_BASE) 3221 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3222 3223 enetc_detect_errata(si); 3224 3225 return 0; 3226 3227 err_ioremap: 3228 enetc_kfree_si(si); 3229 err_alloc_si: 3230 pci_release_mem_regions(pdev); 3231 err_pci_mem_reg: 3232 err_dma: 3233 pci_disable_device(pdev); 3234 3235 return err; 3236 } 3237 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3238 3239 void enetc_pci_remove(struct pci_dev *pdev) 3240 { 3241 struct enetc_si *si = pci_get_drvdata(pdev); 3242 struct enetc_hw *hw = &si->hw; 3243 3244 iounmap(hw->reg); 3245 enetc_kfree_si(si); 3246 pci_release_mem_regions(pdev); 3247 pci_disable_device(pdev); 3248 } 3249 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3250 3251 MODULE_LICENSE("Dual BSD/GPL"); 3252