1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/tcp.h> 7 #include <linux/udp.h> 8 #include <linux/vmalloc.h> 9 #include <linux/ptp_classify.h> 10 #include <net/ip6_checksum.h> 11 #include <net/pkt_sched.h> 12 #include <net/tso.h> 13 14 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 15 { 16 int num_tx_rings = priv->num_tx_rings; 17 int i; 18 19 for (i = 0; i < priv->num_rx_rings; i++) 20 if (priv->rx_ring[i]->xdp.prog) 21 return num_tx_rings - num_possible_cpus(); 22 23 return num_tx_rings; 24 } 25 26 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 27 struct enetc_bdr *tx_ring) 28 { 29 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 30 31 return priv->rx_ring[index]; 32 } 33 34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 35 { 36 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 37 return NULL; 38 39 return tx_swbd->skb; 40 } 41 42 static struct xdp_frame * 43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 44 { 45 if (tx_swbd->is_xdp_redirect) 46 return tx_swbd->xdp_frame; 47 48 return NULL; 49 } 50 51 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 52 struct enetc_tx_swbd *tx_swbd) 53 { 54 /* For XDP_TX, pages come from RX, whereas for the other contexts where 55 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 56 * to match the DMA mapping length, so we need to differentiate those. 57 */ 58 if (tx_swbd->is_dma_page) 59 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 60 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 61 tx_swbd->dir); 62 else 63 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 64 tx_swbd->len, tx_swbd->dir); 65 tx_swbd->dma = 0; 66 } 67 68 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 69 struct enetc_tx_swbd *tx_swbd) 70 { 71 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 72 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 73 74 if (tx_swbd->dma) 75 enetc_unmap_tx_buff(tx_ring, tx_swbd); 76 77 if (xdp_frame) { 78 xdp_return_frame(tx_swbd->xdp_frame); 79 tx_swbd->xdp_frame = NULL; 80 } else if (skb) { 81 dev_kfree_skb_any(skb); 82 tx_swbd->skb = NULL; 83 } 84 } 85 86 /* Let H/W know BD ring has been updated */ 87 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 88 { 89 /* includes wmb() */ 90 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 91 } 92 93 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 94 u8 *msgtype, u8 *twostep, 95 u16 *correction_offset, u16 *body_offset) 96 { 97 unsigned int ptp_class; 98 struct ptp_header *hdr; 99 unsigned int type; 100 u8 *base; 101 102 ptp_class = ptp_classify_raw(skb); 103 if (ptp_class == PTP_CLASS_NONE) 104 return -EINVAL; 105 106 hdr = ptp_parse_header(skb, ptp_class); 107 if (!hdr) 108 return -EINVAL; 109 110 type = ptp_class & PTP_CLASS_PMASK; 111 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 112 *udp = 1; 113 else 114 *udp = 0; 115 116 *msgtype = ptp_get_msgtype(hdr, ptp_class); 117 *twostep = hdr->flag_field[0] & 0x2; 118 119 base = skb_mac_header(skb); 120 *correction_offset = (u8 *)&hdr->correction - base; 121 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 122 123 return 0; 124 } 125 126 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 127 { 128 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 129 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 130 struct enetc_hw *hw = &priv->si->hw; 131 struct enetc_tx_swbd *tx_swbd; 132 int len = skb_headlen(skb); 133 union enetc_tx_bd temp_bd; 134 u8 msgtype, twostep, udp; 135 union enetc_tx_bd *txbd; 136 u16 offset1, offset2; 137 int i, count = 0; 138 skb_frag_t *frag; 139 unsigned int f; 140 dma_addr_t dma; 141 u8 flags = 0; 142 143 i = tx_ring->next_to_use; 144 txbd = ENETC_TXBD(*tx_ring, i); 145 prefetchw(txbd); 146 147 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 148 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 149 goto dma_err; 150 151 temp_bd.addr = cpu_to_le64(dma); 152 temp_bd.buf_len = cpu_to_le16(len); 153 temp_bd.lstatus = 0; 154 155 tx_swbd = &tx_ring->tx_swbd[i]; 156 tx_swbd->dma = dma; 157 tx_swbd->len = len; 158 tx_swbd->is_dma_page = 0; 159 tx_swbd->dir = DMA_TO_DEVICE; 160 count++; 161 162 do_vlan = skb_vlan_tag_present(skb); 163 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 164 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 165 &offset2) || 166 msgtype != PTP_MSGTYPE_SYNC || twostep) 167 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 168 else 169 do_onestep_tstamp = true; 170 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 171 do_twostep_tstamp = true; 172 } 173 174 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 175 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 176 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 177 178 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 179 flags |= ENETC_TXBD_FLAGS_EX; 180 181 if (tx_ring->tsd_enable) 182 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 183 184 /* first BD needs frm_len and offload flags set */ 185 temp_bd.frm_len = cpu_to_le16(skb->len); 186 temp_bd.flags = flags; 187 188 if (flags & ENETC_TXBD_FLAGS_TSE) 189 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 190 flags); 191 192 if (flags & ENETC_TXBD_FLAGS_EX) { 193 u8 e_flags = 0; 194 *txbd = temp_bd; 195 enetc_clear_tx_bd(&temp_bd); 196 197 /* add extension BD for VLAN and/or timestamping */ 198 flags = 0; 199 tx_swbd++; 200 txbd++; 201 i++; 202 if (unlikely(i == tx_ring->bd_count)) { 203 i = 0; 204 tx_swbd = tx_ring->tx_swbd; 205 txbd = ENETC_TXBD(*tx_ring, 0); 206 } 207 prefetchw(txbd); 208 209 if (do_vlan) { 210 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 211 temp_bd.ext.tpid = 0; /* < C-TAG */ 212 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 213 } 214 215 if (do_onestep_tstamp) { 216 u32 lo, hi, val; 217 u64 sec, nsec; 218 u8 *data; 219 220 lo = enetc_rd_hot(hw, ENETC_SICTR0); 221 hi = enetc_rd_hot(hw, ENETC_SICTR1); 222 sec = (u64)hi << 32 | lo; 223 nsec = do_div(sec, 1000000000); 224 225 /* Configure extension BD */ 226 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 227 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 228 229 /* Update originTimestamp field of Sync packet 230 * - 48 bits seconds field 231 * - 32 bits nanseconds field 232 */ 233 data = skb_mac_header(skb); 234 *(__be16 *)(data + offset2) = 235 htons((sec >> 32) & 0xffff); 236 *(__be32 *)(data + offset2 + 2) = 237 htonl(sec & 0xffffffff); 238 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 239 240 /* Configure single-step register */ 241 val = ENETC_PM0_SINGLE_STEP_EN; 242 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 243 if (udp) 244 val |= ENETC_PM0_SINGLE_STEP_CH; 245 246 enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val); 247 enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val); 248 } else if (do_twostep_tstamp) { 249 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 250 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 251 } 252 253 temp_bd.ext.e_flags = e_flags; 254 count++; 255 } 256 257 frag = &skb_shinfo(skb)->frags[0]; 258 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 259 len = skb_frag_size(frag); 260 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 261 DMA_TO_DEVICE); 262 if (dma_mapping_error(tx_ring->dev, dma)) 263 goto dma_err; 264 265 *txbd = temp_bd; 266 enetc_clear_tx_bd(&temp_bd); 267 268 flags = 0; 269 tx_swbd++; 270 txbd++; 271 i++; 272 if (unlikely(i == tx_ring->bd_count)) { 273 i = 0; 274 tx_swbd = tx_ring->tx_swbd; 275 txbd = ENETC_TXBD(*tx_ring, 0); 276 } 277 prefetchw(txbd); 278 279 temp_bd.addr = cpu_to_le64(dma); 280 temp_bd.buf_len = cpu_to_le16(len); 281 282 tx_swbd->dma = dma; 283 tx_swbd->len = len; 284 tx_swbd->is_dma_page = 1; 285 tx_swbd->dir = DMA_TO_DEVICE; 286 count++; 287 } 288 289 /* last BD needs 'F' bit set */ 290 flags |= ENETC_TXBD_FLAGS_F; 291 temp_bd.flags = flags; 292 *txbd = temp_bd; 293 294 tx_ring->tx_swbd[i].is_eof = true; 295 tx_ring->tx_swbd[i].skb = skb; 296 297 enetc_bdr_idx_inc(tx_ring, &i); 298 tx_ring->next_to_use = i; 299 300 skb_tx_timestamp(skb); 301 302 enetc_update_tx_ring_tail(tx_ring); 303 304 return count; 305 306 dma_err: 307 dev_err(tx_ring->dev, "DMA map error"); 308 309 do { 310 tx_swbd = &tx_ring->tx_swbd[i]; 311 enetc_free_tx_frame(tx_ring, tx_swbd); 312 if (i == 0) 313 i = tx_ring->bd_count; 314 i--; 315 } while (count--); 316 317 return 0; 318 } 319 320 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 321 struct enetc_tx_swbd *tx_swbd, 322 union enetc_tx_bd *txbd, int *i, int hdr_len, 323 int data_len) 324 { 325 union enetc_tx_bd txbd_tmp; 326 u8 flags = 0, e_flags = 0; 327 dma_addr_t addr; 328 329 enetc_clear_tx_bd(&txbd_tmp); 330 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 331 332 if (skb_vlan_tag_present(skb)) 333 flags |= ENETC_TXBD_FLAGS_EX; 334 335 txbd_tmp.addr = cpu_to_le64(addr); 336 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 337 338 /* first BD needs frm_len and offload flags set */ 339 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 340 txbd_tmp.flags = flags; 341 342 /* For the TSO header we do not set the dma address since we do not 343 * want it unmapped when we do cleanup. We still set len so that we 344 * count the bytes sent. 345 */ 346 tx_swbd->len = hdr_len; 347 tx_swbd->do_twostep_tstamp = false; 348 tx_swbd->check_wb = false; 349 350 /* Actually write the header in the BD */ 351 *txbd = txbd_tmp; 352 353 /* Add extension BD for VLAN */ 354 if (flags & ENETC_TXBD_FLAGS_EX) { 355 /* Get the next BD */ 356 enetc_bdr_idx_inc(tx_ring, i); 357 txbd = ENETC_TXBD(*tx_ring, *i); 358 tx_swbd = &tx_ring->tx_swbd[*i]; 359 prefetchw(txbd); 360 361 /* Setup the VLAN fields */ 362 enetc_clear_tx_bd(&txbd_tmp); 363 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 364 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 365 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 366 367 /* Write the BD */ 368 txbd_tmp.ext.e_flags = e_flags; 369 *txbd = txbd_tmp; 370 } 371 } 372 373 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 374 struct enetc_tx_swbd *tx_swbd, 375 union enetc_tx_bd *txbd, char *data, 376 int size, bool last_bd) 377 { 378 union enetc_tx_bd txbd_tmp; 379 dma_addr_t addr; 380 u8 flags = 0; 381 382 enetc_clear_tx_bd(&txbd_tmp); 383 384 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 385 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 386 netdev_err(tx_ring->ndev, "DMA map error\n"); 387 return -ENOMEM; 388 } 389 390 if (last_bd) { 391 flags |= ENETC_TXBD_FLAGS_F; 392 tx_swbd->is_eof = 1; 393 } 394 395 txbd_tmp.addr = cpu_to_le64(addr); 396 txbd_tmp.buf_len = cpu_to_le16(size); 397 txbd_tmp.flags = flags; 398 399 tx_swbd->dma = addr; 400 tx_swbd->len = size; 401 tx_swbd->dir = DMA_TO_DEVICE; 402 403 *txbd = txbd_tmp; 404 405 return 0; 406 } 407 408 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 409 char *hdr, int hdr_len, int *l4_hdr_len) 410 { 411 char *l4_hdr = hdr + skb_transport_offset(skb); 412 int mac_hdr_len = skb_network_offset(skb); 413 414 if (tso->tlen != sizeof(struct udphdr)) { 415 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 416 417 tcph->check = 0; 418 } else { 419 struct udphdr *udph = (struct udphdr *)(l4_hdr); 420 421 udph->check = 0; 422 } 423 424 /* Compute the IP checksum. This is necessary since tso_build_hdr() 425 * already incremented the IP ID field. 426 */ 427 if (!tso->ipv6) { 428 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 429 430 iph->check = 0; 431 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 432 } 433 434 /* Compute the checksum over the L4 header. */ 435 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 436 return csum_partial(l4_hdr, *l4_hdr_len, 0); 437 } 438 439 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 440 struct sk_buff *skb, char *hdr, int len, 441 __wsum sum) 442 { 443 char *l4_hdr = hdr + skb_transport_offset(skb); 444 __sum16 csum_final; 445 446 /* Complete the L4 checksum by appending the pseudo-header to the 447 * already computed checksum. 448 */ 449 if (!tso->ipv6) 450 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 451 ip_hdr(skb)->daddr, 452 len, ip_hdr(skb)->protocol, sum); 453 else 454 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 455 &ipv6_hdr(skb)->daddr, 456 len, ipv6_hdr(skb)->nexthdr, sum); 457 458 if (tso->tlen != sizeof(struct udphdr)) { 459 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 460 461 tcph->check = csum_final; 462 } else { 463 struct udphdr *udph = (struct udphdr *)(l4_hdr); 464 465 udph->check = csum_final; 466 } 467 } 468 469 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 470 { 471 int hdr_len, total_len, data_len; 472 struct enetc_tx_swbd *tx_swbd; 473 union enetc_tx_bd *txbd; 474 struct tso_t tso; 475 __wsum csum, csum2; 476 int count = 0, pos; 477 int err, i, bd_data_num; 478 479 /* Initialize the TSO handler, and prepare the first payload */ 480 hdr_len = tso_start(skb, &tso); 481 total_len = skb->len - hdr_len; 482 i = tx_ring->next_to_use; 483 484 while (total_len > 0) { 485 char *hdr; 486 487 /* Get the BD */ 488 txbd = ENETC_TXBD(*tx_ring, i); 489 tx_swbd = &tx_ring->tx_swbd[i]; 490 prefetchw(txbd); 491 492 /* Determine the length of this packet */ 493 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 494 total_len -= data_len; 495 496 /* prepare packet headers: MAC + IP + TCP */ 497 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 498 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 499 500 /* compute the csum over the L4 header */ 501 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 502 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 503 bd_data_num = 0; 504 count++; 505 506 while (data_len > 0) { 507 int size; 508 509 size = min_t(int, tso.size, data_len); 510 511 /* Advance the index in the BDR */ 512 enetc_bdr_idx_inc(tx_ring, &i); 513 txbd = ENETC_TXBD(*tx_ring, i); 514 tx_swbd = &tx_ring->tx_swbd[i]; 515 prefetchw(txbd); 516 517 /* Compute the checksum over this segment of data and 518 * add it to the csum already computed (over the L4 519 * header and possible other data segments). 520 */ 521 csum2 = csum_partial(tso.data, size, 0); 522 csum = csum_block_add(csum, csum2, pos); 523 pos += size; 524 525 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 526 tso.data, size, 527 size == data_len); 528 if (err) 529 goto err_map_data; 530 531 data_len -= size; 532 count++; 533 bd_data_num++; 534 tso_build_data(skb, &tso, size); 535 536 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 537 goto err_chained_bd; 538 } 539 540 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 541 542 if (total_len == 0) 543 tx_swbd->skb = skb; 544 545 /* Go to the next BD */ 546 enetc_bdr_idx_inc(tx_ring, &i); 547 } 548 549 tx_ring->next_to_use = i; 550 enetc_update_tx_ring_tail(tx_ring); 551 552 return count; 553 554 err_map_data: 555 dev_err(tx_ring->dev, "DMA map error"); 556 557 err_chained_bd: 558 do { 559 tx_swbd = &tx_ring->tx_swbd[i]; 560 enetc_free_tx_frame(tx_ring, tx_swbd); 561 if (i == 0) 562 i = tx_ring->bd_count; 563 i--; 564 } while (count--); 565 566 return 0; 567 } 568 569 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 570 struct net_device *ndev) 571 { 572 struct enetc_ndev_priv *priv = netdev_priv(ndev); 573 struct enetc_bdr *tx_ring; 574 int count, err; 575 576 /* Queue one-step Sync packet if already locked */ 577 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 578 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 579 &priv->flags)) { 580 skb_queue_tail(&priv->tx_skbs, skb); 581 return NETDEV_TX_OK; 582 } 583 } 584 585 tx_ring = priv->tx_ring[skb->queue_mapping]; 586 587 if (skb_is_gso(skb)) { 588 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 589 netif_stop_subqueue(ndev, tx_ring->index); 590 return NETDEV_TX_BUSY; 591 } 592 593 enetc_lock_mdio(); 594 count = enetc_map_tx_tso_buffs(tx_ring, skb); 595 enetc_unlock_mdio(); 596 } else { 597 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 598 if (unlikely(skb_linearize(skb))) 599 goto drop_packet_err; 600 601 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 602 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 603 netif_stop_subqueue(ndev, tx_ring->index); 604 return NETDEV_TX_BUSY; 605 } 606 607 if (skb->ip_summed == CHECKSUM_PARTIAL) { 608 err = skb_checksum_help(skb); 609 if (err) 610 goto drop_packet_err; 611 } 612 enetc_lock_mdio(); 613 count = enetc_map_tx_buffs(tx_ring, skb); 614 enetc_unlock_mdio(); 615 } 616 617 if (unlikely(!count)) 618 goto drop_packet_err; 619 620 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 621 netif_stop_subqueue(ndev, tx_ring->index); 622 623 return NETDEV_TX_OK; 624 625 drop_packet_err: 626 dev_kfree_skb_any(skb); 627 return NETDEV_TX_OK; 628 } 629 630 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 631 { 632 struct enetc_ndev_priv *priv = netdev_priv(ndev); 633 u8 udp, msgtype, twostep; 634 u16 offset1, offset2; 635 636 /* Mark tx timestamp type on skb->cb[0] if requires */ 637 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 638 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 639 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 640 } else { 641 skb->cb[0] = 0; 642 } 643 644 /* Fall back to two-step timestamp if not one-step Sync packet */ 645 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 646 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 647 &offset1, &offset2) || 648 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 649 skb->cb[0] = ENETC_F_TX_TSTAMP; 650 } 651 652 return enetc_start_xmit(skb, ndev); 653 } 654 655 static irqreturn_t enetc_msix(int irq, void *data) 656 { 657 struct enetc_int_vector *v = data; 658 int i; 659 660 enetc_lock_mdio(); 661 662 /* disable interrupts */ 663 enetc_wr_reg_hot(v->rbier, 0); 664 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 665 666 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 667 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 668 669 enetc_unlock_mdio(); 670 671 napi_schedule(&v->napi); 672 673 return IRQ_HANDLED; 674 } 675 676 static void enetc_rx_dim_work(struct work_struct *w) 677 { 678 struct dim *dim = container_of(w, struct dim, work); 679 struct dim_cq_moder moder = 680 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 681 struct enetc_int_vector *v = 682 container_of(dim, struct enetc_int_vector, rx_dim); 683 684 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 685 dim->state = DIM_START_MEASURE; 686 } 687 688 static void enetc_rx_net_dim(struct enetc_int_vector *v) 689 { 690 struct dim_sample dim_sample = {}; 691 692 v->comp_cnt++; 693 694 if (!v->rx_napi_work) 695 return; 696 697 dim_update_sample(v->comp_cnt, 698 v->rx_ring.stats.packets, 699 v->rx_ring.stats.bytes, 700 &dim_sample); 701 net_dim(&v->rx_dim, dim_sample); 702 } 703 704 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 705 { 706 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 707 708 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 709 } 710 711 static bool enetc_page_reusable(struct page *page) 712 { 713 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 714 } 715 716 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 717 struct enetc_rx_swbd *old) 718 { 719 struct enetc_rx_swbd *new; 720 721 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 722 723 /* next buf that may reuse a page */ 724 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 725 726 /* copy page reference */ 727 *new = *old; 728 } 729 730 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 731 u64 *tstamp) 732 { 733 u32 lo, hi, tstamp_lo; 734 735 lo = enetc_rd_hot(hw, ENETC_SICTR0); 736 hi = enetc_rd_hot(hw, ENETC_SICTR1); 737 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 738 if (lo <= tstamp_lo) 739 hi -= 1; 740 *tstamp = (u64)hi << 32 | tstamp_lo; 741 } 742 743 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 744 { 745 struct skb_shared_hwtstamps shhwtstamps; 746 747 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 748 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 749 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 750 skb_txtime_consumed(skb); 751 skb_tstamp_tx(skb, &shhwtstamps); 752 } 753 } 754 755 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 756 struct enetc_tx_swbd *tx_swbd) 757 { 758 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 759 struct enetc_rx_swbd rx_swbd = { 760 .dma = tx_swbd->dma, 761 .page = tx_swbd->page, 762 .page_offset = tx_swbd->page_offset, 763 .dir = tx_swbd->dir, 764 .len = tx_swbd->len, 765 }; 766 struct enetc_bdr *rx_ring; 767 768 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 769 770 if (likely(enetc_swbd_unused(rx_ring))) { 771 enetc_reuse_page(rx_ring, &rx_swbd); 772 773 /* sync for use by the device */ 774 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 775 rx_swbd.page_offset, 776 ENETC_RXB_DMA_SIZE_XDP, 777 rx_swbd.dir); 778 779 rx_ring->stats.recycles++; 780 } else { 781 /* RX ring is already full, we need to unmap and free the 782 * page, since there's nothing useful we can do with it. 783 */ 784 rx_ring->stats.recycle_failures++; 785 786 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 787 rx_swbd.dir); 788 __free_page(rx_swbd.page); 789 } 790 791 rx_ring->xdp.xdp_tx_in_flight--; 792 } 793 794 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 795 { 796 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 797 struct net_device *ndev = tx_ring->ndev; 798 struct enetc_ndev_priv *priv = netdev_priv(ndev); 799 struct enetc_tx_swbd *tx_swbd; 800 int i, bds_to_clean; 801 bool do_twostep_tstamp; 802 u64 tstamp = 0; 803 804 i = tx_ring->next_to_clean; 805 tx_swbd = &tx_ring->tx_swbd[i]; 806 807 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 808 809 do_twostep_tstamp = false; 810 811 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 812 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 813 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 814 bool is_eof = tx_swbd->is_eof; 815 816 if (unlikely(tx_swbd->check_wb)) { 817 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 818 819 if (txbd->flags & ENETC_TXBD_FLAGS_W && 820 tx_swbd->do_twostep_tstamp) { 821 enetc_get_tx_tstamp(&priv->si->hw, txbd, 822 &tstamp); 823 do_twostep_tstamp = true; 824 } 825 826 if (tx_swbd->qbv_en && 827 txbd->wb.status & ENETC_TXBD_STATS_WIN) 828 tx_win_drop++; 829 } 830 831 if (tx_swbd->is_xdp_tx) 832 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 833 else if (likely(tx_swbd->dma)) 834 enetc_unmap_tx_buff(tx_ring, tx_swbd); 835 836 if (xdp_frame) { 837 xdp_return_frame(xdp_frame); 838 } else if (skb) { 839 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 840 /* Start work to release lock for next one-step 841 * timestamping packet. And send one skb in 842 * tx_skbs queue if has. 843 */ 844 schedule_work(&priv->tx_onestep_tstamp); 845 } else if (unlikely(do_twostep_tstamp)) { 846 enetc_tstamp_tx(skb, tstamp); 847 do_twostep_tstamp = false; 848 } 849 napi_consume_skb(skb, napi_budget); 850 } 851 852 tx_byte_cnt += tx_swbd->len; 853 /* Scrub the swbd here so we don't have to do that 854 * when we reuse it during xmit 855 */ 856 memset(tx_swbd, 0, sizeof(*tx_swbd)); 857 858 bds_to_clean--; 859 tx_swbd++; 860 i++; 861 if (unlikely(i == tx_ring->bd_count)) { 862 i = 0; 863 tx_swbd = tx_ring->tx_swbd; 864 } 865 866 /* BD iteration loop end */ 867 if (is_eof) { 868 tx_frm_cnt++; 869 /* re-arm interrupt source */ 870 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 871 BIT(16 + tx_ring->index)); 872 } 873 874 if (unlikely(!bds_to_clean)) 875 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 876 } 877 878 tx_ring->next_to_clean = i; 879 tx_ring->stats.packets += tx_frm_cnt; 880 tx_ring->stats.bytes += tx_byte_cnt; 881 tx_ring->stats.win_drop += tx_win_drop; 882 883 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 884 __netif_subqueue_stopped(ndev, tx_ring->index) && 885 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 886 netif_wake_subqueue(ndev, tx_ring->index); 887 } 888 889 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 890 } 891 892 static bool enetc_new_page(struct enetc_bdr *rx_ring, 893 struct enetc_rx_swbd *rx_swbd) 894 { 895 bool xdp = !!(rx_ring->xdp.prog); 896 struct page *page; 897 dma_addr_t addr; 898 899 page = dev_alloc_page(); 900 if (unlikely(!page)) 901 return false; 902 903 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 904 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 905 906 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 907 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 908 __free_page(page); 909 910 return false; 911 } 912 913 rx_swbd->dma = addr; 914 rx_swbd->page = page; 915 rx_swbd->page_offset = rx_ring->buffer_offset; 916 917 return true; 918 } 919 920 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 921 { 922 struct enetc_rx_swbd *rx_swbd; 923 union enetc_rx_bd *rxbd; 924 int i, j; 925 926 i = rx_ring->next_to_use; 927 rx_swbd = &rx_ring->rx_swbd[i]; 928 rxbd = enetc_rxbd(rx_ring, i); 929 930 for (j = 0; j < buff_cnt; j++) { 931 /* try reuse page */ 932 if (unlikely(!rx_swbd->page)) { 933 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 934 rx_ring->stats.rx_alloc_errs++; 935 break; 936 } 937 } 938 939 /* update RxBD */ 940 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 941 rx_swbd->page_offset); 942 /* clear 'R" as well */ 943 rxbd->r.lstatus = 0; 944 945 enetc_rxbd_next(rx_ring, &rxbd, &i); 946 rx_swbd = &rx_ring->rx_swbd[i]; 947 } 948 949 if (likely(j)) { 950 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 951 rx_ring->next_to_use = i; 952 953 /* update ENETC's consumer index */ 954 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 955 } 956 957 return j; 958 } 959 960 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 961 static void enetc_get_rx_tstamp(struct net_device *ndev, 962 union enetc_rx_bd *rxbd, 963 struct sk_buff *skb) 964 { 965 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 966 struct enetc_ndev_priv *priv = netdev_priv(ndev); 967 struct enetc_hw *hw = &priv->si->hw; 968 u32 lo, hi, tstamp_lo; 969 u64 tstamp; 970 971 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 972 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 973 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 974 rxbd = enetc_rxbd_ext(rxbd); 975 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 976 if (lo <= tstamp_lo) 977 hi -= 1; 978 979 tstamp = (u64)hi << 32 | tstamp_lo; 980 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 981 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 982 } 983 } 984 #endif 985 986 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 987 union enetc_rx_bd *rxbd, struct sk_buff *skb) 988 { 989 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 990 991 /* TODO: hashing */ 992 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 993 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 994 995 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 996 skb->ip_summed = CHECKSUM_COMPLETE; 997 } 998 999 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1000 __be16 tpid = 0; 1001 1002 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1003 case 0: 1004 tpid = htons(ETH_P_8021Q); 1005 break; 1006 case 1: 1007 tpid = htons(ETH_P_8021AD); 1008 break; 1009 case 2: 1010 tpid = htons(enetc_port_rd(&priv->si->hw, 1011 ENETC_PCVLANR1)); 1012 break; 1013 case 3: 1014 tpid = htons(enetc_port_rd(&priv->si->hw, 1015 ENETC_PCVLANR2)); 1016 break; 1017 default: 1018 break; 1019 } 1020 1021 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1022 } 1023 1024 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1025 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1026 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1027 #endif 1028 } 1029 1030 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1031 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1032 * mapped buffers. 1033 */ 1034 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1035 int i, u16 size) 1036 { 1037 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1038 1039 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1040 rx_swbd->page_offset, 1041 size, rx_swbd->dir); 1042 return rx_swbd; 1043 } 1044 1045 /* Reuse the current page without performing half-page buffer flipping */ 1046 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1047 struct enetc_rx_swbd *rx_swbd) 1048 { 1049 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1050 1051 enetc_reuse_page(rx_ring, rx_swbd); 1052 1053 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1054 rx_swbd->page_offset, 1055 buffer_size, rx_swbd->dir); 1056 1057 rx_swbd->page = NULL; 1058 } 1059 1060 /* Reuse the current page by performing half-page buffer flipping */ 1061 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1062 struct enetc_rx_swbd *rx_swbd) 1063 { 1064 if (likely(enetc_page_reusable(rx_swbd->page))) { 1065 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1066 page_ref_inc(rx_swbd->page); 1067 1068 enetc_put_rx_buff(rx_ring, rx_swbd); 1069 } else { 1070 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1071 rx_swbd->dir); 1072 rx_swbd->page = NULL; 1073 } 1074 } 1075 1076 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1077 int i, u16 size) 1078 { 1079 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1080 struct sk_buff *skb; 1081 void *ba; 1082 1083 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1084 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1085 if (unlikely(!skb)) { 1086 rx_ring->stats.rx_alloc_errs++; 1087 return NULL; 1088 } 1089 1090 skb_reserve(skb, rx_ring->buffer_offset); 1091 __skb_put(skb, size); 1092 1093 enetc_flip_rx_buff(rx_ring, rx_swbd); 1094 1095 return skb; 1096 } 1097 1098 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1099 u16 size, struct sk_buff *skb) 1100 { 1101 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1102 1103 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1104 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1105 1106 enetc_flip_rx_buff(rx_ring, rx_swbd); 1107 } 1108 1109 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1110 u32 bd_status, 1111 union enetc_rx_bd **rxbd, int *i) 1112 { 1113 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1114 return false; 1115 1116 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1117 enetc_rxbd_next(rx_ring, rxbd, i); 1118 1119 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1120 dma_rmb(); 1121 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1122 1123 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1124 enetc_rxbd_next(rx_ring, rxbd, i); 1125 } 1126 1127 rx_ring->ndev->stats.rx_dropped++; 1128 rx_ring->ndev->stats.rx_errors++; 1129 1130 return true; 1131 } 1132 1133 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1134 u32 bd_status, union enetc_rx_bd **rxbd, 1135 int *i, int *cleaned_cnt, int buffer_size) 1136 { 1137 struct sk_buff *skb; 1138 u16 size; 1139 1140 size = le16_to_cpu((*rxbd)->r.buf_len); 1141 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1142 if (!skb) 1143 return NULL; 1144 1145 enetc_get_offloads(rx_ring, *rxbd, skb); 1146 1147 (*cleaned_cnt)++; 1148 1149 enetc_rxbd_next(rx_ring, rxbd, i); 1150 1151 /* not last BD in frame? */ 1152 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1153 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1154 size = buffer_size; 1155 1156 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1157 dma_rmb(); 1158 size = le16_to_cpu((*rxbd)->r.buf_len); 1159 } 1160 1161 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1162 1163 (*cleaned_cnt)++; 1164 1165 enetc_rxbd_next(rx_ring, rxbd, i); 1166 } 1167 1168 skb_record_rx_queue(skb, rx_ring->index); 1169 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1170 1171 return skb; 1172 } 1173 1174 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1175 1176 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1177 struct napi_struct *napi, int work_limit) 1178 { 1179 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1180 int cleaned_cnt, i; 1181 1182 cleaned_cnt = enetc_bd_unused(rx_ring); 1183 /* next descriptor to process */ 1184 i = rx_ring->next_to_clean; 1185 1186 while (likely(rx_frm_cnt < work_limit)) { 1187 union enetc_rx_bd *rxbd; 1188 struct sk_buff *skb; 1189 u32 bd_status; 1190 1191 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1192 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1193 cleaned_cnt); 1194 1195 rxbd = enetc_rxbd(rx_ring, i); 1196 bd_status = le32_to_cpu(rxbd->r.lstatus); 1197 if (!bd_status) 1198 break; 1199 1200 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1201 dma_rmb(); /* for reading other rxbd fields */ 1202 1203 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1204 &rxbd, &i)) 1205 break; 1206 1207 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1208 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1209 if (!skb) 1210 break; 1211 1212 rx_byte_cnt += skb->len; 1213 rx_frm_cnt++; 1214 1215 napi_gro_receive(napi, skb); 1216 } 1217 1218 rx_ring->next_to_clean = i; 1219 1220 rx_ring->stats.packets += rx_frm_cnt; 1221 rx_ring->stats.bytes += rx_byte_cnt; 1222 1223 return rx_frm_cnt; 1224 } 1225 1226 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1227 struct enetc_tx_swbd *tx_swbd, 1228 int frm_len) 1229 { 1230 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1231 1232 prefetchw(txbd); 1233 1234 enetc_clear_tx_bd(txbd); 1235 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1236 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1237 txbd->frm_len = cpu_to_le16(frm_len); 1238 1239 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1240 } 1241 1242 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1243 * descriptors. 1244 */ 1245 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1246 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1247 { 1248 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1249 int i, k, frm_len = tmp_tx_swbd->len; 1250 1251 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1252 return false; 1253 1254 while (unlikely(!tmp_tx_swbd->is_eof)) { 1255 tmp_tx_swbd++; 1256 frm_len += tmp_tx_swbd->len; 1257 } 1258 1259 i = tx_ring->next_to_use; 1260 1261 for (k = 0; k < num_tx_swbd; k++) { 1262 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1263 1264 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1265 1266 /* last BD needs 'F' bit set */ 1267 if (xdp_tx_swbd->is_eof) { 1268 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1269 1270 txbd->flags = ENETC_TXBD_FLAGS_F; 1271 } 1272 1273 enetc_bdr_idx_inc(tx_ring, &i); 1274 } 1275 1276 tx_ring->next_to_use = i; 1277 1278 return true; 1279 } 1280 1281 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1282 struct enetc_tx_swbd *xdp_tx_arr, 1283 struct xdp_frame *xdp_frame) 1284 { 1285 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1286 struct skb_shared_info *shinfo; 1287 void *data = xdp_frame->data; 1288 int len = xdp_frame->len; 1289 skb_frag_t *frag; 1290 dma_addr_t dma; 1291 unsigned int f; 1292 int n = 0; 1293 1294 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1295 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1296 netdev_err(tx_ring->ndev, "DMA map error\n"); 1297 return -1; 1298 } 1299 1300 xdp_tx_swbd->dma = dma; 1301 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1302 xdp_tx_swbd->len = len; 1303 xdp_tx_swbd->is_xdp_redirect = true; 1304 xdp_tx_swbd->is_eof = false; 1305 xdp_tx_swbd->xdp_frame = NULL; 1306 1307 n++; 1308 xdp_tx_swbd = &xdp_tx_arr[n]; 1309 1310 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1311 1312 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1313 f++, frag++) { 1314 data = skb_frag_address(frag); 1315 len = skb_frag_size(frag); 1316 1317 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1318 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1319 /* Undo the DMA mapping for all fragments */ 1320 while (--n >= 0) 1321 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1322 1323 netdev_err(tx_ring->ndev, "DMA map error\n"); 1324 return -1; 1325 } 1326 1327 xdp_tx_swbd->dma = dma; 1328 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1329 xdp_tx_swbd->len = len; 1330 xdp_tx_swbd->is_xdp_redirect = true; 1331 xdp_tx_swbd->is_eof = false; 1332 xdp_tx_swbd->xdp_frame = NULL; 1333 1334 n++; 1335 xdp_tx_swbd = &xdp_tx_arr[n]; 1336 } 1337 1338 xdp_tx_arr[n - 1].is_eof = true; 1339 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1340 1341 return n; 1342 } 1343 1344 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1345 struct xdp_frame **frames, u32 flags) 1346 { 1347 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1348 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1349 struct enetc_bdr *tx_ring; 1350 int xdp_tx_bd_cnt, i, k; 1351 int xdp_tx_frm_cnt = 0; 1352 1353 enetc_lock_mdio(); 1354 1355 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1356 1357 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1358 1359 for (k = 0; k < num_frames; k++) { 1360 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1361 xdp_redirect_arr, 1362 frames[k]); 1363 if (unlikely(xdp_tx_bd_cnt < 0)) 1364 break; 1365 1366 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1367 xdp_tx_bd_cnt))) { 1368 for (i = 0; i < xdp_tx_bd_cnt; i++) 1369 enetc_unmap_tx_buff(tx_ring, 1370 &xdp_redirect_arr[i]); 1371 tx_ring->stats.xdp_tx_drops++; 1372 break; 1373 } 1374 1375 xdp_tx_frm_cnt++; 1376 } 1377 1378 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1379 enetc_update_tx_ring_tail(tx_ring); 1380 1381 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1382 1383 enetc_unlock_mdio(); 1384 1385 return xdp_tx_frm_cnt; 1386 } 1387 1388 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1389 struct xdp_buff *xdp_buff, u16 size) 1390 { 1391 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1392 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1393 struct skb_shared_info *shinfo; 1394 1395 /* To be used for XDP_TX */ 1396 rx_swbd->len = size; 1397 1398 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1399 rx_ring->buffer_offset, size, false); 1400 1401 shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1402 shinfo->nr_frags = 0; 1403 } 1404 1405 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1406 u16 size, struct xdp_buff *xdp_buff) 1407 { 1408 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1409 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1410 skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags]; 1411 1412 /* To be used for XDP_TX */ 1413 rx_swbd->len = size; 1414 1415 skb_frag_off_set(frag, rx_swbd->page_offset); 1416 skb_frag_size_set(frag, size); 1417 __skb_frag_set_page(frag, rx_swbd->page); 1418 1419 shinfo->nr_frags++; 1420 } 1421 1422 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1423 union enetc_rx_bd **rxbd, int *i, 1424 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1425 { 1426 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1427 1428 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1429 1430 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1431 (*cleaned_cnt)++; 1432 enetc_rxbd_next(rx_ring, rxbd, i); 1433 1434 /* not last BD in frame? */ 1435 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1436 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1437 size = ENETC_RXB_DMA_SIZE_XDP; 1438 1439 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1440 dma_rmb(); 1441 size = le16_to_cpu((*rxbd)->r.buf_len); 1442 } 1443 1444 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1445 (*cleaned_cnt)++; 1446 enetc_rxbd_next(rx_ring, rxbd, i); 1447 } 1448 } 1449 1450 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1451 * recycled back into the RX ring in enetc_clean_tx_ring. 1452 */ 1453 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1454 struct enetc_bdr *rx_ring, 1455 int rx_ring_first, int rx_ring_last) 1456 { 1457 int n = 0; 1458 1459 for (; rx_ring_first != rx_ring_last; 1460 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1461 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1462 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1463 1464 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1465 tx_swbd->dma = rx_swbd->dma; 1466 tx_swbd->dir = rx_swbd->dir; 1467 tx_swbd->page = rx_swbd->page; 1468 tx_swbd->page_offset = rx_swbd->page_offset; 1469 tx_swbd->len = rx_swbd->len; 1470 tx_swbd->is_dma_page = true; 1471 tx_swbd->is_xdp_tx = true; 1472 tx_swbd->is_eof = false; 1473 } 1474 1475 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1476 xdp_tx_arr[n - 1].is_eof = true; 1477 1478 return n; 1479 } 1480 1481 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1482 int rx_ring_last) 1483 { 1484 while (rx_ring_first != rx_ring_last) { 1485 enetc_put_rx_buff(rx_ring, 1486 &rx_ring->rx_swbd[rx_ring_first]); 1487 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1488 } 1489 rx_ring->stats.xdp_drops++; 1490 } 1491 1492 static void enetc_xdp_free(struct enetc_bdr *rx_ring, int rx_ring_first, 1493 int rx_ring_last) 1494 { 1495 while (rx_ring_first != rx_ring_last) { 1496 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1497 1498 if (rx_swbd->page) { 1499 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1500 rx_swbd->dir); 1501 __free_page(rx_swbd->page); 1502 rx_swbd->page = NULL; 1503 } 1504 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1505 } 1506 rx_ring->stats.xdp_redirect_failures++; 1507 } 1508 1509 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1510 struct napi_struct *napi, int work_limit, 1511 struct bpf_prog *prog) 1512 { 1513 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1514 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1515 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1516 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1517 struct enetc_bdr *tx_ring; 1518 int cleaned_cnt, i; 1519 u32 xdp_act; 1520 1521 cleaned_cnt = enetc_bd_unused(rx_ring); 1522 /* next descriptor to process */ 1523 i = rx_ring->next_to_clean; 1524 1525 while (likely(rx_frm_cnt < work_limit)) { 1526 union enetc_rx_bd *rxbd, *orig_rxbd; 1527 int orig_i, orig_cleaned_cnt; 1528 struct xdp_buff xdp_buff; 1529 struct sk_buff *skb; 1530 int tmp_orig_i, err; 1531 u32 bd_status; 1532 1533 rxbd = enetc_rxbd(rx_ring, i); 1534 bd_status = le32_to_cpu(rxbd->r.lstatus); 1535 if (!bd_status) 1536 break; 1537 1538 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1539 dma_rmb(); /* for reading other rxbd fields */ 1540 1541 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1542 &rxbd, &i)) 1543 break; 1544 1545 orig_rxbd = rxbd; 1546 orig_cleaned_cnt = cleaned_cnt; 1547 orig_i = i; 1548 1549 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1550 &cleaned_cnt, &xdp_buff); 1551 1552 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1553 1554 switch (xdp_act) { 1555 default: 1556 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1557 fallthrough; 1558 case XDP_ABORTED: 1559 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1560 fallthrough; 1561 case XDP_DROP: 1562 enetc_xdp_drop(rx_ring, orig_i, i); 1563 break; 1564 case XDP_PASS: 1565 rxbd = orig_rxbd; 1566 cleaned_cnt = orig_cleaned_cnt; 1567 i = orig_i; 1568 1569 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1570 &i, &cleaned_cnt, 1571 ENETC_RXB_DMA_SIZE_XDP); 1572 if (unlikely(!skb)) 1573 goto out; 1574 1575 napi_gro_receive(napi, skb); 1576 break; 1577 case XDP_TX: 1578 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1579 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1580 rx_ring, 1581 orig_i, i); 1582 1583 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1584 enetc_xdp_drop(rx_ring, orig_i, i); 1585 tx_ring->stats.xdp_tx_drops++; 1586 } else { 1587 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1588 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1589 xdp_tx_frm_cnt++; 1590 /* The XDP_TX enqueue was successful, so we 1591 * need to scrub the RX software BDs because 1592 * the ownership of the buffers no longer 1593 * belongs to the RX ring, and we must prevent 1594 * enetc_refill_rx_ring() from reusing 1595 * rx_swbd->page. 1596 */ 1597 while (orig_i != i) { 1598 rx_ring->rx_swbd[orig_i].page = NULL; 1599 enetc_bdr_idx_inc(rx_ring, &orig_i); 1600 } 1601 } 1602 break; 1603 case XDP_REDIRECT: 1604 /* xdp_return_frame does not support S/G in the sense 1605 * that it leaks the fragments (__xdp_return should not 1606 * call page_frag_free only for the initial buffer). 1607 * Until XDP_REDIRECT gains support for S/G let's keep 1608 * the code structure in place, but dead. We drop the 1609 * S/G frames ourselves to avoid memory leaks which 1610 * would otherwise leave the kernel OOM. 1611 */ 1612 if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) { 1613 enetc_xdp_drop(rx_ring, orig_i, i); 1614 rx_ring->stats.xdp_redirect_sg++; 1615 break; 1616 } 1617 1618 tmp_orig_i = orig_i; 1619 1620 while (orig_i != i) { 1621 enetc_flip_rx_buff(rx_ring, 1622 &rx_ring->rx_swbd[orig_i]); 1623 enetc_bdr_idx_inc(rx_ring, &orig_i); 1624 } 1625 1626 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1627 if (unlikely(err)) { 1628 enetc_xdp_free(rx_ring, tmp_orig_i, i); 1629 } else { 1630 xdp_redirect_frm_cnt++; 1631 rx_ring->stats.xdp_redirect++; 1632 } 1633 } 1634 1635 rx_frm_cnt++; 1636 } 1637 1638 out: 1639 rx_ring->next_to_clean = i; 1640 1641 rx_ring->stats.packets += rx_frm_cnt; 1642 rx_ring->stats.bytes += rx_byte_cnt; 1643 1644 if (xdp_redirect_frm_cnt) 1645 xdp_do_flush_map(); 1646 1647 if (xdp_tx_frm_cnt) 1648 enetc_update_tx_ring_tail(tx_ring); 1649 1650 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1651 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1652 rx_ring->xdp.xdp_tx_in_flight); 1653 1654 return rx_frm_cnt; 1655 } 1656 1657 static int enetc_poll(struct napi_struct *napi, int budget) 1658 { 1659 struct enetc_int_vector 1660 *v = container_of(napi, struct enetc_int_vector, napi); 1661 struct enetc_bdr *rx_ring = &v->rx_ring; 1662 struct bpf_prog *prog; 1663 bool complete = true; 1664 int work_done; 1665 int i; 1666 1667 enetc_lock_mdio(); 1668 1669 for (i = 0; i < v->count_tx_rings; i++) 1670 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1671 complete = false; 1672 1673 prog = rx_ring->xdp.prog; 1674 if (prog) 1675 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1676 else 1677 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1678 if (work_done == budget) 1679 complete = false; 1680 if (work_done) 1681 v->rx_napi_work = true; 1682 1683 if (!complete) { 1684 enetc_unlock_mdio(); 1685 return budget; 1686 } 1687 1688 napi_complete_done(napi, work_done); 1689 1690 if (likely(v->rx_dim_en)) 1691 enetc_rx_net_dim(v); 1692 1693 v->rx_napi_work = false; 1694 1695 /* enable interrupts */ 1696 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1697 1698 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1699 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1700 ENETC_TBIER_TXTIE); 1701 1702 enetc_unlock_mdio(); 1703 1704 return work_done; 1705 } 1706 1707 /* Probing and Init */ 1708 #define ENETC_MAX_RFS_SIZE 64 1709 void enetc_get_si_caps(struct enetc_si *si) 1710 { 1711 struct enetc_hw *hw = &si->hw; 1712 u32 val; 1713 1714 /* find out how many of various resources we have to work with */ 1715 val = enetc_rd(hw, ENETC_SICAPR0); 1716 si->num_rx_rings = (val >> 16) & 0xff; 1717 si->num_tx_rings = val & 0xff; 1718 1719 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1720 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1721 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1722 1723 si->num_rss = 0; 1724 val = enetc_rd(hw, ENETC_SIPCAPR0); 1725 if (val & ENETC_SIPCAPR0_RSS) { 1726 u32 rss; 1727 1728 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1729 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1730 } 1731 1732 if (val & ENETC_SIPCAPR0_QBV) 1733 si->hw_features |= ENETC_SI_F_QBV; 1734 1735 if (val & ENETC_SIPCAPR0_PSFP) 1736 si->hw_features |= ENETC_SI_F_PSFP; 1737 } 1738 1739 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 1740 { 1741 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 1742 &r->bd_dma_base, GFP_KERNEL); 1743 if (!r->bd_base) 1744 return -ENOMEM; 1745 1746 /* h/w requires 128B alignment */ 1747 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 1748 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 1749 r->bd_dma_base); 1750 return -EINVAL; 1751 } 1752 1753 return 0; 1754 } 1755 1756 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 1757 { 1758 int err; 1759 1760 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 1761 if (!txr->tx_swbd) 1762 return -ENOMEM; 1763 1764 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 1765 if (err) 1766 goto err_alloc_bdr; 1767 1768 txr->tso_headers = dma_alloc_coherent(txr->dev, 1769 txr->bd_count * TSO_HEADER_SIZE, 1770 &txr->tso_headers_dma, 1771 GFP_KERNEL); 1772 if (!txr->tso_headers) { 1773 err = -ENOMEM; 1774 goto err_alloc_tso; 1775 } 1776 1777 txr->next_to_clean = 0; 1778 txr->next_to_use = 0; 1779 1780 return 0; 1781 1782 err_alloc_tso: 1783 dma_free_coherent(txr->dev, txr->bd_count * sizeof(union enetc_tx_bd), 1784 txr->bd_base, txr->bd_dma_base); 1785 txr->bd_base = NULL; 1786 err_alloc_bdr: 1787 vfree(txr->tx_swbd); 1788 txr->tx_swbd = NULL; 1789 1790 return err; 1791 } 1792 1793 static void enetc_free_txbdr(struct enetc_bdr *txr) 1794 { 1795 int size, i; 1796 1797 for (i = 0; i < txr->bd_count; i++) 1798 enetc_free_tx_frame(txr, &txr->tx_swbd[i]); 1799 1800 size = txr->bd_count * sizeof(union enetc_tx_bd); 1801 1802 dma_free_coherent(txr->dev, txr->bd_count * TSO_HEADER_SIZE, 1803 txr->tso_headers, txr->tso_headers_dma); 1804 txr->tso_headers = NULL; 1805 1806 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 1807 txr->bd_base = NULL; 1808 1809 vfree(txr->tx_swbd); 1810 txr->tx_swbd = NULL; 1811 } 1812 1813 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1814 { 1815 int i, err; 1816 1817 for (i = 0; i < priv->num_tx_rings; i++) { 1818 err = enetc_alloc_txbdr(priv->tx_ring[i]); 1819 1820 if (err) 1821 goto fail; 1822 } 1823 1824 return 0; 1825 1826 fail: 1827 while (i-- > 0) 1828 enetc_free_txbdr(priv->tx_ring[i]); 1829 1830 return err; 1831 } 1832 1833 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 1834 { 1835 int i; 1836 1837 for (i = 0; i < priv->num_tx_rings; i++) 1838 enetc_free_txbdr(priv->tx_ring[i]); 1839 } 1840 1841 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 1842 { 1843 size_t size = sizeof(union enetc_rx_bd); 1844 int err; 1845 1846 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 1847 if (!rxr->rx_swbd) 1848 return -ENOMEM; 1849 1850 if (extended) 1851 size *= 2; 1852 1853 err = enetc_dma_alloc_bdr(rxr, size); 1854 if (err) { 1855 vfree(rxr->rx_swbd); 1856 return err; 1857 } 1858 1859 rxr->next_to_clean = 0; 1860 rxr->next_to_use = 0; 1861 rxr->next_to_alloc = 0; 1862 rxr->ext_en = extended; 1863 1864 return 0; 1865 } 1866 1867 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 1868 { 1869 int size; 1870 1871 size = rxr->bd_count * sizeof(union enetc_rx_bd); 1872 1873 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 1874 rxr->bd_base = NULL; 1875 1876 vfree(rxr->rx_swbd); 1877 rxr->rx_swbd = NULL; 1878 } 1879 1880 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 1881 { 1882 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 1883 int i, err; 1884 1885 for (i = 0; i < priv->num_rx_rings; i++) { 1886 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 1887 1888 if (err) 1889 goto fail; 1890 } 1891 1892 return 0; 1893 1894 fail: 1895 while (i-- > 0) 1896 enetc_free_rxbdr(priv->rx_ring[i]); 1897 1898 return err; 1899 } 1900 1901 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 1902 { 1903 int i; 1904 1905 for (i = 0; i < priv->num_rx_rings; i++) 1906 enetc_free_rxbdr(priv->rx_ring[i]); 1907 } 1908 1909 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 1910 { 1911 int i; 1912 1913 if (!tx_ring->tx_swbd) 1914 return; 1915 1916 for (i = 0; i < tx_ring->bd_count; i++) { 1917 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 1918 1919 enetc_free_tx_frame(tx_ring, tx_swbd); 1920 } 1921 1922 tx_ring->next_to_clean = 0; 1923 tx_ring->next_to_use = 0; 1924 } 1925 1926 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 1927 { 1928 int i; 1929 1930 if (!rx_ring->rx_swbd) 1931 return; 1932 1933 for (i = 0; i < rx_ring->bd_count; i++) { 1934 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1935 1936 if (!rx_swbd->page) 1937 continue; 1938 1939 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1940 rx_swbd->dir); 1941 __free_page(rx_swbd->page); 1942 rx_swbd->page = NULL; 1943 } 1944 1945 rx_ring->next_to_clean = 0; 1946 rx_ring->next_to_use = 0; 1947 rx_ring->next_to_alloc = 0; 1948 } 1949 1950 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 1951 { 1952 int i; 1953 1954 for (i = 0; i < priv->num_rx_rings; i++) 1955 enetc_free_rx_ring(priv->rx_ring[i]); 1956 1957 for (i = 0; i < priv->num_tx_rings; i++) 1958 enetc_free_tx_ring(priv->tx_ring[i]); 1959 } 1960 1961 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1962 { 1963 int *rss_table; 1964 int i; 1965 1966 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1967 if (!rss_table) 1968 return -ENOMEM; 1969 1970 /* Set up RSS table defaults */ 1971 for (i = 0; i < si->num_rss; i++) 1972 rss_table[i] = i % num_groups; 1973 1974 enetc_set_rss_table(si, rss_table, si->num_rss); 1975 1976 kfree(rss_table); 1977 1978 return 0; 1979 } 1980 1981 int enetc_configure_si(struct enetc_ndev_priv *priv) 1982 { 1983 struct enetc_si *si = priv->si; 1984 struct enetc_hw *hw = &si->hw; 1985 int err; 1986 1987 /* set SI cache attributes */ 1988 enetc_wr(hw, ENETC_SICAR0, 1989 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1990 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1991 /* enable SI */ 1992 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1993 1994 if (si->num_rss) { 1995 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1996 if (err) 1997 return err; 1998 } 1999 2000 return 0; 2001 } 2002 2003 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2004 { 2005 struct enetc_si *si = priv->si; 2006 int cpus = num_online_cpus(); 2007 2008 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2009 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2010 2011 /* Enable all available TX rings in order to configure as many 2012 * priorities as possible, when needed. 2013 * TODO: Make # of TX rings run-time configurable 2014 */ 2015 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2016 priv->num_tx_rings = si->num_tx_rings; 2017 priv->bdr_int_num = cpus; 2018 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2019 priv->tx_ictt = ENETC_TXIC_TIMETHR; 2020 } 2021 2022 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2023 { 2024 struct enetc_si *si = priv->si; 2025 2026 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2027 GFP_KERNEL); 2028 if (!priv->cls_rules) 2029 return -ENOMEM; 2030 2031 return 0; 2032 } 2033 2034 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2035 { 2036 kfree(priv->cls_rules); 2037 } 2038 2039 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2040 { 2041 int idx = tx_ring->index; 2042 u32 tbmr; 2043 2044 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2045 lower_32_bits(tx_ring->bd_dma_base)); 2046 2047 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2048 upper_32_bits(tx_ring->bd_dma_base)); 2049 2050 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2051 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2052 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2053 2054 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2055 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2056 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2057 2058 /* enable Tx ints by setting pkt thr to 1 */ 2059 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2060 2061 tbmr = ENETC_TBMR_EN; 2062 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2063 tbmr |= ENETC_TBMR_VIH; 2064 2065 /* enable ring */ 2066 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2067 2068 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2069 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2070 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2071 } 2072 2073 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2074 { 2075 int idx = rx_ring->index; 2076 u32 rbmr; 2077 2078 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2079 lower_32_bits(rx_ring->bd_dma_base)); 2080 2081 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2082 upper_32_bits(rx_ring->bd_dma_base)); 2083 2084 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2085 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2086 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2087 2088 if (rx_ring->xdp.prog) 2089 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2090 else 2091 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2092 2093 /* Also prepare the consumer index in case page allocation never 2094 * succeeds. In that case, hardware will never advance producer index 2095 * to match consumer index, and will drop all frames. 2096 */ 2097 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2098 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2099 2100 /* enable Rx ints by setting pkt thr to 1 */ 2101 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2102 2103 rbmr = ENETC_RBMR_EN; 2104 2105 if (rx_ring->ext_en) 2106 rbmr |= ENETC_RBMR_BDS; 2107 2108 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2109 rbmr |= ENETC_RBMR_VTE; 2110 2111 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2112 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2113 2114 enetc_lock_mdio(); 2115 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2116 enetc_unlock_mdio(); 2117 2118 /* enable ring */ 2119 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2120 } 2121 2122 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 2123 { 2124 struct enetc_hw *hw = &priv->si->hw; 2125 int i; 2126 2127 for (i = 0; i < priv->num_tx_rings; i++) 2128 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2129 2130 for (i = 0; i < priv->num_rx_rings; i++) 2131 enetc_setup_rxbdr(hw, priv->rx_ring[i]); 2132 } 2133 2134 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2135 { 2136 int idx = rx_ring->index; 2137 2138 /* disable EN bit on ring */ 2139 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2140 } 2141 2142 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2143 { 2144 int delay = 8, timeout = 100; 2145 int idx = tx_ring->index; 2146 2147 /* disable EN bit on ring */ 2148 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2149 2150 /* wait for busy to clear */ 2151 while (delay < timeout && 2152 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2153 msleep(delay); 2154 delay *= 2; 2155 } 2156 2157 if (delay >= timeout) 2158 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2159 idx); 2160 } 2161 2162 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 2163 { 2164 struct enetc_hw *hw = &priv->si->hw; 2165 int i; 2166 2167 for (i = 0; i < priv->num_tx_rings; i++) 2168 enetc_clear_txbdr(hw, priv->tx_ring[i]); 2169 2170 for (i = 0; i < priv->num_rx_rings; i++) 2171 enetc_clear_rxbdr(hw, priv->rx_ring[i]); 2172 2173 udelay(1); 2174 } 2175 2176 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2177 { 2178 struct pci_dev *pdev = priv->si->pdev; 2179 struct enetc_hw *hw = &priv->si->hw; 2180 int i, j, err; 2181 2182 for (i = 0; i < priv->bdr_int_num; i++) { 2183 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2184 struct enetc_int_vector *v = priv->int_vector[i]; 2185 int entry = ENETC_BDR_INT_BASE_IDX + i; 2186 2187 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2188 priv->ndev->name, i); 2189 err = request_irq(irq, enetc_msix, 0, v->name, v); 2190 if (err) { 2191 dev_err(priv->dev, "request_irq() failed!\n"); 2192 goto irq_err; 2193 } 2194 disable_irq(irq); 2195 2196 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2197 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2198 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2199 2200 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2201 2202 for (j = 0; j < v->count_tx_rings; j++) { 2203 int idx = v->tx_ring[j].index; 2204 2205 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2206 } 2207 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2208 } 2209 2210 return 0; 2211 2212 irq_err: 2213 while (i--) { 2214 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2215 2216 irq_set_affinity_hint(irq, NULL); 2217 free_irq(irq, priv->int_vector[i]); 2218 } 2219 2220 return err; 2221 } 2222 2223 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2224 { 2225 struct pci_dev *pdev = priv->si->pdev; 2226 int i; 2227 2228 for (i = 0; i < priv->bdr_int_num; i++) { 2229 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2230 2231 irq_set_affinity_hint(irq, NULL); 2232 free_irq(irq, priv->int_vector[i]); 2233 } 2234 } 2235 2236 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2237 { 2238 struct enetc_hw *hw = &priv->si->hw; 2239 u32 icpt, ictt; 2240 int i; 2241 2242 /* enable Tx & Rx event indication */ 2243 if (priv->ic_mode & 2244 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2245 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2246 /* init to non-0 minimum, will be adjusted later */ 2247 ictt = 0x1; 2248 } else { 2249 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2250 ictt = 0; 2251 } 2252 2253 for (i = 0; i < priv->num_rx_rings; i++) { 2254 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2255 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2256 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2257 } 2258 2259 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2260 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2261 else 2262 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2263 2264 for (i = 0; i < priv->num_tx_rings; i++) { 2265 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2266 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2267 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2268 } 2269 } 2270 2271 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2272 { 2273 struct enetc_hw *hw = &priv->si->hw; 2274 int i; 2275 2276 for (i = 0; i < priv->num_tx_rings; i++) 2277 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2278 2279 for (i = 0; i < priv->num_rx_rings; i++) 2280 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2281 } 2282 2283 static int enetc_phylink_connect(struct net_device *ndev) 2284 { 2285 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2286 struct ethtool_eee edata; 2287 int err; 2288 2289 if (!priv->phylink) 2290 return 0; /* phy-less mode */ 2291 2292 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2293 if (err) { 2294 dev_err(&ndev->dev, "could not attach to PHY\n"); 2295 return err; 2296 } 2297 2298 /* disable EEE autoneg, until ENETC driver supports it */ 2299 memset(&edata, 0, sizeof(struct ethtool_eee)); 2300 phylink_ethtool_set_eee(priv->phylink, &edata); 2301 2302 return 0; 2303 } 2304 2305 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2306 { 2307 struct enetc_ndev_priv *priv; 2308 struct sk_buff *skb; 2309 2310 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2311 2312 netif_tx_lock(priv->ndev); 2313 2314 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2315 skb = skb_dequeue(&priv->tx_skbs); 2316 if (skb) 2317 enetc_start_xmit(skb, priv->ndev); 2318 2319 netif_tx_unlock(priv->ndev); 2320 } 2321 2322 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2323 { 2324 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2325 skb_queue_head_init(&priv->tx_skbs); 2326 } 2327 2328 void enetc_start(struct net_device *ndev) 2329 { 2330 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2331 int i; 2332 2333 enetc_setup_interrupts(priv); 2334 2335 for (i = 0; i < priv->bdr_int_num; i++) { 2336 int irq = pci_irq_vector(priv->si->pdev, 2337 ENETC_BDR_INT_BASE_IDX + i); 2338 2339 napi_enable(&priv->int_vector[i]->napi); 2340 enable_irq(irq); 2341 } 2342 2343 if (priv->phylink) 2344 phylink_start(priv->phylink); 2345 else 2346 netif_carrier_on(ndev); 2347 2348 netif_tx_start_all_queues(ndev); 2349 } 2350 2351 int enetc_open(struct net_device *ndev) 2352 { 2353 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2354 int num_stack_tx_queues; 2355 int err; 2356 2357 err = enetc_setup_irqs(priv); 2358 if (err) 2359 return err; 2360 2361 err = enetc_phylink_connect(ndev); 2362 if (err) 2363 goto err_phy_connect; 2364 2365 err = enetc_alloc_tx_resources(priv); 2366 if (err) 2367 goto err_alloc_tx; 2368 2369 err = enetc_alloc_rx_resources(priv); 2370 if (err) 2371 goto err_alloc_rx; 2372 2373 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2374 2375 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2376 if (err) 2377 goto err_set_queues; 2378 2379 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 2380 if (err) 2381 goto err_set_queues; 2382 2383 enetc_tx_onestep_tstamp_init(priv); 2384 enetc_setup_bdrs(priv); 2385 enetc_start(ndev); 2386 2387 return 0; 2388 2389 err_set_queues: 2390 enetc_free_rx_resources(priv); 2391 err_alloc_rx: 2392 enetc_free_tx_resources(priv); 2393 err_alloc_tx: 2394 if (priv->phylink) 2395 phylink_disconnect_phy(priv->phylink); 2396 err_phy_connect: 2397 enetc_free_irqs(priv); 2398 2399 return err; 2400 } 2401 2402 void enetc_stop(struct net_device *ndev) 2403 { 2404 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2405 int i; 2406 2407 netif_tx_stop_all_queues(ndev); 2408 2409 for (i = 0; i < priv->bdr_int_num; i++) { 2410 int irq = pci_irq_vector(priv->si->pdev, 2411 ENETC_BDR_INT_BASE_IDX + i); 2412 2413 disable_irq(irq); 2414 napi_synchronize(&priv->int_vector[i]->napi); 2415 napi_disable(&priv->int_vector[i]->napi); 2416 } 2417 2418 if (priv->phylink) 2419 phylink_stop(priv->phylink); 2420 else 2421 netif_carrier_off(ndev); 2422 2423 enetc_clear_interrupts(priv); 2424 } 2425 2426 int enetc_close(struct net_device *ndev) 2427 { 2428 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2429 2430 enetc_stop(ndev); 2431 enetc_clear_bdrs(priv); 2432 2433 if (priv->phylink) 2434 phylink_disconnect_phy(priv->phylink); 2435 enetc_free_rxtx_rings(priv); 2436 enetc_free_rx_resources(priv); 2437 enetc_free_tx_resources(priv); 2438 enetc_free_irqs(priv); 2439 2440 return 0; 2441 } 2442 2443 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2444 { 2445 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2446 struct tc_mqprio_qopt *mqprio = type_data; 2447 struct enetc_hw *hw = &priv->si->hw; 2448 struct enetc_bdr *tx_ring; 2449 int num_stack_tx_queues; 2450 u8 num_tc; 2451 int i; 2452 2453 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2454 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2455 num_tc = mqprio->num_tc; 2456 2457 if (!num_tc) { 2458 netdev_reset_tc(ndev); 2459 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2460 2461 /* Reset all ring priorities to 0 */ 2462 for (i = 0; i < priv->num_tx_rings; i++) { 2463 tx_ring = priv->tx_ring[i]; 2464 enetc_set_bdr_prio(hw, tx_ring->index, 0); 2465 } 2466 2467 return 0; 2468 } 2469 2470 /* Check if we have enough BD rings available to accommodate all TCs */ 2471 if (num_tc > num_stack_tx_queues) { 2472 netdev_err(ndev, "Max %d traffic classes supported\n", 2473 priv->num_tx_rings); 2474 return -EINVAL; 2475 } 2476 2477 /* For the moment, we use only one BD ring per TC. 2478 * 2479 * Configure num_tc BD rings with increasing priorities. 2480 */ 2481 for (i = 0; i < num_tc; i++) { 2482 tx_ring = priv->tx_ring[i]; 2483 enetc_set_bdr_prio(hw, tx_ring->index, i); 2484 } 2485 2486 /* Reset the number of netdev queues based on the TC count */ 2487 netif_set_real_num_tx_queues(ndev, num_tc); 2488 2489 netdev_set_num_tc(ndev, num_tc); 2490 2491 /* Each TC is associated with one netdev queue */ 2492 for (i = 0; i < num_tc; i++) 2493 netdev_set_tc_queue(ndev, i, 1, i); 2494 2495 return 0; 2496 } 2497 2498 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog, 2499 struct netlink_ext_ack *extack) 2500 { 2501 struct enetc_ndev_priv *priv = netdev_priv(dev); 2502 struct bpf_prog *old_prog; 2503 bool is_up; 2504 int i; 2505 2506 /* The buffer layout is changing, so we need to drain the old 2507 * RX buffers and seed new ones. 2508 */ 2509 is_up = netif_running(dev); 2510 if (is_up) 2511 dev_close(dev); 2512 2513 old_prog = xchg(&priv->xdp_prog, prog); 2514 if (old_prog) 2515 bpf_prog_put(old_prog); 2516 2517 for (i = 0; i < priv->num_rx_rings; i++) { 2518 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2519 2520 rx_ring->xdp.prog = prog; 2521 2522 if (prog) 2523 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2524 else 2525 rx_ring->buffer_offset = ENETC_RXB_PAD; 2526 } 2527 2528 if (is_up) 2529 return dev_open(dev, extack); 2530 2531 return 0; 2532 } 2533 2534 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp) 2535 { 2536 switch (xdp->command) { 2537 case XDP_SETUP_PROG: 2538 return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack); 2539 default: 2540 return -EINVAL; 2541 } 2542 2543 return 0; 2544 } 2545 2546 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2547 { 2548 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2549 struct net_device_stats *stats = &ndev->stats; 2550 unsigned long packets = 0, bytes = 0; 2551 unsigned long tx_dropped = 0; 2552 int i; 2553 2554 for (i = 0; i < priv->num_rx_rings; i++) { 2555 packets += priv->rx_ring[i]->stats.packets; 2556 bytes += priv->rx_ring[i]->stats.bytes; 2557 } 2558 2559 stats->rx_packets = packets; 2560 stats->rx_bytes = bytes; 2561 bytes = 0; 2562 packets = 0; 2563 2564 for (i = 0; i < priv->num_tx_rings; i++) { 2565 packets += priv->tx_ring[i]->stats.packets; 2566 bytes += priv->tx_ring[i]->stats.bytes; 2567 tx_dropped += priv->tx_ring[i]->stats.win_drop; 2568 } 2569 2570 stats->tx_packets = packets; 2571 stats->tx_bytes = bytes; 2572 stats->tx_dropped = tx_dropped; 2573 2574 return stats; 2575 } 2576 2577 static int enetc_set_rss(struct net_device *ndev, int en) 2578 { 2579 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2580 struct enetc_hw *hw = &priv->si->hw; 2581 u32 reg; 2582 2583 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2584 2585 reg = enetc_rd(hw, ENETC_SIMR); 2586 reg &= ~ENETC_SIMR_RSSE; 2587 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2588 enetc_wr(hw, ENETC_SIMR, reg); 2589 2590 return 0; 2591 } 2592 2593 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2594 { 2595 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2596 struct enetc_hw *hw = &priv->si->hw; 2597 int i; 2598 2599 for (i = 0; i < priv->num_rx_rings; i++) 2600 enetc_bdr_enable_rxvlan(hw, i, en); 2601 } 2602 2603 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2604 { 2605 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2606 struct enetc_hw *hw = &priv->si->hw; 2607 int i; 2608 2609 for (i = 0; i < priv->num_tx_rings; i++) 2610 enetc_bdr_enable_txvlan(hw, i, en); 2611 } 2612 2613 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 2614 { 2615 netdev_features_t changed = ndev->features ^ features; 2616 2617 if (changed & NETIF_F_RXHASH) 2618 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2619 2620 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2621 enetc_enable_rxvlan(ndev, 2622 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2623 2624 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2625 enetc_enable_txvlan(ndev, 2626 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2627 } 2628 2629 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2630 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2631 { 2632 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2633 struct hwtstamp_config config; 2634 int ao; 2635 2636 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2637 return -EFAULT; 2638 2639 switch (config.tx_type) { 2640 case HWTSTAMP_TX_OFF: 2641 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2642 break; 2643 case HWTSTAMP_TX_ON: 2644 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2645 priv->active_offloads |= ENETC_F_TX_TSTAMP; 2646 break; 2647 case HWTSTAMP_TX_ONESTEP_SYNC: 2648 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2649 priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2650 break; 2651 default: 2652 return -ERANGE; 2653 } 2654 2655 ao = priv->active_offloads; 2656 switch (config.rx_filter) { 2657 case HWTSTAMP_FILTER_NONE: 2658 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 2659 break; 2660 default: 2661 priv->active_offloads |= ENETC_F_RX_TSTAMP; 2662 config.rx_filter = HWTSTAMP_FILTER_ALL; 2663 } 2664 2665 if (netif_running(ndev) && ao != priv->active_offloads) { 2666 enetc_close(ndev); 2667 enetc_open(ndev); 2668 } 2669 2670 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2671 -EFAULT : 0; 2672 } 2673 2674 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2675 { 2676 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2677 struct hwtstamp_config config; 2678 2679 config.flags = 0; 2680 2681 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2682 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2683 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2684 config.tx_type = HWTSTAMP_TX_ON; 2685 else 2686 config.tx_type = HWTSTAMP_TX_OFF; 2687 2688 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2689 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2690 2691 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2692 -EFAULT : 0; 2693 } 2694 #endif 2695 2696 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2697 { 2698 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2699 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2700 if (cmd == SIOCSHWTSTAMP) 2701 return enetc_hwtstamp_set(ndev, rq); 2702 if (cmd == SIOCGHWTSTAMP) 2703 return enetc_hwtstamp_get(ndev, rq); 2704 #endif 2705 2706 if (!priv->phylink) 2707 return -EOPNOTSUPP; 2708 2709 return phylink_mii_ioctl(priv->phylink, rq, cmd); 2710 } 2711 2712 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 2713 { 2714 struct pci_dev *pdev = priv->si->pdev; 2715 int first_xdp_tx_ring; 2716 int i, n, err, nvec; 2717 int v_tx_rings; 2718 2719 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 2720 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 2721 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 2722 2723 if (n < 0) 2724 return n; 2725 2726 if (n != nvec) 2727 return -EPERM; 2728 2729 /* # of tx rings per int vector */ 2730 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 2731 2732 for (i = 0; i < priv->bdr_int_num; i++) { 2733 struct enetc_int_vector *v; 2734 struct enetc_bdr *bdr; 2735 int j; 2736 2737 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 2738 if (!v) { 2739 err = -ENOMEM; 2740 goto fail; 2741 } 2742 2743 priv->int_vector[i] = v; 2744 2745 bdr = &v->rx_ring; 2746 bdr->index = i; 2747 bdr->ndev = priv->ndev; 2748 bdr->dev = priv->dev; 2749 bdr->bd_count = priv->rx_bd_count; 2750 bdr->buffer_offset = ENETC_RXB_PAD; 2751 priv->rx_ring[i] = bdr; 2752 2753 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 2754 if (err) { 2755 kfree(v); 2756 goto fail; 2757 } 2758 2759 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, 2760 MEM_TYPE_PAGE_SHARED, NULL); 2761 if (err) { 2762 xdp_rxq_info_unreg(&bdr->xdp.rxq); 2763 kfree(v); 2764 goto fail; 2765 } 2766 2767 /* init defaults for adaptive IC */ 2768 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 2769 v->rx_ictt = 0x1; 2770 v->rx_dim_en = true; 2771 } 2772 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 2773 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 2774 v->count_tx_rings = v_tx_rings; 2775 2776 for (j = 0; j < v_tx_rings; j++) { 2777 int idx; 2778 2779 /* default tx ring mapping policy */ 2780 idx = priv->bdr_int_num * j + i; 2781 __set_bit(idx, &v->tx_rings_map); 2782 bdr = &v->tx_ring[j]; 2783 bdr->index = idx; 2784 bdr->ndev = priv->ndev; 2785 bdr->dev = priv->dev; 2786 bdr->bd_count = priv->tx_bd_count; 2787 priv->tx_ring[idx] = bdr; 2788 } 2789 } 2790 2791 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 2792 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 2793 2794 return 0; 2795 2796 fail: 2797 while (i--) { 2798 struct enetc_int_vector *v = priv->int_vector[i]; 2799 struct enetc_bdr *rx_ring = &v->rx_ring; 2800 2801 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 2802 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 2803 netif_napi_del(&v->napi); 2804 cancel_work_sync(&v->rx_dim.work); 2805 kfree(v); 2806 } 2807 2808 pci_free_irq_vectors(pdev); 2809 2810 return err; 2811 } 2812 2813 void enetc_free_msix(struct enetc_ndev_priv *priv) 2814 { 2815 int i; 2816 2817 for (i = 0; i < priv->bdr_int_num; i++) { 2818 struct enetc_int_vector *v = priv->int_vector[i]; 2819 struct enetc_bdr *rx_ring = &v->rx_ring; 2820 2821 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 2822 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 2823 netif_napi_del(&v->napi); 2824 cancel_work_sync(&v->rx_dim.work); 2825 } 2826 2827 for (i = 0; i < priv->num_rx_rings; i++) 2828 priv->rx_ring[i] = NULL; 2829 2830 for (i = 0; i < priv->num_tx_rings; i++) 2831 priv->tx_ring[i] = NULL; 2832 2833 for (i = 0; i < priv->bdr_int_num; i++) { 2834 kfree(priv->int_vector[i]); 2835 priv->int_vector[i] = NULL; 2836 } 2837 2838 /* disable all MSIX for this device */ 2839 pci_free_irq_vectors(priv->si->pdev); 2840 } 2841 2842 static void enetc_kfree_si(struct enetc_si *si) 2843 { 2844 char *p = (char *)si - si->pad; 2845 2846 kfree(p); 2847 } 2848 2849 static void enetc_detect_errata(struct enetc_si *si) 2850 { 2851 if (si->pdev->revision == ENETC_REV1) 2852 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 2853 } 2854 2855 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 2856 { 2857 struct enetc_si *si, *p; 2858 struct enetc_hw *hw; 2859 size_t alloc_size; 2860 int err, len; 2861 2862 pcie_flr(pdev); 2863 err = pci_enable_device_mem(pdev); 2864 if (err) 2865 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 2866 2867 /* set up for high or low dma */ 2868 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2869 if (err) { 2870 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 2871 goto err_dma; 2872 } 2873 2874 err = pci_request_mem_regions(pdev, name); 2875 if (err) { 2876 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 2877 goto err_pci_mem_reg; 2878 } 2879 2880 pci_set_master(pdev); 2881 2882 alloc_size = sizeof(struct enetc_si); 2883 if (sizeof_priv) { 2884 /* align priv to 32B */ 2885 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 2886 alloc_size += sizeof_priv; 2887 } 2888 /* force 32B alignment for enetc_si */ 2889 alloc_size += ENETC_SI_ALIGN - 1; 2890 2891 p = kzalloc(alloc_size, GFP_KERNEL); 2892 if (!p) { 2893 err = -ENOMEM; 2894 goto err_alloc_si; 2895 } 2896 2897 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 2898 si->pad = (char *)si - (char *)p; 2899 2900 pci_set_drvdata(pdev, si); 2901 si->pdev = pdev; 2902 hw = &si->hw; 2903 2904 len = pci_resource_len(pdev, ENETC_BAR_REGS); 2905 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 2906 if (!hw->reg) { 2907 err = -ENXIO; 2908 dev_err(&pdev->dev, "ioremap() failed\n"); 2909 goto err_ioremap; 2910 } 2911 if (len > ENETC_PORT_BASE) 2912 hw->port = hw->reg + ENETC_PORT_BASE; 2913 if (len > ENETC_GLOBAL_BASE) 2914 hw->global = hw->reg + ENETC_GLOBAL_BASE; 2915 2916 enetc_detect_errata(si); 2917 2918 return 0; 2919 2920 err_ioremap: 2921 enetc_kfree_si(si); 2922 err_alloc_si: 2923 pci_release_mem_regions(pdev); 2924 err_pci_mem_reg: 2925 err_dma: 2926 pci_disable_device(pdev); 2927 2928 return err; 2929 } 2930 2931 void enetc_pci_remove(struct pci_dev *pdev) 2932 { 2933 struct enetc_si *si = pci_get_drvdata(pdev); 2934 struct enetc_hw *hw = &si->hw; 2935 2936 iounmap(hw->reg); 2937 enetc_kfree_si(si); 2938 pci_release_mem_regions(pdev); 2939 pci_disable_device(pdev); 2940 } 2941