1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/tcp.h> 7 #include <linux/udp.h> 8 #include <linux/vmalloc.h> 9 #include <linux/ptp_classify.h> 10 #include <net/ip6_checksum.h> 11 #include <net/pkt_sched.h> 12 #include <net/tso.h> 13 14 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 15 { 16 int num_tx_rings = priv->num_tx_rings; 17 int i; 18 19 for (i = 0; i < priv->num_rx_rings; i++) 20 if (priv->rx_ring[i]->xdp.prog) 21 return num_tx_rings - num_possible_cpus(); 22 23 return num_tx_rings; 24 } 25 26 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 27 struct enetc_bdr *tx_ring) 28 { 29 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 30 31 return priv->rx_ring[index]; 32 } 33 34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 35 { 36 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 37 return NULL; 38 39 return tx_swbd->skb; 40 } 41 42 static struct xdp_frame * 43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 44 { 45 if (tx_swbd->is_xdp_redirect) 46 return tx_swbd->xdp_frame; 47 48 return NULL; 49 } 50 51 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 52 struct enetc_tx_swbd *tx_swbd) 53 { 54 /* For XDP_TX, pages come from RX, whereas for the other contexts where 55 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 56 * to match the DMA mapping length, so we need to differentiate those. 57 */ 58 if (tx_swbd->is_dma_page) 59 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 60 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 61 tx_swbd->dir); 62 else 63 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 64 tx_swbd->len, tx_swbd->dir); 65 tx_swbd->dma = 0; 66 } 67 68 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 69 struct enetc_tx_swbd *tx_swbd) 70 { 71 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 72 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 73 74 if (tx_swbd->dma) 75 enetc_unmap_tx_buff(tx_ring, tx_swbd); 76 77 if (xdp_frame) { 78 xdp_return_frame(tx_swbd->xdp_frame); 79 tx_swbd->xdp_frame = NULL; 80 } else if (skb) { 81 dev_kfree_skb_any(skb); 82 tx_swbd->skb = NULL; 83 } 84 } 85 86 /* Let H/W know BD ring has been updated */ 87 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 88 { 89 /* includes wmb() */ 90 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 91 } 92 93 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 94 u8 *msgtype, u8 *twostep, 95 u16 *correction_offset, u16 *body_offset) 96 { 97 unsigned int ptp_class; 98 struct ptp_header *hdr; 99 unsigned int type; 100 u8 *base; 101 102 ptp_class = ptp_classify_raw(skb); 103 if (ptp_class == PTP_CLASS_NONE) 104 return -EINVAL; 105 106 hdr = ptp_parse_header(skb, ptp_class); 107 if (!hdr) 108 return -EINVAL; 109 110 type = ptp_class & PTP_CLASS_PMASK; 111 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 112 *udp = 1; 113 else 114 *udp = 0; 115 116 *msgtype = ptp_get_msgtype(hdr, ptp_class); 117 *twostep = hdr->flag_field[0] & 0x2; 118 119 base = skb_mac_header(skb); 120 *correction_offset = (u8 *)&hdr->correction - base; 121 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 122 123 return 0; 124 } 125 126 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 127 { 128 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 129 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 130 struct enetc_hw *hw = &priv->si->hw; 131 struct enetc_tx_swbd *tx_swbd; 132 int len = skb_headlen(skb); 133 union enetc_tx_bd temp_bd; 134 u8 msgtype, twostep, udp; 135 union enetc_tx_bd *txbd; 136 u16 offset1, offset2; 137 int i, count = 0; 138 skb_frag_t *frag; 139 unsigned int f; 140 dma_addr_t dma; 141 u8 flags = 0; 142 143 i = tx_ring->next_to_use; 144 txbd = ENETC_TXBD(*tx_ring, i); 145 prefetchw(txbd); 146 147 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 148 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 149 goto dma_err; 150 151 temp_bd.addr = cpu_to_le64(dma); 152 temp_bd.buf_len = cpu_to_le16(len); 153 temp_bd.lstatus = 0; 154 155 tx_swbd = &tx_ring->tx_swbd[i]; 156 tx_swbd->dma = dma; 157 tx_swbd->len = len; 158 tx_swbd->is_dma_page = 0; 159 tx_swbd->dir = DMA_TO_DEVICE; 160 count++; 161 162 do_vlan = skb_vlan_tag_present(skb); 163 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 164 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 165 &offset2) || 166 msgtype != PTP_MSGTYPE_SYNC || twostep) 167 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 168 else 169 do_onestep_tstamp = true; 170 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 171 do_twostep_tstamp = true; 172 } 173 174 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 175 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp; 176 177 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 178 flags |= ENETC_TXBD_FLAGS_EX; 179 180 if (tx_ring->tsd_enable) 181 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 182 183 /* first BD needs frm_len and offload flags set */ 184 temp_bd.frm_len = cpu_to_le16(skb->len); 185 temp_bd.flags = flags; 186 187 if (flags & ENETC_TXBD_FLAGS_TSE) 188 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 189 flags); 190 191 if (flags & ENETC_TXBD_FLAGS_EX) { 192 u8 e_flags = 0; 193 *txbd = temp_bd; 194 enetc_clear_tx_bd(&temp_bd); 195 196 /* add extension BD for VLAN and/or timestamping */ 197 flags = 0; 198 tx_swbd++; 199 txbd++; 200 i++; 201 if (unlikely(i == tx_ring->bd_count)) { 202 i = 0; 203 tx_swbd = tx_ring->tx_swbd; 204 txbd = ENETC_TXBD(*tx_ring, 0); 205 } 206 prefetchw(txbd); 207 208 if (do_vlan) { 209 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 210 temp_bd.ext.tpid = 0; /* < C-TAG */ 211 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 212 } 213 214 if (do_onestep_tstamp) { 215 u32 lo, hi, val; 216 u64 sec, nsec; 217 u8 *data; 218 219 lo = enetc_rd_hot(hw, ENETC_SICTR0); 220 hi = enetc_rd_hot(hw, ENETC_SICTR1); 221 sec = (u64)hi << 32 | lo; 222 nsec = do_div(sec, 1000000000); 223 224 /* Configure extension BD */ 225 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 226 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 227 228 /* Update originTimestamp field of Sync packet 229 * - 48 bits seconds field 230 * - 32 bits nanseconds field 231 */ 232 data = skb_mac_header(skb); 233 *(__be16 *)(data + offset2) = 234 htons((sec >> 32) & 0xffff); 235 *(__be32 *)(data + offset2 + 2) = 236 htonl(sec & 0xffffffff); 237 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 238 239 /* Configure single-step register */ 240 val = ENETC_PM0_SINGLE_STEP_EN; 241 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 242 if (udp) 243 val |= ENETC_PM0_SINGLE_STEP_CH; 244 245 enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val); 246 enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val); 247 } else if (do_twostep_tstamp) { 248 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 249 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 250 } 251 252 temp_bd.ext.e_flags = e_flags; 253 count++; 254 } 255 256 frag = &skb_shinfo(skb)->frags[0]; 257 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 258 len = skb_frag_size(frag); 259 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 260 DMA_TO_DEVICE); 261 if (dma_mapping_error(tx_ring->dev, dma)) 262 goto dma_err; 263 264 *txbd = temp_bd; 265 enetc_clear_tx_bd(&temp_bd); 266 267 flags = 0; 268 tx_swbd++; 269 txbd++; 270 i++; 271 if (unlikely(i == tx_ring->bd_count)) { 272 i = 0; 273 tx_swbd = tx_ring->tx_swbd; 274 txbd = ENETC_TXBD(*tx_ring, 0); 275 } 276 prefetchw(txbd); 277 278 temp_bd.addr = cpu_to_le64(dma); 279 temp_bd.buf_len = cpu_to_le16(len); 280 281 tx_swbd->dma = dma; 282 tx_swbd->len = len; 283 tx_swbd->is_dma_page = 1; 284 tx_swbd->dir = DMA_TO_DEVICE; 285 count++; 286 } 287 288 /* last BD needs 'F' bit set */ 289 flags |= ENETC_TXBD_FLAGS_F; 290 temp_bd.flags = flags; 291 *txbd = temp_bd; 292 293 tx_ring->tx_swbd[i].is_eof = true; 294 tx_ring->tx_swbd[i].skb = skb; 295 296 enetc_bdr_idx_inc(tx_ring, &i); 297 tx_ring->next_to_use = i; 298 299 skb_tx_timestamp(skb); 300 301 enetc_update_tx_ring_tail(tx_ring); 302 303 return count; 304 305 dma_err: 306 dev_err(tx_ring->dev, "DMA map error"); 307 308 do { 309 tx_swbd = &tx_ring->tx_swbd[i]; 310 enetc_free_tx_frame(tx_ring, tx_swbd); 311 if (i == 0) 312 i = tx_ring->bd_count; 313 i--; 314 } while (count--); 315 316 return 0; 317 } 318 319 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 320 struct enetc_tx_swbd *tx_swbd, 321 union enetc_tx_bd *txbd, int *i, int hdr_len, 322 int data_len) 323 { 324 union enetc_tx_bd txbd_tmp; 325 u8 flags = 0, e_flags = 0; 326 dma_addr_t addr; 327 328 enetc_clear_tx_bd(&txbd_tmp); 329 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 330 331 if (skb_vlan_tag_present(skb)) 332 flags |= ENETC_TXBD_FLAGS_EX; 333 334 txbd_tmp.addr = cpu_to_le64(addr); 335 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 336 337 /* first BD needs frm_len and offload flags set */ 338 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 339 txbd_tmp.flags = flags; 340 341 /* For the TSO header we do not set the dma address since we do not 342 * want it unmapped when we do cleanup. We still set len so that we 343 * count the bytes sent. 344 */ 345 tx_swbd->len = hdr_len; 346 tx_swbd->do_twostep_tstamp = false; 347 tx_swbd->check_wb = false; 348 349 /* Actually write the header in the BD */ 350 *txbd = txbd_tmp; 351 352 /* Add extension BD for VLAN */ 353 if (flags & ENETC_TXBD_FLAGS_EX) { 354 /* Get the next BD */ 355 enetc_bdr_idx_inc(tx_ring, i); 356 txbd = ENETC_TXBD(*tx_ring, *i); 357 tx_swbd = &tx_ring->tx_swbd[*i]; 358 prefetchw(txbd); 359 360 /* Setup the VLAN fields */ 361 enetc_clear_tx_bd(&txbd_tmp); 362 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 363 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 364 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 365 366 /* Write the BD */ 367 txbd_tmp.ext.e_flags = e_flags; 368 *txbd = txbd_tmp; 369 } 370 } 371 372 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 373 struct enetc_tx_swbd *tx_swbd, 374 union enetc_tx_bd *txbd, char *data, 375 int size, bool last_bd) 376 { 377 union enetc_tx_bd txbd_tmp; 378 dma_addr_t addr; 379 u8 flags = 0; 380 381 enetc_clear_tx_bd(&txbd_tmp); 382 383 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 384 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 385 netdev_err(tx_ring->ndev, "DMA map error\n"); 386 return -ENOMEM; 387 } 388 389 if (last_bd) { 390 flags |= ENETC_TXBD_FLAGS_F; 391 tx_swbd->is_eof = 1; 392 } 393 394 txbd_tmp.addr = cpu_to_le64(addr); 395 txbd_tmp.buf_len = cpu_to_le16(size); 396 txbd_tmp.flags = flags; 397 398 tx_swbd->dma = addr; 399 tx_swbd->len = size; 400 tx_swbd->dir = DMA_TO_DEVICE; 401 402 *txbd = txbd_tmp; 403 404 return 0; 405 } 406 407 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 408 char *hdr, int hdr_len, int *l4_hdr_len) 409 { 410 char *l4_hdr = hdr + skb_transport_offset(skb); 411 int mac_hdr_len = skb_network_offset(skb); 412 413 if (tso->tlen != sizeof(struct udphdr)) { 414 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 415 416 tcph->check = 0; 417 } else { 418 struct udphdr *udph = (struct udphdr *)(l4_hdr); 419 420 udph->check = 0; 421 } 422 423 /* Compute the IP checksum. This is necessary since tso_build_hdr() 424 * already incremented the IP ID field. 425 */ 426 if (!tso->ipv6) { 427 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 428 429 iph->check = 0; 430 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 431 } 432 433 /* Compute the checksum over the L4 header. */ 434 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 435 return csum_partial(l4_hdr, *l4_hdr_len, 0); 436 } 437 438 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 439 struct sk_buff *skb, char *hdr, int len, 440 __wsum sum) 441 { 442 char *l4_hdr = hdr + skb_transport_offset(skb); 443 __sum16 csum_final; 444 445 /* Complete the L4 checksum by appending the pseudo-header to the 446 * already computed checksum. 447 */ 448 if (!tso->ipv6) 449 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 450 ip_hdr(skb)->daddr, 451 len, ip_hdr(skb)->protocol, sum); 452 else 453 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 454 &ipv6_hdr(skb)->daddr, 455 len, ipv6_hdr(skb)->nexthdr, sum); 456 457 if (tso->tlen != sizeof(struct udphdr)) { 458 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 459 460 tcph->check = csum_final; 461 } else { 462 struct udphdr *udph = (struct udphdr *)(l4_hdr); 463 464 udph->check = csum_final; 465 } 466 } 467 468 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 469 { 470 int hdr_len, total_len, data_len; 471 struct enetc_tx_swbd *tx_swbd; 472 union enetc_tx_bd *txbd; 473 struct tso_t tso; 474 __wsum csum, csum2; 475 int count = 0, pos; 476 int err, i, bd_data_num; 477 478 /* Initialize the TSO handler, and prepare the first payload */ 479 hdr_len = tso_start(skb, &tso); 480 total_len = skb->len - hdr_len; 481 i = tx_ring->next_to_use; 482 483 while (total_len > 0) { 484 char *hdr; 485 486 /* Get the BD */ 487 txbd = ENETC_TXBD(*tx_ring, i); 488 tx_swbd = &tx_ring->tx_swbd[i]; 489 prefetchw(txbd); 490 491 /* Determine the length of this packet */ 492 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 493 total_len -= data_len; 494 495 /* prepare packet headers: MAC + IP + TCP */ 496 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 497 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 498 499 /* compute the csum over the L4 header */ 500 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 501 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 502 bd_data_num = 0; 503 count++; 504 505 while (data_len > 0) { 506 int size; 507 508 size = min_t(int, tso.size, data_len); 509 510 /* Advance the index in the BDR */ 511 enetc_bdr_idx_inc(tx_ring, &i); 512 txbd = ENETC_TXBD(*tx_ring, i); 513 tx_swbd = &tx_ring->tx_swbd[i]; 514 prefetchw(txbd); 515 516 /* Compute the checksum over this segment of data and 517 * add it to the csum already computed (over the L4 518 * header and possible other data segments). 519 */ 520 csum2 = csum_partial(tso.data, size, 0); 521 csum = csum_block_add(csum, csum2, pos); 522 pos += size; 523 524 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 525 tso.data, size, 526 size == data_len); 527 if (err) 528 goto err_map_data; 529 530 data_len -= size; 531 count++; 532 bd_data_num++; 533 tso_build_data(skb, &tso, size); 534 535 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 536 goto err_chained_bd; 537 } 538 539 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 540 541 if (total_len == 0) 542 tx_swbd->skb = skb; 543 544 /* Go to the next BD */ 545 enetc_bdr_idx_inc(tx_ring, &i); 546 } 547 548 tx_ring->next_to_use = i; 549 enetc_update_tx_ring_tail(tx_ring); 550 551 return count; 552 553 err_map_data: 554 dev_err(tx_ring->dev, "DMA map error"); 555 556 err_chained_bd: 557 do { 558 tx_swbd = &tx_ring->tx_swbd[i]; 559 enetc_free_tx_frame(tx_ring, tx_swbd); 560 if (i == 0) 561 i = tx_ring->bd_count; 562 i--; 563 } while (count--); 564 565 return 0; 566 } 567 568 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 569 struct net_device *ndev) 570 { 571 struct enetc_ndev_priv *priv = netdev_priv(ndev); 572 struct enetc_bdr *tx_ring; 573 int count, err; 574 575 /* Queue one-step Sync packet if already locked */ 576 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 577 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 578 &priv->flags)) { 579 skb_queue_tail(&priv->tx_skbs, skb); 580 return NETDEV_TX_OK; 581 } 582 } 583 584 tx_ring = priv->tx_ring[skb->queue_mapping]; 585 586 if (skb_is_gso(skb)) { 587 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 588 netif_stop_subqueue(ndev, tx_ring->index); 589 return NETDEV_TX_BUSY; 590 } 591 592 enetc_lock_mdio(); 593 count = enetc_map_tx_tso_buffs(tx_ring, skb); 594 enetc_unlock_mdio(); 595 } else { 596 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 597 if (unlikely(skb_linearize(skb))) 598 goto drop_packet_err; 599 600 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 601 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 602 netif_stop_subqueue(ndev, tx_ring->index); 603 return NETDEV_TX_BUSY; 604 } 605 606 if (skb->ip_summed == CHECKSUM_PARTIAL) { 607 err = skb_checksum_help(skb); 608 if (err) 609 goto drop_packet_err; 610 } 611 enetc_lock_mdio(); 612 count = enetc_map_tx_buffs(tx_ring, skb); 613 enetc_unlock_mdio(); 614 } 615 616 if (unlikely(!count)) 617 goto drop_packet_err; 618 619 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 620 netif_stop_subqueue(ndev, tx_ring->index); 621 622 return NETDEV_TX_OK; 623 624 drop_packet_err: 625 dev_kfree_skb_any(skb); 626 return NETDEV_TX_OK; 627 } 628 629 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 630 { 631 struct enetc_ndev_priv *priv = netdev_priv(ndev); 632 u8 udp, msgtype, twostep; 633 u16 offset1, offset2; 634 635 /* Mark tx timestamp type on skb->cb[0] if requires */ 636 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 637 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 638 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 639 } else { 640 skb->cb[0] = 0; 641 } 642 643 /* Fall back to two-step timestamp if not one-step Sync packet */ 644 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 645 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 646 &offset1, &offset2) || 647 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 648 skb->cb[0] = ENETC_F_TX_TSTAMP; 649 } 650 651 return enetc_start_xmit(skb, ndev); 652 } 653 654 static irqreturn_t enetc_msix(int irq, void *data) 655 { 656 struct enetc_int_vector *v = data; 657 int i; 658 659 enetc_lock_mdio(); 660 661 /* disable interrupts */ 662 enetc_wr_reg_hot(v->rbier, 0); 663 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 664 665 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 666 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 667 668 enetc_unlock_mdio(); 669 670 napi_schedule(&v->napi); 671 672 return IRQ_HANDLED; 673 } 674 675 static void enetc_rx_dim_work(struct work_struct *w) 676 { 677 struct dim *dim = container_of(w, struct dim, work); 678 struct dim_cq_moder moder = 679 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 680 struct enetc_int_vector *v = 681 container_of(dim, struct enetc_int_vector, rx_dim); 682 683 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 684 dim->state = DIM_START_MEASURE; 685 } 686 687 static void enetc_rx_net_dim(struct enetc_int_vector *v) 688 { 689 struct dim_sample dim_sample = {}; 690 691 v->comp_cnt++; 692 693 if (!v->rx_napi_work) 694 return; 695 696 dim_update_sample(v->comp_cnt, 697 v->rx_ring.stats.packets, 698 v->rx_ring.stats.bytes, 699 &dim_sample); 700 net_dim(&v->rx_dim, dim_sample); 701 } 702 703 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 704 { 705 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 706 707 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 708 } 709 710 static bool enetc_page_reusable(struct page *page) 711 { 712 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 713 } 714 715 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 716 struct enetc_rx_swbd *old) 717 { 718 struct enetc_rx_swbd *new; 719 720 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 721 722 /* next buf that may reuse a page */ 723 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 724 725 /* copy page reference */ 726 *new = *old; 727 } 728 729 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 730 u64 *tstamp) 731 { 732 u32 lo, hi, tstamp_lo; 733 734 lo = enetc_rd_hot(hw, ENETC_SICTR0); 735 hi = enetc_rd_hot(hw, ENETC_SICTR1); 736 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 737 if (lo <= tstamp_lo) 738 hi -= 1; 739 *tstamp = (u64)hi << 32 | tstamp_lo; 740 } 741 742 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 743 { 744 struct skb_shared_hwtstamps shhwtstamps; 745 746 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 747 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 748 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 749 skb_txtime_consumed(skb); 750 skb_tstamp_tx(skb, &shhwtstamps); 751 } 752 } 753 754 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 755 struct enetc_tx_swbd *tx_swbd) 756 { 757 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 758 struct enetc_rx_swbd rx_swbd = { 759 .dma = tx_swbd->dma, 760 .page = tx_swbd->page, 761 .page_offset = tx_swbd->page_offset, 762 .dir = tx_swbd->dir, 763 .len = tx_swbd->len, 764 }; 765 struct enetc_bdr *rx_ring; 766 767 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 768 769 if (likely(enetc_swbd_unused(rx_ring))) { 770 enetc_reuse_page(rx_ring, &rx_swbd); 771 772 /* sync for use by the device */ 773 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 774 rx_swbd.page_offset, 775 ENETC_RXB_DMA_SIZE_XDP, 776 rx_swbd.dir); 777 778 rx_ring->stats.recycles++; 779 } else { 780 /* RX ring is already full, we need to unmap and free the 781 * page, since there's nothing useful we can do with it. 782 */ 783 rx_ring->stats.recycle_failures++; 784 785 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 786 rx_swbd.dir); 787 __free_page(rx_swbd.page); 788 } 789 790 rx_ring->xdp.xdp_tx_in_flight--; 791 } 792 793 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 794 { 795 struct net_device *ndev = tx_ring->ndev; 796 struct enetc_ndev_priv *priv = netdev_priv(ndev); 797 int tx_frm_cnt = 0, tx_byte_cnt = 0; 798 struct enetc_tx_swbd *tx_swbd; 799 int i, bds_to_clean; 800 bool do_twostep_tstamp; 801 u64 tstamp = 0; 802 803 i = tx_ring->next_to_clean; 804 tx_swbd = &tx_ring->tx_swbd[i]; 805 806 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 807 808 do_twostep_tstamp = false; 809 810 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 811 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 812 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 813 bool is_eof = tx_swbd->is_eof; 814 815 if (unlikely(tx_swbd->check_wb)) { 816 struct enetc_ndev_priv *priv = netdev_priv(ndev); 817 union enetc_tx_bd *txbd; 818 819 txbd = ENETC_TXBD(*tx_ring, i); 820 821 if (txbd->flags & ENETC_TXBD_FLAGS_W && 822 tx_swbd->do_twostep_tstamp) { 823 enetc_get_tx_tstamp(&priv->si->hw, txbd, 824 &tstamp); 825 do_twostep_tstamp = true; 826 } 827 } 828 829 if (tx_swbd->is_xdp_tx) 830 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 831 else if (likely(tx_swbd->dma)) 832 enetc_unmap_tx_buff(tx_ring, tx_swbd); 833 834 if (xdp_frame) { 835 xdp_return_frame(xdp_frame); 836 } else if (skb) { 837 if (unlikely(tx_swbd->skb->cb[0] & 838 ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 839 /* Start work to release lock for next one-step 840 * timestamping packet. And send one skb in 841 * tx_skbs queue if has. 842 */ 843 schedule_work(&priv->tx_onestep_tstamp); 844 } else if (unlikely(do_twostep_tstamp)) { 845 enetc_tstamp_tx(skb, tstamp); 846 do_twostep_tstamp = false; 847 } 848 napi_consume_skb(skb, napi_budget); 849 } 850 851 tx_byte_cnt += tx_swbd->len; 852 /* Scrub the swbd here so we don't have to do that 853 * when we reuse it during xmit 854 */ 855 memset(tx_swbd, 0, sizeof(*tx_swbd)); 856 857 bds_to_clean--; 858 tx_swbd++; 859 i++; 860 if (unlikely(i == tx_ring->bd_count)) { 861 i = 0; 862 tx_swbd = tx_ring->tx_swbd; 863 } 864 865 /* BD iteration loop end */ 866 if (is_eof) { 867 tx_frm_cnt++; 868 /* re-arm interrupt source */ 869 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 870 BIT(16 + tx_ring->index)); 871 } 872 873 if (unlikely(!bds_to_clean)) 874 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 875 } 876 877 tx_ring->next_to_clean = i; 878 tx_ring->stats.packets += tx_frm_cnt; 879 tx_ring->stats.bytes += tx_byte_cnt; 880 881 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 882 __netif_subqueue_stopped(ndev, tx_ring->index) && 883 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 884 netif_wake_subqueue(ndev, tx_ring->index); 885 } 886 887 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 888 } 889 890 static bool enetc_new_page(struct enetc_bdr *rx_ring, 891 struct enetc_rx_swbd *rx_swbd) 892 { 893 bool xdp = !!(rx_ring->xdp.prog); 894 struct page *page; 895 dma_addr_t addr; 896 897 page = dev_alloc_page(); 898 if (unlikely(!page)) 899 return false; 900 901 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 902 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 903 904 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 905 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 906 __free_page(page); 907 908 return false; 909 } 910 911 rx_swbd->dma = addr; 912 rx_swbd->page = page; 913 rx_swbd->page_offset = rx_ring->buffer_offset; 914 915 return true; 916 } 917 918 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 919 { 920 struct enetc_rx_swbd *rx_swbd; 921 union enetc_rx_bd *rxbd; 922 int i, j; 923 924 i = rx_ring->next_to_use; 925 rx_swbd = &rx_ring->rx_swbd[i]; 926 rxbd = enetc_rxbd(rx_ring, i); 927 928 for (j = 0; j < buff_cnt; j++) { 929 /* try reuse page */ 930 if (unlikely(!rx_swbd->page)) { 931 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 932 rx_ring->stats.rx_alloc_errs++; 933 break; 934 } 935 } 936 937 /* update RxBD */ 938 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 939 rx_swbd->page_offset); 940 /* clear 'R" as well */ 941 rxbd->r.lstatus = 0; 942 943 enetc_rxbd_next(rx_ring, &rxbd, &i); 944 rx_swbd = &rx_ring->rx_swbd[i]; 945 } 946 947 if (likely(j)) { 948 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 949 rx_ring->next_to_use = i; 950 951 /* update ENETC's consumer index */ 952 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 953 } 954 955 return j; 956 } 957 958 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 959 static void enetc_get_rx_tstamp(struct net_device *ndev, 960 union enetc_rx_bd *rxbd, 961 struct sk_buff *skb) 962 { 963 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 964 struct enetc_ndev_priv *priv = netdev_priv(ndev); 965 struct enetc_hw *hw = &priv->si->hw; 966 u32 lo, hi, tstamp_lo; 967 u64 tstamp; 968 969 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 970 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 971 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 972 rxbd = enetc_rxbd_ext(rxbd); 973 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 974 if (lo <= tstamp_lo) 975 hi -= 1; 976 977 tstamp = (u64)hi << 32 | tstamp_lo; 978 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 979 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 980 } 981 } 982 #endif 983 984 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 985 union enetc_rx_bd *rxbd, struct sk_buff *skb) 986 { 987 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 988 989 /* TODO: hashing */ 990 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 991 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 992 993 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 994 skb->ip_summed = CHECKSUM_COMPLETE; 995 } 996 997 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 998 __be16 tpid = 0; 999 1000 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1001 case 0: 1002 tpid = htons(ETH_P_8021Q); 1003 break; 1004 case 1: 1005 tpid = htons(ETH_P_8021AD); 1006 break; 1007 case 2: 1008 tpid = htons(enetc_port_rd(&priv->si->hw, 1009 ENETC_PCVLANR1)); 1010 break; 1011 case 3: 1012 tpid = htons(enetc_port_rd(&priv->si->hw, 1013 ENETC_PCVLANR2)); 1014 break; 1015 default: 1016 break; 1017 } 1018 1019 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1020 } 1021 1022 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1023 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1024 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1025 #endif 1026 } 1027 1028 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1029 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1030 * mapped buffers. 1031 */ 1032 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1033 int i, u16 size) 1034 { 1035 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1036 1037 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1038 rx_swbd->page_offset, 1039 size, rx_swbd->dir); 1040 return rx_swbd; 1041 } 1042 1043 /* Reuse the current page without performing half-page buffer flipping */ 1044 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1045 struct enetc_rx_swbd *rx_swbd) 1046 { 1047 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1048 1049 enetc_reuse_page(rx_ring, rx_swbd); 1050 1051 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1052 rx_swbd->page_offset, 1053 buffer_size, rx_swbd->dir); 1054 1055 rx_swbd->page = NULL; 1056 } 1057 1058 /* Reuse the current page by performing half-page buffer flipping */ 1059 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1060 struct enetc_rx_swbd *rx_swbd) 1061 { 1062 if (likely(enetc_page_reusable(rx_swbd->page))) { 1063 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1064 page_ref_inc(rx_swbd->page); 1065 1066 enetc_put_rx_buff(rx_ring, rx_swbd); 1067 } else { 1068 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1069 rx_swbd->dir); 1070 rx_swbd->page = NULL; 1071 } 1072 } 1073 1074 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1075 int i, u16 size) 1076 { 1077 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1078 struct sk_buff *skb; 1079 void *ba; 1080 1081 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1082 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1083 if (unlikely(!skb)) { 1084 rx_ring->stats.rx_alloc_errs++; 1085 return NULL; 1086 } 1087 1088 skb_reserve(skb, rx_ring->buffer_offset); 1089 __skb_put(skb, size); 1090 1091 enetc_flip_rx_buff(rx_ring, rx_swbd); 1092 1093 return skb; 1094 } 1095 1096 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1097 u16 size, struct sk_buff *skb) 1098 { 1099 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1100 1101 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1102 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1103 1104 enetc_flip_rx_buff(rx_ring, rx_swbd); 1105 } 1106 1107 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1108 u32 bd_status, 1109 union enetc_rx_bd **rxbd, int *i) 1110 { 1111 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1112 return false; 1113 1114 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1115 enetc_rxbd_next(rx_ring, rxbd, i); 1116 1117 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1118 dma_rmb(); 1119 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1120 1121 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1122 enetc_rxbd_next(rx_ring, rxbd, i); 1123 } 1124 1125 rx_ring->ndev->stats.rx_dropped++; 1126 rx_ring->ndev->stats.rx_errors++; 1127 1128 return true; 1129 } 1130 1131 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1132 u32 bd_status, union enetc_rx_bd **rxbd, 1133 int *i, int *cleaned_cnt, int buffer_size) 1134 { 1135 struct sk_buff *skb; 1136 u16 size; 1137 1138 size = le16_to_cpu((*rxbd)->r.buf_len); 1139 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1140 if (!skb) 1141 return NULL; 1142 1143 enetc_get_offloads(rx_ring, *rxbd, skb); 1144 1145 (*cleaned_cnt)++; 1146 1147 enetc_rxbd_next(rx_ring, rxbd, i); 1148 1149 /* not last BD in frame? */ 1150 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1151 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1152 size = buffer_size; 1153 1154 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1155 dma_rmb(); 1156 size = le16_to_cpu((*rxbd)->r.buf_len); 1157 } 1158 1159 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1160 1161 (*cleaned_cnt)++; 1162 1163 enetc_rxbd_next(rx_ring, rxbd, i); 1164 } 1165 1166 skb_record_rx_queue(skb, rx_ring->index); 1167 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1168 1169 return skb; 1170 } 1171 1172 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1173 1174 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1175 struct napi_struct *napi, int work_limit) 1176 { 1177 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1178 int cleaned_cnt, i; 1179 1180 cleaned_cnt = enetc_bd_unused(rx_ring); 1181 /* next descriptor to process */ 1182 i = rx_ring->next_to_clean; 1183 1184 while (likely(rx_frm_cnt < work_limit)) { 1185 union enetc_rx_bd *rxbd; 1186 struct sk_buff *skb; 1187 u32 bd_status; 1188 1189 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1190 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1191 cleaned_cnt); 1192 1193 rxbd = enetc_rxbd(rx_ring, i); 1194 bd_status = le32_to_cpu(rxbd->r.lstatus); 1195 if (!bd_status) 1196 break; 1197 1198 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1199 dma_rmb(); /* for reading other rxbd fields */ 1200 1201 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1202 &rxbd, &i)) 1203 break; 1204 1205 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1206 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1207 if (!skb) 1208 break; 1209 1210 rx_byte_cnt += skb->len; 1211 rx_frm_cnt++; 1212 1213 napi_gro_receive(napi, skb); 1214 } 1215 1216 rx_ring->next_to_clean = i; 1217 1218 rx_ring->stats.packets += rx_frm_cnt; 1219 rx_ring->stats.bytes += rx_byte_cnt; 1220 1221 return rx_frm_cnt; 1222 } 1223 1224 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1225 struct enetc_tx_swbd *tx_swbd, 1226 int frm_len) 1227 { 1228 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1229 1230 prefetchw(txbd); 1231 1232 enetc_clear_tx_bd(txbd); 1233 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1234 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1235 txbd->frm_len = cpu_to_le16(frm_len); 1236 1237 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1238 } 1239 1240 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1241 * descriptors. 1242 */ 1243 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1244 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1245 { 1246 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1247 int i, k, frm_len = tmp_tx_swbd->len; 1248 1249 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1250 return false; 1251 1252 while (unlikely(!tmp_tx_swbd->is_eof)) { 1253 tmp_tx_swbd++; 1254 frm_len += tmp_tx_swbd->len; 1255 } 1256 1257 i = tx_ring->next_to_use; 1258 1259 for (k = 0; k < num_tx_swbd; k++) { 1260 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1261 1262 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1263 1264 /* last BD needs 'F' bit set */ 1265 if (xdp_tx_swbd->is_eof) { 1266 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1267 1268 txbd->flags = ENETC_TXBD_FLAGS_F; 1269 } 1270 1271 enetc_bdr_idx_inc(tx_ring, &i); 1272 } 1273 1274 tx_ring->next_to_use = i; 1275 1276 return true; 1277 } 1278 1279 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1280 struct enetc_tx_swbd *xdp_tx_arr, 1281 struct xdp_frame *xdp_frame) 1282 { 1283 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1284 struct skb_shared_info *shinfo; 1285 void *data = xdp_frame->data; 1286 int len = xdp_frame->len; 1287 skb_frag_t *frag; 1288 dma_addr_t dma; 1289 unsigned int f; 1290 int n = 0; 1291 1292 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1293 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1294 netdev_err(tx_ring->ndev, "DMA map error\n"); 1295 return -1; 1296 } 1297 1298 xdp_tx_swbd->dma = dma; 1299 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1300 xdp_tx_swbd->len = len; 1301 xdp_tx_swbd->is_xdp_redirect = true; 1302 xdp_tx_swbd->is_eof = false; 1303 xdp_tx_swbd->xdp_frame = NULL; 1304 1305 n++; 1306 xdp_tx_swbd = &xdp_tx_arr[n]; 1307 1308 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1309 1310 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1311 f++, frag++) { 1312 data = skb_frag_address(frag); 1313 len = skb_frag_size(frag); 1314 1315 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1316 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1317 /* Undo the DMA mapping for all fragments */ 1318 while (--n >= 0) 1319 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1320 1321 netdev_err(tx_ring->ndev, "DMA map error\n"); 1322 return -1; 1323 } 1324 1325 xdp_tx_swbd->dma = dma; 1326 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1327 xdp_tx_swbd->len = len; 1328 xdp_tx_swbd->is_xdp_redirect = true; 1329 xdp_tx_swbd->is_eof = false; 1330 xdp_tx_swbd->xdp_frame = NULL; 1331 1332 n++; 1333 xdp_tx_swbd = &xdp_tx_arr[n]; 1334 } 1335 1336 xdp_tx_arr[n - 1].is_eof = true; 1337 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1338 1339 return n; 1340 } 1341 1342 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1343 struct xdp_frame **frames, u32 flags) 1344 { 1345 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1346 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1347 struct enetc_bdr *tx_ring; 1348 int xdp_tx_bd_cnt, i, k; 1349 int xdp_tx_frm_cnt = 0; 1350 1351 enetc_lock_mdio(); 1352 1353 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1354 1355 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1356 1357 for (k = 0; k < num_frames; k++) { 1358 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1359 xdp_redirect_arr, 1360 frames[k]); 1361 if (unlikely(xdp_tx_bd_cnt < 0)) 1362 break; 1363 1364 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1365 xdp_tx_bd_cnt))) { 1366 for (i = 0; i < xdp_tx_bd_cnt; i++) 1367 enetc_unmap_tx_buff(tx_ring, 1368 &xdp_redirect_arr[i]); 1369 tx_ring->stats.xdp_tx_drops++; 1370 break; 1371 } 1372 1373 xdp_tx_frm_cnt++; 1374 } 1375 1376 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1377 enetc_update_tx_ring_tail(tx_ring); 1378 1379 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1380 1381 enetc_unlock_mdio(); 1382 1383 return xdp_tx_frm_cnt; 1384 } 1385 1386 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1387 struct xdp_buff *xdp_buff, u16 size) 1388 { 1389 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1390 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1391 struct skb_shared_info *shinfo; 1392 1393 /* To be used for XDP_TX */ 1394 rx_swbd->len = size; 1395 1396 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1397 rx_ring->buffer_offset, size, false); 1398 1399 shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1400 shinfo->nr_frags = 0; 1401 } 1402 1403 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1404 u16 size, struct xdp_buff *xdp_buff) 1405 { 1406 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1407 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1408 skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags]; 1409 1410 /* To be used for XDP_TX */ 1411 rx_swbd->len = size; 1412 1413 skb_frag_off_set(frag, rx_swbd->page_offset); 1414 skb_frag_size_set(frag, size); 1415 __skb_frag_set_page(frag, rx_swbd->page); 1416 1417 shinfo->nr_frags++; 1418 } 1419 1420 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1421 union enetc_rx_bd **rxbd, int *i, 1422 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1423 { 1424 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1425 1426 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1427 1428 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1429 (*cleaned_cnt)++; 1430 enetc_rxbd_next(rx_ring, rxbd, i); 1431 1432 /* not last BD in frame? */ 1433 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1434 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1435 size = ENETC_RXB_DMA_SIZE_XDP; 1436 1437 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1438 dma_rmb(); 1439 size = le16_to_cpu((*rxbd)->r.buf_len); 1440 } 1441 1442 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1443 (*cleaned_cnt)++; 1444 enetc_rxbd_next(rx_ring, rxbd, i); 1445 } 1446 } 1447 1448 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1449 * recycled back into the RX ring in enetc_clean_tx_ring. 1450 */ 1451 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1452 struct enetc_bdr *rx_ring, 1453 int rx_ring_first, int rx_ring_last) 1454 { 1455 int n = 0; 1456 1457 for (; rx_ring_first != rx_ring_last; 1458 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1459 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1460 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1461 1462 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1463 tx_swbd->dma = rx_swbd->dma; 1464 tx_swbd->dir = rx_swbd->dir; 1465 tx_swbd->page = rx_swbd->page; 1466 tx_swbd->page_offset = rx_swbd->page_offset; 1467 tx_swbd->len = rx_swbd->len; 1468 tx_swbd->is_dma_page = true; 1469 tx_swbd->is_xdp_tx = true; 1470 tx_swbd->is_eof = false; 1471 } 1472 1473 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1474 xdp_tx_arr[n - 1].is_eof = true; 1475 1476 return n; 1477 } 1478 1479 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1480 int rx_ring_last) 1481 { 1482 while (rx_ring_first != rx_ring_last) { 1483 enetc_put_rx_buff(rx_ring, 1484 &rx_ring->rx_swbd[rx_ring_first]); 1485 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1486 } 1487 rx_ring->stats.xdp_drops++; 1488 } 1489 1490 static void enetc_xdp_free(struct enetc_bdr *rx_ring, int rx_ring_first, 1491 int rx_ring_last) 1492 { 1493 while (rx_ring_first != rx_ring_last) { 1494 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1495 1496 if (rx_swbd->page) { 1497 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1498 rx_swbd->dir); 1499 __free_page(rx_swbd->page); 1500 rx_swbd->page = NULL; 1501 } 1502 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1503 } 1504 rx_ring->stats.xdp_redirect_failures++; 1505 } 1506 1507 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1508 struct napi_struct *napi, int work_limit, 1509 struct bpf_prog *prog) 1510 { 1511 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1512 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1513 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1514 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1515 struct enetc_bdr *tx_ring; 1516 int cleaned_cnt, i; 1517 u32 xdp_act; 1518 1519 cleaned_cnt = enetc_bd_unused(rx_ring); 1520 /* next descriptor to process */ 1521 i = rx_ring->next_to_clean; 1522 1523 while (likely(rx_frm_cnt < work_limit)) { 1524 union enetc_rx_bd *rxbd, *orig_rxbd; 1525 int orig_i, orig_cleaned_cnt; 1526 struct xdp_buff xdp_buff; 1527 struct sk_buff *skb; 1528 int tmp_orig_i, err; 1529 u32 bd_status; 1530 1531 rxbd = enetc_rxbd(rx_ring, i); 1532 bd_status = le32_to_cpu(rxbd->r.lstatus); 1533 if (!bd_status) 1534 break; 1535 1536 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1537 dma_rmb(); /* for reading other rxbd fields */ 1538 1539 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1540 &rxbd, &i)) 1541 break; 1542 1543 orig_rxbd = rxbd; 1544 orig_cleaned_cnt = cleaned_cnt; 1545 orig_i = i; 1546 1547 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1548 &cleaned_cnt, &xdp_buff); 1549 1550 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1551 1552 switch (xdp_act) { 1553 default: 1554 bpf_warn_invalid_xdp_action(xdp_act); 1555 fallthrough; 1556 case XDP_ABORTED: 1557 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1558 fallthrough; 1559 case XDP_DROP: 1560 enetc_xdp_drop(rx_ring, orig_i, i); 1561 break; 1562 case XDP_PASS: 1563 rxbd = orig_rxbd; 1564 cleaned_cnt = orig_cleaned_cnt; 1565 i = orig_i; 1566 1567 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1568 &i, &cleaned_cnt, 1569 ENETC_RXB_DMA_SIZE_XDP); 1570 if (unlikely(!skb)) 1571 goto out; 1572 1573 napi_gro_receive(napi, skb); 1574 break; 1575 case XDP_TX: 1576 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1577 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1578 rx_ring, 1579 orig_i, i); 1580 1581 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1582 enetc_xdp_drop(rx_ring, orig_i, i); 1583 tx_ring->stats.xdp_tx_drops++; 1584 } else { 1585 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1586 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1587 xdp_tx_frm_cnt++; 1588 /* The XDP_TX enqueue was successful, so we 1589 * need to scrub the RX software BDs because 1590 * the ownership of the buffers no longer 1591 * belongs to the RX ring, and we must prevent 1592 * enetc_refill_rx_ring() from reusing 1593 * rx_swbd->page. 1594 */ 1595 while (orig_i != i) { 1596 rx_ring->rx_swbd[orig_i].page = NULL; 1597 enetc_bdr_idx_inc(rx_ring, &orig_i); 1598 } 1599 } 1600 break; 1601 case XDP_REDIRECT: 1602 /* xdp_return_frame does not support S/G in the sense 1603 * that it leaks the fragments (__xdp_return should not 1604 * call page_frag_free only for the initial buffer). 1605 * Until XDP_REDIRECT gains support for S/G let's keep 1606 * the code structure in place, but dead. We drop the 1607 * S/G frames ourselves to avoid memory leaks which 1608 * would otherwise leave the kernel OOM. 1609 */ 1610 if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) { 1611 enetc_xdp_drop(rx_ring, orig_i, i); 1612 rx_ring->stats.xdp_redirect_sg++; 1613 break; 1614 } 1615 1616 tmp_orig_i = orig_i; 1617 1618 while (orig_i != i) { 1619 enetc_flip_rx_buff(rx_ring, 1620 &rx_ring->rx_swbd[orig_i]); 1621 enetc_bdr_idx_inc(rx_ring, &orig_i); 1622 } 1623 1624 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1625 if (unlikely(err)) { 1626 enetc_xdp_free(rx_ring, tmp_orig_i, i); 1627 } else { 1628 xdp_redirect_frm_cnt++; 1629 rx_ring->stats.xdp_redirect++; 1630 } 1631 } 1632 1633 rx_frm_cnt++; 1634 } 1635 1636 out: 1637 rx_ring->next_to_clean = i; 1638 1639 rx_ring->stats.packets += rx_frm_cnt; 1640 rx_ring->stats.bytes += rx_byte_cnt; 1641 1642 if (xdp_redirect_frm_cnt) 1643 xdp_do_flush_map(); 1644 1645 if (xdp_tx_frm_cnt) 1646 enetc_update_tx_ring_tail(tx_ring); 1647 1648 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1649 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1650 rx_ring->xdp.xdp_tx_in_flight); 1651 1652 return rx_frm_cnt; 1653 } 1654 1655 static int enetc_poll(struct napi_struct *napi, int budget) 1656 { 1657 struct enetc_int_vector 1658 *v = container_of(napi, struct enetc_int_vector, napi); 1659 struct enetc_bdr *rx_ring = &v->rx_ring; 1660 struct bpf_prog *prog; 1661 bool complete = true; 1662 int work_done; 1663 int i; 1664 1665 enetc_lock_mdio(); 1666 1667 for (i = 0; i < v->count_tx_rings; i++) 1668 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1669 complete = false; 1670 1671 prog = rx_ring->xdp.prog; 1672 if (prog) 1673 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1674 else 1675 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1676 if (work_done == budget) 1677 complete = false; 1678 if (work_done) 1679 v->rx_napi_work = true; 1680 1681 if (!complete) { 1682 enetc_unlock_mdio(); 1683 return budget; 1684 } 1685 1686 napi_complete_done(napi, work_done); 1687 1688 if (likely(v->rx_dim_en)) 1689 enetc_rx_net_dim(v); 1690 1691 v->rx_napi_work = false; 1692 1693 /* enable interrupts */ 1694 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1695 1696 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1697 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1698 ENETC_TBIER_TXTIE); 1699 1700 enetc_unlock_mdio(); 1701 1702 return work_done; 1703 } 1704 1705 /* Probing and Init */ 1706 #define ENETC_MAX_RFS_SIZE 64 1707 void enetc_get_si_caps(struct enetc_si *si) 1708 { 1709 struct enetc_hw *hw = &si->hw; 1710 u32 val; 1711 1712 /* find out how many of various resources we have to work with */ 1713 val = enetc_rd(hw, ENETC_SICAPR0); 1714 si->num_rx_rings = (val >> 16) & 0xff; 1715 si->num_tx_rings = val & 0xff; 1716 1717 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1718 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1719 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1720 1721 si->num_rss = 0; 1722 val = enetc_rd(hw, ENETC_SIPCAPR0); 1723 if (val & ENETC_SIPCAPR0_RSS) { 1724 u32 rss; 1725 1726 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1727 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1728 } 1729 1730 if (val & ENETC_SIPCAPR0_QBV) 1731 si->hw_features |= ENETC_SI_F_QBV; 1732 1733 if (val & ENETC_SIPCAPR0_PSFP) 1734 si->hw_features |= ENETC_SI_F_PSFP; 1735 } 1736 1737 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 1738 { 1739 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 1740 &r->bd_dma_base, GFP_KERNEL); 1741 if (!r->bd_base) 1742 return -ENOMEM; 1743 1744 /* h/w requires 128B alignment */ 1745 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 1746 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 1747 r->bd_dma_base); 1748 return -EINVAL; 1749 } 1750 1751 return 0; 1752 } 1753 1754 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 1755 { 1756 int err; 1757 1758 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 1759 if (!txr->tx_swbd) 1760 return -ENOMEM; 1761 1762 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 1763 if (err) 1764 goto err_alloc_bdr; 1765 1766 txr->tso_headers = dma_alloc_coherent(txr->dev, 1767 txr->bd_count * TSO_HEADER_SIZE, 1768 &txr->tso_headers_dma, 1769 GFP_KERNEL); 1770 if (!txr->tso_headers) { 1771 err = -ENOMEM; 1772 goto err_alloc_tso; 1773 } 1774 1775 txr->next_to_clean = 0; 1776 txr->next_to_use = 0; 1777 1778 return 0; 1779 1780 err_alloc_tso: 1781 dma_free_coherent(txr->dev, txr->bd_count * sizeof(union enetc_tx_bd), 1782 txr->bd_base, txr->bd_dma_base); 1783 txr->bd_base = NULL; 1784 err_alloc_bdr: 1785 vfree(txr->tx_swbd); 1786 txr->tx_swbd = NULL; 1787 1788 return err; 1789 } 1790 1791 static void enetc_free_txbdr(struct enetc_bdr *txr) 1792 { 1793 int size, i; 1794 1795 for (i = 0; i < txr->bd_count; i++) 1796 enetc_free_tx_frame(txr, &txr->tx_swbd[i]); 1797 1798 size = txr->bd_count * sizeof(union enetc_tx_bd); 1799 1800 dma_free_coherent(txr->dev, txr->bd_count * TSO_HEADER_SIZE, 1801 txr->tso_headers, txr->tso_headers_dma); 1802 txr->tso_headers = NULL; 1803 1804 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 1805 txr->bd_base = NULL; 1806 1807 vfree(txr->tx_swbd); 1808 txr->tx_swbd = NULL; 1809 } 1810 1811 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1812 { 1813 int i, err; 1814 1815 for (i = 0; i < priv->num_tx_rings; i++) { 1816 err = enetc_alloc_txbdr(priv->tx_ring[i]); 1817 1818 if (err) 1819 goto fail; 1820 } 1821 1822 return 0; 1823 1824 fail: 1825 while (i-- > 0) 1826 enetc_free_txbdr(priv->tx_ring[i]); 1827 1828 return err; 1829 } 1830 1831 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 1832 { 1833 int i; 1834 1835 for (i = 0; i < priv->num_tx_rings; i++) 1836 enetc_free_txbdr(priv->tx_ring[i]); 1837 } 1838 1839 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 1840 { 1841 size_t size = sizeof(union enetc_rx_bd); 1842 int err; 1843 1844 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 1845 if (!rxr->rx_swbd) 1846 return -ENOMEM; 1847 1848 if (extended) 1849 size *= 2; 1850 1851 err = enetc_dma_alloc_bdr(rxr, size); 1852 if (err) { 1853 vfree(rxr->rx_swbd); 1854 return err; 1855 } 1856 1857 rxr->next_to_clean = 0; 1858 rxr->next_to_use = 0; 1859 rxr->next_to_alloc = 0; 1860 rxr->ext_en = extended; 1861 1862 return 0; 1863 } 1864 1865 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 1866 { 1867 int size; 1868 1869 size = rxr->bd_count * sizeof(union enetc_rx_bd); 1870 1871 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 1872 rxr->bd_base = NULL; 1873 1874 vfree(rxr->rx_swbd); 1875 rxr->rx_swbd = NULL; 1876 } 1877 1878 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 1879 { 1880 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 1881 int i, err; 1882 1883 for (i = 0; i < priv->num_rx_rings; i++) { 1884 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 1885 1886 if (err) 1887 goto fail; 1888 } 1889 1890 return 0; 1891 1892 fail: 1893 while (i-- > 0) 1894 enetc_free_rxbdr(priv->rx_ring[i]); 1895 1896 return err; 1897 } 1898 1899 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 1900 { 1901 int i; 1902 1903 for (i = 0; i < priv->num_rx_rings; i++) 1904 enetc_free_rxbdr(priv->rx_ring[i]); 1905 } 1906 1907 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 1908 { 1909 int i; 1910 1911 if (!tx_ring->tx_swbd) 1912 return; 1913 1914 for (i = 0; i < tx_ring->bd_count; i++) { 1915 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 1916 1917 enetc_free_tx_frame(tx_ring, tx_swbd); 1918 } 1919 1920 tx_ring->next_to_clean = 0; 1921 tx_ring->next_to_use = 0; 1922 } 1923 1924 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 1925 { 1926 int i; 1927 1928 if (!rx_ring->rx_swbd) 1929 return; 1930 1931 for (i = 0; i < rx_ring->bd_count; i++) { 1932 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1933 1934 if (!rx_swbd->page) 1935 continue; 1936 1937 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1938 rx_swbd->dir); 1939 __free_page(rx_swbd->page); 1940 rx_swbd->page = NULL; 1941 } 1942 1943 rx_ring->next_to_clean = 0; 1944 rx_ring->next_to_use = 0; 1945 rx_ring->next_to_alloc = 0; 1946 } 1947 1948 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 1949 { 1950 int i; 1951 1952 for (i = 0; i < priv->num_rx_rings; i++) 1953 enetc_free_rx_ring(priv->rx_ring[i]); 1954 1955 for (i = 0; i < priv->num_tx_rings; i++) 1956 enetc_free_tx_ring(priv->tx_ring[i]); 1957 } 1958 1959 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1960 { 1961 int *rss_table; 1962 int i; 1963 1964 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1965 if (!rss_table) 1966 return -ENOMEM; 1967 1968 /* Set up RSS table defaults */ 1969 for (i = 0; i < si->num_rss; i++) 1970 rss_table[i] = i % num_groups; 1971 1972 enetc_set_rss_table(si, rss_table, si->num_rss); 1973 1974 kfree(rss_table); 1975 1976 return 0; 1977 } 1978 1979 int enetc_configure_si(struct enetc_ndev_priv *priv) 1980 { 1981 struct enetc_si *si = priv->si; 1982 struct enetc_hw *hw = &si->hw; 1983 int err; 1984 1985 /* set SI cache attributes */ 1986 enetc_wr(hw, ENETC_SICAR0, 1987 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1988 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1989 /* enable SI */ 1990 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1991 1992 if (si->num_rss) { 1993 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1994 if (err) 1995 return err; 1996 } 1997 1998 return 0; 1999 } 2000 2001 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2002 { 2003 struct enetc_si *si = priv->si; 2004 int cpus = num_online_cpus(); 2005 2006 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2007 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2008 2009 /* Enable all available TX rings in order to configure as many 2010 * priorities as possible, when needed. 2011 * TODO: Make # of TX rings run-time configurable 2012 */ 2013 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2014 priv->num_tx_rings = si->num_tx_rings; 2015 priv->bdr_int_num = cpus; 2016 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2017 priv->tx_ictt = ENETC_TXIC_TIMETHR; 2018 } 2019 2020 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2021 { 2022 struct enetc_si *si = priv->si; 2023 2024 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2025 GFP_KERNEL); 2026 if (!priv->cls_rules) 2027 return -ENOMEM; 2028 2029 return 0; 2030 } 2031 2032 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2033 { 2034 kfree(priv->cls_rules); 2035 } 2036 2037 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2038 { 2039 int idx = tx_ring->index; 2040 u32 tbmr; 2041 2042 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2043 lower_32_bits(tx_ring->bd_dma_base)); 2044 2045 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2046 upper_32_bits(tx_ring->bd_dma_base)); 2047 2048 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2049 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2050 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2051 2052 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2053 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2054 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2055 2056 /* enable Tx ints by setting pkt thr to 1 */ 2057 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2058 2059 tbmr = ENETC_TBMR_EN; 2060 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2061 tbmr |= ENETC_TBMR_VIH; 2062 2063 /* enable ring */ 2064 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2065 2066 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2067 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2068 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2069 } 2070 2071 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2072 { 2073 int idx = rx_ring->index; 2074 u32 rbmr; 2075 2076 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2077 lower_32_bits(rx_ring->bd_dma_base)); 2078 2079 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2080 upper_32_bits(rx_ring->bd_dma_base)); 2081 2082 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2083 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2084 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2085 2086 if (rx_ring->xdp.prog) 2087 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2088 else 2089 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2090 2091 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2092 2093 /* enable Rx ints by setting pkt thr to 1 */ 2094 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2095 2096 rbmr = ENETC_RBMR_EN; 2097 2098 if (rx_ring->ext_en) 2099 rbmr |= ENETC_RBMR_BDS; 2100 2101 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2102 rbmr |= ENETC_RBMR_VTE; 2103 2104 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2105 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2106 2107 enetc_lock_mdio(); 2108 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2109 enetc_unlock_mdio(); 2110 2111 /* enable ring */ 2112 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2113 } 2114 2115 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 2116 { 2117 int i; 2118 2119 for (i = 0; i < priv->num_tx_rings; i++) 2120 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 2121 2122 for (i = 0; i < priv->num_rx_rings; i++) 2123 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 2124 } 2125 2126 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2127 { 2128 int idx = rx_ring->index; 2129 2130 /* disable EN bit on ring */ 2131 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2132 } 2133 2134 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2135 { 2136 int delay = 8, timeout = 100; 2137 int idx = tx_ring->index; 2138 2139 /* disable EN bit on ring */ 2140 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2141 2142 /* wait for busy to clear */ 2143 while (delay < timeout && 2144 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2145 msleep(delay); 2146 delay *= 2; 2147 } 2148 2149 if (delay >= timeout) 2150 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2151 idx); 2152 } 2153 2154 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 2155 { 2156 int i; 2157 2158 for (i = 0; i < priv->num_tx_rings; i++) 2159 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 2160 2161 for (i = 0; i < priv->num_rx_rings; i++) 2162 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 2163 2164 udelay(1); 2165 } 2166 2167 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2168 { 2169 struct pci_dev *pdev = priv->si->pdev; 2170 int i, j, err; 2171 2172 for (i = 0; i < priv->bdr_int_num; i++) { 2173 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2174 struct enetc_int_vector *v = priv->int_vector[i]; 2175 int entry = ENETC_BDR_INT_BASE_IDX + i; 2176 struct enetc_hw *hw = &priv->si->hw; 2177 2178 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2179 priv->ndev->name, i); 2180 err = request_irq(irq, enetc_msix, 0, v->name, v); 2181 if (err) { 2182 dev_err(priv->dev, "request_irq() failed!\n"); 2183 goto irq_err; 2184 } 2185 disable_irq(irq); 2186 2187 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2188 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2189 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2190 2191 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2192 2193 for (j = 0; j < v->count_tx_rings; j++) { 2194 int idx = v->tx_ring[j].index; 2195 2196 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2197 } 2198 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2199 } 2200 2201 return 0; 2202 2203 irq_err: 2204 while (i--) { 2205 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2206 2207 irq_set_affinity_hint(irq, NULL); 2208 free_irq(irq, priv->int_vector[i]); 2209 } 2210 2211 return err; 2212 } 2213 2214 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2215 { 2216 struct pci_dev *pdev = priv->si->pdev; 2217 int i; 2218 2219 for (i = 0; i < priv->bdr_int_num; i++) { 2220 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2221 2222 irq_set_affinity_hint(irq, NULL); 2223 free_irq(irq, priv->int_vector[i]); 2224 } 2225 } 2226 2227 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2228 { 2229 struct enetc_hw *hw = &priv->si->hw; 2230 u32 icpt, ictt; 2231 int i; 2232 2233 /* enable Tx & Rx event indication */ 2234 if (priv->ic_mode & 2235 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2236 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2237 /* init to non-0 minimum, will be adjusted later */ 2238 ictt = 0x1; 2239 } else { 2240 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2241 ictt = 0; 2242 } 2243 2244 for (i = 0; i < priv->num_rx_rings; i++) { 2245 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2246 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2247 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2248 } 2249 2250 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2251 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2252 else 2253 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2254 2255 for (i = 0; i < priv->num_tx_rings; i++) { 2256 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2257 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2258 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2259 } 2260 } 2261 2262 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2263 { 2264 int i; 2265 2266 for (i = 0; i < priv->num_tx_rings; i++) 2267 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 2268 2269 for (i = 0; i < priv->num_rx_rings; i++) 2270 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 2271 } 2272 2273 static int enetc_phylink_connect(struct net_device *ndev) 2274 { 2275 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2276 struct ethtool_eee edata; 2277 int err; 2278 2279 if (!priv->phylink) 2280 return 0; /* phy-less mode */ 2281 2282 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2283 if (err) { 2284 dev_err(&ndev->dev, "could not attach to PHY\n"); 2285 return err; 2286 } 2287 2288 /* disable EEE autoneg, until ENETC driver supports it */ 2289 memset(&edata, 0, sizeof(struct ethtool_eee)); 2290 phylink_ethtool_set_eee(priv->phylink, &edata); 2291 2292 return 0; 2293 } 2294 2295 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2296 { 2297 struct enetc_ndev_priv *priv; 2298 struct sk_buff *skb; 2299 2300 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2301 2302 netif_tx_lock(priv->ndev); 2303 2304 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2305 skb = skb_dequeue(&priv->tx_skbs); 2306 if (skb) 2307 enetc_start_xmit(skb, priv->ndev); 2308 2309 netif_tx_unlock(priv->ndev); 2310 } 2311 2312 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2313 { 2314 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2315 skb_queue_head_init(&priv->tx_skbs); 2316 } 2317 2318 void enetc_start(struct net_device *ndev) 2319 { 2320 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2321 int i; 2322 2323 enetc_setup_interrupts(priv); 2324 2325 for (i = 0; i < priv->bdr_int_num; i++) { 2326 int irq = pci_irq_vector(priv->si->pdev, 2327 ENETC_BDR_INT_BASE_IDX + i); 2328 2329 napi_enable(&priv->int_vector[i]->napi); 2330 enable_irq(irq); 2331 } 2332 2333 if (priv->phylink) 2334 phylink_start(priv->phylink); 2335 else 2336 netif_carrier_on(ndev); 2337 2338 netif_tx_start_all_queues(ndev); 2339 } 2340 2341 int enetc_open(struct net_device *ndev) 2342 { 2343 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2344 int num_stack_tx_queues; 2345 int err; 2346 2347 err = enetc_setup_irqs(priv); 2348 if (err) 2349 return err; 2350 2351 err = enetc_phylink_connect(ndev); 2352 if (err) 2353 goto err_phy_connect; 2354 2355 err = enetc_alloc_tx_resources(priv); 2356 if (err) 2357 goto err_alloc_tx; 2358 2359 err = enetc_alloc_rx_resources(priv); 2360 if (err) 2361 goto err_alloc_rx; 2362 2363 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2364 2365 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2366 if (err) 2367 goto err_set_queues; 2368 2369 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 2370 if (err) 2371 goto err_set_queues; 2372 2373 enetc_tx_onestep_tstamp_init(priv); 2374 enetc_setup_bdrs(priv); 2375 enetc_start(ndev); 2376 2377 return 0; 2378 2379 err_set_queues: 2380 enetc_free_rx_resources(priv); 2381 err_alloc_rx: 2382 enetc_free_tx_resources(priv); 2383 err_alloc_tx: 2384 if (priv->phylink) 2385 phylink_disconnect_phy(priv->phylink); 2386 err_phy_connect: 2387 enetc_free_irqs(priv); 2388 2389 return err; 2390 } 2391 2392 void enetc_stop(struct net_device *ndev) 2393 { 2394 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2395 int i; 2396 2397 netif_tx_stop_all_queues(ndev); 2398 2399 for (i = 0; i < priv->bdr_int_num; i++) { 2400 int irq = pci_irq_vector(priv->si->pdev, 2401 ENETC_BDR_INT_BASE_IDX + i); 2402 2403 disable_irq(irq); 2404 napi_synchronize(&priv->int_vector[i]->napi); 2405 napi_disable(&priv->int_vector[i]->napi); 2406 } 2407 2408 if (priv->phylink) 2409 phylink_stop(priv->phylink); 2410 else 2411 netif_carrier_off(ndev); 2412 2413 enetc_clear_interrupts(priv); 2414 } 2415 2416 int enetc_close(struct net_device *ndev) 2417 { 2418 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2419 2420 enetc_stop(ndev); 2421 enetc_clear_bdrs(priv); 2422 2423 if (priv->phylink) 2424 phylink_disconnect_phy(priv->phylink); 2425 enetc_free_rxtx_rings(priv); 2426 enetc_free_rx_resources(priv); 2427 enetc_free_tx_resources(priv); 2428 enetc_free_irqs(priv); 2429 2430 return 0; 2431 } 2432 2433 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2434 { 2435 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2436 struct tc_mqprio_qopt *mqprio = type_data; 2437 struct enetc_bdr *tx_ring; 2438 int num_stack_tx_queues; 2439 u8 num_tc; 2440 int i; 2441 2442 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2443 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2444 num_tc = mqprio->num_tc; 2445 2446 if (!num_tc) { 2447 netdev_reset_tc(ndev); 2448 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2449 2450 /* Reset all ring priorities to 0 */ 2451 for (i = 0; i < priv->num_tx_rings; i++) { 2452 tx_ring = priv->tx_ring[i]; 2453 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 2454 } 2455 2456 return 0; 2457 } 2458 2459 /* Check if we have enough BD rings available to accommodate all TCs */ 2460 if (num_tc > num_stack_tx_queues) { 2461 netdev_err(ndev, "Max %d traffic classes supported\n", 2462 priv->num_tx_rings); 2463 return -EINVAL; 2464 } 2465 2466 /* For the moment, we use only one BD ring per TC. 2467 * 2468 * Configure num_tc BD rings with increasing priorities. 2469 */ 2470 for (i = 0; i < num_tc; i++) { 2471 tx_ring = priv->tx_ring[i]; 2472 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 2473 } 2474 2475 /* Reset the number of netdev queues based on the TC count */ 2476 netif_set_real_num_tx_queues(ndev, num_tc); 2477 2478 netdev_set_num_tc(ndev, num_tc); 2479 2480 /* Each TC is associated with one netdev queue */ 2481 for (i = 0; i < num_tc; i++) 2482 netdev_set_tc_queue(ndev, i, 1, i); 2483 2484 return 0; 2485 } 2486 2487 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 2488 void *type_data) 2489 { 2490 switch (type) { 2491 case TC_SETUP_QDISC_MQPRIO: 2492 return enetc_setup_tc_mqprio(ndev, type_data); 2493 case TC_SETUP_QDISC_TAPRIO: 2494 return enetc_setup_tc_taprio(ndev, type_data); 2495 case TC_SETUP_QDISC_CBS: 2496 return enetc_setup_tc_cbs(ndev, type_data); 2497 case TC_SETUP_QDISC_ETF: 2498 return enetc_setup_tc_txtime(ndev, type_data); 2499 case TC_SETUP_BLOCK: 2500 return enetc_setup_tc_psfp(ndev, type_data); 2501 default: 2502 return -EOPNOTSUPP; 2503 } 2504 } 2505 2506 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog, 2507 struct netlink_ext_ack *extack) 2508 { 2509 struct enetc_ndev_priv *priv = netdev_priv(dev); 2510 struct bpf_prog *old_prog; 2511 bool is_up; 2512 int i; 2513 2514 /* The buffer layout is changing, so we need to drain the old 2515 * RX buffers and seed new ones. 2516 */ 2517 is_up = netif_running(dev); 2518 if (is_up) 2519 dev_close(dev); 2520 2521 old_prog = xchg(&priv->xdp_prog, prog); 2522 if (old_prog) 2523 bpf_prog_put(old_prog); 2524 2525 for (i = 0; i < priv->num_rx_rings; i++) { 2526 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2527 2528 rx_ring->xdp.prog = prog; 2529 2530 if (prog) 2531 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2532 else 2533 rx_ring->buffer_offset = ENETC_RXB_PAD; 2534 } 2535 2536 if (is_up) 2537 return dev_open(dev, extack); 2538 2539 return 0; 2540 } 2541 2542 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp) 2543 { 2544 switch (xdp->command) { 2545 case XDP_SETUP_PROG: 2546 return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack); 2547 default: 2548 return -EINVAL; 2549 } 2550 2551 return 0; 2552 } 2553 2554 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2555 { 2556 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2557 struct net_device_stats *stats = &ndev->stats; 2558 unsigned long packets = 0, bytes = 0; 2559 int i; 2560 2561 for (i = 0; i < priv->num_rx_rings; i++) { 2562 packets += priv->rx_ring[i]->stats.packets; 2563 bytes += priv->rx_ring[i]->stats.bytes; 2564 } 2565 2566 stats->rx_packets = packets; 2567 stats->rx_bytes = bytes; 2568 bytes = 0; 2569 packets = 0; 2570 2571 for (i = 0; i < priv->num_tx_rings; i++) { 2572 packets += priv->tx_ring[i]->stats.packets; 2573 bytes += priv->tx_ring[i]->stats.bytes; 2574 } 2575 2576 stats->tx_packets = packets; 2577 stats->tx_bytes = bytes; 2578 2579 return stats; 2580 } 2581 2582 static int enetc_set_rss(struct net_device *ndev, int en) 2583 { 2584 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2585 struct enetc_hw *hw = &priv->si->hw; 2586 u32 reg; 2587 2588 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2589 2590 reg = enetc_rd(hw, ENETC_SIMR); 2591 reg &= ~ENETC_SIMR_RSSE; 2592 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2593 enetc_wr(hw, ENETC_SIMR, reg); 2594 2595 return 0; 2596 } 2597 2598 static int enetc_set_psfp(struct net_device *ndev, int en) 2599 { 2600 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2601 int err; 2602 2603 if (en) { 2604 err = enetc_psfp_enable(priv); 2605 if (err) 2606 return err; 2607 2608 priv->active_offloads |= ENETC_F_QCI; 2609 return 0; 2610 } 2611 2612 err = enetc_psfp_disable(priv); 2613 if (err) 2614 return err; 2615 2616 priv->active_offloads &= ~ENETC_F_QCI; 2617 2618 return 0; 2619 } 2620 2621 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2622 { 2623 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2624 int i; 2625 2626 for (i = 0; i < priv->num_rx_rings; i++) 2627 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en); 2628 } 2629 2630 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2631 { 2632 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2633 int i; 2634 2635 for (i = 0; i < priv->num_tx_rings; i++) 2636 enetc_bdr_enable_txvlan(&priv->si->hw, i, en); 2637 } 2638 2639 int enetc_set_features(struct net_device *ndev, 2640 netdev_features_t features) 2641 { 2642 netdev_features_t changed = ndev->features ^ features; 2643 int err = 0; 2644 2645 if (changed & NETIF_F_RXHASH) 2646 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2647 2648 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2649 enetc_enable_rxvlan(ndev, 2650 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2651 2652 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2653 enetc_enable_txvlan(ndev, 2654 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2655 2656 if (changed & NETIF_F_HW_TC) 2657 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); 2658 2659 return err; 2660 } 2661 2662 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2663 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2664 { 2665 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2666 struct hwtstamp_config config; 2667 int ao; 2668 2669 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2670 return -EFAULT; 2671 2672 switch (config.tx_type) { 2673 case HWTSTAMP_TX_OFF: 2674 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2675 break; 2676 case HWTSTAMP_TX_ON: 2677 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2678 priv->active_offloads |= ENETC_F_TX_TSTAMP; 2679 break; 2680 case HWTSTAMP_TX_ONESTEP_SYNC: 2681 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2682 priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2683 break; 2684 default: 2685 return -ERANGE; 2686 } 2687 2688 ao = priv->active_offloads; 2689 switch (config.rx_filter) { 2690 case HWTSTAMP_FILTER_NONE: 2691 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 2692 break; 2693 default: 2694 priv->active_offloads |= ENETC_F_RX_TSTAMP; 2695 config.rx_filter = HWTSTAMP_FILTER_ALL; 2696 } 2697 2698 if (netif_running(ndev) && ao != priv->active_offloads) { 2699 enetc_close(ndev); 2700 enetc_open(ndev); 2701 } 2702 2703 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2704 -EFAULT : 0; 2705 } 2706 2707 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2708 { 2709 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2710 struct hwtstamp_config config; 2711 2712 config.flags = 0; 2713 2714 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2715 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2716 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2717 config.tx_type = HWTSTAMP_TX_ON; 2718 else 2719 config.tx_type = HWTSTAMP_TX_OFF; 2720 2721 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2722 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2723 2724 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2725 -EFAULT : 0; 2726 } 2727 #endif 2728 2729 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2730 { 2731 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2732 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2733 if (cmd == SIOCSHWTSTAMP) 2734 return enetc_hwtstamp_set(ndev, rq); 2735 if (cmd == SIOCGHWTSTAMP) 2736 return enetc_hwtstamp_get(ndev, rq); 2737 #endif 2738 2739 if (!priv->phylink) 2740 return -EOPNOTSUPP; 2741 2742 return phylink_mii_ioctl(priv->phylink, rq, cmd); 2743 } 2744 2745 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 2746 { 2747 struct pci_dev *pdev = priv->si->pdev; 2748 int first_xdp_tx_ring; 2749 int i, n, err, nvec; 2750 int v_tx_rings; 2751 2752 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 2753 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 2754 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 2755 2756 if (n < 0) 2757 return n; 2758 2759 if (n != nvec) 2760 return -EPERM; 2761 2762 /* # of tx rings per int vector */ 2763 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 2764 2765 for (i = 0; i < priv->bdr_int_num; i++) { 2766 struct enetc_int_vector *v; 2767 struct enetc_bdr *bdr; 2768 int j; 2769 2770 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 2771 if (!v) { 2772 err = -ENOMEM; 2773 goto fail; 2774 } 2775 2776 priv->int_vector[i] = v; 2777 2778 bdr = &v->rx_ring; 2779 bdr->index = i; 2780 bdr->ndev = priv->ndev; 2781 bdr->dev = priv->dev; 2782 bdr->bd_count = priv->rx_bd_count; 2783 bdr->buffer_offset = ENETC_RXB_PAD; 2784 priv->rx_ring[i] = bdr; 2785 2786 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 2787 if (err) { 2788 kfree(v); 2789 goto fail; 2790 } 2791 2792 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, 2793 MEM_TYPE_PAGE_SHARED, NULL); 2794 if (err) { 2795 xdp_rxq_info_unreg(&bdr->xdp.rxq); 2796 kfree(v); 2797 goto fail; 2798 } 2799 2800 /* init defaults for adaptive IC */ 2801 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 2802 v->rx_ictt = 0x1; 2803 v->rx_dim_en = true; 2804 } 2805 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 2806 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 2807 NAPI_POLL_WEIGHT); 2808 v->count_tx_rings = v_tx_rings; 2809 2810 for (j = 0; j < v_tx_rings; j++) { 2811 int idx; 2812 2813 /* default tx ring mapping policy */ 2814 idx = priv->bdr_int_num * j + i; 2815 __set_bit(idx, &v->tx_rings_map); 2816 bdr = &v->tx_ring[j]; 2817 bdr->index = idx; 2818 bdr->ndev = priv->ndev; 2819 bdr->dev = priv->dev; 2820 bdr->bd_count = priv->tx_bd_count; 2821 priv->tx_ring[idx] = bdr; 2822 } 2823 } 2824 2825 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 2826 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 2827 2828 return 0; 2829 2830 fail: 2831 while (i--) { 2832 struct enetc_int_vector *v = priv->int_vector[i]; 2833 struct enetc_bdr *rx_ring = &v->rx_ring; 2834 2835 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 2836 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 2837 netif_napi_del(&v->napi); 2838 cancel_work_sync(&v->rx_dim.work); 2839 kfree(v); 2840 } 2841 2842 pci_free_irq_vectors(pdev); 2843 2844 return err; 2845 } 2846 2847 void enetc_free_msix(struct enetc_ndev_priv *priv) 2848 { 2849 int i; 2850 2851 for (i = 0; i < priv->bdr_int_num; i++) { 2852 struct enetc_int_vector *v = priv->int_vector[i]; 2853 struct enetc_bdr *rx_ring = &v->rx_ring; 2854 2855 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 2856 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 2857 netif_napi_del(&v->napi); 2858 cancel_work_sync(&v->rx_dim.work); 2859 } 2860 2861 for (i = 0; i < priv->num_rx_rings; i++) 2862 priv->rx_ring[i] = NULL; 2863 2864 for (i = 0; i < priv->num_tx_rings; i++) 2865 priv->tx_ring[i] = NULL; 2866 2867 for (i = 0; i < priv->bdr_int_num; i++) { 2868 kfree(priv->int_vector[i]); 2869 priv->int_vector[i] = NULL; 2870 } 2871 2872 /* disable all MSIX for this device */ 2873 pci_free_irq_vectors(priv->si->pdev); 2874 } 2875 2876 static void enetc_kfree_si(struct enetc_si *si) 2877 { 2878 char *p = (char *)si - si->pad; 2879 2880 kfree(p); 2881 } 2882 2883 static void enetc_detect_errata(struct enetc_si *si) 2884 { 2885 if (si->pdev->revision == ENETC_REV1) 2886 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 2887 } 2888 2889 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 2890 { 2891 struct enetc_si *si, *p; 2892 struct enetc_hw *hw; 2893 size_t alloc_size; 2894 int err, len; 2895 2896 pcie_flr(pdev); 2897 err = pci_enable_device_mem(pdev); 2898 if (err) 2899 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 2900 2901 /* set up for high or low dma */ 2902 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2903 if (err) { 2904 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 2905 if (err) { 2906 dev_err(&pdev->dev, 2907 "DMA configuration failed: 0x%x\n", err); 2908 goto err_dma; 2909 } 2910 } 2911 2912 err = pci_request_mem_regions(pdev, name); 2913 if (err) { 2914 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 2915 goto err_pci_mem_reg; 2916 } 2917 2918 pci_set_master(pdev); 2919 2920 alloc_size = sizeof(struct enetc_si); 2921 if (sizeof_priv) { 2922 /* align priv to 32B */ 2923 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 2924 alloc_size += sizeof_priv; 2925 } 2926 /* force 32B alignment for enetc_si */ 2927 alloc_size += ENETC_SI_ALIGN - 1; 2928 2929 p = kzalloc(alloc_size, GFP_KERNEL); 2930 if (!p) { 2931 err = -ENOMEM; 2932 goto err_alloc_si; 2933 } 2934 2935 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 2936 si->pad = (char *)si - (char *)p; 2937 2938 pci_set_drvdata(pdev, si); 2939 si->pdev = pdev; 2940 hw = &si->hw; 2941 2942 len = pci_resource_len(pdev, ENETC_BAR_REGS); 2943 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 2944 if (!hw->reg) { 2945 err = -ENXIO; 2946 dev_err(&pdev->dev, "ioremap() failed\n"); 2947 goto err_ioremap; 2948 } 2949 if (len > ENETC_PORT_BASE) 2950 hw->port = hw->reg + ENETC_PORT_BASE; 2951 if (len > ENETC_GLOBAL_BASE) 2952 hw->global = hw->reg + ENETC_GLOBAL_BASE; 2953 2954 enetc_detect_errata(si); 2955 2956 return 0; 2957 2958 err_ioremap: 2959 enetc_kfree_si(si); 2960 err_alloc_si: 2961 pci_release_mem_regions(pdev); 2962 err_pci_mem_reg: 2963 err_dma: 2964 pci_disable_device(pdev); 2965 2966 return err; 2967 } 2968 2969 void enetc_pci_remove(struct pci_dev *pdev) 2970 { 2971 struct enetc_si *si = pci_get_drvdata(pdev); 2972 struct enetc_hw *hw = &si->hw; 2973 2974 iounmap(hw->reg); 2975 enetc_kfree_si(si); 2976 pci_release_mem_regions(pdev); 2977 pci_disable_device(pdev); 2978 } 2979