1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2014-2016 Freescale Semiconductor Inc. 3 * Copyright 2016-2017 NXP 4 */ 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/platform_device.h> 8 #include <linux/etherdevice.h> 9 #include <linux/of_net.h> 10 #include <linux/interrupt.h> 11 #include <linux/msi.h> 12 #include <linux/kthread.h> 13 #include <linux/iommu.h> 14 #include <linux/net_tstamp.h> 15 #include <linux/fsl/mc.h> 16 #include <linux/bpf.h> 17 #include <linux/bpf_trace.h> 18 #include <net/sock.h> 19 20 #include "dpaa2-eth.h" 21 22 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 23 * using trace events only need to #include <trace/events/sched.h> 24 */ 25 #define CREATE_TRACE_POINTS 26 #include "dpaa2-eth-trace.h" 27 28 MODULE_LICENSE("Dual BSD/GPL"); 29 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 30 MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 31 32 static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 33 dma_addr_t iova_addr) 34 { 35 phys_addr_t phys_addr; 36 37 phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 38 39 return phys_to_virt(phys_addr); 40 } 41 42 static void validate_rx_csum(struct dpaa2_eth_priv *priv, 43 u32 fd_status, 44 struct sk_buff *skb) 45 { 46 skb_checksum_none_assert(skb); 47 48 /* HW checksum validation is disabled, nothing to do here */ 49 if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 50 return; 51 52 /* Read checksum validation bits */ 53 if (!((fd_status & DPAA2_FAS_L3CV) && 54 (fd_status & DPAA2_FAS_L4CV))) 55 return; 56 57 /* Inform the stack there's no need to compute L3/L4 csum anymore */ 58 skb->ip_summed = CHECKSUM_UNNECESSARY; 59 } 60 61 /* Free a received FD. 62 * Not to be used for Tx conf FDs or on any other paths. 63 */ 64 static void free_rx_fd(struct dpaa2_eth_priv *priv, 65 const struct dpaa2_fd *fd, 66 void *vaddr) 67 { 68 struct device *dev = priv->net_dev->dev.parent; 69 dma_addr_t addr = dpaa2_fd_get_addr(fd); 70 u8 fd_format = dpaa2_fd_get_format(fd); 71 struct dpaa2_sg_entry *sgt; 72 void *sg_vaddr; 73 int i; 74 75 /* If single buffer frame, just free the data buffer */ 76 if (fd_format == dpaa2_fd_single) 77 goto free_buf; 78 else if (fd_format != dpaa2_fd_sg) 79 /* We don't support any other format */ 80 return; 81 82 /* For S/G frames, we first need to free all SG entries 83 * except the first one, which was taken care of already 84 */ 85 sgt = vaddr + dpaa2_fd_get_offset(fd); 86 for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 87 addr = dpaa2_sg_get_addr(&sgt[i]); 88 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 89 dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 90 DMA_BIDIRECTIONAL); 91 92 free_pages((unsigned long)sg_vaddr, 0); 93 if (dpaa2_sg_is_final(&sgt[i])) 94 break; 95 } 96 97 free_buf: 98 free_pages((unsigned long)vaddr, 0); 99 } 100 101 /* Build a linear skb based on a single-buffer frame descriptor */ 102 static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch, 103 const struct dpaa2_fd *fd, 104 void *fd_vaddr) 105 { 106 struct sk_buff *skb = NULL; 107 u16 fd_offset = dpaa2_fd_get_offset(fd); 108 u32 fd_length = dpaa2_fd_get_len(fd); 109 110 ch->buf_count--; 111 112 skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 113 if (unlikely(!skb)) 114 return NULL; 115 116 skb_reserve(skb, fd_offset); 117 skb_put(skb, fd_length); 118 119 return skb; 120 } 121 122 /* Build a non linear (fragmented) skb based on a S/G table */ 123 static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv, 124 struct dpaa2_eth_channel *ch, 125 struct dpaa2_sg_entry *sgt) 126 { 127 struct sk_buff *skb = NULL; 128 struct device *dev = priv->net_dev->dev.parent; 129 void *sg_vaddr; 130 dma_addr_t sg_addr; 131 u16 sg_offset; 132 u32 sg_length; 133 struct page *page, *head_page; 134 int page_offset; 135 int i; 136 137 for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 138 struct dpaa2_sg_entry *sge = &sgt[i]; 139 140 /* NOTE: We only support SG entries in dpaa2_sg_single format, 141 * but this is the only format we may receive from HW anyway 142 */ 143 144 /* Get the address and length from the S/G entry */ 145 sg_addr = dpaa2_sg_get_addr(sge); 146 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 147 dma_unmap_page(dev, sg_addr, DPAA2_ETH_RX_BUF_SIZE, 148 DMA_BIDIRECTIONAL); 149 150 sg_length = dpaa2_sg_get_len(sge); 151 152 if (i == 0) { 153 /* We build the skb around the first data buffer */ 154 skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 155 if (unlikely(!skb)) { 156 /* Free the first SG entry now, since we already 157 * unmapped it and obtained the virtual address 158 */ 159 free_pages((unsigned long)sg_vaddr, 0); 160 161 /* We still need to subtract the buffers used 162 * by this FD from our software counter 163 */ 164 while (!dpaa2_sg_is_final(&sgt[i]) && 165 i < DPAA2_ETH_MAX_SG_ENTRIES) 166 i++; 167 break; 168 } 169 170 sg_offset = dpaa2_sg_get_offset(sge); 171 skb_reserve(skb, sg_offset); 172 skb_put(skb, sg_length); 173 } else { 174 /* Rest of the data buffers are stored as skb frags */ 175 page = virt_to_page(sg_vaddr); 176 head_page = virt_to_head_page(sg_vaddr); 177 178 /* Offset in page (which may be compound). 179 * Data in subsequent SG entries is stored from the 180 * beginning of the buffer, so we don't need to add the 181 * sg_offset. 182 */ 183 page_offset = ((unsigned long)sg_vaddr & 184 (PAGE_SIZE - 1)) + 185 (page_address(page) - page_address(head_page)); 186 187 skb_add_rx_frag(skb, i - 1, head_page, page_offset, 188 sg_length, DPAA2_ETH_RX_BUF_SIZE); 189 } 190 191 if (dpaa2_sg_is_final(sge)) 192 break; 193 } 194 195 WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 196 197 /* Count all data buffers + SG table buffer */ 198 ch->buf_count -= i + 2; 199 200 return skb; 201 } 202 203 /* Free buffers acquired from the buffer pool or which were meant to 204 * be released in the pool 205 */ 206 static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count) 207 { 208 struct device *dev = priv->net_dev->dev.parent; 209 void *vaddr; 210 int i; 211 212 for (i = 0; i < count; i++) { 213 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 214 dma_unmap_page(dev, buf_array[i], DPAA2_ETH_RX_BUF_SIZE, 215 DMA_BIDIRECTIONAL); 216 free_pages((unsigned long)vaddr, 0); 217 } 218 } 219 220 static void xdp_release_buf(struct dpaa2_eth_priv *priv, 221 struct dpaa2_eth_channel *ch, 222 dma_addr_t addr) 223 { 224 int err; 225 226 ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr; 227 if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD) 228 return; 229 230 while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 231 ch->xdp.drop_bufs, 232 ch->xdp.drop_cnt)) == -EBUSY) 233 cpu_relax(); 234 235 if (err) { 236 free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt); 237 ch->buf_count -= ch->xdp.drop_cnt; 238 } 239 240 ch->xdp.drop_cnt = 0; 241 } 242 243 static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd, 244 void *buf_start, u16 queue_id) 245 { 246 struct dpaa2_eth_fq *fq; 247 struct dpaa2_faead *faead; 248 u32 ctrl, frc; 249 int i, err; 250 251 /* Mark the egress frame hardware annotation area as valid */ 252 frc = dpaa2_fd_get_frc(fd); 253 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 254 dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 255 256 /* Instruct hardware to release the FD buffer directly into 257 * the buffer pool once transmission is completed, instead of 258 * sending a Tx confirmation frame to us 259 */ 260 ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 261 faead = dpaa2_get_faead(buf_start, false); 262 faead->ctrl = cpu_to_le32(ctrl); 263 faead->conf_fqid = 0; 264 265 fq = &priv->fq[queue_id]; 266 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 267 err = priv->enqueue(priv, fq, fd, 0); 268 if (err != -EBUSY) 269 break; 270 } 271 272 return err; 273 } 274 275 static u32 run_xdp(struct dpaa2_eth_priv *priv, 276 struct dpaa2_eth_channel *ch, 277 struct dpaa2_eth_fq *rx_fq, 278 struct dpaa2_fd *fd, void *vaddr) 279 { 280 dma_addr_t addr = dpaa2_fd_get_addr(fd); 281 struct rtnl_link_stats64 *percpu_stats; 282 struct bpf_prog *xdp_prog; 283 struct xdp_buff xdp; 284 u32 xdp_act = XDP_PASS; 285 int err; 286 287 percpu_stats = this_cpu_ptr(priv->percpu_stats); 288 289 rcu_read_lock(); 290 291 xdp_prog = READ_ONCE(ch->xdp.prog); 292 if (!xdp_prog) 293 goto out; 294 295 xdp.data = vaddr + dpaa2_fd_get_offset(fd); 296 xdp.data_end = xdp.data + dpaa2_fd_get_len(fd); 297 xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; 298 xdp_set_data_meta_invalid(&xdp); 299 xdp.rxq = &ch->xdp_rxq; 300 301 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 302 303 /* xdp.data pointer may have changed */ 304 dpaa2_fd_set_offset(fd, xdp.data - vaddr); 305 dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 306 307 switch (xdp_act) { 308 case XDP_PASS: 309 break; 310 case XDP_TX: 311 err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid); 312 if (err) { 313 xdp_release_buf(priv, ch, addr); 314 percpu_stats->tx_errors++; 315 ch->stats.xdp_tx_err++; 316 } else { 317 percpu_stats->tx_packets++; 318 percpu_stats->tx_bytes += dpaa2_fd_get_len(fd); 319 ch->stats.xdp_tx++; 320 } 321 break; 322 default: 323 bpf_warn_invalid_xdp_action(xdp_act); 324 /* fall through */ 325 case XDP_ABORTED: 326 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 327 /* fall through */ 328 case XDP_DROP: 329 xdp_release_buf(priv, ch, addr); 330 ch->stats.xdp_drop++; 331 break; 332 case XDP_REDIRECT: 333 dma_unmap_page(priv->net_dev->dev.parent, addr, 334 DPAA2_ETH_RX_BUF_SIZE, DMA_BIDIRECTIONAL); 335 ch->buf_count--; 336 xdp.data_hard_start = vaddr; 337 err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 338 if (unlikely(err)) 339 ch->stats.xdp_drop++; 340 else 341 ch->stats.xdp_redirect++; 342 break; 343 } 344 345 ch->xdp.res |= xdp_act; 346 out: 347 rcu_read_unlock(); 348 return xdp_act; 349 } 350 351 /* Main Rx frame processing routine */ 352 static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 353 struct dpaa2_eth_channel *ch, 354 const struct dpaa2_fd *fd, 355 struct dpaa2_eth_fq *fq) 356 { 357 dma_addr_t addr = dpaa2_fd_get_addr(fd); 358 u8 fd_format = dpaa2_fd_get_format(fd); 359 void *vaddr; 360 struct sk_buff *skb; 361 struct rtnl_link_stats64 *percpu_stats; 362 struct dpaa2_eth_drv_stats *percpu_extras; 363 struct device *dev = priv->net_dev->dev.parent; 364 struct dpaa2_fas *fas; 365 void *buf_data; 366 u32 status = 0; 367 u32 xdp_act; 368 369 /* Tracing point */ 370 trace_dpaa2_rx_fd(priv->net_dev, fd); 371 372 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 373 dma_sync_single_for_cpu(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 374 DMA_BIDIRECTIONAL); 375 376 fas = dpaa2_get_fas(vaddr, false); 377 prefetch(fas); 378 buf_data = vaddr + dpaa2_fd_get_offset(fd); 379 prefetch(buf_data); 380 381 percpu_stats = this_cpu_ptr(priv->percpu_stats); 382 percpu_extras = this_cpu_ptr(priv->percpu_extras); 383 384 if (fd_format == dpaa2_fd_single) { 385 xdp_act = run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 386 if (xdp_act != XDP_PASS) { 387 percpu_stats->rx_packets++; 388 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 389 return; 390 } 391 392 dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 393 DMA_BIDIRECTIONAL); 394 skb = build_linear_skb(ch, fd, vaddr); 395 } else if (fd_format == dpaa2_fd_sg) { 396 WARN_ON(priv->xdp_prog); 397 398 dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 399 DMA_BIDIRECTIONAL); 400 skb = build_frag_skb(priv, ch, buf_data); 401 free_pages((unsigned long)vaddr, 0); 402 percpu_extras->rx_sg_frames++; 403 percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 404 } else { 405 /* We don't support any other format */ 406 goto err_frame_format; 407 } 408 409 if (unlikely(!skb)) 410 goto err_build_skb; 411 412 prefetch(skb->data); 413 414 /* Get the timestamp value */ 415 if (priv->rx_tstamp) { 416 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 417 __le64 *ts = dpaa2_get_ts(vaddr, false); 418 u64 ns; 419 420 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 421 422 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 423 shhwtstamps->hwtstamp = ns_to_ktime(ns); 424 } 425 426 /* Check if we need to validate the L4 csum */ 427 if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 428 status = le32_to_cpu(fas->status); 429 validate_rx_csum(priv, status, skb); 430 } 431 432 skb->protocol = eth_type_trans(skb, priv->net_dev); 433 skb_record_rx_queue(skb, fq->flowid); 434 435 percpu_stats->rx_packets++; 436 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 437 438 list_add_tail(&skb->list, ch->rx_list); 439 440 return; 441 442 err_build_skb: 443 free_rx_fd(priv, fd, vaddr); 444 err_frame_format: 445 percpu_stats->rx_dropped++; 446 } 447 448 /* Consume all frames pull-dequeued into the store. This is the simplest way to 449 * make sure we don't accidentally issue another volatile dequeue which would 450 * overwrite (leak) frames already in the store. 451 * 452 * Observance of NAPI budget is not our concern, leaving that to the caller. 453 */ 454 static int consume_frames(struct dpaa2_eth_channel *ch, 455 struct dpaa2_eth_fq **src) 456 { 457 struct dpaa2_eth_priv *priv = ch->priv; 458 struct dpaa2_eth_fq *fq = NULL; 459 struct dpaa2_dq *dq; 460 const struct dpaa2_fd *fd; 461 int cleaned = 0; 462 int is_last; 463 464 do { 465 dq = dpaa2_io_store_next(ch->store, &is_last); 466 if (unlikely(!dq)) { 467 /* If we're here, we *must* have placed a 468 * volatile dequeue comnmand, so keep reading through 469 * the store until we get some sort of valid response 470 * token (either a valid frame or an "empty dequeue") 471 */ 472 continue; 473 } 474 475 fd = dpaa2_dq_fd(dq); 476 fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 477 478 fq->consume(priv, ch, fd, fq); 479 cleaned++; 480 } while (!is_last); 481 482 if (!cleaned) 483 return 0; 484 485 fq->stats.frames += cleaned; 486 487 /* A dequeue operation only pulls frames from a single queue 488 * into the store. Return the frame queue as an out param. 489 */ 490 if (src) 491 *src = fq; 492 493 return cleaned; 494 } 495 496 /* Configure the egress frame annotation for timestamp update */ 497 static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start) 498 { 499 struct dpaa2_faead *faead; 500 u32 ctrl, frc; 501 502 /* Mark the egress frame annotation area as valid */ 503 frc = dpaa2_fd_get_frc(fd); 504 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 505 506 /* Set hardware annotation size */ 507 ctrl = dpaa2_fd_get_ctrl(fd); 508 dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 509 510 /* enable UPD (update prepanded data) bit in FAEAD field of 511 * hardware frame annotation area 512 */ 513 ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 514 faead = dpaa2_get_faead(buf_start, true); 515 faead->ctrl = cpu_to_le32(ctrl); 516 } 517 518 /* Create a frame descriptor based on a fragmented skb */ 519 static int build_sg_fd(struct dpaa2_eth_priv *priv, 520 struct sk_buff *skb, 521 struct dpaa2_fd *fd) 522 { 523 struct device *dev = priv->net_dev->dev.parent; 524 void *sgt_buf = NULL; 525 dma_addr_t addr; 526 int nr_frags = skb_shinfo(skb)->nr_frags; 527 struct dpaa2_sg_entry *sgt; 528 int i, err; 529 int sgt_buf_size; 530 struct scatterlist *scl, *crt_scl; 531 int num_sg; 532 int num_dma_bufs; 533 struct dpaa2_eth_swa *swa; 534 535 /* Create and map scatterlist. 536 * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 537 * to go beyond nr_frags+1. 538 * Note: We don't support chained scatterlists 539 */ 540 if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 541 return -EINVAL; 542 543 scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 544 if (unlikely(!scl)) 545 return -ENOMEM; 546 547 sg_init_table(scl, nr_frags + 1); 548 num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 549 num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 550 if (unlikely(!num_dma_bufs)) { 551 err = -ENOMEM; 552 goto dma_map_sg_failed; 553 } 554 555 /* Prepare the HW SGT structure */ 556 sgt_buf_size = priv->tx_data_offset + 557 sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 558 sgt_buf = napi_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN); 559 if (unlikely(!sgt_buf)) { 560 err = -ENOMEM; 561 goto sgt_buf_alloc_failed; 562 } 563 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 564 memset(sgt_buf, 0, sgt_buf_size); 565 566 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 567 568 /* Fill in the HW SGT structure. 569 * 570 * sgt_buf is zeroed out, so the following fields are implicit 571 * in all sgt entries: 572 * - offset is 0 573 * - format is 'dpaa2_sg_single' 574 */ 575 for_each_sg(scl, crt_scl, num_dma_bufs, i) { 576 dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 577 dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 578 } 579 dpaa2_sg_set_final(&sgt[i - 1], true); 580 581 /* Store the skb backpointer in the SGT buffer. 582 * Fit the scatterlist and the number of buffers alongside the 583 * skb backpointer in the software annotation area. We'll need 584 * all of them on Tx Conf. 585 */ 586 swa = (struct dpaa2_eth_swa *)sgt_buf; 587 swa->type = DPAA2_ETH_SWA_SG; 588 swa->sg.skb = skb; 589 swa->sg.scl = scl; 590 swa->sg.num_sg = num_sg; 591 swa->sg.sgt_size = sgt_buf_size; 592 593 /* Separately map the SGT buffer */ 594 addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 595 if (unlikely(dma_mapping_error(dev, addr))) { 596 err = -ENOMEM; 597 goto dma_map_single_failed; 598 } 599 dpaa2_fd_set_offset(fd, priv->tx_data_offset); 600 dpaa2_fd_set_format(fd, dpaa2_fd_sg); 601 dpaa2_fd_set_addr(fd, addr); 602 dpaa2_fd_set_len(fd, skb->len); 603 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 604 605 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 606 enable_tx_tstamp(fd, sgt_buf); 607 608 return 0; 609 610 dma_map_single_failed: 611 skb_free_frag(sgt_buf); 612 sgt_buf_alloc_failed: 613 dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 614 dma_map_sg_failed: 615 kfree(scl); 616 return err; 617 } 618 619 /* Create a frame descriptor based on a linear skb */ 620 static int build_single_fd(struct dpaa2_eth_priv *priv, 621 struct sk_buff *skb, 622 struct dpaa2_fd *fd) 623 { 624 struct device *dev = priv->net_dev->dev.parent; 625 u8 *buffer_start, *aligned_start; 626 struct dpaa2_eth_swa *swa; 627 dma_addr_t addr; 628 629 buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb); 630 631 /* If there's enough room to align the FD address, do it. 632 * It will help hardware optimize accesses. 633 */ 634 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 635 DPAA2_ETH_TX_BUF_ALIGN); 636 if (aligned_start >= skb->head) 637 buffer_start = aligned_start; 638 639 /* Store a backpointer to the skb at the beginning of the buffer 640 * (in the private data area) such that we can release it 641 * on Tx confirm 642 */ 643 swa = (struct dpaa2_eth_swa *)buffer_start; 644 swa->type = DPAA2_ETH_SWA_SINGLE; 645 swa->single.skb = skb; 646 647 addr = dma_map_single(dev, buffer_start, 648 skb_tail_pointer(skb) - buffer_start, 649 DMA_BIDIRECTIONAL); 650 if (unlikely(dma_mapping_error(dev, addr))) 651 return -ENOMEM; 652 653 dpaa2_fd_set_addr(fd, addr); 654 dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 655 dpaa2_fd_set_len(fd, skb->len); 656 dpaa2_fd_set_format(fd, dpaa2_fd_single); 657 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 658 659 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 660 enable_tx_tstamp(fd, buffer_start); 661 662 return 0; 663 } 664 665 /* FD freeing routine on the Tx path 666 * 667 * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 668 * back-pointed to is also freed. 669 * This can be called either from dpaa2_eth_tx_conf() or on the error path of 670 * dpaa2_eth_tx(). 671 */ 672 static void free_tx_fd(const struct dpaa2_eth_priv *priv, 673 struct dpaa2_eth_fq *fq, 674 const struct dpaa2_fd *fd, bool in_napi) 675 { 676 struct device *dev = priv->net_dev->dev.parent; 677 dma_addr_t fd_addr; 678 struct sk_buff *skb = NULL; 679 unsigned char *buffer_start; 680 struct dpaa2_eth_swa *swa; 681 u8 fd_format = dpaa2_fd_get_format(fd); 682 u32 fd_len = dpaa2_fd_get_len(fd); 683 684 fd_addr = dpaa2_fd_get_addr(fd); 685 buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 686 swa = (struct dpaa2_eth_swa *)buffer_start; 687 688 if (fd_format == dpaa2_fd_single) { 689 if (swa->type == DPAA2_ETH_SWA_SINGLE) { 690 skb = swa->single.skb; 691 /* Accessing the skb buffer is safe before dma unmap, 692 * because we didn't map the actual skb shell. 693 */ 694 dma_unmap_single(dev, fd_addr, 695 skb_tail_pointer(skb) - buffer_start, 696 DMA_BIDIRECTIONAL); 697 } else { 698 WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 699 dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 700 DMA_BIDIRECTIONAL); 701 } 702 } else if (fd_format == dpaa2_fd_sg) { 703 skb = swa->sg.skb; 704 705 /* Unmap the scatterlist */ 706 dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 707 DMA_BIDIRECTIONAL); 708 kfree(swa->sg.scl); 709 710 /* Unmap the SGT buffer */ 711 dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 712 DMA_BIDIRECTIONAL); 713 } else { 714 netdev_dbg(priv->net_dev, "Invalid FD format\n"); 715 return; 716 } 717 718 if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 719 fq->dq_frames++; 720 fq->dq_bytes += fd_len; 721 } 722 723 if (swa->type == DPAA2_ETH_SWA_XDP) { 724 xdp_return_frame(swa->xdp.xdpf); 725 return; 726 } 727 728 /* Get the timestamp value */ 729 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { 730 struct skb_shared_hwtstamps shhwtstamps; 731 __le64 *ts = dpaa2_get_ts(buffer_start, true); 732 u64 ns; 733 734 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 735 736 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 737 shhwtstamps.hwtstamp = ns_to_ktime(ns); 738 skb_tstamp_tx(skb, &shhwtstamps); 739 } 740 741 /* Free SGT buffer allocated on tx */ 742 if (fd_format != dpaa2_fd_single) 743 skb_free_frag(buffer_start); 744 745 /* Move on with skb release */ 746 napi_consume_skb(skb, in_napi); 747 } 748 749 static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 750 { 751 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 752 struct dpaa2_fd fd; 753 struct rtnl_link_stats64 *percpu_stats; 754 struct dpaa2_eth_drv_stats *percpu_extras; 755 struct dpaa2_eth_fq *fq; 756 struct netdev_queue *nq; 757 u16 queue_mapping; 758 unsigned int needed_headroom; 759 u32 fd_len; 760 u8 prio = 0; 761 int err, i; 762 763 percpu_stats = this_cpu_ptr(priv->percpu_stats); 764 percpu_extras = this_cpu_ptr(priv->percpu_extras); 765 766 needed_headroom = dpaa2_eth_needed_headroom(priv, skb); 767 if (skb_headroom(skb) < needed_headroom) { 768 struct sk_buff *ns; 769 770 ns = skb_realloc_headroom(skb, needed_headroom); 771 if (unlikely(!ns)) { 772 percpu_stats->tx_dropped++; 773 goto err_alloc_headroom; 774 } 775 percpu_extras->tx_reallocs++; 776 777 if (skb->sk) 778 skb_set_owner_w(ns, skb->sk); 779 780 dev_kfree_skb(skb); 781 skb = ns; 782 } 783 784 /* We'll be holding a back-reference to the skb until Tx Confirmation; 785 * we don't want that overwritten by a concurrent Tx with a cloned skb. 786 */ 787 skb = skb_unshare(skb, GFP_ATOMIC); 788 if (unlikely(!skb)) { 789 /* skb_unshare() has already freed the skb */ 790 percpu_stats->tx_dropped++; 791 return NETDEV_TX_OK; 792 } 793 794 /* Setup the FD fields */ 795 memset(&fd, 0, sizeof(fd)); 796 797 if (skb_is_nonlinear(skb)) { 798 err = build_sg_fd(priv, skb, &fd); 799 percpu_extras->tx_sg_frames++; 800 percpu_extras->tx_sg_bytes += skb->len; 801 } else { 802 err = build_single_fd(priv, skb, &fd); 803 } 804 805 if (unlikely(err)) { 806 percpu_stats->tx_dropped++; 807 goto err_build_fd; 808 } 809 810 /* Tracing point */ 811 trace_dpaa2_tx_fd(net_dev, &fd); 812 813 /* TxConf FQ selection relies on queue id from the stack. 814 * In case of a forwarded frame from another DPNI interface, we choose 815 * a queue affined to the same core that processed the Rx frame 816 */ 817 queue_mapping = skb_get_queue_mapping(skb); 818 819 if (net_dev->num_tc) { 820 prio = netdev_txq_to_tc(net_dev, queue_mapping); 821 /* Hardware interprets priority level 0 as being the highest, 822 * so we need to do a reverse mapping to the netdev tc index 823 */ 824 prio = net_dev->num_tc - prio - 1; 825 /* We have only one FQ array entry for all Tx hardware queues 826 * with the same flow id (but different priority levels) 827 */ 828 queue_mapping %= dpaa2_eth_queue_count(priv); 829 } 830 fq = &priv->fq[queue_mapping]; 831 832 fd_len = dpaa2_fd_get_len(&fd); 833 nq = netdev_get_tx_queue(net_dev, queue_mapping); 834 netdev_tx_sent_queue(nq, fd_len); 835 836 /* Everything that happens after this enqueues might race with 837 * the Tx confirmation callback for this frame 838 */ 839 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 840 err = priv->enqueue(priv, fq, &fd, prio); 841 if (err != -EBUSY) 842 break; 843 } 844 percpu_extras->tx_portal_busy += i; 845 if (unlikely(err < 0)) { 846 percpu_stats->tx_errors++; 847 /* Clean up everything, including freeing the skb */ 848 free_tx_fd(priv, fq, &fd, false); 849 netdev_tx_completed_queue(nq, 1, fd_len); 850 } else { 851 percpu_stats->tx_packets++; 852 percpu_stats->tx_bytes += fd_len; 853 } 854 855 return NETDEV_TX_OK; 856 857 err_build_fd: 858 err_alloc_headroom: 859 dev_kfree_skb(skb); 860 861 return NETDEV_TX_OK; 862 } 863 864 /* Tx confirmation frame processing routine */ 865 static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 866 struct dpaa2_eth_channel *ch __always_unused, 867 const struct dpaa2_fd *fd, 868 struct dpaa2_eth_fq *fq) 869 { 870 struct rtnl_link_stats64 *percpu_stats; 871 struct dpaa2_eth_drv_stats *percpu_extras; 872 u32 fd_len = dpaa2_fd_get_len(fd); 873 u32 fd_errors; 874 875 /* Tracing point */ 876 trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 877 878 percpu_extras = this_cpu_ptr(priv->percpu_extras); 879 percpu_extras->tx_conf_frames++; 880 percpu_extras->tx_conf_bytes += fd_len; 881 882 /* Check frame errors in the FD field */ 883 fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 884 free_tx_fd(priv, fq, fd, true); 885 886 if (likely(!fd_errors)) 887 return; 888 889 if (net_ratelimit()) 890 netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 891 fd_errors); 892 893 percpu_stats = this_cpu_ptr(priv->percpu_stats); 894 /* Tx-conf logically pertains to the egress path. */ 895 percpu_stats->tx_errors++; 896 } 897 898 static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 899 { 900 int err; 901 902 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 903 DPNI_OFF_RX_L3_CSUM, enable); 904 if (err) { 905 netdev_err(priv->net_dev, 906 "dpni_set_offload(RX_L3_CSUM) failed\n"); 907 return err; 908 } 909 910 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 911 DPNI_OFF_RX_L4_CSUM, enable); 912 if (err) { 913 netdev_err(priv->net_dev, 914 "dpni_set_offload(RX_L4_CSUM) failed\n"); 915 return err; 916 } 917 918 return 0; 919 } 920 921 static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 922 { 923 int err; 924 925 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 926 DPNI_OFF_TX_L3_CSUM, enable); 927 if (err) { 928 netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 929 return err; 930 } 931 932 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 933 DPNI_OFF_TX_L4_CSUM, enable); 934 if (err) { 935 netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 936 return err; 937 } 938 939 return 0; 940 } 941 942 /* Perform a single release command to add buffers 943 * to the specified buffer pool 944 */ 945 static int add_bufs(struct dpaa2_eth_priv *priv, 946 struct dpaa2_eth_channel *ch, u16 bpid) 947 { 948 struct device *dev = priv->net_dev->dev.parent; 949 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 950 struct page *page; 951 dma_addr_t addr; 952 int i, err; 953 954 for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 955 /* Allocate buffer visible to WRIOP + skb shared info + 956 * alignment padding 957 */ 958 /* allocate one page for each Rx buffer. WRIOP sees 959 * the entire page except for a tailroom reserved for 960 * skb shared info 961 */ 962 page = dev_alloc_pages(0); 963 if (!page) 964 goto err_alloc; 965 966 addr = dma_map_page(dev, page, 0, DPAA2_ETH_RX_BUF_SIZE, 967 DMA_BIDIRECTIONAL); 968 if (unlikely(dma_mapping_error(dev, addr))) 969 goto err_map; 970 971 buf_array[i] = addr; 972 973 /* tracing point */ 974 trace_dpaa2_eth_buf_seed(priv->net_dev, 975 page, DPAA2_ETH_RX_BUF_RAW_SIZE, 976 addr, DPAA2_ETH_RX_BUF_SIZE, 977 bpid); 978 } 979 980 release_bufs: 981 /* In case the portal is busy, retry until successful */ 982 while ((err = dpaa2_io_service_release(ch->dpio, bpid, 983 buf_array, i)) == -EBUSY) 984 cpu_relax(); 985 986 /* If release command failed, clean up and bail out; 987 * not much else we can do about it 988 */ 989 if (err) { 990 free_bufs(priv, buf_array, i); 991 return 0; 992 } 993 994 return i; 995 996 err_map: 997 __free_pages(page, 0); 998 err_alloc: 999 /* If we managed to allocate at least some buffers, 1000 * release them to hardware 1001 */ 1002 if (i) 1003 goto release_bufs; 1004 1005 return 0; 1006 } 1007 1008 static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 1009 { 1010 int i, j; 1011 int new_count; 1012 1013 for (j = 0; j < priv->num_channels; j++) { 1014 for (i = 0; i < DPAA2_ETH_NUM_BUFS; 1015 i += DPAA2_ETH_BUFS_PER_CMD) { 1016 new_count = add_bufs(priv, priv->channel[j], bpid); 1017 priv->channel[j]->buf_count += new_count; 1018 1019 if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 1020 return -ENOMEM; 1021 } 1022 } 1023 } 1024 1025 return 0; 1026 } 1027 1028 /** 1029 * Drain the specified number of buffers from the DPNI's private buffer pool. 1030 * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 1031 */ 1032 static void drain_bufs(struct dpaa2_eth_priv *priv, int count) 1033 { 1034 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1035 int ret; 1036 1037 do { 1038 ret = dpaa2_io_service_acquire(NULL, priv->bpid, 1039 buf_array, count); 1040 if (ret < 0) { 1041 netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 1042 return; 1043 } 1044 free_bufs(priv, buf_array, ret); 1045 } while (ret); 1046 } 1047 1048 static void drain_pool(struct dpaa2_eth_priv *priv) 1049 { 1050 int i; 1051 1052 drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 1053 drain_bufs(priv, 1); 1054 1055 for (i = 0; i < priv->num_channels; i++) 1056 priv->channel[i]->buf_count = 0; 1057 } 1058 1059 /* Function is called from softirq context only, so we don't need to guard 1060 * the access to percpu count 1061 */ 1062 static int refill_pool(struct dpaa2_eth_priv *priv, 1063 struct dpaa2_eth_channel *ch, 1064 u16 bpid) 1065 { 1066 int new_count; 1067 1068 if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 1069 return 0; 1070 1071 do { 1072 new_count = add_bufs(priv, ch, bpid); 1073 if (unlikely(!new_count)) { 1074 /* Out of memory; abort for now, we'll try later on */ 1075 break; 1076 } 1077 ch->buf_count += new_count; 1078 } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 1079 1080 if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 1081 return -ENOMEM; 1082 1083 return 0; 1084 } 1085 1086 static int pull_channel(struct dpaa2_eth_channel *ch) 1087 { 1088 int err; 1089 int dequeues = -1; 1090 1091 /* Retry while portal is busy */ 1092 do { 1093 err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 1094 ch->store); 1095 dequeues++; 1096 cpu_relax(); 1097 } while (err == -EBUSY); 1098 1099 ch->stats.dequeue_portal_busy += dequeues; 1100 if (unlikely(err)) 1101 ch->stats.pull_err++; 1102 1103 return err; 1104 } 1105 1106 /* NAPI poll routine 1107 * 1108 * Frames are dequeued from the QMan channel associated with this NAPI context. 1109 * Rx, Tx confirmation and (if configured) Rx error frames all count 1110 * towards the NAPI budget. 1111 */ 1112 static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 1113 { 1114 struct dpaa2_eth_channel *ch; 1115 struct dpaa2_eth_priv *priv; 1116 int rx_cleaned = 0, txconf_cleaned = 0; 1117 struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1118 struct netdev_queue *nq; 1119 int store_cleaned, work_done; 1120 struct list_head rx_list; 1121 int err; 1122 1123 ch = container_of(napi, struct dpaa2_eth_channel, napi); 1124 ch->xdp.res = 0; 1125 priv = ch->priv; 1126 1127 INIT_LIST_HEAD(&rx_list); 1128 ch->rx_list = &rx_list; 1129 1130 do { 1131 err = pull_channel(ch); 1132 if (unlikely(err)) 1133 break; 1134 1135 /* Refill pool if appropriate */ 1136 refill_pool(priv, ch, priv->bpid); 1137 1138 store_cleaned = consume_frames(ch, &fq); 1139 if (!store_cleaned) 1140 break; 1141 if (fq->type == DPAA2_RX_FQ) { 1142 rx_cleaned += store_cleaned; 1143 } else { 1144 txconf_cleaned += store_cleaned; 1145 /* We have a single Tx conf FQ on this channel */ 1146 txc_fq = fq; 1147 } 1148 1149 /* If we either consumed the whole NAPI budget with Rx frames 1150 * or we reached the Tx confirmations threshold, we're done. 1151 */ 1152 if (rx_cleaned >= budget || 1153 txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1154 work_done = budget; 1155 goto out; 1156 } 1157 } while (store_cleaned); 1158 1159 /* We didn't consume the entire budget, so finish napi and 1160 * re-enable data availability notifications 1161 */ 1162 napi_complete_done(napi, rx_cleaned); 1163 do { 1164 err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 1165 cpu_relax(); 1166 } while (err == -EBUSY); 1167 WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 1168 ch->nctx.desired_cpu); 1169 1170 work_done = max(rx_cleaned, 1); 1171 1172 out: 1173 netif_receive_skb_list(ch->rx_list); 1174 1175 if (txc_fq && txc_fq->dq_frames) { 1176 nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1177 netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1178 txc_fq->dq_bytes); 1179 txc_fq->dq_frames = 0; 1180 txc_fq->dq_bytes = 0; 1181 } 1182 1183 if (ch->xdp.res & XDP_REDIRECT) 1184 xdp_do_flush_map(); 1185 1186 return work_done; 1187 } 1188 1189 static void enable_ch_napi(struct dpaa2_eth_priv *priv) 1190 { 1191 struct dpaa2_eth_channel *ch; 1192 int i; 1193 1194 for (i = 0; i < priv->num_channels; i++) { 1195 ch = priv->channel[i]; 1196 napi_enable(&ch->napi); 1197 } 1198 } 1199 1200 static void disable_ch_napi(struct dpaa2_eth_priv *priv) 1201 { 1202 struct dpaa2_eth_channel *ch; 1203 int i; 1204 1205 for (i = 0; i < priv->num_channels; i++) { 1206 ch = priv->channel[i]; 1207 napi_disable(&ch->napi); 1208 } 1209 } 1210 1211 static void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, bool enable) 1212 { 1213 struct dpni_taildrop td = {0}; 1214 int i, err; 1215 1216 if (priv->rx_td_enabled == enable) 1217 return; 1218 1219 td.enable = enable; 1220 td.threshold = DPAA2_ETH_TAILDROP_THRESH; 1221 1222 for (i = 0; i < priv->num_fqs; i++) { 1223 if (priv->fq[i].type != DPAA2_RX_FQ) 1224 continue; 1225 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1226 DPNI_CP_QUEUE, DPNI_QUEUE_RX, 0, 1227 priv->fq[i].flowid, &td); 1228 if (err) { 1229 netdev_err(priv->net_dev, 1230 "dpni_set_taildrop() failed\n"); 1231 break; 1232 } 1233 } 1234 1235 priv->rx_td_enabled = enable; 1236 } 1237 1238 static void update_tx_fqids(struct dpaa2_eth_priv *priv); 1239 1240 static int link_state_update(struct dpaa2_eth_priv *priv) 1241 { 1242 struct dpni_link_state state = {0}; 1243 bool tx_pause; 1244 int err; 1245 1246 err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 1247 if (unlikely(err)) { 1248 netdev_err(priv->net_dev, 1249 "dpni_get_link_state() failed\n"); 1250 return err; 1251 } 1252 1253 /* If Tx pause frame settings have changed, we need to update 1254 * Rx FQ taildrop configuration as well. We configure taildrop 1255 * only when pause frame generation is disabled. 1256 */ 1257 tx_pause = !!(state.options & DPNI_LINK_OPT_PAUSE) ^ 1258 !!(state.options & DPNI_LINK_OPT_ASYM_PAUSE); 1259 dpaa2_eth_set_rx_taildrop(priv, !tx_pause); 1260 1261 /* Chech link state; speed / duplex changes are not treated yet */ 1262 if (priv->link_state.up == state.up) 1263 goto out; 1264 1265 if (state.up) { 1266 update_tx_fqids(priv); 1267 netif_carrier_on(priv->net_dev); 1268 netif_tx_start_all_queues(priv->net_dev); 1269 } else { 1270 netif_tx_stop_all_queues(priv->net_dev); 1271 netif_carrier_off(priv->net_dev); 1272 } 1273 1274 netdev_info(priv->net_dev, "Link Event: state %s\n", 1275 state.up ? "up" : "down"); 1276 1277 out: 1278 priv->link_state = state; 1279 1280 return 0; 1281 } 1282 1283 static int dpaa2_eth_open(struct net_device *net_dev) 1284 { 1285 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1286 int err; 1287 1288 err = seed_pool(priv, priv->bpid); 1289 if (err) { 1290 /* Not much to do; the buffer pool, though not filled up, 1291 * may still contain some buffers which would enable us 1292 * to limp on. 1293 */ 1294 netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 1295 priv->dpbp_dev->obj_desc.id, priv->bpid); 1296 } 1297 1298 /* We'll only start the txqs when the link is actually ready; make sure 1299 * we don't race against the link up notification, which may come 1300 * immediately after dpni_enable(); 1301 */ 1302 netif_tx_stop_all_queues(net_dev); 1303 enable_ch_napi(priv); 1304 /* Also, explicitly set carrier off, otherwise netif_carrier_ok() will 1305 * return true and cause 'ip link show' to report the LOWER_UP flag, 1306 * even though the link notification wasn't even received. 1307 */ 1308 netif_carrier_off(net_dev); 1309 1310 err = dpni_enable(priv->mc_io, 0, priv->mc_token); 1311 if (err < 0) { 1312 netdev_err(net_dev, "dpni_enable() failed\n"); 1313 goto enable_err; 1314 } 1315 1316 /* If the DPMAC object has already processed the link up interrupt, 1317 * we have to learn the link state ourselves. 1318 */ 1319 err = link_state_update(priv); 1320 if (err < 0) { 1321 netdev_err(net_dev, "Can't update link state\n"); 1322 goto link_state_err; 1323 } 1324 1325 return 0; 1326 1327 link_state_err: 1328 enable_err: 1329 disable_ch_napi(priv); 1330 drain_pool(priv); 1331 return err; 1332 } 1333 1334 /* Total number of in-flight frames on ingress queues */ 1335 static u32 ingress_fq_count(struct dpaa2_eth_priv *priv) 1336 { 1337 struct dpaa2_eth_fq *fq; 1338 u32 fcnt = 0, bcnt = 0, total = 0; 1339 int i, err; 1340 1341 for (i = 0; i < priv->num_fqs; i++) { 1342 fq = &priv->fq[i]; 1343 err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 1344 if (err) { 1345 netdev_warn(priv->net_dev, "query_fq_count failed"); 1346 break; 1347 } 1348 total += fcnt; 1349 } 1350 1351 return total; 1352 } 1353 1354 static void wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 1355 { 1356 int retries = 10; 1357 u32 pending; 1358 1359 do { 1360 pending = ingress_fq_count(priv); 1361 if (pending) 1362 msleep(100); 1363 } while (pending && --retries); 1364 } 1365 1366 #define DPNI_TX_PENDING_VER_MAJOR 7 1367 #define DPNI_TX_PENDING_VER_MINOR 13 1368 static void wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 1369 { 1370 union dpni_statistics stats; 1371 int retries = 10; 1372 int err; 1373 1374 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 1375 DPNI_TX_PENDING_VER_MINOR) < 0) 1376 goto out; 1377 1378 do { 1379 err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 1380 &stats); 1381 if (err) 1382 goto out; 1383 if (stats.page_6.tx_pending_frames == 0) 1384 return; 1385 } while (--retries); 1386 1387 out: 1388 msleep(500); 1389 } 1390 1391 static int dpaa2_eth_stop(struct net_device *net_dev) 1392 { 1393 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1394 int dpni_enabled = 0; 1395 int retries = 10; 1396 1397 netif_tx_stop_all_queues(net_dev); 1398 netif_carrier_off(net_dev); 1399 1400 /* On dpni_disable(), the MC firmware will: 1401 * - stop MAC Rx and wait for all Rx frames to be enqueued to software 1402 * - cut off WRIOP dequeues from egress FQs and wait until transmission 1403 * of all in flight Tx frames is finished (and corresponding Tx conf 1404 * frames are enqueued back to software) 1405 * 1406 * Before calling dpni_disable(), we wait for all Tx frames to arrive 1407 * on WRIOP. After it finishes, wait until all remaining frames on Rx 1408 * and Tx conf queues are consumed on NAPI poll. 1409 */ 1410 wait_for_egress_fq_empty(priv); 1411 1412 do { 1413 dpni_disable(priv->mc_io, 0, priv->mc_token); 1414 dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 1415 if (dpni_enabled) 1416 /* Allow the hardware some slack */ 1417 msleep(100); 1418 } while (dpni_enabled && --retries); 1419 if (!retries) { 1420 netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 1421 /* Must go on and disable NAPI nonetheless, so we don't crash at 1422 * the next "ifconfig up" 1423 */ 1424 } 1425 1426 wait_for_ingress_fq_empty(priv); 1427 disable_ch_napi(priv); 1428 1429 /* Empty the buffer pool */ 1430 drain_pool(priv); 1431 1432 return 0; 1433 } 1434 1435 static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 1436 { 1437 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1438 struct device *dev = net_dev->dev.parent; 1439 int err; 1440 1441 err = eth_mac_addr(net_dev, addr); 1442 if (err < 0) { 1443 dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 1444 return err; 1445 } 1446 1447 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 1448 net_dev->dev_addr); 1449 if (err) { 1450 dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 1451 return err; 1452 } 1453 1454 return 0; 1455 } 1456 1457 /** Fill in counters maintained by the GPP driver. These may be different from 1458 * the hardware counters obtained by ethtool. 1459 */ 1460 static void dpaa2_eth_get_stats(struct net_device *net_dev, 1461 struct rtnl_link_stats64 *stats) 1462 { 1463 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1464 struct rtnl_link_stats64 *percpu_stats; 1465 u64 *cpustats; 1466 u64 *netstats = (u64 *)stats; 1467 int i, j; 1468 int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 1469 1470 for_each_possible_cpu(i) { 1471 percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 1472 cpustats = (u64 *)percpu_stats; 1473 for (j = 0; j < num; j++) 1474 netstats[j] += cpustats[j]; 1475 } 1476 } 1477 1478 /* Copy mac unicast addresses from @net_dev to @priv. 1479 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 1480 */ 1481 static void add_uc_hw_addr(const struct net_device *net_dev, 1482 struct dpaa2_eth_priv *priv) 1483 { 1484 struct netdev_hw_addr *ha; 1485 int err; 1486 1487 netdev_for_each_uc_addr(ha, net_dev) { 1488 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 1489 ha->addr); 1490 if (err) 1491 netdev_warn(priv->net_dev, 1492 "Could not add ucast MAC %pM to the filtering table (err %d)\n", 1493 ha->addr, err); 1494 } 1495 } 1496 1497 /* Copy mac multicast addresses from @net_dev to @priv 1498 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 1499 */ 1500 static void add_mc_hw_addr(const struct net_device *net_dev, 1501 struct dpaa2_eth_priv *priv) 1502 { 1503 struct netdev_hw_addr *ha; 1504 int err; 1505 1506 netdev_for_each_mc_addr(ha, net_dev) { 1507 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 1508 ha->addr); 1509 if (err) 1510 netdev_warn(priv->net_dev, 1511 "Could not add mcast MAC %pM to the filtering table (err %d)\n", 1512 ha->addr, err); 1513 } 1514 } 1515 1516 static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 1517 { 1518 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1519 int uc_count = netdev_uc_count(net_dev); 1520 int mc_count = netdev_mc_count(net_dev); 1521 u8 max_mac = priv->dpni_attrs.mac_filter_entries; 1522 u32 options = priv->dpni_attrs.options; 1523 u16 mc_token = priv->mc_token; 1524 struct fsl_mc_io *mc_io = priv->mc_io; 1525 int err; 1526 1527 /* Basic sanity checks; these probably indicate a misconfiguration */ 1528 if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 1529 netdev_info(net_dev, 1530 "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 1531 max_mac); 1532 1533 /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 1534 if (uc_count > max_mac) { 1535 netdev_info(net_dev, 1536 "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 1537 uc_count, max_mac); 1538 goto force_promisc; 1539 } 1540 if (mc_count + uc_count > max_mac) { 1541 netdev_info(net_dev, 1542 "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 1543 uc_count + mc_count, max_mac); 1544 goto force_mc_promisc; 1545 } 1546 1547 /* Adjust promisc settings due to flag combinations */ 1548 if (net_dev->flags & IFF_PROMISC) 1549 goto force_promisc; 1550 if (net_dev->flags & IFF_ALLMULTI) { 1551 /* First, rebuild unicast filtering table. This should be done 1552 * in promisc mode, in order to avoid frame loss while we 1553 * progressively add entries to the table. 1554 * We don't know whether we had been in promisc already, and 1555 * making an MC call to find out is expensive; so set uc promisc 1556 * nonetheless. 1557 */ 1558 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1559 if (err) 1560 netdev_warn(net_dev, "Can't set uc promisc\n"); 1561 1562 /* Actual uc table reconstruction. */ 1563 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 1564 if (err) 1565 netdev_warn(net_dev, "Can't clear uc filters\n"); 1566 add_uc_hw_addr(net_dev, priv); 1567 1568 /* Finally, clear uc promisc and set mc promisc as requested. */ 1569 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 1570 if (err) 1571 netdev_warn(net_dev, "Can't clear uc promisc\n"); 1572 goto force_mc_promisc; 1573 } 1574 1575 /* Neither unicast, nor multicast promisc will be on... eventually. 1576 * For now, rebuild mac filtering tables while forcing both of them on. 1577 */ 1578 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1579 if (err) 1580 netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 1581 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 1582 if (err) 1583 netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 1584 1585 /* Actual mac filtering tables reconstruction */ 1586 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 1587 if (err) 1588 netdev_warn(net_dev, "Can't clear mac filters\n"); 1589 add_mc_hw_addr(net_dev, priv); 1590 add_uc_hw_addr(net_dev, priv); 1591 1592 /* Now we can clear both ucast and mcast promisc, without risking 1593 * to drop legitimate frames anymore. 1594 */ 1595 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 1596 if (err) 1597 netdev_warn(net_dev, "Can't clear ucast promisc\n"); 1598 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 1599 if (err) 1600 netdev_warn(net_dev, "Can't clear mcast promisc\n"); 1601 1602 return; 1603 1604 force_promisc: 1605 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1606 if (err) 1607 netdev_warn(net_dev, "Can't set ucast promisc\n"); 1608 force_mc_promisc: 1609 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 1610 if (err) 1611 netdev_warn(net_dev, "Can't set mcast promisc\n"); 1612 } 1613 1614 static int dpaa2_eth_set_features(struct net_device *net_dev, 1615 netdev_features_t features) 1616 { 1617 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1618 netdev_features_t changed = features ^ net_dev->features; 1619 bool enable; 1620 int err; 1621 1622 if (changed & NETIF_F_RXCSUM) { 1623 enable = !!(features & NETIF_F_RXCSUM); 1624 err = set_rx_csum(priv, enable); 1625 if (err) 1626 return err; 1627 } 1628 1629 if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 1630 enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 1631 err = set_tx_csum(priv, enable); 1632 if (err) 1633 return err; 1634 } 1635 1636 return 0; 1637 } 1638 1639 static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 1640 { 1641 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1642 struct hwtstamp_config config; 1643 1644 if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 1645 return -EFAULT; 1646 1647 switch (config.tx_type) { 1648 case HWTSTAMP_TX_OFF: 1649 priv->tx_tstamp = false; 1650 break; 1651 case HWTSTAMP_TX_ON: 1652 priv->tx_tstamp = true; 1653 break; 1654 default: 1655 return -ERANGE; 1656 } 1657 1658 if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 1659 priv->rx_tstamp = false; 1660 } else { 1661 priv->rx_tstamp = true; 1662 /* TS is set for all frame types, not only those requested */ 1663 config.rx_filter = HWTSTAMP_FILTER_ALL; 1664 } 1665 1666 return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 1667 -EFAULT : 0; 1668 } 1669 1670 static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 1671 { 1672 if (cmd == SIOCSHWTSTAMP) 1673 return dpaa2_eth_ts_ioctl(dev, rq, cmd); 1674 1675 return -EINVAL; 1676 } 1677 1678 static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 1679 { 1680 int mfl, linear_mfl; 1681 1682 mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 1683 linear_mfl = DPAA2_ETH_RX_BUF_SIZE - DPAA2_ETH_RX_HWA_SIZE - 1684 dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 1685 1686 if (mfl > linear_mfl) { 1687 netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 1688 linear_mfl - VLAN_ETH_HLEN); 1689 return false; 1690 } 1691 1692 return true; 1693 } 1694 1695 static int set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 1696 { 1697 int mfl, err; 1698 1699 /* We enforce a maximum Rx frame length based on MTU only if we have 1700 * an XDP program attached (in order to avoid Rx S/G frames). 1701 * Otherwise, we accept all incoming frames as long as they are not 1702 * larger than maximum size supported in hardware 1703 */ 1704 if (has_xdp) 1705 mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 1706 else 1707 mfl = DPAA2_ETH_MFL; 1708 1709 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 1710 if (err) { 1711 netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 1712 return err; 1713 } 1714 1715 return 0; 1716 } 1717 1718 static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 1719 { 1720 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1721 int err; 1722 1723 if (!priv->xdp_prog) 1724 goto out; 1725 1726 if (!xdp_mtu_valid(priv, new_mtu)) 1727 return -EINVAL; 1728 1729 err = set_rx_mfl(priv, new_mtu, true); 1730 if (err) 1731 return err; 1732 1733 out: 1734 dev->mtu = new_mtu; 1735 return 0; 1736 } 1737 1738 static int update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 1739 { 1740 struct dpni_buffer_layout buf_layout = {0}; 1741 int err; 1742 1743 err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 1744 DPNI_QUEUE_RX, &buf_layout); 1745 if (err) { 1746 netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 1747 return err; 1748 } 1749 1750 /* Reserve extra headroom for XDP header size changes */ 1751 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 1752 (has_xdp ? XDP_PACKET_HEADROOM : 0); 1753 buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 1754 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 1755 DPNI_QUEUE_RX, &buf_layout); 1756 if (err) { 1757 netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 1758 return err; 1759 } 1760 1761 return 0; 1762 } 1763 1764 static int setup_xdp(struct net_device *dev, struct bpf_prog *prog) 1765 { 1766 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1767 struct dpaa2_eth_channel *ch; 1768 struct bpf_prog *old; 1769 bool up, need_update; 1770 int i, err; 1771 1772 if (prog && !xdp_mtu_valid(priv, dev->mtu)) 1773 return -EINVAL; 1774 1775 if (prog) { 1776 prog = bpf_prog_add(prog, priv->num_channels); 1777 if (IS_ERR(prog)) 1778 return PTR_ERR(prog); 1779 } 1780 1781 up = netif_running(dev); 1782 need_update = (!!priv->xdp_prog != !!prog); 1783 1784 if (up) 1785 dpaa2_eth_stop(dev); 1786 1787 /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 1788 * Also, when switching between xdp/non-xdp modes we need to reconfigure 1789 * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 1790 * so we are sure no old format buffers will be used from now on. 1791 */ 1792 if (need_update) { 1793 err = set_rx_mfl(priv, dev->mtu, !!prog); 1794 if (err) 1795 goto out_err; 1796 err = update_rx_buffer_headroom(priv, !!prog); 1797 if (err) 1798 goto out_err; 1799 } 1800 1801 old = xchg(&priv->xdp_prog, prog); 1802 if (old) 1803 bpf_prog_put(old); 1804 1805 for (i = 0; i < priv->num_channels; i++) { 1806 ch = priv->channel[i]; 1807 old = xchg(&ch->xdp.prog, prog); 1808 if (old) 1809 bpf_prog_put(old); 1810 } 1811 1812 if (up) { 1813 err = dpaa2_eth_open(dev); 1814 if (err) 1815 return err; 1816 } 1817 1818 return 0; 1819 1820 out_err: 1821 if (prog) 1822 bpf_prog_sub(prog, priv->num_channels); 1823 if (up) 1824 dpaa2_eth_open(dev); 1825 1826 return err; 1827 } 1828 1829 static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1830 { 1831 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1832 1833 switch (xdp->command) { 1834 case XDP_SETUP_PROG: 1835 return setup_xdp(dev, xdp->prog); 1836 case XDP_QUERY_PROG: 1837 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0; 1838 break; 1839 default: 1840 return -EINVAL; 1841 } 1842 1843 return 0; 1844 } 1845 1846 static int dpaa2_eth_xdp_xmit_frame(struct net_device *net_dev, 1847 struct xdp_frame *xdpf) 1848 { 1849 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1850 struct device *dev = net_dev->dev.parent; 1851 struct rtnl_link_stats64 *percpu_stats; 1852 struct dpaa2_eth_drv_stats *percpu_extras; 1853 unsigned int needed_headroom; 1854 struct dpaa2_eth_swa *swa; 1855 struct dpaa2_eth_fq *fq; 1856 struct dpaa2_fd fd; 1857 void *buffer_start, *aligned_start; 1858 dma_addr_t addr; 1859 int err, i; 1860 1861 /* We require a minimum headroom to be able to transmit the frame. 1862 * Otherwise return an error and let the original net_device handle it 1863 */ 1864 needed_headroom = dpaa2_eth_needed_headroom(priv, NULL); 1865 if (xdpf->headroom < needed_headroom) 1866 return -EINVAL; 1867 1868 percpu_stats = this_cpu_ptr(priv->percpu_stats); 1869 percpu_extras = this_cpu_ptr(priv->percpu_extras); 1870 1871 /* Setup the FD fields */ 1872 memset(&fd, 0, sizeof(fd)); 1873 1874 /* Align FD address, if possible */ 1875 buffer_start = xdpf->data - needed_headroom; 1876 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 1877 DPAA2_ETH_TX_BUF_ALIGN); 1878 if (aligned_start >= xdpf->data - xdpf->headroom) 1879 buffer_start = aligned_start; 1880 1881 swa = (struct dpaa2_eth_swa *)buffer_start; 1882 /* fill in necessary fields here */ 1883 swa->type = DPAA2_ETH_SWA_XDP; 1884 swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 1885 swa->xdp.xdpf = xdpf; 1886 1887 addr = dma_map_single(dev, buffer_start, 1888 swa->xdp.dma_size, 1889 DMA_BIDIRECTIONAL); 1890 if (unlikely(dma_mapping_error(dev, addr))) { 1891 percpu_stats->tx_dropped++; 1892 return -ENOMEM; 1893 } 1894 1895 dpaa2_fd_set_addr(&fd, addr); 1896 dpaa2_fd_set_offset(&fd, xdpf->data - buffer_start); 1897 dpaa2_fd_set_len(&fd, xdpf->len); 1898 dpaa2_fd_set_format(&fd, dpaa2_fd_single); 1899 dpaa2_fd_set_ctrl(&fd, FD_CTRL_PTA); 1900 1901 fq = &priv->fq[smp_processor_id() % dpaa2_eth_queue_count(priv)]; 1902 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 1903 err = priv->enqueue(priv, fq, &fd, 0); 1904 if (err != -EBUSY) 1905 break; 1906 } 1907 percpu_extras->tx_portal_busy += i; 1908 if (unlikely(err < 0)) { 1909 percpu_stats->tx_errors++; 1910 /* let the Rx device handle the cleanup */ 1911 return err; 1912 } 1913 1914 percpu_stats->tx_packets++; 1915 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fd); 1916 1917 return 0; 1918 } 1919 1920 static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 1921 struct xdp_frame **frames, u32 flags) 1922 { 1923 int drops = 0; 1924 int i, err; 1925 1926 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1927 return -EINVAL; 1928 1929 if (!netif_running(net_dev)) 1930 return -ENETDOWN; 1931 1932 for (i = 0; i < n; i++) { 1933 struct xdp_frame *xdpf = frames[i]; 1934 1935 err = dpaa2_eth_xdp_xmit_frame(net_dev, xdpf); 1936 if (err) { 1937 xdp_return_frame_rx_napi(xdpf); 1938 drops++; 1939 } 1940 } 1941 1942 return n - drops; 1943 } 1944 1945 static int update_xps(struct dpaa2_eth_priv *priv) 1946 { 1947 struct net_device *net_dev = priv->net_dev; 1948 struct cpumask xps_mask; 1949 struct dpaa2_eth_fq *fq; 1950 int i, num_queues, netdev_queues; 1951 int err = 0; 1952 1953 num_queues = dpaa2_eth_queue_count(priv); 1954 netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 1955 1956 /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 1957 * queues, so only process those 1958 */ 1959 for (i = 0; i < netdev_queues; i++) { 1960 fq = &priv->fq[i % num_queues]; 1961 1962 cpumask_clear(&xps_mask); 1963 cpumask_set_cpu(fq->target_cpu, &xps_mask); 1964 1965 err = netif_set_xps_queue(net_dev, &xps_mask, i); 1966 if (err) { 1967 netdev_warn_once(net_dev, "Error setting XPS queue\n"); 1968 break; 1969 } 1970 } 1971 1972 return err; 1973 } 1974 1975 static int dpaa2_eth_setup_tc(struct net_device *net_dev, 1976 enum tc_setup_type type, void *type_data) 1977 { 1978 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1979 struct tc_mqprio_qopt *mqprio = type_data; 1980 u8 num_tc, num_queues; 1981 int i; 1982 1983 if (type != TC_SETUP_QDISC_MQPRIO) 1984 return -EINVAL; 1985 1986 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1987 num_queues = dpaa2_eth_queue_count(priv); 1988 num_tc = mqprio->num_tc; 1989 1990 if (num_tc == net_dev->num_tc) 1991 return 0; 1992 1993 if (num_tc > dpaa2_eth_tc_count(priv)) { 1994 netdev_err(net_dev, "Max %d traffic classes supported\n", 1995 dpaa2_eth_tc_count(priv)); 1996 return -EINVAL; 1997 } 1998 1999 if (!num_tc) { 2000 netdev_reset_tc(net_dev); 2001 netif_set_real_num_tx_queues(net_dev, num_queues); 2002 goto out; 2003 } 2004 2005 netdev_set_num_tc(net_dev, num_tc); 2006 netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2007 2008 for (i = 0; i < num_tc; i++) 2009 netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2010 2011 out: 2012 update_xps(priv); 2013 2014 return 0; 2015 } 2016 2017 static const struct net_device_ops dpaa2_eth_ops = { 2018 .ndo_open = dpaa2_eth_open, 2019 .ndo_start_xmit = dpaa2_eth_tx, 2020 .ndo_stop = dpaa2_eth_stop, 2021 .ndo_set_mac_address = dpaa2_eth_set_addr, 2022 .ndo_get_stats64 = dpaa2_eth_get_stats, 2023 .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 2024 .ndo_set_features = dpaa2_eth_set_features, 2025 .ndo_do_ioctl = dpaa2_eth_ioctl, 2026 .ndo_change_mtu = dpaa2_eth_change_mtu, 2027 .ndo_bpf = dpaa2_eth_xdp, 2028 .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2029 .ndo_setup_tc = dpaa2_eth_setup_tc, 2030 }; 2031 2032 static void cdan_cb(struct dpaa2_io_notification_ctx *ctx) 2033 { 2034 struct dpaa2_eth_channel *ch; 2035 2036 ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 2037 2038 /* Update NAPI statistics */ 2039 ch->stats.cdan++; 2040 2041 napi_schedule_irqoff(&ch->napi); 2042 } 2043 2044 /* Allocate and configure a DPCON object */ 2045 static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv) 2046 { 2047 struct fsl_mc_device *dpcon; 2048 struct device *dev = priv->net_dev->dev.parent; 2049 struct dpcon_attr attrs; 2050 int err; 2051 2052 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 2053 FSL_MC_POOL_DPCON, &dpcon); 2054 if (err) { 2055 if (err == -ENXIO) 2056 err = -EPROBE_DEFER; 2057 else 2058 dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2059 return ERR_PTR(err); 2060 } 2061 2062 err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 2063 if (err) { 2064 dev_err(dev, "dpcon_open() failed\n"); 2065 goto free; 2066 } 2067 2068 err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 2069 if (err) { 2070 dev_err(dev, "dpcon_reset() failed\n"); 2071 goto close; 2072 } 2073 2074 err = dpcon_get_attributes(priv->mc_io, 0, dpcon->mc_handle, &attrs); 2075 if (err) { 2076 dev_err(dev, "dpcon_get_attributes() failed\n"); 2077 goto close; 2078 } 2079 2080 err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 2081 if (err) { 2082 dev_err(dev, "dpcon_enable() failed\n"); 2083 goto close; 2084 } 2085 2086 return dpcon; 2087 2088 close: 2089 dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 2090 free: 2091 fsl_mc_object_free(dpcon); 2092 2093 return NULL; 2094 } 2095 2096 static void free_dpcon(struct dpaa2_eth_priv *priv, 2097 struct fsl_mc_device *dpcon) 2098 { 2099 dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 2100 dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 2101 fsl_mc_object_free(dpcon); 2102 } 2103 2104 static struct dpaa2_eth_channel * 2105 alloc_channel(struct dpaa2_eth_priv *priv) 2106 { 2107 struct dpaa2_eth_channel *channel; 2108 struct dpcon_attr attr; 2109 struct device *dev = priv->net_dev->dev.parent; 2110 int err; 2111 2112 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 2113 if (!channel) 2114 return NULL; 2115 2116 channel->dpcon = setup_dpcon(priv); 2117 if (IS_ERR_OR_NULL(channel->dpcon)) { 2118 err = PTR_ERR_OR_ZERO(channel->dpcon); 2119 goto err_setup; 2120 } 2121 2122 err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 2123 &attr); 2124 if (err) { 2125 dev_err(dev, "dpcon_get_attributes() failed\n"); 2126 goto err_get_attr; 2127 } 2128 2129 channel->dpcon_id = attr.id; 2130 channel->ch_id = attr.qbman_ch_id; 2131 channel->priv = priv; 2132 2133 return channel; 2134 2135 err_get_attr: 2136 free_dpcon(priv, channel->dpcon); 2137 err_setup: 2138 kfree(channel); 2139 return ERR_PTR(err); 2140 } 2141 2142 static void free_channel(struct dpaa2_eth_priv *priv, 2143 struct dpaa2_eth_channel *channel) 2144 { 2145 free_dpcon(priv, channel->dpcon); 2146 kfree(channel); 2147 } 2148 2149 /* DPIO setup: allocate and configure QBMan channels, setup core affinity 2150 * and register data availability notifications 2151 */ 2152 static int setup_dpio(struct dpaa2_eth_priv *priv) 2153 { 2154 struct dpaa2_io_notification_ctx *nctx; 2155 struct dpaa2_eth_channel *channel; 2156 struct dpcon_notification_cfg dpcon_notif_cfg; 2157 struct device *dev = priv->net_dev->dev.parent; 2158 int i, err; 2159 2160 /* We want the ability to spread ingress traffic (RX, TX conf) to as 2161 * many cores as possible, so we need one channel for each core 2162 * (unless there's fewer queues than cores, in which case the extra 2163 * channels would be wasted). 2164 * Allocate one channel per core and register it to the core's 2165 * affine DPIO. If not enough channels are available for all cores 2166 * or if some cores don't have an affine DPIO, there will be no 2167 * ingress frame processing on those cores. 2168 */ 2169 cpumask_clear(&priv->dpio_cpumask); 2170 for_each_online_cpu(i) { 2171 /* Try to allocate a channel */ 2172 channel = alloc_channel(priv); 2173 if (IS_ERR_OR_NULL(channel)) { 2174 err = PTR_ERR_OR_ZERO(channel); 2175 if (err != -EPROBE_DEFER) 2176 dev_info(dev, 2177 "No affine channel for cpu %d and above\n", i); 2178 goto err_alloc_ch; 2179 } 2180 2181 priv->channel[priv->num_channels] = channel; 2182 2183 nctx = &channel->nctx; 2184 nctx->is_cdan = 1; 2185 nctx->cb = cdan_cb; 2186 nctx->id = channel->ch_id; 2187 nctx->desired_cpu = i; 2188 2189 /* Register the new context */ 2190 channel->dpio = dpaa2_io_service_select(i); 2191 err = dpaa2_io_service_register(channel->dpio, nctx, dev); 2192 if (err) { 2193 dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 2194 /* If no affine DPIO for this core, there's probably 2195 * none available for next cores either. Signal we want 2196 * to retry later, in case the DPIO devices weren't 2197 * probed yet. 2198 */ 2199 err = -EPROBE_DEFER; 2200 goto err_service_reg; 2201 } 2202 2203 /* Register DPCON notification with MC */ 2204 dpcon_notif_cfg.dpio_id = nctx->dpio_id; 2205 dpcon_notif_cfg.priority = 0; 2206 dpcon_notif_cfg.user_ctx = nctx->qman64; 2207 err = dpcon_set_notification(priv->mc_io, 0, 2208 channel->dpcon->mc_handle, 2209 &dpcon_notif_cfg); 2210 if (err) { 2211 dev_err(dev, "dpcon_set_notification failed()\n"); 2212 goto err_set_cdan; 2213 } 2214 2215 /* If we managed to allocate a channel and also found an affine 2216 * DPIO for this core, add it to the final mask 2217 */ 2218 cpumask_set_cpu(i, &priv->dpio_cpumask); 2219 priv->num_channels++; 2220 2221 /* Stop if we already have enough channels to accommodate all 2222 * RX and TX conf queues 2223 */ 2224 if (priv->num_channels == priv->dpni_attrs.num_queues) 2225 break; 2226 } 2227 2228 return 0; 2229 2230 err_set_cdan: 2231 dpaa2_io_service_deregister(channel->dpio, nctx, dev); 2232 err_service_reg: 2233 free_channel(priv, channel); 2234 err_alloc_ch: 2235 if (err == -EPROBE_DEFER) 2236 return err; 2237 2238 if (cpumask_empty(&priv->dpio_cpumask)) { 2239 dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 2240 return -ENODEV; 2241 } 2242 2243 dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 2244 cpumask_pr_args(&priv->dpio_cpumask)); 2245 2246 return 0; 2247 } 2248 2249 static void free_dpio(struct dpaa2_eth_priv *priv) 2250 { 2251 struct device *dev = priv->net_dev->dev.parent; 2252 struct dpaa2_eth_channel *ch; 2253 int i; 2254 2255 /* deregister CDAN notifications and free channels */ 2256 for (i = 0; i < priv->num_channels; i++) { 2257 ch = priv->channel[i]; 2258 dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 2259 free_channel(priv, ch); 2260 } 2261 } 2262 2263 static struct dpaa2_eth_channel *get_affine_channel(struct dpaa2_eth_priv *priv, 2264 int cpu) 2265 { 2266 struct device *dev = priv->net_dev->dev.parent; 2267 int i; 2268 2269 for (i = 0; i < priv->num_channels; i++) 2270 if (priv->channel[i]->nctx.desired_cpu == cpu) 2271 return priv->channel[i]; 2272 2273 /* We should never get here. Issue a warning and return 2274 * the first channel, because it's still better than nothing 2275 */ 2276 dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 2277 2278 return priv->channel[0]; 2279 } 2280 2281 static void set_fq_affinity(struct dpaa2_eth_priv *priv) 2282 { 2283 struct device *dev = priv->net_dev->dev.parent; 2284 struct dpaa2_eth_fq *fq; 2285 int rx_cpu, txc_cpu; 2286 int i; 2287 2288 /* For each FQ, pick one channel/CPU to deliver frames to. 2289 * This may well change at runtime, either through irqbalance or 2290 * through direct user intervention. 2291 */ 2292 rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 2293 2294 for (i = 0; i < priv->num_fqs; i++) { 2295 fq = &priv->fq[i]; 2296 switch (fq->type) { 2297 case DPAA2_RX_FQ: 2298 fq->target_cpu = rx_cpu; 2299 rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 2300 if (rx_cpu >= nr_cpu_ids) 2301 rx_cpu = cpumask_first(&priv->dpio_cpumask); 2302 break; 2303 case DPAA2_TX_CONF_FQ: 2304 fq->target_cpu = txc_cpu; 2305 txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 2306 if (txc_cpu >= nr_cpu_ids) 2307 txc_cpu = cpumask_first(&priv->dpio_cpumask); 2308 break; 2309 default: 2310 dev_err(dev, "Unknown FQ type: %d\n", fq->type); 2311 } 2312 fq->channel = get_affine_channel(priv, fq->target_cpu); 2313 } 2314 2315 update_xps(priv); 2316 } 2317 2318 static void setup_fqs(struct dpaa2_eth_priv *priv) 2319 { 2320 int i; 2321 2322 /* We have one TxConf FQ per Tx flow. 2323 * The number of Tx and Rx queues is the same. 2324 * Tx queues come first in the fq array. 2325 */ 2326 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 2327 priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 2328 priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 2329 priv->fq[priv->num_fqs++].flowid = (u16)i; 2330 } 2331 2332 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 2333 priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 2334 priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 2335 priv->fq[priv->num_fqs++].flowid = (u16)i; 2336 } 2337 2338 /* For each FQ, decide on which core to process incoming frames */ 2339 set_fq_affinity(priv); 2340 } 2341 2342 /* Allocate and configure one buffer pool for each interface */ 2343 static int setup_dpbp(struct dpaa2_eth_priv *priv) 2344 { 2345 int err; 2346 struct fsl_mc_device *dpbp_dev; 2347 struct device *dev = priv->net_dev->dev.parent; 2348 struct dpbp_attr dpbp_attrs; 2349 2350 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 2351 &dpbp_dev); 2352 if (err) { 2353 if (err == -ENXIO) 2354 err = -EPROBE_DEFER; 2355 else 2356 dev_err(dev, "DPBP device allocation failed\n"); 2357 return err; 2358 } 2359 2360 priv->dpbp_dev = dpbp_dev; 2361 2362 err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 2363 &dpbp_dev->mc_handle); 2364 if (err) { 2365 dev_err(dev, "dpbp_open() failed\n"); 2366 goto err_open; 2367 } 2368 2369 err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 2370 if (err) { 2371 dev_err(dev, "dpbp_reset() failed\n"); 2372 goto err_reset; 2373 } 2374 2375 err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 2376 if (err) { 2377 dev_err(dev, "dpbp_enable() failed\n"); 2378 goto err_enable; 2379 } 2380 2381 err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 2382 &dpbp_attrs); 2383 if (err) { 2384 dev_err(dev, "dpbp_get_attributes() failed\n"); 2385 goto err_get_attr; 2386 } 2387 priv->bpid = dpbp_attrs.bpid; 2388 2389 return 0; 2390 2391 err_get_attr: 2392 dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 2393 err_enable: 2394 err_reset: 2395 dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 2396 err_open: 2397 fsl_mc_object_free(dpbp_dev); 2398 2399 return err; 2400 } 2401 2402 static void free_dpbp(struct dpaa2_eth_priv *priv) 2403 { 2404 drain_pool(priv); 2405 dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 2406 dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 2407 fsl_mc_object_free(priv->dpbp_dev); 2408 } 2409 2410 static int set_buffer_layout(struct dpaa2_eth_priv *priv) 2411 { 2412 struct device *dev = priv->net_dev->dev.parent; 2413 struct dpni_buffer_layout buf_layout = {0}; 2414 u16 rx_buf_align; 2415 int err; 2416 2417 /* We need to check for WRIOP version 1.0.0, but depending on the MC 2418 * version, this number is not always provided correctly on rev1. 2419 * We need to check for both alternatives in this situation. 2420 */ 2421 if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 2422 priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 2423 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 2424 else 2425 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 2426 2427 /* tx buffer */ 2428 buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 2429 buf_layout.pass_timestamp = true; 2430 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 2431 DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2432 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2433 DPNI_QUEUE_TX, &buf_layout); 2434 if (err) { 2435 dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 2436 return err; 2437 } 2438 2439 /* tx-confirm buffer */ 2440 buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2441 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2442 DPNI_QUEUE_TX_CONFIRM, &buf_layout); 2443 if (err) { 2444 dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 2445 return err; 2446 } 2447 2448 /* Now that we've set our tx buffer layout, retrieve the minimum 2449 * required tx data offset. 2450 */ 2451 err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 2452 &priv->tx_data_offset); 2453 if (err) { 2454 dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 2455 return err; 2456 } 2457 2458 if ((priv->tx_data_offset % 64) != 0) 2459 dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 2460 priv->tx_data_offset); 2461 2462 /* rx buffer */ 2463 buf_layout.pass_frame_status = true; 2464 buf_layout.pass_parser_result = true; 2465 buf_layout.data_align = rx_buf_align; 2466 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 2467 buf_layout.private_data_size = 0; 2468 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 2469 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 2470 DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 2471 DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 2472 DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2473 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2474 DPNI_QUEUE_RX, &buf_layout); 2475 if (err) { 2476 dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 2477 return err; 2478 } 2479 2480 return 0; 2481 } 2482 2483 #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 2484 #define DPNI_ENQUEUE_FQID_VER_MINOR 9 2485 2486 static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 2487 struct dpaa2_eth_fq *fq, 2488 struct dpaa2_fd *fd, u8 prio) 2489 { 2490 return dpaa2_io_service_enqueue_qd(fq->channel->dpio, 2491 priv->tx_qdid, prio, 2492 fq->tx_qdbin, fd); 2493 } 2494 2495 static inline int dpaa2_eth_enqueue_fq(struct dpaa2_eth_priv *priv, 2496 struct dpaa2_eth_fq *fq, 2497 struct dpaa2_fd *fd, u8 prio) 2498 { 2499 return dpaa2_io_service_enqueue_fq(fq->channel->dpio, 2500 fq->tx_fqid[prio], fd); 2501 } 2502 2503 static void set_enqueue_mode(struct dpaa2_eth_priv *priv) 2504 { 2505 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 2506 DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 2507 priv->enqueue = dpaa2_eth_enqueue_qd; 2508 else 2509 priv->enqueue = dpaa2_eth_enqueue_fq; 2510 } 2511 2512 static int set_pause(struct dpaa2_eth_priv *priv) 2513 { 2514 struct device *dev = priv->net_dev->dev.parent; 2515 struct dpni_link_cfg link_cfg = {0}; 2516 int err; 2517 2518 /* Get the default link options so we don't override other flags */ 2519 err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 2520 if (err) { 2521 dev_err(dev, "dpni_get_link_cfg() failed\n"); 2522 return err; 2523 } 2524 2525 /* By default, enable both Rx and Tx pause frames */ 2526 link_cfg.options |= DPNI_LINK_OPT_PAUSE; 2527 link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 2528 err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 2529 if (err) { 2530 dev_err(dev, "dpni_set_link_cfg() failed\n"); 2531 return err; 2532 } 2533 2534 priv->link_state.options = link_cfg.options; 2535 2536 return 0; 2537 } 2538 2539 static void update_tx_fqids(struct dpaa2_eth_priv *priv) 2540 { 2541 struct dpni_queue_id qid = {0}; 2542 struct dpaa2_eth_fq *fq; 2543 struct dpni_queue queue; 2544 int i, j, err; 2545 2546 /* We only use Tx FQIDs for FQID-based enqueue, so check 2547 * if DPNI version supports it before updating FQIDs 2548 */ 2549 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 2550 DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 2551 return; 2552 2553 for (i = 0; i < priv->num_fqs; i++) { 2554 fq = &priv->fq[i]; 2555 if (fq->type != DPAA2_TX_CONF_FQ) 2556 continue; 2557 for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 2558 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 2559 DPNI_QUEUE_TX, j, fq->flowid, 2560 &queue, &qid); 2561 if (err) 2562 goto out_err; 2563 2564 fq->tx_fqid[j] = qid.fqid; 2565 if (fq->tx_fqid[j] == 0) 2566 goto out_err; 2567 } 2568 } 2569 2570 priv->enqueue = dpaa2_eth_enqueue_fq; 2571 2572 return; 2573 2574 out_err: 2575 netdev_info(priv->net_dev, 2576 "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 2577 priv->enqueue = dpaa2_eth_enqueue_qd; 2578 } 2579 2580 /* Configure the DPNI object this interface is associated with */ 2581 static int setup_dpni(struct fsl_mc_device *ls_dev) 2582 { 2583 struct device *dev = &ls_dev->dev; 2584 struct dpaa2_eth_priv *priv; 2585 struct net_device *net_dev; 2586 int err; 2587 2588 net_dev = dev_get_drvdata(dev); 2589 priv = netdev_priv(net_dev); 2590 2591 /* get a handle for the DPNI object */ 2592 err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 2593 if (err) { 2594 dev_err(dev, "dpni_open() failed\n"); 2595 return err; 2596 } 2597 2598 /* Check if we can work with this DPNI object */ 2599 err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 2600 &priv->dpni_ver_minor); 2601 if (err) { 2602 dev_err(dev, "dpni_get_api_version() failed\n"); 2603 goto close; 2604 } 2605 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 2606 dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 2607 priv->dpni_ver_major, priv->dpni_ver_minor, 2608 DPNI_VER_MAJOR, DPNI_VER_MINOR); 2609 err = -ENOTSUPP; 2610 goto close; 2611 } 2612 2613 ls_dev->mc_io = priv->mc_io; 2614 ls_dev->mc_handle = priv->mc_token; 2615 2616 err = dpni_reset(priv->mc_io, 0, priv->mc_token); 2617 if (err) { 2618 dev_err(dev, "dpni_reset() failed\n"); 2619 goto close; 2620 } 2621 2622 err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 2623 &priv->dpni_attrs); 2624 if (err) { 2625 dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 2626 goto close; 2627 } 2628 2629 err = set_buffer_layout(priv); 2630 if (err) 2631 goto close; 2632 2633 set_enqueue_mode(priv); 2634 2635 /* Enable pause frame support */ 2636 if (dpaa2_eth_has_pause_support(priv)) { 2637 err = set_pause(priv); 2638 if (err) 2639 goto close; 2640 } 2641 2642 priv->cls_rules = devm_kzalloc(dev, sizeof(struct dpaa2_eth_cls_rule) * 2643 dpaa2_eth_fs_count(priv), GFP_KERNEL); 2644 if (!priv->cls_rules) 2645 goto close; 2646 2647 return 0; 2648 2649 close: 2650 dpni_close(priv->mc_io, 0, priv->mc_token); 2651 2652 return err; 2653 } 2654 2655 static void free_dpni(struct dpaa2_eth_priv *priv) 2656 { 2657 int err; 2658 2659 err = dpni_reset(priv->mc_io, 0, priv->mc_token); 2660 if (err) 2661 netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 2662 err); 2663 2664 dpni_close(priv->mc_io, 0, priv->mc_token); 2665 } 2666 2667 static int setup_rx_flow(struct dpaa2_eth_priv *priv, 2668 struct dpaa2_eth_fq *fq) 2669 { 2670 struct device *dev = priv->net_dev->dev.parent; 2671 struct dpni_queue queue; 2672 struct dpni_queue_id qid; 2673 int err; 2674 2675 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 2676 DPNI_QUEUE_RX, 0, fq->flowid, &queue, &qid); 2677 if (err) { 2678 dev_err(dev, "dpni_get_queue(RX) failed\n"); 2679 return err; 2680 } 2681 2682 fq->fqid = qid.fqid; 2683 2684 queue.destination.id = fq->channel->dpcon_id; 2685 queue.destination.type = DPNI_DEST_DPCON; 2686 queue.destination.priority = 1; 2687 queue.user_context = (u64)(uintptr_t)fq; 2688 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 2689 DPNI_QUEUE_RX, 0, fq->flowid, 2690 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 2691 &queue); 2692 if (err) { 2693 dev_err(dev, "dpni_set_queue(RX) failed\n"); 2694 return err; 2695 } 2696 2697 /* xdp_rxq setup */ 2698 err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 2699 fq->flowid); 2700 if (err) { 2701 dev_err(dev, "xdp_rxq_info_reg failed\n"); 2702 return err; 2703 } 2704 2705 err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 2706 MEM_TYPE_PAGE_ORDER0, NULL); 2707 if (err) { 2708 dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 2709 return err; 2710 } 2711 2712 return 0; 2713 } 2714 2715 static int setup_tx_flow(struct dpaa2_eth_priv *priv, 2716 struct dpaa2_eth_fq *fq) 2717 { 2718 struct device *dev = priv->net_dev->dev.parent; 2719 struct dpni_queue queue; 2720 struct dpni_queue_id qid; 2721 int i, err; 2722 2723 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 2724 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 2725 DPNI_QUEUE_TX, i, fq->flowid, 2726 &queue, &qid); 2727 if (err) { 2728 dev_err(dev, "dpni_get_queue(TX) failed\n"); 2729 return err; 2730 } 2731 fq->tx_fqid[i] = qid.fqid; 2732 } 2733 2734 /* All Tx queues belonging to the same flowid have the same qdbin */ 2735 fq->tx_qdbin = qid.qdbin; 2736 2737 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 2738 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 2739 &queue, &qid); 2740 if (err) { 2741 dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 2742 return err; 2743 } 2744 2745 fq->fqid = qid.fqid; 2746 2747 queue.destination.id = fq->channel->dpcon_id; 2748 queue.destination.type = DPNI_DEST_DPCON; 2749 queue.destination.priority = 0; 2750 queue.user_context = (u64)(uintptr_t)fq; 2751 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 2752 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 2753 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 2754 &queue); 2755 if (err) { 2756 dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 2757 return err; 2758 } 2759 2760 return 0; 2761 } 2762 2763 /* Supported header fields for Rx hash distribution key */ 2764 static const struct dpaa2_eth_dist_fields dist_fields[] = { 2765 { 2766 /* L2 header */ 2767 .rxnfc_field = RXH_L2DA, 2768 .cls_prot = NET_PROT_ETH, 2769 .cls_field = NH_FLD_ETH_DA, 2770 .id = DPAA2_ETH_DIST_ETHDST, 2771 .size = 6, 2772 }, { 2773 .cls_prot = NET_PROT_ETH, 2774 .cls_field = NH_FLD_ETH_SA, 2775 .id = DPAA2_ETH_DIST_ETHSRC, 2776 .size = 6, 2777 }, { 2778 /* This is the last ethertype field parsed: 2779 * depending on frame format, it can be the MAC ethertype 2780 * or the VLAN etype. 2781 */ 2782 .cls_prot = NET_PROT_ETH, 2783 .cls_field = NH_FLD_ETH_TYPE, 2784 .id = DPAA2_ETH_DIST_ETHTYPE, 2785 .size = 2, 2786 }, { 2787 /* VLAN header */ 2788 .rxnfc_field = RXH_VLAN, 2789 .cls_prot = NET_PROT_VLAN, 2790 .cls_field = NH_FLD_VLAN_TCI, 2791 .id = DPAA2_ETH_DIST_VLAN, 2792 .size = 2, 2793 }, { 2794 /* IP header */ 2795 .rxnfc_field = RXH_IP_SRC, 2796 .cls_prot = NET_PROT_IP, 2797 .cls_field = NH_FLD_IP_SRC, 2798 .id = DPAA2_ETH_DIST_IPSRC, 2799 .size = 4, 2800 }, { 2801 .rxnfc_field = RXH_IP_DST, 2802 .cls_prot = NET_PROT_IP, 2803 .cls_field = NH_FLD_IP_DST, 2804 .id = DPAA2_ETH_DIST_IPDST, 2805 .size = 4, 2806 }, { 2807 .rxnfc_field = RXH_L3_PROTO, 2808 .cls_prot = NET_PROT_IP, 2809 .cls_field = NH_FLD_IP_PROTO, 2810 .id = DPAA2_ETH_DIST_IPPROTO, 2811 .size = 1, 2812 }, { 2813 /* Using UDP ports, this is functionally equivalent to raw 2814 * byte pairs from L4 header. 2815 */ 2816 .rxnfc_field = RXH_L4_B_0_1, 2817 .cls_prot = NET_PROT_UDP, 2818 .cls_field = NH_FLD_UDP_PORT_SRC, 2819 .id = DPAA2_ETH_DIST_L4SRC, 2820 .size = 2, 2821 }, { 2822 .rxnfc_field = RXH_L4_B_2_3, 2823 .cls_prot = NET_PROT_UDP, 2824 .cls_field = NH_FLD_UDP_PORT_DST, 2825 .id = DPAA2_ETH_DIST_L4DST, 2826 .size = 2, 2827 }, 2828 }; 2829 2830 /* Configure the Rx hash key using the legacy API */ 2831 static int config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 2832 { 2833 struct device *dev = priv->net_dev->dev.parent; 2834 struct dpni_rx_tc_dist_cfg dist_cfg; 2835 int err; 2836 2837 memset(&dist_cfg, 0, sizeof(dist_cfg)); 2838 2839 dist_cfg.key_cfg_iova = key; 2840 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 2841 dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 2842 2843 err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 0, &dist_cfg); 2844 if (err) 2845 dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 2846 2847 return err; 2848 } 2849 2850 /* Configure the Rx hash key using the new API */ 2851 static int config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 2852 { 2853 struct device *dev = priv->net_dev->dev.parent; 2854 struct dpni_rx_dist_cfg dist_cfg; 2855 int err; 2856 2857 memset(&dist_cfg, 0, sizeof(dist_cfg)); 2858 2859 dist_cfg.key_cfg_iova = key; 2860 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 2861 dist_cfg.enable = 1; 2862 2863 err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg); 2864 if (err) 2865 dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 2866 2867 return err; 2868 } 2869 2870 /* Configure the Rx flow classification key */ 2871 static int config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 2872 { 2873 struct device *dev = priv->net_dev->dev.parent; 2874 struct dpni_rx_dist_cfg dist_cfg; 2875 int err; 2876 2877 memset(&dist_cfg, 0, sizeof(dist_cfg)); 2878 2879 dist_cfg.key_cfg_iova = key; 2880 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 2881 dist_cfg.enable = 1; 2882 2883 err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg); 2884 if (err) 2885 dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 2886 2887 return err; 2888 } 2889 2890 /* Size of the Rx flow classification key */ 2891 int dpaa2_eth_cls_key_size(u64 fields) 2892 { 2893 int i, size = 0; 2894 2895 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 2896 if (!(fields & dist_fields[i].id)) 2897 continue; 2898 size += dist_fields[i].size; 2899 } 2900 2901 return size; 2902 } 2903 2904 /* Offset of header field in Rx classification key */ 2905 int dpaa2_eth_cls_fld_off(int prot, int field) 2906 { 2907 int i, off = 0; 2908 2909 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 2910 if (dist_fields[i].cls_prot == prot && 2911 dist_fields[i].cls_field == field) 2912 return off; 2913 off += dist_fields[i].size; 2914 } 2915 2916 WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 2917 return 0; 2918 } 2919 2920 /* Prune unused fields from the classification rule. 2921 * Used when masking is not supported 2922 */ 2923 void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 2924 { 2925 int off = 0, new_off = 0; 2926 int i, size; 2927 2928 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 2929 size = dist_fields[i].size; 2930 if (dist_fields[i].id & fields) { 2931 memcpy(key_mem + new_off, key_mem + off, size); 2932 new_off += size; 2933 } 2934 off += size; 2935 } 2936 } 2937 2938 /* Set Rx distribution (hash or flow classification) key 2939 * flags is a combination of RXH_ bits 2940 */ 2941 static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 2942 enum dpaa2_eth_rx_dist type, u64 flags) 2943 { 2944 struct device *dev = net_dev->dev.parent; 2945 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2946 struct dpkg_profile_cfg cls_cfg; 2947 u32 rx_hash_fields = 0; 2948 dma_addr_t key_iova; 2949 u8 *dma_mem; 2950 int i; 2951 int err = 0; 2952 2953 memset(&cls_cfg, 0, sizeof(cls_cfg)); 2954 2955 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 2956 struct dpkg_extract *key = 2957 &cls_cfg.extracts[cls_cfg.num_extracts]; 2958 2959 /* For both Rx hashing and classification keys 2960 * we set only the selected fields. 2961 */ 2962 if (!(flags & dist_fields[i].id)) 2963 continue; 2964 if (type == DPAA2_ETH_RX_DIST_HASH) 2965 rx_hash_fields |= dist_fields[i].rxnfc_field; 2966 2967 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 2968 dev_err(dev, "error adding key extraction rule, too many rules?\n"); 2969 return -E2BIG; 2970 } 2971 2972 key->type = DPKG_EXTRACT_FROM_HDR; 2973 key->extract.from_hdr.prot = dist_fields[i].cls_prot; 2974 key->extract.from_hdr.type = DPKG_FULL_FIELD; 2975 key->extract.from_hdr.field = dist_fields[i].cls_field; 2976 cls_cfg.num_extracts++; 2977 } 2978 2979 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 2980 if (!dma_mem) 2981 return -ENOMEM; 2982 2983 err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 2984 if (err) { 2985 dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 2986 goto free_key; 2987 } 2988 2989 /* Prepare for setting the rx dist */ 2990 key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 2991 DMA_TO_DEVICE); 2992 if (dma_mapping_error(dev, key_iova)) { 2993 dev_err(dev, "DMA mapping failed\n"); 2994 err = -ENOMEM; 2995 goto free_key; 2996 } 2997 2998 if (type == DPAA2_ETH_RX_DIST_HASH) { 2999 if (dpaa2_eth_has_legacy_dist(priv)) 3000 err = config_legacy_hash_key(priv, key_iova); 3001 else 3002 err = config_hash_key(priv, key_iova); 3003 } else { 3004 err = config_cls_key(priv, key_iova); 3005 } 3006 3007 dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3008 DMA_TO_DEVICE); 3009 if (!err && type == DPAA2_ETH_RX_DIST_HASH) 3010 priv->rx_hash_fields = rx_hash_fields; 3011 3012 free_key: 3013 kfree(dma_mem); 3014 return err; 3015 } 3016 3017 int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 3018 { 3019 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3020 u64 key = 0; 3021 int i; 3022 3023 if (!dpaa2_eth_hash_enabled(priv)) 3024 return -EOPNOTSUPP; 3025 3026 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 3027 if (dist_fields[i].rxnfc_field & flags) 3028 key |= dist_fields[i].id; 3029 3030 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 3031 } 3032 3033 int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 3034 { 3035 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 3036 } 3037 3038 static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 3039 { 3040 struct device *dev = priv->net_dev->dev.parent; 3041 int err; 3042 3043 /* Check if we actually support Rx flow classification */ 3044 if (dpaa2_eth_has_legacy_dist(priv)) { 3045 dev_dbg(dev, "Rx cls not supported by current MC version\n"); 3046 return -EOPNOTSUPP; 3047 } 3048 3049 if (!dpaa2_eth_fs_enabled(priv)) { 3050 dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 3051 return -EOPNOTSUPP; 3052 } 3053 3054 if (!dpaa2_eth_hash_enabled(priv)) { 3055 dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 3056 return -EOPNOTSUPP; 3057 } 3058 3059 /* If there is no support for masking in the classification table, 3060 * we don't set a default key, as it will depend on the rules 3061 * added by the user at runtime. 3062 */ 3063 if (!dpaa2_eth_fs_mask_enabled(priv)) 3064 goto out; 3065 3066 err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 3067 if (err) 3068 return err; 3069 3070 out: 3071 priv->rx_cls_enabled = 1; 3072 3073 return 0; 3074 } 3075 3076 /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 3077 * frame queues and channels 3078 */ 3079 static int bind_dpni(struct dpaa2_eth_priv *priv) 3080 { 3081 struct net_device *net_dev = priv->net_dev; 3082 struct device *dev = net_dev->dev.parent; 3083 struct dpni_pools_cfg pools_params; 3084 struct dpni_error_cfg err_cfg; 3085 int err = 0; 3086 int i; 3087 3088 pools_params.num_dpbp = 1; 3089 pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 3090 pools_params.pools[0].backup_pool = 0; 3091 pools_params.pools[0].buffer_size = DPAA2_ETH_RX_BUF_SIZE; 3092 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 3093 if (err) { 3094 dev_err(dev, "dpni_set_pools() failed\n"); 3095 return err; 3096 } 3097 3098 /* have the interface implicitly distribute traffic based on 3099 * the default hash key 3100 */ 3101 err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 3102 if (err && err != -EOPNOTSUPP) 3103 dev_err(dev, "Failed to configure hashing\n"); 3104 3105 /* Configure the flow classification key; it includes all 3106 * supported header fields and cannot be modified at runtime 3107 */ 3108 err = dpaa2_eth_set_default_cls(priv); 3109 if (err && err != -EOPNOTSUPP) 3110 dev_err(dev, "Failed to configure Rx classification key\n"); 3111 3112 /* Configure handling of error frames */ 3113 err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 3114 err_cfg.set_frame_annotation = 1; 3115 err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 3116 err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 3117 &err_cfg); 3118 if (err) { 3119 dev_err(dev, "dpni_set_errors_behavior failed\n"); 3120 return err; 3121 } 3122 3123 /* Configure Rx and Tx conf queues to generate CDANs */ 3124 for (i = 0; i < priv->num_fqs; i++) { 3125 switch (priv->fq[i].type) { 3126 case DPAA2_RX_FQ: 3127 err = setup_rx_flow(priv, &priv->fq[i]); 3128 break; 3129 case DPAA2_TX_CONF_FQ: 3130 err = setup_tx_flow(priv, &priv->fq[i]); 3131 break; 3132 default: 3133 dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 3134 return -EINVAL; 3135 } 3136 if (err) 3137 return err; 3138 } 3139 3140 err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 3141 DPNI_QUEUE_TX, &priv->tx_qdid); 3142 if (err) { 3143 dev_err(dev, "dpni_get_qdid() failed\n"); 3144 return err; 3145 } 3146 3147 return 0; 3148 } 3149 3150 /* Allocate rings for storing incoming frame descriptors */ 3151 static int alloc_rings(struct dpaa2_eth_priv *priv) 3152 { 3153 struct net_device *net_dev = priv->net_dev; 3154 struct device *dev = net_dev->dev.parent; 3155 int i; 3156 3157 for (i = 0; i < priv->num_channels; i++) { 3158 priv->channel[i]->store = 3159 dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 3160 if (!priv->channel[i]->store) { 3161 netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 3162 goto err_ring; 3163 } 3164 } 3165 3166 return 0; 3167 3168 err_ring: 3169 for (i = 0; i < priv->num_channels; i++) { 3170 if (!priv->channel[i]->store) 3171 break; 3172 dpaa2_io_store_destroy(priv->channel[i]->store); 3173 } 3174 3175 return -ENOMEM; 3176 } 3177 3178 static void free_rings(struct dpaa2_eth_priv *priv) 3179 { 3180 int i; 3181 3182 for (i = 0; i < priv->num_channels; i++) 3183 dpaa2_io_store_destroy(priv->channel[i]->store); 3184 } 3185 3186 static int set_mac_addr(struct dpaa2_eth_priv *priv) 3187 { 3188 struct net_device *net_dev = priv->net_dev; 3189 struct device *dev = net_dev->dev.parent; 3190 u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 3191 int err; 3192 3193 /* Get firmware address, if any */ 3194 err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 3195 if (err) { 3196 dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 3197 return err; 3198 } 3199 3200 /* Get DPNI attributes address, if any */ 3201 err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 3202 dpni_mac_addr); 3203 if (err) { 3204 dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 3205 return err; 3206 } 3207 3208 /* First check if firmware has any address configured by bootloader */ 3209 if (!is_zero_ether_addr(mac_addr)) { 3210 /* If the DPMAC addr != DPNI addr, update it */ 3211 if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 3212 err = dpni_set_primary_mac_addr(priv->mc_io, 0, 3213 priv->mc_token, 3214 mac_addr); 3215 if (err) { 3216 dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 3217 return err; 3218 } 3219 } 3220 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); 3221 } else if (is_zero_ether_addr(dpni_mac_addr)) { 3222 /* No MAC address configured, fill in net_dev->dev_addr 3223 * with a random one 3224 */ 3225 eth_hw_addr_random(net_dev); 3226 dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 3227 3228 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 3229 net_dev->dev_addr); 3230 if (err) { 3231 dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 3232 return err; 3233 } 3234 3235 /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 3236 * practical purposes, this will be our "permanent" mac address, 3237 * at least until the next reboot. This move will also permit 3238 * register_netdevice() to properly fill up net_dev->perm_addr. 3239 */ 3240 net_dev->addr_assign_type = NET_ADDR_PERM; 3241 } else { 3242 /* NET_ADDR_PERM is default, all we have to do is 3243 * fill in the device addr. 3244 */ 3245 memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len); 3246 } 3247 3248 return 0; 3249 } 3250 3251 static int netdev_init(struct net_device *net_dev) 3252 { 3253 struct device *dev = net_dev->dev.parent; 3254 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3255 u32 options = priv->dpni_attrs.options; 3256 u64 supported = 0, not_supported = 0; 3257 u8 bcast_addr[ETH_ALEN]; 3258 u8 num_queues; 3259 int err; 3260 3261 net_dev->netdev_ops = &dpaa2_eth_ops; 3262 net_dev->ethtool_ops = &dpaa2_ethtool_ops; 3263 3264 err = set_mac_addr(priv); 3265 if (err) 3266 return err; 3267 3268 /* Explicitly add the broadcast address to the MAC filtering table */ 3269 eth_broadcast_addr(bcast_addr); 3270 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 3271 if (err) { 3272 dev_err(dev, "dpni_add_mac_addr() failed\n"); 3273 return err; 3274 } 3275 3276 /* Set MTU upper limit; lower limit is 68B (default value) */ 3277 net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 3278 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 3279 DPAA2_ETH_MFL); 3280 if (err) { 3281 dev_err(dev, "dpni_set_max_frame_length() failed\n"); 3282 return err; 3283 } 3284 3285 /* Set actual number of queues in the net device */ 3286 num_queues = dpaa2_eth_queue_count(priv); 3287 err = netif_set_real_num_tx_queues(net_dev, num_queues); 3288 if (err) { 3289 dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 3290 return err; 3291 } 3292 err = netif_set_real_num_rx_queues(net_dev, num_queues); 3293 if (err) { 3294 dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 3295 return err; 3296 } 3297 3298 /* Capabilities listing */ 3299 supported |= IFF_LIVE_ADDR_CHANGE; 3300 3301 if (options & DPNI_OPT_NO_MAC_FILTER) 3302 not_supported |= IFF_UNICAST_FLT; 3303 else 3304 supported |= IFF_UNICAST_FLT; 3305 3306 net_dev->priv_flags |= supported; 3307 net_dev->priv_flags &= ~not_supported; 3308 3309 /* Features */ 3310 net_dev->features = NETIF_F_RXCSUM | 3311 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3312 NETIF_F_SG | NETIF_F_HIGHDMA | 3313 NETIF_F_LLTX; 3314 net_dev->hw_features = net_dev->features; 3315 3316 return 0; 3317 } 3318 3319 static int poll_link_state(void *arg) 3320 { 3321 struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 3322 int err; 3323 3324 while (!kthread_should_stop()) { 3325 err = link_state_update(priv); 3326 if (unlikely(err)) 3327 return err; 3328 3329 msleep(DPAA2_ETH_LINK_STATE_REFRESH); 3330 } 3331 3332 return 0; 3333 } 3334 3335 static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 3336 { 3337 u32 status = ~0; 3338 struct device *dev = (struct device *)arg; 3339 struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 3340 struct net_device *net_dev = dev_get_drvdata(dev); 3341 int err; 3342 3343 err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 3344 DPNI_IRQ_INDEX, &status); 3345 if (unlikely(err)) { 3346 netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 3347 return IRQ_HANDLED; 3348 } 3349 3350 if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 3351 link_state_update(netdev_priv(net_dev)); 3352 3353 if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) 3354 set_mac_addr(netdev_priv(net_dev)); 3355 3356 return IRQ_HANDLED; 3357 } 3358 3359 static int setup_irqs(struct fsl_mc_device *ls_dev) 3360 { 3361 int err = 0; 3362 struct fsl_mc_device_irq *irq; 3363 3364 err = fsl_mc_allocate_irqs(ls_dev); 3365 if (err) { 3366 dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 3367 return err; 3368 } 3369 3370 irq = ls_dev->irqs[0]; 3371 err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, 3372 NULL, dpni_irq0_handler_thread, 3373 IRQF_NO_SUSPEND | IRQF_ONESHOT, 3374 dev_name(&ls_dev->dev), &ls_dev->dev); 3375 if (err < 0) { 3376 dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 3377 goto free_mc_irq; 3378 } 3379 3380 err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 3381 DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 3382 DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 3383 if (err < 0) { 3384 dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 3385 goto free_irq; 3386 } 3387 3388 err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 3389 DPNI_IRQ_INDEX, 1); 3390 if (err < 0) { 3391 dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 3392 goto free_irq; 3393 } 3394 3395 return 0; 3396 3397 free_irq: 3398 devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); 3399 free_mc_irq: 3400 fsl_mc_free_irqs(ls_dev); 3401 3402 return err; 3403 } 3404 3405 static void add_ch_napi(struct dpaa2_eth_priv *priv) 3406 { 3407 int i; 3408 struct dpaa2_eth_channel *ch; 3409 3410 for (i = 0; i < priv->num_channels; i++) { 3411 ch = priv->channel[i]; 3412 /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 3413 netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 3414 NAPI_POLL_WEIGHT); 3415 } 3416 } 3417 3418 static void del_ch_napi(struct dpaa2_eth_priv *priv) 3419 { 3420 int i; 3421 struct dpaa2_eth_channel *ch; 3422 3423 for (i = 0; i < priv->num_channels; i++) { 3424 ch = priv->channel[i]; 3425 netif_napi_del(&ch->napi); 3426 } 3427 } 3428 3429 static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 3430 { 3431 struct device *dev; 3432 struct net_device *net_dev = NULL; 3433 struct dpaa2_eth_priv *priv = NULL; 3434 int err = 0; 3435 3436 dev = &dpni_dev->dev; 3437 3438 /* Net device */ 3439 net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 3440 if (!net_dev) { 3441 dev_err(dev, "alloc_etherdev_mq() failed\n"); 3442 return -ENOMEM; 3443 } 3444 3445 SET_NETDEV_DEV(net_dev, dev); 3446 dev_set_drvdata(dev, net_dev); 3447 3448 priv = netdev_priv(net_dev); 3449 priv->net_dev = net_dev; 3450 3451 priv->iommu_domain = iommu_get_domain_for_dev(dev); 3452 3453 /* Obtain a MC portal */ 3454 err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 3455 &priv->mc_io); 3456 if (err) { 3457 if (err == -ENXIO) 3458 err = -EPROBE_DEFER; 3459 else 3460 dev_err(dev, "MC portal allocation failed\n"); 3461 goto err_portal_alloc; 3462 } 3463 3464 /* MC objects initialization and configuration */ 3465 err = setup_dpni(dpni_dev); 3466 if (err) 3467 goto err_dpni_setup; 3468 3469 err = setup_dpio(priv); 3470 if (err) 3471 goto err_dpio_setup; 3472 3473 setup_fqs(priv); 3474 3475 err = setup_dpbp(priv); 3476 if (err) 3477 goto err_dpbp_setup; 3478 3479 err = bind_dpni(priv); 3480 if (err) 3481 goto err_bind; 3482 3483 /* Add a NAPI context for each channel */ 3484 add_ch_napi(priv); 3485 3486 /* Percpu statistics */ 3487 priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 3488 if (!priv->percpu_stats) { 3489 dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 3490 err = -ENOMEM; 3491 goto err_alloc_percpu_stats; 3492 } 3493 priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 3494 if (!priv->percpu_extras) { 3495 dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 3496 err = -ENOMEM; 3497 goto err_alloc_percpu_extras; 3498 } 3499 3500 err = netdev_init(net_dev); 3501 if (err) 3502 goto err_netdev_init; 3503 3504 /* Configure checksum offload based on current interface flags */ 3505 err = set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 3506 if (err) 3507 goto err_csum; 3508 3509 err = set_tx_csum(priv, !!(net_dev->features & 3510 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 3511 if (err) 3512 goto err_csum; 3513 3514 err = alloc_rings(priv); 3515 if (err) 3516 goto err_alloc_rings; 3517 3518 err = setup_irqs(dpni_dev); 3519 if (err) { 3520 netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 3521 priv->poll_thread = kthread_run(poll_link_state, priv, 3522 "%s_poll_link", net_dev->name); 3523 if (IS_ERR(priv->poll_thread)) { 3524 dev_err(dev, "Error starting polling thread\n"); 3525 goto err_poll_thread; 3526 } 3527 priv->do_link_poll = true; 3528 } 3529 3530 err = register_netdev(net_dev); 3531 if (err < 0) { 3532 dev_err(dev, "register_netdev() failed\n"); 3533 goto err_netdev_reg; 3534 } 3535 3536 #ifdef CONFIG_DEBUG_FS 3537 dpaa2_dbg_add(priv); 3538 #endif 3539 3540 dev_info(dev, "Probed interface %s\n", net_dev->name); 3541 return 0; 3542 3543 err_netdev_reg: 3544 if (priv->do_link_poll) 3545 kthread_stop(priv->poll_thread); 3546 else 3547 fsl_mc_free_irqs(dpni_dev); 3548 err_poll_thread: 3549 free_rings(priv); 3550 err_alloc_rings: 3551 err_csum: 3552 err_netdev_init: 3553 free_percpu(priv->percpu_extras); 3554 err_alloc_percpu_extras: 3555 free_percpu(priv->percpu_stats); 3556 err_alloc_percpu_stats: 3557 del_ch_napi(priv); 3558 err_bind: 3559 free_dpbp(priv); 3560 err_dpbp_setup: 3561 free_dpio(priv); 3562 err_dpio_setup: 3563 free_dpni(priv); 3564 err_dpni_setup: 3565 fsl_mc_portal_free(priv->mc_io); 3566 err_portal_alloc: 3567 dev_set_drvdata(dev, NULL); 3568 free_netdev(net_dev); 3569 3570 return err; 3571 } 3572 3573 static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 3574 { 3575 struct device *dev; 3576 struct net_device *net_dev; 3577 struct dpaa2_eth_priv *priv; 3578 3579 dev = &ls_dev->dev; 3580 net_dev = dev_get_drvdata(dev); 3581 priv = netdev_priv(net_dev); 3582 3583 #ifdef CONFIG_DEBUG_FS 3584 dpaa2_dbg_remove(priv); 3585 #endif 3586 unregister_netdev(net_dev); 3587 3588 if (priv->do_link_poll) 3589 kthread_stop(priv->poll_thread); 3590 else 3591 fsl_mc_free_irqs(ls_dev); 3592 3593 free_rings(priv); 3594 free_percpu(priv->percpu_stats); 3595 free_percpu(priv->percpu_extras); 3596 3597 del_ch_napi(priv); 3598 free_dpbp(priv); 3599 free_dpio(priv); 3600 free_dpni(priv); 3601 3602 fsl_mc_portal_free(priv->mc_io); 3603 3604 free_netdev(net_dev); 3605 3606 dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 3607 3608 return 0; 3609 } 3610 3611 static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 3612 { 3613 .vendor = FSL_MC_VENDOR_FREESCALE, 3614 .obj_type = "dpni", 3615 }, 3616 { .vendor = 0x0 } 3617 }; 3618 MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 3619 3620 static struct fsl_mc_driver dpaa2_eth_driver = { 3621 .driver = { 3622 .name = KBUILD_MODNAME, 3623 .owner = THIS_MODULE, 3624 }, 3625 .probe = dpaa2_eth_probe, 3626 .remove = dpaa2_eth_remove, 3627 .match_id_table = dpaa2_eth_match_id_table 3628 }; 3629 3630 static int __init dpaa2_eth_driver_init(void) 3631 { 3632 int err; 3633 3634 dpaa2_eth_dbg_init(); 3635 err = fsl_mc_driver_register(&dpaa2_eth_driver); 3636 if (err) { 3637 dpaa2_eth_dbg_exit(); 3638 return err; 3639 } 3640 3641 return 0; 3642 } 3643 3644 static void __exit dpaa2_eth_driver_exit(void) 3645 { 3646 dpaa2_eth_dbg_exit(); 3647 fsl_mc_driver_unregister(&dpaa2_eth_driver); 3648 } 3649 3650 module_init(dpaa2_eth_driver_init); 3651 module_exit(dpaa2_eth_driver_exit); 3652