1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2014-2016 Freescale Semiconductor Inc. 3 * Copyright 2016-2020 NXP 4 */ 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/platform_device.h> 8 #include <linux/etherdevice.h> 9 #include <linux/of_net.h> 10 #include <linux/interrupt.h> 11 #include <linux/msi.h> 12 #include <linux/kthread.h> 13 #include <linux/iommu.h> 14 #include <linux/fsl/mc.h> 15 #include <linux/bpf.h> 16 #include <linux/bpf_trace.h> 17 #include <linux/fsl/ptp_qoriq.h> 18 #include <linux/ptp_classify.h> 19 #include <net/pkt_cls.h> 20 #include <net/sock.h> 21 22 #include "dpaa2-eth.h" 23 24 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 25 * using trace events only need to #include <trace/events/sched.h> 26 */ 27 #define CREATE_TRACE_POINTS 28 #include "dpaa2-eth-trace.h" 29 30 MODULE_LICENSE("Dual BSD/GPL"); 31 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 32 MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 33 34 struct ptp_qoriq *dpaa2_ptp; 35 EXPORT_SYMBOL(dpaa2_ptp); 36 37 static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 38 dma_addr_t iova_addr) 39 { 40 phys_addr_t phys_addr; 41 42 phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 43 44 return phys_to_virt(phys_addr); 45 } 46 47 static void dpaa2_eth_validate_rx_csum(struct dpaa2_eth_priv *priv, 48 u32 fd_status, 49 struct sk_buff *skb) 50 { 51 skb_checksum_none_assert(skb); 52 53 /* HW checksum validation is disabled, nothing to do here */ 54 if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 55 return; 56 57 /* Read checksum validation bits */ 58 if (!((fd_status & DPAA2_FAS_L3CV) && 59 (fd_status & DPAA2_FAS_L4CV))) 60 return; 61 62 /* Inform the stack there's no need to compute L3/L4 csum anymore */ 63 skb->ip_summed = CHECKSUM_UNNECESSARY; 64 } 65 66 /* Free a received FD. 67 * Not to be used for Tx conf FDs or on any other paths. 68 */ 69 static void dpaa2_eth_free_rx_fd(struct dpaa2_eth_priv *priv, 70 const struct dpaa2_fd *fd, 71 void *vaddr) 72 { 73 struct device *dev = priv->net_dev->dev.parent; 74 dma_addr_t addr = dpaa2_fd_get_addr(fd); 75 u8 fd_format = dpaa2_fd_get_format(fd); 76 struct dpaa2_sg_entry *sgt; 77 void *sg_vaddr; 78 int i; 79 80 /* If single buffer frame, just free the data buffer */ 81 if (fd_format == dpaa2_fd_single) 82 goto free_buf; 83 else if (fd_format != dpaa2_fd_sg) 84 /* We don't support any other format */ 85 return; 86 87 /* For S/G frames, we first need to free all SG entries 88 * except the first one, which was taken care of already 89 */ 90 sgt = vaddr + dpaa2_fd_get_offset(fd); 91 for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 92 addr = dpaa2_sg_get_addr(&sgt[i]); 93 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 94 dma_unmap_page(dev, addr, priv->rx_buf_size, 95 DMA_BIDIRECTIONAL); 96 97 free_pages((unsigned long)sg_vaddr, 0); 98 if (dpaa2_sg_is_final(&sgt[i])) 99 break; 100 } 101 102 free_buf: 103 free_pages((unsigned long)vaddr, 0); 104 } 105 106 /* Build a linear skb based on a single-buffer frame descriptor */ 107 static struct sk_buff *dpaa2_eth_build_linear_skb(struct dpaa2_eth_channel *ch, 108 const struct dpaa2_fd *fd, 109 void *fd_vaddr) 110 { 111 struct sk_buff *skb = NULL; 112 u16 fd_offset = dpaa2_fd_get_offset(fd); 113 u32 fd_length = dpaa2_fd_get_len(fd); 114 115 ch->buf_count--; 116 117 skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 118 if (unlikely(!skb)) 119 return NULL; 120 121 skb_reserve(skb, fd_offset); 122 skb_put(skb, fd_length); 123 124 return skb; 125 } 126 127 /* Build a non linear (fragmented) skb based on a S/G table */ 128 static struct sk_buff *dpaa2_eth_build_frag_skb(struct dpaa2_eth_priv *priv, 129 struct dpaa2_eth_channel *ch, 130 struct dpaa2_sg_entry *sgt) 131 { 132 struct sk_buff *skb = NULL; 133 struct device *dev = priv->net_dev->dev.parent; 134 void *sg_vaddr; 135 dma_addr_t sg_addr; 136 u16 sg_offset; 137 u32 sg_length; 138 struct page *page, *head_page; 139 int page_offset; 140 int i; 141 142 for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 143 struct dpaa2_sg_entry *sge = &sgt[i]; 144 145 /* NOTE: We only support SG entries in dpaa2_sg_single format, 146 * but this is the only format we may receive from HW anyway 147 */ 148 149 /* Get the address and length from the S/G entry */ 150 sg_addr = dpaa2_sg_get_addr(sge); 151 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 152 dma_unmap_page(dev, sg_addr, priv->rx_buf_size, 153 DMA_BIDIRECTIONAL); 154 155 sg_length = dpaa2_sg_get_len(sge); 156 157 if (i == 0) { 158 /* We build the skb around the first data buffer */ 159 skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 160 if (unlikely(!skb)) { 161 /* Free the first SG entry now, since we already 162 * unmapped it and obtained the virtual address 163 */ 164 free_pages((unsigned long)sg_vaddr, 0); 165 166 /* We still need to subtract the buffers used 167 * by this FD from our software counter 168 */ 169 while (!dpaa2_sg_is_final(&sgt[i]) && 170 i < DPAA2_ETH_MAX_SG_ENTRIES) 171 i++; 172 break; 173 } 174 175 sg_offset = dpaa2_sg_get_offset(sge); 176 skb_reserve(skb, sg_offset); 177 skb_put(skb, sg_length); 178 } else { 179 /* Rest of the data buffers are stored as skb frags */ 180 page = virt_to_page(sg_vaddr); 181 head_page = virt_to_head_page(sg_vaddr); 182 183 /* Offset in page (which may be compound). 184 * Data in subsequent SG entries is stored from the 185 * beginning of the buffer, so we don't need to add the 186 * sg_offset. 187 */ 188 page_offset = ((unsigned long)sg_vaddr & 189 (PAGE_SIZE - 1)) + 190 (page_address(page) - page_address(head_page)); 191 192 skb_add_rx_frag(skb, i - 1, head_page, page_offset, 193 sg_length, priv->rx_buf_size); 194 } 195 196 if (dpaa2_sg_is_final(sge)) 197 break; 198 } 199 200 WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 201 202 /* Count all data buffers + SG table buffer */ 203 ch->buf_count -= i + 2; 204 205 return skb; 206 } 207 208 /* Free buffers acquired from the buffer pool or which were meant to 209 * be released in the pool 210 */ 211 static void dpaa2_eth_free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, 212 int count) 213 { 214 struct device *dev = priv->net_dev->dev.parent; 215 void *vaddr; 216 int i; 217 218 for (i = 0; i < count; i++) { 219 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 220 dma_unmap_page(dev, buf_array[i], priv->rx_buf_size, 221 DMA_BIDIRECTIONAL); 222 free_pages((unsigned long)vaddr, 0); 223 } 224 } 225 226 static void dpaa2_eth_xdp_release_buf(struct dpaa2_eth_priv *priv, 227 struct dpaa2_eth_channel *ch, 228 dma_addr_t addr) 229 { 230 int retries = 0; 231 int err; 232 233 ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr; 234 if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD) 235 return; 236 237 while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 238 ch->xdp.drop_bufs, 239 ch->xdp.drop_cnt)) == -EBUSY) { 240 if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 241 break; 242 cpu_relax(); 243 } 244 245 if (err) { 246 dpaa2_eth_free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt); 247 ch->buf_count -= ch->xdp.drop_cnt; 248 } 249 250 ch->xdp.drop_cnt = 0; 251 } 252 253 static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 254 struct dpaa2_eth_fq *fq, 255 struct dpaa2_eth_xdp_fds *xdp_fds) 256 { 257 int total_enqueued = 0, retries = 0, enqueued; 258 struct dpaa2_eth_drv_stats *percpu_extras; 259 int num_fds, err, max_retries; 260 struct dpaa2_fd *fds; 261 262 percpu_extras = this_cpu_ptr(priv->percpu_extras); 263 264 /* try to enqueue all the FDs until the max number of retries is hit */ 265 fds = xdp_fds->fds; 266 num_fds = xdp_fds->num; 267 max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 268 while (total_enqueued < num_fds && retries < max_retries) { 269 err = priv->enqueue(priv, fq, &fds[total_enqueued], 270 0, num_fds - total_enqueued, &enqueued); 271 if (err == -EBUSY) { 272 percpu_extras->tx_portal_busy += ++retries; 273 continue; 274 } 275 total_enqueued += enqueued; 276 } 277 xdp_fds->num = 0; 278 279 return total_enqueued; 280 } 281 282 static void dpaa2_eth_xdp_tx_flush(struct dpaa2_eth_priv *priv, 283 struct dpaa2_eth_channel *ch, 284 struct dpaa2_eth_fq *fq) 285 { 286 struct rtnl_link_stats64 *percpu_stats; 287 struct dpaa2_fd *fds; 288 int enqueued, i; 289 290 percpu_stats = this_cpu_ptr(priv->percpu_stats); 291 292 // enqueue the array of XDP_TX frames 293 enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds); 294 295 /* update statistics */ 296 percpu_stats->tx_packets += enqueued; 297 fds = fq->xdp_tx_fds.fds; 298 for (i = 0; i < enqueued; i++) { 299 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 300 ch->stats.xdp_tx++; 301 } 302 for (i = enqueued; i < fq->xdp_tx_fds.num; i++) { 303 dpaa2_eth_xdp_release_buf(priv, ch, dpaa2_fd_get_addr(&fds[i])); 304 percpu_stats->tx_errors++; 305 ch->stats.xdp_tx_err++; 306 } 307 fq->xdp_tx_fds.num = 0; 308 } 309 310 static void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv, 311 struct dpaa2_eth_channel *ch, 312 struct dpaa2_fd *fd, 313 void *buf_start, u16 queue_id) 314 { 315 struct dpaa2_faead *faead; 316 struct dpaa2_fd *dest_fd; 317 struct dpaa2_eth_fq *fq; 318 u32 ctrl, frc; 319 320 /* Mark the egress frame hardware annotation area as valid */ 321 frc = dpaa2_fd_get_frc(fd); 322 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 323 dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 324 325 /* Instruct hardware to release the FD buffer directly into 326 * the buffer pool once transmission is completed, instead of 327 * sending a Tx confirmation frame to us 328 */ 329 ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 330 faead = dpaa2_get_faead(buf_start, false); 331 faead->ctrl = cpu_to_le32(ctrl); 332 faead->conf_fqid = 0; 333 334 fq = &priv->fq[queue_id]; 335 dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++]; 336 memcpy(dest_fd, fd, sizeof(*dest_fd)); 337 338 if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE) 339 return; 340 341 dpaa2_eth_xdp_tx_flush(priv, ch, fq); 342 } 343 344 static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, 345 struct dpaa2_eth_channel *ch, 346 struct dpaa2_eth_fq *rx_fq, 347 struct dpaa2_fd *fd, void *vaddr) 348 { 349 dma_addr_t addr = dpaa2_fd_get_addr(fd); 350 struct bpf_prog *xdp_prog; 351 struct xdp_buff xdp; 352 u32 xdp_act = XDP_PASS; 353 int err; 354 355 rcu_read_lock(); 356 357 xdp_prog = READ_ONCE(ch->xdp.prog); 358 if (!xdp_prog) 359 goto out; 360 361 xdp.data = vaddr + dpaa2_fd_get_offset(fd); 362 xdp.data_end = xdp.data + dpaa2_fd_get_len(fd); 363 xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; 364 xdp_set_data_meta_invalid(&xdp); 365 xdp.rxq = &ch->xdp_rxq; 366 367 xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE - 368 (dpaa2_fd_get_offset(fd) - XDP_PACKET_HEADROOM); 369 370 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 371 372 /* xdp.data pointer may have changed */ 373 dpaa2_fd_set_offset(fd, xdp.data - vaddr); 374 dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 375 376 switch (xdp_act) { 377 case XDP_PASS: 378 break; 379 case XDP_TX: 380 dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); 381 break; 382 default: 383 bpf_warn_invalid_xdp_action(xdp_act); 384 fallthrough; 385 case XDP_ABORTED: 386 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 387 fallthrough; 388 case XDP_DROP: 389 dpaa2_eth_xdp_release_buf(priv, ch, addr); 390 ch->stats.xdp_drop++; 391 break; 392 case XDP_REDIRECT: 393 dma_unmap_page(priv->net_dev->dev.parent, addr, 394 priv->rx_buf_size, DMA_BIDIRECTIONAL); 395 ch->buf_count--; 396 397 /* Allow redirect use of full headroom */ 398 xdp.data_hard_start = vaddr; 399 xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE; 400 401 err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 402 if (unlikely(err)) 403 ch->stats.xdp_drop++; 404 else 405 ch->stats.xdp_redirect++; 406 break; 407 } 408 409 ch->xdp.res |= xdp_act; 410 out: 411 rcu_read_unlock(); 412 return xdp_act; 413 } 414 415 /* Main Rx frame processing routine */ 416 static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 417 struct dpaa2_eth_channel *ch, 418 const struct dpaa2_fd *fd, 419 struct dpaa2_eth_fq *fq) 420 { 421 dma_addr_t addr = dpaa2_fd_get_addr(fd); 422 u8 fd_format = dpaa2_fd_get_format(fd); 423 void *vaddr; 424 struct sk_buff *skb; 425 struct rtnl_link_stats64 *percpu_stats; 426 struct dpaa2_eth_drv_stats *percpu_extras; 427 struct device *dev = priv->net_dev->dev.parent; 428 struct dpaa2_fas *fas; 429 void *buf_data; 430 u32 status = 0; 431 u32 xdp_act; 432 433 /* Tracing point */ 434 trace_dpaa2_rx_fd(priv->net_dev, fd); 435 436 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 437 dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 438 DMA_BIDIRECTIONAL); 439 440 fas = dpaa2_get_fas(vaddr, false); 441 prefetch(fas); 442 buf_data = vaddr + dpaa2_fd_get_offset(fd); 443 prefetch(buf_data); 444 445 percpu_stats = this_cpu_ptr(priv->percpu_stats); 446 percpu_extras = this_cpu_ptr(priv->percpu_extras); 447 448 if (fd_format == dpaa2_fd_single) { 449 xdp_act = dpaa2_eth_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 450 if (xdp_act != XDP_PASS) { 451 percpu_stats->rx_packets++; 452 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 453 return; 454 } 455 456 dma_unmap_page(dev, addr, priv->rx_buf_size, 457 DMA_BIDIRECTIONAL); 458 skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 459 } else if (fd_format == dpaa2_fd_sg) { 460 WARN_ON(priv->xdp_prog); 461 462 dma_unmap_page(dev, addr, priv->rx_buf_size, 463 DMA_BIDIRECTIONAL); 464 skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 465 free_pages((unsigned long)vaddr, 0); 466 percpu_extras->rx_sg_frames++; 467 percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 468 } else { 469 /* We don't support any other format */ 470 goto err_frame_format; 471 } 472 473 if (unlikely(!skb)) 474 goto err_build_skb; 475 476 prefetch(skb->data); 477 478 /* Get the timestamp value */ 479 if (priv->rx_tstamp) { 480 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 481 __le64 *ts = dpaa2_get_ts(vaddr, false); 482 u64 ns; 483 484 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 485 486 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 487 shhwtstamps->hwtstamp = ns_to_ktime(ns); 488 } 489 490 /* Check if we need to validate the L4 csum */ 491 if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 492 status = le32_to_cpu(fas->status); 493 dpaa2_eth_validate_rx_csum(priv, status, skb); 494 } 495 496 skb->protocol = eth_type_trans(skb, priv->net_dev); 497 skb_record_rx_queue(skb, fq->flowid); 498 499 percpu_stats->rx_packets++; 500 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 501 502 list_add_tail(&skb->list, ch->rx_list); 503 504 return; 505 506 err_build_skb: 507 dpaa2_eth_free_rx_fd(priv, fd, vaddr); 508 err_frame_format: 509 percpu_stats->rx_dropped++; 510 } 511 512 /* Processing of Rx frames received on the error FQ 513 * We check and print the error bits and then free the frame 514 */ 515 static void dpaa2_eth_rx_err(struct dpaa2_eth_priv *priv, 516 struct dpaa2_eth_channel *ch, 517 const struct dpaa2_fd *fd, 518 struct dpaa2_eth_fq *fq __always_unused) 519 { 520 struct device *dev = priv->net_dev->dev.parent; 521 dma_addr_t addr = dpaa2_fd_get_addr(fd); 522 u8 fd_format = dpaa2_fd_get_format(fd); 523 struct rtnl_link_stats64 *percpu_stats; 524 struct dpaa2_eth_trap_item *trap_item; 525 struct dpaa2_fapr *fapr; 526 struct sk_buff *skb; 527 void *buf_data; 528 void *vaddr; 529 530 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 531 dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 532 DMA_BIDIRECTIONAL); 533 534 buf_data = vaddr + dpaa2_fd_get_offset(fd); 535 536 if (fd_format == dpaa2_fd_single) { 537 dma_unmap_page(dev, addr, priv->rx_buf_size, 538 DMA_BIDIRECTIONAL); 539 skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 540 } else if (fd_format == dpaa2_fd_sg) { 541 dma_unmap_page(dev, addr, priv->rx_buf_size, 542 DMA_BIDIRECTIONAL); 543 skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 544 free_pages((unsigned long)vaddr, 0); 545 } else { 546 /* We don't support any other format */ 547 dpaa2_eth_free_rx_fd(priv, fd, vaddr); 548 goto err_frame_format; 549 } 550 551 fapr = dpaa2_get_fapr(vaddr, false); 552 trap_item = dpaa2_eth_dl_get_trap(priv, fapr); 553 if (trap_item) 554 devlink_trap_report(priv->devlink, skb, trap_item->trap_ctx, 555 &priv->devlink_port, NULL); 556 consume_skb(skb); 557 558 err_frame_format: 559 percpu_stats = this_cpu_ptr(priv->percpu_stats); 560 percpu_stats->rx_errors++; 561 ch->buf_count--; 562 } 563 564 /* Consume all frames pull-dequeued into the store. This is the simplest way to 565 * make sure we don't accidentally issue another volatile dequeue which would 566 * overwrite (leak) frames already in the store. 567 * 568 * Observance of NAPI budget is not our concern, leaving that to the caller. 569 */ 570 static int dpaa2_eth_consume_frames(struct dpaa2_eth_channel *ch, 571 struct dpaa2_eth_fq **src) 572 { 573 struct dpaa2_eth_priv *priv = ch->priv; 574 struct dpaa2_eth_fq *fq = NULL; 575 struct dpaa2_dq *dq; 576 const struct dpaa2_fd *fd; 577 int cleaned = 0, retries = 0; 578 int is_last; 579 580 do { 581 dq = dpaa2_io_store_next(ch->store, &is_last); 582 if (unlikely(!dq)) { 583 /* If we're here, we *must* have placed a 584 * volatile dequeue comnmand, so keep reading through 585 * the store until we get some sort of valid response 586 * token (either a valid frame or an "empty dequeue") 587 */ 588 if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) { 589 netdev_err_once(priv->net_dev, 590 "Unable to read a valid dequeue response\n"); 591 return -ETIMEDOUT; 592 } 593 continue; 594 } 595 596 fd = dpaa2_dq_fd(dq); 597 fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 598 599 fq->consume(priv, ch, fd, fq); 600 cleaned++; 601 retries = 0; 602 } while (!is_last); 603 604 if (!cleaned) 605 return 0; 606 607 fq->stats.frames += cleaned; 608 ch->stats.frames += cleaned; 609 610 /* A dequeue operation only pulls frames from a single queue 611 * into the store. Return the frame queue as an out param. 612 */ 613 if (src) 614 *src = fq; 615 616 return cleaned; 617 } 618 619 static int dpaa2_eth_ptp_parse(struct sk_buff *skb, 620 u8 *msgtype, u8 *twostep, u8 *udp, 621 u16 *correction_offset, 622 u16 *origintimestamp_offset) 623 { 624 unsigned int ptp_class; 625 struct ptp_header *hdr; 626 unsigned int type; 627 u8 *base; 628 629 ptp_class = ptp_classify_raw(skb); 630 if (ptp_class == PTP_CLASS_NONE) 631 return -EINVAL; 632 633 hdr = ptp_parse_header(skb, ptp_class); 634 if (!hdr) 635 return -EINVAL; 636 637 *msgtype = ptp_get_msgtype(hdr, ptp_class); 638 *twostep = hdr->flag_field[0] & 0x2; 639 640 type = ptp_class & PTP_CLASS_PMASK; 641 if (type == PTP_CLASS_IPV4 || 642 type == PTP_CLASS_IPV6) 643 *udp = 1; 644 else 645 *udp = 0; 646 647 base = skb_mac_header(skb); 648 *correction_offset = (u8 *)&hdr->correction - base; 649 *origintimestamp_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 650 651 return 0; 652 } 653 654 /* Configure the egress frame annotation for timestamp update */ 655 static void dpaa2_eth_enable_tx_tstamp(struct dpaa2_eth_priv *priv, 656 struct dpaa2_fd *fd, 657 void *buf_start, 658 struct sk_buff *skb) 659 { 660 struct ptp_tstamp origin_timestamp; 661 struct dpni_single_step_cfg cfg; 662 u8 msgtype, twostep, udp; 663 struct dpaa2_faead *faead; 664 struct dpaa2_fas *fas; 665 struct timespec64 ts; 666 u16 offset1, offset2; 667 u32 ctrl, frc; 668 __le64 *ns; 669 u8 *data; 670 671 /* Mark the egress frame annotation area as valid */ 672 frc = dpaa2_fd_get_frc(fd); 673 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 674 675 /* Set hardware annotation size */ 676 ctrl = dpaa2_fd_get_ctrl(fd); 677 dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 678 679 /* enable UPD (update prepanded data) bit in FAEAD field of 680 * hardware frame annotation area 681 */ 682 ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 683 faead = dpaa2_get_faead(buf_start, true); 684 faead->ctrl = cpu_to_le32(ctrl); 685 686 if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 687 if (dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 688 &offset1, &offset2) || 689 msgtype != PTP_MSGTYPE_SYNC || twostep) { 690 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 691 return; 692 } 693 694 /* Mark the frame annotation status as valid */ 695 frc = dpaa2_fd_get_frc(fd); 696 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FASV); 697 698 /* Mark the PTP flag for one step timestamping */ 699 fas = dpaa2_get_fas(buf_start, true); 700 fas->status = cpu_to_le32(DPAA2_FAS_PTP); 701 702 dpaa2_ptp->caps.gettime64(&dpaa2_ptp->caps, &ts); 703 ns = dpaa2_get_ts(buf_start, true); 704 *ns = cpu_to_le64(timespec64_to_ns(&ts) / 705 DPAA2_PTP_CLK_PERIOD_NS); 706 707 /* Update current time to PTP message originTimestamp field */ 708 ns_to_ptp_tstamp(&origin_timestamp, le64_to_cpup(ns)); 709 data = skb_mac_header(skb); 710 *(__be16 *)(data + offset2) = htons(origin_timestamp.sec_msb); 711 *(__be32 *)(data + offset2 + 2) = 712 htonl(origin_timestamp.sec_lsb); 713 *(__be32 *)(data + offset2 + 6) = htonl(origin_timestamp.nsec); 714 715 cfg.en = 1; 716 cfg.ch_update = udp; 717 cfg.offset = offset1; 718 cfg.peer_delay = 0; 719 720 if (dpni_set_single_step_cfg(priv->mc_io, 0, priv->mc_token, 721 &cfg)) 722 WARN_ONCE(1, "Failed to set single step register"); 723 } 724 } 725 726 /* Create a frame descriptor based on a fragmented skb */ 727 static int dpaa2_eth_build_sg_fd(struct dpaa2_eth_priv *priv, 728 struct sk_buff *skb, 729 struct dpaa2_fd *fd, 730 void **swa_addr) 731 { 732 struct device *dev = priv->net_dev->dev.parent; 733 void *sgt_buf = NULL; 734 dma_addr_t addr; 735 int nr_frags = skb_shinfo(skb)->nr_frags; 736 struct dpaa2_sg_entry *sgt; 737 int i, err; 738 int sgt_buf_size; 739 struct scatterlist *scl, *crt_scl; 740 int num_sg; 741 int num_dma_bufs; 742 struct dpaa2_eth_swa *swa; 743 744 /* Create and map scatterlist. 745 * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 746 * to go beyond nr_frags+1. 747 * Note: We don't support chained scatterlists 748 */ 749 if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 750 return -EINVAL; 751 752 scl = kmalloc_array(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 753 if (unlikely(!scl)) 754 return -ENOMEM; 755 756 sg_init_table(scl, nr_frags + 1); 757 num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 758 if (unlikely(num_sg < 0)) { 759 err = -ENOMEM; 760 goto dma_map_sg_failed; 761 } 762 num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 763 if (unlikely(!num_dma_bufs)) { 764 err = -ENOMEM; 765 goto dma_map_sg_failed; 766 } 767 768 /* Prepare the HW SGT structure */ 769 sgt_buf_size = priv->tx_data_offset + 770 sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 771 sgt_buf = napi_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN); 772 if (unlikely(!sgt_buf)) { 773 err = -ENOMEM; 774 goto sgt_buf_alloc_failed; 775 } 776 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 777 memset(sgt_buf, 0, sgt_buf_size); 778 779 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 780 781 /* Fill in the HW SGT structure. 782 * 783 * sgt_buf is zeroed out, so the following fields are implicit 784 * in all sgt entries: 785 * - offset is 0 786 * - format is 'dpaa2_sg_single' 787 */ 788 for_each_sg(scl, crt_scl, num_dma_bufs, i) { 789 dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 790 dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 791 } 792 dpaa2_sg_set_final(&sgt[i - 1], true); 793 794 /* Store the skb backpointer in the SGT buffer. 795 * Fit the scatterlist and the number of buffers alongside the 796 * skb backpointer in the software annotation area. We'll need 797 * all of them on Tx Conf. 798 */ 799 *swa_addr = (void *)sgt_buf; 800 swa = (struct dpaa2_eth_swa *)sgt_buf; 801 swa->type = DPAA2_ETH_SWA_SG; 802 swa->sg.skb = skb; 803 swa->sg.scl = scl; 804 swa->sg.num_sg = num_sg; 805 swa->sg.sgt_size = sgt_buf_size; 806 807 /* Separately map the SGT buffer */ 808 addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 809 if (unlikely(dma_mapping_error(dev, addr))) { 810 err = -ENOMEM; 811 goto dma_map_single_failed; 812 } 813 dpaa2_fd_set_offset(fd, priv->tx_data_offset); 814 dpaa2_fd_set_format(fd, dpaa2_fd_sg); 815 dpaa2_fd_set_addr(fd, addr); 816 dpaa2_fd_set_len(fd, skb->len); 817 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 818 819 return 0; 820 821 dma_map_single_failed: 822 skb_free_frag(sgt_buf); 823 sgt_buf_alloc_failed: 824 dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 825 dma_map_sg_failed: 826 kfree(scl); 827 return err; 828 } 829 830 /* Create a SG frame descriptor based on a linear skb. 831 * 832 * This function is used on the Tx path when the skb headroom is not large 833 * enough for the HW requirements, thus instead of realloc-ing the skb we 834 * create a SG frame descriptor with only one entry. 835 */ 836 static int dpaa2_eth_build_sg_fd_single_buf(struct dpaa2_eth_priv *priv, 837 struct sk_buff *skb, 838 struct dpaa2_fd *fd, 839 void **swa_addr) 840 { 841 struct device *dev = priv->net_dev->dev.parent; 842 struct dpaa2_eth_sgt_cache *sgt_cache; 843 struct dpaa2_sg_entry *sgt; 844 struct dpaa2_eth_swa *swa; 845 dma_addr_t addr, sgt_addr; 846 void *sgt_buf = NULL; 847 int sgt_buf_size; 848 int err; 849 850 /* Prepare the HW SGT structure */ 851 sgt_cache = this_cpu_ptr(priv->sgt_cache); 852 sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry); 853 854 if (sgt_cache->count == 0) 855 sgt_buf = kzalloc(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN, 856 GFP_ATOMIC); 857 else 858 sgt_buf = sgt_cache->buf[--sgt_cache->count]; 859 if (unlikely(!sgt_buf)) 860 return -ENOMEM; 861 862 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 863 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 864 865 addr = dma_map_single(dev, skb->data, skb->len, DMA_BIDIRECTIONAL); 866 if (unlikely(dma_mapping_error(dev, addr))) { 867 err = -ENOMEM; 868 goto data_map_failed; 869 } 870 871 /* Fill in the HW SGT structure */ 872 dpaa2_sg_set_addr(sgt, addr); 873 dpaa2_sg_set_len(sgt, skb->len); 874 dpaa2_sg_set_final(sgt, true); 875 876 /* Store the skb backpointer in the SGT buffer */ 877 *swa_addr = (void *)sgt_buf; 878 swa = (struct dpaa2_eth_swa *)sgt_buf; 879 swa->type = DPAA2_ETH_SWA_SINGLE; 880 swa->single.skb = skb; 881 swa->single.sgt_size = sgt_buf_size; 882 883 /* Separately map the SGT buffer */ 884 sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 885 if (unlikely(dma_mapping_error(dev, sgt_addr))) { 886 err = -ENOMEM; 887 goto sgt_map_failed; 888 } 889 890 dpaa2_fd_set_offset(fd, priv->tx_data_offset); 891 dpaa2_fd_set_format(fd, dpaa2_fd_sg); 892 dpaa2_fd_set_addr(fd, sgt_addr); 893 dpaa2_fd_set_len(fd, skb->len); 894 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 895 896 return 0; 897 898 sgt_map_failed: 899 dma_unmap_single(dev, addr, skb->len, DMA_BIDIRECTIONAL); 900 data_map_failed: 901 if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 902 kfree(sgt_buf); 903 else 904 sgt_cache->buf[sgt_cache->count++] = sgt_buf; 905 906 return err; 907 } 908 909 /* Create a frame descriptor based on a linear skb */ 910 static int dpaa2_eth_build_single_fd(struct dpaa2_eth_priv *priv, 911 struct sk_buff *skb, 912 struct dpaa2_fd *fd, 913 void **swa_addr) 914 { 915 struct device *dev = priv->net_dev->dev.parent; 916 u8 *buffer_start, *aligned_start; 917 struct dpaa2_eth_swa *swa; 918 dma_addr_t addr; 919 920 buffer_start = skb->data - dpaa2_eth_needed_headroom(skb); 921 922 /* If there's enough room to align the FD address, do it. 923 * It will help hardware optimize accesses. 924 */ 925 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 926 DPAA2_ETH_TX_BUF_ALIGN); 927 if (aligned_start >= skb->head) 928 buffer_start = aligned_start; 929 930 /* Store a backpointer to the skb at the beginning of the buffer 931 * (in the private data area) such that we can release it 932 * on Tx confirm 933 */ 934 *swa_addr = (void *)buffer_start; 935 swa = (struct dpaa2_eth_swa *)buffer_start; 936 swa->type = DPAA2_ETH_SWA_SINGLE; 937 swa->single.skb = skb; 938 939 addr = dma_map_single(dev, buffer_start, 940 skb_tail_pointer(skb) - buffer_start, 941 DMA_BIDIRECTIONAL); 942 if (unlikely(dma_mapping_error(dev, addr))) 943 return -ENOMEM; 944 945 dpaa2_fd_set_addr(fd, addr); 946 dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 947 dpaa2_fd_set_len(fd, skb->len); 948 dpaa2_fd_set_format(fd, dpaa2_fd_single); 949 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 950 951 return 0; 952 } 953 954 /* FD freeing routine on the Tx path 955 * 956 * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 957 * back-pointed to is also freed. 958 * This can be called either from dpaa2_eth_tx_conf() or on the error path of 959 * dpaa2_eth_tx(). 960 */ 961 static void dpaa2_eth_free_tx_fd(struct dpaa2_eth_priv *priv, 962 struct dpaa2_eth_fq *fq, 963 const struct dpaa2_fd *fd, bool in_napi) 964 { 965 struct device *dev = priv->net_dev->dev.parent; 966 dma_addr_t fd_addr, sg_addr; 967 struct sk_buff *skb = NULL; 968 unsigned char *buffer_start; 969 struct dpaa2_eth_swa *swa; 970 u8 fd_format = dpaa2_fd_get_format(fd); 971 u32 fd_len = dpaa2_fd_get_len(fd); 972 973 struct dpaa2_eth_sgt_cache *sgt_cache; 974 struct dpaa2_sg_entry *sgt; 975 976 fd_addr = dpaa2_fd_get_addr(fd); 977 buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 978 swa = (struct dpaa2_eth_swa *)buffer_start; 979 980 if (fd_format == dpaa2_fd_single) { 981 if (swa->type == DPAA2_ETH_SWA_SINGLE) { 982 skb = swa->single.skb; 983 /* Accessing the skb buffer is safe before dma unmap, 984 * because we didn't map the actual skb shell. 985 */ 986 dma_unmap_single(dev, fd_addr, 987 skb_tail_pointer(skb) - buffer_start, 988 DMA_BIDIRECTIONAL); 989 } else { 990 WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 991 dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 992 DMA_BIDIRECTIONAL); 993 } 994 } else if (fd_format == dpaa2_fd_sg) { 995 if (swa->type == DPAA2_ETH_SWA_SG) { 996 skb = swa->sg.skb; 997 998 /* Unmap the scatterlist */ 999 dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 1000 DMA_BIDIRECTIONAL); 1001 kfree(swa->sg.scl); 1002 1003 /* Unmap the SGT buffer */ 1004 dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 1005 DMA_BIDIRECTIONAL); 1006 } else { 1007 skb = swa->single.skb; 1008 1009 /* Unmap the SGT Buffer */ 1010 dma_unmap_single(dev, fd_addr, swa->single.sgt_size, 1011 DMA_BIDIRECTIONAL); 1012 1013 sgt = (struct dpaa2_sg_entry *)(buffer_start + 1014 priv->tx_data_offset); 1015 sg_addr = dpaa2_sg_get_addr(sgt); 1016 dma_unmap_single(dev, sg_addr, skb->len, DMA_BIDIRECTIONAL); 1017 } 1018 } else { 1019 netdev_dbg(priv->net_dev, "Invalid FD format\n"); 1020 return; 1021 } 1022 1023 if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 1024 fq->dq_frames++; 1025 fq->dq_bytes += fd_len; 1026 } 1027 1028 if (swa->type == DPAA2_ETH_SWA_XDP) { 1029 xdp_return_frame(swa->xdp.xdpf); 1030 return; 1031 } 1032 1033 /* Get the timestamp value */ 1034 if (skb->cb[0] == TX_TSTAMP) { 1035 struct skb_shared_hwtstamps shhwtstamps; 1036 __le64 *ts = dpaa2_get_ts(buffer_start, true); 1037 u64 ns; 1038 1039 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 1040 1041 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 1042 shhwtstamps.hwtstamp = ns_to_ktime(ns); 1043 skb_tstamp_tx(skb, &shhwtstamps); 1044 } else if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1045 mutex_unlock(&priv->onestep_tstamp_lock); 1046 } 1047 1048 /* Free SGT buffer allocated on tx */ 1049 if (fd_format != dpaa2_fd_single) { 1050 sgt_cache = this_cpu_ptr(priv->sgt_cache); 1051 if (swa->type == DPAA2_ETH_SWA_SG) { 1052 skb_free_frag(buffer_start); 1053 } else { 1054 if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 1055 kfree(buffer_start); 1056 else 1057 sgt_cache->buf[sgt_cache->count++] = buffer_start; 1058 } 1059 } 1060 1061 /* Move on with skb release */ 1062 napi_consume_skb(skb, in_napi); 1063 } 1064 1065 static netdev_tx_t __dpaa2_eth_tx(struct sk_buff *skb, 1066 struct net_device *net_dev) 1067 { 1068 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1069 struct dpaa2_fd fd; 1070 struct rtnl_link_stats64 *percpu_stats; 1071 struct dpaa2_eth_drv_stats *percpu_extras; 1072 struct dpaa2_eth_fq *fq; 1073 struct netdev_queue *nq; 1074 u16 queue_mapping; 1075 unsigned int needed_headroom; 1076 u32 fd_len; 1077 u8 prio = 0; 1078 int err, i; 1079 void *swa; 1080 1081 percpu_stats = this_cpu_ptr(priv->percpu_stats); 1082 percpu_extras = this_cpu_ptr(priv->percpu_extras); 1083 1084 needed_headroom = dpaa2_eth_needed_headroom(skb); 1085 1086 /* We'll be holding a back-reference to the skb until Tx Confirmation; 1087 * we don't want that overwritten by a concurrent Tx with a cloned skb. 1088 */ 1089 skb = skb_unshare(skb, GFP_ATOMIC); 1090 if (unlikely(!skb)) { 1091 /* skb_unshare() has already freed the skb */ 1092 percpu_stats->tx_dropped++; 1093 return NETDEV_TX_OK; 1094 } 1095 1096 /* Setup the FD fields */ 1097 memset(&fd, 0, sizeof(fd)); 1098 1099 if (skb_is_nonlinear(skb)) { 1100 err = dpaa2_eth_build_sg_fd(priv, skb, &fd, &swa); 1101 percpu_extras->tx_sg_frames++; 1102 percpu_extras->tx_sg_bytes += skb->len; 1103 } else if (skb_headroom(skb) < needed_headroom) { 1104 err = dpaa2_eth_build_sg_fd_single_buf(priv, skb, &fd, &swa); 1105 percpu_extras->tx_sg_frames++; 1106 percpu_extras->tx_sg_bytes += skb->len; 1107 percpu_extras->tx_converted_sg_frames++; 1108 percpu_extras->tx_converted_sg_bytes += skb->len; 1109 } else { 1110 err = dpaa2_eth_build_single_fd(priv, skb, &fd, &swa); 1111 } 1112 1113 if (unlikely(err)) { 1114 percpu_stats->tx_dropped++; 1115 goto err_build_fd; 1116 } 1117 1118 if (skb->cb[0]) 1119 dpaa2_eth_enable_tx_tstamp(priv, &fd, swa, skb); 1120 1121 /* Tracing point */ 1122 trace_dpaa2_tx_fd(net_dev, &fd); 1123 1124 /* TxConf FQ selection relies on queue id from the stack. 1125 * In case of a forwarded frame from another DPNI interface, we choose 1126 * a queue affined to the same core that processed the Rx frame 1127 */ 1128 queue_mapping = skb_get_queue_mapping(skb); 1129 1130 if (net_dev->num_tc) { 1131 prio = netdev_txq_to_tc(net_dev, queue_mapping); 1132 /* Hardware interprets priority level 0 as being the highest, 1133 * so we need to do a reverse mapping to the netdev tc index 1134 */ 1135 prio = net_dev->num_tc - prio - 1; 1136 /* We have only one FQ array entry for all Tx hardware queues 1137 * with the same flow id (but different priority levels) 1138 */ 1139 queue_mapping %= dpaa2_eth_queue_count(priv); 1140 } 1141 fq = &priv->fq[queue_mapping]; 1142 1143 fd_len = dpaa2_fd_get_len(&fd); 1144 nq = netdev_get_tx_queue(net_dev, queue_mapping); 1145 netdev_tx_sent_queue(nq, fd_len); 1146 1147 /* Everything that happens after this enqueues might race with 1148 * the Tx confirmation callback for this frame 1149 */ 1150 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 1151 err = priv->enqueue(priv, fq, &fd, prio, 1, NULL); 1152 if (err != -EBUSY) 1153 break; 1154 } 1155 percpu_extras->tx_portal_busy += i; 1156 if (unlikely(err < 0)) { 1157 percpu_stats->tx_errors++; 1158 /* Clean up everything, including freeing the skb */ 1159 dpaa2_eth_free_tx_fd(priv, fq, &fd, false); 1160 netdev_tx_completed_queue(nq, 1, fd_len); 1161 } else { 1162 percpu_stats->tx_packets++; 1163 percpu_stats->tx_bytes += fd_len; 1164 } 1165 1166 return NETDEV_TX_OK; 1167 1168 err_build_fd: 1169 dev_kfree_skb(skb); 1170 1171 return NETDEV_TX_OK; 1172 } 1173 1174 static void dpaa2_eth_tx_onestep_tstamp(struct work_struct *work) 1175 { 1176 struct dpaa2_eth_priv *priv = container_of(work, struct dpaa2_eth_priv, 1177 tx_onestep_tstamp); 1178 struct sk_buff *skb; 1179 1180 while (true) { 1181 skb = skb_dequeue(&priv->tx_skbs); 1182 if (!skb) 1183 return; 1184 1185 /* Lock just before TX one-step timestamping packet, 1186 * and release the lock in dpaa2_eth_free_tx_fd when 1187 * confirm the packet has been sent on hardware, or 1188 * when clean up during transmit failure. 1189 */ 1190 mutex_lock(&priv->onestep_tstamp_lock); 1191 __dpaa2_eth_tx(skb, priv->net_dev); 1192 } 1193 } 1194 1195 static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 1196 { 1197 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1198 u8 msgtype, twostep, udp; 1199 u16 offset1, offset2; 1200 1201 /* Utilize skb->cb[0] for timestamping request per skb */ 1202 skb->cb[0] = 0; 1203 1204 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && dpaa2_ptp) { 1205 if (priv->tx_tstamp_type == HWTSTAMP_TX_ON) 1206 skb->cb[0] = TX_TSTAMP; 1207 else if (priv->tx_tstamp_type == HWTSTAMP_TX_ONESTEP_SYNC) 1208 skb->cb[0] = TX_TSTAMP_ONESTEP_SYNC; 1209 } 1210 1211 /* TX for one-step timestamping PTP Sync packet */ 1212 if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1213 if (!dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 1214 &offset1, &offset2)) 1215 if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0) { 1216 skb_queue_tail(&priv->tx_skbs, skb); 1217 queue_work(priv->dpaa2_ptp_wq, 1218 &priv->tx_onestep_tstamp); 1219 return NETDEV_TX_OK; 1220 } 1221 /* Use two-step timestamping if not one-step timestamping 1222 * PTP Sync packet 1223 */ 1224 skb->cb[0] = TX_TSTAMP; 1225 } 1226 1227 /* TX for other packets */ 1228 return __dpaa2_eth_tx(skb, net_dev); 1229 } 1230 1231 /* Tx confirmation frame processing routine */ 1232 static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 1233 struct dpaa2_eth_channel *ch __always_unused, 1234 const struct dpaa2_fd *fd, 1235 struct dpaa2_eth_fq *fq) 1236 { 1237 struct rtnl_link_stats64 *percpu_stats; 1238 struct dpaa2_eth_drv_stats *percpu_extras; 1239 u32 fd_len = dpaa2_fd_get_len(fd); 1240 u32 fd_errors; 1241 1242 /* Tracing point */ 1243 trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 1244 1245 percpu_extras = this_cpu_ptr(priv->percpu_extras); 1246 percpu_extras->tx_conf_frames++; 1247 percpu_extras->tx_conf_bytes += fd_len; 1248 1249 /* Check frame errors in the FD field */ 1250 fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 1251 dpaa2_eth_free_tx_fd(priv, fq, fd, true); 1252 1253 if (likely(!fd_errors)) 1254 return; 1255 1256 if (net_ratelimit()) 1257 netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 1258 fd_errors); 1259 1260 percpu_stats = this_cpu_ptr(priv->percpu_stats); 1261 /* Tx-conf logically pertains to the egress path. */ 1262 percpu_stats->tx_errors++; 1263 } 1264 1265 static int dpaa2_eth_set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 1266 { 1267 int err; 1268 1269 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1270 DPNI_OFF_RX_L3_CSUM, enable); 1271 if (err) { 1272 netdev_err(priv->net_dev, 1273 "dpni_set_offload(RX_L3_CSUM) failed\n"); 1274 return err; 1275 } 1276 1277 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1278 DPNI_OFF_RX_L4_CSUM, enable); 1279 if (err) { 1280 netdev_err(priv->net_dev, 1281 "dpni_set_offload(RX_L4_CSUM) failed\n"); 1282 return err; 1283 } 1284 1285 return 0; 1286 } 1287 1288 static int dpaa2_eth_set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 1289 { 1290 int err; 1291 1292 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1293 DPNI_OFF_TX_L3_CSUM, enable); 1294 if (err) { 1295 netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 1296 return err; 1297 } 1298 1299 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1300 DPNI_OFF_TX_L4_CSUM, enable); 1301 if (err) { 1302 netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 1303 return err; 1304 } 1305 1306 return 0; 1307 } 1308 1309 /* Perform a single release command to add buffers 1310 * to the specified buffer pool 1311 */ 1312 static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv, 1313 struct dpaa2_eth_channel *ch, u16 bpid) 1314 { 1315 struct device *dev = priv->net_dev->dev.parent; 1316 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1317 struct page *page; 1318 dma_addr_t addr; 1319 int retries = 0; 1320 int i, err; 1321 1322 for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 1323 /* Allocate buffer visible to WRIOP + skb shared info + 1324 * alignment padding 1325 */ 1326 /* allocate one page for each Rx buffer. WRIOP sees 1327 * the entire page except for a tailroom reserved for 1328 * skb shared info 1329 */ 1330 page = dev_alloc_pages(0); 1331 if (!page) 1332 goto err_alloc; 1333 1334 addr = dma_map_page(dev, page, 0, priv->rx_buf_size, 1335 DMA_BIDIRECTIONAL); 1336 if (unlikely(dma_mapping_error(dev, addr))) 1337 goto err_map; 1338 1339 buf_array[i] = addr; 1340 1341 /* tracing point */ 1342 trace_dpaa2_eth_buf_seed(priv->net_dev, 1343 page, DPAA2_ETH_RX_BUF_RAW_SIZE, 1344 addr, priv->rx_buf_size, 1345 bpid); 1346 } 1347 1348 release_bufs: 1349 /* In case the portal is busy, retry until successful */ 1350 while ((err = dpaa2_io_service_release(ch->dpio, bpid, 1351 buf_array, i)) == -EBUSY) { 1352 if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 1353 break; 1354 cpu_relax(); 1355 } 1356 1357 /* If release command failed, clean up and bail out; 1358 * not much else we can do about it 1359 */ 1360 if (err) { 1361 dpaa2_eth_free_bufs(priv, buf_array, i); 1362 return 0; 1363 } 1364 1365 return i; 1366 1367 err_map: 1368 __free_pages(page, 0); 1369 err_alloc: 1370 /* If we managed to allocate at least some buffers, 1371 * release them to hardware 1372 */ 1373 if (i) 1374 goto release_bufs; 1375 1376 return 0; 1377 } 1378 1379 static int dpaa2_eth_seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 1380 { 1381 int i, j; 1382 int new_count; 1383 1384 for (j = 0; j < priv->num_channels; j++) { 1385 for (i = 0; i < DPAA2_ETH_NUM_BUFS; 1386 i += DPAA2_ETH_BUFS_PER_CMD) { 1387 new_count = dpaa2_eth_add_bufs(priv, priv->channel[j], bpid); 1388 priv->channel[j]->buf_count += new_count; 1389 1390 if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 1391 return -ENOMEM; 1392 } 1393 } 1394 } 1395 1396 return 0; 1397 } 1398 1399 /* 1400 * Drain the specified number of buffers from the DPNI's private buffer pool. 1401 * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 1402 */ 1403 static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int count) 1404 { 1405 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1406 int retries = 0; 1407 int ret; 1408 1409 do { 1410 ret = dpaa2_io_service_acquire(NULL, priv->bpid, 1411 buf_array, count); 1412 if (ret < 0) { 1413 if (ret == -EBUSY && 1414 retries++ < DPAA2_ETH_SWP_BUSY_RETRIES) 1415 continue; 1416 netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 1417 return; 1418 } 1419 dpaa2_eth_free_bufs(priv, buf_array, ret); 1420 retries = 0; 1421 } while (ret); 1422 } 1423 1424 static void dpaa2_eth_drain_pool(struct dpaa2_eth_priv *priv) 1425 { 1426 int i; 1427 1428 dpaa2_eth_drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 1429 dpaa2_eth_drain_bufs(priv, 1); 1430 1431 for (i = 0; i < priv->num_channels; i++) 1432 priv->channel[i]->buf_count = 0; 1433 } 1434 1435 /* Function is called from softirq context only, so we don't need to guard 1436 * the access to percpu count 1437 */ 1438 static int dpaa2_eth_refill_pool(struct dpaa2_eth_priv *priv, 1439 struct dpaa2_eth_channel *ch, 1440 u16 bpid) 1441 { 1442 int new_count; 1443 1444 if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 1445 return 0; 1446 1447 do { 1448 new_count = dpaa2_eth_add_bufs(priv, ch, bpid); 1449 if (unlikely(!new_count)) { 1450 /* Out of memory; abort for now, we'll try later on */ 1451 break; 1452 } 1453 ch->buf_count += new_count; 1454 } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 1455 1456 if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 1457 return -ENOMEM; 1458 1459 return 0; 1460 } 1461 1462 static void dpaa2_eth_sgt_cache_drain(struct dpaa2_eth_priv *priv) 1463 { 1464 struct dpaa2_eth_sgt_cache *sgt_cache; 1465 u16 count; 1466 int k, i; 1467 1468 for_each_possible_cpu(k) { 1469 sgt_cache = per_cpu_ptr(priv->sgt_cache, k); 1470 count = sgt_cache->count; 1471 1472 for (i = 0; i < count; i++) 1473 kfree(sgt_cache->buf[i]); 1474 sgt_cache->count = 0; 1475 } 1476 } 1477 1478 static int dpaa2_eth_pull_channel(struct dpaa2_eth_channel *ch) 1479 { 1480 int err; 1481 int dequeues = -1; 1482 1483 /* Retry while portal is busy */ 1484 do { 1485 err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 1486 ch->store); 1487 dequeues++; 1488 cpu_relax(); 1489 } while (err == -EBUSY && dequeues < DPAA2_ETH_SWP_BUSY_RETRIES); 1490 1491 ch->stats.dequeue_portal_busy += dequeues; 1492 if (unlikely(err)) 1493 ch->stats.pull_err++; 1494 1495 return err; 1496 } 1497 1498 /* NAPI poll routine 1499 * 1500 * Frames are dequeued from the QMan channel associated with this NAPI context. 1501 * Rx, Tx confirmation and (if configured) Rx error frames all count 1502 * towards the NAPI budget. 1503 */ 1504 static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 1505 { 1506 struct dpaa2_eth_channel *ch; 1507 struct dpaa2_eth_priv *priv; 1508 int rx_cleaned = 0, txconf_cleaned = 0; 1509 struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1510 struct netdev_queue *nq; 1511 int store_cleaned, work_done; 1512 struct list_head rx_list; 1513 int retries = 0; 1514 u16 flowid; 1515 int err; 1516 1517 ch = container_of(napi, struct dpaa2_eth_channel, napi); 1518 ch->xdp.res = 0; 1519 priv = ch->priv; 1520 1521 INIT_LIST_HEAD(&rx_list); 1522 ch->rx_list = &rx_list; 1523 1524 do { 1525 err = dpaa2_eth_pull_channel(ch); 1526 if (unlikely(err)) 1527 break; 1528 1529 /* Refill pool if appropriate */ 1530 dpaa2_eth_refill_pool(priv, ch, priv->bpid); 1531 1532 store_cleaned = dpaa2_eth_consume_frames(ch, &fq); 1533 if (store_cleaned <= 0) 1534 break; 1535 if (fq->type == DPAA2_RX_FQ) { 1536 rx_cleaned += store_cleaned; 1537 flowid = fq->flowid; 1538 } else { 1539 txconf_cleaned += store_cleaned; 1540 /* We have a single Tx conf FQ on this channel */ 1541 txc_fq = fq; 1542 } 1543 1544 /* If we either consumed the whole NAPI budget with Rx frames 1545 * or we reached the Tx confirmations threshold, we're done. 1546 */ 1547 if (rx_cleaned >= budget || 1548 txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1549 work_done = budget; 1550 goto out; 1551 } 1552 } while (store_cleaned); 1553 1554 /* We didn't consume the entire budget, so finish napi and 1555 * re-enable data availability notifications 1556 */ 1557 napi_complete_done(napi, rx_cleaned); 1558 do { 1559 err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 1560 cpu_relax(); 1561 } while (err == -EBUSY && retries++ < DPAA2_ETH_SWP_BUSY_RETRIES); 1562 WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 1563 ch->nctx.desired_cpu); 1564 1565 work_done = max(rx_cleaned, 1); 1566 1567 out: 1568 netif_receive_skb_list(ch->rx_list); 1569 1570 if (txc_fq && txc_fq->dq_frames) { 1571 nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1572 netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1573 txc_fq->dq_bytes); 1574 txc_fq->dq_frames = 0; 1575 txc_fq->dq_bytes = 0; 1576 } 1577 1578 if (ch->xdp.res & XDP_REDIRECT) 1579 xdp_do_flush_map(); 1580 else if (rx_cleaned && ch->xdp.res & XDP_TX) 1581 dpaa2_eth_xdp_tx_flush(priv, ch, &priv->fq[flowid]); 1582 1583 return work_done; 1584 } 1585 1586 static void dpaa2_eth_enable_ch_napi(struct dpaa2_eth_priv *priv) 1587 { 1588 struct dpaa2_eth_channel *ch; 1589 int i; 1590 1591 for (i = 0; i < priv->num_channels; i++) { 1592 ch = priv->channel[i]; 1593 napi_enable(&ch->napi); 1594 } 1595 } 1596 1597 static void dpaa2_eth_disable_ch_napi(struct dpaa2_eth_priv *priv) 1598 { 1599 struct dpaa2_eth_channel *ch; 1600 int i; 1601 1602 for (i = 0; i < priv->num_channels; i++) { 1603 ch = priv->channel[i]; 1604 napi_disable(&ch->napi); 1605 } 1606 } 1607 1608 void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, 1609 bool tx_pause, bool pfc) 1610 { 1611 struct dpni_taildrop td = {0}; 1612 struct dpaa2_eth_fq *fq; 1613 int i, err; 1614 1615 /* FQ taildrop: threshold is in bytes, per frame queue. Enabled if 1616 * flow control is disabled (as it might interfere with either the 1617 * buffer pool depletion trigger for pause frames or with the group 1618 * congestion trigger for PFC frames) 1619 */ 1620 td.enable = !tx_pause; 1621 if (priv->rx_fqtd_enabled == td.enable) 1622 goto set_cgtd; 1623 1624 td.threshold = DPAA2_ETH_FQ_TAILDROP_THRESH; 1625 td.units = DPNI_CONGESTION_UNIT_BYTES; 1626 1627 for (i = 0; i < priv->num_fqs; i++) { 1628 fq = &priv->fq[i]; 1629 if (fq->type != DPAA2_RX_FQ) 1630 continue; 1631 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1632 DPNI_CP_QUEUE, DPNI_QUEUE_RX, 1633 fq->tc, fq->flowid, &td); 1634 if (err) { 1635 netdev_err(priv->net_dev, 1636 "dpni_set_taildrop(FQ) failed\n"); 1637 return; 1638 } 1639 } 1640 1641 priv->rx_fqtd_enabled = td.enable; 1642 1643 set_cgtd: 1644 /* Congestion group taildrop: threshold is in frames, per group 1645 * of FQs belonging to the same traffic class 1646 * Enabled if general Tx pause disabled or if PFCs are enabled 1647 * (congestion group threhsold for PFC generation is lower than the 1648 * CG taildrop threshold, so it won't interfere with it; we also 1649 * want frames in non-PFC enabled traffic classes to be kept in check) 1650 */ 1651 td.enable = !tx_pause || (tx_pause && pfc); 1652 if (priv->rx_cgtd_enabled == td.enable) 1653 return; 1654 1655 td.threshold = DPAA2_ETH_CG_TAILDROP_THRESH(priv); 1656 td.units = DPNI_CONGESTION_UNIT_FRAMES; 1657 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 1658 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1659 DPNI_CP_GROUP, DPNI_QUEUE_RX, 1660 i, 0, &td); 1661 if (err) { 1662 netdev_err(priv->net_dev, 1663 "dpni_set_taildrop(CG) failed\n"); 1664 return; 1665 } 1666 } 1667 1668 priv->rx_cgtd_enabled = td.enable; 1669 } 1670 1671 static int dpaa2_eth_link_state_update(struct dpaa2_eth_priv *priv) 1672 { 1673 struct dpni_link_state state = {0}; 1674 bool tx_pause; 1675 int err; 1676 1677 err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 1678 if (unlikely(err)) { 1679 netdev_err(priv->net_dev, 1680 "dpni_get_link_state() failed\n"); 1681 return err; 1682 } 1683 1684 /* If Tx pause frame settings have changed, we need to update 1685 * Rx FQ taildrop configuration as well. We configure taildrop 1686 * only when pause frame generation is disabled. 1687 */ 1688 tx_pause = dpaa2_eth_tx_pause_enabled(state.options); 1689 dpaa2_eth_set_rx_taildrop(priv, tx_pause, priv->pfc_enabled); 1690 1691 /* When we manage the MAC/PHY using phylink there is no need 1692 * to manually update the netif_carrier. 1693 */ 1694 if (dpaa2_eth_is_type_phy(priv)) 1695 goto out; 1696 1697 /* Chech link state; speed / duplex changes are not treated yet */ 1698 if (priv->link_state.up == state.up) 1699 goto out; 1700 1701 if (state.up) { 1702 netif_carrier_on(priv->net_dev); 1703 netif_tx_start_all_queues(priv->net_dev); 1704 } else { 1705 netif_tx_stop_all_queues(priv->net_dev); 1706 netif_carrier_off(priv->net_dev); 1707 } 1708 1709 netdev_info(priv->net_dev, "Link Event: state %s\n", 1710 state.up ? "up" : "down"); 1711 1712 out: 1713 priv->link_state = state; 1714 1715 return 0; 1716 } 1717 1718 static int dpaa2_eth_open(struct net_device *net_dev) 1719 { 1720 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1721 int err; 1722 1723 err = dpaa2_eth_seed_pool(priv, priv->bpid); 1724 if (err) { 1725 /* Not much to do; the buffer pool, though not filled up, 1726 * may still contain some buffers which would enable us 1727 * to limp on. 1728 */ 1729 netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 1730 priv->dpbp_dev->obj_desc.id, priv->bpid); 1731 } 1732 1733 if (!dpaa2_eth_is_type_phy(priv)) { 1734 /* We'll only start the txqs when the link is actually ready; 1735 * make sure we don't race against the link up notification, 1736 * which may come immediately after dpni_enable(); 1737 */ 1738 netif_tx_stop_all_queues(net_dev); 1739 1740 /* Also, explicitly set carrier off, otherwise 1741 * netif_carrier_ok() will return true and cause 'ip link show' 1742 * to report the LOWER_UP flag, even though the link 1743 * notification wasn't even received. 1744 */ 1745 netif_carrier_off(net_dev); 1746 } 1747 dpaa2_eth_enable_ch_napi(priv); 1748 1749 err = dpni_enable(priv->mc_io, 0, priv->mc_token); 1750 if (err < 0) { 1751 netdev_err(net_dev, "dpni_enable() failed\n"); 1752 goto enable_err; 1753 } 1754 1755 if (dpaa2_eth_is_type_phy(priv)) 1756 phylink_start(priv->mac->phylink); 1757 1758 return 0; 1759 1760 enable_err: 1761 dpaa2_eth_disable_ch_napi(priv); 1762 dpaa2_eth_drain_pool(priv); 1763 return err; 1764 } 1765 1766 /* Total number of in-flight frames on ingress queues */ 1767 static u32 dpaa2_eth_ingress_fq_count(struct dpaa2_eth_priv *priv) 1768 { 1769 struct dpaa2_eth_fq *fq; 1770 u32 fcnt = 0, bcnt = 0, total = 0; 1771 int i, err; 1772 1773 for (i = 0; i < priv->num_fqs; i++) { 1774 fq = &priv->fq[i]; 1775 err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 1776 if (err) { 1777 netdev_warn(priv->net_dev, "query_fq_count failed"); 1778 break; 1779 } 1780 total += fcnt; 1781 } 1782 1783 return total; 1784 } 1785 1786 static void dpaa2_eth_wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 1787 { 1788 int retries = 10; 1789 u32 pending; 1790 1791 do { 1792 pending = dpaa2_eth_ingress_fq_count(priv); 1793 if (pending) 1794 msleep(100); 1795 } while (pending && --retries); 1796 } 1797 1798 #define DPNI_TX_PENDING_VER_MAJOR 7 1799 #define DPNI_TX_PENDING_VER_MINOR 13 1800 static void dpaa2_eth_wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 1801 { 1802 union dpni_statistics stats; 1803 int retries = 10; 1804 int err; 1805 1806 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 1807 DPNI_TX_PENDING_VER_MINOR) < 0) 1808 goto out; 1809 1810 do { 1811 err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 1812 &stats); 1813 if (err) 1814 goto out; 1815 if (stats.page_6.tx_pending_frames == 0) 1816 return; 1817 } while (--retries); 1818 1819 out: 1820 msleep(500); 1821 } 1822 1823 static int dpaa2_eth_stop(struct net_device *net_dev) 1824 { 1825 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1826 int dpni_enabled = 0; 1827 int retries = 10; 1828 1829 if (dpaa2_eth_is_type_phy(priv)) { 1830 phylink_stop(priv->mac->phylink); 1831 } else { 1832 netif_tx_stop_all_queues(net_dev); 1833 netif_carrier_off(net_dev); 1834 } 1835 1836 /* On dpni_disable(), the MC firmware will: 1837 * - stop MAC Rx and wait for all Rx frames to be enqueued to software 1838 * - cut off WRIOP dequeues from egress FQs and wait until transmission 1839 * of all in flight Tx frames is finished (and corresponding Tx conf 1840 * frames are enqueued back to software) 1841 * 1842 * Before calling dpni_disable(), we wait for all Tx frames to arrive 1843 * on WRIOP. After it finishes, wait until all remaining frames on Rx 1844 * and Tx conf queues are consumed on NAPI poll. 1845 */ 1846 dpaa2_eth_wait_for_egress_fq_empty(priv); 1847 1848 do { 1849 dpni_disable(priv->mc_io, 0, priv->mc_token); 1850 dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 1851 if (dpni_enabled) 1852 /* Allow the hardware some slack */ 1853 msleep(100); 1854 } while (dpni_enabled && --retries); 1855 if (!retries) { 1856 netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 1857 /* Must go on and disable NAPI nonetheless, so we don't crash at 1858 * the next "ifconfig up" 1859 */ 1860 } 1861 1862 dpaa2_eth_wait_for_ingress_fq_empty(priv); 1863 dpaa2_eth_disable_ch_napi(priv); 1864 1865 /* Empty the buffer pool */ 1866 dpaa2_eth_drain_pool(priv); 1867 1868 /* Empty the Scatter-Gather Buffer cache */ 1869 dpaa2_eth_sgt_cache_drain(priv); 1870 1871 return 0; 1872 } 1873 1874 static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 1875 { 1876 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1877 struct device *dev = net_dev->dev.parent; 1878 int err; 1879 1880 err = eth_mac_addr(net_dev, addr); 1881 if (err < 0) { 1882 dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 1883 return err; 1884 } 1885 1886 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 1887 net_dev->dev_addr); 1888 if (err) { 1889 dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 1890 return err; 1891 } 1892 1893 return 0; 1894 } 1895 1896 /** Fill in counters maintained by the GPP driver. These may be different from 1897 * the hardware counters obtained by ethtool. 1898 */ 1899 static void dpaa2_eth_get_stats(struct net_device *net_dev, 1900 struct rtnl_link_stats64 *stats) 1901 { 1902 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1903 struct rtnl_link_stats64 *percpu_stats; 1904 u64 *cpustats; 1905 u64 *netstats = (u64 *)stats; 1906 int i, j; 1907 int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 1908 1909 for_each_possible_cpu(i) { 1910 percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 1911 cpustats = (u64 *)percpu_stats; 1912 for (j = 0; j < num; j++) 1913 netstats[j] += cpustats[j]; 1914 } 1915 } 1916 1917 /* Copy mac unicast addresses from @net_dev to @priv. 1918 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 1919 */ 1920 static void dpaa2_eth_add_uc_hw_addr(const struct net_device *net_dev, 1921 struct dpaa2_eth_priv *priv) 1922 { 1923 struct netdev_hw_addr *ha; 1924 int err; 1925 1926 netdev_for_each_uc_addr(ha, net_dev) { 1927 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 1928 ha->addr); 1929 if (err) 1930 netdev_warn(priv->net_dev, 1931 "Could not add ucast MAC %pM to the filtering table (err %d)\n", 1932 ha->addr, err); 1933 } 1934 } 1935 1936 /* Copy mac multicast addresses from @net_dev to @priv 1937 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 1938 */ 1939 static void dpaa2_eth_add_mc_hw_addr(const struct net_device *net_dev, 1940 struct dpaa2_eth_priv *priv) 1941 { 1942 struct netdev_hw_addr *ha; 1943 int err; 1944 1945 netdev_for_each_mc_addr(ha, net_dev) { 1946 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 1947 ha->addr); 1948 if (err) 1949 netdev_warn(priv->net_dev, 1950 "Could not add mcast MAC %pM to the filtering table (err %d)\n", 1951 ha->addr, err); 1952 } 1953 } 1954 1955 static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 1956 { 1957 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1958 int uc_count = netdev_uc_count(net_dev); 1959 int mc_count = netdev_mc_count(net_dev); 1960 u8 max_mac = priv->dpni_attrs.mac_filter_entries; 1961 u32 options = priv->dpni_attrs.options; 1962 u16 mc_token = priv->mc_token; 1963 struct fsl_mc_io *mc_io = priv->mc_io; 1964 int err; 1965 1966 /* Basic sanity checks; these probably indicate a misconfiguration */ 1967 if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 1968 netdev_info(net_dev, 1969 "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 1970 max_mac); 1971 1972 /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 1973 if (uc_count > max_mac) { 1974 netdev_info(net_dev, 1975 "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 1976 uc_count, max_mac); 1977 goto force_promisc; 1978 } 1979 if (mc_count + uc_count > max_mac) { 1980 netdev_info(net_dev, 1981 "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 1982 uc_count + mc_count, max_mac); 1983 goto force_mc_promisc; 1984 } 1985 1986 /* Adjust promisc settings due to flag combinations */ 1987 if (net_dev->flags & IFF_PROMISC) 1988 goto force_promisc; 1989 if (net_dev->flags & IFF_ALLMULTI) { 1990 /* First, rebuild unicast filtering table. This should be done 1991 * in promisc mode, in order to avoid frame loss while we 1992 * progressively add entries to the table. 1993 * We don't know whether we had been in promisc already, and 1994 * making an MC call to find out is expensive; so set uc promisc 1995 * nonetheless. 1996 */ 1997 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1998 if (err) 1999 netdev_warn(net_dev, "Can't set uc promisc\n"); 2000 2001 /* Actual uc table reconstruction. */ 2002 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 2003 if (err) 2004 netdev_warn(net_dev, "Can't clear uc filters\n"); 2005 dpaa2_eth_add_uc_hw_addr(net_dev, priv); 2006 2007 /* Finally, clear uc promisc and set mc promisc as requested. */ 2008 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 2009 if (err) 2010 netdev_warn(net_dev, "Can't clear uc promisc\n"); 2011 goto force_mc_promisc; 2012 } 2013 2014 /* Neither unicast, nor multicast promisc will be on... eventually. 2015 * For now, rebuild mac filtering tables while forcing both of them on. 2016 */ 2017 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 2018 if (err) 2019 netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 2020 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 2021 if (err) 2022 netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 2023 2024 /* Actual mac filtering tables reconstruction */ 2025 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 2026 if (err) 2027 netdev_warn(net_dev, "Can't clear mac filters\n"); 2028 dpaa2_eth_add_mc_hw_addr(net_dev, priv); 2029 dpaa2_eth_add_uc_hw_addr(net_dev, priv); 2030 2031 /* Now we can clear both ucast and mcast promisc, without risking 2032 * to drop legitimate frames anymore. 2033 */ 2034 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 2035 if (err) 2036 netdev_warn(net_dev, "Can't clear ucast promisc\n"); 2037 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 2038 if (err) 2039 netdev_warn(net_dev, "Can't clear mcast promisc\n"); 2040 2041 return; 2042 2043 force_promisc: 2044 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 2045 if (err) 2046 netdev_warn(net_dev, "Can't set ucast promisc\n"); 2047 force_mc_promisc: 2048 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 2049 if (err) 2050 netdev_warn(net_dev, "Can't set mcast promisc\n"); 2051 } 2052 2053 static int dpaa2_eth_set_features(struct net_device *net_dev, 2054 netdev_features_t features) 2055 { 2056 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2057 netdev_features_t changed = features ^ net_dev->features; 2058 bool enable; 2059 int err; 2060 2061 if (changed & NETIF_F_RXCSUM) { 2062 enable = !!(features & NETIF_F_RXCSUM); 2063 err = dpaa2_eth_set_rx_csum(priv, enable); 2064 if (err) 2065 return err; 2066 } 2067 2068 if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 2069 enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 2070 err = dpaa2_eth_set_tx_csum(priv, enable); 2071 if (err) 2072 return err; 2073 } 2074 2075 return 0; 2076 } 2077 2078 static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 2079 { 2080 struct dpaa2_eth_priv *priv = netdev_priv(dev); 2081 struct hwtstamp_config config; 2082 2083 if (!dpaa2_ptp) 2084 return -EINVAL; 2085 2086 if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 2087 return -EFAULT; 2088 2089 switch (config.tx_type) { 2090 case HWTSTAMP_TX_OFF: 2091 case HWTSTAMP_TX_ON: 2092 case HWTSTAMP_TX_ONESTEP_SYNC: 2093 priv->tx_tstamp_type = config.tx_type; 2094 break; 2095 default: 2096 return -ERANGE; 2097 } 2098 2099 if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 2100 priv->rx_tstamp = false; 2101 } else { 2102 priv->rx_tstamp = true; 2103 /* TS is set for all frame types, not only those requested */ 2104 config.rx_filter = HWTSTAMP_FILTER_ALL; 2105 } 2106 2107 return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 2108 -EFAULT : 0; 2109 } 2110 2111 static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 2112 { 2113 struct dpaa2_eth_priv *priv = netdev_priv(dev); 2114 2115 if (cmd == SIOCSHWTSTAMP) 2116 return dpaa2_eth_ts_ioctl(dev, rq, cmd); 2117 2118 if (dpaa2_eth_is_type_phy(priv)) 2119 return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); 2120 2121 return -EOPNOTSUPP; 2122 } 2123 2124 static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 2125 { 2126 int mfl, linear_mfl; 2127 2128 mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 2129 linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE - 2130 dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 2131 2132 if (mfl > linear_mfl) { 2133 netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 2134 linear_mfl - VLAN_ETH_HLEN); 2135 return false; 2136 } 2137 2138 return true; 2139 } 2140 2141 static int dpaa2_eth_set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 2142 { 2143 int mfl, err; 2144 2145 /* We enforce a maximum Rx frame length based on MTU only if we have 2146 * an XDP program attached (in order to avoid Rx S/G frames). 2147 * Otherwise, we accept all incoming frames as long as they are not 2148 * larger than maximum size supported in hardware 2149 */ 2150 if (has_xdp) 2151 mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 2152 else 2153 mfl = DPAA2_ETH_MFL; 2154 2155 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 2156 if (err) { 2157 netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 2158 return err; 2159 } 2160 2161 return 0; 2162 } 2163 2164 static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 2165 { 2166 struct dpaa2_eth_priv *priv = netdev_priv(dev); 2167 int err; 2168 2169 if (!priv->xdp_prog) 2170 goto out; 2171 2172 if (!xdp_mtu_valid(priv, new_mtu)) 2173 return -EINVAL; 2174 2175 err = dpaa2_eth_set_rx_mfl(priv, new_mtu, true); 2176 if (err) 2177 return err; 2178 2179 out: 2180 dev->mtu = new_mtu; 2181 return 0; 2182 } 2183 2184 static int dpaa2_eth_update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 2185 { 2186 struct dpni_buffer_layout buf_layout = {0}; 2187 int err; 2188 2189 err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 2190 DPNI_QUEUE_RX, &buf_layout); 2191 if (err) { 2192 netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 2193 return err; 2194 } 2195 2196 /* Reserve extra headroom for XDP header size changes */ 2197 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 2198 (has_xdp ? XDP_PACKET_HEADROOM : 0); 2199 buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 2200 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2201 DPNI_QUEUE_RX, &buf_layout); 2202 if (err) { 2203 netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 2204 return err; 2205 } 2206 2207 return 0; 2208 } 2209 2210 static int dpaa2_eth_setup_xdp(struct net_device *dev, struct bpf_prog *prog) 2211 { 2212 struct dpaa2_eth_priv *priv = netdev_priv(dev); 2213 struct dpaa2_eth_channel *ch; 2214 struct bpf_prog *old; 2215 bool up, need_update; 2216 int i, err; 2217 2218 if (prog && !xdp_mtu_valid(priv, dev->mtu)) 2219 return -EINVAL; 2220 2221 if (prog) 2222 bpf_prog_add(prog, priv->num_channels); 2223 2224 up = netif_running(dev); 2225 need_update = (!!priv->xdp_prog != !!prog); 2226 2227 if (up) 2228 dpaa2_eth_stop(dev); 2229 2230 /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 2231 * Also, when switching between xdp/non-xdp modes we need to reconfigure 2232 * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 2233 * so we are sure no old format buffers will be used from now on. 2234 */ 2235 if (need_update) { 2236 err = dpaa2_eth_set_rx_mfl(priv, dev->mtu, !!prog); 2237 if (err) 2238 goto out_err; 2239 err = dpaa2_eth_update_rx_buffer_headroom(priv, !!prog); 2240 if (err) 2241 goto out_err; 2242 } 2243 2244 old = xchg(&priv->xdp_prog, prog); 2245 if (old) 2246 bpf_prog_put(old); 2247 2248 for (i = 0; i < priv->num_channels; i++) { 2249 ch = priv->channel[i]; 2250 old = xchg(&ch->xdp.prog, prog); 2251 if (old) 2252 bpf_prog_put(old); 2253 } 2254 2255 if (up) { 2256 err = dpaa2_eth_open(dev); 2257 if (err) 2258 return err; 2259 } 2260 2261 return 0; 2262 2263 out_err: 2264 if (prog) 2265 bpf_prog_sub(prog, priv->num_channels); 2266 if (up) 2267 dpaa2_eth_open(dev); 2268 2269 return err; 2270 } 2271 2272 static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2273 { 2274 switch (xdp->command) { 2275 case XDP_SETUP_PROG: 2276 return dpaa2_eth_setup_xdp(dev, xdp->prog); 2277 default: 2278 return -EINVAL; 2279 } 2280 2281 return 0; 2282 } 2283 2284 static int dpaa2_eth_xdp_create_fd(struct net_device *net_dev, 2285 struct xdp_frame *xdpf, 2286 struct dpaa2_fd *fd) 2287 { 2288 struct device *dev = net_dev->dev.parent; 2289 unsigned int needed_headroom; 2290 struct dpaa2_eth_swa *swa; 2291 void *buffer_start, *aligned_start; 2292 dma_addr_t addr; 2293 2294 /* We require a minimum headroom to be able to transmit the frame. 2295 * Otherwise return an error and let the original net_device handle it 2296 */ 2297 needed_headroom = dpaa2_eth_needed_headroom(NULL); 2298 if (xdpf->headroom < needed_headroom) 2299 return -EINVAL; 2300 2301 /* Setup the FD fields */ 2302 memset(fd, 0, sizeof(*fd)); 2303 2304 /* Align FD address, if possible */ 2305 buffer_start = xdpf->data - needed_headroom; 2306 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 2307 DPAA2_ETH_TX_BUF_ALIGN); 2308 if (aligned_start >= xdpf->data - xdpf->headroom) 2309 buffer_start = aligned_start; 2310 2311 swa = (struct dpaa2_eth_swa *)buffer_start; 2312 /* fill in necessary fields here */ 2313 swa->type = DPAA2_ETH_SWA_XDP; 2314 swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 2315 swa->xdp.xdpf = xdpf; 2316 2317 addr = dma_map_single(dev, buffer_start, 2318 swa->xdp.dma_size, 2319 DMA_BIDIRECTIONAL); 2320 if (unlikely(dma_mapping_error(dev, addr))) 2321 return -ENOMEM; 2322 2323 dpaa2_fd_set_addr(fd, addr); 2324 dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); 2325 dpaa2_fd_set_len(fd, xdpf->len); 2326 dpaa2_fd_set_format(fd, dpaa2_fd_single); 2327 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 2328 2329 return 0; 2330 } 2331 2332 static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 2333 struct xdp_frame **frames, u32 flags) 2334 { 2335 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2336 struct dpaa2_eth_xdp_fds *xdp_redirect_fds; 2337 struct rtnl_link_stats64 *percpu_stats; 2338 struct dpaa2_eth_fq *fq; 2339 struct dpaa2_fd *fds; 2340 int enqueued, i, err; 2341 2342 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2343 return -EINVAL; 2344 2345 if (!netif_running(net_dev)) 2346 return -ENETDOWN; 2347 2348 fq = &priv->fq[smp_processor_id()]; 2349 xdp_redirect_fds = &fq->xdp_redirect_fds; 2350 fds = xdp_redirect_fds->fds; 2351 2352 percpu_stats = this_cpu_ptr(priv->percpu_stats); 2353 2354 /* create a FD for each xdp_frame in the list received */ 2355 for (i = 0; i < n; i++) { 2356 err = dpaa2_eth_xdp_create_fd(net_dev, frames[i], &fds[i]); 2357 if (err) 2358 break; 2359 } 2360 xdp_redirect_fds->num = i; 2361 2362 /* enqueue all the frame descriptors */ 2363 enqueued = dpaa2_eth_xdp_flush(priv, fq, xdp_redirect_fds); 2364 2365 /* update statistics */ 2366 percpu_stats->tx_packets += enqueued; 2367 for (i = 0; i < enqueued; i++) 2368 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 2369 for (i = enqueued; i < n; i++) 2370 xdp_return_frame_rx_napi(frames[i]); 2371 2372 return enqueued; 2373 } 2374 2375 static int update_xps(struct dpaa2_eth_priv *priv) 2376 { 2377 struct net_device *net_dev = priv->net_dev; 2378 struct cpumask xps_mask; 2379 struct dpaa2_eth_fq *fq; 2380 int i, num_queues, netdev_queues; 2381 int err = 0; 2382 2383 num_queues = dpaa2_eth_queue_count(priv); 2384 netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 2385 2386 /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 2387 * queues, so only process those 2388 */ 2389 for (i = 0; i < netdev_queues; i++) { 2390 fq = &priv->fq[i % num_queues]; 2391 2392 cpumask_clear(&xps_mask); 2393 cpumask_set_cpu(fq->target_cpu, &xps_mask); 2394 2395 err = netif_set_xps_queue(net_dev, &xps_mask, i); 2396 if (err) { 2397 netdev_warn_once(net_dev, "Error setting XPS queue\n"); 2398 break; 2399 } 2400 } 2401 2402 return err; 2403 } 2404 2405 static int dpaa2_eth_setup_mqprio(struct net_device *net_dev, 2406 struct tc_mqprio_qopt *mqprio) 2407 { 2408 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2409 u8 num_tc, num_queues; 2410 int i; 2411 2412 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2413 num_queues = dpaa2_eth_queue_count(priv); 2414 num_tc = mqprio->num_tc; 2415 2416 if (num_tc == net_dev->num_tc) 2417 return 0; 2418 2419 if (num_tc > dpaa2_eth_tc_count(priv)) { 2420 netdev_err(net_dev, "Max %d traffic classes supported\n", 2421 dpaa2_eth_tc_count(priv)); 2422 return -EOPNOTSUPP; 2423 } 2424 2425 if (!num_tc) { 2426 netdev_reset_tc(net_dev); 2427 netif_set_real_num_tx_queues(net_dev, num_queues); 2428 goto out; 2429 } 2430 2431 netdev_set_num_tc(net_dev, num_tc); 2432 netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2433 2434 for (i = 0; i < num_tc; i++) 2435 netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2436 2437 out: 2438 update_xps(priv); 2439 2440 return 0; 2441 } 2442 2443 #define bps_to_mbits(rate) (div_u64((rate), 1000000) * 8) 2444 2445 static int dpaa2_eth_setup_tbf(struct net_device *net_dev, struct tc_tbf_qopt_offload *p) 2446 { 2447 struct tc_tbf_qopt_offload_replace_params *cfg = &p->replace_params; 2448 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2449 struct dpni_tx_shaping_cfg tx_cr_shaper = { 0 }; 2450 struct dpni_tx_shaping_cfg tx_er_shaper = { 0 }; 2451 int err; 2452 2453 if (p->command == TC_TBF_STATS) 2454 return -EOPNOTSUPP; 2455 2456 /* Only per port Tx shaping */ 2457 if (p->parent != TC_H_ROOT) 2458 return -EOPNOTSUPP; 2459 2460 if (p->command == TC_TBF_REPLACE) { 2461 if (cfg->max_size > DPAA2_ETH_MAX_BURST_SIZE) { 2462 netdev_err(net_dev, "burst size cannot be greater than %d\n", 2463 DPAA2_ETH_MAX_BURST_SIZE); 2464 return -EINVAL; 2465 } 2466 2467 tx_cr_shaper.max_burst_size = cfg->max_size; 2468 /* The TBF interface is in bytes/s, whereas DPAA2 expects the 2469 * rate in Mbits/s 2470 */ 2471 tx_cr_shaper.rate_limit = bps_to_mbits(cfg->rate.rate_bytes_ps); 2472 } 2473 2474 err = dpni_set_tx_shaping(priv->mc_io, 0, priv->mc_token, &tx_cr_shaper, 2475 &tx_er_shaper, 0); 2476 if (err) { 2477 netdev_err(net_dev, "dpni_set_tx_shaping() = %d\n", err); 2478 return err; 2479 } 2480 2481 return 0; 2482 } 2483 2484 static int dpaa2_eth_setup_tc(struct net_device *net_dev, 2485 enum tc_setup_type type, void *type_data) 2486 { 2487 switch (type) { 2488 case TC_SETUP_QDISC_MQPRIO: 2489 return dpaa2_eth_setup_mqprio(net_dev, type_data); 2490 case TC_SETUP_QDISC_TBF: 2491 return dpaa2_eth_setup_tbf(net_dev, type_data); 2492 default: 2493 return -EOPNOTSUPP; 2494 } 2495 } 2496 2497 static const struct net_device_ops dpaa2_eth_ops = { 2498 .ndo_open = dpaa2_eth_open, 2499 .ndo_start_xmit = dpaa2_eth_tx, 2500 .ndo_stop = dpaa2_eth_stop, 2501 .ndo_set_mac_address = dpaa2_eth_set_addr, 2502 .ndo_get_stats64 = dpaa2_eth_get_stats, 2503 .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 2504 .ndo_set_features = dpaa2_eth_set_features, 2505 .ndo_do_ioctl = dpaa2_eth_ioctl, 2506 .ndo_change_mtu = dpaa2_eth_change_mtu, 2507 .ndo_bpf = dpaa2_eth_xdp, 2508 .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2509 .ndo_setup_tc = dpaa2_eth_setup_tc, 2510 }; 2511 2512 static void dpaa2_eth_cdan_cb(struct dpaa2_io_notification_ctx *ctx) 2513 { 2514 struct dpaa2_eth_channel *ch; 2515 2516 ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 2517 2518 /* Update NAPI statistics */ 2519 ch->stats.cdan++; 2520 2521 napi_schedule(&ch->napi); 2522 } 2523 2524 /* Allocate and configure a DPCON object */ 2525 static struct fsl_mc_device *dpaa2_eth_setup_dpcon(struct dpaa2_eth_priv *priv) 2526 { 2527 struct fsl_mc_device *dpcon; 2528 struct device *dev = priv->net_dev->dev.parent; 2529 int err; 2530 2531 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 2532 FSL_MC_POOL_DPCON, &dpcon); 2533 if (err) { 2534 if (err == -ENXIO) 2535 err = -EPROBE_DEFER; 2536 else 2537 dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2538 return ERR_PTR(err); 2539 } 2540 2541 err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 2542 if (err) { 2543 dev_err(dev, "dpcon_open() failed\n"); 2544 goto free; 2545 } 2546 2547 err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 2548 if (err) { 2549 dev_err(dev, "dpcon_reset() failed\n"); 2550 goto close; 2551 } 2552 2553 err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 2554 if (err) { 2555 dev_err(dev, "dpcon_enable() failed\n"); 2556 goto close; 2557 } 2558 2559 return dpcon; 2560 2561 close: 2562 dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 2563 free: 2564 fsl_mc_object_free(dpcon); 2565 2566 return ERR_PTR(err); 2567 } 2568 2569 static void dpaa2_eth_free_dpcon(struct dpaa2_eth_priv *priv, 2570 struct fsl_mc_device *dpcon) 2571 { 2572 dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 2573 dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 2574 fsl_mc_object_free(dpcon); 2575 } 2576 2577 static struct dpaa2_eth_channel *dpaa2_eth_alloc_channel(struct dpaa2_eth_priv *priv) 2578 { 2579 struct dpaa2_eth_channel *channel; 2580 struct dpcon_attr attr; 2581 struct device *dev = priv->net_dev->dev.parent; 2582 int err; 2583 2584 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 2585 if (!channel) 2586 return NULL; 2587 2588 channel->dpcon = dpaa2_eth_setup_dpcon(priv); 2589 if (IS_ERR(channel->dpcon)) { 2590 err = PTR_ERR(channel->dpcon); 2591 goto err_setup; 2592 } 2593 2594 err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 2595 &attr); 2596 if (err) { 2597 dev_err(dev, "dpcon_get_attributes() failed\n"); 2598 goto err_get_attr; 2599 } 2600 2601 channel->dpcon_id = attr.id; 2602 channel->ch_id = attr.qbman_ch_id; 2603 channel->priv = priv; 2604 2605 return channel; 2606 2607 err_get_attr: 2608 dpaa2_eth_free_dpcon(priv, channel->dpcon); 2609 err_setup: 2610 kfree(channel); 2611 return ERR_PTR(err); 2612 } 2613 2614 static void dpaa2_eth_free_channel(struct dpaa2_eth_priv *priv, 2615 struct dpaa2_eth_channel *channel) 2616 { 2617 dpaa2_eth_free_dpcon(priv, channel->dpcon); 2618 kfree(channel); 2619 } 2620 2621 /* DPIO setup: allocate and configure QBMan channels, setup core affinity 2622 * and register data availability notifications 2623 */ 2624 static int dpaa2_eth_setup_dpio(struct dpaa2_eth_priv *priv) 2625 { 2626 struct dpaa2_io_notification_ctx *nctx; 2627 struct dpaa2_eth_channel *channel; 2628 struct dpcon_notification_cfg dpcon_notif_cfg; 2629 struct device *dev = priv->net_dev->dev.parent; 2630 int i, err; 2631 2632 /* We want the ability to spread ingress traffic (RX, TX conf) to as 2633 * many cores as possible, so we need one channel for each core 2634 * (unless there's fewer queues than cores, in which case the extra 2635 * channels would be wasted). 2636 * Allocate one channel per core and register it to the core's 2637 * affine DPIO. If not enough channels are available for all cores 2638 * or if some cores don't have an affine DPIO, there will be no 2639 * ingress frame processing on those cores. 2640 */ 2641 cpumask_clear(&priv->dpio_cpumask); 2642 for_each_online_cpu(i) { 2643 /* Try to allocate a channel */ 2644 channel = dpaa2_eth_alloc_channel(priv); 2645 if (IS_ERR_OR_NULL(channel)) { 2646 err = PTR_ERR_OR_ZERO(channel); 2647 if (err != -EPROBE_DEFER) 2648 dev_info(dev, 2649 "No affine channel for cpu %d and above\n", i); 2650 goto err_alloc_ch; 2651 } 2652 2653 priv->channel[priv->num_channels] = channel; 2654 2655 nctx = &channel->nctx; 2656 nctx->is_cdan = 1; 2657 nctx->cb = dpaa2_eth_cdan_cb; 2658 nctx->id = channel->ch_id; 2659 nctx->desired_cpu = i; 2660 2661 /* Register the new context */ 2662 channel->dpio = dpaa2_io_service_select(i); 2663 err = dpaa2_io_service_register(channel->dpio, nctx, dev); 2664 if (err) { 2665 dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 2666 /* If no affine DPIO for this core, there's probably 2667 * none available for next cores either. Signal we want 2668 * to retry later, in case the DPIO devices weren't 2669 * probed yet. 2670 */ 2671 err = -EPROBE_DEFER; 2672 goto err_service_reg; 2673 } 2674 2675 /* Register DPCON notification with MC */ 2676 dpcon_notif_cfg.dpio_id = nctx->dpio_id; 2677 dpcon_notif_cfg.priority = 0; 2678 dpcon_notif_cfg.user_ctx = nctx->qman64; 2679 err = dpcon_set_notification(priv->mc_io, 0, 2680 channel->dpcon->mc_handle, 2681 &dpcon_notif_cfg); 2682 if (err) { 2683 dev_err(dev, "dpcon_set_notification failed()\n"); 2684 goto err_set_cdan; 2685 } 2686 2687 /* If we managed to allocate a channel and also found an affine 2688 * DPIO for this core, add it to the final mask 2689 */ 2690 cpumask_set_cpu(i, &priv->dpio_cpumask); 2691 priv->num_channels++; 2692 2693 /* Stop if we already have enough channels to accommodate all 2694 * RX and TX conf queues 2695 */ 2696 if (priv->num_channels == priv->dpni_attrs.num_queues) 2697 break; 2698 } 2699 2700 return 0; 2701 2702 err_set_cdan: 2703 dpaa2_io_service_deregister(channel->dpio, nctx, dev); 2704 err_service_reg: 2705 dpaa2_eth_free_channel(priv, channel); 2706 err_alloc_ch: 2707 if (err == -EPROBE_DEFER) { 2708 for (i = 0; i < priv->num_channels; i++) { 2709 channel = priv->channel[i]; 2710 nctx = &channel->nctx; 2711 dpaa2_io_service_deregister(channel->dpio, nctx, dev); 2712 dpaa2_eth_free_channel(priv, channel); 2713 } 2714 priv->num_channels = 0; 2715 return err; 2716 } 2717 2718 if (cpumask_empty(&priv->dpio_cpumask)) { 2719 dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 2720 return -ENODEV; 2721 } 2722 2723 dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 2724 cpumask_pr_args(&priv->dpio_cpumask)); 2725 2726 return 0; 2727 } 2728 2729 static void dpaa2_eth_free_dpio(struct dpaa2_eth_priv *priv) 2730 { 2731 struct device *dev = priv->net_dev->dev.parent; 2732 struct dpaa2_eth_channel *ch; 2733 int i; 2734 2735 /* deregister CDAN notifications and free channels */ 2736 for (i = 0; i < priv->num_channels; i++) { 2737 ch = priv->channel[i]; 2738 dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 2739 dpaa2_eth_free_channel(priv, ch); 2740 } 2741 } 2742 2743 static struct dpaa2_eth_channel *dpaa2_eth_get_affine_channel(struct dpaa2_eth_priv *priv, 2744 int cpu) 2745 { 2746 struct device *dev = priv->net_dev->dev.parent; 2747 int i; 2748 2749 for (i = 0; i < priv->num_channels; i++) 2750 if (priv->channel[i]->nctx.desired_cpu == cpu) 2751 return priv->channel[i]; 2752 2753 /* We should never get here. Issue a warning and return 2754 * the first channel, because it's still better than nothing 2755 */ 2756 dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 2757 2758 return priv->channel[0]; 2759 } 2760 2761 static void dpaa2_eth_set_fq_affinity(struct dpaa2_eth_priv *priv) 2762 { 2763 struct device *dev = priv->net_dev->dev.parent; 2764 struct dpaa2_eth_fq *fq; 2765 int rx_cpu, txc_cpu; 2766 int i; 2767 2768 /* For each FQ, pick one channel/CPU to deliver frames to. 2769 * This may well change at runtime, either through irqbalance or 2770 * through direct user intervention. 2771 */ 2772 rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 2773 2774 for (i = 0; i < priv->num_fqs; i++) { 2775 fq = &priv->fq[i]; 2776 switch (fq->type) { 2777 case DPAA2_RX_FQ: 2778 case DPAA2_RX_ERR_FQ: 2779 fq->target_cpu = rx_cpu; 2780 rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 2781 if (rx_cpu >= nr_cpu_ids) 2782 rx_cpu = cpumask_first(&priv->dpio_cpumask); 2783 break; 2784 case DPAA2_TX_CONF_FQ: 2785 fq->target_cpu = txc_cpu; 2786 txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 2787 if (txc_cpu >= nr_cpu_ids) 2788 txc_cpu = cpumask_first(&priv->dpio_cpumask); 2789 break; 2790 default: 2791 dev_err(dev, "Unknown FQ type: %d\n", fq->type); 2792 } 2793 fq->channel = dpaa2_eth_get_affine_channel(priv, fq->target_cpu); 2794 } 2795 2796 update_xps(priv); 2797 } 2798 2799 static void dpaa2_eth_setup_fqs(struct dpaa2_eth_priv *priv) 2800 { 2801 int i, j; 2802 2803 /* We have one TxConf FQ per Tx flow. 2804 * The number of Tx and Rx queues is the same. 2805 * Tx queues come first in the fq array. 2806 */ 2807 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 2808 priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 2809 priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 2810 priv->fq[priv->num_fqs++].flowid = (u16)i; 2811 } 2812 2813 for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 2814 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 2815 priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 2816 priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 2817 priv->fq[priv->num_fqs].tc = (u8)j; 2818 priv->fq[priv->num_fqs++].flowid = (u16)i; 2819 } 2820 } 2821 2822 /* We have exactly one Rx error queue per DPNI */ 2823 priv->fq[priv->num_fqs].type = DPAA2_RX_ERR_FQ; 2824 priv->fq[priv->num_fqs++].consume = dpaa2_eth_rx_err; 2825 2826 /* For each FQ, decide on which core to process incoming frames */ 2827 dpaa2_eth_set_fq_affinity(priv); 2828 } 2829 2830 /* Allocate and configure one buffer pool for each interface */ 2831 static int dpaa2_eth_setup_dpbp(struct dpaa2_eth_priv *priv) 2832 { 2833 int err; 2834 struct fsl_mc_device *dpbp_dev; 2835 struct device *dev = priv->net_dev->dev.parent; 2836 struct dpbp_attr dpbp_attrs; 2837 2838 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 2839 &dpbp_dev); 2840 if (err) { 2841 if (err == -ENXIO) 2842 err = -EPROBE_DEFER; 2843 else 2844 dev_err(dev, "DPBP device allocation failed\n"); 2845 return err; 2846 } 2847 2848 priv->dpbp_dev = dpbp_dev; 2849 2850 err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 2851 &dpbp_dev->mc_handle); 2852 if (err) { 2853 dev_err(dev, "dpbp_open() failed\n"); 2854 goto err_open; 2855 } 2856 2857 err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 2858 if (err) { 2859 dev_err(dev, "dpbp_reset() failed\n"); 2860 goto err_reset; 2861 } 2862 2863 err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 2864 if (err) { 2865 dev_err(dev, "dpbp_enable() failed\n"); 2866 goto err_enable; 2867 } 2868 2869 err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 2870 &dpbp_attrs); 2871 if (err) { 2872 dev_err(dev, "dpbp_get_attributes() failed\n"); 2873 goto err_get_attr; 2874 } 2875 priv->bpid = dpbp_attrs.bpid; 2876 2877 return 0; 2878 2879 err_get_attr: 2880 dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 2881 err_enable: 2882 err_reset: 2883 dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 2884 err_open: 2885 fsl_mc_object_free(dpbp_dev); 2886 2887 return err; 2888 } 2889 2890 static void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv) 2891 { 2892 dpaa2_eth_drain_pool(priv); 2893 dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 2894 dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 2895 fsl_mc_object_free(priv->dpbp_dev); 2896 } 2897 2898 static int dpaa2_eth_set_buffer_layout(struct dpaa2_eth_priv *priv) 2899 { 2900 struct device *dev = priv->net_dev->dev.parent; 2901 struct dpni_buffer_layout buf_layout = {0}; 2902 u16 rx_buf_align; 2903 int err; 2904 2905 /* We need to check for WRIOP version 1.0.0, but depending on the MC 2906 * version, this number is not always provided correctly on rev1. 2907 * We need to check for both alternatives in this situation. 2908 */ 2909 if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 2910 priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 2911 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 2912 else 2913 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 2914 2915 /* We need to ensure that the buffer size seen by WRIOP is a multiple 2916 * of 64 or 256 bytes depending on the WRIOP version. 2917 */ 2918 priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align); 2919 2920 /* tx buffer */ 2921 buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 2922 buf_layout.pass_timestamp = true; 2923 buf_layout.pass_frame_status = true; 2924 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 2925 DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 2926 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 2927 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2928 DPNI_QUEUE_TX, &buf_layout); 2929 if (err) { 2930 dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 2931 return err; 2932 } 2933 2934 /* tx-confirm buffer */ 2935 buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 2936 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 2937 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2938 DPNI_QUEUE_TX_CONFIRM, &buf_layout); 2939 if (err) { 2940 dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 2941 return err; 2942 } 2943 2944 /* Now that we've set our tx buffer layout, retrieve the minimum 2945 * required tx data offset. 2946 */ 2947 err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 2948 &priv->tx_data_offset); 2949 if (err) { 2950 dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 2951 return err; 2952 } 2953 2954 if ((priv->tx_data_offset % 64) != 0) 2955 dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 2956 priv->tx_data_offset); 2957 2958 /* rx buffer */ 2959 buf_layout.pass_frame_status = true; 2960 buf_layout.pass_parser_result = true; 2961 buf_layout.data_align = rx_buf_align; 2962 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 2963 buf_layout.private_data_size = 0; 2964 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 2965 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 2966 DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 2967 DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 2968 DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2969 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2970 DPNI_QUEUE_RX, &buf_layout); 2971 if (err) { 2972 dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 2973 return err; 2974 } 2975 2976 return 0; 2977 } 2978 2979 #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 2980 #define DPNI_ENQUEUE_FQID_VER_MINOR 9 2981 2982 static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 2983 struct dpaa2_eth_fq *fq, 2984 struct dpaa2_fd *fd, u8 prio, 2985 u32 num_frames __always_unused, 2986 int *frames_enqueued) 2987 { 2988 int err; 2989 2990 err = dpaa2_io_service_enqueue_qd(fq->channel->dpio, 2991 priv->tx_qdid, prio, 2992 fq->tx_qdbin, fd); 2993 if (!err && frames_enqueued) 2994 *frames_enqueued = 1; 2995 return err; 2996 } 2997 2998 static inline int dpaa2_eth_enqueue_fq_multiple(struct dpaa2_eth_priv *priv, 2999 struct dpaa2_eth_fq *fq, 3000 struct dpaa2_fd *fd, 3001 u8 prio, u32 num_frames, 3002 int *frames_enqueued) 3003 { 3004 int err; 3005 3006 err = dpaa2_io_service_enqueue_multiple_fq(fq->channel->dpio, 3007 fq->tx_fqid[prio], 3008 fd, num_frames); 3009 3010 if (err == 0) 3011 return -EBUSY; 3012 3013 if (frames_enqueued) 3014 *frames_enqueued = err; 3015 return 0; 3016 } 3017 3018 static void dpaa2_eth_set_enqueue_mode(struct dpaa2_eth_priv *priv) 3019 { 3020 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 3021 DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 3022 priv->enqueue = dpaa2_eth_enqueue_qd; 3023 else 3024 priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 3025 } 3026 3027 static int dpaa2_eth_set_pause(struct dpaa2_eth_priv *priv) 3028 { 3029 struct device *dev = priv->net_dev->dev.parent; 3030 struct dpni_link_cfg link_cfg = {0}; 3031 int err; 3032 3033 /* Get the default link options so we don't override other flags */ 3034 err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 3035 if (err) { 3036 dev_err(dev, "dpni_get_link_cfg() failed\n"); 3037 return err; 3038 } 3039 3040 /* By default, enable both Rx and Tx pause frames */ 3041 link_cfg.options |= DPNI_LINK_OPT_PAUSE; 3042 link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 3043 err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 3044 if (err) { 3045 dev_err(dev, "dpni_set_link_cfg() failed\n"); 3046 return err; 3047 } 3048 3049 priv->link_state.options = link_cfg.options; 3050 3051 return 0; 3052 } 3053 3054 static void dpaa2_eth_update_tx_fqids(struct dpaa2_eth_priv *priv) 3055 { 3056 struct dpni_queue_id qid = {0}; 3057 struct dpaa2_eth_fq *fq; 3058 struct dpni_queue queue; 3059 int i, j, err; 3060 3061 /* We only use Tx FQIDs for FQID-based enqueue, so check 3062 * if DPNI version supports it before updating FQIDs 3063 */ 3064 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 3065 DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 3066 return; 3067 3068 for (i = 0; i < priv->num_fqs; i++) { 3069 fq = &priv->fq[i]; 3070 if (fq->type != DPAA2_TX_CONF_FQ) 3071 continue; 3072 for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 3073 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3074 DPNI_QUEUE_TX, j, fq->flowid, 3075 &queue, &qid); 3076 if (err) 3077 goto out_err; 3078 3079 fq->tx_fqid[j] = qid.fqid; 3080 if (fq->tx_fqid[j] == 0) 3081 goto out_err; 3082 } 3083 } 3084 3085 priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 3086 3087 return; 3088 3089 out_err: 3090 netdev_info(priv->net_dev, 3091 "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 3092 priv->enqueue = dpaa2_eth_enqueue_qd; 3093 } 3094 3095 /* Configure ingress classification based on VLAN PCP */ 3096 static int dpaa2_eth_set_vlan_qos(struct dpaa2_eth_priv *priv) 3097 { 3098 struct device *dev = priv->net_dev->dev.parent; 3099 struct dpkg_profile_cfg kg_cfg = {0}; 3100 struct dpni_qos_tbl_cfg qos_cfg = {0}; 3101 struct dpni_rule_cfg key_params; 3102 void *dma_mem, *key, *mask; 3103 u8 key_size = 2; /* VLAN TCI field */ 3104 int i, pcp, err; 3105 3106 /* VLAN-based classification only makes sense if we have multiple 3107 * traffic classes. 3108 * Also, we need to extract just the 3-bit PCP field from the VLAN 3109 * header and we can only do that by using a mask 3110 */ 3111 if (dpaa2_eth_tc_count(priv) == 1 || !dpaa2_eth_fs_mask_enabled(priv)) { 3112 dev_dbg(dev, "VLAN-based QoS classification not supported\n"); 3113 return -EOPNOTSUPP; 3114 } 3115 3116 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 3117 if (!dma_mem) 3118 return -ENOMEM; 3119 3120 kg_cfg.num_extracts = 1; 3121 kg_cfg.extracts[0].type = DPKG_EXTRACT_FROM_HDR; 3122 kg_cfg.extracts[0].extract.from_hdr.prot = NET_PROT_VLAN; 3123 kg_cfg.extracts[0].extract.from_hdr.type = DPKG_FULL_FIELD; 3124 kg_cfg.extracts[0].extract.from_hdr.field = NH_FLD_VLAN_TCI; 3125 3126 err = dpni_prepare_key_cfg(&kg_cfg, dma_mem); 3127 if (err) { 3128 dev_err(dev, "dpni_prepare_key_cfg failed\n"); 3129 goto out_free_tbl; 3130 } 3131 3132 /* set QoS table */ 3133 qos_cfg.default_tc = 0; 3134 qos_cfg.discard_on_miss = 0; 3135 qos_cfg.key_cfg_iova = dma_map_single(dev, dma_mem, 3136 DPAA2_CLASSIFIER_DMA_SIZE, 3137 DMA_TO_DEVICE); 3138 if (dma_mapping_error(dev, qos_cfg.key_cfg_iova)) { 3139 dev_err(dev, "QoS table DMA mapping failed\n"); 3140 err = -ENOMEM; 3141 goto out_free_tbl; 3142 } 3143 3144 err = dpni_set_qos_table(priv->mc_io, 0, priv->mc_token, &qos_cfg); 3145 if (err) { 3146 dev_err(dev, "dpni_set_qos_table failed\n"); 3147 goto out_unmap_tbl; 3148 } 3149 3150 /* Add QoS table entries */ 3151 key = kzalloc(key_size * 2, GFP_KERNEL); 3152 if (!key) { 3153 err = -ENOMEM; 3154 goto out_unmap_tbl; 3155 } 3156 mask = key + key_size; 3157 *(__be16 *)mask = cpu_to_be16(VLAN_PRIO_MASK); 3158 3159 key_params.key_iova = dma_map_single(dev, key, key_size * 2, 3160 DMA_TO_DEVICE); 3161 if (dma_mapping_error(dev, key_params.key_iova)) { 3162 dev_err(dev, "Qos table entry DMA mapping failed\n"); 3163 err = -ENOMEM; 3164 goto out_free_key; 3165 } 3166 3167 key_params.mask_iova = key_params.key_iova + key_size; 3168 key_params.key_size = key_size; 3169 3170 /* We add rules for PCP-based distribution starting with highest 3171 * priority (VLAN PCP = 7). If this DPNI doesn't have enough traffic 3172 * classes to accommodate all priority levels, the lowest ones end up 3173 * on TC 0 which was configured as default 3174 */ 3175 for (i = dpaa2_eth_tc_count(priv) - 1, pcp = 7; i >= 0; i--, pcp--) { 3176 *(__be16 *)key = cpu_to_be16(pcp << VLAN_PRIO_SHIFT); 3177 dma_sync_single_for_device(dev, key_params.key_iova, 3178 key_size * 2, DMA_TO_DEVICE); 3179 3180 err = dpni_add_qos_entry(priv->mc_io, 0, priv->mc_token, 3181 &key_params, i, i); 3182 if (err) { 3183 dev_err(dev, "dpni_add_qos_entry failed\n"); 3184 dpni_clear_qos_table(priv->mc_io, 0, priv->mc_token); 3185 goto out_unmap_key; 3186 } 3187 } 3188 3189 priv->vlan_cls_enabled = true; 3190 3191 /* Table and key memory is not persistent, clean everything up after 3192 * configuration is finished 3193 */ 3194 out_unmap_key: 3195 dma_unmap_single(dev, key_params.key_iova, key_size * 2, DMA_TO_DEVICE); 3196 out_free_key: 3197 kfree(key); 3198 out_unmap_tbl: 3199 dma_unmap_single(dev, qos_cfg.key_cfg_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3200 DMA_TO_DEVICE); 3201 out_free_tbl: 3202 kfree(dma_mem); 3203 3204 return err; 3205 } 3206 3207 /* Configure the DPNI object this interface is associated with */ 3208 static int dpaa2_eth_setup_dpni(struct fsl_mc_device *ls_dev) 3209 { 3210 struct device *dev = &ls_dev->dev; 3211 struct dpaa2_eth_priv *priv; 3212 struct net_device *net_dev; 3213 int err; 3214 3215 net_dev = dev_get_drvdata(dev); 3216 priv = netdev_priv(net_dev); 3217 3218 /* get a handle for the DPNI object */ 3219 err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 3220 if (err) { 3221 dev_err(dev, "dpni_open() failed\n"); 3222 return err; 3223 } 3224 3225 /* Check if we can work with this DPNI object */ 3226 err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 3227 &priv->dpni_ver_minor); 3228 if (err) { 3229 dev_err(dev, "dpni_get_api_version() failed\n"); 3230 goto close; 3231 } 3232 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 3233 dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 3234 priv->dpni_ver_major, priv->dpni_ver_minor, 3235 DPNI_VER_MAJOR, DPNI_VER_MINOR); 3236 err = -ENOTSUPP; 3237 goto close; 3238 } 3239 3240 ls_dev->mc_io = priv->mc_io; 3241 ls_dev->mc_handle = priv->mc_token; 3242 3243 err = dpni_reset(priv->mc_io, 0, priv->mc_token); 3244 if (err) { 3245 dev_err(dev, "dpni_reset() failed\n"); 3246 goto close; 3247 } 3248 3249 err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 3250 &priv->dpni_attrs); 3251 if (err) { 3252 dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 3253 goto close; 3254 } 3255 3256 err = dpaa2_eth_set_buffer_layout(priv); 3257 if (err) 3258 goto close; 3259 3260 dpaa2_eth_set_enqueue_mode(priv); 3261 3262 /* Enable pause frame support */ 3263 if (dpaa2_eth_has_pause_support(priv)) { 3264 err = dpaa2_eth_set_pause(priv); 3265 if (err) 3266 goto close; 3267 } 3268 3269 err = dpaa2_eth_set_vlan_qos(priv); 3270 if (err && err != -EOPNOTSUPP) 3271 goto close; 3272 3273 priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv), 3274 sizeof(struct dpaa2_eth_cls_rule), 3275 GFP_KERNEL); 3276 if (!priv->cls_rules) { 3277 err = -ENOMEM; 3278 goto close; 3279 } 3280 3281 return 0; 3282 3283 close: 3284 dpni_close(priv->mc_io, 0, priv->mc_token); 3285 3286 return err; 3287 } 3288 3289 static void dpaa2_eth_free_dpni(struct dpaa2_eth_priv *priv) 3290 { 3291 int err; 3292 3293 err = dpni_reset(priv->mc_io, 0, priv->mc_token); 3294 if (err) 3295 netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 3296 err); 3297 3298 dpni_close(priv->mc_io, 0, priv->mc_token); 3299 } 3300 3301 static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, 3302 struct dpaa2_eth_fq *fq) 3303 { 3304 struct device *dev = priv->net_dev->dev.parent; 3305 struct dpni_queue queue; 3306 struct dpni_queue_id qid; 3307 int err; 3308 3309 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3310 DPNI_QUEUE_RX, fq->tc, fq->flowid, &queue, &qid); 3311 if (err) { 3312 dev_err(dev, "dpni_get_queue(RX) failed\n"); 3313 return err; 3314 } 3315 3316 fq->fqid = qid.fqid; 3317 3318 queue.destination.id = fq->channel->dpcon_id; 3319 queue.destination.type = DPNI_DEST_DPCON; 3320 queue.destination.priority = 1; 3321 queue.user_context = (u64)(uintptr_t)fq; 3322 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3323 DPNI_QUEUE_RX, fq->tc, fq->flowid, 3324 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 3325 &queue); 3326 if (err) { 3327 dev_err(dev, "dpni_set_queue(RX) failed\n"); 3328 return err; 3329 } 3330 3331 /* xdp_rxq setup */ 3332 /* only once for each channel */ 3333 if (fq->tc > 0) 3334 return 0; 3335 3336 err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 3337 fq->flowid, 0); 3338 if (err) { 3339 dev_err(dev, "xdp_rxq_info_reg failed\n"); 3340 return err; 3341 } 3342 3343 err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 3344 MEM_TYPE_PAGE_ORDER0, NULL); 3345 if (err) { 3346 dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 3347 return err; 3348 } 3349 3350 return 0; 3351 } 3352 3353 static int dpaa2_eth_setup_tx_flow(struct dpaa2_eth_priv *priv, 3354 struct dpaa2_eth_fq *fq) 3355 { 3356 struct device *dev = priv->net_dev->dev.parent; 3357 struct dpni_queue queue; 3358 struct dpni_queue_id qid; 3359 int i, err; 3360 3361 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3362 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3363 DPNI_QUEUE_TX, i, fq->flowid, 3364 &queue, &qid); 3365 if (err) { 3366 dev_err(dev, "dpni_get_queue(TX) failed\n"); 3367 return err; 3368 } 3369 fq->tx_fqid[i] = qid.fqid; 3370 } 3371 3372 /* All Tx queues belonging to the same flowid have the same qdbin */ 3373 fq->tx_qdbin = qid.qdbin; 3374 3375 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3376 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 3377 &queue, &qid); 3378 if (err) { 3379 dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 3380 return err; 3381 } 3382 3383 fq->fqid = qid.fqid; 3384 3385 queue.destination.id = fq->channel->dpcon_id; 3386 queue.destination.type = DPNI_DEST_DPCON; 3387 queue.destination.priority = 0; 3388 queue.user_context = (u64)(uintptr_t)fq; 3389 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3390 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 3391 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 3392 &queue); 3393 if (err) { 3394 dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 3395 return err; 3396 } 3397 3398 return 0; 3399 } 3400 3401 static int setup_rx_err_flow(struct dpaa2_eth_priv *priv, 3402 struct dpaa2_eth_fq *fq) 3403 { 3404 struct device *dev = priv->net_dev->dev.parent; 3405 struct dpni_queue q = { { 0 } }; 3406 struct dpni_queue_id qid; 3407 u8 q_opt = DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST; 3408 int err; 3409 3410 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3411 DPNI_QUEUE_RX_ERR, 0, 0, &q, &qid); 3412 if (err) { 3413 dev_err(dev, "dpni_get_queue() failed (%d)\n", err); 3414 return err; 3415 } 3416 3417 fq->fqid = qid.fqid; 3418 3419 q.destination.id = fq->channel->dpcon_id; 3420 q.destination.type = DPNI_DEST_DPCON; 3421 q.destination.priority = 1; 3422 q.user_context = (u64)(uintptr_t)fq; 3423 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3424 DPNI_QUEUE_RX_ERR, 0, 0, q_opt, &q); 3425 if (err) { 3426 dev_err(dev, "dpni_set_queue() failed (%d)\n", err); 3427 return err; 3428 } 3429 3430 return 0; 3431 } 3432 3433 /* Supported header fields for Rx hash distribution key */ 3434 static const struct dpaa2_eth_dist_fields dist_fields[] = { 3435 { 3436 /* L2 header */ 3437 .rxnfc_field = RXH_L2DA, 3438 .cls_prot = NET_PROT_ETH, 3439 .cls_field = NH_FLD_ETH_DA, 3440 .id = DPAA2_ETH_DIST_ETHDST, 3441 .size = 6, 3442 }, { 3443 .cls_prot = NET_PROT_ETH, 3444 .cls_field = NH_FLD_ETH_SA, 3445 .id = DPAA2_ETH_DIST_ETHSRC, 3446 .size = 6, 3447 }, { 3448 /* This is the last ethertype field parsed: 3449 * depending on frame format, it can be the MAC ethertype 3450 * or the VLAN etype. 3451 */ 3452 .cls_prot = NET_PROT_ETH, 3453 .cls_field = NH_FLD_ETH_TYPE, 3454 .id = DPAA2_ETH_DIST_ETHTYPE, 3455 .size = 2, 3456 }, { 3457 /* VLAN header */ 3458 .rxnfc_field = RXH_VLAN, 3459 .cls_prot = NET_PROT_VLAN, 3460 .cls_field = NH_FLD_VLAN_TCI, 3461 .id = DPAA2_ETH_DIST_VLAN, 3462 .size = 2, 3463 }, { 3464 /* IP header */ 3465 .rxnfc_field = RXH_IP_SRC, 3466 .cls_prot = NET_PROT_IP, 3467 .cls_field = NH_FLD_IP_SRC, 3468 .id = DPAA2_ETH_DIST_IPSRC, 3469 .size = 4, 3470 }, { 3471 .rxnfc_field = RXH_IP_DST, 3472 .cls_prot = NET_PROT_IP, 3473 .cls_field = NH_FLD_IP_DST, 3474 .id = DPAA2_ETH_DIST_IPDST, 3475 .size = 4, 3476 }, { 3477 .rxnfc_field = RXH_L3_PROTO, 3478 .cls_prot = NET_PROT_IP, 3479 .cls_field = NH_FLD_IP_PROTO, 3480 .id = DPAA2_ETH_DIST_IPPROTO, 3481 .size = 1, 3482 }, { 3483 /* Using UDP ports, this is functionally equivalent to raw 3484 * byte pairs from L4 header. 3485 */ 3486 .rxnfc_field = RXH_L4_B_0_1, 3487 .cls_prot = NET_PROT_UDP, 3488 .cls_field = NH_FLD_UDP_PORT_SRC, 3489 .id = DPAA2_ETH_DIST_L4SRC, 3490 .size = 2, 3491 }, { 3492 .rxnfc_field = RXH_L4_B_2_3, 3493 .cls_prot = NET_PROT_UDP, 3494 .cls_field = NH_FLD_UDP_PORT_DST, 3495 .id = DPAA2_ETH_DIST_L4DST, 3496 .size = 2, 3497 }, 3498 }; 3499 3500 /* Configure the Rx hash key using the legacy API */ 3501 static int dpaa2_eth_config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3502 { 3503 struct device *dev = priv->net_dev->dev.parent; 3504 struct dpni_rx_tc_dist_cfg dist_cfg; 3505 int i, err = 0; 3506 3507 memset(&dist_cfg, 0, sizeof(dist_cfg)); 3508 3509 dist_cfg.key_cfg_iova = key; 3510 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3511 dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 3512 3513 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3514 err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 3515 i, &dist_cfg); 3516 if (err) { 3517 dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 3518 break; 3519 } 3520 } 3521 3522 return err; 3523 } 3524 3525 /* Configure the Rx hash key using the new API */ 3526 static int dpaa2_eth_config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3527 { 3528 struct device *dev = priv->net_dev->dev.parent; 3529 struct dpni_rx_dist_cfg dist_cfg; 3530 int i, err = 0; 3531 3532 memset(&dist_cfg, 0, sizeof(dist_cfg)); 3533 3534 dist_cfg.key_cfg_iova = key; 3535 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3536 dist_cfg.enable = 1; 3537 3538 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3539 dist_cfg.tc = i; 3540 err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, 3541 &dist_cfg); 3542 if (err) { 3543 dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 3544 break; 3545 } 3546 3547 /* If the flow steering / hashing key is shared between all 3548 * traffic classes, install it just once 3549 */ 3550 if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 3551 break; 3552 } 3553 3554 return err; 3555 } 3556 3557 /* Configure the Rx flow classification key */ 3558 static int dpaa2_eth_config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3559 { 3560 struct device *dev = priv->net_dev->dev.parent; 3561 struct dpni_rx_dist_cfg dist_cfg; 3562 int i, err = 0; 3563 3564 memset(&dist_cfg, 0, sizeof(dist_cfg)); 3565 3566 dist_cfg.key_cfg_iova = key; 3567 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3568 dist_cfg.enable = 1; 3569 3570 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3571 dist_cfg.tc = i; 3572 err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, 3573 &dist_cfg); 3574 if (err) { 3575 dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 3576 break; 3577 } 3578 3579 /* If the flow steering / hashing key is shared between all 3580 * traffic classes, install it just once 3581 */ 3582 if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 3583 break; 3584 } 3585 3586 return err; 3587 } 3588 3589 /* Size of the Rx flow classification key */ 3590 int dpaa2_eth_cls_key_size(u64 fields) 3591 { 3592 int i, size = 0; 3593 3594 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3595 if (!(fields & dist_fields[i].id)) 3596 continue; 3597 size += dist_fields[i].size; 3598 } 3599 3600 return size; 3601 } 3602 3603 /* Offset of header field in Rx classification key */ 3604 int dpaa2_eth_cls_fld_off(int prot, int field) 3605 { 3606 int i, off = 0; 3607 3608 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3609 if (dist_fields[i].cls_prot == prot && 3610 dist_fields[i].cls_field == field) 3611 return off; 3612 off += dist_fields[i].size; 3613 } 3614 3615 WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 3616 return 0; 3617 } 3618 3619 /* Prune unused fields from the classification rule. 3620 * Used when masking is not supported 3621 */ 3622 void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 3623 { 3624 int off = 0, new_off = 0; 3625 int i, size; 3626 3627 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3628 size = dist_fields[i].size; 3629 if (dist_fields[i].id & fields) { 3630 memcpy(key_mem + new_off, key_mem + off, size); 3631 new_off += size; 3632 } 3633 off += size; 3634 } 3635 } 3636 3637 /* Set Rx distribution (hash or flow classification) key 3638 * flags is a combination of RXH_ bits 3639 */ 3640 static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 3641 enum dpaa2_eth_rx_dist type, u64 flags) 3642 { 3643 struct device *dev = net_dev->dev.parent; 3644 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3645 struct dpkg_profile_cfg cls_cfg; 3646 u32 rx_hash_fields = 0; 3647 dma_addr_t key_iova; 3648 u8 *dma_mem; 3649 int i; 3650 int err = 0; 3651 3652 memset(&cls_cfg, 0, sizeof(cls_cfg)); 3653 3654 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3655 struct dpkg_extract *key = 3656 &cls_cfg.extracts[cls_cfg.num_extracts]; 3657 3658 /* For both Rx hashing and classification keys 3659 * we set only the selected fields. 3660 */ 3661 if (!(flags & dist_fields[i].id)) 3662 continue; 3663 if (type == DPAA2_ETH_RX_DIST_HASH) 3664 rx_hash_fields |= dist_fields[i].rxnfc_field; 3665 3666 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 3667 dev_err(dev, "error adding key extraction rule, too many rules?\n"); 3668 return -E2BIG; 3669 } 3670 3671 key->type = DPKG_EXTRACT_FROM_HDR; 3672 key->extract.from_hdr.prot = dist_fields[i].cls_prot; 3673 key->extract.from_hdr.type = DPKG_FULL_FIELD; 3674 key->extract.from_hdr.field = dist_fields[i].cls_field; 3675 cls_cfg.num_extracts++; 3676 } 3677 3678 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 3679 if (!dma_mem) 3680 return -ENOMEM; 3681 3682 err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 3683 if (err) { 3684 dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 3685 goto free_key; 3686 } 3687 3688 /* Prepare for setting the rx dist */ 3689 key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 3690 DMA_TO_DEVICE); 3691 if (dma_mapping_error(dev, key_iova)) { 3692 dev_err(dev, "DMA mapping failed\n"); 3693 err = -ENOMEM; 3694 goto free_key; 3695 } 3696 3697 if (type == DPAA2_ETH_RX_DIST_HASH) { 3698 if (dpaa2_eth_has_legacy_dist(priv)) 3699 err = dpaa2_eth_config_legacy_hash_key(priv, key_iova); 3700 else 3701 err = dpaa2_eth_config_hash_key(priv, key_iova); 3702 } else { 3703 err = dpaa2_eth_config_cls_key(priv, key_iova); 3704 } 3705 3706 dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3707 DMA_TO_DEVICE); 3708 if (!err && type == DPAA2_ETH_RX_DIST_HASH) 3709 priv->rx_hash_fields = rx_hash_fields; 3710 3711 free_key: 3712 kfree(dma_mem); 3713 return err; 3714 } 3715 3716 int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 3717 { 3718 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3719 u64 key = 0; 3720 int i; 3721 3722 if (!dpaa2_eth_hash_enabled(priv)) 3723 return -EOPNOTSUPP; 3724 3725 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 3726 if (dist_fields[i].rxnfc_field & flags) 3727 key |= dist_fields[i].id; 3728 3729 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 3730 } 3731 3732 int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 3733 { 3734 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 3735 } 3736 3737 static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 3738 { 3739 struct device *dev = priv->net_dev->dev.parent; 3740 int err; 3741 3742 /* Check if we actually support Rx flow classification */ 3743 if (dpaa2_eth_has_legacy_dist(priv)) { 3744 dev_dbg(dev, "Rx cls not supported by current MC version\n"); 3745 return -EOPNOTSUPP; 3746 } 3747 3748 if (!dpaa2_eth_fs_enabled(priv)) { 3749 dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 3750 return -EOPNOTSUPP; 3751 } 3752 3753 if (!dpaa2_eth_hash_enabled(priv)) { 3754 dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 3755 return -EOPNOTSUPP; 3756 } 3757 3758 /* If there is no support for masking in the classification table, 3759 * we don't set a default key, as it will depend on the rules 3760 * added by the user at runtime. 3761 */ 3762 if (!dpaa2_eth_fs_mask_enabled(priv)) 3763 goto out; 3764 3765 err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 3766 if (err) 3767 return err; 3768 3769 out: 3770 priv->rx_cls_enabled = 1; 3771 3772 return 0; 3773 } 3774 3775 /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 3776 * frame queues and channels 3777 */ 3778 static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv) 3779 { 3780 struct net_device *net_dev = priv->net_dev; 3781 struct device *dev = net_dev->dev.parent; 3782 struct dpni_pools_cfg pools_params; 3783 struct dpni_error_cfg err_cfg; 3784 int err = 0; 3785 int i; 3786 3787 pools_params.num_dpbp = 1; 3788 pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 3789 pools_params.pools[0].backup_pool = 0; 3790 pools_params.pools[0].buffer_size = priv->rx_buf_size; 3791 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 3792 if (err) { 3793 dev_err(dev, "dpni_set_pools() failed\n"); 3794 return err; 3795 } 3796 3797 /* have the interface implicitly distribute traffic based on 3798 * the default hash key 3799 */ 3800 err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 3801 if (err && err != -EOPNOTSUPP) 3802 dev_err(dev, "Failed to configure hashing\n"); 3803 3804 /* Configure the flow classification key; it includes all 3805 * supported header fields and cannot be modified at runtime 3806 */ 3807 err = dpaa2_eth_set_default_cls(priv); 3808 if (err && err != -EOPNOTSUPP) 3809 dev_err(dev, "Failed to configure Rx classification key\n"); 3810 3811 /* Configure handling of error frames */ 3812 err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 3813 err_cfg.set_frame_annotation = 1; 3814 err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 3815 err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 3816 &err_cfg); 3817 if (err) { 3818 dev_err(dev, "dpni_set_errors_behavior failed\n"); 3819 return err; 3820 } 3821 3822 /* Configure Rx and Tx conf queues to generate CDANs */ 3823 for (i = 0; i < priv->num_fqs; i++) { 3824 switch (priv->fq[i].type) { 3825 case DPAA2_RX_FQ: 3826 err = dpaa2_eth_setup_rx_flow(priv, &priv->fq[i]); 3827 break; 3828 case DPAA2_TX_CONF_FQ: 3829 err = dpaa2_eth_setup_tx_flow(priv, &priv->fq[i]); 3830 break; 3831 case DPAA2_RX_ERR_FQ: 3832 err = setup_rx_err_flow(priv, &priv->fq[i]); 3833 break; 3834 default: 3835 dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 3836 return -EINVAL; 3837 } 3838 if (err) 3839 return err; 3840 } 3841 3842 err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 3843 DPNI_QUEUE_TX, &priv->tx_qdid); 3844 if (err) { 3845 dev_err(dev, "dpni_get_qdid() failed\n"); 3846 return err; 3847 } 3848 3849 return 0; 3850 } 3851 3852 /* Allocate rings for storing incoming frame descriptors */ 3853 static int dpaa2_eth_alloc_rings(struct dpaa2_eth_priv *priv) 3854 { 3855 struct net_device *net_dev = priv->net_dev; 3856 struct device *dev = net_dev->dev.parent; 3857 int i; 3858 3859 for (i = 0; i < priv->num_channels; i++) { 3860 priv->channel[i]->store = 3861 dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 3862 if (!priv->channel[i]->store) { 3863 netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 3864 goto err_ring; 3865 } 3866 } 3867 3868 return 0; 3869 3870 err_ring: 3871 for (i = 0; i < priv->num_channels; i++) { 3872 if (!priv->channel[i]->store) 3873 break; 3874 dpaa2_io_store_destroy(priv->channel[i]->store); 3875 } 3876 3877 return -ENOMEM; 3878 } 3879 3880 static void dpaa2_eth_free_rings(struct dpaa2_eth_priv *priv) 3881 { 3882 int i; 3883 3884 for (i = 0; i < priv->num_channels; i++) 3885 dpaa2_io_store_destroy(priv->channel[i]->store); 3886 } 3887 3888 static int dpaa2_eth_set_mac_addr(struct dpaa2_eth_priv *priv) 3889 { 3890 struct net_device *net_dev = priv->net_dev; 3891 struct device *dev = net_dev->dev.parent; 3892 u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 3893 int err; 3894 3895 /* Get firmware address, if any */ 3896 err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 3897 if (err) { 3898 dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 3899 return err; 3900 } 3901 3902 /* Get DPNI attributes address, if any */ 3903 err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 3904 dpni_mac_addr); 3905 if (err) { 3906 dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 3907 return err; 3908 } 3909 3910 /* First check if firmware has any address configured by bootloader */ 3911 if (!is_zero_ether_addr(mac_addr)) { 3912 /* If the DPMAC addr != DPNI addr, update it */ 3913 if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 3914 err = dpni_set_primary_mac_addr(priv->mc_io, 0, 3915 priv->mc_token, 3916 mac_addr); 3917 if (err) { 3918 dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 3919 return err; 3920 } 3921 } 3922 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); 3923 } else if (is_zero_ether_addr(dpni_mac_addr)) { 3924 /* No MAC address configured, fill in net_dev->dev_addr 3925 * with a random one 3926 */ 3927 eth_hw_addr_random(net_dev); 3928 dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 3929 3930 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 3931 net_dev->dev_addr); 3932 if (err) { 3933 dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 3934 return err; 3935 } 3936 3937 /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 3938 * practical purposes, this will be our "permanent" mac address, 3939 * at least until the next reboot. This move will also permit 3940 * register_netdevice() to properly fill up net_dev->perm_addr. 3941 */ 3942 net_dev->addr_assign_type = NET_ADDR_PERM; 3943 } else { 3944 /* NET_ADDR_PERM is default, all we have to do is 3945 * fill in the device addr. 3946 */ 3947 memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len); 3948 } 3949 3950 return 0; 3951 } 3952 3953 static int dpaa2_eth_netdev_init(struct net_device *net_dev) 3954 { 3955 struct device *dev = net_dev->dev.parent; 3956 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3957 u32 options = priv->dpni_attrs.options; 3958 u64 supported = 0, not_supported = 0; 3959 u8 bcast_addr[ETH_ALEN]; 3960 u8 num_queues; 3961 int err; 3962 3963 net_dev->netdev_ops = &dpaa2_eth_ops; 3964 net_dev->ethtool_ops = &dpaa2_ethtool_ops; 3965 3966 err = dpaa2_eth_set_mac_addr(priv); 3967 if (err) 3968 return err; 3969 3970 /* Explicitly add the broadcast address to the MAC filtering table */ 3971 eth_broadcast_addr(bcast_addr); 3972 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 3973 if (err) { 3974 dev_err(dev, "dpni_add_mac_addr() failed\n"); 3975 return err; 3976 } 3977 3978 /* Set MTU upper limit; lower limit is 68B (default value) */ 3979 net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 3980 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 3981 DPAA2_ETH_MFL); 3982 if (err) { 3983 dev_err(dev, "dpni_set_max_frame_length() failed\n"); 3984 return err; 3985 } 3986 3987 /* Set actual number of queues in the net device */ 3988 num_queues = dpaa2_eth_queue_count(priv); 3989 err = netif_set_real_num_tx_queues(net_dev, num_queues); 3990 if (err) { 3991 dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 3992 return err; 3993 } 3994 err = netif_set_real_num_rx_queues(net_dev, num_queues); 3995 if (err) { 3996 dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 3997 return err; 3998 } 3999 4000 /* Capabilities listing */ 4001 supported |= IFF_LIVE_ADDR_CHANGE; 4002 4003 if (options & DPNI_OPT_NO_MAC_FILTER) 4004 not_supported |= IFF_UNICAST_FLT; 4005 else 4006 supported |= IFF_UNICAST_FLT; 4007 4008 net_dev->priv_flags |= supported; 4009 net_dev->priv_flags &= ~not_supported; 4010 4011 /* Features */ 4012 net_dev->features = NETIF_F_RXCSUM | 4013 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4014 NETIF_F_SG | NETIF_F_HIGHDMA | 4015 NETIF_F_LLTX | NETIF_F_HW_TC; 4016 net_dev->hw_features = net_dev->features; 4017 4018 return 0; 4019 } 4020 4021 static int dpaa2_eth_poll_link_state(void *arg) 4022 { 4023 struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 4024 int err; 4025 4026 while (!kthread_should_stop()) { 4027 err = dpaa2_eth_link_state_update(priv); 4028 if (unlikely(err)) 4029 return err; 4030 4031 msleep(DPAA2_ETH_LINK_STATE_REFRESH); 4032 } 4033 4034 return 0; 4035 } 4036 4037 static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) 4038 { 4039 struct fsl_mc_device *dpni_dev, *dpmac_dev; 4040 struct dpaa2_mac *mac; 4041 int err; 4042 4043 dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); 4044 dpmac_dev = fsl_mc_get_endpoint(dpni_dev); 4045 if (IS_ERR_OR_NULL(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) 4046 return 0; 4047 4048 mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); 4049 if (!mac) 4050 return -ENOMEM; 4051 4052 mac->mc_dev = dpmac_dev; 4053 mac->mc_io = priv->mc_io; 4054 mac->net_dev = priv->net_dev; 4055 4056 err = dpaa2_mac_open(mac); 4057 if (err) 4058 goto err_free_mac; 4059 priv->mac = mac; 4060 4061 if (dpaa2_eth_is_type_phy(priv)) { 4062 err = dpaa2_mac_connect(mac); 4063 if (err) { 4064 netdev_err(priv->net_dev, "Error connecting to the MAC endpoint\n"); 4065 goto err_close_mac; 4066 } 4067 } 4068 4069 return 0; 4070 4071 err_close_mac: 4072 dpaa2_mac_close(mac); 4073 priv->mac = NULL; 4074 err_free_mac: 4075 kfree(mac); 4076 return err; 4077 } 4078 4079 static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) 4080 { 4081 if (dpaa2_eth_is_type_phy(priv)) 4082 dpaa2_mac_disconnect(priv->mac); 4083 4084 dpaa2_mac_close(priv->mac); 4085 kfree(priv->mac); 4086 priv->mac = NULL; 4087 } 4088 4089 static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 4090 { 4091 u32 status = ~0; 4092 struct device *dev = (struct device *)arg; 4093 struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 4094 struct net_device *net_dev = dev_get_drvdata(dev); 4095 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 4096 int err; 4097 4098 err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 4099 DPNI_IRQ_INDEX, &status); 4100 if (unlikely(err)) { 4101 netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 4102 return IRQ_HANDLED; 4103 } 4104 4105 if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 4106 dpaa2_eth_link_state_update(netdev_priv(net_dev)); 4107 4108 if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) { 4109 dpaa2_eth_set_mac_addr(netdev_priv(net_dev)); 4110 dpaa2_eth_update_tx_fqids(priv); 4111 4112 rtnl_lock(); 4113 if (dpaa2_eth_has_mac(priv)) 4114 dpaa2_eth_disconnect_mac(priv); 4115 else 4116 dpaa2_eth_connect_mac(priv); 4117 rtnl_unlock(); 4118 } 4119 4120 return IRQ_HANDLED; 4121 } 4122 4123 static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) 4124 { 4125 int err = 0; 4126 struct fsl_mc_device_irq *irq; 4127 4128 err = fsl_mc_allocate_irqs(ls_dev); 4129 if (err) { 4130 dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 4131 return err; 4132 } 4133 4134 irq = ls_dev->irqs[0]; 4135 err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, 4136 NULL, dpni_irq0_handler_thread, 4137 IRQF_NO_SUSPEND | IRQF_ONESHOT, 4138 dev_name(&ls_dev->dev), &ls_dev->dev); 4139 if (err < 0) { 4140 dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 4141 goto free_mc_irq; 4142 } 4143 4144 err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 4145 DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 4146 DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 4147 if (err < 0) { 4148 dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 4149 goto free_irq; 4150 } 4151 4152 err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 4153 DPNI_IRQ_INDEX, 1); 4154 if (err < 0) { 4155 dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 4156 goto free_irq; 4157 } 4158 4159 return 0; 4160 4161 free_irq: 4162 devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); 4163 free_mc_irq: 4164 fsl_mc_free_irqs(ls_dev); 4165 4166 return err; 4167 } 4168 4169 static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv) 4170 { 4171 int i; 4172 struct dpaa2_eth_channel *ch; 4173 4174 for (i = 0; i < priv->num_channels; i++) { 4175 ch = priv->channel[i]; 4176 /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 4177 netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 4178 NAPI_POLL_WEIGHT); 4179 } 4180 } 4181 4182 static void dpaa2_eth_del_ch_napi(struct dpaa2_eth_priv *priv) 4183 { 4184 int i; 4185 struct dpaa2_eth_channel *ch; 4186 4187 for (i = 0; i < priv->num_channels; i++) { 4188 ch = priv->channel[i]; 4189 netif_napi_del(&ch->napi); 4190 } 4191 } 4192 4193 static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 4194 { 4195 struct device *dev; 4196 struct net_device *net_dev = NULL; 4197 struct dpaa2_eth_priv *priv = NULL; 4198 int err = 0; 4199 4200 dev = &dpni_dev->dev; 4201 4202 /* Net device */ 4203 net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 4204 if (!net_dev) { 4205 dev_err(dev, "alloc_etherdev_mq() failed\n"); 4206 return -ENOMEM; 4207 } 4208 4209 SET_NETDEV_DEV(net_dev, dev); 4210 dev_set_drvdata(dev, net_dev); 4211 4212 priv = netdev_priv(net_dev); 4213 priv->net_dev = net_dev; 4214 4215 priv->iommu_domain = iommu_get_domain_for_dev(dev); 4216 4217 priv->tx_tstamp_type = HWTSTAMP_TX_OFF; 4218 priv->rx_tstamp = false; 4219 4220 priv->dpaa2_ptp_wq = alloc_workqueue("dpaa2_ptp_wq", 0, 0); 4221 if (!priv->dpaa2_ptp_wq) { 4222 err = -ENOMEM; 4223 goto err_wq_alloc; 4224 } 4225 4226 INIT_WORK(&priv->tx_onestep_tstamp, dpaa2_eth_tx_onestep_tstamp); 4227 4228 skb_queue_head_init(&priv->tx_skbs); 4229 4230 /* Obtain a MC portal */ 4231 err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 4232 &priv->mc_io); 4233 if (err) { 4234 if (err == -ENXIO) 4235 err = -EPROBE_DEFER; 4236 else 4237 dev_err(dev, "MC portal allocation failed\n"); 4238 goto err_portal_alloc; 4239 } 4240 4241 /* MC objects initialization and configuration */ 4242 err = dpaa2_eth_setup_dpni(dpni_dev); 4243 if (err) 4244 goto err_dpni_setup; 4245 4246 err = dpaa2_eth_setup_dpio(priv); 4247 if (err) 4248 goto err_dpio_setup; 4249 4250 dpaa2_eth_setup_fqs(priv); 4251 4252 err = dpaa2_eth_setup_dpbp(priv); 4253 if (err) 4254 goto err_dpbp_setup; 4255 4256 err = dpaa2_eth_bind_dpni(priv); 4257 if (err) 4258 goto err_bind; 4259 4260 /* Add a NAPI context for each channel */ 4261 dpaa2_eth_add_ch_napi(priv); 4262 4263 /* Percpu statistics */ 4264 priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 4265 if (!priv->percpu_stats) { 4266 dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 4267 err = -ENOMEM; 4268 goto err_alloc_percpu_stats; 4269 } 4270 priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 4271 if (!priv->percpu_extras) { 4272 dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 4273 err = -ENOMEM; 4274 goto err_alloc_percpu_extras; 4275 } 4276 4277 priv->sgt_cache = alloc_percpu(*priv->sgt_cache); 4278 if (!priv->sgt_cache) { 4279 dev_err(dev, "alloc_percpu(sgt_cache) failed\n"); 4280 err = -ENOMEM; 4281 goto err_alloc_sgt_cache; 4282 } 4283 4284 err = dpaa2_eth_netdev_init(net_dev); 4285 if (err) 4286 goto err_netdev_init; 4287 4288 /* Configure checksum offload based on current interface flags */ 4289 err = dpaa2_eth_set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 4290 if (err) 4291 goto err_csum; 4292 4293 err = dpaa2_eth_set_tx_csum(priv, 4294 !!(net_dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 4295 if (err) 4296 goto err_csum; 4297 4298 err = dpaa2_eth_alloc_rings(priv); 4299 if (err) 4300 goto err_alloc_rings; 4301 4302 #ifdef CONFIG_FSL_DPAA2_ETH_DCB 4303 if (dpaa2_eth_has_pause_support(priv) && priv->vlan_cls_enabled) { 4304 priv->dcbx_mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; 4305 net_dev->dcbnl_ops = &dpaa2_eth_dcbnl_ops; 4306 } else { 4307 dev_dbg(dev, "PFC not supported\n"); 4308 } 4309 #endif 4310 4311 err = dpaa2_eth_setup_irqs(dpni_dev); 4312 if (err) { 4313 netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 4314 priv->poll_thread = kthread_run(dpaa2_eth_poll_link_state, priv, 4315 "%s_poll_link", net_dev->name); 4316 if (IS_ERR(priv->poll_thread)) { 4317 dev_err(dev, "Error starting polling thread\n"); 4318 goto err_poll_thread; 4319 } 4320 priv->do_link_poll = true; 4321 } 4322 4323 err = dpaa2_eth_connect_mac(priv); 4324 if (err) 4325 goto err_connect_mac; 4326 4327 err = dpaa2_eth_dl_register(priv); 4328 if (err) 4329 goto err_dl_register; 4330 4331 err = dpaa2_eth_dl_traps_register(priv); 4332 if (err) 4333 goto err_dl_trap_register; 4334 4335 err = dpaa2_eth_dl_port_add(priv); 4336 if (err) 4337 goto err_dl_port_add; 4338 4339 err = register_netdev(net_dev); 4340 if (err < 0) { 4341 dev_err(dev, "register_netdev() failed\n"); 4342 goto err_netdev_reg; 4343 } 4344 4345 #ifdef CONFIG_DEBUG_FS 4346 dpaa2_dbg_add(priv); 4347 #endif 4348 4349 dev_info(dev, "Probed interface %s\n", net_dev->name); 4350 return 0; 4351 4352 err_netdev_reg: 4353 dpaa2_eth_dl_port_del(priv); 4354 err_dl_port_add: 4355 dpaa2_eth_dl_traps_unregister(priv); 4356 err_dl_trap_register: 4357 dpaa2_eth_dl_unregister(priv); 4358 err_dl_register: 4359 dpaa2_eth_disconnect_mac(priv); 4360 err_connect_mac: 4361 if (priv->do_link_poll) 4362 kthread_stop(priv->poll_thread); 4363 else 4364 fsl_mc_free_irqs(dpni_dev); 4365 err_poll_thread: 4366 dpaa2_eth_free_rings(priv); 4367 err_alloc_rings: 4368 err_csum: 4369 err_netdev_init: 4370 free_percpu(priv->sgt_cache); 4371 err_alloc_sgt_cache: 4372 free_percpu(priv->percpu_extras); 4373 err_alloc_percpu_extras: 4374 free_percpu(priv->percpu_stats); 4375 err_alloc_percpu_stats: 4376 dpaa2_eth_del_ch_napi(priv); 4377 err_bind: 4378 dpaa2_eth_free_dpbp(priv); 4379 err_dpbp_setup: 4380 dpaa2_eth_free_dpio(priv); 4381 err_dpio_setup: 4382 dpaa2_eth_free_dpni(priv); 4383 err_dpni_setup: 4384 fsl_mc_portal_free(priv->mc_io); 4385 err_portal_alloc: 4386 destroy_workqueue(priv->dpaa2_ptp_wq); 4387 err_wq_alloc: 4388 dev_set_drvdata(dev, NULL); 4389 free_netdev(net_dev); 4390 4391 return err; 4392 } 4393 4394 static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 4395 { 4396 struct device *dev; 4397 struct net_device *net_dev; 4398 struct dpaa2_eth_priv *priv; 4399 4400 dev = &ls_dev->dev; 4401 net_dev = dev_get_drvdata(dev); 4402 priv = netdev_priv(net_dev); 4403 4404 #ifdef CONFIG_DEBUG_FS 4405 dpaa2_dbg_remove(priv); 4406 #endif 4407 rtnl_lock(); 4408 dpaa2_eth_disconnect_mac(priv); 4409 rtnl_unlock(); 4410 4411 unregister_netdev(net_dev); 4412 4413 dpaa2_eth_dl_port_del(priv); 4414 dpaa2_eth_dl_traps_unregister(priv); 4415 dpaa2_eth_dl_unregister(priv); 4416 4417 if (priv->do_link_poll) 4418 kthread_stop(priv->poll_thread); 4419 else 4420 fsl_mc_free_irqs(ls_dev); 4421 4422 dpaa2_eth_free_rings(priv); 4423 free_percpu(priv->sgt_cache); 4424 free_percpu(priv->percpu_stats); 4425 free_percpu(priv->percpu_extras); 4426 4427 dpaa2_eth_del_ch_napi(priv); 4428 dpaa2_eth_free_dpbp(priv); 4429 dpaa2_eth_free_dpio(priv); 4430 dpaa2_eth_free_dpni(priv); 4431 4432 fsl_mc_portal_free(priv->mc_io); 4433 4434 free_netdev(net_dev); 4435 4436 dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 4437 4438 return 0; 4439 } 4440 4441 static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 4442 { 4443 .vendor = FSL_MC_VENDOR_FREESCALE, 4444 .obj_type = "dpni", 4445 }, 4446 { .vendor = 0x0 } 4447 }; 4448 MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 4449 4450 static struct fsl_mc_driver dpaa2_eth_driver = { 4451 .driver = { 4452 .name = KBUILD_MODNAME, 4453 .owner = THIS_MODULE, 4454 }, 4455 .probe = dpaa2_eth_probe, 4456 .remove = dpaa2_eth_remove, 4457 .match_id_table = dpaa2_eth_match_id_table 4458 }; 4459 4460 static int __init dpaa2_eth_driver_init(void) 4461 { 4462 int err; 4463 4464 dpaa2_eth_dbg_init(); 4465 err = fsl_mc_driver_register(&dpaa2_eth_driver); 4466 if (err) { 4467 dpaa2_eth_dbg_exit(); 4468 return err; 4469 } 4470 4471 return 0; 4472 } 4473 4474 static void __exit dpaa2_eth_driver_exit(void) 4475 { 4476 dpaa2_eth_dbg_exit(); 4477 fsl_mc_driver_unregister(&dpaa2_eth_driver); 4478 } 4479 4480 module_init(dpaa2_eth_driver_init); 4481 module_exit(dpaa2_eth_driver_exit); 4482