134ff6846SIoana Radulescu // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 234ff6846SIoana Radulescu /* Copyright 2014-2016 Freescale Semiconductor Inc. 348c0481eSIoana Ciornei * Copyright 2016-2020 NXP 434ff6846SIoana Radulescu */ 534ff6846SIoana Radulescu #include <linux/init.h> 634ff6846SIoana Radulescu #include <linux/module.h> 734ff6846SIoana Radulescu #include <linux/platform_device.h> 834ff6846SIoana Radulescu #include <linux/etherdevice.h> 934ff6846SIoana Radulescu #include <linux/of_net.h> 1034ff6846SIoana Radulescu #include <linux/interrupt.h> 1134ff6846SIoana Radulescu #include <linux/msi.h> 1234ff6846SIoana Radulescu #include <linux/kthread.h> 1334ff6846SIoana Radulescu #include <linux/iommu.h> 1434ff6846SIoana Radulescu #include <linux/fsl/mc.h> 157e273a8eSIoana Ciocoi Radulescu #include <linux/bpf.h> 167e273a8eSIoana Ciocoi Radulescu #include <linux/bpf_trace.h> 17d21c784cSYangbo Lu #include <linux/fsl/ptp_qoriq.h> 18c5521189SYangbo Lu #include <linux/ptp_classify.h> 193657cdafSIoana Ciornei #include <net/pkt_cls.h> 2034ff6846SIoana Radulescu #include <net/sock.h> 2134ff6846SIoana Radulescu 2234ff6846SIoana Radulescu #include "dpaa2-eth.h" 2334ff6846SIoana Radulescu 2434ff6846SIoana Radulescu /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 2534ff6846SIoana Radulescu * using trace events only need to #include <trace/events/sched.h> 2634ff6846SIoana Radulescu */ 2734ff6846SIoana Radulescu #define CREATE_TRACE_POINTS 2834ff6846SIoana Radulescu #include "dpaa2-eth-trace.h" 2934ff6846SIoana Radulescu 3034ff6846SIoana Radulescu MODULE_LICENSE("Dual BSD/GPL"); 3134ff6846SIoana Radulescu MODULE_AUTHOR("Freescale Semiconductor, Inc"); 3234ff6846SIoana Radulescu MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 3334ff6846SIoana Radulescu 34d21c784cSYangbo Lu struct ptp_qoriq *dpaa2_ptp; 35d21c784cSYangbo Lu EXPORT_SYMBOL(dpaa2_ptp); 36d21c784cSYangbo Lu 3734ff6846SIoana Radulescu static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 3834ff6846SIoana Radulescu dma_addr_t iova_addr) 3934ff6846SIoana Radulescu { 4034ff6846SIoana Radulescu phys_addr_t phys_addr; 4134ff6846SIoana Radulescu 4234ff6846SIoana Radulescu phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 4334ff6846SIoana Radulescu 4434ff6846SIoana Radulescu return phys_to_virt(phys_addr); 4534ff6846SIoana Radulescu } 4634ff6846SIoana Radulescu 475d8dccf8SIoana Ciornei static void dpaa2_eth_validate_rx_csum(struct dpaa2_eth_priv *priv, 4834ff6846SIoana Radulescu u32 fd_status, 4934ff6846SIoana Radulescu struct sk_buff *skb) 5034ff6846SIoana Radulescu { 5134ff6846SIoana Radulescu skb_checksum_none_assert(skb); 5234ff6846SIoana Radulescu 5334ff6846SIoana Radulescu /* HW checksum validation is disabled, nothing to do here */ 5434ff6846SIoana Radulescu if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 5534ff6846SIoana Radulescu return; 5634ff6846SIoana Radulescu 5734ff6846SIoana Radulescu /* Read checksum validation bits */ 5834ff6846SIoana Radulescu if (!((fd_status & DPAA2_FAS_L3CV) && 5934ff6846SIoana Radulescu (fd_status & DPAA2_FAS_L4CV))) 6034ff6846SIoana Radulescu return; 6134ff6846SIoana Radulescu 6234ff6846SIoana Radulescu /* Inform the stack there's no need to compute L3/L4 csum anymore */ 6334ff6846SIoana Radulescu skb->ip_summed = CHECKSUM_UNNECESSARY; 6434ff6846SIoana Radulescu } 6534ff6846SIoana Radulescu 6634ff6846SIoana Radulescu /* Free a received FD. 6734ff6846SIoana Radulescu * Not to be used for Tx conf FDs or on any other paths. 6834ff6846SIoana Radulescu */ 695d8dccf8SIoana Ciornei static void dpaa2_eth_free_rx_fd(struct dpaa2_eth_priv *priv, 7034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 7134ff6846SIoana Radulescu void *vaddr) 7234ff6846SIoana Radulescu { 7334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 7434ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 7534ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 7634ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 7734ff6846SIoana Radulescu void *sg_vaddr; 7834ff6846SIoana Radulescu int i; 7934ff6846SIoana Radulescu 8034ff6846SIoana Radulescu /* If single buffer frame, just free the data buffer */ 8134ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) 8234ff6846SIoana Radulescu goto free_buf; 8334ff6846SIoana Radulescu else if (fd_format != dpaa2_fd_sg) 8434ff6846SIoana Radulescu /* We don't support any other format */ 8534ff6846SIoana Radulescu return; 8634ff6846SIoana Radulescu 8734ff6846SIoana Radulescu /* For S/G frames, we first need to free all SG entries 8834ff6846SIoana Radulescu * except the first one, which was taken care of already 8934ff6846SIoana Radulescu */ 9034ff6846SIoana Radulescu sgt = vaddr + dpaa2_fd_get_offset(fd); 9134ff6846SIoana Radulescu for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 9234ff6846SIoana Radulescu addr = dpaa2_sg_get_addr(&sgt[i]); 9334ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 94efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 9518c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 9634ff6846SIoana Radulescu 9727c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 9834ff6846SIoana Radulescu if (dpaa2_sg_is_final(&sgt[i])) 9934ff6846SIoana Radulescu break; 10034ff6846SIoana Radulescu } 10134ff6846SIoana Radulescu 10234ff6846SIoana Radulescu free_buf: 10327c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 10434ff6846SIoana Radulescu } 10534ff6846SIoana Radulescu 10634ff6846SIoana Radulescu /* Build a linear skb based on a single-buffer frame descriptor */ 1075d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_linear_skb(struct dpaa2_eth_channel *ch, 10834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 10934ff6846SIoana Radulescu void *fd_vaddr) 11034ff6846SIoana Radulescu { 11134ff6846SIoana Radulescu struct sk_buff *skb = NULL; 11234ff6846SIoana Radulescu u16 fd_offset = dpaa2_fd_get_offset(fd); 11334ff6846SIoana Radulescu u32 fd_length = dpaa2_fd_get_len(fd); 11434ff6846SIoana Radulescu 11534ff6846SIoana Radulescu ch->buf_count--; 11634ff6846SIoana Radulescu 11727c87486SIoana Ciocoi Radulescu skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 11834ff6846SIoana Radulescu if (unlikely(!skb)) 11934ff6846SIoana Radulescu return NULL; 12034ff6846SIoana Radulescu 12134ff6846SIoana Radulescu skb_reserve(skb, fd_offset); 12234ff6846SIoana Radulescu skb_put(skb, fd_length); 12334ff6846SIoana Radulescu 12434ff6846SIoana Radulescu return skb; 12534ff6846SIoana Radulescu } 12634ff6846SIoana Radulescu 12734ff6846SIoana Radulescu /* Build a non linear (fragmented) skb based on a S/G table */ 1285d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_frag_skb(struct dpaa2_eth_priv *priv, 12934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 13034ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt) 13134ff6846SIoana Radulescu { 13234ff6846SIoana Radulescu struct sk_buff *skb = NULL; 13334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 13434ff6846SIoana Radulescu void *sg_vaddr; 13534ff6846SIoana Radulescu dma_addr_t sg_addr; 13634ff6846SIoana Radulescu u16 sg_offset; 13734ff6846SIoana Radulescu u32 sg_length; 13834ff6846SIoana Radulescu struct page *page, *head_page; 13934ff6846SIoana Radulescu int page_offset; 14034ff6846SIoana Radulescu int i; 14134ff6846SIoana Radulescu 14234ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 14334ff6846SIoana Radulescu struct dpaa2_sg_entry *sge = &sgt[i]; 14434ff6846SIoana Radulescu 14534ff6846SIoana Radulescu /* NOTE: We only support SG entries in dpaa2_sg_single format, 14634ff6846SIoana Radulescu * but this is the only format we may receive from HW anyway 14734ff6846SIoana Radulescu */ 14834ff6846SIoana Radulescu 14934ff6846SIoana Radulescu /* Get the address and length from the S/G entry */ 15034ff6846SIoana Radulescu sg_addr = dpaa2_sg_get_addr(sge); 15134ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 152efa6a7d0SIoana Ciornei dma_unmap_page(dev, sg_addr, priv->rx_buf_size, 15318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 15434ff6846SIoana Radulescu 15534ff6846SIoana Radulescu sg_length = dpaa2_sg_get_len(sge); 15634ff6846SIoana Radulescu 15734ff6846SIoana Radulescu if (i == 0) { 15834ff6846SIoana Radulescu /* We build the skb around the first data buffer */ 15927c87486SIoana Ciocoi Radulescu skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 16034ff6846SIoana Radulescu if (unlikely(!skb)) { 16134ff6846SIoana Radulescu /* Free the first SG entry now, since we already 16234ff6846SIoana Radulescu * unmapped it and obtained the virtual address 16334ff6846SIoana Radulescu */ 16427c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 16534ff6846SIoana Radulescu 16634ff6846SIoana Radulescu /* We still need to subtract the buffers used 16734ff6846SIoana Radulescu * by this FD from our software counter 16834ff6846SIoana Radulescu */ 16934ff6846SIoana Radulescu while (!dpaa2_sg_is_final(&sgt[i]) && 17034ff6846SIoana Radulescu i < DPAA2_ETH_MAX_SG_ENTRIES) 17134ff6846SIoana Radulescu i++; 17234ff6846SIoana Radulescu break; 17334ff6846SIoana Radulescu } 17434ff6846SIoana Radulescu 17534ff6846SIoana Radulescu sg_offset = dpaa2_sg_get_offset(sge); 17634ff6846SIoana Radulescu skb_reserve(skb, sg_offset); 17734ff6846SIoana Radulescu skb_put(skb, sg_length); 17834ff6846SIoana Radulescu } else { 17934ff6846SIoana Radulescu /* Rest of the data buffers are stored as skb frags */ 18034ff6846SIoana Radulescu page = virt_to_page(sg_vaddr); 18134ff6846SIoana Radulescu head_page = virt_to_head_page(sg_vaddr); 18234ff6846SIoana Radulescu 18334ff6846SIoana Radulescu /* Offset in page (which may be compound). 18434ff6846SIoana Radulescu * Data in subsequent SG entries is stored from the 18534ff6846SIoana Radulescu * beginning of the buffer, so we don't need to add the 18634ff6846SIoana Radulescu * sg_offset. 18734ff6846SIoana Radulescu */ 18834ff6846SIoana Radulescu page_offset = ((unsigned long)sg_vaddr & 18934ff6846SIoana Radulescu (PAGE_SIZE - 1)) + 19034ff6846SIoana Radulescu (page_address(page) - page_address(head_page)); 19134ff6846SIoana Radulescu 19234ff6846SIoana Radulescu skb_add_rx_frag(skb, i - 1, head_page, page_offset, 193efa6a7d0SIoana Ciornei sg_length, priv->rx_buf_size); 19434ff6846SIoana Radulescu } 19534ff6846SIoana Radulescu 19634ff6846SIoana Radulescu if (dpaa2_sg_is_final(sge)) 19734ff6846SIoana Radulescu break; 19834ff6846SIoana Radulescu } 19934ff6846SIoana Radulescu 20034ff6846SIoana Radulescu WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 20134ff6846SIoana Radulescu 20234ff6846SIoana Radulescu /* Count all data buffers + SG table buffer */ 20334ff6846SIoana Radulescu ch->buf_count -= i + 2; 20434ff6846SIoana Radulescu 20534ff6846SIoana Radulescu return skb; 20634ff6846SIoana Radulescu } 20734ff6846SIoana Radulescu 208569375fbSIoana Ciocoi Radulescu /* Free buffers acquired from the buffer pool or which were meant to 209569375fbSIoana Ciocoi Radulescu * be released in the pool 210569375fbSIoana Ciocoi Radulescu */ 2115d8dccf8SIoana Ciornei static void dpaa2_eth_free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, 2125d8dccf8SIoana Ciornei int count) 213569375fbSIoana Ciocoi Radulescu { 214569375fbSIoana Ciocoi Radulescu struct device *dev = priv->net_dev->dev.parent; 215569375fbSIoana Ciocoi Radulescu void *vaddr; 216569375fbSIoana Ciocoi Radulescu int i; 217569375fbSIoana Ciocoi Radulescu 218569375fbSIoana Ciocoi Radulescu for (i = 0; i < count; i++) { 219569375fbSIoana Ciocoi Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 220efa6a7d0SIoana Ciornei dma_unmap_page(dev, buf_array[i], priv->rx_buf_size, 22118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 22227c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 223569375fbSIoana Ciocoi Radulescu } 224569375fbSIoana Ciocoi Radulescu } 225569375fbSIoana Ciocoi Radulescu 2265d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_release_buf(struct dpaa2_eth_priv *priv, 2275d39dc21SIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 2285d39dc21SIoana Ciocoi Radulescu dma_addr_t addr) 2295d39dc21SIoana Ciocoi Radulescu { 230ef17bd7cSIoana Radulescu int retries = 0; 2315d39dc21SIoana Ciocoi Radulescu int err; 2325d39dc21SIoana Ciocoi Radulescu 2335d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr; 2345d39dc21SIoana Ciocoi Radulescu if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD) 2355d39dc21SIoana Ciocoi Radulescu return; 2365d39dc21SIoana Ciocoi Radulescu 2375d39dc21SIoana Ciocoi Radulescu while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 2385d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_bufs, 239ef17bd7cSIoana Radulescu ch->xdp.drop_cnt)) == -EBUSY) { 240ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 241ef17bd7cSIoana Radulescu break; 2425d39dc21SIoana Ciocoi Radulescu cpu_relax(); 243ef17bd7cSIoana Radulescu } 2445d39dc21SIoana Ciocoi Radulescu 2455d39dc21SIoana Ciocoi Radulescu if (err) { 2465d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt); 2475d39dc21SIoana Ciocoi Radulescu ch->buf_count -= ch->xdp.drop_cnt; 2485d39dc21SIoana Ciocoi Radulescu } 2495d39dc21SIoana Ciocoi Radulescu 2505d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_cnt = 0; 2515d39dc21SIoana Ciocoi Radulescu } 2525d39dc21SIoana Ciocoi Radulescu 25338c440b2SIoana Ciornei static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 25438c440b2SIoana Ciornei struct dpaa2_eth_fq *fq, 25538c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_fds) 25638c440b2SIoana Ciornei { 25738c440b2SIoana Ciornei int total_enqueued = 0, retries = 0, enqueued; 25838c440b2SIoana Ciornei struct dpaa2_eth_drv_stats *percpu_extras; 25938c440b2SIoana Ciornei int num_fds, err, max_retries; 26038c440b2SIoana Ciornei struct dpaa2_fd *fds; 26138c440b2SIoana Ciornei 26238c440b2SIoana Ciornei percpu_extras = this_cpu_ptr(priv->percpu_extras); 26338c440b2SIoana Ciornei 26438c440b2SIoana Ciornei /* try to enqueue all the FDs until the max number of retries is hit */ 26538c440b2SIoana Ciornei fds = xdp_fds->fds; 26638c440b2SIoana Ciornei num_fds = xdp_fds->num; 26738c440b2SIoana Ciornei max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 26838c440b2SIoana Ciornei while (total_enqueued < num_fds && retries < max_retries) { 26938c440b2SIoana Ciornei err = priv->enqueue(priv, fq, &fds[total_enqueued], 27038c440b2SIoana Ciornei 0, num_fds - total_enqueued, &enqueued); 27138c440b2SIoana Ciornei if (err == -EBUSY) { 27238c440b2SIoana Ciornei percpu_extras->tx_portal_busy += ++retries; 27338c440b2SIoana Ciornei continue; 27438c440b2SIoana Ciornei } 27538c440b2SIoana Ciornei total_enqueued += enqueued; 27638c440b2SIoana Ciornei } 27738c440b2SIoana Ciornei xdp_fds->num = 0; 27838c440b2SIoana Ciornei 27938c440b2SIoana Ciornei return total_enqueued; 28038c440b2SIoana Ciornei } 28138c440b2SIoana Ciornei 2825d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_tx_flush(struct dpaa2_eth_priv *priv, 28374a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 28474a1c059SIoana Ciornei struct dpaa2_eth_fq *fq) 28574a1c059SIoana Ciornei { 28674a1c059SIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 28774a1c059SIoana Ciornei struct dpaa2_fd *fds; 28874a1c059SIoana Ciornei int enqueued, i; 28974a1c059SIoana Ciornei 29074a1c059SIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 29174a1c059SIoana Ciornei 29274a1c059SIoana Ciornei // enqueue the array of XDP_TX frames 29374a1c059SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds); 29474a1c059SIoana Ciornei 29574a1c059SIoana Ciornei /* update statistics */ 29674a1c059SIoana Ciornei percpu_stats->tx_packets += enqueued; 29774a1c059SIoana Ciornei fds = fq->xdp_tx_fds.fds; 29874a1c059SIoana Ciornei for (i = 0; i < enqueued; i++) { 29974a1c059SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 30074a1c059SIoana Ciornei ch->stats.xdp_tx++; 30174a1c059SIoana Ciornei } 30274a1c059SIoana Ciornei for (i = enqueued; i < fq->xdp_tx_fds.num; i++) { 3035d8dccf8SIoana Ciornei dpaa2_eth_xdp_release_buf(priv, ch, dpaa2_fd_get_addr(&fds[i])); 30474a1c059SIoana Ciornei percpu_stats->tx_errors++; 30574a1c059SIoana Ciornei ch->stats.xdp_tx_err++; 30674a1c059SIoana Ciornei } 30774a1c059SIoana Ciornei fq->xdp_tx_fds.num = 0; 30874a1c059SIoana Ciornei } 30974a1c059SIoana Ciornei 3105d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv, 31174a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 31274a1c059SIoana Ciornei struct dpaa2_fd *fd, 31399e43521SIoana Ciocoi Radulescu void *buf_start, u16 queue_id) 31499e43521SIoana Ciocoi Radulescu { 31599e43521SIoana Ciocoi Radulescu struct dpaa2_faead *faead; 31674a1c059SIoana Ciornei struct dpaa2_fd *dest_fd; 31774a1c059SIoana Ciornei struct dpaa2_eth_fq *fq; 31899e43521SIoana Ciocoi Radulescu u32 ctrl, frc; 31999e43521SIoana Ciocoi Radulescu 32099e43521SIoana Ciocoi Radulescu /* Mark the egress frame hardware annotation area as valid */ 32199e43521SIoana Ciocoi Radulescu frc = dpaa2_fd_get_frc(fd); 32299e43521SIoana Ciocoi Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 32399e43521SIoana Ciocoi Radulescu dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 32499e43521SIoana Ciocoi Radulescu 32599e43521SIoana Ciocoi Radulescu /* Instruct hardware to release the FD buffer directly into 32699e43521SIoana Ciocoi Radulescu * the buffer pool once transmission is completed, instead of 32799e43521SIoana Ciocoi Radulescu * sending a Tx confirmation frame to us 32899e43521SIoana Ciocoi Radulescu */ 32999e43521SIoana Ciocoi Radulescu ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 33099e43521SIoana Ciocoi Radulescu faead = dpaa2_get_faead(buf_start, false); 33199e43521SIoana Ciocoi Radulescu faead->ctrl = cpu_to_le32(ctrl); 33299e43521SIoana Ciocoi Radulescu faead->conf_fqid = 0; 33399e43521SIoana Ciocoi Radulescu 33499e43521SIoana Ciocoi Radulescu fq = &priv->fq[queue_id]; 33574a1c059SIoana Ciornei dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++]; 33674a1c059SIoana Ciornei memcpy(dest_fd, fd, sizeof(*dest_fd)); 33799e43521SIoana Ciocoi Radulescu 33874a1c059SIoana Ciornei if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE) 33974a1c059SIoana Ciornei return; 34074a1c059SIoana Ciornei 3415d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, fq); 34299e43521SIoana Ciocoi Radulescu } 34399e43521SIoana Ciocoi Radulescu 3445d8dccf8SIoana Ciornei static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, 3457e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 34699e43521SIoana Ciocoi Radulescu struct dpaa2_eth_fq *rx_fq, 3477e273a8eSIoana Ciocoi Radulescu struct dpaa2_fd *fd, void *vaddr) 3487e273a8eSIoana Ciocoi Radulescu { 3495d39dc21SIoana Ciocoi Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 3507e273a8eSIoana Ciocoi Radulescu struct bpf_prog *xdp_prog; 3517e273a8eSIoana Ciocoi Radulescu struct xdp_buff xdp; 3527e273a8eSIoana Ciocoi Radulescu u32 xdp_act = XDP_PASS; 35399e43521SIoana Ciocoi Radulescu int err; 35499e43521SIoana Ciocoi Radulescu 3557e273a8eSIoana Ciocoi Radulescu rcu_read_lock(); 3567e273a8eSIoana Ciocoi Radulescu 3577e273a8eSIoana Ciocoi Radulescu xdp_prog = READ_ONCE(ch->xdp.prog); 3587e273a8eSIoana Ciocoi Radulescu if (!xdp_prog) 3597e273a8eSIoana Ciocoi Radulescu goto out; 3607e273a8eSIoana Ciocoi Radulescu 3617e273a8eSIoana Ciocoi Radulescu xdp.data = vaddr + dpaa2_fd_get_offset(fd); 3627e273a8eSIoana Ciocoi Radulescu xdp.data_end = xdp.data + dpaa2_fd_get_len(fd); 3637b1eea1aSIoana Ciocoi Radulescu xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; 3647e273a8eSIoana Ciocoi Radulescu xdp_set_data_meta_invalid(&xdp); 365d678be1dSIoana Radulescu xdp.rxq = &ch->xdp_rxq; 3667e273a8eSIoana Ciocoi Radulescu 3674a9b052aSJesper Dangaard Brouer xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE - 3684a9b052aSJesper Dangaard Brouer (dpaa2_fd_get_offset(fd) - XDP_PACKET_HEADROOM); 3694a9b052aSJesper Dangaard Brouer 3707e273a8eSIoana Ciocoi Radulescu xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 3717e273a8eSIoana Ciocoi Radulescu 3727b1eea1aSIoana Ciocoi Radulescu /* xdp.data pointer may have changed */ 3737b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_offset(fd, xdp.data - vaddr); 3747b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 3757b1eea1aSIoana Ciocoi Radulescu 3767e273a8eSIoana Ciocoi Radulescu switch (xdp_act) { 3777e273a8eSIoana Ciocoi Radulescu case XDP_PASS: 3787e273a8eSIoana Ciocoi Radulescu break; 37999e43521SIoana Ciocoi Radulescu case XDP_TX: 3805d8dccf8SIoana Ciornei dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); 38199e43521SIoana Ciocoi Radulescu break; 3827e273a8eSIoana Ciocoi Radulescu default: 3837e273a8eSIoana Ciocoi Radulescu bpf_warn_invalid_xdp_action(xdp_act); 384df561f66SGustavo A. R. Silva fallthrough; 3857e273a8eSIoana Ciocoi Radulescu case XDP_ABORTED: 3867e273a8eSIoana Ciocoi Radulescu trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 387df561f66SGustavo A. R. Silva fallthrough; 3887e273a8eSIoana Ciocoi Radulescu case XDP_DROP: 3895d8dccf8SIoana Ciornei dpaa2_eth_xdp_release_buf(priv, ch, addr); 390a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_drop++; 3917e273a8eSIoana Ciocoi Radulescu break; 392d678be1dSIoana Radulescu case XDP_REDIRECT: 393d678be1dSIoana Radulescu dma_unmap_page(priv->net_dev->dev.parent, addr, 394efa6a7d0SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 395d678be1dSIoana Radulescu ch->buf_count--; 3964a9b052aSJesper Dangaard Brouer 3974a9b052aSJesper Dangaard Brouer /* Allow redirect use of full headroom */ 398d678be1dSIoana Radulescu xdp.data_hard_start = vaddr; 3994a9b052aSJesper Dangaard Brouer xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE; 4004a9b052aSJesper Dangaard Brouer 401d678be1dSIoana Radulescu err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 402d678be1dSIoana Radulescu if (unlikely(err)) 403d678be1dSIoana Radulescu ch->stats.xdp_drop++; 404d678be1dSIoana Radulescu else 405d678be1dSIoana Radulescu ch->stats.xdp_redirect++; 406d678be1dSIoana Radulescu break; 4077e273a8eSIoana Ciocoi Radulescu } 4087e273a8eSIoana Ciocoi Radulescu 409d678be1dSIoana Radulescu ch->xdp.res |= xdp_act; 4107e273a8eSIoana Ciocoi Radulescu out: 4117e273a8eSIoana Ciocoi Radulescu rcu_read_unlock(); 4127e273a8eSIoana Ciocoi Radulescu return xdp_act; 4137e273a8eSIoana Ciocoi Radulescu } 4147e273a8eSIoana Ciocoi Radulescu 41534ff6846SIoana Radulescu /* Main Rx frame processing routine */ 41634ff6846SIoana Radulescu static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 41734ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 41834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 419dbcdf728SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 42034ff6846SIoana Radulescu { 42134ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 42234ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 42334ff6846SIoana Radulescu void *vaddr; 42434ff6846SIoana Radulescu struct sk_buff *skb; 42534ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 42634ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 42734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 42834ff6846SIoana Radulescu struct dpaa2_fas *fas; 42934ff6846SIoana Radulescu void *buf_data; 43034ff6846SIoana Radulescu u32 status = 0; 4317e273a8eSIoana Ciocoi Radulescu u32 xdp_act; 43234ff6846SIoana Radulescu 43334ff6846SIoana Radulescu /* Tracing point */ 43434ff6846SIoana Radulescu trace_dpaa2_rx_fd(priv->net_dev, fd); 43534ff6846SIoana Radulescu 43634ff6846SIoana Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 437efa6a7d0SIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 43818c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 43934ff6846SIoana Radulescu 44034ff6846SIoana Radulescu fas = dpaa2_get_fas(vaddr, false); 44134ff6846SIoana Radulescu prefetch(fas); 44234ff6846SIoana Radulescu buf_data = vaddr + dpaa2_fd_get_offset(fd); 44334ff6846SIoana Radulescu prefetch(buf_data); 44434ff6846SIoana Radulescu 44534ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 44634ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 44734ff6846SIoana Radulescu 44834ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 4495d8dccf8SIoana Ciornei xdp_act = dpaa2_eth_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 4507e273a8eSIoana Ciocoi Radulescu if (xdp_act != XDP_PASS) { 4517e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_packets++; 4527e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 4537e273a8eSIoana Ciocoi Radulescu return; 4547e273a8eSIoana Ciocoi Radulescu } 4557e273a8eSIoana Ciocoi Radulescu 456efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 45718c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 4585d8dccf8SIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 45934ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 4607e273a8eSIoana Ciocoi Radulescu WARN_ON(priv->xdp_prog); 4617e273a8eSIoana Ciocoi Radulescu 462efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 46318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 4645d8dccf8SIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 46527c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 46634ff6846SIoana Radulescu percpu_extras->rx_sg_frames++; 46734ff6846SIoana Radulescu percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 46834ff6846SIoana Radulescu } else { 46934ff6846SIoana Radulescu /* We don't support any other format */ 47034ff6846SIoana Radulescu goto err_frame_format; 47134ff6846SIoana Radulescu } 47234ff6846SIoana Radulescu 47334ff6846SIoana Radulescu if (unlikely(!skb)) 47434ff6846SIoana Radulescu goto err_build_skb; 47534ff6846SIoana Radulescu 47634ff6846SIoana Radulescu prefetch(skb->data); 47734ff6846SIoana Radulescu 47834ff6846SIoana Radulescu /* Get the timestamp value */ 47934ff6846SIoana Radulescu if (priv->rx_tstamp) { 48034ff6846SIoana Radulescu struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 48134ff6846SIoana Radulescu __le64 *ts = dpaa2_get_ts(vaddr, false); 48234ff6846SIoana Radulescu u64 ns; 48334ff6846SIoana Radulescu 48434ff6846SIoana Radulescu memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 48534ff6846SIoana Radulescu 48634ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 48734ff6846SIoana Radulescu shhwtstamps->hwtstamp = ns_to_ktime(ns); 48834ff6846SIoana Radulescu } 48934ff6846SIoana Radulescu 49034ff6846SIoana Radulescu /* Check if we need to validate the L4 csum */ 49134ff6846SIoana Radulescu if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 49234ff6846SIoana Radulescu status = le32_to_cpu(fas->status); 4935d8dccf8SIoana Ciornei dpaa2_eth_validate_rx_csum(priv, status, skb); 49434ff6846SIoana Radulescu } 49534ff6846SIoana Radulescu 49634ff6846SIoana Radulescu skb->protocol = eth_type_trans(skb, priv->net_dev); 497dbcdf728SIoana Ciocoi Radulescu skb_record_rx_queue(skb, fq->flowid); 49834ff6846SIoana Radulescu 49934ff6846SIoana Radulescu percpu_stats->rx_packets++; 50034ff6846SIoana Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 50134ff6846SIoana Radulescu 5020a25d92cSIoana Ciornei list_add_tail(&skb->list, ch->rx_list); 50334ff6846SIoana Radulescu 50434ff6846SIoana Radulescu return; 50534ff6846SIoana Radulescu 50634ff6846SIoana Radulescu err_build_skb: 5075d8dccf8SIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 50834ff6846SIoana Radulescu err_frame_format: 50934ff6846SIoana Radulescu percpu_stats->rx_dropped++; 51034ff6846SIoana Radulescu } 51134ff6846SIoana Radulescu 51234ff6846SIoana Radulescu /* Consume all frames pull-dequeued into the store. This is the simplest way to 51334ff6846SIoana Radulescu * make sure we don't accidentally issue another volatile dequeue which would 51434ff6846SIoana Radulescu * overwrite (leak) frames already in the store. 51534ff6846SIoana Radulescu * 51634ff6846SIoana Radulescu * Observance of NAPI budget is not our concern, leaving that to the caller. 51734ff6846SIoana Radulescu */ 5185d8dccf8SIoana Ciornei static int dpaa2_eth_consume_frames(struct dpaa2_eth_channel *ch, 519569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq **src) 52034ff6846SIoana Radulescu { 52134ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = ch->priv; 52268049a5fSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq = NULL; 52334ff6846SIoana Radulescu struct dpaa2_dq *dq; 52434ff6846SIoana Radulescu const struct dpaa2_fd *fd; 525ef17bd7cSIoana Radulescu int cleaned = 0, retries = 0; 52634ff6846SIoana Radulescu int is_last; 52734ff6846SIoana Radulescu 52834ff6846SIoana Radulescu do { 52934ff6846SIoana Radulescu dq = dpaa2_io_store_next(ch->store, &is_last); 53034ff6846SIoana Radulescu if (unlikely(!dq)) { 53134ff6846SIoana Radulescu /* If we're here, we *must* have placed a 53234ff6846SIoana Radulescu * volatile dequeue comnmand, so keep reading through 53334ff6846SIoana Radulescu * the store until we get some sort of valid response 53434ff6846SIoana Radulescu * token (either a valid frame or an "empty dequeue") 53534ff6846SIoana Radulescu */ 536ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) { 537ef17bd7cSIoana Radulescu netdev_err_once(priv->net_dev, 538ef17bd7cSIoana Radulescu "Unable to read a valid dequeue response\n"); 539ef17bd7cSIoana Radulescu return -ETIMEDOUT; 540ef17bd7cSIoana Radulescu } 54134ff6846SIoana Radulescu continue; 54234ff6846SIoana Radulescu } 54334ff6846SIoana Radulescu 54434ff6846SIoana Radulescu fd = dpaa2_dq_fd(dq); 54534ff6846SIoana Radulescu fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 54634ff6846SIoana Radulescu 547dbcdf728SIoana Ciocoi Radulescu fq->consume(priv, ch, fd, fq); 54834ff6846SIoana Radulescu cleaned++; 549ef17bd7cSIoana Radulescu retries = 0; 55034ff6846SIoana Radulescu } while (!is_last); 55134ff6846SIoana Radulescu 55268049a5fSIoana Ciocoi Radulescu if (!cleaned) 55368049a5fSIoana Ciocoi Radulescu return 0; 55468049a5fSIoana Ciocoi Radulescu 55568049a5fSIoana Ciocoi Radulescu fq->stats.frames += cleaned; 556460fd830SIoana Ciornei ch->stats.frames += cleaned; 55768049a5fSIoana Ciocoi Radulescu 55868049a5fSIoana Ciocoi Radulescu /* A dequeue operation only pulls frames from a single queue 559569dac6aSIoana Ciocoi Radulescu * into the store. Return the frame queue as an out param. 56068049a5fSIoana Ciocoi Radulescu */ 561569dac6aSIoana Ciocoi Radulescu if (src) 562569dac6aSIoana Ciocoi Radulescu *src = fq; 56368049a5fSIoana Ciocoi Radulescu 56434ff6846SIoana Radulescu return cleaned; 56534ff6846SIoana Radulescu } 56634ff6846SIoana Radulescu 567c5521189SYangbo Lu static int dpaa2_eth_ptp_parse(struct sk_buff *skb, 568c5521189SYangbo Lu u8 *msgtype, u8 *twostep, u8 *udp, 569c5521189SYangbo Lu u16 *correction_offset, 570c5521189SYangbo Lu u16 *origintimestamp_offset) 57134ff6846SIoana Radulescu { 572c5521189SYangbo Lu unsigned int ptp_class; 573c5521189SYangbo Lu struct ptp_header *hdr; 574c5521189SYangbo Lu unsigned int type; 575c5521189SYangbo Lu u8 *base; 576c5521189SYangbo Lu 577c5521189SYangbo Lu ptp_class = ptp_classify_raw(skb); 578c5521189SYangbo Lu if (ptp_class == PTP_CLASS_NONE) 579c5521189SYangbo Lu return -EINVAL; 580c5521189SYangbo Lu 581c5521189SYangbo Lu hdr = ptp_parse_header(skb, ptp_class); 582c5521189SYangbo Lu if (!hdr) 583c5521189SYangbo Lu return -EINVAL; 584c5521189SYangbo Lu 585c5521189SYangbo Lu *msgtype = ptp_get_msgtype(hdr, ptp_class); 586c5521189SYangbo Lu *twostep = hdr->flag_field[0] & 0x2; 587c5521189SYangbo Lu 588c5521189SYangbo Lu type = ptp_class & PTP_CLASS_PMASK; 589c5521189SYangbo Lu if (type == PTP_CLASS_IPV4 || 590c5521189SYangbo Lu type == PTP_CLASS_IPV6) 591c5521189SYangbo Lu *udp = 1; 592c5521189SYangbo Lu else 593c5521189SYangbo Lu *udp = 0; 594c5521189SYangbo Lu 595c5521189SYangbo Lu base = skb_mac_header(skb); 596c5521189SYangbo Lu *correction_offset = (u8 *)&hdr->correction - base; 597c5521189SYangbo Lu *origintimestamp_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 598c5521189SYangbo Lu 599c5521189SYangbo Lu return 0; 600c5521189SYangbo Lu } 601c5521189SYangbo Lu 602c5521189SYangbo Lu /* Configure the egress frame annotation for timestamp update */ 603c5521189SYangbo Lu static void dpaa2_eth_enable_tx_tstamp(struct dpaa2_eth_priv *priv, 604c5521189SYangbo Lu struct dpaa2_fd *fd, 605c5521189SYangbo Lu void *buf_start, 606c5521189SYangbo Lu struct sk_buff *skb) 607c5521189SYangbo Lu { 608c5521189SYangbo Lu struct ptp_tstamp origin_timestamp; 609c5521189SYangbo Lu struct dpni_single_step_cfg cfg; 610c5521189SYangbo Lu u8 msgtype, twostep, udp; 61134ff6846SIoana Radulescu struct dpaa2_faead *faead; 612c5521189SYangbo Lu struct dpaa2_fas *fas; 613c5521189SYangbo Lu struct timespec64 ts; 614c5521189SYangbo Lu u16 offset1, offset2; 61534ff6846SIoana Radulescu u32 ctrl, frc; 616c5521189SYangbo Lu __le64 *ns; 617c5521189SYangbo Lu u8 *data; 61834ff6846SIoana Radulescu 61934ff6846SIoana Radulescu /* Mark the egress frame annotation area as valid */ 62034ff6846SIoana Radulescu frc = dpaa2_fd_get_frc(fd); 62134ff6846SIoana Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 62234ff6846SIoana Radulescu 62334ff6846SIoana Radulescu /* Set hardware annotation size */ 62434ff6846SIoana Radulescu ctrl = dpaa2_fd_get_ctrl(fd); 62534ff6846SIoana Radulescu dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 62634ff6846SIoana Radulescu 62734ff6846SIoana Radulescu /* enable UPD (update prepanded data) bit in FAEAD field of 62834ff6846SIoana Radulescu * hardware frame annotation area 62934ff6846SIoana Radulescu */ 63034ff6846SIoana Radulescu ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 63134ff6846SIoana Radulescu faead = dpaa2_get_faead(buf_start, true); 63234ff6846SIoana Radulescu faead->ctrl = cpu_to_le32(ctrl); 633c5521189SYangbo Lu 634c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 635c5521189SYangbo Lu if (dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 636c5521189SYangbo Lu &offset1, &offset2) || 637c5521189SYangbo Lu msgtype != 0 || twostep) { 638c5521189SYangbo Lu WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 639c5521189SYangbo Lu return; 640c5521189SYangbo Lu } 641c5521189SYangbo Lu 642c5521189SYangbo Lu /* Mark the frame annotation status as valid */ 643c5521189SYangbo Lu frc = dpaa2_fd_get_frc(fd); 644c5521189SYangbo Lu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FASV); 645c5521189SYangbo Lu 646c5521189SYangbo Lu /* Mark the PTP flag for one step timestamping */ 647c5521189SYangbo Lu fas = dpaa2_get_fas(buf_start, true); 648c5521189SYangbo Lu fas->status = cpu_to_le32(DPAA2_FAS_PTP); 649c5521189SYangbo Lu 650c5521189SYangbo Lu dpaa2_ptp->caps.gettime64(&dpaa2_ptp->caps, &ts); 651c5521189SYangbo Lu ns = dpaa2_get_ts(buf_start, true); 652c5521189SYangbo Lu *ns = cpu_to_le64(timespec64_to_ns(&ts) / 653c5521189SYangbo Lu DPAA2_PTP_CLK_PERIOD_NS); 654c5521189SYangbo Lu 655c5521189SYangbo Lu /* Update current time to PTP message originTimestamp field */ 656c5521189SYangbo Lu ns_to_ptp_tstamp(&origin_timestamp, le64_to_cpup(ns)); 657c5521189SYangbo Lu data = skb_mac_header(skb); 658c5521189SYangbo Lu *(__be16 *)(data + offset2) = htons(origin_timestamp.sec_msb); 659c5521189SYangbo Lu *(__be32 *)(data + offset2 + 2) = 660c5521189SYangbo Lu htonl(origin_timestamp.sec_lsb); 661c5521189SYangbo Lu *(__be32 *)(data + offset2 + 6) = htonl(origin_timestamp.nsec); 662c5521189SYangbo Lu 663c5521189SYangbo Lu cfg.en = 1; 664c5521189SYangbo Lu cfg.ch_update = udp; 665c5521189SYangbo Lu cfg.offset = offset1; 666c5521189SYangbo Lu cfg.peer_delay = 0; 667c5521189SYangbo Lu 668c5521189SYangbo Lu if (dpni_set_single_step_cfg(priv->mc_io, 0, priv->mc_token, 669c5521189SYangbo Lu &cfg)) 670c5521189SYangbo Lu WARN_ONCE(1, "Failed to set single step register"); 671c5521189SYangbo Lu } 67234ff6846SIoana Radulescu } 67334ff6846SIoana Radulescu 67434ff6846SIoana Radulescu /* Create a frame descriptor based on a fragmented skb */ 6755d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd(struct dpaa2_eth_priv *priv, 67634ff6846SIoana Radulescu struct sk_buff *skb, 67764a965deSYangbo Lu struct dpaa2_fd *fd, 67864a965deSYangbo Lu void **swa_addr) 67934ff6846SIoana Radulescu { 68034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 68134ff6846SIoana Radulescu void *sgt_buf = NULL; 68234ff6846SIoana Radulescu dma_addr_t addr; 68334ff6846SIoana Radulescu int nr_frags = skb_shinfo(skb)->nr_frags; 68434ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 68534ff6846SIoana Radulescu int i, err; 68634ff6846SIoana Radulescu int sgt_buf_size; 68734ff6846SIoana Radulescu struct scatterlist *scl, *crt_scl; 68834ff6846SIoana Radulescu int num_sg; 68934ff6846SIoana Radulescu int num_dma_bufs; 69034ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 69134ff6846SIoana Radulescu 69234ff6846SIoana Radulescu /* Create and map scatterlist. 69334ff6846SIoana Radulescu * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 69434ff6846SIoana Radulescu * to go beyond nr_frags+1. 69534ff6846SIoana Radulescu * Note: We don't support chained scatterlists 69634ff6846SIoana Radulescu */ 69734ff6846SIoana Radulescu if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 69834ff6846SIoana Radulescu return -EINVAL; 69934ff6846SIoana Radulescu 700d4ceb8deSJulia Lawall scl = kmalloc_array(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 70134ff6846SIoana Radulescu if (unlikely(!scl)) 70234ff6846SIoana Radulescu return -ENOMEM; 70334ff6846SIoana Radulescu 70434ff6846SIoana Radulescu sg_init_table(scl, nr_frags + 1); 70534ff6846SIoana Radulescu num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 70637fbbddaSIoana Ciornei if (unlikely(num_sg < 0)) { 70737fbbddaSIoana Ciornei err = -ENOMEM; 70837fbbddaSIoana Ciornei goto dma_map_sg_failed; 70937fbbddaSIoana Ciornei } 71034ff6846SIoana Radulescu num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 71134ff6846SIoana Radulescu if (unlikely(!num_dma_bufs)) { 71234ff6846SIoana Radulescu err = -ENOMEM; 71334ff6846SIoana Radulescu goto dma_map_sg_failed; 71434ff6846SIoana Radulescu } 71534ff6846SIoana Radulescu 71634ff6846SIoana Radulescu /* Prepare the HW SGT structure */ 71734ff6846SIoana Radulescu sgt_buf_size = priv->tx_data_offset + 71834ff6846SIoana Radulescu sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 71990bc6d4bSSebastian Andrzej Siewior sgt_buf = napi_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN); 72034ff6846SIoana Radulescu if (unlikely(!sgt_buf)) { 72134ff6846SIoana Radulescu err = -ENOMEM; 72234ff6846SIoana Radulescu goto sgt_buf_alloc_failed; 72334ff6846SIoana Radulescu } 72434ff6846SIoana Radulescu sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 72534ff6846SIoana Radulescu memset(sgt_buf, 0, sgt_buf_size); 72634ff6846SIoana Radulescu 72734ff6846SIoana Radulescu sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 72834ff6846SIoana Radulescu 72934ff6846SIoana Radulescu /* Fill in the HW SGT structure. 73034ff6846SIoana Radulescu * 73134ff6846SIoana Radulescu * sgt_buf is zeroed out, so the following fields are implicit 73234ff6846SIoana Radulescu * in all sgt entries: 73334ff6846SIoana Radulescu * - offset is 0 73434ff6846SIoana Radulescu * - format is 'dpaa2_sg_single' 73534ff6846SIoana Radulescu */ 73634ff6846SIoana Radulescu for_each_sg(scl, crt_scl, num_dma_bufs, i) { 73734ff6846SIoana Radulescu dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 73834ff6846SIoana Radulescu dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 73934ff6846SIoana Radulescu } 74034ff6846SIoana Radulescu dpaa2_sg_set_final(&sgt[i - 1], true); 74134ff6846SIoana Radulescu 74234ff6846SIoana Radulescu /* Store the skb backpointer in the SGT buffer. 74334ff6846SIoana Radulescu * Fit the scatterlist and the number of buffers alongside the 74434ff6846SIoana Radulescu * skb backpointer in the software annotation area. We'll need 74534ff6846SIoana Radulescu * all of them on Tx Conf. 74634ff6846SIoana Radulescu */ 74764a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 74834ff6846SIoana Radulescu swa = (struct dpaa2_eth_swa *)sgt_buf; 749e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SG; 750e3fdf6baSIoana Radulescu swa->sg.skb = skb; 751e3fdf6baSIoana Radulescu swa->sg.scl = scl; 752e3fdf6baSIoana Radulescu swa->sg.num_sg = num_sg; 753e3fdf6baSIoana Radulescu swa->sg.sgt_size = sgt_buf_size; 75434ff6846SIoana Radulescu 75534ff6846SIoana Radulescu /* Separately map the SGT buffer */ 75634ff6846SIoana Radulescu addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 75734ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) { 75834ff6846SIoana Radulescu err = -ENOMEM; 75934ff6846SIoana Radulescu goto dma_map_single_failed; 76034ff6846SIoana Radulescu } 76134ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, priv->tx_data_offset); 76234ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_sg); 76334ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 76434ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 765b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 76634ff6846SIoana Radulescu 76734ff6846SIoana Radulescu return 0; 76834ff6846SIoana Radulescu 76934ff6846SIoana Radulescu dma_map_single_failed: 77034ff6846SIoana Radulescu skb_free_frag(sgt_buf); 77134ff6846SIoana Radulescu sgt_buf_alloc_failed: 77234ff6846SIoana Radulescu dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 77334ff6846SIoana Radulescu dma_map_sg_failed: 77434ff6846SIoana Radulescu kfree(scl); 77534ff6846SIoana Radulescu return err; 77634ff6846SIoana Radulescu } 77734ff6846SIoana Radulescu 778d70446eeSIoana Ciornei /* Create a SG frame descriptor based on a linear skb. 779d70446eeSIoana Ciornei * 780d70446eeSIoana Ciornei * This function is used on the Tx path when the skb headroom is not large 781d70446eeSIoana Ciornei * enough for the HW requirements, thus instead of realloc-ing the skb we 782d70446eeSIoana Ciornei * create a SG frame descriptor with only one entry. 783d70446eeSIoana Ciornei */ 7845d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd_single_buf(struct dpaa2_eth_priv *priv, 785d70446eeSIoana Ciornei struct sk_buff *skb, 78664a965deSYangbo Lu struct dpaa2_fd *fd, 78764a965deSYangbo Lu void **swa_addr) 788d70446eeSIoana Ciornei { 789d70446eeSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 790d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 791d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 792d70446eeSIoana Ciornei struct dpaa2_eth_swa *swa; 793d70446eeSIoana Ciornei dma_addr_t addr, sgt_addr; 794d70446eeSIoana Ciornei void *sgt_buf = NULL; 795d70446eeSIoana Ciornei int sgt_buf_size; 796d70446eeSIoana Ciornei int err; 797d70446eeSIoana Ciornei 798d70446eeSIoana Ciornei /* Prepare the HW SGT structure */ 799d70446eeSIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 800d70446eeSIoana Ciornei sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry); 801d70446eeSIoana Ciornei 802d70446eeSIoana Ciornei if (sgt_cache->count == 0) 803d70446eeSIoana Ciornei sgt_buf = kzalloc(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN, 804d70446eeSIoana Ciornei GFP_ATOMIC); 805d70446eeSIoana Ciornei else 806d70446eeSIoana Ciornei sgt_buf = sgt_cache->buf[--sgt_cache->count]; 807d70446eeSIoana Ciornei if (unlikely(!sgt_buf)) 808d70446eeSIoana Ciornei return -ENOMEM; 809d70446eeSIoana Ciornei 810d70446eeSIoana Ciornei sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 811d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 812d70446eeSIoana Ciornei 813d70446eeSIoana Ciornei addr = dma_map_single(dev, skb->data, skb->len, DMA_BIDIRECTIONAL); 814d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) { 815d70446eeSIoana Ciornei err = -ENOMEM; 816d70446eeSIoana Ciornei goto data_map_failed; 817d70446eeSIoana Ciornei } 818d70446eeSIoana Ciornei 819d70446eeSIoana Ciornei /* Fill in the HW SGT structure */ 820d70446eeSIoana Ciornei dpaa2_sg_set_addr(sgt, addr); 821d70446eeSIoana Ciornei dpaa2_sg_set_len(sgt, skb->len); 822d70446eeSIoana Ciornei dpaa2_sg_set_final(sgt, true); 823d70446eeSIoana Ciornei 824d70446eeSIoana Ciornei /* Store the skb backpointer in the SGT buffer */ 82564a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 826d70446eeSIoana Ciornei swa = (struct dpaa2_eth_swa *)sgt_buf; 827d70446eeSIoana Ciornei swa->type = DPAA2_ETH_SWA_SINGLE; 828d70446eeSIoana Ciornei swa->single.skb = skb; 829d70446eeSIoana Ciornei swa->sg.sgt_size = sgt_buf_size; 830d70446eeSIoana Ciornei 831d70446eeSIoana Ciornei /* Separately map the SGT buffer */ 832d70446eeSIoana Ciornei sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 833d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, sgt_addr))) { 834d70446eeSIoana Ciornei err = -ENOMEM; 835d70446eeSIoana Ciornei goto sgt_map_failed; 836d70446eeSIoana Ciornei } 837d70446eeSIoana Ciornei 838d70446eeSIoana Ciornei dpaa2_fd_set_offset(fd, priv->tx_data_offset); 839d70446eeSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_sg); 840d70446eeSIoana Ciornei dpaa2_fd_set_addr(fd, sgt_addr); 841d70446eeSIoana Ciornei dpaa2_fd_set_len(fd, skb->len); 842d70446eeSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 843d70446eeSIoana Ciornei 844d70446eeSIoana Ciornei return 0; 845d70446eeSIoana Ciornei 846d70446eeSIoana Ciornei sgt_map_failed: 847d70446eeSIoana Ciornei dma_unmap_single(dev, addr, skb->len, DMA_BIDIRECTIONAL); 848d70446eeSIoana Ciornei data_map_failed: 849d70446eeSIoana Ciornei if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 850d70446eeSIoana Ciornei kfree(sgt_buf); 851d70446eeSIoana Ciornei else 852d70446eeSIoana Ciornei sgt_cache->buf[sgt_cache->count++] = sgt_buf; 853d70446eeSIoana Ciornei 854d70446eeSIoana Ciornei return err; 855d70446eeSIoana Ciornei } 856d70446eeSIoana Ciornei 85734ff6846SIoana Radulescu /* Create a frame descriptor based on a linear skb */ 8585d8dccf8SIoana Ciornei static int dpaa2_eth_build_single_fd(struct dpaa2_eth_priv *priv, 85934ff6846SIoana Radulescu struct sk_buff *skb, 86064a965deSYangbo Lu struct dpaa2_fd *fd, 86164a965deSYangbo Lu void **swa_addr) 86234ff6846SIoana Radulescu { 86334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 86434ff6846SIoana Radulescu u8 *buffer_start, *aligned_start; 865e3fdf6baSIoana Radulescu struct dpaa2_eth_swa *swa; 86634ff6846SIoana Radulescu dma_addr_t addr; 86734ff6846SIoana Radulescu 8681cf773bdSYangbo Lu buffer_start = skb->data - dpaa2_eth_needed_headroom(skb); 86934ff6846SIoana Radulescu 87034ff6846SIoana Radulescu /* If there's enough room to align the FD address, do it. 87134ff6846SIoana Radulescu * It will help hardware optimize accesses. 87234ff6846SIoana Radulescu */ 87334ff6846SIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 87434ff6846SIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 87534ff6846SIoana Radulescu if (aligned_start >= skb->head) 87634ff6846SIoana Radulescu buffer_start = aligned_start; 87734ff6846SIoana Radulescu 87834ff6846SIoana Radulescu /* Store a backpointer to the skb at the beginning of the buffer 87934ff6846SIoana Radulescu * (in the private data area) such that we can release it 88034ff6846SIoana Radulescu * on Tx confirm 88134ff6846SIoana Radulescu */ 88264a965deSYangbo Lu *swa_addr = (void *)buffer_start; 883e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 884e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SINGLE; 885e3fdf6baSIoana Radulescu swa->single.skb = skb; 88634ff6846SIoana Radulescu 88734ff6846SIoana Radulescu addr = dma_map_single(dev, buffer_start, 88834ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 88934ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 89034ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 89134ff6846SIoana Radulescu return -ENOMEM; 89234ff6846SIoana Radulescu 89334ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 89434ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 89534ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 89634ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_single); 897b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 89834ff6846SIoana Radulescu 89934ff6846SIoana Radulescu return 0; 90034ff6846SIoana Radulescu } 90134ff6846SIoana Radulescu 90234ff6846SIoana Radulescu /* FD freeing routine on the Tx path 90334ff6846SIoana Radulescu * 90434ff6846SIoana Radulescu * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 90534ff6846SIoana Radulescu * back-pointed to is also freed. 90634ff6846SIoana Radulescu * This can be called either from dpaa2_eth_tx_conf() or on the error path of 90734ff6846SIoana Radulescu * dpaa2_eth_tx(). 90834ff6846SIoana Radulescu */ 909c5521189SYangbo Lu static void dpaa2_eth_free_tx_fd(struct dpaa2_eth_priv *priv, 910d678be1dSIoana Radulescu struct dpaa2_eth_fq *fq, 9110723a3aeSIoana Ciocoi Radulescu const struct dpaa2_fd *fd, bool in_napi) 91234ff6846SIoana Radulescu { 91334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 914d70446eeSIoana Ciornei dma_addr_t fd_addr, sg_addr; 915d678be1dSIoana Radulescu struct sk_buff *skb = NULL; 91634ff6846SIoana Radulescu unsigned char *buffer_start; 91734ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 91834ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 919d678be1dSIoana Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 92034ff6846SIoana Radulescu 921d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 922d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 923d70446eeSIoana Ciornei 92434ff6846SIoana Radulescu fd_addr = dpaa2_fd_get_addr(fd); 925e3fdf6baSIoana Radulescu buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 926e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 92734ff6846SIoana Radulescu 92834ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 929d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_SINGLE) { 930e3fdf6baSIoana Radulescu skb = swa->single.skb; 931d678be1dSIoana Radulescu /* Accessing the skb buffer is safe before dma unmap, 932d678be1dSIoana Radulescu * because we didn't map the actual skb shell. 93334ff6846SIoana Radulescu */ 93434ff6846SIoana Radulescu dma_unmap_single(dev, fd_addr, 93534ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 93634ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 937d678be1dSIoana Radulescu } else { 938d678be1dSIoana Radulescu WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 939d678be1dSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 940d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 941d678be1dSIoana Radulescu } 94234ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 943d70446eeSIoana Ciornei if (swa->type == DPAA2_ETH_SWA_SG) { 944e3fdf6baSIoana Radulescu skb = swa->sg.skb; 94534ff6846SIoana Radulescu 94634ff6846SIoana Radulescu /* Unmap the scatterlist */ 947e3fdf6baSIoana Radulescu dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 948e3fdf6baSIoana Radulescu DMA_BIDIRECTIONAL); 949e3fdf6baSIoana Radulescu kfree(swa->sg.scl); 95034ff6846SIoana Radulescu 95134ff6846SIoana Radulescu /* Unmap the SGT buffer */ 952e3fdf6baSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 95334ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 95434ff6846SIoana Radulescu } else { 955d70446eeSIoana Ciornei skb = swa->single.skb; 956d70446eeSIoana Ciornei 957d70446eeSIoana Ciornei /* Unmap the SGT Buffer */ 958d70446eeSIoana Ciornei dma_unmap_single(dev, fd_addr, swa->single.sgt_size, 959d70446eeSIoana Ciornei DMA_BIDIRECTIONAL); 960d70446eeSIoana Ciornei 961d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(buffer_start + 962d70446eeSIoana Ciornei priv->tx_data_offset); 963d70446eeSIoana Ciornei sg_addr = dpaa2_sg_get_addr(sgt); 964d70446eeSIoana Ciornei dma_unmap_single(dev, sg_addr, skb->len, DMA_BIDIRECTIONAL); 965d70446eeSIoana Ciornei } 966d70446eeSIoana Ciornei } else { 96734ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "Invalid FD format\n"); 96834ff6846SIoana Radulescu return; 96934ff6846SIoana Radulescu } 97034ff6846SIoana Radulescu 971d678be1dSIoana Radulescu if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 972d678be1dSIoana Radulescu fq->dq_frames++; 973d678be1dSIoana Radulescu fq->dq_bytes += fd_len; 974d678be1dSIoana Radulescu } 975d678be1dSIoana Radulescu 976d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_XDP) { 977d678be1dSIoana Radulescu xdp_return_frame(swa->xdp.xdpf); 978d678be1dSIoana Radulescu return; 979d678be1dSIoana Radulescu } 980d678be1dSIoana Radulescu 98134ff6846SIoana Radulescu /* Get the timestamp value */ 9821cf773bdSYangbo Lu if (skb->cb[0] == TX_TSTAMP) { 98334ff6846SIoana Radulescu struct skb_shared_hwtstamps shhwtstamps; 984e3fdf6baSIoana Radulescu __le64 *ts = dpaa2_get_ts(buffer_start, true); 98534ff6846SIoana Radulescu u64 ns; 98634ff6846SIoana Radulescu 98734ff6846SIoana Radulescu memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 98834ff6846SIoana Radulescu 98934ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 99034ff6846SIoana Radulescu shhwtstamps.hwtstamp = ns_to_ktime(ns); 99134ff6846SIoana Radulescu skb_tstamp_tx(skb, &shhwtstamps); 992c5521189SYangbo Lu } else if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 993c5521189SYangbo Lu mutex_unlock(&priv->onestep_tstamp_lock); 99434ff6846SIoana Radulescu } 99534ff6846SIoana Radulescu 99634ff6846SIoana Radulescu /* Free SGT buffer allocated on tx */ 997d70446eeSIoana Ciornei if (fd_format != dpaa2_fd_single) { 998d70446eeSIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 999d70446eeSIoana Ciornei if (swa->type == DPAA2_ETH_SWA_SG) { 1000e3fdf6baSIoana Radulescu skb_free_frag(buffer_start); 1001d70446eeSIoana Ciornei } else { 1002d70446eeSIoana Ciornei if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 1003d70446eeSIoana Ciornei kfree(buffer_start); 1004d70446eeSIoana Ciornei else 1005d70446eeSIoana Ciornei sgt_cache->buf[sgt_cache->count++] = buffer_start; 1006d70446eeSIoana Ciornei } 1007d70446eeSIoana Ciornei } 100834ff6846SIoana Radulescu 100934ff6846SIoana Radulescu /* Move on with skb release */ 10100723a3aeSIoana Ciocoi Radulescu napi_consume_skb(skb, in_napi); 101134ff6846SIoana Radulescu } 101234ff6846SIoana Radulescu 1013c5521189SYangbo Lu static netdev_tx_t __dpaa2_eth_tx(struct sk_buff *skb, 1014c5521189SYangbo Lu struct net_device *net_dev) 101534ff6846SIoana Radulescu { 101634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 101734ff6846SIoana Radulescu struct dpaa2_fd fd; 101834ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 101934ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 102034ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 1021569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 102234ff6846SIoana Radulescu u16 queue_mapping; 102334ff6846SIoana Radulescu unsigned int needed_headroom; 1024569dac6aSIoana Ciocoi Radulescu u32 fd_len; 1025ab1e6de2SIoana Radulescu u8 prio = 0; 102634ff6846SIoana Radulescu int err, i; 102764a965deSYangbo Lu void *swa; 102834ff6846SIoana Radulescu 102934ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 103034ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 103134ff6846SIoana Radulescu 10321cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(skb); 103334ff6846SIoana Radulescu 103434ff6846SIoana Radulescu /* We'll be holding a back-reference to the skb until Tx Confirmation; 103534ff6846SIoana Radulescu * we don't want that overwritten by a concurrent Tx with a cloned skb. 103634ff6846SIoana Radulescu */ 103734ff6846SIoana Radulescu skb = skb_unshare(skb, GFP_ATOMIC); 103834ff6846SIoana Radulescu if (unlikely(!skb)) { 103934ff6846SIoana Radulescu /* skb_unshare() has already freed the skb */ 104034ff6846SIoana Radulescu percpu_stats->tx_dropped++; 104134ff6846SIoana Radulescu return NETDEV_TX_OK; 104234ff6846SIoana Radulescu } 104334ff6846SIoana Radulescu 104434ff6846SIoana Radulescu /* Setup the FD fields */ 104534ff6846SIoana Radulescu memset(&fd, 0, sizeof(fd)); 104634ff6846SIoana Radulescu 104734ff6846SIoana Radulescu if (skb_is_nonlinear(skb)) { 104864a965deSYangbo Lu err = dpaa2_eth_build_sg_fd(priv, skb, &fd, &swa); 104934ff6846SIoana Radulescu percpu_extras->tx_sg_frames++; 105034ff6846SIoana Radulescu percpu_extras->tx_sg_bytes += skb->len; 1051d70446eeSIoana Ciornei } else if (skb_headroom(skb) < needed_headroom) { 105264a965deSYangbo Lu err = dpaa2_eth_build_sg_fd_single_buf(priv, skb, &fd, &swa); 1053d70446eeSIoana Ciornei percpu_extras->tx_sg_frames++; 1054d70446eeSIoana Ciornei percpu_extras->tx_sg_bytes += skb->len; 10554c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_frames++; 10564c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_bytes += skb->len; 105734ff6846SIoana Radulescu } else { 105864a965deSYangbo Lu err = dpaa2_eth_build_single_fd(priv, skb, &fd, &swa); 105934ff6846SIoana Radulescu } 106034ff6846SIoana Radulescu 106134ff6846SIoana Radulescu if (unlikely(err)) { 106234ff6846SIoana Radulescu percpu_stats->tx_dropped++; 106334ff6846SIoana Radulescu goto err_build_fd; 106434ff6846SIoana Radulescu } 106534ff6846SIoana Radulescu 1066c5521189SYangbo Lu if (skb->cb[0]) 1067c5521189SYangbo Lu dpaa2_eth_enable_tx_tstamp(priv, &fd, swa, skb); 106864a965deSYangbo Lu 106934ff6846SIoana Radulescu /* Tracing point */ 107034ff6846SIoana Radulescu trace_dpaa2_tx_fd(net_dev, &fd); 107134ff6846SIoana Radulescu 107234ff6846SIoana Radulescu /* TxConf FQ selection relies on queue id from the stack. 107334ff6846SIoana Radulescu * In case of a forwarded frame from another DPNI interface, we choose 107434ff6846SIoana Radulescu * a queue affined to the same core that processed the Rx frame 107534ff6846SIoana Radulescu */ 107634ff6846SIoana Radulescu queue_mapping = skb_get_queue_mapping(skb); 1077ab1e6de2SIoana Radulescu 1078ab1e6de2SIoana Radulescu if (net_dev->num_tc) { 1079ab1e6de2SIoana Radulescu prio = netdev_txq_to_tc(net_dev, queue_mapping); 1080ab1e6de2SIoana Radulescu /* Hardware interprets priority level 0 as being the highest, 1081ab1e6de2SIoana Radulescu * so we need to do a reverse mapping to the netdev tc index 1082ab1e6de2SIoana Radulescu */ 1083ab1e6de2SIoana Radulescu prio = net_dev->num_tc - prio - 1; 1084ab1e6de2SIoana Radulescu /* We have only one FQ array entry for all Tx hardware queues 1085ab1e6de2SIoana Radulescu * with the same flow id (but different priority levels) 1086ab1e6de2SIoana Radulescu */ 1087ab1e6de2SIoana Radulescu queue_mapping %= dpaa2_eth_queue_count(priv); 1088ab1e6de2SIoana Radulescu } 108934ff6846SIoana Radulescu fq = &priv->fq[queue_mapping]; 10908c838f53SIoana Ciornei 10918c838f53SIoana Ciornei fd_len = dpaa2_fd_get_len(&fd); 10928c838f53SIoana Ciornei nq = netdev_get_tx_queue(net_dev, queue_mapping); 10938c838f53SIoana Ciornei netdev_tx_sent_queue(nq, fd_len); 10948c838f53SIoana Ciornei 10958c838f53SIoana Ciornei /* Everything that happens after this enqueues might race with 10968c838f53SIoana Ciornei * the Tx confirmation callback for this frame 10978c838f53SIoana Ciornei */ 109834ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 10996ff80447SIoana Ciornei err = priv->enqueue(priv, fq, &fd, prio, 1, NULL); 110034ff6846SIoana Radulescu if (err != -EBUSY) 110134ff6846SIoana Radulescu break; 110234ff6846SIoana Radulescu } 110334ff6846SIoana Radulescu percpu_extras->tx_portal_busy += i; 110434ff6846SIoana Radulescu if (unlikely(err < 0)) { 110534ff6846SIoana Radulescu percpu_stats->tx_errors++; 110634ff6846SIoana Radulescu /* Clean up everything, including freeing the skb */ 11075d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, &fd, false); 11088c838f53SIoana Ciornei netdev_tx_completed_queue(nq, 1, fd_len); 110934ff6846SIoana Radulescu } else { 111034ff6846SIoana Radulescu percpu_stats->tx_packets++; 1111569dac6aSIoana Ciocoi Radulescu percpu_stats->tx_bytes += fd_len; 111234ff6846SIoana Radulescu } 111334ff6846SIoana Radulescu 111434ff6846SIoana Radulescu return NETDEV_TX_OK; 111534ff6846SIoana Radulescu 111634ff6846SIoana Radulescu err_build_fd: 111734ff6846SIoana Radulescu dev_kfree_skb(skb); 111834ff6846SIoana Radulescu 111934ff6846SIoana Radulescu return NETDEV_TX_OK; 112034ff6846SIoana Radulescu } 112134ff6846SIoana Radulescu 1122c5521189SYangbo Lu static void dpaa2_eth_tx_onestep_tstamp(struct work_struct *work) 1123c5521189SYangbo Lu { 1124c5521189SYangbo Lu struct dpaa2_eth_priv *priv = container_of(work, struct dpaa2_eth_priv, 1125c5521189SYangbo Lu tx_onestep_tstamp); 1126c5521189SYangbo Lu struct sk_buff *skb; 1127c5521189SYangbo Lu 1128c5521189SYangbo Lu while (true) { 1129c5521189SYangbo Lu skb = skb_dequeue(&priv->tx_skbs); 1130c5521189SYangbo Lu if (!skb) 1131c5521189SYangbo Lu return; 1132c5521189SYangbo Lu 1133c5521189SYangbo Lu /* Lock just before TX one-step timestamping packet, 1134c5521189SYangbo Lu * and release the lock in dpaa2_eth_free_tx_fd when 1135c5521189SYangbo Lu * confirm the packet has been sent on hardware, or 1136c5521189SYangbo Lu * when clean up during transmit failure. 1137c5521189SYangbo Lu */ 1138c5521189SYangbo Lu mutex_lock(&priv->onestep_tstamp_lock); 1139c5521189SYangbo Lu __dpaa2_eth_tx(skb, priv->net_dev); 1140c5521189SYangbo Lu } 1141c5521189SYangbo Lu } 1142c5521189SYangbo Lu 1143c5521189SYangbo Lu static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 1144c5521189SYangbo Lu { 1145c5521189SYangbo Lu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1146c5521189SYangbo Lu u8 msgtype, twostep, udp; 1147c5521189SYangbo Lu u16 offset1, offset2; 1148c5521189SYangbo Lu 1149c5521189SYangbo Lu /* Utilize skb->cb[0] for timestamping request per skb */ 1150c5521189SYangbo Lu skb->cb[0] = 0; 1151c5521189SYangbo Lu 1152c5521189SYangbo Lu if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && dpaa2_ptp) { 1153c5521189SYangbo Lu if (priv->tx_tstamp_type == HWTSTAMP_TX_ON) 1154c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1155c5521189SYangbo Lu else if (priv->tx_tstamp_type == HWTSTAMP_TX_ONESTEP_SYNC) 1156c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP_ONESTEP_SYNC; 1157c5521189SYangbo Lu } 1158c5521189SYangbo Lu 1159c5521189SYangbo Lu /* TX for one-step timestamping PTP Sync packet */ 1160c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1161c5521189SYangbo Lu if (!dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 1162c5521189SYangbo Lu &offset1, &offset2)) 1163c5521189SYangbo Lu if (msgtype == 0 && twostep == 0) { 1164c5521189SYangbo Lu skb_queue_tail(&priv->tx_skbs, skb); 1165c5521189SYangbo Lu queue_work(priv->dpaa2_ptp_wq, 1166c5521189SYangbo Lu &priv->tx_onestep_tstamp); 1167c5521189SYangbo Lu return NETDEV_TX_OK; 1168c5521189SYangbo Lu } 1169c5521189SYangbo Lu /* Use two-step timestamping if not one-step timestamping 1170c5521189SYangbo Lu * PTP Sync packet 1171c5521189SYangbo Lu */ 1172c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1173c5521189SYangbo Lu } 1174c5521189SYangbo Lu 1175c5521189SYangbo Lu /* TX for other packets */ 1176c5521189SYangbo Lu return __dpaa2_eth_tx(skb, net_dev); 1177c5521189SYangbo Lu } 1178c5521189SYangbo Lu 117934ff6846SIoana Radulescu /* Tx confirmation frame processing routine */ 118034ff6846SIoana Radulescu static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 1181b00c898cSIoana Ciornei struct dpaa2_eth_channel *ch __always_unused, 118234ff6846SIoana Radulescu const struct dpaa2_fd *fd, 1183569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 118434ff6846SIoana Radulescu { 118534ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 118634ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 1187569dac6aSIoana Ciocoi Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 118834ff6846SIoana Radulescu u32 fd_errors; 118934ff6846SIoana Radulescu 119034ff6846SIoana Radulescu /* Tracing point */ 119134ff6846SIoana Radulescu trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 119234ff6846SIoana Radulescu 119334ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 119434ff6846SIoana Radulescu percpu_extras->tx_conf_frames++; 1195569dac6aSIoana Ciocoi Radulescu percpu_extras->tx_conf_bytes += fd_len; 1196569dac6aSIoana Ciocoi Radulescu 119734ff6846SIoana Radulescu /* Check frame errors in the FD field */ 119834ff6846SIoana Radulescu fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 11995d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, fd, true); 120034ff6846SIoana Radulescu 120134ff6846SIoana Radulescu if (likely(!fd_errors)) 120234ff6846SIoana Radulescu return; 120334ff6846SIoana Radulescu 120434ff6846SIoana Radulescu if (net_ratelimit()) 120534ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 120634ff6846SIoana Radulescu fd_errors); 120734ff6846SIoana Radulescu 120834ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 120934ff6846SIoana Radulescu /* Tx-conf logically pertains to the egress path. */ 121034ff6846SIoana Radulescu percpu_stats->tx_errors++; 121134ff6846SIoana Radulescu } 121234ff6846SIoana Radulescu 12135d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 121434ff6846SIoana Radulescu { 121534ff6846SIoana Radulescu int err; 121634ff6846SIoana Radulescu 121734ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 121834ff6846SIoana Radulescu DPNI_OFF_RX_L3_CSUM, enable); 121934ff6846SIoana Radulescu if (err) { 122034ff6846SIoana Radulescu netdev_err(priv->net_dev, 122134ff6846SIoana Radulescu "dpni_set_offload(RX_L3_CSUM) failed\n"); 122234ff6846SIoana Radulescu return err; 122334ff6846SIoana Radulescu } 122434ff6846SIoana Radulescu 122534ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 122634ff6846SIoana Radulescu DPNI_OFF_RX_L4_CSUM, enable); 122734ff6846SIoana Radulescu if (err) { 122834ff6846SIoana Radulescu netdev_err(priv->net_dev, 122934ff6846SIoana Radulescu "dpni_set_offload(RX_L4_CSUM) failed\n"); 123034ff6846SIoana Radulescu return err; 123134ff6846SIoana Radulescu } 123234ff6846SIoana Radulescu 123334ff6846SIoana Radulescu return 0; 123434ff6846SIoana Radulescu } 123534ff6846SIoana Radulescu 12365d8dccf8SIoana Ciornei static int dpaa2_eth_set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 123734ff6846SIoana Radulescu { 123834ff6846SIoana Radulescu int err; 123934ff6846SIoana Radulescu 124034ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 124134ff6846SIoana Radulescu DPNI_OFF_TX_L3_CSUM, enable); 124234ff6846SIoana Radulescu if (err) { 124334ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 124434ff6846SIoana Radulescu return err; 124534ff6846SIoana Radulescu } 124634ff6846SIoana Radulescu 124734ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 124834ff6846SIoana Radulescu DPNI_OFF_TX_L4_CSUM, enable); 124934ff6846SIoana Radulescu if (err) { 125034ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 125134ff6846SIoana Radulescu return err; 125234ff6846SIoana Radulescu } 125334ff6846SIoana Radulescu 125434ff6846SIoana Radulescu return 0; 125534ff6846SIoana Radulescu } 125634ff6846SIoana Radulescu 125734ff6846SIoana Radulescu /* Perform a single release command to add buffers 125834ff6846SIoana Radulescu * to the specified buffer pool 125934ff6846SIoana Radulescu */ 12605d8dccf8SIoana Ciornei static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv, 126134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, u16 bpid) 126234ff6846SIoana Radulescu { 126334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 126434ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 126527c87486SIoana Ciocoi Radulescu struct page *page; 126634ff6846SIoana Radulescu dma_addr_t addr; 1267ef17bd7cSIoana Radulescu int retries = 0; 126834ff6846SIoana Radulescu int i, err; 126934ff6846SIoana Radulescu 127034ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 127134ff6846SIoana Radulescu /* Allocate buffer visible to WRIOP + skb shared info + 127234ff6846SIoana Radulescu * alignment padding 127334ff6846SIoana Radulescu */ 127427c87486SIoana Ciocoi Radulescu /* allocate one page for each Rx buffer. WRIOP sees 127527c87486SIoana Ciocoi Radulescu * the entire page except for a tailroom reserved for 127627c87486SIoana Ciocoi Radulescu * skb shared info 127727c87486SIoana Ciocoi Radulescu */ 127827c87486SIoana Ciocoi Radulescu page = dev_alloc_pages(0); 127927c87486SIoana Ciocoi Radulescu if (!page) 128034ff6846SIoana Radulescu goto err_alloc; 128134ff6846SIoana Radulescu 1282efa6a7d0SIoana Ciornei addr = dma_map_page(dev, page, 0, priv->rx_buf_size, 128318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 128434ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 128534ff6846SIoana Radulescu goto err_map; 128634ff6846SIoana Radulescu 128734ff6846SIoana Radulescu buf_array[i] = addr; 128834ff6846SIoana Radulescu 128934ff6846SIoana Radulescu /* tracing point */ 129034ff6846SIoana Radulescu trace_dpaa2_eth_buf_seed(priv->net_dev, 129127c87486SIoana Ciocoi Radulescu page, DPAA2_ETH_RX_BUF_RAW_SIZE, 1292efa6a7d0SIoana Ciornei addr, priv->rx_buf_size, 129334ff6846SIoana Radulescu bpid); 129434ff6846SIoana Radulescu } 129534ff6846SIoana Radulescu 129634ff6846SIoana Radulescu release_bufs: 129734ff6846SIoana Radulescu /* In case the portal is busy, retry until successful */ 129834ff6846SIoana Radulescu while ((err = dpaa2_io_service_release(ch->dpio, bpid, 1299ef17bd7cSIoana Radulescu buf_array, i)) == -EBUSY) { 1300ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 1301ef17bd7cSIoana Radulescu break; 130234ff6846SIoana Radulescu cpu_relax(); 1303ef17bd7cSIoana Radulescu } 130434ff6846SIoana Radulescu 130534ff6846SIoana Radulescu /* If release command failed, clean up and bail out; 130634ff6846SIoana Radulescu * not much else we can do about it 130734ff6846SIoana Radulescu */ 130834ff6846SIoana Radulescu if (err) { 13095d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, i); 131034ff6846SIoana Radulescu return 0; 131134ff6846SIoana Radulescu } 131234ff6846SIoana Radulescu 131334ff6846SIoana Radulescu return i; 131434ff6846SIoana Radulescu 131534ff6846SIoana Radulescu err_map: 131627c87486SIoana Ciocoi Radulescu __free_pages(page, 0); 131734ff6846SIoana Radulescu err_alloc: 131834ff6846SIoana Radulescu /* If we managed to allocate at least some buffers, 131934ff6846SIoana Radulescu * release them to hardware 132034ff6846SIoana Radulescu */ 132134ff6846SIoana Radulescu if (i) 132234ff6846SIoana Radulescu goto release_bufs; 132334ff6846SIoana Radulescu 132434ff6846SIoana Radulescu return 0; 132534ff6846SIoana Radulescu } 132634ff6846SIoana Radulescu 13275d8dccf8SIoana Ciornei static int dpaa2_eth_seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 132834ff6846SIoana Radulescu { 132934ff6846SIoana Radulescu int i, j; 133034ff6846SIoana Radulescu int new_count; 133134ff6846SIoana Radulescu 133234ff6846SIoana Radulescu for (j = 0; j < priv->num_channels; j++) { 133334ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_NUM_BUFS; 133434ff6846SIoana Radulescu i += DPAA2_ETH_BUFS_PER_CMD) { 13355d8dccf8SIoana Ciornei new_count = dpaa2_eth_add_bufs(priv, priv->channel[j], bpid); 133634ff6846SIoana Radulescu priv->channel[j]->buf_count += new_count; 133734ff6846SIoana Radulescu 133834ff6846SIoana Radulescu if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 133934ff6846SIoana Radulescu return -ENOMEM; 134034ff6846SIoana Radulescu } 134134ff6846SIoana Radulescu } 134234ff6846SIoana Radulescu } 134334ff6846SIoana Radulescu 134434ff6846SIoana Radulescu return 0; 134534ff6846SIoana Radulescu } 134634ff6846SIoana Radulescu 1347d0ea5cbdSJesse Brandeburg /* 134834ff6846SIoana Radulescu * Drain the specified number of buffers from the DPNI's private buffer pool. 134934ff6846SIoana Radulescu * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 135034ff6846SIoana Radulescu */ 13515d8dccf8SIoana Ciornei static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int count) 135234ff6846SIoana Radulescu { 135334ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1354ef17bd7cSIoana Radulescu int retries = 0; 135534ff6846SIoana Radulescu int ret; 135634ff6846SIoana Radulescu 135734ff6846SIoana Radulescu do { 135834ff6846SIoana Radulescu ret = dpaa2_io_service_acquire(NULL, priv->bpid, 135934ff6846SIoana Radulescu buf_array, count); 136034ff6846SIoana Radulescu if (ret < 0) { 1361ef17bd7cSIoana Radulescu if (ret == -EBUSY && 13620e5ad75bSIoana Ciornei retries++ < DPAA2_ETH_SWP_BUSY_RETRIES) 1363ef17bd7cSIoana Radulescu continue; 136434ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 136534ff6846SIoana Radulescu return; 136634ff6846SIoana Radulescu } 13675d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, ret); 1368ef17bd7cSIoana Radulescu retries = 0; 136934ff6846SIoana Radulescu } while (ret); 137034ff6846SIoana Radulescu } 137134ff6846SIoana Radulescu 13725d8dccf8SIoana Ciornei static void dpaa2_eth_drain_pool(struct dpaa2_eth_priv *priv) 137334ff6846SIoana Radulescu { 137434ff6846SIoana Radulescu int i; 137534ff6846SIoana Radulescu 13765d8dccf8SIoana Ciornei dpaa2_eth_drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 13775d8dccf8SIoana Ciornei dpaa2_eth_drain_bufs(priv, 1); 137834ff6846SIoana Radulescu 137934ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 138034ff6846SIoana Radulescu priv->channel[i]->buf_count = 0; 138134ff6846SIoana Radulescu } 138234ff6846SIoana Radulescu 138334ff6846SIoana Radulescu /* Function is called from softirq context only, so we don't need to guard 138434ff6846SIoana Radulescu * the access to percpu count 138534ff6846SIoana Radulescu */ 13865d8dccf8SIoana Ciornei static int dpaa2_eth_refill_pool(struct dpaa2_eth_priv *priv, 138734ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 138834ff6846SIoana Radulescu u16 bpid) 138934ff6846SIoana Radulescu { 139034ff6846SIoana Radulescu int new_count; 139134ff6846SIoana Radulescu 139234ff6846SIoana Radulescu if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 139334ff6846SIoana Radulescu return 0; 139434ff6846SIoana Radulescu 139534ff6846SIoana Radulescu do { 13965d8dccf8SIoana Ciornei new_count = dpaa2_eth_add_bufs(priv, ch, bpid); 139734ff6846SIoana Radulescu if (unlikely(!new_count)) { 139834ff6846SIoana Radulescu /* Out of memory; abort for now, we'll try later on */ 139934ff6846SIoana Radulescu break; 140034ff6846SIoana Radulescu } 140134ff6846SIoana Radulescu ch->buf_count += new_count; 140234ff6846SIoana Radulescu } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 140334ff6846SIoana Radulescu 140434ff6846SIoana Radulescu if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 140534ff6846SIoana Radulescu return -ENOMEM; 140634ff6846SIoana Radulescu 140734ff6846SIoana Radulescu return 0; 140834ff6846SIoana Radulescu } 140934ff6846SIoana Radulescu 1410d70446eeSIoana Ciornei static void dpaa2_eth_sgt_cache_drain(struct dpaa2_eth_priv *priv) 1411d70446eeSIoana Ciornei { 1412d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 1413d70446eeSIoana Ciornei u16 count; 1414d70446eeSIoana Ciornei int k, i; 1415d70446eeSIoana Ciornei 14160fe665d4SIoana Ciornei for_each_possible_cpu(k) { 1417d70446eeSIoana Ciornei sgt_cache = per_cpu_ptr(priv->sgt_cache, k); 1418d70446eeSIoana Ciornei count = sgt_cache->count; 1419d70446eeSIoana Ciornei 1420d70446eeSIoana Ciornei for (i = 0; i < count; i++) 1421d70446eeSIoana Ciornei kfree(sgt_cache->buf[i]); 1422d70446eeSIoana Ciornei sgt_cache->count = 0; 1423d70446eeSIoana Ciornei } 1424d70446eeSIoana Ciornei } 1425d70446eeSIoana Ciornei 14265d8dccf8SIoana Ciornei static int dpaa2_eth_pull_channel(struct dpaa2_eth_channel *ch) 142734ff6846SIoana Radulescu { 142834ff6846SIoana Radulescu int err; 142934ff6846SIoana Radulescu int dequeues = -1; 143034ff6846SIoana Radulescu 143134ff6846SIoana Radulescu /* Retry while portal is busy */ 143234ff6846SIoana Radulescu do { 143334ff6846SIoana Radulescu err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 143434ff6846SIoana Radulescu ch->store); 143534ff6846SIoana Radulescu dequeues++; 143634ff6846SIoana Radulescu cpu_relax(); 1437ef17bd7cSIoana Radulescu } while (err == -EBUSY && dequeues < DPAA2_ETH_SWP_BUSY_RETRIES); 143834ff6846SIoana Radulescu 143934ff6846SIoana Radulescu ch->stats.dequeue_portal_busy += dequeues; 144034ff6846SIoana Radulescu if (unlikely(err)) 144134ff6846SIoana Radulescu ch->stats.pull_err++; 144234ff6846SIoana Radulescu 144334ff6846SIoana Radulescu return err; 144434ff6846SIoana Radulescu } 144534ff6846SIoana Radulescu 144634ff6846SIoana Radulescu /* NAPI poll routine 144734ff6846SIoana Radulescu * 144834ff6846SIoana Radulescu * Frames are dequeued from the QMan channel associated with this NAPI context. 144934ff6846SIoana Radulescu * Rx, Tx confirmation and (if configured) Rx error frames all count 145034ff6846SIoana Radulescu * towards the NAPI budget. 145134ff6846SIoana Radulescu */ 145234ff6846SIoana Radulescu static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 145334ff6846SIoana Radulescu { 145434ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 145534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 145668049a5fSIoana Ciocoi Radulescu int rx_cleaned = 0, txconf_cleaned = 0; 1457569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1458569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1459569dac6aSIoana Ciocoi Radulescu int store_cleaned, work_done; 14600a25d92cSIoana Ciornei struct list_head rx_list; 1461ef17bd7cSIoana Radulescu int retries = 0; 146274a1c059SIoana Ciornei u16 flowid; 146334ff6846SIoana Radulescu int err; 146434ff6846SIoana Radulescu 146534ff6846SIoana Radulescu ch = container_of(napi, struct dpaa2_eth_channel, napi); 1466d678be1dSIoana Radulescu ch->xdp.res = 0; 146734ff6846SIoana Radulescu priv = ch->priv; 146834ff6846SIoana Radulescu 14690a25d92cSIoana Ciornei INIT_LIST_HEAD(&rx_list); 14700a25d92cSIoana Ciornei ch->rx_list = &rx_list; 14710a25d92cSIoana Ciornei 147268049a5fSIoana Ciocoi Radulescu do { 14735d8dccf8SIoana Ciornei err = dpaa2_eth_pull_channel(ch); 147434ff6846SIoana Radulescu if (unlikely(err)) 147534ff6846SIoana Radulescu break; 147634ff6846SIoana Radulescu 147734ff6846SIoana Radulescu /* Refill pool if appropriate */ 14785d8dccf8SIoana Ciornei dpaa2_eth_refill_pool(priv, ch, priv->bpid); 147934ff6846SIoana Radulescu 14805d8dccf8SIoana Ciornei store_cleaned = dpaa2_eth_consume_frames(ch, &fq); 1481ef17bd7cSIoana Radulescu if (store_cleaned <= 0) 1482569dac6aSIoana Ciocoi Radulescu break; 1483569dac6aSIoana Ciocoi Radulescu if (fq->type == DPAA2_RX_FQ) { 148468049a5fSIoana Ciocoi Radulescu rx_cleaned += store_cleaned; 148574a1c059SIoana Ciornei flowid = fq->flowid; 1486569dac6aSIoana Ciocoi Radulescu } else { 148768049a5fSIoana Ciocoi Radulescu txconf_cleaned += store_cleaned; 1488569dac6aSIoana Ciocoi Radulescu /* We have a single Tx conf FQ on this channel */ 1489569dac6aSIoana Ciocoi Radulescu txc_fq = fq; 1490569dac6aSIoana Ciocoi Radulescu } 149134ff6846SIoana Radulescu 149268049a5fSIoana Ciocoi Radulescu /* If we either consumed the whole NAPI budget with Rx frames 149368049a5fSIoana Ciocoi Radulescu * or we reached the Tx confirmations threshold, we're done. 149434ff6846SIoana Radulescu */ 149568049a5fSIoana Ciocoi Radulescu if (rx_cleaned >= budget || 1496569dac6aSIoana Ciocoi Radulescu txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1497569dac6aSIoana Ciocoi Radulescu work_done = budget; 1498569dac6aSIoana Ciocoi Radulescu goto out; 1499569dac6aSIoana Ciocoi Radulescu } 150068049a5fSIoana Ciocoi Radulescu } while (store_cleaned); 150134ff6846SIoana Radulescu 150268049a5fSIoana Ciocoi Radulescu /* We didn't consume the entire budget, so finish napi and 150368049a5fSIoana Ciocoi Radulescu * re-enable data availability notifications 150468049a5fSIoana Ciocoi Radulescu */ 150568049a5fSIoana Ciocoi Radulescu napi_complete_done(napi, rx_cleaned); 150634ff6846SIoana Radulescu do { 150734ff6846SIoana Radulescu err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 150834ff6846SIoana Radulescu cpu_relax(); 1509ef17bd7cSIoana Radulescu } while (err == -EBUSY && retries++ < DPAA2_ETH_SWP_BUSY_RETRIES); 151034ff6846SIoana Radulescu WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 151134ff6846SIoana Radulescu ch->nctx.desired_cpu); 151234ff6846SIoana Radulescu 1513569dac6aSIoana Ciocoi Radulescu work_done = max(rx_cleaned, 1); 1514569dac6aSIoana Ciocoi Radulescu 1515569dac6aSIoana Ciocoi Radulescu out: 15160a25d92cSIoana Ciornei netif_receive_skb_list(ch->rx_list); 15170a25d92cSIoana Ciornei 1518d678be1dSIoana Radulescu if (txc_fq && txc_fq->dq_frames) { 1519569dac6aSIoana Ciocoi Radulescu nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1520569dac6aSIoana Ciocoi Radulescu netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1521569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes); 1522569dac6aSIoana Ciocoi Radulescu txc_fq->dq_frames = 0; 1523569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes = 0; 1524569dac6aSIoana Ciocoi Radulescu } 1525569dac6aSIoana Ciocoi Radulescu 1526d678be1dSIoana Radulescu if (ch->xdp.res & XDP_REDIRECT) 1527d678be1dSIoana Radulescu xdp_do_flush_map(); 152874a1c059SIoana Ciornei else if (rx_cleaned && ch->xdp.res & XDP_TX) 15295d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, &priv->fq[flowid]); 1530d678be1dSIoana Radulescu 1531569dac6aSIoana Ciocoi Radulescu return work_done; 153234ff6846SIoana Radulescu } 153334ff6846SIoana Radulescu 15345d8dccf8SIoana Ciornei static void dpaa2_eth_enable_ch_napi(struct dpaa2_eth_priv *priv) 153534ff6846SIoana Radulescu { 153634ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 153734ff6846SIoana Radulescu int i; 153834ff6846SIoana Radulescu 153934ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 154034ff6846SIoana Radulescu ch = priv->channel[i]; 154134ff6846SIoana Radulescu napi_enable(&ch->napi); 154234ff6846SIoana Radulescu } 154334ff6846SIoana Radulescu } 154434ff6846SIoana Radulescu 15455d8dccf8SIoana Ciornei static void dpaa2_eth_disable_ch_napi(struct dpaa2_eth_priv *priv) 154634ff6846SIoana Radulescu { 154734ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 154834ff6846SIoana Radulescu int i; 154934ff6846SIoana Radulescu 155034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 155134ff6846SIoana Radulescu ch = priv->channel[i]; 155234ff6846SIoana Radulescu napi_disable(&ch->napi); 155334ff6846SIoana Radulescu } 155434ff6846SIoana Radulescu } 155534ff6846SIoana Radulescu 155607beb165SIoana Ciornei void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, 155707beb165SIoana Ciornei bool tx_pause, bool pfc) 15588eb3cef8SIoana Radulescu { 15598eb3cef8SIoana Radulescu struct dpni_taildrop td = {0}; 1560685e39eaSIoana Radulescu struct dpaa2_eth_fq *fq; 15618eb3cef8SIoana Radulescu int i, err; 15628eb3cef8SIoana Radulescu 156307beb165SIoana Ciornei /* FQ taildrop: threshold is in bytes, per frame queue. Enabled if 156407beb165SIoana Ciornei * flow control is disabled (as it might interfere with either the 156507beb165SIoana Ciornei * buffer pool depletion trigger for pause frames or with the group 156607beb165SIoana Ciornei * congestion trigger for PFC frames) 156707beb165SIoana Ciornei */ 15682c8d1c8dSIoana Radulescu td.enable = !tx_pause; 156907beb165SIoana Ciornei if (priv->rx_fqtd_enabled == td.enable) 157007beb165SIoana Ciornei goto set_cgtd; 15718eb3cef8SIoana Radulescu 15722c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_FQ_TAILDROP_THRESH; 15732c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_BYTES; 15748eb3cef8SIoana Radulescu 15758eb3cef8SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 1576685e39eaSIoana Radulescu fq = &priv->fq[i]; 1577685e39eaSIoana Radulescu if (fq->type != DPAA2_RX_FQ) 15788eb3cef8SIoana Radulescu continue; 15798eb3cef8SIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1580685e39eaSIoana Radulescu DPNI_CP_QUEUE, DPNI_QUEUE_RX, 1581685e39eaSIoana Radulescu fq->tc, fq->flowid, &td); 15828eb3cef8SIoana Radulescu if (err) { 15838eb3cef8SIoana Radulescu netdev_err(priv->net_dev, 15842c8d1c8dSIoana Radulescu "dpni_set_taildrop(FQ) failed\n"); 15852c8d1c8dSIoana Radulescu return; 15868eb3cef8SIoana Radulescu } 15878eb3cef8SIoana Radulescu } 15888eb3cef8SIoana Radulescu 158907beb165SIoana Ciornei priv->rx_fqtd_enabled = td.enable; 159007beb165SIoana Ciornei 159107beb165SIoana Ciornei set_cgtd: 15922c8d1c8dSIoana Radulescu /* Congestion group taildrop: threshold is in frames, per group 15932c8d1c8dSIoana Radulescu * of FQs belonging to the same traffic class 159407beb165SIoana Ciornei * Enabled if general Tx pause disabled or if PFCs are enabled 159507beb165SIoana Ciornei * (congestion group threhsold for PFC generation is lower than the 159607beb165SIoana Ciornei * CG taildrop threshold, so it won't interfere with it; we also 159707beb165SIoana Ciornei * want frames in non-PFC enabled traffic classes to be kept in check) 15982c8d1c8dSIoana Radulescu */ 159907beb165SIoana Ciornei td.enable = !tx_pause || (tx_pause && pfc); 160007beb165SIoana Ciornei if (priv->rx_cgtd_enabled == td.enable) 160107beb165SIoana Ciornei return; 160207beb165SIoana Ciornei 16032c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_CG_TAILDROP_THRESH(priv); 16042c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_FRAMES; 16052c8d1c8dSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 16062c8d1c8dSIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 16072c8d1c8dSIoana Radulescu DPNI_CP_GROUP, DPNI_QUEUE_RX, 16082c8d1c8dSIoana Radulescu i, 0, &td); 16092c8d1c8dSIoana Radulescu if (err) { 16102c8d1c8dSIoana Radulescu netdev_err(priv->net_dev, 16112c8d1c8dSIoana Radulescu "dpni_set_taildrop(CG) failed\n"); 16122c8d1c8dSIoana Radulescu return; 16132c8d1c8dSIoana Radulescu } 16142c8d1c8dSIoana Radulescu } 16152c8d1c8dSIoana Radulescu 161607beb165SIoana Ciornei priv->rx_cgtd_enabled = td.enable; 16178eb3cef8SIoana Radulescu } 16188eb3cef8SIoana Radulescu 16195d8dccf8SIoana Ciornei static int dpaa2_eth_link_state_update(struct dpaa2_eth_priv *priv) 162034ff6846SIoana Radulescu { 162185b7a342SIoana Ciornei struct dpni_link_state state = {0}; 16228eb3cef8SIoana Radulescu bool tx_pause; 162334ff6846SIoana Radulescu int err; 162434ff6846SIoana Radulescu 162534ff6846SIoana Radulescu err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 162634ff6846SIoana Radulescu if (unlikely(err)) { 162734ff6846SIoana Radulescu netdev_err(priv->net_dev, 162834ff6846SIoana Radulescu "dpni_get_link_state() failed\n"); 162934ff6846SIoana Radulescu return err; 163034ff6846SIoana Radulescu } 163134ff6846SIoana Radulescu 16328eb3cef8SIoana Radulescu /* If Tx pause frame settings have changed, we need to update 16338eb3cef8SIoana Radulescu * Rx FQ taildrop configuration as well. We configure taildrop 16348eb3cef8SIoana Radulescu * only when pause frame generation is disabled. 16358eb3cef8SIoana Radulescu */ 1636ad054f26SIoana Radulescu tx_pause = dpaa2_eth_tx_pause_enabled(state.options); 163707beb165SIoana Ciornei dpaa2_eth_set_rx_taildrop(priv, tx_pause, priv->pfc_enabled); 16388eb3cef8SIoana Radulescu 163971947923SIoana Ciornei /* When we manage the MAC/PHY using phylink there is no need 164071947923SIoana Ciornei * to manually update the netif_carrier. 164171947923SIoana Ciornei */ 164271947923SIoana Ciornei if (priv->mac) 164371947923SIoana Ciornei goto out; 164471947923SIoana Ciornei 164534ff6846SIoana Radulescu /* Chech link state; speed / duplex changes are not treated yet */ 164634ff6846SIoana Radulescu if (priv->link_state.up == state.up) 1647cce62943SIoana Radulescu goto out; 164834ff6846SIoana Radulescu 164934ff6846SIoana Radulescu if (state.up) { 165034ff6846SIoana Radulescu netif_carrier_on(priv->net_dev); 165134ff6846SIoana Radulescu netif_tx_start_all_queues(priv->net_dev); 165234ff6846SIoana Radulescu } else { 165334ff6846SIoana Radulescu netif_tx_stop_all_queues(priv->net_dev); 165434ff6846SIoana Radulescu netif_carrier_off(priv->net_dev); 165534ff6846SIoana Radulescu } 165634ff6846SIoana Radulescu 165734ff6846SIoana Radulescu netdev_info(priv->net_dev, "Link Event: state %s\n", 165834ff6846SIoana Radulescu state.up ? "up" : "down"); 165934ff6846SIoana Radulescu 1660cce62943SIoana Radulescu out: 1661cce62943SIoana Radulescu priv->link_state = state; 1662cce62943SIoana Radulescu 166334ff6846SIoana Radulescu return 0; 166434ff6846SIoana Radulescu } 166534ff6846SIoana Radulescu 166634ff6846SIoana Radulescu static int dpaa2_eth_open(struct net_device *net_dev) 166734ff6846SIoana Radulescu { 166834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 166934ff6846SIoana Radulescu int err; 167034ff6846SIoana Radulescu 16715d8dccf8SIoana Ciornei err = dpaa2_eth_seed_pool(priv, priv->bpid); 167234ff6846SIoana Radulescu if (err) { 167334ff6846SIoana Radulescu /* Not much to do; the buffer pool, though not filled up, 167434ff6846SIoana Radulescu * may still contain some buffers which would enable us 167534ff6846SIoana Radulescu * to limp on. 167634ff6846SIoana Radulescu */ 167734ff6846SIoana Radulescu netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 167834ff6846SIoana Radulescu priv->dpbp_dev->obj_desc.id, priv->bpid); 167934ff6846SIoana Radulescu } 168034ff6846SIoana Radulescu 168171947923SIoana Ciornei if (!priv->mac) { 168271947923SIoana Ciornei /* We'll only start the txqs when the link is actually ready; 168371947923SIoana Ciornei * make sure we don't race against the link up notification, 168471947923SIoana Ciornei * which may come immediately after dpni_enable(); 168534ff6846SIoana Radulescu */ 168634ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 168771947923SIoana Ciornei 168871947923SIoana Ciornei /* Also, explicitly set carrier off, otherwise 168971947923SIoana Ciornei * netif_carrier_ok() will return true and cause 'ip link show' 169071947923SIoana Ciornei * to report the LOWER_UP flag, even though the link 169171947923SIoana Ciornei * notification wasn't even received. 169234ff6846SIoana Radulescu */ 169334ff6846SIoana Radulescu netif_carrier_off(net_dev); 169471947923SIoana Ciornei } 16955d8dccf8SIoana Ciornei dpaa2_eth_enable_ch_napi(priv); 169634ff6846SIoana Radulescu 169734ff6846SIoana Radulescu err = dpni_enable(priv->mc_io, 0, priv->mc_token); 169834ff6846SIoana Radulescu if (err < 0) { 169934ff6846SIoana Radulescu netdev_err(net_dev, "dpni_enable() failed\n"); 170034ff6846SIoana Radulescu goto enable_err; 170134ff6846SIoana Radulescu } 170234ff6846SIoana Radulescu 17034c33a5bdSIoana Ciornei if (priv->mac) 170471947923SIoana Ciornei phylink_start(priv->mac->phylink); 170534ff6846SIoana Radulescu 170634ff6846SIoana Radulescu return 0; 170734ff6846SIoana Radulescu 170834ff6846SIoana Radulescu enable_err: 17095d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 17105d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 171134ff6846SIoana Radulescu return err; 171234ff6846SIoana Radulescu } 171334ff6846SIoana Radulescu 171468d74315SIoana Ciocoi Radulescu /* Total number of in-flight frames on ingress queues */ 17155d8dccf8SIoana Ciornei static u32 dpaa2_eth_ingress_fq_count(struct dpaa2_eth_priv *priv) 171634ff6846SIoana Radulescu { 171768d74315SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq; 171868d74315SIoana Ciocoi Radulescu u32 fcnt = 0, bcnt = 0, total = 0; 171968d74315SIoana Ciocoi Radulescu int i, err; 172034ff6846SIoana Radulescu 172168d74315SIoana Ciocoi Radulescu for (i = 0; i < priv->num_fqs; i++) { 172268d74315SIoana Ciocoi Radulescu fq = &priv->fq[i]; 172368d74315SIoana Ciocoi Radulescu err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 172468d74315SIoana Ciocoi Radulescu if (err) { 172568d74315SIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "query_fq_count failed"); 172668d74315SIoana Ciocoi Radulescu break; 172768d74315SIoana Ciocoi Radulescu } 172868d74315SIoana Ciocoi Radulescu total += fcnt; 172968d74315SIoana Ciocoi Radulescu } 173034ff6846SIoana Radulescu 173134ff6846SIoana Radulescu return total; 173234ff6846SIoana Radulescu } 173334ff6846SIoana Radulescu 17345d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 173534ff6846SIoana Radulescu { 173668d74315SIoana Ciocoi Radulescu int retries = 10; 173768d74315SIoana Ciocoi Radulescu u32 pending; 173834ff6846SIoana Radulescu 173968d74315SIoana Ciocoi Radulescu do { 17405d8dccf8SIoana Ciornei pending = dpaa2_eth_ingress_fq_count(priv); 174168d74315SIoana Ciocoi Radulescu if (pending) 174268d74315SIoana Ciocoi Radulescu msleep(100); 174368d74315SIoana Ciocoi Radulescu } while (pending && --retries); 174434ff6846SIoana Radulescu } 174534ff6846SIoana Radulescu 174652b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MAJOR 7 174752b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MINOR 13 17485d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 174952b6a4ffSIoana Radulescu { 175052b6a4ffSIoana Radulescu union dpni_statistics stats; 175152b6a4ffSIoana Radulescu int retries = 10; 175252b6a4ffSIoana Radulescu int err; 175352b6a4ffSIoana Radulescu 175452b6a4ffSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 175552b6a4ffSIoana Radulescu DPNI_TX_PENDING_VER_MINOR) < 0) 175652b6a4ffSIoana Radulescu goto out; 175752b6a4ffSIoana Radulescu 175852b6a4ffSIoana Radulescu do { 175952b6a4ffSIoana Radulescu err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 176052b6a4ffSIoana Radulescu &stats); 176152b6a4ffSIoana Radulescu if (err) 176252b6a4ffSIoana Radulescu goto out; 176352b6a4ffSIoana Radulescu if (stats.page_6.tx_pending_frames == 0) 176452b6a4ffSIoana Radulescu return; 176552b6a4ffSIoana Radulescu } while (--retries); 176652b6a4ffSIoana Radulescu 176752b6a4ffSIoana Radulescu out: 176852b6a4ffSIoana Radulescu msleep(500); 176952b6a4ffSIoana Radulescu } 177052b6a4ffSIoana Radulescu 177134ff6846SIoana Radulescu static int dpaa2_eth_stop(struct net_device *net_dev) 177234ff6846SIoana Radulescu { 177334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 177485b7a342SIoana Ciornei int dpni_enabled = 0; 177534ff6846SIoana Radulescu int retries = 10; 177634ff6846SIoana Radulescu 177771947923SIoana Ciornei if (!priv->mac) { 177834ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 177934ff6846SIoana Radulescu netif_carrier_off(net_dev); 178071947923SIoana Ciornei } else { 178171947923SIoana Ciornei phylink_stop(priv->mac->phylink); 178271947923SIoana Ciornei } 178334ff6846SIoana Radulescu 178468d74315SIoana Ciocoi Radulescu /* On dpni_disable(), the MC firmware will: 178568d74315SIoana Ciocoi Radulescu * - stop MAC Rx and wait for all Rx frames to be enqueued to software 178668d74315SIoana Ciocoi Radulescu * - cut off WRIOP dequeues from egress FQs and wait until transmission 178768d74315SIoana Ciocoi Radulescu * of all in flight Tx frames is finished (and corresponding Tx conf 178868d74315SIoana Ciocoi Radulescu * frames are enqueued back to software) 178968d74315SIoana Ciocoi Radulescu * 179068d74315SIoana Ciocoi Radulescu * Before calling dpni_disable(), we wait for all Tx frames to arrive 179168d74315SIoana Ciocoi Radulescu * on WRIOP. After it finishes, wait until all remaining frames on Rx 179268d74315SIoana Ciocoi Radulescu * and Tx conf queues are consumed on NAPI poll. 179334ff6846SIoana Radulescu */ 17945d8dccf8SIoana Ciornei dpaa2_eth_wait_for_egress_fq_empty(priv); 179568d74315SIoana Ciocoi Radulescu 179634ff6846SIoana Radulescu do { 179734ff6846SIoana Radulescu dpni_disable(priv->mc_io, 0, priv->mc_token); 179834ff6846SIoana Radulescu dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 179934ff6846SIoana Radulescu if (dpni_enabled) 180034ff6846SIoana Radulescu /* Allow the hardware some slack */ 180134ff6846SIoana Radulescu msleep(100); 180234ff6846SIoana Radulescu } while (dpni_enabled && --retries); 180334ff6846SIoana Radulescu if (!retries) { 180434ff6846SIoana Radulescu netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 180534ff6846SIoana Radulescu /* Must go on and disable NAPI nonetheless, so we don't crash at 180634ff6846SIoana Radulescu * the next "ifconfig up" 180734ff6846SIoana Radulescu */ 180834ff6846SIoana Radulescu } 180934ff6846SIoana Radulescu 18105d8dccf8SIoana Ciornei dpaa2_eth_wait_for_ingress_fq_empty(priv); 18115d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 181234ff6846SIoana Radulescu 181334ff6846SIoana Radulescu /* Empty the buffer pool */ 18145d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 181534ff6846SIoana Radulescu 1816d70446eeSIoana Ciornei /* Empty the Scatter-Gather Buffer cache */ 1817d70446eeSIoana Ciornei dpaa2_eth_sgt_cache_drain(priv); 1818d70446eeSIoana Ciornei 181934ff6846SIoana Radulescu return 0; 182034ff6846SIoana Radulescu } 182134ff6846SIoana Radulescu 182234ff6846SIoana Radulescu static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 182334ff6846SIoana Radulescu { 182434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 182534ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 182634ff6846SIoana Radulescu int err; 182734ff6846SIoana Radulescu 182834ff6846SIoana Radulescu err = eth_mac_addr(net_dev, addr); 182934ff6846SIoana Radulescu if (err < 0) { 183034ff6846SIoana Radulescu dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 183134ff6846SIoana Radulescu return err; 183234ff6846SIoana Radulescu } 183334ff6846SIoana Radulescu 183434ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 183534ff6846SIoana Radulescu net_dev->dev_addr); 183634ff6846SIoana Radulescu if (err) { 183734ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 183834ff6846SIoana Radulescu return err; 183934ff6846SIoana Radulescu } 184034ff6846SIoana Radulescu 184134ff6846SIoana Radulescu return 0; 184234ff6846SIoana Radulescu } 184334ff6846SIoana Radulescu 184434ff6846SIoana Radulescu /** Fill in counters maintained by the GPP driver. These may be different from 184534ff6846SIoana Radulescu * the hardware counters obtained by ethtool. 184634ff6846SIoana Radulescu */ 184734ff6846SIoana Radulescu static void dpaa2_eth_get_stats(struct net_device *net_dev, 184834ff6846SIoana Radulescu struct rtnl_link_stats64 *stats) 184934ff6846SIoana Radulescu { 185034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 185134ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 185234ff6846SIoana Radulescu u64 *cpustats; 185334ff6846SIoana Radulescu u64 *netstats = (u64 *)stats; 185434ff6846SIoana Radulescu int i, j; 185534ff6846SIoana Radulescu int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 185634ff6846SIoana Radulescu 185734ff6846SIoana Radulescu for_each_possible_cpu(i) { 185834ff6846SIoana Radulescu percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 185934ff6846SIoana Radulescu cpustats = (u64 *)percpu_stats; 186034ff6846SIoana Radulescu for (j = 0; j < num; j++) 186134ff6846SIoana Radulescu netstats[j] += cpustats[j]; 186234ff6846SIoana Radulescu } 186334ff6846SIoana Radulescu } 186434ff6846SIoana Radulescu 186534ff6846SIoana Radulescu /* Copy mac unicast addresses from @net_dev to @priv. 186634ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 186734ff6846SIoana Radulescu */ 18685d8dccf8SIoana Ciornei static void dpaa2_eth_add_uc_hw_addr(const struct net_device *net_dev, 186934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 187034ff6846SIoana Radulescu { 187134ff6846SIoana Radulescu struct netdev_hw_addr *ha; 187234ff6846SIoana Radulescu int err; 187334ff6846SIoana Radulescu 187434ff6846SIoana Radulescu netdev_for_each_uc_addr(ha, net_dev) { 187534ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 187634ff6846SIoana Radulescu ha->addr); 187734ff6846SIoana Radulescu if (err) 187834ff6846SIoana Radulescu netdev_warn(priv->net_dev, 187934ff6846SIoana Radulescu "Could not add ucast MAC %pM to the filtering table (err %d)\n", 188034ff6846SIoana Radulescu ha->addr, err); 188134ff6846SIoana Radulescu } 188234ff6846SIoana Radulescu } 188334ff6846SIoana Radulescu 188434ff6846SIoana Radulescu /* Copy mac multicast addresses from @net_dev to @priv 188534ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 188634ff6846SIoana Radulescu */ 18875d8dccf8SIoana Ciornei static void dpaa2_eth_add_mc_hw_addr(const struct net_device *net_dev, 188834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 188934ff6846SIoana Radulescu { 189034ff6846SIoana Radulescu struct netdev_hw_addr *ha; 189134ff6846SIoana Radulescu int err; 189234ff6846SIoana Radulescu 189334ff6846SIoana Radulescu netdev_for_each_mc_addr(ha, net_dev) { 189434ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 189534ff6846SIoana Radulescu ha->addr); 189634ff6846SIoana Radulescu if (err) 189734ff6846SIoana Radulescu netdev_warn(priv->net_dev, 189834ff6846SIoana Radulescu "Could not add mcast MAC %pM to the filtering table (err %d)\n", 189934ff6846SIoana Radulescu ha->addr, err); 190034ff6846SIoana Radulescu } 190134ff6846SIoana Radulescu } 190234ff6846SIoana Radulescu 190334ff6846SIoana Radulescu static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 190434ff6846SIoana Radulescu { 190534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 190634ff6846SIoana Radulescu int uc_count = netdev_uc_count(net_dev); 190734ff6846SIoana Radulescu int mc_count = netdev_mc_count(net_dev); 190834ff6846SIoana Radulescu u8 max_mac = priv->dpni_attrs.mac_filter_entries; 190934ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 191034ff6846SIoana Radulescu u16 mc_token = priv->mc_token; 191134ff6846SIoana Radulescu struct fsl_mc_io *mc_io = priv->mc_io; 191234ff6846SIoana Radulescu int err; 191334ff6846SIoana Radulescu 191434ff6846SIoana Radulescu /* Basic sanity checks; these probably indicate a misconfiguration */ 191534ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 191634ff6846SIoana Radulescu netdev_info(net_dev, 191734ff6846SIoana Radulescu "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 191834ff6846SIoana Radulescu max_mac); 191934ff6846SIoana Radulescu 192034ff6846SIoana Radulescu /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 192134ff6846SIoana Radulescu if (uc_count > max_mac) { 192234ff6846SIoana Radulescu netdev_info(net_dev, 192334ff6846SIoana Radulescu "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 192434ff6846SIoana Radulescu uc_count, max_mac); 192534ff6846SIoana Radulescu goto force_promisc; 192634ff6846SIoana Radulescu } 192734ff6846SIoana Radulescu if (mc_count + uc_count > max_mac) { 192834ff6846SIoana Radulescu netdev_info(net_dev, 192934ff6846SIoana Radulescu "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 193034ff6846SIoana Radulescu uc_count + mc_count, max_mac); 193134ff6846SIoana Radulescu goto force_mc_promisc; 193234ff6846SIoana Radulescu } 193334ff6846SIoana Radulescu 193434ff6846SIoana Radulescu /* Adjust promisc settings due to flag combinations */ 193534ff6846SIoana Radulescu if (net_dev->flags & IFF_PROMISC) 193634ff6846SIoana Radulescu goto force_promisc; 193734ff6846SIoana Radulescu if (net_dev->flags & IFF_ALLMULTI) { 193834ff6846SIoana Radulescu /* First, rebuild unicast filtering table. This should be done 193934ff6846SIoana Radulescu * in promisc mode, in order to avoid frame loss while we 194034ff6846SIoana Radulescu * progressively add entries to the table. 194134ff6846SIoana Radulescu * We don't know whether we had been in promisc already, and 194234ff6846SIoana Radulescu * making an MC call to find out is expensive; so set uc promisc 194334ff6846SIoana Radulescu * nonetheless. 194434ff6846SIoana Radulescu */ 194534ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 194634ff6846SIoana Radulescu if (err) 194734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc\n"); 194834ff6846SIoana Radulescu 194934ff6846SIoana Radulescu /* Actual uc table reconstruction. */ 195034ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 195134ff6846SIoana Radulescu if (err) 195234ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc filters\n"); 19535d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 195434ff6846SIoana Radulescu 195534ff6846SIoana Radulescu /* Finally, clear uc promisc and set mc promisc as requested. */ 195634ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 195734ff6846SIoana Radulescu if (err) 195834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc promisc\n"); 195934ff6846SIoana Radulescu goto force_mc_promisc; 196034ff6846SIoana Radulescu } 196134ff6846SIoana Radulescu 196234ff6846SIoana Radulescu /* Neither unicast, nor multicast promisc will be on... eventually. 196334ff6846SIoana Radulescu * For now, rebuild mac filtering tables while forcing both of them on. 196434ff6846SIoana Radulescu */ 196534ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 196634ff6846SIoana Radulescu if (err) 196734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 196834ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 196934ff6846SIoana Radulescu if (err) 197034ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 197134ff6846SIoana Radulescu 197234ff6846SIoana Radulescu /* Actual mac filtering tables reconstruction */ 197334ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 197434ff6846SIoana Radulescu if (err) 197534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mac filters\n"); 19765d8dccf8SIoana Ciornei dpaa2_eth_add_mc_hw_addr(net_dev, priv); 19775d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 197834ff6846SIoana Radulescu 197934ff6846SIoana Radulescu /* Now we can clear both ucast and mcast promisc, without risking 198034ff6846SIoana Radulescu * to drop legitimate frames anymore. 198134ff6846SIoana Radulescu */ 198234ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 198334ff6846SIoana Radulescu if (err) 198434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear ucast promisc\n"); 198534ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 198634ff6846SIoana Radulescu if (err) 198734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mcast promisc\n"); 198834ff6846SIoana Radulescu 198934ff6846SIoana Radulescu return; 199034ff6846SIoana Radulescu 199134ff6846SIoana Radulescu force_promisc: 199234ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 199334ff6846SIoana Radulescu if (err) 199434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set ucast promisc\n"); 199534ff6846SIoana Radulescu force_mc_promisc: 199634ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 199734ff6846SIoana Radulescu if (err) 199834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mcast promisc\n"); 199934ff6846SIoana Radulescu } 200034ff6846SIoana Radulescu 200134ff6846SIoana Radulescu static int dpaa2_eth_set_features(struct net_device *net_dev, 200234ff6846SIoana Radulescu netdev_features_t features) 200334ff6846SIoana Radulescu { 200434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 200534ff6846SIoana Radulescu netdev_features_t changed = features ^ net_dev->features; 200634ff6846SIoana Radulescu bool enable; 200734ff6846SIoana Radulescu int err; 200834ff6846SIoana Radulescu 200934ff6846SIoana Radulescu if (changed & NETIF_F_RXCSUM) { 201034ff6846SIoana Radulescu enable = !!(features & NETIF_F_RXCSUM); 20115d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, enable); 201234ff6846SIoana Radulescu if (err) 201334ff6846SIoana Radulescu return err; 201434ff6846SIoana Radulescu } 201534ff6846SIoana Radulescu 201634ff6846SIoana Radulescu if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 201734ff6846SIoana Radulescu enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 20185d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, enable); 201934ff6846SIoana Radulescu if (err) 202034ff6846SIoana Radulescu return err; 202134ff6846SIoana Radulescu } 202234ff6846SIoana Radulescu 202334ff6846SIoana Radulescu return 0; 202434ff6846SIoana Radulescu } 202534ff6846SIoana Radulescu 202634ff6846SIoana Radulescu static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 202734ff6846SIoana Radulescu { 202834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 202934ff6846SIoana Radulescu struct hwtstamp_config config; 203034ff6846SIoana Radulescu 2031c5521189SYangbo Lu if (!dpaa2_ptp) 2032c5521189SYangbo Lu return -EINVAL; 2033c5521189SYangbo Lu 203434ff6846SIoana Radulescu if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 203534ff6846SIoana Radulescu return -EFAULT; 203634ff6846SIoana Radulescu 203734ff6846SIoana Radulescu switch (config.tx_type) { 203834ff6846SIoana Radulescu case HWTSTAMP_TX_OFF: 203934ff6846SIoana Radulescu case HWTSTAMP_TX_ON: 2040c5521189SYangbo Lu case HWTSTAMP_TX_ONESTEP_SYNC: 20411cf773bdSYangbo Lu priv->tx_tstamp_type = config.tx_type; 204234ff6846SIoana Radulescu break; 204334ff6846SIoana Radulescu default: 204434ff6846SIoana Radulescu return -ERANGE; 204534ff6846SIoana Radulescu } 204634ff6846SIoana Radulescu 204734ff6846SIoana Radulescu if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 204834ff6846SIoana Radulescu priv->rx_tstamp = false; 204934ff6846SIoana Radulescu } else { 205034ff6846SIoana Radulescu priv->rx_tstamp = true; 205134ff6846SIoana Radulescu /* TS is set for all frame types, not only those requested */ 205234ff6846SIoana Radulescu config.rx_filter = HWTSTAMP_FILTER_ALL; 205334ff6846SIoana Radulescu } 205434ff6846SIoana Radulescu 205534ff6846SIoana Radulescu return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 205634ff6846SIoana Radulescu -EFAULT : 0; 205734ff6846SIoana Radulescu } 205834ff6846SIoana Radulescu 205934ff6846SIoana Radulescu static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 206034ff6846SIoana Radulescu { 20614a84182aSRussell King struct dpaa2_eth_priv *priv = netdev_priv(dev); 20624a84182aSRussell King 206334ff6846SIoana Radulescu if (cmd == SIOCSHWTSTAMP) 206434ff6846SIoana Radulescu return dpaa2_eth_ts_ioctl(dev, rq, cmd); 206534ff6846SIoana Radulescu 20664a84182aSRussell King if (priv->mac) 20674a84182aSRussell King return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); 20684a84182aSRussell King 20694a84182aSRussell King return -EOPNOTSUPP; 207034ff6846SIoana Radulescu } 207134ff6846SIoana Radulescu 20727e273a8eSIoana Ciocoi Radulescu static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 20737e273a8eSIoana Ciocoi Radulescu { 20747e273a8eSIoana Ciocoi Radulescu int mfl, linear_mfl; 20757e273a8eSIoana Ciocoi Radulescu 20767e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 2077efa6a7d0SIoana Ciornei linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE - 20787b1eea1aSIoana Ciocoi Radulescu dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 20797e273a8eSIoana Ciocoi Radulescu 20807e273a8eSIoana Ciocoi Radulescu if (mfl > linear_mfl) { 20817e273a8eSIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 20827e273a8eSIoana Ciocoi Radulescu linear_mfl - VLAN_ETH_HLEN); 20837e273a8eSIoana Ciocoi Radulescu return false; 20847e273a8eSIoana Ciocoi Radulescu } 20857e273a8eSIoana Ciocoi Radulescu 20867e273a8eSIoana Ciocoi Radulescu return true; 20877e273a8eSIoana Ciocoi Radulescu } 20887e273a8eSIoana Ciocoi Radulescu 20895d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 20907e273a8eSIoana Ciocoi Radulescu { 20917e273a8eSIoana Ciocoi Radulescu int mfl, err; 20927e273a8eSIoana Ciocoi Radulescu 20937e273a8eSIoana Ciocoi Radulescu /* We enforce a maximum Rx frame length based on MTU only if we have 20947e273a8eSIoana Ciocoi Radulescu * an XDP program attached (in order to avoid Rx S/G frames). 20957e273a8eSIoana Ciocoi Radulescu * Otherwise, we accept all incoming frames as long as they are not 20967e273a8eSIoana Ciocoi Radulescu * larger than maximum size supported in hardware 20977e273a8eSIoana Ciocoi Radulescu */ 20987e273a8eSIoana Ciocoi Radulescu if (has_xdp) 20997e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 21007e273a8eSIoana Ciocoi Radulescu else 21017e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_MFL; 21027e273a8eSIoana Ciocoi Radulescu 21037e273a8eSIoana Ciocoi Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 21047e273a8eSIoana Ciocoi Radulescu if (err) { 21057e273a8eSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 21067e273a8eSIoana Ciocoi Radulescu return err; 21077e273a8eSIoana Ciocoi Radulescu } 21087e273a8eSIoana Ciocoi Radulescu 21097e273a8eSIoana Ciocoi Radulescu return 0; 21107e273a8eSIoana Ciocoi Radulescu } 21117e273a8eSIoana Ciocoi Radulescu 21127e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 21137e273a8eSIoana Ciocoi Radulescu { 21147e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 21157e273a8eSIoana Ciocoi Radulescu int err; 21167e273a8eSIoana Ciocoi Radulescu 21177e273a8eSIoana Ciocoi Radulescu if (!priv->xdp_prog) 21187e273a8eSIoana Ciocoi Radulescu goto out; 21197e273a8eSIoana Ciocoi Radulescu 21207e273a8eSIoana Ciocoi Radulescu if (!xdp_mtu_valid(priv, new_mtu)) 21217e273a8eSIoana Ciocoi Radulescu return -EINVAL; 21227e273a8eSIoana Ciocoi Radulescu 21235d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, new_mtu, true); 21247e273a8eSIoana Ciocoi Radulescu if (err) 21257e273a8eSIoana Ciocoi Radulescu return err; 21267e273a8eSIoana Ciocoi Radulescu 21277e273a8eSIoana Ciocoi Radulescu out: 21287e273a8eSIoana Ciocoi Radulescu dev->mtu = new_mtu; 21297e273a8eSIoana Ciocoi Radulescu return 0; 21307e273a8eSIoana Ciocoi Radulescu } 21317e273a8eSIoana Ciocoi Radulescu 21325d8dccf8SIoana Ciornei static int dpaa2_eth_update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 21337b1eea1aSIoana Ciocoi Radulescu { 21347b1eea1aSIoana Ciocoi Radulescu struct dpni_buffer_layout buf_layout = {0}; 21357b1eea1aSIoana Ciocoi Radulescu int err; 21367b1eea1aSIoana Ciocoi Radulescu 21377b1eea1aSIoana Ciocoi Radulescu err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 21387b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 21397b1eea1aSIoana Ciocoi Radulescu if (err) { 21407b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 21417b1eea1aSIoana Ciocoi Radulescu return err; 21427b1eea1aSIoana Ciocoi Radulescu } 21437b1eea1aSIoana Ciocoi Radulescu 21447b1eea1aSIoana Ciocoi Radulescu /* Reserve extra headroom for XDP header size changes */ 21457b1eea1aSIoana Ciocoi Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 21467b1eea1aSIoana Ciocoi Radulescu (has_xdp ? XDP_PACKET_HEADROOM : 0); 21477b1eea1aSIoana Ciocoi Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 21487b1eea1aSIoana Ciocoi Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 21497b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 21507b1eea1aSIoana Ciocoi Radulescu if (err) { 21517b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 21527b1eea1aSIoana Ciocoi Radulescu return err; 21537b1eea1aSIoana Ciocoi Radulescu } 21547b1eea1aSIoana Ciocoi Radulescu 21557b1eea1aSIoana Ciocoi Radulescu return 0; 21567b1eea1aSIoana Ciocoi Radulescu } 21577b1eea1aSIoana Ciocoi Radulescu 21585d8dccf8SIoana Ciornei static int dpaa2_eth_setup_xdp(struct net_device *dev, struct bpf_prog *prog) 21597e273a8eSIoana Ciocoi Radulescu { 21607e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 21617e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch; 21627e273a8eSIoana Ciocoi Radulescu struct bpf_prog *old; 21637e273a8eSIoana Ciocoi Radulescu bool up, need_update; 21647e273a8eSIoana Ciocoi Radulescu int i, err; 21657e273a8eSIoana Ciocoi Radulescu 21667e273a8eSIoana Ciocoi Radulescu if (prog && !xdp_mtu_valid(priv, dev->mtu)) 21677e273a8eSIoana Ciocoi Radulescu return -EINVAL; 21687e273a8eSIoana Ciocoi Radulescu 216985192dbfSAndrii Nakryiko if (prog) 217085192dbfSAndrii Nakryiko bpf_prog_add(prog, priv->num_channels); 21717e273a8eSIoana Ciocoi Radulescu 21727e273a8eSIoana Ciocoi Radulescu up = netif_running(dev); 21737e273a8eSIoana Ciocoi Radulescu need_update = (!!priv->xdp_prog != !!prog); 21747e273a8eSIoana Ciocoi Radulescu 21757e273a8eSIoana Ciocoi Radulescu if (up) 21767e273a8eSIoana Ciocoi Radulescu dpaa2_eth_stop(dev); 21777e273a8eSIoana Ciocoi Radulescu 21787b1eea1aSIoana Ciocoi Radulescu /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 21797b1eea1aSIoana Ciocoi Radulescu * Also, when switching between xdp/non-xdp modes we need to reconfigure 21807b1eea1aSIoana Ciocoi Radulescu * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 21817b1eea1aSIoana Ciocoi Radulescu * so we are sure no old format buffers will be used from now on. 21827b1eea1aSIoana Ciocoi Radulescu */ 21837e273a8eSIoana Ciocoi Radulescu if (need_update) { 21845d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, dev->mtu, !!prog); 21857e273a8eSIoana Ciocoi Radulescu if (err) 21867e273a8eSIoana Ciocoi Radulescu goto out_err; 21875d8dccf8SIoana Ciornei err = dpaa2_eth_update_rx_buffer_headroom(priv, !!prog); 21887b1eea1aSIoana Ciocoi Radulescu if (err) 21897b1eea1aSIoana Ciocoi Radulescu goto out_err; 21907e273a8eSIoana Ciocoi Radulescu } 21917e273a8eSIoana Ciocoi Radulescu 21927e273a8eSIoana Ciocoi Radulescu old = xchg(&priv->xdp_prog, prog); 21937e273a8eSIoana Ciocoi Radulescu if (old) 21947e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 21957e273a8eSIoana Ciocoi Radulescu 21967e273a8eSIoana Ciocoi Radulescu for (i = 0; i < priv->num_channels; i++) { 21977e273a8eSIoana Ciocoi Radulescu ch = priv->channel[i]; 21987e273a8eSIoana Ciocoi Radulescu old = xchg(&ch->xdp.prog, prog); 21997e273a8eSIoana Ciocoi Radulescu if (old) 22007e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 22017e273a8eSIoana Ciocoi Radulescu } 22027e273a8eSIoana Ciocoi Radulescu 22037e273a8eSIoana Ciocoi Radulescu if (up) { 22047e273a8eSIoana Ciocoi Radulescu err = dpaa2_eth_open(dev); 22057e273a8eSIoana Ciocoi Radulescu if (err) 22067e273a8eSIoana Ciocoi Radulescu return err; 22077e273a8eSIoana Ciocoi Radulescu } 22087e273a8eSIoana Ciocoi Radulescu 22097e273a8eSIoana Ciocoi Radulescu return 0; 22107e273a8eSIoana Ciocoi Radulescu 22117e273a8eSIoana Ciocoi Radulescu out_err: 22127e273a8eSIoana Ciocoi Radulescu if (prog) 22137e273a8eSIoana Ciocoi Radulescu bpf_prog_sub(prog, priv->num_channels); 22147e273a8eSIoana Ciocoi Radulescu if (up) 22157e273a8eSIoana Ciocoi Radulescu dpaa2_eth_open(dev); 22167e273a8eSIoana Ciocoi Radulescu 22177e273a8eSIoana Ciocoi Radulescu return err; 22187e273a8eSIoana Ciocoi Radulescu } 22197e273a8eSIoana Ciocoi Radulescu 22207e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 22217e273a8eSIoana Ciocoi Radulescu { 22227e273a8eSIoana Ciocoi Radulescu switch (xdp->command) { 22237e273a8eSIoana Ciocoi Radulescu case XDP_SETUP_PROG: 22245d8dccf8SIoana Ciornei return dpaa2_eth_setup_xdp(dev, xdp->prog); 22257e273a8eSIoana Ciocoi Radulescu default: 22267e273a8eSIoana Ciocoi Radulescu return -EINVAL; 22277e273a8eSIoana Ciocoi Radulescu } 22287e273a8eSIoana Ciocoi Radulescu 22297e273a8eSIoana Ciocoi Radulescu return 0; 22307e273a8eSIoana Ciocoi Radulescu } 22317e273a8eSIoana Ciocoi Radulescu 22326aa40b9eSIoana Ciornei static int dpaa2_eth_xdp_create_fd(struct net_device *net_dev, 22336aa40b9eSIoana Ciornei struct xdp_frame *xdpf, 22346aa40b9eSIoana Ciornei struct dpaa2_fd *fd) 2235d678be1dSIoana Radulescu { 2236d678be1dSIoana Radulescu struct device *dev = net_dev->dev.parent; 2237d678be1dSIoana Radulescu unsigned int needed_headroom; 2238d678be1dSIoana Radulescu struct dpaa2_eth_swa *swa; 2239d678be1dSIoana Radulescu void *buffer_start, *aligned_start; 2240d678be1dSIoana Radulescu dma_addr_t addr; 2241d678be1dSIoana Radulescu 2242d678be1dSIoana Radulescu /* We require a minimum headroom to be able to transmit the frame. 2243d678be1dSIoana Radulescu * Otherwise return an error and let the original net_device handle it 2244d678be1dSIoana Radulescu */ 22451cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(NULL); 2246d678be1dSIoana Radulescu if (xdpf->headroom < needed_headroom) 2247d678be1dSIoana Radulescu return -EINVAL; 2248d678be1dSIoana Radulescu 2249d678be1dSIoana Radulescu /* Setup the FD fields */ 22506aa40b9eSIoana Ciornei memset(fd, 0, sizeof(*fd)); 2251d678be1dSIoana Radulescu 2252d678be1dSIoana Radulescu /* Align FD address, if possible */ 2253d678be1dSIoana Radulescu buffer_start = xdpf->data - needed_headroom; 2254d678be1dSIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 2255d678be1dSIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 2256d678be1dSIoana Radulescu if (aligned_start >= xdpf->data - xdpf->headroom) 2257d678be1dSIoana Radulescu buffer_start = aligned_start; 2258d678be1dSIoana Radulescu 2259d678be1dSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 2260d678be1dSIoana Radulescu /* fill in necessary fields here */ 2261d678be1dSIoana Radulescu swa->type = DPAA2_ETH_SWA_XDP; 2262d678be1dSIoana Radulescu swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 2263d678be1dSIoana Radulescu swa->xdp.xdpf = xdpf; 2264d678be1dSIoana Radulescu 2265d678be1dSIoana Radulescu addr = dma_map_single(dev, buffer_start, 2266d678be1dSIoana Radulescu swa->xdp.dma_size, 2267d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 22686aa40b9eSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) 2269d678be1dSIoana Radulescu return -ENOMEM; 2270d678be1dSIoana Radulescu 22716aa40b9eSIoana Ciornei dpaa2_fd_set_addr(fd, addr); 22726aa40b9eSIoana Ciornei dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); 22736aa40b9eSIoana Ciornei dpaa2_fd_set_len(fd, xdpf->len); 22746aa40b9eSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_single); 22756aa40b9eSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 2276d678be1dSIoana Radulescu 2277d678be1dSIoana Radulescu return 0; 2278d678be1dSIoana Radulescu } 2279d678be1dSIoana Radulescu 2280d678be1dSIoana Radulescu static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 2281d678be1dSIoana Radulescu struct xdp_frame **frames, u32 flags) 2282d678be1dSIoana Radulescu { 22836aa40b9eSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 228438c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_redirect_fds; 22856aa40b9eSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 22866aa40b9eSIoana Ciornei struct dpaa2_eth_fq *fq; 22878665d978SIoana Ciornei struct dpaa2_fd *fds; 228838c440b2SIoana Ciornei int enqueued, i, err; 2289d678be1dSIoana Radulescu 2290d678be1dSIoana Radulescu if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2291d678be1dSIoana Radulescu return -EINVAL; 2292d678be1dSIoana Radulescu 2293d678be1dSIoana Radulescu if (!netif_running(net_dev)) 2294d678be1dSIoana Radulescu return -ENETDOWN; 2295d678be1dSIoana Radulescu 22968665d978SIoana Ciornei fq = &priv->fq[smp_processor_id()]; 229738c440b2SIoana Ciornei xdp_redirect_fds = &fq->xdp_redirect_fds; 229838c440b2SIoana Ciornei fds = xdp_redirect_fds->fds; 22998665d978SIoana Ciornei 23006aa40b9eSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 23016aa40b9eSIoana Ciornei 23028665d978SIoana Ciornei /* create a FD for each xdp_frame in the list received */ 2303d678be1dSIoana Radulescu for (i = 0; i < n; i++) { 23048665d978SIoana Ciornei err = dpaa2_eth_xdp_create_fd(net_dev, frames[i], &fds[i]); 23058665d978SIoana Ciornei if (err) 23066aa40b9eSIoana Ciornei break; 23076aa40b9eSIoana Ciornei } 230838c440b2SIoana Ciornei xdp_redirect_fds->num = i; 23096aa40b9eSIoana Ciornei 231038c440b2SIoana Ciornei /* enqueue all the frame descriptors */ 231138c440b2SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, xdp_redirect_fds); 2312d678be1dSIoana Radulescu 23138665d978SIoana Ciornei /* update statistics */ 231438c440b2SIoana Ciornei percpu_stats->tx_packets += enqueued; 231538c440b2SIoana Ciornei for (i = 0; i < enqueued; i++) 23168665d978SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 231738c440b2SIoana Ciornei for (i = enqueued; i < n; i++) 23188665d978SIoana Ciornei xdp_return_frame_rx_napi(frames[i]); 23198665d978SIoana Ciornei 232038c440b2SIoana Ciornei return enqueued; 2321d678be1dSIoana Radulescu } 2322d678be1dSIoana Radulescu 232306d5b179SIoana Radulescu static int update_xps(struct dpaa2_eth_priv *priv) 232406d5b179SIoana Radulescu { 232506d5b179SIoana Radulescu struct net_device *net_dev = priv->net_dev; 232606d5b179SIoana Radulescu struct cpumask xps_mask; 232706d5b179SIoana Radulescu struct dpaa2_eth_fq *fq; 2328ab1e6de2SIoana Radulescu int i, num_queues, netdev_queues; 232906d5b179SIoana Radulescu int err = 0; 233006d5b179SIoana Radulescu 233106d5b179SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2332ab1e6de2SIoana Radulescu netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 233306d5b179SIoana Radulescu 233406d5b179SIoana Radulescu /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 233506d5b179SIoana Radulescu * queues, so only process those 233606d5b179SIoana Radulescu */ 2337ab1e6de2SIoana Radulescu for (i = 0; i < netdev_queues; i++) { 2338ab1e6de2SIoana Radulescu fq = &priv->fq[i % num_queues]; 233906d5b179SIoana Radulescu 234006d5b179SIoana Radulescu cpumask_clear(&xps_mask); 234106d5b179SIoana Radulescu cpumask_set_cpu(fq->target_cpu, &xps_mask); 234206d5b179SIoana Radulescu 234306d5b179SIoana Radulescu err = netif_set_xps_queue(net_dev, &xps_mask, i); 234406d5b179SIoana Radulescu if (err) { 234506d5b179SIoana Radulescu netdev_warn_once(net_dev, "Error setting XPS queue\n"); 234606d5b179SIoana Radulescu break; 234706d5b179SIoana Radulescu } 234806d5b179SIoana Radulescu } 234906d5b179SIoana Radulescu 235006d5b179SIoana Radulescu return err; 235106d5b179SIoana Radulescu } 235206d5b179SIoana Radulescu 2353e3ec13beSIoana Ciornei static int dpaa2_eth_setup_mqprio(struct net_device *net_dev, 2354e3ec13beSIoana Ciornei struct tc_mqprio_qopt *mqprio) 2355ab1e6de2SIoana Radulescu { 2356ab1e6de2SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2357ab1e6de2SIoana Radulescu u8 num_tc, num_queues; 2358ab1e6de2SIoana Radulescu int i; 2359ab1e6de2SIoana Radulescu 2360ab1e6de2SIoana Radulescu mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2361ab1e6de2SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2362ab1e6de2SIoana Radulescu num_tc = mqprio->num_tc; 2363ab1e6de2SIoana Radulescu 2364ab1e6de2SIoana Radulescu if (num_tc == net_dev->num_tc) 2365ab1e6de2SIoana Radulescu return 0; 2366ab1e6de2SIoana Radulescu 2367ab1e6de2SIoana Radulescu if (num_tc > dpaa2_eth_tc_count(priv)) { 2368ab1e6de2SIoana Radulescu netdev_err(net_dev, "Max %d traffic classes supported\n", 2369ab1e6de2SIoana Radulescu dpaa2_eth_tc_count(priv)); 2370b89c1e6bSJesper Dangaard Brouer return -EOPNOTSUPP; 2371ab1e6de2SIoana Radulescu } 2372ab1e6de2SIoana Radulescu 2373ab1e6de2SIoana Radulescu if (!num_tc) { 2374ab1e6de2SIoana Radulescu netdev_reset_tc(net_dev); 2375ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_queues); 2376ab1e6de2SIoana Radulescu goto out; 2377ab1e6de2SIoana Radulescu } 2378ab1e6de2SIoana Radulescu 2379ab1e6de2SIoana Radulescu netdev_set_num_tc(net_dev, num_tc); 2380ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2381ab1e6de2SIoana Radulescu 2382ab1e6de2SIoana Radulescu for (i = 0; i < num_tc; i++) 2383ab1e6de2SIoana Radulescu netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2384ab1e6de2SIoana Radulescu 2385ab1e6de2SIoana Radulescu out: 2386ab1e6de2SIoana Radulescu update_xps(priv); 2387ab1e6de2SIoana Radulescu 2388ab1e6de2SIoana Radulescu return 0; 2389ab1e6de2SIoana Radulescu } 2390ab1e6de2SIoana Radulescu 23913657cdafSIoana Ciornei #define bps_to_mbits(rate) (div_u64((rate), 1000000) * 8) 23923657cdafSIoana Ciornei 23933657cdafSIoana Ciornei static int dpaa2_eth_setup_tbf(struct net_device *net_dev, struct tc_tbf_qopt_offload *p) 23943657cdafSIoana Ciornei { 23953657cdafSIoana Ciornei struct tc_tbf_qopt_offload_replace_params *cfg = &p->replace_params; 23963657cdafSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 23973657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_cr_shaper = { 0 }; 23983657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_er_shaper = { 0 }; 23993657cdafSIoana Ciornei int err; 24003657cdafSIoana Ciornei 24013657cdafSIoana Ciornei if (p->command == TC_TBF_STATS) 24023657cdafSIoana Ciornei return -EOPNOTSUPP; 24033657cdafSIoana Ciornei 24043657cdafSIoana Ciornei /* Only per port Tx shaping */ 24053657cdafSIoana Ciornei if (p->parent != TC_H_ROOT) 24063657cdafSIoana Ciornei return -EOPNOTSUPP; 24073657cdafSIoana Ciornei 24083657cdafSIoana Ciornei if (p->command == TC_TBF_REPLACE) { 24093657cdafSIoana Ciornei if (cfg->max_size > DPAA2_ETH_MAX_BURST_SIZE) { 24103657cdafSIoana Ciornei netdev_err(net_dev, "burst size cannot be greater than %d\n", 24113657cdafSIoana Ciornei DPAA2_ETH_MAX_BURST_SIZE); 24123657cdafSIoana Ciornei return -EINVAL; 24133657cdafSIoana Ciornei } 24143657cdafSIoana Ciornei 24153657cdafSIoana Ciornei tx_cr_shaper.max_burst_size = cfg->max_size; 24163657cdafSIoana Ciornei /* The TBF interface is in bytes/s, whereas DPAA2 expects the 24173657cdafSIoana Ciornei * rate in Mbits/s 24183657cdafSIoana Ciornei */ 24193657cdafSIoana Ciornei tx_cr_shaper.rate_limit = bps_to_mbits(cfg->rate.rate_bytes_ps); 24203657cdafSIoana Ciornei } 24213657cdafSIoana Ciornei 24223657cdafSIoana Ciornei err = dpni_set_tx_shaping(priv->mc_io, 0, priv->mc_token, &tx_cr_shaper, 24233657cdafSIoana Ciornei &tx_er_shaper, 0); 24243657cdafSIoana Ciornei if (err) { 24253657cdafSIoana Ciornei netdev_err(net_dev, "dpni_set_tx_shaping() = %d\n", err); 24263657cdafSIoana Ciornei return err; 24273657cdafSIoana Ciornei } 24283657cdafSIoana Ciornei 24293657cdafSIoana Ciornei return 0; 24303657cdafSIoana Ciornei } 24313657cdafSIoana Ciornei 2432e3ec13beSIoana Ciornei static int dpaa2_eth_setup_tc(struct net_device *net_dev, 2433e3ec13beSIoana Ciornei enum tc_setup_type type, void *type_data) 2434e3ec13beSIoana Ciornei { 2435e3ec13beSIoana Ciornei switch (type) { 2436e3ec13beSIoana Ciornei case TC_SETUP_QDISC_MQPRIO: 2437e3ec13beSIoana Ciornei return dpaa2_eth_setup_mqprio(net_dev, type_data); 24383657cdafSIoana Ciornei case TC_SETUP_QDISC_TBF: 24393657cdafSIoana Ciornei return dpaa2_eth_setup_tbf(net_dev, type_data); 2440e3ec13beSIoana Ciornei default: 2441e3ec13beSIoana Ciornei return -EOPNOTSUPP; 2442e3ec13beSIoana Ciornei } 2443e3ec13beSIoana Ciornei } 2444e3ec13beSIoana Ciornei 244534ff6846SIoana Radulescu static const struct net_device_ops dpaa2_eth_ops = { 244634ff6846SIoana Radulescu .ndo_open = dpaa2_eth_open, 244734ff6846SIoana Radulescu .ndo_start_xmit = dpaa2_eth_tx, 244834ff6846SIoana Radulescu .ndo_stop = dpaa2_eth_stop, 244934ff6846SIoana Radulescu .ndo_set_mac_address = dpaa2_eth_set_addr, 245034ff6846SIoana Radulescu .ndo_get_stats64 = dpaa2_eth_get_stats, 245134ff6846SIoana Radulescu .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 245234ff6846SIoana Radulescu .ndo_set_features = dpaa2_eth_set_features, 245334ff6846SIoana Radulescu .ndo_do_ioctl = dpaa2_eth_ioctl, 24547e273a8eSIoana Ciocoi Radulescu .ndo_change_mtu = dpaa2_eth_change_mtu, 24557e273a8eSIoana Ciocoi Radulescu .ndo_bpf = dpaa2_eth_xdp, 2456d678be1dSIoana Radulescu .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2457ab1e6de2SIoana Radulescu .ndo_setup_tc = dpaa2_eth_setup_tc, 245834ff6846SIoana Radulescu }; 245934ff6846SIoana Radulescu 24605d8dccf8SIoana Ciornei static void dpaa2_eth_cdan_cb(struct dpaa2_io_notification_ctx *ctx) 246134ff6846SIoana Radulescu { 246234ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 246334ff6846SIoana Radulescu 246434ff6846SIoana Radulescu ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 246534ff6846SIoana Radulescu 246634ff6846SIoana Radulescu /* Update NAPI statistics */ 246734ff6846SIoana Radulescu ch->stats.cdan++; 246834ff6846SIoana Radulescu 24696c33ae1aSJiafei Pan napi_schedule(&ch->napi); 247034ff6846SIoana Radulescu } 247134ff6846SIoana Radulescu 247234ff6846SIoana Radulescu /* Allocate and configure a DPCON object */ 24735d8dccf8SIoana Ciornei static struct fsl_mc_device *dpaa2_eth_setup_dpcon(struct dpaa2_eth_priv *priv) 247434ff6846SIoana Radulescu { 247534ff6846SIoana Radulescu struct fsl_mc_device *dpcon; 247634ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 247734ff6846SIoana Radulescu int err; 247834ff6846SIoana Radulescu 247934ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 248034ff6846SIoana Radulescu FSL_MC_POOL_DPCON, &dpcon); 248134ff6846SIoana Radulescu if (err) { 2482d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2483d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2484d7f5a9d8SIoana Ciornei else 248534ff6846SIoana Radulescu dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2486d7f5a9d8SIoana Ciornei return ERR_PTR(err); 248734ff6846SIoana Radulescu } 248834ff6846SIoana Radulescu 248934ff6846SIoana Radulescu err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 249034ff6846SIoana Radulescu if (err) { 249134ff6846SIoana Radulescu dev_err(dev, "dpcon_open() failed\n"); 249234ff6846SIoana Radulescu goto free; 249334ff6846SIoana Radulescu } 249434ff6846SIoana Radulescu 249534ff6846SIoana Radulescu err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 249634ff6846SIoana Radulescu if (err) { 249734ff6846SIoana Radulescu dev_err(dev, "dpcon_reset() failed\n"); 249834ff6846SIoana Radulescu goto close; 249934ff6846SIoana Radulescu } 250034ff6846SIoana Radulescu 250134ff6846SIoana Radulescu err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 250234ff6846SIoana Radulescu if (err) { 250334ff6846SIoana Radulescu dev_err(dev, "dpcon_enable() failed\n"); 250434ff6846SIoana Radulescu goto close; 250534ff6846SIoana Radulescu } 250634ff6846SIoana Radulescu 250734ff6846SIoana Radulescu return dpcon; 250834ff6846SIoana Radulescu 250934ff6846SIoana Radulescu close: 251034ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 251134ff6846SIoana Radulescu free: 251234ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 251334ff6846SIoana Radulescu 251402afa9c6SYueHaibing return ERR_PTR(err); 251534ff6846SIoana Radulescu } 251634ff6846SIoana Radulescu 25175d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpcon(struct dpaa2_eth_priv *priv, 251834ff6846SIoana Radulescu struct fsl_mc_device *dpcon) 251934ff6846SIoana Radulescu { 252034ff6846SIoana Radulescu dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 252134ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 252234ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 252334ff6846SIoana Radulescu } 252434ff6846SIoana Radulescu 25255d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_alloc_channel(struct dpaa2_eth_priv *priv) 252634ff6846SIoana Radulescu { 252734ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 252834ff6846SIoana Radulescu struct dpcon_attr attr; 252934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 253034ff6846SIoana Radulescu int err; 253134ff6846SIoana Radulescu 253234ff6846SIoana Radulescu channel = kzalloc(sizeof(*channel), GFP_KERNEL); 253334ff6846SIoana Radulescu if (!channel) 253434ff6846SIoana Radulescu return NULL; 253534ff6846SIoana Radulescu 25365d8dccf8SIoana Ciornei channel->dpcon = dpaa2_eth_setup_dpcon(priv); 253702afa9c6SYueHaibing if (IS_ERR(channel->dpcon)) { 253802afa9c6SYueHaibing err = PTR_ERR(channel->dpcon); 253934ff6846SIoana Radulescu goto err_setup; 2540d7f5a9d8SIoana Ciornei } 254134ff6846SIoana Radulescu 254234ff6846SIoana Radulescu err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 254334ff6846SIoana Radulescu &attr); 254434ff6846SIoana Radulescu if (err) { 254534ff6846SIoana Radulescu dev_err(dev, "dpcon_get_attributes() failed\n"); 254634ff6846SIoana Radulescu goto err_get_attr; 254734ff6846SIoana Radulescu } 254834ff6846SIoana Radulescu 254934ff6846SIoana Radulescu channel->dpcon_id = attr.id; 255034ff6846SIoana Radulescu channel->ch_id = attr.qbman_ch_id; 255134ff6846SIoana Radulescu channel->priv = priv; 255234ff6846SIoana Radulescu 255334ff6846SIoana Radulescu return channel; 255434ff6846SIoana Radulescu 255534ff6846SIoana Radulescu err_get_attr: 25565d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 255734ff6846SIoana Radulescu err_setup: 255834ff6846SIoana Radulescu kfree(channel); 2559d7f5a9d8SIoana Ciornei return ERR_PTR(err); 256034ff6846SIoana Radulescu } 256134ff6846SIoana Radulescu 25625d8dccf8SIoana Ciornei static void dpaa2_eth_free_channel(struct dpaa2_eth_priv *priv, 256334ff6846SIoana Radulescu struct dpaa2_eth_channel *channel) 256434ff6846SIoana Radulescu { 25655d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 256634ff6846SIoana Radulescu kfree(channel); 256734ff6846SIoana Radulescu } 256834ff6846SIoana Radulescu 256934ff6846SIoana Radulescu /* DPIO setup: allocate and configure QBMan channels, setup core affinity 257034ff6846SIoana Radulescu * and register data availability notifications 257134ff6846SIoana Radulescu */ 25725d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpio(struct dpaa2_eth_priv *priv) 257334ff6846SIoana Radulescu { 257434ff6846SIoana Radulescu struct dpaa2_io_notification_ctx *nctx; 257534ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 257634ff6846SIoana Radulescu struct dpcon_notification_cfg dpcon_notif_cfg; 257734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 257834ff6846SIoana Radulescu int i, err; 257934ff6846SIoana Radulescu 258034ff6846SIoana Radulescu /* We want the ability to spread ingress traffic (RX, TX conf) to as 258134ff6846SIoana Radulescu * many cores as possible, so we need one channel for each core 258234ff6846SIoana Radulescu * (unless there's fewer queues than cores, in which case the extra 258334ff6846SIoana Radulescu * channels would be wasted). 258434ff6846SIoana Radulescu * Allocate one channel per core and register it to the core's 258534ff6846SIoana Radulescu * affine DPIO. If not enough channels are available for all cores 258634ff6846SIoana Radulescu * or if some cores don't have an affine DPIO, there will be no 258734ff6846SIoana Radulescu * ingress frame processing on those cores. 258834ff6846SIoana Radulescu */ 258934ff6846SIoana Radulescu cpumask_clear(&priv->dpio_cpumask); 259034ff6846SIoana Radulescu for_each_online_cpu(i) { 259134ff6846SIoana Radulescu /* Try to allocate a channel */ 25925d8dccf8SIoana Ciornei channel = dpaa2_eth_alloc_channel(priv); 2593d7f5a9d8SIoana Ciornei if (IS_ERR_OR_NULL(channel)) { 2594bd8460faSIoana Radulescu err = PTR_ERR_OR_ZERO(channel); 2595d7f5a9d8SIoana Ciornei if (err != -EPROBE_DEFER) 259634ff6846SIoana Radulescu dev_info(dev, 259734ff6846SIoana Radulescu "No affine channel for cpu %d and above\n", i); 259834ff6846SIoana Radulescu goto err_alloc_ch; 259934ff6846SIoana Radulescu } 260034ff6846SIoana Radulescu 260134ff6846SIoana Radulescu priv->channel[priv->num_channels] = channel; 260234ff6846SIoana Radulescu 260334ff6846SIoana Radulescu nctx = &channel->nctx; 260434ff6846SIoana Radulescu nctx->is_cdan = 1; 26055d8dccf8SIoana Ciornei nctx->cb = dpaa2_eth_cdan_cb; 260634ff6846SIoana Radulescu nctx->id = channel->ch_id; 260734ff6846SIoana Radulescu nctx->desired_cpu = i; 260834ff6846SIoana Radulescu 260934ff6846SIoana Radulescu /* Register the new context */ 261034ff6846SIoana Radulescu channel->dpio = dpaa2_io_service_select(i); 261147441f7fSIoana Ciornei err = dpaa2_io_service_register(channel->dpio, nctx, dev); 261234ff6846SIoana Radulescu if (err) { 261334ff6846SIoana Radulescu dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 261434ff6846SIoana Radulescu /* If no affine DPIO for this core, there's probably 261534ff6846SIoana Radulescu * none available for next cores either. Signal we want 261634ff6846SIoana Radulescu * to retry later, in case the DPIO devices weren't 261734ff6846SIoana Radulescu * probed yet. 261834ff6846SIoana Radulescu */ 261934ff6846SIoana Radulescu err = -EPROBE_DEFER; 262034ff6846SIoana Radulescu goto err_service_reg; 262134ff6846SIoana Radulescu } 262234ff6846SIoana Radulescu 262334ff6846SIoana Radulescu /* Register DPCON notification with MC */ 262434ff6846SIoana Radulescu dpcon_notif_cfg.dpio_id = nctx->dpio_id; 262534ff6846SIoana Radulescu dpcon_notif_cfg.priority = 0; 262634ff6846SIoana Radulescu dpcon_notif_cfg.user_ctx = nctx->qman64; 262734ff6846SIoana Radulescu err = dpcon_set_notification(priv->mc_io, 0, 262834ff6846SIoana Radulescu channel->dpcon->mc_handle, 262934ff6846SIoana Radulescu &dpcon_notif_cfg); 263034ff6846SIoana Radulescu if (err) { 263134ff6846SIoana Radulescu dev_err(dev, "dpcon_set_notification failed()\n"); 263234ff6846SIoana Radulescu goto err_set_cdan; 263334ff6846SIoana Radulescu } 263434ff6846SIoana Radulescu 263534ff6846SIoana Radulescu /* If we managed to allocate a channel and also found an affine 263634ff6846SIoana Radulescu * DPIO for this core, add it to the final mask 263734ff6846SIoana Radulescu */ 263834ff6846SIoana Radulescu cpumask_set_cpu(i, &priv->dpio_cpumask); 263934ff6846SIoana Radulescu priv->num_channels++; 264034ff6846SIoana Radulescu 264134ff6846SIoana Radulescu /* Stop if we already have enough channels to accommodate all 264234ff6846SIoana Radulescu * RX and TX conf queues 264334ff6846SIoana Radulescu */ 2644b0e4f37bSIoana Ciocoi Radulescu if (priv->num_channels == priv->dpni_attrs.num_queues) 264534ff6846SIoana Radulescu break; 264634ff6846SIoana Radulescu } 264734ff6846SIoana Radulescu 264834ff6846SIoana Radulescu return 0; 264934ff6846SIoana Radulescu 265034ff6846SIoana Radulescu err_set_cdan: 265147441f7fSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 265234ff6846SIoana Radulescu err_service_reg: 26535d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 265434ff6846SIoana Radulescu err_alloc_ch: 26555aa4277dSIoana Ciornei if (err == -EPROBE_DEFER) { 26565aa4277dSIoana Ciornei for (i = 0; i < priv->num_channels; i++) { 26575aa4277dSIoana Ciornei channel = priv->channel[i]; 26585aa4277dSIoana Ciornei nctx = &channel->nctx; 26595aa4277dSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 26605d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 26615aa4277dSIoana Ciornei } 26625aa4277dSIoana Ciornei priv->num_channels = 0; 2663d7f5a9d8SIoana Ciornei return err; 26645aa4277dSIoana Ciornei } 2665d7f5a9d8SIoana Ciornei 266634ff6846SIoana Radulescu if (cpumask_empty(&priv->dpio_cpumask)) { 266734ff6846SIoana Radulescu dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 2668d7f5a9d8SIoana Ciornei return -ENODEV; 266934ff6846SIoana Radulescu } 267034ff6846SIoana Radulescu 267134ff6846SIoana Radulescu dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 267234ff6846SIoana Radulescu cpumask_pr_args(&priv->dpio_cpumask)); 267334ff6846SIoana Radulescu 267434ff6846SIoana Radulescu return 0; 267534ff6846SIoana Radulescu } 267634ff6846SIoana Radulescu 26775d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpio(struct dpaa2_eth_priv *priv) 267834ff6846SIoana Radulescu { 267947441f7fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 268034ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 268147441f7fSIoana Ciornei int i; 268234ff6846SIoana Radulescu 268334ff6846SIoana Radulescu /* deregister CDAN notifications and free channels */ 268434ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 268534ff6846SIoana Radulescu ch = priv->channel[i]; 268647441f7fSIoana Ciornei dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 26875d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, ch); 268834ff6846SIoana Radulescu } 268934ff6846SIoana Radulescu } 269034ff6846SIoana Radulescu 26915d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_get_affine_channel(struct dpaa2_eth_priv *priv, 269234ff6846SIoana Radulescu int cpu) 269334ff6846SIoana Radulescu { 269434ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 269534ff6846SIoana Radulescu int i; 269634ff6846SIoana Radulescu 269734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 269834ff6846SIoana Radulescu if (priv->channel[i]->nctx.desired_cpu == cpu) 269934ff6846SIoana Radulescu return priv->channel[i]; 270034ff6846SIoana Radulescu 270134ff6846SIoana Radulescu /* We should never get here. Issue a warning and return 270234ff6846SIoana Radulescu * the first channel, because it's still better than nothing 270334ff6846SIoana Radulescu */ 270434ff6846SIoana Radulescu dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 270534ff6846SIoana Radulescu 270634ff6846SIoana Radulescu return priv->channel[0]; 270734ff6846SIoana Radulescu } 270834ff6846SIoana Radulescu 27095d8dccf8SIoana Ciornei static void dpaa2_eth_set_fq_affinity(struct dpaa2_eth_priv *priv) 271034ff6846SIoana Radulescu { 271134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 271234ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 271334ff6846SIoana Radulescu int rx_cpu, txc_cpu; 271406d5b179SIoana Radulescu int i; 271534ff6846SIoana Radulescu 271634ff6846SIoana Radulescu /* For each FQ, pick one channel/CPU to deliver frames to. 271734ff6846SIoana Radulescu * This may well change at runtime, either through irqbalance or 271834ff6846SIoana Radulescu * through direct user intervention. 271934ff6846SIoana Radulescu */ 272034ff6846SIoana Radulescu rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 272134ff6846SIoana Radulescu 272234ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 272334ff6846SIoana Radulescu fq = &priv->fq[i]; 272434ff6846SIoana Radulescu switch (fq->type) { 272534ff6846SIoana Radulescu case DPAA2_RX_FQ: 272634ff6846SIoana Radulescu fq->target_cpu = rx_cpu; 272734ff6846SIoana Radulescu rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 272834ff6846SIoana Radulescu if (rx_cpu >= nr_cpu_ids) 272934ff6846SIoana Radulescu rx_cpu = cpumask_first(&priv->dpio_cpumask); 273034ff6846SIoana Radulescu break; 273134ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 273234ff6846SIoana Radulescu fq->target_cpu = txc_cpu; 273334ff6846SIoana Radulescu txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 273434ff6846SIoana Radulescu if (txc_cpu >= nr_cpu_ids) 273534ff6846SIoana Radulescu txc_cpu = cpumask_first(&priv->dpio_cpumask); 273634ff6846SIoana Radulescu break; 273734ff6846SIoana Radulescu default: 273834ff6846SIoana Radulescu dev_err(dev, "Unknown FQ type: %d\n", fq->type); 273934ff6846SIoana Radulescu } 27405d8dccf8SIoana Ciornei fq->channel = dpaa2_eth_get_affine_channel(priv, fq->target_cpu); 274134ff6846SIoana Radulescu } 274206d5b179SIoana Radulescu 274306d5b179SIoana Radulescu update_xps(priv); 274434ff6846SIoana Radulescu } 274534ff6846SIoana Radulescu 27465d8dccf8SIoana Ciornei static void dpaa2_eth_setup_fqs(struct dpaa2_eth_priv *priv) 274734ff6846SIoana Radulescu { 2748685e39eaSIoana Radulescu int i, j; 274934ff6846SIoana Radulescu 275034ff6846SIoana Radulescu /* We have one TxConf FQ per Tx flow. 275134ff6846SIoana Radulescu * The number of Tx and Rx queues is the same. 275234ff6846SIoana Radulescu * Tx queues come first in the fq array. 275334ff6846SIoana Radulescu */ 275434ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 275534ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 275634ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 275734ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 275834ff6846SIoana Radulescu } 275934ff6846SIoana Radulescu 2760685e39eaSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 276134ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 276234ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 276334ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 2764685e39eaSIoana Radulescu priv->fq[priv->num_fqs].tc = (u8)j; 276534ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 276634ff6846SIoana Radulescu } 2767685e39eaSIoana Radulescu } 276834ff6846SIoana Radulescu 276934ff6846SIoana Radulescu /* For each FQ, decide on which core to process incoming frames */ 27705d8dccf8SIoana Ciornei dpaa2_eth_set_fq_affinity(priv); 277134ff6846SIoana Radulescu } 277234ff6846SIoana Radulescu 277334ff6846SIoana Radulescu /* Allocate and configure one buffer pool for each interface */ 27745d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpbp(struct dpaa2_eth_priv *priv) 277534ff6846SIoana Radulescu { 277634ff6846SIoana Radulescu int err; 277734ff6846SIoana Radulescu struct fsl_mc_device *dpbp_dev; 277834ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 277934ff6846SIoana Radulescu struct dpbp_attr dpbp_attrs; 278034ff6846SIoana Radulescu 278134ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 278234ff6846SIoana Radulescu &dpbp_dev); 278334ff6846SIoana Radulescu if (err) { 2784d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2785d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2786d7f5a9d8SIoana Ciornei else 278734ff6846SIoana Radulescu dev_err(dev, "DPBP device allocation failed\n"); 278834ff6846SIoana Radulescu return err; 278934ff6846SIoana Radulescu } 279034ff6846SIoana Radulescu 279134ff6846SIoana Radulescu priv->dpbp_dev = dpbp_dev; 279234ff6846SIoana Radulescu 279334ff6846SIoana Radulescu err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 279434ff6846SIoana Radulescu &dpbp_dev->mc_handle); 279534ff6846SIoana Radulescu if (err) { 279634ff6846SIoana Radulescu dev_err(dev, "dpbp_open() failed\n"); 279734ff6846SIoana Radulescu goto err_open; 279834ff6846SIoana Radulescu } 279934ff6846SIoana Radulescu 280034ff6846SIoana Radulescu err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 280134ff6846SIoana Radulescu if (err) { 280234ff6846SIoana Radulescu dev_err(dev, "dpbp_reset() failed\n"); 280334ff6846SIoana Radulescu goto err_reset; 280434ff6846SIoana Radulescu } 280534ff6846SIoana Radulescu 280634ff6846SIoana Radulescu err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 280734ff6846SIoana Radulescu if (err) { 280834ff6846SIoana Radulescu dev_err(dev, "dpbp_enable() failed\n"); 280934ff6846SIoana Radulescu goto err_enable; 281034ff6846SIoana Radulescu } 281134ff6846SIoana Radulescu 281234ff6846SIoana Radulescu err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 281334ff6846SIoana Radulescu &dpbp_attrs); 281434ff6846SIoana Radulescu if (err) { 281534ff6846SIoana Radulescu dev_err(dev, "dpbp_get_attributes() failed\n"); 281634ff6846SIoana Radulescu goto err_get_attr; 281734ff6846SIoana Radulescu } 281834ff6846SIoana Radulescu priv->bpid = dpbp_attrs.bpid; 281934ff6846SIoana Radulescu 282034ff6846SIoana Radulescu return 0; 282134ff6846SIoana Radulescu 282234ff6846SIoana Radulescu err_get_attr: 282334ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 282434ff6846SIoana Radulescu err_enable: 282534ff6846SIoana Radulescu err_reset: 282634ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 282734ff6846SIoana Radulescu err_open: 282834ff6846SIoana Radulescu fsl_mc_object_free(dpbp_dev); 282934ff6846SIoana Radulescu 283034ff6846SIoana Radulescu return err; 283134ff6846SIoana Radulescu } 283234ff6846SIoana Radulescu 28335d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv) 283434ff6846SIoana Radulescu { 28355d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 283634ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 283734ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 283834ff6846SIoana Radulescu fsl_mc_object_free(priv->dpbp_dev); 283934ff6846SIoana Radulescu } 284034ff6846SIoana Radulescu 28415d8dccf8SIoana Ciornei static int dpaa2_eth_set_buffer_layout(struct dpaa2_eth_priv *priv) 284234ff6846SIoana Radulescu { 284334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 284434ff6846SIoana Radulescu struct dpni_buffer_layout buf_layout = {0}; 284527c87486SIoana Ciocoi Radulescu u16 rx_buf_align; 284634ff6846SIoana Radulescu int err; 284734ff6846SIoana Radulescu 284834ff6846SIoana Radulescu /* We need to check for WRIOP version 1.0.0, but depending on the MC 284934ff6846SIoana Radulescu * version, this number is not always provided correctly on rev1. 285034ff6846SIoana Radulescu * We need to check for both alternatives in this situation. 285134ff6846SIoana Radulescu */ 285234ff6846SIoana Radulescu if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 285334ff6846SIoana Radulescu priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 285427c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 285534ff6846SIoana Radulescu else 285627c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 285734ff6846SIoana Radulescu 2858efa6a7d0SIoana Ciornei /* We need to ensure that the buffer size seen by WRIOP is a multiple 2859efa6a7d0SIoana Ciornei * of 64 or 256 bytes depending on the WRIOP version. 2860efa6a7d0SIoana Ciornei */ 2861efa6a7d0SIoana Ciornei priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align); 2862efa6a7d0SIoana Ciornei 286334ff6846SIoana Radulescu /* tx buffer */ 286434ff6846SIoana Radulescu buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 286534ff6846SIoana Radulescu buf_layout.pass_timestamp = true; 2866c5521189SYangbo Lu buf_layout.pass_frame_status = true; 286734ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 2868c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 2869c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 287034ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 287134ff6846SIoana Radulescu DPNI_QUEUE_TX, &buf_layout); 287234ff6846SIoana Radulescu if (err) { 287334ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 287434ff6846SIoana Radulescu return err; 287534ff6846SIoana Radulescu } 287634ff6846SIoana Radulescu 287734ff6846SIoana Radulescu /* tx-confirm buffer */ 2878c5521189SYangbo Lu buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 2879c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 288034ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 288134ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, &buf_layout); 288234ff6846SIoana Radulescu if (err) { 288334ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 288434ff6846SIoana Radulescu return err; 288534ff6846SIoana Radulescu } 288634ff6846SIoana Radulescu 288734ff6846SIoana Radulescu /* Now that we've set our tx buffer layout, retrieve the minimum 288834ff6846SIoana Radulescu * required tx data offset. 288934ff6846SIoana Radulescu */ 289034ff6846SIoana Radulescu err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 289134ff6846SIoana Radulescu &priv->tx_data_offset); 289234ff6846SIoana Radulescu if (err) { 289334ff6846SIoana Radulescu dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 289434ff6846SIoana Radulescu return err; 289534ff6846SIoana Radulescu } 289634ff6846SIoana Radulescu 289734ff6846SIoana Radulescu if ((priv->tx_data_offset % 64) != 0) 289834ff6846SIoana Radulescu dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 289934ff6846SIoana Radulescu priv->tx_data_offset); 290034ff6846SIoana Radulescu 290134ff6846SIoana Radulescu /* rx buffer */ 290234ff6846SIoana Radulescu buf_layout.pass_frame_status = true; 290334ff6846SIoana Radulescu buf_layout.pass_parser_result = true; 290427c87486SIoana Ciocoi Radulescu buf_layout.data_align = rx_buf_align; 290534ff6846SIoana Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 290634ff6846SIoana Radulescu buf_layout.private_data_size = 0; 290734ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 290834ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 290934ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 291034ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 291134ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 291234ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 291334ff6846SIoana Radulescu DPNI_QUEUE_RX, &buf_layout); 291434ff6846SIoana Radulescu if (err) { 291534ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 291634ff6846SIoana Radulescu return err; 291734ff6846SIoana Radulescu } 291834ff6846SIoana Radulescu 291934ff6846SIoana Radulescu return 0; 292034ff6846SIoana Radulescu } 292134ff6846SIoana Radulescu 29221fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 29231fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MINOR 9 29241fa0f68cSIoana Ciocoi Radulescu 29251fa0f68cSIoana Ciocoi Radulescu static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 29261fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 292748c0481eSIoana Ciornei struct dpaa2_fd *fd, u8 prio, 29286ff80447SIoana Ciornei u32 num_frames __always_unused, 292948c0481eSIoana Ciornei int *frames_enqueued) 29301fa0f68cSIoana Ciocoi Radulescu { 293148c0481eSIoana Ciornei int err; 293248c0481eSIoana Ciornei 293348c0481eSIoana Ciornei err = dpaa2_io_service_enqueue_qd(fq->channel->dpio, 29341fa0f68cSIoana Ciocoi Radulescu priv->tx_qdid, prio, 29351fa0f68cSIoana Ciocoi Radulescu fq->tx_qdbin, fd); 293648c0481eSIoana Ciornei if (!err && frames_enqueued) 293748c0481eSIoana Ciornei *frames_enqueued = 1; 293848c0481eSIoana Ciornei return err; 29391fa0f68cSIoana Ciocoi Radulescu } 29401fa0f68cSIoana Ciocoi Radulescu 29416ff80447SIoana Ciornei static inline int dpaa2_eth_enqueue_fq_multiple(struct dpaa2_eth_priv *priv, 29421fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 29436ff80447SIoana Ciornei struct dpaa2_fd *fd, 29446ff80447SIoana Ciornei u8 prio, u32 num_frames, 294548c0481eSIoana Ciornei int *frames_enqueued) 29461fa0f68cSIoana Ciocoi Radulescu { 294748c0481eSIoana Ciornei int err; 294848c0481eSIoana Ciornei 29496ff80447SIoana Ciornei err = dpaa2_io_service_enqueue_multiple_fq(fq->channel->dpio, 29506ff80447SIoana Ciornei fq->tx_fqid[prio], 29516ff80447SIoana Ciornei fd, num_frames); 29526ff80447SIoana Ciornei 29536ff80447SIoana Ciornei if (err == 0) 29546ff80447SIoana Ciornei return -EBUSY; 29556ff80447SIoana Ciornei 29566ff80447SIoana Ciornei if (frames_enqueued) 29576ff80447SIoana Ciornei *frames_enqueued = err; 29586ff80447SIoana Ciornei return 0; 29591fa0f68cSIoana Ciocoi Radulescu } 29601fa0f68cSIoana Ciocoi Radulescu 29615d8dccf8SIoana Ciornei static void dpaa2_eth_set_enqueue_mode(struct dpaa2_eth_priv *priv) 29621fa0f68cSIoana Ciocoi Radulescu { 29631fa0f68cSIoana Ciocoi Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 29641fa0f68cSIoana Ciocoi Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 29651fa0f68cSIoana Ciocoi Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 29661fa0f68cSIoana Ciocoi Radulescu else 29676ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 29681fa0f68cSIoana Ciocoi Radulescu } 29691fa0f68cSIoana Ciocoi Radulescu 29705d8dccf8SIoana Ciornei static int dpaa2_eth_set_pause(struct dpaa2_eth_priv *priv) 29718eb3cef8SIoana Radulescu { 29728eb3cef8SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 29738eb3cef8SIoana Radulescu struct dpni_link_cfg link_cfg = {0}; 29748eb3cef8SIoana Radulescu int err; 29758eb3cef8SIoana Radulescu 29768eb3cef8SIoana Radulescu /* Get the default link options so we don't override other flags */ 29778eb3cef8SIoana Radulescu err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 29788eb3cef8SIoana Radulescu if (err) { 29798eb3cef8SIoana Radulescu dev_err(dev, "dpni_get_link_cfg() failed\n"); 29808eb3cef8SIoana Radulescu return err; 29818eb3cef8SIoana Radulescu } 29828eb3cef8SIoana Radulescu 29838eb3cef8SIoana Radulescu /* By default, enable both Rx and Tx pause frames */ 29848eb3cef8SIoana Radulescu link_cfg.options |= DPNI_LINK_OPT_PAUSE; 29858eb3cef8SIoana Radulescu link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 29868eb3cef8SIoana Radulescu err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 29878eb3cef8SIoana Radulescu if (err) { 29888eb3cef8SIoana Radulescu dev_err(dev, "dpni_set_link_cfg() failed\n"); 29898eb3cef8SIoana Radulescu return err; 29908eb3cef8SIoana Radulescu } 29918eb3cef8SIoana Radulescu 29928eb3cef8SIoana Radulescu priv->link_state.options = link_cfg.options; 29938eb3cef8SIoana Radulescu 29948eb3cef8SIoana Radulescu return 0; 29958eb3cef8SIoana Radulescu } 29968eb3cef8SIoana Radulescu 29975d8dccf8SIoana Ciornei static void dpaa2_eth_update_tx_fqids(struct dpaa2_eth_priv *priv) 2998a690af4fSIoana Radulescu { 2999a690af4fSIoana Radulescu struct dpni_queue_id qid = {0}; 3000a690af4fSIoana Radulescu struct dpaa2_eth_fq *fq; 3001a690af4fSIoana Radulescu struct dpni_queue queue; 3002a690af4fSIoana Radulescu int i, j, err; 3003a690af4fSIoana Radulescu 3004a690af4fSIoana Radulescu /* We only use Tx FQIDs for FQID-based enqueue, so check 3005a690af4fSIoana Radulescu * if DPNI version supports it before updating FQIDs 3006a690af4fSIoana Radulescu */ 3007a690af4fSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 3008a690af4fSIoana Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 3009a690af4fSIoana Radulescu return; 3010a690af4fSIoana Radulescu 3011a690af4fSIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 3012a690af4fSIoana Radulescu fq = &priv->fq[i]; 3013a690af4fSIoana Radulescu if (fq->type != DPAA2_TX_CONF_FQ) 3014a690af4fSIoana Radulescu continue; 3015a690af4fSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 3016a690af4fSIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3017a690af4fSIoana Radulescu DPNI_QUEUE_TX, j, fq->flowid, 3018a690af4fSIoana Radulescu &queue, &qid); 3019a690af4fSIoana Radulescu if (err) 3020a690af4fSIoana Radulescu goto out_err; 3021a690af4fSIoana Radulescu 3022a690af4fSIoana Radulescu fq->tx_fqid[j] = qid.fqid; 3023a690af4fSIoana Radulescu if (fq->tx_fqid[j] == 0) 3024a690af4fSIoana Radulescu goto out_err; 3025a690af4fSIoana Radulescu } 3026a690af4fSIoana Radulescu } 3027a690af4fSIoana Radulescu 30286ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 3029a690af4fSIoana Radulescu 3030a690af4fSIoana Radulescu return; 3031a690af4fSIoana Radulescu 3032a690af4fSIoana Radulescu out_err: 3033a690af4fSIoana Radulescu netdev_info(priv->net_dev, 3034a690af4fSIoana Radulescu "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 3035a690af4fSIoana Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 3036a690af4fSIoana Radulescu } 3037a690af4fSIoana Radulescu 30386aa90fe2SIoana Radulescu /* Configure ingress classification based on VLAN PCP */ 30395d8dccf8SIoana Ciornei static int dpaa2_eth_set_vlan_qos(struct dpaa2_eth_priv *priv) 30406aa90fe2SIoana Radulescu { 30416aa90fe2SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 30426aa90fe2SIoana Radulescu struct dpkg_profile_cfg kg_cfg = {0}; 30436aa90fe2SIoana Radulescu struct dpni_qos_tbl_cfg qos_cfg = {0}; 30446aa90fe2SIoana Radulescu struct dpni_rule_cfg key_params; 30456aa90fe2SIoana Radulescu void *dma_mem, *key, *mask; 30466aa90fe2SIoana Radulescu u8 key_size = 2; /* VLAN TCI field */ 30476aa90fe2SIoana Radulescu int i, pcp, err; 30486aa90fe2SIoana Radulescu 30496aa90fe2SIoana Radulescu /* VLAN-based classification only makes sense if we have multiple 30506aa90fe2SIoana Radulescu * traffic classes. 30516aa90fe2SIoana Radulescu * Also, we need to extract just the 3-bit PCP field from the VLAN 30526aa90fe2SIoana Radulescu * header and we can only do that by using a mask 30536aa90fe2SIoana Radulescu */ 30546aa90fe2SIoana Radulescu if (dpaa2_eth_tc_count(priv) == 1 || !dpaa2_eth_fs_mask_enabled(priv)) { 30556aa90fe2SIoana Radulescu dev_dbg(dev, "VLAN-based QoS classification not supported\n"); 30566aa90fe2SIoana Radulescu return -EOPNOTSUPP; 30576aa90fe2SIoana Radulescu } 30586aa90fe2SIoana Radulescu 30596aa90fe2SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 30606aa90fe2SIoana Radulescu if (!dma_mem) 30616aa90fe2SIoana Radulescu return -ENOMEM; 30626aa90fe2SIoana Radulescu 30636aa90fe2SIoana Radulescu kg_cfg.num_extracts = 1; 30646aa90fe2SIoana Radulescu kg_cfg.extracts[0].type = DPKG_EXTRACT_FROM_HDR; 30656aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.prot = NET_PROT_VLAN; 30666aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.type = DPKG_FULL_FIELD; 30676aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.field = NH_FLD_VLAN_TCI; 30686aa90fe2SIoana Radulescu 30696aa90fe2SIoana Radulescu err = dpni_prepare_key_cfg(&kg_cfg, dma_mem); 30706aa90fe2SIoana Radulescu if (err) { 30716aa90fe2SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg failed\n"); 30726aa90fe2SIoana Radulescu goto out_free_tbl; 30736aa90fe2SIoana Radulescu } 30746aa90fe2SIoana Radulescu 30756aa90fe2SIoana Radulescu /* set QoS table */ 30766aa90fe2SIoana Radulescu qos_cfg.default_tc = 0; 30776aa90fe2SIoana Radulescu qos_cfg.discard_on_miss = 0; 30786aa90fe2SIoana Radulescu qos_cfg.key_cfg_iova = dma_map_single(dev, dma_mem, 30796aa90fe2SIoana Radulescu DPAA2_CLASSIFIER_DMA_SIZE, 30806aa90fe2SIoana Radulescu DMA_TO_DEVICE); 30816aa90fe2SIoana Radulescu if (dma_mapping_error(dev, qos_cfg.key_cfg_iova)) { 30826aa90fe2SIoana Radulescu dev_err(dev, "QoS table DMA mapping failed\n"); 30836aa90fe2SIoana Radulescu err = -ENOMEM; 30846aa90fe2SIoana Radulescu goto out_free_tbl; 30856aa90fe2SIoana Radulescu } 30866aa90fe2SIoana Radulescu 30876aa90fe2SIoana Radulescu err = dpni_set_qos_table(priv->mc_io, 0, priv->mc_token, &qos_cfg); 30886aa90fe2SIoana Radulescu if (err) { 30896aa90fe2SIoana Radulescu dev_err(dev, "dpni_set_qos_table failed\n"); 30906aa90fe2SIoana Radulescu goto out_unmap_tbl; 30916aa90fe2SIoana Radulescu } 30926aa90fe2SIoana Radulescu 30936aa90fe2SIoana Radulescu /* Add QoS table entries */ 30946aa90fe2SIoana Radulescu key = kzalloc(key_size * 2, GFP_KERNEL); 30956aa90fe2SIoana Radulescu if (!key) { 30966aa90fe2SIoana Radulescu err = -ENOMEM; 30976aa90fe2SIoana Radulescu goto out_unmap_tbl; 30986aa90fe2SIoana Radulescu } 30996aa90fe2SIoana Radulescu mask = key + key_size; 31006aa90fe2SIoana Radulescu *(__be16 *)mask = cpu_to_be16(VLAN_PRIO_MASK); 31016aa90fe2SIoana Radulescu 31026aa90fe2SIoana Radulescu key_params.key_iova = dma_map_single(dev, key, key_size * 2, 31036aa90fe2SIoana Radulescu DMA_TO_DEVICE); 31046aa90fe2SIoana Radulescu if (dma_mapping_error(dev, key_params.key_iova)) { 31056aa90fe2SIoana Radulescu dev_err(dev, "Qos table entry DMA mapping failed\n"); 31066aa90fe2SIoana Radulescu err = -ENOMEM; 31076aa90fe2SIoana Radulescu goto out_free_key; 31086aa90fe2SIoana Radulescu } 31096aa90fe2SIoana Radulescu 31106aa90fe2SIoana Radulescu key_params.mask_iova = key_params.key_iova + key_size; 31116aa90fe2SIoana Radulescu key_params.key_size = key_size; 31126aa90fe2SIoana Radulescu 31136aa90fe2SIoana Radulescu /* We add rules for PCP-based distribution starting with highest 31146aa90fe2SIoana Radulescu * priority (VLAN PCP = 7). If this DPNI doesn't have enough traffic 31156aa90fe2SIoana Radulescu * classes to accommodate all priority levels, the lowest ones end up 31166aa90fe2SIoana Radulescu * on TC 0 which was configured as default 31176aa90fe2SIoana Radulescu */ 31186aa90fe2SIoana Radulescu for (i = dpaa2_eth_tc_count(priv) - 1, pcp = 7; i >= 0; i--, pcp--) { 31196aa90fe2SIoana Radulescu *(__be16 *)key = cpu_to_be16(pcp << VLAN_PRIO_SHIFT); 31206aa90fe2SIoana Radulescu dma_sync_single_for_device(dev, key_params.key_iova, 31216aa90fe2SIoana Radulescu key_size * 2, DMA_TO_DEVICE); 31226aa90fe2SIoana Radulescu 31236aa90fe2SIoana Radulescu err = dpni_add_qos_entry(priv->mc_io, 0, priv->mc_token, 31246aa90fe2SIoana Radulescu &key_params, i, i); 31256aa90fe2SIoana Radulescu if (err) { 31266aa90fe2SIoana Radulescu dev_err(dev, "dpni_add_qos_entry failed\n"); 31276aa90fe2SIoana Radulescu dpni_clear_qos_table(priv->mc_io, 0, priv->mc_token); 31286aa90fe2SIoana Radulescu goto out_unmap_key; 31296aa90fe2SIoana Radulescu } 31306aa90fe2SIoana Radulescu } 31316aa90fe2SIoana Radulescu 31326aa90fe2SIoana Radulescu priv->vlan_cls_enabled = true; 31336aa90fe2SIoana Radulescu 31346aa90fe2SIoana Radulescu /* Table and key memory is not persistent, clean everything up after 31356aa90fe2SIoana Radulescu * configuration is finished 31366aa90fe2SIoana Radulescu */ 31376aa90fe2SIoana Radulescu out_unmap_key: 31386aa90fe2SIoana Radulescu dma_unmap_single(dev, key_params.key_iova, key_size * 2, DMA_TO_DEVICE); 31396aa90fe2SIoana Radulescu out_free_key: 31406aa90fe2SIoana Radulescu kfree(key); 31416aa90fe2SIoana Radulescu out_unmap_tbl: 31426aa90fe2SIoana Radulescu dma_unmap_single(dev, qos_cfg.key_cfg_iova, DPAA2_CLASSIFIER_DMA_SIZE, 31436aa90fe2SIoana Radulescu DMA_TO_DEVICE); 31446aa90fe2SIoana Radulescu out_free_tbl: 31456aa90fe2SIoana Radulescu kfree(dma_mem); 31466aa90fe2SIoana Radulescu 31476aa90fe2SIoana Radulescu return err; 31486aa90fe2SIoana Radulescu } 31496aa90fe2SIoana Radulescu 315034ff6846SIoana Radulescu /* Configure the DPNI object this interface is associated with */ 31515d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpni(struct fsl_mc_device *ls_dev) 315234ff6846SIoana Radulescu { 315334ff6846SIoana Radulescu struct device *dev = &ls_dev->dev; 315434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 315534ff6846SIoana Radulescu struct net_device *net_dev; 315634ff6846SIoana Radulescu int err; 315734ff6846SIoana Radulescu 315834ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 315934ff6846SIoana Radulescu priv = netdev_priv(net_dev); 316034ff6846SIoana Radulescu 316134ff6846SIoana Radulescu /* get a handle for the DPNI object */ 316234ff6846SIoana Radulescu err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 316334ff6846SIoana Radulescu if (err) { 316434ff6846SIoana Radulescu dev_err(dev, "dpni_open() failed\n"); 316534ff6846SIoana Radulescu return err; 316634ff6846SIoana Radulescu } 316734ff6846SIoana Radulescu 316834ff6846SIoana Radulescu /* Check if we can work with this DPNI object */ 316934ff6846SIoana Radulescu err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 317034ff6846SIoana Radulescu &priv->dpni_ver_minor); 317134ff6846SIoana Radulescu if (err) { 317234ff6846SIoana Radulescu dev_err(dev, "dpni_get_api_version() failed\n"); 317334ff6846SIoana Radulescu goto close; 317434ff6846SIoana Radulescu } 317534ff6846SIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 317634ff6846SIoana Radulescu dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 317734ff6846SIoana Radulescu priv->dpni_ver_major, priv->dpni_ver_minor, 317834ff6846SIoana Radulescu DPNI_VER_MAJOR, DPNI_VER_MINOR); 317934ff6846SIoana Radulescu err = -ENOTSUPP; 318034ff6846SIoana Radulescu goto close; 318134ff6846SIoana Radulescu } 318234ff6846SIoana Radulescu 318334ff6846SIoana Radulescu ls_dev->mc_io = priv->mc_io; 318434ff6846SIoana Radulescu ls_dev->mc_handle = priv->mc_token; 318534ff6846SIoana Radulescu 318634ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 318734ff6846SIoana Radulescu if (err) { 318834ff6846SIoana Radulescu dev_err(dev, "dpni_reset() failed\n"); 318934ff6846SIoana Radulescu goto close; 319034ff6846SIoana Radulescu } 319134ff6846SIoana Radulescu 319234ff6846SIoana Radulescu err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 319334ff6846SIoana Radulescu &priv->dpni_attrs); 319434ff6846SIoana Radulescu if (err) { 319534ff6846SIoana Radulescu dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 319634ff6846SIoana Radulescu goto close; 319734ff6846SIoana Radulescu } 319834ff6846SIoana Radulescu 31995d8dccf8SIoana Ciornei err = dpaa2_eth_set_buffer_layout(priv); 320034ff6846SIoana Radulescu if (err) 320134ff6846SIoana Radulescu goto close; 320234ff6846SIoana Radulescu 32035d8dccf8SIoana Ciornei dpaa2_eth_set_enqueue_mode(priv); 32041fa0f68cSIoana Ciocoi Radulescu 32058eb3cef8SIoana Radulescu /* Enable pause frame support */ 32068eb3cef8SIoana Radulescu if (dpaa2_eth_has_pause_support(priv)) { 32075d8dccf8SIoana Ciornei err = dpaa2_eth_set_pause(priv); 32088eb3cef8SIoana Radulescu if (err) 32098eb3cef8SIoana Radulescu goto close; 32108eb3cef8SIoana Radulescu } 32118eb3cef8SIoana Radulescu 32125d8dccf8SIoana Ciornei err = dpaa2_eth_set_vlan_qos(priv); 32136aa90fe2SIoana Radulescu if (err && err != -EOPNOTSUPP) 32146aa90fe2SIoana Radulescu goto close; 32156aa90fe2SIoana Radulescu 32169334d5baSXu Wang priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv), 32179334d5baSXu Wang sizeof(struct dpaa2_eth_cls_rule), 32189334d5baSXu Wang GFP_KERNEL); 321997fff7c8SWei Yongjun if (!priv->cls_rules) { 322097fff7c8SWei Yongjun err = -ENOMEM; 3221afb90dbbSIoana Radulescu goto close; 322297fff7c8SWei Yongjun } 3223afb90dbbSIoana Radulescu 322434ff6846SIoana Radulescu return 0; 322534ff6846SIoana Radulescu 322634ff6846SIoana Radulescu close: 322734ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 322834ff6846SIoana Radulescu 322934ff6846SIoana Radulescu return err; 323034ff6846SIoana Radulescu } 323134ff6846SIoana Radulescu 32325d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpni(struct dpaa2_eth_priv *priv) 323334ff6846SIoana Radulescu { 323434ff6846SIoana Radulescu int err; 323534ff6846SIoana Radulescu 323634ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 323734ff6846SIoana Radulescu if (err) 323834ff6846SIoana Radulescu netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 323934ff6846SIoana Radulescu err); 324034ff6846SIoana Radulescu 324134ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 324234ff6846SIoana Radulescu } 324334ff6846SIoana Radulescu 32445d8dccf8SIoana Ciornei static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, 324534ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 324634ff6846SIoana Radulescu { 324734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 324834ff6846SIoana Radulescu struct dpni_queue queue; 324934ff6846SIoana Radulescu struct dpni_queue_id qid; 325034ff6846SIoana Radulescu int err; 325134ff6846SIoana Radulescu 325234ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3253685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, &queue, &qid); 325434ff6846SIoana Radulescu if (err) { 325534ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(RX) failed\n"); 325634ff6846SIoana Radulescu return err; 325734ff6846SIoana Radulescu } 325834ff6846SIoana Radulescu 325934ff6846SIoana Radulescu fq->fqid = qid.fqid; 326034ff6846SIoana Radulescu 326134ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 326234ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 326334ff6846SIoana Radulescu queue.destination.priority = 1; 326434ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 326534ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3266685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, 326716fa1cf1SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 326834ff6846SIoana Radulescu &queue); 326934ff6846SIoana Radulescu if (err) { 327034ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(RX) failed\n"); 327134ff6846SIoana Radulescu return err; 327234ff6846SIoana Radulescu } 327334ff6846SIoana Radulescu 3274d678be1dSIoana Radulescu /* xdp_rxq setup */ 3275685e39eaSIoana Radulescu /* only once for each channel */ 3276685e39eaSIoana Radulescu if (fq->tc > 0) 3277685e39eaSIoana Radulescu return 0; 3278685e39eaSIoana Radulescu 3279d678be1dSIoana Radulescu err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 3280d678be1dSIoana Radulescu fq->flowid); 3281d678be1dSIoana Radulescu if (err) { 3282d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg failed\n"); 3283d678be1dSIoana Radulescu return err; 3284d678be1dSIoana Radulescu } 3285d678be1dSIoana Radulescu 3286d678be1dSIoana Radulescu err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 3287d678be1dSIoana Radulescu MEM_TYPE_PAGE_ORDER0, NULL); 3288d678be1dSIoana Radulescu if (err) { 3289d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 3290d678be1dSIoana Radulescu return err; 3291d678be1dSIoana Radulescu } 3292d678be1dSIoana Radulescu 329334ff6846SIoana Radulescu return 0; 329434ff6846SIoana Radulescu } 329534ff6846SIoana Radulescu 32965d8dccf8SIoana Ciornei static int dpaa2_eth_setup_tx_flow(struct dpaa2_eth_priv *priv, 329734ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 329834ff6846SIoana Radulescu { 329934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 330034ff6846SIoana Radulescu struct dpni_queue queue; 330134ff6846SIoana Radulescu struct dpni_queue_id qid; 330215c87f6bSIoana Radulescu int i, err; 330334ff6846SIoana Radulescu 330415c87f6bSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 330534ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 330615c87f6bSIoana Radulescu DPNI_QUEUE_TX, i, fq->flowid, 330715c87f6bSIoana Radulescu &queue, &qid); 330834ff6846SIoana Radulescu if (err) { 330934ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX) failed\n"); 331034ff6846SIoana Radulescu return err; 331134ff6846SIoana Radulescu } 331215c87f6bSIoana Radulescu fq->tx_fqid[i] = qid.fqid; 331315c87f6bSIoana Radulescu } 331434ff6846SIoana Radulescu 331515c87f6bSIoana Radulescu /* All Tx queues belonging to the same flowid have the same qdbin */ 331634ff6846SIoana Radulescu fq->tx_qdbin = qid.qdbin; 331734ff6846SIoana Radulescu 331834ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 331934ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 332034ff6846SIoana Radulescu &queue, &qid); 332134ff6846SIoana Radulescu if (err) { 332234ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 332334ff6846SIoana Radulescu return err; 332434ff6846SIoana Radulescu } 332534ff6846SIoana Radulescu 332634ff6846SIoana Radulescu fq->fqid = qid.fqid; 332734ff6846SIoana Radulescu 332834ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 332934ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 333034ff6846SIoana Radulescu queue.destination.priority = 0; 333134ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 333234ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 333334ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 333434ff6846SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 333534ff6846SIoana Radulescu &queue); 333634ff6846SIoana Radulescu if (err) { 333734ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 333834ff6846SIoana Radulescu return err; 333934ff6846SIoana Radulescu } 334034ff6846SIoana Radulescu 334134ff6846SIoana Radulescu return 0; 334234ff6846SIoana Radulescu } 334334ff6846SIoana Radulescu 3344edad8d26SIoana Ciocoi Radulescu /* Supported header fields for Rx hash distribution key */ 3345f76c483aSIoana Radulescu static const struct dpaa2_eth_dist_fields dist_fields[] = { 334634ff6846SIoana Radulescu { 3347edad8d26SIoana Ciocoi Radulescu /* L2 header */ 3348edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_L2DA, 3349edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_ETH, 3350edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_ETH_DA, 33513a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHDST, 3352edad8d26SIoana Ciocoi Radulescu .size = 6, 3353edad8d26SIoana Ciocoi Radulescu }, { 3354afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3355afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_SA, 33563a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHSRC, 3357afb90dbbSIoana Radulescu .size = 6, 3358afb90dbbSIoana Radulescu }, { 3359afb90dbbSIoana Radulescu /* This is the last ethertype field parsed: 3360afb90dbbSIoana Radulescu * depending on frame format, it can be the MAC ethertype 3361afb90dbbSIoana Radulescu * or the VLAN etype. 3362afb90dbbSIoana Radulescu */ 3363afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3364afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_TYPE, 33653a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHTYPE, 3366afb90dbbSIoana Radulescu .size = 2, 3367afb90dbbSIoana Radulescu }, { 3368edad8d26SIoana Ciocoi Radulescu /* VLAN header */ 3369edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_VLAN, 3370edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_VLAN, 3371edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_VLAN_TCI, 33723a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_VLAN, 3373edad8d26SIoana Ciocoi Radulescu .size = 2, 3374edad8d26SIoana Ciocoi Radulescu }, { 337534ff6846SIoana Radulescu /* IP header */ 337634ff6846SIoana Radulescu .rxnfc_field = RXH_IP_SRC, 337734ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 337834ff6846SIoana Radulescu .cls_field = NH_FLD_IP_SRC, 33793a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPSRC, 338034ff6846SIoana Radulescu .size = 4, 338134ff6846SIoana Radulescu }, { 338234ff6846SIoana Radulescu .rxnfc_field = RXH_IP_DST, 338334ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 338434ff6846SIoana Radulescu .cls_field = NH_FLD_IP_DST, 33853a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPDST, 338634ff6846SIoana Radulescu .size = 4, 338734ff6846SIoana Radulescu }, { 338834ff6846SIoana Radulescu .rxnfc_field = RXH_L3_PROTO, 338934ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 339034ff6846SIoana Radulescu .cls_field = NH_FLD_IP_PROTO, 33913a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPPROTO, 339234ff6846SIoana Radulescu .size = 1, 339334ff6846SIoana Radulescu }, { 339434ff6846SIoana Radulescu /* Using UDP ports, this is functionally equivalent to raw 339534ff6846SIoana Radulescu * byte pairs from L4 header. 339634ff6846SIoana Radulescu */ 339734ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_0_1, 339834ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 339934ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_SRC, 34003a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4SRC, 340134ff6846SIoana Radulescu .size = 2, 340234ff6846SIoana Radulescu }, { 340334ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_2_3, 340434ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 340534ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_DST, 34063a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4DST, 340734ff6846SIoana Radulescu .size = 2, 340834ff6846SIoana Radulescu }, 340934ff6846SIoana Radulescu }; 341034ff6846SIoana Radulescu 3411df85aeb9SIoana Radulescu /* Configure the Rx hash key using the legacy API */ 34125d8dccf8SIoana Ciornei static int dpaa2_eth_config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3413df85aeb9SIoana Radulescu { 3414df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3415df85aeb9SIoana Radulescu struct dpni_rx_tc_dist_cfg dist_cfg; 3416685e39eaSIoana Radulescu int i, err = 0; 3417df85aeb9SIoana Radulescu 3418df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3419df85aeb9SIoana Radulescu 3420df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3421df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3422df85aeb9SIoana Radulescu dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 3423df85aeb9SIoana Radulescu 3424685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3425685e39eaSIoana Radulescu err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 3426685e39eaSIoana Radulescu i, &dist_cfg); 3427685e39eaSIoana Radulescu if (err) { 3428df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 3429685e39eaSIoana Radulescu break; 3430685e39eaSIoana Radulescu } 3431685e39eaSIoana Radulescu } 3432df85aeb9SIoana Radulescu 3433df85aeb9SIoana Radulescu return err; 3434df85aeb9SIoana Radulescu } 3435df85aeb9SIoana Radulescu 3436df85aeb9SIoana Radulescu /* Configure the Rx hash key using the new API */ 34375d8dccf8SIoana Ciornei static int dpaa2_eth_config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3438df85aeb9SIoana Radulescu { 3439df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3440df85aeb9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3441685e39eaSIoana Radulescu int i, err = 0; 3442df85aeb9SIoana Radulescu 3443df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3444df85aeb9SIoana Radulescu 3445df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3446df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3447df85aeb9SIoana Radulescu dist_cfg.enable = 1; 3448df85aeb9SIoana Radulescu 3449685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3450685e39eaSIoana Radulescu dist_cfg.tc = i; 3451685e39eaSIoana Radulescu err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, 3452685e39eaSIoana Radulescu &dist_cfg); 3453685e39eaSIoana Radulescu if (err) { 3454df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 3455685e39eaSIoana Radulescu break; 3456685e39eaSIoana Radulescu } 34575e29c16fSIonut-robert Aron 34585e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 34595e29c16fSIonut-robert Aron * traffic classes, install it just once 34605e29c16fSIonut-robert Aron */ 34615e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 34625e29c16fSIonut-robert Aron break; 3463685e39eaSIoana Radulescu } 3464df85aeb9SIoana Radulescu 3465df85aeb9SIoana Radulescu return err; 3466df85aeb9SIoana Radulescu } 3467df85aeb9SIoana Radulescu 34684aaaf9b9SIoana Radulescu /* Configure the Rx flow classification key */ 34695d8dccf8SIoana Ciornei static int dpaa2_eth_config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 34704aaaf9b9SIoana Radulescu { 34714aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 34724aaaf9b9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3473685e39eaSIoana Radulescu int i, err = 0; 34744aaaf9b9SIoana Radulescu 34754aaaf9b9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 34764aaaf9b9SIoana Radulescu 34774aaaf9b9SIoana Radulescu dist_cfg.key_cfg_iova = key; 34784aaaf9b9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 34794aaaf9b9SIoana Radulescu dist_cfg.enable = 1; 34804aaaf9b9SIoana Radulescu 3481685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3482685e39eaSIoana Radulescu dist_cfg.tc = i; 3483685e39eaSIoana Radulescu err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, 3484685e39eaSIoana Radulescu &dist_cfg); 3485685e39eaSIoana Radulescu if (err) { 34864aaaf9b9SIoana Radulescu dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 3487685e39eaSIoana Radulescu break; 3488685e39eaSIoana Radulescu } 34895e29c16fSIonut-robert Aron 34905e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 34915e29c16fSIonut-robert Aron * traffic classes, install it just once 34925e29c16fSIonut-robert Aron */ 34935e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 34945e29c16fSIonut-robert Aron break; 3495685e39eaSIoana Radulescu } 34964aaaf9b9SIoana Radulescu 34974aaaf9b9SIoana Radulescu return err; 34984aaaf9b9SIoana Radulescu } 34994aaaf9b9SIoana Radulescu 3500afb90dbbSIoana Radulescu /* Size of the Rx flow classification key */ 35012d680237SIoana Ciocoi Radulescu int dpaa2_eth_cls_key_size(u64 fields) 3502afb90dbbSIoana Radulescu { 3503afb90dbbSIoana Radulescu int i, size = 0; 3504afb90dbbSIoana Radulescu 35052d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 35062d680237SIoana Ciocoi Radulescu if (!(fields & dist_fields[i].id)) 35072d680237SIoana Ciocoi Radulescu continue; 3508afb90dbbSIoana Radulescu size += dist_fields[i].size; 35092d680237SIoana Ciocoi Radulescu } 3510afb90dbbSIoana Radulescu 3511afb90dbbSIoana Radulescu return size; 3512afb90dbbSIoana Radulescu } 3513afb90dbbSIoana Radulescu 3514afb90dbbSIoana Radulescu /* Offset of header field in Rx classification key */ 3515afb90dbbSIoana Radulescu int dpaa2_eth_cls_fld_off(int prot, int field) 3516afb90dbbSIoana Radulescu { 3517afb90dbbSIoana Radulescu int i, off = 0; 3518afb90dbbSIoana Radulescu 3519afb90dbbSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3520afb90dbbSIoana Radulescu if (dist_fields[i].cls_prot == prot && 3521afb90dbbSIoana Radulescu dist_fields[i].cls_field == field) 3522afb90dbbSIoana Radulescu return off; 3523afb90dbbSIoana Radulescu off += dist_fields[i].size; 3524afb90dbbSIoana Radulescu } 3525afb90dbbSIoana Radulescu 3526afb90dbbSIoana Radulescu WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 3527afb90dbbSIoana Radulescu return 0; 3528afb90dbbSIoana Radulescu } 3529afb90dbbSIoana Radulescu 35302d680237SIoana Ciocoi Radulescu /* Prune unused fields from the classification rule. 35312d680237SIoana Ciocoi Radulescu * Used when masking is not supported 35322d680237SIoana Ciocoi Radulescu */ 35332d680237SIoana Ciocoi Radulescu void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 35342d680237SIoana Ciocoi Radulescu { 35352d680237SIoana Ciocoi Radulescu int off = 0, new_off = 0; 35362d680237SIoana Ciocoi Radulescu int i, size; 35372d680237SIoana Ciocoi Radulescu 35382d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 35392d680237SIoana Ciocoi Radulescu size = dist_fields[i].size; 35402d680237SIoana Ciocoi Radulescu if (dist_fields[i].id & fields) { 35412d680237SIoana Ciocoi Radulescu memcpy(key_mem + new_off, key_mem + off, size); 35422d680237SIoana Ciocoi Radulescu new_off += size; 35432d680237SIoana Ciocoi Radulescu } 35442d680237SIoana Ciocoi Radulescu off += size; 35452d680237SIoana Ciocoi Radulescu } 35462d680237SIoana Ciocoi Radulescu } 35472d680237SIoana Ciocoi Radulescu 35484aaaf9b9SIoana Radulescu /* Set Rx distribution (hash or flow classification) key 354934ff6846SIoana Radulescu * flags is a combination of RXH_ bits 355034ff6846SIoana Radulescu */ 35513233c151SIoana Ciornei static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 35524aaaf9b9SIoana Radulescu enum dpaa2_eth_rx_dist type, u64 flags) 355334ff6846SIoana Radulescu { 355434ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 355534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 355634ff6846SIoana Radulescu struct dpkg_profile_cfg cls_cfg; 3557edad8d26SIoana Ciocoi Radulescu u32 rx_hash_fields = 0; 3558df85aeb9SIoana Radulescu dma_addr_t key_iova; 355934ff6846SIoana Radulescu u8 *dma_mem; 356034ff6846SIoana Radulescu int i; 356134ff6846SIoana Radulescu int err = 0; 356234ff6846SIoana Radulescu 356334ff6846SIoana Radulescu memset(&cls_cfg, 0, sizeof(cls_cfg)); 356434ff6846SIoana Radulescu 3565f76c483aSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 356634ff6846SIoana Radulescu struct dpkg_extract *key = 356734ff6846SIoana Radulescu &cls_cfg.extracts[cls_cfg.num_extracts]; 356834ff6846SIoana Radulescu 35692d680237SIoana Ciocoi Radulescu /* For both Rx hashing and classification keys 35702d680237SIoana Ciocoi Radulescu * we set only the selected fields. 35714aaaf9b9SIoana Radulescu */ 35723a1e6b84SIoana Ciocoi Radulescu if (!(flags & dist_fields[i].id)) 357334ff6846SIoana Radulescu continue; 35742d680237SIoana Ciocoi Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) 35754aaaf9b9SIoana Radulescu rx_hash_fields |= dist_fields[i].rxnfc_field; 357634ff6846SIoana Radulescu 357734ff6846SIoana Radulescu if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 357834ff6846SIoana Radulescu dev_err(dev, "error adding key extraction rule, too many rules?\n"); 357934ff6846SIoana Radulescu return -E2BIG; 358034ff6846SIoana Radulescu } 358134ff6846SIoana Radulescu 358234ff6846SIoana Radulescu key->type = DPKG_EXTRACT_FROM_HDR; 3583f76c483aSIoana Radulescu key->extract.from_hdr.prot = dist_fields[i].cls_prot; 358434ff6846SIoana Radulescu key->extract.from_hdr.type = DPKG_FULL_FIELD; 3585f76c483aSIoana Radulescu key->extract.from_hdr.field = dist_fields[i].cls_field; 358634ff6846SIoana Radulescu cls_cfg.num_extracts++; 358734ff6846SIoana Radulescu } 358834ff6846SIoana Radulescu 358934ff6846SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 359034ff6846SIoana Radulescu if (!dma_mem) 359134ff6846SIoana Radulescu return -ENOMEM; 359234ff6846SIoana Radulescu 359334ff6846SIoana Radulescu err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 359434ff6846SIoana Radulescu if (err) { 359534ff6846SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 3596df85aeb9SIoana Radulescu goto free_key; 359734ff6846SIoana Radulescu } 359834ff6846SIoana Radulescu 359934ff6846SIoana Radulescu /* Prepare for setting the rx dist */ 3600df85aeb9SIoana Radulescu key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 360134ff6846SIoana Radulescu DMA_TO_DEVICE); 3602df85aeb9SIoana Radulescu if (dma_mapping_error(dev, key_iova)) { 360334ff6846SIoana Radulescu dev_err(dev, "DMA mapping failed\n"); 360434ff6846SIoana Radulescu err = -ENOMEM; 3605df85aeb9SIoana Radulescu goto free_key; 360634ff6846SIoana Radulescu } 360734ff6846SIoana Radulescu 36084aaaf9b9SIoana Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) { 3609df85aeb9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) 36105d8dccf8SIoana Ciornei err = dpaa2_eth_config_legacy_hash_key(priv, key_iova); 3611edad8d26SIoana Ciocoi Radulescu else 36125d8dccf8SIoana Ciornei err = dpaa2_eth_config_hash_key(priv, key_iova); 36134aaaf9b9SIoana Radulescu } else { 36145d8dccf8SIoana Ciornei err = dpaa2_eth_config_cls_key(priv, key_iova); 36154aaaf9b9SIoana Radulescu } 3616df85aeb9SIoana Radulescu 3617df85aeb9SIoana Radulescu dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3618df85aeb9SIoana Radulescu DMA_TO_DEVICE); 36194aaaf9b9SIoana Radulescu if (!err && type == DPAA2_ETH_RX_DIST_HASH) 3620edad8d26SIoana Ciocoi Radulescu priv->rx_hash_fields = rx_hash_fields; 362134ff6846SIoana Radulescu 3622df85aeb9SIoana Radulescu free_key: 362334ff6846SIoana Radulescu kfree(dma_mem); 362434ff6846SIoana Radulescu return err; 362534ff6846SIoana Radulescu } 362634ff6846SIoana Radulescu 36274aaaf9b9SIoana Radulescu int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 36284aaaf9b9SIoana Radulescu { 36294aaaf9b9SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 36303a1e6b84SIoana Ciocoi Radulescu u64 key = 0; 36313a1e6b84SIoana Ciocoi Radulescu int i; 36324aaaf9b9SIoana Radulescu 36334aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) 36344aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 36354aaaf9b9SIoana Radulescu 36363a1e6b84SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 36373a1e6b84SIoana Ciocoi Radulescu if (dist_fields[i].rxnfc_field & flags) 36383a1e6b84SIoana Ciocoi Radulescu key |= dist_fields[i].id; 36393a1e6b84SIoana Ciocoi Radulescu 36403a1e6b84SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 36414aaaf9b9SIoana Radulescu } 36424aaaf9b9SIoana Radulescu 36432d680237SIoana Ciocoi Radulescu int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 36442d680237SIoana Ciocoi Radulescu { 36452d680237SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 36462d680237SIoana Ciocoi Radulescu } 36472d680237SIoana Ciocoi Radulescu 36482d680237SIoana Ciocoi Radulescu static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 36494aaaf9b9SIoana Radulescu { 36504aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3651df8e249bSIoana Ciocoi Radulescu int err; 36524aaaf9b9SIoana Radulescu 36534aaaf9b9SIoana Radulescu /* Check if we actually support Rx flow classification */ 36544aaaf9b9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) { 36554aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls not supported by current MC version\n"); 36564aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 36574aaaf9b9SIoana Radulescu } 36584aaaf9b9SIoana Radulescu 36592d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_enabled(priv)) { 36604aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 36614aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 36624aaaf9b9SIoana Radulescu } 36634aaaf9b9SIoana Radulescu 36644aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) { 36654aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 36664aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 36674aaaf9b9SIoana Radulescu } 36684aaaf9b9SIoana Radulescu 36692d680237SIoana Ciocoi Radulescu /* If there is no support for masking in the classification table, 36702d680237SIoana Ciocoi Radulescu * we don't set a default key, as it will depend on the rules 36712d680237SIoana Ciocoi Radulescu * added by the user at runtime. 36722d680237SIoana Ciocoi Radulescu */ 36732d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_mask_enabled(priv)) 36742d680237SIoana Ciocoi Radulescu goto out; 36752d680237SIoana Ciocoi Radulescu 36762d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 3677df8e249bSIoana Ciocoi Radulescu if (err) 3678df8e249bSIoana Ciocoi Radulescu return err; 3679df8e249bSIoana Ciocoi Radulescu 36802d680237SIoana Ciocoi Radulescu out: 36814aaaf9b9SIoana Radulescu priv->rx_cls_enabled = 1; 36824aaaf9b9SIoana Radulescu 3683df8e249bSIoana Ciocoi Radulescu return 0; 36844aaaf9b9SIoana Radulescu } 36854aaaf9b9SIoana Radulescu 368634ff6846SIoana Radulescu /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 368734ff6846SIoana Radulescu * frame queues and channels 368834ff6846SIoana Radulescu */ 36895d8dccf8SIoana Ciornei static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv) 369034ff6846SIoana Radulescu { 369134ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 369234ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 369334ff6846SIoana Radulescu struct dpni_pools_cfg pools_params; 369434ff6846SIoana Radulescu struct dpni_error_cfg err_cfg; 369534ff6846SIoana Radulescu int err = 0; 369634ff6846SIoana Radulescu int i; 369734ff6846SIoana Radulescu 369834ff6846SIoana Radulescu pools_params.num_dpbp = 1; 369934ff6846SIoana Radulescu pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 370034ff6846SIoana Radulescu pools_params.pools[0].backup_pool = 0; 3701efa6a7d0SIoana Ciornei pools_params.pools[0].buffer_size = priv->rx_buf_size; 370234ff6846SIoana Radulescu err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 370334ff6846SIoana Radulescu if (err) { 370434ff6846SIoana Radulescu dev_err(dev, "dpni_set_pools() failed\n"); 370534ff6846SIoana Radulescu return err; 370634ff6846SIoana Radulescu } 370734ff6846SIoana Radulescu 370834ff6846SIoana Radulescu /* have the interface implicitly distribute traffic based on 370934ff6846SIoana Radulescu * the default hash key 371034ff6846SIoana Radulescu */ 371134ff6846SIoana Radulescu err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 3712edad8d26SIoana Ciocoi Radulescu if (err && err != -EOPNOTSUPP) 371334ff6846SIoana Radulescu dev_err(dev, "Failed to configure hashing\n"); 371434ff6846SIoana Radulescu 37154aaaf9b9SIoana Radulescu /* Configure the flow classification key; it includes all 37164aaaf9b9SIoana Radulescu * supported header fields and cannot be modified at runtime 37174aaaf9b9SIoana Radulescu */ 37182d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_default_cls(priv); 37194aaaf9b9SIoana Radulescu if (err && err != -EOPNOTSUPP) 37204aaaf9b9SIoana Radulescu dev_err(dev, "Failed to configure Rx classification key\n"); 37214aaaf9b9SIoana Radulescu 372234ff6846SIoana Radulescu /* Configure handling of error frames */ 372334ff6846SIoana Radulescu err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 372434ff6846SIoana Radulescu err_cfg.set_frame_annotation = 1; 372534ff6846SIoana Radulescu err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 372634ff6846SIoana Radulescu err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 372734ff6846SIoana Radulescu &err_cfg); 372834ff6846SIoana Radulescu if (err) { 372934ff6846SIoana Radulescu dev_err(dev, "dpni_set_errors_behavior failed\n"); 373034ff6846SIoana Radulescu return err; 373134ff6846SIoana Radulescu } 373234ff6846SIoana Radulescu 373334ff6846SIoana Radulescu /* Configure Rx and Tx conf queues to generate CDANs */ 373434ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 373534ff6846SIoana Radulescu switch (priv->fq[i].type) { 373634ff6846SIoana Radulescu case DPAA2_RX_FQ: 37375d8dccf8SIoana Ciornei err = dpaa2_eth_setup_rx_flow(priv, &priv->fq[i]); 373834ff6846SIoana Radulescu break; 373934ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 37405d8dccf8SIoana Ciornei err = dpaa2_eth_setup_tx_flow(priv, &priv->fq[i]); 374134ff6846SIoana Radulescu break; 374234ff6846SIoana Radulescu default: 374334ff6846SIoana Radulescu dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 374434ff6846SIoana Radulescu return -EINVAL; 374534ff6846SIoana Radulescu } 374634ff6846SIoana Radulescu if (err) 374734ff6846SIoana Radulescu return err; 374834ff6846SIoana Radulescu } 374934ff6846SIoana Radulescu 375034ff6846SIoana Radulescu err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 375134ff6846SIoana Radulescu DPNI_QUEUE_TX, &priv->tx_qdid); 375234ff6846SIoana Radulescu if (err) { 375334ff6846SIoana Radulescu dev_err(dev, "dpni_get_qdid() failed\n"); 375434ff6846SIoana Radulescu return err; 375534ff6846SIoana Radulescu } 375634ff6846SIoana Radulescu 375734ff6846SIoana Radulescu return 0; 375834ff6846SIoana Radulescu } 375934ff6846SIoana Radulescu 376034ff6846SIoana Radulescu /* Allocate rings for storing incoming frame descriptors */ 37615d8dccf8SIoana Ciornei static int dpaa2_eth_alloc_rings(struct dpaa2_eth_priv *priv) 376234ff6846SIoana Radulescu { 376334ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 376434ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 376534ff6846SIoana Radulescu int i; 376634ff6846SIoana Radulescu 376734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 376834ff6846SIoana Radulescu priv->channel[i]->store = 376934ff6846SIoana Radulescu dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 377034ff6846SIoana Radulescu if (!priv->channel[i]->store) { 377134ff6846SIoana Radulescu netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 377234ff6846SIoana Radulescu goto err_ring; 377334ff6846SIoana Radulescu } 377434ff6846SIoana Radulescu } 377534ff6846SIoana Radulescu 377634ff6846SIoana Radulescu return 0; 377734ff6846SIoana Radulescu 377834ff6846SIoana Radulescu err_ring: 377934ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 378034ff6846SIoana Radulescu if (!priv->channel[i]->store) 378134ff6846SIoana Radulescu break; 378234ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 378334ff6846SIoana Radulescu } 378434ff6846SIoana Radulescu 378534ff6846SIoana Radulescu return -ENOMEM; 378634ff6846SIoana Radulescu } 378734ff6846SIoana Radulescu 37885d8dccf8SIoana Ciornei static void dpaa2_eth_free_rings(struct dpaa2_eth_priv *priv) 378934ff6846SIoana Radulescu { 379034ff6846SIoana Radulescu int i; 379134ff6846SIoana Radulescu 379234ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 379334ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 379434ff6846SIoana Radulescu } 379534ff6846SIoana Radulescu 37965d8dccf8SIoana Ciornei static int dpaa2_eth_set_mac_addr(struct dpaa2_eth_priv *priv) 379734ff6846SIoana Radulescu { 379834ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 379934ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 380034ff6846SIoana Radulescu u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 380134ff6846SIoana Radulescu int err; 380234ff6846SIoana Radulescu 380334ff6846SIoana Radulescu /* Get firmware address, if any */ 380434ff6846SIoana Radulescu err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 380534ff6846SIoana Radulescu if (err) { 380634ff6846SIoana Radulescu dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 380734ff6846SIoana Radulescu return err; 380834ff6846SIoana Radulescu } 380934ff6846SIoana Radulescu 381034ff6846SIoana Radulescu /* Get DPNI attributes address, if any */ 381134ff6846SIoana Radulescu err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 381234ff6846SIoana Radulescu dpni_mac_addr); 381334ff6846SIoana Radulescu if (err) { 381434ff6846SIoana Radulescu dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 381534ff6846SIoana Radulescu return err; 381634ff6846SIoana Radulescu } 381734ff6846SIoana Radulescu 381834ff6846SIoana Radulescu /* First check if firmware has any address configured by bootloader */ 381934ff6846SIoana Radulescu if (!is_zero_ether_addr(mac_addr)) { 382034ff6846SIoana Radulescu /* If the DPMAC addr != DPNI addr, update it */ 382134ff6846SIoana Radulescu if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 382234ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, 382334ff6846SIoana Radulescu priv->mc_token, 382434ff6846SIoana Radulescu mac_addr); 382534ff6846SIoana Radulescu if (err) { 382634ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 382734ff6846SIoana Radulescu return err; 382834ff6846SIoana Radulescu } 382934ff6846SIoana Radulescu } 383034ff6846SIoana Radulescu memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); 383134ff6846SIoana Radulescu } else if (is_zero_ether_addr(dpni_mac_addr)) { 383234ff6846SIoana Radulescu /* No MAC address configured, fill in net_dev->dev_addr 383334ff6846SIoana Radulescu * with a random one 383434ff6846SIoana Radulescu */ 383534ff6846SIoana Radulescu eth_hw_addr_random(net_dev); 383634ff6846SIoana Radulescu dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 383734ff6846SIoana Radulescu 383834ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 383934ff6846SIoana Radulescu net_dev->dev_addr); 384034ff6846SIoana Radulescu if (err) { 384134ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 384234ff6846SIoana Radulescu return err; 384334ff6846SIoana Radulescu } 384434ff6846SIoana Radulescu 384534ff6846SIoana Radulescu /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 384634ff6846SIoana Radulescu * practical purposes, this will be our "permanent" mac address, 384734ff6846SIoana Radulescu * at least until the next reboot. This move will also permit 384834ff6846SIoana Radulescu * register_netdevice() to properly fill up net_dev->perm_addr. 384934ff6846SIoana Radulescu */ 385034ff6846SIoana Radulescu net_dev->addr_assign_type = NET_ADDR_PERM; 385134ff6846SIoana Radulescu } else { 385234ff6846SIoana Radulescu /* NET_ADDR_PERM is default, all we have to do is 385334ff6846SIoana Radulescu * fill in the device addr. 385434ff6846SIoana Radulescu */ 385534ff6846SIoana Radulescu memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len); 385634ff6846SIoana Radulescu } 385734ff6846SIoana Radulescu 385834ff6846SIoana Radulescu return 0; 385934ff6846SIoana Radulescu } 386034ff6846SIoana Radulescu 38615d8dccf8SIoana Ciornei static int dpaa2_eth_netdev_init(struct net_device *net_dev) 386234ff6846SIoana Radulescu { 386334ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 386434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 386534ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 386634ff6846SIoana Radulescu u64 supported = 0, not_supported = 0; 386734ff6846SIoana Radulescu u8 bcast_addr[ETH_ALEN]; 386834ff6846SIoana Radulescu u8 num_queues; 386934ff6846SIoana Radulescu int err; 387034ff6846SIoana Radulescu 387134ff6846SIoana Radulescu net_dev->netdev_ops = &dpaa2_eth_ops; 387234ff6846SIoana Radulescu net_dev->ethtool_ops = &dpaa2_ethtool_ops; 387334ff6846SIoana Radulescu 38745d8dccf8SIoana Ciornei err = dpaa2_eth_set_mac_addr(priv); 387534ff6846SIoana Radulescu if (err) 387634ff6846SIoana Radulescu return err; 387734ff6846SIoana Radulescu 387834ff6846SIoana Radulescu /* Explicitly add the broadcast address to the MAC filtering table */ 387934ff6846SIoana Radulescu eth_broadcast_addr(bcast_addr); 388034ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 388134ff6846SIoana Radulescu if (err) { 388234ff6846SIoana Radulescu dev_err(dev, "dpni_add_mac_addr() failed\n"); 388334ff6846SIoana Radulescu return err; 388434ff6846SIoana Radulescu } 388534ff6846SIoana Radulescu 388634ff6846SIoana Radulescu /* Set MTU upper limit; lower limit is 68B (default value) */ 388734ff6846SIoana Radulescu net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 388834ff6846SIoana Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 388934ff6846SIoana Radulescu DPAA2_ETH_MFL); 389034ff6846SIoana Radulescu if (err) { 389134ff6846SIoana Radulescu dev_err(dev, "dpni_set_max_frame_length() failed\n"); 389234ff6846SIoana Radulescu return err; 389334ff6846SIoana Radulescu } 389434ff6846SIoana Radulescu 389534ff6846SIoana Radulescu /* Set actual number of queues in the net device */ 389634ff6846SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 389734ff6846SIoana Radulescu err = netif_set_real_num_tx_queues(net_dev, num_queues); 389834ff6846SIoana Radulescu if (err) { 389934ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 390034ff6846SIoana Radulescu return err; 390134ff6846SIoana Radulescu } 390234ff6846SIoana Radulescu err = netif_set_real_num_rx_queues(net_dev, num_queues); 390334ff6846SIoana Radulescu if (err) { 390434ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 390534ff6846SIoana Radulescu return err; 390634ff6846SIoana Radulescu } 390734ff6846SIoana Radulescu 390834ff6846SIoana Radulescu /* Capabilities listing */ 390934ff6846SIoana Radulescu supported |= IFF_LIVE_ADDR_CHANGE; 391034ff6846SIoana Radulescu 391134ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER) 391234ff6846SIoana Radulescu not_supported |= IFF_UNICAST_FLT; 391334ff6846SIoana Radulescu else 391434ff6846SIoana Radulescu supported |= IFF_UNICAST_FLT; 391534ff6846SIoana Radulescu 391634ff6846SIoana Radulescu net_dev->priv_flags |= supported; 391734ff6846SIoana Radulescu net_dev->priv_flags &= ~not_supported; 391834ff6846SIoana Radulescu 391934ff6846SIoana Radulescu /* Features */ 392034ff6846SIoana Radulescu net_dev->features = NETIF_F_RXCSUM | 392134ff6846SIoana Radulescu NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 392234ff6846SIoana Radulescu NETIF_F_SG | NETIF_F_HIGHDMA | 39233657cdafSIoana Ciornei NETIF_F_LLTX | NETIF_F_HW_TC; 392434ff6846SIoana Radulescu net_dev->hw_features = net_dev->features; 392534ff6846SIoana Radulescu 392634ff6846SIoana Radulescu return 0; 392734ff6846SIoana Radulescu } 392834ff6846SIoana Radulescu 39295d8dccf8SIoana Ciornei static int dpaa2_eth_poll_link_state(void *arg) 393034ff6846SIoana Radulescu { 393134ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 393234ff6846SIoana Radulescu int err; 393334ff6846SIoana Radulescu 393434ff6846SIoana Radulescu while (!kthread_should_stop()) { 39355d8dccf8SIoana Ciornei err = dpaa2_eth_link_state_update(priv); 393634ff6846SIoana Radulescu if (unlikely(err)) 393734ff6846SIoana Radulescu return err; 393834ff6846SIoana Radulescu 393934ff6846SIoana Radulescu msleep(DPAA2_ETH_LINK_STATE_REFRESH); 394034ff6846SIoana Radulescu } 394134ff6846SIoana Radulescu 394234ff6846SIoana Radulescu return 0; 394334ff6846SIoana Radulescu } 394434ff6846SIoana Radulescu 394571947923SIoana Ciornei static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) 394671947923SIoana Ciornei { 394771947923SIoana Ciornei struct fsl_mc_device *dpni_dev, *dpmac_dev; 394871947923SIoana Ciornei struct dpaa2_mac *mac; 394971947923SIoana Ciornei int err; 395071947923SIoana Ciornei 395171947923SIoana Ciornei dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); 395271947923SIoana Ciornei dpmac_dev = fsl_mc_get_endpoint(dpni_dev); 3953841eb401SIoana Ciornei if (IS_ERR_OR_NULL(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) 395471947923SIoana Ciornei return 0; 395571947923SIoana Ciornei 395671947923SIoana Ciornei if (dpaa2_mac_is_type_fixed(dpmac_dev, priv->mc_io)) 395771947923SIoana Ciornei return 0; 395871947923SIoana Ciornei 395971947923SIoana Ciornei mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); 396071947923SIoana Ciornei if (!mac) 396171947923SIoana Ciornei return -ENOMEM; 396271947923SIoana Ciornei 396371947923SIoana Ciornei mac->mc_dev = dpmac_dev; 396471947923SIoana Ciornei mac->mc_io = priv->mc_io; 396571947923SIoana Ciornei mac->net_dev = priv->net_dev; 396671947923SIoana Ciornei 396771947923SIoana Ciornei err = dpaa2_mac_connect(mac); 396871947923SIoana Ciornei if (err) { 396971947923SIoana Ciornei netdev_err(priv->net_dev, "Error connecting to the MAC endpoint\n"); 397071947923SIoana Ciornei kfree(mac); 397171947923SIoana Ciornei return err; 397271947923SIoana Ciornei } 397371947923SIoana Ciornei priv->mac = mac; 397471947923SIoana Ciornei 397571947923SIoana Ciornei return 0; 397671947923SIoana Ciornei } 397771947923SIoana Ciornei 397871947923SIoana Ciornei static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) 397971947923SIoana Ciornei { 398071947923SIoana Ciornei if (!priv->mac) 398171947923SIoana Ciornei return; 398271947923SIoana Ciornei 398371947923SIoana Ciornei dpaa2_mac_disconnect(priv->mac); 398471947923SIoana Ciornei kfree(priv->mac); 398571947923SIoana Ciornei priv->mac = NULL; 398671947923SIoana Ciornei } 398771947923SIoana Ciornei 398834ff6846SIoana Radulescu static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 398934ff6846SIoana Radulescu { 399034ff6846SIoana Radulescu u32 status = ~0; 399134ff6846SIoana Radulescu struct device *dev = (struct device *)arg; 399234ff6846SIoana Radulescu struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 399334ff6846SIoana Radulescu struct net_device *net_dev = dev_get_drvdata(dev); 399471947923SIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 399534ff6846SIoana Radulescu int err; 399634ff6846SIoana Radulescu 399734ff6846SIoana Radulescu err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 399834ff6846SIoana Radulescu DPNI_IRQ_INDEX, &status); 399934ff6846SIoana Radulescu if (unlikely(err)) { 400034ff6846SIoana Radulescu netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 400134ff6846SIoana Radulescu return IRQ_HANDLED; 400234ff6846SIoana Radulescu } 400334ff6846SIoana Radulescu 400434ff6846SIoana Radulescu if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 40055d8dccf8SIoana Ciornei dpaa2_eth_link_state_update(netdev_priv(net_dev)); 400634ff6846SIoana Radulescu 4007f5c3fffaSIoana Ciornei if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) { 40085d8dccf8SIoana Ciornei dpaa2_eth_set_mac_addr(netdev_priv(net_dev)); 40095d8dccf8SIoana Ciornei dpaa2_eth_update_tx_fqids(priv); 401071947923SIoana Ciornei 401171947923SIoana Ciornei rtnl_lock(); 401271947923SIoana Ciornei if (priv->mac) 401371947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 401471947923SIoana Ciornei else 401571947923SIoana Ciornei dpaa2_eth_connect_mac(priv); 401671947923SIoana Ciornei rtnl_unlock(); 4017f5c3fffaSIoana Ciornei } 40188398b375SFlorin Chiculita 401934ff6846SIoana Radulescu return IRQ_HANDLED; 402034ff6846SIoana Radulescu } 402134ff6846SIoana Radulescu 40225d8dccf8SIoana Ciornei static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) 402334ff6846SIoana Radulescu { 402434ff6846SIoana Radulescu int err = 0; 402534ff6846SIoana Radulescu struct fsl_mc_device_irq *irq; 402634ff6846SIoana Radulescu 402734ff6846SIoana Radulescu err = fsl_mc_allocate_irqs(ls_dev); 402834ff6846SIoana Radulescu if (err) { 402934ff6846SIoana Radulescu dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 403034ff6846SIoana Radulescu return err; 403134ff6846SIoana Radulescu } 403234ff6846SIoana Radulescu 403334ff6846SIoana Radulescu irq = ls_dev->irqs[0]; 403434ff6846SIoana Radulescu err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, 403534ff6846SIoana Radulescu NULL, dpni_irq0_handler_thread, 403634ff6846SIoana Radulescu IRQF_NO_SUSPEND | IRQF_ONESHOT, 403734ff6846SIoana Radulescu dev_name(&ls_dev->dev), &ls_dev->dev); 403834ff6846SIoana Radulescu if (err < 0) { 403934ff6846SIoana Radulescu dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 404034ff6846SIoana Radulescu goto free_mc_irq; 404134ff6846SIoana Radulescu } 404234ff6846SIoana Radulescu 404334ff6846SIoana Radulescu err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 40448398b375SFlorin Chiculita DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 40458398b375SFlorin Chiculita DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 404634ff6846SIoana Radulescu if (err < 0) { 404734ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 404834ff6846SIoana Radulescu goto free_irq; 404934ff6846SIoana Radulescu } 405034ff6846SIoana Radulescu 405134ff6846SIoana Radulescu err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 405234ff6846SIoana Radulescu DPNI_IRQ_INDEX, 1); 405334ff6846SIoana Radulescu if (err < 0) { 405434ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 405534ff6846SIoana Radulescu goto free_irq; 405634ff6846SIoana Radulescu } 405734ff6846SIoana Radulescu 405834ff6846SIoana Radulescu return 0; 405934ff6846SIoana Radulescu 406034ff6846SIoana Radulescu free_irq: 406134ff6846SIoana Radulescu devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); 406234ff6846SIoana Radulescu free_mc_irq: 406334ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 406434ff6846SIoana Radulescu 406534ff6846SIoana Radulescu return err; 406634ff6846SIoana Radulescu } 406734ff6846SIoana Radulescu 40685d8dccf8SIoana Ciornei static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv) 406934ff6846SIoana Radulescu { 407034ff6846SIoana Radulescu int i; 407134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 407234ff6846SIoana Radulescu 407334ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 407434ff6846SIoana Radulescu ch = priv->channel[i]; 407534ff6846SIoana Radulescu /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 407634ff6846SIoana Radulescu netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 407734ff6846SIoana Radulescu NAPI_POLL_WEIGHT); 407834ff6846SIoana Radulescu } 407934ff6846SIoana Radulescu } 408034ff6846SIoana Radulescu 40815d8dccf8SIoana Ciornei static void dpaa2_eth_del_ch_napi(struct dpaa2_eth_priv *priv) 408234ff6846SIoana Radulescu { 408334ff6846SIoana Radulescu int i; 408434ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 408534ff6846SIoana Radulescu 408634ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 408734ff6846SIoana Radulescu ch = priv->channel[i]; 408834ff6846SIoana Radulescu netif_napi_del(&ch->napi); 408934ff6846SIoana Radulescu } 409034ff6846SIoana Radulescu } 409134ff6846SIoana Radulescu 409234ff6846SIoana Radulescu static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 409334ff6846SIoana Radulescu { 409434ff6846SIoana Radulescu struct device *dev; 409534ff6846SIoana Radulescu struct net_device *net_dev = NULL; 409634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = NULL; 409734ff6846SIoana Radulescu int err = 0; 409834ff6846SIoana Radulescu 409934ff6846SIoana Radulescu dev = &dpni_dev->dev; 410034ff6846SIoana Radulescu 410134ff6846SIoana Radulescu /* Net device */ 4102ab1e6de2SIoana Radulescu net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 410334ff6846SIoana Radulescu if (!net_dev) { 410434ff6846SIoana Radulescu dev_err(dev, "alloc_etherdev_mq() failed\n"); 410534ff6846SIoana Radulescu return -ENOMEM; 410634ff6846SIoana Radulescu } 410734ff6846SIoana Radulescu 410834ff6846SIoana Radulescu SET_NETDEV_DEV(net_dev, dev); 410934ff6846SIoana Radulescu dev_set_drvdata(dev, net_dev); 411034ff6846SIoana Radulescu 411134ff6846SIoana Radulescu priv = netdev_priv(net_dev); 411234ff6846SIoana Radulescu priv->net_dev = net_dev; 411334ff6846SIoana Radulescu 411434ff6846SIoana Radulescu priv->iommu_domain = iommu_get_domain_for_dev(dev); 411534ff6846SIoana Radulescu 41161cf773bdSYangbo Lu priv->tx_tstamp_type = HWTSTAMP_TX_OFF; 41171cf773bdSYangbo Lu priv->rx_tstamp = false; 41181cf773bdSYangbo Lu 4119c5521189SYangbo Lu priv->dpaa2_ptp_wq = alloc_workqueue("dpaa2_ptp_wq", 0, 0); 4120c5521189SYangbo Lu if (!priv->dpaa2_ptp_wq) { 4121c5521189SYangbo Lu err = -ENOMEM; 4122c5521189SYangbo Lu goto err_wq_alloc; 4123c5521189SYangbo Lu } 4124c5521189SYangbo Lu 4125c5521189SYangbo Lu INIT_WORK(&priv->tx_onestep_tstamp, dpaa2_eth_tx_onestep_tstamp); 4126c5521189SYangbo Lu 4127c5521189SYangbo Lu skb_queue_head_init(&priv->tx_skbs); 4128c5521189SYangbo Lu 412934ff6846SIoana Radulescu /* Obtain a MC portal */ 413034ff6846SIoana Radulescu err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 413134ff6846SIoana Radulescu &priv->mc_io); 413234ff6846SIoana Radulescu if (err) { 413334ff6846SIoana Radulescu if (err == -ENXIO) 413434ff6846SIoana Radulescu err = -EPROBE_DEFER; 413534ff6846SIoana Radulescu else 413634ff6846SIoana Radulescu dev_err(dev, "MC portal allocation failed\n"); 413734ff6846SIoana Radulescu goto err_portal_alloc; 413834ff6846SIoana Radulescu } 413934ff6846SIoana Radulescu 414034ff6846SIoana Radulescu /* MC objects initialization and configuration */ 41415d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpni(dpni_dev); 414234ff6846SIoana Radulescu if (err) 414334ff6846SIoana Radulescu goto err_dpni_setup; 414434ff6846SIoana Radulescu 41455d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpio(priv); 414634ff6846SIoana Radulescu if (err) 414734ff6846SIoana Radulescu goto err_dpio_setup; 414834ff6846SIoana Radulescu 41495d8dccf8SIoana Ciornei dpaa2_eth_setup_fqs(priv); 415034ff6846SIoana Radulescu 41515d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpbp(priv); 415234ff6846SIoana Radulescu if (err) 415334ff6846SIoana Radulescu goto err_dpbp_setup; 415434ff6846SIoana Radulescu 41555d8dccf8SIoana Ciornei err = dpaa2_eth_bind_dpni(priv); 415634ff6846SIoana Radulescu if (err) 415734ff6846SIoana Radulescu goto err_bind; 415834ff6846SIoana Radulescu 415934ff6846SIoana Radulescu /* Add a NAPI context for each channel */ 41605d8dccf8SIoana Ciornei dpaa2_eth_add_ch_napi(priv); 416134ff6846SIoana Radulescu 416234ff6846SIoana Radulescu /* Percpu statistics */ 416334ff6846SIoana Radulescu priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 416434ff6846SIoana Radulescu if (!priv->percpu_stats) { 416534ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 416634ff6846SIoana Radulescu err = -ENOMEM; 416734ff6846SIoana Radulescu goto err_alloc_percpu_stats; 416834ff6846SIoana Radulescu } 416934ff6846SIoana Radulescu priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 417034ff6846SIoana Radulescu if (!priv->percpu_extras) { 417134ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 417234ff6846SIoana Radulescu err = -ENOMEM; 417334ff6846SIoana Radulescu goto err_alloc_percpu_extras; 417434ff6846SIoana Radulescu } 417534ff6846SIoana Radulescu 4176d70446eeSIoana Ciornei priv->sgt_cache = alloc_percpu(*priv->sgt_cache); 4177d70446eeSIoana Ciornei if (!priv->sgt_cache) { 4178d70446eeSIoana Ciornei dev_err(dev, "alloc_percpu(sgt_cache) failed\n"); 4179d70446eeSIoana Ciornei err = -ENOMEM; 4180d70446eeSIoana Ciornei goto err_alloc_sgt_cache; 4181d70446eeSIoana Ciornei } 4182d70446eeSIoana Ciornei 41835d8dccf8SIoana Ciornei err = dpaa2_eth_netdev_init(net_dev); 418434ff6846SIoana Radulescu if (err) 418534ff6846SIoana Radulescu goto err_netdev_init; 418634ff6846SIoana Radulescu 418734ff6846SIoana Radulescu /* Configure checksum offload based on current interface flags */ 41885d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 418934ff6846SIoana Radulescu if (err) 419034ff6846SIoana Radulescu goto err_csum; 419134ff6846SIoana Radulescu 41925d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, 41935d8dccf8SIoana Ciornei !!(net_dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 419434ff6846SIoana Radulescu if (err) 419534ff6846SIoana Radulescu goto err_csum; 419634ff6846SIoana Radulescu 41975d8dccf8SIoana Ciornei err = dpaa2_eth_alloc_rings(priv); 419834ff6846SIoana Radulescu if (err) 419934ff6846SIoana Radulescu goto err_alloc_rings; 420034ff6846SIoana Radulescu 4201f395b69fSIoana Ciornei #ifdef CONFIG_FSL_DPAA2_ETH_DCB 4202f395b69fSIoana Ciornei if (dpaa2_eth_has_pause_support(priv) && priv->vlan_cls_enabled) { 4203f395b69fSIoana Ciornei priv->dcbx_mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; 4204f395b69fSIoana Ciornei net_dev->dcbnl_ops = &dpaa2_eth_dcbnl_ops; 4205f395b69fSIoana Ciornei } else { 4206f395b69fSIoana Ciornei dev_dbg(dev, "PFC not supported\n"); 4207f395b69fSIoana Ciornei } 4208f395b69fSIoana Ciornei #endif 4209f395b69fSIoana Ciornei 42105d8dccf8SIoana Ciornei err = dpaa2_eth_setup_irqs(dpni_dev); 421134ff6846SIoana Radulescu if (err) { 421234ff6846SIoana Radulescu netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 42135d8dccf8SIoana Ciornei priv->poll_thread = kthread_run(dpaa2_eth_poll_link_state, priv, 421434ff6846SIoana Radulescu "%s_poll_link", net_dev->name); 421534ff6846SIoana Radulescu if (IS_ERR(priv->poll_thread)) { 421634ff6846SIoana Radulescu dev_err(dev, "Error starting polling thread\n"); 421734ff6846SIoana Radulescu goto err_poll_thread; 421834ff6846SIoana Radulescu } 421934ff6846SIoana Radulescu priv->do_link_poll = true; 422034ff6846SIoana Radulescu } 422134ff6846SIoana Radulescu 422271947923SIoana Ciornei err = dpaa2_eth_connect_mac(priv); 422371947923SIoana Ciornei if (err) 422471947923SIoana Ciornei goto err_connect_mac; 422571947923SIoana Ciornei 422634ff6846SIoana Radulescu err = register_netdev(net_dev); 422734ff6846SIoana Radulescu if (err < 0) { 422834ff6846SIoana Radulescu dev_err(dev, "register_netdev() failed\n"); 422934ff6846SIoana Radulescu goto err_netdev_reg; 423034ff6846SIoana Radulescu } 423134ff6846SIoana Radulescu 4232091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4233091a19eaSIoana Radulescu dpaa2_dbg_add(priv); 4234091a19eaSIoana Radulescu #endif 4235091a19eaSIoana Radulescu 423634ff6846SIoana Radulescu dev_info(dev, "Probed interface %s\n", net_dev->name); 423734ff6846SIoana Radulescu return 0; 423834ff6846SIoana Radulescu 423934ff6846SIoana Radulescu err_netdev_reg: 424071947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 424171947923SIoana Ciornei err_connect_mac: 424234ff6846SIoana Radulescu if (priv->do_link_poll) 424334ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 424434ff6846SIoana Radulescu else 424534ff6846SIoana Radulescu fsl_mc_free_irqs(dpni_dev); 424634ff6846SIoana Radulescu err_poll_thread: 42475d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 424834ff6846SIoana Radulescu err_alloc_rings: 424934ff6846SIoana Radulescu err_csum: 425034ff6846SIoana Radulescu err_netdev_init: 4251d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 4252d70446eeSIoana Ciornei err_alloc_sgt_cache: 425334ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 425434ff6846SIoana Radulescu err_alloc_percpu_extras: 425534ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 425634ff6846SIoana Radulescu err_alloc_percpu_stats: 42575d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 425834ff6846SIoana Radulescu err_bind: 42595d8dccf8SIoana Ciornei dpaa2_eth_free_dpbp(priv); 426034ff6846SIoana Radulescu err_dpbp_setup: 42615d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 426234ff6846SIoana Radulescu err_dpio_setup: 42635d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 426434ff6846SIoana Radulescu err_dpni_setup: 426534ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 426634ff6846SIoana Radulescu err_portal_alloc: 4267c5521189SYangbo Lu destroy_workqueue(priv->dpaa2_ptp_wq); 4268c5521189SYangbo Lu err_wq_alloc: 426934ff6846SIoana Radulescu dev_set_drvdata(dev, NULL); 427034ff6846SIoana Radulescu free_netdev(net_dev); 427134ff6846SIoana Radulescu 427234ff6846SIoana Radulescu return err; 427334ff6846SIoana Radulescu } 427434ff6846SIoana Radulescu 427534ff6846SIoana Radulescu static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 427634ff6846SIoana Radulescu { 427734ff6846SIoana Radulescu struct device *dev; 427834ff6846SIoana Radulescu struct net_device *net_dev; 427934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 428034ff6846SIoana Radulescu 428134ff6846SIoana Radulescu dev = &ls_dev->dev; 428234ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 428334ff6846SIoana Radulescu priv = netdev_priv(net_dev); 428434ff6846SIoana Radulescu 4285091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4286091a19eaSIoana Radulescu dpaa2_dbg_remove(priv); 4287091a19eaSIoana Radulescu #endif 428871947923SIoana Ciornei rtnl_lock(); 428971947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 429071947923SIoana Ciornei rtnl_unlock(); 429171947923SIoana Ciornei 429234ff6846SIoana Radulescu unregister_netdev(net_dev); 429334ff6846SIoana Radulescu 429434ff6846SIoana Radulescu if (priv->do_link_poll) 429534ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 429634ff6846SIoana Radulescu else 429734ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 429834ff6846SIoana Radulescu 42995d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 4300d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 430134ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 430234ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 430334ff6846SIoana Radulescu 43045d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 43055d8dccf8SIoana Ciornei dpaa2_eth_free_dpbp(priv); 43065d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 43075d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 430834ff6846SIoana Radulescu 430934ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 431034ff6846SIoana Radulescu 431134ff6846SIoana Radulescu free_netdev(net_dev); 431234ff6846SIoana Radulescu 431334ff6846SIoana Radulescu dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 431434ff6846SIoana Radulescu 431534ff6846SIoana Radulescu return 0; 431634ff6846SIoana Radulescu } 431734ff6846SIoana Radulescu 431834ff6846SIoana Radulescu static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 431934ff6846SIoana Radulescu { 432034ff6846SIoana Radulescu .vendor = FSL_MC_VENDOR_FREESCALE, 432134ff6846SIoana Radulescu .obj_type = "dpni", 432234ff6846SIoana Radulescu }, 432334ff6846SIoana Radulescu { .vendor = 0x0 } 432434ff6846SIoana Radulescu }; 432534ff6846SIoana Radulescu MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 432634ff6846SIoana Radulescu 432734ff6846SIoana Radulescu static struct fsl_mc_driver dpaa2_eth_driver = { 432834ff6846SIoana Radulescu .driver = { 432934ff6846SIoana Radulescu .name = KBUILD_MODNAME, 433034ff6846SIoana Radulescu .owner = THIS_MODULE, 433134ff6846SIoana Radulescu }, 433234ff6846SIoana Radulescu .probe = dpaa2_eth_probe, 433334ff6846SIoana Radulescu .remove = dpaa2_eth_remove, 433434ff6846SIoana Radulescu .match_id_table = dpaa2_eth_match_id_table 433534ff6846SIoana Radulescu }; 433634ff6846SIoana Radulescu 4337091a19eaSIoana Radulescu static int __init dpaa2_eth_driver_init(void) 4338091a19eaSIoana Radulescu { 4339091a19eaSIoana Radulescu int err; 4340091a19eaSIoana Radulescu 4341091a19eaSIoana Radulescu dpaa2_eth_dbg_init(); 4342091a19eaSIoana Radulescu err = fsl_mc_driver_register(&dpaa2_eth_driver); 4343091a19eaSIoana Radulescu if (err) { 4344091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4345091a19eaSIoana Radulescu return err; 4346091a19eaSIoana Radulescu } 4347091a19eaSIoana Radulescu 4348091a19eaSIoana Radulescu return 0; 4349091a19eaSIoana Radulescu } 4350091a19eaSIoana Radulescu 4351091a19eaSIoana Radulescu static void __exit dpaa2_eth_driver_exit(void) 4352091a19eaSIoana Radulescu { 4353091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4354091a19eaSIoana Radulescu fsl_mc_driver_unregister(&dpaa2_eth_driver); 4355091a19eaSIoana Radulescu } 4356091a19eaSIoana Radulescu 4357091a19eaSIoana Radulescu module_init(dpaa2_eth_driver_init); 4358091a19eaSIoana Radulescu module_exit(dpaa2_eth_driver_exit); 4359