134ff6846SIoana Radulescu // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 234ff6846SIoana Radulescu /* Copyright 2014-2016 Freescale Semiconductor Inc. 348c0481eSIoana Ciornei * Copyright 2016-2020 NXP 434ff6846SIoana Radulescu */ 534ff6846SIoana Radulescu #include <linux/init.h> 634ff6846SIoana Radulescu #include <linux/module.h> 734ff6846SIoana Radulescu #include <linux/platform_device.h> 834ff6846SIoana Radulescu #include <linux/etherdevice.h> 934ff6846SIoana Radulescu #include <linux/of_net.h> 1034ff6846SIoana Radulescu #include <linux/interrupt.h> 1134ff6846SIoana Radulescu #include <linux/msi.h> 1234ff6846SIoana Radulescu #include <linux/kthread.h> 1334ff6846SIoana Radulescu #include <linux/iommu.h> 1434ff6846SIoana Radulescu #include <linux/fsl/mc.h> 157e273a8eSIoana Ciocoi Radulescu #include <linux/bpf.h> 167e273a8eSIoana Ciocoi Radulescu #include <linux/bpf_trace.h> 17d21c784cSYangbo Lu #include <linux/fsl/ptp_qoriq.h> 18c5521189SYangbo Lu #include <linux/ptp_classify.h> 193657cdafSIoana Ciornei #include <net/pkt_cls.h> 2034ff6846SIoana Radulescu #include <net/sock.h> 2134ff6846SIoana Radulescu 2234ff6846SIoana Radulescu #include "dpaa2-eth.h" 2334ff6846SIoana Radulescu 2434ff6846SIoana Radulescu /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 2534ff6846SIoana Radulescu * using trace events only need to #include <trace/events/sched.h> 2634ff6846SIoana Radulescu */ 2734ff6846SIoana Radulescu #define CREATE_TRACE_POINTS 2834ff6846SIoana Radulescu #include "dpaa2-eth-trace.h" 2934ff6846SIoana Radulescu 3034ff6846SIoana Radulescu MODULE_LICENSE("Dual BSD/GPL"); 3134ff6846SIoana Radulescu MODULE_AUTHOR("Freescale Semiconductor, Inc"); 3234ff6846SIoana Radulescu MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 3334ff6846SIoana Radulescu 34d21c784cSYangbo Lu struct ptp_qoriq *dpaa2_ptp; 35d21c784cSYangbo Lu EXPORT_SYMBOL(dpaa2_ptp); 36d21c784cSYangbo Lu 3734ff6846SIoana Radulescu static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 3834ff6846SIoana Radulescu dma_addr_t iova_addr) 3934ff6846SIoana Radulescu { 4034ff6846SIoana Radulescu phys_addr_t phys_addr; 4134ff6846SIoana Radulescu 4234ff6846SIoana Radulescu phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 4334ff6846SIoana Radulescu 4434ff6846SIoana Radulescu return phys_to_virt(phys_addr); 4534ff6846SIoana Radulescu } 4634ff6846SIoana Radulescu 475d8dccf8SIoana Ciornei static void dpaa2_eth_validate_rx_csum(struct dpaa2_eth_priv *priv, 4834ff6846SIoana Radulescu u32 fd_status, 4934ff6846SIoana Radulescu struct sk_buff *skb) 5034ff6846SIoana Radulescu { 5134ff6846SIoana Radulescu skb_checksum_none_assert(skb); 5234ff6846SIoana Radulescu 5334ff6846SIoana Radulescu /* HW checksum validation is disabled, nothing to do here */ 5434ff6846SIoana Radulescu if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 5534ff6846SIoana Radulescu return; 5634ff6846SIoana Radulescu 5734ff6846SIoana Radulescu /* Read checksum validation bits */ 5834ff6846SIoana Radulescu if (!((fd_status & DPAA2_FAS_L3CV) && 5934ff6846SIoana Radulescu (fd_status & DPAA2_FAS_L4CV))) 6034ff6846SIoana Radulescu return; 6134ff6846SIoana Radulescu 6234ff6846SIoana Radulescu /* Inform the stack there's no need to compute L3/L4 csum anymore */ 6334ff6846SIoana Radulescu skb->ip_summed = CHECKSUM_UNNECESSARY; 6434ff6846SIoana Radulescu } 6534ff6846SIoana Radulescu 6634ff6846SIoana Radulescu /* Free a received FD. 6734ff6846SIoana Radulescu * Not to be used for Tx conf FDs or on any other paths. 6834ff6846SIoana Radulescu */ 695d8dccf8SIoana Ciornei static void dpaa2_eth_free_rx_fd(struct dpaa2_eth_priv *priv, 7034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 7134ff6846SIoana Radulescu void *vaddr) 7234ff6846SIoana Radulescu { 7334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 7434ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 7534ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 7634ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 7734ff6846SIoana Radulescu void *sg_vaddr; 7834ff6846SIoana Radulescu int i; 7934ff6846SIoana Radulescu 8034ff6846SIoana Radulescu /* If single buffer frame, just free the data buffer */ 8134ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) 8234ff6846SIoana Radulescu goto free_buf; 8334ff6846SIoana Radulescu else if (fd_format != dpaa2_fd_sg) 8434ff6846SIoana Radulescu /* We don't support any other format */ 8534ff6846SIoana Radulescu return; 8634ff6846SIoana Radulescu 8734ff6846SIoana Radulescu /* For S/G frames, we first need to free all SG entries 8834ff6846SIoana Radulescu * except the first one, which was taken care of already 8934ff6846SIoana Radulescu */ 9034ff6846SIoana Radulescu sgt = vaddr + dpaa2_fd_get_offset(fd); 9134ff6846SIoana Radulescu for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 9234ff6846SIoana Radulescu addr = dpaa2_sg_get_addr(&sgt[i]); 9334ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 94efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 9518c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 9634ff6846SIoana Radulescu 9727c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 9834ff6846SIoana Radulescu if (dpaa2_sg_is_final(&sgt[i])) 9934ff6846SIoana Radulescu break; 10034ff6846SIoana Radulescu } 10134ff6846SIoana Radulescu 10234ff6846SIoana Radulescu free_buf: 10327c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 10434ff6846SIoana Radulescu } 10534ff6846SIoana Radulescu 10634ff6846SIoana Radulescu /* Build a linear skb based on a single-buffer frame descriptor */ 1075d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_linear_skb(struct dpaa2_eth_channel *ch, 10834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 10934ff6846SIoana Radulescu void *fd_vaddr) 11034ff6846SIoana Radulescu { 11134ff6846SIoana Radulescu struct sk_buff *skb = NULL; 11234ff6846SIoana Radulescu u16 fd_offset = dpaa2_fd_get_offset(fd); 11334ff6846SIoana Radulescu u32 fd_length = dpaa2_fd_get_len(fd); 11434ff6846SIoana Radulescu 11534ff6846SIoana Radulescu ch->buf_count--; 11634ff6846SIoana Radulescu 11727c87486SIoana Ciocoi Radulescu skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 11834ff6846SIoana Radulescu if (unlikely(!skb)) 11934ff6846SIoana Radulescu return NULL; 12034ff6846SIoana Radulescu 12134ff6846SIoana Radulescu skb_reserve(skb, fd_offset); 12234ff6846SIoana Radulescu skb_put(skb, fd_length); 12334ff6846SIoana Radulescu 12434ff6846SIoana Radulescu return skb; 12534ff6846SIoana Radulescu } 12634ff6846SIoana Radulescu 12734ff6846SIoana Radulescu /* Build a non linear (fragmented) skb based on a S/G table */ 1285d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_frag_skb(struct dpaa2_eth_priv *priv, 12934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 13034ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt) 13134ff6846SIoana Radulescu { 13234ff6846SIoana Radulescu struct sk_buff *skb = NULL; 13334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 13434ff6846SIoana Radulescu void *sg_vaddr; 13534ff6846SIoana Radulescu dma_addr_t sg_addr; 13634ff6846SIoana Radulescu u16 sg_offset; 13734ff6846SIoana Radulescu u32 sg_length; 13834ff6846SIoana Radulescu struct page *page, *head_page; 13934ff6846SIoana Radulescu int page_offset; 14034ff6846SIoana Radulescu int i; 14134ff6846SIoana Radulescu 14234ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 14334ff6846SIoana Radulescu struct dpaa2_sg_entry *sge = &sgt[i]; 14434ff6846SIoana Radulescu 14534ff6846SIoana Radulescu /* NOTE: We only support SG entries in dpaa2_sg_single format, 14634ff6846SIoana Radulescu * but this is the only format we may receive from HW anyway 14734ff6846SIoana Radulescu */ 14834ff6846SIoana Radulescu 14934ff6846SIoana Radulescu /* Get the address and length from the S/G entry */ 15034ff6846SIoana Radulescu sg_addr = dpaa2_sg_get_addr(sge); 15134ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 152efa6a7d0SIoana Ciornei dma_unmap_page(dev, sg_addr, priv->rx_buf_size, 15318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 15434ff6846SIoana Radulescu 15534ff6846SIoana Radulescu sg_length = dpaa2_sg_get_len(sge); 15634ff6846SIoana Radulescu 15734ff6846SIoana Radulescu if (i == 0) { 15834ff6846SIoana Radulescu /* We build the skb around the first data buffer */ 15927c87486SIoana Ciocoi Radulescu skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 16034ff6846SIoana Radulescu if (unlikely(!skb)) { 16134ff6846SIoana Radulescu /* Free the first SG entry now, since we already 16234ff6846SIoana Radulescu * unmapped it and obtained the virtual address 16334ff6846SIoana Radulescu */ 16427c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 16534ff6846SIoana Radulescu 16634ff6846SIoana Radulescu /* We still need to subtract the buffers used 16734ff6846SIoana Radulescu * by this FD from our software counter 16834ff6846SIoana Radulescu */ 16934ff6846SIoana Radulescu while (!dpaa2_sg_is_final(&sgt[i]) && 17034ff6846SIoana Radulescu i < DPAA2_ETH_MAX_SG_ENTRIES) 17134ff6846SIoana Radulescu i++; 17234ff6846SIoana Radulescu break; 17334ff6846SIoana Radulescu } 17434ff6846SIoana Radulescu 17534ff6846SIoana Radulescu sg_offset = dpaa2_sg_get_offset(sge); 17634ff6846SIoana Radulescu skb_reserve(skb, sg_offset); 17734ff6846SIoana Radulescu skb_put(skb, sg_length); 17834ff6846SIoana Radulescu } else { 17934ff6846SIoana Radulescu /* Rest of the data buffers are stored as skb frags */ 18034ff6846SIoana Radulescu page = virt_to_page(sg_vaddr); 18134ff6846SIoana Radulescu head_page = virt_to_head_page(sg_vaddr); 18234ff6846SIoana Radulescu 18334ff6846SIoana Radulescu /* Offset in page (which may be compound). 18434ff6846SIoana Radulescu * Data in subsequent SG entries is stored from the 18534ff6846SIoana Radulescu * beginning of the buffer, so we don't need to add the 18634ff6846SIoana Radulescu * sg_offset. 18734ff6846SIoana Radulescu */ 18834ff6846SIoana Radulescu page_offset = ((unsigned long)sg_vaddr & 18934ff6846SIoana Radulescu (PAGE_SIZE - 1)) + 19034ff6846SIoana Radulescu (page_address(page) - page_address(head_page)); 19134ff6846SIoana Radulescu 19234ff6846SIoana Radulescu skb_add_rx_frag(skb, i - 1, head_page, page_offset, 193efa6a7d0SIoana Ciornei sg_length, priv->rx_buf_size); 19434ff6846SIoana Radulescu } 19534ff6846SIoana Radulescu 19634ff6846SIoana Radulescu if (dpaa2_sg_is_final(sge)) 19734ff6846SIoana Radulescu break; 19834ff6846SIoana Radulescu } 19934ff6846SIoana Radulescu 20034ff6846SIoana Radulescu WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 20134ff6846SIoana Radulescu 20234ff6846SIoana Radulescu /* Count all data buffers + SG table buffer */ 20334ff6846SIoana Radulescu ch->buf_count -= i + 2; 20434ff6846SIoana Radulescu 20534ff6846SIoana Radulescu return skb; 20634ff6846SIoana Radulescu } 20734ff6846SIoana Radulescu 208569375fbSIoana Ciocoi Radulescu /* Free buffers acquired from the buffer pool or which were meant to 209569375fbSIoana Ciocoi Radulescu * be released in the pool 210569375fbSIoana Ciocoi Radulescu */ 2115d8dccf8SIoana Ciornei static void dpaa2_eth_free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, 2125d8dccf8SIoana Ciornei int count) 213569375fbSIoana Ciocoi Radulescu { 214569375fbSIoana Ciocoi Radulescu struct device *dev = priv->net_dev->dev.parent; 215569375fbSIoana Ciocoi Radulescu void *vaddr; 216569375fbSIoana Ciocoi Radulescu int i; 217569375fbSIoana Ciocoi Radulescu 218569375fbSIoana Ciocoi Radulescu for (i = 0; i < count; i++) { 219569375fbSIoana Ciocoi Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 220efa6a7d0SIoana Ciornei dma_unmap_page(dev, buf_array[i], priv->rx_buf_size, 22118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 22227c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 223569375fbSIoana Ciocoi Radulescu } 224569375fbSIoana Ciocoi Radulescu } 225569375fbSIoana Ciocoi Radulescu 22628d137ccSIoana Ciornei static void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv, 2275d39dc21SIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 2285d39dc21SIoana Ciocoi Radulescu dma_addr_t addr) 2295d39dc21SIoana Ciocoi Radulescu { 230ef17bd7cSIoana Radulescu int retries = 0; 2315d39dc21SIoana Ciocoi Radulescu int err; 2325d39dc21SIoana Ciocoi Radulescu 23328d137ccSIoana Ciornei ch->recycled_bufs[ch->recycled_bufs_cnt++] = addr; 23428d137ccSIoana Ciornei if (ch->recycled_bufs_cnt < DPAA2_ETH_BUFS_PER_CMD) 2355d39dc21SIoana Ciocoi Radulescu return; 2365d39dc21SIoana Ciocoi Radulescu 2375d39dc21SIoana Ciocoi Radulescu while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 23828d137ccSIoana Ciornei ch->recycled_bufs, 23928d137ccSIoana Ciornei ch->recycled_bufs_cnt)) == -EBUSY) { 240ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 241ef17bd7cSIoana Radulescu break; 2425d39dc21SIoana Ciocoi Radulescu cpu_relax(); 243ef17bd7cSIoana Radulescu } 2445d39dc21SIoana Ciocoi Radulescu 2455d39dc21SIoana Ciocoi Radulescu if (err) { 24628d137ccSIoana Ciornei dpaa2_eth_free_bufs(priv, ch->recycled_bufs, ch->recycled_bufs_cnt); 24728d137ccSIoana Ciornei ch->buf_count -= ch->recycled_bufs_cnt; 2485d39dc21SIoana Ciocoi Radulescu } 2495d39dc21SIoana Ciocoi Radulescu 25028d137ccSIoana Ciornei ch->recycled_bufs_cnt = 0; 2515d39dc21SIoana Ciocoi Radulescu } 2525d39dc21SIoana Ciocoi Radulescu 25338c440b2SIoana Ciornei static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 25438c440b2SIoana Ciornei struct dpaa2_eth_fq *fq, 25538c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_fds) 25638c440b2SIoana Ciornei { 25738c440b2SIoana Ciornei int total_enqueued = 0, retries = 0, enqueued; 25838c440b2SIoana Ciornei struct dpaa2_eth_drv_stats *percpu_extras; 25938c440b2SIoana Ciornei int num_fds, err, max_retries; 26038c440b2SIoana Ciornei struct dpaa2_fd *fds; 26138c440b2SIoana Ciornei 26238c440b2SIoana Ciornei percpu_extras = this_cpu_ptr(priv->percpu_extras); 26338c440b2SIoana Ciornei 26438c440b2SIoana Ciornei /* try to enqueue all the FDs until the max number of retries is hit */ 26538c440b2SIoana Ciornei fds = xdp_fds->fds; 26638c440b2SIoana Ciornei num_fds = xdp_fds->num; 26738c440b2SIoana Ciornei max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 26838c440b2SIoana Ciornei while (total_enqueued < num_fds && retries < max_retries) { 26938c440b2SIoana Ciornei err = priv->enqueue(priv, fq, &fds[total_enqueued], 27038c440b2SIoana Ciornei 0, num_fds - total_enqueued, &enqueued); 27138c440b2SIoana Ciornei if (err == -EBUSY) { 27238c440b2SIoana Ciornei percpu_extras->tx_portal_busy += ++retries; 27338c440b2SIoana Ciornei continue; 27438c440b2SIoana Ciornei } 27538c440b2SIoana Ciornei total_enqueued += enqueued; 27638c440b2SIoana Ciornei } 27738c440b2SIoana Ciornei xdp_fds->num = 0; 27838c440b2SIoana Ciornei 27938c440b2SIoana Ciornei return total_enqueued; 28038c440b2SIoana Ciornei } 28138c440b2SIoana Ciornei 2825d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_tx_flush(struct dpaa2_eth_priv *priv, 28374a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 28474a1c059SIoana Ciornei struct dpaa2_eth_fq *fq) 28574a1c059SIoana Ciornei { 28674a1c059SIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 28774a1c059SIoana Ciornei struct dpaa2_fd *fds; 28874a1c059SIoana Ciornei int enqueued, i; 28974a1c059SIoana Ciornei 29074a1c059SIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 29174a1c059SIoana Ciornei 29274a1c059SIoana Ciornei // enqueue the array of XDP_TX frames 29374a1c059SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds); 29474a1c059SIoana Ciornei 29574a1c059SIoana Ciornei /* update statistics */ 29674a1c059SIoana Ciornei percpu_stats->tx_packets += enqueued; 29774a1c059SIoana Ciornei fds = fq->xdp_tx_fds.fds; 29874a1c059SIoana Ciornei for (i = 0; i < enqueued; i++) { 29974a1c059SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 30074a1c059SIoana Ciornei ch->stats.xdp_tx++; 30174a1c059SIoana Ciornei } 30274a1c059SIoana Ciornei for (i = enqueued; i < fq->xdp_tx_fds.num; i++) { 30328d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, dpaa2_fd_get_addr(&fds[i])); 30474a1c059SIoana Ciornei percpu_stats->tx_errors++; 30574a1c059SIoana Ciornei ch->stats.xdp_tx_err++; 30674a1c059SIoana Ciornei } 30774a1c059SIoana Ciornei fq->xdp_tx_fds.num = 0; 30874a1c059SIoana Ciornei } 30974a1c059SIoana Ciornei 3105d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv, 31174a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 31274a1c059SIoana Ciornei struct dpaa2_fd *fd, 31399e43521SIoana Ciocoi Radulescu void *buf_start, u16 queue_id) 31499e43521SIoana Ciocoi Radulescu { 31599e43521SIoana Ciocoi Radulescu struct dpaa2_faead *faead; 31674a1c059SIoana Ciornei struct dpaa2_fd *dest_fd; 31774a1c059SIoana Ciornei struct dpaa2_eth_fq *fq; 31899e43521SIoana Ciocoi Radulescu u32 ctrl, frc; 31999e43521SIoana Ciocoi Radulescu 32099e43521SIoana Ciocoi Radulescu /* Mark the egress frame hardware annotation area as valid */ 32199e43521SIoana Ciocoi Radulescu frc = dpaa2_fd_get_frc(fd); 32299e43521SIoana Ciocoi Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 32399e43521SIoana Ciocoi Radulescu dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 32499e43521SIoana Ciocoi Radulescu 32599e43521SIoana Ciocoi Radulescu /* Instruct hardware to release the FD buffer directly into 32699e43521SIoana Ciocoi Radulescu * the buffer pool once transmission is completed, instead of 32799e43521SIoana Ciocoi Radulescu * sending a Tx confirmation frame to us 32899e43521SIoana Ciocoi Radulescu */ 32999e43521SIoana Ciocoi Radulescu ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 33099e43521SIoana Ciocoi Radulescu faead = dpaa2_get_faead(buf_start, false); 33199e43521SIoana Ciocoi Radulescu faead->ctrl = cpu_to_le32(ctrl); 33299e43521SIoana Ciocoi Radulescu faead->conf_fqid = 0; 33399e43521SIoana Ciocoi Radulescu 33499e43521SIoana Ciocoi Radulescu fq = &priv->fq[queue_id]; 33574a1c059SIoana Ciornei dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++]; 33674a1c059SIoana Ciornei memcpy(dest_fd, fd, sizeof(*dest_fd)); 33799e43521SIoana Ciocoi Radulescu 33874a1c059SIoana Ciornei if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE) 33974a1c059SIoana Ciornei return; 34074a1c059SIoana Ciornei 3415d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, fq); 34299e43521SIoana Ciocoi Radulescu } 34399e43521SIoana Ciocoi Radulescu 3445d8dccf8SIoana Ciornei static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, 3457e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 34699e43521SIoana Ciocoi Radulescu struct dpaa2_eth_fq *rx_fq, 3477e273a8eSIoana Ciocoi Radulescu struct dpaa2_fd *fd, void *vaddr) 3487e273a8eSIoana Ciocoi Radulescu { 3495d39dc21SIoana Ciocoi Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 3507e273a8eSIoana Ciocoi Radulescu struct bpf_prog *xdp_prog; 3517e273a8eSIoana Ciocoi Radulescu struct xdp_buff xdp; 3527e273a8eSIoana Ciocoi Radulescu u32 xdp_act = XDP_PASS; 353be9df4afSLorenzo Bianconi int err, offset; 35499e43521SIoana Ciocoi Radulescu 3557e273a8eSIoana Ciocoi Radulescu xdp_prog = READ_ONCE(ch->xdp.prog); 3567e273a8eSIoana Ciocoi Radulescu if (!xdp_prog) 3577e273a8eSIoana Ciocoi Radulescu goto out; 3587e273a8eSIoana Ciocoi Radulescu 359be9df4afSLorenzo Bianconi offset = dpaa2_fd_get_offset(fd) - XDP_PACKET_HEADROOM; 360be9df4afSLorenzo Bianconi xdp_init_buff(&xdp, DPAA2_ETH_RX_BUF_RAW_SIZE - offset, &ch->xdp_rxq); 361be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, vaddr + offset, XDP_PACKET_HEADROOM, 362be9df4afSLorenzo Bianconi dpaa2_fd_get_len(fd), false); 3634a9b052aSJesper Dangaard Brouer 3647e273a8eSIoana Ciocoi Radulescu xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 3657e273a8eSIoana Ciocoi Radulescu 3667b1eea1aSIoana Ciocoi Radulescu /* xdp.data pointer may have changed */ 3677b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_offset(fd, xdp.data - vaddr); 3687b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 3697b1eea1aSIoana Ciocoi Radulescu 3707e273a8eSIoana Ciocoi Radulescu switch (xdp_act) { 3717e273a8eSIoana Ciocoi Radulescu case XDP_PASS: 3727e273a8eSIoana Ciocoi Radulescu break; 37399e43521SIoana Ciocoi Radulescu case XDP_TX: 3745d8dccf8SIoana Ciornei dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); 37599e43521SIoana Ciocoi Radulescu break; 3767e273a8eSIoana Ciocoi Radulescu default: 377c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act); 378df561f66SGustavo A. R. Silva fallthrough; 3797e273a8eSIoana Ciocoi Radulescu case XDP_ABORTED: 3807e273a8eSIoana Ciocoi Radulescu trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 381df561f66SGustavo A. R. Silva fallthrough; 3827e273a8eSIoana Ciocoi Radulescu case XDP_DROP: 38328d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, addr); 384a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_drop++; 3857e273a8eSIoana Ciocoi Radulescu break; 386d678be1dSIoana Radulescu case XDP_REDIRECT: 387d678be1dSIoana Radulescu dma_unmap_page(priv->net_dev->dev.parent, addr, 388efa6a7d0SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 389d678be1dSIoana Radulescu ch->buf_count--; 3904a9b052aSJesper Dangaard Brouer 3914a9b052aSJesper Dangaard Brouer /* Allow redirect use of full headroom */ 392d678be1dSIoana Radulescu xdp.data_hard_start = vaddr; 3934a9b052aSJesper Dangaard Brouer xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE; 3944a9b052aSJesper Dangaard Brouer 395d678be1dSIoana Radulescu err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 396e12be913SIoana Ciornei if (unlikely(err)) { 397e12be913SIoana Ciornei addr = dma_map_page(priv->net_dev->dev.parent, 398e12be913SIoana Ciornei virt_to_page(vaddr), 0, 399e12be913SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 400e12be913SIoana Ciornei if (unlikely(dma_mapping_error(priv->net_dev->dev.parent, addr))) { 401e12be913SIoana Ciornei free_pages((unsigned long)vaddr, 0); 402e12be913SIoana Ciornei } else { 403e12be913SIoana Ciornei ch->buf_count++; 40428d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, addr); 405e12be913SIoana Ciornei } 406d678be1dSIoana Radulescu ch->stats.xdp_drop++; 407e12be913SIoana Ciornei } else { 408d678be1dSIoana Radulescu ch->stats.xdp_redirect++; 409e12be913SIoana Ciornei } 410d678be1dSIoana Radulescu break; 4117e273a8eSIoana Ciocoi Radulescu } 4127e273a8eSIoana Ciocoi Radulescu 413d678be1dSIoana Radulescu ch->xdp.res |= xdp_act; 4147e273a8eSIoana Ciocoi Radulescu out: 4157e273a8eSIoana Ciocoi Radulescu return xdp_act; 4167e273a8eSIoana Ciocoi Radulescu } 4177e273a8eSIoana Ciocoi Radulescu 41850f82699SIoana Ciornei static struct sk_buff *dpaa2_eth_copybreak(struct dpaa2_eth_channel *ch, 41950f82699SIoana Ciornei const struct dpaa2_fd *fd, 42050f82699SIoana Ciornei void *fd_vaddr) 42150f82699SIoana Ciornei { 42250f82699SIoana Ciornei u16 fd_offset = dpaa2_fd_get_offset(fd); 4238ed3cefcSIoana Ciornei struct dpaa2_eth_priv *priv = ch->priv; 42450f82699SIoana Ciornei u32 fd_length = dpaa2_fd_get_len(fd); 42550f82699SIoana Ciornei struct sk_buff *skb = NULL; 42650f82699SIoana Ciornei unsigned int skb_len; 42750f82699SIoana Ciornei 4288ed3cefcSIoana Ciornei if (fd_length > priv->rx_copybreak) 42950f82699SIoana Ciornei return NULL; 43050f82699SIoana Ciornei 43150f82699SIoana Ciornei skb_len = fd_length + dpaa2_eth_needed_headroom(NULL); 43250f82699SIoana Ciornei 43350f82699SIoana Ciornei skb = napi_alloc_skb(&ch->napi, skb_len); 43450f82699SIoana Ciornei if (!skb) 43550f82699SIoana Ciornei return NULL; 43650f82699SIoana Ciornei 43750f82699SIoana Ciornei skb_reserve(skb, dpaa2_eth_needed_headroom(NULL)); 43850f82699SIoana Ciornei skb_put(skb, fd_length); 43950f82699SIoana Ciornei 44050f82699SIoana Ciornei memcpy(skb->data, fd_vaddr + fd_offset, fd_length); 44150f82699SIoana Ciornei 4428ed3cefcSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, dpaa2_fd_get_addr(fd)); 44350f82699SIoana Ciornei 44450f82699SIoana Ciornei return skb; 44550f82699SIoana Ciornei } 44650f82699SIoana Ciornei 44734ff6846SIoana Radulescu /* Main Rx frame processing routine */ 44834ff6846SIoana Radulescu static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 44934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 45034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 451dbcdf728SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 45234ff6846SIoana Radulescu { 45334ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 45434ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 45534ff6846SIoana Radulescu void *vaddr; 45634ff6846SIoana Radulescu struct sk_buff *skb; 45734ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 45834ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 45934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 46034ff6846SIoana Radulescu struct dpaa2_fas *fas; 46134ff6846SIoana Radulescu void *buf_data; 46234ff6846SIoana Radulescu u32 status = 0; 4637e273a8eSIoana Ciocoi Radulescu u32 xdp_act; 46434ff6846SIoana Radulescu 46534ff6846SIoana Radulescu /* Tracing point */ 46634ff6846SIoana Radulescu trace_dpaa2_rx_fd(priv->net_dev, fd); 46734ff6846SIoana Radulescu 46834ff6846SIoana Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 469efa6a7d0SIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 47018c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 47134ff6846SIoana Radulescu 47234ff6846SIoana Radulescu fas = dpaa2_get_fas(vaddr, false); 47334ff6846SIoana Radulescu prefetch(fas); 47434ff6846SIoana Radulescu buf_data = vaddr + dpaa2_fd_get_offset(fd); 47534ff6846SIoana Radulescu prefetch(buf_data); 47634ff6846SIoana Radulescu 47734ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 47834ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 47934ff6846SIoana Radulescu 48034ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 4815d8dccf8SIoana Ciornei xdp_act = dpaa2_eth_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 4827e273a8eSIoana Ciocoi Radulescu if (xdp_act != XDP_PASS) { 4837e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_packets++; 4847e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 4857e273a8eSIoana Ciocoi Radulescu return; 4867e273a8eSIoana Ciocoi Radulescu } 4877e273a8eSIoana Ciocoi Radulescu 48850f82699SIoana Ciornei skb = dpaa2_eth_copybreak(ch, fd, vaddr); 48950f82699SIoana Ciornei if (!skb) { 490efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 49118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 4925d8dccf8SIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 49350f82699SIoana Ciornei } 49434ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 4957e273a8eSIoana Ciocoi Radulescu WARN_ON(priv->xdp_prog); 4967e273a8eSIoana Ciocoi Radulescu 497efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 49818c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 4995d8dccf8SIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 50027c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 50134ff6846SIoana Radulescu percpu_extras->rx_sg_frames++; 50234ff6846SIoana Radulescu percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 50334ff6846SIoana Radulescu } else { 50434ff6846SIoana Radulescu /* We don't support any other format */ 50534ff6846SIoana Radulescu goto err_frame_format; 50634ff6846SIoana Radulescu } 50734ff6846SIoana Radulescu 50834ff6846SIoana Radulescu if (unlikely(!skb)) 50934ff6846SIoana Radulescu goto err_build_skb; 51034ff6846SIoana Radulescu 51134ff6846SIoana Radulescu prefetch(skb->data); 51234ff6846SIoana Radulescu 51334ff6846SIoana Radulescu /* Get the timestamp value */ 51434ff6846SIoana Radulescu if (priv->rx_tstamp) { 51534ff6846SIoana Radulescu struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 51634ff6846SIoana Radulescu __le64 *ts = dpaa2_get_ts(vaddr, false); 51734ff6846SIoana Radulescu u64 ns; 51834ff6846SIoana Radulescu 51934ff6846SIoana Radulescu memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 52034ff6846SIoana Radulescu 52134ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 52234ff6846SIoana Radulescu shhwtstamps->hwtstamp = ns_to_ktime(ns); 52334ff6846SIoana Radulescu } 52434ff6846SIoana Radulescu 52534ff6846SIoana Radulescu /* Check if we need to validate the L4 csum */ 52634ff6846SIoana Radulescu if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 52734ff6846SIoana Radulescu status = le32_to_cpu(fas->status); 5285d8dccf8SIoana Ciornei dpaa2_eth_validate_rx_csum(priv, status, skb); 52934ff6846SIoana Radulescu } 53034ff6846SIoana Radulescu 53134ff6846SIoana Radulescu skb->protocol = eth_type_trans(skb, priv->net_dev); 532dbcdf728SIoana Ciocoi Radulescu skb_record_rx_queue(skb, fq->flowid); 53334ff6846SIoana Radulescu 53434ff6846SIoana Radulescu percpu_stats->rx_packets++; 53534ff6846SIoana Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 536fc398becSIoana Ciornei ch->stats.bytes_per_cdan += dpaa2_fd_get_len(fd); 53734ff6846SIoana Radulescu 5380a25d92cSIoana Ciornei list_add_tail(&skb->list, ch->rx_list); 53934ff6846SIoana Radulescu 54034ff6846SIoana Radulescu return; 54134ff6846SIoana Radulescu 54234ff6846SIoana Radulescu err_build_skb: 5435d8dccf8SIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 54434ff6846SIoana Radulescu err_frame_format: 54534ff6846SIoana Radulescu percpu_stats->rx_dropped++; 54634ff6846SIoana Radulescu } 54734ff6846SIoana Radulescu 548061d631fSIoana Ciornei /* Processing of Rx frames received on the error FQ 549061d631fSIoana Ciornei * We check and print the error bits and then free the frame 550061d631fSIoana Ciornei */ 551061d631fSIoana Ciornei static void dpaa2_eth_rx_err(struct dpaa2_eth_priv *priv, 552061d631fSIoana Ciornei struct dpaa2_eth_channel *ch, 553061d631fSIoana Ciornei const struct dpaa2_fd *fd, 554061d631fSIoana Ciornei struct dpaa2_eth_fq *fq __always_unused) 555061d631fSIoana Ciornei { 556061d631fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 557061d631fSIoana Ciornei dma_addr_t addr = dpaa2_fd_get_addr(fd); 558061d631fSIoana Ciornei u8 fd_format = dpaa2_fd_get_format(fd); 559061d631fSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 560061d631fSIoana Ciornei struct dpaa2_eth_trap_item *trap_item; 561061d631fSIoana Ciornei struct dpaa2_fapr *fapr; 562061d631fSIoana Ciornei struct sk_buff *skb; 563061d631fSIoana Ciornei void *buf_data; 564061d631fSIoana Ciornei void *vaddr; 565061d631fSIoana Ciornei 566061d631fSIoana Ciornei vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 567061d631fSIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 568061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 569061d631fSIoana Ciornei 570061d631fSIoana Ciornei buf_data = vaddr + dpaa2_fd_get_offset(fd); 571061d631fSIoana Ciornei 572061d631fSIoana Ciornei if (fd_format == dpaa2_fd_single) { 573061d631fSIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 574061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 575061d631fSIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 576061d631fSIoana Ciornei } else if (fd_format == dpaa2_fd_sg) { 577061d631fSIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 578061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 579061d631fSIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 580061d631fSIoana Ciornei free_pages((unsigned long)vaddr, 0); 581061d631fSIoana Ciornei } else { 582061d631fSIoana Ciornei /* We don't support any other format */ 583061d631fSIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 584061d631fSIoana Ciornei goto err_frame_format; 585061d631fSIoana Ciornei } 586061d631fSIoana Ciornei 587061d631fSIoana Ciornei fapr = dpaa2_get_fapr(vaddr, false); 588061d631fSIoana Ciornei trap_item = dpaa2_eth_dl_get_trap(priv, fapr); 589061d631fSIoana Ciornei if (trap_item) 590061d631fSIoana Ciornei devlink_trap_report(priv->devlink, skb, trap_item->trap_ctx, 591061d631fSIoana Ciornei &priv->devlink_port, NULL); 592061d631fSIoana Ciornei consume_skb(skb); 593061d631fSIoana Ciornei 594061d631fSIoana Ciornei err_frame_format: 595061d631fSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 596061d631fSIoana Ciornei percpu_stats->rx_errors++; 597061d631fSIoana Ciornei ch->buf_count--; 598061d631fSIoana Ciornei } 599061d631fSIoana Ciornei 60034ff6846SIoana Radulescu /* Consume all frames pull-dequeued into the store. This is the simplest way to 60134ff6846SIoana Radulescu * make sure we don't accidentally issue another volatile dequeue which would 60234ff6846SIoana Radulescu * overwrite (leak) frames already in the store. 60334ff6846SIoana Radulescu * 60434ff6846SIoana Radulescu * Observance of NAPI budget is not our concern, leaving that to the caller. 60534ff6846SIoana Radulescu */ 6065d8dccf8SIoana Ciornei static int dpaa2_eth_consume_frames(struct dpaa2_eth_channel *ch, 607569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq **src) 60834ff6846SIoana Radulescu { 60934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = ch->priv; 61068049a5fSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq = NULL; 61134ff6846SIoana Radulescu struct dpaa2_dq *dq; 61234ff6846SIoana Radulescu const struct dpaa2_fd *fd; 613ef17bd7cSIoana Radulescu int cleaned = 0, retries = 0; 61434ff6846SIoana Radulescu int is_last; 61534ff6846SIoana Radulescu 61634ff6846SIoana Radulescu do { 61734ff6846SIoana Radulescu dq = dpaa2_io_store_next(ch->store, &is_last); 61834ff6846SIoana Radulescu if (unlikely(!dq)) { 61934ff6846SIoana Radulescu /* If we're here, we *must* have placed a 62034ff6846SIoana Radulescu * volatile dequeue comnmand, so keep reading through 62134ff6846SIoana Radulescu * the store until we get some sort of valid response 62234ff6846SIoana Radulescu * token (either a valid frame or an "empty dequeue") 62334ff6846SIoana Radulescu */ 624ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) { 625ef17bd7cSIoana Radulescu netdev_err_once(priv->net_dev, 626ef17bd7cSIoana Radulescu "Unable to read a valid dequeue response\n"); 627ef17bd7cSIoana Radulescu return -ETIMEDOUT; 628ef17bd7cSIoana Radulescu } 62934ff6846SIoana Radulescu continue; 63034ff6846SIoana Radulescu } 63134ff6846SIoana Radulescu 63234ff6846SIoana Radulescu fd = dpaa2_dq_fd(dq); 63334ff6846SIoana Radulescu fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 63434ff6846SIoana Radulescu 635dbcdf728SIoana Ciocoi Radulescu fq->consume(priv, ch, fd, fq); 63634ff6846SIoana Radulescu cleaned++; 637ef17bd7cSIoana Radulescu retries = 0; 63834ff6846SIoana Radulescu } while (!is_last); 63934ff6846SIoana Radulescu 64068049a5fSIoana Ciocoi Radulescu if (!cleaned) 64168049a5fSIoana Ciocoi Radulescu return 0; 64268049a5fSIoana Ciocoi Radulescu 64368049a5fSIoana Ciocoi Radulescu fq->stats.frames += cleaned; 644460fd830SIoana Ciornei ch->stats.frames += cleaned; 645fc398becSIoana Ciornei ch->stats.frames_per_cdan += cleaned; 64668049a5fSIoana Ciocoi Radulescu 64768049a5fSIoana Ciocoi Radulescu /* A dequeue operation only pulls frames from a single queue 648569dac6aSIoana Ciocoi Radulescu * into the store. Return the frame queue as an out param. 64968049a5fSIoana Ciocoi Radulescu */ 650569dac6aSIoana Ciocoi Radulescu if (src) 651569dac6aSIoana Ciocoi Radulescu *src = fq; 65268049a5fSIoana Ciocoi Radulescu 65334ff6846SIoana Radulescu return cleaned; 65434ff6846SIoana Radulescu } 65534ff6846SIoana Radulescu 656c5521189SYangbo Lu static int dpaa2_eth_ptp_parse(struct sk_buff *skb, 657c5521189SYangbo Lu u8 *msgtype, u8 *twostep, u8 *udp, 658c5521189SYangbo Lu u16 *correction_offset, 659c5521189SYangbo Lu u16 *origintimestamp_offset) 66034ff6846SIoana Radulescu { 661c5521189SYangbo Lu unsigned int ptp_class; 662c5521189SYangbo Lu struct ptp_header *hdr; 663c5521189SYangbo Lu unsigned int type; 664c5521189SYangbo Lu u8 *base; 665c5521189SYangbo Lu 666c5521189SYangbo Lu ptp_class = ptp_classify_raw(skb); 667c5521189SYangbo Lu if (ptp_class == PTP_CLASS_NONE) 668c5521189SYangbo Lu return -EINVAL; 669c5521189SYangbo Lu 670c5521189SYangbo Lu hdr = ptp_parse_header(skb, ptp_class); 671c5521189SYangbo Lu if (!hdr) 672c5521189SYangbo Lu return -EINVAL; 673c5521189SYangbo Lu 674c5521189SYangbo Lu *msgtype = ptp_get_msgtype(hdr, ptp_class); 675c5521189SYangbo Lu *twostep = hdr->flag_field[0] & 0x2; 676c5521189SYangbo Lu 677c5521189SYangbo Lu type = ptp_class & PTP_CLASS_PMASK; 678c5521189SYangbo Lu if (type == PTP_CLASS_IPV4 || 679c5521189SYangbo Lu type == PTP_CLASS_IPV6) 680c5521189SYangbo Lu *udp = 1; 681c5521189SYangbo Lu else 682c5521189SYangbo Lu *udp = 0; 683c5521189SYangbo Lu 684c5521189SYangbo Lu base = skb_mac_header(skb); 685c5521189SYangbo Lu *correction_offset = (u8 *)&hdr->correction - base; 686c5521189SYangbo Lu *origintimestamp_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 687c5521189SYangbo Lu 688c5521189SYangbo Lu return 0; 689c5521189SYangbo Lu } 690c5521189SYangbo Lu 691c5521189SYangbo Lu /* Configure the egress frame annotation for timestamp update */ 692c5521189SYangbo Lu static void dpaa2_eth_enable_tx_tstamp(struct dpaa2_eth_priv *priv, 693c5521189SYangbo Lu struct dpaa2_fd *fd, 694c5521189SYangbo Lu void *buf_start, 695c5521189SYangbo Lu struct sk_buff *skb) 696c5521189SYangbo Lu { 697c5521189SYangbo Lu struct ptp_tstamp origin_timestamp; 698c5521189SYangbo Lu struct dpni_single_step_cfg cfg; 699c5521189SYangbo Lu u8 msgtype, twostep, udp; 70034ff6846SIoana Radulescu struct dpaa2_faead *faead; 701c5521189SYangbo Lu struct dpaa2_fas *fas; 702c5521189SYangbo Lu struct timespec64 ts; 703c5521189SYangbo Lu u16 offset1, offset2; 70434ff6846SIoana Radulescu u32 ctrl, frc; 705c5521189SYangbo Lu __le64 *ns; 706c5521189SYangbo Lu u8 *data; 70734ff6846SIoana Radulescu 70834ff6846SIoana Radulescu /* Mark the egress frame annotation area as valid */ 70934ff6846SIoana Radulescu frc = dpaa2_fd_get_frc(fd); 71034ff6846SIoana Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 71134ff6846SIoana Radulescu 71234ff6846SIoana Radulescu /* Set hardware annotation size */ 71334ff6846SIoana Radulescu ctrl = dpaa2_fd_get_ctrl(fd); 71434ff6846SIoana Radulescu dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 71534ff6846SIoana Radulescu 71634ff6846SIoana Radulescu /* enable UPD (update prepanded data) bit in FAEAD field of 71734ff6846SIoana Radulescu * hardware frame annotation area 71834ff6846SIoana Radulescu */ 71934ff6846SIoana Radulescu ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 72034ff6846SIoana Radulescu faead = dpaa2_get_faead(buf_start, true); 72134ff6846SIoana Radulescu faead->ctrl = cpu_to_le32(ctrl); 722c5521189SYangbo Lu 723c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 724c5521189SYangbo Lu if (dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 725c5521189SYangbo Lu &offset1, &offset2) || 7266b6817c5SChristian Eggers msgtype != PTP_MSGTYPE_SYNC || twostep) { 727c5521189SYangbo Lu WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 728c5521189SYangbo Lu return; 729c5521189SYangbo Lu } 730c5521189SYangbo Lu 731c5521189SYangbo Lu /* Mark the frame annotation status as valid */ 732c5521189SYangbo Lu frc = dpaa2_fd_get_frc(fd); 733c5521189SYangbo Lu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FASV); 734c5521189SYangbo Lu 735c5521189SYangbo Lu /* Mark the PTP flag for one step timestamping */ 736c5521189SYangbo Lu fas = dpaa2_get_fas(buf_start, true); 737c5521189SYangbo Lu fas->status = cpu_to_le32(DPAA2_FAS_PTP); 738c5521189SYangbo Lu 739c5521189SYangbo Lu dpaa2_ptp->caps.gettime64(&dpaa2_ptp->caps, &ts); 740c5521189SYangbo Lu ns = dpaa2_get_ts(buf_start, true); 741c5521189SYangbo Lu *ns = cpu_to_le64(timespec64_to_ns(&ts) / 742c5521189SYangbo Lu DPAA2_PTP_CLK_PERIOD_NS); 743c5521189SYangbo Lu 744c5521189SYangbo Lu /* Update current time to PTP message originTimestamp field */ 745c5521189SYangbo Lu ns_to_ptp_tstamp(&origin_timestamp, le64_to_cpup(ns)); 746c5521189SYangbo Lu data = skb_mac_header(skb); 747c5521189SYangbo Lu *(__be16 *)(data + offset2) = htons(origin_timestamp.sec_msb); 748c5521189SYangbo Lu *(__be32 *)(data + offset2 + 2) = 749c5521189SYangbo Lu htonl(origin_timestamp.sec_lsb); 750c5521189SYangbo Lu *(__be32 *)(data + offset2 + 6) = htonl(origin_timestamp.nsec); 751c5521189SYangbo Lu 752c5521189SYangbo Lu cfg.en = 1; 753c5521189SYangbo Lu cfg.ch_update = udp; 754c5521189SYangbo Lu cfg.offset = offset1; 755c5521189SYangbo Lu cfg.peer_delay = 0; 756c5521189SYangbo Lu 757c5521189SYangbo Lu if (dpni_set_single_step_cfg(priv->mc_io, 0, priv->mc_token, 758c5521189SYangbo Lu &cfg)) 759c5521189SYangbo Lu WARN_ONCE(1, "Failed to set single step register"); 760c5521189SYangbo Lu } 76134ff6846SIoana Radulescu } 76234ff6846SIoana Radulescu 763ae3b0817SIoana Ciornei static void *dpaa2_eth_sgt_get(struct dpaa2_eth_priv *priv) 764ae3b0817SIoana Ciornei { 765ae3b0817SIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 766ae3b0817SIoana Ciornei void *sgt_buf = NULL; 767ae3b0817SIoana Ciornei int sgt_buf_size; 768ae3b0817SIoana Ciornei 769ae3b0817SIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 770*a4218aefSIoana Ciornei sgt_buf_size = priv->tx_data_offset + 771*a4218aefSIoana Ciornei DPAA2_ETH_SG_ENTRIES_MAX * sizeof(struct dpaa2_sg_entry); 772ae3b0817SIoana Ciornei 773ae3b0817SIoana Ciornei if (sgt_cache->count == 0) 774ae3b0817SIoana Ciornei sgt_buf = napi_alloc_frag_align(sgt_buf_size, DPAA2_ETH_TX_BUF_ALIGN); 775ae3b0817SIoana Ciornei else 776ae3b0817SIoana Ciornei sgt_buf = sgt_cache->buf[--sgt_cache->count]; 777ae3b0817SIoana Ciornei if (!sgt_buf) 778ae3b0817SIoana Ciornei return NULL; 779ae3b0817SIoana Ciornei 780ae3b0817SIoana Ciornei memset(sgt_buf, 0, sgt_buf_size); 781ae3b0817SIoana Ciornei 782ae3b0817SIoana Ciornei return sgt_buf; 783ae3b0817SIoana Ciornei } 784ae3b0817SIoana Ciornei 785ae3b0817SIoana Ciornei static void dpaa2_eth_sgt_recycle(struct dpaa2_eth_priv *priv, void *sgt_buf) 786ae3b0817SIoana Ciornei { 787ae3b0817SIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 788ae3b0817SIoana Ciornei 789ae3b0817SIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 790ae3b0817SIoana Ciornei if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 791ae3b0817SIoana Ciornei skb_free_frag(sgt_buf); 792ae3b0817SIoana Ciornei else 793ae3b0817SIoana Ciornei sgt_cache->buf[sgt_cache->count++] = sgt_buf; 794ae3b0817SIoana Ciornei } 795ae3b0817SIoana Ciornei 79634ff6846SIoana Radulescu /* Create a frame descriptor based on a fragmented skb */ 7975d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd(struct dpaa2_eth_priv *priv, 79834ff6846SIoana Radulescu struct sk_buff *skb, 79964a965deSYangbo Lu struct dpaa2_fd *fd, 80064a965deSYangbo Lu void **swa_addr) 80134ff6846SIoana Radulescu { 80234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 80334ff6846SIoana Radulescu void *sgt_buf = NULL; 80434ff6846SIoana Radulescu dma_addr_t addr; 80534ff6846SIoana Radulescu int nr_frags = skb_shinfo(skb)->nr_frags; 80634ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 80734ff6846SIoana Radulescu int i, err; 80834ff6846SIoana Radulescu int sgt_buf_size; 80934ff6846SIoana Radulescu struct scatterlist *scl, *crt_scl; 81034ff6846SIoana Radulescu int num_sg; 81134ff6846SIoana Radulescu int num_dma_bufs; 81234ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 81334ff6846SIoana Radulescu 81434ff6846SIoana Radulescu /* Create and map scatterlist. 81534ff6846SIoana Radulescu * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 81634ff6846SIoana Radulescu * to go beyond nr_frags+1. 81734ff6846SIoana Radulescu * Note: We don't support chained scatterlists 81834ff6846SIoana Radulescu */ 81934ff6846SIoana Radulescu if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 82034ff6846SIoana Radulescu return -EINVAL; 82134ff6846SIoana Radulescu 822d4ceb8deSJulia Lawall scl = kmalloc_array(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 82334ff6846SIoana Radulescu if (unlikely(!scl)) 82434ff6846SIoana Radulescu return -ENOMEM; 82534ff6846SIoana Radulescu 82634ff6846SIoana Radulescu sg_init_table(scl, nr_frags + 1); 82734ff6846SIoana Radulescu num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 82837fbbddaSIoana Ciornei if (unlikely(num_sg < 0)) { 82937fbbddaSIoana Ciornei err = -ENOMEM; 83037fbbddaSIoana Ciornei goto dma_map_sg_failed; 83137fbbddaSIoana Ciornei } 83234ff6846SIoana Radulescu num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 83334ff6846SIoana Radulescu if (unlikely(!num_dma_bufs)) { 83434ff6846SIoana Radulescu err = -ENOMEM; 83534ff6846SIoana Radulescu goto dma_map_sg_failed; 83634ff6846SIoana Radulescu } 83734ff6846SIoana Radulescu 83834ff6846SIoana Radulescu /* Prepare the HW SGT structure */ 83934ff6846SIoana Radulescu sgt_buf_size = priv->tx_data_offset + 84034ff6846SIoana Radulescu sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 841*a4218aefSIoana Ciornei sgt_buf = dpaa2_eth_sgt_get(priv); 84234ff6846SIoana Radulescu if (unlikely(!sgt_buf)) { 84334ff6846SIoana Radulescu err = -ENOMEM; 84434ff6846SIoana Radulescu goto sgt_buf_alloc_failed; 84534ff6846SIoana Radulescu } 84634ff6846SIoana Radulescu 84734ff6846SIoana Radulescu sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 84834ff6846SIoana Radulescu 84934ff6846SIoana Radulescu /* Fill in the HW SGT structure. 85034ff6846SIoana Radulescu * 85134ff6846SIoana Radulescu * sgt_buf is zeroed out, so the following fields are implicit 85234ff6846SIoana Radulescu * in all sgt entries: 85334ff6846SIoana Radulescu * - offset is 0 85434ff6846SIoana Radulescu * - format is 'dpaa2_sg_single' 85534ff6846SIoana Radulescu */ 85634ff6846SIoana Radulescu for_each_sg(scl, crt_scl, num_dma_bufs, i) { 85734ff6846SIoana Radulescu dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 85834ff6846SIoana Radulescu dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 85934ff6846SIoana Radulescu } 86034ff6846SIoana Radulescu dpaa2_sg_set_final(&sgt[i - 1], true); 86134ff6846SIoana Radulescu 86234ff6846SIoana Radulescu /* Store the skb backpointer in the SGT buffer. 86334ff6846SIoana Radulescu * Fit the scatterlist and the number of buffers alongside the 86434ff6846SIoana Radulescu * skb backpointer in the software annotation area. We'll need 86534ff6846SIoana Radulescu * all of them on Tx Conf. 86634ff6846SIoana Radulescu */ 86764a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 86834ff6846SIoana Radulescu swa = (struct dpaa2_eth_swa *)sgt_buf; 869e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SG; 870e3fdf6baSIoana Radulescu swa->sg.skb = skb; 871e3fdf6baSIoana Radulescu swa->sg.scl = scl; 872e3fdf6baSIoana Radulescu swa->sg.num_sg = num_sg; 873e3fdf6baSIoana Radulescu swa->sg.sgt_size = sgt_buf_size; 87434ff6846SIoana Radulescu 87534ff6846SIoana Radulescu /* Separately map the SGT buffer */ 87634ff6846SIoana Radulescu addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 87734ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) { 87834ff6846SIoana Radulescu err = -ENOMEM; 87934ff6846SIoana Radulescu goto dma_map_single_failed; 88034ff6846SIoana Radulescu } 88134ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, priv->tx_data_offset); 88234ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_sg); 88334ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 88434ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 885b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 88634ff6846SIoana Radulescu 88734ff6846SIoana Radulescu return 0; 88834ff6846SIoana Radulescu 88934ff6846SIoana Radulescu dma_map_single_failed: 890*a4218aefSIoana Ciornei dpaa2_eth_sgt_recycle(priv, sgt_buf); 89134ff6846SIoana Radulescu sgt_buf_alloc_failed: 89234ff6846SIoana Radulescu dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 89334ff6846SIoana Radulescu dma_map_sg_failed: 89434ff6846SIoana Radulescu kfree(scl); 89534ff6846SIoana Radulescu return err; 89634ff6846SIoana Radulescu } 89734ff6846SIoana Radulescu 898d70446eeSIoana Ciornei /* Create a SG frame descriptor based on a linear skb. 899d70446eeSIoana Ciornei * 900d70446eeSIoana Ciornei * This function is used on the Tx path when the skb headroom is not large 901d70446eeSIoana Ciornei * enough for the HW requirements, thus instead of realloc-ing the skb we 902d70446eeSIoana Ciornei * create a SG frame descriptor with only one entry. 903d70446eeSIoana Ciornei */ 9045d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd_single_buf(struct dpaa2_eth_priv *priv, 905d70446eeSIoana Ciornei struct sk_buff *skb, 90664a965deSYangbo Lu struct dpaa2_fd *fd, 90764a965deSYangbo Lu void **swa_addr) 908d70446eeSIoana Ciornei { 909d70446eeSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 910d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 911d70446eeSIoana Ciornei struct dpaa2_eth_swa *swa; 912d70446eeSIoana Ciornei dma_addr_t addr, sgt_addr; 913d70446eeSIoana Ciornei void *sgt_buf = NULL; 914d70446eeSIoana Ciornei int sgt_buf_size; 915d70446eeSIoana Ciornei int err; 916d70446eeSIoana Ciornei 917d70446eeSIoana Ciornei /* Prepare the HW SGT structure */ 918d70446eeSIoana Ciornei sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry); 919ae3b0817SIoana Ciornei sgt_buf = dpaa2_eth_sgt_get(priv); 920d70446eeSIoana Ciornei if (unlikely(!sgt_buf)) 921d70446eeSIoana Ciornei return -ENOMEM; 922d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 923d70446eeSIoana Ciornei 924d70446eeSIoana Ciornei addr = dma_map_single(dev, skb->data, skb->len, DMA_BIDIRECTIONAL); 925d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) { 926d70446eeSIoana Ciornei err = -ENOMEM; 927d70446eeSIoana Ciornei goto data_map_failed; 928d70446eeSIoana Ciornei } 929d70446eeSIoana Ciornei 930d70446eeSIoana Ciornei /* Fill in the HW SGT structure */ 931d70446eeSIoana Ciornei dpaa2_sg_set_addr(sgt, addr); 932d70446eeSIoana Ciornei dpaa2_sg_set_len(sgt, skb->len); 933d70446eeSIoana Ciornei dpaa2_sg_set_final(sgt, true); 934d70446eeSIoana Ciornei 935d70446eeSIoana Ciornei /* Store the skb backpointer in the SGT buffer */ 93664a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 937d70446eeSIoana Ciornei swa = (struct dpaa2_eth_swa *)sgt_buf; 938d70446eeSIoana Ciornei swa->type = DPAA2_ETH_SWA_SINGLE; 939d70446eeSIoana Ciornei swa->single.skb = skb; 94054a57d1cSIoana Ciornei swa->single.sgt_size = sgt_buf_size; 941d70446eeSIoana Ciornei 942d70446eeSIoana Ciornei /* Separately map the SGT buffer */ 943d70446eeSIoana Ciornei sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 944d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, sgt_addr))) { 945d70446eeSIoana Ciornei err = -ENOMEM; 946d70446eeSIoana Ciornei goto sgt_map_failed; 947d70446eeSIoana Ciornei } 948d70446eeSIoana Ciornei 949d70446eeSIoana Ciornei dpaa2_fd_set_offset(fd, priv->tx_data_offset); 950d70446eeSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_sg); 951d70446eeSIoana Ciornei dpaa2_fd_set_addr(fd, sgt_addr); 952d70446eeSIoana Ciornei dpaa2_fd_set_len(fd, skb->len); 953d70446eeSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 954d70446eeSIoana Ciornei 955d70446eeSIoana Ciornei return 0; 956d70446eeSIoana Ciornei 957d70446eeSIoana Ciornei sgt_map_failed: 958d70446eeSIoana Ciornei dma_unmap_single(dev, addr, skb->len, DMA_BIDIRECTIONAL); 959d70446eeSIoana Ciornei data_map_failed: 960ae3b0817SIoana Ciornei dpaa2_eth_sgt_recycle(priv, sgt_buf); 961d70446eeSIoana Ciornei 962d70446eeSIoana Ciornei return err; 963d70446eeSIoana Ciornei } 964d70446eeSIoana Ciornei 96534ff6846SIoana Radulescu /* Create a frame descriptor based on a linear skb */ 9665d8dccf8SIoana Ciornei static int dpaa2_eth_build_single_fd(struct dpaa2_eth_priv *priv, 96734ff6846SIoana Radulescu struct sk_buff *skb, 96864a965deSYangbo Lu struct dpaa2_fd *fd, 96964a965deSYangbo Lu void **swa_addr) 97034ff6846SIoana Radulescu { 97134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 97234ff6846SIoana Radulescu u8 *buffer_start, *aligned_start; 973e3fdf6baSIoana Radulescu struct dpaa2_eth_swa *swa; 97434ff6846SIoana Radulescu dma_addr_t addr; 97534ff6846SIoana Radulescu 9761cf773bdSYangbo Lu buffer_start = skb->data - dpaa2_eth_needed_headroom(skb); 97734ff6846SIoana Radulescu 97834ff6846SIoana Radulescu /* If there's enough room to align the FD address, do it. 97934ff6846SIoana Radulescu * It will help hardware optimize accesses. 98034ff6846SIoana Radulescu */ 98134ff6846SIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 98234ff6846SIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 98334ff6846SIoana Radulescu if (aligned_start >= skb->head) 98434ff6846SIoana Radulescu buffer_start = aligned_start; 98534ff6846SIoana Radulescu 98634ff6846SIoana Radulescu /* Store a backpointer to the skb at the beginning of the buffer 98734ff6846SIoana Radulescu * (in the private data area) such that we can release it 98834ff6846SIoana Radulescu * on Tx confirm 98934ff6846SIoana Radulescu */ 99064a965deSYangbo Lu *swa_addr = (void *)buffer_start; 991e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 992e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SINGLE; 993e3fdf6baSIoana Radulescu swa->single.skb = skb; 99434ff6846SIoana Radulescu 99534ff6846SIoana Radulescu addr = dma_map_single(dev, buffer_start, 99634ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 99734ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 99834ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 99934ff6846SIoana Radulescu return -ENOMEM; 100034ff6846SIoana Radulescu 100134ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 100234ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 100334ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 100434ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_single); 1005b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 100634ff6846SIoana Radulescu 100734ff6846SIoana Radulescu return 0; 100834ff6846SIoana Radulescu } 100934ff6846SIoana Radulescu 101034ff6846SIoana Radulescu /* FD freeing routine on the Tx path 101134ff6846SIoana Radulescu * 101234ff6846SIoana Radulescu * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 101334ff6846SIoana Radulescu * back-pointed to is also freed. 101434ff6846SIoana Radulescu * This can be called either from dpaa2_eth_tx_conf() or on the error path of 101534ff6846SIoana Radulescu * dpaa2_eth_tx(). 101634ff6846SIoana Radulescu */ 1017c5521189SYangbo Lu static void dpaa2_eth_free_tx_fd(struct dpaa2_eth_priv *priv, 1018d678be1dSIoana Radulescu struct dpaa2_eth_fq *fq, 10190723a3aeSIoana Ciocoi Radulescu const struct dpaa2_fd *fd, bool in_napi) 102034ff6846SIoana Radulescu { 102134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 1022d70446eeSIoana Ciornei dma_addr_t fd_addr, sg_addr; 1023d678be1dSIoana Radulescu struct sk_buff *skb = NULL; 102434ff6846SIoana Radulescu unsigned char *buffer_start; 102534ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 102634ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 1027d678be1dSIoana Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 1028d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 1029d70446eeSIoana Ciornei 103034ff6846SIoana Radulescu fd_addr = dpaa2_fd_get_addr(fd); 1031e3fdf6baSIoana Radulescu buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 1032e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 103334ff6846SIoana Radulescu 103434ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 1035d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_SINGLE) { 1036e3fdf6baSIoana Radulescu skb = swa->single.skb; 1037d678be1dSIoana Radulescu /* Accessing the skb buffer is safe before dma unmap, 1038d678be1dSIoana Radulescu * because we didn't map the actual skb shell. 103934ff6846SIoana Radulescu */ 104034ff6846SIoana Radulescu dma_unmap_single(dev, fd_addr, 104134ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 104234ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 1043d678be1dSIoana Radulescu } else { 1044d678be1dSIoana Radulescu WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 1045d678be1dSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 1046d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 1047d678be1dSIoana Radulescu } 104834ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 1049d70446eeSIoana Ciornei if (swa->type == DPAA2_ETH_SWA_SG) { 1050e3fdf6baSIoana Radulescu skb = swa->sg.skb; 105134ff6846SIoana Radulescu 105234ff6846SIoana Radulescu /* Unmap the scatterlist */ 1053e3fdf6baSIoana Radulescu dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 1054e3fdf6baSIoana Radulescu DMA_BIDIRECTIONAL); 1055e3fdf6baSIoana Radulescu kfree(swa->sg.scl); 105634ff6846SIoana Radulescu 105734ff6846SIoana Radulescu /* Unmap the SGT buffer */ 1058e3fdf6baSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 105934ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 106034ff6846SIoana Radulescu } else { 1061d70446eeSIoana Ciornei skb = swa->single.skb; 1062d70446eeSIoana Ciornei 1063d70446eeSIoana Ciornei /* Unmap the SGT Buffer */ 1064d70446eeSIoana Ciornei dma_unmap_single(dev, fd_addr, swa->single.sgt_size, 1065d70446eeSIoana Ciornei DMA_BIDIRECTIONAL); 1066d70446eeSIoana Ciornei 1067d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(buffer_start + 1068d70446eeSIoana Ciornei priv->tx_data_offset); 1069d70446eeSIoana Ciornei sg_addr = dpaa2_sg_get_addr(sgt); 1070d70446eeSIoana Ciornei dma_unmap_single(dev, sg_addr, skb->len, DMA_BIDIRECTIONAL); 1071d70446eeSIoana Ciornei } 1072d70446eeSIoana Ciornei } else { 107334ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "Invalid FD format\n"); 107434ff6846SIoana Radulescu return; 107534ff6846SIoana Radulescu } 107634ff6846SIoana Radulescu 1077d678be1dSIoana Radulescu if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 1078d678be1dSIoana Radulescu fq->dq_frames++; 1079d678be1dSIoana Radulescu fq->dq_bytes += fd_len; 1080d678be1dSIoana Radulescu } 1081d678be1dSIoana Radulescu 1082d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_XDP) { 1083d678be1dSIoana Radulescu xdp_return_frame(swa->xdp.xdpf); 1084d678be1dSIoana Radulescu return; 1085d678be1dSIoana Radulescu } 1086d678be1dSIoana Radulescu 108734ff6846SIoana Radulescu /* Get the timestamp value */ 10881cf773bdSYangbo Lu if (skb->cb[0] == TX_TSTAMP) { 108934ff6846SIoana Radulescu struct skb_shared_hwtstamps shhwtstamps; 1090e3fdf6baSIoana Radulescu __le64 *ts = dpaa2_get_ts(buffer_start, true); 109134ff6846SIoana Radulescu u64 ns; 109234ff6846SIoana Radulescu 109334ff6846SIoana Radulescu memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 109434ff6846SIoana Radulescu 109534ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 109634ff6846SIoana Radulescu shhwtstamps.hwtstamp = ns_to_ktime(ns); 109734ff6846SIoana Radulescu skb_tstamp_tx(skb, &shhwtstamps); 1098c5521189SYangbo Lu } else if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1099c5521189SYangbo Lu mutex_unlock(&priv->onestep_tstamp_lock); 110034ff6846SIoana Radulescu } 110134ff6846SIoana Radulescu 110234ff6846SIoana Radulescu /* Free SGT buffer allocated on tx */ 1103*a4218aefSIoana Ciornei if (fd_format != dpaa2_fd_single) 1104ae3b0817SIoana Ciornei dpaa2_eth_sgt_recycle(priv, buffer_start); 110534ff6846SIoana Radulescu 110634ff6846SIoana Radulescu /* Move on with skb release */ 11070723a3aeSIoana Ciocoi Radulescu napi_consume_skb(skb, in_napi); 110834ff6846SIoana Radulescu } 110934ff6846SIoana Radulescu 1110c5521189SYangbo Lu static netdev_tx_t __dpaa2_eth_tx(struct sk_buff *skb, 1111c5521189SYangbo Lu struct net_device *net_dev) 111234ff6846SIoana Radulescu { 111334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 111434ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 1115035dd64dSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 1116035dd64dSIoana Ciornei unsigned int needed_headroom; 111734ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 1118569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1119035dd64dSIoana Ciornei struct dpaa2_fd fd; 112034ff6846SIoana Radulescu u16 queue_mapping; 1121ab1e6de2SIoana Radulescu u8 prio = 0; 112234ff6846SIoana Radulescu int err, i; 1123035dd64dSIoana Ciornei u32 fd_len; 112464a965deSYangbo Lu void *swa; 112534ff6846SIoana Radulescu 112634ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 112734ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 112834ff6846SIoana Radulescu 11291cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(skb); 113034ff6846SIoana Radulescu 113134ff6846SIoana Radulescu /* We'll be holding a back-reference to the skb until Tx Confirmation; 113234ff6846SIoana Radulescu * we don't want that overwritten by a concurrent Tx with a cloned skb. 113334ff6846SIoana Radulescu */ 113434ff6846SIoana Radulescu skb = skb_unshare(skb, GFP_ATOMIC); 113534ff6846SIoana Radulescu if (unlikely(!skb)) { 113634ff6846SIoana Radulescu /* skb_unshare() has already freed the skb */ 113734ff6846SIoana Radulescu percpu_stats->tx_dropped++; 113834ff6846SIoana Radulescu return NETDEV_TX_OK; 113934ff6846SIoana Radulescu } 114034ff6846SIoana Radulescu 114134ff6846SIoana Radulescu /* Setup the FD fields */ 114234ff6846SIoana Radulescu memset(&fd, 0, sizeof(fd)); 114334ff6846SIoana Radulescu 114434ff6846SIoana Radulescu if (skb_is_nonlinear(skb)) { 114564a965deSYangbo Lu err = dpaa2_eth_build_sg_fd(priv, skb, &fd, &swa); 114634ff6846SIoana Radulescu percpu_extras->tx_sg_frames++; 114734ff6846SIoana Radulescu percpu_extras->tx_sg_bytes += skb->len; 1148d70446eeSIoana Ciornei } else if (skb_headroom(skb) < needed_headroom) { 114964a965deSYangbo Lu err = dpaa2_eth_build_sg_fd_single_buf(priv, skb, &fd, &swa); 1150d70446eeSIoana Ciornei percpu_extras->tx_sg_frames++; 1151d70446eeSIoana Ciornei percpu_extras->tx_sg_bytes += skb->len; 11524c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_frames++; 11534c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_bytes += skb->len; 115434ff6846SIoana Radulescu } else { 115564a965deSYangbo Lu err = dpaa2_eth_build_single_fd(priv, skb, &fd, &swa); 115634ff6846SIoana Radulescu } 115734ff6846SIoana Radulescu 115834ff6846SIoana Radulescu if (unlikely(err)) { 115934ff6846SIoana Radulescu percpu_stats->tx_dropped++; 116034ff6846SIoana Radulescu goto err_build_fd; 116134ff6846SIoana Radulescu } 116234ff6846SIoana Radulescu 1163c5521189SYangbo Lu if (skb->cb[0]) 1164c5521189SYangbo Lu dpaa2_eth_enable_tx_tstamp(priv, &fd, swa, skb); 116564a965deSYangbo Lu 116634ff6846SIoana Radulescu /* Tracing point */ 116734ff6846SIoana Radulescu trace_dpaa2_tx_fd(net_dev, &fd); 116834ff6846SIoana Radulescu 116934ff6846SIoana Radulescu /* TxConf FQ selection relies on queue id from the stack. 117034ff6846SIoana Radulescu * In case of a forwarded frame from another DPNI interface, we choose 117134ff6846SIoana Radulescu * a queue affined to the same core that processed the Rx frame 117234ff6846SIoana Radulescu */ 117334ff6846SIoana Radulescu queue_mapping = skb_get_queue_mapping(skb); 1174ab1e6de2SIoana Radulescu 1175ab1e6de2SIoana Radulescu if (net_dev->num_tc) { 1176ab1e6de2SIoana Radulescu prio = netdev_txq_to_tc(net_dev, queue_mapping); 1177ab1e6de2SIoana Radulescu /* Hardware interprets priority level 0 as being the highest, 1178ab1e6de2SIoana Radulescu * so we need to do a reverse mapping to the netdev tc index 1179ab1e6de2SIoana Radulescu */ 1180ab1e6de2SIoana Radulescu prio = net_dev->num_tc - prio - 1; 1181ab1e6de2SIoana Radulescu /* We have only one FQ array entry for all Tx hardware queues 1182ab1e6de2SIoana Radulescu * with the same flow id (but different priority levels) 1183ab1e6de2SIoana Radulescu */ 1184ab1e6de2SIoana Radulescu queue_mapping %= dpaa2_eth_queue_count(priv); 1185ab1e6de2SIoana Radulescu } 118634ff6846SIoana Radulescu fq = &priv->fq[queue_mapping]; 11878c838f53SIoana Ciornei 11888c838f53SIoana Ciornei fd_len = dpaa2_fd_get_len(&fd); 11898c838f53SIoana Ciornei nq = netdev_get_tx_queue(net_dev, queue_mapping); 11908c838f53SIoana Ciornei netdev_tx_sent_queue(nq, fd_len); 11918c838f53SIoana Ciornei 11928c838f53SIoana Ciornei /* Everything that happens after this enqueues might race with 11938c838f53SIoana Ciornei * the Tx confirmation callback for this frame 11948c838f53SIoana Ciornei */ 119534ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 11966ff80447SIoana Ciornei err = priv->enqueue(priv, fq, &fd, prio, 1, NULL); 119734ff6846SIoana Radulescu if (err != -EBUSY) 119834ff6846SIoana Radulescu break; 119934ff6846SIoana Radulescu } 120034ff6846SIoana Radulescu percpu_extras->tx_portal_busy += i; 120134ff6846SIoana Radulescu if (unlikely(err < 0)) { 120234ff6846SIoana Radulescu percpu_stats->tx_errors++; 120334ff6846SIoana Radulescu /* Clean up everything, including freeing the skb */ 12045d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, &fd, false); 12058c838f53SIoana Ciornei netdev_tx_completed_queue(nq, 1, fd_len); 120634ff6846SIoana Radulescu } else { 120734ff6846SIoana Radulescu percpu_stats->tx_packets++; 1208569dac6aSIoana Ciocoi Radulescu percpu_stats->tx_bytes += fd_len; 120934ff6846SIoana Radulescu } 121034ff6846SIoana Radulescu 121134ff6846SIoana Radulescu return NETDEV_TX_OK; 121234ff6846SIoana Radulescu 121334ff6846SIoana Radulescu err_build_fd: 121434ff6846SIoana Radulescu dev_kfree_skb(skb); 121534ff6846SIoana Radulescu 121634ff6846SIoana Radulescu return NETDEV_TX_OK; 121734ff6846SIoana Radulescu } 121834ff6846SIoana Radulescu 1219c5521189SYangbo Lu static void dpaa2_eth_tx_onestep_tstamp(struct work_struct *work) 1220c5521189SYangbo Lu { 1221c5521189SYangbo Lu struct dpaa2_eth_priv *priv = container_of(work, struct dpaa2_eth_priv, 1222c5521189SYangbo Lu tx_onestep_tstamp); 1223c5521189SYangbo Lu struct sk_buff *skb; 1224c5521189SYangbo Lu 1225c5521189SYangbo Lu while (true) { 1226c5521189SYangbo Lu skb = skb_dequeue(&priv->tx_skbs); 1227c5521189SYangbo Lu if (!skb) 1228c5521189SYangbo Lu return; 1229c5521189SYangbo Lu 1230c5521189SYangbo Lu /* Lock just before TX one-step timestamping packet, 1231c5521189SYangbo Lu * and release the lock in dpaa2_eth_free_tx_fd when 1232c5521189SYangbo Lu * confirm the packet has been sent on hardware, or 1233c5521189SYangbo Lu * when clean up during transmit failure. 1234c5521189SYangbo Lu */ 1235c5521189SYangbo Lu mutex_lock(&priv->onestep_tstamp_lock); 1236c5521189SYangbo Lu __dpaa2_eth_tx(skb, priv->net_dev); 1237c5521189SYangbo Lu } 1238c5521189SYangbo Lu } 1239c5521189SYangbo Lu 1240c5521189SYangbo Lu static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 1241c5521189SYangbo Lu { 1242c5521189SYangbo Lu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1243c5521189SYangbo Lu u8 msgtype, twostep, udp; 1244c5521189SYangbo Lu u16 offset1, offset2; 1245c5521189SYangbo Lu 1246c5521189SYangbo Lu /* Utilize skb->cb[0] for timestamping request per skb */ 1247c5521189SYangbo Lu skb->cb[0] = 0; 1248c5521189SYangbo Lu 1249c5521189SYangbo Lu if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && dpaa2_ptp) { 1250c5521189SYangbo Lu if (priv->tx_tstamp_type == HWTSTAMP_TX_ON) 1251c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1252c5521189SYangbo Lu else if (priv->tx_tstamp_type == HWTSTAMP_TX_ONESTEP_SYNC) 1253c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP_ONESTEP_SYNC; 1254c5521189SYangbo Lu } 1255c5521189SYangbo Lu 1256c5521189SYangbo Lu /* TX for one-step timestamping PTP Sync packet */ 1257c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1258c5521189SYangbo Lu if (!dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 1259c5521189SYangbo Lu &offset1, &offset2)) 12606b6817c5SChristian Eggers if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0) { 1261c5521189SYangbo Lu skb_queue_tail(&priv->tx_skbs, skb); 1262c5521189SYangbo Lu queue_work(priv->dpaa2_ptp_wq, 1263c5521189SYangbo Lu &priv->tx_onestep_tstamp); 1264c5521189SYangbo Lu return NETDEV_TX_OK; 1265c5521189SYangbo Lu } 1266c5521189SYangbo Lu /* Use two-step timestamping if not one-step timestamping 1267c5521189SYangbo Lu * PTP Sync packet 1268c5521189SYangbo Lu */ 1269c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1270c5521189SYangbo Lu } 1271c5521189SYangbo Lu 1272c5521189SYangbo Lu /* TX for other packets */ 1273c5521189SYangbo Lu return __dpaa2_eth_tx(skb, net_dev); 1274c5521189SYangbo Lu } 1275c5521189SYangbo Lu 127634ff6846SIoana Radulescu /* Tx confirmation frame processing routine */ 127734ff6846SIoana Radulescu static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 1278fc398becSIoana Ciornei struct dpaa2_eth_channel *ch, 127934ff6846SIoana Radulescu const struct dpaa2_fd *fd, 1280569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 128134ff6846SIoana Radulescu { 128234ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 128334ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 1284569dac6aSIoana Ciocoi Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 128534ff6846SIoana Radulescu u32 fd_errors; 128634ff6846SIoana Radulescu 128734ff6846SIoana Radulescu /* Tracing point */ 128834ff6846SIoana Radulescu trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 128934ff6846SIoana Radulescu 129034ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 129134ff6846SIoana Radulescu percpu_extras->tx_conf_frames++; 1292569dac6aSIoana Ciocoi Radulescu percpu_extras->tx_conf_bytes += fd_len; 1293fc398becSIoana Ciornei ch->stats.bytes_per_cdan += fd_len; 1294569dac6aSIoana Ciocoi Radulescu 129534ff6846SIoana Radulescu /* Check frame errors in the FD field */ 129634ff6846SIoana Radulescu fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 12975d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, fd, true); 129834ff6846SIoana Radulescu 129934ff6846SIoana Radulescu if (likely(!fd_errors)) 130034ff6846SIoana Radulescu return; 130134ff6846SIoana Radulescu 130234ff6846SIoana Radulescu if (net_ratelimit()) 130334ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 130434ff6846SIoana Radulescu fd_errors); 130534ff6846SIoana Radulescu 130634ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 130734ff6846SIoana Radulescu /* Tx-conf logically pertains to the egress path. */ 130834ff6846SIoana Radulescu percpu_stats->tx_errors++; 130934ff6846SIoana Radulescu } 131034ff6846SIoana Radulescu 131170b32d82SIonut-robert Aron static int dpaa2_eth_set_rx_vlan_filtering(struct dpaa2_eth_priv *priv, 131270b32d82SIonut-robert Aron bool enable) 131370b32d82SIonut-robert Aron { 131470b32d82SIonut-robert Aron int err; 131570b32d82SIonut-robert Aron 131670b32d82SIonut-robert Aron err = dpni_enable_vlan_filter(priv->mc_io, 0, priv->mc_token, enable); 131770b32d82SIonut-robert Aron 131870b32d82SIonut-robert Aron if (err) { 131970b32d82SIonut-robert Aron netdev_err(priv->net_dev, 132070b32d82SIonut-robert Aron "dpni_enable_vlan_filter failed\n"); 132170b32d82SIonut-robert Aron return err; 132270b32d82SIonut-robert Aron } 132370b32d82SIonut-robert Aron 132470b32d82SIonut-robert Aron return 0; 132570b32d82SIonut-robert Aron } 132670b32d82SIonut-robert Aron 13275d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 132834ff6846SIoana Radulescu { 132934ff6846SIoana Radulescu int err; 133034ff6846SIoana Radulescu 133134ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 133234ff6846SIoana Radulescu DPNI_OFF_RX_L3_CSUM, enable); 133334ff6846SIoana Radulescu if (err) { 133434ff6846SIoana Radulescu netdev_err(priv->net_dev, 133534ff6846SIoana Radulescu "dpni_set_offload(RX_L3_CSUM) failed\n"); 133634ff6846SIoana Radulescu return err; 133734ff6846SIoana Radulescu } 133834ff6846SIoana Radulescu 133934ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 134034ff6846SIoana Radulescu DPNI_OFF_RX_L4_CSUM, enable); 134134ff6846SIoana Radulescu if (err) { 134234ff6846SIoana Radulescu netdev_err(priv->net_dev, 134334ff6846SIoana Radulescu "dpni_set_offload(RX_L4_CSUM) failed\n"); 134434ff6846SIoana Radulescu return err; 134534ff6846SIoana Radulescu } 134634ff6846SIoana Radulescu 134734ff6846SIoana Radulescu return 0; 134834ff6846SIoana Radulescu } 134934ff6846SIoana Radulescu 13505d8dccf8SIoana Ciornei static int dpaa2_eth_set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 135134ff6846SIoana Radulescu { 135234ff6846SIoana Radulescu int err; 135334ff6846SIoana Radulescu 135434ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 135534ff6846SIoana Radulescu DPNI_OFF_TX_L3_CSUM, enable); 135634ff6846SIoana Radulescu if (err) { 135734ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 135834ff6846SIoana Radulescu return err; 135934ff6846SIoana Radulescu } 136034ff6846SIoana Radulescu 136134ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 136234ff6846SIoana Radulescu DPNI_OFF_TX_L4_CSUM, enable); 136334ff6846SIoana Radulescu if (err) { 136434ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 136534ff6846SIoana Radulescu return err; 136634ff6846SIoana Radulescu } 136734ff6846SIoana Radulescu 136834ff6846SIoana Radulescu return 0; 136934ff6846SIoana Radulescu } 137034ff6846SIoana Radulescu 137134ff6846SIoana Radulescu /* Perform a single release command to add buffers 137234ff6846SIoana Radulescu * to the specified buffer pool 137334ff6846SIoana Radulescu */ 13745d8dccf8SIoana Ciornei static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv, 137534ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, u16 bpid) 137634ff6846SIoana Radulescu { 137734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 137834ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 137927c87486SIoana Ciocoi Radulescu struct page *page; 138034ff6846SIoana Radulescu dma_addr_t addr; 1381ef17bd7cSIoana Radulescu int retries = 0; 138234ff6846SIoana Radulescu int i, err; 138334ff6846SIoana Radulescu 138434ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 138534ff6846SIoana Radulescu /* Allocate buffer visible to WRIOP + skb shared info + 138634ff6846SIoana Radulescu * alignment padding 138734ff6846SIoana Radulescu */ 138827c87486SIoana Ciocoi Radulescu /* allocate one page for each Rx buffer. WRIOP sees 138927c87486SIoana Ciocoi Radulescu * the entire page except for a tailroom reserved for 139027c87486SIoana Ciocoi Radulescu * skb shared info 139127c87486SIoana Ciocoi Radulescu */ 139227c87486SIoana Ciocoi Radulescu page = dev_alloc_pages(0); 139327c87486SIoana Ciocoi Radulescu if (!page) 139434ff6846SIoana Radulescu goto err_alloc; 139534ff6846SIoana Radulescu 1396efa6a7d0SIoana Ciornei addr = dma_map_page(dev, page, 0, priv->rx_buf_size, 139718c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 139834ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 139934ff6846SIoana Radulescu goto err_map; 140034ff6846SIoana Radulescu 140134ff6846SIoana Radulescu buf_array[i] = addr; 140234ff6846SIoana Radulescu 140334ff6846SIoana Radulescu /* tracing point */ 140434ff6846SIoana Radulescu trace_dpaa2_eth_buf_seed(priv->net_dev, 140527c87486SIoana Ciocoi Radulescu page, DPAA2_ETH_RX_BUF_RAW_SIZE, 1406efa6a7d0SIoana Ciornei addr, priv->rx_buf_size, 140734ff6846SIoana Radulescu bpid); 140834ff6846SIoana Radulescu } 140934ff6846SIoana Radulescu 141034ff6846SIoana Radulescu release_bufs: 141134ff6846SIoana Radulescu /* In case the portal is busy, retry until successful */ 141234ff6846SIoana Radulescu while ((err = dpaa2_io_service_release(ch->dpio, bpid, 1413ef17bd7cSIoana Radulescu buf_array, i)) == -EBUSY) { 1414ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 1415ef17bd7cSIoana Radulescu break; 141634ff6846SIoana Radulescu cpu_relax(); 1417ef17bd7cSIoana Radulescu } 141834ff6846SIoana Radulescu 141934ff6846SIoana Radulescu /* If release command failed, clean up and bail out; 142034ff6846SIoana Radulescu * not much else we can do about it 142134ff6846SIoana Radulescu */ 142234ff6846SIoana Radulescu if (err) { 14235d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, i); 142434ff6846SIoana Radulescu return 0; 142534ff6846SIoana Radulescu } 142634ff6846SIoana Radulescu 142734ff6846SIoana Radulescu return i; 142834ff6846SIoana Radulescu 142934ff6846SIoana Radulescu err_map: 143027c87486SIoana Ciocoi Radulescu __free_pages(page, 0); 143134ff6846SIoana Radulescu err_alloc: 143234ff6846SIoana Radulescu /* If we managed to allocate at least some buffers, 143334ff6846SIoana Radulescu * release them to hardware 143434ff6846SIoana Radulescu */ 143534ff6846SIoana Radulescu if (i) 143634ff6846SIoana Radulescu goto release_bufs; 143734ff6846SIoana Radulescu 143834ff6846SIoana Radulescu return 0; 143934ff6846SIoana Radulescu } 144034ff6846SIoana Radulescu 14415d8dccf8SIoana Ciornei static int dpaa2_eth_seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 144234ff6846SIoana Radulescu { 144334ff6846SIoana Radulescu int i, j; 144434ff6846SIoana Radulescu int new_count; 144534ff6846SIoana Radulescu 144634ff6846SIoana Radulescu for (j = 0; j < priv->num_channels; j++) { 144734ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_NUM_BUFS; 144834ff6846SIoana Radulescu i += DPAA2_ETH_BUFS_PER_CMD) { 14495d8dccf8SIoana Ciornei new_count = dpaa2_eth_add_bufs(priv, priv->channel[j], bpid); 145034ff6846SIoana Radulescu priv->channel[j]->buf_count += new_count; 145134ff6846SIoana Radulescu 145234ff6846SIoana Radulescu if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 145334ff6846SIoana Radulescu return -ENOMEM; 145434ff6846SIoana Radulescu } 145534ff6846SIoana Radulescu } 145634ff6846SIoana Radulescu } 145734ff6846SIoana Radulescu 145834ff6846SIoana Radulescu return 0; 145934ff6846SIoana Radulescu } 146034ff6846SIoana Radulescu 1461d0ea5cbdSJesse Brandeburg /* 146234ff6846SIoana Radulescu * Drain the specified number of buffers from the DPNI's private buffer pool. 146334ff6846SIoana Radulescu * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 146434ff6846SIoana Radulescu */ 14655d8dccf8SIoana Ciornei static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int count) 146634ff6846SIoana Radulescu { 146734ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1468ef17bd7cSIoana Radulescu int retries = 0; 146934ff6846SIoana Radulescu int ret; 147034ff6846SIoana Radulescu 147134ff6846SIoana Radulescu do { 147234ff6846SIoana Radulescu ret = dpaa2_io_service_acquire(NULL, priv->bpid, 147334ff6846SIoana Radulescu buf_array, count); 147434ff6846SIoana Radulescu if (ret < 0) { 1475ef17bd7cSIoana Radulescu if (ret == -EBUSY && 14760e5ad75bSIoana Ciornei retries++ < DPAA2_ETH_SWP_BUSY_RETRIES) 1477ef17bd7cSIoana Radulescu continue; 147834ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 147934ff6846SIoana Radulescu return; 148034ff6846SIoana Radulescu } 14815d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, ret); 1482ef17bd7cSIoana Radulescu retries = 0; 148334ff6846SIoana Radulescu } while (ret); 148434ff6846SIoana Radulescu } 148534ff6846SIoana Radulescu 14865d8dccf8SIoana Ciornei static void dpaa2_eth_drain_pool(struct dpaa2_eth_priv *priv) 148734ff6846SIoana Radulescu { 148834ff6846SIoana Radulescu int i; 148934ff6846SIoana Radulescu 14905d8dccf8SIoana Ciornei dpaa2_eth_drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 14915d8dccf8SIoana Ciornei dpaa2_eth_drain_bufs(priv, 1); 149234ff6846SIoana Radulescu 149334ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 149434ff6846SIoana Radulescu priv->channel[i]->buf_count = 0; 149534ff6846SIoana Radulescu } 149634ff6846SIoana Radulescu 149734ff6846SIoana Radulescu /* Function is called from softirq context only, so we don't need to guard 149834ff6846SIoana Radulescu * the access to percpu count 149934ff6846SIoana Radulescu */ 15005d8dccf8SIoana Ciornei static int dpaa2_eth_refill_pool(struct dpaa2_eth_priv *priv, 150134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 150234ff6846SIoana Radulescu u16 bpid) 150334ff6846SIoana Radulescu { 150434ff6846SIoana Radulescu int new_count; 150534ff6846SIoana Radulescu 150634ff6846SIoana Radulescu if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 150734ff6846SIoana Radulescu return 0; 150834ff6846SIoana Radulescu 150934ff6846SIoana Radulescu do { 15105d8dccf8SIoana Ciornei new_count = dpaa2_eth_add_bufs(priv, ch, bpid); 151134ff6846SIoana Radulescu if (unlikely(!new_count)) { 151234ff6846SIoana Radulescu /* Out of memory; abort for now, we'll try later on */ 151334ff6846SIoana Radulescu break; 151434ff6846SIoana Radulescu } 151534ff6846SIoana Radulescu ch->buf_count += new_count; 151634ff6846SIoana Radulescu } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 151734ff6846SIoana Radulescu 151834ff6846SIoana Radulescu if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 151934ff6846SIoana Radulescu return -ENOMEM; 152034ff6846SIoana Radulescu 152134ff6846SIoana Radulescu return 0; 152234ff6846SIoana Radulescu } 152334ff6846SIoana Radulescu 1524d70446eeSIoana Ciornei static void dpaa2_eth_sgt_cache_drain(struct dpaa2_eth_priv *priv) 1525d70446eeSIoana Ciornei { 1526d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 1527d70446eeSIoana Ciornei u16 count; 1528d70446eeSIoana Ciornei int k, i; 1529d70446eeSIoana Ciornei 15300fe665d4SIoana Ciornei for_each_possible_cpu(k) { 1531d70446eeSIoana Ciornei sgt_cache = per_cpu_ptr(priv->sgt_cache, k); 1532d70446eeSIoana Ciornei count = sgt_cache->count; 1533d70446eeSIoana Ciornei 1534d70446eeSIoana Ciornei for (i = 0; i < count; i++) 15358378a791SIoana Ciornei skb_free_frag(sgt_cache->buf[i]); 1536d70446eeSIoana Ciornei sgt_cache->count = 0; 1537d70446eeSIoana Ciornei } 1538d70446eeSIoana Ciornei } 1539d70446eeSIoana Ciornei 15405d8dccf8SIoana Ciornei static int dpaa2_eth_pull_channel(struct dpaa2_eth_channel *ch) 154134ff6846SIoana Radulescu { 154234ff6846SIoana Radulescu int err; 154334ff6846SIoana Radulescu int dequeues = -1; 154434ff6846SIoana Radulescu 154534ff6846SIoana Radulescu /* Retry while portal is busy */ 154634ff6846SIoana Radulescu do { 154734ff6846SIoana Radulescu err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 154834ff6846SIoana Radulescu ch->store); 154934ff6846SIoana Radulescu dequeues++; 155034ff6846SIoana Radulescu cpu_relax(); 1551ef17bd7cSIoana Radulescu } while (err == -EBUSY && dequeues < DPAA2_ETH_SWP_BUSY_RETRIES); 155234ff6846SIoana Radulescu 155334ff6846SIoana Radulescu ch->stats.dequeue_portal_busy += dequeues; 155434ff6846SIoana Radulescu if (unlikely(err)) 155534ff6846SIoana Radulescu ch->stats.pull_err++; 155634ff6846SIoana Radulescu 155734ff6846SIoana Radulescu return err; 155834ff6846SIoana Radulescu } 155934ff6846SIoana Radulescu 156034ff6846SIoana Radulescu /* NAPI poll routine 156134ff6846SIoana Radulescu * 156234ff6846SIoana Radulescu * Frames are dequeued from the QMan channel associated with this NAPI context. 156334ff6846SIoana Radulescu * Rx, Tx confirmation and (if configured) Rx error frames all count 156434ff6846SIoana Radulescu * towards the NAPI budget. 156534ff6846SIoana Radulescu */ 156634ff6846SIoana Radulescu static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 156734ff6846SIoana Radulescu { 156834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 156934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 157068049a5fSIoana Ciocoi Radulescu int rx_cleaned = 0, txconf_cleaned = 0; 1571569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1572569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1573569dac6aSIoana Ciocoi Radulescu int store_cleaned, work_done; 15740a25d92cSIoana Ciornei struct list_head rx_list; 1575ef17bd7cSIoana Radulescu int retries = 0; 157674a1c059SIoana Ciornei u16 flowid; 157734ff6846SIoana Radulescu int err; 157834ff6846SIoana Radulescu 157934ff6846SIoana Radulescu ch = container_of(napi, struct dpaa2_eth_channel, napi); 1580d678be1dSIoana Radulescu ch->xdp.res = 0; 158134ff6846SIoana Radulescu priv = ch->priv; 158234ff6846SIoana Radulescu 15830a25d92cSIoana Ciornei INIT_LIST_HEAD(&rx_list); 15840a25d92cSIoana Ciornei ch->rx_list = &rx_list; 15850a25d92cSIoana Ciornei 158668049a5fSIoana Ciocoi Radulescu do { 15875d8dccf8SIoana Ciornei err = dpaa2_eth_pull_channel(ch); 158834ff6846SIoana Radulescu if (unlikely(err)) 158934ff6846SIoana Radulescu break; 159034ff6846SIoana Radulescu 159134ff6846SIoana Radulescu /* Refill pool if appropriate */ 15925d8dccf8SIoana Ciornei dpaa2_eth_refill_pool(priv, ch, priv->bpid); 159334ff6846SIoana Radulescu 15945d8dccf8SIoana Ciornei store_cleaned = dpaa2_eth_consume_frames(ch, &fq); 1595ef17bd7cSIoana Radulescu if (store_cleaned <= 0) 1596569dac6aSIoana Ciocoi Radulescu break; 1597569dac6aSIoana Ciocoi Radulescu if (fq->type == DPAA2_RX_FQ) { 159868049a5fSIoana Ciocoi Radulescu rx_cleaned += store_cleaned; 159974a1c059SIoana Ciornei flowid = fq->flowid; 1600569dac6aSIoana Ciocoi Radulescu } else { 160168049a5fSIoana Ciocoi Radulescu txconf_cleaned += store_cleaned; 1602569dac6aSIoana Ciocoi Radulescu /* We have a single Tx conf FQ on this channel */ 1603569dac6aSIoana Ciocoi Radulescu txc_fq = fq; 1604569dac6aSIoana Ciocoi Radulescu } 160534ff6846SIoana Radulescu 160668049a5fSIoana Ciocoi Radulescu /* If we either consumed the whole NAPI budget with Rx frames 160768049a5fSIoana Ciocoi Radulescu * or we reached the Tx confirmations threshold, we're done. 160834ff6846SIoana Radulescu */ 160968049a5fSIoana Ciocoi Radulescu if (rx_cleaned >= budget || 1610569dac6aSIoana Ciocoi Radulescu txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1611569dac6aSIoana Ciocoi Radulescu work_done = budget; 1612569dac6aSIoana Ciocoi Radulescu goto out; 1613569dac6aSIoana Ciocoi Radulescu } 161468049a5fSIoana Ciocoi Radulescu } while (store_cleaned); 161534ff6846SIoana Radulescu 1616fc398becSIoana Ciornei /* Update NET DIM with the values for this CDAN */ 1617fc398becSIoana Ciornei dpaa2_io_update_net_dim(ch->dpio, ch->stats.frames_per_cdan, 1618fc398becSIoana Ciornei ch->stats.bytes_per_cdan); 1619fc398becSIoana Ciornei ch->stats.frames_per_cdan = 0; 1620fc398becSIoana Ciornei ch->stats.bytes_per_cdan = 0; 1621fc398becSIoana Ciornei 162268049a5fSIoana Ciocoi Radulescu /* We didn't consume the entire budget, so finish napi and 162368049a5fSIoana Ciocoi Radulescu * re-enable data availability notifications 162468049a5fSIoana Ciocoi Radulescu */ 162568049a5fSIoana Ciocoi Radulescu napi_complete_done(napi, rx_cleaned); 162634ff6846SIoana Radulescu do { 162734ff6846SIoana Radulescu err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 162834ff6846SIoana Radulescu cpu_relax(); 1629ef17bd7cSIoana Radulescu } while (err == -EBUSY && retries++ < DPAA2_ETH_SWP_BUSY_RETRIES); 163034ff6846SIoana Radulescu WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 163134ff6846SIoana Radulescu ch->nctx.desired_cpu); 163234ff6846SIoana Radulescu 1633569dac6aSIoana Ciocoi Radulescu work_done = max(rx_cleaned, 1); 1634569dac6aSIoana Ciocoi Radulescu 1635569dac6aSIoana Ciocoi Radulescu out: 16360a25d92cSIoana Ciornei netif_receive_skb_list(ch->rx_list); 16370a25d92cSIoana Ciornei 1638d678be1dSIoana Radulescu if (txc_fq && txc_fq->dq_frames) { 1639569dac6aSIoana Ciocoi Radulescu nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1640569dac6aSIoana Ciocoi Radulescu netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1641569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes); 1642569dac6aSIoana Ciocoi Radulescu txc_fq->dq_frames = 0; 1643569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes = 0; 1644569dac6aSIoana Ciocoi Radulescu } 1645569dac6aSIoana Ciocoi Radulescu 1646d678be1dSIoana Radulescu if (ch->xdp.res & XDP_REDIRECT) 1647d678be1dSIoana Radulescu xdp_do_flush_map(); 164874a1c059SIoana Ciornei else if (rx_cleaned && ch->xdp.res & XDP_TX) 16495d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, &priv->fq[flowid]); 1650d678be1dSIoana Radulescu 1651569dac6aSIoana Ciocoi Radulescu return work_done; 165234ff6846SIoana Radulescu } 165334ff6846SIoana Radulescu 16545d8dccf8SIoana Ciornei static void dpaa2_eth_enable_ch_napi(struct dpaa2_eth_priv *priv) 165534ff6846SIoana Radulescu { 165634ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 165734ff6846SIoana Radulescu int i; 165834ff6846SIoana Radulescu 165934ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 166034ff6846SIoana Radulescu ch = priv->channel[i]; 166134ff6846SIoana Radulescu napi_enable(&ch->napi); 166234ff6846SIoana Radulescu } 166334ff6846SIoana Radulescu } 166434ff6846SIoana Radulescu 16655d8dccf8SIoana Ciornei static void dpaa2_eth_disable_ch_napi(struct dpaa2_eth_priv *priv) 166634ff6846SIoana Radulescu { 166734ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 166834ff6846SIoana Radulescu int i; 166934ff6846SIoana Radulescu 167034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 167134ff6846SIoana Radulescu ch = priv->channel[i]; 167234ff6846SIoana Radulescu napi_disable(&ch->napi); 167334ff6846SIoana Radulescu } 167434ff6846SIoana Radulescu } 167534ff6846SIoana Radulescu 167607beb165SIoana Ciornei void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, 167707beb165SIoana Ciornei bool tx_pause, bool pfc) 16788eb3cef8SIoana Radulescu { 16798eb3cef8SIoana Radulescu struct dpni_taildrop td = {0}; 1680685e39eaSIoana Radulescu struct dpaa2_eth_fq *fq; 16818eb3cef8SIoana Radulescu int i, err; 16828eb3cef8SIoana Radulescu 168307beb165SIoana Ciornei /* FQ taildrop: threshold is in bytes, per frame queue. Enabled if 168407beb165SIoana Ciornei * flow control is disabled (as it might interfere with either the 168507beb165SIoana Ciornei * buffer pool depletion trigger for pause frames or with the group 168607beb165SIoana Ciornei * congestion trigger for PFC frames) 168707beb165SIoana Ciornei */ 16882c8d1c8dSIoana Radulescu td.enable = !tx_pause; 168907beb165SIoana Ciornei if (priv->rx_fqtd_enabled == td.enable) 169007beb165SIoana Ciornei goto set_cgtd; 16918eb3cef8SIoana Radulescu 16922c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_FQ_TAILDROP_THRESH; 16932c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_BYTES; 16948eb3cef8SIoana Radulescu 16958eb3cef8SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 1696685e39eaSIoana Radulescu fq = &priv->fq[i]; 1697685e39eaSIoana Radulescu if (fq->type != DPAA2_RX_FQ) 16988eb3cef8SIoana Radulescu continue; 16998eb3cef8SIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1700685e39eaSIoana Radulescu DPNI_CP_QUEUE, DPNI_QUEUE_RX, 1701685e39eaSIoana Radulescu fq->tc, fq->flowid, &td); 17028eb3cef8SIoana Radulescu if (err) { 17038eb3cef8SIoana Radulescu netdev_err(priv->net_dev, 17042c8d1c8dSIoana Radulescu "dpni_set_taildrop(FQ) failed\n"); 17052c8d1c8dSIoana Radulescu return; 17068eb3cef8SIoana Radulescu } 17078eb3cef8SIoana Radulescu } 17088eb3cef8SIoana Radulescu 170907beb165SIoana Ciornei priv->rx_fqtd_enabled = td.enable; 171007beb165SIoana Ciornei 171107beb165SIoana Ciornei set_cgtd: 17122c8d1c8dSIoana Radulescu /* Congestion group taildrop: threshold is in frames, per group 17132c8d1c8dSIoana Radulescu * of FQs belonging to the same traffic class 171407beb165SIoana Ciornei * Enabled if general Tx pause disabled or if PFCs are enabled 171507beb165SIoana Ciornei * (congestion group threhsold for PFC generation is lower than the 171607beb165SIoana Ciornei * CG taildrop threshold, so it won't interfere with it; we also 171707beb165SIoana Ciornei * want frames in non-PFC enabled traffic classes to be kept in check) 17182c8d1c8dSIoana Radulescu */ 1719b91b3a21SJiapeng Chong td.enable = !tx_pause || pfc; 172007beb165SIoana Ciornei if (priv->rx_cgtd_enabled == td.enable) 172107beb165SIoana Ciornei return; 172207beb165SIoana Ciornei 17232c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_CG_TAILDROP_THRESH(priv); 17242c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_FRAMES; 17252c8d1c8dSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 17262c8d1c8dSIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 17272c8d1c8dSIoana Radulescu DPNI_CP_GROUP, DPNI_QUEUE_RX, 17282c8d1c8dSIoana Radulescu i, 0, &td); 17292c8d1c8dSIoana Radulescu if (err) { 17302c8d1c8dSIoana Radulescu netdev_err(priv->net_dev, 17312c8d1c8dSIoana Radulescu "dpni_set_taildrop(CG) failed\n"); 17322c8d1c8dSIoana Radulescu return; 17332c8d1c8dSIoana Radulescu } 17342c8d1c8dSIoana Radulescu } 17352c8d1c8dSIoana Radulescu 173607beb165SIoana Ciornei priv->rx_cgtd_enabled = td.enable; 17378eb3cef8SIoana Radulescu } 17388eb3cef8SIoana Radulescu 17395d8dccf8SIoana Ciornei static int dpaa2_eth_link_state_update(struct dpaa2_eth_priv *priv) 174034ff6846SIoana Radulescu { 174185b7a342SIoana Ciornei struct dpni_link_state state = {0}; 17428eb3cef8SIoana Radulescu bool tx_pause; 174334ff6846SIoana Radulescu int err; 174434ff6846SIoana Radulescu 174534ff6846SIoana Radulescu err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 174634ff6846SIoana Radulescu if (unlikely(err)) { 174734ff6846SIoana Radulescu netdev_err(priv->net_dev, 174834ff6846SIoana Radulescu "dpni_get_link_state() failed\n"); 174934ff6846SIoana Radulescu return err; 175034ff6846SIoana Radulescu } 175134ff6846SIoana Radulescu 17528eb3cef8SIoana Radulescu /* If Tx pause frame settings have changed, we need to update 17538eb3cef8SIoana Radulescu * Rx FQ taildrop configuration as well. We configure taildrop 17548eb3cef8SIoana Radulescu * only when pause frame generation is disabled. 17558eb3cef8SIoana Radulescu */ 1756ad054f26SIoana Radulescu tx_pause = dpaa2_eth_tx_pause_enabled(state.options); 175707beb165SIoana Ciornei dpaa2_eth_set_rx_taildrop(priv, tx_pause, priv->pfc_enabled); 17588eb3cef8SIoana Radulescu 175971947923SIoana Ciornei /* When we manage the MAC/PHY using phylink there is no need 176071947923SIoana Ciornei * to manually update the netif_carrier. 176171947923SIoana Ciornei */ 1762d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 176371947923SIoana Ciornei goto out; 176471947923SIoana Ciornei 176534ff6846SIoana Radulescu /* Chech link state; speed / duplex changes are not treated yet */ 176634ff6846SIoana Radulescu if (priv->link_state.up == state.up) 1767cce62943SIoana Radulescu goto out; 176834ff6846SIoana Radulescu 176934ff6846SIoana Radulescu if (state.up) { 177034ff6846SIoana Radulescu netif_carrier_on(priv->net_dev); 177134ff6846SIoana Radulescu netif_tx_start_all_queues(priv->net_dev); 177234ff6846SIoana Radulescu } else { 177334ff6846SIoana Radulescu netif_tx_stop_all_queues(priv->net_dev); 177434ff6846SIoana Radulescu netif_carrier_off(priv->net_dev); 177534ff6846SIoana Radulescu } 177634ff6846SIoana Radulescu 177734ff6846SIoana Radulescu netdev_info(priv->net_dev, "Link Event: state %s\n", 177834ff6846SIoana Radulescu state.up ? "up" : "down"); 177934ff6846SIoana Radulescu 1780cce62943SIoana Radulescu out: 1781cce62943SIoana Radulescu priv->link_state = state; 1782cce62943SIoana Radulescu 178334ff6846SIoana Radulescu return 0; 178434ff6846SIoana Radulescu } 178534ff6846SIoana Radulescu 178634ff6846SIoana Radulescu static int dpaa2_eth_open(struct net_device *net_dev) 178734ff6846SIoana Radulescu { 178834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 178934ff6846SIoana Radulescu int err; 179034ff6846SIoana Radulescu 17915d8dccf8SIoana Ciornei err = dpaa2_eth_seed_pool(priv, priv->bpid); 179234ff6846SIoana Radulescu if (err) { 179334ff6846SIoana Radulescu /* Not much to do; the buffer pool, though not filled up, 179434ff6846SIoana Radulescu * may still contain some buffers which would enable us 179534ff6846SIoana Radulescu * to limp on. 179634ff6846SIoana Radulescu */ 179734ff6846SIoana Radulescu netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 179834ff6846SIoana Radulescu priv->dpbp_dev->obj_desc.id, priv->bpid); 179934ff6846SIoana Radulescu } 180034ff6846SIoana Radulescu 1801d87e6063SIoana Ciornei if (!dpaa2_eth_is_type_phy(priv)) { 180271947923SIoana Ciornei /* We'll only start the txqs when the link is actually ready; 180371947923SIoana Ciornei * make sure we don't race against the link up notification, 180471947923SIoana Ciornei * which may come immediately after dpni_enable(); 180534ff6846SIoana Radulescu */ 180634ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 180771947923SIoana Ciornei 180871947923SIoana Ciornei /* Also, explicitly set carrier off, otherwise 180971947923SIoana Ciornei * netif_carrier_ok() will return true and cause 'ip link show' 181071947923SIoana Ciornei * to report the LOWER_UP flag, even though the link 181171947923SIoana Ciornei * notification wasn't even received. 181234ff6846SIoana Radulescu */ 181334ff6846SIoana Radulescu netif_carrier_off(net_dev); 181471947923SIoana Ciornei } 18155d8dccf8SIoana Ciornei dpaa2_eth_enable_ch_napi(priv); 181634ff6846SIoana Radulescu 181734ff6846SIoana Radulescu err = dpni_enable(priv->mc_io, 0, priv->mc_token); 181834ff6846SIoana Radulescu if (err < 0) { 181934ff6846SIoana Radulescu netdev_err(net_dev, "dpni_enable() failed\n"); 182034ff6846SIoana Radulescu goto enable_err; 182134ff6846SIoana Radulescu } 182234ff6846SIoana Radulescu 1823d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 182471947923SIoana Ciornei phylink_start(priv->mac->phylink); 182534ff6846SIoana Radulescu 182634ff6846SIoana Radulescu return 0; 182734ff6846SIoana Radulescu 182834ff6846SIoana Radulescu enable_err: 18295d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 18305d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 183134ff6846SIoana Radulescu return err; 183234ff6846SIoana Radulescu } 183334ff6846SIoana Radulescu 183468d74315SIoana Ciocoi Radulescu /* Total number of in-flight frames on ingress queues */ 18355d8dccf8SIoana Ciornei static u32 dpaa2_eth_ingress_fq_count(struct dpaa2_eth_priv *priv) 183634ff6846SIoana Radulescu { 183768d74315SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq; 183868d74315SIoana Ciocoi Radulescu u32 fcnt = 0, bcnt = 0, total = 0; 183968d74315SIoana Ciocoi Radulescu int i, err; 184034ff6846SIoana Radulescu 184168d74315SIoana Ciocoi Radulescu for (i = 0; i < priv->num_fqs; i++) { 184268d74315SIoana Ciocoi Radulescu fq = &priv->fq[i]; 184368d74315SIoana Ciocoi Radulescu err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 184468d74315SIoana Ciocoi Radulescu if (err) { 184568d74315SIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "query_fq_count failed"); 184668d74315SIoana Ciocoi Radulescu break; 184768d74315SIoana Ciocoi Radulescu } 184868d74315SIoana Ciocoi Radulescu total += fcnt; 184968d74315SIoana Ciocoi Radulescu } 185034ff6846SIoana Radulescu 185134ff6846SIoana Radulescu return total; 185234ff6846SIoana Radulescu } 185334ff6846SIoana Radulescu 18545d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 185534ff6846SIoana Radulescu { 185668d74315SIoana Ciocoi Radulescu int retries = 10; 185768d74315SIoana Ciocoi Radulescu u32 pending; 185834ff6846SIoana Radulescu 185968d74315SIoana Ciocoi Radulescu do { 18605d8dccf8SIoana Ciornei pending = dpaa2_eth_ingress_fq_count(priv); 186168d74315SIoana Ciocoi Radulescu if (pending) 186268d74315SIoana Ciocoi Radulescu msleep(100); 186368d74315SIoana Ciocoi Radulescu } while (pending && --retries); 186434ff6846SIoana Radulescu } 186534ff6846SIoana Radulescu 186652b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MAJOR 7 186752b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MINOR 13 18685d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 186952b6a4ffSIoana Radulescu { 187052b6a4ffSIoana Radulescu union dpni_statistics stats; 187152b6a4ffSIoana Radulescu int retries = 10; 187252b6a4ffSIoana Radulescu int err; 187352b6a4ffSIoana Radulescu 187452b6a4ffSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 187552b6a4ffSIoana Radulescu DPNI_TX_PENDING_VER_MINOR) < 0) 187652b6a4ffSIoana Radulescu goto out; 187752b6a4ffSIoana Radulescu 187852b6a4ffSIoana Radulescu do { 187952b6a4ffSIoana Radulescu err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 188052b6a4ffSIoana Radulescu &stats); 188152b6a4ffSIoana Radulescu if (err) 188252b6a4ffSIoana Radulescu goto out; 188352b6a4ffSIoana Radulescu if (stats.page_6.tx_pending_frames == 0) 188452b6a4ffSIoana Radulescu return; 188552b6a4ffSIoana Radulescu } while (--retries); 188652b6a4ffSIoana Radulescu 188752b6a4ffSIoana Radulescu out: 188852b6a4ffSIoana Radulescu msleep(500); 188952b6a4ffSIoana Radulescu } 189052b6a4ffSIoana Radulescu 189134ff6846SIoana Radulescu static int dpaa2_eth_stop(struct net_device *net_dev) 189234ff6846SIoana Radulescu { 189334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 189485b7a342SIoana Ciornei int dpni_enabled = 0; 189534ff6846SIoana Radulescu int retries = 10; 189634ff6846SIoana Radulescu 1897d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 1898d87e6063SIoana Ciornei phylink_stop(priv->mac->phylink); 1899d87e6063SIoana Ciornei } else { 190034ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 190134ff6846SIoana Radulescu netif_carrier_off(net_dev); 190271947923SIoana Ciornei } 190334ff6846SIoana Radulescu 190468d74315SIoana Ciocoi Radulescu /* On dpni_disable(), the MC firmware will: 190568d74315SIoana Ciocoi Radulescu * - stop MAC Rx and wait for all Rx frames to be enqueued to software 190668d74315SIoana Ciocoi Radulescu * - cut off WRIOP dequeues from egress FQs and wait until transmission 190768d74315SIoana Ciocoi Radulescu * of all in flight Tx frames is finished (and corresponding Tx conf 190868d74315SIoana Ciocoi Radulescu * frames are enqueued back to software) 190968d74315SIoana Ciocoi Radulescu * 191068d74315SIoana Ciocoi Radulescu * Before calling dpni_disable(), we wait for all Tx frames to arrive 191168d74315SIoana Ciocoi Radulescu * on WRIOP. After it finishes, wait until all remaining frames on Rx 191268d74315SIoana Ciocoi Radulescu * and Tx conf queues are consumed on NAPI poll. 191334ff6846SIoana Radulescu */ 19145d8dccf8SIoana Ciornei dpaa2_eth_wait_for_egress_fq_empty(priv); 191568d74315SIoana Ciocoi Radulescu 191634ff6846SIoana Radulescu do { 191734ff6846SIoana Radulescu dpni_disable(priv->mc_io, 0, priv->mc_token); 191834ff6846SIoana Radulescu dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 191934ff6846SIoana Radulescu if (dpni_enabled) 192034ff6846SIoana Radulescu /* Allow the hardware some slack */ 192134ff6846SIoana Radulescu msleep(100); 192234ff6846SIoana Radulescu } while (dpni_enabled && --retries); 192334ff6846SIoana Radulescu if (!retries) { 192434ff6846SIoana Radulescu netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 192534ff6846SIoana Radulescu /* Must go on and disable NAPI nonetheless, so we don't crash at 192634ff6846SIoana Radulescu * the next "ifconfig up" 192734ff6846SIoana Radulescu */ 192834ff6846SIoana Radulescu } 192934ff6846SIoana Radulescu 19305d8dccf8SIoana Ciornei dpaa2_eth_wait_for_ingress_fq_empty(priv); 19315d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 193234ff6846SIoana Radulescu 193334ff6846SIoana Radulescu /* Empty the buffer pool */ 19345d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 193534ff6846SIoana Radulescu 1936d70446eeSIoana Ciornei /* Empty the Scatter-Gather Buffer cache */ 1937d70446eeSIoana Ciornei dpaa2_eth_sgt_cache_drain(priv); 1938d70446eeSIoana Ciornei 193934ff6846SIoana Radulescu return 0; 194034ff6846SIoana Radulescu } 194134ff6846SIoana Radulescu 194234ff6846SIoana Radulescu static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 194334ff6846SIoana Radulescu { 194434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 194534ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 194634ff6846SIoana Radulescu int err; 194734ff6846SIoana Radulescu 194834ff6846SIoana Radulescu err = eth_mac_addr(net_dev, addr); 194934ff6846SIoana Radulescu if (err < 0) { 195034ff6846SIoana Radulescu dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 195134ff6846SIoana Radulescu return err; 195234ff6846SIoana Radulescu } 195334ff6846SIoana Radulescu 195434ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 195534ff6846SIoana Radulescu net_dev->dev_addr); 195634ff6846SIoana Radulescu if (err) { 195734ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 195834ff6846SIoana Radulescu return err; 195934ff6846SIoana Radulescu } 196034ff6846SIoana Radulescu 196134ff6846SIoana Radulescu return 0; 196234ff6846SIoana Radulescu } 196334ff6846SIoana Radulescu 196434ff6846SIoana Radulescu /** Fill in counters maintained by the GPP driver. These may be different from 196534ff6846SIoana Radulescu * the hardware counters obtained by ethtool. 196634ff6846SIoana Radulescu */ 196734ff6846SIoana Radulescu static void dpaa2_eth_get_stats(struct net_device *net_dev, 196834ff6846SIoana Radulescu struct rtnl_link_stats64 *stats) 196934ff6846SIoana Radulescu { 197034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 197134ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 197234ff6846SIoana Radulescu u64 *cpustats; 197334ff6846SIoana Radulescu u64 *netstats = (u64 *)stats; 197434ff6846SIoana Radulescu int i, j; 197534ff6846SIoana Radulescu int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 197634ff6846SIoana Radulescu 197734ff6846SIoana Radulescu for_each_possible_cpu(i) { 197834ff6846SIoana Radulescu percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 197934ff6846SIoana Radulescu cpustats = (u64 *)percpu_stats; 198034ff6846SIoana Radulescu for (j = 0; j < num; j++) 198134ff6846SIoana Radulescu netstats[j] += cpustats[j]; 198234ff6846SIoana Radulescu } 198334ff6846SIoana Radulescu } 198434ff6846SIoana Radulescu 198534ff6846SIoana Radulescu /* Copy mac unicast addresses from @net_dev to @priv. 198634ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 198734ff6846SIoana Radulescu */ 19885d8dccf8SIoana Ciornei static void dpaa2_eth_add_uc_hw_addr(const struct net_device *net_dev, 198934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 199034ff6846SIoana Radulescu { 199134ff6846SIoana Radulescu struct netdev_hw_addr *ha; 199234ff6846SIoana Radulescu int err; 199334ff6846SIoana Radulescu 199434ff6846SIoana Radulescu netdev_for_each_uc_addr(ha, net_dev) { 199534ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 199634ff6846SIoana Radulescu ha->addr); 199734ff6846SIoana Radulescu if (err) 199834ff6846SIoana Radulescu netdev_warn(priv->net_dev, 199934ff6846SIoana Radulescu "Could not add ucast MAC %pM to the filtering table (err %d)\n", 200034ff6846SIoana Radulescu ha->addr, err); 200134ff6846SIoana Radulescu } 200234ff6846SIoana Radulescu } 200334ff6846SIoana Radulescu 200434ff6846SIoana Radulescu /* Copy mac multicast addresses from @net_dev to @priv 200534ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 200634ff6846SIoana Radulescu */ 20075d8dccf8SIoana Ciornei static void dpaa2_eth_add_mc_hw_addr(const struct net_device *net_dev, 200834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 200934ff6846SIoana Radulescu { 201034ff6846SIoana Radulescu struct netdev_hw_addr *ha; 201134ff6846SIoana Radulescu int err; 201234ff6846SIoana Radulescu 201334ff6846SIoana Radulescu netdev_for_each_mc_addr(ha, net_dev) { 201434ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 201534ff6846SIoana Radulescu ha->addr); 201634ff6846SIoana Radulescu if (err) 201734ff6846SIoana Radulescu netdev_warn(priv->net_dev, 201834ff6846SIoana Radulescu "Could not add mcast MAC %pM to the filtering table (err %d)\n", 201934ff6846SIoana Radulescu ha->addr, err); 202034ff6846SIoana Radulescu } 202134ff6846SIoana Radulescu } 202234ff6846SIoana Radulescu 202370b32d82SIonut-robert Aron static int dpaa2_eth_rx_add_vid(struct net_device *net_dev, 202470b32d82SIonut-robert Aron __be16 vlan_proto, u16 vid) 202570b32d82SIonut-robert Aron { 202670b32d82SIonut-robert Aron struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 202770b32d82SIonut-robert Aron int err; 202870b32d82SIonut-robert Aron 202970b32d82SIonut-robert Aron err = dpni_add_vlan_id(priv->mc_io, 0, priv->mc_token, 203070b32d82SIonut-robert Aron vid, 0, 0, 0); 203170b32d82SIonut-robert Aron 203270b32d82SIonut-robert Aron if (err) { 203370b32d82SIonut-robert Aron netdev_warn(priv->net_dev, 203470b32d82SIonut-robert Aron "Could not add the vlan id %u\n", 203570b32d82SIonut-robert Aron vid); 203670b32d82SIonut-robert Aron return err; 203770b32d82SIonut-robert Aron } 203870b32d82SIonut-robert Aron 203970b32d82SIonut-robert Aron return 0; 204070b32d82SIonut-robert Aron } 204170b32d82SIonut-robert Aron 204270b32d82SIonut-robert Aron static int dpaa2_eth_rx_kill_vid(struct net_device *net_dev, 204370b32d82SIonut-robert Aron __be16 vlan_proto, u16 vid) 204470b32d82SIonut-robert Aron { 204570b32d82SIonut-robert Aron struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 204670b32d82SIonut-robert Aron int err; 204770b32d82SIonut-robert Aron 204870b32d82SIonut-robert Aron err = dpni_remove_vlan_id(priv->mc_io, 0, priv->mc_token, vid); 204970b32d82SIonut-robert Aron 205070b32d82SIonut-robert Aron if (err) { 205170b32d82SIonut-robert Aron netdev_warn(priv->net_dev, 205270b32d82SIonut-robert Aron "Could not remove the vlan id %u\n", 205370b32d82SIonut-robert Aron vid); 205470b32d82SIonut-robert Aron return err; 205570b32d82SIonut-robert Aron } 205670b32d82SIonut-robert Aron 205770b32d82SIonut-robert Aron return 0; 205870b32d82SIonut-robert Aron } 205970b32d82SIonut-robert Aron 206034ff6846SIoana Radulescu static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 206134ff6846SIoana Radulescu { 206234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 206334ff6846SIoana Radulescu int uc_count = netdev_uc_count(net_dev); 206434ff6846SIoana Radulescu int mc_count = netdev_mc_count(net_dev); 206534ff6846SIoana Radulescu u8 max_mac = priv->dpni_attrs.mac_filter_entries; 206634ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 206734ff6846SIoana Radulescu u16 mc_token = priv->mc_token; 206834ff6846SIoana Radulescu struct fsl_mc_io *mc_io = priv->mc_io; 206934ff6846SIoana Radulescu int err; 207034ff6846SIoana Radulescu 207134ff6846SIoana Radulescu /* Basic sanity checks; these probably indicate a misconfiguration */ 207234ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 207334ff6846SIoana Radulescu netdev_info(net_dev, 207434ff6846SIoana Radulescu "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 207534ff6846SIoana Radulescu max_mac); 207634ff6846SIoana Radulescu 207734ff6846SIoana Radulescu /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 207834ff6846SIoana Radulescu if (uc_count > max_mac) { 207934ff6846SIoana Radulescu netdev_info(net_dev, 208034ff6846SIoana Radulescu "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 208134ff6846SIoana Radulescu uc_count, max_mac); 208234ff6846SIoana Radulescu goto force_promisc; 208334ff6846SIoana Radulescu } 208434ff6846SIoana Radulescu if (mc_count + uc_count > max_mac) { 208534ff6846SIoana Radulescu netdev_info(net_dev, 208634ff6846SIoana Radulescu "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 208734ff6846SIoana Radulescu uc_count + mc_count, max_mac); 208834ff6846SIoana Radulescu goto force_mc_promisc; 208934ff6846SIoana Radulescu } 209034ff6846SIoana Radulescu 209134ff6846SIoana Radulescu /* Adjust promisc settings due to flag combinations */ 209234ff6846SIoana Radulescu if (net_dev->flags & IFF_PROMISC) 209334ff6846SIoana Radulescu goto force_promisc; 209434ff6846SIoana Radulescu if (net_dev->flags & IFF_ALLMULTI) { 209534ff6846SIoana Radulescu /* First, rebuild unicast filtering table. This should be done 209634ff6846SIoana Radulescu * in promisc mode, in order to avoid frame loss while we 209734ff6846SIoana Radulescu * progressively add entries to the table. 209834ff6846SIoana Radulescu * We don't know whether we had been in promisc already, and 209934ff6846SIoana Radulescu * making an MC call to find out is expensive; so set uc promisc 210034ff6846SIoana Radulescu * nonetheless. 210134ff6846SIoana Radulescu */ 210234ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 210334ff6846SIoana Radulescu if (err) 210434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc\n"); 210534ff6846SIoana Radulescu 210634ff6846SIoana Radulescu /* Actual uc table reconstruction. */ 210734ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 210834ff6846SIoana Radulescu if (err) 210934ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc filters\n"); 21105d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 211134ff6846SIoana Radulescu 211234ff6846SIoana Radulescu /* Finally, clear uc promisc and set mc promisc as requested. */ 211334ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 211434ff6846SIoana Radulescu if (err) 211534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc promisc\n"); 211634ff6846SIoana Radulescu goto force_mc_promisc; 211734ff6846SIoana Radulescu } 211834ff6846SIoana Radulescu 211934ff6846SIoana Radulescu /* Neither unicast, nor multicast promisc will be on... eventually. 212034ff6846SIoana Radulescu * For now, rebuild mac filtering tables while forcing both of them on. 212134ff6846SIoana Radulescu */ 212234ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 212334ff6846SIoana Radulescu if (err) 212434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 212534ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 212634ff6846SIoana Radulescu if (err) 212734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 212834ff6846SIoana Radulescu 212934ff6846SIoana Radulescu /* Actual mac filtering tables reconstruction */ 213034ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 213134ff6846SIoana Radulescu if (err) 213234ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mac filters\n"); 21335d8dccf8SIoana Ciornei dpaa2_eth_add_mc_hw_addr(net_dev, priv); 21345d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 213534ff6846SIoana Radulescu 213634ff6846SIoana Radulescu /* Now we can clear both ucast and mcast promisc, without risking 213734ff6846SIoana Radulescu * to drop legitimate frames anymore. 213834ff6846SIoana Radulescu */ 213934ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 214034ff6846SIoana Radulescu if (err) 214134ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear ucast promisc\n"); 214234ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 214334ff6846SIoana Radulescu if (err) 214434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mcast promisc\n"); 214534ff6846SIoana Radulescu 214634ff6846SIoana Radulescu return; 214734ff6846SIoana Radulescu 214834ff6846SIoana Radulescu force_promisc: 214934ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 215034ff6846SIoana Radulescu if (err) 215134ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set ucast promisc\n"); 215234ff6846SIoana Radulescu force_mc_promisc: 215334ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 215434ff6846SIoana Radulescu if (err) 215534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mcast promisc\n"); 215634ff6846SIoana Radulescu } 215734ff6846SIoana Radulescu 215834ff6846SIoana Radulescu static int dpaa2_eth_set_features(struct net_device *net_dev, 215934ff6846SIoana Radulescu netdev_features_t features) 216034ff6846SIoana Radulescu { 216134ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 216234ff6846SIoana Radulescu netdev_features_t changed = features ^ net_dev->features; 216334ff6846SIoana Radulescu bool enable; 216434ff6846SIoana Radulescu int err; 216534ff6846SIoana Radulescu 216670b32d82SIonut-robert Aron if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) { 216770b32d82SIonut-robert Aron enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER); 216870b32d82SIonut-robert Aron err = dpaa2_eth_set_rx_vlan_filtering(priv, enable); 216970b32d82SIonut-robert Aron if (err) 217070b32d82SIonut-robert Aron return err; 217170b32d82SIonut-robert Aron } 217270b32d82SIonut-robert Aron 217334ff6846SIoana Radulescu if (changed & NETIF_F_RXCSUM) { 217434ff6846SIoana Radulescu enable = !!(features & NETIF_F_RXCSUM); 21755d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, enable); 217634ff6846SIoana Radulescu if (err) 217734ff6846SIoana Radulescu return err; 217834ff6846SIoana Radulescu } 217934ff6846SIoana Radulescu 218034ff6846SIoana Radulescu if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 218134ff6846SIoana Radulescu enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 21825d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, enable); 218334ff6846SIoana Radulescu if (err) 218434ff6846SIoana Radulescu return err; 218534ff6846SIoana Radulescu } 218634ff6846SIoana Radulescu 218734ff6846SIoana Radulescu return 0; 218834ff6846SIoana Radulescu } 218934ff6846SIoana Radulescu 219034ff6846SIoana Radulescu static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 219134ff6846SIoana Radulescu { 219234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 219334ff6846SIoana Radulescu struct hwtstamp_config config; 219434ff6846SIoana Radulescu 2195c5521189SYangbo Lu if (!dpaa2_ptp) 2196c5521189SYangbo Lu return -EINVAL; 2197c5521189SYangbo Lu 219834ff6846SIoana Radulescu if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 219934ff6846SIoana Radulescu return -EFAULT; 220034ff6846SIoana Radulescu 220134ff6846SIoana Radulescu switch (config.tx_type) { 220234ff6846SIoana Radulescu case HWTSTAMP_TX_OFF: 220334ff6846SIoana Radulescu case HWTSTAMP_TX_ON: 2204c5521189SYangbo Lu case HWTSTAMP_TX_ONESTEP_SYNC: 22051cf773bdSYangbo Lu priv->tx_tstamp_type = config.tx_type; 220634ff6846SIoana Radulescu break; 220734ff6846SIoana Radulescu default: 220834ff6846SIoana Radulescu return -ERANGE; 220934ff6846SIoana Radulescu } 221034ff6846SIoana Radulescu 221134ff6846SIoana Radulescu if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 221234ff6846SIoana Radulescu priv->rx_tstamp = false; 221334ff6846SIoana Radulescu } else { 221434ff6846SIoana Radulescu priv->rx_tstamp = true; 221534ff6846SIoana Radulescu /* TS is set for all frame types, not only those requested */ 221634ff6846SIoana Radulescu config.rx_filter = HWTSTAMP_FILTER_ALL; 221734ff6846SIoana Radulescu } 221834ff6846SIoana Radulescu 221934ff6846SIoana Radulescu return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 222034ff6846SIoana Radulescu -EFAULT : 0; 222134ff6846SIoana Radulescu } 222234ff6846SIoana Radulescu 222334ff6846SIoana Radulescu static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 222434ff6846SIoana Radulescu { 22254a84182aSRussell King struct dpaa2_eth_priv *priv = netdev_priv(dev); 22264a84182aSRussell King 222734ff6846SIoana Radulescu if (cmd == SIOCSHWTSTAMP) 222834ff6846SIoana Radulescu return dpaa2_eth_ts_ioctl(dev, rq, cmd); 222934ff6846SIoana Radulescu 2230d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 22314a84182aSRussell King return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); 22324a84182aSRussell King 22334a84182aSRussell King return -EOPNOTSUPP; 223434ff6846SIoana Radulescu } 223534ff6846SIoana Radulescu 22367e273a8eSIoana Ciocoi Radulescu static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 22377e273a8eSIoana Ciocoi Radulescu { 22387e273a8eSIoana Ciocoi Radulescu int mfl, linear_mfl; 22397e273a8eSIoana Ciocoi Radulescu 22407e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 2241efa6a7d0SIoana Ciornei linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE - 22427b1eea1aSIoana Ciocoi Radulescu dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 22437e273a8eSIoana Ciocoi Radulescu 22447e273a8eSIoana Ciocoi Radulescu if (mfl > linear_mfl) { 22457e273a8eSIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 22467e273a8eSIoana Ciocoi Radulescu linear_mfl - VLAN_ETH_HLEN); 22477e273a8eSIoana Ciocoi Radulescu return false; 22487e273a8eSIoana Ciocoi Radulescu } 22497e273a8eSIoana Ciocoi Radulescu 22507e273a8eSIoana Ciocoi Radulescu return true; 22517e273a8eSIoana Ciocoi Radulescu } 22527e273a8eSIoana Ciocoi Radulescu 22535d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 22547e273a8eSIoana Ciocoi Radulescu { 22557e273a8eSIoana Ciocoi Radulescu int mfl, err; 22567e273a8eSIoana Ciocoi Radulescu 22577e273a8eSIoana Ciocoi Radulescu /* We enforce a maximum Rx frame length based on MTU only if we have 22587e273a8eSIoana Ciocoi Radulescu * an XDP program attached (in order to avoid Rx S/G frames). 22597e273a8eSIoana Ciocoi Radulescu * Otherwise, we accept all incoming frames as long as they are not 22607e273a8eSIoana Ciocoi Radulescu * larger than maximum size supported in hardware 22617e273a8eSIoana Ciocoi Radulescu */ 22627e273a8eSIoana Ciocoi Radulescu if (has_xdp) 22637e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 22647e273a8eSIoana Ciocoi Radulescu else 22657e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_MFL; 22667e273a8eSIoana Ciocoi Radulescu 22677e273a8eSIoana Ciocoi Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 22687e273a8eSIoana Ciocoi Radulescu if (err) { 22697e273a8eSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 22707e273a8eSIoana Ciocoi Radulescu return err; 22717e273a8eSIoana Ciocoi Radulescu } 22727e273a8eSIoana Ciocoi Radulescu 22737e273a8eSIoana Ciocoi Radulescu return 0; 22747e273a8eSIoana Ciocoi Radulescu } 22757e273a8eSIoana Ciocoi Radulescu 22767e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 22777e273a8eSIoana Ciocoi Radulescu { 22787e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 22797e273a8eSIoana Ciocoi Radulescu int err; 22807e273a8eSIoana Ciocoi Radulescu 22817e273a8eSIoana Ciocoi Radulescu if (!priv->xdp_prog) 22827e273a8eSIoana Ciocoi Radulescu goto out; 22837e273a8eSIoana Ciocoi Radulescu 22847e273a8eSIoana Ciocoi Radulescu if (!xdp_mtu_valid(priv, new_mtu)) 22857e273a8eSIoana Ciocoi Radulescu return -EINVAL; 22867e273a8eSIoana Ciocoi Radulescu 22875d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, new_mtu, true); 22887e273a8eSIoana Ciocoi Radulescu if (err) 22897e273a8eSIoana Ciocoi Radulescu return err; 22907e273a8eSIoana Ciocoi Radulescu 22917e273a8eSIoana Ciocoi Radulescu out: 22927e273a8eSIoana Ciocoi Radulescu dev->mtu = new_mtu; 22937e273a8eSIoana Ciocoi Radulescu return 0; 22947e273a8eSIoana Ciocoi Radulescu } 22957e273a8eSIoana Ciocoi Radulescu 22965d8dccf8SIoana Ciornei static int dpaa2_eth_update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 22977b1eea1aSIoana Ciocoi Radulescu { 22987b1eea1aSIoana Ciocoi Radulescu struct dpni_buffer_layout buf_layout = {0}; 22997b1eea1aSIoana Ciocoi Radulescu int err; 23007b1eea1aSIoana Ciocoi Radulescu 23017b1eea1aSIoana Ciocoi Radulescu err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 23027b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 23037b1eea1aSIoana Ciocoi Radulescu if (err) { 23047b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 23057b1eea1aSIoana Ciocoi Radulescu return err; 23067b1eea1aSIoana Ciocoi Radulescu } 23077b1eea1aSIoana Ciocoi Radulescu 23087b1eea1aSIoana Ciocoi Radulescu /* Reserve extra headroom for XDP header size changes */ 23097b1eea1aSIoana Ciocoi Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 23107b1eea1aSIoana Ciocoi Radulescu (has_xdp ? XDP_PACKET_HEADROOM : 0); 23117b1eea1aSIoana Ciocoi Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 23127b1eea1aSIoana Ciocoi Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 23137b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 23147b1eea1aSIoana Ciocoi Radulescu if (err) { 23157b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 23167b1eea1aSIoana Ciocoi Radulescu return err; 23177b1eea1aSIoana Ciocoi Radulescu } 23187b1eea1aSIoana Ciocoi Radulescu 23197b1eea1aSIoana Ciocoi Radulescu return 0; 23207b1eea1aSIoana Ciocoi Radulescu } 23217b1eea1aSIoana Ciocoi Radulescu 23225d8dccf8SIoana Ciornei static int dpaa2_eth_setup_xdp(struct net_device *dev, struct bpf_prog *prog) 23237e273a8eSIoana Ciocoi Radulescu { 23247e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 23257e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch; 23267e273a8eSIoana Ciocoi Radulescu struct bpf_prog *old; 23277e273a8eSIoana Ciocoi Radulescu bool up, need_update; 23287e273a8eSIoana Ciocoi Radulescu int i, err; 23297e273a8eSIoana Ciocoi Radulescu 23307e273a8eSIoana Ciocoi Radulescu if (prog && !xdp_mtu_valid(priv, dev->mtu)) 23317e273a8eSIoana Ciocoi Radulescu return -EINVAL; 23327e273a8eSIoana Ciocoi Radulescu 233385192dbfSAndrii Nakryiko if (prog) 233485192dbfSAndrii Nakryiko bpf_prog_add(prog, priv->num_channels); 23357e273a8eSIoana Ciocoi Radulescu 23367e273a8eSIoana Ciocoi Radulescu up = netif_running(dev); 23377e273a8eSIoana Ciocoi Radulescu need_update = (!!priv->xdp_prog != !!prog); 23387e273a8eSIoana Ciocoi Radulescu 23397e273a8eSIoana Ciocoi Radulescu if (up) 23407e273a8eSIoana Ciocoi Radulescu dpaa2_eth_stop(dev); 23417e273a8eSIoana Ciocoi Radulescu 23427b1eea1aSIoana Ciocoi Radulescu /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 23437b1eea1aSIoana Ciocoi Radulescu * Also, when switching between xdp/non-xdp modes we need to reconfigure 23447b1eea1aSIoana Ciocoi Radulescu * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 23457b1eea1aSIoana Ciocoi Radulescu * so we are sure no old format buffers will be used from now on. 23467b1eea1aSIoana Ciocoi Radulescu */ 23477e273a8eSIoana Ciocoi Radulescu if (need_update) { 23485d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, dev->mtu, !!prog); 23497e273a8eSIoana Ciocoi Radulescu if (err) 23507e273a8eSIoana Ciocoi Radulescu goto out_err; 23515d8dccf8SIoana Ciornei err = dpaa2_eth_update_rx_buffer_headroom(priv, !!prog); 23527b1eea1aSIoana Ciocoi Radulescu if (err) 23537b1eea1aSIoana Ciocoi Radulescu goto out_err; 23547e273a8eSIoana Ciocoi Radulescu } 23557e273a8eSIoana Ciocoi Radulescu 23567e273a8eSIoana Ciocoi Radulescu old = xchg(&priv->xdp_prog, prog); 23577e273a8eSIoana Ciocoi Radulescu if (old) 23587e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 23597e273a8eSIoana Ciocoi Radulescu 23607e273a8eSIoana Ciocoi Radulescu for (i = 0; i < priv->num_channels; i++) { 23617e273a8eSIoana Ciocoi Radulescu ch = priv->channel[i]; 23627e273a8eSIoana Ciocoi Radulescu old = xchg(&ch->xdp.prog, prog); 23637e273a8eSIoana Ciocoi Radulescu if (old) 23647e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 23657e273a8eSIoana Ciocoi Radulescu } 23667e273a8eSIoana Ciocoi Radulescu 23677e273a8eSIoana Ciocoi Radulescu if (up) { 23687e273a8eSIoana Ciocoi Radulescu err = dpaa2_eth_open(dev); 23697e273a8eSIoana Ciocoi Radulescu if (err) 23707e273a8eSIoana Ciocoi Radulescu return err; 23717e273a8eSIoana Ciocoi Radulescu } 23727e273a8eSIoana Ciocoi Radulescu 23737e273a8eSIoana Ciocoi Radulescu return 0; 23747e273a8eSIoana Ciocoi Radulescu 23757e273a8eSIoana Ciocoi Radulescu out_err: 23767e273a8eSIoana Ciocoi Radulescu if (prog) 23777e273a8eSIoana Ciocoi Radulescu bpf_prog_sub(prog, priv->num_channels); 23787e273a8eSIoana Ciocoi Radulescu if (up) 23797e273a8eSIoana Ciocoi Radulescu dpaa2_eth_open(dev); 23807e273a8eSIoana Ciocoi Radulescu 23817e273a8eSIoana Ciocoi Radulescu return err; 23827e273a8eSIoana Ciocoi Radulescu } 23837e273a8eSIoana Ciocoi Radulescu 23847e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 23857e273a8eSIoana Ciocoi Radulescu { 23867e273a8eSIoana Ciocoi Radulescu switch (xdp->command) { 23877e273a8eSIoana Ciocoi Radulescu case XDP_SETUP_PROG: 23885d8dccf8SIoana Ciornei return dpaa2_eth_setup_xdp(dev, xdp->prog); 23897e273a8eSIoana Ciocoi Radulescu default: 23907e273a8eSIoana Ciocoi Radulescu return -EINVAL; 23917e273a8eSIoana Ciocoi Radulescu } 23927e273a8eSIoana Ciocoi Radulescu 23937e273a8eSIoana Ciocoi Radulescu return 0; 23947e273a8eSIoana Ciocoi Radulescu } 23957e273a8eSIoana Ciocoi Radulescu 23966aa40b9eSIoana Ciornei static int dpaa2_eth_xdp_create_fd(struct net_device *net_dev, 23976aa40b9eSIoana Ciornei struct xdp_frame *xdpf, 23986aa40b9eSIoana Ciornei struct dpaa2_fd *fd) 2399d678be1dSIoana Radulescu { 2400d678be1dSIoana Radulescu struct device *dev = net_dev->dev.parent; 2401d678be1dSIoana Radulescu unsigned int needed_headroom; 2402d678be1dSIoana Radulescu struct dpaa2_eth_swa *swa; 2403d678be1dSIoana Radulescu void *buffer_start, *aligned_start; 2404d678be1dSIoana Radulescu dma_addr_t addr; 2405d678be1dSIoana Radulescu 2406d678be1dSIoana Radulescu /* We require a minimum headroom to be able to transmit the frame. 2407d678be1dSIoana Radulescu * Otherwise return an error and let the original net_device handle it 2408d678be1dSIoana Radulescu */ 24091cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(NULL); 2410d678be1dSIoana Radulescu if (xdpf->headroom < needed_headroom) 2411d678be1dSIoana Radulescu return -EINVAL; 2412d678be1dSIoana Radulescu 2413d678be1dSIoana Radulescu /* Setup the FD fields */ 24146aa40b9eSIoana Ciornei memset(fd, 0, sizeof(*fd)); 2415d678be1dSIoana Radulescu 2416d678be1dSIoana Radulescu /* Align FD address, if possible */ 2417d678be1dSIoana Radulescu buffer_start = xdpf->data - needed_headroom; 2418d678be1dSIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 2419d678be1dSIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 2420d678be1dSIoana Radulescu if (aligned_start >= xdpf->data - xdpf->headroom) 2421d678be1dSIoana Radulescu buffer_start = aligned_start; 2422d678be1dSIoana Radulescu 2423d678be1dSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 2424d678be1dSIoana Radulescu /* fill in necessary fields here */ 2425d678be1dSIoana Radulescu swa->type = DPAA2_ETH_SWA_XDP; 2426d678be1dSIoana Radulescu swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 2427d678be1dSIoana Radulescu swa->xdp.xdpf = xdpf; 2428d678be1dSIoana Radulescu 2429d678be1dSIoana Radulescu addr = dma_map_single(dev, buffer_start, 2430d678be1dSIoana Radulescu swa->xdp.dma_size, 2431d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 24326aa40b9eSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) 2433d678be1dSIoana Radulescu return -ENOMEM; 2434d678be1dSIoana Radulescu 24356aa40b9eSIoana Ciornei dpaa2_fd_set_addr(fd, addr); 24366aa40b9eSIoana Ciornei dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); 24376aa40b9eSIoana Ciornei dpaa2_fd_set_len(fd, xdpf->len); 24386aa40b9eSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_single); 24396aa40b9eSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 2440d678be1dSIoana Radulescu 2441d678be1dSIoana Radulescu return 0; 2442d678be1dSIoana Radulescu } 2443d678be1dSIoana Radulescu 2444d678be1dSIoana Radulescu static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 2445d678be1dSIoana Radulescu struct xdp_frame **frames, u32 flags) 2446d678be1dSIoana Radulescu { 24476aa40b9eSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 244838c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_redirect_fds; 24496aa40b9eSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 24506aa40b9eSIoana Ciornei struct dpaa2_eth_fq *fq; 24518665d978SIoana Ciornei struct dpaa2_fd *fds; 245238c440b2SIoana Ciornei int enqueued, i, err; 2453d678be1dSIoana Radulescu 2454d678be1dSIoana Radulescu if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2455d678be1dSIoana Radulescu return -EINVAL; 2456d678be1dSIoana Radulescu 2457d678be1dSIoana Radulescu if (!netif_running(net_dev)) 2458d678be1dSIoana Radulescu return -ENETDOWN; 2459d678be1dSIoana Radulescu 24608665d978SIoana Ciornei fq = &priv->fq[smp_processor_id()]; 246138c440b2SIoana Ciornei xdp_redirect_fds = &fq->xdp_redirect_fds; 246238c440b2SIoana Ciornei fds = xdp_redirect_fds->fds; 24638665d978SIoana Ciornei 24646aa40b9eSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 24656aa40b9eSIoana Ciornei 24668665d978SIoana Ciornei /* create a FD for each xdp_frame in the list received */ 2467d678be1dSIoana Radulescu for (i = 0; i < n; i++) { 24688665d978SIoana Ciornei err = dpaa2_eth_xdp_create_fd(net_dev, frames[i], &fds[i]); 24698665d978SIoana Ciornei if (err) 24706aa40b9eSIoana Ciornei break; 24716aa40b9eSIoana Ciornei } 247238c440b2SIoana Ciornei xdp_redirect_fds->num = i; 24736aa40b9eSIoana Ciornei 247438c440b2SIoana Ciornei /* enqueue all the frame descriptors */ 247538c440b2SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, xdp_redirect_fds); 2476d678be1dSIoana Radulescu 24778665d978SIoana Ciornei /* update statistics */ 247838c440b2SIoana Ciornei percpu_stats->tx_packets += enqueued; 247938c440b2SIoana Ciornei for (i = 0; i < enqueued; i++) 24808665d978SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 24818665d978SIoana Ciornei 248238c440b2SIoana Ciornei return enqueued; 2483d678be1dSIoana Radulescu } 2484d678be1dSIoana Radulescu 248506d5b179SIoana Radulescu static int update_xps(struct dpaa2_eth_priv *priv) 248606d5b179SIoana Radulescu { 248706d5b179SIoana Radulescu struct net_device *net_dev = priv->net_dev; 248806d5b179SIoana Radulescu struct cpumask xps_mask; 248906d5b179SIoana Radulescu struct dpaa2_eth_fq *fq; 2490ab1e6de2SIoana Radulescu int i, num_queues, netdev_queues; 249106d5b179SIoana Radulescu int err = 0; 249206d5b179SIoana Radulescu 249306d5b179SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2494ab1e6de2SIoana Radulescu netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 249506d5b179SIoana Radulescu 249606d5b179SIoana Radulescu /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 249706d5b179SIoana Radulescu * queues, so only process those 249806d5b179SIoana Radulescu */ 2499ab1e6de2SIoana Radulescu for (i = 0; i < netdev_queues; i++) { 2500ab1e6de2SIoana Radulescu fq = &priv->fq[i % num_queues]; 250106d5b179SIoana Radulescu 250206d5b179SIoana Radulescu cpumask_clear(&xps_mask); 250306d5b179SIoana Radulescu cpumask_set_cpu(fq->target_cpu, &xps_mask); 250406d5b179SIoana Radulescu 250506d5b179SIoana Radulescu err = netif_set_xps_queue(net_dev, &xps_mask, i); 250606d5b179SIoana Radulescu if (err) { 250706d5b179SIoana Radulescu netdev_warn_once(net_dev, "Error setting XPS queue\n"); 250806d5b179SIoana Radulescu break; 250906d5b179SIoana Radulescu } 251006d5b179SIoana Radulescu } 251106d5b179SIoana Radulescu 251206d5b179SIoana Radulescu return err; 251306d5b179SIoana Radulescu } 251406d5b179SIoana Radulescu 2515e3ec13beSIoana Ciornei static int dpaa2_eth_setup_mqprio(struct net_device *net_dev, 2516e3ec13beSIoana Ciornei struct tc_mqprio_qopt *mqprio) 2517ab1e6de2SIoana Radulescu { 2518ab1e6de2SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2519ab1e6de2SIoana Radulescu u8 num_tc, num_queues; 2520ab1e6de2SIoana Radulescu int i; 2521ab1e6de2SIoana Radulescu 2522ab1e6de2SIoana Radulescu mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2523ab1e6de2SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2524ab1e6de2SIoana Radulescu num_tc = mqprio->num_tc; 2525ab1e6de2SIoana Radulescu 2526ab1e6de2SIoana Radulescu if (num_tc == net_dev->num_tc) 2527ab1e6de2SIoana Radulescu return 0; 2528ab1e6de2SIoana Radulescu 2529ab1e6de2SIoana Radulescu if (num_tc > dpaa2_eth_tc_count(priv)) { 2530ab1e6de2SIoana Radulescu netdev_err(net_dev, "Max %d traffic classes supported\n", 2531ab1e6de2SIoana Radulescu dpaa2_eth_tc_count(priv)); 2532b89c1e6bSJesper Dangaard Brouer return -EOPNOTSUPP; 2533ab1e6de2SIoana Radulescu } 2534ab1e6de2SIoana Radulescu 2535ab1e6de2SIoana Radulescu if (!num_tc) { 2536ab1e6de2SIoana Radulescu netdev_reset_tc(net_dev); 2537ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_queues); 2538ab1e6de2SIoana Radulescu goto out; 2539ab1e6de2SIoana Radulescu } 2540ab1e6de2SIoana Radulescu 2541ab1e6de2SIoana Radulescu netdev_set_num_tc(net_dev, num_tc); 2542ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2543ab1e6de2SIoana Radulescu 2544ab1e6de2SIoana Radulescu for (i = 0; i < num_tc; i++) 2545ab1e6de2SIoana Radulescu netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2546ab1e6de2SIoana Radulescu 2547ab1e6de2SIoana Radulescu out: 2548ab1e6de2SIoana Radulescu update_xps(priv); 2549ab1e6de2SIoana Radulescu 2550ab1e6de2SIoana Radulescu return 0; 2551ab1e6de2SIoana Radulescu } 2552ab1e6de2SIoana Radulescu 25533657cdafSIoana Ciornei #define bps_to_mbits(rate) (div_u64((rate), 1000000) * 8) 25543657cdafSIoana Ciornei 25553657cdafSIoana Ciornei static int dpaa2_eth_setup_tbf(struct net_device *net_dev, struct tc_tbf_qopt_offload *p) 25563657cdafSIoana Ciornei { 25573657cdafSIoana Ciornei struct tc_tbf_qopt_offload_replace_params *cfg = &p->replace_params; 25583657cdafSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 25593657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_cr_shaper = { 0 }; 25603657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_er_shaper = { 0 }; 25613657cdafSIoana Ciornei int err; 25623657cdafSIoana Ciornei 25633657cdafSIoana Ciornei if (p->command == TC_TBF_STATS) 25643657cdafSIoana Ciornei return -EOPNOTSUPP; 25653657cdafSIoana Ciornei 25663657cdafSIoana Ciornei /* Only per port Tx shaping */ 25673657cdafSIoana Ciornei if (p->parent != TC_H_ROOT) 25683657cdafSIoana Ciornei return -EOPNOTSUPP; 25693657cdafSIoana Ciornei 25703657cdafSIoana Ciornei if (p->command == TC_TBF_REPLACE) { 25713657cdafSIoana Ciornei if (cfg->max_size > DPAA2_ETH_MAX_BURST_SIZE) { 25723657cdafSIoana Ciornei netdev_err(net_dev, "burst size cannot be greater than %d\n", 25733657cdafSIoana Ciornei DPAA2_ETH_MAX_BURST_SIZE); 25743657cdafSIoana Ciornei return -EINVAL; 25753657cdafSIoana Ciornei } 25763657cdafSIoana Ciornei 25773657cdafSIoana Ciornei tx_cr_shaper.max_burst_size = cfg->max_size; 25783657cdafSIoana Ciornei /* The TBF interface is in bytes/s, whereas DPAA2 expects the 25793657cdafSIoana Ciornei * rate in Mbits/s 25803657cdafSIoana Ciornei */ 25813657cdafSIoana Ciornei tx_cr_shaper.rate_limit = bps_to_mbits(cfg->rate.rate_bytes_ps); 25823657cdafSIoana Ciornei } 25833657cdafSIoana Ciornei 25843657cdafSIoana Ciornei err = dpni_set_tx_shaping(priv->mc_io, 0, priv->mc_token, &tx_cr_shaper, 25853657cdafSIoana Ciornei &tx_er_shaper, 0); 25863657cdafSIoana Ciornei if (err) { 25873657cdafSIoana Ciornei netdev_err(net_dev, "dpni_set_tx_shaping() = %d\n", err); 25883657cdafSIoana Ciornei return err; 25893657cdafSIoana Ciornei } 25903657cdafSIoana Ciornei 25913657cdafSIoana Ciornei return 0; 25923657cdafSIoana Ciornei } 25933657cdafSIoana Ciornei 2594e3ec13beSIoana Ciornei static int dpaa2_eth_setup_tc(struct net_device *net_dev, 2595e3ec13beSIoana Ciornei enum tc_setup_type type, void *type_data) 2596e3ec13beSIoana Ciornei { 2597e3ec13beSIoana Ciornei switch (type) { 2598e3ec13beSIoana Ciornei case TC_SETUP_QDISC_MQPRIO: 2599e3ec13beSIoana Ciornei return dpaa2_eth_setup_mqprio(net_dev, type_data); 26003657cdafSIoana Ciornei case TC_SETUP_QDISC_TBF: 26013657cdafSIoana Ciornei return dpaa2_eth_setup_tbf(net_dev, type_data); 2602e3ec13beSIoana Ciornei default: 2603e3ec13beSIoana Ciornei return -EOPNOTSUPP; 2604e3ec13beSIoana Ciornei } 2605e3ec13beSIoana Ciornei } 2606e3ec13beSIoana Ciornei 260734ff6846SIoana Radulescu static const struct net_device_ops dpaa2_eth_ops = { 260834ff6846SIoana Radulescu .ndo_open = dpaa2_eth_open, 260934ff6846SIoana Radulescu .ndo_start_xmit = dpaa2_eth_tx, 261034ff6846SIoana Radulescu .ndo_stop = dpaa2_eth_stop, 261134ff6846SIoana Radulescu .ndo_set_mac_address = dpaa2_eth_set_addr, 261234ff6846SIoana Radulescu .ndo_get_stats64 = dpaa2_eth_get_stats, 261334ff6846SIoana Radulescu .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 261434ff6846SIoana Radulescu .ndo_set_features = dpaa2_eth_set_features, 2615a7605370SArnd Bergmann .ndo_eth_ioctl = dpaa2_eth_ioctl, 26167e273a8eSIoana Ciocoi Radulescu .ndo_change_mtu = dpaa2_eth_change_mtu, 26177e273a8eSIoana Ciocoi Radulescu .ndo_bpf = dpaa2_eth_xdp, 2618d678be1dSIoana Radulescu .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2619ab1e6de2SIoana Radulescu .ndo_setup_tc = dpaa2_eth_setup_tc, 262070b32d82SIonut-robert Aron .ndo_vlan_rx_add_vid = dpaa2_eth_rx_add_vid, 262170b32d82SIonut-robert Aron .ndo_vlan_rx_kill_vid = dpaa2_eth_rx_kill_vid 262234ff6846SIoana Radulescu }; 262334ff6846SIoana Radulescu 26245d8dccf8SIoana Ciornei static void dpaa2_eth_cdan_cb(struct dpaa2_io_notification_ctx *ctx) 262534ff6846SIoana Radulescu { 262634ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 262734ff6846SIoana Radulescu 262834ff6846SIoana Radulescu ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 262934ff6846SIoana Radulescu 263034ff6846SIoana Radulescu /* Update NAPI statistics */ 263134ff6846SIoana Radulescu ch->stats.cdan++; 263234ff6846SIoana Radulescu 26336c33ae1aSJiafei Pan napi_schedule(&ch->napi); 263434ff6846SIoana Radulescu } 263534ff6846SIoana Radulescu 263634ff6846SIoana Radulescu /* Allocate and configure a DPCON object */ 26375d8dccf8SIoana Ciornei static struct fsl_mc_device *dpaa2_eth_setup_dpcon(struct dpaa2_eth_priv *priv) 263834ff6846SIoana Radulescu { 263934ff6846SIoana Radulescu struct fsl_mc_device *dpcon; 264034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 264134ff6846SIoana Radulescu int err; 264234ff6846SIoana Radulescu 264334ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 264434ff6846SIoana Radulescu FSL_MC_POOL_DPCON, &dpcon); 264534ff6846SIoana Radulescu if (err) { 2646d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2647d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2648d7f5a9d8SIoana Ciornei else 264934ff6846SIoana Radulescu dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2650d7f5a9d8SIoana Ciornei return ERR_PTR(err); 265134ff6846SIoana Radulescu } 265234ff6846SIoana Radulescu 265334ff6846SIoana Radulescu err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 265434ff6846SIoana Radulescu if (err) { 265534ff6846SIoana Radulescu dev_err(dev, "dpcon_open() failed\n"); 265634ff6846SIoana Radulescu goto free; 265734ff6846SIoana Radulescu } 265834ff6846SIoana Radulescu 265934ff6846SIoana Radulescu err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 266034ff6846SIoana Radulescu if (err) { 266134ff6846SIoana Radulescu dev_err(dev, "dpcon_reset() failed\n"); 266234ff6846SIoana Radulescu goto close; 266334ff6846SIoana Radulescu } 266434ff6846SIoana Radulescu 266534ff6846SIoana Radulescu err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 266634ff6846SIoana Radulescu if (err) { 266734ff6846SIoana Radulescu dev_err(dev, "dpcon_enable() failed\n"); 266834ff6846SIoana Radulescu goto close; 266934ff6846SIoana Radulescu } 267034ff6846SIoana Radulescu 267134ff6846SIoana Radulescu return dpcon; 267234ff6846SIoana Radulescu 267334ff6846SIoana Radulescu close: 267434ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 267534ff6846SIoana Radulescu free: 267634ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 267734ff6846SIoana Radulescu 267802afa9c6SYueHaibing return ERR_PTR(err); 267934ff6846SIoana Radulescu } 268034ff6846SIoana Radulescu 26815d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpcon(struct dpaa2_eth_priv *priv, 268234ff6846SIoana Radulescu struct fsl_mc_device *dpcon) 268334ff6846SIoana Radulescu { 268434ff6846SIoana Radulescu dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 268534ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 268634ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 268734ff6846SIoana Radulescu } 268834ff6846SIoana Radulescu 26895d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_alloc_channel(struct dpaa2_eth_priv *priv) 269034ff6846SIoana Radulescu { 269134ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 269234ff6846SIoana Radulescu struct dpcon_attr attr; 269334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 269434ff6846SIoana Radulescu int err; 269534ff6846SIoana Radulescu 269634ff6846SIoana Radulescu channel = kzalloc(sizeof(*channel), GFP_KERNEL); 269734ff6846SIoana Radulescu if (!channel) 269834ff6846SIoana Radulescu return NULL; 269934ff6846SIoana Radulescu 27005d8dccf8SIoana Ciornei channel->dpcon = dpaa2_eth_setup_dpcon(priv); 270102afa9c6SYueHaibing if (IS_ERR(channel->dpcon)) { 270202afa9c6SYueHaibing err = PTR_ERR(channel->dpcon); 270334ff6846SIoana Radulescu goto err_setup; 2704d7f5a9d8SIoana Ciornei } 270534ff6846SIoana Radulescu 270634ff6846SIoana Radulescu err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 270734ff6846SIoana Radulescu &attr); 270834ff6846SIoana Radulescu if (err) { 270934ff6846SIoana Radulescu dev_err(dev, "dpcon_get_attributes() failed\n"); 271034ff6846SIoana Radulescu goto err_get_attr; 271134ff6846SIoana Radulescu } 271234ff6846SIoana Radulescu 271334ff6846SIoana Radulescu channel->dpcon_id = attr.id; 271434ff6846SIoana Radulescu channel->ch_id = attr.qbman_ch_id; 271534ff6846SIoana Radulescu channel->priv = priv; 271634ff6846SIoana Radulescu 271734ff6846SIoana Radulescu return channel; 271834ff6846SIoana Radulescu 271934ff6846SIoana Radulescu err_get_attr: 27205d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 272134ff6846SIoana Radulescu err_setup: 272234ff6846SIoana Radulescu kfree(channel); 2723d7f5a9d8SIoana Ciornei return ERR_PTR(err); 272434ff6846SIoana Radulescu } 272534ff6846SIoana Radulescu 27265d8dccf8SIoana Ciornei static void dpaa2_eth_free_channel(struct dpaa2_eth_priv *priv, 272734ff6846SIoana Radulescu struct dpaa2_eth_channel *channel) 272834ff6846SIoana Radulescu { 27295d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 273034ff6846SIoana Radulescu kfree(channel); 273134ff6846SIoana Radulescu } 273234ff6846SIoana Radulescu 273334ff6846SIoana Radulescu /* DPIO setup: allocate and configure QBMan channels, setup core affinity 273434ff6846SIoana Radulescu * and register data availability notifications 273534ff6846SIoana Radulescu */ 27365d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpio(struct dpaa2_eth_priv *priv) 273734ff6846SIoana Radulescu { 273834ff6846SIoana Radulescu struct dpaa2_io_notification_ctx *nctx; 273934ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 274034ff6846SIoana Radulescu struct dpcon_notification_cfg dpcon_notif_cfg; 274134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 274234ff6846SIoana Radulescu int i, err; 274334ff6846SIoana Radulescu 274434ff6846SIoana Radulescu /* We want the ability to spread ingress traffic (RX, TX conf) to as 274534ff6846SIoana Radulescu * many cores as possible, so we need one channel for each core 274634ff6846SIoana Radulescu * (unless there's fewer queues than cores, in which case the extra 274734ff6846SIoana Radulescu * channels would be wasted). 274834ff6846SIoana Radulescu * Allocate one channel per core and register it to the core's 274934ff6846SIoana Radulescu * affine DPIO. If not enough channels are available for all cores 275034ff6846SIoana Radulescu * or if some cores don't have an affine DPIO, there will be no 275134ff6846SIoana Radulescu * ingress frame processing on those cores. 275234ff6846SIoana Radulescu */ 275334ff6846SIoana Radulescu cpumask_clear(&priv->dpio_cpumask); 275434ff6846SIoana Radulescu for_each_online_cpu(i) { 275534ff6846SIoana Radulescu /* Try to allocate a channel */ 27565d8dccf8SIoana Ciornei channel = dpaa2_eth_alloc_channel(priv); 2757d7f5a9d8SIoana Ciornei if (IS_ERR_OR_NULL(channel)) { 2758bd8460faSIoana Radulescu err = PTR_ERR_OR_ZERO(channel); 2759d7f5a9d8SIoana Ciornei if (err != -EPROBE_DEFER) 276034ff6846SIoana Radulescu dev_info(dev, 276134ff6846SIoana Radulescu "No affine channel for cpu %d and above\n", i); 276234ff6846SIoana Radulescu goto err_alloc_ch; 276334ff6846SIoana Radulescu } 276434ff6846SIoana Radulescu 276534ff6846SIoana Radulescu priv->channel[priv->num_channels] = channel; 276634ff6846SIoana Radulescu 276734ff6846SIoana Radulescu nctx = &channel->nctx; 276834ff6846SIoana Radulescu nctx->is_cdan = 1; 27695d8dccf8SIoana Ciornei nctx->cb = dpaa2_eth_cdan_cb; 277034ff6846SIoana Radulescu nctx->id = channel->ch_id; 277134ff6846SIoana Radulescu nctx->desired_cpu = i; 277234ff6846SIoana Radulescu 277334ff6846SIoana Radulescu /* Register the new context */ 277434ff6846SIoana Radulescu channel->dpio = dpaa2_io_service_select(i); 277547441f7fSIoana Ciornei err = dpaa2_io_service_register(channel->dpio, nctx, dev); 277634ff6846SIoana Radulescu if (err) { 277734ff6846SIoana Radulescu dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 277834ff6846SIoana Radulescu /* If no affine DPIO for this core, there's probably 277934ff6846SIoana Radulescu * none available for next cores either. Signal we want 278034ff6846SIoana Radulescu * to retry later, in case the DPIO devices weren't 278134ff6846SIoana Radulescu * probed yet. 278234ff6846SIoana Radulescu */ 278334ff6846SIoana Radulescu err = -EPROBE_DEFER; 278434ff6846SIoana Radulescu goto err_service_reg; 278534ff6846SIoana Radulescu } 278634ff6846SIoana Radulescu 278734ff6846SIoana Radulescu /* Register DPCON notification with MC */ 278834ff6846SIoana Radulescu dpcon_notif_cfg.dpio_id = nctx->dpio_id; 278934ff6846SIoana Radulescu dpcon_notif_cfg.priority = 0; 279034ff6846SIoana Radulescu dpcon_notif_cfg.user_ctx = nctx->qman64; 279134ff6846SIoana Radulescu err = dpcon_set_notification(priv->mc_io, 0, 279234ff6846SIoana Radulescu channel->dpcon->mc_handle, 279334ff6846SIoana Radulescu &dpcon_notif_cfg); 279434ff6846SIoana Radulescu if (err) { 279534ff6846SIoana Radulescu dev_err(dev, "dpcon_set_notification failed()\n"); 279634ff6846SIoana Radulescu goto err_set_cdan; 279734ff6846SIoana Radulescu } 279834ff6846SIoana Radulescu 279934ff6846SIoana Radulescu /* If we managed to allocate a channel and also found an affine 280034ff6846SIoana Radulescu * DPIO for this core, add it to the final mask 280134ff6846SIoana Radulescu */ 280234ff6846SIoana Radulescu cpumask_set_cpu(i, &priv->dpio_cpumask); 280334ff6846SIoana Radulescu priv->num_channels++; 280434ff6846SIoana Radulescu 280534ff6846SIoana Radulescu /* Stop if we already have enough channels to accommodate all 280634ff6846SIoana Radulescu * RX and TX conf queues 280734ff6846SIoana Radulescu */ 2808b0e4f37bSIoana Ciocoi Radulescu if (priv->num_channels == priv->dpni_attrs.num_queues) 280934ff6846SIoana Radulescu break; 281034ff6846SIoana Radulescu } 281134ff6846SIoana Radulescu 281234ff6846SIoana Radulescu return 0; 281334ff6846SIoana Radulescu 281434ff6846SIoana Radulescu err_set_cdan: 281547441f7fSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 281634ff6846SIoana Radulescu err_service_reg: 28175d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 281834ff6846SIoana Radulescu err_alloc_ch: 28195aa4277dSIoana Ciornei if (err == -EPROBE_DEFER) { 28205aa4277dSIoana Ciornei for (i = 0; i < priv->num_channels; i++) { 28215aa4277dSIoana Ciornei channel = priv->channel[i]; 28225aa4277dSIoana Ciornei nctx = &channel->nctx; 28235aa4277dSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 28245d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 28255aa4277dSIoana Ciornei } 28265aa4277dSIoana Ciornei priv->num_channels = 0; 2827d7f5a9d8SIoana Ciornei return err; 28285aa4277dSIoana Ciornei } 2829d7f5a9d8SIoana Ciornei 283034ff6846SIoana Radulescu if (cpumask_empty(&priv->dpio_cpumask)) { 283134ff6846SIoana Radulescu dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 2832d7f5a9d8SIoana Ciornei return -ENODEV; 283334ff6846SIoana Radulescu } 283434ff6846SIoana Radulescu 283534ff6846SIoana Radulescu dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 283634ff6846SIoana Radulescu cpumask_pr_args(&priv->dpio_cpumask)); 283734ff6846SIoana Radulescu 283834ff6846SIoana Radulescu return 0; 283934ff6846SIoana Radulescu } 284034ff6846SIoana Radulescu 28415d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpio(struct dpaa2_eth_priv *priv) 284234ff6846SIoana Radulescu { 284347441f7fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 284434ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 284547441f7fSIoana Ciornei int i; 284634ff6846SIoana Radulescu 284734ff6846SIoana Radulescu /* deregister CDAN notifications and free channels */ 284834ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 284934ff6846SIoana Radulescu ch = priv->channel[i]; 285047441f7fSIoana Ciornei dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 28515d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, ch); 285234ff6846SIoana Radulescu } 285334ff6846SIoana Radulescu } 285434ff6846SIoana Radulescu 28555d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_get_affine_channel(struct dpaa2_eth_priv *priv, 285634ff6846SIoana Radulescu int cpu) 285734ff6846SIoana Radulescu { 285834ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 285934ff6846SIoana Radulescu int i; 286034ff6846SIoana Radulescu 286134ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 286234ff6846SIoana Radulescu if (priv->channel[i]->nctx.desired_cpu == cpu) 286334ff6846SIoana Radulescu return priv->channel[i]; 286434ff6846SIoana Radulescu 286534ff6846SIoana Radulescu /* We should never get here. Issue a warning and return 286634ff6846SIoana Radulescu * the first channel, because it's still better than nothing 286734ff6846SIoana Radulescu */ 286834ff6846SIoana Radulescu dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 286934ff6846SIoana Radulescu 287034ff6846SIoana Radulescu return priv->channel[0]; 287134ff6846SIoana Radulescu } 287234ff6846SIoana Radulescu 28735d8dccf8SIoana Ciornei static void dpaa2_eth_set_fq_affinity(struct dpaa2_eth_priv *priv) 287434ff6846SIoana Radulescu { 287534ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 287634ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 287734ff6846SIoana Radulescu int rx_cpu, txc_cpu; 287806d5b179SIoana Radulescu int i; 287934ff6846SIoana Radulescu 288034ff6846SIoana Radulescu /* For each FQ, pick one channel/CPU to deliver frames to. 288134ff6846SIoana Radulescu * This may well change at runtime, either through irqbalance or 288234ff6846SIoana Radulescu * through direct user intervention. 288334ff6846SIoana Radulescu */ 288434ff6846SIoana Radulescu rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 288534ff6846SIoana Radulescu 288634ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 288734ff6846SIoana Radulescu fq = &priv->fq[i]; 288834ff6846SIoana Radulescu switch (fq->type) { 288934ff6846SIoana Radulescu case DPAA2_RX_FQ: 2890061d631fSIoana Ciornei case DPAA2_RX_ERR_FQ: 289134ff6846SIoana Radulescu fq->target_cpu = rx_cpu; 289234ff6846SIoana Radulescu rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 289334ff6846SIoana Radulescu if (rx_cpu >= nr_cpu_ids) 289434ff6846SIoana Radulescu rx_cpu = cpumask_first(&priv->dpio_cpumask); 289534ff6846SIoana Radulescu break; 289634ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 289734ff6846SIoana Radulescu fq->target_cpu = txc_cpu; 289834ff6846SIoana Radulescu txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 289934ff6846SIoana Radulescu if (txc_cpu >= nr_cpu_ids) 290034ff6846SIoana Radulescu txc_cpu = cpumask_first(&priv->dpio_cpumask); 290134ff6846SIoana Radulescu break; 290234ff6846SIoana Radulescu default: 290334ff6846SIoana Radulescu dev_err(dev, "Unknown FQ type: %d\n", fq->type); 290434ff6846SIoana Radulescu } 29055d8dccf8SIoana Ciornei fq->channel = dpaa2_eth_get_affine_channel(priv, fq->target_cpu); 290634ff6846SIoana Radulescu } 290706d5b179SIoana Radulescu 290806d5b179SIoana Radulescu update_xps(priv); 290934ff6846SIoana Radulescu } 291034ff6846SIoana Radulescu 29115d8dccf8SIoana Ciornei static void dpaa2_eth_setup_fqs(struct dpaa2_eth_priv *priv) 291234ff6846SIoana Radulescu { 2913685e39eaSIoana Radulescu int i, j; 291434ff6846SIoana Radulescu 291534ff6846SIoana Radulescu /* We have one TxConf FQ per Tx flow. 291634ff6846SIoana Radulescu * The number of Tx and Rx queues is the same. 291734ff6846SIoana Radulescu * Tx queues come first in the fq array. 291834ff6846SIoana Radulescu */ 291934ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 292034ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 292134ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 292234ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 292334ff6846SIoana Radulescu } 292434ff6846SIoana Radulescu 2925685e39eaSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 292634ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 292734ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 292834ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 2929685e39eaSIoana Radulescu priv->fq[priv->num_fqs].tc = (u8)j; 293034ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 293134ff6846SIoana Radulescu } 2932685e39eaSIoana Radulescu } 293334ff6846SIoana Radulescu 2934061d631fSIoana Ciornei /* We have exactly one Rx error queue per DPNI */ 2935061d631fSIoana Ciornei priv->fq[priv->num_fqs].type = DPAA2_RX_ERR_FQ; 2936061d631fSIoana Ciornei priv->fq[priv->num_fqs++].consume = dpaa2_eth_rx_err; 2937061d631fSIoana Ciornei 293834ff6846SIoana Radulescu /* For each FQ, decide on which core to process incoming frames */ 29395d8dccf8SIoana Ciornei dpaa2_eth_set_fq_affinity(priv); 294034ff6846SIoana Radulescu } 294134ff6846SIoana Radulescu 294234ff6846SIoana Radulescu /* Allocate and configure one buffer pool for each interface */ 29435d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpbp(struct dpaa2_eth_priv *priv) 294434ff6846SIoana Radulescu { 294534ff6846SIoana Radulescu int err; 294634ff6846SIoana Radulescu struct fsl_mc_device *dpbp_dev; 294734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 294834ff6846SIoana Radulescu struct dpbp_attr dpbp_attrs; 294934ff6846SIoana Radulescu 295034ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 295134ff6846SIoana Radulescu &dpbp_dev); 295234ff6846SIoana Radulescu if (err) { 2953d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2954d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2955d7f5a9d8SIoana Ciornei else 295634ff6846SIoana Radulescu dev_err(dev, "DPBP device allocation failed\n"); 295734ff6846SIoana Radulescu return err; 295834ff6846SIoana Radulescu } 295934ff6846SIoana Radulescu 296034ff6846SIoana Radulescu priv->dpbp_dev = dpbp_dev; 296134ff6846SIoana Radulescu 296234ff6846SIoana Radulescu err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 296334ff6846SIoana Radulescu &dpbp_dev->mc_handle); 296434ff6846SIoana Radulescu if (err) { 296534ff6846SIoana Radulescu dev_err(dev, "dpbp_open() failed\n"); 296634ff6846SIoana Radulescu goto err_open; 296734ff6846SIoana Radulescu } 296834ff6846SIoana Radulescu 296934ff6846SIoana Radulescu err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 297034ff6846SIoana Radulescu if (err) { 297134ff6846SIoana Radulescu dev_err(dev, "dpbp_reset() failed\n"); 297234ff6846SIoana Radulescu goto err_reset; 297334ff6846SIoana Radulescu } 297434ff6846SIoana Radulescu 297534ff6846SIoana Radulescu err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 297634ff6846SIoana Radulescu if (err) { 297734ff6846SIoana Radulescu dev_err(dev, "dpbp_enable() failed\n"); 297834ff6846SIoana Radulescu goto err_enable; 297934ff6846SIoana Radulescu } 298034ff6846SIoana Radulescu 298134ff6846SIoana Radulescu err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 298234ff6846SIoana Radulescu &dpbp_attrs); 298334ff6846SIoana Radulescu if (err) { 298434ff6846SIoana Radulescu dev_err(dev, "dpbp_get_attributes() failed\n"); 298534ff6846SIoana Radulescu goto err_get_attr; 298634ff6846SIoana Radulescu } 298734ff6846SIoana Radulescu priv->bpid = dpbp_attrs.bpid; 298834ff6846SIoana Radulescu 298934ff6846SIoana Radulescu return 0; 299034ff6846SIoana Radulescu 299134ff6846SIoana Radulescu err_get_attr: 299234ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 299334ff6846SIoana Radulescu err_enable: 299434ff6846SIoana Radulescu err_reset: 299534ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 299634ff6846SIoana Radulescu err_open: 299734ff6846SIoana Radulescu fsl_mc_object_free(dpbp_dev); 299834ff6846SIoana Radulescu 299934ff6846SIoana Radulescu return err; 300034ff6846SIoana Radulescu } 300134ff6846SIoana Radulescu 30025d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv) 300334ff6846SIoana Radulescu { 30045d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 300534ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 300634ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 300734ff6846SIoana Radulescu fsl_mc_object_free(priv->dpbp_dev); 300834ff6846SIoana Radulescu } 300934ff6846SIoana Radulescu 30105d8dccf8SIoana Ciornei static int dpaa2_eth_set_buffer_layout(struct dpaa2_eth_priv *priv) 301134ff6846SIoana Radulescu { 301234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 301334ff6846SIoana Radulescu struct dpni_buffer_layout buf_layout = {0}; 301427c87486SIoana Ciocoi Radulescu u16 rx_buf_align; 301534ff6846SIoana Radulescu int err; 301634ff6846SIoana Radulescu 301734ff6846SIoana Radulescu /* We need to check for WRIOP version 1.0.0, but depending on the MC 301834ff6846SIoana Radulescu * version, this number is not always provided correctly on rev1. 301934ff6846SIoana Radulescu * We need to check for both alternatives in this situation. 302034ff6846SIoana Radulescu */ 302134ff6846SIoana Radulescu if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 302234ff6846SIoana Radulescu priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 302327c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 302434ff6846SIoana Radulescu else 302527c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 302634ff6846SIoana Radulescu 3027efa6a7d0SIoana Ciornei /* We need to ensure that the buffer size seen by WRIOP is a multiple 3028efa6a7d0SIoana Ciornei * of 64 or 256 bytes depending on the WRIOP version. 3029efa6a7d0SIoana Ciornei */ 3030efa6a7d0SIoana Ciornei priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align); 3031efa6a7d0SIoana Ciornei 303234ff6846SIoana Radulescu /* tx buffer */ 303334ff6846SIoana Radulescu buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 303434ff6846SIoana Radulescu buf_layout.pass_timestamp = true; 3035c5521189SYangbo Lu buf_layout.pass_frame_status = true; 303634ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 3037c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 3038c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 303934ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 304034ff6846SIoana Radulescu DPNI_QUEUE_TX, &buf_layout); 304134ff6846SIoana Radulescu if (err) { 304234ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 304334ff6846SIoana Radulescu return err; 304434ff6846SIoana Radulescu } 304534ff6846SIoana Radulescu 304634ff6846SIoana Radulescu /* tx-confirm buffer */ 3047c5521189SYangbo Lu buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 3048c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 304934ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 305034ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, &buf_layout); 305134ff6846SIoana Radulescu if (err) { 305234ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 305334ff6846SIoana Radulescu return err; 305434ff6846SIoana Radulescu } 305534ff6846SIoana Radulescu 305634ff6846SIoana Radulescu /* Now that we've set our tx buffer layout, retrieve the minimum 305734ff6846SIoana Radulescu * required tx data offset. 305834ff6846SIoana Radulescu */ 305934ff6846SIoana Radulescu err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 306034ff6846SIoana Radulescu &priv->tx_data_offset); 306134ff6846SIoana Radulescu if (err) { 306234ff6846SIoana Radulescu dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 306334ff6846SIoana Radulescu return err; 306434ff6846SIoana Radulescu } 306534ff6846SIoana Radulescu 306634ff6846SIoana Radulescu if ((priv->tx_data_offset % 64) != 0) 306734ff6846SIoana Radulescu dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 306834ff6846SIoana Radulescu priv->tx_data_offset); 306934ff6846SIoana Radulescu 307034ff6846SIoana Radulescu /* rx buffer */ 307134ff6846SIoana Radulescu buf_layout.pass_frame_status = true; 307234ff6846SIoana Radulescu buf_layout.pass_parser_result = true; 307327c87486SIoana Ciocoi Radulescu buf_layout.data_align = rx_buf_align; 307434ff6846SIoana Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 307534ff6846SIoana Radulescu buf_layout.private_data_size = 0; 307634ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 307734ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 307834ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 307934ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 308034ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 308134ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 308234ff6846SIoana Radulescu DPNI_QUEUE_RX, &buf_layout); 308334ff6846SIoana Radulescu if (err) { 308434ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 308534ff6846SIoana Radulescu return err; 308634ff6846SIoana Radulescu } 308734ff6846SIoana Radulescu 308834ff6846SIoana Radulescu return 0; 308934ff6846SIoana Radulescu } 309034ff6846SIoana Radulescu 30911fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 30921fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MINOR 9 30931fa0f68cSIoana Ciocoi Radulescu 30941fa0f68cSIoana Ciocoi Radulescu static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 30951fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 309648c0481eSIoana Ciornei struct dpaa2_fd *fd, u8 prio, 30976ff80447SIoana Ciornei u32 num_frames __always_unused, 309848c0481eSIoana Ciornei int *frames_enqueued) 30991fa0f68cSIoana Ciocoi Radulescu { 310048c0481eSIoana Ciornei int err; 310148c0481eSIoana Ciornei 310248c0481eSIoana Ciornei err = dpaa2_io_service_enqueue_qd(fq->channel->dpio, 31031fa0f68cSIoana Ciocoi Radulescu priv->tx_qdid, prio, 31041fa0f68cSIoana Ciocoi Radulescu fq->tx_qdbin, fd); 310548c0481eSIoana Ciornei if (!err && frames_enqueued) 310648c0481eSIoana Ciornei *frames_enqueued = 1; 310748c0481eSIoana Ciornei return err; 31081fa0f68cSIoana Ciocoi Radulescu } 31091fa0f68cSIoana Ciocoi Radulescu 31106ff80447SIoana Ciornei static inline int dpaa2_eth_enqueue_fq_multiple(struct dpaa2_eth_priv *priv, 31111fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 31126ff80447SIoana Ciornei struct dpaa2_fd *fd, 31136ff80447SIoana Ciornei u8 prio, u32 num_frames, 311448c0481eSIoana Ciornei int *frames_enqueued) 31151fa0f68cSIoana Ciocoi Radulescu { 311648c0481eSIoana Ciornei int err; 311748c0481eSIoana Ciornei 31186ff80447SIoana Ciornei err = dpaa2_io_service_enqueue_multiple_fq(fq->channel->dpio, 31196ff80447SIoana Ciornei fq->tx_fqid[prio], 31206ff80447SIoana Ciornei fd, num_frames); 31216ff80447SIoana Ciornei 31226ff80447SIoana Ciornei if (err == 0) 31236ff80447SIoana Ciornei return -EBUSY; 31246ff80447SIoana Ciornei 31256ff80447SIoana Ciornei if (frames_enqueued) 31266ff80447SIoana Ciornei *frames_enqueued = err; 31276ff80447SIoana Ciornei return 0; 31281fa0f68cSIoana Ciocoi Radulescu } 31291fa0f68cSIoana Ciocoi Radulescu 31305d8dccf8SIoana Ciornei static void dpaa2_eth_set_enqueue_mode(struct dpaa2_eth_priv *priv) 31311fa0f68cSIoana Ciocoi Radulescu { 31321fa0f68cSIoana Ciocoi Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 31331fa0f68cSIoana Ciocoi Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 31341fa0f68cSIoana Ciocoi Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 31351fa0f68cSIoana Ciocoi Radulescu else 31366ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 31371fa0f68cSIoana Ciocoi Radulescu } 31381fa0f68cSIoana Ciocoi Radulescu 31395d8dccf8SIoana Ciornei static int dpaa2_eth_set_pause(struct dpaa2_eth_priv *priv) 31408eb3cef8SIoana Radulescu { 31418eb3cef8SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 31428eb3cef8SIoana Radulescu struct dpni_link_cfg link_cfg = {0}; 31438eb3cef8SIoana Radulescu int err; 31448eb3cef8SIoana Radulescu 31458eb3cef8SIoana Radulescu /* Get the default link options so we don't override other flags */ 31468eb3cef8SIoana Radulescu err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 31478eb3cef8SIoana Radulescu if (err) { 31488eb3cef8SIoana Radulescu dev_err(dev, "dpni_get_link_cfg() failed\n"); 31498eb3cef8SIoana Radulescu return err; 31508eb3cef8SIoana Radulescu } 31518eb3cef8SIoana Radulescu 31528eb3cef8SIoana Radulescu /* By default, enable both Rx and Tx pause frames */ 31538eb3cef8SIoana Radulescu link_cfg.options |= DPNI_LINK_OPT_PAUSE; 31548eb3cef8SIoana Radulescu link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 31558eb3cef8SIoana Radulescu err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 31568eb3cef8SIoana Radulescu if (err) { 31578eb3cef8SIoana Radulescu dev_err(dev, "dpni_set_link_cfg() failed\n"); 31588eb3cef8SIoana Radulescu return err; 31598eb3cef8SIoana Radulescu } 31608eb3cef8SIoana Radulescu 31618eb3cef8SIoana Radulescu priv->link_state.options = link_cfg.options; 31628eb3cef8SIoana Radulescu 31638eb3cef8SIoana Radulescu return 0; 31648eb3cef8SIoana Radulescu } 31658eb3cef8SIoana Radulescu 31665d8dccf8SIoana Ciornei static void dpaa2_eth_update_tx_fqids(struct dpaa2_eth_priv *priv) 3167a690af4fSIoana Radulescu { 3168a690af4fSIoana Radulescu struct dpni_queue_id qid = {0}; 3169a690af4fSIoana Radulescu struct dpaa2_eth_fq *fq; 3170a690af4fSIoana Radulescu struct dpni_queue queue; 3171a690af4fSIoana Radulescu int i, j, err; 3172a690af4fSIoana Radulescu 3173a690af4fSIoana Radulescu /* We only use Tx FQIDs for FQID-based enqueue, so check 3174a690af4fSIoana Radulescu * if DPNI version supports it before updating FQIDs 3175a690af4fSIoana Radulescu */ 3176a690af4fSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 3177a690af4fSIoana Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 3178a690af4fSIoana Radulescu return; 3179a690af4fSIoana Radulescu 3180a690af4fSIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 3181a690af4fSIoana Radulescu fq = &priv->fq[i]; 3182a690af4fSIoana Radulescu if (fq->type != DPAA2_TX_CONF_FQ) 3183a690af4fSIoana Radulescu continue; 3184a690af4fSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 3185a690af4fSIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3186a690af4fSIoana Radulescu DPNI_QUEUE_TX, j, fq->flowid, 3187a690af4fSIoana Radulescu &queue, &qid); 3188a690af4fSIoana Radulescu if (err) 3189a690af4fSIoana Radulescu goto out_err; 3190a690af4fSIoana Radulescu 3191a690af4fSIoana Radulescu fq->tx_fqid[j] = qid.fqid; 3192a690af4fSIoana Radulescu if (fq->tx_fqid[j] == 0) 3193a690af4fSIoana Radulescu goto out_err; 3194a690af4fSIoana Radulescu } 3195a690af4fSIoana Radulescu } 3196a690af4fSIoana Radulescu 31976ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 3198a690af4fSIoana Radulescu 3199a690af4fSIoana Radulescu return; 3200a690af4fSIoana Radulescu 3201a690af4fSIoana Radulescu out_err: 3202a690af4fSIoana Radulescu netdev_info(priv->net_dev, 3203a690af4fSIoana Radulescu "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 3204a690af4fSIoana Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 3205a690af4fSIoana Radulescu } 3206a690af4fSIoana Radulescu 32076aa90fe2SIoana Radulescu /* Configure ingress classification based on VLAN PCP */ 32085d8dccf8SIoana Ciornei static int dpaa2_eth_set_vlan_qos(struct dpaa2_eth_priv *priv) 32096aa90fe2SIoana Radulescu { 32106aa90fe2SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 32116aa90fe2SIoana Radulescu struct dpkg_profile_cfg kg_cfg = {0}; 32126aa90fe2SIoana Radulescu struct dpni_qos_tbl_cfg qos_cfg = {0}; 32136aa90fe2SIoana Radulescu struct dpni_rule_cfg key_params; 32146aa90fe2SIoana Radulescu void *dma_mem, *key, *mask; 32156aa90fe2SIoana Radulescu u8 key_size = 2; /* VLAN TCI field */ 32166aa90fe2SIoana Radulescu int i, pcp, err; 32176aa90fe2SIoana Radulescu 32186aa90fe2SIoana Radulescu /* VLAN-based classification only makes sense if we have multiple 32196aa90fe2SIoana Radulescu * traffic classes. 32206aa90fe2SIoana Radulescu * Also, we need to extract just the 3-bit PCP field from the VLAN 32216aa90fe2SIoana Radulescu * header and we can only do that by using a mask 32226aa90fe2SIoana Radulescu */ 32236aa90fe2SIoana Radulescu if (dpaa2_eth_tc_count(priv) == 1 || !dpaa2_eth_fs_mask_enabled(priv)) { 32246aa90fe2SIoana Radulescu dev_dbg(dev, "VLAN-based QoS classification not supported\n"); 32256aa90fe2SIoana Radulescu return -EOPNOTSUPP; 32266aa90fe2SIoana Radulescu } 32276aa90fe2SIoana Radulescu 32286aa90fe2SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 32296aa90fe2SIoana Radulescu if (!dma_mem) 32306aa90fe2SIoana Radulescu return -ENOMEM; 32316aa90fe2SIoana Radulescu 32326aa90fe2SIoana Radulescu kg_cfg.num_extracts = 1; 32336aa90fe2SIoana Radulescu kg_cfg.extracts[0].type = DPKG_EXTRACT_FROM_HDR; 32346aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.prot = NET_PROT_VLAN; 32356aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.type = DPKG_FULL_FIELD; 32366aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.field = NH_FLD_VLAN_TCI; 32376aa90fe2SIoana Radulescu 32386aa90fe2SIoana Radulescu err = dpni_prepare_key_cfg(&kg_cfg, dma_mem); 32396aa90fe2SIoana Radulescu if (err) { 32406aa90fe2SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg failed\n"); 32416aa90fe2SIoana Radulescu goto out_free_tbl; 32426aa90fe2SIoana Radulescu } 32436aa90fe2SIoana Radulescu 32446aa90fe2SIoana Radulescu /* set QoS table */ 32456aa90fe2SIoana Radulescu qos_cfg.default_tc = 0; 32466aa90fe2SIoana Radulescu qos_cfg.discard_on_miss = 0; 32476aa90fe2SIoana Radulescu qos_cfg.key_cfg_iova = dma_map_single(dev, dma_mem, 32486aa90fe2SIoana Radulescu DPAA2_CLASSIFIER_DMA_SIZE, 32496aa90fe2SIoana Radulescu DMA_TO_DEVICE); 32506aa90fe2SIoana Radulescu if (dma_mapping_error(dev, qos_cfg.key_cfg_iova)) { 32516aa90fe2SIoana Radulescu dev_err(dev, "QoS table DMA mapping failed\n"); 32526aa90fe2SIoana Radulescu err = -ENOMEM; 32536aa90fe2SIoana Radulescu goto out_free_tbl; 32546aa90fe2SIoana Radulescu } 32556aa90fe2SIoana Radulescu 32566aa90fe2SIoana Radulescu err = dpni_set_qos_table(priv->mc_io, 0, priv->mc_token, &qos_cfg); 32576aa90fe2SIoana Radulescu if (err) { 32586aa90fe2SIoana Radulescu dev_err(dev, "dpni_set_qos_table failed\n"); 32596aa90fe2SIoana Radulescu goto out_unmap_tbl; 32606aa90fe2SIoana Radulescu } 32616aa90fe2SIoana Radulescu 32626aa90fe2SIoana Radulescu /* Add QoS table entries */ 32636aa90fe2SIoana Radulescu key = kzalloc(key_size * 2, GFP_KERNEL); 32646aa90fe2SIoana Radulescu if (!key) { 32656aa90fe2SIoana Radulescu err = -ENOMEM; 32666aa90fe2SIoana Radulescu goto out_unmap_tbl; 32676aa90fe2SIoana Radulescu } 32686aa90fe2SIoana Radulescu mask = key + key_size; 32696aa90fe2SIoana Radulescu *(__be16 *)mask = cpu_to_be16(VLAN_PRIO_MASK); 32706aa90fe2SIoana Radulescu 32716aa90fe2SIoana Radulescu key_params.key_iova = dma_map_single(dev, key, key_size * 2, 32726aa90fe2SIoana Radulescu DMA_TO_DEVICE); 32736aa90fe2SIoana Radulescu if (dma_mapping_error(dev, key_params.key_iova)) { 32746aa90fe2SIoana Radulescu dev_err(dev, "Qos table entry DMA mapping failed\n"); 32756aa90fe2SIoana Radulescu err = -ENOMEM; 32766aa90fe2SIoana Radulescu goto out_free_key; 32776aa90fe2SIoana Radulescu } 32786aa90fe2SIoana Radulescu 32796aa90fe2SIoana Radulescu key_params.mask_iova = key_params.key_iova + key_size; 32806aa90fe2SIoana Radulescu key_params.key_size = key_size; 32816aa90fe2SIoana Radulescu 32826aa90fe2SIoana Radulescu /* We add rules for PCP-based distribution starting with highest 32836aa90fe2SIoana Radulescu * priority (VLAN PCP = 7). If this DPNI doesn't have enough traffic 32846aa90fe2SIoana Radulescu * classes to accommodate all priority levels, the lowest ones end up 32856aa90fe2SIoana Radulescu * on TC 0 which was configured as default 32866aa90fe2SIoana Radulescu */ 32876aa90fe2SIoana Radulescu for (i = dpaa2_eth_tc_count(priv) - 1, pcp = 7; i >= 0; i--, pcp--) { 32886aa90fe2SIoana Radulescu *(__be16 *)key = cpu_to_be16(pcp << VLAN_PRIO_SHIFT); 32896aa90fe2SIoana Radulescu dma_sync_single_for_device(dev, key_params.key_iova, 32906aa90fe2SIoana Radulescu key_size * 2, DMA_TO_DEVICE); 32916aa90fe2SIoana Radulescu 32926aa90fe2SIoana Radulescu err = dpni_add_qos_entry(priv->mc_io, 0, priv->mc_token, 32936aa90fe2SIoana Radulescu &key_params, i, i); 32946aa90fe2SIoana Radulescu if (err) { 32956aa90fe2SIoana Radulescu dev_err(dev, "dpni_add_qos_entry failed\n"); 32966aa90fe2SIoana Radulescu dpni_clear_qos_table(priv->mc_io, 0, priv->mc_token); 32976aa90fe2SIoana Radulescu goto out_unmap_key; 32986aa90fe2SIoana Radulescu } 32996aa90fe2SIoana Radulescu } 33006aa90fe2SIoana Radulescu 33016aa90fe2SIoana Radulescu priv->vlan_cls_enabled = true; 33026aa90fe2SIoana Radulescu 33036aa90fe2SIoana Radulescu /* Table and key memory is not persistent, clean everything up after 33046aa90fe2SIoana Radulescu * configuration is finished 33056aa90fe2SIoana Radulescu */ 33066aa90fe2SIoana Radulescu out_unmap_key: 33076aa90fe2SIoana Radulescu dma_unmap_single(dev, key_params.key_iova, key_size * 2, DMA_TO_DEVICE); 33086aa90fe2SIoana Radulescu out_free_key: 33096aa90fe2SIoana Radulescu kfree(key); 33106aa90fe2SIoana Radulescu out_unmap_tbl: 33116aa90fe2SIoana Radulescu dma_unmap_single(dev, qos_cfg.key_cfg_iova, DPAA2_CLASSIFIER_DMA_SIZE, 33126aa90fe2SIoana Radulescu DMA_TO_DEVICE); 33136aa90fe2SIoana Radulescu out_free_tbl: 33146aa90fe2SIoana Radulescu kfree(dma_mem); 33156aa90fe2SIoana Radulescu 33166aa90fe2SIoana Radulescu return err; 33176aa90fe2SIoana Radulescu } 33186aa90fe2SIoana Radulescu 331934ff6846SIoana Radulescu /* Configure the DPNI object this interface is associated with */ 33205d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpni(struct fsl_mc_device *ls_dev) 332134ff6846SIoana Radulescu { 332234ff6846SIoana Radulescu struct device *dev = &ls_dev->dev; 332334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 332434ff6846SIoana Radulescu struct net_device *net_dev; 332534ff6846SIoana Radulescu int err; 332634ff6846SIoana Radulescu 332734ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 332834ff6846SIoana Radulescu priv = netdev_priv(net_dev); 332934ff6846SIoana Radulescu 333034ff6846SIoana Radulescu /* get a handle for the DPNI object */ 333134ff6846SIoana Radulescu err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 333234ff6846SIoana Radulescu if (err) { 333334ff6846SIoana Radulescu dev_err(dev, "dpni_open() failed\n"); 333434ff6846SIoana Radulescu return err; 333534ff6846SIoana Radulescu } 333634ff6846SIoana Radulescu 333734ff6846SIoana Radulescu /* Check if we can work with this DPNI object */ 333834ff6846SIoana Radulescu err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 333934ff6846SIoana Radulescu &priv->dpni_ver_minor); 334034ff6846SIoana Radulescu if (err) { 334134ff6846SIoana Radulescu dev_err(dev, "dpni_get_api_version() failed\n"); 334234ff6846SIoana Radulescu goto close; 334334ff6846SIoana Radulescu } 334434ff6846SIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 334534ff6846SIoana Radulescu dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 334634ff6846SIoana Radulescu priv->dpni_ver_major, priv->dpni_ver_minor, 334734ff6846SIoana Radulescu DPNI_VER_MAJOR, DPNI_VER_MINOR); 334834ff6846SIoana Radulescu err = -ENOTSUPP; 334934ff6846SIoana Radulescu goto close; 335034ff6846SIoana Radulescu } 335134ff6846SIoana Radulescu 335234ff6846SIoana Radulescu ls_dev->mc_io = priv->mc_io; 335334ff6846SIoana Radulescu ls_dev->mc_handle = priv->mc_token; 335434ff6846SIoana Radulescu 335534ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 335634ff6846SIoana Radulescu if (err) { 335734ff6846SIoana Radulescu dev_err(dev, "dpni_reset() failed\n"); 335834ff6846SIoana Radulescu goto close; 335934ff6846SIoana Radulescu } 336034ff6846SIoana Radulescu 336134ff6846SIoana Radulescu err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 336234ff6846SIoana Radulescu &priv->dpni_attrs); 336334ff6846SIoana Radulescu if (err) { 336434ff6846SIoana Radulescu dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 336534ff6846SIoana Radulescu goto close; 336634ff6846SIoana Radulescu } 336734ff6846SIoana Radulescu 33685d8dccf8SIoana Ciornei err = dpaa2_eth_set_buffer_layout(priv); 336934ff6846SIoana Radulescu if (err) 337034ff6846SIoana Radulescu goto close; 337134ff6846SIoana Radulescu 33725d8dccf8SIoana Ciornei dpaa2_eth_set_enqueue_mode(priv); 33731fa0f68cSIoana Ciocoi Radulescu 33748eb3cef8SIoana Radulescu /* Enable pause frame support */ 33758eb3cef8SIoana Radulescu if (dpaa2_eth_has_pause_support(priv)) { 33765d8dccf8SIoana Ciornei err = dpaa2_eth_set_pause(priv); 33778eb3cef8SIoana Radulescu if (err) 33788eb3cef8SIoana Radulescu goto close; 33798eb3cef8SIoana Radulescu } 33808eb3cef8SIoana Radulescu 33815d8dccf8SIoana Ciornei err = dpaa2_eth_set_vlan_qos(priv); 33826aa90fe2SIoana Radulescu if (err && err != -EOPNOTSUPP) 33836aa90fe2SIoana Radulescu goto close; 33846aa90fe2SIoana Radulescu 33859334d5baSXu Wang priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv), 33869334d5baSXu Wang sizeof(struct dpaa2_eth_cls_rule), 33879334d5baSXu Wang GFP_KERNEL); 338897fff7c8SWei Yongjun if (!priv->cls_rules) { 338997fff7c8SWei Yongjun err = -ENOMEM; 3390afb90dbbSIoana Radulescu goto close; 339197fff7c8SWei Yongjun } 3392afb90dbbSIoana Radulescu 339334ff6846SIoana Radulescu return 0; 339434ff6846SIoana Radulescu 339534ff6846SIoana Radulescu close: 339634ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 339734ff6846SIoana Radulescu 339834ff6846SIoana Radulescu return err; 339934ff6846SIoana Radulescu } 340034ff6846SIoana Radulescu 34015d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpni(struct dpaa2_eth_priv *priv) 340234ff6846SIoana Radulescu { 340334ff6846SIoana Radulescu int err; 340434ff6846SIoana Radulescu 340534ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 340634ff6846SIoana Radulescu if (err) 340734ff6846SIoana Radulescu netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 340834ff6846SIoana Radulescu err); 340934ff6846SIoana Radulescu 341034ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 341134ff6846SIoana Radulescu } 341234ff6846SIoana Radulescu 34135d8dccf8SIoana Ciornei static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, 341434ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 341534ff6846SIoana Radulescu { 341634ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 341734ff6846SIoana Radulescu struct dpni_queue queue; 341834ff6846SIoana Radulescu struct dpni_queue_id qid; 341934ff6846SIoana Radulescu int err; 342034ff6846SIoana Radulescu 342134ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3422685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, &queue, &qid); 342334ff6846SIoana Radulescu if (err) { 342434ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(RX) failed\n"); 342534ff6846SIoana Radulescu return err; 342634ff6846SIoana Radulescu } 342734ff6846SIoana Radulescu 342834ff6846SIoana Radulescu fq->fqid = qid.fqid; 342934ff6846SIoana Radulescu 343034ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 343134ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 343234ff6846SIoana Radulescu queue.destination.priority = 1; 343334ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 343434ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3435685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, 343616fa1cf1SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 343734ff6846SIoana Radulescu &queue); 343834ff6846SIoana Radulescu if (err) { 343934ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(RX) failed\n"); 344034ff6846SIoana Radulescu return err; 344134ff6846SIoana Radulescu } 344234ff6846SIoana Radulescu 3443d678be1dSIoana Radulescu /* xdp_rxq setup */ 3444685e39eaSIoana Radulescu /* only once for each channel */ 3445685e39eaSIoana Radulescu if (fq->tc > 0) 3446685e39eaSIoana Radulescu return 0; 3447685e39eaSIoana Radulescu 3448d678be1dSIoana Radulescu err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 3449b02e5a0eSBjörn Töpel fq->flowid, 0); 3450d678be1dSIoana Radulescu if (err) { 3451d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg failed\n"); 3452d678be1dSIoana Radulescu return err; 3453d678be1dSIoana Radulescu } 3454d678be1dSIoana Radulescu 3455d678be1dSIoana Radulescu err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 3456d678be1dSIoana Radulescu MEM_TYPE_PAGE_ORDER0, NULL); 3457d678be1dSIoana Radulescu if (err) { 3458d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 3459d678be1dSIoana Radulescu return err; 3460d678be1dSIoana Radulescu } 3461d678be1dSIoana Radulescu 346234ff6846SIoana Radulescu return 0; 346334ff6846SIoana Radulescu } 346434ff6846SIoana Radulescu 34655d8dccf8SIoana Ciornei static int dpaa2_eth_setup_tx_flow(struct dpaa2_eth_priv *priv, 346634ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 346734ff6846SIoana Radulescu { 346834ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 346934ff6846SIoana Radulescu struct dpni_queue queue; 347034ff6846SIoana Radulescu struct dpni_queue_id qid; 347115c87f6bSIoana Radulescu int i, err; 347234ff6846SIoana Radulescu 347315c87f6bSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 347434ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 347515c87f6bSIoana Radulescu DPNI_QUEUE_TX, i, fq->flowid, 347615c87f6bSIoana Radulescu &queue, &qid); 347734ff6846SIoana Radulescu if (err) { 347834ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX) failed\n"); 347934ff6846SIoana Radulescu return err; 348034ff6846SIoana Radulescu } 348115c87f6bSIoana Radulescu fq->tx_fqid[i] = qid.fqid; 348215c87f6bSIoana Radulescu } 348334ff6846SIoana Radulescu 348415c87f6bSIoana Radulescu /* All Tx queues belonging to the same flowid have the same qdbin */ 348534ff6846SIoana Radulescu fq->tx_qdbin = qid.qdbin; 348634ff6846SIoana Radulescu 348734ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 348834ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 348934ff6846SIoana Radulescu &queue, &qid); 349034ff6846SIoana Radulescu if (err) { 349134ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 349234ff6846SIoana Radulescu return err; 349334ff6846SIoana Radulescu } 349434ff6846SIoana Radulescu 349534ff6846SIoana Radulescu fq->fqid = qid.fqid; 349634ff6846SIoana Radulescu 349734ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 349834ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 349934ff6846SIoana Radulescu queue.destination.priority = 0; 350034ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 350134ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 350234ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 350334ff6846SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 350434ff6846SIoana Radulescu &queue); 350534ff6846SIoana Radulescu if (err) { 350634ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 350734ff6846SIoana Radulescu return err; 350834ff6846SIoana Radulescu } 350934ff6846SIoana Radulescu 351034ff6846SIoana Radulescu return 0; 351134ff6846SIoana Radulescu } 351234ff6846SIoana Radulescu 3513061d631fSIoana Ciornei static int setup_rx_err_flow(struct dpaa2_eth_priv *priv, 3514061d631fSIoana Ciornei struct dpaa2_eth_fq *fq) 3515061d631fSIoana Ciornei { 3516061d631fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 3517061d631fSIoana Ciornei struct dpni_queue q = { { 0 } }; 3518061d631fSIoana Ciornei struct dpni_queue_id qid; 3519061d631fSIoana Ciornei u8 q_opt = DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST; 3520061d631fSIoana Ciornei int err; 3521061d631fSIoana Ciornei 3522061d631fSIoana Ciornei err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3523061d631fSIoana Ciornei DPNI_QUEUE_RX_ERR, 0, 0, &q, &qid); 3524061d631fSIoana Ciornei if (err) { 3525061d631fSIoana Ciornei dev_err(dev, "dpni_get_queue() failed (%d)\n", err); 3526061d631fSIoana Ciornei return err; 3527061d631fSIoana Ciornei } 3528061d631fSIoana Ciornei 3529061d631fSIoana Ciornei fq->fqid = qid.fqid; 3530061d631fSIoana Ciornei 3531061d631fSIoana Ciornei q.destination.id = fq->channel->dpcon_id; 3532061d631fSIoana Ciornei q.destination.type = DPNI_DEST_DPCON; 3533061d631fSIoana Ciornei q.destination.priority = 1; 3534061d631fSIoana Ciornei q.user_context = (u64)(uintptr_t)fq; 3535061d631fSIoana Ciornei err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3536061d631fSIoana Ciornei DPNI_QUEUE_RX_ERR, 0, 0, q_opt, &q); 3537061d631fSIoana Ciornei if (err) { 3538061d631fSIoana Ciornei dev_err(dev, "dpni_set_queue() failed (%d)\n", err); 3539061d631fSIoana Ciornei return err; 3540061d631fSIoana Ciornei } 3541061d631fSIoana Ciornei 3542061d631fSIoana Ciornei return 0; 3543061d631fSIoana Ciornei } 3544061d631fSIoana Ciornei 3545edad8d26SIoana Ciocoi Radulescu /* Supported header fields for Rx hash distribution key */ 3546f76c483aSIoana Radulescu static const struct dpaa2_eth_dist_fields dist_fields[] = { 354734ff6846SIoana Radulescu { 3548edad8d26SIoana Ciocoi Radulescu /* L2 header */ 3549edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_L2DA, 3550edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_ETH, 3551edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_ETH_DA, 35523a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHDST, 3553edad8d26SIoana Ciocoi Radulescu .size = 6, 3554edad8d26SIoana Ciocoi Radulescu }, { 3555afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3556afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_SA, 35573a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHSRC, 3558afb90dbbSIoana Radulescu .size = 6, 3559afb90dbbSIoana Radulescu }, { 3560afb90dbbSIoana Radulescu /* This is the last ethertype field parsed: 3561afb90dbbSIoana Radulescu * depending on frame format, it can be the MAC ethertype 3562afb90dbbSIoana Radulescu * or the VLAN etype. 3563afb90dbbSIoana Radulescu */ 3564afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3565afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_TYPE, 35663a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHTYPE, 3567afb90dbbSIoana Radulescu .size = 2, 3568afb90dbbSIoana Radulescu }, { 3569edad8d26SIoana Ciocoi Radulescu /* VLAN header */ 3570edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_VLAN, 3571edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_VLAN, 3572edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_VLAN_TCI, 35733a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_VLAN, 3574edad8d26SIoana Ciocoi Radulescu .size = 2, 3575edad8d26SIoana Ciocoi Radulescu }, { 357634ff6846SIoana Radulescu /* IP header */ 357734ff6846SIoana Radulescu .rxnfc_field = RXH_IP_SRC, 357834ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 357934ff6846SIoana Radulescu .cls_field = NH_FLD_IP_SRC, 35803a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPSRC, 358134ff6846SIoana Radulescu .size = 4, 358234ff6846SIoana Radulescu }, { 358334ff6846SIoana Radulescu .rxnfc_field = RXH_IP_DST, 358434ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 358534ff6846SIoana Radulescu .cls_field = NH_FLD_IP_DST, 35863a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPDST, 358734ff6846SIoana Radulescu .size = 4, 358834ff6846SIoana Radulescu }, { 358934ff6846SIoana Radulescu .rxnfc_field = RXH_L3_PROTO, 359034ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 359134ff6846SIoana Radulescu .cls_field = NH_FLD_IP_PROTO, 35923a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPPROTO, 359334ff6846SIoana Radulescu .size = 1, 359434ff6846SIoana Radulescu }, { 359534ff6846SIoana Radulescu /* Using UDP ports, this is functionally equivalent to raw 359634ff6846SIoana Radulescu * byte pairs from L4 header. 359734ff6846SIoana Radulescu */ 359834ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_0_1, 359934ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 360034ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_SRC, 36013a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4SRC, 360234ff6846SIoana Radulescu .size = 2, 360334ff6846SIoana Radulescu }, { 360434ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_2_3, 360534ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 360634ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_DST, 36073a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4DST, 360834ff6846SIoana Radulescu .size = 2, 360934ff6846SIoana Radulescu }, 361034ff6846SIoana Radulescu }; 361134ff6846SIoana Radulescu 3612df85aeb9SIoana Radulescu /* Configure the Rx hash key using the legacy API */ 36135d8dccf8SIoana Ciornei static int dpaa2_eth_config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3614df85aeb9SIoana Radulescu { 3615df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3616df85aeb9SIoana Radulescu struct dpni_rx_tc_dist_cfg dist_cfg; 3617685e39eaSIoana Radulescu int i, err = 0; 3618df85aeb9SIoana Radulescu 3619df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3620df85aeb9SIoana Radulescu 3621df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3622df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3623df85aeb9SIoana Radulescu dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 3624df85aeb9SIoana Radulescu 3625685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3626685e39eaSIoana Radulescu err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 3627685e39eaSIoana Radulescu i, &dist_cfg); 3628685e39eaSIoana Radulescu if (err) { 3629df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 3630685e39eaSIoana Radulescu break; 3631685e39eaSIoana Radulescu } 3632685e39eaSIoana Radulescu } 3633df85aeb9SIoana Radulescu 3634df85aeb9SIoana Radulescu return err; 3635df85aeb9SIoana Radulescu } 3636df85aeb9SIoana Radulescu 3637df85aeb9SIoana Radulescu /* Configure the Rx hash key using the new API */ 36385d8dccf8SIoana Ciornei static int dpaa2_eth_config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3639df85aeb9SIoana Radulescu { 3640df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3641df85aeb9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3642685e39eaSIoana Radulescu int i, err = 0; 3643df85aeb9SIoana Radulescu 3644df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3645df85aeb9SIoana Radulescu 3646df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3647df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3648df85aeb9SIoana Radulescu dist_cfg.enable = 1; 3649df85aeb9SIoana Radulescu 3650685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3651685e39eaSIoana Radulescu dist_cfg.tc = i; 3652685e39eaSIoana Radulescu err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, 3653685e39eaSIoana Radulescu &dist_cfg); 3654685e39eaSIoana Radulescu if (err) { 3655df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 3656685e39eaSIoana Radulescu break; 3657685e39eaSIoana Radulescu } 36585e29c16fSIonut-robert Aron 36595e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 36605e29c16fSIonut-robert Aron * traffic classes, install it just once 36615e29c16fSIonut-robert Aron */ 36625e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 36635e29c16fSIonut-robert Aron break; 3664685e39eaSIoana Radulescu } 3665df85aeb9SIoana Radulescu 3666df85aeb9SIoana Radulescu return err; 3667df85aeb9SIoana Radulescu } 3668df85aeb9SIoana Radulescu 36694aaaf9b9SIoana Radulescu /* Configure the Rx flow classification key */ 36705d8dccf8SIoana Ciornei static int dpaa2_eth_config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 36714aaaf9b9SIoana Radulescu { 36724aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 36734aaaf9b9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3674685e39eaSIoana Radulescu int i, err = 0; 36754aaaf9b9SIoana Radulescu 36764aaaf9b9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 36774aaaf9b9SIoana Radulescu 36784aaaf9b9SIoana Radulescu dist_cfg.key_cfg_iova = key; 36794aaaf9b9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 36804aaaf9b9SIoana Radulescu dist_cfg.enable = 1; 36814aaaf9b9SIoana Radulescu 3682685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3683685e39eaSIoana Radulescu dist_cfg.tc = i; 3684685e39eaSIoana Radulescu err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, 3685685e39eaSIoana Radulescu &dist_cfg); 3686685e39eaSIoana Radulescu if (err) { 36874aaaf9b9SIoana Radulescu dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 3688685e39eaSIoana Radulescu break; 3689685e39eaSIoana Radulescu } 36905e29c16fSIonut-robert Aron 36915e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 36925e29c16fSIonut-robert Aron * traffic classes, install it just once 36935e29c16fSIonut-robert Aron */ 36945e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 36955e29c16fSIonut-robert Aron break; 3696685e39eaSIoana Radulescu } 36974aaaf9b9SIoana Radulescu 36984aaaf9b9SIoana Radulescu return err; 36994aaaf9b9SIoana Radulescu } 37004aaaf9b9SIoana Radulescu 3701afb90dbbSIoana Radulescu /* Size of the Rx flow classification key */ 37022d680237SIoana Ciocoi Radulescu int dpaa2_eth_cls_key_size(u64 fields) 3703afb90dbbSIoana Radulescu { 3704afb90dbbSIoana Radulescu int i, size = 0; 3705afb90dbbSIoana Radulescu 37062d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 37072d680237SIoana Ciocoi Radulescu if (!(fields & dist_fields[i].id)) 37082d680237SIoana Ciocoi Radulescu continue; 3709afb90dbbSIoana Radulescu size += dist_fields[i].size; 37102d680237SIoana Ciocoi Radulescu } 3711afb90dbbSIoana Radulescu 3712afb90dbbSIoana Radulescu return size; 3713afb90dbbSIoana Radulescu } 3714afb90dbbSIoana Radulescu 3715afb90dbbSIoana Radulescu /* Offset of header field in Rx classification key */ 3716afb90dbbSIoana Radulescu int dpaa2_eth_cls_fld_off(int prot, int field) 3717afb90dbbSIoana Radulescu { 3718afb90dbbSIoana Radulescu int i, off = 0; 3719afb90dbbSIoana Radulescu 3720afb90dbbSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3721afb90dbbSIoana Radulescu if (dist_fields[i].cls_prot == prot && 3722afb90dbbSIoana Radulescu dist_fields[i].cls_field == field) 3723afb90dbbSIoana Radulescu return off; 3724afb90dbbSIoana Radulescu off += dist_fields[i].size; 3725afb90dbbSIoana Radulescu } 3726afb90dbbSIoana Radulescu 3727afb90dbbSIoana Radulescu WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 3728afb90dbbSIoana Radulescu return 0; 3729afb90dbbSIoana Radulescu } 3730afb90dbbSIoana Radulescu 37312d680237SIoana Ciocoi Radulescu /* Prune unused fields from the classification rule. 37322d680237SIoana Ciocoi Radulescu * Used when masking is not supported 37332d680237SIoana Ciocoi Radulescu */ 37342d680237SIoana Ciocoi Radulescu void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 37352d680237SIoana Ciocoi Radulescu { 37362d680237SIoana Ciocoi Radulescu int off = 0, new_off = 0; 37372d680237SIoana Ciocoi Radulescu int i, size; 37382d680237SIoana Ciocoi Radulescu 37392d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 37402d680237SIoana Ciocoi Radulescu size = dist_fields[i].size; 37412d680237SIoana Ciocoi Radulescu if (dist_fields[i].id & fields) { 37422d680237SIoana Ciocoi Radulescu memcpy(key_mem + new_off, key_mem + off, size); 37432d680237SIoana Ciocoi Radulescu new_off += size; 37442d680237SIoana Ciocoi Radulescu } 37452d680237SIoana Ciocoi Radulescu off += size; 37462d680237SIoana Ciocoi Radulescu } 37472d680237SIoana Ciocoi Radulescu } 37482d680237SIoana Ciocoi Radulescu 37494aaaf9b9SIoana Radulescu /* Set Rx distribution (hash or flow classification) key 375034ff6846SIoana Radulescu * flags is a combination of RXH_ bits 375134ff6846SIoana Radulescu */ 37523233c151SIoana Ciornei static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 37534aaaf9b9SIoana Radulescu enum dpaa2_eth_rx_dist type, u64 flags) 375434ff6846SIoana Radulescu { 375534ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 375634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 375734ff6846SIoana Radulescu struct dpkg_profile_cfg cls_cfg; 3758edad8d26SIoana Ciocoi Radulescu u32 rx_hash_fields = 0; 3759df85aeb9SIoana Radulescu dma_addr_t key_iova; 376034ff6846SIoana Radulescu u8 *dma_mem; 376134ff6846SIoana Radulescu int i; 376234ff6846SIoana Radulescu int err = 0; 376334ff6846SIoana Radulescu 376434ff6846SIoana Radulescu memset(&cls_cfg, 0, sizeof(cls_cfg)); 376534ff6846SIoana Radulescu 3766f76c483aSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 376734ff6846SIoana Radulescu struct dpkg_extract *key = 376834ff6846SIoana Radulescu &cls_cfg.extracts[cls_cfg.num_extracts]; 376934ff6846SIoana Radulescu 37702d680237SIoana Ciocoi Radulescu /* For both Rx hashing and classification keys 37712d680237SIoana Ciocoi Radulescu * we set only the selected fields. 37724aaaf9b9SIoana Radulescu */ 37733a1e6b84SIoana Ciocoi Radulescu if (!(flags & dist_fields[i].id)) 377434ff6846SIoana Radulescu continue; 37752d680237SIoana Ciocoi Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) 37764aaaf9b9SIoana Radulescu rx_hash_fields |= dist_fields[i].rxnfc_field; 377734ff6846SIoana Radulescu 377834ff6846SIoana Radulescu if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 377934ff6846SIoana Radulescu dev_err(dev, "error adding key extraction rule, too many rules?\n"); 378034ff6846SIoana Radulescu return -E2BIG; 378134ff6846SIoana Radulescu } 378234ff6846SIoana Radulescu 378334ff6846SIoana Radulescu key->type = DPKG_EXTRACT_FROM_HDR; 3784f76c483aSIoana Radulescu key->extract.from_hdr.prot = dist_fields[i].cls_prot; 378534ff6846SIoana Radulescu key->extract.from_hdr.type = DPKG_FULL_FIELD; 3786f76c483aSIoana Radulescu key->extract.from_hdr.field = dist_fields[i].cls_field; 378734ff6846SIoana Radulescu cls_cfg.num_extracts++; 378834ff6846SIoana Radulescu } 378934ff6846SIoana Radulescu 379034ff6846SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 379134ff6846SIoana Radulescu if (!dma_mem) 379234ff6846SIoana Radulescu return -ENOMEM; 379334ff6846SIoana Radulescu 379434ff6846SIoana Radulescu err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 379534ff6846SIoana Radulescu if (err) { 379634ff6846SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 3797df85aeb9SIoana Radulescu goto free_key; 379834ff6846SIoana Radulescu } 379934ff6846SIoana Radulescu 380034ff6846SIoana Radulescu /* Prepare for setting the rx dist */ 3801df85aeb9SIoana Radulescu key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 380234ff6846SIoana Radulescu DMA_TO_DEVICE); 3803df85aeb9SIoana Radulescu if (dma_mapping_error(dev, key_iova)) { 380434ff6846SIoana Radulescu dev_err(dev, "DMA mapping failed\n"); 380534ff6846SIoana Radulescu err = -ENOMEM; 3806df85aeb9SIoana Radulescu goto free_key; 380734ff6846SIoana Radulescu } 380834ff6846SIoana Radulescu 38094aaaf9b9SIoana Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) { 3810df85aeb9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) 38115d8dccf8SIoana Ciornei err = dpaa2_eth_config_legacy_hash_key(priv, key_iova); 3812edad8d26SIoana Ciocoi Radulescu else 38135d8dccf8SIoana Ciornei err = dpaa2_eth_config_hash_key(priv, key_iova); 38144aaaf9b9SIoana Radulescu } else { 38155d8dccf8SIoana Ciornei err = dpaa2_eth_config_cls_key(priv, key_iova); 38164aaaf9b9SIoana Radulescu } 3817df85aeb9SIoana Radulescu 3818df85aeb9SIoana Radulescu dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3819df85aeb9SIoana Radulescu DMA_TO_DEVICE); 38204aaaf9b9SIoana Radulescu if (!err && type == DPAA2_ETH_RX_DIST_HASH) 3821edad8d26SIoana Ciocoi Radulescu priv->rx_hash_fields = rx_hash_fields; 382234ff6846SIoana Radulescu 3823df85aeb9SIoana Radulescu free_key: 382434ff6846SIoana Radulescu kfree(dma_mem); 382534ff6846SIoana Radulescu return err; 382634ff6846SIoana Radulescu } 382734ff6846SIoana Radulescu 38284aaaf9b9SIoana Radulescu int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 38294aaaf9b9SIoana Radulescu { 38304aaaf9b9SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 38313a1e6b84SIoana Ciocoi Radulescu u64 key = 0; 38323a1e6b84SIoana Ciocoi Radulescu int i; 38334aaaf9b9SIoana Radulescu 38344aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) 38354aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38364aaaf9b9SIoana Radulescu 38373a1e6b84SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 38383a1e6b84SIoana Ciocoi Radulescu if (dist_fields[i].rxnfc_field & flags) 38393a1e6b84SIoana Ciocoi Radulescu key |= dist_fields[i].id; 38403a1e6b84SIoana Ciocoi Radulescu 38413a1e6b84SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 38424aaaf9b9SIoana Radulescu } 38434aaaf9b9SIoana Radulescu 38442d680237SIoana Ciocoi Radulescu int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 38452d680237SIoana Ciocoi Radulescu { 38462d680237SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 38472d680237SIoana Ciocoi Radulescu } 38482d680237SIoana Ciocoi Radulescu 38492d680237SIoana Ciocoi Radulescu static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 38504aaaf9b9SIoana Radulescu { 38514aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3852df8e249bSIoana Ciocoi Radulescu int err; 38534aaaf9b9SIoana Radulescu 38544aaaf9b9SIoana Radulescu /* Check if we actually support Rx flow classification */ 38554aaaf9b9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) { 38564aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls not supported by current MC version\n"); 38574aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38584aaaf9b9SIoana Radulescu } 38594aaaf9b9SIoana Radulescu 38602d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_enabled(priv)) { 38614aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 38624aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38634aaaf9b9SIoana Radulescu } 38644aaaf9b9SIoana Radulescu 38654aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) { 38664aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 38674aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38684aaaf9b9SIoana Radulescu } 38694aaaf9b9SIoana Radulescu 38702d680237SIoana Ciocoi Radulescu /* If there is no support for masking in the classification table, 38712d680237SIoana Ciocoi Radulescu * we don't set a default key, as it will depend on the rules 38722d680237SIoana Ciocoi Radulescu * added by the user at runtime. 38732d680237SIoana Ciocoi Radulescu */ 38742d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_mask_enabled(priv)) 38752d680237SIoana Ciocoi Radulescu goto out; 38762d680237SIoana Ciocoi Radulescu 38772d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 3878df8e249bSIoana Ciocoi Radulescu if (err) 3879df8e249bSIoana Ciocoi Radulescu return err; 3880df8e249bSIoana Ciocoi Radulescu 38812d680237SIoana Ciocoi Radulescu out: 38824aaaf9b9SIoana Radulescu priv->rx_cls_enabled = 1; 38834aaaf9b9SIoana Radulescu 3884df8e249bSIoana Ciocoi Radulescu return 0; 38854aaaf9b9SIoana Radulescu } 38864aaaf9b9SIoana Radulescu 388734ff6846SIoana Radulescu /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 388834ff6846SIoana Radulescu * frame queues and channels 388934ff6846SIoana Radulescu */ 38905d8dccf8SIoana Ciornei static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv) 389134ff6846SIoana Radulescu { 389234ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 389334ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 389434ff6846SIoana Radulescu struct dpni_pools_cfg pools_params; 389534ff6846SIoana Radulescu struct dpni_error_cfg err_cfg; 389634ff6846SIoana Radulescu int err = 0; 389734ff6846SIoana Radulescu int i; 389834ff6846SIoana Radulescu 389934ff6846SIoana Radulescu pools_params.num_dpbp = 1; 390034ff6846SIoana Radulescu pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 390134ff6846SIoana Radulescu pools_params.pools[0].backup_pool = 0; 3902efa6a7d0SIoana Ciornei pools_params.pools[0].buffer_size = priv->rx_buf_size; 390334ff6846SIoana Radulescu err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 390434ff6846SIoana Radulescu if (err) { 390534ff6846SIoana Radulescu dev_err(dev, "dpni_set_pools() failed\n"); 390634ff6846SIoana Radulescu return err; 390734ff6846SIoana Radulescu } 390834ff6846SIoana Radulescu 390934ff6846SIoana Radulescu /* have the interface implicitly distribute traffic based on 391034ff6846SIoana Radulescu * the default hash key 391134ff6846SIoana Radulescu */ 391234ff6846SIoana Radulescu err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 3913edad8d26SIoana Ciocoi Radulescu if (err && err != -EOPNOTSUPP) 391434ff6846SIoana Radulescu dev_err(dev, "Failed to configure hashing\n"); 391534ff6846SIoana Radulescu 39164aaaf9b9SIoana Radulescu /* Configure the flow classification key; it includes all 39174aaaf9b9SIoana Radulescu * supported header fields and cannot be modified at runtime 39184aaaf9b9SIoana Radulescu */ 39192d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_default_cls(priv); 39204aaaf9b9SIoana Radulescu if (err && err != -EOPNOTSUPP) 39214aaaf9b9SIoana Radulescu dev_err(dev, "Failed to configure Rx classification key\n"); 39224aaaf9b9SIoana Radulescu 392334ff6846SIoana Radulescu /* Configure handling of error frames */ 392434ff6846SIoana Radulescu err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 392534ff6846SIoana Radulescu err_cfg.set_frame_annotation = 1; 392634ff6846SIoana Radulescu err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 392734ff6846SIoana Radulescu err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 392834ff6846SIoana Radulescu &err_cfg); 392934ff6846SIoana Radulescu if (err) { 393034ff6846SIoana Radulescu dev_err(dev, "dpni_set_errors_behavior failed\n"); 393134ff6846SIoana Radulescu return err; 393234ff6846SIoana Radulescu } 393334ff6846SIoana Radulescu 393434ff6846SIoana Radulescu /* Configure Rx and Tx conf queues to generate CDANs */ 393534ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 393634ff6846SIoana Radulescu switch (priv->fq[i].type) { 393734ff6846SIoana Radulescu case DPAA2_RX_FQ: 39385d8dccf8SIoana Ciornei err = dpaa2_eth_setup_rx_flow(priv, &priv->fq[i]); 393934ff6846SIoana Radulescu break; 394034ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 39415d8dccf8SIoana Ciornei err = dpaa2_eth_setup_tx_flow(priv, &priv->fq[i]); 394234ff6846SIoana Radulescu break; 3943061d631fSIoana Ciornei case DPAA2_RX_ERR_FQ: 3944061d631fSIoana Ciornei err = setup_rx_err_flow(priv, &priv->fq[i]); 3945061d631fSIoana Ciornei break; 394634ff6846SIoana Radulescu default: 394734ff6846SIoana Radulescu dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 394834ff6846SIoana Radulescu return -EINVAL; 394934ff6846SIoana Radulescu } 395034ff6846SIoana Radulescu if (err) 395134ff6846SIoana Radulescu return err; 395234ff6846SIoana Radulescu } 395334ff6846SIoana Radulescu 395434ff6846SIoana Radulescu err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 395534ff6846SIoana Radulescu DPNI_QUEUE_TX, &priv->tx_qdid); 395634ff6846SIoana Radulescu if (err) { 395734ff6846SIoana Radulescu dev_err(dev, "dpni_get_qdid() failed\n"); 395834ff6846SIoana Radulescu return err; 395934ff6846SIoana Radulescu } 396034ff6846SIoana Radulescu 396134ff6846SIoana Radulescu return 0; 396234ff6846SIoana Radulescu } 396334ff6846SIoana Radulescu 396434ff6846SIoana Radulescu /* Allocate rings for storing incoming frame descriptors */ 39655d8dccf8SIoana Ciornei static int dpaa2_eth_alloc_rings(struct dpaa2_eth_priv *priv) 396634ff6846SIoana Radulescu { 396734ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 396834ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 396934ff6846SIoana Radulescu int i; 397034ff6846SIoana Radulescu 397134ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 397234ff6846SIoana Radulescu priv->channel[i]->store = 397334ff6846SIoana Radulescu dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 397434ff6846SIoana Radulescu if (!priv->channel[i]->store) { 397534ff6846SIoana Radulescu netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 397634ff6846SIoana Radulescu goto err_ring; 397734ff6846SIoana Radulescu } 397834ff6846SIoana Radulescu } 397934ff6846SIoana Radulescu 398034ff6846SIoana Radulescu return 0; 398134ff6846SIoana Radulescu 398234ff6846SIoana Radulescu err_ring: 398334ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 398434ff6846SIoana Radulescu if (!priv->channel[i]->store) 398534ff6846SIoana Radulescu break; 398634ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 398734ff6846SIoana Radulescu } 398834ff6846SIoana Radulescu 398934ff6846SIoana Radulescu return -ENOMEM; 399034ff6846SIoana Radulescu } 399134ff6846SIoana Radulescu 39925d8dccf8SIoana Ciornei static void dpaa2_eth_free_rings(struct dpaa2_eth_priv *priv) 399334ff6846SIoana Radulescu { 399434ff6846SIoana Radulescu int i; 399534ff6846SIoana Radulescu 399634ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 399734ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 399834ff6846SIoana Radulescu } 399934ff6846SIoana Radulescu 40005d8dccf8SIoana Ciornei static int dpaa2_eth_set_mac_addr(struct dpaa2_eth_priv *priv) 400134ff6846SIoana Radulescu { 400234ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 400334ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 400434ff6846SIoana Radulescu u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 400534ff6846SIoana Radulescu int err; 400634ff6846SIoana Radulescu 400734ff6846SIoana Radulescu /* Get firmware address, if any */ 400834ff6846SIoana Radulescu err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 400934ff6846SIoana Radulescu if (err) { 401034ff6846SIoana Radulescu dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 401134ff6846SIoana Radulescu return err; 401234ff6846SIoana Radulescu } 401334ff6846SIoana Radulescu 401434ff6846SIoana Radulescu /* Get DPNI attributes address, if any */ 401534ff6846SIoana Radulescu err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 401634ff6846SIoana Radulescu dpni_mac_addr); 401734ff6846SIoana Radulescu if (err) { 401834ff6846SIoana Radulescu dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 401934ff6846SIoana Radulescu return err; 402034ff6846SIoana Radulescu } 402134ff6846SIoana Radulescu 402234ff6846SIoana Radulescu /* First check if firmware has any address configured by bootloader */ 402334ff6846SIoana Radulescu if (!is_zero_ether_addr(mac_addr)) { 402434ff6846SIoana Radulescu /* If the DPMAC addr != DPNI addr, update it */ 402534ff6846SIoana Radulescu if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 402634ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, 402734ff6846SIoana Radulescu priv->mc_token, 402834ff6846SIoana Radulescu mac_addr); 402934ff6846SIoana Radulescu if (err) { 403034ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 403134ff6846SIoana Radulescu return err; 403234ff6846SIoana Radulescu } 403334ff6846SIoana Radulescu } 4034a05e4c0aSJakub Kicinski eth_hw_addr_set(net_dev, mac_addr); 403534ff6846SIoana Radulescu } else if (is_zero_ether_addr(dpni_mac_addr)) { 403634ff6846SIoana Radulescu /* No MAC address configured, fill in net_dev->dev_addr 403734ff6846SIoana Radulescu * with a random one 403834ff6846SIoana Radulescu */ 403934ff6846SIoana Radulescu eth_hw_addr_random(net_dev); 404034ff6846SIoana Radulescu dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 404134ff6846SIoana Radulescu 404234ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 404334ff6846SIoana Radulescu net_dev->dev_addr); 404434ff6846SIoana Radulescu if (err) { 404534ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 404634ff6846SIoana Radulescu return err; 404734ff6846SIoana Radulescu } 404834ff6846SIoana Radulescu 404934ff6846SIoana Radulescu /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 405034ff6846SIoana Radulescu * practical purposes, this will be our "permanent" mac address, 405134ff6846SIoana Radulescu * at least until the next reboot. This move will also permit 405234ff6846SIoana Radulescu * register_netdevice() to properly fill up net_dev->perm_addr. 405334ff6846SIoana Radulescu */ 405434ff6846SIoana Radulescu net_dev->addr_assign_type = NET_ADDR_PERM; 405534ff6846SIoana Radulescu } else { 405634ff6846SIoana Radulescu /* NET_ADDR_PERM is default, all we have to do is 405734ff6846SIoana Radulescu * fill in the device addr. 405834ff6846SIoana Radulescu */ 4059a05e4c0aSJakub Kicinski eth_hw_addr_set(net_dev, dpni_mac_addr); 406034ff6846SIoana Radulescu } 406134ff6846SIoana Radulescu 406234ff6846SIoana Radulescu return 0; 406334ff6846SIoana Radulescu } 406434ff6846SIoana Radulescu 40655d8dccf8SIoana Ciornei static int dpaa2_eth_netdev_init(struct net_device *net_dev) 406634ff6846SIoana Radulescu { 406734ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 406834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 406934ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 407034ff6846SIoana Radulescu u64 supported = 0, not_supported = 0; 407134ff6846SIoana Radulescu u8 bcast_addr[ETH_ALEN]; 407234ff6846SIoana Radulescu u8 num_queues; 407334ff6846SIoana Radulescu int err; 407434ff6846SIoana Radulescu 407534ff6846SIoana Radulescu net_dev->netdev_ops = &dpaa2_eth_ops; 407634ff6846SIoana Radulescu net_dev->ethtool_ops = &dpaa2_ethtool_ops; 407734ff6846SIoana Radulescu 40785d8dccf8SIoana Ciornei err = dpaa2_eth_set_mac_addr(priv); 407934ff6846SIoana Radulescu if (err) 408034ff6846SIoana Radulescu return err; 408134ff6846SIoana Radulescu 408234ff6846SIoana Radulescu /* Explicitly add the broadcast address to the MAC filtering table */ 408334ff6846SIoana Radulescu eth_broadcast_addr(bcast_addr); 408434ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 408534ff6846SIoana Radulescu if (err) { 408634ff6846SIoana Radulescu dev_err(dev, "dpni_add_mac_addr() failed\n"); 408734ff6846SIoana Radulescu return err; 408834ff6846SIoana Radulescu } 408934ff6846SIoana Radulescu 409034ff6846SIoana Radulescu /* Set MTU upper limit; lower limit is 68B (default value) */ 409134ff6846SIoana Radulescu net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 409234ff6846SIoana Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 409334ff6846SIoana Radulescu DPAA2_ETH_MFL); 409434ff6846SIoana Radulescu if (err) { 409534ff6846SIoana Radulescu dev_err(dev, "dpni_set_max_frame_length() failed\n"); 409634ff6846SIoana Radulescu return err; 409734ff6846SIoana Radulescu } 409834ff6846SIoana Radulescu 409934ff6846SIoana Radulescu /* Set actual number of queues in the net device */ 410034ff6846SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 410134ff6846SIoana Radulescu err = netif_set_real_num_tx_queues(net_dev, num_queues); 410234ff6846SIoana Radulescu if (err) { 410334ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 410434ff6846SIoana Radulescu return err; 410534ff6846SIoana Radulescu } 410634ff6846SIoana Radulescu err = netif_set_real_num_rx_queues(net_dev, num_queues); 410734ff6846SIoana Radulescu if (err) { 410834ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 410934ff6846SIoana Radulescu return err; 411034ff6846SIoana Radulescu } 411134ff6846SIoana Radulescu 411234ff6846SIoana Radulescu /* Capabilities listing */ 411334ff6846SIoana Radulescu supported |= IFF_LIVE_ADDR_CHANGE; 411434ff6846SIoana Radulescu 411534ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER) 411634ff6846SIoana Radulescu not_supported |= IFF_UNICAST_FLT; 411734ff6846SIoana Radulescu else 411834ff6846SIoana Radulescu supported |= IFF_UNICAST_FLT; 411934ff6846SIoana Radulescu 412034ff6846SIoana Radulescu net_dev->priv_flags |= supported; 412134ff6846SIoana Radulescu net_dev->priv_flags &= ~not_supported; 412234ff6846SIoana Radulescu 412334ff6846SIoana Radulescu /* Features */ 412434ff6846SIoana Radulescu net_dev->features = NETIF_F_RXCSUM | 412534ff6846SIoana Radulescu NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 412634ff6846SIoana Radulescu NETIF_F_SG | NETIF_F_HIGHDMA | 41273657cdafSIoana Ciornei NETIF_F_LLTX | NETIF_F_HW_TC; 412834ff6846SIoana Radulescu net_dev->hw_features = net_dev->features; 412934ff6846SIoana Radulescu 413070b32d82SIonut-robert Aron if (priv->dpni_attrs.vlan_filter_entries) 413170b32d82SIonut-robert Aron net_dev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 413270b32d82SIonut-robert Aron 413334ff6846SIoana Radulescu return 0; 413434ff6846SIoana Radulescu } 413534ff6846SIoana Radulescu 41365d8dccf8SIoana Ciornei static int dpaa2_eth_poll_link_state(void *arg) 413734ff6846SIoana Radulescu { 413834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 413934ff6846SIoana Radulescu int err; 414034ff6846SIoana Radulescu 414134ff6846SIoana Radulescu while (!kthread_should_stop()) { 41425d8dccf8SIoana Ciornei err = dpaa2_eth_link_state_update(priv); 414334ff6846SIoana Radulescu if (unlikely(err)) 414434ff6846SIoana Radulescu return err; 414534ff6846SIoana Radulescu 414634ff6846SIoana Radulescu msleep(DPAA2_ETH_LINK_STATE_REFRESH); 414734ff6846SIoana Radulescu } 414834ff6846SIoana Radulescu 414934ff6846SIoana Radulescu return 0; 415034ff6846SIoana Radulescu } 415134ff6846SIoana Radulescu 415271947923SIoana Ciornei static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) 415371947923SIoana Ciornei { 415471947923SIoana Ciornei struct fsl_mc_device *dpni_dev, *dpmac_dev; 415571947923SIoana Ciornei struct dpaa2_mac *mac; 415671947923SIoana Ciornei int err; 415771947923SIoana Ciornei 415871947923SIoana Ciornei dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); 415927cfdaddSIoana Ciornei dpmac_dev = fsl_mc_get_endpoint(dpni_dev, 0); 416047325da2SIoana Ciornei 416147325da2SIoana Ciornei if (PTR_ERR(dpmac_dev) == -EPROBE_DEFER) 416247325da2SIoana Ciornei return PTR_ERR(dpmac_dev); 416347325da2SIoana Ciornei 416447325da2SIoana Ciornei if (IS_ERR(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) 416571947923SIoana Ciornei return 0; 416671947923SIoana Ciornei 416771947923SIoana Ciornei mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); 416871947923SIoana Ciornei if (!mac) 416971947923SIoana Ciornei return -ENOMEM; 417071947923SIoana Ciornei 417171947923SIoana Ciornei mac->mc_dev = dpmac_dev; 417271947923SIoana Ciornei mac->mc_io = priv->mc_io; 417371947923SIoana Ciornei mac->net_dev = priv->net_dev; 417471947923SIoana Ciornei 4175095dca16SIoana Ciornei err = dpaa2_mac_open(mac); 4176095dca16SIoana Ciornei if (err) 4177095dca16SIoana Ciornei goto err_free_mac; 4178d87e6063SIoana Ciornei priv->mac = mac; 4179095dca16SIoana Ciornei 4180d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 418171947923SIoana Ciornei err = dpaa2_mac_connect(mac); 4182f5120f59SVladimir Oltean if (err && err != -EPROBE_DEFER) 4183f5120f59SVladimir Oltean netdev_err(priv->net_dev, "Error connecting to the MAC endpoint: %pe", 4184f5120f59SVladimir Oltean ERR_PTR(err)); 4185f5120f59SVladimir Oltean if (err) 4186095dca16SIoana Ciornei goto err_close_mac; 418771947923SIoana Ciornei } 418871947923SIoana Ciornei 418971947923SIoana Ciornei return 0; 4190095dca16SIoana Ciornei 4191095dca16SIoana Ciornei err_close_mac: 4192095dca16SIoana Ciornei dpaa2_mac_close(mac); 4193d87e6063SIoana Ciornei priv->mac = NULL; 4194095dca16SIoana Ciornei err_free_mac: 4195095dca16SIoana Ciornei kfree(mac); 4196095dca16SIoana Ciornei return err; 419771947923SIoana Ciornei } 419871947923SIoana Ciornei 419971947923SIoana Ciornei static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) 420071947923SIoana Ciornei { 4201d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 420271947923SIoana Ciornei dpaa2_mac_disconnect(priv->mac); 4203d87e6063SIoana Ciornei 4204848c1903SIoana Ciornei if (!dpaa2_eth_has_mac(priv)) 4205848c1903SIoana Ciornei return; 4206848c1903SIoana Ciornei 4207095dca16SIoana Ciornei dpaa2_mac_close(priv->mac); 420871947923SIoana Ciornei kfree(priv->mac); 420971947923SIoana Ciornei priv->mac = NULL; 421071947923SIoana Ciornei } 421171947923SIoana Ciornei 421234ff6846SIoana Radulescu static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 421334ff6846SIoana Radulescu { 421434ff6846SIoana Radulescu u32 status = ~0; 421534ff6846SIoana Radulescu struct device *dev = (struct device *)arg; 421634ff6846SIoana Radulescu struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 421734ff6846SIoana Radulescu struct net_device *net_dev = dev_get_drvdata(dev); 421871947923SIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 421934ff6846SIoana Radulescu int err; 422034ff6846SIoana Radulescu 422134ff6846SIoana Radulescu err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 422234ff6846SIoana Radulescu DPNI_IRQ_INDEX, &status); 422334ff6846SIoana Radulescu if (unlikely(err)) { 422434ff6846SIoana Radulescu netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 422534ff6846SIoana Radulescu return IRQ_HANDLED; 422634ff6846SIoana Radulescu } 422734ff6846SIoana Radulescu 422834ff6846SIoana Radulescu if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 42295d8dccf8SIoana Ciornei dpaa2_eth_link_state_update(netdev_priv(net_dev)); 423034ff6846SIoana Radulescu 4231f5c3fffaSIoana Ciornei if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) { 42325d8dccf8SIoana Ciornei dpaa2_eth_set_mac_addr(netdev_priv(net_dev)); 42335d8dccf8SIoana Ciornei dpaa2_eth_update_tx_fqids(priv); 423471947923SIoana Ciornei 423571947923SIoana Ciornei rtnl_lock(); 4236d87e6063SIoana Ciornei if (dpaa2_eth_has_mac(priv)) 423771947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 423871947923SIoana Ciornei else 423971947923SIoana Ciornei dpaa2_eth_connect_mac(priv); 424071947923SIoana Ciornei rtnl_unlock(); 4241f5c3fffaSIoana Ciornei } 42428398b375SFlorin Chiculita 424334ff6846SIoana Radulescu return IRQ_HANDLED; 424434ff6846SIoana Radulescu } 424534ff6846SIoana Radulescu 42465d8dccf8SIoana Ciornei static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) 424734ff6846SIoana Radulescu { 424834ff6846SIoana Radulescu int err = 0; 424934ff6846SIoana Radulescu struct fsl_mc_device_irq *irq; 425034ff6846SIoana Radulescu 425134ff6846SIoana Radulescu err = fsl_mc_allocate_irqs(ls_dev); 425234ff6846SIoana Radulescu if (err) { 425334ff6846SIoana Radulescu dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 425434ff6846SIoana Radulescu return err; 425534ff6846SIoana Radulescu } 425634ff6846SIoana Radulescu 425734ff6846SIoana Radulescu irq = ls_dev->irqs[0]; 4258d86a6d47SThomas Gleixner err = devm_request_threaded_irq(&ls_dev->dev, irq->virq, 425934ff6846SIoana Radulescu NULL, dpni_irq0_handler_thread, 426034ff6846SIoana Radulescu IRQF_NO_SUSPEND | IRQF_ONESHOT, 426134ff6846SIoana Radulescu dev_name(&ls_dev->dev), &ls_dev->dev); 426234ff6846SIoana Radulescu if (err < 0) { 426334ff6846SIoana Radulescu dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 426434ff6846SIoana Radulescu goto free_mc_irq; 426534ff6846SIoana Radulescu } 426634ff6846SIoana Radulescu 426734ff6846SIoana Radulescu err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 42688398b375SFlorin Chiculita DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 42698398b375SFlorin Chiculita DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 427034ff6846SIoana Radulescu if (err < 0) { 427134ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 427234ff6846SIoana Radulescu goto free_irq; 427334ff6846SIoana Radulescu } 427434ff6846SIoana Radulescu 427534ff6846SIoana Radulescu err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 427634ff6846SIoana Radulescu DPNI_IRQ_INDEX, 1); 427734ff6846SIoana Radulescu if (err < 0) { 427834ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 427934ff6846SIoana Radulescu goto free_irq; 428034ff6846SIoana Radulescu } 428134ff6846SIoana Radulescu 428234ff6846SIoana Radulescu return 0; 428334ff6846SIoana Radulescu 428434ff6846SIoana Radulescu free_irq: 4285d86a6d47SThomas Gleixner devm_free_irq(&ls_dev->dev, irq->virq, &ls_dev->dev); 428634ff6846SIoana Radulescu free_mc_irq: 428734ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 428834ff6846SIoana Radulescu 428934ff6846SIoana Radulescu return err; 429034ff6846SIoana Radulescu } 429134ff6846SIoana Radulescu 42925d8dccf8SIoana Ciornei static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv) 429334ff6846SIoana Radulescu { 429434ff6846SIoana Radulescu int i; 429534ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 429634ff6846SIoana Radulescu 429734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 429834ff6846SIoana Radulescu ch = priv->channel[i]; 429934ff6846SIoana Radulescu /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 430034ff6846SIoana Radulescu netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 430134ff6846SIoana Radulescu NAPI_POLL_WEIGHT); 430234ff6846SIoana Radulescu } 430334ff6846SIoana Radulescu } 430434ff6846SIoana Radulescu 43055d8dccf8SIoana Ciornei static void dpaa2_eth_del_ch_napi(struct dpaa2_eth_priv *priv) 430634ff6846SIoana Radulescu { 430734ff6846SIoana Radulescu int i; 430834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 430934ff6846SIoana Radulescu 431034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 431134ff6846SIoana Radulescu ch = priv->channel[i]; 431234ff6846SIoana Radulescu netif_napi_del(&ch->napi); 431334ff6846SIoana Radulescu } 431434ff6846SIoana Radulescu } 431534ff6846SIoana Radulescu 431634ff6846SIoana Radulescu static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 431734ff6846SIoana Radulescu { 431834ff6846SIoana Radulescu struct device *dev; 431934ff6846SIoana Radulescu struct net_device *net_dev = NULL; 432034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = NULL; 432134ff6846SIoana Radulescu int err = 0; 432234ff6846SIoana Radulescu 432334ff6846SIoana Radulescu dev = &dpni_dev->dev; 432434ff6846SIoana Radulescu 432534ff6846SIoana Radulescu /* Net device */ 4326ab1e6de2SIoana Radulescu net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 432734ff6846SIoana Radulescu if (!net_dev) { 432834ff6846SIoana Radulescu dev_err(dev, "alloc_etherdev_mq() failed\n"); 432934ff6846SIoana Radulescu return -ENOMEM; 433034ff6846SIoana Radulescu } 433134ff6846SIoana Radulescu 433234ff6846SIoana Radulescu SET_NETDEV_DEV(net_dev, dev); 433334ff6846SIoana Radulescu dev_set_drvdata(dev, net_dev); 433434ff6846SIoana Radulescu 433534ff6846SIoana Radulescu priv = netdev_priv(net_dev); 433634ff6846SIoana Radulescu priv->net_dev = net_dev; 433734ff6846SIoana Radulescu 433834ff6846SIoana Radulescu priv->iommu_domain = iommu_get_domain_for_dev(dev); 433934ff6846SIoana Radulescu 43401cf773bdSYangbo Lu priv->tx_tstamp_type = HWTSTAMP_TX_OFF; 43411cf773bdSYangbo Lu priv->rx_tstamp = false; 43421cf773bdSYangbo Lu 4343c5521189SYangbo Lu priv->dpaa2_ptp_wq = alloc_workqueue("dpaa2_ptp_wq", 0, 0); 4344c5521189SYangbo Lu if (!priv->dpaa2_ptp_wq) { 4345c5521189SYangbo Lu err = -ENOMEM; 4346c5521189SYangbo Lu goto err_wq_alloc; 4347c5521189SYangbo Lu } 4348c5521189SYangbo Lu 4349c5521189SYangbo Lu INIT_WORK(&priv->tx_onestep_tstamp, dpaa2_eth_tx_onestep_tstamp); 4350c5521189SYangbo Lu 4351c5521189SYangbo Lu skb_queue_head_init(&priv->tx_skbs); 4352c5521189SYangbo Lu 43538ed3cefcSIoana Ciornei priv->rx_copybreak = DPAA2_ETH_DEFAULT_COPYBREAK; 43548ed3cefcSIoana Ciornei 435534ff6846SIoana Radulescu /* Obtain a MC portal */ 435634ff6846SIoana Radulescu err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 435734ff6846SIoana Radulescu &priv->mc_io); 435834ff6846SIoana Radulescu if (err) { 435934ff6846SIoana Radulescu if (err == -ENXIO) 436034ff6846SIoana Radulescu err = -EPROBE_DEFER; 436134ff6846SIoana Radulescu else 436234ff6846SIoana Radulescu dev_err(dev, "MC portal allocation failed\n"); 436334ff6846SIoana Radulescu goto err_portal_alloc; 436434ff6846SIoana Radulescu } 436534ff6846SIoana Radulescu 436634ff6846SIoana Radulescu /* MC objects initialization and configuration */ 43675d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpni(dpni_dev); 436834ff6846SIoana Radulescu if (err) 436934ff6846SIoana Radulescu goto err_dpni_setup; 437034ff6846SIoana Radulescu 43715d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpio(priv); 437234ff6846SIoana Radulescu if (err) 437334ff6846SIoana Radulescu goto err_dpio_setup; 437434ff6846SIoana Radulescu 43755d8dccf8SIoana Ciornei dpaa2_eth_setup_fqs(priv); 437634ff6846SIoana Radulescu 43775d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpbp(priv); 437834ff6846SIoana Radulescu if (err) 437934ff6846SIoana Radulescu goto err_dpbp_setup; 438034ff6846SIoana Radulescu 43815d8dccf8SIoana Ciornei err = dpaa2_eth_bind_dpni(priv); 438234ff6846SIoana Radulescu if (err) 438334ff6846SIoana Radulescu goto err_bind; 438434ff6846SIoana Radulescu 438534ff6846SIoana Radulescu /* Add a NAPI context for each channel */ 43865d8dccf8SIoana Ciornei dpaa2_eth_add_ch_napi(priv); 438734ff6846SIoana Radulescu 438834ff6846SIoana Radulescu /* Percpu statistics */ 438934ff6846SIoana Radulescu priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 439034ff6846SIoana Radulescu if (!priv->percpu_stats) { 439134ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 439234ff6846SIoana Radulescu err = -ENOMEM; 439334ff6846SIoana Radulescu goto err_alloc_percpu_stats; 439434ff6846SIoana Radulescu } 439534ff6846SIoana Radulescu priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 439634ff6846SIoana Radulescu if (!priv->percpu_extras) { 439734ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 439834ff6846SIoana Radulescu err = -ENOMEM; 439934ff6846SIoana Radulescu goto err_alloc_percpu_extras; 440034ff6846SIoana Radulescu } 440134ff6846SIoana Radulescu 4402d70446eeSIoana Ciornei priv->sgt_cache = alloc_percpu(*priv->sgt_cache); 4403d70446eeSIoana Ciornei if (!priv->sgt_cache) { 4404d70446eeSIoana Ciornei dev_err(dev, "alloc_percpu(sgt_cache) failed\n"); 4405d70446eeSIoana Ciornei err = -ENOMEM; 4406d70446eeSIoana Ciornei goto err_alloc_sgt_cache; 4407d70446eeSIoana Ciornei } 4408d70446eeSIoana Ciornei 44095d8dccf8SIoana Ciornei err = dpaa2_eth_netdev_init(net_dev); 441034ff6846SIoana Radulescu if (err) 441134ff6846SIoana Radulescu goto err_netdev_init; 441234ff6846SIoana Radulescu 441334ff6846SIoana Radulescu /* Configure checksum offload based on current interface flags */ 44145d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 441534ff6846SIoana Radulescu if (err) 441634ff6846SIoana Radulescu goto err_csum; 441734ff6846SIoana Radulescu 44185d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, 44195d8dccf8SIoana Ciornei !!(net_dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 442034ff6846SIoana Radulescu if (err) 442134ff6846SIoana Radulescu goto err_csum; 442234ff6846SIoana Radulescu 44235d8dccf8SIoana Ciornei err = dpaa2_eth_alloc_rings(priv); 442434ff6846SIoana Radulescu if (err) 442534ff6846SIoana Radulescu goto err_alloc_rings; 442634ff6846SIoana Radulescu 4427f395b69fSIoana Ciornei #ifdef CONFIG_FSL_DPAA2_ETH_DCB 4428f395b69fSIoana Ciornei if (dpaa2_eth_has_pause_support(priv) && priv->vlan_cls_enabled) { 4429f395b69fSIoana Ciornei priv->dcbx_mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; 4430f395b69fSIoana Ciornei net_dev->dcbnl_ops = &dpaa2_eth_dcbnl_ops; 4431f395b69fSIoana Ciornei } else { 4432f395b69fSIoana Ciornei dev_dbg(dev, "PFC not supported\n"); 4433f395b69fSIoana Ciornei } 4434f395b69fSIoana Ciornei #endif 4435f395b69fSIoana Ciornei 44365d8dccf8SIoana Ciornei err = dpaa2_eth_setup_irqs(dpni_dev); 443734ff6846SIoana Radulescu if (err) { 443834ff6846SIoana Radulescu netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 44395d8dccf8SIoana Ciornei priv->poll_thread = kthread_run(dpaa2_eth_poll_link_state, priv, 444034ff6846SIoana Radulescu "%s_poll_link", net_dev->name); 444134ff6846SIoana Radulescu if (IS_ERR(priv->poll_thread)) { 444234ff6846SIoana Radulescu dev_err(dev, "Error starting polling thread\n"); 444334ff6846SIoana Radulescu goto err_poll_thread; 444434ff6846SIoana Radulescu } 444534ff6846SIoana Radulescu priv->do_link_poll = true; 444634ff6846SIoana Radulescu } 444734ff6846SIoana Radulescu 444871947923SIoana Ciornei err = dpaa2_eth_connect_mac(priv); 444971947923SIoana Ciornei if (err) 445071947923SIoana Ciornei goto err_connect_mac; 445171947923SIoana Ciornei 4452bbb9ae25SLeon Romanovsky err = dpaa2_eth_dl_alloc(priv); 4453ceeb03adSIoana Ciornei if (err) 4454ceeb03adSIoana Ciornei goto err_dl_register; 4455ceeb03adSIoana Ciornei 4456061d631fSIoana Ciornei err = dpaa2_eth_dl_traps_register(priv); 4457061d631fSIoana Ciornei if (err) 4458061d631fSIoana Ciornei goto err_dl_trap_register; 4459061d631fSIoana Ciornei 4460ceeb03adSIoana Ciornei err = dpaa2_eth_dl_port_add(priv); 4461ceeb03adSIoana Ciornei if (err) 4462ceeb03adSIoana Ciornei goto err_dl_port_add; 4463ceeb03adSIoana Ciornei 446434ff6846SIoana Radulescu err = register_netdev(net_dev); 446534ff6846SIoana Radulescu if (err < 0) { 446634ff6846SIoana Radulescu dev_err(dev, "register_netdev() failed\n"); 446734ff6846SIoana Radulescu goto err_netdev_reg; 446834ff6846SIoana Radulescu } 446934ff6846SIoana Radulescu 4470091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4471091a19eaSIoana Radulescu dpaa2_dbg_add(priv); 4472091a19eaSIoana Radulescu #endif 4473091a19eaSIoana Radulescu 4474bbb9ae25SLeon Romanovsky dpaa2_eth_dl_register(priv); 447534ff6846SIoana Radulescu dev_info(dev, "Probed interface %s\n", net_dev->name); 447634ff6846SIoana Radulescu return 0; 447734ff6846SIoana Radulescu 447834ff6846SIoana Radulescu err_netdev_reg: 4479ceeb03adSIoana Ciornei dpaa2_eth_dl_port_del(priv); 4480ceeb03adSIoana Ciornei err_dl_port_add: 4481061d631fSIoana Ciornei dpaa2_eth_dl_traps_unregister(priv); 4482061d631fSIoana Ciornei err_dl_trap_register: 4483bbb9ae25SLeon Romanovsky dpaa2_eth_dl_free(priv); 4484ceeb03adSIoana Ciornei err_dl_register: 448571947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 448671947923SIoana Ciornei err_connect_mac: 448734ff6846SIoana Radulescu if (priv->do_link_poll) 448834ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 448934ff6846SIoana Radulescu else 449034ff6846SIoana Radulescu fsl_mc_free_irqs(dpni_dev); 449134ff6846SIoana Radulescu err_poll_thread: 44925d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 449334ff6846SIoana Radulescu err_alloc_rings: 449434ff6846SIoana Radulescu err_csum: 449534ff6846SIoana Radulescu err_netdev_init: 4496d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 4497d70446eeSIoana Ciornei err_alloc_sgt_cache: 449834ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 449934ff6846SIoana Radulescu err_alloc_percpu_extras: 450034ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 450134ff6846SIoana Radulescu err_alloc_percpu_stats: 45025d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 450334ff6846SIoana Radulescu err_bind: 45045d8dccf8SIoana Ciornei dpaa2_eth_free_dpbp(priv); 450534ff6846SIoana Radulescu err_dpbp_setup: 45065d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 450734ff6846SIoana Radulescu err_dpio_setup: 45085d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 450934ff6846SIoana Radulescu err_dpni_setup: 451034ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 451134ff6846SIoana Radulescu err_portal_alloc: 4512c5521189SYangbo Lu destroy_workqueue(priv->dpaa2_ptp_wq); 4513c5521189SYangbo Lu err_wq_alloc: 451434ff6846SIoana Radulescu dev_set_drvdata(dev, NULL); 451534ff6846SIoana Radulescu free_netdev(net_dev); 451634ff6846SIoana Radulescu 451734ff6846SIoana Radulescu return err; 451834ff6846SIoana Radulescu } 451934ff6846SIoana Radulescu 452034ff6846SIoana Radulescu static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 452134ff6846SIoana Radulescu { 452234ff6846SIoana Radulescu struct device *dev; 452334ff6846SIoana Radulescu struct net_device *net_dev; 452434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 452534ff6846SIoana Radulescu 452634ff6846SIoana Radulescu dev = &ls_dev->dev; 452734ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 452834ff6846SIoana Radulescu priv = netdev_priv(net_dev); 452934ff6846SIoana Radulescu 4530bbb9ae25SLeon Romanovsky dpaa2_eth_dl_unregister(priv); 4531bbb9ae25SLeon Romanovsky 4532091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4533091a19eaSIoana Radulescu dpaa2_dbg_remove(priv); 4534091a19eaSIoana Radulescu #endif 453571947923SIoana Ciornei rtnl_lock(); 453671947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 453771947923SIoana Ciornei rtnl_unlock(); 453871947923SIoana Ciornei 453934ff6846SIoana Radulescu unregister_netdev(net_dev); 454034ff6846SIoana Radulescu 4541ceeb03adSIoana Ciornei dpaa2_eth_dl_port_del(priv); 4542061d631fSIoana Ciornei dpaa2_eth_dl_traps_unregister(priv); 4543bbb9ae25SLeon Romanovsky dpaa2_eth_dl_free(priv); 4544ceeb03adSIoana Ciornei 454534ff6846SIoana Radulescu if (priv->do_link_poll) 454634ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 454734ff6846SIoana Radulescu else 454834ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 454934ff6846SIoana Radulescu 45505d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 4551d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 455234ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 455334ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 455434ff6846SIoana Radulescu 45555d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 45565d8dccf8SIoana Ciornei dpaa2_eth_free_dpbp(priv); 45575d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 45585d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 455934ff6846SIoana Radulescu 456034ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 456134ff6846SIoana Radulescu 4562f4a8adbfSDongliang Mu destroy_workqueue(priv->dpaa2_ptp_wq); 4563f4a8adbfSDongliang Mu 456434ff6846SIoana Radulescu dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 456534ff6846SIoana Radulescu 45669b5a3332SPavel Skripkin free_netdev(net_dev); 45679b5a3332SPavel Skripkin 456834ff6846SIoana Radulescu return 0; 456934ff6846SIoana Radulescu } 457034ff6846SIoana Radulescu 457134ff6846SIoana Radulescu static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 457234ff6846SIoana Radulescu { 457334ff6846SIoana Radulescu .vendor = FSL_MC_VENDOR_FREESCALE, 457434ff6846SIoana Radulescu .obj_type = "dpni", 457534ff6846SIoana Radulescu }, 457634ff6846SIoana Radulescu { .vendor = 0x0 } 457734ff6846SIoana Radulescu }; 457834ff6846SIoana Radulescu MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 457934ff6846SIoana Radulescu 458034ff6846SIoana Radulescu static struct fsl_mc_driver dpaa2_eth_driver = { 458134ff6846SIoana Radulescu .driver = { 458234ff6846SIoana Radulescu .name = KBUILD_MODNAME, 458334ff6846SIoana Radulescu .owner = THIS_MODULE, 458434ff6846SIoana Radulescu }, 458534ff6846SIoana Radulescu .probe = dpaa2_eth_probe, 458634ff6846SIoana Radulescu .remove = dpaa2_eth_remove, 458734ff6846SIoana Radulescu .match_id_table = dpaa2_eth_match_id_table 458834ff6846SIoana Radulescu }; 458934ff6846SIoana Radulescu 4590091a19eaSIoana Radulescu static int __init dpaa2_eth_driver_init(void) 4591091a19eaSIoana Radulescu { 4592091a19eaSIoana Radulescu int err; 4593091a19eaSIoana Radulescu 4594091a19eaSIoana Radulescu dpaa2_eth_dbg_init(); 4595091a19eaSIoana Radulescu err = fsl_mc_driver_register(&dpaa2_eth_driver); 4596091a19eaSIoana Radulescu if (err) { 4597091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4598091a19eaSIoana Radulescu return err; 4599091a19eaSIoana Radulescu } 4600091a19eaSIoana Radulescu 4601091a19eaSIoana Radulescu return 0; 4602091a19eaSIoana Radulescu } 4603091a19eaSIoana Radulescu 4604091a19eaSIoana Radulescu static void __exit dpaa2_eth_driver_exit(void) 4605091a19eaSIoana Radulescu { 4606091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4607091a19eaSIoana Radulescu fsl_mc_driver_unregister(&dpaa2_eth_driver); 4608091a19eaSIoana Radulescu } 4609091a19eaSIoana Radulescu 4610091a19eaSIoana Radulescu module_init(dpaa2_eth_driver_init); 4611091a19eaSIoana Radulescu module_exit(dpaa2_eth_driver_exit); 4612