134ff6846SIoana Radulescu // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 234ff6846SIoana Radulescu /* Copyright 2014-2016 Freescale Semiconductor Inc. 348c0481eSIoana Ciornei * Copyright 2016-2020 NXP 434ff6846SIoana Radulescu */ 534ff6846SIoana Radulescu #include <linux/init.h> 634ff6846SIoana Radulescu #include <linux/module.h> 734ff6846SIoana Radulescu #include <linux/platform_device.h> 834ff6846SIoana Radulescu #include <linux/etherdevice.h> 934ff6846SIoana Radulescu #include <linux/of_net.h> 1034ff6846SIoana Radulescu #include <linux/interrupt.h> 1134ff6846SIoana Radulescu #include <linux/msi.h> 1234ff6846SIoana Radulescu #include <linux/kthread.h> 1334ff6846SIoana Radulescu #include <linux/iommu.h> 1434ff6846SIoana Radulescu #include <linux/fsl/mc.h> 157e273a8eSIoana Ciocoi Radulescu #include <linux/bpf.h> 167e273a8eSIoana Ciocoi Radulescu #include <linux/bpf_trace.h> 17d21c784cSYangbo Lu #include <linux/fsl/ptp_qoriq.h> 18c5521189SYangbo Lu #include <linux/ptp_classify.h> 193657cdafSIoana Ciornei #include <net/pkt_cls.h> 2034ff6846SIoana Radulescu #include <net/sock.h> 2134ff6846SIoana Radulescu 2234ff6846SIoana Radulescu #include "dpaa2-eth.h" 2334ff6846SIoana Radulescu 2434ff6846SIoana Radulescu /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 2534ff6846SIoana Radulescu * using trace events only need to #include <trace/events/sched.h> 2634ff6846SIoana Radulescu */ 2734ff6846SIoana Radulescu #define CREATE_TRACE_POINTS 2834ff6846SIoana Radulescu #include "dpaa2-eth-trace.h" 2934ff6846SIoana Radulescu 3034ff6846SIoana Radulescu MODULE_LICENSE("Dual BSD/GPL"); 3134ff6846SIoana Radulescu MODULE_AUTHOR("Freescale Semiconductor, Inc"); 3234ff6846SIoana Radulescu MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 3334ff6846SIoana Radulescu 34d21c784cSYangbo Lu struct ptp_qoriq *dpaa2_ptp; 35d21c784cSYangbo Lu EXPORT_SYMBOL(dpaa2_ptp); 36d21c784cSYangbo Lu 3734ff6846SIoana Radulescu static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 3834ff6846SIoana Radulescu dma_addr_t iova_addr) 3934ff6846SIoana Radulescu { 4034ff6846SIoana Radulescu phys_addr_t phys_addr; 4134ff6846SIoana Radulescu 4234ff6846SIoana Radulescu phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 4334ff6846SIoana Radulescu 4434ff6846SIoana Radulescu return phys_to_virt(phys_addr); 4534ff6846SIoana Radulescu } 4634ff6846SIoana Radulescu 475d8dccf8SIoana Ciornei static void dpaa2_eth_validate_rx_csum(struct dpaa2_eth_priv *priv, 4834ff6846SIoana Radulescu u32 fd_status, 4934ff6846SIoana Radulescu struct sk_buff *skb) 5034ff6846SIoana Radulescu { 5134ff6846SIoana Radulescu skb_checksum_none_assert(skb); 5234ff6846SIoana Radulescu 5334ff6846SIoana Radulescu /* HW checksum validation is disabled, nothing to do here */ 5434ff6846SIoana Radulescu if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 5534ff6846SIoana Radulescu return; 5634ff6846SIoana Radulescu 5734ff6846SIoana Radulescu /* Read checksum validation bits */ 5834ff6846SIoana Radulescu if (!((fd_status & DPAA2_FAS_L3CV) && 5934ff6846SIoana Radulescu (fd_status & DPAA2_FAS_L4CV))) 6034ff6846SIoana Radulescu return; 6134ff6846SIoana Radulescu 6234ff6846SIoana Radulescu /* Inform the stack there's no need to compute L3/L4 csum anymore */ 6334ff6846SIoana Radulescu skb->ip_summed = CHECKSUM_UNNECESSARY; 6434ff6846SIoana Radulescu } 6534ff6846SIoana Radulescu 6634ff6846SIoana Radulescu /* Free a received FD. 6734ff6846SIoana Radulescu * Not to be used for Tx conf FDs or on any other paths. 6834ff6846SIoana Radulescu */ 695d8dccf8SIoana Ciornei static void dpaa2_eth_free_rx_fd(struct dpaa2_eth_priv *priv, 7034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 7134ff6846SIoana Radulescu void *vaddr) 7234ff6846SIoana Radulescu { 7334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 7434ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 7534ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 7634ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 7734ff6846SIoana Radulescu void *sg_vaddr; 7834ff6846SIoana Radulescu int i; 7934ff6846SIoana Radulescu 8034ff6846SIoana Radulescu /* If single buffer frame, just free the data buffer */ 8134ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) 8234ff6846SIoana Radulescu goto free_buf; 8334ff6846SIoana Radulescu else if (fd_format != dpaa2_fd_sg) 8434ff6846SIoana Radulescu /* We don't support any other format */ 8534ff6846SIoana Radulescu return; 8634ff6846SIoana Radulescu 8734ff6846SIoana Radulescu /* For S/G frames, we first need to free all SG entries 8834ff6846SIoana Radulescu * except the first one, which was taken care of already 8934ff6846SIoana Radulescu */ 9034ff6846SIoana Radulescu sgt = vaddr + dpaa2_fd_get_offset(fd); 9134ff6846SIoana Radulescu for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 9234ff6846SIoana Radulescu addr = dpaa2_sg_get_addr(&sgt[i]); 9334ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 94efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 9518c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 9634ff6846SIoana Radulescu 9727c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 9834ff6846SIoana Radulescu if (dpaa2_sg_is_final(&sgt[i])) 9934ff6846SIoana Radulescu break; 10034ff6846SIoana Radulescu } 10134ff6846SIoana Radulescu 10234ff6846SIoana Radulescu free_buf: 10327c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 10434ff6846SIoana Radulescu } 10534ff6846SIoana Radulescu 10634ff6846SIoana Radulescu /* Build a linear skb based on a single-buffer frame descriptor */ 1075d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_linear_skb(struct dpaa2_eth_channel *ch, 10834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 10934ff6846SIoana Radulescu void *fd_vaddr) 11034ff6846SIoana Radulescu { 11134ff6846SIoana Radulescu struct sk_buff *skb = NULL; 11234ff6846SIoana Radulescu u16 fd_offset = dpaa2_fd_get_offset(fd); 11334ff6846SIoana Radulescu u32 fd_length = dpaa2_fd_get_len(fd); 11434ff6846SIoana Radulescu 11534ff6846SIoana Radulescu ch->buf_count--; 11634ff6846SIoana Radulescu 11727c87486SIoana Ciocoi Radulescu skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 11834ff6846SIoana Radulescu if (unlikely(!skb)) 11934ff6846SIoana Radulescu return NULL; 12034ff6846SIoana Radulescu 12134ff6846SIoana Radulescu skb_reserve(skb, fd_offset); 12234ff6846SIoana Radulescu skb_put(skb, fd_length); 12334ff6846SIoana Radulescu 12434ff6846SIoana Radulescu return skb; 12534ff6846SIoana Radulescu } 12634ff6846SIoana Radulescu 12734ff6846SIoana Radulescu /* Build a non linear (fragmented) skb based on a S/G table */ 1285d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_frag_skb(struct dpaa2_eth_priv *priv, 12934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 13034ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt) 13134ff6846SIoana Radulescu { 13234ff6846SIoana Radulescu struct sk_buff *skb = NULL; 13334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 13434ff6846SIoana Radulescu void *sg_vaddr; 13534ff6846SIoana Radulescu dma_addr_t sg_addr; 13634ff6846SIoana Radulescu u16 sg_offset; 13734ff6846SIoana Radulescu u32 sg_length; 13834ff6846SIoana Radulescu struct page *page, *head_page; 13934ff6846SIoana Radulescu int page_offset; 14034ff6846SIoana Radulescu int i; 14134ff6846SIoana Radulescu 14234ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 14334ff6846SIoana Radulescu struct dpaa2_sg_entry *sge = &sgt[i]; 14434ff6846SIoana Radulescu 14534ff6846SIoana Radulescu /* NOTE: We only support SG entries in dpaa2_sg_single format, 14634ff6846SIoana Radulescu * but this is the only format we may receive from HW anyway 14734ff6846SIoana Radulescu */ 14834ff6846SIoana Radulescu 14934ff6846SIoana Radulescu /* Get the address and length from the S/G entry */ 15034ff6846SIoana Radulescu sg_addr = dpaa2_sg_get_addr(sge); 15134ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 152efa6a7d0SIoana Ciornei dma_unmap_page(dev, sg_addr, priv->rx_buf_size, 15318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 15434ff6846SIoana Radulescu 15534ff6846SIoana Radulescu sg_length = dpaa2_sg_get_len(sge); 15634ff6846SIoana Radulescu 15734ff6846SIoana Radulescu if (i == 0) { 15834ff6846SIoana Radulescu /* We build the skb around the first data buffer */ 15927c87486SIoana Ciocoi Radulescu skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 16034ff6846SIoana Radulescu if (unlikely(!skb)) { 16134ff6846SIoana Radulescu /* Free the first SG entry now, since we already 16234ff6846SIoana Radulescu * unmapped it and obtained the virtual address 16334ff6846SIoana Radulescu */ 16427c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 16534ff6846SIoana Radulescu 16634ff6846SIoana Radulescu /* We still need to subtract the buffers used 16734ff6846SIoana Radulescu * by this FD from our software counter 16834ff6846SIoana Radulescu */ 16934ff6846SIoana Radulescu while (!dpaa2_sg_is_final(&sgt[i]) && 17034ff6846SIoana Radulescu i < DPAA2_ETH_MAX_SG_ENTRIES) 17134ff6846SIoana Radulescu i++; 17234ff6846SIoana Radulescu break; 17334ff6846SIoana Radulescu } 17434ff6846SIoana Radulescu 17534ff6846SIoana Radulescu sg_offset = dpaa2_sg_get_offset(sge); 17634ff6846SIoana Radulescu skb_reserve(skb, sg_offset); 17734ff6846SIoana Radulescu skb_put(skb, sg_length); 17834ff6846SIoana Radulescu } else { 17934ff6846SIoana Radulescu /* Rest of the data buffers are stored as skb frags */ 18034ff6846SIoana Radulescu page = virt_to_page(sg_vaddr); 18134ff6846SIoana Radulescu head_page = virt_to_head_page(sg_vaddr); 18234ff6846SIoana Radulescu 18334ff6846SIoana Radulescu /* Offset in page (which may be compound). 18434ff6846SIoana Radulescu * Data in subsequent SG entries is stored from the 18534ff6846SIoana Radulescu * beginning of the buffer, so we don't need to add the 18634ff6846SIoana Radulescu * sg_offset. 18734ff6846SIoana Radulescu */ 18834ff6846SIoana Radulescu page_offset = ((unsigned long)sg_vaddr & 18934ff6846SIoana Radulescu (PAGE_SIZE - 1)) + 19034ff6846SIoana Radulescu (page_address(page) - page_address(head_page)); 19134ff6846SIoana Radulescu 19234ff6846SIoana Radulescu skb_add_rx_frag(skb, i - 1, head_page, page_offset, 193efa6a7d0SIoana Ciornei sg_length, priv->rx_buf_size); 19434ff6846SIoana Radulescu } 19534ff6846SIoana Radulescu 19634ff6846SIoana Radulescu if (dpaa2_sg_is_final(sge)) 19734ff6846SIoana Radulescu break; 19834ff6846SIoana Radulescu } 19934ff6846SIoana Radulescu 20034ff6846SIoana Radulescu WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 20134ff6846SIoana Radulescu 20234ff6846SIoana Radulescu /* Count all data buffers + SG table buffer */ 20334ff6846SIoana Radulescu ch->buf_count -= i + 2; 20434ff6846SIoana Radulescu 20534ff6846SIoana Radulescu return skb; 20634ff6846SIoana Radulescu } 20734ff6846SIoana Radulescu 208569375fbSIoana Ciocoi Radulescu /* Free buffers acquired from the buffer pool or which were meant to 209569375fbSIoana Ciocoi Radulescu * be released in the pool 210569375fbSIoana Ciocoi Radulescu */ 2115d8dccf8SIoana Ciornei static void dpaa2_eth_free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, 2125d8dccf8SIoana Ciornei int count) 213569375fbSIoana Ciocoi Radulescu { 214569375fbSIoana Ciocoi Radulescu struct device *dev = priv->net_dev->dev.parent; 215569375fbSIoana Ciocoi Radulescu void *vaddr; 216569375fbSIoana Ciocoi Radulescu int i; 217569375fbSIoana Ciocoi Radulescu 218569375fbSIoana Ciocoi Radulescu for (i = 0; i < count; i++) { 219569375fbSIoana Ciocoi Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 220efa6a7d0SIoana Ciornei dma_unmap_page(dev, buf_array[i], priv->rx_buf_size, 22118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 22227c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 223569375fbSIoana Ciocoi Radulescu } 224569375fbSIoana Ciocoi Radulescu } 225569375fbSIoana Ciocoi Radulescu 22628d137ccSIoana Ciornei static void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv, 2275d39dc21SIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 2285d39dc21SIoana Ciocoi Radulescu dma_addr_t addr) 2295d39dc21SIoana Ciocoi Radulescu { 230ef17bd7cSIoana Radulescu int retries = 0; 2315d39dc21SIoana Ciocoi Radulescu int err; 2325d39dc21SIoana Ciocoi Radulescu 23328d137ccSIoana Ciornei ch->recycled_bufs[ch->recycled_bufs_cnt++] = addr; 23428d137ccSIoana Ciornei if (ch->recycled_bufs_cnt < DPAA2_ETH_BUFS_PER_CMD) 2355d39dc21SIoana Ciocoi Radulescu return; 2365d39dc21SIoana Ciocoi Radulescu 2375d39dc21SIoana Ciocoi Radulescu while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 23828d137ccSIoana Ciornei ch->recycled_bufs, 23928d137ccSIoana Ciornei ch->recycled_bufs_cnt)) == -EBUSY) { 240ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 241ef17bd7cSIoana Radulescu break; 2425d39dc21SIoana Ciocoi Radulescu cpu_relax(); 243ef17bd7cSIoana Radulescu } 2445d39dc21SIoana Ciocoi Radulescu 2455d39dc21SIoana Ciocoi Radulescu if (err) { 24628d137ccSIoana Ciornei dpaa2_eth_free_bufs(priv, ch->recycled_bufs, ch->recycled_bufs_cnt); 24728d137ccSIoana Ciornei ch->buf_count -= ch->recycled_bufs_cnt; 2485d39dc21SIoana Ciocoi Radulescu } 2495d39dc21SIoana Ciocoi Radulescu 25028d137ccSIoana Ciornei ch->recycled_bufs_cnt = 0; 2515d39dc21SIoana Ciocoi Radulescu } 2525d39dc21SIoana Ciocoi Radulescu 25338c440b2SIoana Ciornei static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 25438c440b2SIoana Ciornei struct dpaa2_eth_fq *fq, 25538c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_fds) 25638c440b2SIoana Ciornei { 25738c440b2SIoana Ciornei int total_enqueued = 0, retries = 0, enqueued; 25838c440b2SIoana Ciornei struct dpaa2_eth_drv_stats *percpu_extras; 25938c440b2SIoana Ciornei int num_fds, err, max_retries; 26038c440b2SIoana Ciornei struct dpaa2_fd *fds; 26138c440b2SIoana Ciornei 26238c440b2SIoana Ciornei percpu_extras = this_cpu_ptr(priv->percpu_extras); 26338c440b2SIoana Ciornei 26438c440b2SIoana Ciornei /* try to enqueue all the FDs until the max number of retries is hit */ 26538c440b2SIoana Ciornei fds = xdp_fds->fds; 26638c440b2SIoana Ciornei num_fds = xdp_fds->num; 26738c440b2SIoana Ciornei max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 26838c440b2SIoana Ciornei while (total_enqueued < num_fds && retries < max_retries) { 26938c440b2SIoana Ciornei err = priv->enqueue(priv, fq, &fds[total_enqueued], 27038c440b2SIoana Ciornei 0, num_fds - total_enqueued, &enqueued); 27138c440b2SIoana Ciornei if (err == -EBUSY) { 27238c440b2SIoana Ciornei percpu_extras->tx_portal_busy += ++retries; 27338c440b2SIoana Ciornei continue; 27438c440b2SIoana Ciornei } 27538c440b2SIoana Ciornei total_enqueued += enqueued; 27638c440b2SIoana Ciornei } 27738c440b2SIoana Ciornei xdp_fds->num = 0; 27838c440b2SIoana Ciornei 27938c440b2SIoana Ciornei return total_enqueued; 28038c440b2SIoana Ciornei } 28138c440b2SIoana Ciornei 2825d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_tx_flush(struct dpaa2_eth_priv *priv, 28374a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 28474a1c059SIoana Ciornei struct dpaa2_eth_fq *fq) 28574a1c059SIoana Ciornei { 28674a1c059SIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 28774a1c059SIoana Ciornei struct dpaa2_fd *fds; 28874a1c059SIoana Ciornei int enqueued, i; 28974a1c059SIoana Ciornei 29074a1c059SIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 29174a1c059SIoana Ciornei 29274a1c059SIoana Ciornei // enqueue the array of XDP_TX frames 29374a1c059SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds); 29474a1c059SIoana Ciornei 29574a1c059SIoana Ciornei /* update statistics */ 29674a1c059SIoana Ciornei percpu_stats->tx_packets += enqueued; 29774a1c059SIoana Ciornei fds = fq->xdp_tx_fds.fds; 29874a1c059SIoana Ciornei for (i = 0; i < enqueued; i++) { 29974a1c059SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 30074a1c059SIoana Ciornei ch->stats.xdp_tx++; 30174a1c059SIoana Ciornei } 30274a1c059SIoana Ciornei for (i = enqueued; i < fq->xdp_tx_fds.num; i++) { 30328d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, dpaa2_fd_get_addr(&fds[i])); 30474a1c059SIoana Ciornei percpu_stats->tx_errors++; 30574a1c059SIoana Ciornei ch->stats.xdp_tx_err++; 30674a1c059SIoana Ciornei } 30774a1c059SIoana Ciornei fq->xdp_tx_fds.num = 0; 30874a1c059SIoana Ciornei } 30974a1c059SIoana Ciornei 3105d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv, 31174a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 31274a1c059SIoana Ciornei struct dpaa2_fd *fd, 31399e43521SIoana Ciocoi Radulescu void *buf_start, u16 queue_id) 31499e43521SIoana Ciocoi Radulescu { 31599e43521SIoana Ciocoi Radulescu struct dpaa2_faead *faead; 31674a1c059SIoana Ciornei struct dpaa2_fd *dest_fd; 31774a1c059SIoana Ciornei struct dpaa2_eth_fq *fq; 31899e43521SIoana Ciocoi Radulescu u32 ctrl, frc; 31999e43521SIoana Ciocoi Radulescu 32099e43521SIoana Ciocoi Radulescu /* Mark the egress frame hardware annotation area as valid */ 32199e43521SIoana Ciocoi Radulescu frc = dpaa2_fd_get_frc(fd); 32299e43521SIoana Ciocoi Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 32399e43521SIoana Ciocoi Radulescu dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 32499e43521SIoana Ciocoi Radulescu 32599e43521SIoana Ciocoi Radulescu /* Instruct hardware to release the FD buffer directly into 32699e43521SIoana Ciocoi Radulescu * the buffer pool once transmission is completed, instead of 32799e43521SIoana Ciocoi Radulescu * sending a Tx confirmation frame to us 32899e43521SIoana Ciocoi Radulescu */ 32999e43521SIoana Ciocoi Radulescu ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 33099e43521SIoana Ciocoi Radulescu faead = dpaa2_get_faead(buf_start, false); 33199e43521SIoana Ciocoi Radulescu faead->ctrl = cpu_to_le32(ctrl); 33299e43521SIoana Ciocoi Radulescu faead->conf_fqid = 0; 33399e43521SIoana Ciocoi Radulescu 33499e43521SIoana Ciocoi Radulescu fq = &priv->fq[queue_id]; 33574a1c059SIoana Ciornei dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++]; 33674a1c059SIoana Ciornei memcpy(dest_fd, fd, sizeof(*dest_fd)); 33799e43521SIoana Ciocoi Radulescu 33874a1c059SIoana Ciornei if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE) 33974a1c059SIoana Ciornei return; 34074a1c059SIoana Ciornei 3415d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, fq); 34299e43521SIoana Ciocoi Radulescu } 34399e43521SIoana Ciocoi Radulescu 3445d8dccf8SIoana Ciornei static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, 3457e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 34699e43521SIoana Ciocoi Radulescu struct dpaa2_eth_fq *rx_fq, 3477e273a8eSIoana Ciocoi Radulescu struct dpaa2_fd *fd, void *vaddr) 3487e273a8eSIoana Ciocoi Radulescu { 3495d39dc21SIoana Ciocoi Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 3507e273a8eSIoana Ciocoi Radulescu struct bpf_prog *xdp_prog; 3517e273a8eSIoana Ciocoi Radulescu struct xdp_buff xdp; 3527e273a8eSIoana Ciocoi Radulescu u32 xdp_act = XDP_PASS; 353be9df4afSLorenzo Bianconi int err, offset; 35499e43521SIoana Ciocoi Radulescu 3557e273a8eSIoana Ciocoi Radulescu rcu_read_lock(); 3567e273a8eSIoana Ciocoi Radulescu 3577e273a8eSIoana Ciocoi Radulescu xdp_prog = READ_ONCE(ch->xdp.prog); 3587e273a8eSIoana Ciocoi Radulescu if (!xdp_prog) 3597e273a8eSIoana Ciocoi Radulescu goto out; 3607e273a8eSIoana Ciocoi Radulescu 361be9df4afSLorenzo Bianconi offset = dpaa2_fd_get_offset(fd) - XDP_PACKET_HEADROOM; 362be9df4afSLorenzo Bianconi xdp_init_buff(&xdp, DPAA2_ETH_RX_BUF_RAW_SIZE - offset, &ch->xdp_rxq); 363be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, vaddr + offset, XDP_PACKET_HEADROOM, 364be9df4afSLorenzo Bianconi dpaa2_fd_get_len(fd), false); 3654a9b052aSJesper Dangaard Brouer 3667e273a8eSIoana Ciocoi Radulescu xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 3677e273a8eSIoana Ciocoi Radulescu 3687b1eea1aSIoana Ciocoi Radulescu /* xdp.data pointer may have changed */ 3697b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_offset(fd, xdp.data - vaddr); 3707b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 3717b1eea1aSIoana Ciocoi Radulescu 3727e273a8eSIoana Ciocoi Radulescu switch (xdp_act) { 3737e273a8eSIoana Ciocoi Radulescu case XDP_PASS: 3747e273a8eSIoana Ciocoi Radulescu break; 37599e43521SIoana Ciocoi Radulescu case XDP_TX: 3765d8dccf8SIoana Ciornei dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); 37799e43521SIoana Ciocoi Radulescu break; 3787e273a8eSIoana Ciocoi Radulescu default: 3797e273a8eSIoana Ciocoi Radulescu bpf_warn_invalid_xdp_action(xdp_act); 380df561f66SGustavo A. R. Silva fallthrough; 3817e273a8eSIoana Ciocoi Radulescu case XDP_ABORTED: 3827e273a8eSIoana Ciocoi Radulescu trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 383df561f66SGustavo A. R. Silva fallthrough; 3847e273a8eSIoana Ciocoi Radulescu case XDP_DROP: 38528d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, addr); 386a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_drop++; 3877e273a8eSIoana Ciocoi Radulescu break; 388d678be1dSIoana Radulescu case XDP_REDIRECT: 389d678be1dSIoana Radulescu dma_unmap_page(priv->net_dev->dev.parent, addr, 390efa6a7d0SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 391d678be1dSIoana Radulescu ch->buf_count--; 3924a9b052aSJesper Dangaard Brouer 3934a9b052aSJesper Dangaard Brouer /* Allow redirect use of full headroom */ 394d678be1dSIoana Radulescu xdp.data_hard_start = vaddr; 3954a9b052aSJesper Dangaard Brouer xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE; 3964a9b052aSJesper Dangaard Brouer 397d678be1dSIoana Radulescu err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 398e12be913SIoana Ciornei if (unlikely(err)) { 399e12be913SIoana Ciornei addr = dma_map_page(priv->net_dev->dev.parent, 400e12be913SIoana Ciornei virt_to_page(vaddr), 0, 401e12be913SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 402e12be913SIoana Ciornei if (unlikely(dma_mapping_error(priv->net_dev->dev.parent, addr))) { 403e12be913SIoana Ciornei free_pages((unsigned long)vaddr, 0); 404e12be913SIoana Ciornei } else { 405e12be913SIoana Ciornei ch->buf_count++; 40628d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, addr); 407e12be913SIoana Ciornei } 408d678be1dSIoana Radulescu ch->stats.xdp_drop++; 409e12be913SIoana Ciornei } else { 410d678be1dSIoana Radulescu ch->stats.xdp_redirect++; 411e12be913SIoana Ciornei } 412d678be1dSIoana Radulescu break; 4137e273a8eSIoana Ciocoi Radulescu } 4147e273a8eSIoana Ciocoi Radulescu 415d678be1dSIoana Radulescu ch->xdp.res |= xdp_act; 4167e273a8eSIoana Ciocoi Radulescu out: 4177e273a8eSIoana Ciocoi Radulescu rcu_read_unlock(); 4187e273a8eSIoana Ciocoi Radulescu return xdp_act; 4197e273a8eSIoana Ciocoi Radulescu } 4207e273a8eSIoana Ciocoi Radulescu 421*50f82699SIoana Ciornei static struct sk_buff *dpaa2_eth_copybreak(struct dpaa2_eth_channel *ch, 422*50f82699SIoana Ciornei const struct dpaa2_fd *fd, 423*50f82699SIoana Ciornei void *fd_vaddr) 424*50f82699SIoana Ciornei { 425*50f82699SIoana Ciornei u16 fd_offset = dpaa2_fd_get_offset(fd); 426*50f82699SIoana Ciornei u32 fd_length = dpaa2_fd_get_len(fd); 427*50f82699SIoana Ciornei struct sk_buff *skb = NULL; 428*50f82699SIoana Ciornei unsigned int skb_len; 429*50f82699SIoana Ciornei 430*50f82699SIoana Ciornei if (fd_length > DPAA2_ETH_DEFAULT_COPYBREAK) 431*50f82699SIoana Ciornei return NULL; 432*50f82699SIoana Ciornei 433*50f82699SIoana Ciornei skb_len = fd_length + dpaa2_eth_needed_headroom(NULL); 434*50f82699SIoana Ciornei 435*50f82699SIoana Ciornei skb = napi_alloc_skb(&ch->napi, skb_len); 436*50f82699SIoana Ciornei if (!skb) 437*50f82699SIoana Ciornei return NULL; 438*50f82699SIoana Ciornei 439*50f82699SIoana Ciornei skb_reserve(skb, dpaa2_eth_needed_headroom(NULL)); 440*50f82699SIoana Ciornei skb_put(skb, fd_length); 441*50f82699SIoana Ciornei 442*50f82699SIoana Ciornei memcpy(skb->data, fd_vaddr + fd_offset, fd_length); 443*50f82699SIoana Ciornei 444*50f82699SIoana Ciornei dpaa2_eth_recycle_buf(ch->priv, ch, dpaa2_fd_get_addr(fd)); 445*50f82699SIoana Ciornei 446*50f82699SIoana Ciornei return skb; 447*50f82699SIoana Ciornei } 448*50f82699SIoana Ciornei 44934ff6846SIoana Radulescu /* Main Rx frame processing routine */ 45034ff6846SIoana Radulescu static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 45134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 45234ff6846SIoana Radulescu const struct dpaa2_fd *fd, 453dbcdf728SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 45434ff6846SIoana Radulescu { 45534ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 45634ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 45734ff6846SIoana Radulescu void *vaddr; 45834ff6846SIoana Radulescu struct sk_buff *skb; 45934ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 46034ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 46134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 46234ff6846SIoana Radulescu struct dpaa2_fas *fas; 46334ff6846SIoana Radulescu void *buf_data; 46434ff6846SIoana Radulescu u32 status = 0; 4657e273a8eSIoana Ciocoi Radulescu u32 xdp_act; 46634ff6846SIoana Radulescu 46734ff6846SIoana Radulescu /* Tracing point */ 46834ff6846SIoana Radulescu trace_dpaa2_rx_fd(priv->net_dev, fd); 46934ff6846SIoana Radulescu 47034ff6846SIoana Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 471efa6a7d0SIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 47218c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 47334ff6846SIoana Radulescu 47434ff6846SIoana Radulescu fas = dpaa2_get_fas(vaddr, false); 47534ff6846SIoana Radulescu prefetch(fas); 47634ff6846SIoana Radulescu buf_data = vaddr + dpaa2_fd_get_offset(fd); 47734ff6846SIoana Radulescu prefetch(buf_data); 47834ff6846SIoana Radulescu 47934ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 48034ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 48134ff6846SIoana Radulescu 48234ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 4835d8dccf8SIoana Ciornei xdp_act = dpaa2_eth_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 4847e273a8eSIoana Ciocoi Radulescu if (xdp_act != XDP_PASS) { 4857e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_packets++; 4867e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 4877e273a8eSIoana Ciocoi Radulescu return; 4887e273a8eSIoana Ciocoi Radulescu } 4897e273a8eSIoana Ciocoi Radulescu 490*50f82699SIoana Ciornei skb = dpaa2_eth_copybreak(ch, fd, vaddr); 491*50f82699SIoana Ciornei if (!skb) { 492efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 49318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 4945d8dccf8SIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 495*50f82699SIoana Ciornei } 49634ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 4977e273a8eSIoana Ciocoi Radulescu WARN_ON(priv->xdp_prog); 4987e273a8eSIoana Ciocoi Radulescu 499efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 50018c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 5015d8dccf8SIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 50227c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 50334ff6846SIoana Radulescu percpu_extras->rx_sg_frames++; 50434ff6846SIoana Radulescu percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 50534ff6846SIoana Radulescu } else { 50634ff6846SIoana Radulescu /* We don't support any other format */ 50734ff6846SIoana Radulescu goto err_frame_format; 50834ff6846SIoana Radulescu } 50934ff6846SIoana Radulescu 51034ff6846SIoana Radulescu if (unlikely(!skb)) 51134ff6846SIoana Radulescu goto err_build_skb; 51234ff6846SIoana Radulescu 51334ff6846SIoana Radulescu prefetch(skb->data); 51434ff6846SIoana Radulescu 51534ff6846SIoana Radulescu /* Get the timestamp value */ 51634ff6846SIoana Radulescu if (priv->rx_tstamp) { 51734ff6846SIoana Radulescu struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 51834ff6846SIoana Radulescu __le64 *ts = dpaa2_get_ts(vaddr, false); 51934ff6846SIoana Radulescu u64 ns; 52034ff6846SIoana Radulescu 52134ff6846SIoana Radulescu memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 52234ff6846SIoana Radulescu 52334ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 52434ff6846SIoana Radulescu shhwtstamps->hwtstamp = ns_to_ktime(ns); 52534ff6846SIoana Radulescu } 52634ff6846SIoana Radulescu 52734ff6846SIoana Radulescu /* Check if we need to validate the L4 csum */ 52834ff6846SIoana Radulescu if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 52934ff6846SIoana Radulescu status = le32_to_cpu(fas->status); 5305d8dccf8SIoana Ciornei dpaa2_eth_validate_rx_csum(priv, status, skb); 53134ff6846SIoana Radulescu } 53234ff6846SIoana Radulescu 53334ff6846SIoana Radulescu skb->protocol = eth_type_trans(skb, priv->net_dev); 534dbcdf728SIoana Ciocoi Radulescu skb_record_rx_queue(skb, fq->flowid); 53534ff6846SIoana Radulescu 53634ff6846SIoana Radulescu percpu_stats->rx_packets++; 53734ff6846SIoana Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 53834ff6846SIoana Radulescu 5390a25d92cSIoana Ciornei list_add_tail(&skb->list, ch->rx_list); 54034ff6846SIoana Radulescu 54134ff6846SIoana Radulescu return; 54234ff6846SIoana Radulescu 54334ff6846SIoana Radulescu err_build_skb: 5445d8dccf8SIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 54534ff6846SIoana Radulescu err_frame_format: 54634ff6846SIoana Radulescu percpu_stats->rx_dropped++; 54734ff6846SIoana Radulescu } 54834ff6846SIoana Radulescu 549061d631fSIoana Ciornei /* Processing of Rx frames received on the error FQ 550061d631fSIoana Ciornei * We check and print the error bits and then free the frame 551061d631fSIoana Ciornei */ 552061d631fSIoana Ciornei static void dpaa2_eth_rx_err(struct dpaa2_eth_priv *priv, 553061d631fSIoana Ciornei struct dpaa2_eth_channel *ch, 554061d631fSIoana Ciornei const struct dpaa2_fd *fd, 555061d631fSIoana Ciornei struct dpaa2_eth_fq *fq __always_unused) 556061d631fSIoana Ciornei { 557061d631fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 558061d631fSIoana Ciornei dma_addr_t addr = dpaa2_fd_get_addr(fd); 559061d631fSIoana Ciornei u8 fd_format = dpaa2_fd_get_format(fd); 560061d631fSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 561061d631fSIoana Ciornei struct dpaa2_eth_trap_item *trap_item; 562061d631fSIoana Ciornei struct dpaa2_fapr *fapr; 563061d631fSIoana Ciornei struct sk_buff *skb; 564061d631fSIoana Ciornei void *buf_data; 565061d631fSIoana Ciornei void *vaddr; 566061d631fSIoana Ciornei 567061d631fSIoana Ciornei vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 568061d631fSIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 569061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 570061d631fSIoana Ciornei 571061d631fSIoana Ciornei buf_data = vaddr + dpaa2_fd_get_offset(fd); 572061d631fSIoana Ciornei 573061d631fSIoana Ciornei if (fd_format == dpaa2_fd_single) { 574061d631fSIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 575061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 576061d631fSIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 577061d631fSIoana Ciornei } else if (fd_format == dpaa2_fd_sg) { 578061d631fSIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 579061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 580061d631fSIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 581061d631fSIoana Ciornei free_pages((unsigned long)vaddr, 0); 582061d631fSIoana Ciornei } else { 583061d631fSIoana Ciornei /* We don't support any other format */ 584061d631fSIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 585061d631fSIoana Ciornei goto err_frame_format; 586061d631fSIoana Ciornei } 587061d631fSIoana Ciornei 588061d631fSIoana Ciornei fapr = dpaa2_get_fapr(vaddr, false); 589061d631fSIoana Ciornei trap_item = dpaa2_eth_dl_get_trap(priv, fapr); 590061d631fSIoana Ciornei if (trap_item) 591061d631fSIoana Ciornei devlink_trap_report(priv->devlink, skb, trap_item->trap_ctx, 592061d631fSIoana Ciornei &priv->devlink_port, NULL); 593061d631fSIoana Ciornei consume_skb(skb); 594061d631fSIoana Ciornei 595061d631fSIoana Ciornei err_frame_format: 596061d631fSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 597061d631fSIoana Ciornei percpu_stats->rx_errors++; 598061d631fSIoana Ciornei ch->buf_count--; 599061d631fSIoana Ciornei } 600061d631fSIoana Ciornei 60134ff6846SIoana Radulescu /* Consume all frames pull-dequeued into the store. This is the simplest way to 60234ff6846SIoana Radulescu * make sure we don't accidentally issue another volatile dequeue which would 60334ff6846SIoana Radulescu * overwrite (leak) frames already in the store. 60434ff6846SIoana Radulescu * 60534ff6846SIoana Radulescu * Observance of NAPI budget is not our concern, leaving that to the caller. 60634ff6846SIoana Radulescu */ 6075d8dccf8SIoana Ciornei static int dpaa2_eth_consume_frames(struct dpaa2_eth_channel *ch, 608569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq **src) 60934ff6846SIoana Radulescu { 61034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = ch->priv; 61168049a5fSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq = NULL; 61234ff6846SIoana Radulescu struct dpaa2_dq *dq; 61334ff6846SIoana Radulescu const struct dpaa2_fd *fd; 614ef17bd7cSIoana Radulescu int cleaned = 0, retries = 0; 61534ff6846SIoana Radulescu int is_last; 61634ff6846SIoana Radulescu 61734ff6846SIoana Radulescu do { 61834ff6846SIoana Radulescu dq = dpaa2_io_store_next(ch->store, &is_last); 61934ff6846SIoana Radulescu if (unlikely(!dq)) { 62034ff6846SIoana Radulescu /* If we're here, we *must* have placed a 62134ff6846SIoana Radulescu * volatile dequeue comnmand, so keep reading through 62234ff6846SIoana Radulescu * the store until we get some sort of valid response 62334ff6846SIoana Radulescu * token (either a valid frame or an "empty dequeue") 62434ff6846SIoana Radulescu */ 625ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) { 626ef17bd7cSIoana Radulescu netdev_err_once(priv->net_dev, 627ef17bd7cSIoana Radulescu "Unable to read a valid dequeue response\n"); 628ef17bd7cSIoana Radulescu return -ETIMEDOUT; 629ef17bd7cSIoana Radulescu } 63034ff6846SIoana Radulescu continue; 63134ff6846SIoana Radulescu } 63234ff6846SIoana Radulescu 63334ff6846SIoana Radulescu fd = dpaa2_dq_fd(dq); 63434ff6846SIoana Radulescu fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 63534ff6846SIoana Radulescu 636dbcdf728SIoana Ciocoi Radulescu fq->consume(priv, ch, fd, fq); 63734ff6846SIoana Radulescu cleaned++; 638ef17bd7cSIoana Radulescu retries = 0; 63934ff6846SIoana Radulescu } while (!is_last); 64034ff6846SIoana Radulescu 64168049a5fSIoana Ciocoi Radulescu if (!cleaned) 64268049a5fSIoana Ciocoi Radulescu return 0; 64368049a5fSIoana Ciocoi Radulescu 64468049a5fSIoana Ciocoi Radulescu fq->stats.frames += cleaned; 645460fd830SIoana Ciornei ch->stats.frames += cleaned; 64668049a5fSIoana Ciocoi Radulescu 64768049a5fSIoana Ciocoi Radulescu /* A dequeue operation only pulls frames from a single queue 648569dac6aSIoana Ciocoi Radulescu * into the store. Return the frame queue as an out param. 64968049a5fSIoana Ciocoi Radulescu */ 650569dac6aSIoana Ciocoi Radulescu if (src) 651569dac6aSIoana Ciocoi Radulescu *src = fq; 65268049a5fSIoana Ciocoi Radulescu 65334ff6846SIoana Radulescu return cleaned; 65434ff6846SIoana Radulescu } 65534ff6846SIoana Radulescu 656c5521189SYangbo Lu static int dpaa2_eth_ptp_parse(struct sk_buff *skb, 657c5521189SYangbo Lu u8 *msgtype, u8 *twostep, u8 *udp, 658c5521189SYangbo Lu u16 *correction_offset, 659c5521189SYangbo Lu u16 *origintimestamp_offset) 66034ff6846SIoana Radulescu { 661c5521189SYangbo Lu unsigned int ptp_class; 662c5521189SYangbo Lu struct ptp_header *hdr; 663c5521189SYangbo Lu unsigned int type; 664c5521189SYangbo Lu u8 *base; 665c5521189SYangbo Lu 666c5521189SYangbo Lu ptp_class = ptp_classify_raw(skb); 667c5521189SYangbo Lu if (ptp_class == PTP_CLASS_NONE) 668c5521189SYangbo Lu return -EINVAL; 669c5521189SYangbo Lu 670c5521189SYangbo Lu hdr = ptp_parse_header(skb, ptp_class); 671c5521189SYangbo Lu if (!hdr) 672c5521189SYangbo Lu return -EINVAL; 673c5521189SYangbo Lu 674c5521189SYangbo Lu *msgtype = ptp_get_msgtype(hdr, ptp_class); 675c5521189SYangbo Lu *twostep = hdr->flag_field[0] & 0x2; 676c5521189SYangbo Lu 677c5521189SYangbo Lu type = ptp_class & PTP_CLASS_PMASK; 678c5521189SYangbo Lu if (type == PTP_CLASS_IPV4 || 679c5521189SYangbo Lu type == PTP_CLASS_IPV6) 680c5521189SYangbo Lu *udp = 1; 681c5521189SYangbo Lu else 682c5521189SYangbo Lu *udp = 0; 683c5521189SYangbo Lu 684c5521189SYangbo Lu base = skb_mac_header(skb); 685c5521189SYangbo Lu *correction_offset = (u8 *)&hdr->correction - base; 686c5521189SYangbo Lu *origintimestamp_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 687c5521189SYangbo Lu 688c5521189SYangbo Lu return 0; 689c5521189SYangbo Lu } 690c5521189SYangbo Lu 691c5521189SYangbo Lu /* Configure the egress frame annotation for timestamp update */ 692c5521189SYangbo Lu static void dpaa2_eth_enable_tx_tstamp(struct dpaa2_eth_priv *priv, 693c5521189SYangbo Lu struct dpaa2_fd *fd, 694c5521189SYangbo Lu void *buf_start, 695c5521189SYangbo Lu struct sk_buff *skb) 696c5521189SYangbo Lu { 697c5521189SYangbo Lu struct ptp_tstamp origin_timestamp; 698c5521189SYangbo Lu struct dpni_single_step_cfg cfg; 699c5521189SYangbo Lu u8 msgtype, twostep, udp; 70034ff6846SIoana Radulescu struct dpaa2_faead *faead; 701c5521189SYangbo Lu struct dpaa2_fas *fas; 702c5521189SYangbo Lu struct timespec64 ts; 703c5521189SYangbo Lu u16 offset1, offset2; 70434ff6846SIoana Radulescu u32 ctrl, frc; 705c5521189SYangbo Lu __le64 *ns; 706c5521189SYangbo Lu u8 *data; 70734ff6846SIoana Radulescu 70834ff6846SIoana Radulescu /* Mark the egress frame annotation area as valid */ 70934ff6846SIoana Radulescu frc = dpaa2_fd_get_frc(fd); 71034ff6846SIoana Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 71134ff6846SIoana Radulescu 71234ff6846SIoana Radulescu /* Set hardware annotation size */ 71334ff6846SIoana Radulescu ctrl = dpaa2_fd_get_ctrl(fd); 71434ff6846SIoana Radulescu dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 71534ff6846SIoana Radulescu 71634ff6846SIoana Radulescu /* enable UPD (update prepanded data) bit in FAEAD field of 71734ff6846SIoana Radulescu * hardware frame annotation area 71834ff6846SIoana Radulescu */ 71934ff6846SIoana Radulescu ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 72034ff6846SIoana Radulescu faead = dpaa2_get_faead(buf_start, true); 72134ff6846SIoana Radulescu faead->ctrl = cpu_to_le32(ctrl); 722c5521189SYangbo Lu 723c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 724c5521189SYangbo Lu if (dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 725c5521189SYangbo Lu &offset1, &offset2) || 7266b6817c5SChristian Eggers msgtype != PTP_MSGTYPE_SYNC || twostep) { 727c5521189SYangbo Lu WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 728c5521189SYangbo Lu return; 729c5521189SYangbo Lu } 730c5521189SYangbo Lu 731c5521189SYangbo Lu /* Mark the frame annotation status as valid */ 732c5521189SYangbo Lu frc = dpaa2_fd_get_frc(fd); 733c5521189SYangbo Lu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FASV); 734c5521189SYangbo Lu 735c5521189SYangbo Lu /* Mark the PTP flag for one step timestamping */ 736c5521189SYangbo Lu fas = dpaa2_get_fas(buf_start, true); 737c5521189SYangbo Lu fas->status = cpu_to_le32(DPAA2_FAS_PTP); 738c5521189SYangbo Lu 739c5521189SYangbo Lu dpaa2_ptp->caps.gettime64(&dpaa2_ptp->caps, &ts); 740c5521189SYangbo Lu ns = dpaa2_get_ts(buf_start, true); 741c5521189SYangbo Lu *ns = cpu_to_le64(timespec64_to_ns(&ts) / 742c5521189SYangbo Lu DPAA2_PTP_CLK_PERIOD_NS); 743c5521189SYangbo Lu 744c5521189SYangbo Lu /* Update current time to PTP message originTimestamp field */ 745c5521189SYangbo Lu ns_to_ptp_tstamp(&origin_timestamp, le64_to_cpup(ns)); 746c5521189SYangbo Lu data = skb_mac_header(skb); 747c5521189SYangbo Lu *(__be16 *)(data + offset2) = htons(origin_timestamp.sec_msb); 748c5521189SYangbo Lu *(__be32 *)(data + offset2 + 2) = 749c5521189SYangbo Lu htonl(origin_timestamp.sec_lsb); 750c5521189SYangbo Lu *(__be32 *)(data + offset2 + 6) = htonl(origin_timestamp.nsec); 751c5521189SYangbo Lu 752c5521189SYangbo Lu cfg.en = 1; 753c5521189SYangbo Lu cfg.ch_update = udp; 754c5521189SYangbo Lu cfg.offset = offset1; 755c5521189SYangbo Lu cfg.peer_delay = 0; 756c5521189SYangbo Lu 757c5521189SYangbo Lu if (dpni_set_single_step_cfg(priv->mc_io, 0, priv->mc_token, 758c5521189SYangbo Lu &cfg)) 759c5521189SYangbo Lu WARN_ONCE(1, "Failed to set single step register"); 760c5521189SYangbo Lu } 76134ff6846SIoana Radulescu } 76234ff6846SIoana Radulescu 76334ff6846SIoana Radulescu /* Create a frame descriptor based on a fragmented skb */ 7645d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd(struct dpaa2_eth_priv *priv, 76534ff6846SIoana Radulescu struct sk_buff *skb, 76664a965deSYangbo Lu struct dpaa2_fd *fd, 76764a965deSYangbo Lu void **swa_addr) 76834ff6846SIoana Radulescu { 76934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 77034ff6846SIoana Radulescu void *sgt_buf = NULL; 77134ff6846SIoana Radulescu dma_addr_t addr; 77234ff6846SIoana Radulescu int nr_frags = skb_shinfo(skb)->nr_frags; 77334ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 77434ff6846SIoana Radulescu int i, err; 77534ff6846SIoana Radulescu int sgt_buf_size; 77634ff6846SIoana Radulescu struct scatterlist *scl, *crt_scl; 77734ff6846SIoana Radulescu int num_sg; 77834ff6846SIoana Radulescu int num_dma_bufs; 77934ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 78034ff6846SIoana Radulescu 78134ff6846SIoana Radulescu /* Create and map scatterlist. 78234ff6846SIoana Radulescu * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 78334ff6846SIoana Radulescu * to go beyond nr_frags+1. 78434ff6846SIoana Radulescu * Note: We don't support chained scatterlists 78534ff6846SIoana Radulescu */ 78634ff6846SIoana Radulescu if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 78734ff6846SIoana Radulescu return -EINVAL; 78834ff6846SIoana Radulescu 789d4ceb8deSJulia Lawall scl = kmalloc_array(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 79034ff6846SIoana Radulescu if (unlikely(!scl)) 79134ff6846SIoana Radulescu return -ENOMEM; 79234ff6846SIoana Radulescu 79334ff6846SIoana Radulescu sg_init_table(scl, nr_frags + 1); 79434ff6846SIoana Radulescu num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 79537fbbddaSIoana Ciornei if (unlikely(num_sg < 0)) { 79637fbbddaSIoana Ciornei err = -ENOMEM; 79737fbbddaSIoana Ciornei goto dma_map_sg_failed; 79837fbbddaSIoana Ciornei } 79934ff6846SIoana Radulescu num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 80034ff6846SIoana Radulescu if (unlikely(!num_dma_bufs)) { 80134ff6846SIoana Radulescu err = -ENOMEM; 80234ff6846SIoana Radulescu goto dma_map_sg_failed; 80334ff6846SIoana Radulescu } 80434ff6846SIoana Radulescu 80534ff6846SIoana Radulescu /* Prepare the HW SGT structure */ 80634ff6846SIoana Radulescu sgt_buf_size = priv->tx_data_offset + 80734ff6846SIoana Radulescu sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 808d0dfbb99SKevin Hao sgt_buf = napi_alloc_frag_align(sgt_buf_size, DPAA2_ETH_TX_BUF_ALIGN); 80934ff6846SIoana Radulescu if (unlikely(!sgt_buf)) { 81034ff6846SIoana Radulescu err = -ENOMEM; 81134ff6846SIoana Radulescu goto sgt_buf_alloc_failed; 81234ff6846SIoana Radulescu } 81334ff6846SIoana Radulescu memset(sgt_buf, 0, sgt_buf_size); 81434ff6846SIoana Radulescu 81534ff6846SIoana Radulescu sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 81634ff6846SIoana Radulescu 81734ff6846SIoana Radulescu /* Fill in the HW SGT structure. 81834ff6846SIoana Radulescu * 81934ff6846SIoana Radulescu * sgt_buf is zeroed out, so the following fields are implicit 82034ff6846SIoana Radulescu * in all sgt entries: 82134ff6846SIoana Radulescu * - offset is 0 82234ff6846SIoana Radulescu * - format is 'dpaa2_sg_single' 82334ff6846SIoana Radulescu */ 82434ff6846SIoana Radulescu for_each_sg(scl, crt_scl, num_dma_bufs, i) { 82534ff6846SIoana Radulescu dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 82634ff6846SIoana Radulescu dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 82734ff6846SIoana Radulescu } 82834ff6846SIoana Radulescu dpaa2_sg_set_final(&sgt[i - 1], true); 82934ff6846SIoana Radulescu 83034ff6846SIoana Radulescu /* Store the skb backpointer in the SGT buffer. 83134ff6846SIoana Radulescu * Fit the scatterlist and the number of buffers alongside the 83234ff6846SIoana Radulescu * skb backpointer in the software annotation area. We'll need 83334ff6846SIoana Radulescu * all of them on Tx Conf. 83434ff6846SIoana Radulescu */ 83564a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 83634ff6846SIoana Radulescu swa = (struct dpaa2_eth_swa *)sgt_buf; 837e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SG; 838e3fdf6baSIoana Radulescu swa->sg.skb = skb; 839e3fdf6baSIoana Radulescu swa->sg.scl = scl; 840e3fdf6baSIoana Radulescu swa->sg.num_sg = num_sg; 841e3fdf6baSIoana Radulescu swa->sg.sgt_size = sgt_buf_size; 84234ff6846SIoana Radulescu 84334ff6846SIoana Radulescu /* Separately map the SGT buffer */ 84434ff6846SIoana Radulescu addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 84534ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) { 84634ff6846SIoana Radulescu err = -ENOMEM; 84734ff6846SIoana Radulescu goto dma_map_single_failed; 84834ff6846SIoana Radulescu } 84934ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, priv->tx_data_offset); 85034ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_sg); 85134ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 85234ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 853b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 85434ff6846SIoana Radulescu 85534ff6846SIoana Radulescu return 0; 85634ff6846SIoana Radulescu 85734ff6846SIoana Radulescu dma_map_single_failed: 85834ff6846SIoana Radulescu skb_free_frag(sgt_buf); 85934ff6846SIoana Radulescu sgt_buf_alloc_failed: 86034ff6846SIoana Radulescu dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 86134ff6846SIoana Radulescu dma_map_sg_failed: 86234ff6846SIoana Radulescu kfree(scl); 86334ff6846SIoana Radulescu return err; 86434ff6846SIoana Radulescu } 86534ff6846SIoana Radulescu 866d70446eeSIoana Ciornei /* Create a SG frame descriptor based on a linear skb. 867d70446eeSIoana Ciornei * 868d70446eeSIoana Ciornei * This function is used on the Tx path when the skb headroom is not large 869d70446eeSIoana Ciornei * enough for the HW requirements, thus instead of realloc-ing the skb we 870d70446eeSIoana Ciornei * create a SG frame descriptor with only one entry. 871d70446eeSIoana Ciornei */ 8725d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd_single_buf(struct dpaa2_eth_priv *priv, 873d70446eeSIoana Ciornei struct sk_buff *skb, 87464a965deSYangbo Lu struct dpaa2_fd *fd, 87564a965deSYangbo Lu void **swa_addr) 876d70446eeSIoana Ciornei { 877d70446eeSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 878d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 879d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 880d70446eeSIoana Ciornei struct dpaa2_eth_swa *swa; 881d70446eeSIoana Ciornei dma_addr_t addr, sgt_addr; 882d70446eeSIoana Ciornei void *sgt_buf = NULL; 883d70446eeSIoana Ciornei int sgt_buf_size; 884d70446eeSIoana Ciornei int err; 885d70446eeSIoana Ciornei 886d70446eeSIoana Ciornei /* Prepare the HW SGT structure */ 887d70446eeSIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 888d70446eeSIoana Ciornei sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry); 889d70446eeSIoana Ciornei 890d70446eeSIoana Ciornei if (sgt_cache->count == 0) 891d70446eeSIoana Ciornei sgt_buf = kzalloc(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN, 892d70446eeSIoana Ciornei GFP_ATOMIC); 893d70446eeSIoana Ciornei else 894d70446eeSIoana Ciornei sgt_buf = sgt_cache->buf[--sgt_cache->count]; 895d70446eeSIoana Ciornei if (unlikely(!sgt_buf)) 896d70446eeSIoana Ciornei return -ENOMEM; 897d70446eeSIoana Ciornei 898d70446eeSIoana Ciornei sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 899d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 900d70446eeSIoana Ciornei 901d70446eeSIoana Ciornei addr = dma_map_single(dev, skb->data, skb->len, DMA_BIDIRECTIONAL); 902d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) { 903d70446eeSIoana Ciornei err = -ENOMEM; 904d70446eeSIoana Ciornei goto data_map_failed; 905d70446eeSIoana Ciornei } 906d70446eeSIoana Ciornei 907d70446eeSIoana Ciornei /* Fill in the HW SGT structure */ 908d70446eeSIoana Ciornei dpaa2_sg_set_addr(sgt, addr); 909d70446eeSIoana Ciornei dpaa2_sg_set_len(sgt, skb->len); 910d70446eeSIoana Ciornei dpaa2_sg_set_final(sgt, true); 911d70446eeSIoana Ciornei 912d70446eeSIoana Ciornei /* Store the skb backpointer in the SGT buffer */ 91364a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 914d70446eeSIoana Ciornei swa = (struct dpaa2_eth_swa *)sgt_buf; 915d70446eeSIoana Ciornei swa->type = DPAA2_ETH_SWA_SINGLE; 916d70446eeSIoana Ciornei swa->single.skb = skb; 91754a57d1cSIoana Ciornei swa->single.sgt_size = sgt_buf_size; 918d70446eeSIoana Ciornei 919d70446eeSIoana Ciornei /* Separately map the SGT buffer */ 920d70446eeSIoana Ciornei sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 921d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, sgt_addr))) { 922d70446eeSIoana Ciornei err = -ENOMEM; 923d70446eeSIoana Ciornei goto sgt_map_failed; 924d70446eeSIoana Ciornei } 925d70446eeSIoana Ciornei 926d70446eeSIoana Ciornei dpaa2_fd_set_offset(fd, priv->tx_data_offset); 927d70446eeSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_sg); 928d70446eeSIoana Ciornei dpaa2_fd_set_addr(fd, sgt_addr); 929d70446eeSIoana Ciornei dpaa2_fd_set_len(fd, skb->len); 930d70446eeSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 931d70446eeSIoana Ciornei 932d70446eeSIoana Ciornei return 0; 933d70446eeSIoana Ciornei 934d70446eeSIoana Ciornei sgt_map_failed: 935d70446eeSIoana Ciornei dma_unmap_single(dev, addr, skb->len, DMA_BIDIRECTIONAL); 936d70446eeSIoana Ciornei data_map_failed: 937d70446eeSIoana Ciornei if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 938d70446eeSIoana Ciornei kfree(sgt_buf); 939d70446eeSIoana Ciornei else 940d70446eeSIoana Ciornei sgt_cache->buf[sgt_cache->count++] = sgt_buf; 941d70446eeSIoana Ciornei 942d70446eeSIoana Ciornei return err; 943d70446eeSIoana Ciornei } 944d70446eeSIoana Ciornei 94534ff6846SIoana Radulescu /* Create a frame descriptor based on a linear skb */ 9465d8dccf8SIoana Ciornei static int dpaa2_eth_build_single_fd(struct dpaa2_eth_priv *priv, 94734ff6846SIoana Radulescu struct sk_buff *skb, 94864a965deSYangbo Lu struct dpaa2_fd *fd, 94964a965deSYangbo Lu void **swa_addr) 95034ff6846SIoana Radulescu { 95134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 95234ff6846SIoana Radulescu u8 *buffer_start, *aligned_start; 953e3fdf6baSIoana Radulescu struct dpaa2_eth_swa *swa; 95434ff6846SIoana Radulescu dma_addr_t addr; 95534ff6846SIoana Radulescu 9561cf773bdSYangbo Lu buffer_start = skb->data - dpaa2_eth_needed_headroom(skb); 95734ff6846SIoana Radulescu 95834ff6846SIoana Radulescu /* If there's enough room to align the FD address, do it. 95934ff6846SIoana Radulescu * It will help hardware optimize accesses. 96034ff6846SIoana Radulescu */ 96134ff6846SIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 96234ff6846SIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 96334ff6846SIoana Radulescu if (aligned_start >= skb->head) 96434ff6846SIoana Radulescu buffer_start = aligned_start; 96534ff6846SIoana Radulescu 96634ff6846SIoana Radulescu /* Store a backpointer to the skb at the beginning of the buffer 96734ff6846SIoana Radulescu * (in the private data area) such that we can release it 96834ff6846SIoana Radulescu * on Tx confirm 96934ff6846SIoana Radulescu */ 97064a965deSYangbo Lu *swa_addr = (void *)buffer_start; 971e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 972e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SINGLE; 973e3fdf6baSIoana Radulescu swa->single.skb = skb; 97434ff6846SIoana Radulescu 97534ff6846SIoana Radulescu addr = dma_map_single(dev, buffer_start, 97634ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 97734ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 97834ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 97934ff6846SIoana Radulescu return -ENOMEM; 98034ff6846SIoana Radulescu 98134ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 98234ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 98334ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 98434ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_single); 985b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 98634ff6846SIoana Radulescu 98734ff6846SIoana Radulescu return 0; 98834ff6846SIoana Radulescu } 98934ff6846SIoana Radulescu 99034ff6846SIoana Radulescu /* FD freeing routine on the Tx path 99134ff6846SIoana Radulescu * 99234ff6846SIoana Radulescu * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 99334ff6846SIoana Radulescu * back-pointed to is also freed. 99434ff6846SIoana Radulescu * This can be called either from dpaa2_eth_tx_conf() or on the error path of 99534ff6846SIoana Radulescu * dpaa2_eth_tx(). 99634ff6846SIoana Radulescu */ 997c5521189SYangbo Lu static void dpaa2_eth_free_tx_fd(struct dpaa2_eth_priv *priv, 998d678be1dSIoana Radulescu struct dpaa2_eth_fq *fq, 9990723a3aeSIoana Ciocoi Radulescu const struct dpaa2_fd *fd, bool in_napi) 100034ff6846SIoana Radulescu { 100134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 1002d70446eeSIoana Ciornei dma_addr_t fd_addr, sg_addr; 1003d678be1dSIoana Radulescu struct sk_buff *skb = NULL; 100434ff6846SIoana Radulescu unsigned char *buffer_start; 100534ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 100634ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 1007d678be1dSIoana Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 100834ff6846SIoana Radulescu 1009d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 1010d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 1011d70446eeSIoana Ciornei 101234ff6846SIoana Radulescu fd_addr = dpaa2_fd_get_addr(fd); 1013e3fdf6baSIoana Radulescu buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 1014e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 101534ff6846SIoana Radulescu 101634ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 1017d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_SINGLE) { 1018e3fdf6baSIoana Radulescu skb = swa->single.skb; 1019d678be1dSIoana Radulescu /* Accessing the skb buffer is safe before dma unmap, 1020d678be1dSIoana Radulescu * because we didn't map the actual skb shell. 102134ff6846SIoana Radulescu */ 102234ff6846SIoana Radulescu dma_unmap_single(dev, fd_addr, 102334ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 102434ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 1025d678be1dSIoana Radulescu } else { 1026d678be1dSIoana Radulescu WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 1027d678be1dSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 1028d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 1029d678be1dSIoana Radulescu } 103034ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 1031d70446eeSIoana Ciornei if (swa->type == DPAA2_ETH_SWA_SG) { 1032e3fdf6baSIoana Radulescu skb = swa->sg.skb; 103334ff6846SIoana Radulescu 103434ff6846SIoana Radulescu /* Unmap the scatterlist */ 1035e3fdf6baSIoana Radulescu dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 1036e3fdf6baSIoana Radulescu DMA_BIDIRECTIONAL); 1037e3fdf6baSIoana Radulescu kfree(swa->sg.scl); 103834ff6846SIoana Radulescu 103934ff6846SIoana Radulescu /* Unmap the SGT buffer */ 1040e3fdf6baSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 104134ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 104234ff6846SIoana Radulescu } else { 1043d70446eeSIoana Ciornei skb = swa->single.skb; 1044d70446eeSIoana Ciornei 1045d70446eeSIoana Ciornei /* Unmap the SGT Buffer */ 1046d70446eeSIoana Ciornei dma_unmap_single(dev, fd_addr, swa->single.sgt_size, 1047d70446eeSIoana Ciornei DMA_BIDIRECTIONAL); 1048d70446eeSIoana Ciornei 1049d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(buffer_start + 1050d70446eeSIoana Ciornei priv->tx_data_offset); 1051d70446eeSIoana Ciornei sg_addr = dpaa2_sg_get_addr(sgt); 1052d70446eeSIoana Ciornei dma_unmap_single(dev, sg_addr, skb->len, DMA_BIDIRECTIONAL); 1053d70446eeSIoana Ciornei } 1054d70446eeSIoana Ciornei } else { 105534ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "Invalid FD format\n"); 105634ff6846SIoana Radulescu return; 105734ff6846SIoana Radulescu } 105834ff6846SIoana Radulescu 1059d678be1dSIoana Radulescu if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 1060d678be1dSIoana Radulescu fq->dq_frames++; 1061d678be1dSIoana Radulescu fq->dq_bytes += fd_len; 1062d678be1dSIoana Radulescu } 1063d678be1dSIoana Radulescu 1064d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_XDP) { 1065d678be1dSIoana Radulescu xdp_return_frame(swa->xdp.xdpf); 1066d678be1dSIoana Radulescu return; 1067d678be1dSIoana Radulescu } 1068d678be1dSIoana Radulescu 106934ff6846SIoana Radulescu /* Get the timestamp value */ 10701cf773bdSYangbo Lu if (skb->cb[0] == TX_TSTAMP) { 107134ff6846SIoana Radulescu struct skb_shared_hwtstamps shhwtstamps; 1072e3fdf6baSIoana Radulescu __le64 *ts = dpaa2_get_ts(buffer_start, true); 107334ff6846SIoana Radulescu u64 ns; 107434ff6846SIoana Radulescu 107534ff6846SIoana Radulescu memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 107634ff6846SIoana Radulescu 107734ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 107834ff6846SIoana Radulescu shhwtstamps.hwtstamp = ns_to_ktime(ns); 107934ff6846SIoana Radulescu skb_tstamp_tx(skb, &shhwtstamps); 1080c5521189SYangbo Lu } else if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1081c5521189SYangbo Lu mutex_unlock(&priv->onestep_tstamp_lock); 108234ff6846SIoana Radulescu } 108334ff6846SIoana Radulescu 108434ff6846SIoana Radulescu /* Free SGT buffer allocated on tx */ 1085d70446eeSIoana Ciornei if (fd_format != dpaa2_fd_single) { 1086d70446eeSIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 1087d70446eeSIoana Ciornei if (swa->type == DPAA2_ETH_SWA_SG) { 1088e3fdf6baSIoana Radulescu skb_free_frag(buffer_start); 1089d70446eeSIoana Ciornei } else { 1090d70446eeSIoana Ciornei if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 1091d70446eeSIoana Ciornei kfree(buffer_start); 1092d70446eeSIoana Ciornei else 1093d70446eeSIoana Ciornei sgt_cache->buf[sgt_cache->count++] = buffer_start; 1094d70446eeSIoana Ciornei } 1095d70446eeSIoana Ciornei } 109634ff6846SIoana Radulescu 109734ff6846SIoana Radulescu /* Move on with skb release */ 10980723a3aeSIoana Ciocoi Radulescu napi_consume_skb(skb, in_napi); 109934ff6846SIoana Radulescu } 110034ff6846SIoana Radulescu 1101c5521189SYangbo Lu static netdev_tx_t __dpaa2_eth_tx(struct sk_buff *skb, 1102c5521189SYangbo Lu struct net_device *net_dev) 110334ff6846SIoana Radulescu { 110434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 110534ff6846SIoana Radulescu struct dpaa2_fd fd; 110634ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 110734ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 110834ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 1109569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 111034ff6846SIoana Radulescu u16 queue_mapping; 111134ff6846SIoana Radulescu unsigned int needed_headroom; 1112569dac6aSIoana Ciocoi Radulescu u32 fd_len; 1113ab1e6de2SIoana Radulescu u8 prio = 0; 111434ff6846SIoana Radulescu int err, i; 111564a965deSYangbo Lu void *swa; 111634ff6846SIoana Radulescu 111734ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 111834ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 111934ff6846SIoana Radulescu 11201cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(skb); 112134ff6846SIoana Radulescu 112234ff6846SIoana Radulescu /* We'll be holding a back-reference to the skb until Tx Confirmation; 112334ff6846SIoana Radulescu * we don't want that overwritten by a concurrent Tx with a cloned skb. 112434ff6846SIoana Radulescu */ 112534ff6846SIoana Radulescu skb = skb_unshare(skb, GFP_ATOMIC); 112634ff6846SIoana Radulescu if (unlikely(!skb)) { 112734ff6846SIoana Radulescu /* skb_unshare() has already freed the skb */ 112834ff6846SIoana Radulescu percpu_stats->tx_dropped++; 112934ff6846SIoana Radulescu return NETDEV_TX_OK; 113034ff6846SIoana Radulescu } 113134ff6846SIoana Radulescu 113234ff6846SIoana Radulescu /* Setup the FD fields */ 113334ff6846SIoana Radulescu memset(&fd, 0, sizeof(fd)); 113434ff6846SIoana Radulescu 113534ff6846SIoana Radulescu if (skb_is_nonlinear(skb)) { 113664a965deSYangbo Lu err = dpaa2_eth_build_sg_fd(priv, skb, &fd, &swa); 113734ff6846SIoana Radulescu percpu_extras->tx_sg_frames++; 113834ff6846SIoana Radulescu percpu_extras->tx_sg_bytes += skb->len; 1139d70446eeSIoana Ciornei } else if (skb_headroom(skb) < needed_headroom) { 114064a965deSYangbo Lu err = dpaa2_eth_build_sg_fd_single_buf(priv, skb, &fd, &swa); 1141d70446eeSIoana Ciornei percpu_extras->tx_sg_frames++; 1142d70446eeSIoana Ciornei percpu_extras->tx_sg_bytes += skb->len; 11434c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_frames++; 11444c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_bytes += skb->len; 114534ff6846SIoana Radulescu } else { 114664a965deSYangbo Lu err = dpaa2_eth_build_single_fd(priv, skb, &fd, &swa); 114734ff6846SIoana Radulescu } 114834ff6846SIoana Radulescu 114934ff6846SIoana Radulescu if (unlikely(err)) { 115034ff6846SIoana Radulescu percpu_stats->tx_dropped++; 115134ff6846SIoana Radulescu goto err_build_fd; 115234ff6846SIoana Radulescu } 115334ff6846SIoana Radulescu 1154c5521189SYangbo Lu if (skb->cb[0]) 1155c5521189SYangbo Lu dpaa2_eth_enable_tx_tstamp(priv, &fd, swa, skb); 115664a965deSYangbo Lu 115734ff6846SIoana Radulescu /* Tracing point */ 115834ff6846SIoana Radulescu trace_dpaa2_tx_fd(net_dev, &fd); 115934ff6846SIoana Radulescu 116034ff6846SIoana Radulescu /* TxConf FQ selection relies on queue id from the stack. 116134ff6846SIoana Radulescu * In case of a forwarded frame from another DPNI interface, we choose 116234ff6846SIoana Radulescu * a queue affined to the same core that processed the Rx frame 116334ff6846SIoana Radulescu */ 116434ff6846SIoana Radulescu queue_mapping = skb_get_queue_mapping(skb); 1165ab1e6de2SIoana Radulescu 1166ab1e6de2SIoana Radulescu if (net_dev->num_tc) { 1167ab1e6de2SIoana Radulescu prio = netdev_txq_to_tc(net_dev, queue_mapping); 1168ab1e6de2SIoana Radulescu /* Hardware interprets priority level 0 as being the highest, 1169ab1e6de2SIoana Radulescu * so we need to do a reverse mapping to the netdev tc index 1170ab1e6de2SIoana Radulescu */ 1171ab1e6de2SIoana Radulescu prio = net_dev->num_tc - prio - 1; 1172ab1e6de2SIoana Radulescu /* We have only one FQ array entry for all Tx hardware queues 1173ab1e6de2SIoana Radulescu * with the same flow id (but different priority levels) 1174ab1e6de2SIoana Radulescu */ 1175ab1e6de2SIoana Radulescu queue_mapping %= dpaa2_eth_queue_count(priv); 1176ab1e6de2SIoana Radulescu } 117734ff6846SIoana Radulescu fq = &priv->fq[queue_mapping]; 11788c838f53SIoana Ciornei 11798c838f53SIoana Ciornei fd_len = dpaa2_fd_get_len(&fd); 11808c838f53SIoana Ciornei nq = netdev_get_tx_queue(net_dev, queue_mapping); 11818c838f53SIoana Ciornei netdev_tx_sent_queue(nq, fd_len); 11828c838f53SIoana Ciornei 11838c838f53SIoana Ciornei /* Everything that happens after this enqueues might race with 11848c838f53SIoana Ciornei * the Tx confirmation callback for this frame 11858c838f53SIoana Ciornei */ 118634ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 11876ff80447SIoana Ciornei err = priv->enqueue(priv, fq, &fd, prio, 1, NULL); 118834ff6846SIoana Radulescu if (err != -EBUSY) 118934ff6846SIoana Radulescu break; 119034ff6846SIoana Radulescu } 119134ff6846SIoana Radulescu percpu_extras->tx_portal_busy += i; 119234ff6846SIoana Radulescu if (unlikely(err < 0)) { 119334ff6846SIoana Radulescu percpu_stats->tx_errors++; 119434ff6846SIoana Radulescu /* Clean up everything, including freeing the skb */ 11955d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, &fd, false); 11968c838f53SIoana Ciornei netdev_tx_completed_queue(nq, 1, fd_len); 119734ff6846SIoana Radulescu } else { 119834ff6846SIoana Radulescu percpu_stats->tx_packets++; 1199569dac6aSIoana Ciocoi Radulescu percpu_stats->tx_bytes += fd_len; 120034ff6846SIoana Radulescu } 120134ff6846SIoana Radulescu 120234ff6846SIoana Radulescu return NETDEV_TX_OK; 120334ff6846SIoana Radulescu 120434ff6846SIoana Radulescu err_build_fd: 120534ff6846SIoana Radulescu dev_kfree_skb(skb); 120634ff6846SIoana Radulescu 120734ff6846SIoana Radulescu return NETDEV_TX_OK; 120834ff6846SIoana Radulescu } 120934ff6846SIoana Radulescu 1210c5521189SYangbo Lu static void dpaa2_eth_tx_onestep_tstamp(struct work_struct *work) 1211c5521189SYangbo Lu { 1212c5521189SYangbo Lu struct dpaa2_eth_priv *priv = container_of(work, struct dpaa2_eth_priv, 1213c5521189SYangbo Lu tx_onestep_tstamp); 1214c5521189SYangbo Lu struct sk_buff *skb; 1215c5521189SYangbo Lu 1216c5521189SYangbo Lu while (true) { 1217c5521189SYangbo Lu skb = skb_dequeue(&priv->tx_skbs); 1218c5521189SYangbo Lu if (!skb) 1219c5521189SYangbo Lu return; 1220c5521189SYangbo Lu 1221c5521189SYangbo Lu /* Lock just before TX one-step timestamping packet, 1222c5521189SYangbo Lu * and release the lock in dpaa2_eth_free_tx_fd when 1223c5521189SYangbo Lu * confirm the packet has been sent on hardware, or 1224c5521189SYangbo Lu * when clean up during transmit failure. 1225c5521189SYangbo Lu */ 1226c5521189SYangbo Lu mutex_lock(&priv->onestep_tstamp_lock); 1227c5521189SYangbo Lu __dpaa2_eth_tx(skb, priv->net_dev); 1228c5521189SYangbo Lu } 1229c5521189SYangbo Lu } 1230c5521189SYangbo Lu 1231c5521189SYangbo Lu static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 1232c5521189SYangbo Lu { 1233c5521189SYangbo Lu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1234c5521189SYangbo Lu u8 msgtype, twostep, udp; 1235c5521189SYangbo Lu u16 offset1, offset2; 1236c5521189SYangbo Lu 1237c5521189SYangbo Lu /* Utilize skb->cb[0] for timestamping request per skb */ 1238c5521189SYangbo Lu skb->cb[0] = 0; 1239c5521189SYangbo Lu 1240c5521189SYangbo Lu if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && dpaa2_ptp) { 1241c5521189SYangbo Lu if (priv->tx_tstamp_type == HWTSTAMP_TX_ON) 1242c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1243c5521189SYangbo Lu else if (priv->tx_tstamp_type == HWTSTAMP_TX_ONESTEP_SYNC) 1244c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP_ONESTEP_SYNC; 1245c5521189SYangbo Lu } 1246c5521189SYangbo Lu 1247c5521189SYangbo Lu /* TX for one-step timestamping PTP Sync packet */ 1248c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1249c5521189SYangbo Lu if (!dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 1250c5521189SYangbo Lu &offset1, &offset2)) 12516b6817c5SChristian Eggers if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0) { 1252c5521189SYangbo Lu skb_queue_tail(&priv->tx_skbs, skb); 1253c5521189SYangbo Lu queue_work(priv->dpaa2_ptp_wq, 1254c5521189SYangbo Lu &priv->tx_onestep_tstamp); 1255c5521189SYangbo Lu return NETDEV_TX_OK; 1256c5521189SYangbo Lu } 1257c5521189SYangbo Lu /* Use two-step timestamping if not one-step timestamping 1258c5521189SYangbo Lu * PTP Sync packet 1259c5521189SYangbo Lu */ 1260c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1261c5521189SYangbo Lu } 1262c5521189SYangbo Lu 1263c5521189SYangbo Lu /* TX for other packets */ 1264c5521189SYangbo Lu return __dpaa2_eth_tx(skb, net_dev); 1265c5521189SYangbo Lu } 1266c5521189SYangbo Lu 126734ff6846SIoana Radulescu /* Tx confirmation frame processing routine */ 126834ff6846SIoana Radulescu static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 1269b00c898cSIoana Ciornei struct dpaa2_eth_channel *ch __always_unused, 127034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 1271569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 127234ff6846SIoana Radulescu { 127334ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 127434ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 1275569dac6aSIoana Ciocoi Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 127634ff6846SIoana Radulescu u32 fd_errors; 127734ff6846SIoana Radulescu 127834ff6846SIoana Radulescu /* Tracing point */ 127934ff6846SIoana Radulescu trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 128034ff6846SIoana Radulescu 128134ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 128234ff6846SIoana Radulescu percpu_extras->tx_conf_frames++; 1283569dac6aSIoana Ciocoi Radulescu percpu_extras->tx_conf_bytes += fd_len; 1284569dac6aSIoana Ciocoi Radulescu 128534ff6846SIoana Radulescu /* Check frame errors in the FD field */ 128634ff6846SIoana Radulescu fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 12875d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, fd, true); 128834ff6846SIoana Radulescu 128934ff6846SIoana Radulescu if (likely(!fd_errors)) 129034ff6846SIoana Radulescu return; 129134ff6846SIoana Radulescu 129234ff6846SIoana Radulescu if (net_ratelimit()) 129334ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 129434ff6846SIoana Radulescu fd_errors); 129534ff6846SIoana Radulescu 129634ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 129734ff6846SIoana Radulescu /* Tx-conf logically pertains to the egress path. */ 129834ff6846SIoana Radulescu percpu_stats->tx_errors++; 129934ff6846SIoana Radulescu } 130034ff6846SIoana Radulescu 130170b32d82SIonut-robert Aron static int dpaa2_eth_set_rx_vlan_filtering(struct dpaa2_eth_priv *priv, 130270b32d82SIonut-robert Aron bool enable) 130370b32d82SIonut-robert Aron { 130470b32d82SIonut-robert Aron int err; 130570b32d82SIonut-robert Aron 130670b32d82SIonut-robert Aron err = dpni_enable_vlan_filter(priv->mc_io, 0, priv->mc_token, enable); 130770b32d82SIonut-robert Aron 130870b32d82SIonut-robert Aron if (err) { 130970b32d82SIonut-robert Aron netdev_err(priv->net_dev, 131070b32d82SIonut-robert Aron "dpni_enable_vlan_filter failed\n"); 131170b32d82SIonut-robert Aron return err; 131270b32d82SIonut-robert Aron } 131370b32d82SIonut-robert Aron 131470b32d82SIonut-robert Aron return 0; 131570b32d82SIonut-robert Aron } 131670b32d82SIonut-robert Aron 13175d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 131834ff6846SIoana Radulescu { 131934ff6846SIoana Radulescu int err; 132034ff6846SIoana Radulescu 132134ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 132234ff6846SIoana Radulescu DPNI_OFF_RX_L3_CSUM, enable); 132334ff6846SIoana Radulescu if (err) { 132434ff6846SIoana Radulescu netdev_err(priv->net_dev, 132534ff6846SIoana Radulescu "dpni_set_offload(RX_L3_CSUM) failed\n"); 132634ff6846SIoana Radulescu return err; 132734ff6846SIoana Radulescu } 132834ff6846SIoana Radulescu 132934ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 133034ff6846SIoana Radulescu DPNI_OFF_RX_L4_CSUM, enable); 133134ff6846SIoana Radulescu if (err) { 133234ff6846SIoana Radulescu netdev_err(priv->net_dev, 133334ff6846SIoana Radulescu "dpni_set_offload(RX_L4_CSUM) failed\n"); 133434ff6846SIoana Radulescu return err; 133534ff6846SIoana Radulescu } 133634ff6846SIoana Radulescu 133734ff6846SIoana Radulescu return 0; 133834ff6846SIoana Radulescu } 133934ff6846SIoana Radulescu 13405d8dccf8SIoana Ciornei static int dpaa2_eth_set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 134134ff6846SIoana Radulescu { 134234ff6846SIoana Radulescu int err; 134334ff6846SIoana Radulescu 134434ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 134534ff6846SIoana Radulescu DPNI_OFF_TX_L3_CSUM, enable); 134634ff6846SIoana Radulescu if (err) { 134734ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 134834ff6846SIoana Radulescu return err; 134934ff6846SIoana Radulescu } 135034ff6846SIoana Radulescu 135134ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 135234ff6846SIoana Radulescu DPNI_OFF_TX_L4_CSUM, enable); 135334ff6846SIoana Radulescu if (err) { 135434ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 135534ff6846SIoana Radulescu return err; 135634ff6846SIoana Radulescu } 135734ff6846SIoana Radulescu 135834ff6846SIoana Radulescu return 0; 135934ff6846SIoana Radulescu } 136034ff6846SIoana Radulescu 136134ff6846SIoana Radulescu /* Perform a single release command to add buffers 136234ff6846SIoana Radulescu * to the specified buffer pool 136334ff6846SIoana Radulescu */ 13645d8dccf8SIoana Ciornei static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv, 136534ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, u16 bpid) 136634ff6846SIoana Radulescu { 136734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 136834ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 136927c87486SIoana Ciocoi Radulescu struct page *page; 137034ff6846SIoana Radulescu dma_addr_t addr; 1371ef17bd7cSIoana Radulescu int retries = 0; 137234ff6846SIoana Radulescu int i, err; 137334ff6846SIoana Radulescu 137434ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 137534ff6846SIoana Radulescu /* Allocate buffer visible to WRIOP + skb shared info + 137634ff6846SIoana Radulescu * alignment padding 137734ff6846SIoana Radulescu */ 137827c87486SIoana Ciocoi Radulescu /* allocate one page for each Rx buffer. WRIOP sees 137927c87486SIoana Ciocoi Radulescu * the entire page except for a tailroom reserved for 138027c87486SIoana Ciocoi Radulescu * skb shared info 138127c87486SIoana Ciocoi Radulescu */ 138227c87486SIoana Ciocoi Radulescu page = dev_alloc_pages(0); 138327c87486SIoana Ciocoi Radulescu if (!page) 138434ff6846SIoana Radulescu goto err_alloc; 138534ff6846SIoana Radulescu 1386efa6a7d0SIoana Ciornei addr = dma_map_page(dev, page, 0, priv->rx_buf_size, 138718c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 138834ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 138934ff6846SIoana Radulescu goto err_map; 139034ff6846SIoana Radulescu 139134ff6846SIoana Radulescu buf_array[i] = addr; 139234ff6846SIoana Radulescu 139334ff6846SIoana Radulescu /* tracing point */ 139434ff6846SIoana Radulescu trace_dpaa2_eth_buf_seed(priv->net_dev, 139527c87486SIoana Ciocoi Radulescu page, DPAA2_ETH_RX_BUF_RAW_SIZE, 1396efa6a7d0SIoana Ciornei addr, priv->rx_buf_size, 139734ff6846SIoana Radulescu bpid); 139834ff6846SIoana Radulescu } 139934ff6846SIoana Radulescu 140034ff6846SIoana Radulescu release_bufs: 140134ff6846SIoana Radulescu /* In case the portal is busy, retry until successful */ 140234ff6846SIoana Radulescu while ((err = dpaa2_io_service_release(ch->dpio, bpid, 1403ef17bd7cSIoana Radulescu buf_array, i)) == -EBUSY) { 1404ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 1405ef17bd7cSIoana Radulescu break; 140634ff6846SIoana Radulescu cpu_relax(); 1407ef17bd7cSIoana Radulescu } 140834ff6846SIoana Radulescu 140934ff6846SIoana Radulescu /* If release command failed, clean up and bail out; 141034ff6846SIoana Radulescu * not much else we can do about it 141134ff6846SIoana Radulescu */ 141234ff6846SIoana Radulescu if (err) { 14135d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, i); 141434ff6846SIoana Radulescu return 0; 141534ff6846SIoana Radulescu } 141634ff6846SIoana Radulescu 141734ff6846SIoana Radulescu return i; 141834ff6846SIoana Radulescu 141934ff6846SIoana Radulescu err_map: 142027c87486SIoana Ciocoi Radulescu __free_pages(page, 0); 142134ff6846SIoana Radulescu err_alloc: 142234ff6846SIoana Radulescu /* If we managed to allocate at least some buffers, 142334ff6846SIoana Radulescu * release them to hardware 142434ff6846SIoana Radulescu */ 142534ff6846SIoana Radulescu if (i) 142634ff6846SIoana Radulescu goto release_bufs; 142734ff6846SIoana Radulescu 142834ff6846SIoana Radulescu return 0; 142934ff6846SIoana Radulescu } 143034ff6846SIoana Radulescu 14315d8dccf8SIoana Ciornei static int dpaa2_eth_seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 143234ff6846SIoana Radulescu { 143334ff6846SIoana Radulescu int i, j; 143434ff6846SIoana Radulescu int new_count; 143534ff6846SIoana Radulescu 143634ff6846SIoana Radulescu for (j = 0; j < priv->num_channels; j++) { 143734ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_NUM_BUFS; 143834ff6846SIoana Radulescu i += DPAA2_ETH_BUFS_PER_CMD) { 14395d8dccf8SIoana Ciornei new_count = dpaa2_eth_add_bufs(priv, priv->channel[j], bpid); 144034ff6846SIoana Radulescu priv->channel[j]->buf_count += new_count; 144134ff6846SIoana Radulescu 144234ff6846SIoana Radulescu if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 144334ff6846SIoana Radulescu return -ENOMEM; 144434ff6846SIoana Radulescu } 144534ff6846SIoana Radulescu } 144634ff6846SIoana Radulescu } 144734ff6846SIoana Radulescu 144834ff6846SIoana Radulescu return 0; 144934ff6846SIoana Radulescu } 145034ff6846SIoana Radulescu 1451d0ea5cbdSJesse Brandeburg /* 145234ff6846SIoana Radulescu * Drain the specified number of buffers from the DPNI's private buffer pool. 145334ff6846SIoana Radulescu * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 145434ff6846SIoana Radulescu */ 14555d8dccf8SIoana Ciornei static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int count) 145634ff6846SIoana Radulescu { 145734ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1458ef17bd7cSIoana Radulescu int retries = 0; 145934ff6846SIoana Radulescu int ret; 146034ff6846SIoana Radulescu 146134ff6846SIoana Radulescu do { 146234ff6846SIoana Radulescu ret = dpaa2_io_service_acquire(NULL, priv->bpid, 146334ff6846SIoana Radulescu buf_array, count); 146434ff6846SIoana Radulescu if (ret < 0) { 1465ef17bd7cSIoana Radulescu if (ret == -EBUSY && 14660e5ad75bSIoana Ciornei retries++ < DPAA2_ETH_SWP_BUSY_RETRIES) 1467ef17bd7cSIoana Radulescu continue; 146834ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 146934ff6846SIoana Radulescu return; 147034ff6846SIoana Radulescu } 14715d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, ret); 1472ef17bd7cSIoana Radulescu retries = 0; 147334ff6846SIoana Radulescu } while (ret); 147434ff6846SIoana Radulescu } 147534ff6846SIoana Radulescu 14765d8dccf8SIoana Ciornei static void dpaa2_eth_drain_pool(struct dpaa2_eth_priv *priv) 147734ff6846SIoana Radulescu { 147834ff6846SIoana Radulescu int i; 147934ff6846SIoana Radulescu 14805d8dccf8SIoana Ciornei dpaa2_eth_drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 14815d8dccf8SIoana Ciornei dpaa2_eth_drain_bufs(priv, 1); 148234ff6846SIoana Radulescu 148334ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 148434ff6846SIoana Radulescu priv->channel[i]->buf_count = 0; 148534ff6846SIoana Radulescu } 148634ff6846SIoana Radulescu 148734ff6846SIoana Radulescu /* Function is called from softirq context only, so we don't need to guard 148834ff6846SIoana Radulescu * the access to percpu count 148934ff6846SIoana Radulescu */ 14905d8dccf8SIoana Ciornei static int dpaa2_eth_refill_pool(struct dpaa2_eth_priv *priv, 149134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 149234ff6846SIoana Radulescu u16 bpid) 149334ff6846SIoana Radulescu { 149434ff6846SIoana Radulescu int new_count; 149534ff6846SIoana Radulescu 149634ff6846SIoana Radulescu if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 149734ff6846SIoana Radulescu return 0; 149834ff6846SIoana Radulescu 149934ff6846SIoana Radulescu do { 15005d8dccf8SIoana Ciornei new_count = dpaa2_eth_add_bufs(priv, ch, bpid); 150134ff6846SIoana Radulescu if (unlikely(!new_count)) { 150234ff6846SIoana Radulescu /* Out of memory; abort for now, we'll try later on */ 150334ff6846SIoana Radulescu break; 150434ff6846SIoana Radulescu } 150534ff6846SIoana Radulescu ch->buf_count += new_count; 150634ff6846SIoana Radulescu } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 150734ff6846SIoana Radulescu 150834ff6846SIoana Radulescu if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 150934ff6846SIoana Radulescu return -ENOMEM; 151034ff6846SIoana Radulescu 151134ff6846SIoana Radulescu return 0; 151234ff6846SIoana Radulescu } 151334ff6846SIoana Radulescu 1514d70446eeSIoana Ciornei static void dpaa2_eth_sgt_cache_drain(struct dpaa2_eth_priv *priv) 1515d70446eeSIoana Ciornei { 1516d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 1517d70446eeSIoana Ciornei u16 count; 1518d70446eeSIoana Ciornei int k, i; 1519d70446eeSIoana Ciornei 15200fe665d4SIoana Ciornei for_each_possible_cpu(k) { 1521d70446eeSIoana Ciornei sgt_cache = per_cpu_ptr(priv->sgt_cache, k); 1522d70446eeSIoana Ciornei count = sgt_cache->count; 1523d70446eeSIoana Ciornei 1524d70446eeSIoana Ciornei for (i = 0; i < count; i++) 1525d70446eeSIoana Ciornei kfree(sgt_cache->buf[i]); 1526d70446eeSIoana Ciornei sgt_cache->count = 0; 1527d70446eeSIoana Ciornei } 1528d70446eeSIoana Ciornei } 1529d70446eeSIoana Ciornei 15305d8dccf8SIoana Ciornei static int dpaa2_eth_pull_channel(struct dpaa2_eth_channel *ch) 153134ff6846SIoana Radulescu { 153234ff6846SIoana Radulescu int err; 153334ff6846SIoana Radulescu int dequeues = -1; 153434ff6846SIoana Radulescu 153534ff6846SIoana Radulescu /* Retry while portal is busy */ 153634ff6846SIoana Radulescu do { 153734ff6846SIoana Radulescu err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 153834ff6846SIoana Radulescu ch->store); 153934ff6846SIoana Radulescu dequeues++; 154034ff6846SIoana Radulescu cpu_relax(); 1541ef17bd7cSIoana Radulescu } while (err == -EBUSY && dequeues < DPAA2_ETH_SWP_BUSY_RETRIES); 154234ff6846SIoana Radulescu 154334ff6846SIoana Radulescu ch->stats.dequeue_portal_busy += dequeues; 154434ff6846SIoana Radulescu if (unlikely(err)) 154534ff6846SIoana Radulescu ch->stats.pull_err++; 154634ff6846SIoana Radulescu 154734ff6846SIoana Radulescu return err; 154834ff6846SIoana Radulescu } 154934ff6846SIoana Radulescu 155034ff6846SIoana Radulescu /* NAPI poll routine 155134ff6846SIoana Radulescu * 155234ff6846SIoana Radulescu * Frames are dequeued from the QMan channel associated with this NAPI context. 155334ff6846SIoana Radulescu * Rx, Tx confirmation and (if configured) Rx error frames all count 155434ff6846SIoana Radulescu * towards the NAPI budget. 155534ff6846SIoana Radulescu */ 155634ff6846SIoana Radulescu static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 155734ff6846SIoana Radulescu { 155834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 155934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 156068049a5fSIoana Ciocoi Radulescu int rx_cleaned = 0, txconf_cleaned = 0; 1561569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1562569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1563569dac6aSIoana Ciocoi Radulescu int store_cleaned, work_done; 15640a25d92cSIoana Ciornei struct list_head rx_list; 1565ef17bd7cSIoana Radulescu int retries = 0; 156674a1c059SIoana Ciornei u16 flowid; 156734ff6846SIoana Radulescu int err; 156834ff6846SIoana Radulescu 156934ff6846SIoana Radulescu ch = container_of(napi, struct dpaa2_eth_channel, napi); 1570d678be1dSIoana Radulescu ch->xdp.res = 0; 157134ff6846SIoana Radulescu priv = ch->priv; 157234ff6846SIoana Radulescu 15730a25d92cSIoana Ciornei INIT_LIST_HEAD(&rx_list); 15740a25d92cSIoana Ciornei ch->rx_list = &rx_list; 15750a25d92cSIoana Ciornei 157668049a5fSIoana Ciocoi Radulescu do { 15775d8dccf8SIoana Ciornei err = dpaa2_eth_pull_channel(ch); 157834ff6846SIoana Radulescu if (unlikely(err)) 157934ff6846SIoana Radulescu break; 158034ff6846SIoana Radulescu 158134ff6846SIoana Radulescu /* Refill pool if appropriate */ 15825d8dccf8SIoana Ciornei dpaa2_eth_refill_pool(priv, ch, priv->bpid); 158334ff6846SIoana Radulescu 15845d8dccf8SIoana Ciornei store_cleaned = dpaa2_eth_consume_frames(ch, &fq); 1585ef17bd7cSIoana Radulescu if (store_cleaned <= 0) 1586569dac6aSIoana Ciocoi Radulescu break; 1587569dac6aSIoana Ciocoi Radulescu if (fq->type == DPAA2_RX_FQ) { 158868049a5fSIoana Ciocoi Radulescu rx_cleaned += store_cleaned; 158974a1c059SIoana Ciornei flowid = fq->flowid; 1590569dac6aSIoana Ciocoi Radulescu } else { 159168049a5fSIoana Ciocoi Radulescu txconf_cleaned += store_cleaned; 1592569dac6aSIoana Ciocoi Radulescu /* We have a single Tx conf FQ on this channel */ 1593569dac6aSIoana Ciocoi Radulescu txc_fq = fq; 1594569dac6aSIoana Ciocoi Radulescu } 159534ff6846SIoana Radulescu 159668049a5fSIoana Ciocoi Radulescu /* If we either consumed the whole NAPI budget with Rx frames 159768049a5fSIoana Ciocoi Radulescu * or we reached the Tx confirmations threshold, we're done. 159834ff6846SIoana Radulescu */ 159968049a5fSIoana Ciocoi Radulescu if (rx_cleaned >= budget || 1600569dac6aSIoana Ciocoi Radulescu txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1601569dac6aSIoana Ciocoi Radulescu work_done = budget; 1602569dac6aSIoana Ciocoi Radulescu goto out; 1603569dac6aSIoana Ciocoi Radulescu } 160468049a5fSIoana Ciocoi Radulescu } while (store_cleaned); 160534ff6846SIoana Radulescu 160668049a5fSIoana Ciocoi Radulescu /* We didn't consume the entire budget, so finish napi and 160768049a5fSIoana Ciocoi Radulescu * re-enable data availability notifications 160868049a5fSIoana Ciocoi Radulescu */ 160968049a5fSIoana Ciocoi Radulescu napi_complete_done(napi, rx_cleaned); 161034ff6846SIoana Radulescu do { 161134ff6846SIoana Radulescu err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 161234ff6846SIoana Radulescu cpu_relax(); 1613ef17bd7cSIoana Radulescu } while (err == -EBUSY && retries++ < DPAA2_ETH_SWP_BUSY_RETRIES); 161434ff6846SIoana Radulescu WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 161534ff6846SIoana Radulescu ch->nctx.desired_cpu); 161634ff6846SIoana Radulescu 1617569dac6aSIoana Ciocoi Radulescu work_done = max(rx_cleaned, 1); 1618569dac6aSIoana Ciocoi Radulescu 1619569dac6aSIoana Ciocoi Radulescu out: 16200a25d92cSIoana Ciornei netif_receive_skb_list(ch->rx_list); 16210a25d92cSIoana Ciornei 1622d678be1dSIoana Radulescu if (txc_fq && txc_fq->dq_frames) { 1623569dac6aSIoana Ciocoi Radulescu nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1624569dac6aSIoana Ciocoi Radulescu netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1625569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes); 1626569dac6aSIoana Ciocoi Radulescu txc_fq->dq_frames = 0; 1627569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes = 0; 1628569dac6aSIoana Ciocoi Radulescu } 1629569dac6aSIoana Ciocoi Radulescu 1630d678be1dSIoana Radulescu if (ch->xdp.res & XDP_REDIRECT) 1631d678be1dSIoana Radulescu xdp_do_flush_map(); 163274a1c059SIoana Ciornei else if (rx_cleaned && ch->xdp.res & XDP_TX) 16335d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, &priv->fq[flowid]); 1634d678be1dSIoana Radulescu 1635569dac6aSIoana Ciocoi Radulescu return work_done; 163634ff6846SIoana Radulescu } 163734ff6846SIoana Radulescu 16385d8dccf8SIoana Ciornei static void dpaa2_eth_enable_ch_napi(struct dpaa2_eth_priv *priv) 163934ff6846SIoana Radulescu { 164034ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 164134ff6846SIoana Radulescu int i; 164234ff6846SIoana Radulescu 164334ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 164434ff6846SIoana Radulescu ch = priv->channel[i]; 164534ff6846SIoana Radulescu napi_enable(&ch->napi); 164634ff6846SIoana Radulescu } 164734ff6846SIoana Radulescu } 164834ff6846SIoana Radulescu 16495d8dccf8SIoana Ciornei static void dpaa2_eth_disable_ch_napi(struct dpaa2_eth_priv *priv) 165034ff6846SIoana Radulescu { 165134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 165234ff6846SIoana Radulescu int i; 165334ff6846SIoana Radulescu 165434ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 165534ff6846SIoana Radulescu ch = priv->channel[i]; 165634ff6846SIoana Radulescu napi_disable(&ch->napi); 165734ff6846SIoana Radulescu } 165834ff6846SIoana Radulescu } 165934ff6846SIoana Radulescu 166007beb165SIoana Ciornei void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, 166107beb165SIoana Ciornei bool tx_pause, bool pfc) 16628eb3cef8SIoana Radulescu { 16638eb3cef8SIoana Radulescu struct dpni_taildrop td = {0}; 1664685e39eaSIoana Radulescu struct dpaa2_eth_fq *fq; 16658eb3cef8SIoana Radulescu int i, err; 16668eb3cef8SIoana Radulescu 166707beb165SIoana Ciornei /* FQ taildrop: threshold is in bytes, per frame queue. Enabled if 166807beb165SIoana Ciornei * flow control is disabled (as it might interfere with either the 166907beb165SIoana Ciornei * buffer pool depletion trigger for pause frames or with the group 167007beb165SIoana Ciornei * congestion trigger for PFC frames) 167107beb165SIoana Ciornei */ 16722c8d1c8dSIoana Radulescu td.enable = !tx_pause; 167307beb165SIoana Ciornei if (priv->rx_fqtd_enabled == td.enable) 167407beb165SIoana Ciornei goto set_cgtd; 16758eb3cef8SIoana Radulescu 16762c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_FQ_TAILDROP_THRESH; 16772c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_BYTES; 16788eb3cef8SIoana Radulescu 16798eb3cef8SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 1680685e39eaSIoana Radulescu fq = &priv->fq[i]; 1681685e39eaSIoana Radulescu if (fq->type != DPAA2_RX_FQ) 16828eb3cef8SIoana Radulescu continue; 16838eb3cef8SIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1684685e39eaSIoana Radulescu DPNI_CP_QUEUE, DPNI_QUEUE_RX, 1685685e39eaSIoana Radulescu fq->tc, fq->flowid, &td); 16868eb3cef8SIoana Radulescu if (err) { 16878eb3cef8SIoana Radulescu netdev_err(priv->net_dev, 16882c8d1c8dSIoana Radulescu "dpni_set_taildrop(FQ) failed\n"); 16892c8d1c8dSIoana Radulescu return; 16908eb3cef8SIoana Radulescu } 16918eb3cef8SIoana Radulescu } 16928eb3cef8SIoana Radulescu 169307beb165SIoana Ciornei priv->rx_fqtd_enabled = td.enable; 169407beb165SIoana Ciornei 169507beb165SIoana Ciornei set_cgtd: 16962c8d1c8dSIoana Radulescu /* Congestion group taildrop: threshold is in frames, per group 16972c8d1c8dSIoana Radulescu * of FQs belonging to the same traffic class 169807beb165SIoana Ciornei * Enabled if general Tx pause disabled or if PFCs are enabled 169907beb165SIoana Ciornei * (congestion group threhsold for PFC generation is lower than the 170007beb165SIoana Ciornei * CG taildrop threshold, so it won't interfere with it; we also 170107beb165SIoana Ciornei * want frames in non-PFC enabled traffic classes to be kept in check) 17022c8d1c8dSIoana Radulescu */ 1703b91b3a21SJiapeng Chong td.enable = !tx_pause || pfc; 170407beb165SIoana Ciornei if (priv->rx_cgtd_enabled == td.enable) 170507beb165SIoana Ciornei return; 170607beb165SIoana Ciornei 17072c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_CG_TAILDROP_THRESH(priv); 17082c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_FRAMES; 17092c8d1c8dSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 17102c8d1c8dSIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 17112c8d1c8dSIoana Radulescu DPNI_CP_GROUP, DPNI_QUEUE_RX, 17122c8d1c8dSIoana Radulescu i, 0, &td); 17132c8d1c8dSIoana Radulescu if (err) { 17142c8d1c8dSIoana Radulescu netdev_err(priv->net_dev, 17152c8d1c8dSIoana Radulescu "dpni_set_taildrop(CG) failed\n"); 17162c8d1c8dSIoana Radulescu return; 17172c8d1c8dSIoana Radulescu } 17182c8d1c8dSIoana Radulescu } 17192c8d1c8dSIoana Radulescu 172007beb165SIoana Ciornei priv->rx_cgtd_enabled = td.enable; 17218eb3cef8SIoana Radulescu } 17228eb3cef8SIoana Radulescu 17235d8dccf8SIoana Ciornei static int dpaa2_eth_link_state_update(struct dpaa2_eth_priv *priv) 172434ff6846SIoana Radulescu { 172585b7a342SIoana Ciornei struct dpni_link_state state = {0}; 17268eb3cef8SIoana Radulescu bool tx_pause; 172734ff6846SIoana Radulescu int err; 172834ff6846SIoana Radulescu 172934ff6846SIoana Radulescu err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 173034ff6846SIoana Radulescu if (unlikely(err)) { 173134ff6846SIoana Radulescu netdev_err(priv->net_dev, 173234ff6846SIoana Radulescu "dpni_get_link_state() failed\n"); 173334ff6846SIoana Radulescu return err; 173434ff6846SIoana Radulescu } 173534ff6846SIoana Radulescu 17368eb3cef8SIoana Radulescu /* If Tx pause frame settings have changed, we need to update 17378eb3cef8SIoana Radulescu * Rx FQ taildrop configuration as well. We configure taildrop 17388eb3cef8SIoana Radulescu * only when pause frame generation is disabled. 17398eb3cef8SIoana Radulescu */ 1740ad054f26SIoana Radulescu tx_pause = dpaa2_eth_tx_pause_enabled(state.options); 174107beb165SIoana Ciornei dpaa2_eth_set_rx_taildrop(priv, tx_pause, priv->pfc_enabled); 17428eb3cef8SIoana Radulescu 174371947923SIoana Ciornei /* When we manage the MAC/PHY using phylink there is no need 174471947923SIoana Ciornei * to manually update the netif_carrier. 174571947923SIoana Ciornei */ 1746d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 174771947923SIoana Ciornei goto out; 174871947923SIoana Ciornei 174934ff6846SIoana Radulescu /* Chech link state; speed / duplex changes are not treated yet */ 175034ff6846SIoana Radulescu if (priv->link_state.up == state.up) 1751cce62943SIoana Radulescu goto out; 175234ff6846SIoana Radulescu 175334ff6846SIoana Radulescu if (state.up) { 175434ff6846SIoana Radulescu netif_carrier_on(priv->net_dev); 175534ff6846SIoana Radulescu netif_tx_start_all_queues(priv->net_dev); 175634ff6846SIoana Radulescu } else { 175734ff6846SIoana Radulescu netif_tx_stop_all_queues(priv->net_dev); 175834ff6846SIoana Radulescu netif_carrier_off(priv->net_dev); 175934ff6846SIoana Radulescu } 176034ff6846SIoana Radulescu 176134ff6846SIoana Radulescu netdev_info(priv->net_dev, "Link Event: state %s\n", 176234ff6846SIoana Radulescu state.up ? "up" : "down"); 176334ff6846SIoana Radulescu 1764cce62943SIoana Radulescu out: 1765cce62943SIoana Radulescu priv->link_state = state; 1766cce62943SIoana Radulescu 176734ff6846SIoana Radulescu return 0; 176834ff6846SIoana Radulescu } 176934ff6846SIoana Radulescu 177034ff6846SIoana Radulescu static int dpaa2_eth_open(struct net_device *net_dev) 177134ff6846SIoana Radulescu { 177234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 177334ff6846SIoana Radulescu int err; 177434ff6846SIoana Radulescu 17755d8dccf8SIoana Ciornei err = dpaa2_eth_seed_pool(priv, priv->bpid); 177634ff6846SIoana Radulescu if (err) { 177734ff6846SIoana Radulescu /* Not much to do; the buffer pool, though not filled up, 177834ff6846SIoana Radulescu * may still contain some buffers which would enable us 177934ff6846SIoana Radulescu * to limp on. 178034ff6846SIoana Radulescu */ 178134ff6846SIoana Radulescu netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 178234ff6846SIoana Radulescu priv->dpbp_dev->obj_desc.id, priv->bpid); 178334ff6846SIoana Radulescu } 178434ff6846SIoana Radulescu 1785d87e6063SIoana Ciornei if (!dpaa2_eth_is_type_phy(priv)) { 178671947923SIoana Ciornei /* We'll only start the txqs when the link is actually ready; 178771947923SIoana Ciornei * make sure we don't race against the link up notification, 178871947923SIoana Ciornei * which may come immediately after dpni_enable(); 178934ff6846SIoana Radulescu */ 179034ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 179171947923SIoana Ciornei 179271947923SIoana Ciornei /* Also, explicitly set carrier off, otherwise 179371947923SIoana Ciornei * netif_carrier_ok() will return true and cause 'ip link show' 179471947923SIoana Ciornei * to report the LOWER_UP flag, even though the link 179571947923SIoana Ciornei * notification wasn't even received. 179634ff6846SIoana Radulescu */ 179734ff6846SIoana Radulescu netif_carrier_off(net_dev); 179871947923SIoana Ciornei } 17995d8dccf8SIoana Ciornei dpaa2_eth_enable_ch_napi(priv); 180034ff6846SIoana Radulescu 180134ff6846SIoana Radulescu err = dpni_enable(priv->mc_io, 0, priv->mc_token); 180234ff6846SIoana Radulescu if (err < 0) { 180334ff6846SIoana Radulescu netdev_err(net_dev, "dpni_enable() failed\n"); 180434ff6846SIoana Radulescu goto enable_err; 180534ff6846SIoana Radulescu } 180634ff6846SIoana Radulescu 1807d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 180871947923SIoana Ciornei phylink_start(priv->mac->phylink); 180934ff6846SIoana Radulescu 181034ff6846SIoana Radulescu return 0; 181134ff6846SIoana Radulescu 181234ff6846SIoana Radulescu enable_err: 18135d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 18145d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 181534ff6846SIoana Radulescu return err; 181634ff6846SIoana Radulescu } 181734ff6846SIoana Radulescu 181868d74315SIoana Ciocoi Radulescu /* Total number of in-flight frames on ingress queues */ 18195d8dccf8SIoana Ciornei static u32 dpaa2_eth_ingress_fq_count(struct dpaa2_eth_priv *priv) 182034ff6846SIoana Radulescu { 182168d74315SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq; 182268d74315SIoana Ciocoi Radulescu u32 fcnt = 0, bcnt = 0, total = 0; 182368d74315SIoana Ciocoi Radulescu int i, err; 182434ff6846SIoana Radulescu 182568d74315SIoana Ciocoi Radulescu for (i = 0; i < priv->num_fqs; i++) { 182668d74315SIoana Ciocoi Radulescu fq = &priv->fq[i]; 182768d74315SIoana Ciocoi Radulescu err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 182868d74315SIoana Ciocoi Radulescu if (err) { 182968d74315SIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "query_fq_count failed"); 183068d74315SIoana Ciocoi Radulescu break; 183168d74315SIoana Ciocoi Radulescu } 183268d74315SIoana Ciocoi Radulescu total += fcnt; 183368d74315SIoana Ciocoi Radulescu } 183434ff6846SIoana Radulescu 183534ff6846SIoana Radulescu return total; 183634ff6846SIoana Radulescu } 183734ff6846SIoana Radulescu 18385d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 183934ff6846SIoana Radulescu { 184068d74315SIoana Ciocoi Radulescu int retries = 10; 184168d74315SIoana Ciocoi Radulescu u32 pending; 184234ff6846SIoana Radulescu 184368d74315SIoana Ciocoi Radulescu do { 18445d8dccf8SIoana Ciornei pending = dpaa2_eth_ingress_fq_count(priv); 184568d74315SIoana Ciocoi Radulescu if (pending) 184668d74315SIoana Ciocoi Radulescu msleep(100); 184768d74315SIoana Ciocoi Radulescu } while (pending && --retries); 184834ff6846SIoana Radulescu } 184934ff6846SIoana Radulescu 185052b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MAJOR 7 185152b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MINOR 13 18525d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 185352b6a4ffSIoana Radulescu { 185452b6a4ffSIoana Radulescu union dpni_statistics stats; 185552b6a4ffSIoana Radulescu int retries = 10; 185652b6a4ffSIoana Radulescu int err; 185752b6a4ffSIoana Radulescu 185852b6a4ffSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 185952b6a4ffSIoana Radulescu DPNI_TX_PENDING_VER_MINOR) < 0) 186052b6a4ffSIoana Radulescu goto out; 186152b6a4ffSIoana Radulescu 186252b6a4ffSIoana Radulescu do { 186352b6a4ffSIoana Radulescu err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 186452b6a4ffSIoana Radulescu &stats); 186552b6a4ffSIoana Radulescu if (err) 186652b6a4ffSIoana Radulescu goto out; 186752b6a4ffSIoana Radulescu if (stats.page_6.tx_pending_frames == 0) 186852b6a4ffSIoana Radulescu return; 186952b6a4ffSIoana Radulescu } while (--retries); 187052b6a4ffSIoana Radulescu 187152b6a4ffSIoana Radulescu out: 187252b6a4ffSIoana Radulescu msleep(500); 187352b6a4ffSIoana Radulescu } 187452b6a4ffSIoana Radulescu 187534ff6846SIoana Radulescu static int dpaa2_eth_stop(struct net_device *net_dev) 187634ff6846SIoana Radulescu { 187734ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 187885b7a342SIoana Ciornei int dpni_enabled = 0; 187934ff6846SIoana Radulescu int retries = 10; 188034ff6846SIoana Radulescu 1881d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 1882d87e6063SIoana Ciornei phylink_stop(priv->mac->phylink); 1883d87e6063SIoana Ciornei } else { 188434ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 188534ff6846SIoana Radulescu netif_carrier_off(net_dev); 188671947923SIoana Ciornei } 188734ff6846SIoana Radulescu 188868d74315SIoana Ciocoi Radulescu /* On dpni_disable(), the MC firmware will: 188968d74315SIoana Ciocoi Radulescu * - stop MAC Rx and wait for all Rx frames to be enqueued to software 189068d74315SIoana Ciocoi Radulescu * - cut off WRIOP dequeues from egress FQs and wait until transmission 189168d74315SIoana Ciocoi Radulescu * of all in flight Tx frames is finished (and corresponding Tx conf 189268d74315SIoana Ciocoi Radulescu * frames are enqueued back to software) 189368d74315SIoana Ciocoi Radulescu * 189468d74315SIoana Ciocoi Radulescu * Before calling dpni_disable(), we wait for all Tx frames to arrive 189568d74315SIoana Ciocoi Radulescu * on WRIOP. After it finishes, wait until all remaining frames on Rx 189668d74315SIoana Ciocoi Radulescu * and Tx conf queues are consumed on NAPI poll. 189734ff6846SIoana Radulescu */ 18985d8dccf8SIoana Ciornei dpaa2_eth_wait_for_egress_fq_empty(priv); 189968d74315SIoana Ciocoi Radulescu 190034ff6846SIoana Radulescu do { 190134ff6846SIoana Radulescu dpni_disable(priv->mc_io, 0, priv->mc_token); 190234ff6846SIoana Radulescu dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 190334ff6846SIoana Radulescu if (dpni_enabled) 190434ff6846SIoana Radulescu /* Allow the hardware some slack */ 190534ff6846SIoana Radulescu msleep(100); 190634ff6846SIoana Radulescu } while (dpni_enabled && --retries); 190734ff6846SIoana Radulescu if (!retries) { 190834ff6846SIoana Radulescu netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 190934ff6846SIoana Radulescu /* Must go on and disable NAPI nonetheless, so we don't crash at 191034ff6846SIoana Radulescu * the next "ifconfig up" 191134ff6846SIoana Radulescu */ 191234ff6846SIoana Radulescu } 191334ff6846SIoana Radulescu 19145d8dccf8SIoana Ciornei dpaa2_eth_wait_for_ingress_fq_empty(priv); 19155d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 191634ff6846SIoana Radulescu 191734ff6846SIoana Radulescu /* Empty the buffer pool */ 19185d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 191934ff6846SIoana Radulescu 1920d70446eeSIoana Ciornei /* Empty the Scatter-Gather Buffer cache */ 1921d70446eeSIoana Ciornei dpaa2_eth_sgt_cache_drain(priv); 1922d70446eeSIoana Ciornei 192334ff6846SIoana Radulescu return 0; 192434ff6846SIoana Radulescu } 192534ff6846SIoana Radulescu 192634ff6846SIoana Radulescu static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 192734ff6846SIoana Radulescu { 192834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 192934ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 193034ff6846SIoana Radulescu int err; 193134ff6846SIoana Radulescu 193234ff6846SIoana Radulescu err = eth_mac_addr(net_dev, addr); 193334ff6846SIoana Radulescu if (err < 0) { 193434ff6846SIoana Radulescu dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 193534ff6846SIoana Radulescu return err; 193634ff6846SIoana Radulescu } 193734ff6846SIoana Radulescu 193834ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 193934ff6846SIoana Radulescu net_dev->dev_addr); 194034ff6846SIoana Radulescu if (err) { 194134ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 194234ff6846SIoana Radulescu return err; 194334ff6846SIoana Radulescu } 194434ff6846SIoana Radulescu 194534ff6846SIoana Radulescu return 0; 194634ff6846SIoana Radulescu } 194734ff6846SIoana Radulescu 194834ff6846SIoana Radulescu /** Fill in counters maintained by the GPP driver. These may be different from 194934ff6846SIoana Radulescu * the hardware counters obtained by ethtool. 195034ff6846SIoana Radulescu */ 195134ff6846SIoana Radulescu static void dpaa2_eth_get_stats(struct net_device *net_dev, 195234ff6846SIoana Radulescu struct rtnl_link_stats64 *stats) 195334ff6846SIoana Radulescu { 195434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 195534ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 195634ff6846SIoana Radulescu u64 *cpustats; 195734ff6846SIoana Radulescu u64 *netstats = (u64 *)stats; 195834ff6846SIoana Radulescu int i, j; 195934ff6846SIoana Radulescu int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 196034ff6846SIoana Radulescu 196134ff6846SIoana Radulescu for_each_possible_cpu(i) { 196234ff6846SIoana Radulescu percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 196334ff6846SIoana Radulescu cpustats = (u64 *)percpu_stats; 196434ff6846SIoana Radulescu for (j = 0; j < num; j++) 196534ff6846SIoana Radulescu netstats[j] += cpustats[j]; 196634ff6846SIoana Radulescu } 196734ff6846SIoana Radulescu } 196834ff6846SIoana Radulescu 196934ff6846SIoana Radulescu /* Copy mac unicast addresses from @net_dev to @priv. 197034ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 197134ff6846SIoana Radulescu */ 19725d8dccf8SIoana Ciornei static void dpaa2_eth_add_uc_hw_addr(const struct net_device *net_dev, 197334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 197434ff6846SIoana Radulescu { 197534ff6846SIoana Radulescu struct netdev_hw_addr *ha; 197634ff6846SIoana Radulescu int err; 197734ff6846SIoana Radulescu 197834ff6846SIoana Radulescu netdev_for_each_uc_addr(ha, net_dev) { 197934ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 198034ff6846SIoana Radulescu ha->addr); 198134ff6846SIoana Radulescu if (err) 198234ff6846SIoana Radulescu netdev_warn(priv->net_dev, 198334ff6846SIoana Radulescu "Could not add ucast MAC %pM to the filtering table (err %d)\n", 198434ff6846SIoana Radulescu ha->addr, err); 198534ff6846SIoana Radulescu } 198634ff6846SIoana Radulescu } 198734ff6846SIoana Radulescu 198834ff6846SIoana Radulescu /* Copy mac multicast addresses from @net_dev to @priv 198934ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 199034ff6846SIoana Radulescu */ 19915d8dccf8SIoana Ciornei static void dpaa2_eth_add_mc_hw_addr(const struct net_device *net_dev, 199234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 199334ff6846SIoana Radulescu { 199434ff6846SIoana Radulescu struct netdev_hw_addr *ha; 199534ff6846SIoana Radulescu int err; 199634ff6846SIoana Radulescu 199734ff6846SIoana Radulescu netdev_for_each_mc_addr(ha, net_dev) { 199834ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 199934ff6846SIoana Radulescu ha->addr); 200034ff6846SIoana Radulescu if (err) 200134ff6846SIoana Radulescu netdev_warn(priv->net_dev, 200234ff6846SIoana Radulescu "Could not add mcast MAC %pM to the filtering table (err %d)\n", 200334ff6846SIoana Radulescu ha->addr, err); 200434ff6846SIoana Radulescu } 200534ff6846SIoana Radulescu } 200634ff6846SIoana Radulescu 200770b32d82SIonut-robert Aron static int dpaa2_eth_rx_add_vid(struct net_device *net_dev, 200870b32d82SIonut-robert Aron __be16 vlan_proto, u16 vid) 200970b32d82SIonut-robert Aron { 201070b32d82SIonut-robert Aron struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 201170b32d82SIonut-robert Aron int err; 201270b32d82SIonut-robert Aron 201370b32d82SIonut-robert Aron err = dpni_add_vlan_id(priv->mc_io, 0, priv->mc_token, 201470b32d82SIonut-robert Aron vid, 0, 0, 0); 201570b32d82SIonut-robert Aron 201670b32d82SIonut-robert Aron if (err) { 201770b32d82SIonut-robert Aron netdev_warn(priv->net_dev, 201870b32d82SIonut-robert Aron "Could not add the vlan id %u\n", 201970b32d82SIonut-robert Aron vid); 202070b32d82SIonut-robert Aron return err; 202170b32d82SIonut-robert Aron } 202270b32d82SIonut-robert Aron 202370b32d82SIonut-robert Aron return 0; 202470b32d82SIonut-robert Aron } 202570b32d82SIonut-robert Aron 202670b32d82SIonut-robert Aron static int dpaa2_eth_rx_kill_vid(struct net_device *net_dev, 202770b32d82SIonut-robert Aron __be16 vlan_proto, u16 vid) 202870b32d82SIonut-robert Aron { 202970b32d82SIonut-robert Aron struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 203070b32d82SIonut-robert Aron int err; 203170b32d82SIonut-robert Aron 203270b32d82SIonut-robert Aron err = dpni_remove_vlan_id(priv->mc_io, 0, priv->mc_token, vid); 203370b32d82SIonut-robert Aron 203470b32d82SIonut-robert Aron if (err) { 203570b32d82SIonut-robert Aron netdev_warn(priv->net_dev, 203670b32d82SIonut-robert Aron "Could not remove the vlan id %u\n", 203770b32d82SIonut-robert Aron vid); 203870b32d82SIonut-robert Aron return err; 203970b32d82SIonut-robert Aron } 204070b32d82SIonut-robert Aron 204170b32d82SIonut-robert Aron return 0; 204270b32d82SIonut-robert Aron } 204370b32d82SIonut-robert Aron 204434ff6846SIoana Radulescu static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 204534ff6846SIoana Radulescu { 204634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 204734ff6846SIoana Radulescu int uc_count = netdev_uc_count(net_dev); 204834ff6846SIoana Radulescu int mc_count = netdev_mc_count(net_dev); 204934ff6846SIoana Radulescu u8 max_mac = priv->dpni_attrs.mac_filter_entries; 205034ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 205134ff6846SIoana Radulescu u16 mc_token = priv->mc_token; 205234ff6846SIoana Radulescu struct fsl_mc_io *mc_io = priv->mc_io; 205334ff6846SIoana Radulescu int err; 205434ff6846SIoana Radulescu 205534ff6846SIoana Radulescu /* Basic sanity checks; these probably indicate a misconfiguration */ 205634ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 205734ff6846SIoana Radulescu netdev_info(net_dev, 205834ff6846SIoana Radulescu "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 205934ff6846SIoana Radulescu max_mac); 206034ff6846SIoana Radulescu 206134ff6846SIoana Radulescu /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 206234ff6846SIoana Radulescu if (uc_count > max_mac) { 206334ff6846SIoana Radulescu netdev_info(net_dev, 206434ff6846SIoana Radulescu "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 206534ff6846SIoana Radulescu uc_count, max_mac); 206634ff6846SIoana Radulescu goto force_promisc; 206734ff6846SIoana Radulescu } 206834ff6846SIoana Radulescu if (mc_count + uc_count > max_mac) { 206934ff6846SIoana Radulescu netdev_info(net_dev, 207034ff6846SIoana Radulescu "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 207134ff6846SIoana Radulescu uc_count + mc_count, max_mac); 207234ff6846SIoana Radulescu goto force_mc_promisc; 207334ff6846SIoana Radulescu } 207434ff6846SIoana Radulescu 207534ff6846SIoana Radulescu /* Adjust promisc settings due to flag combinations */ 207634ff6846SIoana Radulescu if (net_dev->flags & IFF_PROMISC) 207734ff6846SIoana Radulescu goto force_promisc; 207834ff6846SIoana Radulescu if (net_dev->flags & IFF_ALLMULTI) { 207934ff6846SIoana Radulescu /* First, rebuild unicast filtering table. This should be done 208034ff6846SIoana Radulescu * in promisc mode, in order to avoid frame loss while we 208134ff6846SIoana Radulescu * progressively add entries to the table. 208234ff6846SIoana Radulescu * We don't know whether we had been in promisc already, and 208334ff6846SIoana Radulescu * making an MC call to find out is expensive; so set uc promisc 208434ff6846SIoana Radulescu * nonetheless. 208534ff6846SIoana Radulescu */ 208634ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 208734ff6846SIoana Radulescu if (err) 208834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc\n"); 208934ff6846SIoana Radulescu 209034ff6846SIoana Radulescu /* Actual uc table reconstruction. */ 209134ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 209234ff6846SIoana Radulescu if (err) 209334ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc filters\n"); 20945d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 209534ff6846SIoana Radulescu 209634ff6846SIoana Radulescu /* Finally, clear uc promisc and set mc promisc as requested. */ 209734ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 209834ff6846SIoana Radulescu if (err) 209934ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc promisc\n"); 210034ff6846SIoana Radulescu goto force_mc_promisc; 210134ff6846SIoana Radulescu } 210234ff6846SIoana Radulescu 210334ff6846SIoana Radulescu /* Neither unicast, nor multicast promisc will be on... eventually. 210434ff6846SIoana Radulescu * For now, rebuild mac filtering tables while forcing both of them on. 210534ff6846SIoana Radulescu */ 210634ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 210734ff6846SIoana Radulescu if (err) 210834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 210934ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 211034ff6846SIoana Radulescu if (err) 211134ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 211234ff6846SIoana Radulescu 211334ff6846SIoana Radulescu /* Actual mac filtering tables reconstruction */ 211434ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 211534ff6846SIoana Radulescu if (err) 211634ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mac filters\n"); 21175d8dccf8SIoana Ciornei dpaa2_eth_add_mc_hw_addr(net_dev, priv); 21185d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 211934ff6846SIoana Radulescu 212034ff6846SIoana Radulescu /* Now we can clear both ucast and mcast promisc, without risking 212134ff6846SIoana Radulescu * to drop legitimate frames anymore. 212234ff6846SIoana Radulescu */ 212334ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 212434ff6846SIoana Radulescu if (err) 212534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear ucast promisc\n"); 212634ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 212734ff6846SIoana Radulescu if (err) 212834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mcast promisc\n"); 212934ff6846SIoana Radulescu 213034ff6846SIoana Radulescu return; 213134ff6846SIoana Radulescu 213234ff6846SIoana Radulescu force_promisc: 213334ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 213434ff6846SIoana Radulescu if (err) 213534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set ucast promisc\n"); 213634ff6846SIoana Radulescu force_mc_promisc: 213734ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 213834ff6846SIoana Radulescu if (err) 213934ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mcast promisc\n"); 214034ff6846SIoana Radulescu } 214134ff6846SIoana Radulescu 214234ff6846SIoana Radulescu static int dpaa2_eth_set_features(struct net_device *net_dev, 214334ff6846SIoana Radulescu netdev_features_t features) 214434ff6846SIoana Radulescu { 214534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 214634ff6846SIoana Radulescu netdev_features_t changed = features ^ net_dev->features; 214734ff6846SIoana Radulescu bool enable; 214834ff6846SIoana Radulescu int err; 214934ff6846SIoana Radulescu 215070b32d82SIonut-robert Aron if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) { 215170b32d82SIonut-robert Aron enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER); 215270b32d82SIonut-robert Aron err = dpaa2_eth_set_rx_vlan_filtering(priv, enable); 215370b32d82SIonut-robert Aron if (err) 215470b32d82SIonut-robert Aron return err; 215570b32d82SIonut-robert Aron } 215670b32d82SIonut-robert Aron 215734ff6846SIoana Radulescu if (changed & NETIF_F_RXCSUM) { 215834ff6846SIoana Radulescu enable = !!(features & NETIF_F_RXCSUM); 21595d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, enable); 216034ff6846SIoana Radulescu if (err) 216134ff6846SIoana Radulescu return err; 216234ff6846SIoana Radulescu } 216334ff6846SIoana Radulescu 216434ff6846SIoana Radulescu if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 216534ff6846SIoana Radulescu enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 21665d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, enable); 216734ff6846SIoana Radulescu if (err) 216834ff6846SIoana Radulescu return err; 216934ff6846SIoana Radulescu } 217034ff6846SIoana Radulescu 217134ff6846SIoana Radulescu return 0; 217234ff6846SIoana Radulescu } 217334ff6846SIoana Radulescu 217434ff6846SIoana Radulescu static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 217534ff6846SIoana Radulescu { 217634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 217734ff6846SIoana Radulescu struct hwtstamp_config config; 217834ff6846SIoana Radulescu 2179c5521189SYangbo Lu if (!dpaa2_ptp) 2180c5521189SYangbo Lu return -EINVAL; 2181c5521189SYangbo Lu 218234ff6846SIoana Radulescu if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 218334ff6846SIoana Radulescu return -EFAULT; 218434ff6846SIoana Radulescu 218534ff6846SIoana Radulescu switch (config.tx_type) { 218634ff6846SIoana Radulescu case HWTSTAMP_TX_OFF: 218734ff6846SIoana Radulescu case HWTSTAMP_TX_ON: 2188c5521189SYangbo Lu case HWTSTAMP_TX_ONESTEP_SYNC: 21891cf773bdSYangbo Lu priv->tx_tstamp_type = config.tx_type; 219034ff6846SIoana Radulescu break; 219134ff6846SIoana Radulescu default: 219234ff6846SIoana Radulescu return -ERANGE; 219334ff6846SIoana Radulescu } 219434ff6846SIoana Radulescu 219534ff6846SIoana Radulescu if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 219634ff6846SIoana Radulescu priv->rx_tstamp = false; 219734ff6846SIoana Radulescu } else { 219834ff6846SIoana Radulescu priv->rx_tstamp = true; 219934ff6846SIoana Radulescu /* TS is set for all frame types, not only those requested */ 220034ff6846SIoana Radulescu config.rx_filter = HWTSTAMP_FILTER_ALL; 220134ff6846SIoana Radulescu } 220234ff6846SIoana Radulescu 220334ff6846SIoana Radulescu return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 220434ff6846SIoana Radulescu -EFAULT : 0; 220534ff6846SIoana Radulescu } 220634ff6846SIoana Radulescu 220734ff6846SIoana Radulescu static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 220834ff6846SIoana Radulescu { 22094a84182aSRussell King struct dpaa2_eth_priv *priv = netdev_priv(dev); 22104a84182aSRussell King 221134ff6846SIoana Radulescu if (cmd == SIOCSHWTSTAMP) 221234ff6846SIoana Radulescu return dpaa2_eth_ts_ioctl(dev, rq, cmd); 221334ff6846SIoana Radulescu 2214d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 22154a84182aSRussell King return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); 22164a84182aSRussell King 22174a84182aSRussell King return -EOPNOTSUPP; 221834ff6846SIoana Radulescu } 221934ff6846SIoana Radulescu 22207e273a8eSIoana Ciocoi Radulescu static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 22217e273a8eSIoana Ciocoi Radulescu { 22227e273a8eSIoana Ciocoi Radulescu int mfl, linear_mfl; 22237e273a8eSIoana Ciocoi Radulescu 22247e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 2225efa6a7d0SIoana Ciornei linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE - 22267b1eea1aSIoana Ciocoi Radulescu dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 22277e273a8eSIoana Ciocoi Radulescu 22287e273a8eSIoana Ciocoi Radulescu if (mfl > linear_mfl) { 22297e273a8eSIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 22307e273a8eSIoana Ciocoi Radulescu linear_mfl - VLAN_ETH_HLEN); 22317e273a8eSIoana Ciocoi Radulescu return false; 22327e273a8eSIoana Ciocoi Radulescu } 22337e273a8eSIoana Ciocoi Radulescu 22347e273a8eSIoana Ciocoi Radulescu return true; 22357e273a8eSIoana Ciocoi Radulescu } 22367e273a8eSIoana Ciocoi Radulescu 22375d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 22387e273a8eSIoana Ciocoi Radulescu { 22397e273a8eSIoana Ciocoi Radulescu int mfl, err; 22407e273a8eSIoana Ciocoi Radulescu 22417e273a8eSIoana Ciocoi Radulescu /* We enforce a maximum Rx frame length based on MTU only if we have 22427e273a8eSIoana Ciocoi Radulescu * an XDP program attached (in order to avoid Rx S/G frames). 22437e273a8eSIoana Ciocoi Radulescu * Otherwise, we accept all incoming frames as long as they are not 22447e273a8eSIoana Ciocoi Radulescu * larger than maximum size supported in hardware 22457e273a8eSIoana Ciocoi Radulescu */ 22467e273a8eSIoana Ciocoi Radulescu if (has_xdp) 22477e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 22487e273a8eSIoana Ciocoi Radulescu else 22497e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_MFL; 22507e273a8eSIoana Ciocoi Radulescu 22517e273a8eSIoana Ciocoi Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 22527e273a8eSIoana Ciocoi Radulescu if (err) { 22537e273a8eSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 22547e273a8eSIoana Ciocoi Radulescu return err; 22557e273a8eSIoana Ciocoi Radulescu } 22567e273a8eSIoana Ciocoi Radulescu 22577e273a8eSIoana Ciocoi Radulescu return 0; 22587e273a8eSIoana Ciocoi Radulescu } 22597e273a8eSIoana Ciocoi Radulescu 22607e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 22617e273a8eSIoana Ciocoi Radulescu { 22627e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 22637e273a8eSIoana Ciocoi Radulescu int err; 22647e273a8eSIoana Ciocoi Radulescu 22657e273a8eSIoana Ciocoi Radulescu if (!priv->xdp_prog) 22667e273a8eSIoana Ciocoi Radulescu goto out; 22677e273a8eSIoana Ciocoi Radulescu 22687e273a8eSIoana Ciocoi Radulescu if (!xdp_mtu_valid(priv, new_mtu)) 22697e273a8eSIoana Ciocoi Radulescu return -EINVAL; 22707e273a8eSIoana Ciocoi Radulescu 22715d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, new_mtu, true); 22727e273a8eSIoana Ciocoi Radulescu if (err) 22737e273a8eSIoana Ciocoi Radulescu return err; 22747e273a8eSIoana Ciocoi Radulescu 22757e273a8eSIoana Ciocoi Radulescu out: 22767e273a8eSIoana Ciocoi Radulescu dev->mtu = new_mtu; 22777e273a8eSIoana Ciocoi Radulescu return 0; 22787e273a8eSIoana Ciocoi Radulescu } 22797e273a8eSIoana Ciocoi Radulescu 22805d8dccf8SIoana Ciornei static int dpaa2_eth_update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 22817b1eea1aSIoana Ciocoi Radulescu { 22827b1eea1aSIoana Ciocoi Radulescu struct dpni_buffer_layout buf_layout = {0}; 22837b1eea1aSIoana Ciocoi Radulescu int err; 22847b1eea1aSIoana Ciocoi Radulescu 22857b1eea1aSIoana Ciocoi Radulescu err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 22867b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 22877b1eea1aSIoana Ciocoi Radulescu if (err) { 22887b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 22897b1eea1aSIoana Ciocoi Radulescu return err; 22907b1eea1aSIoana Ciocoi Radulescu } 22917b1eea1aSIoana Ciocoi Radulescu 22927b1eea1aSIoana Ciocoi Radulescu /* Reserve extra headroom for XDP header size changes */ 22937b1eea1aSIoana Ciocoi Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 22947b1eea1aSIoana Ciocoi Radulescu (has_xdp ? XDP_PACKET_HEADROOM : 0); 22957b1eea1aSIoana Ciocoi Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 22967b1eea1aSIoana Ciocoi Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 22977b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 22987b1eea1aSIoana Ciocoi Radulescu if (err) { 22997b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 23007b1eea1aSIoana Ciocoi Radulescu return err; 23017b1eea1aSIoana Ciocoi Radulescu } 23027b1eea1aSIoana Ciocoi Radulescu 23037b1eea1aSIoana Ciocoi Radulescu return 0; 23047b1eea1aSIoana Ciocoi Radulescu } 23057b1eea1aSIoana Ciocoi Radulescu 23065d8dccf8SIoana Ciornei static int dpaa2_eth_setup_xdp(struct net_device *dev, struct bpf_prog *prog) 23077e273a8eSIoana Ciocoi Radulescu { 23087e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 23097e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch; 23107e273a8eSIoana Ciocoi Radulescu struct bpf_prog *old; 23117e273a8eSIoana Ciocoi Radulescu bool up, need_update; 23127e273a8eSIoana Ciocoi Radulescu int i, err; 23137e273a8eSIoana Ciocoi Radulescu 23147e273a8eSIoana Ciocoi Radulescu if (prog && !xdp_mtu_valid(priv, dev->mtu)) 23157e273a8eSIoana Ciocoi Radulescu return -EINVAL; 23167e273a8eSIoana Ciocoi Radulescu 231785192dbfSAndrii Nakryiko if (prog) 231885192dbfSAndrii Nakryiko bpf_prog_add(prog, priv->num_channels); 23197e273a8eSIoana Ciocoi Radulescu 23207e273a8eSIoana Ciocoi Radulescu up = netif_running(dev); 23217e273a8eSIoana Ciocoi Radulescu need_update = (!!priv->xdp_prog != !!prog); 23227e273a8eSIoana Ciocoi Radulescu 23237e273a8eSIoana Ciocoi Radulescu if (up) 23247e273a8eSIoana Ciocoi Radulescu dpaa2_eth_stop(dev); 23257e273a8eSIoana Ciocoi Radulescu 23267b1eea1aSIoana Ciocoi Radulescu /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 23277b1eea1aSIoana Ciocoi Radulescu * Also, when switching between xdp/non-xdp modes we need to reconfigure 23287b1eea1aSIoana Ciocoi Radulescu * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 23297b1eea1aSIoana Ciocoi Radulescu * so we are sure no old format buffers will be used from now on. 23307b1eea1aSIoana Ciocoi Radulescu */ 23317e273a8eSIoana Ciocoi Radulescu if (need_update) { 23325d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, dev->mtu, !!prog); 23337e273a8eSIoana Ciocoi Radulescu if (err) 23347e273a8eSIoana Ciocoi Radulescu goto out_err; 23355d8dccf8SIoana Ciornei err = dpaa2_eth_update_rx_buffer_headroom(priv, !!prog); 23367b1eea1aSIoana Ciocoi Radulescu if (err) 23377b1eea1aSIoana Ciocoi Radulescu goto out_err; 23387e273a8eSIoana Ciocoi Radulescu } 23397e273a8eSIoana Ciocoi Radulescu 23407e273a8eSIoana Ciocoi Radulescu old = xchg(&priv->xdp_prog, prog); 23417e273a8eSIoana Ciocoi Radulescu if (old) 23427e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 23437e273a8eSIoana Ciocoi Radulescu 23447e273a8eSIoana Ciocoi Radulescu for (i = 0; i < priv->num_channels; i++) { 23457e273a8eSIoana Ciocoi Radulescu ch = priv->channel[i]; 23467e273a8eSIoana Ciocoi Radulescu old = xchg(&ch->xdp.prog, prog); 23477e273a8eSIoana Ciocoi Radulescu if (old) 23487e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 23497e273a8eSIoana Ciocoi Radulescu } 23507e273a8eSIoana Ciocoi Radulescu 23517e273a8eSIoana Ciocoi Radulescu if (up) { 23527e273a8eSIoana Ciocoi Radulescu err = dpaa2_eth_open(dev); 23537e273a8eSIoana Ciocoi Radulescu if (err) 23547e273a8eSIoana Ciocoi Radulescu return err; 23557e273a8eSIoana Ciocoi Radulescu } 23567e273a8eSIoana Ciocoi Radulescu 23577e273a8eSIoana Ciocoi Radulescu return 0; 23587e273a8eSIoana Ciocoi Radulescu 23597e273a8eSIoana Ciocoi Radulescu out_err: 23607e273a8eSIoana Ciocoi Radulescu if (prog) 23617e273a8eSIoana Ciocoi Radulescu bpf_prog_sub(prog, priv->num_channels); 23627e273a8eSIoana Ciocoi Radulescu if (up) 23637e273a8eSIoana Ciocoi Radulescu dpaa2_eth_open(dev); 23647e273a8eSIoana Ciocoi Radulescu 23657e273a8eSIoana Ciocoi Radulescu return err; 23667e273a8eSIoana Ciocoi Radulescu } 23677e273a8eSIoana Ciocoi Radulescu 23687e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 23697e273a8eSIoana Ciocoi Radulescu { 23707e273a8eSIoana Ciocoi Radulescu switch (xdp->command) { 23717e273a8eSIoana Ciocoi Radulescu case XDP_SETUP_PROG: 23725d8dccf8SIoana Ciornei return dpaa2_eth_setup_xdp(dev, xdp->prog); 23737e273a8eSIoana Ciocoi Radulescu default: 23747e273a8eSIoana Ciocoi Radulescu return -EINVAL; 23757e273a8eSIoana Ciocoi Radulescu } 23767e273a8eSIoana Ciocoi Radulescu 23777e273a8eSIoana Ciocoi Radulescu return 0; 23787e273a8eSIoana Ciocoi Radulescu } 23797e273a8eSIoana Ciocoi Radulescu 23806aa40b9eSIoana Ciornei static int dpaa2_eth_xdp_create_fd(struct net_device *net_dev, 23816aa40b9eSIoana Ciornei struct xdp_frame *xdpf, 23826aa40b9eSIoana Ciornei struct dpaa2_fd *fd) 2383d678be1dSIoana Radulescu { 2384d678be1dSIoana Radulescu struct device *dev = net_dev->dev.parent; 2385d678be1dSIoana Radulescu unsigned int needed_headroom; 2386d678be1dSIoana Radulescu struct dpaa2_eth_swa *swa; 2387d678be1dSIoana Radulescu void *buffer_start, *aligned_start; 2388d678be1dSIoana Radulescu dma_addr_t addr; 2389d678be1dSIoana Radulescu 2390d678be1dSIoana Radulescu /* We require a minimum headroom to be able to transmit the frame. 2391d678be1dSIoana Radulescu * Otherwise return an error and let the original net_device handle it 2392d678be1dSIoana Radulescu */ 23931cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(NULL); 2394d678be1dSIoana Radulescu if (xdpf->headroom < needed_headroom) 2395d678be1dSIoana Radulescu return -EINVAL; 2396d678be1dSIoana Radulescu 2397d678be1dSIoana Radulescu /* Setup the FD fields */ 23986aa40b9eSIoana Ciornei memset(fd, 0, sizeof(*fd)); 2399d678be1dSIoana Radulescu 2400d678be1dSIoana Radulescu /* Align FD address, if possible */ 2401d678be1dSIoana Radulescu buffer_start = xdpf->data - needed_headroom; 2402d678be1dSIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 2403d678be1dSIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 2404d678be1dSIoana Radulescu if (aligned_start >= xdpf->data - xdpf->headroom) 2405d678be1dSIoana Radulescu buffer_start = aligned_start; 2406d678be1dSIoana Radulescu 2407d678be1dSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 2408d678be1dSIoana Radulescu /* fill in necessary fields here */ 2409d678be1dSIoana Radulescu swa->type = DPAA2_ETH_SWA_XDP; 2410d678be1dSIoana Radulescu swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 2411d678be1dSIoana Radulescu swa->xdp.xdpf = xdpf; 2412d678be1dSIoana Radulescu 2413d678be1dSIoana Radulescu addr = dma_map_single(dev, buffer_start, 2414d678be1dSIoana Radulescu swa->xdp.dma_size, 2415d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 24166aa40b9eSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) 2417d678be1dSIoana Radulescu return -ENOMEM; 2418d678be1dSIoana Radulescu 24196aa40b9eSIoana Ciornei dpaa2_fd_set_addr(fd, addr); 24206aa40b9eSIoana Ciornei dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); 24216aa40b9eSIoana Ciornei dpaa2_fd_set_len(fd, xdpf->len); 24226aa40b9eSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_single); 24236aa40b9eSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 2424d678be1dSIoana Radulescu 2425d678be1dSIoana Radulescu return 0; 2426d678be1dSIoana Radulescu } 2427d678be1dSIoana Radulescu 2428d678be1dSIoana Radulescu static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 2429d678be1dSIoana Radulescu struct xdp_frame **frames, u32 flags) 2430d678be1dSIoana Radulescu { 24316aa40b9eSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 243238c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_redirect_fds; 24336aa40b9eSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 24346aa40b9eSIoana Ciornei struct dpaa2_eth_fq *fq; 24358665d978SIoana Ciornei struct dpaa2_fd *fds; 243638c440b2SIoana Ciornei int enqueued, i, err; 2437d678be1dSIoana Radulescu 2438d678be1dSIoana Radulescu if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2439d678be1dSIoana Radulescu return -EINVAL; 2440d678be1dSIoana Radulescu 2441d678be1dSIoana Radulescu if (!netif_running(net_dev)) 2442d678be1dSIoana Radulescu return -ENETDOWN; 2443d678be1dSIoana Radulescu 24448665d978SIoana Ciornei fq = &priv->fq[smp_processor_id()]; 244538c440b2SIoana Ciornei xdp_redirect_fds = &fq->xdp_redirect_fds; 244638c440b2SIoana Ciornei fds = xdp_redirect_fds->fds; 24478665d978SIoana Ciornei 24486aa40b9eSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 24496aa40b9eSIoana Ciornei 24508665d978SIoana Ciornei /* create a FD for each xdp_frame in the list received */ 2451d678be1dSIoana Radulescu for (i = 0; i < n; i++) { 24528665d978SIoana Ciornei err = dpaa2_eth_xdp_create_fd(net_dev, frames[i], &fds[i]); 24538665d978SIoana Ciornei if (err) 24546aa40b9eSIoana Ciornei break; 24556aa40b9eSIoana Ciornei } 245638c440b2SIoana Ciornei xdp_redirect_fds->num = i; 24576aa40b9eSIoana Ciornei 245838c440b2SIoana Ciornei /* enqueue all the frame descriptors */ 245938c440b2SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, xdp_redirect_fds); 2460d678be1dSIoana Radulescu 24618665d978SIoana Ciornei /* update statistics */ 246238c440b2SIoana Ciornei percpu_stats->tx_packets += enqueued; 246338c440b2SIoana Ciornei for (i = 0; i < enqueued; i++) 24648665d978SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 24658665d978SIoana Ciornei 246638c440b2SIoana Ciornei return enqueued; 2467d678be1dSIoana Radulescu } 2468d678be1dSIoana Radulescu 246906d5b179SIoana Radulescu static int update_xps(struct dpaa2_eth_priv *priv) 247006d5b179SIoana Radulescu { 247106d5b179SIoana Radulescu struct net_device *net_dev = priv->net_dev; 247206d5b179SIoana Radulescu struct cpumask xps_mask; 247306d5b179SIoana Radulescu struct dpaa2_eth_fq *fq; 2474ab1e6de2SIoana Radulescu int i, num_queues, netdev_queues; 247506d5b179SIoana Radulescu int err = 0; 247606d5b179SIoana Radulescu 247706d5b179SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2478ab1e6de2SIoana Radulescu netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 247906d5b179SIoana Radulescu 248006d5b179SIoana Radulescu /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 248106d5b179SIoana Radulescu * queues, so only process those 248206d5b179SIoana Radulescu */ 2483ab1e6de2SIoana Radulescu for (i = 0; i < netdev_queues; i++) { 2484ab1e6de2SIoana Radulescu fq = &priv->fq[i % num_queues]; 248506d5b179SIoana Radulescu 248606d5b179SIoana Radulescu cpumask_clear(&xps_mask); 248706d5b179SIoana Radulescu cpumask_set_cpu(fq->target_cpu, &xps_mask); 248806d5b179SIoana Radulescu 248906d5b179SIoana Radulescu err = netif_set_xps_queue(net_dev, &xps_mask, i); 249006d5b179SIoana Radulescu if (err) { 249106d5b179SIoana Radulescu netdev_warn_once(net_dev, "Error setting XPS queue\n"); 249206d5b179SIoana Radulescu break; 249306d5b179SIoana Radulescu } 249406d5b179SIoana Radulescu } 249506d5b179SIoana Radulescu 249606d5b179SIoana Radulescu return err; 249706d5b179SIoana Radulescu } 249806d5b179SIoana Radulescu 2499e3ec13beSIoana Ciornei static int dpaa2_eth_setup_mqprio(struct net_device *net_dev, 2500e3ec13beSIoana Ciornei struct tc_mqprio_qopt *mqprio) 2501ab1e6de2SIoana Radulescu { 2502ab1e6de2SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2503ab1e6de2SIoana Radulescu u8 num_tc, num_queues; 2504ab1e6de2SIoana Radulescu int i; 2505ab1e6de2SIoana Radulescu 2506ab1e6de2SIoana Radulescu mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2507ab1e6de2SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2508ab1e6de2SIoana Radulescu num_tc = mqprio->num_tc; 2509ab1e6de2SIoana Radulescu 2510ab1e6de2SIoana Radulescu if (num_tc == net_dev->num_tc) 2511ab1e6de2SIoana Radulescu return 0; 2512ab1e6de2SIoana Radulescu 2513ab1e6de2SIoana Radulescu if (num_tc > dpaa2_eth_tc_count(priv)) { 2514ab1e6de2SIoana Radulescu netdev_err(net_dev, "Max %d traffic classes supported\n", 2515ab1e6de2SIoana Radulescu dpaa2_eth_tc_count(priv)); 2516b89c1e6bSJesper Dangaard Brouer return -EOPNOTSUPP; 2517ab1e6de2SIoana Radulescu } 2518ab1e6de2SIoana Radulescu 2519ab1e6de2SIoana Radulescu if (!num_tc) { 2520ab1e6de2SIoana Radulescu netdev_reset_tc(net_dev); 2521ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_queues); 2522ab1e6de2SIoana Radulescu goto out; 2523ab1e6de2SIoana Radulescu } 2524ab1e6de2SIoana Radulescu 2525ab1e6de2SIoana Radulescu netdev_set_num_tc(net_dev, num_tc); 2526ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2527ab1e6de2SIoana Radulescu 2528ab1e6de2SIoana Radulescu for (i = 0; i < num_tc; i++) 2529ab1e6de2SIoana Radulescu netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2530ab1e6de2SIoana Radulescu 2531ab1e6de2SIoana Radulescu out: 2532ab1e6de2SIoana Radulescu update_xps(priv); 2533ab1e6de2SIoana Radulescu 2534ab1e6de2SIoana Radulescu return 0; 2535ab1e6de2SIoana Radulescu } 2536ab1e6de2SIoana Radulescu 25373657cdafSIoana Ciornei #define bps_to_mbits(rate) (div_u64((rate), 1000000) * 8) 25383657cdafSIoana Ciornei 25393657cdafSIoana Ciornei static int dpaa2_eth_setup_tbf(struct net_device *net_dev, struct tc_tbf_qopt_offload *p) 25403657cdafSIoana Ciornei { 25413657cdafSIoana Ciornei struct tc_tbf_qopt_offload_replace_params *cfg = &p->replace_params; 25423657cdafSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 25433657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_cr_shaper = { 0 }; 25443657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_er_shaper = { 0 }; 25453657cdafSIoana Ciornei int err; 25463657cdafSIoana Ciornei 25473657cdafSIoana Ciornei if (p->command == TC_TBF_STATS) 25483657cdafSIoana Ciornei return -EOPNOTSUPP; 25493657cdafSIoana Ciornei 25503657cdafSIoana Ciornei /* Only per port Tx shaping */ 25513657cdafSIoana Ciornei if (p->parent != TC_H_ROOT) 25523657cdafSIoana Ciornei return -EOPNOTSUPP; 25533657cdafSIoana Ciornei 25543657cdafSIoana Ciornei if (p->command == TC_TBF_REPLACE) { 25553657cdafSIoana Ciornei if (cfg->max_size > DPAA2_ETH_MAX_BURST_SIZE) { 25563657cdafSIoana Ciornei netdev_err(net_dev, "burst size cannot be greater than %d\n", 25573657cdafSIoana Ciornei DPAA2_ETH_MAX_BURST_SIZE); 25583657cdafSIoana Ciornei return -EINVAL; 25593657cdafSIoana Ciornei } 25603657cdafSIoana Ciornei 25613657cdafSIoana Ciornei tx_cr_shaper.max_burst_size = cfg->max_size; 25623657cdafSIoana Ciornei /* The TBF interface is in bytes/s, whereas DPAA2 expects the 25633657cdafSIoana Ciornei * rate in Mbits/s 25643657cdafSIoana Ciornei */ 25653657cdafSIoana Ciornei tx_cr_shaper.rate_limit = bps_to_mbits(cfg->rate.rate_bytes_ps); 25663657cdafSIoana Ciornei } 25673657cdafSIoana Ciornei 25683657cdafSIoana Ciornei err = dpni_set_tx_shaping(priv->mc_io, 0, priv->mc_token, &tx_cr_shaper, 25693657cdafSIoana Ciornei &tx_er_shaper, 0); 25703657cdafSIoana Ciornei if (err) { 25713657cdafSIoana Ciornei netdev_err(net_dev, "dpni_set_tx_shaping() = %d\n", err); 25723657cdafSIoana Ciornei return err; 25733657cdafSIoana Ciornei } 25743657cdafSIoana Ciornei 25753657cdafSIoana Ciornei return 0; 25763657cdafSIoana Ciornei } 25773657cdafSIoana Ciornei 2578e3ec13beSIoana Ciornei static int dpaa2_eth_setup_tc(struct net_device *net_dev, 2579e3ec13beSIoana Ciornei enum tc_setup_type type, void *type_data) 2580e3ec13beSIoana Ciornei { 2581e3ec13beSIoana Ciornei switch (type) { 2582e3ec13beSIoana Ciornei case TC_SETUP_QDISC_MQPRIO: 2583e3ec13beSIoana Ciornei return dpaa2_eth_setup_mqprio(net_dev, type_data); 25843657cdafSIoana Ciornei case TC_SETUP_QDISC_TBF: 25853657cdafSIoana Ciornei return dpaa2_eth_setup_tbf(net_dev, type_data); 2586e3ec13beSIoana Ciornei default: 2587e3ec13beSIoana Ciornei return -EOPNOTSUPP; 2588e3ec13beSIoana Ciornei } 2589e3ec13beSIoana Ciornei } 2590e3ec13beSIoana Ciornei 259134ff6846SIoana Radulescu static const struct net_device_ops dpaa2_eth_ops = { 259234ff6846SIoana Radulescu .ndo_open = dpaa2_eth_open, 259334ff6846SIoana Radulescu .ndo_start_xmit = dpaa2_eth_tx, 259434ff6846SIoana Radulescu .ndo_stop = dpaa2_eth_stop, 259534ff6846SIoana Radulescu .ndo_set_mac_address = dpaa2_eth_set_addr, 259634ff6846SIoana Radulescu .ndo_get_stats64 = dpaa2_eth_get_stats, 259734ff6846SIoana Radulescu .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 259834ff6846SIoana Radulescu .ndo_set_features = dpaa2_eth_set_features, 259934ff6846SIoana Radulescu .ndo_do_ioctl = dpaa2_eth_ioctl, 26007e273a8eSIoana Ciocoi Radulescu .ndo_change_mtu = dpaa2_eth_change_mtu, 26017e273a8eSIoana Ciocoi Radulescu .ndo_bpf = dpaa2_eth_xdp, 2602d678be1dSIoana Radulescu .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2603ab1e6de2SIoana Radulescu .ndo_setup_tc = dpaa2_eth_setup_tc, 260470b32d82SIonut-robert Aron .ndo_vlan_rx_add_vid = dpaa2_eth_rx_add_vid, 260570b32d82SIonut-robert Aron .ndo_vlan_rx_kill_vid = dpaa2_eth_rx_kill_vid 260634ff6846SIoana Radulescu }; 260734ff6846SIoana Radulescu 26085d8dccf8SIoana Ciornei static void dpaa2_eth_cdan_cb(struct dpaa2_io_notification_ctx *ctx) 260934ff6846SIoana Radulescu { 261034ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 261134ff6846SIoana Radulescu 261234ff6846SIoana Radulescu ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 261334ff6846SIoana Radulescu 261434ff6846SIoana Radulescu /* Update NAPI statistics */ 261534ff6846SIoana Radulescu ch->stats.cdan++; 261634ff6846SIoana Radulescu 26176c33ae1aSJiafei Pan napi_schedule(&ch->napi); 261834ff6846SIoana Radulescu } 261934ff6846SIoana Radulescu 262034ff6846SIoana Radulescu /* Allocate and configure a DPCON object */ 26215d8dccf8SIoana Ciornei static struct fsl_mc_device *dpaa2_eth_setup_dpcon(struct dpaa2_eth_priv *priv) 262234ff6846SIoana Radulescu { 262334ff6846SIoana Radulescu struct fsl_mc_device *dpcon; 262434ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 262534ff6846SIoana Radulescu int err; 262634ff6846SIoana Radulescu 262734ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 262834ff6846SIoana Radulescu FSL_MC_POOL_DPCON, &dpcon); 262934ff6846SIoana Radulescu if (err) { 2630d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2631d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2632d7f5a9d8SIoana Ciornei else 263334ff6846SIoana Radulescu dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2634d7f5a9d8SIoana Ciornei return ERR_PTR(err); 263534ff6846SIoana Radulescu } 263634ff6846SIoana Radulescu 263734ff6846SIoana Radulescu err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 263834ff6846SIoana Radulescu if (err) { 263934ff6846SIoana Radulescu dev_err(dev, "dpcon_open() failed\n"); 264034ff6846SIoana Radulescu goto free; 264134ff6846SIoana Radulescu } 264234ff6846SIoana Radulescu 264334ff6846SIoana Radulescu err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 264434ff6846SIoana Radulescu if (err) { 264534ff6846SIoana Radulescu dev_err(dev, "dpcon_reset() failed\n"); 264634ff6846SIoana Radulescu goto close; 264734ff6846SIoana Radulescu } 264834ff6846SIoana Radulescu 264934ff6846SIoana Radulescu err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 265034ff6846SIoana Radulescu if (err) { 265134ff6846SIoana Radulescu dev_err(dev, "dpcon_enable() failed\n"); 265234ff6846SIoana Radulescu goto close; 265334ff6846SIoana Radulescu } 265434ff6846SIoana Radulescu 265534ff6846SIoana Radulescu return dpcon; 265634ff6846SIoana Radulescu 265734ff6846SIoana Radulescu close: 265834ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 265934ff6846SIoana Radulescu free: 266034ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 266134ff6846SIoana Radulescu 266202afa9c6SYueHaibing return ERR_PTR(err); 266334ff6846SIoana Radulescu } 266434ff6846SIoana Radulescu 26655d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpcon(struct dpaa2_eth_priv *priv, 266634ff6846SIoana Radulescu struct fsl_mc_device *dpcon) 266734ff6846SIoana Radulescu { 266834ff6846SIoana Radulescu dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 266934ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 267034ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 267134ff6846SIoana Radulescu } 267234ff6846SIoana Radulescu 26735d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_alloc_channel(struct dpaa2_eth_priv *priv) 267434ff6846SIoana Radulescu { 267534ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 267634ff6846SIoana Radulescu struct dpcon_attr attr; 267734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 267834ff6846SIoana Radulescu int err; 267934ff6846SIoana Radulescu 268034ff6846SIoana Radulescu channel = kzalloc(sizeof(*channel), GFP_KERNEL); 268134ff6846SIoana Radulescu if (!channel) 268234ff6846SIoana Radulescu return NULL; 268334ff6846SIoana Radulescu 26845d8dccf8SIoana Ciornei channel->dpcon = dpaa2_eth_setup_dpcon(priv); 268502afa9c6SYueHaibing if (IS_ERR(channel->dpcon)) { 268602afa9c6SYueHaibing err = PTR_ERR(channel->dpcon); 268734ff6846SIoana Radulescu goto err_setup; 2688d7f5a9d8SIoana Ciornei } 268934ff6846SIoana Radulescu 269034ff6846SIoana Radulescu err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 269134ff6846SIoana Radulescu &attr); 269234ff6846SIoana Radulescu if (err) { 269334ff6846SIoana Radulescu dev_err(dev, "dpcon_get_attributes() failed\n"); 269434ff6846SIoana Radulescu goto err_get_attr; 269534ff6846SIoana Radulescu } 269634ff6846SIoana Radulescu 269734ff6846SIoana Radulescu channel->dpcon_id = attr.id; 269834ff6846SIoana Radulescu channel->ch_id = attr.qbman_ch_id; 269934ff6846SIoana Radulescu channel->priv = priv; 270034ff6846SIoana Radulescu 270134ff6846SIoana Radulescu return channel; 270234ff6846SIoana Radulescu 270334ff6846SIoana Radulescu err_get_attr: 27045d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 270534ff6846SIoana Radulescu err_setup: 270634ff6846SIoana Radulescu kfree(channel); 2707d7f5a9d8SIoana Ciornei return ERR_PTR(err); 270834ff6846SIoana Radulescu } 270934ff6846SIoana Radulescu 27105d8dccf8SIoana Ciornei static void dpaa2_eth_free_channel(struct dpaa2_eth_priv *priv, 271134ff6846SIoana Radulescu struct dpaa2_eth_channel *channel) 271234ff6846SIoana Radulescu { 27135d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 271434ff6846SIoana Radulescu kfree(channel); 271534ff6846SIoana Radulescu } 271634ff6846SIoana Radulescu 271734ff6846SIoana Radulescu /* DPIO setup: allocate and configure QBMan channels, setup core affinity 271834ff6846SIoana Radulescu * and register data availability notifications 271934ff6846SIoana Radulescu */ 27205d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpio(struct dpaa2_eth_priv *priv) 272134ff6846SIoana Radulescu { 272234ff6846SIoana Radulescu struct dpaa2_io_notification_ctx *nctx; 272334ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 272434ff6846SIoana Radulescu struct dpcon_notification_cfg dpcon_notif_cfg; 272534ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 272634ff6846SIoana Radulescu int i, err; 272734ff6846SIoana Radulescu 272834ff6846SIoana Radulescu /* We want the ability to spread ingress traffic (RX, TX conf) to as 272934ff6846SIoana Radulescu * many cores as possible, so we need one channel for each core 273034ff6846SIoana Radulescu * (unless there's fewer queues than cores, in which case the extra 273134ff6846SIoana Radulescu * channels would be wasted). 273234ff6846SIoana Radulescu * Allocate one channel per core and register it to the core's 273334ff6846SIoana Radulescu * affine DPIO. If not enough channels are available for all cores 273434ff6846SIoana Radulescu * or if some cores don't have an affine DPIO, there will be no 273534ff6846SIoana Radulescu * ingress frame processing on those cores. 273634ff6846SIoana Radulescu */ 273734ff6846SIoana Radulescu cpumask_clear(&priv->dpio_cpumask); 273834ff6846SIoana Radulescu for_each_online_cpu(i) { 273934ff6846SIoana Radulescu /* Try to allocate a channel */ 27405d8dccf8SIoana Ciornei channel = dpaa2_eth_alloc_channel(priv); 2741d7f5a9d8SIoana Ciornei if (IS_ERR_OR_NULL(channel)) { 2742bd8460faSIoana Radulescu err = PTR_ERR_OR_ZERO(channel); 2743d7f5a9d8SIoana Ciornei if (err != -EPROBE_DEFER) 274434ff6846SIoana Radulescu dev_info(dev, 274534ff6846SIoana Radulescu "No affine channel for cpu %d and above\n", i); 274634ff6846SIoana Radulescu goto err_alloc_ch; 274734ff6846SIoana Radulescu } 274834ff6846SIoana Radulescu 274934ff6846SIoana Radulescu priv->channel[priv->num_channels] = channel; 275034ff6846SIoana Radulescu 275134ff6846SIoana Radulescu nctx = &channel->nctx; 275234ff6846SIoana Radulescu nctx->is_cdan = 1; 27535d8dccf8SIoana Ciornei nctx->cb = dpaa2_eth_cdan_cb; 275434ff6846SIoana Radulescu nctx->id = channel->ch_id; 275534ff6846SIoana Radulescu nctx->desired_cpu = i; 275634ff6846SIoana Radulescu 275734ff6846SIoana Radulescu /* Register the new context */ 275834ff6846SIoana Radulescu channel->dpio = dpaa2_io_service_select(i); 275947441f7fSIoana Ciornei err = dpaa2_io_service_register(channel->dpio, nctx, dev); 276034ff6846SIoana Radulescu if (err) { 276134ff6846SIoana Radulescu dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 276234ff6846SIoana Radulescu /* If no affine DPIO for this core, there's probably 276334ff6846SIoana Radulescu * none available for next cores either. Signal we want 276434ff6846SIoana Radulescu * to retry later, in case the DPIO devices weren't 276534ff6846SIoana Radulescu * probed yet. 276634ff6846SIoana Radulescu */ 276734ff6846SIoana Radulescu err = -EPROBE_DEFER; 276834ff6846SIoana Radulescu goto err_service_reg; 276934ff6846SIoana Radulescu } 277034ff6846SIoana Radulescu 277134ff6846SIoana Radulescu /* Register DPCON notification with MC */ 277234ff6846SIoana Radulescu dpcon_notif_cfg.dpio_id = nctx->dpio_id; 277334ff6846SIoana Radulescu dpcon_notif_cfg.priority = 0; 277434ff6846SIoana Radulescu dpcon_notif_cfg.user_ctx = nctx->qman64; 277534ff6846SIoana Radulescu err = dpcon_set_notification(priv->mc_io, 0, 277634ff6846SIoana Radulescu channel->dpcon->mc_handle, 277734ff6846SIoana Radulescu &dpcon_notif_cfg); 277834ff6846SIoana Radulescu if (err) { 277934ff6846SIoana Radulescu dev_err(dev, "dpcon_set_notification failed()\n"); 278034ff6846SIoana Radulescu goto err_set_cdan; 278134ff6846SIoana Radulescu } 278234ff6846SIoana Radulescu 278334ff6846SIoana Radulescu /* If we managed to allocate a channel and also found an affine 278434ff6846SIoana Radulescu * DPIO for this core, add it to the final mask 278534ff6846SIoana Radulescu */ 278634ff6846SIoana Radulescu cpumask_set_cpu(i, &priv->dpio_cpumask); 278734ff6846SIoana Radulescu priv->num_channels++; 278834ff6846SIoana Radulescu 278934ff6846SIoana Radulescu /* Stop if we already have enough channels to accommodate all 279034ff6846SIoana Radulescu * RX and TX conf queues 279134ff6846SIoana Radulescu */ 2792b0e4f37bSIoana Ciocoi Radulescu if (priv->num_channels == priv->dpni_attrs.num_queues) 279334ff6846SIoana Radulescu break; 279434ff6846SIoana Radulescu } 279534ff6846SIoana Radulescu 279634ff6846SIoana Radulescu return 0; 279734ff6846SIoana Radulescu 279834ff6846SIoana Radulescu err_set_cdan: 279947441f7fSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 280034ff6846SIoana Radulescu err_service_reg: 28015d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 280234ff6846SIoana Radulescu err_alloc_ch: 28035aa4277dSIoana Ciornei if (err == -EPROBE_DEFER) { 28045aa4277dSIoana Ciornei for (i = 0; i < priv->num_channels; i++) { 28055aa4277dSIoana Ciornei channel = priv->channel[i]; 28065aa4277dSIoana Ciornei nctx = &channel->nctx; 28075aa4277dSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 28085d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 28095aa4277dSIoana Ciornei } 28105aa4277dSIoana Ciornei priv->num_channels = 0; 2811d7f5a9d8SIoana Ciornei return err; 28125aa4277dSIoana Ciornei } 2813d7f5a9d8SIoana Ciornei 281434ff6846SIoana Radulescu if (cpumask_empty(&priv->dpio_cpumask)) { 281534ff6846SIoana Radulescu dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 2816d7f5a9d8SIoana Ciornei return -ENODEV; 281734ff6846SIoana Radulescu } 281834ff6846SIoana Radulescu 281934ff6846SIoana Radulescu dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 282034ff6846SIoana Radulescu cpumask_pr_args(&priv->dpio_cpumask)); 282134ff6846SIoana Radulescu 282234ff6846SIoana Radulescu return 0; 282334ff6846SIoana Radulescu } 282434ff6846SIoana Radulescu 28255d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpio(struct dpaa2_eth_priv *priv) 282634ff6846SIoana Radulescu { 282747441f7fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 282834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 282947441f7fSIoana Ciornei int i; 283034ff6846SIoana Radulescu 283134ff6846SIoana Radulescu /* deregister CDAN notifications and free channels */ 283234ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 283334ff6846SIoana Radulescu ch = priv->channel[i]; 283447441f7fSIoana Ciornei dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 28355d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, ch); 283634ff6846SIoana Radulescu } 283734ff6846SIoana Radulescu } 283834ff6846SIoana Radulescu 28395d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_get_affine_channel(struct dpaa2_eth_priv *priv, 284034ff6846SIoana Radulescu int cpu) 284134ff6846SIoana Radulescu { 284234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 284334ff6846SIoana Radulescu int i; 284434ff6846SIoana Radulescu 284534ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 284634ff6846SIoana Radulescu if (priv->channel[i]->nctx.desired_cpu == cpu) 284734ff6846SIoana Radulescu return priv->channel[i]; 284834ff6846SIoana Radulescu 284934ff6846SIoana Radulescu /* We should never get here. Issue a warning and return 285034ff6846SIoana Radulescu * the first channel, because it's still better than nothing 285134ff6846SIoana Radulescu */ 285234ff6846SIoana Radulescu dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 285334ff6846SIoana Radulescu 285434ff6846SIoana Radulescu return priv->channel[0]; 285534ff6846SIoana Radulescu } 285634ff6846SIoana Radulescu 28575d8dccf8SIoana Ciornei static void dpaa2_eth_set_fq_affinity(struct dpaa2_eth_priv *priv) 285834ff6846SIoana Radulescu { 285934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 286034ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 286134ff6846SIoana Radulescu int rx_cpu, txc_cpu; 286206d5b179SIoana Radulescu int i; 286334ff6846SIoana Radulescu 286434ff6846SIoana Radulescu /* For each FQ, pick one channel/CPU to deliver frames to. 286534ff6846SIoana Radulescu * This may well change at runtime, either through irqbalance or 286634ff6846SIoana Radulescu * through direct user intervention. 286734ff6846SIoana Radulescu */ 286834ff6846SIoana Radulescu rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 286934ff6846SIoana Radulescu 287034ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 287134ff6846SIoana Radulescu fq = &priv->fq[i]; 287234ff6846SIoana Radulescu switch (fq->type) { 287334ff6846SIoana Radulescu case DPAA2_RX_FQ: 2874061d631fSIoana Ciornei case DPAA2_RX_ERR_FQ: 287534ff6846SIoana Radulescu fq->target_cpu = rx_cpu; 287634ff6846SIoana Radulescu rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 287734ff6846SIoana Radulescu if (rx_cpu >= nr_cpu_ids) 287834ff6846SIoana Radulescu rx_cpu = cpumask_first(&priv->dpio_cpumask); 287934ff6846SIoana Radulescu break; 288034ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 288134ff6846SIoana Radulescu fq->target_cpu = txc_cpu; 288234ff6846SIoana Radulescu txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 288334ff6846SIoana Radulescu if (txc_cpu >= nr_cpu_ids) 288434ff6846SIoana Radulescu txc_cpu = cpumask_first(&priv->dpio_cpumask); 288534ff6846SIoana Radulescu break; 288634ff6846SIoana Radulescu default: 288734ff6846SIoana Radulescu dev_err(dev, "Unknown FQ type: %d\n", fq->type); 288834ff6846SIoana Radulescu } 28895d8dccf8SIoana Ciornei fq->channel = dpaa2_eth_get_affine_channel(priv, fq->target_cpu); 289034ff6846SIoana Radulescu } 289106d5b179SIoana Radulescu 289206d5b179SIoana Radulescu update_xps(priv); 289334ff6846SIoana Radulescu } 289434ff6846SIoana Radulescu 28955d8dccf8SIoana Ciornei static void dpaa2_eth_setup_fqs(struct dpaa2_eth_priv *priv) 289634ff6846SIoana Radulescu { 2897685e39eaSIoana Radulescu int i, j; 289834ff6846SIoana Radulescu 289934ff6846SIoana Radulescu /* We have one TxConf FQ per Tx flow. 290034ff6846SIoana Radulescu * The number of Tx and Rx queues is the same. 290134ff6846SIoana Radulescu * Tx queues come first in the fq array. 290234ff6846SIoana Radulescu */ 290334ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 290434ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 290534ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 290634ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 290734ff6846SIoana Radulescu } 290834ff6846SIoana Radulescu 2909685e39eaSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 291034ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 291134ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 291234ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 2913685e39eaSIoana Radulescu priv->fq[priv->num_fqs].tc = (u8)j; 291434ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 291534ff6846SIoana Radulescu } 2916685e39eaSIoana Radulescu } 291734ff6846SIoana Radulescu 2918061d631fSIoana Ciornei /* We have exactly one Rx error queue per DPNI */ 2919061d631fSIoana Ciornei priv->fq[priv->num_fqs].type = DPAA2_RX_ERR_FQ; 2920061d631fSIoana Ciornei priv->fq[priv->num_fqs++].consume = dpaa2_eth_rx_err; 2921061d631fSIoana Ciornei 292234ff6846SIoana Radulescu /* For each FQ, decide on which core to process incoming frames */ 29235d8dccf8SIoana Ciornei dpaa2_eth_set_fq_affinity(priv); 292434ff6846SIoana Radulescu } 292534ff6846SIoana Radulescu 292634ff6846SIoana Radulescu /* Allocate and configure one buffer pool for each interface */ 29275d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpbp(struct dpaa2_eth_priv *priv) 292834ff6846SIoana Radulescu { 292934ff6846SIoana Radulescu int err; 293034ff6846SIoana Radulescu struct fsl_mc_device *dpbp_dev; 293134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 293234ff6846SIoana Radulescu struct dpbp_attr dpbp_attrs; 293334ff6846SIoana Radulescu 293434ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 293534ff6846SIoana Radulescu &dpbp_dev); 293634ff6846SIoana Radulescu if (err) { 2937d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2938d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2939d7f5a9d8SIoana Ciornei else 294034ff6846SIoana Radulescu dev_err(dev, "DPBP device allocation failed\n"); 294134ff6846SIoana Radulescu return err; 294234ff6846SIoana Radulescu } 294334ff6846SIoana Radulescu 294434ff6846SIoana Radulescu priv->dpbp_dev = dpbp_dev; 294534ff6846SIoana Radulescu 294634ff6846SIoana Radulescu err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 294734ff6846SIoana Radulescu &dpbp_dev->mc_handle); 294834ff6846SIoana Radulescu if (err) { 294934ff6846SIoana Radulescu dev_err(dev, "dpbp_open() failed\n"); 295034ff6846SIoana Radulescu goto err_open; 295134ff6846SIoana Radulescu } 295234ff6846SIoana Radulescu 295334ff6846SIoana Radulescu err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 295434ff6846SIoana Radulescu if (err) { 295534ff6846SIoana Radulescu dev_err(dev, "dpbp_reset() failed\n"); 295634ff6846SIoana Radulescu goto err_reset; 295734ff6846SIoana Radulescu } 295834ff6846SIoana Radulescu 295934ff6846SIoana Radulescu err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 296034ff6846SIoana Radulescu if (err) { 296134ff6846SIoana Radulescu dev_err(dev, "dpbp_enable() failed\n"); 296234ff6846SIoana Radulescu goto err_enable; 296334ff6846SIoana Radulescu } 296434ff6846SIoana Radulescu 296534ff6846SIoana Radulescu err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 296634ff6846SIoana Radulescu &dpbp_attrs); 296734ff6846SIoana Radulescu if (err) { 296834ff6846SIoana Radulescu dev_err(dev, "dpbp_get_attributes() failed\n"); 296934ff6846SIoana Radulescu goto err_get_attr; 297034ff6846SIoana Radulescu } 297134ff6846SIoana Radulescu priv->bpid = dpbp_attrs.bpid; 297234ff6846SIoana Radulescu 297334ff6846SIoana Radulescu return 0; 297434ff6846SIoana Radulescu 297534ff6846SIoana Radulescu err_get_attr: 297634ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 297734ff6846SIoana Radulescu err_enable: 297834ff6846SIoana Radulescu err_reset: 297934ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 298034ff6846SIoana Radulescu err_open: 298134ff6846SIoana Radulescu fsl_mc_object_free(dpbp_dev); 298234ff6846SIoana Radulescu 298334ff6846SIoana Radulescu return err; 298434ff6846SIoana Radulescu } 298534ff6846SIoana Radulescu 29865d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv) 298734ff6846SIoana Radulescu { 29885d8dccf8SIoana Ciornei dpaa2_eth_drain_pool(priv); 298934ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 299034ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 299134ff6846SIoana Radulescu fsl_mc_object_free(priv->dpbp_dev); 299234ff6846SIoana Radulescu } 299334ff6846SIoana Radulescu 29945d8dccf8SIoana Ciornei static int dpaa2_eth_set_buffer_layout(struct dpaa2_eth_priv *priv) 299534ff6846SIoana Radulescu { 299634ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 299734ff6846SIoana Radulescu struct dpni_buffer_layout buf_layout = {0}; 299827c87486SIoana Ciocoi Radulescu u16 rx_buf_align; 299934ff6846SIoana Radulescu int err; 300034ff6846SIoana Radulescu 300134ff6846SIoana Radulescu /* We need to check for WRIOP version 1.0.0, but depending on the MC 300234ff6846SIoana Radulescu * version, this number is not always provided correctly on rev1. 300334ff6846SIoana Radulescu * We need to check for both alternatives in this situation. 300434ff6846SIoana Radulescu */ 300534ff6846SIoana Radulescu if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 300634ff6846SIoana Radulescu priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 300727c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 300834ff6846SIoana Radulescu else 300927c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 301034ff6846SIoana Radulescu 3011efa6a7d0SIoana Ciornei /* We need to ensure that the buffer size seen by WRIOP is a multiple 3012efa6a7d0SIoana Ciornei * of 64 or 256 bytes depending on the WRIOP version. 3013efa6a7d0SIoana Ciornei */ 3014efa6a7d0SIoana Ciornei priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align); 3015efa6a7d0SIoana Ciornei 301634ff6846SIoana Radulescu /* tx buffer */ 301734ff6846SIoana Radulescu buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 301834ff6846SIoana Radulescu buf_layout.pass_timestamp = true; 3019c5521189SYangbo Lu buf_layout.pass_frame_status = true; 302034ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 3021c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 3022c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 302334ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 302434ff6846SIoana Radulescu DPNI_QUEUE_TX, &buf_layout); 302534ff6846SIoana Radulescu if (err) { 302634ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 302734ff6846SIoana Radulescu return err; 302834ff6846SIoana Radulescu } 302934ff6846SIoana Radulescu 303034ff6846SIoana Radulescu /* tx-confirm buffer */ 3031c5521189SYangbo Lu buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 3032c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 303334ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 303434ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, &buf_layout); 303534ff6846SIoana Radulescu if (err) { 303634ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 303734ff6846SIoana Radulescu return err; 303834ff6846SIoana Radulescu } 303934ff6846SIoana Radulescu 304034ff6846SIoana Radulescu /* Now that we've set our tx buffer layout, retrieve the minimum 304134ff6846SIoana Radulescu * required tx data offset. 304234ff6846SIoana Radulescu */ 304334ff6846SIoana Radulescu err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 304434ff6846SIoana Radulescu &priv->tx_data_offset); 304534ff6846SIoana Radulescu if (err) { 304634ff6846SIoana Radulescu dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 304734ff6846SIoana Radulescu return err; 304834ff6846SIoana Radulescu } 304934ff6846SIoana Radulescu 305034ff6846SIoana Radulescu if ((priv->tx_data_offset % 64) != 0) 305134ff6846SIoana Radulescu dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 305234ff6846SIoana Radulescu priv->tx_data_offset); 305334ff6846SIoana Radulescu 305434ff6846SIoana Radulescu /* rx buffer */ 305534ff6846SIoana Radulescu buf_layout.pass_frame_status = true; 305634ff6846SIoana Radulescu buf_layout.pass_parser_result = true; 305727c87486SIoana Ciocoi Radulescu buf_layout.data_align = rx_buf_align; 305834ff6846SIoana Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 305934ff6846SIoana Radulescu buf_layout.private_data_size = 0; 306034ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 306134ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 306234ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 306334ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 306434ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 306534ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 306634ff6846SIoana Radulescu DPNI_QUEUE_RX, &buf_layout); 306734ff6846SIoana Radulescu if (err) { 306834ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 306934ff6846SIoana Radulescu return err; 307034ff6846SIoana Radulescu } 307134ff6846SIoana Radulescu 307234ff6846SIoana Radulescu return 0; 307334ff6846SIoana Radulescu } 307434ff6846SIoana Radulescu 30751fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 30761fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MINOR 9 30771fa0f68cSIoana Ciocoi Radulescu 30781fa0f68cSIoana Ciocoi Radulescu static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 30791fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 308048c0481eSIoana Ciornei struct dpaa2_fd *fd, u8 prio, 30816ff80447SIoana Ciornei u32 num_frames __always_unused, 308248c0481eSIoana Ciornei int *frames_enqueued) 30831fa0f68cSIoana Ciocoi Radulescu { 308448c0481eSIoana Ciornei int err; 308548c0481eSIoana Ciornei 308648c0481eSIoana Ciornei err = dpaa2_io_service_enqueue_qd(fq->channel->dpio, 30871fa0f68cSIoana Ciocoi Radulescu priv->tx_qdid, prio, 30881fa0f68cSIoana Ciocoi Radulescu fq->tx_qdbin, fd); 308948c0481eSIoana Ciornei if (!err && frames_enqueued) 309048c0481eSIoana Ciornei *frames_enqueued = 1; 309148c0481eSIoana Ciornei return err; 30921fa0f68cSIoana Ciocoi Radulescu } 30931fa0f68cSIoana Ciocoi Radulescu 30946ff80447SIoana Ciornei static inline int dpaa2_eth_enqueue_fq_multiple(struct dpaa2_eth_priv *priv, 30951fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 30966ff80447SIoana Ciornei struct dpaa2_fd *fd, 30976ff80447SIoana Ciornei u8 prio, u32 num_frames, 309848c0481eSIoana Ciornei int *frames_enqueued) 30991fa0f68cSIoana Ciocoi Radulescu { 310048c0481eSIoana Ciornei int err; 310148c0481eSIoana Ciornei 31026ff80447SIoana Ciornei err = dpaa2_io_service_enqueue_multiple_fq(fq->channel->dpio, 31036ff80447SIoana Ciornei fq->tx_fqid[prio], 31046ff80447SIoana Ciornei fd, num_frames); 31056ff80447SIoana Ciornei 31066ff80447SIoana Ciornei if (err == 0) 31076ff80447SIoana Ciornei return -EBUSY; 31086ff80447SIoana Ciornei 31096ff80447SIoana Ciornei if (frames_enqueued) 31106ff80447SIoana Ciornei *frames_enqueued = err; 31116ff80447SIoana Ciornei return 0; 31121fa0f68cSIoana Ciocoi Radulescu } 31131fa0f68cSIoana Ciocoi Radulescu 31145d8dccf8SIoana Ciornei static void dpaa2_eth_set_enqueue_mode(struct dpaa2_eth_priv *priv) 31151fa0f68cSIoana Ciocoi Radulescu { 31161fa0f68cSIoana Ciocoi Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 31171fa0f68cSIoana Ciocoi Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 31181fa0f68cSIoana Ciocoi Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 31191fa0f68cSIoana Ciocoi Radulescu else 31206ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 31211fa0f68cSIoana Ciocoi Radulescu } 31221fa0f68cSIoana Ciocoi Radulescu 31235d8dccf8SIoana Ciornei static int dpaa2_eth_set_pause(struct dpaa2_eth_priv *priv) 31248eb3cef8SIoana Radulescu { 31258eb3cef8SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 31268eb3cef8SIoana Radulescu struct dpni_link_cfg link_cfg = {0}; 31278eb3cef8SIoana Radulescu int err; 31288eb3cef8SIoana Radulescu 31298eb3cef8SIoana Radulescu /* Get the default link options so we don't override other flags */ 31308eb3cef8SIoana Radulescu err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 31318eb3cef8SIoana Radulescu if (err) { 31328eb3cef8SIoana Radulescu dev_err(dev, "dpni_get_link_cfg() failed\n"); 31338eb3cef8SIoana Radulescu return err; 31348eb3cef8SIoana Radulescu } 31358eb3cef8SIoana Radulescu 31368eb3cef8SIoana Radulescu /* By default, enable both Rx and Tx pause frames */ 31378eb3cef8SIoana Radulescu link_cfg.options |= DPNI_LINK_OPT_PAUSE; 31388eb3cef8SIoana Radulescu link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 31398eb3cef8SIoana Radulescu err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 31408eb3cef8SIoana Radulescu if (err) { 31418eb3cef8SIoana Radulescu dev_err(dev, "dpni_set_link_cfg() failed\n"); 31428eb3cef8SIoana Radulescu return err; 31438eb3cef8SIoana Radulescu } 31448eb3cef8SIoana Radulescu 31458eb3cef8SIoana Radulescu priv->link_state.options = link_cfg.options; 31468eb3cef8SIoana Radulescu 31478eb3cef8SIoana Radulescu return 0; 31488eb3cef8SIoana Radulescu } 31498eb3cef8SIoana Radulescu 31505d8dccf8SIoana Ciornei static void dpaa2_eth_update_tx_fqids(struct dpaa2_eth_priv *priv) 3151a690af4fSIoana Radulescu { 3152a690af4fSIoana Radulescu struct dpni_queue_id qid = {0}; 3153a690af4fSIoana Radulescu struct dpaa2_eth_fq *fq; 3154a690af4fSIoana Radulescu struct dpni_queue queue; 3155a690af4fSIoana Radulescu int i, j, err; 3156a690af4fSIoana Radulescu 3157a690af4fSIoana Radulescu /* We only use Tx FQIDs for FQID-based enqueue, so check 3158a690af4fSIoana Radulescu * if DPNI version supports it before updating FQIDs 3159a690af4fSIoana Radulescu */ 3160a690af4fSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 3161a690af4fSIoana Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 3162a690af4fSIoana Radulescu return; 3163a690af4fSIoana Radulescu 3164a690af4fSIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 3165a690af4fSIoana Radulescu fq = &priv->fq[i]; 3166a690af4fSIoana Radulescu if (fq->type != DPAA2_TX_CONF_FQ) 3167a690af4fSIoana Radulescu continue; 3168a690af4fSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 3169a690af4fSIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3170a690af4fSIoana Radulescu DPNI_QUEUE_TX, j, fq->flowid, 3171a690af4fSIoana Radulescu &queue, &qid); 3172a690af4fSIoana Radulescu if (err) 3173a690af4fSIoana Radulescu goto out_err; 3174a690af4fSIoana Radulescu 3175a690af4fSIoana Radulescu fq->tx_fqid[j] = qid.fqid; 3176a690af4fSIoana Radulescu if (fq->tx_fqid[j] == 0) 3177a690af4fSIoana Radulescu goto out_err; 3178a690af4fSIoana Radulescu } 3179a690af4fSIoana Radulescu } 3180a690af4fSIoana Radulescu 31816ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 3182a690af4fSIoana Radulescu 3183a690af4fSIoana Radulescu return; 3184a690af4fSIoana Radulescu 3185a690af4fSIoana Radulescu out_err: 3186a690af4fSIoana Radulescu netdev_info(priv->net_dev, 3187a690af4fSIoana Radulescu "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 3188a690af4fSIoana Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 3189a690af4fSIoana Radulescu } 3190a690af4fSIoana Radulescu 31916aa90fe2SIoana Radulescu /* Configure ingress classification based on VLAN PCP */ 31925d8dccf8SIoana Ciornei static int dpaa2_eth_set_vlan_qos(struct dpaa2_eth_priv *priv) 31936aa90fe2SIoana Radulescu { 31946aa90fe2SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 31956aa90fe2SIoana Radulescu struct dpkg_profile_cfg kg_cfg = {0}; 31966aa90fe2SIoana Radulescu struct dpni_qos_tbl_cfg qos_cfg = {0}; 31976aa90fe2SIoana Radulescu struct dpni_rule_cfg key_params; 31986aa90fe2SIoana Radulescu void *dma_mem, *key, *mask; 31996aa90fe2SIoana Radulescu u8 key_size = 2; /* VLAN TCI field */ 32006aa90fe2SIoana Radulescu int i, pcp, err; 32016aa90fe2SIoana Radulescu 32026aa90fe2SIoana Radulescu /* VLAN-based classification only makes sense if we have multiple 32036aa90fe2SIoana Radulescu * traffic classes. 32046aa90fe2SIoana Radulescu * Also, we need to extract just the 3-bit PCP field from the VLAN 32056aa90fe2SIoana Radulescu * header and we can only do that by using a mask 32066aa90fe2SIoana Radulescu */ 32076aa90fe2SIoana Radulescu if (dpaa2_eth_tc_count(priv) == 1 || !dpaa2_eth_fs_mask_enabled(priv)) { 32086aa90fe2SIoana Radulescu dev_dbg(dev, "VLAN-based QoS classification not supported\n"); 32096aa90fe2SIoana Radulescu return -EOPNOTSUPP; 32106aa90fe2SIoana Radulescu } 32116aa90fe2SIoana Radulescu 32126aa90fe2SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 32136aa90fe2SIoana Radulescu if (!dma_mem) 32146aa90fe2SIoana Radulescu return -ENOMEM; 32156aa90fe2SIoana Radulescu 32166aa90fe2SIoana Radulescu kg_cfg.num_extracts = 1; 32176aa90fe2SIoana Radulescu kg_cfg.extracts[0].type = DPKG_EXTRACT_FROM_HDR; 32186aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.prot = NET_PROT_VLAN; 32196aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.type = DPKG_FULL_FIELD; 32206aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.field = NH_FLD_VLAN_TCI; 32216aa90fe2SIoana Radulescu 32226aa90fe2SIoana Radulescu err = dpni_prepare_key_cfg(&kg_cfg, dma_mem); 32236aa90fe2SIoana Radulescu if (err) { 32246aa90fe2SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg failed\n"); 32256aa90fe2SIoana Radulescu goto out_free_tbl; 32266aa90fe2SIoana Radulescu } 32276aa90fe2SIoana Radulescu 32286aa90fe2SIoana Radulescu /* set QoS table */ 32296aa90fe2SIoana Radulescu qos_cfg.default_tc = 0; 32306aa90fe2SIoana Radulescu qos_cfg.discard_on_miss = 0; 32316aa90fe2SIoana Radulescu qos_cfg.key_cfg_iova = dma_map_single(dev, dma_mem, 32326aa90fe2SIoana Radulescu DPAA2_CLASSIFIER_DMA_SIZE, 32336aa90fe2SIoana Radulescu DMA_TO_DEVICE); 32346aa90fe2SIoana Radulescu if (dma_mapping_error(dev, qos_cfg.key_cfg_iova)) { 32356aa90fe2SIoana Radulescu dev_err(dev, "QoS table DMA mapping failed\n"); 32366aa90fe2SIoana Radulescu err = -ENOMEM; 32376aa90fe2SIoana Radulescu goto out_free_tbl; 32386aa90fe2SIoana Radulescu } 32396aa90fe2SIoana Radulescu 32406aa90fe2SIoana Radulescu err = dpni_set_qos_table(priv->mc_io, 0, priv->mc_token, &qos_cfg); 32416aa90fe2SIoana Radulescu if (err) { 32426aa90fe2SIoana Radulescu dev_err(dev, "dpni_set_qos_table failed\n"); 32436aa90fe2SIoana Radulescu goto out_unmap_tbl; 32446aa90fe2SIoana Radulescu } 32456aa90fe2SIoana Radulescu 32466aa90fe2SIoana Radulescu /* Add QoS table entries */ 32476aa90fe2SIoana Radulescu key = kzalloc(key_size * 2, GFP_KERNEL); 32486aa90fe2SIoana Radulescu if (!key) { 32496aa90fe2SIoana Radulescu err = -ENOMEM; 32506aa90fe2SIoana Radulescu goto out_unmap_tbl; 32516aa90fe2SIoana Radulescu } 32526aa90fe2SIoana Radulescu mask = key + key_size; 32536aa90fe2SIoana Radulescu *(__be16 *)mask = cpu_to_be16(VLAN_PRIO_MASK); 32546aa90fe2SIoana Radulescu 32556aa90fe2SIoana Radulescu key_params.key_iova = dma_map_single(dev, key, key_size * 2, 32566aa90fe2SIoana Radulescu DMA_TO_DEVICE); 32576aa90fe2SIoana Radulescu if (dma_mapping_error(dev, key_params.key_iova)) { 32586aa90fe2SIoana Radulescu dev_err(dev, "Qos table entry DMA mapping failed\n"); 32596aa90fe2SIoana Radulescu err = -ENOMEM; 32606aa90fe2SIoana Radulescu goto out_free_key; 32616aa90fe2SIoana Radulescu } 32626aa90fe2SIoana Radulescu 32636aa90fe2SIoana Radulescu key_params.mask_iova = key_params.key_iova + key_size; 32646aa90fe2SIoana Radulescu key_params.key_size = key_size; 32656aa90fe2SIoana Radulescu 32666aa90fe2SIoana Radulescu /* We add rules for PCP-based distribution starting with highest 32676aa90fe2SIoana Radulescu * priority (VLAN PCP = 7). If this DPNI doesn't have enough traffic 32686aa90fe2SIoana Radulescu * classes to accommodate all priority levels, the lowest ones end up 32696aa90fe2SIoana Radulescu * on TC 0 which was configured as default 32706aa90fe2SIoana Radulescu */ 32716aa90fe2SIoana Radulescu for (i = dpaa2_eth_tc_count(priv) - 1, pcp = 7; i >= 0; i--, pcp--) { 32726aa90fe2SIoana Radulescu *(__be16 *)key = cpu_to_be16(pcp << VLAN_PRIO_SHIFT); 32736aa90fe2SIoana Radulescu dma_sync_single_for_device(dev, key_params.key_iova, 32746aa90fe2SIoana Radulescu key_size * 2, DMA_TO_DEVICE); 32756aa90fe2SIoana Radulescu 32766aa90fe2SIoana Radulescu err = dpni_add_qos_entry(priv->mc_io, 0, priv->mc_token, 32776aa90fe2SIoana Radulescu &key_params, i, i); 32786aa90fe2SIoana Radulescu if (err) { 32796aa90fe2SIoana Radulescu dev_err(dev, "dpni_add_qos_entry failed\n"); 32806aa90fe2SIoana Radulescu dpni_clear_qos_table(priv->mc_io, 0, priv->mc_token); 32816aa90fe2SIoana Radulescu goto out_unmap_key; 32826aa90fe2SIoana Radulescu } 32836aa90fe2SIoana Radulescu } 32846aa90fe2SIoana Radulescu 32856aa90fe2SIoana Radulescu priv->vlan_cls_enabled = true; 32866aa90fe2SIoana Radulescu 32876aa90fe2SIoana Radulescu /* Table and key memory is not persistent, clean everything up after 32886aa90fe2SIoana Radulescu * configuration is finished 32896aa90fe2SIoana Radulescu */ 32906aa90fe2SIoana Radulescu out_unmap_key: 32916aa90fe2SIoana Radulescu dma_unmap_single(dev, key_params.key_iova, key_size * 2, DMA_TO_DEVICE); 32926aa90fe2SIoana Radulescu out_free_key: 32936aa90fe2SIoana Radulescu kfree(key); 32946aa90fe2SIoana Radulescu out_unmap_tbl: 32956aa90fe2SIoana Radulescu dma_unmap_single(dev, qos_cfg.key_cfg_iova, DPAA2_CLASSIFIER_DMA_SIZE, 32966aa90fe2SIoana Radulescu DMA_TO_DEVICE); 32976aa90fe2SIoana Radulescu out_free_tbl: 32986aa90fe2SIoana Radulescu kfree(dma_mem); 32996aa90fe2SIoana Radulescu 33006aa90fe2SIoana Radulescu return err; 33016aa90fe2SIoana Radulescu } 33026aa90fe2SIoana Radulescu 330334ff6846SIoana Radulescu /* Configure the DPNI object this interface is associated with */ 33045d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpni(struct fsl_mc_device *ls_dev) 330534ff6846SIoana Radulescu { 330634ff6846SIoana Radulescu struct device *dev = &ls_dev->dev; 330734ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 330834ff6846SIoana Radulescu struct net_device *net_dev; 330934ff6846SIoana Radulescu int err; 331034ff6846SIoana Radulescu 331134ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 331234ff6846SIoana Radulescu priv = netdev_priv(net_dev); 331334ff6846SIoana Radulescu 331434ff6846SIoana Radulescu /* get a handle for the DPNI object */ 331534ff6846SIoana Radulescu err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 331634ff6846SIoana Radulescu if (err) { 331734ff6846SIoana Radulescu dev_err(dev, "dpni_open() failed\n"); 331834ff6846SIoana Radulescu return err; 331934ff6846SIoana Radulescu } 332034ff6846SIoana Radulescu 332134ff6846SIoana Radulescu /* Check if we can work with this DPNI object */ 332234ff6846SIoana Radulescu err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 332334ff6846SIoana Radulescu &priv->dpni_ver_minor); 332434ff6846SIoana Radulescu if (err) { 332534ff6846SIoana Radulescu dev_err(dev, "dpni_get_api_version() failed\n"); 332634ff6846SIoana Radulescu goto close; 332734ff6846SIoana Radulescu } 332834ff6846SIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 332934ff6846SIoana Radulescu dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 333034ff6846SIoana Radulescu priv->dpni_ver_major, priv->dpni_ver_minor, 333134ff6846SIoana Radulescu DPNI_VER_MAJOR, DPNI_VER_MINOR); 333234ff6846SIoana Radulescu err = -ENOTSUPP; 333334ff6846SIoana Radulescu goto close; 333434ff6846SIoana Radulescu } 333534ff6846SIoana Radulescu 333634ff6846SIoana Radulescu ls_dev->mc_io = priv->mc_io; 333734ff6846SIoana Radulescu ls_dev->mc_handle = priv->mc_token; 333834ff6846SIoana Radulescu 333934ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 334034ff6846SIoana Radulescu if (err) { 334134ff6846SIoana Radulescu dev_err(dev, "dpni_reset() failed\n"); 334234ff6846SIoana Radulescu goto close; 334334ff6846SIoana Radulescu } 334434ff6846SIoana Radulescu 334534ff6846SIoana Radulescu err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 334634ff6846SIoana Radulescu &priv->dpni_attrs); 334734ff6846SIoana Radulescu if (err) { 334834ff6846SIoana Radulescu dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 334934ff6846SIoana Radulescu goto close; 335034ff6846SIoana Radulescu } 335134ff6846SIoana Radulescu 33525d8dccf8SIoana Ciornei err = dpaa2_eth_set_buffer_layout(priv); 335334ff6846SIoana Radulescu if (err) 335434ff6846SIoana Radulescu goto close; 335534ff6846SIoana Radulescu 33565d8dccf8SIoana Ciornei dpaa2_eth_set_enqueue_mode(priv); 33571fa0f68cSIoana Ciocoi Radulescu 33588eb3cef8SIoana Radulescu /* Enable pause frame support */ 33598eb3cef8SIoana Radulescu if (dpaa2_eth_has_pause_support(priv)) { 33605d8dccf8SIoana Ciornei err = dpaa2_eth_set_pause(priv); 33618eb3cef8SIoana Radulescu if (err) 33628eb3cef8SIoana Radulescu goto close; 33638eb3cef8SIoana Radulescu } 33648eb3cef8SIoana Radulescu 33655d8dccf8SIoana Ciornei err = dpaa2_eth_set_vlan_qos(priv); 33666aa90fe2SIoana Radulescu if (err && err != -EOPNOTSUPP) 33676aa90fe2SIoana Radulescu goto close; 33686aa90fe2SIoana Radulescu 33699334d5baSXu Wang priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv), 33709334d5baSXu Wang sizeof(struct dpaa2_eth_cls_rule), 33719334d5baSXu Wang GFP_KERNEL); 337297fff7c8SWei Yongjun if (!priv->cls_rules) { 337397fff7c8SWei Yongjun err = -ENOMEM; 3374afb90dbbSIoana Radulescu goto close; 337597fff7c8SWei Yongjun } 3376afb90dbbSIoana Radulescu 337734ff6846SIoana Radulescu return 0; 337834ff6846SIoana Radulescu 337934ff6846SIoana Radulescu close: 338034ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 338134ff6846SIoana Radulescu 338234ff6846SIoana Radulescu return err; 338334ff6846SIoana Radulescu } 338434ff6846SIoana Radulescu 33855d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpni(struct dpaa2_eth_priv *priv) 338634ff6846SIoana Radulescu { 338734ff6846SIoana Radulescu int err; 338834ff6846SIoana Radulescu 338934ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 339034ff6846SIoana Radulescu if (err) 339134ff6846SIoana Radulescu netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 339234ff6846SIoana Radulescu err); 339334ff6846SIoana Radulescu 339434ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 339534ff6846SIoana Radulescu } 339634ff6846SIoana Radulescu 33975d8dccf8SIoana Ciornei static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, 339834ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 339934ff6846SIoana Radulescu { 340034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 340134ff6846SIoana Radulescu struct dpni_queue queue; 340234ff6846SIoana Radulescu struct dpni_queue_id qid; 340334ff6846SIoana Radulescu int err; 340434ff6846SIoana Radulescu 340534ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3406685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, &queue, &qid); 340734ff6846SIoana Radulescu if (err) { 340834ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(RX) failed\n"); 340934ff6846SIoana Radulescu return err; 341034ff6846SIoana Radulescu } 341134ff6846SIoana Radulescu 341234ff6846SIoana Radulescu fq->fqid = qid.fqid; 341334ff6846SIoana Radulescu 341434ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 341534ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 341634ff6846SIoana Radulescu queue.destination.priority = 1; 341734ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 341834ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3419685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, 342016fa1cf1SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 342134ff6846SIoana Radulescu &queue); 342234ff6846SIoana Radulescu if (err) { 342334ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(RX) failed\n"); 342434ff6846SIoana Radulescu return err; 342534ff6846SIoana Radulescu } 342634ff6846SIoana Radulescu 3427d678be1dSIoana Radulescu /* xdp_rxq setup */ 3428685e39eaSIoana Radulescu /* only once for each channel */ 3429685e39eaSIoana Radulescu if (fq->tc > 0) 3430685e39eaSIoana Radulescu return 0; 3431685e39eaSIoana Radulescu 3432d678be1dSIoana Radulescu err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 3433b02e5a0eSBjörn Töpel fq->flowid, 0); 3434d678be1dSIoana Radulescu if (err) { 3435d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg failed\n"); 3436d678be1dSIoana Radulescu return err; 3437d678be1dSIoana Radulescu } 3438d678be1dSIoana Radulescu 3439d678be1dSIoana Radulescu err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 3440d678be1dSIoana Radulescu MEM_TYPE_PAGE_ORDER0, NULL); 3441d678be1dSIoana Radulescu if (err) { 3442d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 3443d678be1dSIoana Radulescu return err; 3444d678be1dSIoana Radulescu } 3445d678be1dSIoana Radulescu 344634ff6846SIoana Radulescu return 0; 344734ff6846SIoana Radulescu } 344834ff6846SIoana Radulescu 34495d8dccf8SIoana Ciornei static int dpaa2_eth_setup_tx_flow(struct dpaa2_eth_priv *priv, 345034ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 345134ff6846SIoana Radulescu { 345234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 345334ff6846SIoana Radulescu struct dpni_queue queue; 345434ff6846SIoana Radulescu struct dpni_queue_id qid; 345515c87f6bSIoana Radulescu int i, err; 345634ff6846SIoana Radulescu 345715c87f6bSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 345834ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 345915c87f6bSIoana Radulescu DPNI_QUEUE_TX, i, fq->flowid, 346015c87f6bSIoana Radulescu &queue, &qid); 346134ff6846SIoana Radulescu if (err) { 346234ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX) failed\n"); 346334ff6846SIoana Radulescu return err; 346434ff6846SIoana Radulescu } 346515c87f6bSIoana Radulescu fq->tx_fqid[i] = qid.fqid; 346615c87f6bSIoana Radulescu } 346734ff6846SIoana Radulescu 346815c87f6bSIoana Radulescu /* All Tx queues belonging to the same flowid have the same qdbin */ 346934ff6846SIoana Radulescu fq->tx_qdbin = qid.qdbin; 347034ff6846SIoana Radulescu 347134ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 347234ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 347334ff6846SIoana Radulescu &queue, &qid); 347434ff6846SIoana Radulescu if (err) { 347534ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 347634ff6846SIoana Radulescu return err; 347734ff6846SIoana Radulescu } 347834ff6846SIoana Radulescu 347934ff6846SIoana Radulescu fq->fqid = qid.fqid; 348034ff6846SIoana Radulescu 348134ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 348234ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 348334ff6846SIoana Radulescu queue.destination.priority = 0; 348434ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 348534ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 348634ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 348734ff6846SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 348834ff6846SIoana Radulescu &queue); 348934ff6846SIoana Radulescu if (err) { 349034ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 349134ff6846SIoana Radulescu return err; 349234ff6846SIoana Radulescu } 349334ff6846SIoana Radulescu 349434ff6846SIoana Radulescu return 0; 349534ff6846SIoana Radulescu } 349634ff6846SIoana Radulescu 3497061d631fSIoana Ciornei static int setup_rx_err_flow(struct dpaa2_eth_priv *priv, 3498061d631fSIoana Ciornei struct dpaa2_eth_fq *fq) 3499061d631fSIoana Ciornei { 3500061d631fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 3501061d631fSIoana Ciornei struct dpni_queue q = { { 0 } }; 3502061d631fSIoana Ciornei struct dpni_queue_id qid; 3503061d631fSIoana Ciornei u8 q_opt = DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST; 3504061d631fSIoana Ciornei int err; 3505061d631fSIoana Ciornei 3506061d631fSIoana Ciornei err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3507061d631fSIoana Ciornei DPNI_QUEUE_RX_ERR, 0, 0, &q, &qid); 3508061d631fSIoana Ciornei if (err) { 3509061d631fSIoana Ciornei dev_err(dev, "dpni_get_queue() failed (%d)\n", err); 3510061d631fSIoana Ciornei return err; 3511061d631fSIoana Ciornei } 3512061d631fSIoana Ciornei 3513061d631fSIoana Ciornei fq->fqid = qid.fqid; 3514061d631fSIoana Ciornei 3515061d631fSIoana Ciornei q.destination.id = fq->channel->dpcon_id; 3516061d631fSIoana Ciornei q.destination.type = DPNI_DEST_DPCON; 3517061d631fSIoana Ciornei q.destination.priority = 1; 3518061d631fSIoana Ciornei q.user_context = (u64)(uintptr_t)fq; 3519061d631fSIoana Ciornei err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3520061d631fSIoana Ciornei DPNI_QUEUE_RX_ERR, 0, 0, q_opt, &q); 3521061d631fSIoana Ciornei if (err) { 3522061d631fSIoana Ciornei dev_err(dev, "dpni_set_queue() failed (%d)\n", err); 3523061d631fSIoana Ciornei return err; 3524061d631fSIoana Ciornei } 3525061d631fSIoana Ciornei 3526061d631fSIoana Ciornei return 0; 3527061d631fSIoana Ciornei } 3528061d631fSIoana Ciornei 3529edad8d26SIoana Ciocoi Radulescu /* Supported header fields for Rx hash distribution key */ 3530f76c483aSIoana Radulescu static const struct dpaa2_eth_dist_fields dist_fields[] = { 353134ff6846SIoana Radulescu { 3532edad8d26SIoana Ciocoi Radulescu /* L2 header */ 3533edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_L2DA, 3534edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_ETH, 3535edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_ETH_DA, 35363a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHDST, 3537edad8d26SIoana Ciocoi Radulescu .size = 6, 3538edad8d26SIoana Ciocoi Radulescu }, { 3539afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3540afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_SA, 35413a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHSRC, 3542afb90dbbSIoana Radulescu .size = 6, 3543afb90dbbSIoana Radulescu }, { 3544afb90dbbSIoana Radulescu /* This is the last ethertype field parsed: 3545afb90dbbSIoana Radulescu * depending on frame format, it can be the MAC ethertype 3546afb90dbbSIoana Radulescu * or the VLAN etype. 3547afb90dbbSIoana Radulescu */ 3548afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3549afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_TYPE, 35503a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHTYPE, 3551afb90dbbSIoana Radulescu .size = 2, 3552afb90dbbSIoana Radulescu }, { 3553edad8d26SIoana Ciocoi Radulescu /* VLAN header */ 3554edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_VLAN, 3555edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_VLAN, 3556edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_VLAN_TCI, 35573a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_VLAN, 3558edad8d26SIoana Ciocoi Radulescu .size = 2, 3559edad8d26SIoana Ciocoi Radulescu }, { 356034ff6846SIoana Radulescu /* IP header */ 356134ff6846SIoana Radulescu .rxnfc_field = RXH_IP_SRC, 356234ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 356334ff6846SIoana Radulescu .cls_field = NH_FLD_IP_SRC, 35643a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPSRC, 356534ff6846SIoana Radulescu .size = 4, 356634ff6846SIoana Radulescu }, { 356734ff6846SIoana Radulescu .rxnfc_field = RXH_IP_DST, 356834ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 356934ff6846SIoana Radulescu .cls_field = NH_FLD_IP_DST, 35703a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPDST, 357134ff6846SIoana Radulescu .size = 4, 357234ff6846SIoana Radulescu }, { 357334ff6846SIoana Radulescu .rxnfc_field = RXH_L3_PROTO, 357434ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 357534ff6846SIoana Radulescu .cls_field = NH_FLD_IP_PROTO, 35763a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPPROTO, 357734ff6846SIoana Radulescu .size = 1, 357834ff6846SIoana Radulescu }, { 357934ff6846SIoana Radulescu /* Using UDP ports, this is functionally equivalent to raw 358034ff6846SIoana Radulescu * byte pairs from L4 header. 358134ff6846SIoana Radulescu */ 358234ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_0_1, 358334ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 358434ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_SRC, 35853a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4SRC, 358634ff6846SIoana Radulescu .size = 2, 358734ff6846SIoana Radulescu }, { 358834ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_2_3, 358934ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 359034ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_DST, 35913a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4DST, 359234ff6846SIoana Radulescu .size = 2, 359334ff6846SIoana Radulescu }, 359434ff6846SIoana Radulescu }; 359534ff6846SIoana Radulescu 3596df85aeb9SIoana Radulescu /* Configure the Rx hash key using the legacy API */ 35975d8dccf8SIoana Ciornei static int dpaa2_eth_config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3598df85aeb9SIoana Radulescu { 3599df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3600df85aeb9SIoana Radulescu struct dpni_rx_tc_dist_cfg dist_cfg; 3601685e39eaSIoana Radulescu int i, err = 0; 3602df85aeb9SIoana Radulescu 3603df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3604df85aeb9SIoana Radulescu 3605df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3606df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3607df85aeb9SIoana Radulescu dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 3608df85aeb9SIoana Radulescu 3609685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3610685e39eaSIoana Radulescu err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 3611685e39eaSIoana Radulescu i, &dist_cfg); 3612685e39eaSIoana Radulescu if (err) { 3613df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 3614685e39eaSIoana Radulescu break; 3615685e39eaSIoana Radulescu } 3616685e39eaSIoana Radulescu } 3617df85aeb9SIoana Radulescu 3618df85aeb9SIoana Radulescu return err; 3619df85aeb9SIoana Radulescu } 3620df85aeb9SIoana Radulescu 3621df85aeb9SIoana Radulescu /* Configure the Rx hash key using the new API */ 36225d8dccf8SIoana Ciornei static int dpaa2_eth_config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3623df85aeb9SIoana Radulescu { 3624df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3625df85aeb9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3626685e39eaSIoana Radulescu int i, err = 0; 3627df85aeb9SIoana Radulescu 3628df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3629df85aeb9SIoana Radulescu 3630df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3631df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3632df85aeb9SIoana Radulescu dist_cfg.enable = 1; 3633df85aeb9SIoana Radulescu 3634685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3635685e39eaSIoana Radulescu dist_cfg.tc = i; 3636685e39eaSIoana Radulescu err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, 3637685e39eaSIoana Radulescu &dist_cfg); 3638685e39eaSIoana Radulescu if (err) { 3639df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 3640685e39eaSIoana Radulescu break; 3641685e39eaSIoana Radulescu } 36425e29c16fSIonut-robert Aron 36435e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 36445e29c16fSIonut-robert Aron * traffic classes, install it just once 36455e29c16fSIonut-robert Aron */ 36465e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 36475e29c16fSIonut-robert Aron break; 3648685e39eaSIoana Radulescu } 3649df85aeb9SIoana Radulescu 3650df85aeb9SIoana Radulescu return err; 3651df85aeb9SIoana Radulescu } 3652df85aeb9SIoana Radulescu 36534aaaf9b9SIoana Radulescu /* Configure the Rx flow classification key */ 36545d8dccf8SIoana Ciornei static int dpaa2_eth_config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 36554aaaf9b9SIoana Radulescu { 36564aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 36574aaaf9b9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3658685e39eaSIoana Radulescu int i, err = 0; 36594aaaf9b9SIoana Radulescu 36604aaaf9b9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 36614aaaf9b9SIoana Radulescu 36624aaaf9b9SIoana Radulescu dist_cfg.key_cfg_iova = key; 36634aaaf9b9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 36644aaaf9b9SIoana Radulescu dist_cfg.enable = 1; 36654aaaf9b9SIoana Radulescu 3666685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3667685e39eaSIoana Radulescu dist_cfg.tc = i; 3668685e39eaSIoana Radulescu err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, 3669685e39eaSIoana Radulescu &dist_cfg); 3670685e39eaSIoana Radulescu if (err) { 36714aaaf9b9SIoana Radulescu dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 3672685e39eaSIoana Radulescu break; 3673685e39eaSIoana Radulescu } 36745e29c16fSIonut-robert Aron 36755e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 36765e29c16fSIonut-robert Aron * traffic classes, install it just once 36775e29c16fSIonut-robert Aron */ 36785e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 36795e29c16fSIonut-robert Aron break; 3680685e39eaSIoana Radulescu } 36814aaaf9b9SIoana Radulescu 36824aaaf9b9SIoana Radulescu return err; 36834aaaf9b9SIoana Radulescu } 36844aaaf9b9SIoana Radulescu 3685afb90dbbSIoana Radulescu /* Size of the Rx flow classification key */ 36862d680237SIoana Ciocoi Radulescu int dpaa2_eth_cls_key_size(u64 fields) 3687afb90dbbSIoana Radulescu { 3688afb90dbbSIoana Radulescu int i, size = 0; 3689afb90dbbSIoana Radulescu 36902d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 36912d680237SIoana Ciocoi Radulescu if (!(fields & dist_fields[i].id)) 36922d680237SIoana Ciocoi Radulescu continue; 3693afb90dbbSIoana Radulescu size += dist_fields[i].size; 36942d680237SIoana Ciocoi Radulescu } 3695afb90dbbSIoana Radulescu 3696afb90dbbSIoana Radulescu return size; 3697afb90dbbSIoana Radulescu } 3698afb90dbbSIoana Radulescu 3699afb90dbbSIoana Radulescu /* Offset of header field in Rx classification key */ 3700afb90dbbSIoana Radulescu int dpaa2_eth_cls_fld_off(int prot, int field) 3701afb90dbbSIoana Radulescu { 3702afb90dbbSIoana Radulescu int i, off = 0; 3703afb90dbbSIoana Radulescu 3704afb90dbbSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3705afb90dbbSIoana Radulescu if (dist_fields[i].cls_prot == prot && 3706afb90dbbSIoana Radulescu dist_fields[i].cls_field == field) 3707afb90dbbSIoana Radulescu return off; 3708afb90dbbSIoana Radulescu off += dist_fields[i].size; 3709afb90dbbSIoana Radulescu } 3710afb90dbbSIoana Radulescu 3711afb90dbbSIoana Radulescu WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 3712afb90dbbSIoana Radulescu return 0; 3713afb90dbbSIoana Radulescu } 3714afb90dbbSIoana Radulescu 37152d680237SIoana Ciocoi Radulescu /* Prune unused fields from the classification rule. 37162d680237SIoana Ciocoi Radulescu * Used when masking is not supported 37172d680237SIoana Ciocoi Radulescu */ 37182d680237SIoana Ciocoi Radulescu void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 37192d680237SIoana Ciocoi Radulescu { 37202d680237SIoana Ciocoi Radulescu int off = 0, new_off = 0; 37212d680237SIoana Ciocoi Radulescu int i, size; 37222d680237SIoana Ciocoi Radulescu 37232d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 37242d680237SIoana Ciocoi Radulescu size = dist_fields[i].size; 37252d680237SIoana Ciocoi Radulescu if (dist_fields[i].id & fields) { 37262d680237SIoana Ciocoi Radulescu memcpy(key_mem + new_off, key_mem + off, size); 37272d680237SIoana Ciocoi Radulescu new_off += size; 37282d680237SIoana Ciocoi Radulescu } 37292d680237SIoana Ciocoi Radulescu off += size; 37302d680237SIoana Ciocoi Radulescu } 37312d680237SIoana Ciocoi Radulescu } 37322d680237SIoana Ciocoi Radulescu 37334aaaf9b9SIoana Radulescu /* Set Rx distribution (hash or flow classification) key 373434ff6846SIoana Radulescu * flags is a combination of RXH_ bits 373534ff6846SIoana Radulescu */ 37363233c151SIoana Ciornei static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 37374aaaf9b9SIoana Radulescu enum dpaa2_eth_rx_dist type, u64 flags) 373834ff6846SIoana Radulescu { 373934ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 374034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 374134ff6846SIoana Radulescu struct dpkg_profile_cfg cls_cfg; 3742edad8d26SIoana Ciocoi Radulescu u32 rx_hash_fields = 0; 3743df85aeb9SIoana Radulescu dma_addr_t key_iova; 374434ff6846SIoana Radulescu u8 *dma_mem; 374534ff6846SIoana Radulescu int i; 374634ff6846SIoana Radulescu int err = 0; 374734ff6846SIoana Radulescu 374834ff6846SIoana Radulescu memset(&cls_cfg, 0, sizeof(cls_cfg)); 374934ff6846SIoana Radulescu 3750f76c483aSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 375134ff6846SIoana Radulescu struct dpkg_extract *key = 375234ff6846SIoana Radulescu &cls_cfg.extracts[cls_cfg.num_extracts]; 375334ff6846SIoana Radulescu 37542d680237SIoana Ciocoi Radulescu /* For both Rx hashing and classification keys 37552d680237SIoana Ciocoi Radulescu * we set only the selected fields. 37564aaaf9b9SIoana Radulescu */ 37573a1e6b84SIoana Ciocoi Radulescu if (!(flags & dist_fields[i].id)) 375834ff6846SIoana Radulescu continue; 37592d680237SIoana Ciocoi Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) 37604aaaf9b9SIoana Radulescu rx_hash_fields |= dist_fields[i].rxnfc_field; 376134ff6846SIoana Radulescu 376234ff6846SIoana Radulescu if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 376334ff6846SIoana Radulescu dev_err(dev, "error adding key extraction rule, too many rules?\n"); 376434ff6846SIoana Radulescu return -E2BIG; 376534ff6846SIoana Radulescu } 376634ff6846SIoana Radulescu 376734ff6846SIoana Radulescu key->type = DPKG_EXTRACT_FROM_HDR; 3768f76c483aSIoana Radulescu key->extract.from_hdr.prot = dist_fields[i].cls_prot; 376934ff6846SIoana Radulescu key->extract.from_hdr.type = DPKG_FULL_FIELD; 3770f76c483aSIoana Radulescu key->extract.from_hdr.field = dist_fields[i].cls_field; 377134ff6846SIoana Radulescu cls_cfg.num_extracts++; 377234ff6846SIoana Radulescu } 377334ff6846SIoana Radulescu 377434ff6846SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 377534ff6846SIoana Radulescu if (!dma_mem) 377634ff6846SIoana Radulescu return -ENOMEM; 377734ff6846SIoana Radulescu 377834ff6846SIoana Radulescu err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 377934ff6846SIoana Radulescu if (err) { 378034ff6846SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 3781df85aeb9SIoana Radulescu goto free_key; 378234ff6846SIoana Radulescu } 378334ff6846SIoana Radulescu 378434ff6846SIoana Radulescu /* Prepare for setting the rx dist */ 3785df85aeb9SIoana Radulescu key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 378634ff6846SIoana Radulescu DMA_TO_DEVICE); 3787df85aeb9SIoana Radulescu if (dma_mapping_error(dev, key_iova)) { 378834ff6846SIoana Radulescu dev_err(dev, "DMA mapping failed\n"); 378934ff6846SIoana Radulescu err = -ENOMEM; 3790df85aeb9SIoana Radulescu goto free_key; 379134ff6846SIoana Radulescu } 379234ff6846SIoana Radulescu 37934aaaf9b9SIoana Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) { 3794df85aeb9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) 37955d8dccf8SIoana Ciornei err = dpaa2_eth_config_legacy_hash_key(priv, key_iova); 3796edad8d26SIoana Ciocoi Radulescu else 37975d8dccf8SIoana Ciornei err = dpaa2_eth_config_hash_key(priv, key_iova); 37984aaaf9b9SIoana Radulescu } else { 37995d8dccf8SIoana Ciornei err = dpaa2_eth_config_cls_key(priv, key_iova); 38004aaaf9b9SIoana Radulescu } 3801df85aeb9SIoana Radulescu 3802df85aeb9SIoana Radulescu dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3803df85aeb9SIoana Radulescu DMA_TO_DEVICE); 38044aaaf9b9SIoana Radulescu if (!err && type == DPAA2_ETH_RX_DIST_HASH) 3805edad8d26SIoana Ciocoi Radulescu priv->rx_hash_fields = rx_hash_fields; 380634ff6846SIoana Radulescu 3807df85aeb9SIoana Radulescu free_key: 380834ff6846SIoana Radulescu kfree(dma_mem); 380934ff6846SIoana Radulescu return err; 381034ff6846SIoana Radulescu } 381134ff6846SIoana Radulescu 38124aaaf9b9SIoana Radulescu int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 38134aaaf9b9SIoana Radulescu { 38144aaaf9b9SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 38153a1e6b84SIoana Ciocoi Radulescu u64 key = 0; 38163a1e6b84SIoana Ciocoi Radulescu int i; 38174aaaf9b9SIoana Radulescu 38184aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) 38194aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38204aaaf9b9SIoana Radulescu 38213a1e6b84SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 38223a1e6b84SIoana Ciocoi Radulescu if (dist_fields[i].rxnfc_field & flags) 38233a1e6b84SIoana Ciocoi Radulescu key |= dist_fields[i].id; 38243a1e6b84SIoana Ciocoi Radulescu 38253a1e6b84SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 38264aaaf9b9SIoana Radulescu } 38274aaaf9b9SIoana Radulescu 38282d680237SIoana Ciocoi Radulescu int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 38292d680237SIoana Ciocoi Radulescu { 38302d680237SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 38312d680237SIoana Ciocoi Radulescu } 38322d680237SIoana Ciocoi Radulescu 38332d680237SIoana Ciocoi Radulescu static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 38344aaaf9b9SIoana Radulescu { 38354aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3836df8e249bSIoana Ciocoi Radulescu int err; 38374aaaf9b9SIoana Radulescu 38384aaaf9b9SIoana Radulescu /* Check if we actually support Rx flow classification */ 38394aaaf9b9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) { 38404aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls not supported by current MC version\n"); 38414aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38424aaaf9b9SIoana Radulescu } 38434aaaf9b9SIoana Radulescu 38442d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_enabled(priv)) { 38454aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 38464aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38474aaaf9b9SIoana Radulescu } 38484aaaf9b9SIoana Radulescu 38494aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) { 38504aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 38514aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 38524aaaf9b9SIoana Radulescu } 38534aaaf9b9SIoana Radulescu 38542d680237SIoana Ciocoi Radulescu /* If there is no support for masking in the classification table, 38552d680237SIoana Ciocoi Radulescu * we don't set a default key, as it will depend on the rules 38562d680237SIoana Ciocoi Radulescu * added by the user at runtime. 38572d680237SIoana Ciocoi Radulescu */ 38582d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_mask_enabled(priv)) 38592d680237SIoana Ciocoi Radulescu goto out; 38602d680237SIoana Ciocoi Radulescu 38612d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 3862df8e249bSIoana Ciocoi Radulescu if (err) 3863df8e249bSIoana Ciocoi Radulescu return err; 3864df8e249bSIoana Ciocoi Radulescu 38652d680237SIoana Ciocoi Radulescu out: 38664aaaf9b9SIoana Radulescu priv->rx_cls_enabled = 1; 38674aaaf9b9SIoana Radulescu 3868df8e249bSIoana Ciocoi Radulescu return 0; 38694aaaf9b9SIoana Radulescu } 38704aaaf9b9SIoana Radulescu 387134ff6846SIoana Radulescu /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 387234ff6846SIoana Radulescu * frame queues and channels 387334ff6846SIoana Radulescu */ 38745d8dccf8SIoana Ciornei static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv) 387534ff6846SIoana Radulescu { 387634ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 387734ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 387834ff6846SIoana Radulescu struct dpni_pools_cfg pools_params; 387934ff6846SIoana Radulescu struct dpni_error_cfg err_cfg; 388034ff6846SIoana Radulescu int err = 0; 388134ff6846SIoana Radulescu int i; 388234ff6846SIoana Radulescu 388334ff6846SIoana Radulescu pools_params.num_dpbp = 1; 388434ff6846SIoana Radulescu pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 388534ff6846SIoana Radulescu pools_params.pools[0].backup_pool = 0; 3886efa6a7d0SIoana Ciornei pools_params.pools[0].buffer_size = priv->rx_buf_size; 388734ff6846SIoana Radulescu err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 388834ff6846SIoana Radulescu if (err) { 388934ff6846SIoana Radulescu dev_err(dev, "dpni_set_pools() failed\n"); 389034ff6846SIoana Radulescu return err; 389134ff6846SIoana Radulescu } 389234ff6846SIoana Radulescu 389334ff6846SIoana Radulescu /* have the interface implicitly distribute traffic based on 389434ff6846SIoana Radulescu * the default hash key 389534ff6846SIoana Radulescu */ 389634ff6846SIoana Radulescu err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 3897edad8d26SIoana Ciocoi Radulescu if (err && err != -EOPNOTSUPP) 389834ff6846SIoana Radulescu dev_err(dev, "Failed to configure hashing\n"); 389934ff6846SIoana Radulescu 39004aaaf9b9SIoana Radulescu /* Configure the flow classification key; it includes all 39014aaaf9b9SIoana Radulescu * supported header fields and cannot be modified at runtime 39024aaaf9b9SIoana Radulescu */ 39032d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_default_cls(priv); 39044aaaf9b9SIoana Radulescu if (err && err != -EOPNOTSUPP) 39054aaaf9b9SIoana Radulescu dev_err(dev, "Failed to configure Rx classification key\n"); 39064aaaf9b9SIoana Radulescu 390734ff6846SIoana Radulescu /* Configure handling of error frames */ 390834ff6846SIoana Radulescu err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 390934ff6846SIoana Radulescu err_cfg.set_frame_annotation = 1; 391034ff6846SIoana Radulescu err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 391134ff6846SIoana Radulescu err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 391234ff6846SIoana Radulescu &err_cfg); 391334ff6846SIoana Radulescu if (err) { 391434ff6846SIoana Radulescu dev_err(dev, "dpni_set_errors_behavior failed\n"); 391534ff6846SIoana Radulescu return err; 391634ff6846SIoana Radulescu } 391734ff6846SIoana Radulescu 391834ff6846SIoana Radulescu /* Configure Rx and Tx conf queues to generate CDANs */ 391934ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 392034ff6846SIoana Radulescu switch (priv->fq[i].type) { 392134ff6846SIoana Radulescu case DPAA2_RX_FQ: 39225d8dccf8SIoana Ciornei err = dpaa2_eth_setup_rx_flow(priv, &priv->fq[i]); 392334ff6846SIoana Radulescu break; 392434ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 39255d8dccf8SIoana Ciornei err = dpaa2_eth_setup_tx_flow(priv, &priv->fq[i]); 392634ff6846SIoana Radulescu break; 3927061d631fSIoana Ciornei case DPAA2_RX_ERR_FQ: 3928061d631fSIoana Ciornei err = setup_rx_err_flow(priv, &priv->fq[i]); 3929061d631fSIoana Ciornei break; 393034ff6846SIoana Radulescu default: 393134ff6846SIoana Radulescu dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 393234ff6846SIoana Radulescu return -EINVAL; 393334ff6846SIoana Radulescu } 393434ff6846SIoana Radulescu if (err) 393534ff6846SIoana Radulescu return err; 393634ff6846SIoana Radulescu } 393734ff6846SIoana Radulescu 393834ff6846SIoana Radulescu err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 393934ff6846SIoana Radulescu DPNI_QUEUE_TX, &priv->tx_qdid); 394034ff6846SIoana Radulescu if (err) { 394134ff6846SIoana Radulescu dev_err(dev, "dpni_get_qdid() failed\n"); 394234ff6846SIoana Radulescu return err; 394334ff6846SIoana Radulescu } 394434ff6846SIoana Radulescu 394534ff6846SIoana Radulescu return 0; 394634ff6846SIoana Radulescu } 394734ff6846SIoana Radulescu 394834ff6846SIoana Radulescu /* Allocate rings for storing incoming frame descriptors */ 39495d8dccf8SIoana Ciornei static int dpaa2_eth_alloc_rings(struct dpaa2_eth_priv *priv) 395034ff6846SIoana Radulescu { 395134ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 395234ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 395334ff6846SIoana Radulescu int i; 395434ff6846SIoana Radulescu 395534ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 395634ff6846SIoana Radulescu priv->channel[i]->store = 395734ff6846SIoana Radulescu dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 395834ff6846SIoana Radulescu if (!priv->channel[i]->store) { 395934ff6846SIoana Radulescu netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 396034ff6846SIoana Radulescu goto err_ring; 396134ff6846SIoana Radulescu } 396234ff6846SIoana Radulescu } 396334ff6846SIoana Radulescu 396434ff6846SIoana Radulescu return 0; 396534ff6846SIoana Radulescu 396634ff6846SIoana Radulescu err_ring: 396734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 396834ff6846SIoana Radulescu if (!priv->channel[i]->store) 396934ff6846SIoana Radulescu break; 397034ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 397134ff6846SIoana Radulescu } 397234ff6846SIoana Radulescu 397334ff6846SIoana Radulescu return -ENOMEM; 397434ff6846SIoana Radulescu } 397534ff6846SIoana Radulescu 39765d8dccf8SIoana Ciornei static void dpaa2_eth_free_rings(struct dpaa2_eth_priv *priv) 397734ff6846SIoana Radulescu { 397834ff6846SIoana Radulescu int i; 397934ff6846SIoana Radulescu 398034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 398134ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 398234ff6846SIoana Radulescu } 398334ff6846SIoana Radulescu 39845d8dccf8SIoana Ciornei static int dpaa2_eth_set_mac_addr(struct dpaa2_eth_priv *priv) 398534ff6846SIoana Radulescu { 398634ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 398734ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 398834ff6846SIoana Radulescu u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 398934ff6846SIoana Radulescu int err; 399034ff6846SIoana Radulescu 399134ff6846SIoana Radulescu /* Get firmware address, if any */ 399234ff6846SIoana Radulescu err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 399334ff6846SIoana Radulescu if (err) { 399434ff6846SIoana Radulescu dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 399534ff6846SIoana Radulescu return err; 399634ff6846SIoana Radulescu } 399734ff6846SIoana Radulescu 399834ff6846SIoana Radulescu /* Get DPNI attributes address, if any */ 399934ff6846SIoana Radulescu err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 400034ff6846SIoana Radulescu dpni_mac_addr); 400134ff6846SIoana Radulescu if (err) { 400234ff6846SIoana Radulescu dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 400334ff6846SIoana Radulescu return err; 400434ff6846SIoana Radulescu } 400534ff6846SIoana Radulescu 400634ff6846SIoana Radulescu /* First check if firmware has any address configured by bootloader */ 400734ff6846SIoana Radulescu if (!is_zero_ether_addr(mac_addr)) { 400834ff6846SIoana Radulescu /* If the DPMAC addr != DPNI addr, update it */ 400934ff6846SIoana Radulescu if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 401034ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, 401134ff6846SIoana Radulescu priv->mc_token, 401234ff6846SIoana Radulescu mac_addr); 401334ff6846SIoana Radulescu if (err) { 401434ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 401534ff6846SIoana Radulescu return err; 401634ff6846SIoana Radulescu } 401734ff6846SIoana Radulescu } 401834ff6846SIoana Radulescu memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); 401934ff6846SIoana Radulescu } else if (is_zero_ether_addr(dpni_mac_addr)) { 402034ff6846SIoana Radulescu /* No MAC address configured, fill in net_dev->dev_addr 402134ff6846SIoana Radulescu * with a random one 402234ff6846SIoana Radulescu */ 402334ff6846SIoana Radulescu eth_hw_addr_random(net_dev); 402434ff6846SIoana Radulescu dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 402534ff6846SIoana Radulescu 402634ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 402734ff6846SIoana Radulescu net_dev->dev_addr); 402834ff6846SIoana Radulescu if (err) { 402934ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 403034ff6846SIoana Radulescu return err; 403134ff6846SIoana Radulescu } 403234ff6846SIoana Radulescu 403334ff6846SIoana Radulescu /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 403434ff6846SIoana Radulescu * practical purposes, this will be our "permanent" mac address, 403534ff6846SIoana Radulescu * at least until the next reboot. This move will also permit 403634ff6846SIoana Radulescu * register_netdevice() to properly fill up net_dev->perm_addr. 403734ff6846SIoana Radulescu */ 403834ff6846SIoana Radulescu net_dev->addr_assign_type = NET_ADDR_PERM; 403934ff6846SIoana Radulescu } else { 404034ff6846SIoana Radulescu /* NET_ADDR_PERM is default, all we have to do is 404134ff6846SIoana Radulescu * fill in the device addr. 404234ff6846SIoana Radulescu */ 404334ff6846SIoana Radulescu memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len); 404434ff6846SIoana Radulescu } 404534ff6846SIoana Radulescu 404634ff6846SIoana Radulescu return 0; 404734ff6846SIoana Radulescu } 404834ff6846SIoana Radulescu 40495d8dccf8SIoana Ciornei static int dpaa2_eth_netdev_init(struct net_device *net_dev) 405034ff6846SIoana Radulescu { 405134ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 405234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 405334ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 405434ff6846SIoana Radulescu u64 supported = 0, not_supported = 0; 405534ff6846SIoana Radulescu u8 bcast_addr[ETH_ALEN]; 405634ff6846SIoana Radulescu u8 num_queues; 405734ff6846SIoana Radulescu int err; 405834ff6846SIoana Radulescu 405934ff6846SIoana Radulescu net_dev->netdev_ops = &dpaa2_eth_ops; 406034ff6846SIoana Radulescu net_dev->ethtool_ops = &dpaa2_ethtool_ops; 406134ff6846SIoana Radulescu 40625d8dccf8SIoana Ciornei err = dpaa2_eth_set_mac_addr(priv); 406334ff6846SIoana Radulescu if (err) 406434ff6846SIoana Radulescu return err; 406534ff6846SIoana Radulescu 406634ff6846SIoana Radulescu /* Explicitly add the broadcast address to the MAC filtering table */ 406734ff6846SIoana Radulescu eth_broadcast_addr(bcast_addr); 406834ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 406934ff6846SIoana Radulescu if (err) { 407034ff6846SIoana Radulescu dev_err(dev, "dpni_add_mac_addr() failed\n"); 407134ff6846SIoana Radulescu return err; 407234ff6846SIoana Radulescu } 407334ff6846SIoana Radulescu 407434ff6846SIoana Radulescu /* Set MTU upper limit; lower limit is 68B (default value) */ 407534ff6846SIoana Radulescu net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 407634ff6846SIoana Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 407734ff6846SIoana Radulescu DPAA2_ETH_MFL); 407834ff6846SIoana Radulescu if (err) { 407934ff6846SIoana Radulescu dev_err(dev, "dpni_set_max_frame_length() failed\n"); 408034ff6846SIoana Radulescu return err; 408134ff6846SIoana Radulescu } 408234ff6846SIoana Radulescu 408334ff6846SIoana Radulescu /* Set actual number of queues in the net device */ 408434ff6846SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 408534ff6846SIoana Radulescu err = netif_set_real_num_tx_queues(net_dev, num_queues); 408634ff6846SIoana Radulescu if (err) { 408734ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 408834ff6846SIoana Radulescu return err; 408934ff6846SIoana Radulescu } 409034ff6846SIoana Radulescu err = netif_set_real_num_rx_queues(net_dev, num_queues); 409134ff6846SIoana Radulescu if (err) { 409234ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 409334ff6846SIoana Radulescu return err; 409434ff6846SIoana Radulescu } 409534ff6846SIoana Radulescu 409634ff6846SIoana Radulescu /* Capabilities listing */ 409734ff6846SIoana Radulescu supported |= IFF_LIVE_ADDR_CHANGE; 409834ff6846SIoana Radulescu 409934ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER) 410034ff6846SIoana Radulescu not_supported |= IFF_UNICAST_FLT; 410134ff6846SIoana Radulescu else 410234ff6846SIoana Radulescu supported |= IFF_UNICAST_FLT; 410334ff6846SIoana Radulescu 410434ff6846SIoana Radulescu net_dev->priv_flags |= supported; 410534ff6846SIoana Radulescu net_dev->priv_flags &= ~not_supported; 410634ff6846SIoana Radulescu 410734ff6846SIoana Radulescu /* Features */ 410834ff6846SIoana Radulescu net_dev->features = NETIF_F_RXCSUM | 410934ff6846SIoana Radulescu NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 411034ff6846SIoana Radulescu NETIF_F_SG | NETIF_F_HIGHDMA | 41113657cdafSIoana Ciornei NETIF_F_LLTX | NETIF_F_HW_TC; 411234ff6846SIoana Radulescu net_dev->hw_features = net_dev->features; 411334ff6846SIoana Radulescu 411470b32d82SIonut-robert Aron if (priv->dpni_attrs.vlan_filter_entries) 411570b32d82SIonut-robert Aron net_dev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 411670b32d82SIonut-robert Aron 411734ff6846SIoana Radulescu return 0; 411834ff6846SIoana Radulescu } 411934ff6846SIoana Radulescu 41205d8dccf8SIoana Ciornei static int dpaa2_eth_poll_link_state(void *arg) 412134ff6846SIoana Radulescu { 412234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 412334ff6846SIoana Radulescu int err; 412434ff6846SIoana Radulescu 412534ff6846SIoana Radulescu while (!kthread_should_stop()) { 41265d8dccf8SIoana Ciornei err = dpaa2_eth_link_state_update(priv); 412734ff6846SIoana Radulescu if (unlikely(err)) 412834ff6846SIoana Radulescu return err; 412934ff6846SIoana Radulescu 413034ff6846SIoana Radulescu msleep(DPAA2_ETH_LINK_STATE_REFRESH); 413134ff6846SIoana Radulescu } 413234ff6846SIoana Radulescu 413334ff6846SIoana Radulescu return 0; 413434ff6846SIoana Radulescu } 413534ff6846SIoana Radulescu 413671947923SIoana Ciornei static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) 413771947923SIoana Ciornei { 413871947923SIoana Ciornei struct fsl_mc_device *dpni_dev, *dpmac_dev; 413971947923SIoana Ciornei struct dpaa2_mac *mac; 414071947923SIoana Ciornei int err; 414171947923SIoana Ciornei 414271947923SIoana Ciornei dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); 414371947923SIoana Ciornei dpmac_dev = fsl_mc_get_endpoint(dpni_dev); 414447325da2SIoana Ciornei 414547325da2SIoana Ciornei if (PTR_ERR(dpmac_dev) == -EPROBE_DEFER) 414647325da2SIoana Ciornei return PTR_ERR(dpmac_dev); 414747325da2SIoana Ciornei 414847325da2SIoana Ciornei if (IS_ERR(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) 414971947923SIoana Ciornei return 0; 415071947923SIoana Ciornei 415171947923SIoana Ciornei mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); 415271947923SIoana Ciornei if (!mac) 415371947923SIoana Ciornei return -ENOMEM; 415471947923SIoana Ciornei 415571947923SIoana Ciornei mac->mc_dev = dpmac_dev; 415671947923SIoana Ciornei mac->mc_io = priv->mc_io; 415771947923SIoana Ciornei mac->net_dev = priv->net_dev; 415871947923SIoana Ciornei 4159095dca16SIoana Ciornei err = dpaa2_mac_open(mac); 4160095dca16SIoana Ciornei if (err) 4161095dca16SIoana Ciornei goto err_free_mac; 4162d87e6063SIoana Ciornei priv->mac = mac; 4163095dca16SIoana Ciornei 4164d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 416571947923SIoana Ciornei err = dpaa2_mac_connect(mac); 416671947923SIoana Ciornei if (err) { 416771947923SIoana Ciornei netdev_err(priv->net_dev, "Error connecting to the MAC endpoint\n"); 4168095dca16SIoana Ciornei goto err_close_mac; 416971947923SIoana Ciornei } 4170d87e6063SIoana Ciornei } 417171947923SIoana Ciornei 417271947923SIoana Ciornei return 0; 4173095dca16SIoana Ciornei 4174095dca16SIoana Ciornei err_close_mac: 4175095dca16SIoana Ciornei dpaa2_mac_close(mac); 4176d87e6063SIoana Ciornei priv->mac = NULL; 4177095dca16SIoana Ciornei err_free_mac: 4178095dca16SIoana Ciornei kfree(mac); 4179095dca16SIoana Ciornei return err; 418071947923SIoana Ciornei } 418171947923SIoana Ciornei 418271947923SIoana Ciornei static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) 418371947923SIoana Ciornei { 4184d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 418571947923SIoana Ciornei dpaa2_mac_disconnect(priv->mac); 4186d87e6063SIoana Ciornei 4187848c1903SIoana Ciornei if (!dpaa2_eth_has_mac(priv)) 4188848c1903SIoana Ciornei return; 4189848c1903SIoana Ciornei 4190095dca16SIoana Ciornei dpaa2_mac_close(priv->mac); 419171947923SIoana Ciornei kfree(priv->mac); 419271947923SIoana Ciornei priv->mac = NULL; 419371947923SIoana Ciornei } 419471947923SIoana Ciornei 419534ff6846SIoana Radulescu static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 419634ff6846SIoana Radulescu { 419734ff6846SIoana Radulescu u32 status = ~0; 419834ff6846SIoana Radulescu struct device *dev = (struct device *)arg; 419934ff6846SIoana Radulescu struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 420034ff6846SIoana Radulescu struct net_device *net_dev = dev_get_drvdata(dev); 420171947923SIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 420234ff6846SIoana Radulescu int err; 420334ff6846SIoana Radulescu 420434ff6846SIoana Radulescu err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 420534ff6846SIoana Radulescu DPNI_IRQ_INDEX, &status); 420634ff6846SIoana Radulescu if (unlikely(err)) { 420734ff6846SIoana Radulescu netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 420834ff6846SIoana Radulescu return IRQ_HANDLED; 420934ff6846SIoana Radulescu } 421034ff6846SIoana Radulescu 421134ff6846SIoana Radulescu if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 42125d8dccf8SIoana Ciornei dpaa2_eth_link_state_update(netdev_priv(net_dev)); 421334ff6846SIoana Radulescu 4214f5c3fffaSIoana Ciornei if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) { 42155d8dccf8SIoana Ciornei dpaa2_eth_set_mac_addr(netdev_priv(net_dev)); 42165d8dccf8SIoana Ciornei dpaa2_eth_update_tx_fqids(priv); 421771947923SIoana Ciornei 421871947923SIoana Ciornei rtnl_lock(); 4219d87e6063SIoana Ciornei if (dpaa2_eth_has_mac(priv)) 422071947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 422171947923SIoana Ciornei else 422271947923SIoana Ciornei dpaa2_eth_connect_mac(priv); 422371947923SIoana Ciornei rtnl_unlock(); 4224f5c3fffaSIoana Ciornei } 42258398b375SFlorin Chiculita 422634ff6846SIoana Radulescu return IRQ_HANDLED; 422734ff6846SIoana Radulescu } 422834ff6846SIoana Radulescu 42295d8dccf8SIoana Ciornei static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) 423034ff6846SIoana Radulescu { 423134ff6846SIoana Radulescu int err = 0; 423234ff6846SIoana Radulescu struct fsl_mc_device_irq *irq; 423334ff6846SIoana Radulescu 423434ff6846SIoana Radulescu err = fsl_mc_allocate_irqs(ls_dev); 423534ff6846SIoana Radulescu if (err) { 423634ff6846SIoana Radulescu dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 423734ff6846SIoana Radulescu return err; 423834ff6846SIoana Radulescu } 423934ff6846SIoana Radulescu 424034ff6846SIoana Radulescu irq = ls_dev->irqs[0]; 424134ff6846SIoana Radulescu err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, 424234ff6846SIoana Radulescu NULL, dpni_irq0_handler_thread, 424334ff6846SIoana Radulescu IRQF_NO_SUSPEND | IRQF_ONESHOT, 424434ff6846SIoana Radulescu dev_name(&ls_dev->dev), &ls_dev->dev); 424534ff6846SIoana Radulescu if (err < 0) { 424634ff6846SIoana Radulescu dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 424734ff6846SIoana Radulescu goto free_mc_irq; 424834ff6846SIoana Radulescu } 424934ff6846SIoana Radulescu 425034ff6846SIoana Radulescu err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 42518398b375SFlorin Chiculita DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 42528398b375SFlorin Chiculita DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 425334ff6846SIoana Radulescu if (err < 0) { 425434ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 425534ff6846SIoana Radulescu goto free_irq; 425634ff6846SIoana Radulescu } 425734ff6846SIoana Radulescu 425834ff6846SIoana Radulescu err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 425934ff6846SIoana Radulescu DPNI_IRQ_INDEX, 1); 426034ff6846SIoana Radulescu if (err < 0) { 426134ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 426234ff6846SIoana Radulescu goto free_irq; 426334ff6846SIoana Radulescu } 426434ff6846SIoana Radulescu 426534ff6846SIoana Radulescu return 0; 426634ff6846SIoana Radulescu 426734ff6846SIoana Radulescu free_irq: 426834ff6846SIoana Radulescu devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); 426934ff6846SIoana Radulescu free_mc_irq: 427034ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 427134ff6846SIoana Radulescu 427234ff6846SIoana Radulescu return err; 427334ff6846SIoana Radulescu } 427434ff6846SIoana Radulescu 42755d8dccf8SIoana Ciornei static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv) 427634ff6846SIoana Radulescu { 427734ff6846SIoana Radulescu int i; 427834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 427934ff6846SIoana Radulescu 428034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 428134ff6846SIoana Radulescu ch = priv->channel[i]; 428234ff6846SIoana Radulescu /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 428334ff6846SIoana Radulescu netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 428434ff6846SIoana Radulescu NAPI_POLL_WEIGHT); 428534ff6846SIoana Radulescu } 428634ff6846SIoana Radulescu } 428734ff6846SIoana Radulescu 42885d8dccf8SIoana Ciornei static void dpaa2_eth_del_ch_napi(struct dpaa2_eth_priv *priv) 428934ff6846SIoana Radulescu { 429034ff6846SIoana Radulescu int i; 429134ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 429234ff6846SIoana Radulescu 429334ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 429434ff6846SIoana Radulescu ch = priv->channel[i]; 429534ff6846SIoana Radulescu netif_napi_del(&ch->napi); 429634ff6846SIoana Radulescu } 429734ff6846SIoana Radulescu } 429834ff6846SIoana Radulescu 429934ff6846SIoana Radulescu static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 430034ff6846SIoana Radulescu { 430134ff6846SIoana Radulescu struct device *dev; 430234ff6846SIoana Radulescu struct net_device *net_dev = NULL; 430334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = NULL; 430434ff6846SIoana Radulescu int err = 0; 430534ff6846SIoana Radulescu 430634ff6846SIoana Radulescu dev = &dpni_dev->dev; 430734ff6846SIoana Radulescu 430834ff6846SIoana Radulescu /* Net device */ 4309ab1e6de2SIoana Radulescu net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 431034ff6846SIoana Radulescu if (!net_dev) { 431134ff6846SIoana Radulescu dev_err(dev, "alloc_etherdev_mq() failed\n"); 431234ff6846SIoana Radulescu return -ENOMEM; 431334ff6846SIoana Radulescu } 431434ff6846SIoana Radulescu 431534ff6846SIoana Radulescu SET_NETDEV_DEV(net_dev, dev); 431634ff6846SIoana Radulescu dev_set_drvdata(dev, net_dev); 431734ff6846SIoana Radulescu 431834ff6846SIoana Radulescu priv = netdev_priv(net_dev); 431934ff6846SIoana Radulescu priv->net_dev = net_dev; 432034ff6846SIoana Radulescu 432134ff6846SIoana Radulescu priv->iommu_domain = iommu_get_domain_for_dev(dev); 432234ff6846SIoana Radulescu 43231cf773bdSYangbo Lu priv->tx_tstamp_type = HWTSTAMP_TX_OFF; 43241cf773bdSYangbo Lu priv->rx_tstamp = false; 43251cf773bdSYangbo Lu 4326c5521189SYangbo Lu priv->dpaa2_ptp_wq = alloc_workqueue("dpaa2_ptp_wq", 0, 0); 4327c5521189SYangbo Lu if (!priv->dpaa2_ptp_wq) { 4328c5521189SYangbo Lu err = -ENOMEM; 4329c5521189SYangbo Lu goto err_wq_alloc; 4330c5521189SYangbo Lu } 4331c5521189SYangbo Lu 4332c5521189SYangbo Lu INIT_WORK(&priv->tx_onestep_tstamp, dpaa2_eth_tx_onestep_tstamp); 4333c5521189SYangbo Lu 4334c5521189SYangbo Lu skb_queue_head_init(&priv->tx_skbs); 4335c5521189SYangbo Lu 433634ff6846SIoana Radulescu /* Obtain a MC portal */ 433734ff6846SIoana Radulescu err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 433834ff6846SIoana Radulescu &priv->mc_io); 433934ff6846SIoana Radulescu if (err) { 434034ff6846SIoana Radulescu if (err == -ENXIO) 434134ff6846SIoana Radulescu err = -EPROBE_DEFER; 434234ff6846SIoana Radulescu else 434334ff6846SIoana Radulescu dev_err(dev, "MC portal allocation failed\n"); 434434ff6846SIoana Radulescu goto err_portal_alloc; 434534ff6846SIoana Radulescu } 434634ff6846SIoana Radulescu 434734ff6846SIoana Radulescu /* MC objects initialization and configuration */ 43485d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpni(dpni_dev); 434934ff6846SIoana Radulescu if (err) 435034ff6846SIoana Radulescu goto err_dpni_setup; 435134ff6846SIoana Radulescu 43525d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpio(priv); 435334ff6846SIoana Radulescu if (err) 435434ff6846SIoana Radulescu goto err_dpio_setup; 435534ff6846SIoana Radulescu 43565d8dccf8SIoana Ciornei dpaa2_eth_setup_fqs(priv); 435734ff6846SIoana Radulescu 43585d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpbp(priv); 435934ff6846SIoana Radulescu if (err) 436034ff6846SIoana Radulescu goto err_dpbp_setup; 436134ff6846SIoana Radulescu 43625d8dccf8SIoana Ciornei err = dpaa2_eth_bind_dpni(priv); 436334ff6846SIoana Radulescu if (err) 436434ff6846SIoana Radulescu goto err_bind; 436534ff6846SIoana Radulescu 436634ff6846SIoana Radulescu /* Add a NAPI context for each channel */ 43675d8dccf8SIoana Ciornei dpaa2_eth_add_ch_napi(priv); 436834ff6846SIoana Radulescu 436934ff6846SIoana Radulescu /* Percpu statistics */ 437034ff6846SIoana Radulescu priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 437134ff6846SIoana Radulescu if (!priv->percpu_stats) { 437234ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 437334ff6846SIoana Radulescu err = -ENOMEM; 437434ff6846SIoana Radulescu goto err_alloc_percpu_stats; 437534ff6846SIoana Radulescu } 437634ff6846SIoana Radulescu priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 437734ff6846SIoana Radulescu if (!priv->percpu_extras) { 437834ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 437934ff6846SIoana Radulescu err = -ENOMEM; 438034ff6846SIoana Radulescu goto err_alloc_percpu_extras; 438134ff6846SIoana Radulescu } 438234ff6846SIoana Radulescu 4383d70446eeSIoana Ciornei priv->sgt_cache = alloc_percpu(*priv->sgt_cache); 4384d70446eeSIoana Ciornei if (!priv->sgt_cache) { 4385d70446eeSIoana Ciornei dev_err(dev, "alloc_percpu(sgt_cache) failed\n"); 4386d70446eeSIoana Ciornei err = -ENOMEM; 4387d70446eeSIoana Ciornei goto err_alloc_sgt_cache; 4388d70446eeSIoana Ciornei } 4389d70446eeSIoana Ciornei 43905d8dccf8SIoana Ciornei err = dpaa2_eth_netdev_init(net_dev); 439134ff6846SIoana Radulescu if (err) 439234ff6846SIoana Radulescu goto err_netdev_init; 439334ff6846SIoana Radulescu 439434ff6846SIoana Radulescu /* Configure checksum offload based on current interface flags */ 43955d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 439634ff6846SIoana Radulescu if (err) 439734ff6846SIoana Radulescu goto err_csum; 439834ff6846SIoana Radulescu 43995d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, 44005d8dccf8SIoana Ciornei !!(net_dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 440134ff6846SIoana Radulescu if (err) 440234ff6846SIoana Radulescu goto err_csum; 440334ff6846SIoana Radulescu 44045d8dccf8SIoana Ciornei err = dpaa2_eth_alloc_rings(priv); 440534ff6846SIoana Radulescu if (err) 440634ff6846SIoana Radulescu goto err_alloc_rings; 440734ff6846SIoana Radulescu 4408f395b69fSIoana Ciornei #ifdef CONFIG_FSL_DPAA2_ETH_DCB 4409f395b69fSIoana Ciornei if (dpaa2_eth_has_pause_support(priv) && priv->vlan_cls_enabled) { 4410f395b69fSIoana Ciornei priv->dcbx_mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; 4411f395b69fSIoana Ciornei net_dev->dcbnl_ops = &dpaa2_eth_dcbnl_ops; 4412f395b69fSIoana Ciornei } else { 4413f395b69fSIoana Ciornei dev_dbg(dev, "PFC not supported\n"); 4414f395b69fSIoana Ciornei } 4415f395b69fSIoana Ciornei #endif 4416f395b69fSIoana Ciornei 44175d8dccf8SIoana Ciornei err = dpaa2_eth_setup_irqs(dpni_dev); 441834ff6846SIoana Radulescu if (err) { 441934ff6846SIoana Radulescu netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 44205d8dccf8SIoana Ciornei priv->poll_thread = kthread_run(dpaa2_eth_poll_link_state, priv, 442134ff6846SIoana Radulescu "%s_poll_link", net_dev->name); 442234ff6846SIoana Radulescu if (IS_ERR(priv->poll_thread)) { 442334ff6846SIoana Radulescu dev_err(dev, "Error starting polling thread\n"); 442434ff6846SIoana Radulescu goto err_poll_thread; 442534ff6846SIoana Radulescu } 442634ff6846SIoana Radulescu priv->do_link_poll = true; 442734ff6846SIoana Radulescu } 442834ff6846SIoana Radulescu 442971947923SIoana Ciornei err = dpaa2_eth_connect_mac(priv); 443071947923SIoana Ciornei if (err) 443171947923SIoana Ciornei goto err_connect_mac; 443271947923SIoana Ciornei 4433ceeb03adSIoana Ciornei err = dpaa2_eth_dl_register(priv); 4434ceeb03adSIoana Ciornei if (err) 4435ceeb03adSIoana Ciornei goto err_dl_register; 4436ceeb03adSIoana Ciornei 4437061d631fSIoana Ciornei err = dpaa2_eth_dl_traps_register(priv); 4438061d631fSIoana Ciornei if (err) 4439061d631fSIoana Ciornei goto err_dl_trap_register; 4440061d631fSIoana Ciornei 4441ceeb03adSIoana Ciornei err = dpaa2_eth_dl_port_add(priv); 4442ceeb03adSIoana Ciornei if (err) 4443ceeb03adSIoana Ciornei goto err_dl_port_add; 4444ceeb03adSIoana Ciornei 444534ff6846SIoana Radulescu err = register_netdev(net_dev); 444634ff6846SIoana Radulescu if (err < 0) { 444734ff6846SIoana Radulescu dev_err(dev, "register_netdev() failed\n"); 444834ff6846SIoana Radulescu goto err_netdev_reg; 444934ff6846SIoana Radulescu } 445034ff6846SIoana Radulescu 4451091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4452091a19eaSIoana Radulescu dpaa2_dbg_add(priv); 4453091a19eaSIoana Radulescu #endif 4454091a19eaSIoana Radulescu 445534ff6846SIoana Radulescu dev_info(dev, "Probed interface %s\n", net_dev->name); 445634ff6846SIoana Radulescu return 0; 445734ff6846SIoana Radulescu 445834ff6846SIoana Radulescu err_netdev_reg: 4459ceeb03adSIoana Ciornei dpaa2_eth_dl_port_del(priv); 4460ceeb03adSIoana Ciornei err_dl_port_add: 4461061d631fSIoana Ciornei dpaa2_eth_dl_traps_unregister(priv); 4462061d631fSIoana Ciornei err_dl_trap_register: 4463ceeb03adSIoana Ciornei dpaa2_eth_dl_unregister(priv); 4464ceeb03adSIoana Ciornei err_dl_register: 446571947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 446671947923SIoana Ciornei err_connect_mac: 446734ff6846SIoana Radulescu if (priv->do_link_poll) 446834ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 446934ff6846SIoana Radulescu else 447034ff6846SIoana Radulescu fsl_mc_free_irqs(dpni_dev); 447134ff6846SIoana Radulescu err_poll_thread: 44725d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 447334ff6846SIoana Radulescu err_alloc_rings: 447434ff6846SIoana Radulescu err_csum: 447534ff6846SIoana Radulescu err_netdev_init: 4476d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 4477d70446eeSIoana Ciornei err_alloc_sgt_cache: 447834ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 447934ff6846SIoana Radulescu err_alloc_percpu_extras: 448034ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 448134ff6846SIoana Radulescu err_alloc_percpu_stats: 44825d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 448334ff6846SIoana Radulescu err_bind: 44845d8dccf8SIoana Ciornei dpaa2_eth_free_dpbp(priv); 448534ff6846SIoana Radulescu err_dpbp_setup: 44865d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 448734ff6846SIoana Radulescu err_dpio_setup: 44885d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 448934ff6846SIoana Radulescu err_dpni_setup: 449034ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 449134ff6846SIoana Radulescu err_portal_alloc: 4492c5521189SYangbo Lu destroy_workqueue(priv->dpaa2_ptp_wq); 4493c5521189SYangbo Lu err_wq_alloc: 449434ff6846SIoana Radulescu dev_set_drvdata(dev, NULL); 449534ff6846SIoana Radulescu free_netdev(net_dev); 449634ff6846SIoana Radulescu 449734ff6846SIoana Radulescu return err; 449834ff6846SIoana Radulescu } 449934ff6846SIoana Radulescu 450034ff6846SIoana Radulescu static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 450134ff6846SIoana Radulescu { 450234ff6846SIoana Radulescu struct device *dev; 450334ff6846SIoana Radulescu struct net_device *net_dev; 450434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 450534ff6846SIoana Radulescu 450634ff6846SIoana Radulescu dev = &ls_dev->dev; 450734ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 450834ff6846SIoana Radulescu priv = netdev_priv(net_dev); 450934ff6846SIoana Radulescu 4510091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4511091a19eaSIoana Radulescu dpaa2_dbg_remove(priv); 4512091a19eaSIoana Radulescu #endif 451371947923SIoana Ciornei rtnl_lock(); 451471947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 451571947923SIoana Ciornei rtnl_unlock(); 451671947923SIoana Ciornei 451734ff6846SIoana Radulescu unregister_netdev(net_dev); 451834ff6846SIoana Radulescu 4519ceeb03adSIoana Ciornei dpaa2_eth_dl_port_del(priv); 4520061d631fSIoana Ciornei dpaa2_eth_dl_traps_unregister(priv); 4521ceeb03adSIoana Ciornei dpaa2_eth_dl_unregister(priv); 4522ceeb03adSIoana Ciornei 452334ff6846SIoana Radulescu if (priv->do_link_poll) 452434ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 452534ff6846SIoana Radulescu else 452634ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 452734ff6846SIoana Radulescu 45285d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 4529d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 453034ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 453134ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 453234ff6846SIoana Radulescu 45335d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 45345d8dccf8SIoana Ciornei dpaa2_eth_free_dpbp(priv); 45355d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 45365d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 453734ff6846SIoana Radulescu 453834ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 453934ff6846SIoana Radulescu 454034ff6846SIoana Radulescu free_netdev(net_dev); 454134ff6846SIoana Radulescu 454234ff6846SIoana Radulescu dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 454334ff6846SIoana Radulescu 454434ff6846SIoana Radulescu return 0; 454534ff6846SIoana Radulescu } 454634ff6846SIoana Radulescu 454734ff6846SIoana Radulescu static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 454834ff6846SIoana Radulescu { 454934ff6846SIoana Radulescu .vendor = FSL_MC_VENDOR_FREESCALE, 455034ff6846SIoana Radulescu .obj_type = "dpni", 455134ff6846SIoana Radulescu }, 455234ff6846SIoana Radulescu { .vendor = 0x0 } 455334ff6846SIoana Radulescu }; 455434ff6846SIoana Radulescu MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 455534ff6846SIoana Radulescu 455634ff6846SIoana Radulescu static struct fsl_mc_driver dpaa2_eth_driver = { 455734ff6846SIoana Radulescu .driver = { 455834ff6846SIoana Radulescu .name = KBUILD_MODNAME, 455934ff6846SIoana Radulescu .owner = THIS_MODULE, 456034ff6846SIoana Radulescu }, 456134ff6846SIoana Radulescu .probe = dpaa2_eth_probe, 456234ff6846SIoana Radulescu .remove = dpaa2_eth_remove, 456334ff6846SIoana Radulescu .match_id_table = dpaa2_eth_match_id_table 456434ff6846SIoana Radulescu }; 456534ff6846SIoana Radulescu 4566091a19eaSIoana Radulescu static int __init dpaa2_eth_driver_init(void) 4567091a19eaSIoana Radulescu { 4568091a19eaSIoana Radulescu int err; 4569091a19eaSIoana Radulescu 4570091a19eaSIoana Radulescu dpaa2_eth_dbg_init(); 4571091a19eaSIoana Radulescu err = fsl_mc_driver_register(&dpaa2_eth_driver); 4572091a19eaSIoana Radulescu if (err) { 4573091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4574091a19eaSIoana Radulescu return err; 4575091a19eaSIoana Radulescu } 4576091a19eaSIoana Radulescu 4577091a19eaSIoana Radulescu return 0; 4578091a19eaSIoana Radulescu } 4579091a19eaSIoana Radulescu 4580091a19eaSIoana Radulescu static void __exit dpaa2_eth_driver_exit(void) 4581091a19eaSIoana Radulescu { 4582091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4583091a19eaSIoana Radulescu fsl_mc_driver_unregister(&dpaa2_eth_driver); 4584091a19eaSIoana Radulescu } 4585091a19eaSIoana Radulescu 4586091a19eaSIoana Radulescu module_init(dpaa2_eth_driver_init); 4587091a19eaSIoana Radulescu module_exit(dpaa2_eth_driver_exit); 4588