134ff6846SIoana Radulescu // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 234ff6846SIoana Radulescu /* Copyright 2014-2016 Freescale Semiconductor Inc. 3095174daSRobert-Ionut Alexa * Copyright 2016-2022 NXP 434ff6846SIoana Radulescu */ 534ff6846SIoana Radulescu #include <linux/init.h> 634ff6846SIoana Radulescu #include <linux/module.h> 734ff6846SIoana Radulescu #include <linux/platform_device.h> 834ff6846SIoana Radulescu #include <linux/etherdevice.h> 934ff6846SIoana Radulescu #include <linux/of_net.h> 1034ff6846SIoana Radulescu #include <linux/interrupt.h> 1134ff6846SIoana Radulescu #include <linux/msi.h> 1234ff6846SIoana Radulescu #include <linux/kthread.h> 1334ff6846SIoana Radulescu #include <linux/iommu.h> 1434ff6846SIoana Radulescu #include <linux/fsl/mc.h> 157e273a8eSIoana Ciocoi Radulescu #include <linux/bpf.h> 167e273a8eSIoana Ciocoi Radulescu #include <linux/bpf_trace.h> 17d21c784cSYangbo Lu #include <linux/fsl/ptp_qoriq.h> 18c5521189SYangbo Lu #include <linux/ptp_classify.h> 193657cdafSIoana Ciornei #include <net/pkt_cls.h> 2034ff6846SIoana Radulescu #include <net/sock.h> 213dc709e0SIoana Ciornei #include <net/tso.h> 2234ff6846SIoana Radulescu 2334ff6846SIoana Radulescu #include "dpaa2-eth.h" 2434ff6846SIoana Radulescu 2534ff6846SIoana Radulescu /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 2634ff6846SIoana Radulescu * using trace events only need to #include <trace/events/sched.h> 2734ff6846SIoana Radulescu */ 2834ff6846SIoana Radulescu #define CREATE_TRACE_POINTS 2934ff6846SIoana Radulescu #include "dpaa2-eth-trace.h" 3034ff6846SIoana Radulescu 3134ff6846SIoana Radulescu MODULE_LICENSE("Dual BSD/GPL"); 3234ff6846SIoana Radulescu MODULE_AUTHOR("Freescale Semiconductor, Inc"); 3334ff6846SIoana Radulescu MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 3434ff6846SIoana Radulescu 35d21c784cSYangbo Lu struct ptp_qoriq *dpaa2_ptp; 36d21c784cSYangbo Lu EXPORT_SYMBOL(dpaa2_ptp); 37d21c784cSYangbo Lu 38c4680c97SRadu Bulie static void dpaa2_eth_detect_features(struct dpaa2_eth_priv *priv) 39c4680c97SRadu Bulie { 40c4680c97SRadu Bulie priv->features = 0; 41c4680c97SRadu Bulie 42c4680c97SRadu Bulie if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_PTP_ONESTEP_VER_MAJOR, 43c4680c97SRadu Bulie DPNI_PTP_ONESTEP_VER_MINOR) >= 0) 44c4680c97SRadu Bulie priv->features |= DPAA2_ETH_FEATURE_ONESTEP_CFG_DIRECT; 45c4680c97SRadu Bulie } 46c4680c97SRadu Bulie 47c4680c97SRadu Bulie static void dpaa2_update_ptp_onestep_indirect(struct dpaa2_eth_priv *priv, 48c4680c97SRadu Bulie u32 offset, u8 udp) 49c4680c97SRadu Bulie { 50c4680c97SRadu Bulie struct dpni_single_step_cfg cfg; 51c4680c97SRadu Bulie 52c4680c97SRadu Bulie cfg.en = 1; 53c4680c97SRadu Bulie cfg.ch_update = udp; 54c4680c97SRadu Bulie cfg.offset = offset; 55c4680c97SRadu Bulie cfg.peer_delay = 0; 56c4680c97SRadu Bulie 57c4680c97SRadu Bulie if (dpni_set_single_step_cfg(priv->mc_io, 0, priv->mc_token, &cfg)) 58c4680c97SRadu Bulie WARN_ONCE(1, "Failed to set single step register"); 59c4680c97SRadu Bulie } 60c4680c97SRadu Bulie 61c4680c97SRadu Bulie static void dpaa2_update_ptp_onestep_direct(struct dpaa2_eth_priv *priv, 62c4680c97SRadu Bulie u32 offset, u8 udp) 63c4680c97SRadu Bulie { 64c4680c97SRadu Bulie u32 val = 0; 65c4680c97SRadu Bulie 66c4680c97SRadu Bulie val = DPAA2_PTP_SINGLE_STEP_ENABLE | 67c4680c97SRadu Bulie DPAA2_PTP_SINGLE_CORRECTION_OFF(offset); 68c4680c97SRadu Bulie 69c4680c97SRadu Bulie if (udp) 70c4680c97SRadu Bulie val |= DPAA2_PTP_SINGLE_STEP_CH; 71c4680c97SRadu Bulie 72c4680c97SRadu Bulie if (priv->onestep_reg_base) 73c4680c97SRadu Bulie writel(val, priv->onestep_reg_base); 74c4680c97SRadu Bulie } 75c4680c97SRadu Bulie 76c4680c97SRadu Bulie static void dpaa2_ptp_onestep_reg_update_method(struct dpaa2_eth_priv *priv) 77c4680c97SRadu Bulie { 78c4680c97SRadu Bulie struct device *dev = priv->net_dev->dev.parent; 79c4680c97SRadu Bulie struct dpni_single_step_cfg ptp_cfg; 80c4680c97SRadu Bulie 81c4680c97SRadu Bulie priv->dpaa2_set_onestep_params_cb = dpaa2_update_ptp_onestep_indirect; 82c4680c97SRadu Bulie 83c4680c97SRadu Bulie if (!(priv->features & DPAA2_ETH_FEATURE_ONESTEP_CFG_DIRECT)) 84c4680c97SRadu Bulie return; 85c4680c97SRadu Bulie 86c4680c97SRadu Bulie if (dpni_get_single_step_cfg(priv->mc_io, 0, 87c4680c97SRadu Bulie priv->mc_token, &ptp_cfg)) { 88c4680c97SRadu Bulie dev_err(dev, "dpni_get_single_step_cfg cannot retrieve onestep reg, falling back to indirect update\n"); 89c4680c97SRadu Bulie return; 90c4680c97SRadu Bulie } 91c4680c97SRadu Bulie 92c4680c97SRadu Bulie if (!ptp_cfg.ptp_onestep_reg_base) { 93c4680c97SRadu Bulie dev_err(dev, "1588 onestep reg not available, falling back to indirect update\n"); 94c4680c97SRadu Bulie return; 95c4680c97SRadu Bulie } 96c4680c97SRadu Bulie 97c4680c97SRadu Bulie priv->onestep_reg_base = ioremap(ptp_cfg.ptp_onestep_reg_base, 98c4680c97SRadu Bulie sizeof(u32)); 99c4680c97SRadu Bulie if (!priv->onestep_reg_base) { 100c4680c97SRadu Bulie dev_err(dev, "1588 onestep reg cannot be mapped, falling back to indirect update\n"); 101c4680c97SRadu Bulie return; 102c4680c97SRadu Bulie } 103c4680c97SRadu Bulie 104c4680c97SRadu Bulie priv->dpaa2_set_onestep_params_cb = dpaa2_update_ptp_onestep_direct; 105c4680c97SRadu Bulie } 106c4680c97SRadu Bulie 10734ff6846SIoana Radulescu static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 10834ff6846SIoana Radulescu dma_addr_t iova_addr) 10934ff6846SIoana Radulescu { 11034ff6846SIoana Radulescu phys_addr_t phys_addr; 11134ff6846SIoana Radulescu 11234ff6846SIoana Radulescu phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 11334ff6846SIoana Radulescu 11434ff6846SIoana Radulescu return phys_to_virt(phys_addr); 11534ff6846SIoana Radulescu } 11634ff6846SIoana Radulescu 1175d8dccf8SIoana Ciornei static void dpaa2_eth_validate_rx_csum(struct dpaa2_eth_priv *priv, 11834ff6846SIoana Radulescu u32 fd_status, 11934ff6846SIoana Radulescu struct sk_buff *skb) 12034ff6846SIoana Radulescu { 12134ff6846SIoana Radulescu skb_checksum_none_assert(skb); 12234ff6846SIoana Radulescu 12334ff6846SIoana Radulescu /* HW checksum validation is disabled, nothing to do here */ 12434ff6846SIoana Radulescu if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 12534ff6846SIoana Radulescu return; 12634ff6846SIoana Radulescu 12734ff6846SIoana Radulescu /* Read checksum validation bits */ 12834ff6846SIoana Radulescu if (!((fd_status & DPAA2_FAS_L3CV) && 12934ff6846SIoana Radulescu (fd_status & DPAA2_FAS_L4CV))) 13034ff6846SIoana Radulescu return; 13134ff6846SIoana Radulescu 13234ff6846SIoana Radulescu /* Inform the stack there's no need to compute L3/L4 csum anymore */ 13334ff6846SIoana Radulescu skb->ip_summed = CHECKSUM_UNNECESSARY; 13434ff6846SIoana Radulescu } 13534ff6846SIoana Radulescu 13634ff6846SIoana Radulescu /* Free a received FD. 13734ff6846SIoana Radulescu * Not to be used for Tx conf FDs or on any other paths. 13834ff6846SIoana Radulescu */ 1395d8dccf8SIoana Ciornei static void dpaa2_eth_free_rx_fd(struct dpaa2_eth_priv *priv, 14034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 14134ff6846SIoana Radulescu void *vaddr) 14234ff6846SIoana Radulescu { 14334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 14434ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 14534ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 14634ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 14734ff6846SIoana Radulescu void *sg_vaddr; 14834ff6846SIoana Radulescu int i; 14934ff6846SIoana Radulescu 15034ff6846SIoana Radulescu /* If single buffer frame, just free the data buffer */ 15134ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) 15234ff6846SIoana Radulescu goto free_buf; 15334ff6846SIoana Radulescu else if (fd_format != dpaa2_fd_sg) 15434ff6846SIoana Radulescu /* We don't support any other format */ 15534ff6846SIoana Radulescu return; 15634ff6846SIoana Radulescu 15734ff6846SIoana Radulescu /* For S/G frames, we first need to free all SG entries 15834ff6846SIoana Radulescu * except the first one, which was taken care of already 15934ff6846SIoana Radulescu */ 16034ff6846SIoana Radulescu sgt = vaddr + dpaa2_fd_get_offset(fd); 16134ff6846SIoana Radulescu for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 16234ff6846SIoana Radulescu addr = dpaa2_sg_get_addr(&sgt[i]); 16334ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 164efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 16518c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 16634ff6846SIoana Radulescu 16727c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 16834ff6846SIoana Radulescu if (dpaa2_sg_is_final(&sgt[i])) 16934ff6846SIoana Radulescu break; 17034ff6846SIoana Radulescu } 17134ff6846SIoana Radulescu 17234ff6846SIoana Radulescu free_buf: 17327c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 17434ff6846SIoana Radulescu } 17534ff6846SIoana Radulescu 17634ff6846SIoana Radulescu /* Build a linear skb based on a single-buffer frame descriptor */ 1775d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_linear_skb(struct dpaa2_eth_channel *ch, 17834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 17934ff6846SIoana Radulescu void *fd_vaddr) 18034ff6846SIoana Radulescu { 18134ff6846SIoana Radulescu struct sk_buff *skb = NULL; 18234ff6846SIoana Radulescu u16 fd_offset = dpaa2_fd_get_offset(fd); 18334ff6846SIoana Radulescu u32 fd_length = dpaa2_fd_get_len(fd); 18434ff6846SIoana Radulescu 18534ff6846SIoana Radulescu ch->buf_count--; 18634ff6846SIoana Radulescu 18727c87486SIoana Ciocoi Radulescu skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 18834ff6846SIoana Radulescu if (unlikely(!skb)) 18934ff6846SIoana Radulescu return NULL; 19034ff6846SIoana Radulescu 19134ff6846SIoana Radulescu skb_reserve(skb, fd_offset); 19234ff6846SIoana Radulescu skb_put(skb, fd_length); 19334ff6846SIoana Radulescu 19434ff6846SIoana Radulescu return skb; 19534ff6846SIoana Radulescu } 19634ff6846SIoana Radulescu 19734ff6846SIoana Radulescu /* Build a non linear (fragmented) skb based on a S/G table */ 1985d8dccf8SIoana Ciornei static struct sk_buff *dpaa2_eth_build_frag_skb(struct dpaa2_eth_priv *priv, 19934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 20034ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt) 20134ff6846SIoana Radulescu { 20234ff6846SIoana Radulescu struct sk_buff *skb = NULL; 20334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 20434ff6846SIoana Radulescu void *sg_vaddr; 20534ff6846SIoana Radulescu dma_addr_t sg_addr; 20634ff6846SIoana Radulescu u16 sg_offset; 20734ff6846SIoana Radulescu u32 sg_length; 20834ff6846SIoana Radulescu struct page *page, *head_page; 20934ff6846SIoana Radulescu int page_offset; 21034ff6846SIoana Radulescu int i; 21134ff6846SIoana Radulescu 21234ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 21334ff6846SIoana Radulescu struct dpaa2_sg_entry *sge = &sgt[i]; 21434ff6846SIoana Radulescu 21534ff6846SIoana Radulescu /* NOTE: We only support SG entries in dpaa2_sg_single format, 21634ff6846SIoana Radulescu * but this is the only format we may receive from HW anyway 21734ff6846SIoana Radulescu */ 21834ff6846SIoana Radulescu 21934ff6846SIoana Radulescu /* Get the address and length from the S/G entry */ 22034ff6846SIoana Radulescu sg_addr = dpaa2_sg_get_addr(sge); 22134ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 222efa6a7d0SIoana Ciornei dma_unmap_page(dev, sg_addr, priv->rx_buf_size, 22318c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 22434ff6846SIoana Radulescu 22534ff6846SIoana Radulescu sg_length = dpaa2_sg_get_len(sge); 22634ff6846SIoana Radulescu 22734ff6846SIoana Radulescu if (i == 0) { 22834ff6846SIoana Radulescu /* We build the skb around the first data buffer */ 22927c87486SIoana Ciocoi Radulescu skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 23034ff6846SIoana Radulescu if (unlikely(!skb)) { 23134ff6846SIoana Radulescu /* Free the first SG entry now, since we already 23234ff6846SIoana Radulescu * unmapped it and obtained the virtual address 23334ff6846SIoana Radulescu */ 23427c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 23534ff6846SIoana Radulescu 23634ff6846SIoana Radulescu /* We still need to subtract the buffers used 23734ff6846SIoana Radulescu * by this FD from our software counter 23834ff6846SIoana Radulescu */ 23934ff6846SIoana Radulescu while (!dpaa2_sg_is_final(&sgt[i]) && 24034ff6846SIoana Radulescu i < DPAA2_ETH_MAX_SG_ENTRIES) 24134ff6846SIoana Radulescu i++; 24234ff6846SIoana Radulescu break; 24334ff6846SIoana Radulescu } 24434ff6846SIoana Radulescu 24534ff6846SIoana Radulescu sg_offset = dpaa2_sg_get_offset(sge); 24634ff6846SIoana Radulescu skb_reserve(skb, sg_offset); 24734ff6846SIoana Radulescu skb_put(skb, sg_length); 24834ff6846SIoana Radulescu } else { 24934ff6846SIoana Radulescu /* Rest of the data buffers are stored as skb frags */ 25034ff6846SIoana Radulescu page = virt_to_page(sg_vaddr); 25134ff6846SIoana Radulescu head_page = virt_to_head_page(sg_vaddr); 25234ff6846SIoana Radulescu 25334ff6846SIoana Radulescu /* Offset in page (which may be compound). 25434ff6846SIoana Radulescu * Data in subsequent SG entries is stored from the 25534ff6846SIoana Radulescu * beginning of the buffer, so we don't need to add the 25634ff6846SIoana Radulescu * sg_offset. 25734ff6846SIoana Radulescu */ 25834ff6846SIoana Radulescu page_offset = ((unsigned long)sg_vaddr & 25934ff6846SIoana Radulescu (PAGE_SIZE - 1)) + 26034ff6846SIoana Radulescu (page_address(page) - page_address(head_page)); 26134ff6846SIoana Radulescu 26234ff6846SIoana Radulescu skb_add_rx_frag(skb, i - 1, head_page, page_offset, 263efa6a7d0SIoana Ciornei sg_length, priv->rx_buf_size); 26434ff6846SIoana Radulescu } 26534ff6846SIoana Radulescu 26634ff6846SIoana Radulescu if (dpaa2_sg_is_final(sge)) 26734ff6846SIoana Radulescu break; 26834ff6846SIoana Radulescu } 26934ff6846SIoana Radulescu 27034ff6846SIoana Radulescu WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 27134ff6846SIoana Radulescu 27234ff6846SIoana Radulescu /* Count all data buffers + SG table buffer */ 27334ff6846SIoana Radulescu ch->buf_count -= i + 2; 27434ff6846SIoana Radulescu 27534ff6846SIoana Radulescu return skb; 27634ff6846SIoana Radulescu } 27734ff6846SIoana Radulescu 278569375fbSIoana Ciocoi Radulescu /* Free buffers acquired from the buffer pool or which were meant to 279569375fbSIoana Ciocoi Radulescu * be released in the pool 280569375fbSIoana Ciocoi Radulescu */ 2815d8dccf8SIoana Ciornei static void dpaa2_eth_free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, 2825d8dccf8SIoana Ciornei int count) 283569375fbSIoana Ciocoi Radulescu { 284569375fbSIoana Ciocoi Radulescu struct device *dev = priv->net_dev->dev.parent; 285569375fbSIoana Ciocoi Radulescu void *vaddr; 286569375fbSIoana Ciocoi Radulescu int i; 287569375fbSIoana Ciocoi Radulescu 288569375fbSIoana Ciocoi Radulescu for (i = 0; i < count; i++) { 289569375fbSIoana Ciocoi Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 290efa6a7d0SIoana Ciornei dma_unmap_page(dev, buf_array[i], priv->rx_buf_size, 29118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 29227c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 293569375fbSIoana Ciocoi Radulescu } 294569375fbSIoana Ciocoi Radulescu } 295569375fbSIoana Ciocoi Radulescu 29628d137ccSIoana Ciornei static void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv, 2975d39dc21SIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 2985d39dc21SIoana Ciocoi Radulescu dma_addr_t addr) 2995d39dc21SIoana Ciocoi Radulescu { 300ef17bd7cSIoana Radulescu int retries = 0; 3015d39dc21SIoana Ciocoi Radulescu int err; 3025d39dc21SIoana Ciocoi Radulescu 30328d137ccSIoana Ciornei ch->recycled_bufs[ch->recycled_bufs_cnt++] = addr; 30428d137ccSIoana Ciornei if (ch->recycled_bufs_cnt < DPAA2_ETH_BUFS_PER_CMD) 3055d39dc21SIoana Ciocoi Radulescu return; 3065d39dc21SIoana Ciocoi Radulescu 307095174daSRobert-Ionut Alexa while ((err = dpaa2_io_service_release(ch->dpio, ch->bp->bpid, 30828d137ccSIoana Ciornei ch->recycled_bufs, 30928d137ccSIoana Ciornei ch->recycled_bufs_cnt)) == -EBUSY) { 310ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 311ef17bd7cSIoana Radulescu break; 3125d39dc21SIoana Ciocoi Radulescu cpu_relax(); 313ef17bd7cSIoana Radulescu } 3145d39dc21SIoana Ciocoi Radulescu 3155d39dc21SIoana Ciocoi Radulescu if (err) { 31628d137ccSIoana Ciornei dpaa2_eth_free_bufs(priv, ch->recycled_bufs, ch->recycled_bufs_cnt); 31728d137ccSIoana Ciornei ch->buf_count -= ch->recycled_bufs_cnt; 3185d39dc21SIoana Ciocoi Radulescu } 3195d39dc21SIoana Ciocoi Radulescu 32028d137ccSIoana Ciornei ch->recycled_bufs_cnt = 0; 3215d39dc21SIoana Ciocoi Radulescu } 3225d39dc21SIoana Ciocoi Radulescu 32338c440b2SIoana Ciornei static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 32438c440b2SIoana Ciornei struct dpaa2_eth_fq *fq, 32538c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_fds) 32638c440b2SIoana Ciornei { 32738c440b2SIoana Ciornei int total_enqueued = 0, retries = 0, enqueued; 32838c440b2SIoana Ciornei struct dpaa2_eth_drv_stats *percpu_extras; 32938c440b2SIoana Ciornei int num_fds, err, max_retries; 33038c440b2SIoana Ciornei struct dpaa2_fd *fds; 33138c440b2SIoana Ciornei 33238c440b2SIoana Ciornei percpu_extras = this_cpu_ptr(priv->percpu_extras); 33338c440b2SIoana Ciornei 33438c440b2SIoana Ciornei /* try to enqueue all the FDs until the max number of retries is hit */ 33538c440b2SIoana Ciornei fds = xdp_fds->fds; 33638c440b2SIoana Ciornei num_fds = xdp_fds->num; 33738c440b2SIoana Ciornei max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 33838c440b2SIoana Ciornei while (total_enqueued < num_fds && retries < max_retries) { 33938c440b2SIoana Ciornei err = priv->enqueue(priv, fq, &fds[total_enqueued], 34038c440b2SIoana Ciornei 0, num_fds - total_enqueued, &enqueued); 34138c440b2SIoana Ciornei if (err == -EBUSY) { 34238c440b2SIoana Ciornei percpu_extras->tx_portal_busy += ++retries; 34338c440b2SIoana Ciornei continue; 34438c440b2SIoana Ciornei } 34538c440b2SIoana Ciornei total_enqueued += enqueued; 34638c440b2SIoana Ciornei } 34738c440b2SIoana Ciornei xdp_fds->num = 0; 34838c440b2SIoana Ciornei 34938c440b2SIoana Ciornei return total_enqueued; 35038c440b2SIoana Ciornei } 35138c440b2SIoana Ciornei 3525d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_tx_flush(struct dpaa2_eth_priv *priv, 35374a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 35474a1c059SIoana Ciornei struct dpaa2_eth_fq *fq) 35574a1c059SIoana Ciornei { 35674a1c059SIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 35774a1c059SIoana Ciornei struct dpaa2_fd *fds; 35874a1c059SIoana Ciornei int enqueued, i; 35974a1c059SIoana Ciornei 36074a1c059SIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 36174a1c059SIoana Ciornei 36274a1c059SIoana Ciornei // enqueue the array of XDP_TX frames 36374a1c059SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds); 36474a1c059SIoana Ciornei 36574a1c059SIoana Ciornei /* update statistics */ 36674a1c059SIoana Ciornei percpu_stats->tx_packets += enqueued; 36774a1c059SIoana Ciornei fds = fq->xdp_tx_fds.fds; 36874a1c059SIoana Ciornei for (i = 0; i < enqueued; i++) { 36974a1c059SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 37074a1c059SIoana Ciornei ch->stats.xdp_tx++; 37174a1c059SIoana Ciornei } 37274a1c059SIoana Ciornei for (i = enqueued; i < fq->xdp_tx_fds.num; i++) { 37328d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, dpaa2_fd_get_addr(&fds[i])); 37474a1c059SIoana Ciornei percpu_stats->tx_errors++; 37574a1c059SIoana Ciornei ch->stats.xdp_tx_err++; 37674a1c059SIoana Ciornei } 37774a1c059SIoana Ciornei fq->xdp_tx_fds.num = 0; 37874a1c059SIoana Ciornei } 37974a1c059SIoana Ciornei 3805d8dccf8SIoana Ciornei static void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv, 38174a1c059SIoana Ciornei struct dpaa2_eth_channel *ch, 38274a1c059SIoana Ciornei struct dpaa2_fd *fd, 38399e43521SIoana Ciocoi Radulescu void *buf_start, u16 queue_id) 38499e43521SIoana Ciocoi Radulescu { 38599e43521SIoana Ciocoi Radulescu struct dpaa2_faead *faead; 38674a1c059SIoana Ciornei struct dpaa2_fd *dest_fd; 38774a1c059SIoana Ciornei struct dpaa2_eth_fq *fq; 38899e43521SIoana Ciocoi Radulescu u32 ctrl, frc; 38999e43521SIoana Ciocoi Radulescu 39099e43521SIoana Ciocoi Radulescu /* Mark the egress frame hardware annotation area as valid */ 39199e43521SIoana Ciocoi Radulescu frc = dpaa2_fd_get_frc(fd); 39299e43521SIoana Ciocoi Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 39399e43521SIoana Ciocoi Radulescu dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 39499e43521SIoana Ciocoi Radulescu 39599e43521SIoana Ciocoi Radulescu /* Instruct hardware to release the FD buffer directly into 39699e43521SIoana Ciocoi Radulescu * the buffer pool once transmission is completed, instead of 39799e43521SIoana Ciocoi Radulescu * sending a Tx confirmation frame to us 39899e43521SIoana Ciocoi Radulescu */ 39999e43521SIoana Ciocoi Radulescu ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 40099e43521SIoana Ciocoi Radulescu faead = dpaa2_get_faead(buf_start, false); 40199e43521SIoana Ciocoi Radulescu faead->ctrl = cpu_to_le32(ctrl); 40299e43521SIoana Ciocoi Radulescu faead->conf_fqid = 0; 40399e43521SIoana Ciocoi Radulescu 40499e43521SIoana Ciocoi Radulescu fq = &priv->fq[queue_id]; 40574a1c059SIoana Ciornei dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++]; 40674a1c059SIoana Ciornei memcpy(dest_fd, fd, sizeof(*dest_fd)); 40799e43521SIoana Ciocoi Radulescu 40874a1c059SIoana Ciornei if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE) 40974a1c059SIoana Ciornei return; 41074a1c059SIoana Ciornei 4115d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, fq); 41299e43521SIoana Ciocoi Radulescu } 41399e43521SIoana Ciocoi Radulescu 4145d8dccf8SIoana Ciornei static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, 4157e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 41699e43521SIoana Ciocoi Radulescu struct dpaa2_eth_fq *rx_fq, 4177e273a8eSIoana Ciocoi Radulescu struct dpaa2_fd *fd, void *vaddr) 4187e273a8eSIoana Ciocoi Radulescu { 4195d39dc21SIoana Ciocoi Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 4207e273a8eSIoana Ciocoi Radulescu struct bpf_prog *xdp_prog; 4217e273a8eSIoana Ciocoi Radulescu struct xdp_buff xdp; 4227e273a8eSIoana Ciocoi Radulescu u32 xdp_act = XDP_PASS; 423be9df4afSLorenzo Bianconi int err, offset; 42499e43521SIoana Ciocoi Radulescu 4257e273a8eSIoana Ciocoi Radulescu xdp_prog = READ_ONCE(ch->xdp.prog); 4267e273a8eSIoana Ciocoi Radulescu if (!xdp_prog) 4277e273a8eSIoana Ciocoi Radulescu goto out; 4287e273a8eSIoana Ciocoi Radulescu 429be9df4afSLorenzo Bianconi offset = dpaa2_fd_get_offset(fd) - XDP_PACKET_HEADROOM; 430be9df4afSLorenzo Bianconi xdp_init_buff(&xdp, DPAA2_ETH_RX_BUF_RAW_SIZE - offset, &ch->xdp_rxq); 431be9df4afSLorenzo Bianconi xdp_prepare_buff(&xdp, vaddr + offset, XDP_PACKET_HEADROOM, 432be9df4afSLorenzo Bianconi dpaa2_fd_get_len(fd), false); 4334a9b052aSJesper Dangaard Brouer 4347e273a8eSIoana Ciocoi Radulescu xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 4357e273a8eSIoana Ciocoi Radulescu 4367b1eea1aSIoana Ciocoi Radulescu /* xdp.data pointer may have changed */ 4377b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_offset(fd, xdp.data - vaddr); 4387b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 4397b1eea1aSIoana Ciocoi Radulescu 4407e273a8eSIoana Ciocoi Radulescu switch (xdp_act) { 4417e273a8eSIoana Ciocoi Radulescu case XDP_PASS: 4427e273a8eSIoana Ciocoi Radulescu break; 44399e43521SIoana Ciocoi Radulescu case XDP_TX: 4445d8dccf8SIoana Ciornei dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); 44599e43521SIoana Ciocoi Radulescu break; 4467e273a8eSIoana Ciocoi Radulescu default: 447c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act); 448df561f66SGustavo A. R. Silva fallthrough; 4497e273a8eSIoana Ciocoi Radulescu case XDP_ABORTED: 4507e273a8eSIoana Ciocoi Radulescu trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 451df561f66SGustavo A. R. Silva fallthrough; 4527e273a8eSIoana Ciocoi Radulescu case XDP_DROP: 45328d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, addr); 454a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_drop++; 4557e273a8eSIoana Ciocoi Radulescu break; 456d678be1dSIoana Radulescu case XDP_REDIRECT: 457d678be1dSIoana Radulescu dma_unmap_page(priv->net_dev->dev.parent, addr, 458efa6a7d0SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 459d678be1dSIoana Radulescu ch->buf_count--; 4604a9b052aSJesper Dangaard Brouer 4614a9b052aSJesper Dangaard Brouer /* Allow redirect use of full headroom */ 462d678be1dSIoana Radulescu xdp.data_hard_start = vaddr; 4634a9b052aSJesper Dangaard Brouer xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE; 4644a9b052aSJesper Dangaard Brouer 465d678be1dSIoana Radulescu err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 466e12be913SIoana Ciornei if (unlikely(err)) { 467e12be913SIoana Ciornei addr = dma_map_page(priv->net_dev->dev.parent, 468e12be913SIoana Ciornei virt_to_page(vaddr), 0, 469e12be913SIoana Ciornei priv->rx_buf_size, DMA_BIDIRECTIONAL); 470e12be913SIoana Ciornei if (unlikely(dma_mapping_error(priv->net_dev->dev.parent, addr))) { 471e12be913SIoana Ciornei free_pages((unsigned long)vaddr, 0); 472e12be913SIoana Ciornei } else { 473e12be913SIoana Ciornei ch->buf_count++; 47428d137ccSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, addr); 475e12be913SIoana Ciornei } 476d678be1dSIoana Radulescu ch->stats.xdp_drop++; 477e12be913SIoana Ciornei } else { 478d678be1dSIoana Radulescu ch->stats.xdp_redirect++; 479e12be913SIoana Ciornei } 480d678be1dSIoana Radulescu break; 4817e273a8eSIoana Ciocoi Radulescu } 4827e273a8eSIoana Ciocoi Radulescu 483d678be1dSIoana Radulescu ch->xdp.res |= xdp_act; 4847e273a8eSIoana Ciocoi Radulescu out: 4857e273a8eSIoana Ciocoi Radulescu return xdp_act; 4867e273a8eSIoana Ciocoi Radulescu } 4877e273a8eSIoana Ciocoi Radulescu 48850f82699SIoana Ciornei static struct sk_buff *dpaa2_eth_copybreak(struct dpaa2_eth_channel *ch, 48950f82699SIoana Ciornei const struct dpaa2_fd *fd, 49050f82699SIoana Ciornei void *fd_vaddr) 49150f82699SIoana Ciornei { 49250f82699SIoana Ciornei u16 fd_offset = dpaa2_fd_get_offset(fd); 4938ed3cefcSIoana Ciornei struct dpaa2_eth_priv *priv = ch->priv; 49450f82699SIoana Ciornei u32 fd_length = dpaa2_fd_get_len(fd); 49550f82699SIoana Ciornei struct sk_buff *skb = NULL; 49650f82699SIoana Ciornei unsigned int skb_len; 49750f82699SIoana Ciornei 4988ed3cefcSIoana Ciornei if (fd_length > priv->rx_copybreak) 49950f82699SIoana Ciornei return NULL; 50050f82699SIoana Ciornei 50150f82699SIoana Ciornei skb_len = fd_length + dpaa2_eth_needed_headroom(NULL); 50250f82699SIoana Ciornei 50350f82699SIoana Ciornei skb = napi_alloc_skb(&ch->napi, skb_len); 50450f82699SIoana Ciornei if (!skb) 50550f82699SIoana Ciornei return NULL; 50650f82699SIoana Ciornei 50750f82699SIoana Ciornei skb_reserve(skb, dpaa2_eth_needed_headroom(NULL)); 50850f82699SIoana Ciornei skb_put(skb, fd_length); 50950f82699SIoana Ciornei 51050f82699SIoana Ciornei memcpy(skb->data, fd_vaddr + fd_offset, fd_length); 51150f82699SIoana Ciornei 5128ed3cefcSIoana Ciornei dpaa2_eth_recycle_buf(priv, ch, dpaa2_fd_get_addr(fd)); 51350f82699SIoana Ciornei 51450f82699SIoana Ciornei return skb; 51550f82699SIoana Ciornei } 51650f82699SIoana Ciornei 51734ff6846SIoana Radulescu /* Main Rx frame processing routine */ 51834ff6846SIoana Radulescu static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 51934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 52034ff6846SIoana Radulescu const struct dpaa2_fd *fd, 521dbcdf728SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 52234ff6846SIoana Radulescu { 52334ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 52434ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 52534ff6846SIoana Radulescu void *vaddr; 52634ff6846SIoana Radulescu struct sk_buff *skb; 52734ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 52834ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 52934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 53034ff6846SIoana Radulescu struct dpaa2_fas *fas; 53134ff6846SIoana Radulescu void *buf_data; 53234ff6846SIoana Radulescu u32 status = 0; 5337e273a8eSIoana Ciocoi Radulescu u32 xdp_act; 53434ff6846SIoana Radulescu 53534ff6846SIoana Radulescu /* Tracing point */ 53634ff6846SIoana Radulescu trace_dpaa2_rx_fd(priv->net_dev, fd); 53734ff6846SIoana Radulescu 53834ff6846SIoana Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 539efa6a7d0SIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 54018c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 54134ff6846SIoana Radulescu 54234ff6846SIoana Radulescu fas = dpaa2_get_fas(vaddr, false); 54334ff6846SIoana Radulescu prefetch(fas); 54434ff6846SIoana Radulescu buf_data = vaddr + dpaa2_fd_get_offset(fd); 54534ff6846SIoana Radulescu prefetch(buf_data); 54634ff6846SIoana Radulescu 54734ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 54834ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 54934ff6846SIoana Radulescu 55034ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 5515d8dccf8SIoana Ciornei xdp_act = dpaa2_eth_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 5527e273a8eSIoana Ciocoi Radulescu if (xdp_act != XDP_PASS) { 5537e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_packets++; 5547e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 5557e273a8eSIoana Ciocoi Radulescu return; 5567e273a8eSIoana Ciocoi Radulescu } 5577e273a8eSIoana Ciocoi Radulescu 55850f82699SIoana Ciornei skb = dpaa2_eth_copybreak(ch, fd, vaddr); 55950f82699SIoana Ciornei if (!skb) { 560efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 56118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 5625d8dccf8SIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 56350f82699SIoana Ciornei } 56434ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 5657e273a8eSIoana Ciocoi Radulescu WARN_ON(priv->xdp_prog); 5667e273a8eSIoana Ciocoi Radulescu 567efa6a7d0SIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 56818c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 5695d8dccf8SIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 57027c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 57134ff6846SIoana Radulescu percpu_extras->rx_sg_frames++; 57234ff6846SIoana Radulescu percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 57334ff6846SIoana Radulescu } else { 57434ff6846SIoana Radulescu /* We don't support any other format */ 57534ff6846SIoana Radulescu goto err_frame_format; 57634ff6846SIoana Radulescu } 57734ff6846SIoana Radulescu 57834ff6846SIoana Radulescu if (unlikely(!skb)) 57934ff6846SIoana Radulescu goto err_build_skb; 58034ff6846SIoana Radulescu 58134ff6846SIoana Radulescu prefetch(skb->data); 58234ff6846SIoana Radulescu 58334ff6846SIoana Radulescu /* Get the timestamp value */ 58434ff6846SIoana Radulescu if (priv->rx_tstamp) { 58534ff6846SIoana Radulescu struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 58634ff6846SIoana Radulescu __le64 *ts = dpaa2_get_ts(vaddr, false); 58734ff6846SIoana Radulescu u64 ns; 58834ff6846SIoana Radulescu 58934ff6846SIoana Radulescu memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 59034ff6846SIoana Radulescu 59134ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 59234ff6846SIoana Radulescu shhwtstamps->hwtstamp = ns_to_ktime(ns); 59334ff6846SIoana Radulescu } 59434ff6846SIoana Radulescu 59534ff6846SIoana Radulescu /* Check if we need to validate the L4 csum */ 59634ff6846SIoana Radulescu if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 59734ff6846SIoana Radulescu status = le32_to_cpu(fas->status); 5985d8dccf8SIoana Ciornei dpaa2_eth_validate_rx_csum(priv, status, skb); 59934ff6846SIoana Radulescu } 60034ff6846SIoana Radulescu 60134ff6846SIoana Radulescu skb->protocol = eth_type_trans(skb, priv->net_dev); 602dbcdf728SIoana Ciocoi Radulescu skb_record_rx_queue(skb, fq->flowid); 60334ff6846SIoana Radulescu 60434ff6846SIoana Radulescu percpu_stats->rx_packets++; 60534ff6846SIoana Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 606fc398becSIoana Ciornei ch->stats.bytes_per_cdan += dpaa2_fd_get_len(fd); 60734ff6846SIoana Radulescu 6080a25d92cSIoana Ciornei list_add_tail(&skb->list, ch->rx_list); 60934ff6846SIoana Radulescu 61034ff6846SIoana Radulescu return; 61134ff6846SIoana Radulescu 61234ff6846SIoana Radulescu err_build_skb: 6135d8dccf8SIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 61434ff6846SIoana Radulescu err_frame_format: 61534ff6846SIoana Radulescu percpu_stats->rx_dropped++; 61634ff6846SIoana Radulescu } 61734ff6846SIoana Radulescu 618061d631fSIoana Ciornei /* Processing of Rx frames received on the error FQ 619061d631fSIoana Ciornei * We check and print the error bits and then free the frame 620061d631fSIoana Ciornei */ 621061d631fSIoana Ciornei static void dpaa2_eth_rx_err(struct dpaa2_eth_priv *priv, 622061d631fSIoana Ciornei struct dpaa2_eth_channel *ch, 623061d631fSIoana Ciornei const struct dpaa2_fd *fd, 624061d631fSIoana Ciornei struct dpaa2_eth_fq *fq __always_unused) 625061d631fSIoana Ciornei { 626061d631fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 627061d631fSIoana Ciornei dma_addr_t addr = dpaa2_fd_get_addr(fd); 628061d631fSIoana Ciornei u8 fd_format = dpaa2_fd_get_format(fd); 629061d631fSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 630061d631fSIoana Ciornei struct dpaa2_eth_trap_item *trap_item; 631061d631fSIoana Ciornei struct dpaa2_fapr *fapr; 632061d631fSIoana Ciornei struct sk_buff *skb; 633061d631fSIoana Ciornei void *buf_data; 634061d631fSIoana Ciornei void *vaddr; 635061d631fSIoana Ciornei 636061d631fSIoana Ciornei vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 637061d631fSIoana Ciornei dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 638061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 639061d631fSIoana Ciornei 640061d631fSIoana Ciornei buf_data = vaddr + dpaa2_fd_get_offset(fd); 641061d631fSIoana Ciornei 642061d631fSIoana Ciornei if (fd_format == dpaa2_fd_single) { 643061d631fSIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 644061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 645061d631fSIoana Ciornei skb = dpaa2_eth_build_linear_skb(ch, fd, vaddr); 646061d631fSIoana Ciornei } else if (fd_format == dpaa2_fd_sg) { 647061d631fSIoana Ciornei dma_unmap_page(dev, addr, priv->rx_buf_size, 648061d631fSIoana Ciornei DMA_BIDIRECTIONAL); 649061d631fSIoana Ciornei skb = dpaa2_eth_build_frag_skb(priv, ch, buf_data); 650061d631fSIoana Ciornei free_pages((unsigned long)vaddr, 0); 651061d631fSIoana Ciornei } else { 652061d631fSIoana Ciornei /* We don't support any other format */ 653061d631fSIoana Ciornei dpaa2_eth_free_rx_fd(priv, fd, vaddr); 654061d631fSIoana Ciornei goto err_frame_format; 655061d631fSIoana Ciornei } 656061d631fSIoana Ciornei 657061d631fSIoana Ciornei fapr = dpaa2_get_fapr(vaddr, false); 658061d631fSIoana Ciornei trap_item = dpaa2_eth_dl_get_trap(priv, fapr); 659061d631fSIoana Ciornei if (trap_item) 660061d631fSIoana Ciornei devlink_trap_report(priv->devlink, skb, trap_item->trap_ctx, 661061d631fSIoana Ciornei &priv->devlink_port, NULL); 662061d631fSIoana Ciornei consume_skb(skb); 663061d631fSIoana Ciornei 664061d631fSIoana Ciornei err_frame_format: 665061d631fSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 666061d631fSIoana Ciornei percpu_stats->rx_errors++; 667061d631fSIoana Ciornei ch->buf_count--; 668061d631fSIoana Ciornei } 669061d631fSIoana Ciornei 67034ff6846SIoana Radulescu /* Consume all frames pull-dequeued into the store. This is the simplest way to 67134ff6846SIoana Radulescu * make sure we don't accidentally issue another volatile dequeue which would 67234ff6846SIoana Radulescu * overwrite (leak) frames already in the store. 67334ff6846SIoana Radulescu * 67434ff6846SIoana Radulescu * Observance of NAPI budget is not our concern, leaving that to the caller. 67534ff6846SIoana Radulescu */ 6765d8dccf8SIoana Ciornei static int dpaa2_eth_consume_frames(struct dpaa2_eth_channel *ch, 677569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq **src) 67834ff6846SIoana Radulescu { 67934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = ch->priv; 68068049a5fSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq = NULL; 68134ff6846SIoana Radulescu struct dpaa2_dq *dq; 68234ff6846SIoana Radulescu const struct dpaa2_fd *fd; 683ef17bd7cSIoana Radulescu int cleaned = 0, retries = 0; 68434ff6846SIoana Radulescu int is_last; 68534ff6846SIoana Radulescu 68634ff6846SIoana Radulescu do { 68734ff6846SIoana Radulescu dq = dpaa2_io_store_next(ch->store, &is_last); 68834ff6846SIoana Radulescu if (unlikely(!dq)) { 68934ff6846SIoana Radulescu /* If we're here, we *must* have placed a 69034ff6846SIoana Radulescu * volatile dequeue comnmand, so keep reading through 69134ff6846SIoana Radulescu * the store until we get some sort of valid response 69234ff6846SIoana Radulescu * token (either a valid frame or an "empty dequeue") 69334ff6846SIoana Radulescu */ 694ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) { 695ef17bd7cSIoana Radulescu netdev_err_once(priv->net_dev, 696ef17bd7cSIoana Radulescu "Unable to read a valid dequeue response\n"); 697ef17bd7cSIoana Radulescu return -ETIMEDOUT; 698ef17bd7cSIoana Radulescu } 69934ff6846SIoana Radulescu continue; 70034ff6846SIoana Radulescu } 70134ff6846SIoana Radulescu 70234ff6846SIoana Radulescu fd = dpaa2_dq_fd(dq); 70334ff6846SIoana Radulescu fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 70434ff6846SIoana Radulescu 705dbcdf728SIoana Ciocoi Radulescu fq->consume(priv, ch, fd, fq); 70634ff6846SIoana Radulescu cleaned++; 707ef17bd7cSIoana Radulescu retries = 0; 70834ff6846SIoana Radulescu } while (!is_last); 70934ff6846SIoana Radulescu 71068049a5fSIoana Ciocoi Radulescu if (!cleaned) 71168049a5fSIoana Ciocoi Radulescu return 0; 71268049a5fSIoana Ciocoi Radulescu 71368049a5fSIoana Ciocoi Radulescu fq->stats.frames += cleaned; 714460fd830SIoana Ciornei ch->stats.frames += cleaned; 715fc398becSIoana Ciornei ch->stats.frames_per_cdan += cleaned; 71668049a5fSIoana Ciocoi Radulescu 71768049a5fSIoana Ciocoi Radulescu /* A dequeue operation only pulls frames from a single queue 718569dac6aSIoana Ciocoi Radulescu * into the store. Return the frame queue as an out param. 71968049a5fSIoana Ciocoi Radulescu */ 720569dac6aSIoana Ciocoi Radulescu if (src) 721569dac6aSIoana Ciocoi Radulescu *src = fq; 72268049a5fSIoana Ciocoi Radulescu 72334ff6846SIoana Radulescu return cleaned; 72434ff6846SIoana Radulescu } 72534ff6846SIoana Radulescu 726c5521189SYangbo Lu static int dpaa2_eth_ptp_parse(struct sk_buff *skb, 727c5521189SYangbo Lu u8 *msgtype, u8 *twostep, u8 *udp, 728c5521189SYangbo Lu u16 *correction_offset, 729c5521189SYangbo Lu u16 *origintimestamp_offset) 73034ff6846SIoana Radulescu { 731c5521189SYangbo Lu unsigned int ptp_class; 732c5521189SYangbo Lu struct ptp_header *hdr; 733c5521189SYangbo Lu unsigned int type; 734c5521189SYangbo Lu u8 *base; 735c5521189SYangbo Lu 736c5521189SYangbo Lu ptp_class = ptp_classify_raw(skb); 737c5521189SYangbo Lu if (ptp_class == PTP_CLASS_NONE) 738c5521189SYangbo Lu return -EINVAL; 739c5521189SYangbo Lu 740c5521189SYangbo Lu hdr = ptp_parse_header(skb, ptp_class); 741c5521189SYangbo Lu if (!hdr) 742c5521189SYangbo Lu return -EINVAL; 743c5521189SYangbo Lu 744c5521189SYangbo Lu *msgtype = ptp_get_msgtype(hdr, ptp_class); 745c5521189SYangbo Lu *twostep = hdr->flag_field[0] & 0x2; 746c5521189SYangbo Lu 747c5521189SYangbo Lu type = ptp_class & PTP_CLASS_PMASK; 748c5521189SYangbo Lu if (type == PTP_CLASS_IPV4 || 749c5521189SYangbo Lu type == PTP_CLASS_IPV6) 750c5521189SYangbo Lu *udp = 1; 751c5521189SYangbo Lu else 752c5521189SYangbo Lu *udp = 0; 753c5521189SYangbo Lu 754c5521189SYangbo Lu base = skb_mac_header(skb); 755c5521189SYangbo Lu *correction_offset = (u8 *)&hdr->correction - base; 756c5521189SYangbo Lu *origintimestamp_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 757c5521189SYangbo Lu 758c5521189SYangbo Lu return 0; 759c5521189SYangbo Lu } 760c5521189SYangbo Lu 761c5521189SYangbo Lu /* Configure the egress frame annotation for timestamp update */ 762c5521189SYangbo Lu static void dpaa2_eth_enable_tx_tstamp(struct dpaa2_eth_priv *priv, 763c5521189SYangbo Lu struct dpaa2_fd *fd, 764c5521189SYangbo Lu void *buf_start, 765c5521189SYangbo Lu struct sk_buff *skb) 766c5521189SYangbo Lu { 767c5521189SYangbo Lu struct ptp_tstamp origin_timestamp; 768c5521189SYangbo Lu u8 msgtype, twostep, udp; 76934ff6846SIoana Radulescu struct dpaa2_faead *faead; 770c5521189SYangbo Lu struct dpaa2_fas *fas; 771c5521189SYangbo Lu struct timespec64 ts; 772c5521189SYangbo Lu u16 offset1, offset2; 77334ff6846SIoana Radulescu u32 ctrl, frc; 774c5521189SYangbo Lu __le64 *ns; 775c5521189SYangbo Lu u8 *data; 77634ff6846SIoana Radulescu 77734ff6846SIoana Radulescu /* Mark the egress frame annotation area as valid */ 77834ff6846SIoana Radulescu frc = dpaa2_fd_get_frc(fd); 77934ff6846SIoana Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 78034ff6846SIoana Radulescu 78134ff6846SIoana Radulescu /* Set hardware annotation size */ 78234ff6846SIoana Radulescu ctrl = dpaa2_fd_get_ctrl(fd); 78334ff6846SIoana Radulescu dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 78434ff6846SIoana Radulescu 78534ff6846SIoana Radulescu /* enable UPD (update prepanded data) bit in FAEAD field of 78634ff6846SIoana Radulescu * hardware frame annotation area 78734ff6846SIoana Radulescu */ 78834ff6846SIoana Radulescu ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 78934ff6846SIoana Radulescu faead = dpaa2_get_faead(buf_start, true); 79034ff6846SIoana Radulescu faead->ctrl = cpu_to_le32(ctrl); 791c5521189SYangbo Lu 792c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 793c5521189SYangbo Lu if (dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 794c5521189SYangbo Lu &offset1, &offset2) || 7956b6817c5SChristian Eggers msgtype != PTP_MSGTYPE_SYNC || twostep) { 796c5521189SYangbo Lu WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 797c5521189SYangbo Lu return; 798c5521189SYangbo Lu } 799c5521189SYangbo Lu 800c5521189SYangbo Lu /* Mark the frame annotation status as valid */ 801c5521189SYangbo Lu frc = dpaa2_fd_get_frc(fd); 802c5521189SYangbo Lu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FASV); 803c5521189SYangbo Lu 804c5521189SYangbo Lu /* Mark the PTP flag for one step timestamping */ 805c5521189SYangbo Lu fas = dpaa2_get_fas(buf_start, true); 806c5521189SYangbo Lu fas->status = cpu_to_le32(DPAA2_FAS_PTP); 807c5521189SYangbo Lu 808c5521189SYangbo Lu dpaa2_ptp->caps.gettime64(&dpaa2_ptp->caps, &ts); 809c5521189SYangbo Lu ns = dpaa2_get_ts(buf_start, true); 810c5521189SYangbo Lu *ns = cpu_to_le64(timespec64_to_ns(&ts) / 811c5521189SYangbo Lu DPAA2_PTP_CLK_PERIOD_NS); 812c5521189SYangbo Lu 813c5521189SYangbo Lu /* Update current time to PTP message originTimestamp field */ 814c5521189SYangbo Lu ns_to_ptp_tstamp(&origin_timestamp, le64_to_cpup(ns)); 815c5521189SYangbo Lu data = skb_mac_header(skb); 816c5521189SYangbo Lu *(__be16 *)(data + offset2) = htons(origin_timestamp.sec_msb); 817c5521189SYangbo Lu *(__be32 *)(data + offset2 + 2) = 818c5521189SYangbo Lu htonl(origin_timestamp.sec_lsb); 819c5521189SYangbo Lu *(__be32 *)(data + offset2 + 6) = htonl(origin_timestamp.nsec); 820c5521189SYangbo Lu 821c4680c97SRadu Bulie if (priv->ptp_correction_off == offset1) 822c4680c97SRadu Bulie return; 823c5521189SYangbo Lu 824c4680c97SRadu Bulie priv->dpaa2_set_onestep_params_cb(priv, offset1, udp); 825c4680c97SRadu Bulie priv->ptp_correction_off = offset1; 826c4680c97SRadu Bulie 827c5521189SYangbo Lu } 82834ff6846SIoana Radulescu } 82934ff6846SIoana Radulescu 830ae3b0817SIoana Ciornei static void *dpaa2_eth_sgt_get(struct dpaa2_eth_priv *priv) 831ae3b0817SIoana Ciornei { 832ae3b0817SIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 833ae3b0817SIoana Ciornei void *sgt_buf = NULL; 834ae3b0817SIoana Ciornei int sgt_buf_size; 835ae3b0817SIoana Ciornei 836ae3b0817SIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 837a4218aefSIoana Ciornei sgt_buf_size = priv->tx_data_offset + 838a4218aefSIoana Ciornei DPAA2_ETH_SG_ENTRIES_MAX * sizeof(struct dpaa2_sg_entry); 839ae3b0817SIoana Ciornei 840ae3b0817SIoana Ciornei if (sgt_cache->count == 0) 841ae3b0817SIoana Ciornei sgt_buf = napi_alloc_frag_align(sgt_buf_size, DPAA2_ETH_TX_BUF_ALIGN); 842ae3b0817SIoana Ciornei else 843ae3b0817SIoana Ciornei sgt_buf = sgt_cache->buf[--sgt_cache->count]; 844ae3b0817SIoana Ciornei if (!sgt_buf) 845ae3b0817SIoana Ciornei return NULL; 846ae3b0817SIoana Ciornei 847ae3b0817SIoana Ciornei memset(sgt_buf, 0, sgt_buf_size); 848ae3b0817SIoana Ciornei 849ae3b0817SIoana Ciornei return sgt_buf; 850ae3b0817SIoana Ciornei } 851ae3b0817SIoana Ciornei 852ae3b0817SIoana Ciornei static void dpaa2_eth_sgt_recycle(struct dpaa2_eth_priv *priv, void *sgt_buf) 853ae3b0817SIoana Ciornei { 854ae3b0817SIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 855ae3b0817SIoana Ciornei 856ae3b0817SIoana Ciornei sgt_cache = this_cpu_ptr(priv->sgt_cache); 857ae3b0817SIoana Ciornei if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 858ae3b0817SIoana Ciornei skb_free_frag(sgt_buf); 859ae3b0817SIoana Ciornei else 860ae3b0817SIoana Ciornei sgt_cache->buf[sgt_cache->count++] = sgt_buf; 861ae3b0817SIoana Ciornei } 862ae3b0817SIoana Ciornei 86334ff6846SIoana Radulescu /* Create a frame descriptor based on a fragmented skb */ 8645d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd(struct dpaa2_eth_priv *priv, 86534ff6846SIoana Radulescu struct sk_buff *skb, 86664a965deSYangbo Lu struct dpaa2_fd *fd, 86764a965deSYangbo Lu void **swa_addr) 86834ff6846SIoana Radulescu { 86934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 87034ff6846SIoana Radulescu void *sgt_buf = NULL; 87134ff6846SIoana Radulescu dma_addr_t addr; 87234ff6846SIoana Radulescu int nr_frags = skb_shinfo(skb)->nr_frags; 87334ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 87434ff6846SIoana Radulescu int i, err; 87534ff6846SIoana Radulescu int sgt_buf_size; 87634ff6846SIoana Radulescu struct scatterlist *scl, *crt_scl; 87734ff6846SIoana Radulescu int num_sg; 87834ff6846SIoana Radulescu int num_dma_bufs; 87934ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 88034ff6846SIoana Radulescu 88134ff6846SIoana Radulescu /* Create and map scatterlist. 88234ff6846SIoana Radulescu * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 88334ff6846SIoana Radulescu * to go beyond nr_frags+1. 88434ff6846SIoana Radulescu * Note: We don't support chained scatterlists 88534ff6846SIoana Radulescu */ 88634ff6846SIoana Radulescu if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 88734ff6846SIoana Radulescu return -EINVAL; 88834ff6846SIoana Radulescu 889d4ceb8deSJulia Lawall scl = kmalloc_array(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 89034ff6846SIoana Radulescu if (unlikely(!scl)) 89134ff6846SIoana Radulescu return -ENOMEM; 89234ff6846SIoana Radulescu 89334ff6846SIoana Radulescu sg_init_table(scl, nr_frags + 1); 89434ff6846SIoana Radulescu num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 89537fbbddaSIoana Ciornei if (unlikely(num_sg < 0)) { 89637fbbddaSIoana Ciornei err = -ENOMEM; 89737fbbddaSIoana Ciornei goto dma_map_sg_failed; 89837fbbddaSIoana Ciornei } 89934ff6846SIoana Radulescu num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 90034ff6846SIoana Radulescu if (unlikely(!num_dma_bufs)) { 90134ff6846SIoana Radulescu err = -ENOMEM; 90234ff6846SIoana Radulescu goto dma_map_sg_failed; 90334ff6846SIoana Radulescu } 90434ff6846SIoana Radulescu 90534ff6846SIoana Radulescu /* Prepare the HW SGT structure */ 90634ff6846SIoana Radulescu sgt_buf_size = priv->tx_data_offset + 90734ff6846SIoana Radulescu sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 908a4218aefSIoana Ciornei sgt_buf = dpaa2_eth_sgt_get(priv); 90934ff6846SIoana Radulescu if (unlikely(!sgt_buf)) { 91034ff6846SIoana Radulescu err = -ENOMEM; 91134ff6846SIoana Radulescu goto sgt_buf_alloc_failed; 91234ff6846SIoana Radulescu } 91334ff6846SIoana Radulescu 91434ff6846SIoana Radulescu sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 91534ff6846SIoana Radulescu 91634ff6846SIoana Radulescu /* Fill in the HW SGT structure. 91734ff6846SIoana Radulescu * 91834ff6846SIoana Radulescu * sgt_buf is zeroed out, so the following fields are implicit 91934ff6846SIoana Radulescu * in all sgt entries: 92034ff6846SIoana Radulescu * - offset is 0 92134ff6846SIoana Radulescu * - format is 'dpaa2_sg_single' 92234ff6846SIoana Radulescu */ 92334ff6846SIoana Radulescu for_each_sg(scl, crt_scl, num_dma_bufs, i) { 92434ff6846SIoana Radulescu dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 92534ff6846SIoana Radulescu dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 92634ff6846SIoana Radulescu } 92734ff6846SIoana Radulescu dpaa2_sg_set_final(&sgt[i - 1], true); 92834ff6846SIoana Radulescu 92934ff6846SIoana Radulescu /* Store the skb backpointer in the SGT buffer. 93034ff6846SIoana Radulescu * Fit the scatterlist and the number of buffers alongside the 93134ff6846SIoana Radulescu * skb backpointer in the software annotation area. We'll need 93234ff6846SIoana Radulescu * all of them on Tx Conf. 93334ff6846SIoana Radulescu */ 93464a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 93534ff6846SIoana Radulescu swa = (struct dpaa2_eth_swa *)sgt_buf; 936e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SG; 937e3fdf6baSIoana Radulescu swa->sg.skb = skb; 938e3fdf6baSIoana Radulescu swa->sg.scl = scl; 939e3fdf6baSIoana Radulescu swa->sg.num_sg = num_sg; 940e3fdf6baSIoana Radulescu swa->sg.sgt_size = sgt_buf_size; 94134ff6846SIoana Radulescu 94234ff6846SIoana Radulescu /* Separately map the SGT buffer */ 94334ff6846SIoana Radulescu addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 94434ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) { 94534ff6846SIoana Radulescu err = -ENOMEM; 94634ff6846SIoana Radulescu goto dma_map_single_failed; 94734ff6846SIoana Radulescu } 948a4ca448eSIoana Ciornei memset(fd, 0, sizeof(struct dpaa2_fd)); 94934ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, priv->tx_data_offset); 95034ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_sg); 95134ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 95234ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 953b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 95434ff6846SIoana Radulescu 95534ff6846SIoana Radulescu return 0; 95634ff6846SIoana Radulescu 95734ff6846SIoana Radulescu dma_map_single_failed: 958a4218aefSIoana Ciornei dpaa2_eth_sgt_recycle(priv, sgt_buf); 95934ff6846SIoana Radulescu sgt_buf_alloc_failed: 96034ff6846SIoana Radulescu dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 96134ff6846SIoana Radulescu dma_map_sg_failed: 96234ff6846SIoana Radulescu kfree(scl); 96334ff6846SIoana Radulescu return err; 96434ff6846SIoana Radulescu } 96534ff6846SIoana Radulescu 966d70446eeSIoana Ciornei /* Create a SG frame descriptor based on a linear skb. 967d70446eeSIoana Ciornei * 968d70446eeSIoana Ciornei * This function is used on the Tx path when the skb headroom is not large 969d70446eeSIoana Ciornei * enough for the HW requirements, thus instead of realloc-ing the skb we 970d70446eeSIoana Ciornei * create a SG frame descriptor with only one entry. 971d70446eeSIoana Ciornei */ 9725d8dccf8SIoana Ciornei static int dpaa2_eth_build_sg_fd_single_buf(struct dpaa2_eth_priv *priv, 973d70446eeSIoana Ciornei struct sk_buff *skb, 97464a965deSYangbo Lu struct dpaa2_fd *fd, 97564a965deSYangbo Lu void **swa_addr) 976d70446eeSIoana Ciornei { 977d70446eeSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 978d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 979d70446eeSIoana Ciornei struct dpaa2_eth_swa *swa; 980d70446eeSIoana Ciornei dma_addr_t addr, sgt_addr; 981d70446eeSIoana Ciornei void *sgt_buf = NULL; 982d70446eeSIoana Ciornei int sgt_buf_size; 983d70446eeSIoana Ciornei int err; 984d70446eeSIoana Ciornei 985d70446eeSIoana Ciornei /* Prepare the HW SGT structure */ 986d70446eeSIoana Ciornei sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry); 987ae3b0817SIoana Ciornei sgt_buf = dpaa2_eth_sgt_get(priv); 988d70446eeSIoana Ciornei if (unlikely(!sgt_buf)) 989d70446eeSIoana Ciornei return -ENOMEM; 990d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 991d70446eeSIoana Ciornei 992d70446eeSIoana Ciornei addr = dma_map_single(dev, skb->data, skb->len, DMA_BIDIRECTIONAL); 993d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) { 994d70446eeSIoana Ciornei err = -ENOMEM; 995d70446eeSIoana Ciornei goto data_map_failed; 996d70446eeSIoana Ciornei } 997d70446eeSIoana Ciornei 998d70446eeSIoana Ciornei /* Fill in the HW SGT structure */ 999d70446eeSIoana Ciornei dpaa2_sg_set_addr(sgt, addr); 1000d70446eeSIoana Ciornei dpaa2_sg_set_len(sgt, skb->len); 1001d70446eeSIoana Ciornei dpaa2_sg_set_final(sgt, true); 1002d70446eeSIoana Ciornei 1003d70446eeSIoana Ciornei /* Store the skb backpointer in the SGT buffer */ 100464a965deSYangbo Lu *swa_addr = (void *)sgt_buf; 1005d70446eeSIoana Ciornei swa = (struct dpaa2_eth_swa *)sgt_buf; 1006d70446eeSIoana Ciornei swa->type = DPAA2_ETH_SWA_SINGLE; 1007d70446eeSIoana Ciornei swa->single.skb = skb; 100854a57d1cSIoana Ciornei swa->single.sgt_size = sgt_buf_size; 1009d70446eeSIoana Ciornei 1010d70446eeSIoana Ciornei /* Separately map the SGT buffer */ 1011d70446eeSIoana Ciornei sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 1012d70446eeSIoana Ciornei if (unlikely(dma_mapping_error(dev, sgt_addr))) { 1013d70446eeSIoana Ciornei err = -ENOMEM; 1014d70446eeSIoana Ciornei goto sgt_map_failed; 1015d70446eeSIoana Ciornei } 1016d70446eeSIoana Ciornei 1017a4ca448eSIoana Ciornei memset(fd, 0, sizeof(struct dpaa2_fd)); 1018d70446eeSIoana Ciornei dpaa2_fd_set_offset(fd, priv->tx_data_offset); 1019d70446eeSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_sg); 1020d70446eeSIoana Ciornei dpaa2_fd_set_addr(fd, sgt_addr); 1021d70446eeSIoana Ciornei dpaa2_fd_set_len(fd, skb->len); 1022d70446eeSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 1023d70446eeSIoana Ciornei 1024d70446eeSIoana Ciornei return 0; 1025d70446eeSIoana Ciornei 1026d70446eeSIoana Ciornei sgt_map_failed: 1027d70446eeSIoana Ciornei dma_unmap_single(dev, addr, skb->len, DMA_BIDIRECTIONAL); 1028d70446eeSIoana Ciornei data_map_failed: 1029ae3b0817SIoana Ciornei dpaa2_eth_sgt_recycle(priv, sgt_buf); 1030d70446eeSIoana Ciornei 1031d70446eeSIoana Ciornei return err; 1032d70446eeSIoana Ciornei } 1033d70446eeSIoana Ciornei 103434ff6846SIoana Radulescu /* Create a frame descriptor based on a linear skb */ 10355d8dccf8SIoana Ciornei static int dpaa2_eth_build_single_fd(struct dpaa2_eth_priv *priv, 103634ff6846SIoana Radulescu struct sk_buff *skb, 103764a965deSYangbo Lu struct dpaa2_fd *fd, 103864a965deSYangbo Lu void **swa_addr) 103934ff6846SIoana Radulescu { 104034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 104134ff6846SIoana Radulescu u8 *buffer_start, *aligned_start; 1042e3fdf6baSIoana Radulescu struct dpaa2_eth_swa *swa; 104334ff6846SIoana Radulescu dma_addr_t addr; 104434ff6846SIoana Radulescu 10451cf773bdSYangbo Lu buffer_start = skb->data - dpaa2_eth_needed_headroom(skb); 104634ff6846SIoana Radulescu 104734ff6846SIoana Radulescu /* If there's enough room to align the FD address, do it. 104834ff6846SIoana Radulescu * It will help hardware optimize accesses. 104934ff6846SIoana Radulescu */ 105034ff6846SIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 105134ff6846SIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 105234ff6846SIoana Radulescu if (aligned_start >= skb->head) 105334ff6846SIoana Radulescu buffer_start = aligned_start; 105434ff6846SIoana Radulescu 105534ff6846SIoana Radulescu /* Store a backpointer to the skb at the beginning of the buffer 105634ff6846SIoana Radulescu * (in the private data area) such that we can release it 105734ff6846SIoana Radulescu * on Tx confirm 105834ff6846SIoana Radulescu */ 105964a965deSYangbo Lu *swa_addr = (void *)buffer_start; 1060e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 1061e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SINGLE; 1062e3fdf6baSIoana Radulescu swa->single.skb = skb; 106334ff6846SIoana Radulescu 106434ff6846SIoana Radulescu addr = dma_map_single(dev, buffer_start, 106534ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 106634ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 106734ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 106834ff6846SIoana Radulescu return -ENOMEM; 106934ff6846SIoana Radulescu 1070a4ca448eSIoana Ciornei memset(fd, 0, sizeof(struct dpaa2_fd)); 107134ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 107234ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 107334ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 107434ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_single); 1075b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 107634ff6846SIoana Radulescu 107734ff6846SIoana Radulescu return 0; 107834ff6846SIoana Radulescu } 107934ff6846SIoana Radulescu 108034ff6846SIoana Radulescu /* FD freeing routine on the Tx path 108134ff6846SIoana Radulescu * 108234ff6846SIoana Radulescu * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 108334ff6846SIoana Radulescu * back-pointed to is also freed. 108434ff6846SIoana Radulescu * This can be called either from dpaa2_eth_tx_conf() or on the error path of 108534ff6846SIoana Radulescu * dpaa2_eth_tx(). 108634ff6846SIoana Radulescu */ 1087c5521189SYangbo Lu static void dpaa2_eth_free_tx_fd(struct dpaa2_eth_priv *priv, 1088d678be1dSIoana Radulescu struct dpaa2_eth_fq *fq, 10890723a3aeSIoana Ciocoi Radulescu const struct dpaa2_fd *fd, bool in_napi) 109034ff6846SIoana Radulescu { 109134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 1092d70446eeSIoana Ciornei dma_addr_t fd_addr, sg_addr; 1093d678be1dSIoana Radulescu struct sk_buff *skb = NULL; 109434ff6846SIoana Radulescu unsigned char *buffer_start; 109534ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 109634ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 1097d678be1dSIoana Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 1098d70446eeSIoana Ciornei struct dpaa2_sg_entry *sgt; 10993dc709e0SIoana Ciornei int should_free_skb = 1; 110006d12994SIoana Ciornei void *tso_hdr; 11013dc709e0SIoana Ciornei int i; 1102d70446eeSIoana Ciornei 110334ff6846SIoana Radulescu fd_addr = dpaa2_fd_get_addr(fd); 1104e3fdf6baSIoana Radulescu buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 1105e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 110634ff6846SIoana Radulescu 110734ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 1108d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_SINGLE) { 1109e3fdf6baSIoana Radulescu skb = swa->single.skb; 1110d678be1dSIoana Radulescu /* Accessing the skb buffer is safe before dma unmap, 1111d678be1dSIoana Radulescu * because we didn't map the actual skb shell. 111234ff6846SIoana Radulescu */ 111334ff6846SIoana Radulescu dma_unmap_single(dev, fd_addr, 111434ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 111534ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 1116d678be1dSIoana Radulescu } else { 1117d678be1dSIoana Radulescu WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 1118d678be1dSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 1119d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 1120d678be1dSIoana Radulescu } 112134ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 1122d70446eeSIoana Ciornei if (swa->type == DPAA2_ETH_SWA_SG) { 1123e3fdf6baSIoana Radulescu skb = swa->sg.skb; 112434ff6846SIoana Radulescu 112534ff6846SIoana Radulescu /* Unmap the scatterlist */ 1126e3fdf6baSIoana Radulescu dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 1127e3fdf6baSIoana Radulescu DMA_BIDIRECTIONAL); 1128e3fdf6baSIoana Radulescu kfree(swa->sg.scl); 112934ff6846SIoana Radulescu 113034ff6846SIoana Radulescu /* Unmap the SGT buffer */ 1131e3fdf6baSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 113234ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 11333dc709e0SIoana Ciornei } else if (swa->type == DPAA2_ETH_SWA_SW_TSO) { 11343dc709e0SIoana Ciornei skb = swa->tso.skb; 11353dc709e0SIoana Ciornei 11363dc709e0SIoana Ciornei sgt = (struct dpaa2_sg_entry *)(buffer_start + 11373dc709e0SIoana Ciornei priv->tx_data_offset); 11383dc709e0SIoana Ciornei 11390a09c5b8SIoana Ciornei /* Unmap the SGT buffer */ 11400a09c5b8SIoana Ciornei dma_unmap_single(dev, fd_addr, swa->tso.sgt_size, 11410a09c5b8SIoana Ciornei DMA_BIDIRECTIONAL); 11420a09c5b8SIoana Ciornei 11433dc709e0SIoana Ciornei /* Unmap and free the header */ 114406d12994SIoana Ciornei tso_hdr = dpaa2_iova_to_virt(priv->iommu_domain, dpaa2_sg_get_addr(sgt)); 11453dc709e0SIoana Ciornei dma_unmap_single(dev, dpaa2_sg_get_addr(sgt), TSO_HEADER_SIZE, 11463dc709e0SIoana Ciornei DMA_TO_DEVICE); 114706d12994SIoana Ciornei kfree(tso_hdr); 11483dc709e0SIoana Ciornei 11493dc709e0SIoana Ciornei /* Unmap the other SG entries for the data */ 11503dc709e0SIoana Ciornei for (i = 1; i < swa->tso.num_sg; i++) 11513dc709e0SIoana Ciornei dma_unmap_single(dev, dpaa2_sg_get_addr(&sgt[i]), 11523dc709e0SIoana Ciornei dpaa2_sg_get_len(&sgt[i]), DMA_TO_DEVICE); 11533dc709e0SIoana Ciornei 11543dc709e0SIoana Ciornei if (!swa->tso.is_last_fd) 11553dc709e0SIoana Ciornei should_free_skb = 0; 115634ff6846SIoana Radulescu } else { 1157d70446eeSIoana Ciornei skb = swa->single.skb; 1158d70446eeSIoana Ciornei 1159d70446eeSIoana Ciornei /* Unmap the SGT Buffer */ 1160d70446eeSIoana Ciornei dma_unmap_single(dev, fd_addr, swa->single.sgt_size, 1161d70446eeSIoana Ciornei DMA_BIDIRECTIONAL); 1162d70446eeSIoana Ciornei 1163d70446eeSIoana Ciornei sgt = (struct dpaa2_sg_entry *)(buffer_start + 1164d70446eeSIoana Ciornei priv->tx_data_offset); 1165d70446eeSIoana Ciornei sg_addr = dpaa2_sg_get_addr(sgt); 1166d70446eeSIoana Ciornei dma_unmap_single(dev, sg_addr, skb->len, DMA_BIDIRECTIONAL); 1167d70446eeSIoana Ciornei } 1168d70446eeSIoana Ciornei } else { 116934ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "Invalid FD format\n"); 117034ff6846SIoana Radulescu return; 117134ff6846SIoana Radulescu } 117234ff6846SIoana Radulescu 1173d678be1dSIoana Radulescu if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 1174d678be1dSIoana Radulescu fq->dq_frames++; 1175d678be1dSIoana Radulescu fq->dq_bytes += fd_len; 1176d678be1dSIoana Radulescu } 1177d678be1dSIoana Radulescu 1178d678be1dSIoana Radulescu if (swa->type == DPAA2_ETH_SWA_XDP) { 1179d678be1dSIoana Radulescu xdp_return_frame(swa->xdp.xdpf); 1180d678be1dSIoana Radulescu return; 1181d678be1dSIoana Radulescu } 1182d678be1dSIoana Radulescu 118334ff6846SIoana Radulescu /* Get the timestamp value */ 11843dc709e0SIoana Ciornei if (swa->type != DPAA2_ETH_SWA_SW_TSO) { 11851cf773bdSYangbo Lu if (skb->cb[0] == TX_TSTAMP) { 118634ff6846SIoana Radulescu struct skb_shared_hwtstamps shhwtstamps; 1187e3fdf6baSIoana Radulescu __le64 *ts = dpaa2_get_ts(buffer_start, true); 118834ff6846SIoana Radulescu u64 ns; 118934ff6846SIoana Radulescu 119034ff6846SIoana Radulescu memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 119134ff6846SIoana Radulescu 119234ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 119334ff6846SIoana Radulescu shhwtstamps.hwtstamp = ns_to_ktime(ns); 119434ff6846SIoana Radulescu skb_tstamp_tx(skb, &shhwtstamps); 1195c5521189SYangbo Lu } else if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1196c5521189SYangbo Lu mutex_unlock(&priv->onestep_tstamp_lock); 119734ff6846SIoana Radulescu } 11983dc709e0SIoana Ciornei } 119934ff6846SIoana Radulescu 120034ff6846SIoana Radulescu /* Free SGT buffer allocated on tx */ 1201a4218aefSIoana Ciornei if (fd_format != dpaa2_fd_single) 1202ae3b0817SIoana Ciornei dpaa2_eth_sgt_recycle(priv, buffer_start); 120334ff6846SIoana Radulescu 12043dc709e0SIoana Ciornei /* Move on with skb release. If we are just confirming multiple FDs 12053dc709e0SIoana Ciornei * from the same TSO skb then only the last one will need to free the 12063dc709e0SIoana Ciornei * skb. 12073dc709e0SIoana Ciornei */ 12083dc709e0SIoana Ciornei if (should_free_skb) 12090723a3aeSIoana Ciocoi Radulescu napi_consume_skb(skb, in_napi); 121034ff6846SIoana Radulescu } 121134ff6846SIoana Radulescu 12123dc709e0SIoana Ciornei static int dpaa2_eth_build_gso_fd(struct dpaa2_eth_priv *priv, 12133dc709e0SIoana Ciornei struct sk_buff *skb, struct dpaa2_fd *fd, 12143dc709e0SIoana Ciornei int *num_fds, u32 *total_fds_len) 12153dc709e0SIoana Ciornei { 12163dc709e0SIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 12173dc709e0SIoana Ciornei int hdr_len, total_len, data_left, fd_len; 12183dc709e0SIoana Ciornei int num_sge, err, i, sgt_buf_size; 12193dc709e0SIoana Ciornei struct dpaa2_fd *fd_start = fd; 12203dc709e0SIoana Ciornei struct dpaa2_sg_entry *sgt; 12213dc709e0SIoana Ciornei struct dpaa2_eth_swa *swa; 12223dc709e0SIoana Ciornei dma_addr_t sgt_addr, addr; 12233dc709e0SIoana Ciornei dma_addr_t tso_hdr_dma; 12243dc709e0SIoana Ciornei unsigned int index = 0; 12253dc709e0SIoana Ciornei struct tso_t tso; 12263dc709e0SIoana Ciornei char *tso_hdr; 12273dc709e0SIoana Ciornei void *sgt_buf; 12283dc709e0SIoana Ciornei 12293dc709e0SIoana Ciornei /* Initialize the TSO handler, and prepare the first payload */ 12303dc709e0SIoana Ciornei hdr_len = tso_start(skb, &tso); 12313dc709e0SIoana Ciornei *total_fds_len = 0; 12323dc709e0SIoana Ciornei 12333dc709e0SIoana Ciornei total_len = skb->len - hdr_len; 12343dc709e0SIoana Ciornei while (total_len > 0) { 12353dc709e0SIoana Ciornei /* Prepare the HW SGT structure for this frame */ 12363dc709e0SIoana Ciornei sgt_buf = dpaa2_eth_sgt_get(priv); 12373dc709e0SIoana Ciornei if (unlikely(!sgt_buf)) { 12383dc709e0SIoana Ciornei netdev_err(priv->net_dev, "dpaa2_eth_sgt_get() failed\n"); 12393dc709e0SIoana Ciornei err = -ENOMEM; 12403dc709e0SIoana Ciornei goto err_sgt_get; 12413dc709e0SIoana Ciornei } 12423dc709e0SIoana Ciornei sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 12433dc709e0SIoana Ciornei 12443dc709e0SIoana Ciornei /* Determine the data length of this frame */ 12453dc709e0SIoana Ciornei data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); 12463dc709e0SIoana Ciornei total_len -= data_left; 12473dc709e0SIoana Ciornei fd_len = data_left + hdr_len; 12483dc709e0SIoana Ciornei 12493dc709e0SIoana Ciornei /* Prepare packet headers: MAC + IP + TCP */ 12503dc709e0SIoana Ciornei tso_hdr = kmalloc(TSO_HEADER_SIZE, GFP_ATOMIC); 12513dc709e0SIoana Ciornei if (!tso_hdr) { 12523dc709e0SIoana Ciornei err = -ENOMEM; 12533dc709e0SIoana Ciornei goto err_alloc_tso_hdr; 12543dc709e0SIoana Ciornei } 12553dc709e0SIoana Ciornei 12563dc709e0SIoana Ciornei tso_build_hdr(skb, tso_hdr, &tso, data_left, total_len == 0); 12573dc709e0SIoana Ciornei tso_hdr_dma = dma_map_single(dev, tso_hdr, TSO_HEADER_SIZE, DMA_TO_DEVICE); 12583dc709e0SIoana Ciornei if (dma_mapping_error(dev, tso_hdr_dma)) { 12593dc709e0SIoana Ciornei netdev_err(priv->net_dev, "dma_map_single(tso_hdr) failed\n"); 12603dc709e0SIoana Ciornei err = -ENOMEM; 12613dc709e0SIoana Ciornei goto err_map_tso_hdr; 12623dc709e0SIoana Ciornei } 12633dc709e0SIoana Ciornei 12643dc709e0SIoana Ciornei /* Setup the SG entry for the header */ 12653dc709e0SIoana Ciornei dpaa2_sg_set_addr(sgt, tso_hdr_dma); 12663dc709e0SIoana Ciornei dpaa2_sg_set_len(sgt, hdr_len); 126799cd6a64SYang Li dpaa2_sg_set_final(sgt, data_left <= 0); 12683dc709e0SIoana Ciornei 12693dc709e0SIoana Ciornei /* Compose the SG entries for each fragment of data */ 12703dc709e0SIoana Ciornei num_sge = 1; 12713dc709e0SIoana Ciornei while (data_left > 0) { 12723dc709e0SIoana Ciornei int size; 12733dc709e0SIoana Ciornei 12743dc709e0SIoana Ciornei /* Move to the next SG entry */ 12753dc709e0SIoana Ciornei sgt++; 12763dc709e0SIoana Ciornei size = min_t(int, tso.size, data_left); 12773dc709e0SIoana Ciornei 12783dc709e0SIoana Ciornei addr = dma_map_single(dev, tso.data, size, DMA_TO_DEVICE); 12793dc709e0SIoana Ciornei if (dma_mapping_error(dev, addr)) { 12803dc709e0SIoana Ciornei netdev_err(priv->net_dev, "dma_map_single(tso.data) failed\n"); 12813dc709e0SIoana Ciornei err = -ENOMEM; 12823dc709e0SIoana Ciornei goto err_map_data; 12833dc709e0SIoana Ciornei } 12843dc709e0SIoana Ciornei dpaa2_sg_set_addr(sgt, addr); 12853dc709e0SIoana Ciornei dpaa2_sg_set_len(sgt, size); 128699cd6a64SYang Li dpaa2_sg_set_final(sgt, size == data_left); 12873dc709e0SIoana Ciornei 12883dc709e0SIoana Ciornei num_sge++; 12893dc709e0SIoana Ciornei 12903dc709e0SIoana Ciornei /* Build the data for the __next__ fragment */ 12913dc709e0SIoana Ciornei data_left -= size; 12923dc709e0SIoana Ciornei tso_build_data(skb, &tso, size); 12933dc709e0SIoana Ciornei } 12943dc709e0SIoana Ciornei 12953dc709e0SIoana Ciornei /* Store the skb backpointer in the SGT buffer */ 12963dc709e0SIoana Ciornei sgt_buf_size = priv->tx_data_offset + num_sge * sizeof(struct dpaa2_sg_entry); 12973dc709e0SIoana Ciornei swa = (struct dpaa2_eth_swa *)sgt_buf; 12983dc709e0SIoana Ciornei swa->type = DPAA2_ETH_SWA_SW_TSO; 12993dc709e0SIoana Ciornei swa->tso.skb = skb; 13003dc709e0SIoana Ciornei swa->tso.num_sg = num_sge; 13013dc709e0SIoana Ciornei swa->tso.sgt_size = sgt_buf_size; 13023dc709e0SIoana Ciornei swa->tso.is_last_fd = total_len == 0 ? 1 : 0; 13033dc709e0SIoana Ciornei 13043dc709e0SIoana Ciornei /* Separately map the SGT buffer */ 13053dc709e0SIoana Ciornei sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 13063dc709e0SIoana Ciornei if (unlikely(dma_mapping_error(dev, sgt_addr))) { 13073dc709e0SIoana Ciornei netdev_err(priv->net_dev, "dma_map_single(sgt_buf) failed\n"); 13083dc709e0SIoana Ciornei err = -ENOMEM; 13093dc709e0SIoana Ciornei goto err_map_sgt; 13103dc709e0SIoana Ciornei } 13113dc709e0SIoana Ciornei 13123dc709e0SIoana Ciornei /* Setup the frame descriptor */ 13133dc709e0SIoana Ciornei memset(fd, 0, sizeof(struct dpaa2_fd)); 13143dc709e0SIoana Ciornei dpaa2_fd_set_offset(fd, priv->tx_data_offset); 13153dc709e0SIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_sg); 13163dc709e0SIoana Ciornei dpaa2_fd_set_addr(fd, sgt_addr); 13173dc709e0SIoana Ciornei dpaa2_fd_set_len(fd, fd_len); 13183dc709e0SIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 13193dc709e0SIoana Ciornei 13203dc709e0SIoana Ciornei *total_fds_len += fd_len; 13213dc709e0SIoana Ciornei /* Advance to the next frame descriptor */ 13223dc709e0SIoana Ciornei fd++; 13233dc709e0SIoana Ciornei index++; 13243dc709e0SIoana Ciornei } 13253dc709e0SIoana Ciornei 13263dc709e0SIoana Ciornei *num_fds = index; 13273dc709e0SIoana Ciornei 13283dc709e0SIoana Ciornei return 0; 13293dc709e0SIoana Ciornei 13303dc709e0SIoana Ciornei err_map_sgt: 13313dc709e0SIoana Ciornei err_map_data: 13323dc709e0SIoana Ciornei /* Unmap all the data S/G entries for the current FD */ 13333dc709e0SIoana Ciornei sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 13343dc709e0SIoana Ciornei for (i = 1; i < num_sge; i++) 13353dc709e0SIoana Ciornei dma_unmap_single(dev, dpaa2_sg_get_addr(&sgt[i]), 13363dc709e0SIoana Ciornei dpaa2_sg_get_len(&sgt[i]), DMA_TO_DEVICE); 13373dc709e0SIoana Ciornei 13383dc709e0SIoana Ciornei /* Unmap the header entry */ 13393dc709e0SIoana Ciornei dma_unmap_single(dev, tso_hdr_dma, TSO_HEADER_SIZE, DMA_TO_DEVICE); 13403dc709e0SIoana Ciornei err_map_tso_hdr: 13413dc709e0SIoana Ciornei kfree(tso_hdr); 13423dc709e0SIoana Ciornei err_alloc_tso_hdr: 13433dc709e0SIoana Ciornei dpaa2_eth_sgt_recycle(priv, sgt_buf); 13443dc709e0SIoana Ciornei err_sgt_get: 13453dc709e0SIoana Ciornei /* Free all the other FDs that were already fully created */ 13463dc709e0SIoana Ciornei for (i = 0; i < index; i++) 13473dc709e0SIoana Ciornei dpaa2_eth_free_tx_fd(priv, NULL, &fd_start[i], false); 13483dc709e0SIoana Ciornei 13493dc709e0SIoana Ciornei return err; 13503dc709e0SIoana Ciornei } 13513dc709e0SIoana Ciornei 1352c5521189SYangbo Lu static netdev_tx_t __dpaa2_eth_tx(struct sk_buff *skb, 1353c5521189SYangbo Lu struct net_device *net_dev) 135434ff6846SIoana Radulescu { 135534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1356a4ca448eSIoana Ciornei int total_enqueued = 0, retries = 0, enqueued; 135734ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 1358035dd64dSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 1359035dd64dSIoana Ciornei unsigned int needed_headroom; 1360a4ca448eSIoana Ciornei int num_fds = 1, max_retries; 136134ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 1362569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1363a4ca448eSIoana Ciornei struct dpaa2_fd *fd; 136434ff6846SIoana Radulescu u16 queue_mapping; 13653dc709e0SIoana Ciornei void *swa = NULL; 1366ab1e6de2SIoana Radulescu u8 prio = 0; 136734ff6846SIoana Radulescu int err, i; 1368035dd64dSIoana Ciornei u32 fd_len; 136934ff6846SIoana Radulescu 137034ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 137134ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 1372a4ca448eSIoana Ciornei fd = (this_cpu_ptr(priv->fd))->array; 137334ff6846SIoana Radulescu 13741cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(skb); 137534ff6846SIoana Radulescu 137634ff6846SIoana Radulescu /* We'll be holding a back-reference to the skb until Tx Confirmation; 137734ff6846SIoana Radulescu * we don't want that overwritten by a concurrent Tx with a cloned skb. 137834ff6846SIoana Radulescu */ 137934ff6846SIoana Radulescu skb = skb_unshare(skb, GFP_ATOMIC); 138034ff6846SIoana Radulescu if (unlikely(!skb)) { 138134ff6846SIoana Radulescu /* skb_unshare() has already freed the skb */ 138234ff6846SIoana Radulescu percpu_stats->tx_dropped++; 138334ff6846SIoana Radulescu return NETDEV_TX_OK; 138434ff6846SIoana Radulescu } 138534ff6846SIoana Radulescu 138634ff6846SIoana Radulescu /* Setup the FD fields */ 138734ff6846SIoana Radulescu 13883dc709e0SIoana Ciornei if (skb_is_gso(skb)) { 13893dc709e0SIoana Ciornei err = dpaa2_eth_build_gso_fd(priv, skb, fd, &num_fds, &fd_len); 13903dc709e0SIoana Ciornei percpu_extras->tx_sg_frames += num_fds; 13913dc709e0SIoana Ciornei percpu_extras->tx_sg_bytes += fd_len; 13923dc709e0SIoana Ciornei percpu_extras->tx_tso_frames += num_fds; 13933dc709e0SIoana Ciornei percpu_extras->tx_tso_bytes += fd_len; 13943dc709e0SIoana Ciornei } else if (skb_is_nonlinear(skb)) { 1395a4ca448eSIoana Ciornei err = dpaa2_eth_build_sg_fd(priv, skb, fd, &swa); 139634ff6846SIoana Radulescu percpu_extras->tx_sg_frames++; 139734ff6846SIoana Radulescu percpu_extras->tx_sg_bytes += skb->len; 1398a4ca448eSIoana Ciornei fd_len = dpaa2_fd_get_len(fd); 1399d70446eeSIoana Ciornei } else if (skb_headroom(skb) < needed_headroom) { 1400a4ca448eSIoana Ciornei err = dpaa2_eth_build_sg_fd_single_buf(priv, skb, fd, &swa); 1401d70446eeSIoana Ciornei percpu_extras->tx_sg_frames++; 1402d70446eeSIoana Ciornei percpu_extras->tx_sg_bytes += skb->len; 14034c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_frames++; 14044c96c0acSIoana Ciornei percpu_extras->tx_converted_sg_bytes += skb->len; 1405a4ca448eSIoana Ciornei fd_len = dpaa2_fd_get_len(fd); 140634ff6846SIoana Radulescu } else { 1407a4ca448eSIoana Ciornei err = dpaa2_eth_build_single_fd(priv, skb, fd, &swa); 1408a4ca448eSIoana Ciornei fd_len = dpaa2_fd_get_len(fd); 140934ff6846SIoana Radulescu } 141034ff6846SIoana Radulescu 141134ff6846SIoana Radulescu if (unlikely(err)) { 141234ff6846SIoana Radulescu percpu_stats->tx_dropped++; 141334ff6846SIoana Radulescu goto err_build_fd; 141434ff6846SIoana Radulescu } 141534ff6846SIoana Radulescu 14163dc709e0SIoana Ciornei if (swa && skb->cb[0]) 1417a4ca448eSIoana Ciornei dpaa2_eth_enable_tx_tstamp(priv, fd, swa, skb); 141864a965deSYangbo Lu 141934ff6846SIoana Radulescu /* Tracing point */ 1420a4ca448eSIoana Ciornei for (i = 0; i < num_fds; i++) 1421a4ca448eSIoana Ciornei trace_dpaa2_tx_fd(net_dev, &fd[i]); 142234ff6846SIoana Radulescu 142334ff6846SIoana Radulescu /* TxConf FQ selection relies on queue id from the stack. 142434ff6846SIoana Radulescu * In case of a forwarded frame from another DPNI interface, we choose 142534ff6846SIoana Radulescu * a queue affined to the same core that processed the Rx frame 142634ff6846SIoana Radulescu */ 142734ff6846SIoana Radulescu queue_mapping = skb_get_queue_mapping(skb); 1428ab1e6de2SIoana Radulescu 1429ab1e6de2SIoana Radulescu if (net_dev->num_tc) { 1430ab1e6de2SIoana Radulescu prio = netdev_txq_to_tc(net_dev, queue_mapping); 1431ab1e6de2SIoana Radulescu /* Hardware interprets priority level 0 as being the highest, 1432ab1e6de2SIoana Radulescu * so we need to do a reverse mapping to the netdev tc index 1433ab1e6de2SIoana Radulescu */ 1434ab1e6de2SIoana Radulescu prio = net_dev->num_tc - prio - 1; 1435ab1e6de2SIoana Radulescu /* We have only one FQ array entry for all Tx hardware queues 1436ab1e6de2SIoana Radulescu * with the same flow id (but different priority levels) 1437ab1e6de2SIoana Radulescu */ 1438ab1e6de2SIoana Radulescu queue_mapping %= dpaa2_eth_queue_count(priv); 1439ab1e6de2SIoana Radulescu } 144034ff6846SIoana Radulescu fq = &priv->fq[queue_mapping]; 14418c838f53SIoana Ciornei nq = netdev_get_tx_queue(net_dev, queue_mapping); 14428c838f53SIoana Ciornei netdev_tx_sent_queue(nq, fd_len); 14438c838f53SIoana Ciornei 14448c838f53SIoana Ciornei /* Everything that happens after this enqueues might race with 14458c838f53SIoana Ciornei * the Tx confirmation callback for this frame 14468c838f53SIoana Ciornei */ 1447a4ca448eSIoana Ciornei max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 1448a4ca448eSIoana Ciornei while (total_enqueued < num_fds && retries < max_retries) { 1449a4ca448eSIoana Ciornei err = priv->enqueue(priv, fq, &fd[total_enqueued], 1450a4ca448eSIoana Ciornei prio, num_fds - total_enqueued, &enqueued); 1451a4ca448eSIoana Ciornei if (err == -EBUSY) { 1452a4ca448eSIoana Ciornei retries++; 1453a4ca448eSIoana Ciornei continue; 145434ff6846SIoana Radulescu } 1455a4ca448eSIoana Ciornei 1456a4ca448eSIoana Ciornei total_enqueued += enqueued; 1457a4ca448eSIoana Ciornei } 1458a4ca448eSIoana Ciornei percpu_extras->tx_portal_busy += retries; 1459a4ca448eSIoana Ciornei 146034ff6846SIoana Radulescu if (unlikely(err < 0)) { 146134ff6846SIoana Radulescu percpu_stats->tx_errors++; 146234ff6846SIoana Radulescu /* Clean up everything, including freeing the skb */ 1463a4ca448eSIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, fd, false); 14648c838f53SIoana Ciornei netdev_tx_completed_queue(nq, 1, fd_len); 146534ff6846SIoana Radulescu } else { 1466a4ca448eSIoana Ciornei percpu_stats->tx_packets += total_enqueued; 1467569dac6aSIoana Ciocoi Radulescu percpu_stats->tx_bytes += fd_len; 146834ff6846SIoana Radulescu } 146934ff6846SIoana Radulescu 147034ff6846SIoana Radulescu return NETDEV_TX_OK; 147134ff6846SIoana Radulescu 147234ff6846SIoana Radulescu err_build_fd: 147334ff6846SIoana Radulescu dev_kfree_skb(skb); 147434ff6846SIoana Radulescu 147534ff6846SIoana Radulescu return NETDEV_TX_OK; 147634ff6846SIoana Radulescu } 147734ff6846SIoana Radulescu 1478c5521189SYangbo Lu static void dpaa2_eth_tx_onestep_tstamp(struct work_struct *work) 1479c5521189SYangbo Lu { 1480c5521189SYangbo Lu struct dpaa2_eth_priv *priv = container_of(work, struct dpaa2_eth_priv, 1481c5521189SYangbo Lu tx_onestep_tstamp); 1482c5521189SYangbo Lu struct sk_buff *skb; 1483c5521189SYangbo Lu 1484c5521189SYangbo Lu while (true) { 1485c5521189SYangbo Lu skb = skb_dequeue(&priv->tx_skbs); 1486c5521189SYangbo Lu if (!skb) 1487c5521189SYangbo Lu return; 1488c5521189SYangbo Lu 1489c5521189SYangbo Lu /* Lock just before TX one-step timestamping packet, 1490c5521189SYangbo Lu * and release the lock in dpaa2_eth_free_tx_fd when 1491c5521189SYangbo Lu * confirm the packet has been sent on hardware, or 1492c5521189SYangbo Lu * when clean up during transmit failure. 1493c5521189SYangbo Lu */ 1494c5521189SYangbo Lu mutex_lock(&priv->onestep_tstamp_lock); 1495c5521189SYangbo Lu __dpaa2_eth_tx(skb, priv->net_dev); 1496c5521189SYangbo Lu } 1497c5521189SYangbo Lu } 1498c5521189SYangbo Lu 1499c5521189SYangbo Lu static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 1500c5521189SYangbo Lu { 1501c5521189SYangbo Lu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1502c5521189SYangbo Lu u8 msgtype, twostep, udp; 1503c5521189SYangbo Lu u16 offset1, offset2; 1504c5521189SYangbo Lu 1505c5521189SYangbo Lu /* Utilize skb->cb[0] for timestamping request per skb */ 1506c5521189SYangbo Lu skb->cb[0] = 0; 1507c5521189SYangbo Lu 1508c5521189SYangbo Lu if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && dpaa2_ptp) { 1509c5521189SYangbo Lu if (priv->tx_tstamp_type == HWTSTAMP_TX_ON) 1510c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1511c5521189SYangbo Lu else if (priv->tx_tstamp_type == HWTSTAMP_TX_ONESTEP_SYNC) 1512c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP_ONESTEP_SYNC; 1513c5521189SYangbo Lu } 1514c5521189SYangbo Lu 1515c5521189SYangbo Lu /* TX for one-step timestamping PTP Sync packet */ 1516c5521189SYangbo Lu if (skb->cb[0] == TX_TSTAMP_ONESTEP_SYNC) { 1517c5521189SYangbo Lu if (!dpaa2_eth_ptp_parse(skb, &msgtype, &twostep, &udp, 1518c5521189SYangbo Lu &offset1, &offset2)) 15196b6817c5SChristian Eggers if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0) { 1520c5521189SYangbo Lu skb_queue_tail(&priv->tx_skbs, skb); 1521c5521189SYangbo Lu queue_work(priv->dpaa2_ptp_wq, 1522c5521189SYangbo Lu &priv->tx_onestep_tstamp); 1523c5521189SYangbo Lu return NETDEV_TX_OK; 1524c5521189SYangbo Lu } 1525c5521189SYangbo Lu /* Use two-step timestamping if not one-step timestamping 1526c5521189SYangbo Lu * PTP Sync packet 1527c5521189SYangbo Lu */ 1528c5521189SYangbo Lu skb->cb[0] = TX_TSTAMP; 1529c5521189SYangbo Lu } 1530c5521189SYangbo Lu 1531c5521189SYangbo Lu /* TX for other packets */ 1532c5521189SYangbo Lu return __dpaa2_eth_tx(skb, net_dev); 1533c5521189SYangbo Lu } 1534c5521189SYangbo Lu 153534ff6846SIoana Radulescu /* Tx confirmation frame processing routine */ 153634ff6846SIoana Radulescu static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 1537fc398becSIoana Ciornei struct dpaa2_eth_channel *ch, 153834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 1539569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 154034ff6846SIoana Radulescu { 154134ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 154234ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 1543569dac6aSIoana Ciocoi Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 154434ff6846SIoana Radulescu u32 fd_errors; 154534ff6846SIoana Radulescu 154634ff6846SIoana Radulescu /* Tracing point */ 154734ff6846SIoana Radulescu trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 154834ff6846SIoana Radulescu 154934ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 155034ff6846SIoana Radulescu percpu_extras->tx_conf_frames++; 1551569dac6aSIoana Ciocoi Radulescu percpu_extras->tx_conf_bytes += fd_len; 1552fc398becSIoana Ciornei ch->stats.bytes_per_cdan += fd_len; 1553569dac6aSIoana Ciocoi Radulescu 155434ff6846SIoana Radulescu /* Check frame errors in the FD field */ 155534ff6846SIoana Radulescu fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 15565d8dccf8SIoana Ciornei dpaa2_eth_free_tx_fd(priv, fq, fd, true); 155734ff6846SIoana Radulescu 155834ff6846SIoana Radulescu if (likely(!fd_errors)) 155934ff6846SIoana Radulescu return; 156034ff6846SIoana Radulescu 156134ff6846SIoana Radulescu if (net_ratelimit()) 156234ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 156334ff6846SIoana Radulescu fd_errors); 156434ff6846SIoana Radulescu 156534ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 156634ff6846SIoana Radulescu /* Tx-conf logically pertains to the egress path. */ 156734ff6846SIoana Radulescu percpu_stats->tx_errors++; 156834ff6846SIoana Radulescu } 156934ff6846SIoana Radulescu 157070b32d82SIonut-robert Aron static int dpaa2_eth_set_rx_vlan_filtering(struct dpaa2_eth_priv *priv, 157170b32d82SIonut-robert Aron bool enable) 157270b32d82SIonut-robert Aron { 157370b32d82SIonut-robert Aron int err; 157470b32d82SIonut-robert Aron 157570b32d82SIonut-robert Aron err = dpni_enable_vlan_filter(priv->mc_io, 0, priv->mc_token, enable); 157670b32d82SIonut-robert Aron 157770b32d82SIonut-robert Aron if (err) { 157870b32d82SIonut-robert Aron netdev_err(priv->net_dev, 157970b32d82SIonut-robert Aron "dpni_enable_vlan_filter failed\n"); 158070b32d82SIonut-robert Aron return err; 158170b32d82SIonut-robert Aron } 158270b32d82SIonut-robert Aron 158370b32d82SIonut-robert Aron return 0; 158470b32d82SIonut-robert Aron } 158570b32d82SIonut-robert Aron 15865d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 158734ff6846SIoana Radulescu { 158834ff6846SIoana Radulescu int err; 158934ff6846SIoana Radulescu 159034ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 159134ff6846SIoana Radulescu DPNI_OFF_RX_L3_CSUM, enable); 159234ff6846SIoana Radulescu if (err) { 159334ff6846SIoana Radulescu netdev_err(priv->net_dev, 159434ff6846SIoana Radulescu "dpni_set_offload(RX_L3_CSUM) failed\n"); 159534ff6846SIoana Radulescu return err; 159634ff6846SIoana Radulescu } 159734ff6846SIoana Radulescu 159834ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 159934ff6846SIoana Radulescu DPNI_OFF_RX_L4_CSUM, enable); 160034ff6846SIoana Radulescu if (err) { 160134ff6846SIoana Radulescu netdev_err(priv->net_dev, 160234ff6846SIoana Radulescu "dpni_set_offload(RX_L4_CSUM) failed\n"); 160334ff6846SIoana Radulescu return err; 160434ff6846SIoana Radulescu } 160534ff6846SIoana Radulescu 160634ff6846SIoana Radulescu return 0; 160734ff6846SIoana Radulescu } 160834ff6846SIoana Radulescu 16095d8dccf8SIoana Ciornei static int dpaa2_eth_set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 161034ff6846SIoana Radulescu { 161134ff6846SIoana Radulescu int err; 161234ff6846SIoana Radulescu 161334ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 161434ff6846SIoana Radulescu DPNI_OFF_TX_L3_CSUM, enable); 161534ff6846SIoana Radulescu if (err) { 161634ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 161734ff6846SIoana Radulescu return err; 161834ff6846SIoana Radulescu } 161934ff6846SIoana Radulescu 162034ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 162134ff6846SIoana Radulescu DPNI_OFF_TX_L4_CSUM, enable); 162234ff6846SIoana Radulescu if (err) { 162334ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 162434ff6846SIoana Radulescu return err; 162534ff6846SIoana Radulescu } 162634ff6846SIoana Radulescu 162734ff6846SIoana Radulescu return 0; 162834ff6846SIoana Radulescu } 162934ff6846SIoana Radulescu 163034ff6846SIoana Radulescu /* Perform a single release command to add buffers 163134ff6846SIoana Radulescu * to the specified buffer pool 163234ff6846SIoana Radulescu */ 16335d8dccf8SIoana Ciornei static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv, 1634095174daSRobert-Ionut Alexa struct dpaa2_eth_channel *ch) 163534ff6846SIoana Radulescu { 163634ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 163734ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 163827c87486SIoana Ciocoi Radulescu struct page *page; 163934ff6846SIoana Radulescu dma_addr_t addr; 1640ef17bd7cSIoana Radulescu int retries = 0; 164134ff6846SIoana Radulescu int i, err; 164234ff6846SIoana Radulescu 164334ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 164434ff6846SIoana Radulescu /* Allocate buffer visible to WRIOP + skb shared info + 164534ff6846SIoana Radulescu * alignment padding 164634ff6846SIoana Radulescu */ 164727c87486SIoana Ciocoi Radulescu /* allocate one page for each Rx buffer. WRIOP sees 164827c87486SIoana Ciocoi Radulescu * the entire page except for a tailroom reserved for 164927c87486SIoana Ciocoi Radulescu * skb shared info 165027c87486SIoana Ciocoi Radulescu */ 165127c87486SIoana Ciocoi Radulescu page = dev_alloc_pages(0); 165227c87486SIoana Ciocoi Radulescu if (!page) 165334ff6846SIoana Radulescu goto err_alloc; 165434ff6846SIoana Radulescu 1655efa6a7d0SIoana Ciornei addr = dma_map_page(dev, page, 0, priv->rx_buf_size, 165618c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 165734ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 165834ff6846SIoana Radulescu goto err_map; 165934ff6846SIoana Radulescu 166034ff6846SIoana Radulescu buf_array[i] = addr; 166134ff6846SIoana Radulescu 166234ff6846SIoana Radulescu /* tracing point */ 1663e34f4934SChen Lin trace_dpaa2_eth_buf_seed(priv->net_dev, page_address(page), 1664e34f4934SChen Lin DPAA2_ETH_RX_BUF_RAW_SIZE, 1665efa6a7d0SIoana Ciornei addr, priv->rx_buf_size, 1666095174daSRobert-Ionut Alexa ch->bp->bpid); 166734ff6846SIoana Radulescu } 166834ff6846SIoana Radulescu 166934ff6846SIoana Radulescu release_bufs: 167034ff6846SIoana Radulescu /* In case the portal is busy, retry until successful */ 1671095174daSRobert-Ionut Alexa while ((err = dpaa2_io_service_release(ch->dpio, ch->bp->bpid, 1672ef17bd7cSIoana Radulescu buf_array, i)) == -EBUSY) { 1673ef17bd7cSIoana Radulescu if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 1674ef17bd7cSIoana Radulescu break; 167534ff6846SIoana Radulescu cpu_relax(); 1676ef17bd7cSIoana Radulescu } 167734ff6846SIoana Radulescu 167834ff6846SIoana Radulescu /* If release command failed, clean up and bail out; 167934ff6846SIoana Radulescu * not much else we can do about it 168034ff6846SIoana Radulescu */ 168134ff6846SIoana Radulescu if (err) { 16825d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, i); 168334ff6846SIoana Radulescu return 0; 168434ff6846SIoana Radulescu } 168534ff6846SIoana Radulescu 168634ff6846SIoana Radulescu return i; 168734ff6846SIoana Radulescu 168834ff6846SIoana Radulescu err_map: 168927c87486SIoana Ciocoi Radulescu __free_pages(page, 0); 169034ff6846SIoana Radulescu err_alloc: 169134ff6846SIoana Radulescu /* If we managed to allocate at least some buffers, 169234ff6846SIoana Radulescu * release them to hardware 169334ff6846SIoana Radulescu */ 169434ff6846SIoana Radulescu if (i) 169534ff6846SIoana Radulescu goto release_bufs; 169634ff6846SIoana Radulescu 169734ff6846SIoana Radulescu return 0; 169834ff6846SIoana Radulescu } 169934ff6846SIoana Radulescu 1700095174daSRobert-Ionut Alexa static int dpaa2_eth_seed_pool(struct dpaa2_eth_priv *priv, 1701095174daSRobert-Ionut Alexa struct dpaa2_eth_channel *ch) 170234ff6846SIoana Radulescu { 1703095174daSRobert-Ionut Alexa int i; 170434ff6846SIoana Radulescu int new_count; 170534ff6846SIoana Radulescu 1706095174daSRobert-Ionut Alexa for (i = 0; i < DPAA2_ETH_NUM_BUFS; i += DPAA2_ETH_BUFS_PER_CMD) { 1707095174daSRobert-Ionut Alexa new_count = dpaa2_eth_add_bufs(priv, ch); 1708095174daSRobert-Ionut Alexa ch->buf_count += new_count; 170934ff6846SIoana Radulescu 1710095174daSRobert-Ionut Alexa if (new_count < DPAA2_ETH_BUFS_PER_CMD) 171134ff6846SIoana Radulescu return -ENOMEM; 171234ff6846SIoana Radulescu } 171334ff6846SIoana Radulescu 171434ff6846SIoana Radulescu return 0; 171534ff6846SIoana Radulescu } 171634ff6846SIoana Radulescu 1717095174daSRobert-Ionut Alexa static void dpaa2_eth_seed_pools(struct dpaa2_eth_priv *priv) 1718095174daSRobert-Ionut Alexa { 1719095174daSRobert-Ionut Alexa struct net_device *net_dev = priv->net_dev; 1720095174daSRobert-Ionut Alexa struct dpaa2_eth_channel *channel; 1721095174daSRobert-Ionut Alexa int i, err = 0; 1722095174daSRobert-Ionut Alexa 1723095174daSRobert-Ionut Alexa for (i = 0; i < priv->num_channels; i++) { 1724095174daSRobert-Ionut Alexa channel = priv->channel[i]; 1725095174daSRobert-Ionut Alexa 1726095174daSRobert-Ionut Alexa err = dpaa2_eth_seed_pool(priv, channel); 1727095174daSRobert-Ionut Alexa 1728095174daSRobert-Ionut Alexa /* Not much to do; the buffer pool, though not filled up, 1729095174daSRobert-Ionut Alexa * may still contain some buffers which would enable us 1730095174daSRobert-Ionut Alexa * to limp on. 1731095174daSRobert-Ionut Alexa */ 1732095174daSRobert-Ionut Alexa if (err) 1733095174daSRobert-Ionut Alexa netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 1734095174daSRobert-Ionut Alexa channel->bp->dev->obj_desc.id, 1735095174daSRobert-Ionut Alexa channel->bp->bpid); 1736095174daSRobert-Ionut Alexa } 1737095174daSRobert-Ionut Alexa } 1738095174daSRobert-Ionut Alexa 1739d0ea5cbdSJesse Brandeburg /* 1740095174daSRobert-Ionut Alexa * Drain the specified number of buffers from one of the DPNI's private buffer 1741095174daSRobert-Ionut Alexa * pools. 174234ff6846SIoana Radulescu * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 174334ff6846SIoana Radulescu */ 1744095174daSRobert-Ionut Alexa static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int bpid, 1745095174daSRobert-Ionut Alexa int count) 174634ff6846SIoana Radulescu { 174734ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1748ef17bd7cSIoana Radulescu int retries = 0; 174934ff6846SIoana Radulescu int ret; 175034ff6846SIoana Radulescu 175134ff6846SIoana Radulescu do { 1752095174daSRobert-Ionut Alexa ret = dpaa2_io_service_acquire(NULL, bpid, buf_array, count); 175334ff6846SIoana Radulescu if (ret < 0) { 1754ef17bd7cSIoana Radulescu if (ret == -EBUSY && 17550e5ad75bSIoana Ciornei retries++ < DPAA2_ETH_SWP_BUSY_RETRIES) 1756ef17bd7cSIoana Radulescu continue; 175734ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 175834ff6846SIoana Radulescu return; 175934ff6846SIoana Radulescu } 17605d8dccf8SIoana Ciornei dpaa2_eth_free_bufs(priv, buf_array, ret); 1761ef17bd7cSIoana Radulescu retries = 0; 176234ff6846SIoana Radulescu } while (ret); 176334ff6846SIoana Radulescu } 176434ff6846SIoana Radulescu 1765095174daSRobert-Ionut Alexa static void dpaa2_eth_drain_pool(struct dpaa2_eth_priv *priv, int bpid) 176634ff6846SIoana Radulescu { 176734ff6846SIoana Radulescu int i; 176834ff6846SIoana Radulescu 1769095174daSRobert-Ionut Alexa /* Drain the buffer pool */ 1770095174daSRobert-Ionut Alexa dpaa2_eth_drain_bufs(priv, bpid, DPAA2_ETH_BUFS_PER_CMD); 1771095174daSRobert-Ionut Alexa dpaa2_eth_drain_bufs(priv, bpid, 1); 177234ff6846SIoana Radulescu 1773095174daSRobert-Ionut Alexa /* Setup to zero the buffer count of all channels which were 1774095174daSRobert-Ionut Alexa * using this buffer pool. 1775095174daSRobert-Ionut Alexa */ 177634ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 1777095174daSRobert-Ionut Alexa if (priv->channel[i]->bp->bpid == bpid) 177834ff6846SIoana Radulescu priv->channel[i]->buf_count = 0; 177934ff6846SIoana Radulescu } 178034ff6846SIoana Radulescu 1781095174daSRobert-Ionut Alexa static void dpaa2_eth_drain_pools(struct dpaa2_eth_priv *priv) 1782095174daSRobert-Ionut Alexa { 1783095174daSRobert-Ionut Alexa int i; 1784095174daSRobert-Ionut Alexa 1785095174daSRobert-Ionut Alexa for (i = 0; i < priv->num_bps; i++) 1786095174daSRobert-Ionut Alexa dpaa2_eth_drain_pool(priv, priv->bp[i]->bpid); 1787095174daSRobert-Ionut Alexa } 1788095174daSRobert-Ionut Alexa 178934ff6846SIoana Radulescu /* Function is called from softirq context only, so we don't need to guard 179034ff6846SIoana Radulescu * the access to percpu count 179134ff6846SIoana Radulescu */ 17925d8dccf8SIoana Ciornei static int dpaa2_eth_refill_pool(struct dpaa2_eth_priv *priv, 1793095174daSRobert-Ionut Alexa struct dpaa2_eth_channel *ch) 179434ff6846SIoana Radulescu { 179534ff6846SIoana Radulescu int new_count; 179634ff6846SIoana Radulescu 179734ff6846SIoana Radulescu if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 179834ff6846SIoana Radulescu return 0; 179934ff6846SIoana Radulescu 180034ff6846SIoana Radulescu do { 1801095174daSRobert-Ionut Alexa new_count = dpaa2_eth_add_bufs(priv, ch); 180234ff6846SIoana Radulescu if (unlikely(!new_count)) { 180334ff6846SIoana Radulescu /* Out of memory; abort for now, we'll try later on */ 180434ff6846SIoana Radulescu break; 180534ff6846SIoana Radulescu } 180634ff6846SIoana Radulescu ch->buf_count += new_count; 180734ff6846SIoana Radulescu } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 180834ff6846SIoana Radulescu 180934ff6846SIoana Radulescu if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 181034ff6846SIoana Radulescu return -ENOMEM; 181134ff6846SIoana Radulescu 181234ff6846SIoana Radulescu return 0; 181334ff6846SIoana Radulescu } 181434ff6846SIoana Radulescu 1815d70446eeSIoana Ciornei static void dpaa2_eth_sgt_cache_drain(struct dpaa2_eth_priv *priv) 1816d70446eeSIoana Ciornei { 1817d70446eeSIoana Ciornei struct dpaa2_eth_sgt_cache *sgt_cache; 1818d70446eeSIoana Ciornei u16 count; 1819d70446eeSIoana Ciornei int k, i; 1820d70446eeSIoana Ciornei 18210fe665d4SIoana Ciornei for_each_possible_cpu(k) { 1822d70446eeSIoana Ciornei sgt_cache = per_cpu_ptr(priv->sgt_cache, k); 1823d70446eeSIoana Ciornei count = sgt_cache->count; 1824d70446eeSIoana Ciornei 1825d70446eeSIoana Ciornei for (i = 0; i < count; i++) 18268378a791SIoana Ciornei skb_free_frag(sgt_cache->buf[i]); 1827d70446eeSIoana Ciornei sgt_cache->count = 0; 1828d70446eeSIoana Ciornei } 1829d70446eeSIoana Ciornei } 1830d70446eeSIoana Ciornei 18315d8dccf8SIoana Ciornei static int dpaa2_eth_pull_channel(struct dpaa2_eth_channel *ch) 183234ff6846SIoana Radulescu { 183334ff6846SIoana Radulescu int err; 183434ff6846SIoana Radulescu int dequeues = -1; 183534ff6846SIoana Radulescu 183634ff6846SIoana Radulescu /* Retry while portal is busy */ 183734ff6846SIoana Radulescu do { 183834ff6846SIoana Radulescu err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 183934ff6846SIoana Radulescu ch->store); 184034ff6846SIoana Radulescu dequeues++; 184134ff6846SIoana Radulescu cpu_relax(); 1842ef17bd7cSIoana Radulescu } while (err == -EBUSY && dequeues < DPAA2_ETH_SWP_BUSY_RETRIES); 184334ff6846SIoana Radulescu 184434ff6846SIoana Radulescu ch->stats.dequeue_portal_busy += dequeues; 184534ff6846SIoana Radulescu if (unlikely(err)) 184634ff6846SIoana Radulescu ch->stats.pull_err++; 184734ff6846SIoana Radulescu 184834ff6846SIoana Radulescu return err; 184934ff6846SIoana Radulescu } 185034ff6846SIoana Radulescu 185134ff6846SIoana Radulescu /* NAPI poll routine 185234ff6846SIoana Radulescu * 185334ff6846SIoana Radulescu * Frames are dequeued from the QMan channel associated with this NAPI context. 185434ff6846SIoana Radulescu * Rx, Tx confirmation and (if configured) Rx error frames all count 185534ff6846SIoana Radulescu * towards the NAPI budget. 185634ff6846SIoana Radulescu */ 185734ff6846SIoana Radulescu static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 185834ff6846SIoana Radulescu { 185934ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 186034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 186168049a5fSIoana Ciocoi Radulescu int rx_cleaned = 0, txconf_cleaned = 0; 1862569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1863569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1864569dac6aSIoana Ciocoi Radulescu int store_cleaned, work_done; 18650a25d92cSIoana Ciornei struct list_head rx_list; 1866ef17bd7cSIoana Radulescu int retries = 0; 186774a1c059SIoana Ciornei u16 flowid; 186834ff6846SIoana Radulescu int err; 186934ff6846SIoana Radulescu 187034ff6846SIoana Radulescu ch = container_of(napi, struct dpaa2_eth_channel, napi); 1871d678be1dSIoana Radulescu ch->xdp.res = 0; 187234ff6846SIoana Radulescu priv = ch->priv; 187334ff6846SIoana Radulescu 18740a25d92cSIoana Ciornei INIT_LIST_HEAD(&rx_list); 18750a25d92cSIoana Ciornei ch->rx_list = &rx_list; 18760a25d92cSIoana Ciornei 187768049a5fSIoana Ciocoi Radulescu do { 18785d8dccf8SIoana Ciornei err = dpaa2_eth_pull_channel(ch); 187934ff6846SIoana Radulescu if (unlikely(err)) 188034ff6846SIoana Radulescu break; 188134ff6846SIoana Radulescu 188234ff6846SIoana Radulescu /* Refill pool if appropriate */ 1883095174daSRobert-Ionut Alexa dpaa2_eth_refill_pool(priv, ch); 188434ff6846SIoana Radulescu 18855d8dccf8SIoana Ciornei store_cleaned = dpaa2_eth_consume_frames(ch, &fq); 1886ef17bd7cSIoana Radulescu if (store_cleaned <= 0) 1887569dac6aSIoana Ciocoi Radulescu break; 1888569dac6aSIoana Ciocoi Radulescu if (fq->type == DPAA2_RX_FQ) { 188968049a5fSIoana Ciocoi Radulescu rx_cleaned += store_cleaned; 189074a1c059SIoana Ciornei flowid = fq->flowid; 1891569dac6aSIoana Ciocoi Radulescu } else { 189268049a5fSIoana Ciocoi Radulescu txconf_cleaned += store_cleaned; 1893569dac6aSIoana Ciocoi Radulescu /* We have a single Tx conf FQ on this channel */ 1894569dac6aSIoana Ciocoi Radulescu txc_fq = fq; 1895569dac6aSIoana Ciocoi Radulescu } 189634ff6846SIoana Radulescu 189768049a5fSIoana Ciocoi Radulescu /* If we either consumed the whole NAPI budget with Rx frames 189868049a5fSIoana Ciocoi Radulescu * or we reached the Tx confirmations threshold, we're done. 189934ff6846SIoana Radulescu */ 190068049a5fSIoana Ciocoi Radulescu if (rx_cleaned >= budget || 1901569dac6aSIoana Ciocoi Radulescu txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1902569dac6aSIoana Ciocoi Radulescu work_done = budget; 1903569dac6aSIoana Ciocoi Radulescu goto out; 1904569dac6aSIoana Ciocoi Radulescu } 190568049a5fSIoana Ciocoi Radulescu } while (store_cleaned); 190634ff6846SIoana Radulescu 1907fc398becSIoana Ciornei /* Update NET DIM with the values for this CDAN */ 1908fc398becSIoana Ciornei dpaa2_io_update_net_dim(ch->dpio, ch->stats.frames_per_cdan, 1909fc398becSIoana Ciornei ch->stats.bytes_per_cdan); 1910fc398becSIoana Ciornei ch->stats.frames_per_cdan = 0; 1911fc398becSIoana Ciornei ch->stats.bytes_per_cdan = 0; 1912fc398becSIoana Ciornei 191368049a5fSIoana Ciocoi Radulescu /* We didn't consume the entire budget, so finish napi and 191468049a5fSIoana Ciocoi Radulescu * re-enable data availability notifications 191568049a5fSIoana Ciocoi Radulescu */ 191668049a5fSIoana Ciocoi Radulescu napi_complete_done(napi, rx_cleaned); 191734ff6846SIoana Radulescu do { 191834ff6846SIoana Radulescu err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 191934ff6846SIoana Radulescu cpu_relax(); 1920ef17bd7cSIoana Radulescu } while (err == -EBUSY && retries++ < DPAA2_ETH_SWP_BUSY_RETRIES); 192134ff6846SIoana Radulescu WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 192234ff6846SIoana Radulescu ch->nctx.desired_cpu); 192334ff6846SIoana Radulescu 1924569dac6aSIoana Ciocoi Radulescu work_done = max(rx_cleaned, 1); 1925569dac6aSIoana Ciocoi Radulescu 1926569dac6aSIoana Ciocoi Radulescu out: 19270a25d92cSIoana Ciornei netif_receive_skb_list(ch->rx_list); 19280a25d92cSIoana Ciornei 1929d678be1dSIoana Radulescu if (txc_fq && txc_fq->dq_frames) { 1930569dac6aSIoana Ciocoi Radulescu nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1931569dac6aSIoana Ciocoi Radulescu netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1932569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes); 1933569dac6aSIoana Ciocoi Radulescu txc_fq->dq_frames = 0; 1934569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes = 0; 1935569dac6aSIoana Ciocoi Radulescu } 1936569dac6aSIoana Ciocoi Radulescu 1937d678be1dSIoana Radulescu if (ch->xdp.res & XDP_REDIRECT) 1938d678be1dSIoana Radulescu xdp_do_flush_map(); 193974a1c059SIoana Ciornei else if (rx_cleaned && ch->xdp.res & XDP_TX) 19405d8dccf8SIoana Ciornei dpaa2_eth_xdp_tx_flush(priv, ch, &priv->fq[flowid]); 1941d678be1dSIoana Radulescu 1942569dac6aSIoana Ciocoi Radulescu return work_done; 194334ff6846SIoana Radulescu } 194434ff6846SIoana Radulescu 19455d8dccf8SIoana Ciornei static void dpaa2_eth_enable_ch_napi(struct dpaa2_eth_priv *priv) 194634ff6846SIoana Radulescu { 194734ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 194834ff6846SIoana Radulescu int i; 194934ff6846SIoana Radulescu 195034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 195134ff6846SIoana Radulescu ch = priv->channel[i]; 195234ff6846SIoana Radulescu napi_enable(&ch->napi); 195334ff6846SIoana Radulescu } 195434ff6846SIoana Radulescu } 195534ff6846SIoana Radulescu 19565d8dccf8SIoana Ciornei static void dpaa2_eth_disable_ch_napi(struct dpaa2_eth_priv *priv) 195734ff6846SIoana Radulescu { 195834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 195934ff6846SIoana Radulescu int i; 196034ff6846SIoana Radulescu 196134ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 196234ff6846SIoana Radulescu ch = priv->channel[i]; 196334ff6846SIoana Radulescu napi_disable(&ch->napi); 196434ff6846SIoana Radulescu } 196534ff6846SIoana Radulescu } 196634ff6846SIoana Radulescu 196707beb165SIoana Ciornei void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, 196807beb165SIoana Ciornei bool tx_pause, bool pfc) 19698eb3cef8SIoana Radulescu { 19708eb3cef8SIoana Radulescu struct dpni_taildrop td = {0}; 1971685e39eaSIoana Radulescu struct dpaa2_eth_fq *fq; 19728eb3cef8SIoana Radulescu int i, err; 19738eb3cef8SIoana Radulescu 197407beb165SIoana Ciornei /* FQ taildrop: threshold is in bytes, per frame queue. Enabled if 197507beb165SIoana Ciornei * flow control is disabled (as it might interfere with either the 197607beb165SIoana Ciornei * buffer pool depletion trigger for pause frames or with the group 197707beb165SIoana Ciornei * congestion trigger for PFC frames) 197807beb165SIoana Ciornei */ 19792c8d1c8dSIoana Radulescu td.enable = !tx_pause; 198007beb165SIoana Ciornei if (priv->rx_fqtd_enabled == td.enable) 198107beb165SIoana Ciornei goto set_cgtd; 19828eb3cef8SIoana Radulescu 19832c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_FQ_TAILDROP_THRESH; 19842c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_BYTES; 19858eb3cef8SIoana Radulescu 19868eb3cef8SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 1987685e39eaSIoana Radulescu fq = &priv->fq[i]; 1988685e39eaSIoana Radulescu if (fq->type != DPAA2_RX_FQ) 19898eb3cef8SIoana Radulescu continue; 19908eb3cef8SIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1991685e39eaSIoana Radulescu DPNI_CP_QUEUE, DPNI_QUEUE_RX, 1992685e39eaSIoana Radulescu fq->tc, fq->flowid, &td); 19938eb3cef8SIoana Radulescu if (err) { 19948eb3cef8SIoana Radulescu netdev_err(priv->net_dev, 19952c8d1c8dSIoana Radulescu "dpni_set_taildrop(FQ) failed\n"); 19962c8d1c8dSIoana Radulescu return; 19978eb3cef8SIoana Radulescu } 19988eb3cef8SIoana Radulescu } 19998eb3cef8SIoana Radulescu 200007beb165SIoana Ciornei priv->rx_fqtd_enabled = td.enable; 200107beb165SIoana Ciornei 200207beb165SIoana Ciornei set_cgtd: 20032c8d1c8dSIoana Radulescu /* Congestion group taildrop: threshold is in frames, per group 20042c8d1c8dSIoana Radulescu * of FQs belonging to the same traffic class 200507beb165SIoana Ciornei * Enabled if general Tx pause disabled or if PFCs are enabled 200607beb165SIoana Ciornei * (congestion group threhsold for PFC generation is lower than the 200707beb165SIoana Ciornei * CG taildrop threshold, so it won't interfere with it; we also 200807beb165SIoana Ciornei * want frames in non-PFC enabled traffic classes to be kept in check) 20092c8d1c8dSIoana Radulescu */ 2010b91b3a21SJiapeng Chong td.enable = !tx_pause || pfc; 201107beb165SIoana Ciornei if (priv->rx_cgtd_enabled == td.enable) 201207beb165SIoana Ciornei return; 201307beb165SIoana Ciornei 20142c8d1c8dSIoana Radulescu td.threshold = DPAA2_ETH_CG_TAILDROP_THRESH(priv); 20152c8d1c8dSIoana Radulescu td.units = DPNI_CONGESTION_UNIT_FRAMES; 20162c8d1c8dSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 20172c8d1c8dSIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 20182c8d1c8dSIoana Radulescu DPNI_CP_GROUP, DPNI_QUEUE_RX, 20192c8d1c8dSIoana Radulescu i, 0, &td); 20202c8d1c8dSIoana Radulescu if (err) { 20212c8d1c8dSIoana Radulescu netdev_err(priv->net_dev, 20222c8d1c8dSIoana Radulescu "dpni_set_taildrop(CG) failed\n"); 20232c8d1c8dSIoana Radulescu return; 20242c8d1c8dSIoana Radulescu } 20252c8d1c8dSIoana Radulescu } 20262c8d1c8dSIoana Radulescu 202707beb165SIoana Ciornei priv->rx_cgtd_enabled = td.enable; 20288eb3cef8SIoana Radulescu } 20298eb3cef8SIoana Radulescu 20305d8dccf8SIoana Ciornei static int dpaa2_eth_link_state_update(struct dpaa2_eth_priv *priv) 203134ff6846SIoana Radulescu { 203285b7a342SIoana Ciornei struct dpni_link_state state = {0}; 20338eb3cef8SIoana Radulescu bool tx_pause; 203434ff6846SIoana Radulescu int err; 203534ff6846SIoana Radulescu 203634ff6846SIoana Radulescu err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 203734ff6846SIoana Radulescu if (unlikely(err)) { 203834ff6846SIoana Radulescu netdev_err(priv->net_dev, 203934ff6846SIoana Radulescu "dpni_get_link_state() failed\n"); 204034ff6846SIoana Radulescu return err; 204134ff6846SIoana Radulescu } 204234ff6846SIoana Radulescu 20438eb3cef8SIoana Radulescu /* If Tx pause frame settings have changed, we need to update 20448eb3cef8SIoana Radulescu * Rx FQ taildrop configuration as well. We configure taildrop 20458eb3cef8SIoana Radulescu * only when pause frame generation is disabled. 20468eb3cef8SIoana Radulescu */ 2047ad054f26SIoana Radulescu tx_pause = dpaa2_eth_tx_pause_enabled(state.options); 204807beb165SIoana Ciornei dpaa2_eth_set_rx_taildrop(priv, tx_pause, priv->pfc_enabled); 20498eb3cef8SIoana Radulescu 205071947923SIoana Ciornei /* When we manage the MAC/PHY using phylink there is no need 205171947923SIoana Ciornei * to manually update the netif_carrier. 205271947923SIoana Ciornei */ 2053d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 205471947923SIoana Ciornei goto out; 205571947923SIoana Ciornei 205634ff6846SIoana Radulescu /* Chech link state; speed / duplex changes are not treated yet */ 205734ff6846SIoana Radulescu if (priv->link_state.up == state.up) 2058cce62943SIoana Radulescu goto out; 205934ff6846SIoana Radulescu 206034ff6846SIoana Radulescu if (state.up) { 206134ff6846SIoana Radulescu netif_carrier_on(priv->net_dev); 206234ff6846SIoana Radulescu netif_tx_start_all_queues(priv->net_dev); 206334ff6846SIoana Radulescu } else { 206434ff6846SIoana Radulescu netif_tx_stop_all_queues(priv->net_dev); 206534ff6846SIoana Radulescu netif_carrier_off(priv->net_dev); 206634ff6846SIoana Radulescu } 206734ff6846SIoana Radulescu 206834ff6846SIoana Radulescu netdev_info(priv->net_dev, "Link Event: state %s\n", 206934ff6846SIoana Radulescu state.up ? "up" : "down"); 207034ff6846SIoana Radulescu 2071cce62943SIoana Radulescu out: 2072cce62943SIoana Radulescu priv->link_state = state; 2073cce62943SIoana Radulescu 207434ff6846SIoana Radulescu return 0; 207534ff6846SIoana Radulescu } 207634ff6846SIoana Radulescu 207734ff6846SIoana Radulescu static int dpaa2_eth_open(struct net_device *net_dev) 207834ff6846SIoana Radulescu { 207934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 208034ff6846SIoana Radulescu int err; 208134ff6846SIoana Radulescu 2082095174daSRobert-Ionut Alexa dpaa2_eth_seed_pools(priv); 208334ff6846SIoana Radulescu 2084d87e6063SIoana Ciornei if (!dpaa2_eth_is_type_phy(priv)) { 208571947923SIoana Ciornei /* We'll only start the txqs when the link is actually ready; 208671947923SIoana Ciornei * make sure we don't race against the link up notification, 208771947923SIoana Ciornei * which may come immediately after dpni_enable(); 208834ff6846SIoana Radulescu */ 208934ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 209071947923SIoana Ciornei 209171947923SIoana Ciornei /* Also, explicitly set carrier off, otherwise 209271947923SIoana Ciornei * netif_carrier_ok() will return true and cause 'ip link show' 209371947923SIoana Ciornei * to report the LOWER_UP flag, even though the link 209471947923SIoana Ciornei * notification wasn't even received. 209534ff6846SIoana Radulescu */ 209634ff6846SIoana Radulescu netif_carrier_off(net_dev); 209771947923SIoana Ciornei } 20985d8dccf8SIoana Ciornei dpaa2_eth_enable_ch_napi(priv); 209934ff6846SIoana Radulescu 210034ff6846SIoana Radulescu err = dpni_enable(priv->mc_io, 0, priv->mc_token); 210134ff6846SIoana Radulescu if (err < 0) { 210234ff6846SIoana Radulescu netdev_err(net_dev, "dpni_enable() failed\n"); 210334ff6846SIoana Radulescu goto enable_err; 210434ff6846SIoana Radulescu } 210534ff6846SIoana Radulescu 2106f978fe85SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 2107f978fe85SIoana Ciornei dpaa2_mac_start(priv->mac); 210871947923SIoana Ciornei phylink_start(priv->mac->phylink); 2109f978fe85SIoana Ciornei } 211034ff6846SIoana Radulescu 211134ff6846SIoana Radulescu return 0; 211234ff6846SIoana Radulescu 211334ff6846SIoana Radulescu enable_err: 21145d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 2115095174daSRobert-Ionut Alexa dpaa2_eth_drain_pools(priv); 211634ff6846SIoana Radulescu return err; 211734ff6846SIoana Radulescu } 211834ff6846SIoana Radulescu 211968d74315SIoana Ciocoi Radulescu /* Total number of in-flight frames on ingress queues */ 21205d8dccf8SIoana Ciornei static u32 dpaa2_eth_ingress_fq_count(struct dpaa2_eth_priv *priv) 212134ff6846SIoana Radulescu { 212268d74315SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq; 212368d74315SIoana Ciocoi Radulescu u32 fcnt = 0, bcnt = 0, total = 0; 212468d74315SIoana Ciocoi Radulescu int i, err; 212534ff6846SIoana Radulescu 212668d74315SIoana Ciocoi Radulescu for (i = 0; i < priv->num_fqs; i++) { 212768d74315SIoana Ciocoi Radulescu fq = &priv->fq[i]; 212868d74315SIoana Ciocoi Radulescu err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 212968d74315SIoana Ciocoi Radulescu if (err) { 213068d74315SIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "query_fq_count failed"); 213168d74315SIoana Ciocoi Radulescu break; 213268d74315SIoana Ciocoi Radulescu } 213368d74315SIoana Ciocoi Radulescu total += fcnt; 213468d74315SIoana Ciocoi Radulescu } 213534ff6846SIoana Radulescu 213634ff6846SIoana Radulescu return total; 213734ff6846SIoana Radulescu } 213834ff6846SIoana Radulescu 21395d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 214034ff6846SIoana Radulescu { 214168d74315SIoana Ciocoi Radulescu int retries = 10; 214268d74315SIoana Ciocoi Radulescu u32 pending; 214334ff6846SIoana Radulescu 214468d74315SIoana Ciocoi Radulescu do { 21455d8dccf8SIoana Ciornei pending = dpaa2_eth_ingress_fq_count(priv); 214668d74315SIoana Ciocoi Radulescu if (pending) 214768d74315SIoana Ciocoi Radulescu msleep(100); 214868d74315SIoana Ciocoi Radulescu } while (pending && --retries); 214934ff6846SIoana Radulescu } 215034ff6846SIoana Radulescu 215152b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MAJOR 7 215252b6a4ffSIoana Radulescu #define DPNI_TX_PENDING_VER_MINOR 13 21535d8dccf8SIoana Ciornei static void dpaa2_eth_wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 215452b6a4ffSIoana Radulescu { 215552b6a4ffSIoana Radulescu union dpni_statistics stats; 215652b6a4ffSIoana Radulescu int retries = 10; 215752b6a4ffSIoana Radulescu int err; 215852b6a4ffSIoana Radulescu 215952b6a4ffSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 216052b6a4ffSIoana Radulescu DPNI_TX_PENDING_VER_MINOR) < 0) 216152b6a4ffSIoana Radulescu goto out; 216252b6a4ffSIoana Radulescu 216352b6a4ffSIoana Radulescu do { 216452b6a4ffSIoana Radulescu err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 216552b6a4ffSIoana Radulescu &stats); 216652b6a4ffSIoana Radulescu if (err) 216752b6a4ffSIoana Radulescu goto out; 216852b6a4ffSIoana Radulescu if (stats.page_6.tx_pending_frames == 0) 216952b6a4ffSIoana Radulescu return; 217052b6a4ffSIoana Radulescu } while (--retries); 217152b6a4ffSIoana Radulescu 217252b6a4ffSIoana Radulescu out: 217352b6a4ffSIoana Radulescu msleep(500); 217452b6a4ffSIoana Radulescu } 217552b6a4ffSIoana Radulescu 217634ff6846SIoana Radulescu static int dpaa2_eth_stop(struct net_device *net_dev) 217734ff6846SIoana Radulescu { 217834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 217985b7a342SIoana Ciornei int dpni_enabled = 0; 218034ff6846SIoana Radulescu int retries = 10; 218134ff6846SIoana Radulescu 2182d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 2183d87e6063SIoana Ciornei phylink_stop(priv->mac->phylink); 2184f978fe85SIoana Ciornei dpaa2_mac_stop(priv->mac); 2185d87e6063SIoana Ciornei } else { 218634ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 218734ff6846SIoana Radulescu netif_carrier_off(net_dev); 218871947923SIoana Ciornei } 218934ff6846SIoana Radulescu 219068d74315SIoana Ciocoi Radulescu /* On dpni_disable(), the MC firmware will: 219168d74315SIoana Ciocoi Radulescu * - stop MAC Rx and wait for all Rx frames to be enqueued to software 219268d74315SIoana Ciocoi Radulescu * - cut off WRIOP dequeues from egress FQs and wait until transmission 219368d74315SIoana Ciocoi Radulescu * of all in flight Tx frames is finished (and corresponding Tx conf 219468d74315SIoana Ciocoi Radulescu * frames are enqueued back to software) 219568d74315SIoana Ciocoi Radulescu * 219668d74315SIoana Ciocoi Radulescu * Before calling dpni_disable(), we wait for all Tx frames to arrive 219768d74315SIoana Ciocoi Radulescu * on WRIOP. After it finishes, wait until all remaining frames on Rx 219868d74315SIoana Ciocoi Radulescu * and Tx conf queues are consumed on NAPI poll. 219934ff6846SIoana Radulescu */ 22005d8dccf8SIoana Ciornei dpaa2_eth_wait_for_egress_fq_empty(priv); 220168d74315SIoana Ciocoi Radulescu 220234ff6846SIoana Radulescu do { 220334ff6846SIoana Radulescu dpni_disable(priv->mc_io, 0, priv->mc_token); 220434ff6846SIoana Radulescu dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 220534ff6846SIoana Radulescu if (dpni_enabled) 220634ff6846SIoana Radulescu /* Allow the hardware some slack */ 220734ff6846SIoana Radulescu msleep(100); 220834ff6846SIoana Radulescu } while (dpni_enabled && --retries); 220934ff6846SIoana Radulescu if (!retries) { 221034ff6846SIoana Radulescu netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 221134ff6846SIoana Radulescu /* Must go on and disable NAPI nonetheless, so we don't crash at 221234ff6846SIoana Radulescu * the next "ifconfig up" 221334ff6846SIoana Radulescu */ 221434ff6846SIoana Radulescu } 221534ff6846SIoana Radulescu 22165d8dccf8SIoana Ciornei dpaa2_eth_wait_for_ingress_fq_empty(priv); 22175d8dccf8SIoana Ciornei dpaa2_eth_disable_ch_napi(priv); 221834ff6846SIoana Radulescu 221934ff6846SIoana Radulescu /* Empty the buffer pool */ 2220095174daSRobert-Ionut Alexa dpaa2_eth_drain_pools(priv); 222134ff6846SIoana Radulescu 2222d70446eeSIoana Ciornei /* Empty the Scatter-Gather Buffer cache */ 2223d70446eeSIoana Ciornei dpaa2_eth_sgt_cache_drain(priv); 2224d70446eeSIoana Ciornei 222534ff6846SIoana Radulescu return 0; 222634ff6846SIoana Radulescu } 222734ff6846SIoana Radulescu 222834ff6846SIoana Radulescu static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 222934ff6846SIoana Radulescu { 223034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 223134ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 223234ff6846SIoana Radulescu int err; 223334ff6846SIoana Radulescu 223434ff6846SIoana Radulescu err = eth_mac_addr(net_dev, addr); 223534ff6846SIoana Radulescu if (err < 0) { 223634ff6846SIoana Radulescu dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 223734ff6846SIoana Radulescu return err; 223834ff6846SIoana Radulescu } 223934ff6846SIoana Radulescu 224034ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 224134ff6846SIoana Radulescu net_dev->dev_addr); 224234ff6846SIoana Radulescu if (err) { 224334ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 224434ff6846SIoana Radulescu return err; 224534ff6846SIoana Radulescu } 224634ff6846SIoana Radulescu 224734ff6846SIoana Radulescu return 0; 224834ff6846SIoana Radulescu } 224934ff6846SIoana Radulescu 225034ff6846SIoana Radulescu /** Fill in counters maintained by the GPP driver. These may be different from 225134ff6846SIoana Radulescu * the hardware counters obtained by ethtool. 225234ff6846SIoana Radulescu */ 225334ff6846SIoana Radulescu static void dpaa2_eth_get_stats(struct net_device *net_dev, 225434ff6846SIoana Radulescu struct rtnl_link_stats64 *stats) 225534ff6846SIoana Radulescu { 225634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 225734ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 225834ff6846SIoana Radulescu u64 *cpustats; 225934ff6846SIoana Radulescu u64 *netstats = (u64 *)stats; 226034ff6846SIoana Radulescu int i, j; 226134ff6846SIoana Radulescu int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 226234ff6846SIoana Radulescu 226334ff6846SIoana Radulescu for_each_possible_cpu(i) { 226434ff6846SIoana Radulescu percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 226534ff6846SIoana Radulescu cpustats = (u64 *)percpu_stats; 226634ff6846SIoana Radulescu for (j = 0; j < num; j++) 226734ff6846SIoana Radulescu netstats[j] += cpustats[j]; 226834ff6846SIoana Radulescu } 226934ff6846SIoana Radulescu } 227034ff6846SIoana Radulescu 227134ff6846SIoana Radulescu /* Copy mac unicast addresses from @net_dev to @priv. 227234ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 227334ff6846SIoana Radulescu */ 22745d8dccf8SIoana Ciornei static void dpaa2_eth_add_uc_hw_addr(const struct net_device *net_dev, 227534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 227634ff6846SIoana Radulescu { 227734ff6846SIoana Radulescu struct netdev_hw_addr *ha; 227834ff6846SIoana Radulescu int err; 227934ff6846SIoana Radulescu 228034ff6846SIoana Radulescu netdev_for_each_uc_addr(ha, net_dev) { 228134ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 228234ff6846SIoana Radulescu ha->addr); 228334ff6846SIoana Radulescu if (err) 228434ff6846SIoana Radulescu netdev_warn(priv->net_dev, 228534ff6846SIoana Radulescu "Could not add ucast MAC %pM to the filtering table (err %d)\n", 228634ff6846SIoana Radulescu ha->addr, err); 228734ff6846SIoana Radulescu } 228834ff6846SIoana Radulescu } 228934ff6846SIoana Radulescu 229034ff6846SIoana Radulescu /* Copy mac multicast addresses from @net_dev to @priv 229134ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 229234ff6846SIoana Radulescu */ 22935d8dccf8SIoana Ciornei static void dpaa2_eth_add_mc_hw_addr(const struct net_device *net_dev, 229434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 229534ff6846SIoana Radulescu { 229634ff6846SIoana Radulescu struct netdev_hw_addr *ha; 229734ff6846SIoana Radulescu int err; 229834ff6846SIoana Radulescu 229934ff6846SIoana Radulescu netdev_for_each_mc_addr(ha, net_dev) { 230034ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 230134ff6846SIoana Radulescu ha->addr); 230234ff6846SIoana Radulescu if (err) 230334ff6846SIoana Radulescu netdev_warn(priv->net_dev, 230434ff6846SIoana Radulescu "Could not add mcast MAC %pM to the filtering table (err %d)\n", 230534ff6846SIoana Radulescu ha->addr, err); 230634ff6846SIoana Radulescu } 230734ff6846SIoana Radulescu } 230834ff6846SIoana Radulescu 230970b32d82SIonut-robert Aron static int dpaa2_eth_rx_add_vid(struct net_device *net_dev, 231070b32d82SIonut-robert Aron __be16 vlan_proto, u16 vid) 231170b32d82SIonut-robert Aron { 231270b32d82SIonut-robert Aron struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 231370b32d82SIonut-robert Aron int err; 231470b32d82SIonut-robert Aron 231570b32d82SIonut-robert Aron err = dpni_add_vlan_id(priv->mc_io, 0, priv->mc_token, 231670b32d82SIonut-robert Aron vid, 0, 0, 0); 231770b32d82SIonut-robert Aron 231870b32d82SIonut-robert Aron if (err) { 231970b32d82SIonut-robert Aron netdev_warn(priv->net_dev, 232070b32d82SIonut-robert Aron "Could not add the vlan id %u\n", 232170b32d82SIonut-robert Aron vid); 232270b32d82SIonut-robert Aron return err; 232370b32d82SIonut-robert Aron } 232470b32d82SIonut-robert Aron 232570b32d82SIonut-robert Aron return 0; 232670b32d82SIonut-robert Aron } 232770b32d82SIonut-robert Aron 232870b32d82SIonut-robert Aron static int dpaa2_eth_rx_kill_vid(struct net_device *net_dev, 232970b32d82SIonut-robert Aron __be16 vlan_proto, u16 vid) 233070b32d82SIonut-robert Aron { 233170b32d82SIonut-robert Aron struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 233270b32d82SIonut-robert Aron int err; 233370b32d82SIonut-robert Aron 233470b32d82SIonut-robert Aron err = dpni_remove_vlan_id(priv->mc_io, 0, priv->mc_token, vid); 233570b32d82SIonut-robert Aron 233670b32d82SIonut-robert Aron if (err) { 233770b32d82SIonut-robert Aron netdev_warn(priv->net_dev, 233870b32d82SIonut-robert Aron "Could not remove the vlan id %u\n", 233970b32d82SIonut-robert Aron vid); 234070b32d82SIonut-robert Aron return err; 234170b32d82SIonut-robert Aron } 234270b32d82SIonut-robert Aron 234370b32d82SIonut-robert Aron return 0; 234470b32d82SIonut-robert Aron } 234570b32d82SIonut-robert Aron 234634ff6846SIoana Radulescu static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 234734ff6846SIoana Radulescu { 234834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 234934ff6846SIoana Radulescu int uc_count = netdev_uc_count(net_dev); 235034ff6846SIoana Radulescu int mc_count = netdev_mc_count(net_dev); 235134ff6846SIoana Radulescu u8 max_mac = priv->dpni_attrs.mac_filter_entries; 235234ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 235334ff6846SIoana Radulescu u16 mc_token = priv->mc_token; 235434ff6846SIoana Radulescu struct fsl_mc_io *mc_io = priv->mc_io; 235534ff6846SIoana Radulescu int err; 235634ff6846SIoana Radulescu 235734ff6846SIoana Radulescu /* Basic sanity checks; these probably indicate a misconfiguration */ 235834ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 235934ff6846SIoana Radulescu netdev_info(net_dev, 236034ff6846SIoana Radulescu "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 236134ff6846SIoana Radulescu max_mac); 236234ff6846SIoana Radulescu 236334ff6846SIoana Radulescu /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 236434ff6846SIoana Radulescu if (uc_count > max_mac) { 236534ff6846SIoana Radulescu netdev_info(net_dev, 236634ff6846SIoana Radulescu "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 236734ff6846SIoana Radulescu uc_count, max_mac); 236834ff6846SIoana Radulescu goto force_promisc; 236934ff6846SIoana Radulescu } 237034ff6846SIoana Radulescu if (mc_count + uc_count > max_mac) { 237134ff6846SIoana Radulescu netdev_info(net_dev, 237234ff6846SIoana Radulescu "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 237334ff6846SIoana Radulescu uc_count + mc_count, max_mac); 237434ff6846SIoana Radulescu goto force_mc_promisc; 237534ff6846SIoana Radulescu } 237634ff6846SIoana Radulescu 237734ff6846SIoana Radulescu /* Adjust promisc settings due to flag combinations */ 237834ff6846SIoana Radulescu if (net_dev->flags & IFF_PROMISC) 237934ff6846SIoana Radulescu goto force_promisc; 238034ff6846SIoana Radulescu if (net_dev->flags & IFF_ALLMULTI) { 238134ff6846SIoana Radulescu /* First, rebuild unicast filtering table. This should be done 238234ff6846SIoana Radulescu * in promisc mode, in order to avoid frame loss while we 238334ff6846SIoana Radulescu * progressively add entries to the table. 238434ff6846SIoana Radulescu * We don't know whether we had been in promisc already, and 238534ff6846SIoana Radulescu * making an MC call to find out is expensive; so set uc promisc 238634ff6846SIoana Radulescu * nonetheless. 238734ff6846SIoana Radulescu */ 238834ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 238934ff6846SIoana Radulescu if (err) 239034ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc\n"); 239134ff6846SIoana Radulescu 239234ff6846SIoana Radulescu /* Actual uc table reconstruction. */ 239334ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 239434ff6846SIoana Radulescu if (err) 239534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc filters\n"); 23965d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 239734ff6846SIoana Radulescu 239834ff6846SIoana Radulescu /* Finally, clear uc promisc and set mc promisc as requested. */ 239934ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 240034ff6846SIoana Radulescu if (err) 240134ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc promisc\n"); 240234ff6846SIoana Radulescu goto force_mc_promisc; 240334ff6846SIoana Radulescu } 240434ff6846SIoana Radulescu 240534ff6846SIoana Radulescu /* Neither unicast, nor multicast promisc will be on... eventually. 240634ff6846SIoana Radulescu * For now, rebuild mac filtering tables while forcing both of them on. 240734ff6846SIoana Radulescu */ 240834ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 240934ff6846SIoana Radulescu if (err) 241034ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 241134ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 241234ff6846SIoana Radulescu if (err) 241334ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 241434ff6846SIoana Radulescu 241534ff6846SIoana Radulescu /* Actual mac filtering tables reconstruction */ 241634ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 241734ff6846SIoana Radulescu if (err) 241834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mac filters\n"); 24195d8dccf8SIoana Ciornei dpaa2_eth_add_mc_hw_addr(net_dev, priv); 24205d8dccf8SIoana Ciornei dpaa2_eth_add_uc_hw_addr(net_dev, priv); 242134ff6846SIoana Radulescu 242234ff6846SIoana Radulescu /* Now we can clear both ucast and mcast promisc, without risking 242334ff6846SIoana Radulescu * to drop legitimate frames anymore. 242434ff6846SIoana Radulescu */ 242534ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 242634ff6846SIoana Radulescu if (err) 242734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear ucast promisc\n"); 242834ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 242934ff6846SIoana Radulescu if (err) 243034ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mcast promisc\n"); 243134ff6846SIoana Radulescu 243234ff6846SIoana Radulescu return; 243334ff6846SIoana Radulescu 243434ff6846SIoana Radulescu force_promisc: 243534ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 243634ff6846SIoana Radulescu if (err) 243734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set ucast promisc\n"); 243834ff6846SIoana Radulescu force_mc_promisc: 243934ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 244034ff6846SIoana Radulescu if (err) 244134ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mcast promisc\n"); 244234ff6846SIoana Radulescu } 244334ff6846SIoana Radulescu 244434ff6846SIoana Radulescu static int dpaa2_eth_set_features(struct net_device *net_dev, 244534ff6846SIoana Radulescu netdev_features_t features) 244634ff6846SIoana Radulescu { 244734ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 244834ff6846SIoana Radulescu netdev_features_t changed = features ^ net_dev->features; 244934ff6846SIoana Radulescu bool enable; 245034ff6846SIoana Radulescu int err; 245134ff6846SIoana Radulescu 245270b32d82SIonut-robert Aron if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) { 245370b32d82SIonut-robert Aron enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER); 245470b32d82SIonut-robert Aron err = dpaa2_eth_set_rx_vlan_filtering(priv, enable); 245570b32d82SIonut-robert Aron if (err) 245670b32d82SIonut-robert Aron return err; 245770b32d82SIonut-robert Aron } 245870b32d82SIonut-robert Aron 245934ff6846SIoana Radulescu if (changed & NETIF_F_RXCSUM) { 246034ff6846SIoana Radulescu enable = !!(features & NETIF_F_RXCSUM); 24615d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, enable); 246234ff6846SIoana Radulescu if (err) 246334ff6846SIoana Radulescu return err; 246434ff6846SIoana Radulescu } 246534ff6846SIoana Radulescu 246634ff6846SIoana Radulescu if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 246734ff6846SIoana Radulescu enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 24685d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, enable); 246934ff6846SIoana Radulescu if (err) 247034ff6846SIoana Radulescu return err; 247134ff6846SIoana Radulescu } 247234ff6846SIoana Radulescu 247334ff6846SIoana Radulescu return 0; 247434ff6846SIoana Radulescu } 247534ff6846SIoana Radulescu 247634ff6846SIoana Radulescu static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 247734ff6846SIoana Radulescu { 247834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 247934ff6846SIoana Radulescu struct hwtstamp_config config; 248034ff6846SIoana Radulescu 2481c5521189SYangbo Lu if (!dpaa2_ptp) 2482c5521189SYangbo Lu return -EINVAL; 2483c5521189SYangbo Lu 248434ff6846SIoana Radulescu if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 248534ff6846SIoana Radulescu return -EFAULT; 248634ff6846SIoana Radulescu 248734ff6846SIoana Radulescu switch (config.tx_type) { 248834ff6846SIoana Radulescu case HWTSTAMP_TX_OFF: 248934ff6846SIoana Radulescu case HWTSTAMP_TX_ON: 2490c5521189SYangbo Lu case HWTSTAMP_TX_ONESTEP_SYNC: 24911cf773bdSYangbo Lu priv->tx_tstamp_type = config.tx_type; 249234ff6846SIoana Radulescu break; 249334ff6846SIoana Radulescu default: 249434ff6846SIoana Radulescu return -ERANGE; 249534ff6846SIoana Radulescu } 249634ff6846SIoana Radulescu 249734ff6846SIoana Radulescu if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 249834ff6846SIoana Radulescu priv->rx_tstamp = false; 249934ff6846SIoana Radulescu } else { 250034ff6846SIoana Radulescu priv->rx_tstamp = true; 250134ff6846SIoana Radulescu /* TS is set for all frame types, not only those requested */ 250234ff6846SIoana Radulescu config.rx_filter = HWTSTAMP_FILTER_ALL; 250334ff6846SIoana Radulescu } 250434ff6846SIoana Radulescu 2505c4680c97SRadu Bulie if (priv->tx_tstamp_type == HWTSTAMP_TX_ONESTEP_SYNC) 2506c4680c97SRadu Bulie dpaa2_ptp_onestep_reg_update_method(priv); 2507c4680c97SRadu Bulie 250834ff6846SIoana Radulescu return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 250934ff6846SIoana Radulescu -EFAULT : 0; 251034ff6846SIoana Radulescu } 251134ff6846SIoana Radulescu 251234ff6846SIoana Radulescu static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 251334ff6846SIoana Radulescu { 25144a84182aSRussell King struct dpaa2_eth_priv *priv = netdev_priv(dev); 25154a84182aSRussell King 251634ff6846SIoana Radulescu if (cmd == SIOCSHWTSTAMP) 251734ff6846SIoana Radulescu return dpaa2_eth_ts_ioctl(dev, rq, cmd); 251834ff6846SIoana Radulescu 2519d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 25204a84182aSRussell King return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); 25214a84182aSRussell King 25224a84182aSRussell King return -EOPNOTSUPP; 252334ff6846SIoana Radulescu } 252434ff6846SIoana Radulescu 25257e273a8eSIoana Ciocoi Radulescu static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 25267e273a8eSIoana Ciocoi Radulescu { 25277e273a8eSIoana Ciocoi Radulescu int mfl, linear_mfl; 25287e273a8eSIoana Ciocoi Radulescu 25297e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 2530efa6a7d0SIoana Ciornei linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE - 25317b1eea1aSIoana Ciocoi Radulescu dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 25327e273a8eSIoana Ciocoi Radulescu 25337e273a8eSIoana Ciocoi Radulescu if (mfl > linear_mfl) { 25347e273a8eSIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 25357e273a8eSIoana Ciocoi Radulescu linear_mfl - VLAN_ETH_HLEN); 25367e273a8eSIoana Ciocoi Radulescu return false; 25377e273a8eSIoana Ciocoi Radulescu } 25387e273a8eSIoana Ciocoi Radulescu 25397e273a8eSIoana Ciocoi Radulescu return true; 25407e273a8eSIoana Ciocoi Radulescu } 25417e273a8eSIoana Ciocoi Radulescu 25425d8dccf8SIoana Ciornei static int dpaa2_eth_set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 25437e273a8eSIoana Ciocoi Radulescu { 25447e273a8eSIoana Ciocoi Radulescu int mfl, err; 25457e273a8eSIoana Ciocoi Radulescu 25467e273a8eSIoana Ciocoi Radulescu /* We enforce a maximum Rx frame length based on MTU only if we have 25477e273a8eSIoana Ciocoi Radulescu * an XDP program attached (in order to avoid Rx S/G frames). 25487e273a8eSIoana Ciocoi Radulescu * Otherwise, we accept all incoming frames as long as they are not 25497e273a8eSIoana Ciocoi Radulescu * larger than maximum size supported in hardware 25507e273a8eSIoana Ciocoi Radulescu */ 25517e273a8eSIoana Ciocoi Radulescu if (has_xdp) 25527e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 25537e273a8eSIoana Ciocoi Radulescu else 25547e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_MFL; 25557e273a8eSIoana Ciocoi Radulescu 25567e273a8eSIoana Ciocoi Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 25577e273a8eSIoana Ciocoi Radulescu if (err) { 25587e273a8eSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 25597e273a8eSIoana Ciocoi Radulescu return err; 25607e273a8eSIoana Ciocoi Radulescu } 25617e273a8eSIoana Ciocoi Radulescu 25627e273a8eSIoana Ciocoi Radulescu return 0; 25637e273a8eSIoana Ciocoi Radulescu } 25647e273a8eSIoana Ciocoi Radulescu 25657e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 25667e273a8eSIoana Ciocoi Radulescu { 25677e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 25687e273a8eSIoana Ciocoi Radulescu int err; 25697e273a8eSIoana Ciocoi Radulescu 25707e273a8eSIoana Ciocoi Radulescu if (!priv->xdp_prog) 25717e273a8eSIoana Ciocoi Radulescu goto out; 25727e273a8eSIoana Ciocoi Radulescu 25737e273a8eSIoana Ciocoi Radulescu if (!xdp_mtu_valid(priv, new_mtu)) 25747e273a8eSIoana Ciocoi Radulescu return -EINVAL; 25757e273a8eSIoana Ciocoi Radulescu 25765d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, new_mtu, true); 25777e273a8eSIoana Ciocoi Radulescu if (err) 25787e273a8eSIoana Ciocoi Radulescu return err; 25797e273a8eSIoana Ciocoi Radulescu 25807e273a8eSIoana Ciocoi Radulescu out: 25817e273a8eSIoana Ciocoi Radulescu dev->mtu = new_mtu; 25827e273a8eSIoana Ciocoi Radulescu return 0; 25837e273a8eSIoana Ciocoi Radulescu } 25847e273a8eSIoana Ciocoi Radulescu 25855d8dccf8SIoana Ciornei static int dpaa2_eth_update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 25867b1eea1aSIoana Ciocoi Radulescu { 25877b1eea1aSIoana Ciocoi Radulescu struct dpni_buffer_layout buf_layout = {0}; 25887b1eea1aSIoana Ciocoi Radulescu int err; 25897b1eea1aSIoana Ciocoi Radulescu 25907b1eea1aSIoana Ciocoi Radulescu err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 25917b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 25927b1eea1aSIoana Ciocoi Radulescu if (err) { 25937b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 25947b1eea1aSIoana Ciocoi Radulescu return err; 25957b1eea1aSIoana Ciocoi Radulescu } 25967b1eea1aSIoana Ciocoi Radulescu 25977b1eea1aSIoana Ciocoi Radulescu /* Reserve extra headroom for XDP header size changes */ 25987b1eea1aSIoana Ciocoi Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 25997b1eea1aSIoana Ciocoi Radulescu (has_xdp ? XDP_PACKET_HEADROOM : 0); 26007b1eea1aSIoana Ciocoi Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 26017b1eea1aSIoana Ciocoi Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 26027b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 26037b1eea1aSIoana Ciocoi Radulescu if (err) { 26047b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 26057b1eea1aSIoana Ciocoi Radulescu return err; 26067b1eea1aSIoana Ciocoi Radulescu } 26077b1eea1aSIoana Ciocoi Radulescu 26087b1eea1aSIoana Ciocoi Radulescu return 0; 26097b1eea1aSIoana Ciocoi Radulescu } 26107b1eea1aSIoana Ciocoi Radulescu 26115d8dccf8SIoana Ciornei static int dpaa2_eth_setup_xdp(struct net_device *dev, struct bpf_prog *prog) 26127e273a8eSIoana Ciocoi Radulescu { 26137e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 26147e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch; 26157e273a8eSIoana Ciocoi Radulescu struct bpf_prog *old; 26167e273a8eSIoana Ciocoi Radulescu bool up, need_update; 26177e273a8eSIoana Ciocoi Radulescu int i, err; 26187e273a8eSIoana Ciocoi Radulescu 26197e273a8eSIoana Ciocoi Radulescu if (prog && !xdp_mtu_valid(priv, dev->mtu)) 26207e273a8eSIoana Ciocoi Radulescu return -EINVAL; 26217e273a8eSIoana Ciocoi Radulescu 262285192dbfSAndrii Nakryiko if (prog) 262385192dbfSAndrii Nakryiko bpf_prog_add(prog, priv->num_channels); 26247e273a8eSIoana Ciocoi Radulescu 26257e273a8eSIoana Ciocoi Radulescu up = netif_running(dev); 26267e273a8eSIoana Ciocoi Radulescu need_update = (!!priv->xdp_prog != !!prog); 26277e273a8eSIoana Ciocoi Radulescu 26287e273a8eSIoana Ciocoi Radulescu if (up) 2629*e3caeb2dSIoana Ciornei dev_close(dev); 26307e273a8eSIoana Ciocoi Radulescu 26317b1eea1aSIoana Ciocoi Radulescu /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 26327b1eea1aSIoana Ciocoi Radulescu * Also, when switching between xdp/non-xdp modes we need to reconfigure 26337b1eea1aSIoana Ciocoi Radulescu * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 26347b1eea1aSIoana Ciocoi Radulescu * so we are sure no old format buffers will be used from now on. 26357b1eea1aSIoana Ciocoi Radulescu */ 26367e273a8eSIoana Ciocoi Radulescu if (need_update) { 26375d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_mfl(priv, dev->mtu, !!prog); 26387e273a8eSIoana Ciocoi Radulescu if (err) 26397e273a8eSIoana Ciocoi Radulescu goto out_err; 26405d8dccf8SIoana Ciornei err = dpaa2_eth_update_rx_buffer_headroom(priv, !!prog); 26417b1eea1aSIoana Ciocoi Radulescu if (err) 26427b1eea1aSIoana Ciocoi Radulescu goto out_err; 26437e273a8eSIoana Ciocoi Radulescu } 26447e273a8eSIoana Ciocoi Radulescu 26457e273a8eSIoana Ciocoi Radulescu old = xchg(&priv->xdp_prog, prog); 26467e273a8eSIoana Ciocoi Radulescu if (old) 26477e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 26487e273a8eSIoana Ciocoi Radulescu 26497e273a8eSIoana Ciocoi Radulescu for (i = 0; i < priv->num_channels; i++) { 26507e273a8eSIoana Ciocoi Radulescu ch = priv->channel[i]; 26517e273a8eSIoana Ciocoi Radulescu old = xchg(&ch->xdp.prog, prog); 26527e273a8eSIoana Ciocoi Radulescu if (old) 26537e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 26547e273a8eSIoana Ciocoi Radulescu } 26557e273a8eSIoana Ciocoi Radulescu 26567e273a8eSIoana Ciocoi Radulescu if (up) { 2657*e3caeb2dSIoana Ciornei err = dev_open(dev, NULL); 26587e273a8eSIoana Ciocoi Radulescu if (err) 26597e273a8eSIoana Ciocoi Radulescu return err; 26607e273a8eSIoana Ciocoi Radulescu } 26617e273a8eSIoana Ciocoi Radulescu 26627e273a8eSIoana Ciocoi Radulescu return 0; 26637e273a8eSIoana Ciocoi Radulescu 26647e273a8eSIoana Ciocoi Radulescu out_err: 26657e273a8eSIoana Ciocoi Radulescu if (prog) 26667e273a8eSIoana Ciocoi Radulescu bpf_prog_sub(prog, priv->num_channels); 26677e273a8eSIoana Ciocoi Radulescu if (up) 2668*e3caeb2dSIoana Ciornei dev_open(dev, NULL); 26697e273a8eSIoana Ciocoi Radulescu 26707e273a8eSIoana Ciocoi Radulescu return err; 26717e273a8eSIoana Ciocoi Radulescu } 26727e273a8eSIoana Ciocoi Radulescu 26737e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 26747e273a8eSIoana Ciocoi Radulescu { 26757e273a8eSIoana Ciocoi Radulescu switch (xdp->command) { 26767e273a8eSIoana Ciocoi Radulescu case XDP_SETUP_PROG: 26775d8dccf8SIoana Ciornei return dpaa2_eth_setup_xdp(dev, xdp->prog); 26787e273a8eSIoana Ciocoi Radulescu default: 26797e273a8eSIoana Ciocoi Radulescu return -EINVAL; 26807e273a8eSIoana Ciocoi Radulescu } 26817e273a8eSIoana Ciocoi Radulescu 26827e273a8eSIoana Ciocoi Radulescu return 0; 26837e273a8eSIoana Ciocoi Radulescu } 26847e273a8eSIoana Ciocoi Radulescu 26856aa40b9eSIoana Ciornei static int dpaa2_eth_xdp_create_fd(struct net_device *net_dev, 26866aa40b9eSIoana Ciornei struct xdp_frame *xdpf, 26876aa40b9eSIoana Ciornei struct dpaa2_fd *fd) 2688d678be1dSIoana Radulescu { 2689d678be1dSIoana Radulescu struct device *dev = net_dev->dev.parent; 2690d678be1dSIoana Radulescu unsigned int needed_headroom; 2691d678be1dSIoana Radulescu struct dpaa2_eth_swa *swa; 2692d678be1dSIoana Radulescu void *buffer_start, *aligned_start; 2693d678be1dSIoana Radulescu dma_addr_t addr; 2694d678be1dSIoana Radulescu 2695d678be1dSIoana Radulescu /* We require a minimum headroom to be able to transmit the frame. 2696d678be1dSIoana Radulescu * Otherwise return an error and let the original net_device handle it 2697d678be1dSIoana Radulescu */ 26981cf773bdSYangbo Lu needed_headroom = dpaa2_eth_needed_headroom(NULL); 2699d678be1dSIoana Radulescu if (xdpf->headroom < needed_headroom) 2700d678be1dSIoana Radulescu return -EINVAL; 2701d678be1dSIoana Radulescu 2702d678be1dSIoana Radulescu /* Setup the FD fields */ 27036aa40b9eSIoana Ciornei memset(fd, 0, sizeof(*fd)); 2704d678be1dSIoana Radulescu 2705d678be1dSIoana Radulescu /* Align FD address, if possible */ 2706d678be1dSIoana Radulescu buffer_start = xdpf->data - needed_headroom; 2707d678be1dSIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 2708d678be1dSIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 2709d678be1dSIoana Radulescu if (aligned_start >= xdpf->data - xdpf->headroom) 2710d678be1dSIoana Radulescu buffer_start = aligned_start; 2711d678be1dSIoana Radulescu 2712d678be1dSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 2713d678be1dSIoana Radulescu /* fill in necessary fields here */ 2714d678be1dSIoana Radulescu swa->type = DPAA2_ETH_SWA_XDP; 2715d678be1dSIoana Radulescu swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 2716d678be1dSIoana Radulescu swa->xdp.xdpf = xdpf; 2717d678be1dSIoana Radulescu 2718d678be1dSIoana Radulescu addr = dma_map_single(dev, buffer_start, 2719d678be1dSIoana Radulescu swa->xdp.dma_size, 2720d678be1dSIoana Radulescu DMA_BIDIRECTIONAL); 27216aa40b9eSIoana Ciornei if (unlikely(dma_mapping_error(dev, addr))) 2722d678be1dSIoana Radulescu return -ENOMEM; 2723d678be1dSIoana Radulescu 27246aa40b9eSIoana Ciornei dpaa2_fd_set_addr(fd, addr); 27256aa40b9eSIoana Ciornei dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); 27266aa40b9eSIoana Ciornei dpaa2_fd_set_len(fd, xdpf->len); 27276aa40b9eSIoana Ciornei dpaa2_fd_set_format(fd, dpaa2_fd_single); 27286aa40b9eSIoana Ciornei dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 2729d678be1dSIoana Radulescu 2730d678be1dSIoana Radulescu return 0; 2731d678be1dSIoana Radulescu } 2732d678be1dSIoana Radulescu 2733d678be1dSIoana Radulescu static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 2734d678be1dSIoana Radulescu struct xdp_frame **frames, u32 flags) 2735d678be1dSIoana Radulescu { 27366aa40b9eSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 273738c440b2SIoana Ciornei struct dpaa2_eth_xdp_fds *xdp_redirect_fds; 27386aa40b9eSIoana Ciornei struct rtnl_link_stats64 *percpu_stats; 27396aa40b9eSIoana Ciornei struct dpaa2_eth_fq *fq; 27408665d978SIoana Ciornei struct dpaa2_fd *fds; 274138c440b2SIoana Ciornei int enqueued, i, err; 2742d678be1dSIoana Radulescu 2743d678be1dSIoana Radulescu if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2744d678be1dSIoana Radulescu return -EINVAL; 2745d678be1dSIoana Radulescu 2746d678be1dSIoana Radulescu if (!netif_running(net_dev)) 2747d678be1dSIoana Radulescu return -ENETDOWN; 2748d678be1dSIoana Radulescu 27498665d978SIoana Ciornei fq = &priv->fq[smp_processor_id()]; 275038c440b2SIoana Ciornei xdp_redirect_fds = &fq->xdp_redirect_fds; 275138c440b2SIoana Ciornei fds = xdp_redirect_fds->fds; 27528665d978SIoana Ciornei 27536aa40b9eSIoana Ciornei percpu_stats = this_cpu_ptr(priv->percpu_stats); 27546aa40b9eSIoana Ciornei 27558665d978SIoana Ciornei /* create a FD for each xdp_frame in the list received */ 2756d678be1dSIoana Radulescu for (i = 0; i < n; i++) { 27578665d978SIoana Ciornei err = dpaa2_eth_xdp_create_fd(net_dev, frames[i], &fds[i]); 27588665d978SIoana Ciornei if (err) 27596aa40b9eSIoana Ciornei break; 27606aa40b9eSIoana Ciornei } 276138c440b2SIoana Ciornei xdp_redirect_fds->num = i; 27626aa40b9eSIoana Ciornei 276338c440b2SIoana Ciornei /* enqueue all the frame descriptors */ 276438c440b2SIoana Ciornei enqueued = dpaa2_eth_xdp_flush(priv, fq, xdp_redirect_fds); 2765d678be1dSIoana Radulescu 27668665d978SIoana Ciornei /* update statistics */ 276738c440b2SIoana Ciornei percpu_stats->tx_packets += enqueued; 276838c440b2SIoana Ciornei for (i = 0; i < enqueued; i++) 27698665d978SIoana Ciornei percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 27708665d978SIoana Ciornei 277138c440b2SIoana Ciornei return enqueued; 2772d678be1dSIoana Radulescu } 2773d678be1dSIoana Radulescu 277406d5b179SIoana Radulescu static int update_xps(struct dpaa2_eth_priv *priv) 277506d5b179SIoana Radulescu { 277606d5b179SIoana Radulescu struct net_device *net_dev = priv->net_dev; 277706d5b179SIoana Radulescu struct cpumask xps_mask; 277806d5b179SIoana Radulescu struct dpaa2_eth_fq *fq; 2779ab1e6de2SIoana Radulescu int i, num_queues, netdev_queues; 278006d5b179SIoana Radulescu int err = 0; 278106d5b179SIoana Radulescu 278206d5b179SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2783ab1e6de2SIoana Radulescu netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 278406d5b179SIoana Radulescu 278506d5b179SIoana Radulescu /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 278606d5b179SIoana Radulescu * queues, so only process those 278706d5b179SIoana Radulescu */ 2788ab1e6de2SIoana Radulescu for (i = 0; i < netdev_queues; i++) { 2789ab1e6de2SIoana Radulescu fq = &priv->fq[i % num_queues]; 279006d5b179SIoana Radulescu 279106d5b179SIoana Radulescu cpumask_clear(&xps_mask); 279206d5b179SIoana Radulescu cpumask_set_cpu(fq->target_cpu, &xps_mask); 279306d5b179SIoana Radulescu 279406d5b179SIoana Radulescu err = netif_set_xps_queue(net_dev, &xps_mask, i); 279506d5b179SIoana Radulescu if (err) { 279606d5b179SIoana Radulescu netdev_warn_once(net_dev, "Error setting XPS queue\n"); 279706d5b179SIoana Radulescu break; 279806d5b179SIoana Radulescu } 279906d5b179SIoana Radulescu } 280006d5b179SIoana Radulescu 280106d5b179SIoana Radulescu return err; 280206d5b179SIoana Radulescu } 280306d5b179SIoana Radulescu 2804e3ec13beSIoana Ciornei static int dpaa2_eth_setup_mqprio(struct net_device *net_dev, 2805e3ec13beSIoana Ciornei struct tc_mqprio_qopt *mqprio) 2806ab1e6de2SIoana Radulescu { 2807ab1e6de2SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2808ab1e6de2SIoana Radulescu u8 num_tc, num_queues; 2809ab1e6de2SIoana Radulescu int i; 2810ab1e6de2SIoana Radulescu 2811ab1e6de2SIoana Radulescu mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2812ab1e6de2SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 2813ab1e6de2SIoana Radulescu num_tc = mqprio->num_tc; 2814ab1e6de2SIoana Radulescu 2815ab1e6de2SIoana Radulescu if (num_tc == net_dev->num_tc) 2816ab1e6de2SIoana Radulescu return 0; 2817ab1e6de2SIoana Radulescu 2818ab1e6de2SIoana Radulescu if (num_tc > dpaa2_eth_tc_count(priv)) { 2819ab1e6de2SIoana Radulescu netdev_err(net_dev, "Max %d traffic classes supported\n", 2820ab1e6de2SIoana Radulescu dpaa2_eth_tc_count(priv)); 2821b89c1e6bSJesper Dangaard Brouer return -EOPNOTSUPP; 2822ab1e6de2SIoana Radulescu } 2823ab1e6de2SIoana Radulescu 2824ab1e6de2SIoana Radulescu if (!num_tc) { 2825ab1e6de2SIoana Radulescu netdev_reset_tc(net_dev); 2826ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_queues); 2827ab1e6de2SIoana Radulescu goto out; 2828ab1e6de2SIoana Radulescu } 2829ab1e6de2SIoana Radulescu 2830ab1e6de2SIoana Radulescu netdev_set_num_tc(net_dev, num_tc); 2831ab1e6de2SIoana Radulescu netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2832ab1e6de2SIoana Radulescu 2833ab1e6de2SIoana Radulescu for (i = 0; i < num_tc; i++) 2834ab1e6de2SIoana Radulescu netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2835ab1e6de2SIoana Radulescu 2836ab1e6de2SIoana Radulescu out: 2837ab1e6de2SIoana Radulescu update_xps(priv); 2838ab1e6de2SIoana Radulescu 2839ab1e6de2SIoana Radulescu return 0; 2840ab1e6de2SIoana Radulescu } 2841ab1e6de2SIoana Radulescu 28423657cdafSIoana Ciornei #define bps_to_mbits(rate) (div_u64((rate), 1000000) * 8) 28433657cdafSIoana Ciornei 28443657cdafSIoana Ciornei static int dpaa2_eth_setup_tbf(struct net_device *net_dev, struct tc_tbf_qopt_offload *p) 28453657cdafSIoana Ciornei { 28463657cdafSIoana Ciornei struct tc_tbf_qopt_offload_replace_params *cfg = &p->replace_params; 28473657cdafSIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 28483657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_cr_shaper = { 0 }; 28493657cdafSIoana Ciornei struct dpni_tx_shaping_cfg tx_er_shaper = { 0 }; 28503657cdafSIoana Ciornei int err; 28513657cdafSIoana Ciornei 28523657cdafSIoana Ciornei if (p->command == TC_TBF_STATS) 28533657cdafSIoana Ciornei return -EOPNOTSUPP; 28543657cdafSIoana Ciornei 28553657cdafSIoana Ciornei /* Only per port Tx shaping */ 28563657cdafSIoana Ciornei if (p->parent != TC_H_ROOT) 28573657cdafSIoana Ciornei return -EOPNOTSUPP; 28583657cdafSIoana Ciornei 28593657cdafSIoana Ciornei if (p->command == TC_TBF_REPLACE) { 28603657cdafSIoana Ciornei if (cfg->max_size > DPAA2_ETH_MAX_BURST_SIZE) { 28613657cdafSIoana Ciornei netdev_err(net_dev, "burst size cannot be greater than %d\n", 28623657cdafSIoana Ciornei DPAA2_ETH_MAX_BURST_SIZE); 28633657cdafSIoana Ciornei return -EINVAL; 28643657cdafSIoana Ciornei } 28653657cdafSIoana Ciornei 28663657cdafSIoana Ciornei tx_cr_shaper.max_burst_size = cfg->max_size; 28673657cdafSIoana Ciornei /* The TBF interface is in bytes/s, whereas DPAA2 expects the 28683657cdafSIoana Ciornei * rate in Mbits/s 28693657cdafSIoana Ciornei */ 28703657cdafSIoana Ciornei tx_cr_shaper.rate_limit = bps_to_mbits(cfg->rate.rate_bytes_ps); 28713657cdafSIoana Ciornei } 28723657cdafSIoana Ciornei 28733657cdafSIoana Ciornei err = dpni_set_tx_shaping(priv->mc_io, 0, priv->mc_token, &tx_cr_shaper, 28743657cdafSIoana Ciornei &tx_er_shaper, 0); 28753657cdafSIoana Ciornei if (err) { 28763657cdafSIoana Ciornei netdev_err(net_dev, "dpni_set_tx_shaping() = %d\n", err); 28773657cdafSIoana Ciornei return err; 28783657cdafSIoana Ciornei } 28793657cdafSIoana Ciornei 28803657cdafSIoana Ciornei return 0; 28813657cdafSIoana Ciornei } 28823657cdafSIoana Ciornei 2883e3ec13beSIoana Ciornei static int dpaa2_eth_setup_tc(struct net_device *net_dev, 2884e3ec13beSIoana Ciornei enum tc_setup_type type, void *type_data) 2885e3ec13beSIoana Ciornei { 2886e3ec13beSIoana Ciornei switch (type) { 2887e3ec13beSIoana Ciornei case TC_SETUP_QDISC_MQPRIO: 2888e3ec13beSIoana Ciornei return dpaa2_eth_setup_mqprio(net_dev, type_data); 28893657cdafSIoana Ciornei case TC_SETUP_QDISC_TBF: 28903657cdafSIoana Ciornei return dpaa2_eth_setup_tbf(net_dev, type_data); 2891e3ec13beSIoana Ciornei default: 2892e3ec13beSIoana Ciornei return -EOPNOTSUPP; 2893e3ec13beSIoana Ciornei } 2894e3ec13beSIoana Ciornei } 2895e3ec13beSIoana Ciornei 289634ff6846SIoana Radulescu static const struct net_device_ops dpaa2_eth_ops = { 289734ff6846SIoana Radulescu .ndo_open = dpaa2_eth_open, 289834ff6846SIoana Radulescu .ndo_start_xmit = dpaa2_eth_tx, 289934ff6846SIoana Radulescu .ndo_stop = dpaa2_eth_stop, 290034ff6846SIoana Radulescu .ndo_set_mac_address = dpaa2_eth_set_addr, 290134ff6846SIoana Radulescu .ndo_get_stats64 = dpaa2_eth_get_stats, 290234ff6846SIoana Radulescu .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 290334ff6846SIoana Radulescu .ndo_set_features = dpaa2_eth_set_features, 2904a7605370SArnd Bergmann .ndo_eth_ioctl = dpaa2_eth_ioctl, 29057e273a8eSIoana Ciocoi Radulescu .ndo_change_mtu = dpaa2_eth_change_mtu, 29067e273a8eSIoana Ciocoi Radulescu .ndo_bpf = dpaa2_eth_xdp, 2907d678be1dSIoana Radulescu .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2908ab1e6de2SIoana Radulescu .ndo_setup_tc = dpaa2_eth_setup_tc, 290970b32d82SIonut-robert Aron .ndo_vlan_rx_add_vid = dpaa2_eth_rx_add_vid, 291070b32d82SIonut-robert Aron .ndo_vlan_rx_kill_vid = dpaa2_eth_rx_kill_vid 291134ff6846SIoana Radulescu }; 291234ff6846SIoana Radulescu 29135d8dccf8SIoana Ciornei static void dpaa2_eth_cdan_cb(struct dpaa2_io_notification_ctx *ctx) 291434ff6846SIoana Radulescu { 291534ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 291634ff6846SIoana Radulescu 291734ff6846SIoana Radulescu ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 291834ff6846SIoana Radulescu 291934ff6846SIoana Radulescu /* Update NAPI statistics */ 292034ff6846SIoana Radulescu ch->stats.cdan++; 292134ff6846SIoana Radulescu 29226c33ae1aSJiafei Pan napi_schedule(&ch->napi); 292334ff6846SIoana Radulescu } 292434ff6846SIoana Radulescu 292534ff6846SIoana Radulescu /* Allocate and configure a DPCON object */ 29265d8dccf8SIoana Ciornei static struct fsl_mc_device *dpaa2_eth_setup_dpcon(struct dpaa2_eth_priv *priv) 292734ff6846SIoana Radulescu { 292834ff6846SIoana Radulescu struct fsl_mc_device *dpcon; 292934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 293034ff6846SIoana Radulescu int err; 293134ff6846SIoana Radulescu 293234ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 293334ff6846SIoana Radulescu FSL_MC_POOL_DPCON, &dpcon); 293434ff6846SIoana Radulescu if (err) { 2935d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2936d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2937d7f5a9d8SIoana Ciornei else 293834ff6846SIoana Radulescu dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2939d7f5a9d8SIoana Ciornei return ERR_PTR(err); 294034ff6846SIoana Radulescu } 294134ff6846SIoana Radulescu 294234ff6846SIoana Radulescu err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 294334ff6846SIoana Radulescu if (err) { 294434ff6846SIoana Radulescu dev_err(dev, "dpcon_open() failed\n"); 294534ff6846SIoana Radulescu goto free; 294634ff6846SIoana Radulescu } 294734ff6846SIoana Radulescu 294834ff6846SIoana Radulescu err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 294934ff6846SIoana Radulescu if (err) { 295034ff6846SIoana Radulescu dev_err(dev, "dpcon_reset() failed\n"); 295134ff6846SIoana Radulescu goto close; 295234ff6846SIoana Radulescu } 295334ff6846SIoana Radulescu 295434ff6846SIoana Radulescu err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 295534ff6846SIoana Radulescu if (err) { 295634ff6846SIoana Radulescu dev_err(dev, "dpcon_enable() failed\n"); 295734ff6846SIoana Radulescu goto close; 295834ff6846SIoana Radulescu } 295934ff6846SIoana Radulescu 296034ff6846SIoana Radulescu return dpcon; 296134ff6846SIoana Radulescu 296234ff6846SIoana Radulescu close: 296334ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 296434ff6846SIoana Radulescu free: 296534ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 296634ff6846SIoana Radulescu 296702afa9c6SYueHaibing return ERR_PTR(err); 296834ff6846SIoana Radulescu } 296934ff6846SIoana Radulescu 29705d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpcon(struct dpaa2_eth_priv *priv, 297134ff6846SIoana Radulescu struct fsl_mc_device *dpcon) 297234ff6846SIoana Radulescu { 297334ff6846SIoana Radulescu dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 297434ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 297534ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 297634ff6846SIoana Radulescu } 297734ff6846SIoana Radulescu 29785d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_alloc_channel(struct dpaa2_eth_priv *priv) 297934ff6846SIoana Radulescu { 298034ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 298134ff6846SIoana Radulescu struct dpcon_attr attr; 298234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 298334ff6846SIoana Radulescu int err; 298434ff6846SIoana Radulescu 298534ff6846SIoana Radulescu channel = kzalloc(sizeof(*channel), GFP_KERNEL); 298634ff6846SIoana Radulescu if (!channel) 298734ff6846SIoana Radulescu return NULL; 298834ff6846SIoana Radulescu 29895d8dccf8SIoana Ciornei channel->dpcon = dpaa2_eth_setup_dpcon(priv); 299002afa9c6SYueHaibing if (IS_ERR(channel->dpcon)) { 299102afa9c6SYueHaibing err = PTR_ERR(channel->dpcon); 299234ff6846SIoana Radulescu goto err_setup; 2993d7f5a9d8SIoana Ciornei } 299434ff6846SIoana Radulescu 299534ff6846SIoana Radulescu err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 299634ff6846SIoana Radulescu &attr); 299734ff6846SIoana Radulescu if (err) { 299834ff6846SIoana Radulescu dev_err(dev, "dpcon_get_attributes() failed\n"); 299934ff6846SIoana Radulescu goto err_get_attr; 300034ff6846SIoana Radulescu } 300134ff6846SIoana Radulescu 300234ff6846SIoana Radulescu channel->dpcon_id = attr.id; 300334ff6846SIoana Radulescu channel->ch_id = attr.qbman_ch_id; 300434ff6846SIoana Radulescu channel->priv = priv; 300534ff6846SIoana Radulescu 300634ff6846SIoana Radulescu return channel; 300734ff6846SIoana Radulescu 300834ff6846SIoana Radulescu err_get_attr: 30095d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 301034ff6846SIoana Radulescu err_setup: 301134ff6846SIoana Radulescu kfree(channel); 3012d7f5a9d8SIoana Ciornei return ERR_PTR(err); 301334ff6846SIoana Radulescu } 301434ff6846SIoana Radulescu 30155d8dccf8SIoana Ciornei static void dpaa2_eth_free_channel(struct dpaa2_eth_priv *priv, 301634ff6846SIoana Radulescu struct dpaa2_eth_channel *channel) 301734ff6846SIoana Radulescu { 30185d8dccf8SIoana Ciornei dpaa2_eth_free_dpcon(priv, channel->dpcon); 301934ff6846SIoana Radulescu kfree(channel); 302034ff6846SIoana Radulescu } 302134ff6846SIoana Radulescu 302234ff6846SIoana Radulescu /* DPIO setup: allocate and configure QBMan channels, setup core affinity 302334ff6846SIoana Radulescu * and register data availability notifications 302434ff6846SIoana Radulescu */ 30255d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpio(struct dpaa2_eth_priv *priv) 302634ff6846SIoana Radulescu { 302734ff6846SIoana Radulescu struct dpaa2_io_notification_ctx *nctx; 302834ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 302934ff6846SIoana Radulescu struct dpcon_notification_cfg dpcon_notif_cfg; 303034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 303134ff6846SIoana Radulescu int i, err; 303234ff6846SIoana Radulescu 303334ff6846SIoana Radulescu /* We want the ability to spread ingress traffic (RX, TX conf) to as 303434ff6846SIoana Radulescu * many cores as possible, so we need one channel for each core 303534ff6846SIoana Radulescu * (unless there's fewer queues than cores, in which case the extra 303634ff6846SIoana Radulescu * channels would be wasted). 303734ff6846SIoana Radulescu * Allocate one channel per core and register it to the core's 303834ff6846SIoana Radulescu * affine DPIO. If not enough channels are available for all cores 303934ff6846SIoana Radulescu * or if some cores don't have an affine DPIO, there will be no 304034ff6846SIoana Radulescu * ingress frame processing on those cores. 304134ff6846SIoana Radulescu */ 304234ff6846SIoana Radulescu cpumask_clear(&priv->dpio_cpumask); 304334ff6846SIoana Radulescu for_each_online_cpu(i) { 304434ff6846SIoana Radulescu /* Try to allocate a channel */ 30455d8dccf8SIoana Ciornei channel = dpaa2_eth_alloc_channel(priv); 3046d7f5a9d8SIoana Ciornei if (IS_ERR_OR_NULL(channel)) { 3047bd8460faSIoana Radulescu err = PTR_ERR_OR_ZERO(channel); 3048d7f5a9d8SIoana Ciornei if (err != -EPROBE_DEFER) 304934ff6846SIoana Radulescu dev_info(dev, 305034ff6846SIoana Radulescu "No affine channel for cpu %d and above\n", i); 305134ff6846SIoana Radulescu goto err_alloc_ch; 305234ff6846SIoana Radulescu } 305334ff6846SIoana Radulescu 305434ff6846SIoana Radulescu priv->channel[priv->num_channels] = channel; 305534ff6846SIoana Radulescu 305634ff6846SIoana Radulescu nctx = &channel->nctx; 305734ff6846SIoana Radulescu nctx->is_cdan = 1; 30585d8dccf8SIoana Ciornei nctx->cb = dpaa2_eth_cdan_cb; 305934ff6846SIoana Radulescu nctx->id = channel->ch_id; 306034ff6846SIoana Radulescu nctx->desired_cpu = i; 306134ff6846SIoana Radulescu 306234ff6846SIoana Radulescu /* Register the new context */ 306334ff6846SIoana Radulescu channel->dpio = dpaa2_io_service_select(i); 306447441f7fSIoana Ciornei err = dpaa2_io_service_register(channel->dpio, nctx, dev); 306534ff6846SIoana Radulescu if (err) { 306634ff6846SIoana Radulescu dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 306734ff6846SIoana Radulescu /* If no affine DPIO for this core, there's probably 306834ff6846SIoana Radulescu * none available for next cores either. Signal we want 306934ff6846SIoana Radulescu * to retry later, in case the DPIO devices weren't 307034ff6846SIoana Radulescu * probed yet. 307134ff6846SIoana Radulescu */ 307234ff6846SIoana Radulescu err = -EPROBE_DEFER; 307334ff6846SIoana Radulescu goto err_service_reg; 307434ff6846SIoana Radulescu } 307534ff6846SIoana Radulescu 307634ff6846SIoana Radulescu /* Register DPCON notification with MC */ 307734ff6846SIoana Radulescu dpcon_notif_cfg.dpio_id = nctx->dpio_id; 307834ff6846SIoana Radulescu dpcon_notif_cfg.priority = 0; 307934ff6846SIoana Radulescu dpcon_notif_cfg.user_ctx = nctx->qman64; 308034ff6846SIoana Radulescu err = dpcon_set_notification(priv->mc_io, 0, 308134ff6846SIoana Radulescu channel->dpcon->mc_handle, 308234ff6846SIoana Radulescu &dpcon_notif_cfg); 308334ff6846SIoana Radulescu if (err) { 308434ff6846SIoana Radulescu dev_err(dev, "dpcon_set_notification failed()\n"); 308534ff6846SIoana Radulescu goto err_set_cdan; 308634ff6846SIoana Radulescu } 308734ff6846SIoana Radulescu 308834ff6846SIoana Radulescu /* If we managed to allocate a channel and also found an affine 308934ff6846SIoana Radulescu * DPIO for this core, add it to the final mask 309034ff6846SIoana Radulescu */ 309134ff6846SIoana Radulescu cpumask_set_cpu(i, &priv->dpio_cpumask); 309234ff6846SIoana Radulescu priv->num_channels++; 309334ff6846SIoana Radulescu 309434ff6846SIoana Radulescu /* Stop if we already have enough channels to accommodate all 309534ff6846SIoana Radulescu * RX and TX conf queues 309634ff6846SIoana Radulescu */ 3097b0e4f37bSIoana Ciocoi Radulescu if (priv->num_channels == priv->dpni_attrs.num_queues) 309834ff6846SIoana Radulescu break; 309934ff6846SIoana Radulescu } 310034ff6846SIoana Radulescu 310134ff6846SIoana Radulescu return 0; 310234ff6846SIoana Radulescu 310334ff6846SIoana Radulescu err_set_cdan: 310447441f7fSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 310534ff6846SIoana Radulescu err_service_reg: 31065d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 310734ff6846SIoana Radulescu err_alloc_ch: 31085aa4277dSIoana Ciornei if (err == -EPROBE_DEFER) { 31095aa4277dSIoana Ciornei for (i = 0; i < priv->num_channels; i++) { 31105aa4277dSIoana Ciornei channel = priv->channel[i]; 31115aa4277dSIoana Ciornei nctx = &channel->nctx; 31125aa4277dSIoana Ciornei dpaa2_io_service_deregister(channel->dpio, nctx, dev); 31135d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, channel); 31145aa4277dSIoana Ciornei } 31155aa4277dSIoana Ciornei priv->num_channels = 0; 3116d7f5a9d8SIoana Ciornei return err; 31175aa4277dSIoana Ciornei } 3118d7f5a9d8SIoana Ciornei 311934ff6846SIoana Radulescu if (cpumask_empty(&priv->dpio_cpumask)) { 312034ff6846SIoana Radulescu dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 3121d7f5a9d8SIoana Ciornei return -ENODEV; 312234ff6846SIoana Radulescu } 312334ff6846SIoana Radulescu 312434ff6846SIoana Radulescu dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 312534ff6846SIoana Radulescu cpumask_pr_args(&priv->dpio_cpumask)); 312634ff6846SIoana Radulescu 312734ff6846SIoana Radulescu return 0; 312834ff6846SIoana Radulescu } 312934ff6846SIoana Radulescu 31305d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpio(struct dpaa2_eth_priv *priv) 313134ff6846SIoana Radulescu { 313247441f7fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 313334ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 313447441f7fSIoana Ciornei int i; 313534ff6846SIoana Radulescu 313634ff6846SIoana Radulescu /* deregister CDAN notifications and free channels */ 313734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 313834ff6846SIoana Radulescu ch = priv->channel[i]; 313947441f7fSIoana Ciornei dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 31405d8dccf8SIoana Ciornei dpaa2_eth_free_channel(priv, ch); 314134ff6846SIoana Radulescu } 314234ff6846SIoana Radulescu } 314334ff6846SIoana Radulescu 31445d8dccf8SIoana Ciornei static struct dpaa2_eth_channel *dpaa2_eth_get_affine_channel(struct dpaa2_eth_priv *priv, 314534ff6846SIoana Radulescu int cpu) 314634ff6846SIoana Radulescu { 314734ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 314834ff6846SIoana Radulescu int i; 314934ff6846SIoana Radulescu 315034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 315134ff6846SIoana Radulescu if (priv->channel[i]->nctx.desired_cpu == cpu) 315234ff6846SIoana Radulescu return priv->channel[i]; 315334ff6846SIoana Radulescu 315434ff6846SIoana Radulescu /* We should never get here. Issue a warning and return 315534ff6846SIoana Radulescu * the first channel, because it's still better than nothing 315634ff6846SIoana Radulescu */ 315734ff6846SIoana Radulescu dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 315834ff6846SIoana Radulescu 315934ff6846SIoana Radulescu return priv->channel[0]; 316034ff6846SIoana Radulescu } 316134ff6846SIoana Radulescu 31625d8dccf8SIoana Ciornei static void dpaa2_eth_set_fq_affinity(struct dpaa2_eth_priv *priv) 316334ff6846SIoana Radulescu { 316434ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 316534ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 316634ff6846SIoana Radulescu int rx_cpu, txc_cpu; 316706d5b179SIoana Radulescu int i; 316834ff6846SIoana Radulescu 316934ff6846SIoana Radulescu /* For each FQ, pick one channel/CPU to deliver frames to. 317034ff6846SIoana Radulescu * This may well change at runtime, either through irqbalance or 317134ff6846SIoana Radulescu * through direct user intervention. 317234ff6846SIoana Radulescu */ 317334ff6846SIoana Radulescu rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 317434ff6846SIoana Radulescu 317534ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 317634ff6846SIoana Radulescu fq = &priv->fq[i]; 317734ff6846SIoana Radulescu switch (fq->type) { 317834ff6846SIoana Radulescu case DPAA2_RX_FQ: 3179061d631fSIoana Ciornei case DPAA2_RX_ERR_FQ: 318034ff6846SIoana Radulescu fq->target_cpu = rx_cpu; 318134ff6846SIoana Radulescu rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 318234ff6846SIoana Radulescu if (rx_cpu >= nr_cpu_ids) 318334ff6846SIoana Radulescu rx_cpu = cpumask_first(&priv->dpio_cpumask); 318434ff6846SIoana Radulescu break; 318534ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 318634ff6846SIoana Radulescu fq->target_cpu = txc_cpu; 318734ff6846SIoana Radulescu txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 318834ff6846SIoana Radulescu if (txc_cpu >= nr_cpu_ids) 318934ff6846SIoana Radulescu txc_cpu = cpumask_first(&priv->dpio_cpumask); 319034ff6846SIoana Radulescu break; 319134ff6846SIoana Radulescu default: 319234ff6846SIoana Radulescu dev_err(dev, "Unknown FQ type: %d\n", fq->type); 319334ff6846SIoana Radulescu } 31945d8dccf8SIoana Ciornei fq->channel = dpaa2_eth_get_affine_channel(priv, fq->target_cpu); 319534ff6846SIoana Radulescu } 319606d5b179SIoana Radulescu 319706d5b179SIoana Radulescu update_xps(priv); 319834ff6846SIoana Radulescu } 319934ff6846SIoana Radulescu 32005d8dccf8SIoana Ciornei static void dpaa2_eth_setup_fqs(struct dpaa2_eth_priv *priv) 320134ff6846SIoana Radulescu { 3202685e39eaSIoana Radulescu int i, j; 320334ff6846SIoana Radulescu 320434ff6846SIoana Radulescu /* We have one TxConf FQ per Tx flow. 320534ff6846SIoana Radulescu * The number of Tx and Rx queues is the same. 320634ff6846SIoana Radulescu * Tx queues come first in the fq array. 320734ff6846SIoana Radulescu */ 320834ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 320934ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 321034ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 321134ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 321234ff6846SIoana Radulescu } 321334ff6846SIoana Radulescu 3214685e39eaSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 321534ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 321634ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 321734ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 3218685e39eaSIoana Radulescu priv->fq[priv->num_fqs].tc = (u8)j; 321934ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 322034ff6846SIoana Radulescu } 3221685e39eaSIoana Radulescu } 322234ff6846SIoana Radulescu 3223061d631fSIoana Ciornei /* We have exactly one Rx error queue per DPNI */ 3224061d631fSIoana Ciornei priv->fq[priv->num_fqs].type = DPAA2_RX_ERR_FQ; 3225061d631fSIoana Ciornei priv->fq[priv->num_fqs++].consume = dpaa2_eth_rx_err; 3226061d631fSIoana Ciornei 322734ff6846SIoana Radulescu /* For each FQ, decide on which core to process incoming frames */ 32285d8dccf8SIoana Ciornei dpaa2_eth_set_fq_affinity(priv); 322934ff6846SIoana Radulescu } 323034ff6846SIoana Radulescu 3231095174daSRobert-Ionut Alexa /* Allocate and configure a buffer pool */ 3232095174daSRobert-Ionut Alexa struct dpaa2_eth_bp *dpaa2_eth_allocate_dpbp(struct dpaa2_eth_priv *priv) 323334ff6846SIoana Radulescu { 323434ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3235095174daSRobert-Ionut Alexa struct fsl_mc_device *dpbp_dev; 323634ff6846SIoana Radulescu struct dpbp_attr dpbp_attrs; 3237095174daSRobert-Ionut Alexa struct dpaa2_eth_bp *bp; 3238095174daSRobert-Ionut Alexa int err; 323934ff6846SIoana Radulescu 324034ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 324134ff6846SIoana Radulescu &dpbp_dev); 324234ff6846SIoana Radulescu if (err) { 3243d7f5a9d8SIoana Ciornei if (err == -ENXIO) 3244d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 3245d7f5a9d8SIoana Ciornei else 324634ff6846SIoana Radulescu dev_err(dev, "DPBP device allocation failed\n"); 3247095174daSRobert-Ionut Alexa return ERR_PTR(err); 324834ff6846SIoana Radulescu } 324934ff6846SIoana Radulescu 3250095174daSRobert-Ionut Alexa bp = kzalloc(sizeof(*bp), GFP_KERNEL); 3251095174daSRobert-Ionut Alexa if (!bp) { 3252095174daSRobert-Ionut Alexa err = -ENOMEM; 3253095174daSRobert-Ionut Alexa goto err_alloc; 3254095174daSRobert-Ionut Alexa } 325534ff6846SIoana Radulescu 3256095174daSRobert-Ionut Alexa err = dpbp_open(priv->mc_io, 0, dpbp_dev->obj_desc.id, 325734ff6846SIoana Radulescu &dpbp_dev->mc_handle); 325834ff6846SIoana Radulescu if (err) { 325934ff6846SIoana Radulescu dev_err(dev, "dpbp_open() failed\n"); 326034ff6846SIoana Radulescu goto err_open; 326134ff6846SIoana Radulescu } 326234ff6846SIoana Radulescu 326334ff6846SIoana Radulescu err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 326434ff6846SIoana Radulescu if (err) { 326534ff6846SIoana Radulescu dev_err(dev, "dpbp_reset() failed\n"); 326634ff6846SIoana Radulescu goto err_reset; 326734ff6846SIoana Radulescu } 326834ff6846SIoana Radulescu 326934ff6846SIoana Radulescu err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 327034ff6846SIoana Radulescu if (err) { 327134ff6846SIoana Radulescu dev_err(dev, "dpbp_enable() failed\n"); 327234ff6846SIoana Radulescu goto err_enable; 327334ff6846SIoana Radulescu } 327434ff6846SIoana Radulescu 327534ff6846SIoana Radulescu err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 327634ff6846SIoana Radulescu &dpbp_attrs); 327734ff6846SIoana Radulescu if (err) { 327834ff6846SIoana Radulescu dev_err(dev, "dpbp_get_attributes() failed\n"); 327934ff6846SIoana Radulescu goto err_get_attr; 328034ff6846SIoana Radulescu } 328134ff6846SIoana Radulescu 3282095174daSRobert-Ionut Alexa bp->dev = dpbp_dev; 3283095174daSRobert-Ionut Alexa bp->bpid = dpbp_attrs.bpid; 3284095174daSRobert-Ionut Alexa 3285095174daSRobert-Ionut Alexa return bp; 328634ff6846SIoana Radulescu 328734ff6846SIoana Radulescu err_get_attr: 328834ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 328934ff6846SIoana Radulescu err_enable: 329034ff6846SIoana Radulescu err_reset: 329134ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 329234ff6846SIoana Radulescu err_open: 3293095174daSRobert-Ionut Alexa kfree(bp); 3294095174daSRobert-Ionut Alexa err_alloc: 329534ff6846SIoana Radulescu fsl_mc_object_free(dpbp_dev); 329634ff6846SIoana Radulescu 3297095174daSRobert-Ionut Alexa return ERR_PTR(err); 329834ff6846SIoana Radulescu } 329934ff6846SIoana Radulescu 3300095174daSRobert-Ionut Alexa static int dpaa2_eth_setup_default_dpbp(struct dpaa2_eth_priv *priv) 330134ff6846SIoana Radulescu { 3302095174daSRobert-Ionut Alexa struct dpaa2_eth_bp *bp; 3303095174daSRobert-Ionut Alexa int i; 3304095174daSRobert-Ionut Alexa 3305095174daSRobert-Ionut Alexa bp = dpaa2_eth_allocate_dpbp(priv); 3306095174daSRobert-Ionut Alexa if (IS_ERR(bp)) 3307095174daSRobert-Ionut Alexa return PTR_ERR(bp); 3308095174daSRobert-Ionut Alexa 3309095174daSRobert-Ionut Alexa priv->bp[DPAA2_ETH_DEFAULT_BP_IDX] = bp; 3310095174daSRobert-Ionut Alexa priv->num_bps++; 3311095174daSRobert-Ionut Alexa 3312095174daSRobert-Ionut Alexa for (i = 0; i < priv->num_channels; i++) 3313095174daSRobert-Ionut Alexa priv->channel[i]->bp = bp; 3314095174daSRobert-Ionut Alexa 3315095174daSRobert-Ionut Alexa return 0; 3316095174daSRobert-Ionut Alexa } 3317095174daSRobert-Ionut Alexa 3318095174daSRobert-Ionut Alexa void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv, struct dpaa2_eth_bp *bp) 3319095174daSRobert-Ionut Alexa { 3320095174daSRobert-Ionut Alexa int idx_bp; 3321095174daSRobert-Ionut Alexa 3322095174daSRobert-Ionut Alexa /* Find the index at which this BP is stored */ 3323095174daSRobert-Ionut Alexa for (idx_bp = 0; idx_bp < priv->num_bps; idx_bp++) 3324095174daSRobert-Ionut Alexa if (priv->bp[idx_bp] == bp) 3325095174daSRobert-Ionut Alexa break; 3326095174daSRobert-Ionut Alexa 3327095174daSRobert-Ionut Alexa /* Drain the pool and disable the associated MC object */ 3328095174daSRobert-Ionut Alexa dpaa2_eth_drain_pool(priv, bp->bpid); 3329095174daSRobert-Ionut Alexa dpbp_disable(priv->mc_io, 0, bp->dev->mc_handle); 3330095174daSRobert-Ionut Alexa dpbp_close(priv->mc_io, 0, bp->dev->mc_handle); 3331095174daSRobert-Ionut Alexa fsl_mc_object_free(bp->dev); 3332095174daSRobert-Ionut Alexa kfree(bp); 3333095174daSRobert-Ionut Alexa 3334095174daSRobert-Ionut Alexa /* Move the last in use DPBP over in this position */ 3335095174daSRobert-Ionut Alexa priv->bp[idx_bp] = priv->bp[priv->num_bps - 1]; 3336095174daSRobert-Ionut Alexa priv->num_bps--; 3337095174daSRobert-Ionut Alexa } 3338095174daSRobert-Ionut Alexa 3339095174daSRobert-Ionut Alexa static void dpaa2_eth_free_dpbps(struct dpaa2_eth_priv *priv) 3340095174daSRobert-Ionut Alexa { 3341095174daSRobert-Ionut Alexa int i; 3342095174daSRobert-Ionut Alexa 3343095174daSRobert-Ionut Alexa for (i = 0; i < priv->num_bps; i++) 3344095174daSRobert-Ionut Alexa dpaa2_eth_free_dpbp(priv, priv->bp[i]); 334534ff6846SIoana Radulescu } 334634ff6846SIoana Radulescu 33475d8dccf8SIoana Ciornei static int dpaa2_eth_set_buffer_layout(struct dpaa2_eth_priv *priv) 334834ff6846SIoana Radulescu { 334934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 335034ff6846SIoana Radulescu struct dpni_buffer_layout buf_layout = {0}; 335127c87486SIoana Ciocoi Radulescu u16 rx_buf_align; 335234ff6846SIoana Radulescu int err; 335334ff6846SIoana Radulescu 335434ff6846SIoana Radulescu /* We need to check for WRIOP version 1.0.0, but depending on the MC 335534ff6846SIoana Radulescu * version, this number is not always provided correctly on rev1. 335634ff6846SIoana Radulescu * We need to check for both alternatives in this situation. 335734ff6846SIoana Radulescu */ 335834ff6846SIoana Radulescu if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 335934ff6846SIoana Radulescu priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 336027c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 336134ff6846SIoana Radulescu else 336227c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 336334ff6846SIoana Radulescu 3364efa6a7d0SIoana Ciornei /* We need to ensure that the buffer size seen by WRIOP is a multiple 3365efa6a7d0SIoana Ciornei * of 64 or 256 bytes depending on the WRIOP version. 3366efa6a7d0SIoana Ciornei */ 3367efa6a7d0SIoana Ciornei priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align); 3368efa6a7d0SIoana Ciornei 336934ff6846SIoana Radulescu /* tx buffer */ 337034ff6846SIoana Radulescu buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 337134ff6846SIoana Radulescu buf_layout.pass_timestamp = true; 3372c5521189SYangbo Lu buf_layout.pass_frame_status = true; 337334ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 3374c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 3375c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 337634ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 337734ff6846SIoana Radulescu DPNI_QUEUE_TX, &buf_layout); 337834ff6846SIoana Radulescu if (err) { 337934ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 338034ff6846SIoana Radulescu return err; 338134ff6846SIoana Radulescu } 338234ff6846SIoana Radulescu 338334ff6846SIoana Radulescu /* tx-confirm buffer */ 3384c5521189SYangbo Lu buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP | 3385c5521189SYangbo Lu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; 338634ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 338734ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, &buf_layout); 338834ff6846SIoana Radulescu if (err) { 338934ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 339034ff6846SIoana Radulescu return err; 339134ff6846SIoana Radulescu } 339234ff6846SIoana Radulescu 339334ff6846SIoana Radulescu /* Now that we've set our tx buffer layout, retrieve the minimum 339434ff6846SIoana Radulescu * required tx data offset. 339534ff6846SIoana Radulescu */ 339634ff6846SIoana Radulescu err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 339734ff6846SIoana Radulescu &priv->tx_data_offset); 339834ff6846SIoana Radulescu if (err) { 339934ff6846SIoana Radulescu dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 340034ff6846SIoana Radulescu return err; 340134ff6846SIoana Radulescu } 340234ff6846SIoana Radulescu 340334ff6846SIoana Radulescu if ((priv->tx_data_offset % 64) != 0) 340434ff6846SIoana Radulescu dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 340534ff6846SIoana Radulescu priv->tx_data_offset); 340634ff6846SIoana Radulescu 340734ff6846SIoana Radulescu /* rx buffer */ 340834ff6846SIoana Radulescu buf_layout.pass_frame_status = true; 340934ff6846SIoana Radulescu buf_layout.pass_parser_result = true; 341027c87486SIoana Ciocoi Radulescu buf_layout.data_align = rx_buf_align; 341134ff6846SIoana Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 341234ff6846SIoana Radulescu buf_layout.private_data_size = 0; 341334ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 341434ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 341534ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 341634ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 341734ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 341834ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 341934ff6846SIoana Radulescu DPNI_QUEUE_RX, &buf_layout); 342034ff6846SIoana Radulescu if (err) { 342134ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 342234ff6846SIoana Radulescu return err; 342334ff6846SIoana Radulescu } 342434ff6846SIoana Radulescu 342534ff6846SIoana Radulescu return 0; 342634ff6846SIoana Radulescu } 342734ff6846SIoana Radulescu 34281fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 34291fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MINOR 9 34301fa0f68cSIoana Ciocoi Radulescu 34311fa0f68cSIoana Ciocoi Radulescu static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 34321fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 343348c0481eSIoana Ciornei struct dpaa2_fd *fd, u8 prio, 34346ff80447SIoana Ciornei u32 num_frames __always_unused, 343548c0481eSIoana Ciornei int *frames_enqueued) 34361fa0f68cSIoana Ciocoi Radulescu { 343748c0481eSIoana Ciornei int err; 343848c0481eSIoana Ciornei 343948c0481eSIoana Ciornei err = dpaa2_io_service_enqueue_qd(fq->channel->dpio, 34401fa0f68cSIoana Ciocoi Radulescu priv->tx_qdid, prio, 34411fa0f68cSIoana Ciocoi Radulescu fq->tx_qdbin, fd); 344248c0481eSIoana Ciornei if (!err && frames_enqueued) 344348c0481eSIoana Ciornei *frames_enqueued = 1; 344448c0481eSIoana Ciornei return err; 34451fa0f68cSIoana Ciocoi Radulescu } 34461fa0f68cSIoana Ciocoi Radulescu 34476ff80447SIoana Ciornei static inline int dpaa2_eth_enqueue_fq_multiple(struct dpaa2_eth_priv *priv, 34481fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 34496ff80447SIoana Ciornei struct dpaa2_fd *fd, 34506ff80447SIoana Ciornei u8 prio, u32 num_frames, 345148c0481eSIoana Ciornei int *frames_enqueued) 34521fa0f68cSIoana Ciocoi Radulescu { 345348c0481eSIoana Ciornei int err; 345448c0481eSIoana Ciornei 34556ff80447SIoana Ciornei err = dpaa2_io_service_enqueue_multiple_fq(fq->channel->dpio, 34566ff80447SIoana Ciornei fq->tx_fqid[prio], 34576ff80447SIoana Ciornei fd, num_frames); 34586ff80447SIoana Ciornei 34596ff80447SIoana Ciornei if (err == 0) 34606ff80447SIoana Ciornei return -EBUSY; 34616ff80447SIoana Ciornei 34626ff80447SIoana Ciornei if (frames_enqueued) 34636ff80447SIoana Ciornei *frames_enqueued = err; 34646ff80447SIoana Ciornei return 0; 34651fa0f68cSIoana Ciocoi Radulescu } 34661fa0f68cSIoana Ciocoi Radulescu 34675d8dccf8SIoana Ciornei static void dpaa2_eth_set_enqueue_mode(struct dpaa2_eth_priv *priv) 34681fa0f68cSIoana Ciocoi Radulescu { 34691fa0f68cSIoana Ciocoi Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 34701fa0f68cSIoana Ciocoi Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 34711fa0f68cSIoana Ciocoi Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 34721fa0f68cSIoana Ciocoi Radulescu else 34736ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 34741fa0f68cSIoana Ciocoi Radulescu } 34751fa0f68cSIoana Ciocoi Radulescu 34765d8dccf8SIoana Ciornei static int dpaa2_eth_set_pause(struct dpaa2_eth_priv *priv) 34778eb3cef8SIoana Radulescu { 34788eb3cef8SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 34798eb3cef8SIoana Radulescu struct dpni_link_cfg link_cfg = {0}; 34808eb3cef8SIoana Radulescu int err; 34818eb3cef8SIoana Radulescu 34828eb3cef8SIoana Radulescu /* Get the default link options so we don't override other flags */ 34838eb3cef8SIoana Radulescu err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 34848eb3cef8SIoana Radulescu if (err) { 34858eb3cef8SIoana Radulescu dev_err(dev, "dpni_get_link_cfg() failed\n"); 34868eb3cef8SIoana Radulescu return err; 34878eb3cef8SIoana Radulescu } 34888eb3cef8SIoana Radulescu 34898eb3cef8SIoana Radulescu /* By default, enable both Rx and Tx pause frames */ 34908eb3cef8SIoana Radulescu link_cfg.options |= DPNI_LINK_OPT_PAUSE; 34918eb3cef8SIoana Radulescu link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 34928eb3cef8SIoana Radulescu err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 34938eb3cef8SIoana Radulescu if (err) { 34948eb3cef8SIoana Radulescu dev_err(dev, "dpni_set_link_cfg() failed\n"); 34958eb3cef8SIoana Radulescu return err; 34968eb3cef8SIoana Radulescu } 34978eb3cef8SIoana Radulescu 34988eb3cef8SIoana Radulescu priv->link_state.options = link_cfg.options; 34998eb3cef8SIoana Radulescu 35008eb3cef8SIoana Radulescu return 0; 35018eb3cef8SIoana Radulescu } 35028eb3cef8SIoana Radulescu 35035d8dccf8SIoana Ciornei static void dpaa2_eth_update_tx_fqids(struct dpaa2_eth_priv *priv) 3504a690af4fSIoana Radulescu { 3505a690af4fSIoana Radulescu struct dpni_queue_id qid = {0}; 3506a690af4fSIoana Radulescu struct dpaa2_eth_fq *fq; 3507a690af4fSIoana Radulescu struct dpni_queue queue; 3508a690af4fSIoana Radulescu int i, j, err; 3509a690af4fSIoana Radulescu 3510a690af4fSIoana Radulescu /* We only use Tx FQIDs for FQID-based enqueue, so check 3511a690af4fSIoana Radulescu * if DPNI version supports it before updating FQIDs 3512a690af4fSIoana Radulescu */ 3513a690af4fSIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 3514a690af4fSIoana Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 3515a690af4fSIoana Radulescu return; 3516a690af4fSIoana Radulescu 3517a690af4fSIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 3518a690af4fSIoana Radulescu fq = &priv->fq[i]; 3519a690af4fSIoana Radulescu if (fq->type != DPAA2_TX_CONF_FQ) 3520a690af4fSIoana Radulescu continue; 3521a690af4fSIoana Radulescu for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 3522a690af4fSIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3523a690af4fSIoana Radulescu DPNI_QUEUE_TX, j, fq->flowid, 3524a690af4fSIoana Radulescu &queue, &qid); 3525a690af4fSIoana Radulescu if (err) 3526a690af4fSIoana Radulescu goto out_err; 3527a690af4fSIoana Radulescu 3528a690af4fSIoana Radulescu fq->tx_fqid[j] = qid.fqid; 3529a690af4fSIoana Radulescu if (fq->tx_fqid[j] == 0) 3530a690af4fSIoana Radulescu goto out_err; 3531a690af4fSIoana Radulescu } 3532a690af4fSIoana Radulescu } 3533a690af4fSIoana Radulescu 35346ff80447SIoana Ciornei priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 3535a690af4fSIoana Radulescu 3536a690af4fSIoana Radulescu return; 3537a690af4fSIoana Radulescu 3538a690af4fSIoana Radulescu out_err: 3539a690af4fSIoana Radulescu netdev_info(priv->net_dev, 3540a690af4fSIoana Radulescu "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 3541a690af4fSIoana Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 3542a690af4fSIoana Radulescu } 3543a690af4fSIoana Radulescu 35446aa90fe2SIoana Radulescu /* Configure ingress classification based on VLAN PCP */ 35455d8dccf8SIoana Ciornei static int dpaa2_eth_set_vlan_qos(struct dpaa2_eth_priv *priv) 35466aa90fe2SIoana Radulescu { 35476aa90fe2SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 35486aa90fe2SIoana Radulescu struct dpkg_profile_cfg kg_cfg = {0}; 35496aa90fe2SIoana Radulescu struct dpni_qos_tbl_cfg qos_cfg = {0}; 35506aa90fe2SIoana Radulescu struct dpni_rule_cfg key_params; 35516aa90fe2SIoana Radulescu void *dma_mem, *key, *mask; 35526aa90fe2SIoana Radulescu u8 key_size = 2; /* VLAN TCI field */ 35536aa90fe2SIoana Radulescu int i, pcp, err; 35546aa90fe2SIoana Radulescu 35556aa90fe2SIoana Radulescu /* VLAN-based classification only makes sense if we have multiple 35566aa90fe2SIoana Radulescu * traffic classes. 35576aa90fe2SIoana Radulescu * Also, we need to extract just the 3-bit PCP field from the VLAN 35586aa90fe2SIoana Radulescu * header and we can only do that by using a mask 35596aa90fe2SIoana Radulescu */ 35606aa90fe2SIoana Radulescu if (dpaa2_eth_tc_count(priv) == 1 || !dpaa2_eth_fs_mask_enabled(priv)) { 35616aa90fe2SIoana Radulescu dev_dbg(dev, "VLAN-based QoS classification not supported\n"); 35626aa90fe2SIoana Radulescu return -EOPNOTSUPP; 35636aa90fe2SIoana Radulescu } 35646aa90fe2SIoana Radulescu 35656aa90fe2SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 35666aa90fe2SIoana Radulescu if (!dma_mem) 35676aa90fe2SIoana Radulescu return -ENOMEM; 35686aa90fe2SIoana Radulescu 35696aa90fe2SIoana Radulescu kg_cfg.num_extracts = 1; 35706aa90fe2SIoana Radulescu kg_cfg.extracts[0].type = DPKG_EXTRACT_FROM_HDR; 35716aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.prot = NET_PROT_VLAN; 35726aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.type = DPKG_FULL_FIELD; 35736aa90fe2SIoana Radulescu kg_cfg.extracts[0].extract.from_hdr.field = NH_FLD_VLAN_TCI; 35746aa90fe2SIoana Radulescu 35756aa90fe2SIoana Radulescu err = dpni_prepare_key_cfg(&kg_cfg, dma_mem); 35766aa90fe2SIoana Radulescu if (err) { 35776aa90fe2SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg failed\n"); 35786aa90fe2SIoana Radulescu goto out_free_tbl; 35796aa90fe2SIoana Radulescu } 35806aa90fe2SIoana Radulescu 35816aa90fe2SIoana Radulescu /* set QoS table */ 35826aa90fe2SIoana Radulescu qos_cfg.default_tc = 0; 35836aa90fe2SIoana Radulescu qos_cfg.discard_on_miss = 0; 35846aa90fe2SIoana Radulescu qos_cfg.key_cfg_iova = dma_map_single(dev, dma_mem, 35856aa90fe2SIoana Radulescu DPAA2_CLASSIFIER_DMA_SIZE, 35866aa90fe2SIoana Radulescu DMA_TO_DEVICE); 35876aa90fe2SIoana Radulescu if (dma_mapping_error(dev, qos_cfg.key_cfg_iova)) { 35886aa90fe2SIoana Radulescu dev_err(dev, "QoS table DMA mapping failed\n"); 35896aa90fe2SIoana Radulescu err = -ENOMEM; 35906aa90fe2SIoana Radulescu goto out_free_tbl; 35916aa90fe2SIoana Radulescu } 35926aa90fe2SIoana Radulescu 35936aa90fe2SIoana Radulescu err = dpni_set_qos_table(priv->mc_io, 0, priv->mc_token, &qos_cfg); 35946aa90fe2SIoana Radulescu if (err) { 35956aa90fe2SIoana Radulescu dev_err(dev, "dpni_set_qos_table failed\n"); 35966aa90fe2SIoana Radulescu goto out_unmap_tbl; 35976aa90fe2SIoana Radulescu } 35986aa90fe2SIoana Radulescu 35996aa90fe2SIoana Radulescu /* Add QoS table entries */ 36006aa90fe2SIoana Radulescu key = kzalloc(key_size * 2, GFP_KERNEL); 36016aa90fe2SIoana Radulescu if (!key) { 36026aa90fe2SIoana Radulescu err = -ENOMEM; 36036aa90fe2SIoana Radulescu goto out_unmap_tbl; 36046aa90fe2SIoana Radulescu } 36056aa90fe2SIoana Radulescu mask = key + key_size; 36066aa90fe2SIoana Radulescu *(__be16 *)mask = cpu_to_be16(VLAN_PRIO_MASK); 36076aa90fe2SIoana Radulescu 36086aa90fe2SIoana Radulescu key_params.key_iova = dma_map_single(dev, key, key_size * 2, 36096aa90fe2SIoana Radulescu DMA_TO_DEVICE); 36106aa90fe2SIoana Radulescu if (dma_mapping_error(dev, key_params.key_iova)) { 36116aa90fe2SIoana Radulescu dev_err(dev, "Qos table entry DMA mapping failed\n"); 36126aa90fe2SIoana Radulescu err = -ENOMEM; 36136aa90fe2SIoana Radulescu goto out_free_key; 36146aa90fe2SIoana Radulescu } 36156aa90fe2SIoana Radulescu 36166aa90fe2SIoana Radulescu key_params.mask_iova = key_params.key_iova + key_size; 36176aa90fe2SIoana Radulescu key_params.key_size = key_size; 36186aa90fe2SIoana Radulescu 36196aa90fe2SIoana Radulescu /* We add rules for PCP-based distribution starting with highest 36206aa90fe2SIoana Radulescu * priority (VLAN PCP = 7). If this DPNI doesn't have enough traffic 36216aa90fe2SIoana Radulescu * classes to accommodate all priority levels, the lowest ones end up 36226aa90fe2SIoana Radulescu * on TC 0 which was configured as default 36236aa90fe2SIoana Radulescu */ 36246aa90fe2SIoana Radulescu for (i = dpaa2_eth_tc_count(priv) - 1, pcp = 7; i >= 0; i--, pcp--) { 36256aa90fe2SIoana Radulescu *(__be16 *)key = cpu_to_be16(pcp << VLAN_PRIO_SHIFT); 36266aa90fe2SIoana Radulescu dma_sync_single_for_device(dev, key_params.key_iova, 36276aa90fe2SIoana Radulescu key_size * 2, DMA_TO_DEVICE); 36286aa90fe2SIoana Radulescu 36296aa90fe2SIoana Radulescu err = dpni_add_qos_entry(priv->mc_io, 0, priv->mc_token, 36306aa90fe2SIoana Radulescu &key_params, i, i); 36316aa90fe2SIoana Radulescu if (err) { 36326aa90fe2SIoana Radulescu dev_err(dev, "dpni_add_qos_entry failed\n"); 36336aa90fe2SIoana Radulescu dpni_clear_qos_table(priv->mc_io, 0, priv->mc_token); 36346aa90fe2SIoana Radulescu goto out_unmap_key; 36356aa90fe2SIoana Radulescu } 36366aa90fe2SIoana Radulescu } 36376aa90fe2SIoana Radulescu 36386aa90fe2SIoana Radulescu priv->vlan_cls_enabled = true; 36396aa90fe2SIoana Radulescu 36406aa90fe2SIoana Radulescu /* Table and key memory is not persistent, clean everything up after 36416aa90fe2SIoana Radulescu * configuration is finished 36426aa90fe2SIoana Radulescu */ 36436aa90fe2SIoana Radulescu out_unmap_key: 36446aa90fe2SIoana Radulescu dma_unmap_single(dev, key_params.key_iova, key_size * 2, DMA_TO_DEVICE); 36456aa90fe2SIoana Radulescu out_free_key: 36466aa90fe2SIoana Radulescu kfree(key); 36476aa90fe2SIoana Radulescu out_unmap_tbl: 36486aa90fe2SIoana Radulescu dma_unmap_single(dev, qos_cfg.key_cfg_iova, DPAA2_CLASSIFIER_DMA_SIZE, 36496aa90fe2SIoana Radulescu DMA_TO_DEVICE); 36506aa90fe2SIoana Radulescu out_free_tbl: 36516aa90fe2SIoana Radulescu kfree(dma_mem); 36526aa90fe2SIoana Radulescu 36536aa90fe2SIoana Radulescu return err; 36546aa90fe2SIoana Radulescu } 36556aa90fe2SIoana Radulescu 365634ff6846SIoana Radulescu /* Configure the DPNI object this interface is associated with */ 36575d8dccf8SIoana Ciornei static int dpaa2_eth_setup_dpni(struct fsl_mc_device *ls_dev) 365834ff6846SIoana Radulescu { 365934ff6846SIoana Radulescu struct device *dev = &ls_dev->dev; 366034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 366134ff6846SIoana Radulescu struct net_device *net_dev; 366234ff6846SIoana Radulescu int err; 366334ff6846SIoana Radulescu 366434ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 366534ff6846SIoana Radulescu priv = netdev_priv(net_dev); 366634ff6846SIoana Radulescu 366734ff6846SIoana Radulescu /* get a handle for the DPNI object */ 366834ff6846SIoana Radulescu err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 366934ff6846SIoana Radulescu if (err) { 367034ff6846SIoana Radulescu dev_err(dev, "dpni_open() failed\n"); 367134ff6846SIoana Radulescu return err; 367234ff6846SIoana Radulescu } 367334ff6846SIoana Radulescu 367434ff6846SIoana Radulescu /* Check if we can work with this DPNI object */ 367534ff6846SIoana Radulescu err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 367634ff6846SIoana Radulescu &priv->dpni_ver_minor); 367734ff6846SIoana Radulescu if (err) { 367834ff6846SIoana Radulescu dev_err(dev, "dpni_get_api_version() failed\n"); 367934ff6846SIoana Radulescu goto close; 368034ff6846SIoana Radulescu } 368134ff6846SIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 368234ff6846SIoana Radulescu dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 368334ff6846SIoana Radulescu priv->dpni_ver_major, priv->dpni_ver_minor, 368434ff6846SIoana Radulescu DPNI_VER_MAJOR, DPNI_VER_MINOR); 368534ff6846SIoana Radulescu err = -ENOTSUPP; 368634ff6846SIoana Radulescu goto close; 368734ff6846SIoana Radulescu } 368834ff6846SIoana Radulescu 368934ff6846SIoana Radulescu ls_dev->mc_io = priv->mc_io; 369034ff6846SIoana Radulescu ls_dev->mc_handle = priv->mc_token; 369134ff6846SIoana Radulescu 369234ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 369334ff6846SIoana Radulescu if (err) { 369434ff6846SIoana Radulescu dev_err(dev, "dpni_reset() failed\n"); 369534ff6846SIoana Radulescu goto close; 369634ff6846SIoana Radulescu } 369734ff6846SIoana Radulescu 369834ff6846SIoana Radulescu err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 369934ff6846SIoana Radulescu &priv->dpni_attrs); 370034ff6846SIoana Radulescu if (err) { 370134ff6846SIoana Radulescu dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 370234ff6846SIoana Radulescu goto close; 370334ff6846SIoana Radulescu } 370434ff6846SIoana Radulescu 37055d8dccf8SIoana Ciornei err = dpaa2_eth_set_buffer_layout(priv); 370634ff6846SIoana Radulescu if (err) 370734ff6846SIoana Radulescu goto close; 370834ff6846SIoana Radulescu 37095d8dccf8SIoana Ciornei dpaa2_eth_set_enqueue_mode(priv); 37101fa0f68cSIoana Ciocoi Radulescu 37118eb3cef8SIoana Radulescu /* Enable pause frame support */ 37128eb3cef8SIoana Radulescu if (dpaa2_eth_has_pause_support(priv)) { 37135d8dccf8SIoana Ciornei err = dpaa2_eth_set_pause(priv); 37148eb3cef8SIoana Radulescu if (err) 37158eb3cef8SIoana Radulescu goto close; 37168eb3cef8SIoana Radulescu } 37178eb3cef8SIoana Radulescu 37185d8dccf8SIoana Ciornei err = dpaa2_eth_set_vlan_qos(priv); 37196aa90fe2SIoana Radulescu if (err && err != -EOPNOTSUPP) 37206aa90fe2SIoana Radulescu goto close; 37216aa90fe2SIoana Radulescu 37229334d5baSXu Wang priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv), 37239334d5baSXu Wang sizeof(struct dpaa2_eth_cls_rule), 37249334d5baSXu Wang GFP_KERNEL); 372597fff7c8SWei Yongjun if (!priv->cls_rules) { 372697fff7c8SWei Yongjun err = -ENOMEM; 3727afb90dbbSIoana Radulescu goto close; 372897fff7c8SWei Yongjun } 3729afb90dbbSIoana Radulescu 373034ff6846SIoana Radulescu return 0; 373134ff6846SIoana Radulescu 373234ff6846SIoana Radulescu close: 373334ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 373434ff6846SIoana Radulescu 373534ff6846SIoana Radulescu return err; 373634ff6846SIoana Radulescu } 373734ff6846SIoana Radulescu 37385d8dccf8SIoana Ciornei static void dpaa2_eth_free_dpni(struct dpaa2_eth_priv *priv) 373934ff6846SIoana Radulescu { 374034ff6846SIoana Radulescu int err; 374134ff6846SIoana Radulescu 374234ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 374334ff6846SIoana Radulescu if (err) 374434ff6846SIoana Radulescu netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 374534ff6846SIoana Radulescu err); 374634ff6846SIoana Radulescu 374734ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 374834ff6846SIoana Radulescu } 374934ff6846SIoana Radulescu 37505d8dccf8SIoana Ciornei static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, 375134ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 375234ff6846SIoana Radulescu { 375334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 375434ff6846SIoana Radulescu struct dpni_queue queue; 375534ff6846SIoana Radulescu struct dpni_queue_id qid; 375634ff6846SIoana Radulescu int err; 375734ff6846SIoana Radulescu 375834ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3759685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, &queue, &qid); 376034ff6846SIoana Radulescu if (err) { 376134ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(RX) failed\n"); 376234ff6846SIoana Radulescu return err; 376334ff6846SIoana Radulescu } 376434ff6846SIoana Radulescu 376534ff6846SIoana Radulescu fq->fqid = qid.fqid; 376634ff6846SIoana Radulescu 376734ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 376834ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 376934ff6846SIoana Radulescu queue.destination.priority = 1; 377034ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 377134ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3772685e39eaSIoana Radulescu DPNI_QUEUE_RX, fq->tc, fq->flowid, 377316fa1cf1SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 377434ff6846SIoana Radulescu &queue); 377534ff6846SIoana Radulescu if (err) { 377634ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(RX) failed\n"); 377734ff6846SIoana Radulescu return err; 377834ff6846SIoana Radulescu } 377934ff6846SIoana Radulescu 3780d678be1dSIoana Radulescu /* xdp_rxq setup */ 3781685e39eaSIoana Radulescu /* only once for each channel */ 3782685e39eaSIoana Radulescu if (fq->tc > 0) 3783685e39eaSIoana Radulescu return 0; 3784685e39eaSIoana Radulescu 3785d678be1dSIoana Radulescu err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 3786b02e5a0eSBjörn Töpel fq->flowid, 0); 3787d678be1dSIoana Radulescu if (err) { 3788d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg failed\n"); 3789d678be1dSIoana Radulescu return err; 3790d678be1dSIoana Radulescu } 3791d678be1dSIoana Radulescu 3792d678be1dSIoana Radulescu err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 3793d678be1dSIoana Radulescu MEM_TYPE_PAGE_ORDER0, NULL); 3794d678be1dSIoana Radulescu if (err) { 3795d678be1dSIoana Radulescu dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 3796d678be1dSIoana Radulescu return err; 3797d678be1dSIoana Radulescu } 3798d678be1dSIoana Radulescu 379934ff6846SIoana Radulescu return 0; 380034ff6846SIoana Radulescu } 380134ff6846SIoana Radulescu 38025d8dccf8SIoana Ciornei static int dpaa2_eth_setup_tx_flow(struct dpaa2_eth_priv *priv, 380334ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 380434ff6846SIoana Radulescu { 380534ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 380634ff6846SIoana Radulescu struct dpni_queue queue; 380734ff6846SIoana Radulescu struct dpni_queue_id qid; 380815c87f6bSIoana Radulescu int i, err; 380934ff6846SIoana Radulescu 381015c87f6bSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 381134ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 381215c87f6bSIoana Radulescu DPNI_QUEUE_TX, i, fq->flowid, 381315c87f6bSIoana Radulescu &queue, &qid); 381434ff6846SIoana Radulescu if (err) { 381534ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX) failed\n"); 381634ff6846SIoana Radulescu return err; 381734ff6846SIoana Radulescu } 381815c87f6bSIoana Radulescu fq->tx_fqid[i] = qid.fqid; 381915c87f6bSIoana Radulescu } 382034ff6846SIoana Radulescu 382115c87f6bSIoana Radulescu /* All Tx queues belonging to the same flowid have the same qdbin */ 382234ff6846SIoana Radulescu fq->tx_qdbin = qid.qdbin; 382334ff6846SIoana Radulescu 382434ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 382534ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 382634ff6846SIoana Radulescu &queue, &qid); 382734ff6846SIoana Radulescu if (err) { 382834ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 382934ff6846SIoana Radulescu return err; 383034ff6846SIoana Radulescu } 383134ff6846SIoana Radulescu 383234ff6846SIoana Radulescu fq->fqid = qid.fqid; 383334ff6846SIoana Radulescu 383434ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 383534ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 383634ff6846SIoana Radulescu queue.destination.priority = 0; 383734ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 383834ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 383934ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 384034ff6846SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 384134ff6846SIoana Radulescu &queue); 384234ff6846SIoana Radulescu if (err) { 384334ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 384434ff6846SIoana Radulescu return err; 384534ff6846SIoana Radulescu } 384634ff6846SIoana Radulescu 384734ff6846SIoana Radulescu return 0; 384834ff6846SIoana Radulescu } 384934ff6846SIoana Radulescu 3850061d631fSIoana Ciornei static int setup_rx_err_flow(struct dpaa2_eth_priv *priv, 3851061d631fSIoana Ciornei struct dpaa2_eth_fq *fq) 3852061d631fSIoana Ciornei { 3853061d631fSIoana Ciornei struct device *dev = priv->net_dev->dev.parent; 3854061d631fSIoana Ciornei struct dpni_queue q = { { 0 } }; 3855061d631fSIoana Ciornei struct dpni_queue_id qid; 3856061d631fSIoana Ciornei u8 q_opt = DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST; 3857061d631fSIoana Ciornei int err; 3858061d631fSIoana Ciornei 3859061d631fSIoana Ciornei err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3860061d631fSIoana Ciornei DPNI_QUEUE_RX_ERR, 0, 0, &q, &qid); 3861061d631fSIoana Ciornei if (err) { 3862061d631fSIoana Ciornei dev_err(dev, "dpni_get_queue() failed (%d)\n", err); 3863061d631fSIoana Ciornei return err; 3864061d631fSIoana Ciornei } 3865061d631fSIoana Ciornei 3866061d631fSIoana Ciornei fq->fqid = qid.fqid; 3867061d631fSIoana Ciornei 3868061d631fSIoana Ciornei q.destination.id = fq->channel->dpcon_id; 3869061d631fSIoana Ciornei q.destination.type = DPNI_DEST_DPCON; 3870061d631fSIoana Ciornei q.destination.priority = 1; 3871061d631fSIoana Ciornei q.user_context = (u64)(uintptr_t)fq; 3872061d631fSIoana Ciornei err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3873061d631fSIoana Ciornei DPNI_QUEUE_RX_ERR, 0, 0, q_opt, &q); 3874061d631fSIoana Ciornei if (err) { 3875061d631fSIoana Ciornei dev_err(dev, "dpni_set_queue() failed (%d)\n", err); 3876061d631fSIoana Ciornei return err; 3877061d631fSIoana Ciornei } 3878061d631fSIoana Ciornei 3879061d631fSIoana Ciornei return 0; 3880061d631fSIoana Ciornei } 3881061d631fSIoana Ciornei 3882edad8d26SIoana Ciocoi Radulescu /* Supported header fields for Rx hash distribution key */ 3883f76c483aSIoana Radulescu static const struct dpaa2_eth_dist_fields dist_fields[] = { 388434ff6846SIoana Radulescu { 3885edad8d26SIoana Ciocoi Radulescu /* L2 header */ 3886edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_L2DA, 3887edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_ETH, 3888edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_ETH_DA, 38893a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHDST, 3890edad8d26SIoana Ciocoi Radulescu .size = 6, 3891edad8d26SIoana Ciocoi Radulescu }, { 3892afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3893afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_SA, 38943a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHSRC, 3895afb90dbbSIoana Radulescu .size = 6, 3896afb90dbbSIoana Radulescu }, { 3897afb90dbbSIoana Radulescu /* This is the last ethertype field parsed: 3898afb90dbbSIoana Radulescu * depending on frame format, it can be the MAC ethertype 3899afb90dbbSIoana Radulescu * or the VLAN etype. 3900afb90dbbSIoana Radulescu */ 3901afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 3902afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_TYPE, 39033a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_ETHTYPE, 3904afb90dbbSIoana Radulescu .size = 2, 3905afb90dbbSIoana Radulescu }, { 3906edad8d26SIoana Ciocoi Radulescu /* VLAN header */ 3907edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_VLAN, 3908edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_VLAN, 3909edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_VLAN_TCI, 39103a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_VLAN, 3911edad8d26SIoana Ciocoi Radulescu .size = 2, 3912edad8d26SIoana Ciocoi Radulescu }, { 391334ff6846SIoana Radulescu /* IP header */ 391434ff6846SIoana Radulescu .rxnfc_field = RXH_IP_SRC, 391534ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 391634ff6846SIoana Radulescu .cls_field = NH_FLD_IP_SRC, 39173a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPSRC, 391834ff6846SIoana Radulescu .size = 4, 391934ff6846SIoana Radulescu }, { 392034ff6846SIoana Radulescu .rxnfc_field = RXH_IP_DST, 392134ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 392234ff6846SIoana Radulescu .cls_field = NH_FLD_IP_DST, 39233a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPDST, 392434ff6846SIoana Radulescu .size = 4, 392534ff6846SIoana Radulescu }, { 392634ff6846SIoana Radulescu .rxnfc_field = RXH_L3_PROTO, 392734ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 392834ff6846SIoana Radulescu .cls_field = NH_FLD_IP_PROTO, 39293a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_IPPROTO, 393034ff6846SIoana Radulescu .size = 1, 393134ff6846SIoana Radulescu }, { 393234ff6846SIoana Radulescu /* Using UDP ports, this is functionally equivalent to raw 393334ff6846SIoana Radulescu * byte pairs from L4 header. 393434ff6846SIoana Radulescu */ 393534ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_0_1, 393634ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 393734ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_SRC, 39383a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4SRC, 393934ff6846SIoana Radulescu .size = 2, 394034ff6846SIoana Radulescu }, { 394134ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_2_3, 394234ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 394334ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_DST, 39443a1e6b84SIoana Ciocoi Radulescu .id = DPAA2_ETH_DIST_L4DST, 394534ff6846SIoana Radulescu .size = 2, 394634ff6846SIoana Radulescu }, 394734ff6846SIoana Radulescu }; 394834ff6846SIoana Radulescu 3949df85aeb9SIoana Radulescu /* Configure the Rx hash key using the legacy API */ 39505d8dccf8SIoana Ciornei static int dpaa2_eth_config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3951df85aeb9SIoana Radulescu { 3952df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3953df85aeb9SIoana Radulescu struct dpni_rx_tc_dist_cfg dist_cfg; 3954685e39eaSIoana Radulescu int i, err = 0; 3955df85aeb9SIoana Radulescu 3956df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3957df85aeb9SIoana Radulescu 3958df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3959df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3960df85aeb9SIoana Radulescu dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 3961df85aeb9SIoana Radulescu 3962685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3963685e39eaSIoana Radulescu err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 3964685e39eaSIoana Radulescu i, &dist_cfg); 3965685e39eaSIoana Radulescu if (err) { 3966df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 3967685e39eaSIoana Radulescu break; 3968685e39eaSIoana Radulescu } 3969685e39eaSIoana Radulescu } 3970df85aeb9SIoana Radulescu 3971df85aeb9SIoana Radulescu return err; 3972df85aeb9SIoana Radulescu } 3973df85aeb9SIoana Radulescu 3974df85aeb9SIoana Radulescu /* Configure the Rx hash key using the new API */ 39755d8dccf8SIoana Ciornei static int dpaa2_eth_config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3976df85aeb9SIoana Radulescu { 3977df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 3978df85aeb9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 3979685e39eaSIoana Radulescu int i, err = 0; 3980df85aeb9SIoana Radulescu 3981df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 3982df85aeb9SIoana Radulescu 3983df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 3984df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3985df85aeb9SIoana Radulescu dist_cfg.enable = 1; 3986df85aeb9SIoana Radulescu 3987685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3988685e39eaSIoana Radulescu dist_cfg.tc = i; 3989685e39eaSIoana Radulescu err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, 3990685e39eaSIoana Radulescu &dist_cfg); 3991685e39eaSIoana Radulescu if (err) { 3992df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 3993685e39eaSIoana Radulescu break; 3994685e39eaSIoana Radulescu } 39955e29c16fSIonut-robert Aron 39965e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 39975e29c16fSIonut-robert Aron * traffic classes, install it just once 39985e29c16fSIonut-robert Aron */ 39995e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 40005e29c16fSIonut-robert Aron break; 4001685e39eaSIoana Radulescu } 4002df85aeb9SIoana Radulescu 4003df85aeb9SIoana Radulescu return err; 4004df85aeb9SIoana Radulescu } 4005df85aeb9SIoana Radulescu 40064aaaf9b9SIoana Radulescu /* Configure the Rx flow classification key */ 40075d8dccf8SIoana Ciornei static int dpaa2_eth_config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 40084aaaf9b9SIoana Radulescu { 40094aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 40104aaaf9b9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 4011685e39eaSIoana Radulescu int i, err = 0; 40124aaaf9b9SIoana Radulescu 40134aaaf9b9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 40144aaaf9b9SIoana Radulescu 40154aaaf9b9SIoana Radulescu dist_cfg.key_cfg_iova = key; 40164aaaf9b9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 40174aaaf9b9SIoana Radulescu dist_cfg.enable = 1; 40184aaaf9b9SIoana Radulescu 4019685e39eaSIoana Radulescu for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 4020685e39eaSIoana Radulescu dist_cfg.tc = i; 4021685e39eaSIoana Radulescu err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, 4022685e39eaSIoana Radulescu &dist_cfg); 4023685e39eaSIoana Radulescu if (err) { 40244aaaf9b9SIoana Radulescu dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 4025685e39eaSIoana Radulescu break; 4026685e39eaSIoana Radulescu } 40275e29c16fSIonut-robert Aron 40285e29c16fSIonut-robert Aron /* If the flow steering / hashing key is shared between all 40295e29c16fSIonut-robert Aron * traffic classes, install it just once 40305e29c16fSIonut-robert Aron */ 40315e29c16fSIonut-robert Aron if (priv->dpni_attrs.options & DPNI_OPT_SHARED_FS) 40325e29c16fSIonut-robert Aron break; 4033685e39eaSIoana Radulescu } 40344aaaf9b9SIoana Radulescu 40354aaaf9b9SIoana Radulescu return err; 40364aaaf9b9SIoana Radulescu } 40374aaaf9b9SIoana Radulescu 4038afb90dbbSIoana Radulescu /* Size of the Rx flow classification key */ 40392d680237SIoana Ciocoi Radulescu int dpaa2_eth_cls_key_size(u64 fields) 4040afb90dbbSIoana Radulescu { 4041afb90dbbSIoana Radulescu int i, size = 0; 4042afb90dbbSIoana Radulescu 40432d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 40442d680237SIoana Ciocoi Radulescu if (!(fields & dist_fields[i].id)) 40452d680237SIoana Ciocoi Radulescu continue; 4046afb90dbbSIoana Radulescu size += dist_fields[i].size; 40472d680237SIoana Ciocoi Radulescu } 4048afb90dbbSIoana Radulescu 4049afb90dbbSIoana Radulescu return size; 4050afb90dbbSIoana Radulescu } 4051afb90dbbSIoana Radulescu 4052afb90dbbSIoana Radulescu /* Offset of header field in Rx classification key */ 4053afb90dbbSIoana Radulescu int dpaa2_eth_cls_fld_off(int prot, int field) 4054afb90dbbSIoana Radulescu { 4055afb90dbbSIoana Radulescu int i, off = 0; 4056afb90dbbSIoana Radulescu 4057afb90dbbSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 4058afb90dbbSIoana Radulescu if (dist_fields[i].cls_prot == prot && 4059afb90dbbSIoana Radulescu dist_fields[i].cls_field == field) 4060afb90dbbSIoana Radulescu return off; 4061afb90dbbSIoana Radulescu off += dist_fields[i].size; 4062afb90dbbSIoana Radulescu } 4063afb90dbbSIoana Radulescu 4064afb90dbbSIoana Radulescu WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 4065afb90dbbSIoana Radulescu return 0; 4066afb90dbbSIoana Radulescu } 4067afb90dbbSIoana Radulescu 40682d680237SIoana Ciocoi Radulescu /* Prune unused fields from the classification rule. 40692d680237SIoana Ciocoi Radulescu * Used when masking is not supported 40702d680237SIoana Ciocoi Radulescu */ 40712d680237SIoana Ciocoi Radulescu void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 40722d680237SIoana Ciocoi Radulescu { 40732d680237SIoana Ciocoi Radulescu int off = 0, new_off = 0; 40742d680237SIoana Ciocoi Radulescu int i, size; 40752d680237SIoana Ciocoi Radulescu 40762d680237SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 40772d680237SIoana Ciocoi Radulescu size = dist_fields[i].size; 40782d680237SIoana Ciocoi Radulescu if (dist_fields[i].id & fields) { 40792d680237SIoana Ciocoi Radulescu memcpy(key_mem + new_off, key_mem + off, size); 40802d680237SIoana Ciocoi Radulescu new_off += size; 40812d680237SIoana Ciocoi Radulescu } 40822d680237SIoana Ciocoi Radulescu off += size; 40832d680237SIoana Ciocoi Radulescu } 40842d680237SIoana Ciocoi Radulescu } 40852d680237SIoana Ciocoi Radulescu 40864aaaf9b9SIoana Radulescu /* Set Rx distribution (hash or flow classification) key 408734ff6846SIoana Radulescu * flags is a combination of RXH_ bits 408834ff6846SIoana Radulescu */ 40893233c151SIoana Ciornei static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 40904aaaf9b9SIoana Radulescu enum dpaa2_eth_rx_dist type, u64 flags) 409134ff6846SIoana Radulescu { 409234ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 409334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 409434ff6846SIoana Radulescu struct dpkg_profile_cfg cls_cfg; 4095edad8d26SIoana Ciocoi Radulescu u32 rx_hash_fields = 0; 4096df85aeb9SIoana Radulescu dma_addr_t key_iova; 409734ff6846SIoana Radulescu u8 *dma_mem; 409834ff6846SIoana Radulescu int i; 409934ff6846SIoana Radulescu int err = 0; 410034ff6846SIoana Radulescu 410134ff6846SIoana Radulescu memset(&cls_cfg, 0, sizeof(cls_cfg)); 410234ff6846SIoana Radulescu 4103f76c483aSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 410434ff6846SIoana Radulescu struct dpkg_extract *key = 410534ff6846SIoana Radulescu &cls_cfg.extracts[cls_cfg.num_extracts]; 410634ff6846SIoana Radulescu 41072d680237SIoana Ciocoi Radulescu /* For both Rx hashing and classification keys 41082d680237SIoana Ciocoi Radulescu * we set only the selected fields. 41094aaaf9b9SIoana Radulescu */ 41103a1e6b84SIoana Ciocoi Radulescu if (!(flags & dist_fields[i].id)) 411134ff6846SIoana Radulescu continue; 41122d680237SIoana Ciocoi Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) 41134aaaf9b9SIoana Radulescu rx_hash_fields |= dist_fields[i].rxnfc_field; 411434ff6846SIoana Radulescu 411534ff6846SIoana Radulescu if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 411634ff6846SIoana Radulescu dev_err(dev, "error adding key extraction rule, too many rules?\n"); 411734ff6846SIoana Radulescu return -E2BIG; 411834ff6846SIoana Radulescu } 411934ff6846SIoana Radulescu 412034ff6846SIoana Radulescu key->type = DPKG_EXTRACT_FROM_HDR; 4121f76c483aSIoana Radulescu key->extract.from_hdr.prot = dist_fields[i].cls_prot; 412234ff6846SIoana Radulescu key->extract.from_hdr.type = DPKG_FULL_FIELD; 4123f76c483aSIoana Radulescu key->extract.from_hdr.field = dist_fields[i].cls_field; 412434ff6846SIoana Radulescu cls_cfg.num_extracts++; 412534ff6846SIoana Radulescu } 412634ff6846SIoana Radulescu 412734ff6846SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 412834ff6846SIoana Radulescu if (!dma_mem) 412934ff6846SIoana Radulescu return -ENOMEM; 413034ff6846SIoana Radulescu 413134ff6846SIoana Radulescu err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 413234ff6846SIoana Radulescu if (err) { 413334ff6846SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 4134df85aeb9SIoana Radulescu goto free_key; 413534ff6846SIoana Radulescu } 413634ff6846SIoana Radulescu 413734ff6846SIoana Radulescu /* Prepare for setting the rx dist */ 4138df85aeb9SIoana Radulescu key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 413934ff6846SIoana Radulescu DMA_TO_DEVICE); 4140df85aeb9SIoana Radulescu if (dma_mapping_error(dev, key_iova)) { 414134ff6846SIoana Radulescu dev_err(dev, "DMA mapping failed\n"); 414234ff6846SIoana Radulescu err = -ENOMEM; 4143df85aeb9SIoana Radulescu goto free_key; 414434ff6846SIoana Radulescu } 414534ff6846SIoana Radulescu 41464aaaf9b9SIoana Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) { 4147df85aeb9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) 41485d8dccf8SIoana Ciornei err = dpaa2_eth_config_legacy_hash_key(priv, key_iova); 4149edad8d26SIoana Ciocoi Radulescu else 41505d8dccf8SIoana Ciornei err = dpaa2_eth_config_hash_key(priv, key_iova); 41514aaaf9b9SIoana Radulescu } else { 41525d8dccf8SIoana Ciornei err = dpaa2_eth_config_cls_key(priv, key_iova); 41534aaaf9b9SIoana Radulescu } 4154df85aeb9SIoana Radulescu 4155df85aeb9SIoana Radulescu dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 4156df85aeb9SIoana Radulescu DMA_TO_DEVICE); 41574aaaf9b9SIoana Radulescu if (!err && type == DPAA2_ETH_RX_DIST_HASH) 4158edad8d26SIoana Ciocoi Radulescu priv->rx_hash_fields = rx_hash_fields; 415934ff6846SIoana Radulescu 4160df85aeb9SIoana Radulescu free_key: 416134ff6846SIoana Radulescu kfree(dma_mem); 416234ff6846SIoana Radulescu return err; 416334ff6846SIoana Radulescu } 416434ff6846SIoana Radulescu 41654aaaf9b9SIoana Radulescu int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 41664aaaf9b9SIoana Radulescu { 41674aaaf9b9SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 41683a1e6b84SIoana Ciocoi Radulescu u64 key = 0; 41693a1e6b84SIoana Ciocoi Radulescu int i; 41704aaaf9b9SIoana Radulescu 41714aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) 41724aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 41734aaaf9b9SIoana Radulescu 41743a1e6b84SIoana Ciocoi Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 41753a1e6b84SIoana Ciocoi Radulescu if (dist_fields[i].rxnfc_field & flags) 41763a1e6b84SIoana Ciocoi Radulescu key |= dist_fields[i].id; 41773a1e6b84SIoana Ciocoi Radulescu 41783a1e6b84SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 41794aaaf9b9SIoana Radulescu } 41804aaaf9b9SIoana Radulescu 41812d680237SIoana Ciocoi Radulescu int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 41822d680237SIoana Ciocoi Radulescu { 41832d680237SIoana Ciocoi Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 41842d680237SIoana Ciocoi Radulescu } 41852d680237SIoana Ciocoi Radulescu 41862d680237SIoana Ciocoi Radulescu static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 41874aaaf9b9SIoana Radulescu { 41884aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 4189df8e249bSIoana Ciocoi Radulescu int err; 41904aaaf9b9SIoana Radulescu 41914aaaf9b9SIoana Radulescu /* Check if we actually support Rx flow classification */ 41924aaaf9b9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) { 41934aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls not supported by current MC version\n"); 41944aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 41954aaaf9b9SIoana Radulescu } 41964aaaf9b9SIoana Radulescu 41972d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_enabled(priv)) { 41984aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 41994aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 42004aaaf9b9SIoana Radulescu } 42014aaaf9b9SIoana Radulescu 42024aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) { 42034aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 42044aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 42054aaaf9b9SIoana Radulescu } 42064aaaf9b9SIoana Radulescu 42072d680237SIoana Ciocoi Radulescu /* If there is no support for masking in the classification table, 42082d680237SIoana Ciocoi Radulescu * we don't set a default key, as it will depend on the rules 42092d680237SIoana Ciocoi Radulescu * added by the user at runtime. 42102d680237SIoana Ciocoi Radulescu */ 42112d680237SIoana Ciocoi Radulescu if (!dpaa2_eth_fs_mask_enabled(priv)) 42122d680237SIoana Ciocoi Radulescu goto out; 42132d680237SIoana Ciocoi Radulescu 42142d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 4215df8e249bSIoana Ciocoi Radulescu if (err) 4216df8e249bSIoana Ciocoi Radulescu return err; 4217df8e249bSIoana Ciocoi Radulescu 42182d680237SIoana Ciocoi Radulescu out: 42194aaaf9b9SIoana Radulescu priv->rx_cls_enabled = 1; 42204aaaf9b9SIoana Radulescu 4221df8e249bSIoana Ciocoi Radulescu return 0; 42224aaaf9b9SIoana Radulescu } 42234aaaf9b9SIoana Radulescu 422434ff6846SIoana Radulescu /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 422534ff6846SIoana Radulescu * frame queues and channels 422634ff6846SIoana Radulescu */ 42275d8dccf8SIoana Ciornei static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv) 422834ff6846SIoana Radulescu { 4229095174daSRobert-Ionut Alexa struct dpaa2_eth_bp *bp = priv->bp[DPAA2_ETH_DEFAULT_BP_IDX]; 423034ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 423134ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 423234ff6846SIoana Radulescu struct dpni_pools_cfg pools_params; 423334ff6846SIoana Radulescu struct dpni_error_cfg err_cfg; 423434ff6846SIoana Radulescu int err = 0; 423534ff6846SIoana Radulescu int i; 423634ff6846SIoana Radulescu 423734ff6846SIoana Radulescu pools_params.num_dpbp = 1; 4238095174daSRobert-Ionut Alexa pools_params.pools[0].dpbp_id = bp->dev->obj_desc.id; 423934ff6846SIoana Radulescu pools_params.pools[0].backup_pool = 0; 4240efa6a7d0SIoana Ciornei pools_params.pools[0].buffer_size = priv->rx_buf_size; 424134ff6846SIoana Radulescu err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 424234ff6846SIoana Radulescu if (err) { 424334ff6846SIoana Radulescu dev_err(dev, "dpni_set_pools() failed\n"); 424434ff6846SIoana Radulescu return err; 424534ff6846SIoana Radulescu } 424634ff6846SIoana Radulescu 424734ff6846SIoana Radulescu /* have the interface implicitly distribute traffic based on 424834ff6846SIoana Radulescu * the default hash key 424934ff6846SIoana Radulescu */ 425034ff6846SIoana Radulescu err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 4251edad8d26SIoana Ciocoi Radulescu if (err && err != -EOPNOTSUPP) 425234ff6846SIoana Radulescu dev_err(dev, "Failed to configure hashing\n"); 425334ff6846SIoana Radulescu 42544aaaf9b9SIoana Radulescu /* Configure the flow classification key; it includes all 42554aaaf9b9SIoana Radulescu * supported header fields and cannot be modified at runtime 42564aaaf9b9SIoana Radulescu */ 42572d680237SIoana Ciocoi Radulescu err = dpaa2_eth_set_default_cls(priv); 42584aaaf9b9SIoana Radulescu if (err && err != -EOPNOTSUPP) 42594aaaf9b9SIoana Radulescu dev_err(dev, "Failed to configure Rx classification key\n"); 42604aaaf9b9SIoana Radulescu 426134ff6846SIoana Radulescu /* Configure handling of error frames */ 426234ff6846SIoana Radulescu err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 426334ff6846SIoana Radulescu err_cfg.set_frame_annotation = 1; 426434ff6846SIoana Radulescu err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 426534ff6846SIoana Radulescu err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 426634ff6846SIoana Radulescu &err_cfg); 426734ff6846SIoana Radulescu if (err) { 426834ff6846SIoana Radulescu dev_err(dev, "dpni_set_errors_behavior failed\n"); 426934ff6846SIoana Radulescu return err; 427034ff6846SIoana Radulescu } 427134ff6846SIoana Radulescu 427234ff6846SIoana Radulescu /* Configure Rx and Tx conf queues to generate CDANs */ 427334ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 427434ff6846SIoana Radulescu switch (priv->fq[i].type) { 427534ff6846SIoana Radulescu case DPAA2_RX_FQ: 42765d8dccf8SIoana Ciornei err = dpaa2_eth_setup_rx_flow(priv, &priv->fq[i]); 427734ff6846SIoana Radulescu break; 427834ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 42795d8dccf8SIoana Ciornei err = dpaa2_eth_setup_tx_flow(priv, &priv->fq[i]); 428034ff6846SIoana Radulescu break; 4281061d631fSIoana Ciornei case DPAA2_RX_ERR_FQ: 4282061d631fSIoana Ciornei err = setup_rx_err_flow(priv, &priv->fq[i]); 4283061d631fSIoana Ciornei break; 428434ff6846SIoana Radulescu default: 428534ff6846SIoana Radulescu dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 428634ff6846SIoana Radulescu return -EINVAL; 428734ff6846SIoana Radulescu } 428834ff6846SIoana Radulescu if (err) 428934ff6846SIoana Radulescu return err; 429034ff6846SIoana Radulescu } 429134ff6846SIoana Radulescu 429234ff6846SIoana Radulescu err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 429334ff6846SIoana Radulescu DPNI_QUEUE_TX, &priv->tx_qdid); 429434ff6846SIoana Radulescu if (err) { 429534ff6846SIoana Radulescu dev_err(dev, "dpni_get_qdid() failed\n"); 429634ff6846SIoana Radulescu return err; 429734ff6846SIoana Radulescu } 429834ff6846SIoana Radulescu 429934ff6846SIoana Radulescu return 0; 430034ff6846SIoana Radulescu } 430134ff6846SIoana Radulescu 430234ff6846SIoana Radulescu /* Allocate rings for storing incoming frame descriptors */ 43035d8dccf8SIoana Ciornei static int dpaa2_eth_alloc_rings(struct dpaa2_eth_priv *priv) 430434ff6846SIoana Radulescu { 430534ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 430634ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 430734ff6846SIoana Radulescu int i; 430834ff6846SIoana Radulescu 430934ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 431034ff6846SIoana Radulescu priv->channel[i]->store = 431134ff6846SIoana Radulescu dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 431234ff6846SIoana Radulescu if (!priv->channel[i]->store) { 431334ff6846SIoana Radulescu netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 431434ff6846SIoana Radulescu goto err_ring; 431534ff6846SIoana Radulescu } 431634ff6846SIoana Radulescu } 431734ff6846SIoana Radulescu 431834ff6846SIoana Radulescu return 0; 431934ff6846SIoana Radulescu 432034ff6846SIoana Radulescu err_ring: 432134ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 432234ff6846SIoana Radulescu if (!priv->channel[i]->store) 432334ff6846SIoana Radulescu break; 432434ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 432534ff6846SIoana Radulescu } 432634ff6846SIoana Radulescu 432734ff6846SIoana Radulescu return -ENOMEM; 432834ff6846SIoana Radulescu } 432934ff6846SIoana Radulescu 43305d8dccf8SIoana Ciornei static void dpaa2_eth_free_rings(struct dpaa2_eth_priv *priv) 433134ff6846SIoana Radulescu { 433234ff6846SIoana Radulescu int i; 433334ff6846SIoana Radulescu 433434ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 433534ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 433634ff6846SIoana Radulescu } 433734ff6846SIoana Radulescu 43385d8dccf8SIoana Ciornei static int dpaa2_eth_set_mac_addr(struct dpaa2_eth_priv *priv) 433934ff6846SIoana Radulescu { 434034ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 434134ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 434234ff6846SIoana Radulescu u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 434334ff6846SIoana Radulescu int err; 434434ff6846SIoana Radulescu 434534ff6846SIoana Radulescu /* Get firmware address, if any */ 434634ff6846SIoana Radulescu err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 434734ff6846SIoana Radulescu if (err) { 434834ff6846SIoana Radulescu dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 434934ff6846SIoana Radulescu return err; 435034ff6846SIoana Radulescu } 435134ff6846SIoana Radulescu 435234ff6846SIoana Radulescu /* Get DPNI attributes address, if any */ 435334ff6846SIoana Radulescu err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 435434ff6846SIoana Radulescu dpni_mac_addr); 435534ff6846SIoana Radulescu if (err) { 435634ff6846SIoana Radulescu dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 435734ff6846SIoana Radulescu return err; 435834ff6846SIoana Radulescu } 435934ff6846SIoana Radulescu 436034ff6846SIoana Radulescu /* First check if firmware has any address configured by bootloader */ 436134ff6846SIoana Radulescu if (!is_zero_ether_addr(mac_addr)) { 436234ff6846SIoana Radulescu /* If the DPMAC addr != DPNI addr, update it */ 436334ff6846SIoana Radulescu if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 436434ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, 436534ff6846SIoana Radulescu priv->mc_token, 436634ff6846SIoana Radulescu mac_addr); 436734ff6846SIoana Radulescu if (err) { 436834ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 436934ff6846SIoana Radulescu return err; 437034ff6846SIoana Radulescu } 437134ff6846SIoana Radulescu } 4372a05e4c0aSJakub Kicinski eth_hw_addr_set(net_dev, mac_addr); 437334ff6846SIoana Radulescu } else if (is_zero_ether_addr(dpni_mac_addr)) { 437434ff6846SIoana Radulescu /* No MAC address configured, fill in net_dev->dev_addr 437534ff6846SIoana Radulescu * with a random one 437634ff6846SIoana Radulescu */ 437734ff6846SIoana Radulescu eth_hw_addr_random(net_dev); 437834ff6846SIoana Radulescu dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 437934ff6846SIoana Radulescu 438034ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 438134ff6846SIoana Radulescu net_dev->dev_addr); 438234ff6846SIoana Radulescu if (err) { 438334ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 438434ff6846SIoana Radulescu return err; 438534ff6846SIoana Radulescu } 438634ff6846SIoana Radulescu 438734ff6846SIoana Radulescu /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 438834ff6846SIoana Radulescu * practical purposes, this will be our "permanent" mac address, 438934ff6846SIoana Radulescu * at least until the next reboot. This move will also permit 439034ff6846SIoana Radulescu * register_netdevice() to properly fill up net_dev->perm_addr. 439134ff6846SIoana Radulescu */ 439234ff6846SIoana Radulescu net_dev->addr_assign_type = NET_ADDR_PERM; 439334ff6846SIoana Radulescu } else { 439434ff6846SIoana Radulescu /* NET_ADDR_PERM is default, all we have to do is 439534ff6846SIoana Radulescu * fill in the device addr. 439634ff6846SIoana Radulescu */ 4397a05e4c0aSJakub Kicinski eth_hw_addr_set(net_dev, dpni_mac_addr); 439834ff6846SIoana Radulescu } 439934ff6846SIoana Radulescu 440034ff6846SIoana Radulescu return 0; 440134ff6846SIoana Radulescu } 440234ff6846SIoana Radulescu 44035d8dccf8SIoana Ciornei static int dpaa2_eth_netdev_init(struct net_device *net_dev) 440434ff6846SIoana Radulescu { 440534ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 440634ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 440734ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 440834ff6846SIoana Radulescu u64 supported = 0, not_supported = 0; 440934ff6846SIoana Radulescu u8 bcast_addr[ETH_ALEN]; 441034ff6846SIoana Radulescu u8 num_queues; 441134ff6846SIoana Radulescu int err; 441234ff6846SIoana Radulescu 441334ff6846SIoana Radulescu net_dev->netdev_ops = &dpaa2_eth_ops; 441434ff6846SIoana Radulescu net_dev->ethtool_ops = &dpaa2_ethtool_ops; 441534ff6846SIoana Radulescu 44165d8dccf8SIoana Ciornei err = dpaa2_eth_set_mac_addr(priv); 441734ff6846SIoana Radulescu if (err) 441834ff6846SIoana Radulescu return err; 441934ff6846SIoana Radulescu 442034ff6846SIoana Radulescu /* Explicitly add the broadcast address to the MAC filtering table */ 442134ff6846SIoana Radulescu eth_broadcast_addr(bcast_addr); 442234ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 442334ff6846SIoana Radulescu if (err) { 442434ff6846SIoana Radulescu dev_err(dev, "dpni_add_mac_addr() failed\n"); 442534ff6846SIoana Radulescu return err; 442634ff6846SIoana Radulescu } 442734ff6846SIoana Radulescu 442834ff6846SIoana Radulescu /* Set MTU upper limit; lower limit is 68B (default value) */ 442934ff6846SIoana Radulescu net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 443034ff6846SIoana Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 443134ff6846SIoana Radulescu DPAA2_ETH_MFL); 443234ff6846SIoana Radulescu if (err) { 443334ff6846SIoana Radulescu dev_err(dev, "dpni_set_max_frame_length() failed\n"); 443434ff6846SIoana Radulescu return err; 443534ff6846SIoana Radulescu } 443634ff6846SIoana Radulescu 443734ff6846SIoana Radulescu /* Set actual number of queues in the net device */ 443834ff6846SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 443934ff6846SIoana Radulescu err = netif_set_real_num_tx_queues(net_dev, num_queues); 444034ff6846SIoana Radulescu if (err) { 444134ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 444234ff6846SIoana Radulescu return err; 444334ff6846SIoana Radulescu } 444434ff6846SIoana Radulescu err = netif_set_real_num_rx_queues(net_dev, num_queues); 444534ff6846SIoana Radulescu if (err) { 444634ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 444734ff6846SIoana Radulescu return err; 444834ff6846SIoana Radulescu } 444934ff6846SIoana Radulescu 4450c4680c97SRadu Bulie dpaa2_eth_detect_features(priv); 4451c4680c97SRadu Bulie 445234ff6846SIoana Radulescu /* Capabilities listing */ 445334ff6846SIoana Radulescu supported |= IFF_LIVE_ADDR_CHANGE; 445434ff6846SIoana Radulescu 445534ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER) 445634ff6846SIoana Radulescu not_supported |= IFF_UNICAST_FLT; 445734ff6846SIoana Radulescu else 445834ff6846SIoana Radulescu supported |= IFF_UNICAST_FLT; 445934ff6846SIoana Radulescu 446034ff6846SIoana Radulescu net_dev->priv_flags |= supported; 446134ff6846SIoana Radulescu net_dev->priv_flags &= ~not_supported; 446234ff6846SIoana Radulescu 446334ff6846SIoana Radulescu /* Features */ 446434ff6846SIoana Radulescu net_dev->features = NETIF_F_RXCSUM | 446534ff6846SIoana Radulescu NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 446634ff6846SIoana Radulescu NETIF_F_SG | NETIF_F_HIGHDMA | 44673dc709e0SIoana Ciornei NETIF_F_LLTX | NETIF_F_HW_TC | NETIF_F_TSO; 44683dc709e0SIoana Ciornei net_dev->gso_max_segs = DPAA2_ETH_ENQUEUE_MAX_FDS; 446934ff6846SIoana Radulescu net_dev->hw_features = net_dev->features; 447034ff6846SIoana Radulescu 447170b32d82SIonut-robert Aron if (priv->dpni_attrs.vlan_filter_entries) 447270b32d82SIonut-robert Aron net_dev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 447370b32d82SIonut-robert Aron 447434ff6846SIoana Radulescu return 0; 447534ff6846SIoana Radulescu } 447634ff6846SIoana Radulescu 44775d8dccf8SIoana Ciornei static int dpaa2_eth_poll_link_state(void *arg) 447834ff6846SIoana Radulescu { 447934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 448034ff6846SIoana Radulescu int err; 448134ff6846SIoana Radulescu 448234ff6846SIoana Radulescu while (!kthread_should_stop()) { 44835d8dccf8SIoana Ciornei err = dpaa2_eth_link_state_update(priv); 448434ff6846SIoana Radulescu if (unlikely(err)) 448534ff6846SIoana Radulescu return err; 448634ff6846SIoana Radulescu 448734ff6846SIoana Radulescu msleep(DPAA2_ETH_LINK_STATE_REFRESH); 448834ff6846SIoana Radulescu } 448934ff6846SIoana Radulescu 449034ff6846SIoana Radulescu return 0; 449134ff6846SIoana Radulescu } 449234ff6846SIoana Radulescu 449371947923SIoana Ciornei static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) 449471947923SIoana Ciornei { 449571947923SIoana Ciornei struct fsl_mc_device *dpni_dev, *dpmac_dev; 449671947923SIoana Ciornei struct dpaa2_mac *mac; 449771947923SIoana Ciornei int err; 449871947923SIoana Ciornei 449971947923SIoana Ciornei dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); 450027cfdaddSIoana Ciornei dpmac_dev = fsl_mc_get_endpoint(dpni_dev, 0); 450147325da2SIoana Ciornei 450247325da2SIoana Ciornei if (PTR_ERR(dpmac_dev) == -EPROBE_DEFER) 450347325da2SIoana Ciornei return PTR_ERR(dpmac_dev); 450447325da2SIoana Ciornei 450547325da2SIoana Ciornei if (IS_ERR(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) 450671947923SIoana Ciornei return 0; 450771947923SIoana Ciornei 450871947923SIoana Ciornei mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); 450971947923SIoana Ciornei if (!mac) 451071947923SIoana Ciornei return -ENOMEM; 451171947923SIoana Ciornei 451271947923SIoana Ciornei mac->mc_dev = dpmac_dev; 451371947923SIoana Ciornei mac->mc_io = priv->mc_io; 451471947923SIoana Ciornei mac->net_dev = priv->net_dev; 451571947923SIoana Ciornei 4516095dca16SIoana Ciornei err = dpaa2_mac_open(mac); 4517095dca16SIoana Ciornei if (err) 4518095dca16SIoana Ciornei goto err_free_mac; 4519d87e6063SIoana Ciornei priv->mac = mac; 4520095dca16SIoana Ciornei 4521d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) { 452271947923SIoana Ciornei err = dpaa2_mac_connect(mac); 4523f5120f59SVladimir Oltean if (err && err != -EPROBE_DEFER) 4524f5120f59SVladimir Oltean netdev_err(priv->net_dev, "Error connecting to the MAC endpoint: %pe", 4525f5120f59SVladimir Oltean ERR_PTR(err)); 4526f5120f59SVladimir Oltean if (err) 4527095dca16SIoana Ciornei goto err_close_mac; 452871947923SIoana Ciornei } 452971947923SIoana Ciornei 453071947923SIoana Ciornei return 0; 4531095dca16SIoana Ciornei 4532095dca16SIoana Ciornei err_close_mac: 4533095dca16SIoana Ciornei dpaa2_mac_close(mac); 4534d87e6063SIoana Ciornei priv->mac = NULL; 4535095dca16SIoana Ciornei err_free_mac: 4536095dca16SIoana Ciornei kfree(mac); 4537095dca16SIoana Ciornei return err; 453871947923SIoana Ciornei } 453971947923SIoana Ciornei 454071947923SIoana Ciornei static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) 454171947923SIoana Ciornei { 4542d87e6063SIoana Ciornei if (dpaa2_eth_is_type_phy(priv)) 454371947923SIoana Ciornei dpaa2_mac_disconnect(priv->mac); 4544d87e6063SIoana Ciornei 4545848c1903SIoana Ciornei if (!dpaa2_eth_has_mac(priv)) 4546848c1903SIoana Ciornei return; 4547848c1903SIoana Ciornei 4548095dca16SIoana Ciornei dpaa2_mac_close(priv->mac); 454971947923SIoana Ciornei kfree(priv->mac); 455071947923SIoana Ciornei priv->mac = NULL; 455171947923SIoana Ciornei } 455271947923SIoana Ciornei 455334ff6846SIoana Radulescu static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 455434ff6846SIoana Radulescu { 455534ff6846SIoana Radulescu u32 status = ~0; 455634ff6846SIoana Radulescu struct device *dev = (struct device *)arg; 455734ff6846SIoana Radulescu struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 455834ff6846SIoana Radulescu struct net_device *net_dev = dev_get_drvdata(dev); 455971947923SIoana Ciornei struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 456034ff6846SIoana Radulescu int err; 456134ff6846SIoana Radulescu 456234ff6846SIoana Radulescu err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 456334ff6846SIoana Radulescu DPNI_IRQ_INDEX, &status); 456434ff6846SIoana Radulescu if (unlikely(err)) { 456534ff6846SIoana Radulescu netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 456634ff6846SIoana Radulescu return IRQ_HANDLED; 456734ff6846SIoana Radulescu } 456834ff6846SIoana Radulescu 456934ff6846SIoana Radulescu if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 45705d8dccf8SIoana Ciornei dpaa2_eth_link_state_update(netdev_priv(net_dev)); 457134ff6846SIoana Radulescu 4572f5c3fffaSIoana Ciornei if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) { 45735d8dccf8SIoana Ciornei dpaa2_eth_set_mac_addr(netdev_priv(net_dev)); 45745d8dccf8SIoana Ciornei dpaa2_eth_update_tx_fqids(priv); 457571947923SIoana Ciornei 457671947923SIoana Ciornei rtnl_lock(); 4577d87e6063SIoana Ciornei if (dpaa2_eth_has_mac(priv)) 457871947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 457971947923SIoana Ciornei else 458071947923SIoana Ciornei dpaa2_eth_connect_mac(priv); 458171947923SIoana Ciornei rtnl_unlock(); 4582f5c3fffaSIoana Ciornei } 45838398b375SFlorin Chiculita 458434ff6846SIoana Radulescu return IRQ_HANDLED; 458534ff6846SIoana Radulescu } 458634ff6846SIoana Radulescu 45875d8dccf8SIoana Ciornei static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) 458834ff6846SIoana Radulescu { 458934ff6846SIoana Radulescu int err = 0; 459034ff6846SIoana Radulescu struct fsl_mc_device_irq *irq; 459134ff6846SIoana Radulescu 459234ff6846SIoana Radulescu err = fsl_mc_allocate_irqs(ls_dev); 459334ff6846SIoana Radulescu if (err) { 459434ff6846SIoana Radulescu dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 459534ff6846SIoana Radulescu return err; 459634ff6846SIoana Radulescu } 459734ff6846SIoana Radulescu 459834ff6846SIoana Radulescu irq = ls_dev->irqs[0]; 4599d86a6d47SThomas Gleixner err = devm_request_threaded_irq(&ls_dev->dev, irq->virq, 460034ff6846SIoana Radulescu NULL, dpni_irq0_handler_thread, 460134ff6846SIoana Radulescu IRQF_NO_SUSPEND | IRQF_ONESHOT, 460234ff6846SIoana Radulescu dev_name(&ls_dev->dev), &ls_dev->dev); 460334ff6846SIoana Radulescu if (err < 0) { 460434ff6846SIoana Radulescu dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 460534ff6846SIoana Radulescu goto free_mc_irq; 460634ff6846SIoana Radulescu } 460734ff6846SIoana Radulescu 460834ff6846SIoana Radulescu err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 46098398b375SFlorin Chiculita DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 46108398b375SFlorin Chiculita DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 461134ff6846SIoana Radulescu if (err < 0) { 461234ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 461334ff6846SIoana Radulescu goto free_irq; 461434ff6846SIoana Radulescu } 461534ff6846SIoana Radulescu 461634ff6846SIoana Radulescu err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 461734ff6846SIoana Radulescu DPNI_IRQ_INDEX, 1); 461834ff6846SIoana Radulescu if (err < 0) { 461934ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 462034ff6846SIoana Radulescu goto free_irq; 462134ff6846SIoana Radulescu } 462234ff6846SIoana Radulescu 462334ff6846SIoana Radulescu return 0; 462434ff6846SIoana Radulescu 462534ff6846SIoana Radulescu free_irq: 4626d86a6d47SThomas Gleixner devm_free_irq(&ls_dev->dev, irq->virq, &ls_dev->dev); 462734ff6846SIoana Radulescu free_mc_irq: 462834ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 462934ff6846SIoana Radulescu 463034ff6846SIoana Radulescu return err; 463134ff6846SIoana Radulescu } 463234ff6846SIoana Radulescu 46335d8dccf8SIoana Ciornei static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv) 463434ff6846SIoana Radulescu { 463534ff6846SIoana Radulescu int i; 463634ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 463734ff6846SIoana Radulescu 463834ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 463934ff6846SIoana Radulescu ch = priv->channel[i]; 464034ff6846SIoana Radulescu /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 4641b48b89f9SJakub Kicinski netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll); 464234ff6846SIoana Radulescu } 464334ff6846SIoana Radulescu } 464434ff6846SIoana Radulescu 46455d8dccf8SIoana Ciornei static void dpaa2_eth_del_ch_napi(struct dpaa2_eth_priv *priv) 464634ff6846SIoana Radulescu { 464734ff6846SIoana Radulescu int i; 464834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 464934ff6846SIoana Radulescu 465034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 465134ff6846SIoana Radulescu ch = priv->channel[i]; 465234ff6846SIoana Radulescu netif_napi_del(&ch->napi); 465334ff6846SIoana Radulescu } 465434ff6846SIoana Radulescu } 465534ff6846SIoana Radulescu 465634ff6846SIoana Radulescu static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 465734ff6846SIoana Radulescu { 465834ff6846SIoana Radulescu struct device *dev; 465934ff6846SIoana Radulescu struct net_device *net_dev = NULL; 466034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = NULL; 466134ff6846SIoana Radulescu int err = 0; 466234ff6846SIoana Radulescu 466334ff6846SIoana Radulescu dev = &dpni_dev->dev; 466434ff6846SIoana Radulescu 466534ff6846SIoana Radulescu /* Net device */ 4666ab1e6de2SIoana Radulescu net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 466734ff6846SIoana Radulescu if (!net_dev) { 466834ff6846SIoana Radulescu dev_err(dev, "alloc_etherdev_mq() failed\n"); 466934ff6846SIoana Radulescu return -ENOMEM; 467034ff6846SIoana Radulescu } 467134ff6846SIoana Radulescu 467234ff6846SIoana Radulescu SET_NETDEV_DEV(net_dev, dev); 467334ff6846SIoana Radulescu dev_set_drvdata(dev, net_dev); 467434ff6846SIoana Radulescu 467534ff6846SIoana Radulescu priv = netdev_priv(net_dev); 467634ff6846SIoana Radulescu priv->net_dev = net_dev; 467734ff6846SIoana Radulescu 467834ff6846SIoana Radulescu priv->iommu_domain = iommu_get_domain_for_dev(dev); 467934ff6846SIoana Radulescu 46801cf773bdSYangbo Lu priv->tx_tstamp_type = HWTSTAMP_TX_OFF; 46811cf773bdSYangbo Lu priv->rx_tstamp = false; 46821cf773bdSYangbo Lu 4683c5521189SYangbo Lu priv->dpaa2_ptp_wq = alloc_workqueue("dpaa2_ptp_wq", 0, 0); 4684c5521189SYangbo Lu if (!priv->dpaa2_ptp_wq) { 4685c5521189SYangbo Lu err = -ENOMEM; 4686c5521189SYangbo Lu goto err_wq_alloc; 4687c5521189SYangbo Lu } 4688c5521189SYangbo Lu 4689c5521189SYangbo Lu INIT_WORK(&priv->tx_onestep_tstamp, dpaa2_eth_tx_onestep_tstamp); 469007dd4485SRadu Bulie mutex_init(&priv->onestep_tstamp_lock); 4691c5521189SYangbo Lu skb_queue_head_init(&priv->tx_skbs); 4692c5521189SYangbo Lu 46938ed3cefcSIoana Ciornei priv->rx_copybreak = DPAA2_ETH_DEFAULT_COPYBREAK; 46948ed3cefcSIoana Ciornei 469534ff6846SIoana Radulescu /* Obtain a MC portal */ 469634ff6846SIoana Radulescu err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 469734ff6846SIoana Radulescu &priv->mc_io); 469834ff6846SIoana Radulescu if (err) { 469934ff6846SIoana Radulescu if (err == -ENXIO) 470034ff6846SIoana Radulescu err = -EPROBE_DEFER; 470134ff6846SIoana Radulescu else 470234ff6846SIoana Radulescu dev_err(dev, "MC portal allocation failed\n"); 470334ff6846SIoana Radulescu goto err_portal_alloc; 470434ff6846SIoana Radulescu } 470534ff6846SIoana Radulescu 470634ff6846SIoana Radulescu /* MC objects initialization and configuration */ 47075d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpni(dpni_dev); 470834ff6846SIoana Radulescu if (err) 470934ff6846SIoana Radulescu goto err_dpni_setup; 471034ff6846SIoana Radulescu 47115d8dccf8SIoana Ciornei err = dpaa2_eth_setup_dpio(priv); 471234ff6846SIoana Radulescu if (err) 471334ff6846SIoana Radulescu goto err_dpio_setup; 471434ff6846SIoana Radulescu 47155d8dccf8SIoana Ciornei dpaa2_eth_setup_fqs(priv); 471634ff6846SIoana Radulescu 4717095174daSRobert-Ionut Alexa err = dpaa2_eth_setup_default_dpbp(priv); 471834ff6846SIoana Radulescu if (err) 471934ff6846SIoana Radulescu goto err_dpbp_setup; 472034ff6846SIoana Radulescu 47215d8dccf8SIoana Ciornei err = dpaa2_eth_bind_dpni(priv); 472234ff6846SIoana Radulescu if (err) 472334ff6846SIoana Radulescu goto err_bind; 472434ff6846SIoana Radulescu 472534ff6846SIoana Radulescu /* Add a NAPI context for each channel */ 47265d8dccf8SIoana Ciornei dpaa2_eth_add_ch_napi(priv); 472734ff6846SIoana Radulescu 472834ff6846SIoana Radulescu /* Percpu statistics */ 472934ff6846SIoana Radulescu priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 473034ff6846SIoana Radulescu if (!priv->percpu_stats) { 473134ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 473234ff6846SIoana Radulescu err = -ENOMEM; 473334ff6846SIoana Radulescu goto err_alloc_percpu_stats; 473434ff6846SIoana Radulescu } 473534ff6846SIoana Radulescu priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 473634ff6846SIoana Radulescu if (!priv->percpu_extras) { 473734ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 473834ff6846SIoana Radulescu err = -ENOMEM; 473934ff6846SIoana Radulescu goto err_alloc_percpu_extras; 474034ff6846SIoana Radulescu } 474134ff6846SIoana Radulescu 4742d70446eeSIoana Ciornei priv->sgt_cache = alloc_percpu(*priv->sgt_cache); 4743d70446eeSIoana Ciornei if (!priv->sgt_cache) { 4744d70446eeSIoana Ciornei dev_err(dev, "alloc_percpu(sgt_cache) failed\n"); 4745d70446eeSIoana Ciornei err = -ENOMEM; 4746d70446eeSIoana Ciornei goto err_alloc_sgt_cache; 4747d70446eeSIoana Ciornei } 4748d70446eeSIoana Ciornei 4749a4ca448eSIoana Ciornei priv->fd = alloc_percpu(*priv->fd); 4750a4ca448eSIoana Ciornei if (!priv->fd) { 4751a4ca448eSIoana Ciornei dev_err(dev, "alloc_percpu(fds) failed\n"); 4752a4ca448eSIoana Ciornei err = -ENOMEM; 4753a4ca448eSIoana Ciornei goto err_alloc_fds; 4754a4ca448eSIoana Ciornei } 4755a4ca448eSIoana Ciornei 47565d8dccf8SIoana Ciornei err = dpaa2_eth_netdev_init(net_dev); 475734ff6846SIoana Radulescu if (err) 475834ff6846SIoana Radulescu goto err_netdev_init; 475934ff6846SIoana Radulescu 476034ff6846SIoana Radulescu /* Configure checksum offload based on current interface flags */ 47615d8dccf8SIoana Ciornei err = dpaa2_eth_set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 476234ff6846SIoana Radulescu if (err) 476334ff6846SIoana Radulescu goto err_csum; 476434ff6846SIoana Radulescu 47655d8dccf8SIoana Ciornei err = dpaa2_eth_set_tx_csum(priv, 47665d8dccf8SIoana Ciornei !!(net_dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 476734ff6846SIoana Radulescu if (err) 476834ff6846SIoana Radulescu goto err_csum; 476934ff6846SIoana Radulescu 47705d8dccf8SIoana Ciornei err = dpaa2_eth_alloc_rings(priv); 477134ff6846SIoana Radulescu if (err) 477234ff6846SIoana Radulescu goto err_alloc_rings; 477334ff6846SIoana Radulescu 4774f395b69fSIoana Ciornei #ifdef CONFIG_FSL_DPAA2_ETH_DCB 4775f395b69fSIoana Ciornei if (dpaa2_eth_has_pause_support(priv) && priv->vlan_cls_enabled) { 4776f395b69fSIoana Ciornei priv->dcbx_mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; 4777f395b69fSIoana Ciornei net_dev->dcbnl_ops = &dpaa2_eth_dcbnl_ops; 4778f395b69fSIoana Ciornei } else { 4779f395b69fSIoana Ciornei dev_dbg(dev, "PFC not supported\n"); 4780f395b69fSIoana Ciornei } 4781f395b69fSIoana Ciornei #endif 4782f395b69fSIoana Ciornei 47835d8dccf8SIoana Ciornei err = dpaa2_eth_setup_irqs(dpni_dev); 478434ff6846SIoana Radulescu if (err) { 478534ff6846SIoana Radulescu netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 47865d8dccf8SIoana Ciornei priv->poll_thread = kthread_run(dpaa2_eth_poll_link_state, priv, 478734ff6846SIoana Radulescu "%s_poll_link", net_dev->name); 478834ff6846SIoana Radulescu if (IS_ERR(priv->poll_thread)) { 478934ff6846SIoana Radulescu dev_err(dev, "Error starting polling thread\n"); 479034ff6846SIoana Radulescu goto err_poll_thread; 479134ff6846SIoana Radulescu } 479234ff6846SIoana Radulescu priv->do_link_poll = true; 479334ff6846SIoana Radulescu } 479434ff6846SIoana Radulescu 479571947923SIoana Ciornei err = dpaa2_eth_connect_mac(priv); 479671947923SIoana Ciornei if (err) 479771947923SIoana Ciornei goto err_connect_mac; 479871947923SIoana Ciornei 4799bbb9ae25SLeon Romanovsky err = dpaa2_eth_dl_alloc(priv); 4800ceeb03adSIoana Ciornei if (err) 4801ceeb03adSIoana Ciornei goto err_dl_register; 4802ceeb03adSIoana Ciornei 4803061d631fSIoana Ciornei err = dpaa2_eth_dl_traps_register(priv); 4804061d631fSIoana Ciornei if (err) 4805061d631fSIoana Ciornei goto err_dl_trap_register; 4806061d631fSIoana Ciornei 4807ceeb03adSIoana Ciornei err = dpaa2_eth_dl_port_add(priv); 4808ceeb03adSIoana Ciornei if (err) 4809ceeb03adSIoana Ciornei goto err_dl_port_add; 4810ceeb03adSIoana Ciornei 481134ff6846SIoana Radulescu err = register_netdev(net_dev); 481234ff6846SIoana Radulescu if (err < 0) { 481334ff6846SIoana Radulescu dev_err(dev, "register_netdev() failed\n"); 481434ff6846SIoana Radulescu goto err_netdev_reg; 481534ff6846SIoana Radulescu } 481634ff6846SIoana Radulescu 4817091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4818091a19eaSIoana Radulescu dpaa2_dbg_add(priv); 4819091a19eaSIoana Radulescu #endif 4820091a19eaSIoana Radulescu 4821bbb9ae25SLeon Romanovsky dpaa2_eth_dl_register(priv); 482234ff6846SIoana Radulescu dev_info(dev, "Probed interface %s\n", net_dev->name); 482334ff6846SIoana Radulescu return 0; 482434ff6846SIoana Radulescu 482534ff6846SIoana Radulescu err_netdev_reg: 4826ceeb03adSIoana Ciornei dpaa2_eth_dl_port_del(priv); 4827ceeb03adSIoana Ciornei err_dl_port_add: 4828061d631fSIoana Ciornei dpaa2_eth_dl_traps_unregister(priv); 4829061d631fSIoana Ciornei err_dl_trap_register: 4830bbb9ae25SLeon Romanovsky dpaa2_eth_dl_free(priv); 4831ceeb03adSIoana Ciornei err_dl_register: 483271947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 483371947923SIoana Ciornei err_connect_mac: 483434ff6846SIoana Radulescu if (priv->do_link_poll) 483534ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 483634ff6846SIoana Radulescu else 483734ff6846SIoana Radulescu fsl_mc_free_irqs(dpni_dev); 483834ff6846SIoana Radulescu err_poll_thread: 48395d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 484034ff6846SIoana Radulescu err_alloc_rings: 484134ff6846SIoana Radulescu err_csum: 484234ff6846SIoana Radulescu err_netdev_init: 4843a4ca448eSIoana Ciornei free_percpu(priv->fd); 4844a4ca448eSIoana Ciornei err_alloc_fds: 4845d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 4846d70446eeSIoana Ciornei err_alloc_sgt_cache: 484734ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 484834ff6846SIoana Radulescu err_alloc_percpu_extras: 484934ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 485034ff6846SIoana Radulescu err_alloc_percpu_stats: 48515d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 485234ff6846SIoana Radulescu err_bind: 4853095174daSRobert-Ionut Alexa dpaa2_eth_free_dpbps(priv); 485434ff6846SIoana Radulescu err_dpbp_setup: 48555d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 485634ff6846SIoana Radulescu err_dpio_setup: 48575d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 485834ff6846SIoana Radulescu err_dpni_setup: 485934ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 486034ff6846SIoana Radulescu err_portal_alloc: 4861c5521189SYangbo Lu destroy_workqueue(priv->dpaa2_ptp_wq); 4862c5521189SYangbo Lu err_wq_alloc: 486334ff6846SIoana Radulescu dev_set_drvdata(dev, NULL); 486434ff6846SIoana Radulescu free_netdev(net_dev); 486534ff6846SIoana Radulescu 486634ff6846SIoana Radulescu return err; 486734ff6846SIoana Radulescu } 486834ff6846SIoana Radulescu 486934ff6846SIoana Radulescu static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 487034ff6846SIoana Radulescu { 487134ff6846SIoana Radulescu struct device *dev; 487234ff6846SIoana Radulescu struct net_device *net_dev; 487334ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 487434ff6846SIoana Radulescu 487534ff6846SIoana Radulescu dev = &ls_dev->dev; 487634ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 487734ff6846SIoana Radulescu priv = netdev_priv(net_dev); 487834ff6846SIoana Radulescu 4879bbb9ae25SLeon Romanovsky dpaa2_eth_dl_unregister(priv); 4880bbb9ae25SLeon Romanovsky 4881091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 4882091a19eaSIoana Radulescu dpaa2_dbg_remove(priv); 4883091a19eaSIoana Radulescu #endif 48849ccc6e0cSRobert-Ionut Alexa 48859ccc6e0cSRobert-Ionut Alexa unregister_netdev(net_dev); 488671947923SIoana Ciornei rtnl_lock(); 488771947923SIoana Ciornei dpaa2_eth_disconnect_mac(priv); 488871947923SIoana Ciornei rtnl_unlock(); 488971947923SIoana Ciornei 4890ceeb03adSIoana Ciornei dpaa2_eth_dl_port_del(priv); 4891061d631fSIoana Ciornei dpaa2_eth_dl_traps_unregister(priv); 4892bbb9ae25SLeon Romanovsky dpaa2_eth_dl_free(priv); 4893ceeb03adSIoana Ciornei 489434ff6846SIoana Radulescu if (priv->do_link_poll) 489534ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 489634ff6846SIoana Radulescu else 489734ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 489834ff6846SIoana Radulescu 48995d8dccf8SIoana Ciornei dpaa2_eth_free_rings(priv); 4900a4ca448eSIoana Ciornei free_percpu(priv->fd); 4901d70446eeSIoana Ciornei free_percpu(priv->sgt_cache); 490234ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 490334ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 490434ff6846SIoana Radulescu 49055d8dccf8SIoana Ciornei dpaa2_eth_del_ch_napi(priv); 4906095174daSRobert-Ionut Alexa dpaa2_eth_free_dpbps(priv); 49075d8dccf8SIoana Ciornei dpaa2_eth_free_dpio(priv); 49085d8dccf8SIoana Ciornei dpaa2_eth_free_dpni(priv); 4909c4680c97SRadu Bulie if (priv->onestep_reg_base) 4910c4680c97SRadu Bulie iounmap(priv->onestep_reg_base); 491134ff6846SIoana Radulescu 491234ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 491334ff6846SIoana Radulescu 4914f4a8adbfSDongliang Mu destroy_workqueue(priv->dpaa2_ptp_wq); 4915f4a8adbfSDongliang Mu 491634ff6846SIoana Radulescu dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 491734ff6846SIoana Radulescu 49189b5a3332SPavel Skripkin free_netdev(net_dev); 49199b5a3332SPavel Skripkin 492034ff6846SIoana Radulescu return 0; 492134ff6846SIoana Radulescu } 492234ff6846SIoana Radulescu 492334ff6846SIoana Radulescu static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 492434ff6846SIoana Radulescu { 492534ff6846SIoana Radulescu .vendor = FSL_MC_VENDOR_FREESCALE, 492634ff6846SIoana Radulescu .obj_type = "dpni", 492734ff6846SIoana Radulescu }, 492834ff6846SIoana Radulescu { .vendor = 0x0 } 492934ff6846SIoana Radulescu }; 493034ff6846SIoana Radulescu MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 493134ff6846SIoana Radulescu 493234ff6846SIoana Radulescu static struct fsl_mc_driver dpaa2_eth_driver = { 493334ff6846SIoana Radulescu .driver = { 493434ff6846SIoana Radulescu .name = KBUILD_MODNAME, 493534ff6846SIoana Radulescu .owner = THIS_MODULE, 493634ff6846SIoana Radulescu }, 493734ff6846SIoana Radulescu .probe = dpaa2_eth_probe, 493834ff6846SIoana Radulescu .remove = dpaa2_eth_remove, 493934ff6846SIoana Radulescu .match_id_table = dpaa2_eth_match_id_table 494034ff6846SIoana Radulescu }; 494134ff6846SIoana Radulescu 4942091a19eaSIoana Radulescu static int __init dpaa2_eth_driver_init(void) 4943091a19eaSIoana Radulescu { 4944091a19eaSIoana Radulescu int err; 4945091a19eaSIoana Radulescu 4946091a19eaSIoana Radulescu dpaa2_eth_dbg_init(); 4947091a19eaSIoana Radulescu err = fsl_mc_driver_register(&dpaa2_eth_driver); 4948091a19eaSIoana Radulescu if (err) { 4949091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4950091a19eaSIoana Radulescu return err; 4951091a19eaSIoana Radulescu } 4952091a19eaSIoana Radulescu 4953091a19eaSIoana Radulescu return 0; 4954091a19eaSIoana Radulescu } 4955091a19eaSIoana Radulescu 4956091a19eaSIoana Radulescu static void __exit dpaa2_eth_driver_exit(void) 4957091a19eaSIoana Radulescu { 4958091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 4959091a19eaSIoana Radulescu fsl_mc_driver_unregister(&dpaa2_eth_driver); 4960091a19eaSIoana Radulescu } 4961091a19eaSIoana Radulescu 4962091a19eaSIoana Radulescu module_init(dpaa2_eth_driver_init); 4963091a19eaSIoana Radulescu module_exit(dpaa2_eth_driver_exit); 4964