134ff6846SIoana Radulescu // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 234ff6846SIoana Radulescu /* Copyright 2014-2016 Freescale Semiconductor Inc. 334ff6846SIoana Radulescu * Copyright 2016-2017 NXP 434ff6846SIoana Radulescu */ 534ff6846SIoana Radulescu #include <linux/init.h> 634ff6846SIoana Radulescu #include <linux/module.h> 734ff6846SIoana Radulescu #include <linux/platform_device.h> 834ff6846SIoana Radulescu #include <linux/etherdevice.h> 934ff6846SIoana Radulescu #include <linux/of_net.h> 1034ff6846SIoana Radulescu #include <linux/interrupt.h> 1134ff6846SIoana Radulescu #include <linux/msi.h> 1234ff6846SIoana Radulescu #include <linux/kthread.h> 1334ff6846SIoana Radulescu #include <linux/iommu.h> 1434ff6846SIoana Radulescu #include <linux/net_tstamp.h> 1534ff6846SIoana Radulescu #include <linux/fsl/mc.h> 167e273a8eSIoana Ciocoi Radulescu #include <linux/bpf.h> 177e273a8eSIoana Ciocoi Radulescu #include <linux/bpf_trace.h> 1834ff6846SIoana Radulescu #include <net/sock.h> 1934ff6846SIoana Radulescu 2034ff6846SIoana Radulescu #include "dpaa2-eth.h" 2134ff6846SIoana Radulescu 2234ff6846SIoana Radulescu /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 2334ff6846SIoana Radulescu * using trace events only need to #include <trace/events/sched.h> 2434ff6846SIoana Radulescu */ 2534ff6846SIoana Radulescu #define CREATE_TRACE_POINTS 2634ff6846SIoana Radulescu #include "dpaa2-eth-trace.h" 2734ff6846SIoana Radulescu 2834ff6846SIoana Radulescu MODULE_LICENSE("Dual BSD/GPL"); 2934ff6846SIoana Radulescu MODULE_AUTHOR("Freescale Semiconductor, Inc"); 3034ff6846SIoana Radulescu MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 3134ff6846SIoana Radulescu 3234ff6846SIoana Radulescu static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 3334ff6846SIoana Radulescu dma_addr_t iova_addr) 3434ff6846SIoana Radulescu { 3534ff6846SIoana Radulescu phys_addr_t phys_addr; 3634ff6846SIoana Radulescu 3734ff6846SIoana Radulescu phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 3834ff6846SIoana Radulescu 3934ff6846SIoana Radulescu return phys_to_virt(phys_addr); 4034ff6846SIoana Radulescu } 4134ff6846SIoana Radulescu 4234ff6846SIoana Radulescu static void validate_rx_csum(struct dpaa2_eth_priv *priv, 4334ff6846SIoana Radulescu u32 fd_status, 4434ff6846SIoana Radulescu struct sk_buff *skb) 4534ff6846SIoana Radulescu { 4634ff6846SIoana Radulescu skb_checksum_none_assert(skb); 4734ff6846SIoana Radulescu 4834ff6846SIoana Radulescu /* HW checksum validation is disabled, nothing to do here */ 4934ff6846SIoana Radulescu if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 5034ff6846SIoana Radulescu return; 5134ff6846SIoana Radulescu 5234ff6846SIoana Radulescu /* Read checksum validation bits */ 5334ff6846SIoana Radulescu if (!((fd_status & DPAA2_FAS_L3CV) && 5434ff6846SIoana Radulescu (fd_status & DPAA2_FAS_L4CV))) 5534ff6846SIoana Radulescu return; 5634ff6846SIoana Radulescu 5734ff6846SIoana Radulescu /* Inform the stack there's no need to compute L3/L4 csum anymore */ 5834ff6846SIoana Radulescu skb->ip_summed = CHECKSUM_UNNECESSARY; 5934ff6846SIoana Radulescu } 6034ff6846SIoana Radulescu 6134ff6846SIoana Radulescu /* Free a received FD. 6234ff6846SIoana Radulescu * Not to be used for Tx conf FDs or on any other paths. 6334ff6846SIoana Radulescu */ 6434ff6846SIoana Radulescu static void free_rx_fd(struct dpaa2_eth_priv *priv, 6534ff6846SIoana Radulescu const struct dpaa2_fd *fd, 6634ff6846SIoana Radulescu void *vaddr) 6734ff6846SIoana Radulescu { 6834ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 6934ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 7034ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 7134ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 7234ff6846SIoana Radulescu void *sg_vaddr; 7334ff6846SIoana Radulescu int i; 7434ff6846SIoana Radulescu 7534ff6846SIoana Radulescu /* If single buffer frame, just free the data buffer */ 7634ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) 7734ff6846SIoana Radulescu goto free_buf; 7834ff6846SIoana Radulescu else if (fd_format != dpaa2_fd_sg) 7934ff6846SIoana Radulescu /* We don't support any other format */ 8034ff6846SIoana Radulescu return; 8134ff6846SIoana Radulescu 8234ff6846SIoana Radulescu /* For S/G frames, we first need to free all SG entries 8334ff6846SIoana Radulescu * except the first one, which was taken care of already 8434ff6846SIoana Radulescu */ 8534ff6846SIoana Radulescu sgt = vaddr + dpaa2_fd_get_offset(fd); 8634ff6846SIoana Radulescu for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 8734ff6846SIoana Radulescu addr = dpaa2_sg_get_addr(&sgt[i]); 8834ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 8927c87486SIoana Ciocoi Radulescu dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 9018c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 9134ff6846SIoana Radulescu 9227c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 9334ff6846SIoana Radulescu if (dpaa2_sg_is_final(&sgt[i])) 9434ff6846SIoana Radulescu break; 9534ff6846SIoana Radulescu } 9634ff6846SIoana Radulescu 9734ff6846SIoana Radulescu free_buf: 9827c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 9934ff6846SIoana Radulescu } 10034ff6846SIoana Radulescu 10134ff6846SIoana Radulescu /* Build a linear skb based on a single-buffer frame descriptor */ 102fdb6ca9eSIoana Ciornei static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch, 10334ff6846SIoana Radulescu const struct dpaa2_fd *fd, 10434ff6846SIoana Radulescu void *fd_vaddr) 10534ff6846SIoana Radulescu { 10634ff6846SIoana Radulescu struct sk_buff *skb = NULL; 10734ff6846SIoana Radulescu u16 fd_offset = dpaa2_fd_get_offset(fd); 10834ff6846SIoana Radulescu u32 fd_length = dpaa2_fd_get_len(fd); 10934ff6846SIoana Radulescu 11034ff6846SIoana Radulescu ch->buf_count--; 11134ff6846SIoana Radulescu 11227c87486SIoana Ciocoi Radulescu skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 11334ff6846SIoana Radulescu if (unlikely(!skb)) 11434ff6846SIoana Radulescu return NULL; 11534ff6846SIoana Radulescu 11634ff6846SIoana Radulescu skb_reserve(skb, fd_offset); 11734ff6846SIoana Radulescu skb_put(skb, fd_length); 11834ff6846SIoana Radulescu 11934ff6846SIoana Radulescu return skb; 12034ff6846SIoana Radulescu } 12134ff6846SIoana Radulescu 12234ff6846SIoana Radulescu /* Build a non linear (fragmented) skb based on a S/G table */ 12334ff6846SIoana Radulescu static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv, 12434ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 12534ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt) 12634ff6846SIoana Radulescu { 12734ff6846SIoana Radulescu struct sk_buff *skb = NULL; 12834ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 12934ff6846SIoana Radulescu void *sg_vaddr; 13034ff6846SIoana Radulescu dma_addr_t sg_addr; 13134ff6846SIoana Radulescu u16 sg_offset; 13234ff6846SIoana Radulescu u32 sg_length; 13334ff6846SIoana Radulescu struct page *page, *head_page; 13434ff6846SIoana Radulescu int page_offset; 13534ff6846SIoana Radulescu int i; 13634ff6846SIoana Radulescu 13734ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 13834ff6846SIoana Radulescu struct dpaa2_sg_entry *sge = &sgt[i]; 13934ff6846SIoana Radulescu 14034ff6846SIoana Radulescu /* NOTE: We only support SG entries in dpaa2_sg_single format, 14134ff6846SIoana Radulescu * but this is the only format we may receive from HW anyway 14234ff6846SIoana Radulescu */ 14334ff6846SIoana Radulescu 14434ff6846SIoana Radulescu /* Get the address and length from the S/G entry */ 14534ff6846SIoana Radulescu sg_addr = dpaa2_sg_get_addr(sge); 14634ff6846SIoana Radulescu sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 14727c87486SIoana Ciocoi Radulescu dma_unmap_page(dev, sg_addr, DPAA2_ETH_RX_BUF_SIZE, 14818c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 14934ff6846SIoana Radulescu 15034ff6846SIoana Radulescu sg_length = dpaa2_sg_get_len(sge); 15134ff6846SIoana Radulescu 15234ff6846SIoana Radulescu if (i == 0) { 15334ff6846SIoana Radulescu /* We build the skb around the first data buffer */ 15427c87486SIoana Ciocoi Radulescu skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 15534ff6846SIoana Radulescu if (unlikely(!skb)) { 15634ff6846SIoana Radulescu /* Free the first SG entry now, since we already 15734ff6846SIoana Radulescu * unmapped it and obtained the virtual address 15834ff6846SIoana Radulescu */ 15927c87486SIoana Ciocoi Radulescu free_pages((unsigned long)sg_vaddr, 0); 16034ff6846SIoana Radulescu 16134ff6846SIoana Radulescu /* We still need to subtract the buffers used 16234ff6846SIoana Radulescu * by this FD from our software counter 16334ff6846SIoana Radulescu */ 16434ff6846SIoana Radulescu while (!dpaa2_sg_is_final(&sgt[i]) && 16534ff6846SIoana Radulescu i < DPAA2_ETH_MAX_SG_ENTRIES) 16634ff6846SIoana Radulescu i++; 16734ff6846SIoana Radulescu break; 16834ff6846SIoana Radulescu } 16934ff6846SIoana Radulescu 17034ff6846SIoana Radulescu sg_offset = dpaa2_sg_get_offset(sge); 17134ff6846SIoana Radulescu skb_reserve(skb, sg_offset); 17234ff6846SIoana Radulescu skb_put(skb, sg_length); 17334ff6846SIoana Radulescu } else { 17434ff6846SIoana Radulescu /* Rest of the data buffers are stored as skb frags */ 17534ff6846SIoana Radulescu page = virt_to_page(sg_vaddr); 17634ff6846SIoana Radulescu head_page = virt_to_head_page(sg_vaddr); 17734ff6846SIoana Radulescu 17834ff6846SIoana Radulescu /* Offset in page (which may be compound). 17934ff6846SIoana Radulescu * Data in subsequent SG entries is stored from the 18034ff6846SIoana Radulescu * beginning of the buffer, so we don't need to add the 18134ff6846SIoana Radulescu * sg_offset. 18234ff6846SIoana Radulescu */ 18334ff6846SIoana Radulescu page_offset = ((unsigned long)sg_vaddr & 18434ff6846SIoana Radulescu (PAGE_SIZE - 1)) + 18534ff6846SIoana Radulescu (page_address(page) - page_address(head_page)); 18634ff6846SIoana Radulescu 18734ff6846SIoana Radulescu skb_add_rx_frag(skb, i - 1, head_page, page_offset, 18834ff6846SIoana Radulescu sg_length, DPAA2_ETH_RX_BUF_SIZE); 18934ff6846SIoana Radulescu } 19034ff6846SIoana Radulescu 19134ff6846SIoana Radulescu if (dpaa2_sg_is_final(sge)) 19234ff6846SIoana Radulescu break; 19334ff6846SIoana Radulescu } 19434ff6846SIoana Radulescu 19534ff6846SIoana Radulescu WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 19634ff6846SIoana Radulescu 19734ff6846SIoana Radulescu /* Count all data buffers + SG table buffer */ 19834ff6846SIoana Radulescu ch->buf_count -= i + 2; 19934ff6846SIoana Radulescu 20034ff6846SIoana Radulescu return skb; 20134ff6846SIoana Radulescu } 20234ff6846SIoana Radulescu 203569375fbSIoana Ciocoi Radulescu /* Free buffers acquired from the buffer pool or which were meant to 204569375fbSIoana Ciocoi Radulescu * be released in the pool 205569375fbSIoana Ciocoi Radulescu */ 206569375fbSIoana Ciocoi Radulescu static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count) 207569375fbSIoana Ciocoi Radulescu { 208569375fbSIoana Ciocoi Radulescu struct device *dev = priv->net_dev->dev.parent; 209569375fbSIoana Ciocoi Radulescu void *vaddr; 210569375fbSIoana Ciocoi Radulescu int i; 211569375fbSIoana Ciocoi Radulescu 212569375fbSIoana Ciocoi Radulescu for (i = 0; i < count; i++) { 213569375fbSIoana Ciocoi Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 21427c87486SIoana Ciocoi Radulescu dma_unmap_page(dev, buf_array[i], DPAA2_ETH_RX_BUF_SIZE, 21518c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 21627c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 217569375fbSIoana Ciocoi Radulescu } 218569375fbSIoana Ciocoi Radulescu } 219569375fbSIoana Ciocoi Radulescu 2205d39dc21SIoana Ciocoi Radulescu static void xdp_release_buf(struct dpaa2_eth_priv *priv, 2215d39dc21SIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 2225d39dc21SIoana Ciocoi Radulescu dma_addr_t addr) 2235d39dc21SIoana Ciocoi Radulescu { 2245d39dc21SIoana Ciocoi Radulescu int err; 2255d39dc21SIoana Ciocoi Radulescu 2265d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr; 2275d39dc21SIoana Ciocoi Radulescu if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD) 2285d39dc21SIoana Ciocoi Radulescu return; 2295d39dc21SIoana Ciocoi Radulescu 2305d39dc21SIoana Ciocoi Radulescu while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 2315d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_bufs, 2325d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_cnt)) == -EBUSY) 2335d39dc21SIoana Ciocoi Radulescu cpu_relax(); 2345d39dc21SIoana Ciocoi Radulescu 2355d39dc21SIoana Ciocoi Radulescu if (err) { 2365d39dc21SIoana Ciocoi Radulescu free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt); 2375d39dc21SIoana Ciocoi Radulescu ch->buf_count -= ch->xdp.drop_cnt; 2385d39dc21SIoana Ciocoi Radulescu } 2395d39dc21SIoana Ciocoi Radulescu 2405d39dc21SIoana Ciocoi Radulescu ch->xdp.drop_cnt = 0; 2415d39dc21SIoana Ciocoi Radulescu } 2425d39dc21SIoana Ciocoi Radulescu 24399e43521SIoana Ciocoi Radulescu static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd, 24499e43521SIoana Ciocoi Radulescu void *buf_start, u16 queue_id) 24599e43521SIoana Ciocoi Radulescu { 24699e43521SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq; 24799e43521SIoana Ciocoi Radulescu struct dpaa2_faead *faead; 24899e43521SIoana Ciocoi Radulescu u32 ctrl, frc; 24999e43521SIoana Ciocoi Radulescu int i, err; 25099e43521SIoana Ciocoi Radulescu 25199e43521SIoana Ciocoi Radulescu /* Mark the egress frame hardware annotation area as valid */ 25299e43521SIoana Ciocoi Radulescu frc = dpaa2_fd_get_frc(fd); 25399e43521SIoana Ciocoi Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 25499e43521SIoana Ciocoi Radulescu dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 25599e43521SIoana Ciocoi Radulescu 25699e43521SIoana Ciocoi Radulescu /* Instruct hardware to release the FD buffer directly into 25799e43521SIoana Ciocoi Radulescu * the buffer pool once transmission is completed, instead of 25899e43521SIoana Ciocoi Radulescu * sending a Tx confirmation frame to us 25999e43521SIoana Ciocoi Radulescu */ 26099e43521SIoana Ciocoi Radulescu ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 26199e43521SIoana Ciocoi Radulescu faead = dpaa2_get_faead(buf_start, false); 26299e43521SIoana Ciocoi Radulescu faead->ctrl = cpu_to_le32(ctrl); 26399e43521SIoana Ciocoi Radulescu faead->conf_fqid = 0; 26499e43521SIoana Ciocoi Radulescu 26599e43521SIoana Ciocoi Radulescu fq = &priv->fq[queue_id]; 26699e43521SIoana Ciocoi Radulescu for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 2671fa0f68cSIoana Ciocoi Radulescu err = priv->enqueue(priv, fq, fd, 0); 26899e43521SIoana Ciocoi Radulescu if (err != -EBUSY) 26999e43521SIoana Ciocoi Radulescu break; 27099e43521SIoana Ciocoi Radulescu } 27199e43521SIoana Ciocoi Radulescu 27299e43521SIoana Ciocoi Radulescu return err; 27399e43521SIoana Ciocoi Radulescu } 27499e43521SIoana Ciocoi Radulescu 2757e273a8eSIoana Ciocoi Radulescu static u32 run_xdp(struct dpaa2_eth_priv *priv, 2767e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch, 27799e43521SIoana Ciocoi Radulescu struct dpaa2_eth_fq *rx_fq, 2787e273a8eSIoana Ciocoi Radulescu struct dpaa2_fd *fd, void *vaddr) 2797e273a8eSIoana Ciocoi Radulescu { 2805d39dc21SIoana Ciocoi Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 28199e43521SIoana Ciocoi Radulescu struct rtnl_link_stats64 *percpu_stats; 2827e273a8eSIoana Ciocoi Radulescu struct bpf_prog *xdp_prog; 2837e273a8eSIoana Ciocoi Radulescu struct xdp_buff xdp; 2847e273a8eSIoana Ciocoi Radulescu u32 xdp_act = XDP_PASS; 28599e43521SIoana Ciocoi Radulescu int err; 28699e43521SIoana Ciocoi Radulescu 28799e43521SIoana Ciocoi Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 2887e273a8eSIoana Ciocoi Radulescu 2897e273a8eSIoana Ciocoi Radulescu rcu_read_lock(); 2907e273a8eSIoana Ciocoi Radulescu 2917e273a8eSIoana Ciocoi Radulescu xdp_prog = READ_ONCE(ch->xdp.prog); 2927e273a8eSIoana Ciocoi Radulescu if (!xdp_prog) 2937e273a8eSIoana Ciocoi Radulescu goto out; 2947e273a8eSIoana Ciocoi Radulescu 2957e273a8eSIoana Ciocoi Radulescu xdp.data = vaddr + dpaa2_fd_get_offset(fd); 2967e273a8eSIoana Ciocoi Radulescu xdp.data_end = xdp.data + dpaa2_fd_get_len(fd); 2977b1eea1aSIoana Ciocoi Radulescu xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; 2987e273a8eSIoana Ciocoi Radulescu xdp_set_data_meta_invalid(&xdp); 2997e273a8eSIoana Ciocoi Radulescu 3007e273a8eSIoana Ciocoi Radulescu xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 3017e273a8eSIoana Ciocoi Radulescu 3027b1eea1aSIoana Ciocoi Radulescu /* xdp.data pointer may have changed */ 3037b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_offset(fd, xdp.data - vaddr); 3047b1eea1aSIoana Ciocoi Radulescu dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 3057b1eea1aSIoana Ciocoi Radulescu 3067e273a8eSIoana Ciocoi Radulescu switch (xdp_act) { 3077e273a8eSIoana Ciocoi Radulescu case XDP_PASS: 3087e273a8eSIoana Ciocoi Radulescu break; 30999e43521SIoana Ciocoi Radulescu case XDP_TX: 31099e43521SIoana Ciocoi Radulescu err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid); 31199e43521SIoana Ciocoi Radulescu if (err) { 31299e43521SIoana Ciocoi Radulescu xdp_release_buf(priv, ch, addr); 31399e43521SIoana Ciocoi Radulescu percpu_stats->tx_errors++; 314a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_tx_err++; 31599e43521SIoana Ciocoi Radulescu } else { 31699e43521SIoana Ciocoi Radulescu percpu_stats->tx_packets++; 31799e43521SIoana Ciocoi Radulescu percpu_stats->tx_bytes += dpaa2_fd_get_len(fd); 318a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_tx++; 31999e43521SIoana Ciocoi Radulescu } 32099e43521SIoana Ciocoi Radulescu break; 3217e273a8eSIoana Ciocoi Radulescu default: 3227e273a8eSIoana Ciocoi Radulescu bpf_warn_invalid_xdp_action(xdp_act); 323c1cb11bcSIoana Ciocoi Radulescu /* fall through */ 3247e273a8eSIoana Ciocoi Radulescu case XDP_ABORTED: 3257e273a8eSIoana Ciocoi Radulescu trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 326c1cb11bcSIoana Ciocoi Radulescu /* fall through */ 3277e273a8eSIoana Ciocoi Radulescu case XDP_DROP: 3285d39dc21SIoana Ciocoi Radulescu xdp_release_buf(priv, ch, addr); 329a4a7b762SIoana Ciocoi Radulescu ch->stats.xdp_drop++; 3307e273a8eSIoana Ciocoi Radulescu break; 3317e273a8eSIoana Ciocoi Radulescu } 3327e273a8eSIoana Ciocoi Radulescu 3337e273a8eSIoana Ciocoi Radulescu out: 3347e273a8eSIoana Ciocoi Radulescu rcu_read_unlock(); 3357e273a8eSIoana Ciocoi Radulescu return xdp_act; 3367e273a8eSIoana Ciocoi Radulescu } 3377e273a8eSIoana Ciocoi Radulescu 33834ff6846SIoana Radulescu /* Main Rx frame processing routine */ 33934ff6846SIoana Radulescu static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 34034ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 34134ff6846SIoana Radulescu const struct dpaa2_fd *fd, 342dbcdf728SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 34334ff6846SIoana Radulescu { 34434ff6846SIoana Radulescu dma_addr_t addr = dpaa2_fd_get_addr(fd); 34534ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 34634ff6846SIoana Radulescu void *vaddr; 34734ff6846SIoana Radulescu struct sk_buff *skb; 34834ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 34934ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 35034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 35134ff6846SIoana Radulescu struct dpaa2_fas *fas; 35234ff6846SIoana Radulescu void *buf_data; 35334ff6846SIoana Radulescu u32 status = 0; 3547e273a8eSIoana Ciocoi Radulescu u32 xdp_act; 35534ff6846SIoana Radulescu 35634ff6846SIoana Radulescu /* Tracing point */ 35734ff6846SIoana Radulescu trace_dpaa2_rx_fd(priv->net_dev, fd); 35834ff6846SIoana Radulescu 35934ff6846SIoana Radulescu vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 3605d39dc21SIoana Ciocoi Radulescu dma_sync_single_for_cpu(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 36118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 36234ff6846SIoana Radulescu 36334ff6846SIoana Radulescu fas = dpaa2_get_fas(vaddr, false); 36434ff6846SIoana Radulescu prefetch(fas); 36534ff6846SIoana Radulescu buf_data = vaddr + dpaa2_fd_get_offset(fd); 36634ff6846SIoana Radulescu prefetch(buf_data); 36734ff6846SIoana Radulescu 36834ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 36934ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 37034ff6846SIoana Radulescu 37134ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 37299e43521SIoana Ciocoi Radulescu xdp_act = run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 3737e273a8eSIoana Ciocoi Radulescu if (xdp_act != XDP_PASS) { 3747e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_packets++; 3757e273a8eSIoana Ciocoi Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 3767e273a8eSIoana Ciocoi Radulescu return; 3777e273a8eSIoana Ciocoi Radulescu } 3787e273a8eSIoana Ciocoi Radulescu 37927c87486SIoana Ciocoi Radulescu dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 38018c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 381fdb6ca9eSIoana Ciornei skb = build_linear_skb(ch, fd, vaddr); 38234ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 3837e273a8eSIoana Ciocoi Radulescu WARN_ON(priv->xdp_prog); 3847e273a8eSIoana Ciocoi Radulescu 38527c87486SIoana Ciocoi Radulescu dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE, 38618c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 38734ff6846SIoana Radulescu skb = build_frag_skb(priv, ch, buf_data); 38827c87486SIoana Ciocoi Radulescu free_pages((unsigned long)vaddr, 0); 38934ff6846SIoana Radulescu percpu_extras->rx_sg_frames++; 39034ff6846SIoana Radulescu percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 39134ff6846SIoana Radulescu } else { 39234ff6846SIoana Radulescu /* We don't support any other format */ 39334ff6846SIoana Radulescu goto err_frame_format; 39434ff6846SIoana Radulescu } 39534ff6846SIoana Radulescu 39634ff6846SIoana Radulescu if (unlikely(!skb)) 39734ff6846SIoana Radulescu goto err_build_skb; 39834ff6846SIoana Radulescu 39934ff6846SIoana Radulescu prefetch(skb->data); 40034ff6846SIoana Radulescu 40134ff6846SIoana Radulescu /* Get the timestamp value */ 40234ff6846SIoana Radulescu if (priv->rx_tstamp) { 40334ff6846SIoana Radulescu struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 40434ff6846SIoana Radulescu __le64 *ts = dpaa2_get_ts(vaddr, false); 40534ff6846SIoana Radulescu u64 ns; 40634ff6846SIoana Radulescu 40734ff6846SIoana Radulescu memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 40834ff6846SIoana Radulescu 40934ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 41034ff6846SIoana Radulescu shhwtstamps->hwtstamp = ns_to_ktime(ns); 41134ff6846SIoana Radulescu } 41234ff6846SIoana Radulescu 41334ff6846SIoana Radulescu /* Check if we need to validate the L4 csum */ 41434ff6846SIoana Radulescu if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 41534ff6846SIoana Radulescu status = le32_to_cpu(fas->status); 41634ff6846SIoana Radulescu validate_rx_csum(priv, status, skb); 41734ff6846SIoana Radulescu } 41834ff6846SIoana Radulescu 41934ff6846SIoana Radulescu skb->protocol = eth_type_trans(skb, priv->net_dev); 420dbcdf728SIoana Ciocoi Radulescu skb_record_rx_queue(skb, fq->flowid); 42134ff6846SIoana Radulescu 42234ff6846SIoana Radulescu percpu_stats->rx_packets++; 42334ff6846SIoana Radulescu percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 42434ff6846SIoana Radulescu 425dbcdf728SIoana Ciocoi Radulescu napi_gro_receive(&ch->napi, skb); 42634ff6846SIoana Radulescu 42734ff6846SIoana Radulescu return; 42834ff6846SIoana Radulescu 42934ff6846SIoana Radulescu err_build_skb: 43034ff6846SIoana Radulescu free_rx_fd(priv, fd, vaddr); 43134ff6846SIoana Radulescu err_frame_format: 43234ff6846SIoana Radulescu percpu_stats->rx_dropped++; 43334ff6846SIoana Radulescu } 43434ff6846SIoana Radulescu 43534ff6846SIoana Radulescu /* Consume all frames pull-dequeued into the store. This is the simplest way to 43634ff6846SIoana Radulescu * make sure we don't accidentally issue another volatile dequeue which would 43734ff6846SIoana Radulescu * overwrite (leak) frames already in the store. 43834ff6846SIoana Radulescu * 43934ff6846SIoana Radulescu * Observance of NAPI budget is not our concern, leaving that to the caller. 44034ff6846SIoana Radulescu */ 44168049a5fSIoana Ciocoi Radulescu static int consume_frames(struct dpaa2_eth_channel *ch, 442569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq **src) 44334ff6846SIoana Radulescu { 44434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = ch->priv; 44568049a5fSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq = NULL; 44634ff6846SIoana Radulescu struct dpaa2_dq *dq; 44734ff6846SIoana Radulescu const struct dpaa2_fd *fd; 44834ff6846SIoana Radulescu int cleaned = 0; 44934ff6846SIoana Radulescu int is_last; 45034ff6846SIoana Radulescu 45134ff6846SIoana Radulescu do { 45234ff6846SIoana Radulescu dq = dpaa2_io_store_next(ch->store, &is_last); 45334ff6846SIoana Radulescu if (unlikely(!dq)) { 45434ff6846SIoana Radulescu /* If we're here, we *must* have placed a 45534ff6846SIoana Radulescu * volatile dequeue comnmand, so keep reading through 45634ff6846SIoana Radulescu * the store until we get some sort of valid response 45734ff6846SIoana Radulescu * token (either a valid frame or an "empty dequeue") 45834ff6846SIoana Radulescu */ 45934ff6846SIoana Radulescu continue; 46034ff6846SIoana Radulescu } 46134ff6846SIoana Radulescu 46234ff6846SIoana Radulescu fd = dpaa2_dq_fd(dq); 46334ff6846SIoana Radulescu fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 46434ff6846SIoana Radulescu 465dbcdf728SIoana Ciocoi Radulescu fq->consume(priv, ch, fd, fq); 46634ff6846SIoana Radulescu cleaned++; 46734ff6846SIoana Radulescu } while (!is_last); 46834ff6846SIoana Radulescu 46968049a5fSIoana Ciocoi Radulescu if (!cleaned) 47068049a5fSIoana Ciocoi Radulescu return 0; 47168049a5fSIoana Ciocoi Radulescu 47268049a5fSIoana Ciocoi Radulescu fq->stats.frames += cleaned; 47368049a5fSIoana Ciocoi Radulescu 47468049a5fSIoana Ciocoi Radulescu /* A dequeue operation only pulls frames from a single queue 475569dac6aSIoana Ciocoi Radulescu * into the store. Return the frame queue as an out param. 47668049a5fSIoana Ciocoi Radulescu */ 477569dac6aSIoana Ciocoi Radulescu if (src) 478569dac6aSIoana Ciocoi Radulescu *src = fq; 47968049a5fSIoana Ciocoi Radulescu 48034ff6846SIoana Radulescu return cleaned; 48134ff6846SIoana Radulescu } 48234ff6846SIoana Radulescu 48334ff6846SIoana Radulescu /* Configure the egress frame annotation for timestamp update */ 48434ff6846SIoana Radulescu static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start) 48534ff6846SIoana Radulescu { 48634ff6846SIoana Radulescu struct dpaa2_faead *faead; 48734ff6846SIoana Radulescu u32 ctrl, frc; 48834ff6846SIoana Radulescu 48934ff6846SIoana Radulescu /* Mark the egress frame annotation area as valid */ 49034ff6846SIoana Radulescu frc = dpaa2_fd_get_frc(fd); 49134ff6846SIoana Radulescu dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 49234ff6846SIoana Radulescu 49334ff6846SIoana Radulescu /* Set hardware annotation size */ 49434ff6846SIoana Radulescu ctrl = dpaa2_fd_get_ctrl(fd); 49534ff6846SIoana Radulescu dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 49634ff6846SIoana Radulescu 49734ff6846SIoana Radulescu /* enable UPD (update prepanded data) bit in FAEAD field of 49834ff6846SIoana Radulescu * hardware frame annotation area 49934ff6846SIoana Radulescu */ 50034ff6846SIoana Radulescu ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 50134ff6846SIoana Radulescu faead = dpaa2_get_faead(buf_start, true); 50234ff6846SIoana Radulescu faead->ctrl = cpu_to_le32(ctrl); 50334ff6846SIoana Radulescu } 50434ff6846SIoana Radulescu 50534ff6846SIoana Radulescu /* Create a frame descriptor based on a fragmented skb */ 50634ff6846SIoana Radulescu static int build_sg_fd(struct dpaa2_eth_priv *priv, 50734ff6846SIoana Radulescu struct sk_buff *skb, 50834ff6846SIoana Radulescu struct dpaa2_fd *fd) 50934ff6846SIoana Radulescu { 51034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 51134ff6846SIoana Radulescu void *sgt_buf = NULL; 51234ff6846SIoana Radulescu dma_addr_t addr; 51334ff6846SIoana Radulescu int nr_frags = skb_shinfo(skb)->nr_frags; 51434ff6846SIoana Radulescu struct dpaa2_sg_entry *sgt; 51534ff6846SIoana Radulescu int i, err; 51634ff6846SIoana Radulescu int sgt_buf_size; 51734ff6846SIoana Radulescu struct scatterlist *scl, *crt_scl; 51834ff6846SIoana Radulescu int num_sg; 51934ff6846SIoana Radulescu int num_dma_bufs; 52034ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 52134ff6846SIoana Radulescu 52234ff6846SIoana Radulescu /* Create and map scatterlist. 52334ff6846SIoana Radulescu * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 52434ff6846SIoana Radulescu * to go beyond nr_frags+1. 52534ff6846SIoana Radulescu * Note: We don't support chained scatterlists 52634ff6846SIoana Radulescu */ 52734ff6846SIoana Radulescu if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 52834ff6846SIoana Radulescu return -EINVAL; 52934ff6846SIoana Radulescu 53034ff6846SIoana Radulescu scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 53134ff6846SIoana Radulescu if (unlikely(!scl)) 53234ff6846SIoana Radulescu return -ENOMEM; 53334ff6846SIoana Radulescu 53434ff6846SIoana Radulescu sg_init_table(scl, nr_frags + 1); 53534ff6846SIoana Radulescu num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 53634ff6846SIoana Radulescu num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 53734ff6846SIoana Radulescu if (unlikely(!num_dma_bufs)) { 53834ff6846SIoana Radulescu err = -ENOMEM; 53934ff6846SIoana Radulescu goto dma_map_sg_failed; 54034ff6846SIoana Radulescu } 54134ff6846SIoana Radulescu 54234ff6846SIoana Radulescu /* Prepare the HW SGT structure */ 54334ff6846SIoana Radulescu sgt_buf_size = priv->tx_data_offset + 54434ff6846SIoana Radulescu sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 54534ff6846SIoana Radulescu sgt_buf = netdev_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN); 54634ff6846SIoana Radulescu if (unlikely(!sgt_buf)) { 54734ff6846SIoana Radulescu err = -ENOMEM; 54834ff6846SIoana Radulescu goto sgt_buf_alloc_failed; 54934ff6846SIoana Radulescu } 55034ff6846SIoana Radulescu sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 55134ff6846SIoana Radulescu memset(sgt_buf, 0, sgt_buf_size); 55234ff6846SIoana Radulescu 55334ff6846SIoana Radulescu sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 55434ff6846SIoana Radulescu 55534ff6846SIoana Radulescu /* Fill in the HW SGT structure. 55634ff6846SIoana Radulescu * 55734ff6846SIoana Radulescu * sgt_buf is zeroed out, so the following fields are implicit 55834ff6846SIoana Radulescu * in all sgt entries: 55934ff6846SIoana Radulescu * - offset is 0 56034ff6846SIoana Radulescu * - format is 'dpaa2_sg_single' 56134ff6846SIoana Radulescu */ 56234ff6846SIoana Radulescu for_each_sg(scl, crt_scl, num_dma_bufs, i) { 56334ff6846SIoana Radulescu dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 56434ff6846SIoana Radulescu dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 56534ff6846SIoana Radulescu } 56634ff6846SIoana Radulescu dpaa2_sg_set_final(&sgt[i - 1], true); 56734ff6846SIoana Radulescu 56834ff6846SIoana Radulescu /* Store the skb backpointer in the SGT buffer. 56934ff6846SIoana Radulescu * Fit the scatterlist and the number of buffers alongside the 57034ff6846SIoana Radulescu * skb backpointer in the software annotation area. We'll need 57134ff6846SIoana Radulescu * all of them on Tx Conf. 57234ff6846SIoana Radulescu */ 57334ff6846SIoana Radulescu swa = (struct dpaa2_eth_swa *)sgt_buf; 574e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SG; 575e3fdf6baSIoana Radulescu swa->sg.skb = skb; 576e3fdf6baSIoana Radulescu swa->sg.scl = scl; 577e3fdf6baSIoana Radulescu swa->sg.num_sg = num_sg; 578e3fdf6baSIoana Radulescu swa->sg.sgt_size = sgt_buf_size; 57934ff6846SIoana Radulescu 58034ff6846SIoana Radulescu /* Separately map the SGT buffer */ 58134ff6846SIoana Radulescu addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 58234ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) { 58334ff6846SIoana Radulescu err = -ENOMEM; 58434ff6846SIoana Radulescu goto dma_map_single_failed; 58534ff6846SIoana Radulescu } 58634ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, priv->tx_data_offset); 58734ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_sg); 58834ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 58934ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 590b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 59134ff6846SIoana Radulescu 59234ff6846SIoana Radulescu if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 59334ff6846SIoana Radulescu enable_tx_tstamp(fd, sgt_buf); 59434ff6846SIoana Radulescu 59534ff6846SIoana Radulescu return 0; 59634ff6846SIoana Radulescu 59734ff6846SIoana Radulescu dma_map_single_failed: 59834ff6846SIoana Radulescu skb_free_frag(sgt_buf); 59934ff6846SIoana Radulescu sgt_buf_alloc_failed: 60034ff6846SIoana Radulescu dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 60134ff6846SIoana Radulescu dma_map_sg_failed: 60234ff6846SIoana Radulescu kfree(scl); 60334ff6846SIoana Radulescu return err; 60434ff6846SIoana Radulescu } 60534ff6846SIoana Radulescu 60634ff6846SIoana Radulescu /* Create a frame descriptor based on a linear skb */ 60734ff6846SIoana Radulescu static int build_single_fd(struct dpaa2_eth_priv *priv, 60834ff6846SIoana Radulescu struct sk_buff *skb, 60934ff6846SIoana Radulescu struct dpaa2_fd *fd) 61034ff6846SIoana Radulescu { 61134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 61234ff6846SIoana Radulescu u8 *buffer_start, *aligned_start; 613e3fdf6baSIoana Radulescu struct dpaa2_eth_swa *swa; 61434ff6846SIoana Radulescu dma_addr_t addr; 61534ff6846SIoana Radulescu 61634ff6846SIoana Radulescu buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb); 61734ff6846SIoana Radulescu 61834ff6846SIoana Radulescu /* If there's enough room to align the FD address, do it. 61934ff6846SIoana Radulescu * It will help hardware optimize accesses. 62034ff6846SIoana Radulescu */ 62134ff6846SIoana Radulescu aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 62234ff6846SIoana Radulescu DPAA2_ETH_TX_BUF_ALIGN); 62334ff6846SIoana Radulescu if (aligned_start >= skb->head) 62434ff6846SIoana Radulescu buffer_start = aligned_start; 62534ff6846SIoana Radulescu 62634ff6846SIoana Radulescu /* Store a backpointer to the skb at the beginning of the buffer 62734ff6846SIoana Radulescu * (in the private data area) such that we can release it 62834ff6846SIoana Radulescu * on Tx confirm 62934ff6846SIoana Radulescu */ 630e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 631e3fdf6baSIoana Radulescu swa->type = DPAA2_ETH_SWA_SINGLE; 632e3fdf6baSIoana Radulescu swa->single.skb = skb; 63334ff6846SIoana Radulescu 63434ff6846SIoana Radulescu addr = dma_map_single(dev, buffer_start, 63534ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 63634ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 63734ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 63834ff6846SIoana Radulescu return -ENOMEM; 63934ff6846SIoana Radulescu 64034ff6846SIoana Radulescu dpaa2_fd_set_addr(fd, addr); 64134ff6846SIoana Radulescu dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 64234ff6846SIoana Radulescu dpaa2_fd_set_len(fd, skb->len); 64334ff6846SIoana Radulescu dpaa2_fd_set_format(fd, dpaa2_fd_single); 644b948c8c6SIoana Radulescu dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 64534ff6846SIoana Radulescu 64634ff6846SIoana Radulescu if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 64734ff6846SIoana Radulescu enable_tx_tstamp(fd, buffer_start); 64834ff6846SIoana Radulescu 64934ff6846SIoana Radulescu return 0; 65034ff6846SIoana Radulescu } 65134ff6846SIoana Radulescu 65234ff6846SIoana Radulescu /* FD freeing routine on the Tx path 65334ff6846SIoana Radulescu * 65434ff6846SIoana Radulescu * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 65534ff6846SIoana Radulescu * back-pointed to is also freed. 65634ff6846SIoana Radulescu * This can be called either from dpaa2_eth_tx_conf() or on the error path of 65734ff6846SIoana Radulescu * dpaa2_eth_tx(). 65834ff6846SIoana Radulescu */ 65934ff6846SIoana Radulescu static void free_tx_fd(const struct dpaa2_eth_priv *priv, 6600723a3aeSIoana Ciocoi Radulescu const struct dpaa2_fd *fd, bool in_napi) 66134ff6846SIoana Radulescu { 66234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 66334ff6846SIoana Radulescu dma_addr_t fd_addr; 664e3fdf6baSIoana Radulescu struct sk_buff *skb; 66534ff6846SIoana Radulescu unsigned char *buffer_start; 66634ff6846SIoana Radulescu struct dpaa2_eth_swa *swa; 66734ff6846SIoana Radulescu u8 fd_format = dpaa2_fd_get_format(fd); 66834ff6846SIoana Radulescu 66934ff6846SIoana Radulescu fd_addr = dpaa2_fd_get_addr(fd); 670e3fdf6baSIoana Radulescu buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 671e3fdf6baSIoana Radulescu swa = (struct dpaa2_eth_swa *)buffer_start; 67234ff6846SIoana Radulescu 67334ff6846SIoana Radulescu if (fd_format == dpaa2_fd_single) { 674e3fdf6baSIoana Radulescu skb = swa->single.skb; 67534ff6846SIoana Radulescu /* Accessing the skb buffer is safe before dma unmap, because 67634ff6846SIoana Radulescu * we didn't map the actual skb shell. 67734ff6846SIoana Radulescu */ 67834ff6846SIoana Radulescu dma_unmap_single(dev, fd_addr, 67934ff6846SIoana Radulescu skb_tail_pointer(skb) - buffer_start, 68034ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 68134ff6846SIoana Radulescu } else if (fd_format == dpaa2_fd_sg) { 682e3fdf6baSIoana Radulescu skb = swa->sg.skb; 68334ff6846SIoana Radulescu 68434ff6846SIoana Radulescu /* Unmap the scatterlist */ 685e3fdf6baSIoana Radulescu dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 686e3fdf6baSIoana Radulescu DMA_BIDIRECTIONAL); 687e3fdf6baSIoana Radulescu kfree(swa->sg.scl); 68834ff6846SIoana Radulescu 68934ff6846SIoana Radulescu /* Unmap the SGT buffer */ 690e3fdf6baSIoana Radulescu dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 69134ff6846SIoana Radulescu DMA_BIDIRECTIONAL); 69234ff6846SIoana Radulescu } else { 69334ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "Invalid FD format\n"); 69434ff6846SIoana Radulescu return; 69534ff6846SIoana Radulescu } 69634ff6846SIoana Radulescu 69734ff6846SIoana Radulescu /* Get the timestamp value */ 69834ff6846SIoana Radulescu if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { 69934ff6846SIoana Radulescu struct skb_shared_hwtstamps shhwtstamps; 700e3fdf6baSIoana Radulescu __le64 *ts = dpaa2_get_ts(buffer_start, true); 70134ff6846SIoana Radulescu u64 ns; 70234ff6846SIoana Radulescu 70334ff6846SIoana Radulescu memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 70434ff6846SIoana Radulescu 70534ff6846SIoana Radulescu ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 70634ff6846SIoana Radulescu shhwtstamps.hwtstamp = ns_to_ktime(ns); 70734ff6846SIoana Radulescu skb_tstamp_tx(skb, &shhwtstamps); 70834ff6846SIoana Radulescu } 70934ff6846SIoana Radulescu 71034ff6846SIoana Radulescu /* Free SGT buffer allocated on tx */ 71134ff6846SIoana Radulescu if (fd_format != dpaa2_fd_single) 712e3fdf6baSIoana Radulescu skb_free_frag(buffer_start); 71334ff6846SIoana Radulescu 71434ff6846SIoana Radulescu /* Move on with skb release */ 7150723a3aeSIoana Ciocoi Radulescu napi_consume_skb(skb, in_napi); 71634ff6846SIoana Radulescu } 71734ff6846SIoana Radulescu 71834ff6846SIoana Radulescu static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 71934ff6846SIoana Radulescu { 72034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 72134ff6846SIoana Radulescu struct dpaa2_fd fd; 72234ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 72334ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 72434ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 725569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 72634ff6846SIoana Radulescu u16 queue_mapping; 72734ff6846SIoana Radulescu unsigned int needed_headroom; 728569dac6aSIoana Ciocoi Radulescu u32 fd_len; 72934ff6846SIoana Radulescu int err, i; 73034ff6846SIoana Radulescu 73134ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 73234ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 73334ff6846SIoana Radulescu 73434ff6846SIoana Radulescu needed_headroom = dpaa2_eth_needed_headroom(priv, skb); 73534ff6846SIoana Radulescu if (skb_headroom(skb) < needed_headroom) { 73634ff6846SIoana Radulescu struct sk_buff *ns; 73734ff6846SIoana Radulescu 73834ff6846SIoana Radulescu ns = skb_realloc_headroom(skb, needed_headroom); 73934ff6846SIoana Radulescu if (unlikely(!ns)) { 74034ff6846SIoana Radulescu percpu_stats->tx_dropped++; 74134ff6846SIoana Radulescu goto err_alloc_headroom; 74234ff6846SIoana Radulescu } 74334ff6846SIoana Radulescu percpu_extras->tx_reallocs++; 74434ff6846SIoana Radulescu 74534ff6846SIoana Radulescu if (skb->sk) 74634ff6846SIoana Radulescu skb_set_owner_w(ns, skb->sk); 74734ff6846SIoana Radulescu 74834ff6846SIoana Radulescu dev_kfree_skb(skb); 74934ff6846SIoana Radulescu skb = ns; 75034ff6846SIoana Radulescu } 75134ff6846SIoana Radulescu 75234ff6846SIoana Radulescu /* We'll be holding a back-reference to the skb until Tx Confirmation; 75334ff6846SIoana Radulescu * we don't want that overwritten by a concurrent Tx with a cloned skb. 75434ff6846SIoana Radulescu */ 75534ff6846SIoana Radulescu skb = skb_unshare(skb, GFP_ATOMIC); 75634ff6846SIoana Radulescu if (unlikely(!skb)) { 75734ff6846SIoana Radulescu /* skb_unshare() has already freed the skb */ 75834ff6846SIoana Radulescu percpu_stats->tx_dropped++; 75934ff6846SIoana Radulescu return NETDEV_TX_OK; 76034ff6846SIoana Radulescu } 76134ff6846SIoana Radulescu 76234ff6846SIoana Radulescu /* Setup the FD fields */ 76334ff6846SIoana Radulescu memset(&fd, 0, sizeof(fd)); 76434ff6846SIoana Radulescu 76534ff6846SIoana Radulescu if (skb_is_nonlinear(skb)) { 76634ff6846SIoana Radulescu err = build_sg_fd(priv, skb, &fd); 76734ff6846SIoana Radulescu percpu_extras->tx_sg_frames++; 76834ff6846SIoana Radulescu percpu_extras->tx_sg_bytes += skb->len; 76934ff6846SIoana Radulescu } else { 77034ff6846SIoana Radulescu err = build_single_fd(priv, skb, &fd); 77134ff6846SIoana Radulescu } 77234ff6846SIoana Radulescu 77334ff6846SIoana Radulescu if (unlikely(err)) { 77434ff6846SIoana Radulescu percpu_stats->tx_dropped++; 77534ff6846SIoana Radulescu goto err_build_fd; 77634ff6846SIoana Radulescu } 77734ff6846SIoana Radulescu 77834ff6846SIoana Radulescu /* Tracing point */ 77934ff6846SIoana Radulescu trace_dpaa2_tx_fd(net_dev, &fd); 78034ff6846SIoana Radulescu 78134ff6846SIoana Radulescu /* TxConf FQ selection relies on queue id from the stack. 78234ff6846SIoana Radulescu * In case of a forwarded frame from another DPNI interface, we choose 78334ff6846SIoana Radulescu * a queue affined to the same core that processed the Rx frame 78434ff6846SIoana Radulescu */ 78534ff6846SIoana Radulescu queue_mapping = skb_get_queue_mapping(skb); 78634ff6846SIoana Radulescu fq = &priv->fq[queue_mapping]; 78734ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 7881fa0f68cSIoana Ciocoi Radulescu err = priv->enqueue(priv, fq, &fd, 0); 78934ff6846SIoana Radulescu if (err != -EBUSY) 79034ff6846SIoana Radulescu break; 79134ff6846SIoana Radulescu } 79234ff6846SIoana Radulescu percpu_extras->tx_portal_busy += i; 79334ff6846SIoana Radulescu if (unlikely(err < 0)) { 79434ff6846SIoana Radulescu percpu_stats->tx_errors++; 79534ff6846SIoana Radulescu /* Clean up everything, including freeing the skb */ 7960723a3aeSIoana Ciocoi Radulescu free_tx_fd(priv, &fd, false); 79734ff6846SIoana Radulescu } else { 798569dac6aSIoana Ciocoi Radulescu fd_len = dpaa2_fd_get_len(&fd); 79934ff6846SIoana Radulescu percpu_stats->tx_packets++; 800569dac6aSIoana Ciocoi Radulescu percpu_stats->tx_bytes += fd_len; 801569dac6aSIoana Ciocoi Radulescu 802569dac6aSIoana Ciocoi Radulescu nq = netdev_get_tx_queue(net_dev, queue_mapping); 803569dac6aSIoana Ciocoi Radulescu netdev_tx_sent_queue(nq, fd_len); 80434ff6846SIoana Radulescu } 80534ff6846SIoana Radulescu 80634ff6846SIoana Radulescu return NETDEV_TX_OK; 80734ff6846SIoana Radulescu 80834ff6846SIoana Radulescu err_build_fd: 80934ff6846SIoana Radulescu err_alloc_headroom: 81034ff6846SIoana Radulescu dev_kfree_skb(skb); 81134ff6846SIoana Radulescu 81234ff6846SIoana Radulescu return NETDEV_TX_OK; 81334ff6846SIoana Radulescu } 81434ff6846SIoana Radulescu 81534ff6846SIoana Radulescu /* Tx confirmation frame processing routine */ 81634ff6846SIoana Radulescu static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 817b00c898cSIoana Ciornei struct dpaa2_eth_channel *ch __always_unused, 81834ff6846SIoana Radulescu const struct dpaa2_fd *fd, 819569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq) 82034ff6846SIoana Radulescu { 82134ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 82234ff6846SIoana Radulescu struct dpaa2_eth_drv_stats *percpu_extras; 823569dac6aSIoana Ciocoi Radulescu u32 fd_len = dpaa2_fd_get_len(fd); 82434ff6846SIoana Radulescu u32 fd_errors; 82534ff6846SIoana Radulescu 82634ff6846SIoana Radulescu /* Tracing point */ 82734ff6846SIoana Radulescu trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 82834ff6846SIoana Radulescu 82934ff6846SIoana Radulescu percpu_extras = this_cpu_ptr(priv->percpu_extras); 83034ff6846SIoana Radulescu percpu_extras->tx_conf_frames++; 831569dac6aSIoana Ciocoi Radulescu percpu_extras->tx_conf_bytes += fd_len; 832569dac6aSIoana Ciocoi Radulescu 833569dac6aSIoana Ciocoi Radulescu fq->dq_frames++; 834569dac6aSIoana Ciocoi Radulescu fq->dq_bytes += fd_len; 83534ff6846SIoana Radulescu 83634ff6846SIoana Radulescu /* Check frame errors in the FD field */ 83734ff6846SIoana Radulescu fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 8380723a3aeSIoana Ciocoi Radulescu free_tx_fd(priv, fd, true); 83934ff6846SIoana Radulescu 84034ff6846SIoana Radulescu if (likely(!fd_errors)) 84134ff6846SIoana Radulescu return; 84234ff6846SIoana Radulescu 84334ff6846SIoana Radulescu if (net_ratelimit()) 84434ff6846SIoana Radulescu netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 84534ff6846SIoana Radulescu fd_errors); 84634ff6846SIoana Radulescu 84734ff6846SIoana Radulescu percpu_stats = this_cpu_ptr(priv->percpu_stats); 84834ff6846SIoana Radulescu /* Tx-conf logically pertains to the egress path. */ 84934ff6846SIoana Radulescu percpu_stats->tx_errors++; 85034ff6846SIoana Radulescu } 85134ff6846SIoana Radulescu 85234ff6846SIoana Radulescu static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 85334ff6846SIoana Radulescu { 85434ff6846SIoana Radulescu int err; 85534ff6846SIoana Radulescu 85634ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 85734ff6846SIoana Radulescu DPNI_OFF_RX_L3_CSUM, enable); 85834ff6846SIoana Radulescu if (err) { 85934ff6846SIoana Radulescu netdev_err(priv->net_dev, 86034ff6846SIoana Radulescu "dpni_set_offload(RX_L3_CSUM) failed\n"); 86134ff6846SIoana Radulescu return err; 86234ff6846SIoana Radulescu } 86334ff6846SIoana Radulescu 86434ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 86534ff6846SIoana Radulescu DPNI_OFF_RX_L4_CSUM, enable); 86634ff6846SIoana Radulescu if (err) { 86734ff6846SIoana Radulescu netdev_err(priv->net_dev, 86834ff6846SIoana Radulescu "dpni_set_offload(RX_L4_CSUM) failed\n"); 86934ff6846SIoana Radulescu return err; 87034ff6846SIoana Radulescu } 87134ff6846SIoana Radulescu 87234ff6846SIoana Radulescu return 0; 87334ff6846SIoana Radulescu } 87434ff6846SIoana Radulescu 87534ff6846SIoana Radulescu static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 87634ff6846SIoana Radulescu { 87734ff6846SIoana Radulescu int err; 87834ff6846SIoana Radulescu 87934ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 88034ff6846SIoana Radulescu DPNI_OFF_TX_L3_CSUM, enable); 88134ff6846SIoana Radulescu if (err) { 88234ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 88334ff6846SIoana Radulescu return err; 88434ff6846SIoana Radulescu } 88534ff6846SIoana Radulescu 88634ff6846SIoana Radulescu err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 88734ff6846SIoana Radulescu DPNI_OFF_TX_L4_CSUM, enable); 88834ff6846SIoana Radulescu if (err) { 88934ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 89034ff6846SIoana Radulescu return err; 89134ff6846SIoana Radulescu } 89234ff6846SIoana Radulescu 89334ff6846SIoana Radulescu return 0; 89434ff6846SIoana Radulescu } 89534ff6846SIoana Radulescu 89634ff6846SIoana Radulescu /* Perform a single release command to add buffers 89734ff6846SIoana Radulescu * to the specified buffer pool 89834ff6846SIoana Radulescu */ 89934ff6846SIoana Radulescu static int add_bufs(struct dpaa2_eth_priv *priv, 90034ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, u16 bpid) 90134ff6846SIoana Radulescu { 90234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 90334ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 90427c87486SIoana Ciocoi Radulescu struct page *page; 90534ff6846SIoana Radulescu dma_addr_t addr; 90634ff6846SIoana Radulescu int i, err; 90734ff6846SIoana Radulescu 90834ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 90934ff6846SIoana Radulescu /* Allocate buffer visible to WRIOP + skb shared info + 91034ff6846SIoana Radulescu * alignment padding 91134ff6846SIoana Radulescu */ 91227c87486SIoana Ciocoi Radulescu /* allocate one page for each Rx buffer. WRIOP sees 91327c87486SIoana Ciocoi Radulescu * the entire page except for a tailroom reserved for 91427c87486SIoana Ciocoi Radulescu * skb shared info 91527c87486SIoana Ciocoi Radulescu */ 91627c87486SIoana Ciocoi Radulescu page = dev_alloc_pages(0); 91727c87486SIoana Ciocoi Radulescu if (!page) 91834ff6846SIoana Radulescu goto err_alloc; 91934ff6846SIoana Radulescu 92027c87486SIoana Ciocoi Radulescu addr = dma_map_page(dev, page, 0, DPAA2_ETH_RX_BUF_SIZE, 92118c2e770SIoana Ciocoi Radulescu DMA_BIDIRECTIONAL); 92234ff6846SIoana Radulescu if (unlikely(dma_mapping_error(dev, addr))) 92334ff6846SIoana Radulescu goto err_map; 92434ff6846SIoana Radulescu 92534ff6846SIoana Radulescu buf_array[i] = addr; 92634ff6846SIoana Radulescu 92734ff6846SIoana Radulescu /* tracing point */ 92834ff6846SIoana Radulescu trace_dpaa2_eth_buf_seed(priv->net_dev, 92927c87486SIoana Ciocoi Radulescu page, DPAA2_ETH_RX_BUF_RAW_SIZE, 93034ff6846SIoana Radulescu addr, DPAA2_ETH_RX_BUF_SIZE, 93134ff6846SIoana Radulescu bpid); 93234ff6846SIoana Radulescu } 93334ff6846SIoana Radulescu 93434ff6846SIoana Radulescu release_bufs: 93534ff6846SIoana Radulescu /* In case the portal is busy, retry until successful */ 93634ff6846SIoana Radulescu while ((err = dpaa2_io_service_release(ch->dpio, bpid, 93734ff6846SIoana Radulescu buf_array, i)) == -EBUSY) 93834ff6846SIoana Radulescu cpu_relax(); 93934ff6846SIoana Radulescu 94034ff6846SIoana Radulescu /* If release command failed, clean up and bail out; 94134ff6846SIoana Radulescu * not much else we can do about it 94234ff6846SIoana Radulescu */ 94334ff6846SIoana Radulescu if (err) { 94434ff6846SIoana Radulescu free_bufs(priv, buf_array, i); 94534ff6846SIoana Radulescu return 0; 94634ff6846SIoana Radulescu } 94734ff6846SIoana Radulescu 94834ff6846SIoana Radulescu return i; 94934ff6846SIoana Radulescu 95034ff6846SIoana Radulescu err_map: 95127c87486SIoana Ciocoi Radulescu __free_pages(page, 0); 95234ff6846SIoana Radulescu err_alloc: 95334ff6846SIoana Radulescu /* If we managed to allocate at least some buffers, 95434ff6846SIoana Radulescu * release them to hardware 95534ff6846SIoana Radulescu */ 95634ff6846SIoana Radulescu if (i) 95734ff6846SIoana Radulescu goto release_bufs; 95834ff6846SIoana Radulescu 95934ff6846SIoana Radulescu return 0; 96034ff6846SIoana Radulescu } 96134ff6846SIoana Radulescu 96234ff6846SIoana Radulescu static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 96334ff6846SIoana Radulescu { 96434ff6846SIoana Radulescu int i, j; 96534ff6846SIoana Radulescu int new_count; 96634ff6846SIoana Radulescu 96734ff6846SIoana Radulescu /* This is the lazy seeding of Rx buffer pools. 96834ff6846SIoana Radulescu * dpaa2_add_bufs() is also used on the Rx hotpath and calls 96934ff6846SIoana Radulescu * napi_alloc_frag(). The trouble with that is that it in turn ends up 97034ff6846SIoana Radulescu * calling this_cpu_ptr(), which mandates execution in atomic context. 97134ff6846SIoana Radulescu * Rather than splitting up the code, do a one-off preempt disable. 97234ff6846SIoana Radulescu */ 97334ff6846SIoana Radulescu preempt_disable(); 97434ff6846SIoana Radulescu for (j = 0; j < priv->num_channels; j++) { 97534ff6846SIoana Radulescu for (i = 0; i < DPAA2_ETH_NUM_BUFS; 97634ff6846SIoana Radulescu i += DPAA2_ETH_BUFS_PER_CMD) { 97734ff6846SIoana Radulescu new_count = add_bufs(priv, priv->channel[j], bpid); 97834ff6846SIoana Radulescu priv->channel[j]->buf_count += new_count; 97934ff6846SIoana Radulescu 98034ff6846SIoana Radulescu if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 98134ff6846SIoana Radulescu preempt_enable(); 98234ff6846SIoana Radulescu return -ENOMEM; 98334ff6846SIoana Radulescu } 98434ff6846SIoana Radulescu } 98534ff6846SIoana Radulescu } 98634ff6846SIoana Radulescu preempt_enable(); 98734ff6846SIoana Radulescu 98834ff6846SIoana Radulescu return 0; 98934ff6846SIoana Radulescu } 99034ff6846SIoana Radulescu 99134ff6846SIoana Radulescu /** 99234ff6846SIoana Radulescu * Drain the specified number of buffers from the DPNI's private buffer pool. 99334ff6846SIoana Radulescu * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 99434ff6846SIoana Radulescu */ 99534ff6846SIoana Radulescu static void drain_bufs(struct dpaa2_eth_priv *priv, int count) 99634ff6846SIoana Radulescu { 99734ff6846SIoana Radulescu u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 99834ff6846SIoana Radulescu int ret; 99934ff6846SIoana Radulescu 100034ff6846SIoana Radulescu do { 100134ff6846SIoana Radulescu ret = dpaa2_io_service_acquire(NULL, priv->bpid, 100234ff6846SIoana Radulescu buf_array, count); 100334ff6846SIoana Radulescu if (ret < 0) { 100434ff6846SIoana Radulescu netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 100534ff6846SIoana Radulescu return; 100634ff6846SIoana Radulescu } 100734ff6846SIoana Radulescu free_bufs(priv, buf_array, ret); 100834ff6846SIoana Radulescu } while (ret); 100934ff6846SIoana Radulescu } 101034ff6846SIoana Radulescu 101134ff6846SIoana Radulescu static void drain_pool(struct dpaa2_eth_priv *priv) 101234ff6846SIoana Radulescu { 101334ff6846SIoana Radulescu int i; 101434ff6846SIoana Radulescu 101534ff6846SIoana Radulescu drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 101634ff6846SIoana Radulescu drain_bufs(priv, 1); 101734ff6846SIoana Radulescu 101834ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 101934ff6846SIoana Radulescu priv->channel[i]->buf_count = 0; 102034ff6846SIoana Radulescu } 102134ff6846SIoana Radulescu 102234ff6846SIoana Radulescu /* Function is called from softirq context only, so we don't need to guard 102334ff6846SIoana Radulescu * the access to percpu count 102434ff6846SIoana Radulescu */ 102534ff6846SIoana Radulescu static int refill_pool(struct dpaa2_eth_priv *priv, 102634ff6846SIoana Radulescu struct dpaa2_eth_channel *ch, 102734ff6846SIoana Radulescu u16 bpid) 102834ff6846SIoana Radulescu { 102934ff6846SIoana Radulescu int new_count; 103034ff6846SIoana Radulescu 103134ff6846SIoana Radulescu if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 103234ff6846SIoana Radulescu return 0; 103334ff6846SIoana Radulescu 103434ff6846SIoana Radulescu do { 103534ff6846SIoana Radulescu new_count = add_bufs(priv, ch, bpid); 103634ff6846SIoana Radulescu if (unlikely(!new_count)) { 103734ff6846SIoana Radulescu /* Out of memory; abort for now, we'll try later on */ 103834ff6846SIoana Radulescu break; 103934ff6846SIoana Radulescu } 104034ff6846SIoana Radulescu ch->buf_count += new_count; 104134ff6846SIoana Radulescu } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 104234ff6846SIoana Radulescu 104334ff6846SIoana Radulescu if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 104434ff6846SIoana Radulescu return -ENOMEM; 104534ff6846SIoana Radulescu 104634ff6846SIoana Radulescu return 0; 104734ff6846SIoana Radulescu } 104834ff6846SIoana Radulescu 104934ff6846SIoana Radulescu static int pull_channel(struct dpaa2_eth_channel *ch) 105034ff6846SIoana Radulescu { 105134ff6846SIoana Radulescu int err; 105234ff6846SIoana Radulescu int dequeues = -1; 105334ff6846SIoana Radulescu 105434ff6846SIoana Radulescu /* Retry while portal is busy */ 105534ff6846SIoana Radulescu do { 105634ff6846SIoana Radulescu err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 105734ff6846SIoana Radulescu ch->store); 105834ff6846SIoana Radulescu dequeues++; 105934ff6846SIoana Radulescu cpu_relax(); 106034ff6846SIoana Radulescu } while (err == -EBUSY); 106134ff6846SIoana Radulescu 106234ff6846SIoana Radulescu ch->stats.dequeue_portal_busy += dequeues; 106334ff6846SIoana Radulescu if (unlikely(err)) 106434ff6846SIoana Radulescu ch->stats.pull_err++; 106534ff6846SIoana Radulescu 106634ff6846SIoana Radulescu return err; 106734ff6846SIoana Radulescu } 106834ff6846SIoana Radulescu 106934ff6846SIoana Radulescu /* NAPI poll routine 107034ff6846SIoana Radulescu * 107134ff6846SIoana Radulescu * Frames are dequeued from the QMan channel associated with this NAPI context. 107234ff6846SIoana Radulescu * Rx, Tx confirmation and (if configured) Rx error frames all count 107334ff6846SIoana Radulescu * towards the NAPI budget. 107434ff6846SIoana Radulescu */ 107534ff6846SIoana Radulescu static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 107634ff6846SIoana Radulescu { 107734ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 107834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 107968049a5fSIoana Ciocoi Radulescu int rx_cleaned = 0, txconf_cleaned = 0; 1080569dac6aSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1081569dac6aSIoana Ciocoi Radulescu struct netdev_queue *nq; 1082569dac6aSIoana Ciocoi Radulescu int store_cleaned, work_done; 108334ff6846SIoana Radulescu int err; 108434ff6846SIoana Radulescu 108534ff6846SIoana Radulescu ch = container_of(napi, struct dpaa2_eth_channel, napi); 108634ff6846SIoana Radulescu priv = ch->priv; 108734ff6846SIoana Radulescu 108868049a5fSIoana Ciocoi Radulescu do { 108934ff6846SIoana Radulescu err = pull_channel(ch); 109034ff6846SIoana Radulescu if (unlikely(err)) 109134ff6846SIoana Radulescu break; 109234ff6846SIoana Radulescu 109334ff6846SIoana Radulescu /* Refill pool if appropriate */ 109434ff6846SIoana Radulescu refill_pool(priv, ch, priv->bpid); 109534ff6846SIoana Radulescu 1096569dac6aSIoana Ciocoi Radulescu store_cleaned = consume_frames(ch, &fq); 1097569dac6aSIoana Ciocoi Radulescu if (!store_cleaned) 1098569dac6aSIoana Ciocoi Radulescu break; 1099569dac6aSIoana Ciocoi Radulescu if (fq->type == DPAA2_RX_FQ) { 110068049a5fSIoana Ciocoi Radulescu rx_cleaned += store_cleaned; 1101569dac6aSIoana Ciocoi Radulescu } else { 110268049a5fSIoana Ciocoi Radulescu txconf_cleaned += store_cleaned; 1103569dac6aSIoana Ciocoi Radulescu /* We have a single Tx conf FQ on this channel */ 1104569dac6aSIoana Ciocoi Radulescu txc_fq = fq; 1105569dac6aSIoana Ciocoi Radulescu } 110634ff6846SIoana Radulescu 110768049a5fSIoana Ciocoi Radulescu /* If we either consumed the whole NAPI budget with Rx frames 110868049a5fSIoana Ciocoi Radulescu * or we reached the Tx confirmations threshold, we're done. 110934ff6846SIoana Radulescu */ 111068049a5fSIoana Ciocoi Radulescu if (rx_cleaned >= budget || 1111569dac6aSIoana Ciocoi Radulescu txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1112569dac6aSIoana Ciocoi Radulescu work_done = budget; 1113569dac6aSIoana Ciocoi Radulescu goto out; 1114569dac6aSIoana Ciocoi Radulescu } 111568049a5fSIoana Ciocoi Radulescu } while (store_cleaned); 111634ff6846SIoana Radulescu 111768049a5fSIoana Ciocoi Radulescu /* We didn't consume the entire budget, so finish napi and 111868049a5fSIoana Ciocoi Radulescu * re-enable data availability notifications 111968049a5fSIoana Ciocoi Radulescu */ 112068049a5fSIoana Ciocoi Radulescu napi_complete_done(napi, rx_cleaned); 112134ff6846SIoana Radulescu do { 112234ff6846SIoana Radulescu err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 112334ff6846SIoana Radulescu cpu_relax(); 112434ff6846SIoana Radulescu } while (err == -EBUSY); 112534ff6846SIoana Radulescu WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 112634ff6846SIoana Radulescu ch->nctx.desired_cpu); 112734ff6846SIoana Radulescu 1128569dac6aSIoana Ciocoi Radulescu work_done = max(rx_cleaned, 1); 1129569dac6aSIoana Ciocoi Radulescu 1130569dac6aSIoana Ciocoi Radulescu out: 1131569dac6aSIoana Ciocoi Radulescu if (txc_fq) { 1132569dac6aSIoana Ciocoi Radulescu nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1133569dac6aSIoana Ciocoi Radulescu netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1134569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes); 1135569dac6aSIoana Ciocoi Radulescu txc_fq->dq_frames = 0; 1136569dac6aSIoana Ciocoi Radulescu txc_fq->dq_bytes = 0; 1137569dac6aSIoana Ciocoi Radulescu } 1138569dac6aSIoana Ciocoi Radulescu 1139569dac6aSIoana Ciocoi Radulescu return work_done; 114034ff6846SIoana Radulescu } 114134ff6846SIoana Radulescu 114234ff6846SIoana Radulescu static void enable_ch_napi(struct dpaa2_eth_priv *priv) 114334ff6846SIoana Radulescu { 114434ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 114534ff6846SIoana Radulescu int i; 114634ff6846SIoana Radulescu 114734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 114834ff6846SIoana Radulescu ch = priv->channel[i]; 114934ff6846SIoana Radulescu napi_enable(&ch->napi); 115034ff6846SIoana Radulescu } 115134ff6846SIoana Radulescu } 115234ff6846SIoana Radulescu 115334ff6846SIoana Radulescu static void disable_ch_napi(struct dpaa2_eth_priv *priv) 115434ff6846SIoana Radulescu { 115534ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 115634ff6846SIoana Radulescu int i; 115734ff6846SIoana Radulescu 115834ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 115934ff6846SIoana Radulescu ch = priv->channel[i]; 116034ff6846SIoana Radulescu napi_disable(&ch->napi); 116134ff6846SIoana Radulescu } 116234ff6846SIoana Radulescu } 116334ff6846SIoana Radulescu 116434ff6846SIoana Radulescu static int link_state_update(struct dpaa2_eth_priv *priv) 116534ff6846SIoana Radulescu { 116685b7a342SIoana Ciornei struct dpni_link_state state = {0}; 116734ff6846SIoana Radulescu int err; 116834ff6846SIoana Radulescu 116934ff6846SIoana Radulescu err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 117034ff6846SIoana Radulescu if (unlikely(err)) { 117134ff6846SIoana Radulescu netdev_err(priv->net_dev, 117234ff6846SIoana Radulescu "dpni_get_link_state() failed\n"); 117334ff6846SIoana Radulescu return err; 117434ff6846SIoana Radulescu } 117534ff6846SIoana Radulescu 117634ff6846SIoana Radulescu /* Chech link state; speed / duplex changes are not treated yet */ 117734ff6846SIoana Radulescu if (priv->link_state.up == state.up) 117834ff6846SIoana Radulescu return 0; 117934ff6846SIoana Radulescu 118034ff6846SIoana Radulescu priv->link_state = state; 118134ff6846SIoana Radulescu if (state.up) { 118234ff6846SIoana Radulescu netif_carrier_on(priv->net_dev); 118334ff6846SIoana Radulescu netif_tx_start_all_queues(priv->net_dev); 118434ff6846SIoana Radulescu } else { 118534ff6846SIoana Radulescu netif_tx_stop_all_queues(priv->net_dev); 118634ff6846SIoana Radulescu netif_carrier_off(priv->net_dev); 118734ff6846SIoana Radulescu } 118834ff6846SIoana Radulescu 118934ff6846SIoana Radulescu netdev_info(priv->net_dev, "Link Event: state %s\n", 119034ff6846SIoana Radulescu state.up ? "up" : "down"); 119134ff6846SIoana Radulescu 119234ff6846SIoana Radulescu return 0; 119334ff6846SIoana Radulescu } 119434ff6846SIoana Radulescu 119534ff6846SIoana Radulescu static int dpaa2_eth_open(struct net_device *net_dev) 119634ff6846SIoana Radulescu { 119734ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 119834ff6846SIoana Radulescu int err; 119934ff6846SIoana Radulescu 120034ff6846SIoana Radulescu err = seed_pool(priv, priv->bpid); 120134ff6846SIoana Radulescu if (err) { 120234ff6846SIoana Radulescu /* Not much to do; the buffer pool, though not filled up, 120334ff6846SIoana Radulescu * may still contain some buffers which would enable us 120434ff6846SIoana Radulescu * to limp on. 120534ff6846SIoana Radulescu */ 120634ff6846SIoana Radulescu netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 120734ff6846SIoana Radulescu priv->dpbp_dev->obj_desc.id, priv->bpid); 120834ff6846SIoana Radulescu } 120934ff6846SIoana Radulescu 121034ff6846SIoana Radulescu /* We'll only start the txqs when the link is actually ready; make sure 121134ff6846SIoana Radulescu * we don't race against the link up notification, which may come 121234ff6846SIoana Radulescu * immediately after dpni_enable(); 121334ff6846SIoana Radulescu */ 121434ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 121534ff6846SIoana Radulescu enable_ch_napi(priv); 121634ff6846SIoana Radulescu /* Also, explicitly set carrier off, otherwise netif_carrier_ok() will 121734ff6846SIoana Radulescu * return true and cause 'ip link show' to report the LOWER_UP flag, 121834ff6846SIoana Radulescu * even though the link notification wasn't even received. 121934ff6846SIoana Radulescu */ 122034ff6846SIoana Radulescu netif_carrier_off(net_dev); 122134ff6846SIoana Radulescu 122234ff6846SIoana Radulescu err = dpni_enable(priv->mc_io, 0, priv->mc_token); 122334ff6846SIoana Radulescu if (err < 0) { 122434ff6846SIoana Radulescu netdev_err(net_dev, "dpni_enable() failed\n"); 122534ff6846SIoana Radulescu goto enable_err; 122634ff6846SIoana Radulescu } 122734ff6846SIoana Radulescu 122834ff6846SIoana Radulescu /* If the DPMAC object has already processed the link up interrupt, 122934ff6846SIoana Radulescu * we have to learn the link state ourselves. 123034ff6846SIoana Radulescu */ 123134ff6846SIoana Radulescu err = link_state_update(priv); 123234ff6846SIoana Radulescu if (err < 0) { 123334ff6846SIoana Radulescu netdev_err(net_dev, "Can't update link state\n"); 123434ff6846SIoana Radulescu goto link_state_err; 123534ff6846SIoana Radulescu } 123634ff6846SIoana Radulescu 123734ff6846SIoana Radulescu return 0; 123834ff6846SIoana Radulescu 123934ff6846SIoana Radulescu link_state_err: 124034ff6846SIoana Radulescu enable_err: 124134ff6846SIoana Radulescu disable_ch_napi(priv); 124234ff6846SIoana Radulescu drain_pool(priv); 124334ff6846SIoana Radulescu return err; 124434ff6846SIoana Radulescu } 124534ff6846SIoana Radulescu 124668d74315SIoana Ciocoi Radulescu /* Total number of in-flight frames on ingress queues */ 124768d74315SIoana Ciocoi Radulescu static u32 ingress_fq_count(struct dpaa2_eth_priv *priv) 124834ff6846SIoana Radulescu { 124968d74315SIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq; 125068d74315SIoana Ciocoi Radulescu u32 fcnt = 0, bcnt = 0, total = 0; 125168d74315SIoana Ciocoi Radulescu int i, err; 125234ff6846SIoana Radulescu 125368d74315SIoana Ciocoi Radulescu for (i = 0; i < priv->num_fqs; i++) { 125468d74315SIoana Ciocoi Radulescu fq = &priv->fq[i]; 125568d74315SIoana Ciocoi Radulescu err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 125668d74315SIoana Ciocoi Radulescu if (err) { 125768d74315SIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "query_fq_count failed"); 125868d74315SIoana Ciocoi Radulescu break; 125968d74315SIoana Ciocoi Radulescu } 126068d74315SIoana Ciocoi Radulescu total += fcnt; 126168d74315SIoana Ciocoi Radulescu } 126234ff6846SIoana Radulescu 126334ff6846SIoana Radulescu return total; 126434ff6846SIoana Radulescu } 126534ff6846SIoana Radulescu 126668d74315SIoana Ciocoi Radulescu static void wait_for_fq_empty(struct dpaa2_eth_priv *priv) 126734ff6846SIoana Radulescu { 126868d74315SIoana Ciocoi Radulescu int retries = 10; 126968d74315SIoana Ciocoi Radulescu u32 pending; 127034ff6846SIoana Radulescu 127168d74315SIoana Ciocoi Radulescu do { 127268d74315SIoana Ciocoi Radulescu pending = ingress_fq_count(priv); 127368d74315SIoana Ciocoi Radulescu if (pending) 127468d74315SIoana Ciocoi Radulescu msleep(100); 127568d74315SIoana Ciocoi Radulescu } while (pending && --retries); 127634ff6846SIoana Radulescu } 127734ff6846SIoana Radulescu 127834ff6846SIoana Radulescu static int dpaa2_eth_stop(struct net_device *net_dev) 127934ff6846SIoana Radulescu { 128034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 128185b7a342SIoana Ciornei int dpni_enabled = 0; 128234ff6846SIoana Radulescu int retries = 10; 128334ff6846SIoana Radulescu 128434ff6846SIoana Radulescu netif_tx_stop_all_queues(net_dev); 128534ff6846SIoana Radulescu netif_carrier_off(net_dev); 128634ff6846SIoana Radulescu 128768d74315SIoana Ciocoi Radulescu /* On dpni_disable(), the MC firmware will: 128868d74315SIoana Ciocoi Radulescu * - stop MAC Rx and wait for all Rx frames to be enqueued to software 128968d74315SIoana Ciocoi Radulescu * - cut off WRIOP dequeues from egress FQs and wait until transmission 129068d74315SIoana Ciocoi Radulescu * of all in flight Tx frames is finished (and corresponding Tx conf 129168d74315SIoana Ciocoi Radulescu * frames are enqueued back to software) 129268d74315SIoana Ciocoi Radulescu * 129368d74315SIoana Ciocoi Radulescu * Before calling dpni_disable(), we wait for all Tx frames to arrive 129468d74315SIoana Ciocoi Radulescu * on WRIOP. After it finishes, wait until all remaining frames on Rx 129568d74315SIoana Ciocoi Radulescu * and Tx conf queues are consumed on NAPI poll. 129634ff6846SIoana Radulescu */ 129768d74315SIoana Ciocoi Radulescu msleep(500); 129868d74315SIoana Ciocoi Radulescu 129934ff6846SIoana Radulescu do { 130034ff6846SIoana Radulescu dpni_disable(priv->mc_io, 0, priv->mc_token); 130134ff6846SIoana Radulescu dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 130234ff6846SIoana Radulescu if (dpni_enabled) 130334ff6846SIoana Radulescu /* Allow the hardware some slack */ 130434ff6846SIoana Radulescu msleep(100); 130534ff6846SIoana Radulescu } while (dpni_enabled && --retries); 130634ff6846SIoana Radulescu if (!retries) { 130734ff6846SIoana Radulescu netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 130834ff6846SIoana Radulescu /* Must go on and disable NAPI nonetheless, so we don't crash at 130934ff6846SIoana Radulescu * the next "ifconfig up" 131034ff6846SIoana Radulescu */ 131134ff6846SIoana Radulescu } 131234ff6846SIoana Radulescu 131368d74315SIoana Ciocoi Radulescu wait_for_fq_empty(priv); 131434ff6846SIoana Radulescu disable_ch_napi(priv); 131534ff6846SIoana Radulescu 131634ff6846SIoana Radulescu /* Empty the buffer pool */ 131734ff6846SIoana Radulescu drain_pool(priv); 131834ff6846SIoana Radulescu 131934ff6846SIoana Radulescu return 0; 132034ff6846SIoana Radulescu } 132134ff6846SIoana Radulescu 132234ff6846SIoana Radulescu static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 132334ff6846SIoana Radulescu { 132434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 132534ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 132634ff6846SIoana Radulescu int err; 132734ff6846SIoana Radulescu 132834ff6846SIoana Radulescu err = eth_mac_addr(net_dev, addr); 132934ff6846SIoana Radulescu if (err < 0) { 133034ff6846SIoana Radulescu dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 133134ff6846SIoana Radulescu return err; 133234ff6846SIoana Radulescu } 133334ff6846SIoana Radulescu 133434ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 133534ff6846SIoana Radulescu net_dev->dev_addr); 133634ff6846SIoana Radulescu if (err) { 133734ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 133834ff6846SIoana Radulescu return err; 133934ff6846SIoana Radulescu } 134034ff6846SIoana Radulescu 134134ff6846SIoana Radulescu return 0; 134234ff6846SIoana Radulescu } 134334ff6846SIoana Radulescu 134434ff6846SIoana Radulescu /** Fill in counters maintained by the GPP driver. These may be different from 134534ff6846SIoana Radulescu * the hardware counters obtained by ethtool. 134634ff6846SIoana Radulescu */ 134734ff6846SIoana Radulescu static void dpaa2_eth_get_stats(struct net_device *net_dev, 134834ff6846SIoana Radulescu struct rtnl_link_stats64 *stats) 134934ff6846SIoana Radulescu { 135034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 135134ff6846SIoana Radulescu struct rtnl_link_stats64 *percpu_stats; 135234ff6846SIoana Radulescu u64 *cpustats; 135334ff6846SIoana Radulescu u64 *netstats = (u64 *)stats; 135434ff6846SIoana Radulescu int i, j; 135534ff6846SIoana Radulescu int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 135634ff6846SIoana Radulescu 135734ff6846SIoana Radulescu for_each_possible_cpu(i) { 135834ff6846SIoana Radulescu percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 135934ff6846SIoana Radulescu cpustats = (u64 *)percpu_stats; 136034ff6846SIoana Radulescu for (j = 0; j < num; j++) 136134ff6846SIoana Radulescu netstats[j] += cpustats[j]; 136234ff6846SIoana Radulescu } 136334ff6846SIoana Radulescu } 136434ff6846SIoana Radulescu 136534ff6846SIoana Radulescu /* Copy mac unicast addresses from @net_dev to @priv. 136634ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 136734ff6846SIoana Radulescu */ 136834ff6846SIoana Radulescu static void add_uc_hw_addr(const struct net_device *net_dev, 136934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 137034ff6846SIoana Radulescu { 137134ff6846SIoana Radulescu struct netdev_hw_addr *ha; 137234ff6846SIoana Radulescu int err; 137334ff6846SIoana Radulescu 137434ff6846SIoana Radulescu netdev_for_each_uc_addr(ha, net_dev) { 137534ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 137634ff6846SIoana Radulescu ha->addr); 137734ff6846SIoana Radulescu if (err) 137834ff6846SIoana Radulescu netdev_warn(priv->net_dev, 137934ff6846SIoana Radulescu "Could not add ucast MAC %pM to the filtering table (err %d)\n", 138034ff6846SIoana Radulescu ha->addr, err); 138134ff6846SIoana Radulescu } 138234ff6846SIoana Radulescu } 138334ff6846SIoana Radulescu 138434ff6846SIoana Radulescu /* Copy mac multicast addresses from @net_dev to @priv 138534ff6846SIoana Radulescu * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 138634ff6846SIoana Radulescu */ 138734ff6846SIoana Radulescu static void add_mc_hw_addr(const struct net_device *net_dev, 138834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv) 138934ff6846SIoana Radulescu { 139034ff6846SIoana Radulescu struct netdev_hw_addr *ha; 139134ff6846SIoana Radulescu int err; 139234ff6846SIoana Radulescu 139334ff6846SIoana Radulescu netdev_for_each_mc_addr(ha, net_dev) { 139434ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 139534ff6846SIoana Radulescu ha->addr); 139634ff6846SIoana Radulescu if (err) 139734ff6846SIoana Radulescu netdev_warn(priv->net_dev, 139834ff6846SIoana Radulescu "Could not add mcast MAC %pM to the filtering table (err %d)\n", 139934ff6846SIoana Radulescu ha->addr, err); 140034ff6846SIoana Radulescu } 140134ff6846SIoana Radulescu } 140234ff6846SIoana Radulescu 140334ff6846SIoana Radulescu static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 140434ff6846SIoana Radulescu { 140534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 140634ff6846SIoana Radulescu int uc_count = netdev_uc_count(net_dev); 140734ff6846SIoana Radulescu int mc_count = netdev_mc_count(net_dev); 140834ff6846SIoana Radulescu u8 max_mac = priv->dpni_attrs.mac_filter_entries; 140934ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 141034ff6846SIoana Radulescu u16 mc_token = priv->mc_token; 141134ff6846SIoana Radulescu struct fsl_mc_io *mc_io = priv->mc_io; 141234ff6846SIoana Radulescu int err; 141334ff6846SIoana Radulescu 141434ff6846SIoana Radulescu /* Basic sanity checks; these probably indicate a misconfiguration */ 141534ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 141634ff6846SIoana Radulescu netdev_info(net_dev, 141734ff6846SIoana Radulescu "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 141834ff6846SIoana Radulescu max_mac); 141934ff6846SIoana Radulescu 142034ff6846SIoana Radulescu /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 142134ff6846SIoana Radulescu if (uc_count > max_mac) { 142234ff6846SIoana Radulescu netdev_info(net_dev, 142334ff6846SIoana Radulescu "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 142434ff6846SIoana Radulescu uc_count, max_mac); 142534ff6846SIoana Radulescu goto force_promisc; 142634ff6846SIoana Radulescu } 142734ff6846SIoana Radulescu if (mc_count + uc_count > max_mac) { 142834ff6846SIoana Radulescu netdev_info(net_dev, 142934ff6846SIoana Radulescu "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 143034ff6846SIoana Radulescu uc_count + mc_count, max_mac); 143134ff6846SIoana Radulescu goto force_mc_promisc; 143234ff6846SIoana Radulescu } 143334ff6846SIoana Radulescu 143434ff6846SIoana Radulescu /* Adjust promisc settings due to flag combinations */ 143534ff6846SIoana Radulescu if (net_dev->flags & IFF_PROMISC) 143634ff6846SIoana Radulescu goto force_promisc; 143734ff6846SIoana Radulescu if (net_dev->flags & IFF_ALLMULTI) { 143834ff6846SIoana Radulescu /* First, rebuild unicast filtering table. This should be done 143934ff6846SIoana Radulescu * in promisc mode, in order to avoid frame loss while we 144034ff6846SIoana Radulescu * progressively add entries to the table. 144134ff6846SIoana Radulescu * We don't know whether we had been in promisc already, and 144234ff6846SIoana Radulescu * making an MC call to find out is expensive; so set uc promisc 144334ff6846SIoana Radulescu * nonetheless. 144434ff6846SIoana Radulescu */ 144534ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 144634ff6846SIoana Radulescu if (err) 144734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc\n"); 144834ff6846SIoana Radulescu 144934ff6846SIoana Radulescu /* Actual uc table reconstruction. */ 145034ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 145134ff6846SIoana Radulescu if (err) 145234ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc filters\n"); 145334ff6846SIoana Radulescu add_uc_hw_addr(net_dev, priv); 145434ff6846SIoana Radulescu 145534ff6846SIoana Radulescu /* Finally, clear uc promisc and set mc promisc as requested. */ 145634ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 145734ff6846SIoana Radulescu if (err) 145834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear uc promisc\n"); 145934ff6846SIoana Radulescu goto force_mc_promisc; 146034ff6846SIoana Radulescu } 146134ff6846SIoana Radulescu 146234ff6846SIoana Radulescu /* Neither unicast, nor multicast promisc will be on... eventually. 146334ff6846SIoana Radulescu * For now, rebuild mac filtering tables while forcing both of them on. 146434ff6846SIoana Radulescu */ 146534ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 146634ff6846SIoana Radulescu if (err) 146734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 146834ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 146934ff6846SIoana Radulescu if (err) 147034ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 147134ff6846SIoana Radulescu 147234ff6846SIoana Radulescu /* Actual mac filtering tables reconstruction */ 147334ff6846SIoana Radulescu err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 147434ff6846SIoana Radulescu if (err) 147534ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mac filters\n"); 147634ff6846SIoana Radulescu add_mc_hw_addr(net_dev, priv); 147734ff6846SIoana Radulescu add_uc_hw_addr(net_dev, priv); 147834ff6846SIoana Radulescu 147934ff6846SIoana Radulescu /* Now we can clear both ucast and mcast promisc, without risking 148034ff6846SIoana Radulescu * to drop legitimate frames anymore. 148134ff6846SIoana Radulescu */ 148234ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 148334ff6846SIoana Radulescu if (err) 148434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear ucast promisc\n"); 148534ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 148634ff6846SIoana Radulescu if (err) 148734ff6846SIoana Radulescu netdev_warn(net_dev, "Can't clear mcast promisc\n"); 148834ff6846SIoana Radulescu 148934ff6846SIoana Radulescu return; 149034ff6846SIoana Radulescu 149134ff6846SIoana Radulescu force_promisc: 149234ff6846SIoana Radulescu err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 149334ff6846SIoana Radulescu if (err) 149434ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set ucast promisc\n"); 149534ff6846SIoana Radulescu force_mc_promisc: 149634ff6846SIoana Radulescu err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 149734ff6846SIoana Radulescu if (err) 149834ff6846SIoana Radulescu netdev_warn(net_dev, "Can't set mcast promisc\n"); 149934ff6846SIoana Radulescu } 150034ff6846SIoana Radulescu 150134ff6846SIoana Radulescu static int dpaa2_eth_set_features(struct net_device *net_dev, 150234ff6846SIoana Radulescu netdev_features_t features) 150334ff6846SIoana Radulescu { 150434ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 150534ff6846SIoana Radulescu netdev_features_t changed = features ^ net_dev->features; 150634ff6846SIoana Radulescu bool enable; 150734ff6846SIoana Radulescu int err; 150834ff6846SIoana Radulescu 150934ff6846SIoana Radulescu if (changed & NETIF_F_RXCSUM) { 151034ff6846SIoana Radulescu enable = !!(features & NETIF_F_RXCSUM); 151134ff6846SIoana Radulescu err = set_rx_csum(priv, enable); 151234ff6846SIoana Radulescu if (err) 151334ff6846SIoana Radulescu return err; 151434ff6846SIoana Radulescu } 151534ff6846SIoana Radulescu 151634ff6846SIoana Radulescu if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 151734ff6846SIoana Radulescu enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 151834ff6846SIoana Radulescu err = set_tx_csum(priv, enable); 151934ff6846SIoana Radulescu if (err) 152034ff6846SIoana Radulescu return err; 152134ff6846SIoana Radulescu } 152234ff6846SIoana Radulescu 152334ff6846SIoana Radulescu return 0; 152434ff6846SIoana Radulescu } 152534ff6846SIoana Radulescu 152634ff6846SIoana Radulescu static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 152734ff6846SIoana Radulescu { 152834ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 152934ff6846SIoana Radulescu struct hwtstamp_config config; 153034ff6846SIoana Radulescu 153134ff6846SIoana Radulescu if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 153234ff6846SIoana Radulescu return -EFAULT; 153334ff6846SIoana Radulescu 153434ff6846SIoana Radulescu switch (config.tx_type) { 153534ff6846SIoana Radulescu case HWTSTAMP_TX_OFF: 153634ff6846SIoana Radulescu priv->tx_tstamp = false; 153734ff6846SIoana Radulescu break; 153834ff6846SIoana Radulescu case HWTSTAMP_TX_ON: 153934ff6846SIoana Radulescu priv->tx_tstamp = true; 154034ff6846SIoana Radulescu break; 154134ff6846SIoana Radulescu default: 154234ff6846SIoana Radulescu return -ERANGE; 154334ff6846SIoana Radulescu } 154434ff6846SIoana Radulescu 154534ff6846SIoana Radulescu if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 154634ff6846SIoana Radulescu priv->rx_tstamp = false; 154734ff6846SIoana Radulescu } else { 154834ff6846SIoana Radulescu priv->rx_tstamp = true; 154934ff6846SIoana Radulescu /* TS is set for all frame types, not only those requested */ 155034ff6846SIoana Radulescu config.rx_filter = HWTSTAMP_FILTER_ALL; 155134ff6846SIoana Radulescu } 155234ff6846SIoana Radulescu 155334ff6846SIoana Radulescu return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 155434ff6846SIoana Radulescu -EFAULT : 0; 155534ff6846SIoana Radulescu } 155634ff6846SIoana Radulescu 155734ff6846SIoana Radulescu static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 155834ff6846SIoana Radulescu { 155934ff6846SIoana Radulescu if (cmd == SIOCSHWTSTAMP) 156034ff6846SIoana Radulescu return dpaa2_eth_ts_ioctl(dev, rq, cmd); 156134ff6846SIoana Radulescu 156234ff6846SIoana Radulescu return -EINVAL; 156334ff6846SIoana Radulescu } 156434ff6846SIoana Radulescu 15657e273a8eSIoana Ciocoi Radulescu static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 15667e273a8eSIoana Ciocoi Radulescu { 15677e273a8eSIoana Ciocoi Radulescu int mfl, linear_mfl; 15687e273a8eSIoana Ciocoi Radulescu 15697e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 15707e273a8eSIoana Ciocoi Radulescu linear_mfl = DPAA2_ETH_RX_BUF_SIZE - DPAA2_ETH_RX_HWA_SIZE - 15717b1eea1aSIoana Ciocoi Radulescu dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 15727e273a8eSIoana Ciocoi Radulescu 15737e273a8eSIoana Ciocoi Radulescu if (mfl > linear_mfl) { 15747e273a8eSIoana Ciocoi Radulescu netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 15757e273a8eSIoana Ciocoi Radulescu linear_mfl - VLAN_ETH_HLEN); 15767e273a8eSIoana Ciocoi Radulescu return false; 15777e273a8eSIoana Ciocoi Radulescu } 15787e273a8eSIoana Ciocoi Radulescu 15797e273a8eSIoana Ciocoi Radulescu return true; 15807e273a8eSIoana Ciocoi Radulescu } 15817e273a8eSIoana Ciocoi Radulescu 15827e273a8eSIoana Ciocoi Radulescu static int set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 15837e273a8eSIoana Ciocoi Radulescu { 15847e273a8eSIoana Ciocoi Radulescu int mfl, err; 15857e273a8eSIoana Ciocoi Radulescu 15867e273a8eSIoana Ciocoi Radulescu /* We enforce a maximum Rx frame length based on MTU only if we have 15877e273a8eSIoana Ciocoi Radulescu * an XDP program attached (in order to avoid Rx S/G frames). 15887e273a8eSIoana Ciocoi Radulescu * Otherwise, we accept all incoming frames as long as they are not 15897e273a8eSIoana Ciocoi Radulescu * larger than maximum size supported in hardware 15907e273a8eSIoana Ciocoi Radulescu */ 15917e273a8eSIoana Ciocoi Radulescu if (has_xdp) 15927e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 15937e273a8eSIoana Ciocoi Radulescu else 15947e273a8eSIoana Ciocoi Radulescu mfl = DPAA2_ETH_MFL; 15957e273a8eSIoana Ciocoi Radulescu 15967e273a8eSIoana Ciocoi Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 15977e273a8eSIoana Ciocoi Radulescu if (err) { 15987e273a8eSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 15997e273a8eSIoana Ciocoi Radulescu return err; 16007e273a8eSIoana Ciocoi Radulescu } 16017e273a8eSIoana Ciocoi Radulescu 16027e273a8eSIoana Ciocoi Radulescu return 0; 16037e273a8eSIoana Ciocoi Radulescu } 16047e273a8eSIoana Ciocoi Radulescu 16057e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 16067e273a8eSIoana Ciocoi Radulescu { 16077e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 16087e273a8eSIoana Ciocoi Radulescu int err; 16097e273a8eSIoana Ciocoi Radulescu 16107e273a8eSIoana Ciocoi Radulescu if (!priv->xdp_prog) 16117e273a8eSIoana Ciocoi Radulescu goto out; 16127e273a8eSIoana Ciocoi Radulescu 16137e273a8eSIoana Ciocoi Radulescu if (!xdp_mtu_valid(priv, new_mtu)) 16147e273a8eSIoana Ciocoi Radulescu return -EINVAL; 16157e273a8eSIoana Ciocoi Radulescu 16167e273a8eSIoana Ciocoi Radulescu err = set_rx_mfl(priv, new_mtu, true); 16177e273a8eSIoana Ciocoi Radulescu if (err) 16187e273a8eSIoana Ciocoi Radulescu return err; 16197e273a8eSIoana Ciocoi Radulescu 16207e273a8eSIoana Ciocoi Radulescu out: 16217e273a8eSIoana Ciocoi Radulescu dev->mtu = new_mtu; 16227e273a8eSIoana Ciocoi Radulescu return 0; 16237e273a8eSIoana Ciocoi Radulescu } 16247e273a8eSIoana Ciocoi Radulescu 16257b1eea1aSIoana Ciocoi Radulescu static int update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 16267b1eea1aSIoana Ciocoi Radulescu { 16277b1eea1aSIoana Ciocoi Radulescu struct dpni_buffer_layout buf_layout = {0}; 16287b1eea1aSIoana Ciocoi Radulescu int err; 16297b1eea1aSIoana Ciocoi Radulescu 16307b1eea1aSIoana Ciocoi Radulescu err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 16317b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 16327b1eea1aSIoana Ciocoi Radulescu if (err) { 16337b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 16347b1eea1aSIoana Ciocoi Radulescu return err; 16357b1eea1aSIoana Ciocoi Radulescu } 16367b1eea1aSIoana Ciocoi Radulescu 16377b1eea1aSIoana Ciocoi Radulescu /* Reserve extra headroom for XDP header size changes */ 16387b1eea1aSIoana Ciocoi Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 16397b1eea1aSIoana Ciocoi Radulescu (has_xdp ? XDP_PACKET_HEADROOM : 0); 16407b1eea1aSIoana Ciocoi Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 16417b1eea1aSIoana Ciocoi Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 16427b1eea1aSIoana Ciocoi Radulescu DPNI_QUEUE_RX, &buf_layout); 16437b1eea1aSIoana Ciocoi Radulescu if (err) { 16447b1eea1aSIoana Ciocoi Radulescu netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 16457b1eea1aSIoana Ciocoi Radulescu return err; 16467b1eea1aSIoana Ciocoi Radulescu } 16477b1eea1aSIoana Ciocoi Radulescu 16487b1eea1aSIoana Ciocoi Radulescu return 0; 16497b1eea1aSIoana Ciocoi Radulescu } 16507b1eea1aSIoana Ciocoi Radulescu 16517e273a8eSIoana Ciocoi Radulescu static int setup_xdp(struct net_device *dev, struct bpf_prog *prog) 16527e273a8eSIoana Ciocoi Radulescu { 16537e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 16547e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_channel *ch; 16557e273a8eSIoana Ciocoi Radulescu struct bpf_prog *old; 16567e273a8eSIoana Ciocoi Radulescu bool up, need_update; 16577e273a8eSIoana Ciocoi Radulescu int i, err; 16587e273a8eSIoana Ciocoi Radulescu 16597e273a8eSIoana Ciocoi Radulescu if (prog && !xdp_mtu_valid(priv, dev->mtu)) 16607e273a8eSIoana Ciocoi Radulescu return -EINVAL; 16617e273a8eSIoana Ciocoi Radulescu 16627e273a8eSIoana Ciocoi Radulescu if (prog) { 16637e273a8eSIoana Ciocoi Radulescu prog = bpf_prog_add(prog, priv->num_channels); 16647e273a8eSIoana Ciocoi Radulescu if (IS_ERR(prog)) 16657e273a8eSIoana Ciocoi Radulescu return PTR_ERR(prog); 16667e273a8eSIoana Ciocoi Radulescu } 16677e273a8eSIoana Ciocoi Radulescu 16687e273a8eSIoana Ciocoi Radulescu up = netif_running(dev); 16697e273a8eSIoana Ciocoi Radulescu need_update = (!!priv->xdp_prog != !!prog); 16707e273a8eSIoana Ciocoi Radulescu 16717e273a8eSIoana Ciocoi Radulescu if (up) 16727e273a8eSIoana Ciocoi Radulescu dpaa2_eth_stop(dev); 16737e273a8eSIoana Ciocoi Radulescu 16747b1eea1aSIoana Ciocoi Radulescu /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 16757b1eea1aSIoana Ciocoi Radulescu * Also, when switching between xdp/non-xdp modes we need to reconfigure 16767b1eea1aSIoana Ciocoi Radulescu * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 16777b1eea1aSIoana Ciocoi Radulescu * so we are sure no old format buffers will be used from now on. 16787b1eea1aSIoana Ciocoi Radulescu */ 16797e273a8eSIoana Ciocoi Radulescu if (need_update) { 16807e273a8eSIoana Ciocoi Radulescu err = set_rx_mfl(priv, dev->mtu, !!prog); 16817e273a8eSIoana Ciocoi Radulescu if (err) 16827e273a8eSIoana Ciocoi Radulescu goto out_err; 16837b1eea1aSIoana Ciocoi Radulescu err = update_rx_buffer_headroom(priv, !!prog); 16847b1eea1aSIoana Ciocoi Radulescu if (err) 16857b1eea1aSIoana Ciocoi Radulescu goto out_err; 16867e273a8eSIoana Ciocoi Radulescu } 16877e273a8eSIoana Ciocoi Radulescu 16887e273a8eSIoana Ciocoi Radulescu old = xchg(&priv->xdp_prog, prog); 16897e273a8eSIoana Ciocoi Radulescu if (old) 16907e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 16917e273a8eSIoana Ciocoi Radulescu 16927e273a8eSIoana Ciocoi Radulescu for (i = 0; i < priv->num_channels; i++) { 16937e273a8eSIoana Ciocoi Radulescu ch = priv->channel[i]; 16947e273a8eSIoana Ciocoi Radulescu old = xchg(&ch->xdp.prog, prog); 16957e273a8eSIoana Ciocoi Radulescu if (old) 16967e273a8eSIoana Ciocoi Radulescu bpf_prog_put(old); 16977e273a8eSIoana Ciocoi Radulescu } 16987e273a8eSIoana Ciocoi Radulescu 16997e273a8eSIoana Ciocoi Radulescu if (up) { 17007e273a8eSIoana Ciocoi Radulescu err = dpaa2_eth_open(dev); 17017e273a8eSIoana Ciocoi Radulescu if (err) 17027e273a8eSIoana Ciocoi Radulescu return err; 17037e273a8eSIoana Ciocoi Radulescu } 17047e273a8eSIoana Ciocoi Radulescu 17057e273a8eSIoana Ciocoi Radulescu return 0; 17067e273a8eSIoana Ciocoi Radulescu 17077e273a8eSIoana Ciocoi Radulescu out_err: 17087e273a8eSIoana Ciocoi Radulescu if (prog) 17097e273a8eSIoana Ciocoi Radulescu bpf_prog_sub(prog, priv->num_channels); 17107e273a8eSIoana Ciocoi Radulescu if (up) 17117e273a8eSIoana Ciocoi Radulescu dpaa2_eth_open(dev); 17127e273a8eSIoana Ciocoi Radulescu 17137e273a8eSIoana Ciocoi Radulescu return err; 17147e273a8eSIoana Ciocoi Radulescu } 17157e273a8eSIoana Ciocoi Radulescu 17167e273a8eSIoana Ciocoi Radulescu static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 17177e273a8eSIoana Ciocoi Radulescu { 17187e273a8eSIoana Ciocoi Radulescu struct dpaa2_eth_priv *priv = netdev_priv(dev); 17197e273a8eSIoana Ciocoi Radulescu 17207e273a8eSIoana Ciocoi Radulescu switch (xdp->command) { 17217e273a8eSIoana Ciocoi Radulescu case XDP_SETUP_PROG: 17227e273a8eSIoana Ciocoi Radulescu return setup_xdp(dev, xdp->prog); 17237e273a8eSIoana Ciocoi Radulescu case XDP_QUERY_PROG: 17247e273a8eSIoana Ciocoi Radulescu xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0; 17257e273a8eSIoana Ciocoi Radulescu break; 17267e273a8eSIoana Ciocoi Radulescu default: 17277e273a8eSIoana Ciocoi Radulescu return -EINVAL; 17287e273a8eSIoana Ciocoi Radulescu } 17297e273a8eSIoana Ciocoi Radulescu 17307e273a8eSIoana Ciocoi Radulescu return 0; 17317e273a8eSIoana Ciocoi Radulescu } 17327e273a8eSIoana Ciocoi Radulescu 173334ff6846SIoana Radulescu static const struct net_device_ops dpaa2_eth_ops = { 173434ff6846SIoana Radulescu .ndo_open = dpaa2_eth_open, 173534ff6846SIoana Radulescu .ndo_start_xmit = dpaa2_eth_tx, 173634ff6846SIoana Radulescu .ndo_stop = dpaa2_eth_stop, 173734ff6846SIoana Radulescu .ndo_set_mac_address = dpaa2_eth_set_addr, 173834ff6846SIoana Radulescu .ndo_get_stats64 = dpaa2_eth_get_stats, 173934ff6846SIoana Radulescu .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 174034ff6846SIoana Radulescu .ndo_set_features = dpaa2_eth_set_features, 174134ff6846SIoana Radulescu .ndo_do_ioctl = dpaa2_eth_ioctl, 17427e273a8eSIoana Ciocoi Radulescu .ndo_change_mtu = dpaa2_eth_change_mtu, 17437e273a8eSIoana Ciocoi Radulescu .ndo_bpf = dpaa2_eth_xdp, 174434ff6846SIoana Radulescu }; 174534ff6846SIoana Radulescu 174634ff6846SIoana Radulescu static void cdan_cb(struct dpaa2_io_notification_ctx *ctx) 174734ff6846SIoana Radulescu { 174834ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 174934ff6846SIoana Radulescu 175034ff6846SIoana Radulescu ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 175134ff6846SIoana Radulescu 175234ff6846SIoana Radulescu /* Update NAPI statistics */ 175334ff6846SIoana Radulescu ch->stats.cdan++; 175434ff6846SIoana Radulescu 175534ff6846SIoana Radulescu napi_schedule_irqoff(&ch->napi); 175634ff6846SIoana Radulescu } 175734ff6846SIoana Radulescu 175834ff6846SIoana Radulescu /* Allocate and configure a DPCON object */ 175934ff6846SIoana Radulescu static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv) 176034ff6846SIoana Radulescu { 176134ff6846SIoana Radulescu struct fsl_mc_device *dpcon; 176234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 176334ff6846SIoana Radulescu struct dpcon_attr attrs; 176434ff6846SIoana Radulescu int err; 176534ff6846SIoana Radulescu 176634ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 176734ff6846SIoana Radulescu FSL_MC_POOL_DPCON, &dpcon); 176834ff6846SIoana Radulescu if (err) { 1769d7f5a9d8SIoana Ciornei if (err == -ENXIO) 1770d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 1771d7f5a9d8SIoana Ciornei else 177234ff6846SIoana Radulescu dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 1773d7f5a9d8SIoana Ciornei return ERR_PTR(err); 177434ff6846SIoana Radulescu } 177534ff6846SIoana Radulescu 177634ff6846SIoana Radulescu err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 177734ff6846SIoana Radulescu if (err) { 177834ff6846SIoana Radulescu dev_err(dev, "dpcon_open() failed\n"); 177934ff6846SIoana Radulescu goto free; 178034ff6846SIoana Radulescu } 178134ff6846SIoana Radulescu 178234ff6846SIoana Radulescu err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 178334ff6846SIoana Radulescu if (err) { 178434ff6846SIoana Radulescu dev_err(dev, "dpcon_reset() failed\n"); 178534ff6846SIoana Radulescu goto close; 178634ff6846SIoana Radulescu } 178734ff6846SIoana Radulescu 178834ff6846SIoana Radulescu err = dpcon_get_attributes(priv->mc_io, 0, dpcon->mc_handle, &attrs); 178934ff6846SIoana Radulescu if (err) { 179034ff6846SIoana Radulescu dev_err(dev, "dpcon_get_attributes() failed\n"); 179134ff6846SIoana Radulescu goto close; 179234ff6846SIoana Radulescu } 179334ff6846SIoana Radulescu 179434ff6846SIoana Radulescu err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 179534ff6846SIoana Radulescu if (err) { 179634ff6846SIoana Radulescu dev_err(dev, "dpcon_enable() failed\n"); 179734ff6846SIoana Radulescu goto close; 179834ff6846SIoana Radulescu } 179934ff6846SIoana Radulescu 180034ff6846SIoana Radulescu return dpcon; 180134ff6846SIoana Radulescu 180234ff6846SIoana Radulescu close: 180334ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 180434ff6846SIoana Radulescu free: 180534ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 180634ff6846SIoana Radulescu 180734ff6846SIoana Radulescu return NULL; 180834ff6846SIoana Radulescu } 180934ff6846SIoana Radulescu 181034ff6846SIoana Radulescu static void free_dpcon(struct dpaa2_eth_priv *priv, 181134ff6846SIoana Radulescu struct fsl_mc_device *dpcon) 181234ff6846SIoana Radulescu { 181334ff6846SIoana Radulescu dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 181434ff6846SIoana Radulescu dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 181534ff6846SIoana Radulescu fsl_mc_object_free(dpcon); 181634ff6846SIoana Radulescu } 181734ff6846SIoana Radulescu 181834ff6846SIoana Radulescu static struct dpaa2_eth_channel * 181934ff6846SIoana Radulescu alloc_channel(struct dpaa2_eth_priv *priv) 182034ff6846SIoana Radulescu { 182134ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 182234ff6846SIoana Radulescu struct dpcon_attr attr; 182334ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 182434ff6846SIoana Radulescu int err; 182534ff6846SIoana Radulescu 182634ff6846SIoana Radulescu channel = kzalloc(sizeof(*channel), GFP_KERNEL); 182734ff6846SIoana Radulescu if (!channel) 182834ff6846SIoana Radulescu return NULL; 182934ff6846SIoana Radulescu 183034ff6846SIoana Radulescu channel->dpcon = setup_dpcon(priv); 1831d7f5a9d8SIoana Ciornei if (IS_ERR_OR_NULL(channel->dpcon)) { 1832d7f5a9d8SIoana Ciornei err = PTR_ERR(channel->dpcon); 183334ff6846SIoana Radulescu goto err_setup; 1834d7f5a9d8SIoana Ciornei } 183534ff6846SIoana Radulescu 183634ff6846SIoana Radulescu err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 183734ff6846SIoana Radulescu &attr); 183834ff6846SIoana Radulescu if (err) { 183934ff6846SIoana Radulescu dev_err(dev, "dpcon_get_attributes() failed\n"); 184034ff6846SIoana Radulescu goto err_get_attr; 184134ff6846SIoana Radulescu } 184234ff6846SIoana Radulescu 184334ff6846SIoana Radulescu channel->dpcon_id = attr.id; 184434ff6846SIoana Radulescu channel->ch_id = attr.qbman_ch_id; 184534ff6846SIoana Radulescu channel->priv = priv; 184634ff6846SIoana Radulescu 184734ff6846SIoana Radulescu return channel; 184834ff6846SIoana Radulescu 184934ff6846SIoana Radulescu err_get_attr: 185034ff6846SIoana Radulescu free_dpcon(priv, channel->dpcon); 185134ff6846SIoana Radulescu err_setup: 185234ff6846SIoana Radulescu kfree(channel); 1853d7f5a9d8SIoana Ciornei return ERR_PTR(err); 185434ff6846SIoana Radulescu } 185534ff6846SIoana Radulescu 185634ff6846SIoana Radulescu static void free_channel(struct dpaa2_eth_priv *priv, 185734ff6846SIoana Radulescu struct dpaa2_eth_channel *channel) 185834ff6846SIoana Radulescu { 185934ff6846SIoana Radulescu free_dpcon(priv, channel->dpcon); 186034ff6846SIoana Radulescu kfree(channel); 186134ff6846SIoana Radulescu } 186234ff6846SIoana Radulescu 186334ff6846SIoana Radulescu /* DPIO setup: allocate and configure QBMan channels, setup core affinity 186434ff6846SIoana Radulescu * and register data availability notifications 186534ff6846SIoana Radulescu */ 186634ff6846SIoana Radulescu static int setup_dpio(struct dpaa2_eth_priv *priv) 186734ff6846SIoana Radulescu { 186834ff6846SIoana Radulescu struct dpaa2_io_notification_ctx *nctx; 186934ff6846SIoana Radulescu struct dpaa2_eth_channel *channel; 187034ff6846SIoana Radulescu struct dpcon_notification_cfg dpcon_notif_cfg; 187134ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 187234ff6846SIoana Radulescu int i, err; 187334ff6846SIoana Radulescu 187434ff6846SIoana Radulescu /* We want the ability to spread ingress traffic (RX, TX conf) to as 187534ff6846SIoana Radulescu * many cores as possible, so we need one channel for each core 187634ff6846SIoana Radulescu * (unless there's fewer queues than cores, in which case the extra 187734ff6846SIoana Radulescu * channels would be wasted). 187834ff6846SIoana Radulescu * Allocate one channel per core and register it to the core's 187934ff6846SIoana Radulescu * affine DPIO. If not enough channels are available for all cores 188034ff6846SIoana Radulescu * or if some cores don't have an affine DPIO, there will be no 188134ff6846SIoana Radulescu * ingress frame processing on those cores. 188234ff6846SIoana Radulescu */ 188334ff6846SIoana Radulescu cpumask_clear(&priv->dpio_cpumask); 188434ff6846SIoana Radulescu for_each_online_cpu(i) { 188534ff6846SIoana Radulescu /* Try to allocate a channel */ 188634ff6846SIoana Radulescu channel = alloc_channel(priv); 1887d7f5a9d8SIoana Ciornei if (IS_ERR_OR_NULL(channel)) { 1888d7f5a9d8SIoana Ciornei err = PTR_ERR(channel); 1889d7f5a9d8SIoana Ciornei if (err != -EPROBE_DEFER) 189034ff6846SIoana Radulescu dev_info(dev, 189134ff6846SIoana Radulescu "No affine channel for cpu %d and above\n", i); 189234ff6846SIoana Radulescu goto err_alloc_ch; 189334ff6846SIoana Radulescu } 189434ff6846SIoana Radulescu 189534ff6846SIoana Radulescu priv->channel[priv->num_channels] = channel; 189634ff6846SIoana Radulescu 189734ff6846SIoana Radulescu nctx = &channel->nctx; 189834ff6846SIoana Radulescu nctx->is_cdan = 1; 189934ff6846SIoana Radulescu nctx->cb = cdan_cb; 190034ff6846SIoana Radulescu nctx->id = channel->ch_id; 190134ff6846SIoana Radulescu nctx->desired_cpu = i; 190234ff6846SIoana Radulescu 190334ff6846SIoana Radulescu /* Register the new context */ 190434ff6846SIoana Radulescu channel->dpio = dpaa2_io_service_select(i); 190534ff6846SIoana Radulescu err = dpaa2_io_service_register(channel->dpio, nctx); 190634ff6846SIoana Radulescu if (err) { 190734ff6846SIoana Radulescu dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 190834ff6846SIoana Radulescu /* If no affine DPIO for this core, there's probably 190934ff6846SIoana Radulescu * none available for next cores either. Signal we want 191034ff6846SIoana Radulescu * to retry later, in case the DPIO devices weren't 191134ff6846SIoana Radulescu * probed yet. 191234ff6846SIoana Radulescu */ 191334ff6846SIoana Radulescu err = -EPROBE_DEFER; 191434ff6846SIoana Radulescu goto err_service_reg; 191534ff6846SIoana Radulescu } 191634ff6846SIoana Radulescu 191734ff6846SIoana Radulescu /* Register DPCON notification with MC */ 191834ff6846SIoana Radulescu dpcon_notif_cfg.dpio_id = nctx->dpio_id; 191934ff6846SIoana Radulescu dpcon_notif_cfg.priority = 0; 192034ff6846SIoana Radulescu dpcon_notif_cfg.user_ctx = nctx->qman64; 192134ff6846SIoana Radulescu err = dpcon_set_notification(priv->mc_io, 0, 192234ff6846SIoana Radulescu channel->dpcon->mc_handle, 192334ff6846SIoana Radulescu &dpcon_notif_cfg); 192434ff6846SIoana Radulescu if (err) { 192534ff6846SIoana Radulescu dev_err(dev, "dpcon_set_notification failed()\n"); 192634ff6846SIoana Radulescu goto err_set_cdan; 192734ff6846SIoana Radulescu } 192834ff6846SIoana Radulescu 192934ff6846SIoana Radulescu /* If we managed to allocate a channel and also found an affine 193034ff6846SIoana Radulescu * DPIO for this core, add it to the final mask 193134ff6846SIoana Radulescu */ 193234ff6846SIoana Radulescu cpumask_set_cpu(i, &priv->dpio_cpumask); 193334ff6846SIoana Radulescu priv->num_channels++; 193434ff6846SIoana Radulescu 193534ff6846SIoana Radulescu /* Stop if we already have enough channels to accommodate all 193634ff6846SIoana Radulescu * RX and TX conf queues 193734ff6846SIoana Radulescu */ 1938b0e4f37bSIoana Ciocoi Radulescu if (priv->num_channels == priv->dpni_attrs.num_queues) 193934ff6846SIoana Radulescu break; 194034ff6846SIoana Radulescu } 194134ff6846SIoana Radulescu 194234ff6846SIoana Radulescu return 0; 194334ff6846SIoana Radulescu 194434ff6846SIoana Radulescu err_set_cdan: 194534ff6846SIoana Radulescu dpaa2_io_service_deregister(channel->dpio, nctx); 194634ff6846SIoana Radulescu err_service_reg: 194734ff6846SIoana Radulescu free_channel(priv, channel); 194834ff6846SIoana Radulescu err_alloc_ch: 1949d7f5a9d8SIoana Ciornei if (err == -EPROBE_DEFER) 1950d7f5a9d8SIoana Ciornei return err; 1951d7f5a9d8SIoana Ciornei 195234ff6846SIoana Radulescu if (cpumask_empty(&priv->dpio_cpumask)) { 195334ff6846SIoana Radulescu dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 1954d7f5a9d8SIoana Ciornei return -ENODEV; 195534ff6846SIoana Radulescu } 195634ff6846SIoana Radulescu 195734ff6846SIoana Radulescu dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 195834ff6846SIoana Radulescu cpumask_pr_args(&priv->dpio_cpumask)); 195934ff6846SIoana Radulescu 196034ff6846SIoana Radulescu return 0; 196134ff6846SIoana Radulescu } 196234ff6846SIoana Radulescu 196334ff6846SIoana Radulescu static void free_dpio(struct dpaa2_eth_priv *priv) 196434ff6846SIoana Radulescu { 196534ff6846SIoana Radulescu int i; 196634ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 196734ff6846SIoana Radulescu 196834ff6846SIoana Radulescu /* deregister CDAN notifications and free channels */ 196934ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 197034ff6846SIoana Radulescu ch = priv->channel[i]; 197134ff6846SIoana Radulescu dpaa2_io_service_deregister(ch->dpio, &ch->nctx); 197234ff6846SIoana Radulescu free_channel(priv, ch); 197334ff6846SIoana Radulescu } 197434ff6846SIoana Radulescu } 197534ff6846SIoana Radulescu 197634ff6846SIoana Radulescu static struct dpaa2_eth_channel *get_affine_channel(struct dpaa2_eth_priv *priv, 197734ff6846SIoana Radulescu int cpu) 197834ff6846SIoana Radulescu { 197934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 198034ff6846SIoana Radulescu int i; 198134ff6846SIoana Radulescu 198234ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 198334ff6846SIoana Radulescu if (priv->channel[i]->nctx.desired_cpu == cpu) 198434ff6846SIoana Radulescu return priv->channel[i]; 198534ff6846SIoana Radulescu 198634ff6846SIoana Radulescu /* We should never get here. Issue a warning and return 198734ff6846SIoana Radulescu * the first channel, because it's still better than nothing 198834ff6846SIoana Radulescu */ 198934ff6846SIoana Radulescu dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 199034ff6846SIoana Radulescu 199134ff6846SIoana Radulescu return priv->channel[0]; 199234ff6846SIoana Radulescu } 199334ff6846SIoana Radulescu 199434ff6846SIoana Radulescu static void set_fq_affinity(struct dpaa2_eth_priv *priv) 199534ff6846SIoana Radulescu { 199634ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 199734ff6846SIoana Radulescu struct cpumask xps_mask; 199834ff6846SIoana Radulescu struct dpaa2_eth_fq *fq; 199934ff6846SIoana Radulescu int rx_cpu, txc_cpu; 200034ff6846SIoana Radulescu int i, err; 200134ff6846SIoana Radulescu 200234ff6846SIoana Radulescu /* For each FQ, pick one channel/CPU to deliver frames to. 200334ff6846SIoana Radulescu * This may well change at runtime, either through irqbalance or 200434ff6846SIoana Radulescu * through direct user intervention. 200534ff6846SIoana Radulescu */ 200634ff6846SIoana Radulescu rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 200734ff6846SIoana Radulescu 200834ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 200934ff6846SIoana Radulescu fq = &priv->fq[i]; 201034ff6846SIoana Radulescu switch (fq->type) { 201134ff6846SIoana Radulescu case DPAA2_RX_FQ: 201234ff6846SIoana Radulescu fq->target_cpu = rx_cpu; 201334ff6846SIoana Radulescu rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 201434ff6846SIoana Radulescu if (rx_cpu >= nr_cpu_ids) 201534ff6846SIoana Radulescu rx_cpu = cpumask_first(&priv->dpio_cpumask); 201634ff6846SIoana Radulescu break; 201734ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 201834ff6846SIoana Radulescu fq->target_cpu = txc_cpu; 201934ff6846SIoana Radulescu 202034ff6846SIoana Radulescu /* Tell the stack to affine to txc_cpu the Tx queue 202134ff6846SIoana Radulescu * associated with the confirmation one 202234ff6846SIoana Radulescu */ 202334ff6846SIoana Radulescu cpumask_clear(&xps_mask); 202434ff6846SIoana Radulescu cpumask_set_cpu(txc_cpu, &xps_mask); 202534ff6846SIoana Radulescu err = netif_set_xps_queue(priv->net_dev, &xps_mask, 202634ff6846SIoana Radulescu fq->flowid); 202734ff6846SIoana Radulescu if (err) 202834ff6846SIoana Radulescu dev_err(dev, "Error setting XPS queue\n"); 202934ff6846SIoana Radulescu 203034ff6846SIoana Radulescu txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 203134ff6846SIoana Radulescu if (txc_cpu >= nr_cpu_ids) 203234ff6846SIoana Radulescu txc_cpu = cpumask_first(&priv->dpio_cpumask); 203334ff6846SIoana Radulescu break; 203434ff6846SIoana Radulescu default: 203534ff6846SIoana Radulescu dev_err(dev, "Unknown FQ type: %d\n", fq->type); 203634ff6846SIoana Radulescu } 203734ff6846SIoana Radulescu fq->channel = get_affine_channel(priv, fq->target_cpu); 203834ff6846SIoana Radulescu } 203934ff6846SIoana Radulescu } 204034ff6846SIoana Radulescu 204134ff6846SIoana Radulescu static void setup_fqs(struct dpaa2_eth_priv *priv) 204234ff6846SIoana Radulescu { 204334ff6846SIoana Radulescu int i; 204434ff6846SIoana Radulescu 204534ff6846SIoana Radulescu /* We have one TxConf FQ per Tx flow. 204634ff6846SIoana Radulescu * The number of Tx and Rx queues is the same. 204734ff6846SIoana Radulescu * Tx queues come first in the fq array. 204834ff6846SIoana Radulescu */ 204934ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 205034ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 205134ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 205234ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 205334ff6846SIoana Radulescu } 205434ff6846SIoana Radulescu 205534ff6846SIoana Radulescu for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 205634ff6846SIoana Radulescu priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 205734ff6846SIoana Radulescu priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 205834ff6846SIoana Radulescu priv->fq[priv->num_fqs++].flowid = (u16)i; 205934ff6846SIoana Radulescu } 206034ff6846SIoana Radulescu 206134ff6846SIoana Radulescu /* For each FQ, decide on which core to process incoming frames */ 206234ff6846SIoana Radulescu set_fq_affinity(priv); 206334ff6846SIoana Radulescu } 206434ff6846SIoana Radulescu 206534ff6846SIoana Radulescu /* Allocate and configure one buffer pool for each interface */ 206634ff6846SIoana Radulescu static int setup_dpbp(struct dpaa2_eth_priv *priv) 206734ff6846SIoana Radulescu { 206834ff6846SIoana Radulescu int err; 206934ff6846SIoana Radulescu struct fsl_mc_device *dpbp_dev; 207034ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 207134ff6846SIoana Radulescu struct dpbp_attr dpbp_attrs; 207234ff6846SIoana Radulescu 207334ff6846SIoana Radulescu err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 207434ff6846SIoana Radulescu &dpbp_dev); 207534ff6846SIoana Radulescu if (err) { 2076d7f5a9d8SIoana Ciornei if (err == -ENXIO) 2077d7f5a9d8SIoana Ciornei err = -EPROBE_DEFER; 2078d7f5a9d8SIoana Ciornei else 207934ff6846SIoana Radulescu dev_err(dev, "DPBP device allocation failed\n"); 208034ff6846SIoana Radulescu return err; 208134ff6846SIoana Radulescu } 208234ff6846SIoana Radulescu 208334ff6846SIoana Radulescu priv->dpbp_dev = dpbp_dev; 208434ff6846SIoana Radulescu 208534ff6846SIoana Radulescu err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 208634ff6846SIoana Radulescu &dpbp_dev->mc_handle); 208734ff6846SIoana Radulescu if (err) { 208834ff6846SIoana Radulescu dev_err(dev, "dpbp_open() failed\n"); 208934ff6846SIoana Radulescu goto err_open; 209034ff6846SIoana Radulescu } 209134ff6846SIoana Radulescu 209234ff6846SIoana Radulescu err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 209334ff6846SIoana Radulescu if (err) { 209434ff6846SIoana Radulescu dev_err(dev, "dpbp_reset() failed\n"); 209534ff6846SIoana Radulescu goto err_reset; 209634ff6846SIoana Radulescu } 209734ff6846SIoana Radulescu 209834ff6846SIoana Radulescu err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 209934ff6846SIoana Radulescu if (err) { 210034ff6846SIoana Radulescu dev_err(dev, "dpbp_enable() failed\n"); 210134ff6846SIoana Radulescu goto err_enable; 210234ff6846SIoana Radulescu } 210334ff6846SIoana Radulescu 210434ff6846SIoana Radulescu err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 210534ff6846SIoana Radulescu &dpbp_attrs); 210634ff6846SIoana Radulescu if (err) { 210734ff6846SIoana Radulescu dev_err(dev, "dpbp_get_attributes() failed\n"); 210834ff6846SIoana Radulescu goto err_get_attr; 210934ff6846SIoana Radulescu } 211034ff6846SIoana Radulescu priv->bpid = dpbp_attrs.bpid; 211134ff6846SIoana Radulescu 211234ff6846SIoana Radulescu return 0; 211334ff6846SIoana Radulescu 211434ff6846SIoana Radulescu err_get_attr: 211534ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 211634ff6846SIoana Radulescu err_enable: 211734ff6846SIoana Radulescu err_reset: 211834ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 211934ff6846SIoana Radulescu err_open: 212034ff6846SIoana Radulescu fsl_mc_object_free(dpbp_dev); 212134ff6846SIoana Radulescu 212234ff6846SIoana Radulescu return err; 212334ff6846SIoana Radulescu } 212434ff6846SIoana Radulescu 212534ff6846SIoana Radulescu static void free_dpbp(struct dpaa2_eth_priv *priv) 212634ff6846SIoana Radulescu { 212734ff6846SIoana Radulescu drain_pool(priv); 212834ff6846SIoana Radulescu dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 212934ff6846SIoana Radulescu dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 213034ff6846SIoana Radulescu fsl_mc_object_free(priv->dpbp_dev); 213134ff6846SIoana Radulescu } 213234ff6846SIoana Radulescu 213334ff6846SIoana Radulescu static int set_buffer_layout(struct dpaa2_eth_priv *priv) 213434ff6846SIoana Radulescu { 213534ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 213634ff6846SIoana Radulescu struct dpni_buffer_layout buf_layout = {0}; 213727c87486SIoana Ciocoi Radulescu u16 rx_buf_align; 213834ff6846SIoana Radulescu int err; 213934ff6846SIoana Radulescu 214034ff6846SIoana Radulescu /* We need to check for WRIOP version 1.0.0, but depending on the MC 214134ff6846SIoana Radulescu * version, this number is not always provided correctly on rev1. 214234ff6846SIoana Radulescu * We need to check for both alternatives in this situation. 214334ff6846SIoana Radulescu */ 214434ff6846SIoana Radulescu if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 214534ff6846SIoana Radulescu priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 214627c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 214734ff6846SIoana Radulescu else 214827c87486SIoana Ciocoi Radulescu rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 214934ff6846SIoana Radulescu 215034ff6846SIoana Radulescu /* tx buffer */ 215134ff6846SIoana Radulescu buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 215234ff6846SIoana Radulescu buf_layout.pass_timestamp = true; 215334ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 215434ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 215534ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 215634ff6846SIoana Radulescu DPNI_QUEUE_TX, &buf_layout); 215734ff6846SIoana Radulescu if (err) { 215834ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 215934ff6846SIoana Radulescu return err; 216034ff6846SIoana Radulescu } 216134ff6846SIoana Radulescu 216234ff6846SIoana Radulescu /* tx-confirm buffer */ 216334ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 216434ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 216534ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, &buf_layout); 216634ff6846SIoana Radulescu if (err) { 216734ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 216834ff6846SIoana Radulescu return err; 216934ff6846SIoana Radulescu } 217034ff6846SIoana Radulescu 217134ff6846SIoana Radulescu /* Now that we've set our tx buffer layout, retrieve the minimum 217234ff6846SIoana Radulescu * required tx data offset. 217334ff6846SIoana Radulescu */ 217434ff6846SIoana Radulescu err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 217534ff6846SIoana Radulescu &priv->tx_data_offset); 217634ff6846SIoana Radulescu if (err) { 217734ff6846SIoana Radulescu dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 217834ff6846SIoana Radulescu return err; 217934ff6846SIoana Radulescu } 218034ff6846SIoana Radulescu 218134ff6846SIoana Radulescu if ((priv->tx_data_offset % 64) != 0) 218234ff6846SIoana Radulescu dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 218334ff6846SIoana Radulescu priv->tx_data_offset); 218434ff6846SIoana Radulescu 218534ff6846SIoana Radulescu /* rx buffer */ 218634ff6846SIoana Radulescu buf_layout.pass_frame_status = true; 218734ff6846SIoana Radulescu buf_layout.pass_parser_result = true; 218827c87486SIoana Ciocoi Radulescu buf_layout.data_align = rx_buf_align; 218934ff6846SIoana Radulescu buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 219034ff6846SIoana Radulescu buf_layout.private_data_size = 0; 219134ff6846SIoana Radulescu buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 219234ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 219334ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 219434ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 219534ff6846SIoana Radulescu DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 219634ff6846SIoana Radulescu err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 219734ff6846SIoana Radulescu DPNI_QUEUE_RX, &buf_layout); 219834ff6846SIoana Radulescu if (err) { 219934ff6846SIoana Radulescu dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 220034ff6846SIoana Radulescu return err; 220134ff6846SIoana Radulescu } 220234ff6846SIoana Radulescu 220334ff6846SIoana Radulescu return 0; 220434ff6846SIoana Radulescu } 220534ff6846SIoana Radulescu 22061fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 22071fa0f68cSIoana Ciocoi Radulescu #define DPNI_ENQUEUE_FQID_VER_MINOR 9 22081fa0f68cSIoana Ciocoi Radulescu 22091fa0f68cSIoana Ciocoi Radulescu static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 22101fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 22111fa0f68cSIoana Ciocoi Radulescu struct dpaa2_fd *fd, u8 prio) 22121fa0f68cSIoana Ciocoi Radulescu { 22131fa0f68cSIoana Ciocoi Radulescu return dpaa2_io_service_enqueue_qd(fq->channel->dpio, 22141fa0f68cSIoana Ciocoi Radulescu priv->tx_qdid, prio, 22151fa0f68cSIoana Ciocoi Radulescu fq->tx_qdbin, fd); 22161fa0f68cSIoana Ciocoi Radulescu } 22171fa0f68cSIoana Ciocoi Radulescu 22181fa0f68cSIoana Ciocoi Radulescu static inline int dpaa2_eth_enqueue_fq(struct dpaa2_eth_priv *priv, 22191fa0f68cSIoana Ciocoi Radulescu struct dpaa2_eth_fq *fq, 22201fa0f68cSIoana Ciocoi Radulescu struct dpaa2_fd *fd, 22211fa0f68cSIoana Ciocoi Radulescu u8 prio __always_unused) 22221fa0f68cSIoana Ciocoi Radulescu { 22231fa0f68cSIoana Ciocoi Radulescu return dpaa2_io_service_enqueue_fq(fq->channel->dpio, 22241fa0f68cSIoana Ciocoi Radulescu fq->tx_fqid, fd); 22251fa0f68cSIoana Ciocoi Radulescu } 22261fa0f68cSIoana Ciocoi Radulescu 22271fa0f68cSIoana Ciocoi Radulescu static void set_enqueue_mode(struct dpaa2_eth_priv *priv) 22281fa0f68cSIoana Ciocoi Radulescu { 22291fa0f68cSIoana Ciocoi Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 22301fa0f68cSIoana Ciocoi Radulescu DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 22311fa0f68cSIoana Ciocoi Radulescu priv->enqueue = dpaa2_eth_enqueue_qd; 22321fa0f68cSIoana Ciocoi Radulescu else 22331fa0f68cSIoana Ciocoi Radulescu priv->enqueue = dpaa2_eth_enqueue_fq; 22341fa0f68cSIoana Ciocoi Radulescu } 22351fa0f68cSIoana Ciocoi Radulescu 223634ff6846SIoana Radulescu /* Configure the DPNI object this interface is associated with */ 223734ff6846SIoana Radulescu static int setup_dpni(struct fsl_mc_device *ls_dev) 223834ff6846SIoana Radulescu { 223934ff6846SIoana Radulescu struct device *dev = &ls_dev->dev; 224034ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 224134ff6846SIoana Radulescu struct net_device *net_dev; 224234ff6846SIoana Radulescu int err; 224334ff6846SIoana Radulescu 224434ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 224534ff6846SIoana Radulescu priv = netdev_priv(net_dev); 224634ff6846SIoana Radulescu 224734ff6846SIoana Radulescu /* get a handle for the DPNI object */ 224834ff6846SIoana Radulescu err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 224934ff6846SIoana Radulescu if (err) { 225034ff6846SIoana Radulescu dev_err(dev, "dpni_open() failed\n"); 225134ff6846SIoana Radulescu return err; 225234ff6846SIoana Radulescu } 225334ff6846SIoana Radulescu 225434ff6846SIoana Radulescu /* Check if we can work with this DPNI object */ 225534ff6846SIoana Radulescu err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 225634ff6846SIoana Radulescu &priv->dpni_ver_minor); 225734ff6846SIoana Radulescu if (err) { 225834ff6846SIoana Radulescu dev_err(dev, "dpni_get_api_version() failed\n"); 225934ff6846SIoana Radulescu goto close; 226034ff6846SIoana Radulescu } 226134ff6846SIoana Radulescu if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 226234ff6846SIoana Radulescu dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 226334ff6846SIoana Radulescu priv->dpni_ver_major, priv->dpni_ver_minor, 226434ff6846SIoana Radulescu DPNI_VER_MAJOR, DPNI_VER_MINOR); 226534ff6846SIoana Radulescu err = -ENOTSUPP; 226634ff6846SIoana Radulescu goto close; 226734ff6846SIoana Radulescu } 226834ff6846SIoana Radulescu 226934ff6846SIoana Radulescu ls_dev->mc_io = priv->mc_io; 227034ff6846SIoana Radulescu ls_dev->mc_handle = priv->mc_token; 227134ff6846SIoana Radulescu 227234ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 227334ff6846SIoana Radulescu if (err) { 227434ff6846SIoana Radulescu dev_err(dev, "dpni_reset() failed\n"); 227534ff6846SIoana Radulescu goto close; 227634ff6846SIoana Radulescu } 227734ff6846SIoana Radulescu 227834ff6846SIoana Radulescu err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 227934ff6846SIoana Radulescu &priv->dpni_attrs); 228034ff6846SIoana Radulescu if (err) { 228134ff6846SIoana Radulescu dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 228234ff6846SIoana Radulescu goto close; 228334ff6846SIoana Radulescu } 228434ff6846SIoana Radulescu 228534ff6846SIoana Radulescu err = set_buffer_layout(priv); 228634ff6846SIoana Radulescu if (err) 228734ff6846SIoana Radulescu goto close; 228834ff6846SIoana Radulescu 22891fa0f68cSIoana Ciocoi Radulescu set_enqueue_mode(priv); 22901fa0f68cSIoana Ciocoi Radulescu 2291afb90dbbSIoana Radulescu priv->cls_rules = devm_kzalloc(dev, sizeof(struct dpaa2_eth_cls_rule) * 2292afb90dbbSIoana Radulescu dpaa2_eth_fs_count(priv), GFP_KERNEL); 2293afb90dbbSIoana Radulescu if (!priv->cls_rules) 2294afb90dbbSIoana Radulescu goto close; 2295afb90dbbSIoana Radulescu 229634ff6846SIoana Radulescu return 0; 229734ff6846SIoana Radulescu 229834ff6846SIoana Radulescu close: 229934ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 230034ff6846SIoana Radulescu 230134ff6846SIoana Radulescu return err; 230234ff6846SIoana Radulescu } 230334ff6846SIoana Radulescu 230434ff6846SIoana Radulescu static void free_dpni(struct dpaa2_eth_priv *priv) 230534ff6846SIoana Radulescu { 230634ff6846SIoana Radulescu int err; 230734ff6846SIoana Radulescu 230834ff6846SIoana Radulescu err = dpni_reset(priv->mc_io, 0, priv->mc_token); 230934ff6846SIoana Radulescu if (err) 231034ff6846SIoana Radulescu netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 231134ff6846SIoana Radulescu err); 231234ff6846SIoana Radulescu 231334ff6846SIoana Radulescu dpni_close(priv->mc_io, 0, priv->mc_token); 231434ff6846SIoana Radulescu } 231534ff6846SIoana Radulescu 231634ff6846SIoana Radulescu static int setup_rx_flow(struct dpaa2_eth_priv *priv, 231734ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 231834ff6846SIoana Radulescu { 231934ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 232034ff6846SIoana Radulescu struct dpni_queue queue; 232134ff6846SIoana Radulescu struct dpni_queue_id qid; 232234ff6846SIoana Radulescu struct dpni_taildrop td; 232334ff6846SIoana Radulescu int err; 232434ff6846SIoana Radulescu 232534ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 232634ff6846SIoana Radulescu DPNI_QUEUE_RX, 0, fq->flowid, &queue, &qid); 232734ff6846SIoana Radulescu if (err) { 232834ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(RX) failed\n"); 232934ff6846SIoana Radulescu return err; 233034ff6846SIoana Radulescu } 233134ff6846SIoana Radulescu 233234ff6846SIoana Radulescu fq->fqid = qid.fqid; 233334ff6846SIoana Radulescu 233434ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 233534ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 233634ff6846SIoana Radulescu queue.destination.priority = 1; 233734ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 233834ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 233934ff6846SIoana Radulescu DPNI_QUEUE_RX, 0, fq->flowid, 234034ff6846SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 234134ff6846SIoana Radulescu &queue); 234234ff6846SIoana Radulescu if (err) { 234334ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(RX) failed\n"); 234434ff6846SIoana Radulescu return err; 234534ff6846SIoana Radulescu } 234634ff6846SIoana Radulescu 234734ff6846SIoana Radulescu td.enable = 1; 234834ff6846SIoana Radulescu td.threshold = DPAA2_ETH_TAILDROP_THRESH; 234934ff6846SIoana Radulescu err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, DPNI_CP_QUEUE, 235034ff6846SIoana Radulescu DPNI_QUEUE_RX, 0, fq->flowid, &td); 235134ff6846SIoana Radulescu if (err) { 235234ff6846SIoana Radulescu dev_err(dev, "dpni_set_threshold() failed\n"); 235334ff6846SIoana Radulescu return err; 235434ff6846SIoana Radulescu } 235534ff6846SIoana Radulescu 235634ff6846SIoana Radulescu return 0; 235734ff6846SIoana Radulescu } 235834ff6846SIoana Radulescu 235934ff6846SIoana Radulescu static int setup_tx_flow(struct dpaa2_eth_priv *priv, 236034ff6846SIoana Radulescu struct dpaa2_eth_fq *fq) 236134ff6846SIoana Radulescu { 236234ff6846SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 236334ff6846SIoana Radulescu struct dpni_queue queue; 236434ff6846SIoana Radulescu struct dpni_queue_id qid; 236534ff6846SIoana Radulescu int err; 236634ff6846SIoana Radulescu 236734ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 236834ff6846SIoana Radulescu DPNI_QUEUE_TX, 0, fq->flowid, &queue, &qid); 236934ff6846SIoana Radulescu if (err) { 237034ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX) failed\n"); 237134ff6846SIoana Radulescu return err; 237234ff6846SIoana Radulescu } 237334ff6846SIoana Radulescu 237434ff6846SIoana Radulescu fq->tx_qdbin = qid.qdbin; 23751fa0f68cSIoana Ciocoi Radulescu fq->tx_fqid = qid.fqid; 237634ff6846SIoana Radulescu 237734ff6846SIoana Radulescu err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 237834ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 237934ff6846SIoana Radulescu &queue, &qid); 238034ff6846SIoana Radulescu if (err) { 238134ff6846SIoana Radulescu dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 238234ff6846SIoana Radulescu return err; 238334ff6846SIoana Radulescu } 238434ff6846SIoana Radulescu 238534ff6846SIoana Radulescu fq->fqid = qid.fqid; 238634ff6846SIoana Radulescu 238734ff6846SIoana Radulescu queue.destination.id = fq->channel->dpcon_id; 238834ff6846SIoana Radulescu queue.destination.type = DPNI_DEST_DPCON; 238934ff6846SIoana Radulescu queue.destination.priority = 0; 239034ff6846SIoana Radulescu queue.user_context = (u64)(uintptr_t)fq; 239134ff6846SIoana Radulescu err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 239234ff6846SIoana Radulescu DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 239334ff6846SIoana Radulescu DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 239434ff6846SIoana Radulescu &queue); 239534ff6846SIoana Radulescu if (err) { 239634ff6846SIoana Radulescu dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 239734ff6846SIoana Radulescu return err; 239834ff6846SIoana Radulescu } 239934ff6846SIoana Radulescu 240034ff6846SIoana Radulescu return 0; 240134ff6846SIoana Radulescu } 240234ff6846SIoana Radulescu 2403edad8d26SIoana Ciocoi Radulescu /* Supported header fields for Rx hash distribution key */ 2404f76c483aSIoana Radulescu static const struct dpaa2_eth_dist_fields dist_fields[] = { 240534ff6846SIoana Radulescu { 2406edad8d26SIoana Ciocoi Radulescu /* L2 header */ 2407edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_L2DA, 2408edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_ETH, 2409edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_ETH_DA, 2410edad8d26SIoana Ciocoi Radulescu .size = 6, 2411edad8d26SIoana Ciocoi Radulescu }, { 2412afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 2413afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_SA, 2414afb90dbbSIoana Radulescu .size = 6, 2415afb90dbbSIoana Radulescu }, { 2416afb90dbbSIoana Radulescu /* This is the last ethertype field parsed: 2417afb90dbbSIoana Radulescu * depending on frame format, it can be the MAC ethertype 2418afb90dbbSIoana Radulescu * or the VLAN etype. 2419afb90dbbSIoana Radulescu */ 2420afb90dbbSIoana Radulescu .cls_prot = NET_PROT_ETH, 2421afb90dbbSIoana Radulescu .cls_field = NH_FLD_ETH_TYPE, 2422afb90dbbSIoana Radulescu .size = 2, 2423afb90dbbSIoana Radulescu }, { 2424edad8d26SIoana Ciocoi Radulescu /* VLAN header */ 2425edad8d26SIoana Ciocoi Radulescu .rxnfc_field = RXH_VLAN, 2426edad8d26SIoana Ciocoi Radulescu .cls_prot = NET_PROT_VLAN, 2427edad8d26SIoana Ciocoi Radulescu .cls_field = NH_FLD_VLAN_TCI, 2428edad8d26SIoana Ciocoi Radulescu .size = 2, 2429edad8d26SIoana Ciocoi Radulescu }, { 243034ff6846SIoana Radulescu /* IP header */ 243134ff6846SIoana Radulescu .rxnfc_field = RXH_IP_SRC, 243234ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 243334ff6846SIoana Radulescu .cls_field = NH_FLD_IP_SRC, 243434ff6846SIoana Radulescu .size = 4, 243534ff6846SIoana Radulescu }, { 243634ff6846SIoana Radulescu .rxnfc_field = RXH_IP_DST, 243734ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 243834ff6846SIoana Radulescu .cls_field = NH_FLD_IP_DST, 243934ff6846SIoana Radulescu .size = 4, 244034ff6846SIoana Radulescu }, { 244134ff6846SIoana Radulescu .rxnfc_field = RXH_L3_PROTO, 244234ff6846SIoana Radulescu .cls_prot = NET_PROT_IP, 244334ff6846SIoana Radulescu .cls_field = NH_FLD_IP_PROTO, 244434ff6846SIoana Radulescu .size = 1, 244534ff6846SIoana Radulescu }, { 244634ff6846SIoana Radulescu /* Using UDP ports, this is functionally equivalent to raw 244734ff6846SIoana Radulescu * byte pairs from L4 header. 244834ff6846SIoana Radulescu */ 244934ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_0_1, 245034ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 245134ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_SRC, 245234ff6846SIoana Radulescu .size = 2, 245334ff6846SIoana Radulescu }, { 245434ff6846SIoana Radulescu .rxnfc_field = RXH_L4_B_2_3, 245534ff6846SIoana Radulescu .cls_prot = NET_PROT_UDP, 245634ff6846SIoana Radulescu .cls_field = NH_FLD_UDP_PORT_DST, 245734ff6846SIoana Radulescu .size = 2, 245834ff6846SIoana Radulescu }, 245934ff6846SIoana Radulescu }; 246034ff6846SIoana Radulescu 2461df85aeb9SIoana Radulescu /* Configure the Rx hash key using the legacy API */ 2462df85aeb9SIoana Radulescu static int config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 2463df85aeb9SIoana Radulescu { 2464df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 2465df85aeb9SIoana Radulescu struct dpni_rx_tc_dist_cfg dist_cfg; 2466df85aeb9SIoana Radulescu int err; 2467df85aeb9SIoana Radulescu 2468df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 2469df85aeb9SIoana Radulescu 2470df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 2471df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 2472df85aeb9SIoana Radulescu dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 2473df85aeb9SIoana Radulescu 2474df85aeb9SIoana Radulescu err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 0, &dist_cfg); 2475df85aeb9SIoana Radulescu if (err) 2476df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 2477df85aeb9SIoana Radulescu 2478df85aeb9SIoana Radulescu return err; 2479df85aeb9SIoana Radulescu } 2480df85aeb9SIoana Radulescu 2481df85aeb9SIoana Radulescu /* Configure the Rx hash key using the new API */ 2482df85aeb9SIoana Radulescu static int config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 2483df85aeb9SIoana Radulescu { 2484df85aeb9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 2485df85aeb9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 2486df85aeb9SIoana Radulescu int err; 2487df85aeb9SIoana Radulescu 2488df85aeb9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 2489df85aeb9SIoana Radulescu 2490df85aeb9SIoana Radulescu dist_cfg.key_cfg_iova = key; 2491df85aeb9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 2492df85aeb9SIoana Radulescu dist_cfg.enable = 1; 2493df85aeb9SIoana Radulescu 2494df85aeb9SIoana Radulescu err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg); 2495df85aeb9SIoana Radulescu if (err) 2496df85aeb9SIoana Radulescu dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 2497df85aeb9SIoana Radulescu 2498df85aeb9SIoana Radulescu return err; 2499df85aeb9SIoana Radulescu } 2500df85aeb9SIoana Radulescu 25014aaaf9b9SIoana Radulescu /* Configure the Rx flow classification key */ 25024aaaf9b9SIoana Radulescu static int config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 25034aaaf9b9SIoana Radulescu { 25044aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 25054aaaf9b9SIoana Radulescu struct dpni_rx_dist_cfg dist_cfg; 25064aaaf9b9SIoana Radulescu int err; 25074aaaf9b9SIoana Radulescu 25084aaaf9b9SIoana Radulescu memset(&dist_cfg, 0, sizeof(dist_cfg)); 25094aaaf9b9SIoana Radulescu 25104aaaf9b9SIoana Radulescu dist_cfg.key_cfg_iova = key; 25114aaaf9b9SIoana Radulescu dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 25124aaaf9b9SIoana Radulescu dist_cfg.enable = 1; 25134aaaf9b9SIoana Radulescu 25144aaaf9b9SIoana Radulescu err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg); 25154aaaf9b9SIoana Radulescu if (err) 25164aaaf9b9SIoana Radulescu dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 25174aaaf9b9SIoana Radulescu 25184aaaf9b9SIoana Radulescu return err; 25194aaaf9b9SIoana Radulescu } 25204aaaf9b9SIoana Radulescu 2521afb90dbbSIoana Radulescu /* Size of the Rx flow classification key */ 2522afb90dbbSIoana Radulescu int dpaa2_eth_cls_key_size(void) 2523afb90dbbSIoana Radulescu { 2524afb90dbbSIoana Radulescu int i, size = 0; 2525afb90dbbSIoana Radulescu 2526afb90dbbSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 2527afb90dbbSIoana Radulescu size += dist_fields[i].size; 2528afb90dbbSIoana Radulescu 2529afb90dbbSIoana Radulescu return size; 2530afb90dbbSIoana Radulescu } 2531afb90dbbSIoana Radulescu 2532afb90dbbSIoana Radulescu /* Offset of header field in Rx classification key */ 2533afb90dbbSIoana Radulescu int dpaa2_eth_cls_fld_off(int prot, int field) 2534afb90dbbSIoana Radulescu { 2535afb90dbbSIoana Radulescu int i, off = 0; 2536afb90dbbSIoana Radulescu 2537afb90dbbSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 2538afb90dbbSIoana Radulescu if (dist_fields[i].cls_prot == prot && 2539afb90dbbSIoana Radulescu dist_fields[i].cls_field == field) 2540afb90dbbSIoana Radulescu return off; 2541afb90dbbSIoana Radulescu off += dist_fields[i].size; 2542afb90dbbSIoana Radulescu } 2543afb90dbbSIoana Radulescu 2544afb90dbbSIoana Radulescu WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 2545afb90dbbSIoana Radulescu return 0; 2546afb90dbbSIoana Radulescu } 2547afb90dbbSIoana Radulescu 25484aaaf9b9SIoana Radulescu /* Set Rx distribution (hash or flow classification) key 254934ff6846SIoana Radulescu * flags is a combination of RXH_ bits 255034ff6846SIoana Radulescu */ 25513233c151SIoana Ciornei static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 25524aaaf9b9SIoana Radulescu enum dpaa2_eth_rx_dist type, u64 flags) 255334ff6846SIoana Radulescu { 255434ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 255534ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 255634ff6846SIoana Radulescu struct dpkg_profile_cfg cls_cfg; 2557edad8d26SIoana Ciocoi Radulescu u32 rx_hash_fields = 0; 2558df85aeb9SIoana Radulescu dma_addr_t key_iova; 255934ff6846SIoana Radulescu u8 *dma_mem; 256034ff6846SIoana Radulescu int i; 256134ff6846SIoana Radulescu int err = 0; 256234ff6846SIoana Radulescu 256334ff6846SIoana Radulescu memset(&cls_cfg, 0, sizeof(cls_cfg)); 256434ff6846SIoana Radulescu 2565f76c483aSIoana Radulescu for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 256634ff6846SIoana Radulescu struct dpkg_extract *key = 256734ff6846SIoana Radulescu &cls_cfg.extracts[cls_cfg.num_extracts]; 256834ff6846SIoana Radulescu 25694aaaf9b9SIoana Radulescu /* For Rx hashing key we set only the selected fields. 25704aaaf9b9SIoana Radulescu * For Rx flow classification key we set all supported fields 25714aaaf9b9SIoana Radulescu */ 25724aaaf9b9SIoana Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) { 2573f76c483aSIoana Radulescu if (!(flags & dist_fields[i].rxnfc_field)) 257434ff6846SIoana Radulescu continue; 25754aaaf9b9SIoana Radulescu rx_hash_fields |= dist_fields[i].rxnfc_field; 25764aaaf9b9SIoana Radulescu } 257734ff6846SIoana Radulescu 257834ff6846SIoana Radulescu if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 257934ff6846SIoana Radulescu dev_err(dev, "error adding key extraction rule, too many rules?\n"); 258034ff6846SIoana Radulescu return -E2BIG; 258134ff6846SIoana Radulescu } 258234ff6846SIoana Radulescu 258334ff6846SIoana Radulescu key->type = DPKG_EXTRACT_FROM_HDR; 2584f76c483aSIoana Radulescu key->extract.from_hdr.prot = dist_fields[i].cls_prot; 258534ff6846SIoana Radulescu key->extract.from_hdr.type = DPKG_FULL_FIELD; 2586f76c483aSIoana Radulescu key->extract.from_hdr.field = dist_fields[i].cls_field; 258734ff6846SIoana Radulescu cls_cfg.num_extracts++; 258834ff6846SIoana Radulescu } 258934ff6846SIoana Radulescu 259034ff6846SIoana Radulescu dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 259134ff6846SIoana Radulescu if (!dma_mem) 259234ff6846SIoana Radulescu return -ENOMEM; 259334ff6846SIoana Radulescu 259434ff6846SIoana Radulescu err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 259534ff6846SIoana Radulescu if (err) { 259634ff6846SIoana Radulescu dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 2597df85aeb9SIoana Radulescu goto free_key; 259834ff6846SIoana Radulescu } 259934ff6846SIoana Radulescu 260034ff6846SIoana Radulescu /* Prepare for setting the rx dist */ 2601df85aeb9SIoana Radulescu key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 260234ff6846SIoana Radulescu DMA_TO_DEVICE); 2603df85aeb9SIoana Radulescu if (dma_mapping_error(dev, key_iova)) { 260434ff6846SIoana Radulescu dev_err(dev, "DMA mapping failed\n"); 260534ff6846SIoana Radulescu err = -ENOMEM; 2606df85aeb9SIoana Radulescu goto free_key; 260734ff6846SIoana Radulescu } 260834ff6846SIoana Radulescu 26094aaaf9b9SIoana Radulescu if (type == DPAA2_ETH_RX_DIST_HASH) { 2610df85aeb9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) 2611df85aeb9SIoana Radulescu err = config_legacy_hash_key(priv, key_iova); 2612edad8d26SIoana Ciocoi Radulescu else 2613df85aeb9SIoana Radulescu err = config_hash_key(priv, key_iova); 26144aaaf9b9SIoana Radulescu } else { 26154aaaf9b9SIoana Radulescu err = config_cls_key(priv, key_iova); 26164aaaf9b9SIoana Radulescu } 2617df85aeb9SIoana Radulescu 2618df85aeb9SIoana Radulescu dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 2619df85aeb9SIoana Radulescu DMA_TO_DEVICE); 26204aaaf9b9SIoana Radulescu if (!err && type == DPAA2_ETH_RX_DIST_HASH) 2621edad8d26SIoana Ciocoi Radulescu priv->rx_hash_fields = rx_hash_fields; 262234ff6846SIoana Radulescu 2623df85aeb9SIoana Radulescu free_key: 262434ff6846SIoana Radulescu kfree(dma_mem); 262534ff6846SIoana Radulescu return err; 262634ff6846SIoana Radulescu } 262734ff6846SIoana Radulescu 26284aaaf9b9SIoana Radulescu int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 26294aaaf9b9SIoana Radulescu { 26304aaaf9b9SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 26314aaaf9b9SIoana Radulescu 26324aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) 26334aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 26344aaaf9b9SIoana Radulescu 26354aaaf9b9SIoana Radulescu return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, flags); 26364aaaf9b9SIoana Radulescu } 26374aaaf9b9SIoana Radulescu 26384aaaf9b9SIoana Radulescu static int dpaa2_eth_set_cls(struct dpaa2_eth_priv *priv) 26394aaaf9b9SIoana Radulescu { 26404aaaf9b9SIoana Radulescu struct device *dev = priv->net_dev->dev.parent; 26414aaaf9b9SIoana Radulescu 26424aaaf9b9SIoana Radulescu /* Check if we actually support Rx flow classification */ 26434aaaf9b9SIoana Radulescu if (dpaa2_eth_has_legacy_dist(priv)) { 26444aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls not supported by current MC version\n"); 26454aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 26464aaaf9b9SIoana Radulescu } 26474aaaf9b9SIoana Radulescu 26484aaaf9b9SIoana Radulescu if (priv->dpni_attrs.options & DPNI_OPT_NO_FS || 26494aaaf9b9SIoana Radulescu !(priv->dpni_attrs.options & DPNI_OPT_HAS_KEY_MASKING)) { 26504aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 26514aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 26524aaaf9b9SIoana Radulescu } 26534aaaf9b9SIoana Radulescu 26544aaaf9b9SIoana Radulescu if (!dpaa2_eth_hash_enabled(priv)) { 26554aaaf9b9SIoana Radulescu dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 26564aaaf9b9SIoana Radulescu return -EOPNOTSUPP; 26574aaaf9b9SIoana Radulescu } 26584aaaf9b9SIoana Radulescu 26594aaaf9b9SIoana Radulescu priv->rx_cls_enabled = 1; 26604aaaf9b9SIoana Radulescu 26614aaaf9b9SIoana Radulescu return dpaa2_eth_set_dist_key(priv->net_dev, DPAA2_ETH_RX_DIST_CLS, 0); 26624aaaf9b9SIoana Radulescu } 26634aaaf9b9SIoana Radulescu 266434ff6846SIoana Radulescu /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 266534ff6846SIoana Radulescu * frame queues and channels 266634ff6846SIoana Radulescu */ 266734ff6846SIoana Radulescu static int bind_dpni(struct dpaa2_eth_priv *priv) 266834ff6846SIoana Radulescu { 266934ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 267034ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 267134ff6846SIoana Radulescu struct dpni_pools_cfg pools_params; 267234ff6846SIoana Radulescu struct dpni_error_cfg err_cfg; 267334ff6846SIoana Radulescu int err = 0; 267434ff6846SIoana Radulescu int i; 267534ff6846SIoana Radulescu 267634ff6846SIoana Radulescu pools_params.num_dpbp = 1; 267734ff6846SIoana Radulescu pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 267834ff6846SIoana Radulescu pools_params.pools[0].backup_pool = 0; 267934ff6846SIoana Radulescu pools_params.pools[0].buffer_size = DPAA2_ETH_RX_BUF_SIZE; 268034ff6846SIoana Radulescu err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 268134ff6846SIoana Radulescu if (err) { 268234ff6846SIoana Radulescu dev_err(dev, "dpni_set_pools() failed\n"); 268334ff6846SIoana Radulescu return err; 268434ff6846SIoana Radulescu } 268534ff6846SIoana Radulescu 268634ff6846SIoana Radulescu /* have the interface implicitly distribute traffic based on 268734ff6846SIoana Radulescu * the default hash key 268834ff6846SIoana Radulescu */ 268934ff6846SIoana Radulescu err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 2690edad8d26SIoana Ciocoi Radulescu if (err && err != -EOPNOTSUPP) 269134ff6846SIoana Radulescu dev_err(dev, "Failed to configure hashing\n"); 269234ff6846SIoana Radulescu 26934aaaf9b9SIoana Radulescu /* Configure the flow classification key; it includes all 26944aaaf9b9SIoana Radulescu * supported header fields and cannot be modified at runtime 26954aaaf9b9SIoana Radulescu */ 26964aaaf9b9SIoana Radulescu err = dpaa2_eth_set_cls(priv); 26974aaaf9b9SIoana Radulescu if (err && err != -EOPNOTSUPP) 26984aaaf9b9SIoana Radulescu dev_err(dev, "Failed to configure Rx classification key\n"); 26994aaaf9b9SIoana Radulescu 270034ff6846SIoana Radulescu /* Configure handling of error frames */ 270134ff6846SIoana Radulescu err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 270234ff6846SIoana Radulescu err_cfg.set_frame_annotation = 1; 270334ff6846SIoana Radulescu err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 270434ff6846SIoana Radulescu err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 270534ff6846SIoana Radulescu &err_cfg); 270634ff6846SIoana Radulescu if (err) { 270734ff6846SIoana Radulescu dev_err(dev, "dpni_set_errors_behavior failed\n"); 270834ff6846SIoana Radulescu return err; 270934ff6846SIoana Radulescu } 271034ff6846SIoana Radulescu 271134ff6846SIoana Radulescu /* Configure Rx and Tx conf queues to generate CDANs */ 271234ff6846SIoana Radulescu for (i = 0; i < priv->num_fqs; i++) { 271334ff6846SIoana Radulescu switch (priv->fq[i].type) { 271434ff6846SIoana Radulescu case DPAA2_RX_FQ: 271534ff6846SIoana Radulescu err = setup_rx_flow(priv, &priv->fq[i]); 271634ff6846SIoana Radulescu break; 271734ff6846SIoana Radulescu case DPAA2_TX_CONF_FQ: 271834ff6846SIoana Radulescu err = setup_tx_flow(priv, &priv->fq[i]); 271934ff6846SIoana Radulescu break; 272034ff6846SIoana Radulescu default: 272134ff6846SIoana Radulescu dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 272234ff6846SIoana Radulescu return -EINVAL; 272334ff6846SIoana Radulescu } 272434ff6846SIoana Radulescu if (err) 272534ff6846SIoana Radulescu return err; 272634ff6846SIoana Radulescu } 272734ff6846SIoana Radulescu 272834ff6846SIoana Radulescu err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 272934ff6846SIoana Radulescu DPNI_QUEUE_TX, &priv->tx_qdid); 273034ff6846SIoana Radulescu if (err) { 273134ff6846SIoana Radulescu dev_err(dev, "dpni_get_qdid() failed\n"); 273234ff6846SIoana Radulescu return err; 273334ff6846SIoana Radulescu } 273434ff6846SIoana Radulescu 273534ff6846SIoana Radulescu return 0; 273634ff6846SIoana Radulescu } 273734ff6846SIoana Radulescu 273834ff6846SIoana Radulescu /* Allocate rings for storing incoming frame descriptors */ 273934ff6846SIoana Radulescu static int alloc_rings(struct dpaa2_eth_priv *priv) 274034ff6846SIoana Radulescu { 274134ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 274234ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 274334ff6846SIoana Radulescu int i; 274434ff6846SIoana Radulescu 274534ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 274634ff6846SIoana Radulescu priv->channel[i]->store = 274734ff6846SIoana Radulescu dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 274834ff6846SIoana Radulescu if (!priv->channel[i]->store) { 274934ff6846SIoana Radulescu netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 275034ff6846SIoana Radulescu goto err_ring; 275134ff6846SIoana Radulescu } 275234ff6846SIoana Radulescu } 275334ff6846SIoana Radulescu 275434ff6846SIoana Radulescu return 0; 275534ff6846SIoana Radulescu 275634ff6846SIoana Radulescu err_ring: 275734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 275834ff6846SIoana Radulescu if (!priv->channel[i]->store) 275934ff6846SIoana Radulescu break; 276034ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 276134ff6846SIoana Radulescu } 276234ff6846SIoana Radulescu 276334ff6846SIoana Radulescu return -ENOMEM; 276434ff6846SIoana Radulescu } 276534ff6846SIoana Radulescu 276634ff6846SIoana Radulescu static void free_rings(struct dpaa2_eth_priv *priv) 276734ff6846SIoana Radulescu { 276834ff6846SIoana Radulescu int i; 276934ff6846SIoana Radulescu 277034ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) 277134ff6846SIoana Radulescu dpaa2_io_store_destroy(priv->channel[i]->store); 277234ff6846SIoana Radulescu } 277334ff6846SIoana Radulescu 277434ff6846SIoana Radulescu static int set_mac_addr(struct dpaa2_eth_priv *priv) 277534ff6846SIoana Radulescu { 277634ff6846SIoana Radulescu struct net_device *net_dev = priv->net_dev; 277734ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 277834ff6846SIoana Radulescu u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 277934ff6846SIoana Radulescu int err; 278034ff6846SIoana Radulescu 278134ff6846SIoana Radulescu /* Get firmware address, if any */ 278234ff6846SIoana Radulescu err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 278334ff6846SIoana Radulescu if (err) { 278434ff6846SIoana Radulescu dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 278534ff6846SIoana Radulescu return err; 278634ff6846SIoana Radulescu } 278734ff6846SIoana Radulescu 278834ff6846SIoana Radulescu /* Get DPNI attributes address, if any */ 278934ff6846SIoana Radulescu err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 279034ff6846SIoana Radulescu dpni_mac_addr); 279134ff6846SIoana Radulescu if (err) { 279234ff6846SIoana Radulescu dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 279334ff6846SIoana Radulescu return err; 279434ff6846SIoana Radulescu } 279534ff6846SIoana Radulescu 279634ff6846SIoana Radulescu /* First check if firmware has any address configured by bootloader */ 279734ff6846SIoana Radulescu if (!is_zero_ether_addr(mac_addr)) { 279834ff6846SIoana Radulescu /* If the DPMAC addr != DPNI addr, update it */ 279934ff6846SIoana Radulescu if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 280034ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, 280134ff6846SIoana Radulescu priv->mc_token, 280234ff6846SIoana Radulescu mac_addr); 280334ff6846SIoana Radulescu if (err) { 280434ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 280534ff6846SIoana Radulescu return err; 280634ff6846SIoana Radulescu } 280734ff6846SIoana Radulescu } 280834ff6846SIoana Radulescu memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); 280934ff6846SIoana Radulescu } else if (is_zero_ether_addr(dpni_mac_addr)) { 281034ff6846SIoana Radulescu /* No MAC address configured, fill in net_dev->dev_addr 281134ff6846SIoana Radulescu * with a random one 281234ff6846SIoana Radulescu */ 281334ff6846SIoana Radulescu eth_hw_addr_random(net_dev); 281434ff6846SIoana Radulescu dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 281534ff6846SIoana Radulescu 281634ff6846SIoana Radulescu err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 281734ff6846SIoana Radulescu net_dev->dev_addr); 281834ff6846SIoana Radulescu if (err) { 281934ff6846SIoana Radulescu dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 282034ff6846SIoana Radulescu return err; 282134ff6846SIoana Radulescu } 282234ff6846SIoana Radulescu 282334ff6846SIoana Radulescu /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 282434ff6846SIoana Radulescu * practical purposes, this will be our "permanent" mac address, 282534ff6846SIoana Radulescu * at least until the next reboot. This move will also permit 282634ff6846SIoana Radulescu * register_netdevice() to properly fill up net_dev->perm_addr. 282734ff6846SIoana Radulescu */ 282834ff6846SIoana Radulescu net_dev->addr_assign_type = NET_ADDR_PERM; 282934ff6846SIoana Radulescu } else { 283034ff6846SIoana Radulescu /* NET_ADDR_PERM is default, all we have to do is 283134ff6846SIoana Radulescu * fill in the device addr. 283234ff6846SIoana Radulescu */ 283334ff6846SIoana Radulescu memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len); 283434ff6846SIoana Radulescu } 283534ff6846SIoana Radulescu 283634ff6846SIoana Radulescu return 0; 283734ff6846SIoana Radulescu } 283834ff6846SIoana Radulescu 283934ff6846SIoana Radulescu static int netdev_init(struct net_device *net_dev) 284034ff6846SIoana Radulescu { 284134ff6846SIoana Radulescu struct device *dev = net_dev->dev.parent; 284234ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 284334ff6846SIoana Radulescu u32 options = priv->dpni_attrs.options; 284434ff6846SIoana Radulescu u64 supported = 0, not_supported = 0; 284534ff6846SIoana Radulescu u8 bcast_addr[ETH_ALEN]; 284634ff6846SIoana Radulescu u8 num_queues; 284734ff6846SIoana Radulescu int err; 284834ff6846SIoana Radulescu 284934ff6846SIoana Radulescu net_dev->netdev_ops = &dpaa2_eth_ops; 285034ff6846SIoana Radulescu net_dev->ethtool_ops = &dpaa2_ethtool_ops; 285134ff6846SIoana Radulescu 285234ff6846SIoana Radulescu err = set_mac_addr(priv); 285334ff6846SIoana Radulescu if (err) 285434ff6846SIoana Radulescu return err; 285534ff6846SIoana Radulescu 285634ff6846SIoana Radulescu /* Explicitly add the broadcast address to the MAC filtering table */ 285734ff6846SIoana Radulescu eth_broadcast_addr(bcast_addr); 285834ff6846SIoana Radulescu err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 285934ff6846SIoana Radulescu if (err) { 286034ff6846SIoana Radulescu dev_err(dev, "dpni_add_mac_addr() failed\n"); 286134ff6846SIoana Radulescu return err; 286234ff6846SIoana Radulescu } 286334ff6846SIoana Radulescu 286434ff6846SIoana Radulescu /* Set MTU upper limit; lower limit is 68B (default value) */ 286534ff6846SIoana Radulescu net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 286634ff6846SIoana Radulescu err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 286734ff6846SIoana Radulescu DPAA2_ETH_MFL); 286834ff6846SIoana Radulescu if (err) { 286934ff6846SIoana Radulescu dev_err(dev, "dpni_set_max_frame_length() failed\n"); 287034ff6846SIoana Radulescu return err; 287134ff6846SIoana Radulescu } 287234ff6846SIoana Radulescu 287334ff6846SIoana Radulescu /* Set actual number of queues in the net device */ 287434ff6846SIoana Radulescu num_queues = dpaa2_eth_queue_count(priv); 287534ff6846SIoana Radulescu err = netif_set_real_num_tx_queues(net_dev, num_queues); 287634ff6846SIoana Radulescu if (err) { 287734ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 287834ff6846SIoana Radulescu return err; 287934ff6846SIoana Radulescu } 288034ff6846SIoana Radulescu err = netif_set_real_num_rx_queues(net_dev, num_queues); 288134ff6846SIoana Radulescu if (err) { 288234ff6846SIoana Radulescu dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 288334ff6846SIoana Radulescu return err; 288434ff6846SIoana Radulescu } 288534ff6846SIoana Radulescu 288634ff6846SIoana Radulescu /* Capabilities listing */ 288734ff6846SIoana Radulescu supported |= IFF_LIVE_ADDR_CHANGE; 288834ff6846SIoana Radulescu 288934ff6846SIoana Radulescu if (options & DPNI_OPT_NO_MAC_FILTER) 289034ff6846SIoana Radulescu not_supported |= IFF_UNICAST_FLT; 289134ff6846SIoana Radulescu else 289234ff6846SIoana Radulescu supported |= IFF_UNICAST_FLT; 289334ff6846SIoana Radulescu 289434ff6846SIoana Radulescu net_dev->priv_flags |= supported; 289534ff6846SIoana Radulescu net_dev->priv_flags &= ~not_supported; 289634ff6846SIoana Radulescu 289734ff6846SIoana Radulescu /* Features */ 289834ff6846SIoana Radulescu net_dev->features = NETIF_F_RXCSUM | 289934ff6846SIoana Radulescu NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 290034ff6846SIoana Radulescu NETIF_F_SG | NETIF_F_HIGHDMA | 290134ff6846SIoana Radulescu NETIF_F_LLTX; 290234ff6846SIoana Radulescu net_dev->hw_features = net_dev->features; 290334ff6846SIoana Radulescu 290434ff6846SIoana Radulescu return 0; 290534ff6846SIoana Radulescu } 290634ff6846SIoana Radulescu 290734ff6846SIoana Radulescu static int poll_link_state(void *arg) 290834ff6846SIoana Radulescu { 290934ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 291034ff6846SIoana Radulescu int err; 291134ff6846SIoana Radulescu 291234ff6846SIoana Radulescu while (!kthread_should_stop()) { 291334ff6846SIoana Radulescu err = link_state_update(priv); 291434ff6846SIoana Radulescu if (unlikely(err)) 291534ff6846SIoana Radulescu return err; 291634ff6846SIoana Radulescu 291734ff6846SIoana Radulescu msleep(DPAA2_ETH_LINK_STATE_REFRESH); 291834ff6846SIoana Radulescu } 291934ff6846SIoana Radulescu 292034ff6846SIoana Radulescu return 0; 292134ff6846SIoana Radulescu } 292234ff6846SIoana Radulescu 292334ff6846SIoana Radulescu static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 292434ff6846SIoana Radulescu { 292534ff6846SIoana Radulescu u32 status = ~0; 292634ff6846SIoana Radulescu struct device *dev = (struct device *)arg; 292734ff6846SIoana Radulescu struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 292834ff6846SIoana Radulescu struct net_device *net_dev = dev_get_drvdata(dev); 292934ff6846SIoana Radulescu int err; 293034ff6846SIoana Radulescu 293134ff6846SIoana Radulescu err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 293234ff6846SIoana Radulescu DPNI_IRQ_INDEX, &status); 293334ff6846SIoana Radulescu if (unlikely(err)) { 293434ff6846SIoana Radulescu netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 293534ff6846SIoana Radulescu return IRQ_HANDLED; 293634ff6846SIoana Radulescu } 293734ff6846SIoana Radulescu 293834ff6846SIoana Radulescu if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 293934ff6846SIoana Radulescu link_state_update(netdev_priv(net_dev)); 294034ff6846SIoana Radulescu 294134ff6846SIoana Radulescu return IRQ_HANDLED; 294234ff6846SIoana Radulescu } 294334ff6846SIoana Radulescu 294434ff6846SIoana Radulescu static int setup_irqs(struct fsl_mc_device *ls_dev) 294534ff6846SIoana Radulescu { 294634ff6846SIoana Radulescu int err = 0; 294734ff6846SIoana Radulescu struct fsl_mc_device_irq *irq; 294834ff6846SIoana Radulescu 294934ff6846SIoana Radulescu err = fsl_mc_allocate_irqs(ls_dev); 295034ff6846SIoana Radulescu if (err) { 295134ff6846SIoana Radulescu dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 295234ff6846SIoana Radulescu return err; 295334ff6846SIoana Radulescu } 295434ff6846SIoana Radulescu 295534ff6846SIoana Radulescu irq = ls_dev->irqs[0]; 295634ff6846SIoana Radulescu err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, 295734ff6846SIoana Radulescu NULL, dpni_irq0_handler_thread, 295834ff6846SIoana Radulescu IRQF_NO_SUSPEND | IRQF_ONESHOT, 295934ff6846SIoana Radulescu dev_name(&ls_dev->dev), &ls_dev->dev); 296034ff6846SIoana Radulescu if (err < 0) { 296134ff6846SIoana Radulescu dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 296234ff6846SIoana Radulescu goto free_mc_irq; 296334ff6846SIoana Radulescu } 296434ff6846SIoana Radulescu 296534ff6846SIoana Radulescu err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 296634ff6846SIoana Radulescu DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED); 296734ff6846SIoana Radulescu if (err < 0) { 296834ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 296934ff6846SIoana Radulescu goto free_irq; 297034ff6846SIoana Radulescu } 297134ff6846SIoana Radulescu 297234ff6846SIoana Radulescu err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 297334ff6846SIoana Radulescu DPNI_IRQ_INDEX, 1); 297434ff6846SIoana Radulescu if (err < 0) { 297534ff6846SIoana Radulescu dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 297634ff6846SIoana Radulescu goto free_irq; 297734ff6846SIoana Radulescu } 297834ff6846SIoana Radulescu 297934ff6846SIoana Radulescu return 0; 298034ff6846SIoana Radulescu 298134ff6846SIoana Radulescu free_irq: 298234ff6846SIoana Radulescu devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); 298334ff6846SIoana Radulescu free_mc_irq: 298434ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 298534ff6846SIoana Radulescu 298634ff6846SIoana Radulescu return err; 298734ff6846SIoana Radulescu } 298834ff6846SIoana Radulescu 298934ff6846SIoana Radulescu static void add_ch_napi(struct dpaa2_eth_priv *priv) 299034ff6846SIoana Radulescu { 299134ff6846SIoana Radulescu int i; 299234ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 299334ff6846SIoana Radulescu 299434ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 299534ff6846SIoana Radulescu ch = priv->channel[i]; 299634ff6846SIoana Radulescu /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 299734ff6846SIoana Radulescu netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 299834ff6846SIoana Radulescu NAPI_POLL_WEIGHT); 299934ff6846SIoana Radulescu } 300034ff6846SIoana Radulescu } 300134ff6846SIoana Radulescu 300234ff6846SIoana Radulescu static void del_ch_napi(struct dpaa2_eth_priv *priv) 300334ff6846SIoana Radulescu { 300434ff6846SIoana Radulescu int i; 300534ff6846SIoana Radulescu struct dpaa2_eth_channel *ch; 300634ff6846SIoana Radulescu 300734ff6846SIoana Radulescu for (i = 0; i < priv->num_channels; i++) { 300834ff6846SIoana Radulescu ch = priv->channel[i]; 300934ff6846SIoana Radulescu netif_napi_del(&ch->napi); 301034ff6846SIoana Radulescu } 301134ff6846SIoana Radulescu } 301234ff6846SIoana Radulescu 301334ff6846SIoana Radulescu static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 301434ff6846SIoana Radulescu { 301534ff6846SIoana Radulescu struct device *dev; 301634ff6846SIoana Radulescu struct net_device *net_dev = NULL; 301734ff6846SIoana Radulescu struct dpaa2_eth_priv *priv = NULL; 301834ff6846SIoana Radulescu int err = 0; 301934ff6846SIoana Radulescu 302034ff6846SIoana Radulescu dev = &dpni_dev->dev; 302134ff6846SIoana Radulescu 302234ff6846SIoana Radulescu /* Net device */ 302334ff6846SIoana Radulescu net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_TX_QUEUES); 302434ff6846SIoana Radulescu if (!net_dev) { 302534ff6846SIoana Radulescu dev_err(dev, "alloc_etherdev_mq() failed\n"); 302634ff6846SIoana Radulescu return -ENOMEM; 302734ff6846SIoana Radulescu } 302834ff6846SIoana Radulescu 302934ff6846SIoana Radulescu SET_NETDEV_DEV(net_dev, dev); 303034ff6846SIoana Radulescu dev_set_drvdata(dev, net_dev); 303134ff6846SIoana Radulescu 303234ff6846SIoana Radulescu priv = netdev_priv(net_dev); 303334ff6846SIoana Radulescu priv->net_dev = net_dev; 303434ff6846SIoana Radulescu 303534ff6846SIoana Radulescu priv->iommu_domain = iommu_get_domain_for_dev(dev); 303634ff6846SIoana Radulescu 303734ff6846SIoana Radulescu /* Obtain a MC portal */ 303834ff6846SIoana Radulescu err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 303934ff6846SIoana Radulescu &priv->mc_io); 304034ff6846SIoana Radulescu if (err) { 304134ff6846SIoana Radulescu if (err == -ENXIO) 304234ff6846SIoana Radulescu err = -EPROBE_DEFER; 304334ff6846SIoana Radulescu else 304434ff6846SIoana Radulescu dev_err(dev, "MC portal allocation failed\n"); 304534ff6846SIoana Radulescu goto err_portal_alloc; 304634ff6846SIoana Radulescu } 304734ff6846SIoana Radulescu 304834ff6846SIoana Radulescu /* MC objects initialization and configuration */ 304934ff6846SIoana Radulescu err = setup_dpni(dpni_dev); 305034ff6846SIoana Radulescu if (err) 305134ff6846SIoana Radulescu goto err_dpni_setup; 305234ff6846SIoana Radulescu 305334ff6846SIoana Radulescu err = setup_dpio(priv); 305434ff6846SIoana Radulescu if (err) 305534ff6846SIoana Radulescu goto err_dpio_setup; 305634ff6846SIoana Radulescu 305734ff6846SIoana Radulescu setup_fqs(priv); 305834ff6846SIoana Radulescu 305934ff6846SIoana Radulescu err = setup_dpbp(priv); 306034ff6846SIoana Radulescu if (err) 306134ff6846SIoana Radulescu goto err_dpbp_setup; 306234ff6846SIoana Radulescu 306334ff6846SIoana Radulescu err = bind_dpni(priv); 306434ff6846SIoana Radulescu if (err) 306534ff6846SIoana Radulescu goto err_bind; 306634ff6846SIoana Radulescu 306734ff6846SIoana Radulescu /* Add a NAPI context for each channel */ 306834ff6846SIoana Radulescu add_ch_napi(priv); 306934ff6846SIoana Radulescu 307034ff6846SIoana Radulescu /* Percpu statistics */ 307134ff6846SIoana Radulescu priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 307234ff6846SIoana Radulescu if (!priv->percpu_stats) { 307334ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 307434ff6846SIoana Radulescu err = -ENOMEM; 307534ff6846SIoana Radulescu goto err_alloc_percpu_stats; 307634ff6846SIoana Radulescu } 307734ff6846SIoana Radulescu priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 307834ff6846SIoana Radulescu if (!priv->percpu_extras) { 307934ff6846SIoana Radulescu dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 308034ff6846SIoana Radulescu err = -ENOMEM; 308134ff6846SIoana Radulescu goto err_alloc_percpu_extras; 308234ff6846SIoana Radulescu } 308334ff6846SIoana Radulescu 308434ff6846SIoana Radulescu err = netdev_init(net_dev); 308534ff6846SIoana Radulescu if (err) 308634ff6846SIoana Radulescu goto err_netdev_init; 308734ff6846SIoana Radulescu 308834ff6846SIoana Radulescu /* Configure checksum offload based on current interface flags */ 308934ff6846SIoana Radulescu err = set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 309034ff6846SIoana Radulescu if (err) 309134ff6846SIoana Radulescu goto err_csum; 309234ff6846SIoana Radulescu 309334ff6846SIoana Radulescu err = set_tx_csum(priv, !!(net_dev->features & 309434ff6846SIoana Radulescu (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 309534ff6846SIoana Radulescu if (err) 309634ff6846SIoana Radulescu goto err_csum; 309734ff6846SIoana Radulescu 309834ff6846SIoana Radulescu err = alloc_rings(priv); 309934ff6846SIoana Radulescu if (err) 310034ff6846SIoana Radulescu goto err_alloc_rings; 310134ff6846SIoana Radulescu 310234ff6846SIoana Radulescu err = setup_irqs(dpni_dev); 310334ff6846SIoana Radulescu if (err) { 310434ff6846SIoana Radulescu netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 310534ff6846SIoana Radulescu priv->poll_thread = kthread_run(poll_link_state, priv, 310634ff6846SIoana Radulescu "%s_poll_link", net_dev->name); 310734ff6846SIoana Radulescu if (IS_ERR(priv->poll_thread)) { 310834ff6846SIoana Radulescu dev_err(dev, "Error starting polling thread\n"); 310934ff6846SIoana Radulescu goto err_poll_thread; 311034ff6846SIoana Radulescu } 311134ff6846SIoana Radulescu priv->do_link_poll = true; 311234ff6846SIoana Radulescu } 311334ff6846SIoana Radulescu 311434ff6846SIoana Radulescu err = register_netdev(net_dev); 311534ff6846SIoana Radulescu if (err < 0) { 311634ff6846SIoana Radulescu dev_err(dev, "register_netdev() failed\n"); 311734ff6846SIoana Radulescu goto err_netdev_reg; 311834ff6846SIoana Radulescu } 311934ff6846SIoana Radulescu 3120091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 3121091a19eaSIoana Radulescu dpaa2_dbg_add(priv); 3122091a19eaSIoana Radulescu #endif 3123091a19eaSIoana Radulescu 312434ff6846SIoana Radulescu dev_info(dev, "Probed interface %s\n", net_dev->name); 312534ff6846SIoana Radulescu return 0; 312634ff6846SIoana Radulescu 312734ff6846SIoana Radulescu err_netdev_reg: 312834ff6846SIoana Radulescu if (priv->do_link_poll) 312934ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 313034ff6846SIoana Radulescu else 313134ff6846SIoana Radulescu fsl_mc_free_irqs(dpni_dev); 313234ff6846SIoana Radulescu err_poll_thread: 313334ff6846SIoana Radulescu free_rings(priv); 313434ff6846SIoana Radulescu err_alloc_rings: 313534ff6846SIoana Radulescu err_csum: 313634ff6846SIoana Radulescu err_netdev_init: 313734ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 313834ff6846SIoana Radulescu err_alloc_percpu_extras: 313934ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 314034ff6846SIoana Radulescu err_alloc_percpu_stats: 314134ff6846SIoana Radulescu del_ch_napi(priv); 314234ff6846SIoana Radulescu err_bind: 314334ff6846SIoana Radulescu free_dpbp(priv); 314434ff6846SIoana Radulescu err_dpbp_setup: 314534ff6846SIoana Radulescu free_dpio(priv); 314634ff6846SIoana Radulescu err_dpio_setup: 314734ff6846SIoana Radulescu free_dpni(priv); 314834ff6846SIoana Radulescu err_dpni_setup: 314934ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 315034ff6846SIoana Radulescu err_portal_alloc: 315134ff6846SIoana Radulescu dev_set_drvdata(dev, NULL); 315234ff6846SIoana Radulescu free_netdev(net_dev); 315334ff6846SIoana Radulescu 315434ff6846SIoana Radulescu return err; 315534ff6846SIoana Radulescu } 315634ff6846SIoana Radulescu 315734ff6846SIoana Radulescu static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 315834ff6846SIoana Radulescu { 315934ff6846SIoana Radulescu struct device *dev; 316034ff6846SIoana Radulescu struct net_device *net_dev; 316134ff6846SIoana Radulescu struct dpaa2_eth_priv *priv; 316234ff6846SIoana Radulescu 316334ff6846SIoana Radulescu dev = &ls_dev->dev; 316434ff6846SIoana Radulescu net_dev = dev_get_drvdata(dev); 316534ff6846SIoana Radulescu priv = netdev_priv(net_dev); 316634ff6846SIoana Radulescu 3167091a19eaSIoana Radulescu #ifdef CONFIG_DEBUG_FS 3168091a19eaSIoana Radulescu dpaa2_dbg_remove(priv); 3169091a19eaSIoana Radulescu #endif 317034ff6846SIoana Radulescu unregister_netdev(net_dev); 317134ff6846SIoana Radulescu 317234ff6846SIoana Radulescu if (priv->do_link_poll) 317334ff6846SIoana Radulescu kthread_stop(priv->poll_thread); 317434ff6846SIoana Radulescu else 317534ff6846SIoana Radulescu fsl_mc_free_irqs(ls_dev); 317634ff6846SIoana Radulescu 317734ff6846SIoana Radulescu free_rings(priv); 317834ff6846SIoana Radulescu free_percpu(priv->percpu_stats); 317934ff6846SIoana Radulescu free_percpu(priv->percpu_extras); 318034ff6846SIoana Radulescu 318134ff6846SIoana Radulescu del_ch_napi(priv); 318234ff6846SIoana Radulescu free_dpbp(priv); 318334ff6846SIoana Radulescu free_dpio(priv); 318434ff6846SIoana Radulescu free_dpni(priv); 318534ff6846SIoana Radulescu 318634ff6846SIoana Radulescu fsl_mc_portal_free(priv->mc_io); 318734ff6846SIoana Radulescu 318834ff6846SIoana Radulescu free_netdev(net_dev); 318934ff6846SIoana Radulescu 319034ff6846SIoana Radulescu dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 319134ff6846SIoana Radulescu 319234ff6846SIoana Radulescu return 0; 319334ff6846SIoana Radulescu } 319434ff6846SIoana Radulescu 319534ff6846SIoana Radulescu static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 319634ff6846SIoana Radulescu { 319734ff6846SIoana Radulescu .vendor = FSL_MC_VENDOR_FREESCALE, 319834ff6846SIoana Radulescu .obj_type = "dpni", 319934ff6846SIoana Radulescu }, 320034ff6846SIoana Radulescu { .vendor = 0x0 } 320134ff6846SIoana Radulescu }; 320234ff6846SIoana Radulescu MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 320334ff6846SIoana Radulescu 320434ff6846SIoana Radulescu static struct fsl_mc_driver dpaa2_eth_driver = { 320534ff6846SIoana Radulescu .driver = { 320634ff6846SIoana Radulescu .name = KBUILD_MODNAME, 320734ff6846SIoana Radulescu .owner = THIS_MODULE, 320834ff6846SIoana Radulescu }, 320934ff6846SIoana Radulescu .probe = dpaa2_eth_probe, 321034ff6846SIoana Radulescu .remove = dpaa2_eth_remove, 321134ff6846SIoana Radulescu .match_id_table = dpaa2_eth_match_id_table 321234ff6846SIoana Radulescu }; 321334ff6846SIoana Radulescu 3214091a19eaSIoana Radulescu static int __init dpaa2_eth_driver_init(void) 3215091a19eaSIoana Radulescu { 3216091a19eaSIoana Radulescu int err; 3217091a19eaSIoana Radulescu 3218091a19eaSIoana Radulescu dpaa2_eth_dbg_init(); 3219091a19eaSIoana Radulescu err = fsl_mc_driver_register(&dpaa2_eth_driver); 3220091a19eaSIoana Radulescu if (err) { 3221091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 3222091a19eaSIoana Radulescu return err; 3223091a19eaSIoana Radulescu } 3224091a19eaSIoana Radulescu 3225091a19eaSIoana Radulescu return 0; 3226091a19eaSIoana Radulescu } 3227091a19eaSIoana Radulescu 3228091a19eaSIoana Radulescu static void __exit dpaa2_eth_driver_exit(void) 3229091a19eaSIoana Radulescu { 3230091a19eaSIoana Radulescu dpaa2_eth_dbg_exit(); 3231091a19eaSIoana Radulescu fsl_mc_driver_unregister(&dpaa2_eth_driver); 3232091a19eaSIoana Radulescu } 3233091a19eaSIoana Radulescu 3234091a19eaSIoana Radulescu module_init(dpaa2_eth_driver_init); 3235091a19eaSIoana Radulescu module_exit(dpaa2_eth_driver_exit); 3236