1f7917c00SJeff Kirsher /* 2f7917c00SJeff Kirsher * This file is part of the Chelsio T4 Ethernet driver for Linux. 3f7917c00SJeff Kirsher * 4ce100b8bSAnish Bhatt * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. 5f7917c00SJeff Kirsher * 6f7917c00SJeff Kirsher * This software is available to you under a choice of one of two 7f7917c00SJeff Kirsher * licenses. You may choose to be licensed under the terms of the GNU 8f7917c00SJeff Kirsher * General Public License (GPL) Version 2, available from the file 9f7917c00SJeff Kirsher * COPYING in the main directory of this source tree, or the 10f7917c00SJeff Kirsher * OpenIB.org BSD license below: 11f7917c00SJeff Kirsher * 12f7917c00SJeff Kirsher * Redistribution and use in source and binary forms, with or 13f7917c00SJeff Kirsher * without modification, are permitted provided that the following 14f7917c00SJeff Kirsher * conditions are met: 15f7917c00SJeff Kirsher * 16f7917c00SJeff Kirsher * - Redistributions of source code must retain the above 17f7917c00SJeff Kirsher * copyright notice, this list of conditions and the following 18f7917c00SJeff Kirsher * disclaimer. 19f7917c00SJeff Kirsher * 20f7917c00SJeff Kirsher * - Redistributions in binary form must reproduce the above 21f7917c00SJeff Kirsher * copyright notice, this list of conditions and the following 22f7917c00SJeff Kirsher * disclaimer in the documentation and/or other materials 23f7917c00SJeff Kirsher * provided with the distribution. 24f7917c00SJeff Kirsher * 25f7917c00SJeff Kirsher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26f7917c00SJeff Kirsher * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27f7917c00SJeff Kirsher * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28f7917c00SJeff Kirsher * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29f7917c00SJeff Kirsher * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30f7917c00SJeff Kirsher * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31f7917c00SJeff Kirsher * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32f7917c00SJeff Kirsher * SOFTWARE. 33f7917c00SJeff Kirsher */ 34f7917c00SJeff Kirsher 35f7917c00SJeff Kirsher #include <linux/skbuff.h> 36f7917c00SJeff Kirsher #include <linux/netdevice.h> 37f7917c00SJeff Kirsher #include <linux/etherdevice.h> 38f7917c00SJeff Kirsher #include <linux/if_vlan.h> 39f7917c00SJeff Kirsher #include <linux/ip.h> 40f7917c00SJeff Kirsher #include <linux/dma-mapping.h> 41f7917c00SJeff Kirsher #include <linux/jiffies.h> 42f7917c00SJeff Kirsher #include <linux/prefetch.h> 43ee40fa06SPaul Gortmaker #include <linux/export.h> 44f7917c00SJeff Kirsher #include <net/ipv6.h> 45f7917c00SJeff Kirsher #include <net/tcp.h> 463a336cb1SHariprasad Shenai #ifdef CONFIG_NET_RX_BUSY_POLL 473a336cb1SHariprasad Shenai #include <net/busy_poll.h> 483a336cb1SHariprasad Shenai #endif /* CONFIG_NET_RX_BUSY_POLL */ 4984a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 5084a200b3SVarun Prakash #include <scsi/fc/fc_fcoe.h> 5184a200b3SVarun Prakash #endif /* CONFIG_CHELSIO_T4_FCOE */ 52f7917c00SJeff Kirsher #include "cxgb4.h" 53f7917c00SJeff Kirsher #include "t4_regs.h" 54f612b815SHariprasad Shenai #include "t4_values.h" 55f7917c00SJeff Kirsher #include "t4_msg.h" 56f7917c00SJeff Kirsher #include "t4fw_api.h" 57f7917c00SJeff Kirsher 58f7917c00SJeff Kirsher /* 59f7917c00SJeff Kirsher * Rx buffer size. We use largish buffers if possible but settle for single 60f7917c00SJeff Kirsher * pages under memory shortage. 61f7917c00SJeff Kirsher */ 62f7917c00SJeff Kirsher #if PAGE_SHIFT >= 16 63f7917c00SJeff Kirsher # define FL_PG_ORDER 0 64f7917c00SJeff Kirsher #else 65f7917c00SJeff Kirsher # define FL_PG_ORDER (16 - PAGE_SHIFT) 66f7917c00SJeff Kirsher #endif 67f7917c00SJeff Kirsher 68f7917c00SJeff Kirsher /* RX_PULL_LEN should be <= RX_COPY_THRES */ 69f7917c00SJeff Kirsher #define RX_COPY_THRES 256 70f7917c00SJeff Kirsher #define RX_PULL_LEN 128 71f7917c00SJeff Kirsher 72f7917c00SJeff Kirsher /* 73f7917c00SJeff Kirsher * Main body length for sk_buffs used for Rx Ethernet packets with fragments. 74f7917c00SJeff Kirsher * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room. 75f7917c00SJeff Kirsher */ 76f7917c00SJeff Kirsher #define RX_PKT_SKB_LEN 512 77f7917c00SJeff Kirsher 78f7917c00SJeff Kirsher /* 79f7917c00SJeff Kirsher * Max number of Tx descriptors we clean up at a time. Should be modest as 80f7917c00SJeff Kirsher * freeing skbs isn't cheap and it happens while holding locks. We just need 81f7917c00SJeff Kirsher * to free packets faster than they arrive, we eventually catch up and keep 82f7917c00SJeff Kirsher * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. 83f7917c00SJeff Kirsher */ 84f7917c00SJeff Kirsher #define MAX_TX_RECLAIM 16 85f7917c00SJeff Kirsher 86f7917c00SJeff Kirsher /* 87f7917c00SJeff Kirsher * Max number of Rx buffers we replenish at a time. Again keep this modest, 88f7917c00SJeff Kirsher * allocating buffers isn't cheap either. 89f7917c00SJeff Kirsher */ 90f7917c00SJeff Kirsher #define MAX_RX_REFILL 16U 91f7917c00SJeff Kirsher 92f7917c00SJeff Kirsher /* 93f7917c00SJeff Kirsher * Period of the Rx queue check timer. This timer is infrequent as it has 94f7917c00SJeff Kirsher * something to do only when the system experiences severe memory shortage. 95f7917c00SJeff Kirsher */ 96f7917c00SJeff Kirsher #define RX_QCHECK_PERIOD (HZ / 2) 97f7917c00SJeff Kirsher 98f7917c00SJeff Kirsher /* 99f7917c00SJeff Kirsher * Period of the Tx queue check timer. 100f7917c00SJeff Kirsher */ 101f7917c00SJeff Kirsher #define TX_QCHECK_PERIOD (HZ / 2) 102f7917c00SJeff Kirsher 103f7917c00SJeff Kirsher /* 104f7917c00SJeff Kirsher * Max number of Tx descriptors to be reclaimed by the Tx timer. 105f7917c00SJeff Kirsher */ 106f7917c00SJeff Kirsher #define MAX_TIMER_TX_RECLAIM 100 107f7917c00SJeff Kirsher 108f7917c00SJeff Kirsher /* 109f7917c00SJeff Kirsher * Timer index used when backing off due to memory shortage. 110f7917c00SJeff Kirsher */ 111f7917c00SJeff Kirsher #define NOMEM_TMR_IDX (SGE_NTIMERS - 1) 112f7917c00SJeff Kirsher 113f7917c00SJeff Kirsher /* 114f7917c00SJeff Kirsher * Suspend an Ethernet Tx queue with fewer available descriptors than this. 115f7917c00SJeff Kirsher * This is the same as calc_tx_descs() for a TSO packet with 116f7917c00SJeff Kirsher * nr_frags == MAX_SKB_FRAGS. 117f7917c00SJeff Kirsher */ 118f7917c00SJeff Kirsher #define ETHTXQ_STOP_THRES \ 119f7917c00SJeff Kirsher (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8)) 120f7917c00SJeff Kirsher 121f7917c00SJeff Kirsher /* 122f7917c00SJeff Kirsher * Suspension threshold for non-Ethernet Tx queues. We require enough room 123f7917c00SJeff Kirsher * for a full sized WR. 124f7917c00SJeff Kirsher */ 125f7917c00SJeff Kirsher #define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc)) 126f7917c00SJeff Kirsher 127f7917c00SJeff Kirsher /* 128f7917c00SJeff Kirsher * Max Tx descriptor space we allow for an Ethernet packet to be inlined 129f7917c00SJeff Kirsher * into a WR. 130f7917c00SJeff Kirsher */ 13121dcfad6SHariprasad Shenai #define MAX_IMM_TX_PKT_LEN 256 132f7917c00SJeff Kirsher 133f7917c00SJeff Kirsher /* 134f7917c00SJeff Kirsher * Max size of a WR sent through a control Tx queue. 135f7917c00SJeff Kirsher */ 136f7917c00SJeff Kirsher #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN 137f7917c00SJeff Kirsher 138f7917c00SJeff Kirsher struct tx_sw_desc { /* SW state per Tx descriptor */ 139f7917c00SJeff Kirsher struct sk_buff *skb; 140f7917c00SJeff Kirsher struct ulptx_sgl *sgl; 141f7917c00SJeff Kirsher }; 142f7917c00SJeff Kirsher 143f7917c00SJeff Kirsher struct rx_sw_desc { /* SW state per Rx descriptor */ 144f7917c00SJeff Kirsher struct page *page; 145f7917c00SJeff Kirsher dma_addr_t dma_addr; 146f7917c00SJeff Kirsher }; 147f7917c00SJeff Kirsher 148f7917c00SJeff Kirsher /* 14952367a76SVipul Pandya * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb 15052367a76SVipul Pandya * buffer). We currently only support two sizes for 1500- and 9000-byte MTUs. 15152367a76SVipul Pandya * We could easily support more but there doesn't seem to be much need for 15252367a76SVipul Pandya * that ... 15352367a76SVipul Pandya */ 15452367a76SVipul Pandya #define FL_MTU_SMALL 1500 15552367a76SVipul Pandya #define FL_MTU_LARGE 9000 15652367a76SVipul Pandya 15752367a76SVipul Pandya static inline unsigned int fl_mtu_bufsize(struct adapter *adapter, 15852367a76SVipul Pandya unsigned int mtu) 15952367a76SVipul Pandya { 16052367a76SVipul Pandya struct sge *s = &adapter->sge; 16152367a76SVipul Pandya 16252367a76SVipul Pandya return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align); 16352367a76SVipul Pandya } 16452367a76SVipul Pandya 16552367a76SVipul Pandya #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL) 16652367a76SVipul Pandya #define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE) 16752367a76SVipul Pandya 16852367a76SVipul Pandya /* 16952367a76SVipul Pandya * Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses 17052367a76SVipul Pandya * these to specify the buffer size as an index into the SGE Free List Buffer 17152367a76SVipul Pandya * Size register array. We also use bit 4, when the buffer has been unmapped 17252367a76SVipul Pandya * for DMA, but this is of course never sent to the hardware and is only used 17352367a76SVipul Pandya * to prevent double unmappings. All of the above requires that the Free List 17452367a76SVipul Pandya * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are 17552367a76SVipul Pandya * 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal 17652367a76SVipul Pandya * Free List Buffer alignment is 32 bytes, this works out for us ... 177f7917c00SJeff Kirsher */ 178f7917c00SJeff Kirsher enum { 17952367a76SVipul Pandya RX_BUF_FLAGS = 0x1f, /* bottom five bits are special */ 18052367a76SVipul Pandya RX_BUF_SIZE = 0x0f, /* bottom three bits are for buf sizes */ 18152367a76SVipul Pandya RX_UNMAPPED_BUF = 0x10, /* buffer is not mapped */ 18252367a76SVipul Pandya 18352367a76SVipul Pandya /* 18452367a76SVipul Pandya * XXX We shouldn't depend on being able to use these indices. 18552367a76SVipul Pandya * XXX Especially when some other Master PF has initialized the 18652367a76SVipul Pandya * XXX adapter or we use the Firmware Configuration File. We 18752367a76SVipul Pandya * XXX should really search through the Host Buffer Size register 18852367a76SVipul Pandya * XXX array for the appropriately sized buffer indices. 18952367a76SVipul Pandya */ 19052367a76SVipul Pandya RX_SMALL_PG_BUF = 0x0, /* small (PAGE_SIZE) page buffer */ 19152367a76SVipul Pandya RX_LARGE_PG_BUF = 0x1, /* buffer large (FL_PG_ORDER) page buffer */ 19252367a76SVipul Pandya 19352367a76SVipul Pandya RX_SMALL_MTU_BUF = 0x2, /* small MTU buffer */ 19452367a76SVipul Pandya RX_LARGE_MTU_BUF = 0x3, /* large MTU buffer */ 195f7917c00SJeff Kirsher }; 196f7917c00SJeff Kirsher 197e553ec3fSHariprasad Shenai static int timer_pkt_quota[] = {1, 1, 2, 3, 4, 5}; 198e553ec3fSHariprasad Shenai #define MIN_NAPI_WORK 1 199e553ec3fSHariprasad Shenai 200f7917c00SJeff Kirsher static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d) 201f7917c00SJeff Kirsher { 20252367a76SVipul Pandya return d->dma_addr & ~(dma_addr_t)RX_BUF_FLAGS; 203f7917c00SJeff Kirsher } 204f7917c00SJeff Kirsher 205f7917c00SJeff Kirsher static inline bool is_buf_mapped(const struct rx_sw_desc *d) 206f7917c00SJeff Kirsher { 207f7917c00SJeff Kirsher return !(d->dma_addr & RX_UNMAPPED_BUF); 208f7917c00SJeff Kirsher } 209f7917c00SJeff Kirsher 210f7917c00SJeff Kirsher /** 211f7917c00SJeff Kirsher * txq_avail - return the number of available slots in a Tx queue 212f7917c00SJeff Kirsher * @q: the Tx queue 213f7917c00SJeff Kirsher * 214f7917c00SJeff Kirsher * Returns the number of descriptors in a Tx queue available to write new 215f7917c00SJeff Kirsher * packets. 216f7917c00SJeff Kirsher */ 217f7917c00SJeff Kirsher static inline unsigned int txq_avail(const struct sge_txq *q) 218f7917c00SJeff Kirsher { 219f7917c00SJeff Kirsher return q->size - 1 - q->in_use; 220f7917c00SJeff Kirsher } 221f7917c00SJeff Kirsher 222f7917c00SJeff Kirsher /** 223f7917c00SJeff Kirsher * fl_cap - return the capacity of a free-buffer list 224f7917c00SJeff Kirsher * @fl: the FL 225f7917c00SJeff Kirsher * 226f7917c00SJeff Kirsher * Returns the capacity of a free-buffer list. The capacity is less than 227f7917c00SJeff Kirsher * the size because one descriptor needs to be left unpopulated, otherwise 228f7917c00SJeff Kirsher * HW will think the FL is empty. 229f7917c00SJeff Kirsher */ 230f7917c00SJeff Kirsher static inline unsigned int fl_cap(const struct sge_fl *fl) 231f7917c00SJeff Kirsher { 232f7917c00SJeff Kirsher return fl->size - 8; /* 1 descriptor = 8 buffers */ 233f7917c00SJeff Kirsher } 234f7917c00SJeff Kirsher 235c098b026SHariprasad Shenai /** 236c098b026SHariprasad Shenai * fl_starving - return whether a Free List is starving. 237c098b026SHariprasad Shenai * @adapter: pointer to the adapter 238c098b026SHariprasad Shenai * @fl: the Free List 239c098b026SHariprasad Shenai * 240c098b026SHariprasad Shenai * Tests specified Free List to see whether the number of buffers 241c098b026SHariprasad Shenai * available to the hardware has falled below our "starvation" 242c098b026SHariprasad Shenai * threshold. 243c098b026SHariprasad Shenai */ 244c098b026SHariprasad Shenai static inline bool fl_starving(const struct adapter *adapter, 245c098b026SHariprasad Shenai const struct sge_fl *fl) 246f7917c00SJeff Kirsher { 247c098b026SHariprasad Shenai const struct sge *s = &adapter->sge; 248c098b026SHariprasad Shenai 249c098b026SHariprasad Shenai return fl->avail - fl->pend_cred <= s->fl_starve_thres; 250f7917c00SJeff Kirsher } 251f7917c00SJeff Kirsher 252f7917c00SJeff Kirsher static int map_skb(struct device *dev, const struct sk_buff *skb, 253f7917c00SJeff Kirsher dma_addr_t *addr) 254f7917c00SJeff Kirsher { 255f7917c00SJeff Kirsher const skb_frag_t *fp, *end; 256f7917c00SJeff Kirsher const struct skb_shared_info *si; 257f7917c00SJeff Kirsher 258f7917c00SJeff Kirsher *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 259f7917c00SJeff Kirsher if (dma_mapping_error(dev, *addr)) 260f7917c00SJeff Kirsher goto out_err; 261f7917c00SJeff Kirsher 262f7917c00SJeff Kirsher si = skb_shinfo(skb); 263f7917c00SJeff Kirsher end = &si->frags[si->nr_frags]; 264f7917c00SJeff Kirsher 265f7917c00SJeff Kirsher for (fp = si->frags; fp < end; fp++) { 266e91b0f24SIan Campbell *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp), 267e91b0f24SIan Campbell DMA_TO_DEVICE); 268f7917c00SJeff Kirsher if (dma_mapping_error(dev, *addr)) 269f7917c00SJeff Kirsher goto unwind; 270f7917c00SJeff Kirsher } 271f7917c00SJeff Kirsher return 0; 272f7917c00SJeff Kirsher 273f7917c00SJeff Kirsher unwind: 274f7917c00SJeff Kirsher while (fp-- > si->frags) 2759e903e08SEric Dumazet dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE); 276f7917c00SJeff Kirsher 277f7917c00SJeff Kirsher dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE); 278f7917c00SJeff Kirsher out_err: 279f7917c00SJeff Kirsher return -ENOMEM; 280f7917c00SJeff Kirsher } 281f7917c00SJeff Kirsher 282f7917c00SJeff Kirsher #ifdef CONFIG_NEED_DMA_MAP_STATE 283f7917c00SJeff Kirsher static void unmap_skb(struct device *dev, const struct sk_buff *skb, 284f7917c00SJeff Kirsher const dma_addr_t *addr) 285f7917c00SJeff Kirsher { 286f7917c00SJeff Kirsher const skb_frag_t *fp, *end; 287f7917c00SJeff Kirsher const struct skb_shared_info *si; 288f7917c00SJeff Kirsher 289f7917c00SJeff Kirsher dma_unmap_single(dev, *addr++, skb_headlen(skb), DMA_TO_DEVICE); 290f7917c00SJeff Kirsher 291f7917c00SJeff Kirsher si = skb_shinfo(skb); 292f7917c00SJeff Kirsher end = &si->frags[si->nr_frags]; 293f7917c00SJeff Kirsher for (fp = si->frags; fp < end; fp++) 2949e903e08SEric Dumazet dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE); 295f7917c00SJeff Kirsher } 296f7917c00SJeff Kirsher 297f7917c00SJeff Kirsher /** 298f7917c00SJeff Kirsher * deferred_unmap_destructor - unmap a packet when it is freed 299f7917c00SJeff Kirsher * @skb: the packet 300f7917c00SJeff Kirsher * 301f7917c00SJeff Kirsher * This is the packet destructor used for Tx packets that need to remain 302f7917c00SJeff Kirsher * mapped until they are freed rather than until their Tx descriptors are 303f7917c00SJeff Kirsher * freed. 304f7917c00SJeff Kirsher */ 305f7917c00SJeff Kirsher static void deferred_unmap_destructor(struct sk_buff *skb) 306f7917c00SJeff Kirsher { 307f7917c00SJeff Kirsher unmap_skb(skb->dev->dev.parent, skb, (dma_addr_t *)skb->head); 308f7917c00SJeff Kirsher } 309f7917c00SJeff Kirsher #endif 310f7917c00SJeff Kirsher 311f7917c00SJeff Kirsher static void unmap_sgl(struct device *dev, const struct sk_buff *skb, 312f7917c00SJeff Kirsher const struct ulptx_sgl *sgl, const struct sge_txq *q) 313f7917c00SJeff Kirsher { 314f7917c00SJeff Kirsher const struct ulptx_sge_pair *p; 315f7917c00SJeff Kirsher unsigned int nfrags = skb_shinfo(skb)->nr_frags; 316f7917c00SJeff Kirsher 317f7917c00SJeff Kirsher if (likely(skb_headlen(skb))) 318f7917c00SJeff Kirsher dma_unmap_single(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0), 319f7917c00SJeff Kirsher DMA_TO_DEVICE); 320f7917c00SJeff Kirsher else { 321f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0), 322f7917c00SJeff Kirsher DMA_TO_DEVICE); 323f7917c00SJeff Kirsher nfrags--; 324f7917c00SJeff Kirsher } 325f7917c00SJeff Kirsher 326f7917c00SJeff Kirsher /* 327f7917c00SJeff Kirsher * the complexity below is because of the possibility of a wrap-around 328f7917c00SJeff Kirsher * in the middle of an SGL 329f7917c00SJeff Kirsher */ 330f7917c00SJeff Kirsher for (p = sgl->sge; nfrags >= 2; nfrags -= 2) { 331f7917c00SJeff Kirsher if (likely((u8 *)(p + 1) <= (u8 *)q->stat)) { 332f7917c00SJeff Kirsher unmap: dma_unmap_page(dev, be64_to_cpu(p->addr[0]), 333f7917c00SJeff Kirsher ntohl(p->len[0]), DMA_TO_DEVICE); 334f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(p->addr[1]), 335f7917c00SJeff Kirsher ntohl(p->len[1]), DMA_TO_DEVICE); 336f7917c00SJeff Kirsher p++; 337f7917c00SJeff Kirsher } else if ((u8 *)p == (u8 *)q->stat) { 338f7917c00SJeff Kirsher p = (const struct ulptx_sge_pair *)q->desc; 339f7917c00SJeff Kirsher goto unmap; 340f7917c00SJeff Kirsher } else if ((u8 *)p + 8 == (u8 *)q->stat) { 341f7917c00SJeff Kirsher const __be64 *addr = (const __be64 *)q->desc; 342f7917c00SJeff Kirsher 343f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(addr[0]), 344f7917c00SJeff Kirsher ntohl(p->len[0]), DMA_TO_DEVICE); 345f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(addr[1]), 346f7917c00SJeff Kirsher ntohl(p->len[1]), DMA_TO_DEVICE); 347f7917c00SJeff Kirsher p = (const struct ulptx_sge_pair *)&addr[2]; 348f7917c00SJeff Kirsher } else { 349f7917c00SJeff Kirsher const __be64 *addr = (const __be64 *)q->desc; 350f7917c00SJeff Kirsher 351f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(p->addr[0]), 352f7917c00SJeff Kirsher ntohl(p->len[0]), DMA_TO_DEVICE); 353f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(addr[0]), 354f7917c00SJeff Kirsher ntohl(p->len[1]), DMA_TO_DEVICE); 355f7917c00SJeff Kirsher p = (const struct ulptx_sge_pair *)&addr[1]; 356f7917c00SJeff Kirsher } 357f7917c00SJeff Kirsher } 358f7917c00SJeff Kirsher if (nfrags) { 359f7917c00SJeff Kirsher __be64 addr; 360f7917c00SJeff Kirsher 361f7917c00SJeff Kirsher if ((u8 *)p == (u8 *)q->stat) 362f7917c00SJeff Kirsher p = (const struct ulptx_sge_pair *)q->desc; 363f7917c00SJeff Kirsher addr = (u8 *)p + 16 <= (u8 *)q->stat ? p->addr[0] : 364f7917c00SJeff Kirsher *(const __be64 *)q->desc; 365f7917c00SJeff Kirsher dma_unmap_page(dev, be64_to_cpu(addr), ntohl(p->len[0]), 366f7917c00SJeff Kirsher DMA_TO_DEVICE); 367f7917c00SJeff Kirsher } 368f7917c00SJeff Kirsher } 369f7917c00SJeff Kirsher 370f7917c00SJeff Kirsher /** 371f7917c00SJeff Kirsher * free_tx_desc - reclaims Tx descriptors and their buffers 372f7917c00SJeff Kirsher * @adapter: the adapter 373f7917c00SJeff Kirsher * @q: the Tx queue to reclaim descriptors from 374f7917c00SJeff Kirsher * @n: the number of descriptors to reclaim 375f7917c00SJeff Kirsher * @unmap: whether the buffers should be unmapped for DMA 376f7917c00SJeff Kirsher * 377f7917c00SJeff Kirsher * Reclaims Tx descriptors from an SGE Tx queue and frees the associated 378f7917c00SJeff Kirsher * Tx buffers. Called with the Tx queue lock held. 379f7917c00SJeff Kirsher */ 380f7917c00SJeff Kirsher static void free_tx_desc(struct adapter *adap, struct sge_txq *q, 381f7917c00SJeff Kirsher unsigned int n, bool unmap) 382f7917c00SJeff Kirsher { 383f7917c00SJeff Kirsher struct tx_sw_desc *d; 384f7917c00SJeff Kirsher unsigned int cidx = q->cidx; 385f7917c00SJeff Kirsher struct device *dev = adap->pdev_dev; 386f7917c00SJeff Kirsher 387f7917c00SJeff Kirsher d = &q->sdesc[cidx]; 388f7917c00SJeff Kirsher while (n--) { 389f7917c00SJeff Kirsher if (d->skb) { /* an SGL is present */ 390f7917c00SJeff Kirsher if (unmap) 391f7917c00SJeff Kirsher unmap_sgl(dev, d->skb, d->sgl, q); 392a7525198SEric W. Biederman dev_consume_skb_any(d->skb); 393f7917c00SJeff Kirsher d->skb = NULL; 394f7917c00SJeff Kirsher } 395f7917c00SJeff Kirsher ++d; 396f7917c00SJeff Kirsher if (++cidx == q->size) { 397f7917c00SJeff Kirsher cidx = 0; 398f7917c00SJeff Kirsher d = q->sdesc; 399f7917c00SJeff Kirsher } 400f7917c00SJeff Kirsher } 401f7917c00SJeff Kirsher q->cidx = cidx; 402f7917c00SJeff Kirsher } 403f7917c00SJeff Kirsher 404f7917c00SJeff Kirsher /* 405f7917c00SJeff Kirsher * Return the number of reclaimable descriptors in a Tx queue. 406f7917c00SJeff Kirsher */ 407f7917c00SJeff Kirsher static inline int reclaimable(const struct sge_txq *q) 408f7917c00SJeff Kirsher { 409f7917c00SJeff Kirsher int hw_cidx = ntohs(q->stat->cidx); 410f7917c00SJeff Kirsher hw_cidx -= q->cidx; 411f7917c00SJeff Kirsher return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx; 412f7917c00SJeff Kirsher } 413f7917c00SJeff Kirsher 414f7917c00SJeff Kirsher /** 415f7917c00SJeff Kirsher * reclaim_completed_tx - reclaims completed Tx descriptors 416f7917c00SJeff Kirsher * @adap: the adapter 417f7917c00SJeff Kirsher * @q: the Tx queue to reclaim completed descriptors from 418f7917c00SJeff Kirsher * @unmap: whether the buffers should be unmapped for DMA 419f7917c00SJeff Kirsher * 420f7917c00SJeff Kirsher * Reclaims Tx descriptors that the SGE has indicated it has processed, 421f7917c00SJeff Kirsher * and frees the associated buffers if possible. Called with the Tx 422f7917c00SJeff Kirsher * queue locked. 423f7917c00SJeff Kirsher */ 424f7917c00SJeff Kirsher static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, 425f7917c00SJeff Kirsher bool unmap) 426f7917c00SJeff Kirsher { 427f7917c00SJeff Kirsher int avail = reclaimable(q); 428f7917c00SJeff Kirsher 429f7917c00SJeff Kirsher if (avail) { 430f7917c00SJeff Kirsher /* 431f7917c00SJeff Kirsher * Limit the amount of clean up work we do at a time to keep 432f7917c00SJeff Kirsher * the Tx lock hold time O(1). 433f7917c00SJeff Kirsher */ 434f7917c00SJeff Kirsher if (avail > MAX_TX_RECLAIM) 435f7917c00SJeff Kirsher avail = MAX_TX_RECLAIM; 436f7917c00SJeff Kirsher 437f7917c00SJeff Kirsher free_tx_desc(adap, q, avail, unmap); 438f7917c00SJeff Kirsher q->in_use -= avail; 439f7917c00SJeff Kirsher } 440f7917c00SJeff Kirsher } 441f7917c00SJeff Kirsher 44252367a76SVipul Pandya static inline int get_buf_size(struct adapter *adapter, 44352367a76SVipul Pandya const struct rx_sw_desc *d) 444f7917c00SJeff Kirsher { 44552367a76SVipul Pandya struct sge *s = &adapter->sge; 44652367a76SVipul Pandya unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE; 44752367a76SVipul Pandya int buf_size; 44852367a76SVipul Pandya 44952367a76SVipul Pandya switch (rx_buf_size_idx) { 45052367a76SVipul Pandya case RX_SMALL_PG_BUF: 45152367a76SVipul Pandya buf_size = PAGE_SIZE; 45252367a76SVipul Pandya break; 45352367a76SVipul Pandya 45452367a76SVipul Pandya case RX_LARGE_PG_BUF: 45552367a76SVipul Pandya buf_size = PAGE_SIZE << s->fl_pg_order; 45652367a76SVipul Pandya break; 45752367a76SVipul Pandya 45852367a76SVipul Pandya case RX_SMALL_MTU_BUF: 45952367a76SVipul Pandya buf_size = FL_MTU_SMALL_BUFSIZE(adapter); 46052367a76SVipul Pandya break; 46152367a76SVipul Pandya 46252367a76SVipul Pandya case RX_LARGE_MTU_BUF: 46352367a76SVipul Pandya buf_size = FL_MTU_LARGE_BUFSIZE(adapter); 46452367a76SVipul Pandya break; 46552367a76SVipul Pandya 46652367a76SVipul Pandya default: 46752367a76SVipul Pandya BUG_ON(1); 46852367a76SVipul Pandya } 46952367a76SVipul Pandya 47052367a76SVipul Pandya return buf_size; 471f7917c00SJeff Kirsher } 472f7917c00SJeff Kirsher 473f7917c00SJeff Kirsher /** 474f7917c00SJeff Kirsher * free_rx_bufs - free the Rx buffers on an SGE free list 475f7917c00SJeff Kirsher * @adap: the adapter 476f7917c00SJeff Kirsher * @q: the SGE free list to free buffers from 477f7917c00SJeff Kirsher * @n: how many buffers to free 478f7917c00SJeff Kirsher * 479f7917c00SJeff Kirsher * Release the next @n buffers on an SGE free-buffer Rx queue. The 480f7917c00SJeff Kirsher * buffers must be made inaccessible to HW before calling this function. 481f7917c00SJeff Kirsher */ 482f7917c00SJeff Kirsher static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n) 483f7917c00SJeff Kirsher { 484f7917c00SJeff Kirsher while (n--) { 485f7917c00SJeff Kirsher struct rx_sw_desc *d = &q->sdesc[q->cidx]; 486f7917c00SJeff Kirsher 487f7917c00SJeff Kirsher if (is_buf_mapped(d)) 488f7917c00SJeff Kirsher dma_unmap_page(adap->pdev_dev, get_buf_addr(d), 48952367a76SVipul Pandya get_buf_size(adap, d), 49052367a76SVipul Pandya PCI_DMA_FROMDEVICE); 491f7917c00SJeff Kirsher put_page(d->page); 492f7917c00SJeff Kirsher d->page = NULL; 493f7917c00SJeff Kirsher if (++q->cidx == q->size) 494f7917c00SJeff Kirsher q->cidx = 0; 495f7917c00SJeff Kirsher q->avail--; 496f7917c00SJeff Kirsher } 497f7917c00SJeff Kirsher } 498f7917c00SJeff Kirsher 499f7917c00SJeff Kirsher /** 500f7917c00SJeff Kirsher * unmap_rx_buf - unmap the current Rx buffer on an SGE free list 501f7917c00SJeff Kirsher * @adap: the adapter 502f7917c00SJeff Kirsher * @q: the SGE free list 503f7917c00SJeff Kirsher * 504f7917c00SJeff Kirsher * Unmap the current buffer on an SGE free-buffer Rx queue. The 505f7917c00SJeff Kirsher * buffer must be made inaccessible to HW before calling this function. 506f7917c00SJeff Kirsher * 507f7917c00SJeff Kirsher * This is similar to @free_rx_bufs above but does not free the buffer. 508f7917c00SJeff Kirsher * Do note that the FL still loses any further access to the buffer. 509f7917c00SJeff Kirsher */ 510f7917c00SJeff Kirsher static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q) 511f7917c00SJeff Kirsher { 512f7917c00SJeff Kirsher struct rx_sw_desc *d = &q->sdesc[q->cidx]; 513f7917c00SJeff Kirsher 514f7917c00SJeff Kirsher if (is_buf_mapped(d)) 515f7917c00SJeff Kirsher dma_unmap_page(adap->pdev_dev, get_buf_addr(d), 51652367a76SVipul Pandya get_buf_size(adap, d), PCI_DMA_FROMDEVICE); 517f7917c00SJeff Kirsher d->page = NULL; 518f7917c00SJeff Kirsher if (++q->cidx == q->size) 519f7917c00SJeff Kirsher q->cidx = 0; 520f7917c00SJeff Kirsher q->avail--; 521f7917c00SJeff Kirsher } 522f7917c00SJeff Kirsher 523f7917c00SJeff Kirsher static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) 524f7917c00SJeff Kirsher { 525f7917c00SJeff Kirsher if (q->pend_cred >= 8) { 5263ccc6cf7SHariprasad Shenai u32 val = adap->params.arch.sge_fl_db; 5273ccc6cf7SHariprasad Shenai 528f612b815SHariprasad Shenai if (is_t4(adap->params.chip)) 5293ccc6cf7SHariprasad Shenai val |= PIDX_V(q->pend_cred / 8); 530f612b815SHariprasad Shenai else 5313ccc6cf7SHariprasad Shenai val |= PIDX_T5_V(q->pend_cred / 8); 5321ecc7b7aSHariprasad Shenai 5331ecc7b7aSHariprasad Shenai /* Make sure all memory writes to the Free List queue are 5341ecc7b7aSHariprasad Shenai * committed before we tell the hardware about them. 5351ecc7b7aSHariprasad Shenai */ 536f7917c00SJeff Kirsher wmb(); 537d63a6dcfSHariprasad Shenai 538df64e4d3SHariprasad Shenai /* If we don't have access to the new User Doorbell (T5+), use 539df64e4d3SHariprasad Shenai * the old doorbell mechanism; otherwise use the new BAR2 540df64e4d3SHariprasad Shenai * mechanism. 541d63a6dcfSHariprasad Shenai */ 542df64e4d3SHariprasad Shenai if (unlikely(q->bar2_addr == NULL)) { 543f612b815SHariprasad Shenai t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 544f612b815SHariprasad Shenai val | QID_V(q->cntxt_id)); 545d63a6dcfSHariprasad Shenai } else { 546f612b815SHariprasad Shenai writel(val | QID_V(q->bar2_qid), 547df64e4d3SHariprasad Shenai q->bar2_addr + SGE_UDB_KDOORBELL); 548d63a6dcfSHariprasad Shenai 549d63a6dcfSHariprasad Shenai /* This Write memory Barrier will force the write to 550d63a6dcfSHariprasad Shenai * the User Doorbell area to be flushed. 551d63a6dcfSHariprasad Shenai */ 552d63a6dcfSHariprasad Shenai wmb(); 553d63a6dcfSHariprasad Shenai } 554f7917c00SJeff Kirsher q->pend_cred &= 7; 555f7917c00SJeff Kirsher } 556f7917c00SJeff Kirsher } 557f7917c00SJeff Kirsher 558f7917c00SJeff Kirsher static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg, 559f7917c00SJeff Kirsher dma_addr_t mapping) 560f7917c00SJeff Kirsher { 561f7917c00SJeff Kirsher sd->page = pg; 562f7917c00SJeff Kirsher sd->dma_addr = mapping; /* includes size low bits */ 563f7917c00SJeff Kirsher } 564f7917c00SJeff Kirsher 565f7917c00SJeff Kirsher /** 566f7917c00SJeff Kirsher * refill_fl - refill an SGE Rx buffer ring 567f7917c00SJeff Kirsher * @adap: the adapter 568f7917c00SJeff Kirsher * @q: the ring to refill 569f7917c00SJeff Kirsher * @n: the number of new buffers to allocate 570f7917c00SJeff Kirsher * @gfp: the gfp flags for the allocations 571f7917c00SJeff Kirsher * 572f7917c00SJeff Kirsher * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, 573f7917c00SJeff Kirsher * allocated with the supplied gfp flags. The caller must assure that 574f7917c00SJeff Kirsher * @n does not exceed the queue's capacity. If afterwards the queue is 575f7917c00SJeff Kirsher * found critically low mark it as starving in the bitmap of starving FLs. 576f7917c00SJeff Kirsher * 577f7917c00SJeff Kirsher * Returns the number of buffers allocated. 578f7917c00SJeff Kirsher */ 579f7917c00SJeff Kirsher static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n, 580f7917c00SJeff Kirsher gfp_t gfp) 581f7917c00SJeff Kirsher { 58252367a76SVipul Pandya struct sge *s = &adap->sge; 583f7917c00SJeff Kirsher struct page *pg; 584f7917c00SJeff Kirsher dma_addr_t mapping; 585f7917c00SJeff Kirsher unsigned int cred = q->avail; 586f7917c00SJeff Kirsher __be64 *d = &q->desc[q->pidx]; 587f7917c00SJeff Kirsher struct rx_sw_desc *sd = &q->sdesc[q->pidx]; 588d52ce920SHariprasad Shenai int node; 589f7917c00SJeff Kirsher 5905b377d11SHariprasad Shenai #ifdef CONFIG_DEBUG_FS 5915b377d11SHariprasad Shenai if (test_bit(q->cntxt_id - adap->sge.egr_start, adap->sge.blocked_fl)) 5925b377d11SHariprasad Shenai goto out; 5935b377d11SHariprasad Shenai #endif 5945b377d11SHariprasad Shenai 595aa9cd31cSAlexander Duyck gfp |= __GFP_NOWARN; 596d52ce920SHariprasad Shenai node = dev_to_node(adap->pdev_dev); 597f7917c00SJeff Kirsher 59852367a76SVipul Pandya if (s->fl_pg_order == 0) 59952367a76SVipul Pandya goto alloc_small_pages; 60052367a76SVipul Pandya 601f7917c00SJeff Kirsher /* 602f7917c00SJeff Kirsher * Prefer large buffers 603f7917c00SJeff Kirsher */ 604f7917c00SJeff Kirsher while (n) { 605d52ce920SHariprasad Shenai pg = alloc_pages_node(node, gfp | __GFP_COMP, s->fl_pg_order); 606f7917c00SJeff Kirsher if (unlikely(!pg)) { 607f7917c00SJeff Kirsher q->large_alloc_failed++; 608f7917c00SJeff Kirsher break; /* fall back to single pages */ 609f7917c00SJeff Kirsher } 610f7917c00SJeff Kirsher 611f7917c00SJeff Kirsher mapping = dma_map_page(adap->pdev_dev, pg, 0, 61252367a76SVipul Pandya PAGE_SIZE << s->fl_pg_order, 613f7917c00SJeff Kirsher PCI_DMA_FROMDEVICE); 614f7917c00SJeff Kirsher if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) { 61552367a76SVipul Pandya __free_pages(pg, s->fl_pg_order); 616f7917c00SJeff Kirsher goto out; /* do not try small pages for this error */ 617f7917c00SJeff Kirsher } 61852367a76SVipul Pandya mapping |= RX_LARGE_PG_BUF; 619f7917c00SJeff Kirsher *d++ = cpu_to_be64(mapping); 620f7917c00SJeff Kirsher 621f7917c00SJeff Kirsher set_rx_sw_desc(sd, pg, mapping); 622f7917c00SJeff Kirsher sd++; 623f7917c00SJeff Kirsher 624f7917c00SJeff Kirsher q->avail++; 625f7917c00SJeff Kirsher if (++q->pidx == q->size) { 626f7917c00SJeff Kirsher q->pidx = 0; 627f7917c00SJeff Kirsher sd = q->sdesc; 628f7917c00SJeff Kirsher d = q->desc; 629f7917c00SJeff Kirsher } 630f7917c00SJeff Kirsher n--; 631f7917c00SJeff Kirsher } 632f7917c00SJeff Kirsher 63352367a76SVipul Pandya alloc_small_pages: 634f7917c00SJeff Kirsher while (n--) { 635d52ce920SHariprasad Shenai pg = alloc_pages_node(node, gfp, 0); 636f7917c00SJeff Kirsher if (unlikely(!pg)) { 637f7917c00SJeff Kirsher q->alloc_failed++; 638f7917c00SJeff Kirsher break; 639f7917c00SJeff Kirsher } 640f7917c00SJeff Kirsher 641f7917c00SJeff Kirsher mapping = dma_map_page(adap->pdev_dev, pg, 0, PAGE_SIZE, 642f7917c00SJeff Kirsher PCI_DMA_FROMDEVICE); 643f7917c00SJeff Kirsher if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) { 6441f2149c1SEric Dumazet put_page(pg); 645f7917c00SJeff Kirsher goto out; 646f7917c00SJeff Kirsher } 647f7917c00SJeff Kirsher *d++ = cpu_to_be64(mapping); 648f7917c00SJeff Kirsher 649f7917c00SJeff Kirsher set_rx_sw_desc(sd, pg, mapping); 650f7917c00SJeff Kirsher sd++; 651f7917c00SJeff Kirsher 652f7917c00SJeff Kirsher q->avail++; 653f7917c00SJeff Kirsher if (++q->pidx == q->size) { 654f7917c00SJeff Kirsher q->pidx = 0; 655f7917c00SJeff Kirsher sd = q->sdesc; 656f7917c00SJeff Kirsher d = q->desc; 657f7917c00SJeff Kirsher } 658f7917c00SJeff Kirsher } 659f7917c00SJeff Kirsher 660f7917c00SJeff Kirsher out: cred = q->avail - cred; 661f7917c00SJeff Kirsher q->pend_cred += cred; 662f7917c00SJeff Kirsher ring_fl_db(adap, q); 663f7917c00SJeff Kirsher 664c098b026SHariprasad Shenai if (unlikely(fl_starving(adap, q))) { 665f7917c00SJeff Kirsher smp_wmb(); 666f7917c00SJeff Kirsher set_bit(q->cntxt_id - adap->sge.egr_start, 667f7917c00SJeff Kirsher adap->sge.starving_fl); 668f7917c00SJeff Kirsher } 669f7917c00SJeff Kirsher 670f7917c00SJeff Kirsher return cred; 671f7917c00SJeff Kirsher } 672f7917c00SJeff Kirsher 673f7917c00SJeff Kirsher static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl) 674f7917c00SJeff Kirsher { 675f7917c00SJeff Kirsher refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail), 676f7917c00SJeff Kirsher GFP_ATOMIC); 677f7917c00SJeff Kirsher } 678f7917c00SJeff Kirsher 679f7917c00SJeff Kirsher /** 680f7917c00SJeff Kirsher * alloc_ring - allocate resources for an SGE descriptor ring 681f7917c00SJeff Kirsher * @dev: the PCI device's core device 682f7917c00SJeff Kirsher * @nelem: the number of descriptors 683f7917c00SJeff Kirsher * @elem_size: the size of each descriptor 684f7917c00SJeff Kirsher * @sw_size: the size of the SW state associated with each ring element 685f7917c00SJeff Kirsher * @phys: the physical address of the allocated ring 686f7917c00SJeff Kirsher * @metadata: address of the array holding the SW state for the ring 687f7917c00SJeff Kirsher * @stat_size: extra space in HW ring for status information 688f7917c00SJeff Kirsher * @node: preferred node for memory allocations 689f7917c00SJeff Kirsher * 690f7917c00SJeff Kirsher * Allocates resources for an SGE descriptor ring, such as Tx queues, 691f7917c00SJeff Kirsher * free buffer lists, or response queues. Each SGE ring requires 692f7917c00SJeff Kirsher * space for its HW descriptors plus, optionally, space for the SW state 693f7917c00SJeff Kirsher * associated with each HW entry (the metadata). The function returns 694f7917c00SJeff Kirsher * three values: the virtual address for the HW ring (the return value 695f7917c00SJeff Kirsher * of the function), the bus address of the HW ring, and the address 696f7917c00SJeff Kirsher * of the SW ring. 697f7917c00SJeff Kirsher */ 698f7917c00SJeff Kirsher static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size, 699f7917c00SJeff Kirsher size_t sw_size, dma_addr_t *phys, void *metadata, 700f7917c00SJeff Kirsher size_t stat_size, int node) 701f7917c00SJeff Kirsher { 702f7917c00SJeff Kirsher size_t len = nelem * elem_size + stat_size; 703f7917c00SJeff Kirsher void *s = NULL; 704f7917c00SJeff Kirsher void *p = dma_alloc_coherent(dev, len, phys, GFP_KERNEL); 705f7917c00SJeff Kirsher 706f7917c00SJeff Kirsher if (!p) 707f7917c00SJeff Kirsher return NULL; 708f7917c00SJeff Kirsher if (sw_size) { 709f7917c00SJeff Kirsher s = kzalloc_node(nelem * sw_size, GFP_KERNEL, node); 710f7917c00SJeff Kirsher 711f7917c00SJeff Kirsher if (!s) { 712f7917c00SJeff Kirsher dma_free_coherent(dev, len, p, *phys); 713f7917c00SJeff Kirsher return NULL; 714f7917c00SJeff Kirsher } 715f7917c00SJeff Kirsher } 716f7917c00SJeff Kirsher if (metadata) 717f7917c00SJeff Kirsher *(void **)metadata = s; 718f7917c00SJeff Kirsher memset(p, 0, len); 719f7917c00SJeff Kirsher return p; 720f7917c00SJeff Kirsher } 721f7917c00SJeff Kirsher 722f7917c00SJeff Kirsher /** 723f7917c00SJeff Kirsher * sgl_len - calculates the size of an SGL of the given capacity 724f7917c00SJeff Kirsher * @n: the number of SGL entries 725f7917c00SJeff Kirsher * 726f7917c00SJeff Kirsher * Calculates the number of flits needed for a scatter/gather list that 727f7917c00SJeff Kirsher * can hold the given number of entries. 728f7917c00SJeff Kirsher */ 729f7917c00SJeff Kirsher static inline unsigned int sgl_len(unsigned int n) 730f7917c00SJeff Kirsher { 7310aac3f56SHariprasad Shenai /* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA 7320aac3f56SHariprasad Shenai * addresses. The DSGL Work Request starts off with a 32-bit DSGL 7330aac3f56SHariprasad Shenai * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N, 7340aac3f56SHariprasad Shenai * repeated sequences of { Length[i], Length[i+1], Address[i], 7350aac3f56SHariprasad Shenai * Address[i+1] } (this ensures that all addresses are on 64-bit 7360aac3f56SHariprasad Shenai * boundaries). If N is even, then Length[N+1] should be set to 0 and 7370aac3f56SHariprasad Shenai * Address[N+1] is omitted. 7380aac3f56SHariprasad Shenai * 7390aac3f56SHariprasad Shenai * The following calculation incorporates all of the above. It's 7400aac3f56SHariprasad Shenai * somewhat hard to follow but, briefly: the "+2" accounts for the 7410aac3f56SHariprasad Shenai * first two flits which include the DSGL header, Length0 and 7420aac3f56SHariprasad Shenai * Address0; the "(3*(n-1))/2" covers the main body of list entries (3 7430aac3f56SHariprasad Shenai * flits for every pair of the remaining N) +1 if (n-1) is odd; and 7440aac3f56SHariprasad Shenai * finally the "+((n-1)&1)" adds the one remaining flit needed if 7450aac3f56SHariprasad Shenai * (n-1) is odd ... 7460aac3f56SHariprasad Shenai */ 747f7917c00SJeff Kirsher n--; 748f7917c00SJeff Kirsher return (3 * n) / 2 + (n & 1) + 2; 749f7917c00SJeff Kirsher } 750f7917c00SJeff Kirsher 751f7917c00SJeff Kirsher /** 752f7917c00SJeff Kirsher * flits_to_desc - returns the num of Tx descriptors for the given flits 753f7917c00SJeff Kirsher * @n: the number of flits 754f7917c00SJeff Kirsher * 755f7917c00SJeff Kirsher * Returns the number of Tx descriptors needed for the supplied number 756f7917c00SJeff Kirsher * of flits. 757f7917c00SJeff Kirsher */ 758f7917c00SJeff Kirsher static inline unsigned int flits_to_desc(unsigned int n) 759f7917c00SJeff Kirsher { 760f7917c00SJeff Kirsher BUG_ON(n > SGE_MAX_WR_LEN / 8); 761f7917c00SJeff Kirsher return DIV_ROUND_UP(n, 8); 762f7917c00SJeff Kirsher } 763f7917c00SJeff Kirsher 764f7917c00SJeff Kirsher /** 765f7917c00SJeff Kirsher * is_eth_imm - can an Ethernet packet be sent as immediate data? 766f7917c00SJeff Kirsher * @skb: the packet 767f7917c00SJeff Kirsher * 768f7917c00SJeff Kirsher * Returns whether an Ethernet packet is small enough to fit as 7690034b298SKumar Sanghvi * immediate data. Return value corresponds to headroom required. 770f7917c00SJeff Kirsher */ 771f7917c00SJeff Kirsher static inline int is_eth_imm(const struct sk_buff *skb) 772f7917c00SJeff Kirsher { 7730034b298SKumar Sanghvi int hdrlen = skb_shinfo(skb)->gso_size ? 7740034b298SKumar Sanghvi sizeof(struct cpl_tx_pkt_lso_core) : 0; 7750034b298SKumar Sanghvi 7760034b298SKumar Sanghvi hdrlen += sizeof(struct cpl_tx_pkt); 7770034b298SKumar Sanghvi if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) 7780034b298SKumar Sanghvi return hdrlen; 7790034b298SKumar Sanghvi return 0; 780f7917c00SJeff Kirsher } 781f7917c00SJeff Kirsher 782f7917c00SJeff Kirsher /** 783f7917c00SJeff Kirsher * calc_tx_flits - calculate the number of flits for a packet Tx WR 784f7917c00SJeff Kirsher * @skb: the packet 785f7917c00SJeff Kirsher * 786f7917c00SJeff Kirsher * Returns the number of flits needed for a Tx WR for the given Ethernet 787f7917c00SJeff Kirsher * packet, including the needed WR and CPL headers. 788f7917c00SJeff Kirsher */ 789f7917c00SJeff Kirsher static inline unsigned int calc_tx_flits(const struct sk_buff *skb) 790f7917c00SJeff Kirsher { 791f7917c00SJeff Kirsher unsigned int flits; 7920034b298SKumar Sanghvi int hdrlen = is_eth_imm(skb); 793f7917c00SJeff Kirsher 7940aac3f56SHariprasad Shenai /* If the skb is small enough, we can pump it out as a work request 7950aac3f56SHariprasad Shenai * with only immediate data. In that case we just have to have the 7960aac3f56SHariprasad Shenai * TX Packet header plus the skb data in the Work Request. 7970aac3f56SHariprasad Shenai */ 7980aac3f56SHariprasad Shenai 7990034b298SKumar Sanghvi if (hdrlen) 8000034b298SKumar Sanghvi return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64)); 801f7917c00SJeff Kirsher 8020aac3f56SHariprasad Shenai /* Otherwise, we're going to have to construct a Scatter gather list 8030aac3f56SHariprasad Shenai * of the skb body and fragments. We also include the flits necessary 8040aac3f56SHariprasad Shenai * for the TX Packet Work Request and CPL. We always have a firmware 8050aac3f56SHariprasad Shenai * Write Header (incorporated as part of the cpl_tx_pkt_lso and 8060aac3f56SHariprasad Shenai * cpl_tx_pkt structures), followed by either a TX Packet Write CPL 8070aac3f56SHariprasad Shenai * message or, if we're doing a Large Send Offload, an LSO CPL message 8080aac3f56SHariprasad Shenai * with an embedded TX Packet Write CPL message. 8090aac3f56SHariprasad Shenai */ 810f7917c00SJeff Kirsher flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 4; 811f7917c00SJeff Kirsher if (skb_shinfo(skb)->gso_size) 8120aac3f56SHariprasad Shenai flits += (sizeof(struct fw_eth_tx_pkt_wr) + 8130aac3f56SHariprasad Shenai sizeof(struct cpl_tx_pkt_lso_core) + 8140aac3f56SHariprasad Shenai sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); 8150aac3f56SHariprasad Shenai else 8160aac3f56SHariprasad Shenai flits += (sizeof(struct fw_eth_tx_pkt_wr) + 8170aac3f56SHariprasad Shenai sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); 818f7917c00SJeff Kirsher return flits; 819f7917c00SJeff Kirsher } 820f7917c00SJeff Kirsher 821f7917c00SJeff Kirsher /** 822f7917c00SJeff Kirsher * calc_tx_descs - calculate the number of Tx descriptors for a packet 823f7917c00SJeff Kirsher * @skb: the packet 824f7917c00SJeff Kirsher * 825f7917c00SJeff Kirsher * Returns the number of Tx descriptors needed for the given Ethernet 826f7917c00SJeff Kirsher * packet, including the needed WR and CPL headers. 827f7917c00SJeff Kirsher */ 828f7917c00SJeff Kirsher static inline unsigned int calc_tx_descs(const struct sk_buff *skb) 829f7917c00SJeff Kirsher { 830f7917c00SJeff Kirsher return flits_to_desc(calc_tx_flits(skb)); 831f7917c00SJeff Kirsher } 832f7917c00SJeff Kirsher 833f7917c00SJeff Kirsher /** 834f7917c00SJeff Kirsher * write_sgl - populate a scatter/gather list for a packet 835f7917c00SJeff Kirsher * @skb: the packet 836f7917c00SJeff Kirsher * @q: the Tx queue we are writing into 837f7917c00SJeff Kirsher * @sgl: starting location for writing the SGL 838f7917c00SJeff Kirsher * @end: points right after the end of the SGL 839f7917c00SJeff Kirsher * @start: start offset into skb main-body data to include in the SGL 840f7917c00SJeff Kirsher * @addr: the list of bus addresses for the SGL elements 841f7917c00SJeff Kirsher * 842f7917c00SJeff Kirsher * Generates a gather list for the buffers that make up a packet. 843f7917c00SJeff Kirsher * The caller must provide adequate space for the SGL that will be written. 844f7917c00SJeff Kirsher * The SGL includes all of the packet's page fragments and the data in its 845f7917c00SJeff Kirsher * main body except for the first @start bytes. @sgl must be 16-byte 846f7917c00SJeff Kirsher * aligned and within a Tx descriptor with available space. @end points 847f7917c00SJeff Kirsher * right after the end of the SGL but does not account for any potential 848f7917c00SJeff Kirsher * wrap around, i.e., @end > @sgl. 849f7917c00SJeff Kirsher */ 850f7917c00SJeff Kirsher static void write_sgl(const struct sk_buff *skb, struct sge_txq *q, 851f7917c00SJeff Kirsher struct ulptx_sgl *sgl, u64 *end, unsigned int start, 852f7917c00SJeff Kirsher const dma_addr_t *addr) 853f7917c00SJeff Kirsher { 854f7917c00SJeff Kirsher unsigned int i, len; 855f7917c00SJeff Kirsher struct ulptx_sge_pair *to; 856f7917c00SJeff Kirsher const struct skb_shared_info *si = skb_shinfo(skb); 857f7917c00SJeff Kirsher unsigned int nfrags = si->nr_frags; 858f7917c00SJeff Kirsher struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1]; 859f7917c00SJeff Kirsher 860f7917c00SJeff Kirsher len = skb_headlen(skb) - start; 861f7917c00SJeff Kirsher if (likely(len)) { 862f7917c00SJeff Kirsher sgl->len0 = htonl(len); 863f7917c00SJeff Kirsher sgl->addr0 = cpu_to_be64(addr[0] + start); 864f7917c00SJeff Kirsher nfrags++; 865f7917c00SJeff Kirsher } else { 8669e903e08SEric Dumazet sgl->len0 = htonl(skb_frag_size(&si->frags[0])); 867f7917c00SJeff Kirsher sgl->addr0 = cpu_to_be64(addr[1]); 868f7917c00SJeff Kirsher } 869f7917c00SJeff Kirsher 870bdc590b9SHariprasad Shenai sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | 871bdc590b9SHariprasad Shenai ULPTX_NSGE_V(nfrags)); 872f7917c00SJeff Kirsher if (likely(--nfrags == 0)) 873f7917c00SJeff Kirsher return; 874f7917c00SJeff Kirsher /* 875f7917c00SJeff Kirsher * Most of the complexity below deals with the possibility we hit the 876f7917c00SJeff Kirsher * end of the queue in the middle of writing the SGL. For this case 877f7917c00SJeff Kirsher * only we create the SGL in a temporary buffer and then copy it. 878f7917c00SJeff Kirsher */ 879f7917c00SJeff Kirsher to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge; 880f7917c00SJeff Kirsher 881f7917c00SJeff Kirsher for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) { 8829e903e08SEric Dumazet to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); 8839e903e08SEric Dumazet to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i])); 884f7917c00SJeff Kirsher to->addr[0] = cpu_to_be64(addr[i]); 885f7917c00SJeff Kirsher to->addr[1] = cpu_to_be64(addr[++i]); 886f7917c00SJeff Kirsher } 887f7917c00SJeff Kirsher if (nfrags) { 8889e903e08SEric Dumazet to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); 889f7917c00SJeff Kirsher to->len[1] = cpu_to_be32(0); 890f7917c00SJeff Kirsher to->addr[0] = cpu_to_be64(addr[i + 1]); 891f7917c00SJeff Kirsher } 892f7917c00SJeff Kirsher if (unlikely((u8 *)end > (u8 *)q->stat)) { 893f7917c00SJeff Kirsher unsigned int part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1; 894f7917c00SJeff Kirsher 895f7917c00SJeff Kirsher if (likely(part0)) 896f7917c00SJeff Kirsher memcpy(sgl->sge, buf, part0); 897f7917c00SJeff Kirsher part1 = (u8 *)end - (u8 *)q->stat; 898f7917c00SJeff Kirsher memcpy(q->desc, (u8 *)buf + part0, part1); 899f7917c00SJeff Kirsher end = (void *)q->desc + part1; 900f7917c00SJeff Kirsher } 901f7917c00SJeff Kirsher if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */ 90264699336SJoe Perches *end = 0; 903f7917c00SJeff Kirsher } 904f7917c00SJeff Kirsher 905df64e4d3SHariprasad Shenai /* This function copies 64 byte coalesced work request to 906df64e4d3SHariprasad Shenai * memory mapped BAR2 space. For coalesced WR SGE fetches 907df64e4d3SHariprasad Shenai * data from the FIFO instead of from Host. 90822adfe0aSSantosh Rastapur */ 909df64e4d3SHariprasad Shenai static void cxgb_pio_copy(u64 __iomem *dst, u64 *src) 91022adfe0aSSantosh Rastapur { 911df64e4d3SHariprasad Shenai int count = 8; 91222adfe0aSSantosh Rastapur 91322adfe0aSSantosh Rastapur while (count) { 91422adfe0aSSantosh Rastapur writeq(*src, dst); 91522adfe0aSSantosh Rastapur src++; 91622adfe0aSSantosh Rastapur dst++; 91722adfe0aSSantosh Rastapur count--; 91822adfe0aSSantosh Rastapur } 91922adfe0aSSantosh Rastapur } 92022adfe0aSSantosh Rastapur 921f7917c00SJeff Kirsher /** 922f7917c00SJeff Kirsher * ring_tx_db - check and potentially ring a Tx queue's doorbell 923f7917c00SJeff Kirsher * @adap: the adapter 924f7917c00SJeff Kirsher * @q: the Tx queue 925f7917c00SJeff Kirsher * @n: number of new descriptors to give to HW 926f7917c00SJeff Kirsher * 927f7917c00SJeff Kirsher * Ring the doorbel for a Tx queue. 928f7917c00SJeff Kirsher */ 929f7917c00SJeff Kirsher static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) 930f7917c00SJeff Kirsher { 9311ecc7b7aSHariprasad Shenai /* Make sure that all writes to the TX Descriptors are committed 9321ecc7b7aSHariprasad Shenai * before we tell the hardware about them. 9331ecc7b7aSHariprasad Shenai */ 9341ecc7b7aSHariprasad Shenai wmb(); 935d63a6dcfSHariprasad Shenai 936df64e4d3SHariprasad Shenai /* If we don't have access to the new User Doorbell (T5+), use the old 937df64e4d3SHariprasad Shenai * doorbell mechanism; otherwise use the new BAR2 mechanism. 938df64e4d3SHariprasad Shenai */ 939df64e4d3SHariprasad Shenai if (unlikely(q->bar2_addr == NULL)) { 940f612b815SHariprasad Shenai u32 val = PIDX_V(n); 94105eb2389SSteve Wise unsigned long flags; 94222adfe0aSSantosh Rastapur 943d63a6dcfSHariprasad Shenai /* For T4 we need to participate in the Doorbell Recovery 944d63a6dcfSHariprasad Shenai * mechanism. 945d63a6dcfSHariprasad Shenai */ 94605eb2389SSteve Wise spin_lock_irqsave(&q->db_lock, flags); 947d63a6dcfSHariprasad Shenai if (!q->db_disabled) 948f612b815SHariprasad Shenai t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 949f612b815SHariprasad Shenai QID_V(q->cntxt_id) | val); 950d63a6dcfSHariprasad Shenai else 95105eb2389SSteve Wise q->db_pidx_inc += n; 9523069ee9bSVipul Pandya q->db_pidx = q->pidx; 95305eb2389SSteve Wise spin_unlock_irqrestore(&q->db_lock, flags); 954d63a6dcfSHariprasad Shenai } else { 955f612b815SHariprasad Shenai u32 val = PIDX_T5_V(n); 956d63a6dcfSHariprasad Shenai 957d63a6dcfSHariprasad Shenai /* T4 and later chips share the same PIDX field offset within 958d63a6dcfSHariprasad Shenai * the doorbell, but T5 and later shrank the field in order to 959d63a6dcfSHariprasad Shenai * gain a bit for Doorbell Priority. The field was absurdly 960d63a6dcfSHariprasad Shenai * large in the first place (14 bits) so we just use the T5 961d63a6dcfSHariprasad Shenai * and later limits and warn if a Queue ID is too large. 962d63a6dcfSHariprasad Shenai */ 963f612b815SHariprasad Shenai WARN_ON(val & DBPRIO_F); 964d63a6dcfSHariprasad Shenai 965df64e4d3SHariprasad Shenai /* If we're only writing a single TX Descriptor and we can use 966df64e4d3SHariprasad Shenai * Inferred QID registers, we can use the Write Combining 967df64e4d3SHariprasad Shenai * Gather Buffer; otherwise we use the simple doorbell. 968d63a6dcfSHariprasad Shenai */ 969df64e4d3SHariprasad Shenai if (n == 1 && q->bar2_qid == 0) { 970d63a6dcfSHariprasad Shenai int index = (q->pidx 971d63a6dcfSHariprasad Shenai ? (q->pidx - 1) 972d63a6dcfSHariprasad Shenai : (q->size - 1)); 973df64e4d3SHariprasad Shenai u64 *wr = (u64 *)&q->desc[index]; 974d63a6dcfSHariprasad Shenai 975df64e4d3SHariprasad Shenai cxgb_pio_copy((u64 __iomem *) 976df64e4d3SHariprasad Shenai (q->bar2_addr + SGE_UDB_WCDOORBELL), 977df64e4d3SHariprasad Shenai wr); 978d63a6dcfSHariprasad Shenai } else { 979f612b815SHariprasad Shenai writel(val | QID_V(q->bar2_qid), 980df64e4d3SHariprasad Shenai q->bar2_addr + SGE_UDB_KDOORBELL); 981d63a6dcfSHariprasad Shenai } 982d63a6dcfSHariprasad Shenai 983d63a6dcfSHariprasad Shenai /* This Write Memory Barrier will force the write to the User 984d63a6dcfSHariprasad Shenai * Doorbell area to be flushed. This is needed to prevent 985d63a6dcfSHariprasad Shenai * writes on different CPUs for the same queue from hitting 986d63a6dcfSHariprasad Shenai * the adapter out of order. This is required when some Work 987d63a6dcfSHariprasad Shenai * Requests take the Write Combine Gather Buffer path (user 988d63a6dcfSHariprasad Shenai * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some 989d63a6dcfSHariprasad Shenai * take the traditional path where we simply increment the 990d63a6dcfSHariprasad Shenai * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the 991d63a6dcfSHariprasad Shenai * hardware DMA read the actual Work Request. 992d63a6dcfSHariprasad Shenai */ 993d63a6dcfSHariprasad Shenai wmb(); 994d63a6dcfSHariprasad Shenai } 995f7917c00SJeff Kirsher } 996f7917c00SJeff Kirsher 997f7917c00SJeff Kirsher /** 998f7917c00SJeff Kirsher * inline_tx_skb - inline a packet's data into Tx descriptors 999f7917c00SJeff Kirsher * @skb: the packet 1000f7917c00SJeff Kirsher * @q: the Tx queue where the packet will be inlined 1001f7917c00SJeff Kirsher * @pos: starting position in the Tx queue where to inline the packet 1002f7917c00SJeff Kirsher * 1003f7917c00SJeff Kirsher * Inline a packet's contents directly into Tx descriptors, starting at 1004f7917c00SJeff Kirsher * the given position within the Tx DMA ring. 1005f7917c00SJeff Kirsher * Most of the complexity of this operation is dealing with wrap arounds 1006f7917c00SJeff Kirsher * in the middle of the packet we want to inline. 1007f7917c00SJeff Kirsher */ 1008f7917c00SJeff Kirsher static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, 1009f7917c00SJeff Kirsher void *pos) 1010f7917c00SJeff Kirsher { 1011f7917c00SJeff Kirsher u64 *p; 1012f7917c00SJeff Kirsher int left = (void *)q->stat - pos; 1013f7917c00SJeff Kirsher 1014f7917c00SJeff Kirsher if (likely(skb->len <= left)) { 1015f7917c00SJeff Kirsher if (likely(!skb->data_len)) 1016f7917c00SJeff Kirsher skb_copy_from_linear_data(skb, pos, skb->len); 1017f7917c00SJeff Kirsher else 1018f7917c00SJeff Kirsher skb_copy_bits(skb, 0, pos, skb->len); 1019f7917c00SJeff Kirsher pos += skb->len; 1020f7917c00SJeff Kirsher } else { 1021f7917c00SJeff Kirsher skb_copy_bits(skb, 0, pos, left); 1022f7917c00SJeff Kirsher skb_copy_bits(skb, left, q->desc, skb->len - left); 1023f7917c00SJeff Kirsher pos = (void *)q->desc + (skb->len - left); 1024f7917c00SJeff Kirsher } 1025f7917c00SJeff Kirsher 1026f7917c00SJeff Kirsher /* 0-pad to multiple of 16 */ 1027f7917c00SJeff Kirsher p = PTR_ALIGN(pos, 8); 1028f7917c00SJeff Kirsher if ((uintptr_t)p & 8) 1029f7917c00SJeff Kirsher *p = 0; 1030f7917c00SJeff Kirsher } 1031f7917c00SJeff Kirsher 1032f7917c00SJeff Kirsher /* 1033f7917c00SJeff Kirsher * Figure out what HW csum a packet wants and return the appropriate control 1034f7917c00SJeff Kirsher * bits. 1035f7917c00SJeff Kirsher */ 10363ccc6cf7SHariprasad Shenai static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb) 1037f7917c00SJeff Kirsher { 1038f7917c00SJeff Kirsher int csum_type; 1039f7917c00SJeff Kirsher const struct iphdr *iph = ip_hdr(skb); 1040f7917c00SJeff Kirsher 1041f7917c00SJeff Kirsher if (iph->version == 4) { 1042f7917c00SJeff Kirsher if (iph->protocol == IPPROTO_TCP) 1043f7917c00SJeff Kirsher csum_type = TX_CSUM_TCPIP; 1044f7917c00SJeff Kirsher else if (iph->protocol == IPPROTO_UDP) 1045f7917c00SJeff Kirsher csum_type = TX_CSUM_UDPIP; 1046f7917c00SJeff Kirsher else { 1047f7917c00SJeff Kirsher nocsum: /* 1048f7917c00SJeff Kirsher * unknown protocol, disable HW csum 1049f7917c00SJeff Kirsher * and hope a bad packet is detected 1050f7917c00SJeff Kirsher */ 10511ecc7b7aSHariprasad Shenai return TXPKT_L4CSUM_DIS_F; 1052f7917c00SJeff Kirsher } 1053f7917c00SJeff Kirsher } else { 1054f7917c00SJeff Kirsher /* 1055f7917c00SJeff Kirsher * this doesn't work with extension headers 1056f7917c00SJeff Kirsher */ 1057f7917c00SJeff Kirsher const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph; 1058f7917c00SJeff Kirsher 1059f7917c00SJeff Kirsher if (ip6h->nexthdr == IPPROTO_TCP) 1060f7917c00SJeff Kirsher csum_type = TX_CSUM_TCPIP6; 1061f7917c00SJeff Kirsher else if (ip6h->nexthdr == IPPROTO_UDP) 1062f7917c00SJeff Kirsher csum_type = TX_CSUM_UDPIP6; 1063f7917c00SJeff Kirsher else 1064f7917c00SJeff Kirsher goto nocsum; 1065f7917c00SJeff Kirsher } 1066f7917c00SJeff Kirsher 10673ccc6cf7SHariprasad Shenai if (likely(csum_type >= TX_CSUM_TCPIP)) { 10683ccc6cf7SHariprasad Shenai u64 hdr_len = TXPKT_IPHDR_LEN_V(skb_network_header_len(skb)); 10693ccc6cf7SHariprasad Shenai int eth_hdr_len = skb_network_offset(skb) - ETH_HLEN; 10703ccc6cf7SHariprasad Shenai 10713ccc6cf7SHariprasad Shenai if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) 10723ccc6cf7SHariprasad Shenai hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len); 10733ccc6cf7SHariprasad Shenai else 10743ccc6cf7SHariprasad Shenai hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len); 10753ccc6cf7SHariprasad Shenai return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len; 10763ccc6cf7SHariprasad Shenai } else { 1077f7917c00SJeff Kirsher int start = skb_transport_offset(skb); 1078f7917c00SJeff Kirsher 10791ecc7b7aSHariprasad Shenai return TXPKT_CSUM_TYPE_V(csum_type) | 10801ecc7b7aSHariprasad Shenai TXPKT_CSUM_START_V(start) | 10811ecc7b7aSHariprasad Shenai TXPKT_CSUM_LOC_V(start + skb->csum_offset); 1082f7917c00SJeff Kirsher } 1083f7917c00SJeff Kirsher } 1084f7917c00SJeff Kirsher 1085f7917c00SJeff Kirsher static void eth_txq_stop(struct sge_eth_txq *q) 1086f7917c00SJeff Kirsher { 1087f7917c00SJeff Kirsher netif_tx_stop_queue(q->txq); 1088f7917c00SJeff Kirsher q->q.stops++; 1089f7917c00SJeff Kirsher } 1090f7917c00SJeff Kirsher 1091f7917c00SJeff Kirsher static inline void txq_advance(struct sge_txq *q, unsigned int n) 1092f7917c00SJeff Kirsher { 1093f7917c00SJeff Kirsher q->in_use += n; 1094f7917c00SJeff Kirsher q->pidx += n; 1095f7917c00SJeff Kirsher if (q->pidx >= q->size) 1096f7917c00SJeff Kirsher q->pidx -= q->size; 1097f7917c00SJeff Kirsher } 1098f7917c00SJeff Kirsher 109984a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 110084a200b3SVarun Prakash static inline int 110184a200b3SVarun Prakash cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap, 110284a200b3SVarun Prakash const struct port_info *pi, u64 *cntrl) 110384a200b3SVarun Prakash { 110484a200b3SVarun Prakash const struct cxgb_fcoe *fcoe = &pi->fcoe; 110584a200b3SVarun Prakash 110684a200b3SVarun Prakash if (!(fcoe->flags & CXGB_FCOE_ENABLED)) 110784a200b3SVarun Prakash return 0; 110884a200b3SVarun Prakash 110984a200b3SVarun Prakash if (skb->protocol != htons(ETH_P_FCOE)) 111084a200b3SVarun Prakash return 0; 111184a200b3SVarun Prakash 111284a200b3SVarun Prakash skb_reset_mac_header(skb); 111384a200b3SVarun Prakash skb->mac_len = sizeof(struct ethhdr); 111484a200b3SVarun Prakash 111584a200b3SVarun Prakash skb_set_network_header(skb, skb->mac_len); 111684a200b3SVarun Prakash skb_set_transport_header(skb, skb->mac_len + sizeof(struct fcoe_hdr)); 111784a200b3SVarun Prakash 111884a200b3SVarun Prakash if (!cxgb_fcoe_sof_eof_supported(adap, skb)) 111984a200b3SVarun Prakash return -ENOTSUPP; 112084a200b3SVarun Prakash 112184a200b3SVarun Prakash /* FC CRC offload */ 11221ecc7b7aSHariprasad Shenai *cntrl = TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE) | 11231ecc7b7aSHariprasad Shenai TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F | 11241ecc7b7aSHariprasad Shenai TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START) | 11251ecc7b7aSHariprasad Shenai TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END) | 11261ecc7b7aSHariprasad Shenai TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END); 112784a200b3SVarun Prakash return 0; 112884a200b3SVarun Prakash } 112984a200b3SVarun Prakash #endif /* CONFIG_CHELSIO_T4_FCOE */ 113084a200b3SVarun Prakash 1131f7917c00SJeff Kirsher /** 1132f7917c00SJeff Kirsher * t4_eth_xmit - add a packet to an Ethernet Tx queue 1133f7917c00SJeff Kirsher * @skb: the packet 1134f7917c00SJeff Kirsher * @dev: the egress net device 1135f7917c00SJeff Kirsher * 1136f7917c00SJeff Kirsher * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled. 1137f7917c00SJeff Kirsher */ 1138f7917c00SJeff Kirsher netdev_tx_t t4_eth_xmit(struct sk_buff *skb, struct net_device *dev) 1139f7917c00SJeff Kirsher { 1140f7917c00SJeff Kirsher u32 wr_mid; 1141f7917c00SJeff Kirsher u64 cntrl, *end; 1142f7917c00SJeff Kirsher int qidx, credits; 1143f7917c00SJeff Kirsher unsigned int flits, ndesc; 1144f7917c00SJeff Kirsher struct adapter *adap; 1145f7917c00SJeff Kirsher struct sge_eth_txq *q; 1146f7917c00SJeff Kirsher const struct port_info *pi; 1147f7917c00SJeff Kirsher struct fw_eth_tx_pkt_wr *wr; 1148f7917c00SJeff Kirsher struct cpl_tx_pkt_core *cpl; 1149f7917c00SJeff Kirsher const struct skb_shared_info *ssi; 1150f7917c00SJeff Kirsher dma_addr_t addr[MAX_SKB_FRAGS + 1]; 11510034b298SKumar Sanghvi bool immediate = false; 1152637d3e99SHariprasad Shenai int len, max_pkt_len; 115384a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 115484a200b3SVarun Prakash int err; 115584a200b3SVarun Prakash #endif /* CONFIG_CHELSIO_T4_FCOE */ 1156f7917c00SJeff Kirsher 1157f7917c00SJeff Kirsher /* 1158f7917c00SJeff Kirsher * The chip min packet length is 10 octets but play safe and reject 1159f7917c00SJeff Kirsher * anything shorter than an Ethernet header. 1160f7917c00SJeff Kirsher */ 1161f7917c00SJeff Kirsher if (unlikely(skb->len < ETH_HLEN)) { 1162a7525198SEric W. Biederman out_free: dev_kfree_skb_any(skb); 1163f7917c00SJeff Kirsher return NETDEV_TX_OK; 1164f7917c00SJeff Kirsher } 1165f7917c00SJeff Kirsher 1166637d3e99SHariprasad Shenai /* Discard the packet if the length is greater than mtu */ 1167637d3e99SHariprasad Shenai max_pkt_len = ETH_HLEN + dev->mtu; 1168637d3e99SHariprasad Shenai if (skb_vlan_tag_present(skb)) 1169637d3e99SHariprasad Shenai max_pkt_len += VLAN_HLEN; 1170637d3e99SHariprasad Shenai if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len))) 1171637d3e99SHariprasad Shenai goto out_free; 1172637d3e99SHariprasad Shenai 1173f7917c00SJeff Kirsher pi = netdev_priv(dev); 1174f7917c00SJeff Kirsher adap = pi->adapter; 1175f7917c00SJeff Kirsher qidx = skb_get_queue_mapping(skb); 1176f7917c00SJeff Kirsher q = &adap->sge.ethtxq[qidx + pi->first_qset]; 1177f7917c00SJeff Kirsher 1178f7917c00SJeff Kirsher reclaim_completed_tx(adap, &q->q, true); 11791ecc7b7aSHariprasad Shenai cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; 118084a200b3SVarun Prakash 118184a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 118284a200b3SVarun Prakash err = cxgb_fcoe_offload(skb, adap, pi, &cntrl); 118384a200b3SVarun Prakash if (unlikely(err == -ENOTSUPP)) 118484a200b3SVarun Prakash goto out_free; 118584a200b3SVarun Prakash #endif /* CONFIG_CHELSIO_T4_FCOE */ 1186f7917c00SJeff Kirsher 1187f7917c00SJeff Kirsher flits = calc_tx_flits(skb); 1188f7917c00SJeff Kirsher ndesc = flits_to_desc(flits); 1189f7917c00SJeff Kirsher credits = txq_avail(&q->q) - ndesc; 1190f7917c00SJeff Kirsher 1191f7917c00SJeff Kirsher if (unlikely(credits < 0)) { 1192f7917c00SJeff Kirsher eth_txq_stop(q); 1193f7917c00SJeff Kirsher dev_err(adap->pdev_dev, 1194f7917c00SJeff Kirsher "%s: Tx ring %u full while queue awake!\n", 1195f7917c00SJeff Kirsher dev->name, qidx); 1196f7917c00SJeff Kirsher return NETDEV_TX_BUSY; 1197f7917c00SJeff Kirsher } 1198f7917c00SJeff Kirsher 11990034b298SKumar Sanghvi if (is_eth_imm(skb)) 12000034b298SKumar Sanghvi immediate = true; 12010034b298SKumar Sanghvi 12020034b298SKumar Sanghvi if (!immediate && 1203f7917c00SJeff Kirsher unlikely(map_skb(adap->pdev_dev, skb, addr) < 0)) { 1204f7917c00SJeff Kirsher q->mapping_err++; 1205f7917c00SJeff Kirsher goto out_free; 1206f7917c00SJeff Kirsher } 1207f7917c00SJeff Kirsher 1208e2ac9628SHariprasad Shenai wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2)); 1209f7917c00SJeff Kirsher if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1210f7917c00SJeff Kirsher eth_txq_stop(q); 1211e2ac9628SHariprasad Shenai wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; 1212f7917c00SJeff Kirsher } 1213f7917c00SJeff Kirsher 1214f7917c00SJeff Kirsher wr = (void *)&q->q.desc[q->q.pidx]; 1215f7917c00SJeff Kirsher wr->equiq_to_len16 = htonl(wr_mid); 1216f7917c00SJeff Kirsher wr->r3 = cpu_to_be64(0); 1217f7917c00SJeff Kirsher end = (u64 *)wr + flits; 1218f7917c00SJeff Kirsher 12190034b298SKumar Sanghvi len = immediate ? skb->len : 0; 1220f7917c00SJeff Kirsher ssi = skb_shinfo(skb); 1221f7917c00SJeff Kirsher if (ssi->gso_size) { 1222f7917c00SJeff Kirsher struct cpl_tx_pkt_lso *lso = (void *)wr; 1223f7917c00SJeff Kirsher bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0; 1224f7917c00SJeff Kirsher int l3hdr_len = skb_network_header_len(skb); 1225f7917c00SJeff Kirsher int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; 1226f7917c00SJeff Kirsher 12270034b298SKumar Sanghvi len += sizeof(*lso); 1228e2ac9628SHariprasad Shenai wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | 1229e2ac9628SHariprasad Shenai FW_WR_IMMDLEN_V(len)); 12301ecc7b7aSHariprasad Shenai lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) | 12311ecc7b7aSHariprasad Shenai LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F | 12321ecc7b7aSHariprasad Shenai LSO_IPV6_V(v6) | 12331ecc7b7aSHariprasad Shenai LSO_ETHHDR_LEN_V(eth_xtra_len / 4) | 12341ecc7b7aSHariprasad Shenai LSO_IPHDR_LEN_V(l3hdr_len / 4) | 12351ecc7b7aSHariprasad Shenai LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff)); 1236f7917c00SJeff Kirsher lso->c.ipid_ofst = htons(0); 1237f7917c00SJeff Kirsher lso->c.mss = htons(ssi->gso_size); 1238f7917c00SJeff Kirsher lso->c.seqno_offset = htonl(0); 12397207c0d1SHariprasad Shenai if (is_t4(adap->params.chip)) 1240f7917c00SJeff Kirsher lso->c.len = htonl(skb->len); 12417207c0d1SHariprasad Shenai else 12421ecc7b7aSHariprasad Shenai lso->c.len = htonl(LSO_T5_XFER_SIZE_V(skb->len)); 1243f7917c00SJeff Kirsher cpl = (void *)(lso + 1); 12443ccc6cf7SHariprasad Shenai 12453ccc6cf7SHariprasad Shenai if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 12463ccc6cf7SHariprasad Shenai cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len); 12473ccc6cf7SHariprasad Shenai else 12483ccc6cf7SHariprasad Shenai cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len); 12493ccc6cf7SHariprasad Shenai 12503ccc6cf7SHariprasad Shenai cntrl |= TXPKT_CSUM_TYPE_V(v6 ? 12513ccc6cf7SHariprasad Shenai TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | 12523ccc6cf7SHariprasad Shenai TXPKT_IPHDR_LEN_V(l3hdr_len); 1253f7917c00SJeff Kirsher q->tso++; 1254f7917c00SJeff Kirsher q->tx_cso += ssi->gso_segs; 1255f7917c00SJeff Kirsher } else { 1256ca71de6bSKumar Sanghvi len += sizeof(*cpl); 1257e2ac9628SHariprasad Shenai wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | 1258e2ac9628SHariprasad Shenai FW_WR_IMMDLEN_V(len)); 1259f7917c00SJeff Kirsher cpl = (void *)(wr + 1); 1260f7917c00SJeff Kirsher if (skb->ip_summed == CHECKSUM_PARTIAL) { 12613ccc6cf7SHariprasad Shenai cntrl = hwcsum(adap->params.chip, skb) | 12623ccc6cf7SHariprasad Shenai TXPKT_IPCSUM_DIS_F; 1263f7917c00SJeff Kirsher q->tx_cso++; 126484a200b3SVarun Prakash } 1265f7917c00SJeff Kirsher } 1266f7917c00SJeff Kirsher 1267df8a39deSJiri Pirko if (skb_vlan_tag_present(skb)) { 1268f7917c00SJeff Kirsher q->vlan_ins++; 12691ecc7b7aSHariprasad Shenai cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); 127084a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 127184a200b3SVarun Prakash if (skb->protocol == htons(ETH_P_FCOE)) 12721ecc7b7aSHariprasad Shenai cntrl |= TXPKT_VLAN_V( 127384a200b3SVarun Prakash ((skb->priority & 0x7) << VLAN_PRIO_SHIFT)); 127484a200b3SVarun Prakash #endif /* CONFIG_CHELSIO_T4_FCOE */ 1275f7917c00SJeff Kirsher } 1276f7917c00SJeff Kirsher 12771ecc7b7aSHariprasad Shenai cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT_XT) | 12781ecc7b7aSHariprasad Shenai TXPKT_INTF_V(pi->tx_chan) | 1279b2612722SHariprasad Shenai TXPKT_PF_V(adap->pf)); 1280f7917c00SJeff Kirsher cpl->pack = htons(0); 1281f7917c00SJeff Kirsher cpl->len = htons(skb->len); 1282f7917c00SJeff Kirsher cpl->ctrl1 = cpu_to_be64(cntrl); 1283f7917c00SJeff Kirsher 12840034b298SKumar Sanghvi if (immediate) { 1285f7917c00SJeff Kirsher inline_tx_skb(skb, &q->q, cpl + 1); 1286a7525198SEric W. Biederman dev_consume_skb_any(skb); 1287f7917c00SJeff Kirsher } else { 1288f7917c00SJeff Kirsher int last_desc; 1289f7917c00SJeff Kirsher 1290f7917c00SJeff Kirsher write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), end, 0, 1291f7917c00SJeff Kirsher addr); 1292f7917c00SJeff Kirsher skb_orphan(skb); 1293f7917c00SJeff Kirsher 1294f7917c00SJeff Kirsher last_desc = q->q.pidx + ndesc - 1; 1295f7917c00SJeff Kirsher if (last_desc >= q->q.size) 1296f7917c00SJeff Kirsher last_desc -= q->q.size; 1297f7917c00SJeff Kirsher q->q.sdesc[last_desc].skb = skb; 1298f7917c00SJeff Kirsher q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)(cpl + 1); 1299f7917c00SJeff Kirsher } 1300f7917c00SJeff Kirsher 1301f7917c00SJeff Kirsher txq_advance(&q->q, ndesc); 1302f7917c00SJeff Kirsher 1303f7917c00SJeff Kirsher ring_tx_db(adap, &q->q, ndesc); 1304f7917c00SJeff Kirsher return NETDEV_TX_OK; 1305f7917c00SJeff Kirsher } 1306f7917c00SJeff Kirsher 1307f7917c00SJeff Kirsher /** 1308f7917c00SJeff Kirsher * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs 1309f7917c00SJeff Kirsher * @q: the SGE control Tx queue 1310f7917c00SJeff Kirsher * 1311f7917c00SJeff Kirsher * This is a variant of reclaim_completed_tx() that is used for Tx queues 1312f7917c00SJeff Kirsher * that send only immediate data (presently just the control queues) and 1313f7917c00SJeff Kirsher * thus do not have any sk_buffs to release. 1314f7917c00SJeff Kirsher */ 1315f7917c00SJeff Kirsher static inline void reclaim_completed_tx_imm(struct sge_txq *q) 1316f7917c00SJeff Kirsher { 1317f7917c00SJeff Kirsher int hw_cidx = ntohs(q->stat->cidx); 1318f7917c00SJeff Kirsher int reclaim = hw_cidx - q->cidx; 1319f7917c00SJeff Kirsher 1320f7917c00SJeff Kirsher if (reclaim < 0) 1321f7917c00SJeff Kirsher reclaim += q->size; 1322f7917c00SJeff Kirsher 1323f7917c00SJeff Kirsher q->in_use -= reclaim; 1324f7917c00SJeff Kirsher q->cidx = hw_cidx; 1325f7917c00SJeff Kirsher } 1326f7917c00SJeff Kirsher 1327f7917c00SJeff Kirsher /** 1328f7917c00SJeff Kirsher * is_imm - check whether a packet can be sent as immediate data 1329f7917c00SJeff Kirsher * @skb: the packet 1330f7917c00SJeff Kirsher * 1331f7917c00SJeff Kirsher * Returns true if a packet can be sent as a WR with immediate data. 1332f7917c00SJeff Kirsher */ 1333f7917c00SJeff Kirsher static inline int is_imm(const struct sk_buff *skb) 1334f7917c00SJeff Kirsher { 1335f7917c00SJeff Kirsher return skb->len <= MAX_CTRL_WR_LEN; 1336f7917c00SJeff Kirsher } 1337f7917c00SJeff Kirsher 1338f7917c00SJeff Kirsher /** 1339f7917c00SJeff Kirsher * ctrlq_check_stop - check if a control queue is full and should stop 1340f7917c00SJeff Kirsher * @q: the queue 1341f7917c00SJeff Kirsher * @wr: most recent WR written to the queue 1342f7917c00SJeff Kirsher * 1343f7917c00SJeff Kirsher * Check if a control queue has become full and should be stopped. 1344f7917c00SJeff Kirsher * We clean up control queue descriptors very lazily, only when we are out. 1345f7917c00SJeff Kirsher * If the queue is still full after reclaiming any completed descriptors 1346f7917c00SJeff Kirsher * we suspend it and have the last WR wake it up. 1347f7917c00SJeff Kirsher */ 1348f7917c00SJeff Kirsher static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr) 1349f7917c00SJeff Kirsher { 1350f7917c00SJeff Kirsher reclaim_completed_tx_imm(&q->q); 1351f7917c00SJeff Kirsher if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) { 1352e2ac9628SHariprasad Shenai wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F); 1353f7917c00SJeff Kirsher q->q.stops++; 1354f7917c00SJeff Kirsher q->full = 1; 1355f7917c00SJeff Kirsher } 1356f7917c00SJeff Kirsher } 1357f7917c00SJeff Kirsher 1358f7917c00SJeff Kirsher /** 1359f7917c00SJeff Kirsher * ctrl_xmit - send a packet through an SGE control Tx queue 1360f7917c00SJeff Kirsher * @q: the control queue 1361f7917c00SJeff Kirsher * @skb: the packet 1362f7917c00SJeff Kirsher * 1363f7917c00SJeff Kirsher * Send a packet through an SGE control Tx queue. Packets sent through 1364f7917c00SJeff Kirsher * a control queue must fit entirely as immediate data. 1365f7917c00SJeff Kirsher */ 1366f7917c00SJeff Kirsher static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb) 1367f7917c00SJeff Kirsher { 1368f7917c00SJeff Kirsher unsigned int ndesc; 1369f7917c00SJeff Kirsher struct fw_wr_hdr *wr; 1370f7917c00SJeff Kirsher 1371f7917c00SJeff Kirsher if (unlikely(!is_imm(skb))) { 1372f7917c00SJeff Kirsher WARN_ON(1); 1373f7917c00SJeff Kirsher dev_kfree_skb(skb); 1374f7917c00SJeff Kirsher return NET_XMIT_DROP; 1375f7917c00SJeff Kirsher } 1376f7917c00SJeff Kirsher 1377f7917c00SJeff Kirsher ndesc = DIV_ROUND_UP(skb->len, sizeof(struct tx_desc)); 1378f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1379f7917c00SJeff Kirsher 1380f7917c00SJeff Kirsher if (unlikely(q->full)) { 1381f7917c00SJeff Kirsher skb->priority = ndesc; /* save for restart */ 1382f7917c00SJeff Kirsher __skb_queue_tail(&q->sendq, skb); 1383f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1384f7917c00SJeff Kirsher return NET_XMIT_CN; 1385f7917c00SJeff Kirsher } 1386f7917c00SJeff Kirsher 1387f7917c00SJeff Kirsher wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx]; 1388f7917c00SJeff Kirsher inline_tx_skb(skb, &q->q, wr); 1389f7917c00SJeff Kirsher 1390f7917c00SJeff Kirsher txq_advance(&q->q, ndesc); 1391f7917c00SJeff Kirsher if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) 1392f7917c00SJeff Kirsher ctrlq_check_stop(q, wr); 1393f7917c00SJeff Kirsher 1394f7917c00SJeff Kirsher ring_tx_db(q->adap, &q->q, ndesc); 1395f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1396f7917c00SJeff Kirsher 1397f7917c00SJeff Kirsher kfree_skb(skb); 1398f7917c00SJeff Kirsher return NET_XMIT_SUCCESS; 1399f7917c00SJeff Kirsher } 1400f7917c00SJeff Kirsher 1401f7917c00SJeff Kirsher /** 1402f7917c00SJeff Kirsher * restart_ctrlq - restart a suspended control queue 1403f7917c00SJeff Kirsher * @data: the control queue to restart 1404f7917c00SJeff Kirsher * 1405f7917c00SJeff Kirsher * Resumes transmission on a suspended Tx control queue. 1406f7917c00SJeff Kirsher */ 1407f7917c00SJeff Kirsher static void restart_ctrlq(unsigned long data) 1408f7917c00SJeff Kirsher { 1409f7917c00SJeff Kirsher struct sk_buff *skb; 1410f7917c00SJeff Kirsher unsigned int written = 0; 1411f7917c00SJeff Kirsher struct sge_ctrl_txq *q = (struct sge_ctrl_txq *)data; 1412f7917c00SJeff Kirsher 1413f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1414f7917c00SJeff Kirsher reclaim_completed_tx_imm(&q->q); 1415f7917c00SJeff Kirsher BUG_ON(txq_avail(&q->q) < TXQ_STOP_THRES); /* q should be empty */ 1416f7917c00SJeff Kirsher 1417f7917c00SJeff Kirsher while ((skb = __skb_dequeue(&q->sendq)) != NULL) { 1418f7917c00SJeff Kirsher struct fw_wr_hdr *wr; 1419f7917c00SJeff Kirsher unsigned int ndesc = skb->priority; /* previously saved */ 1420f7917c00SJeff Kirsher 1421f7917c00SJeff Kirsher /* 1422f7917c00SJeff Kirsher * Write descriptors and free skbs outside the lock to limit 1423f7917c00SJeff Kirsher * wait times. q->full is still set so new skbs will be queued. 1424f7917c00SJeff Kirsher */ 1425f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1426f7917c00SJeff Kirsher 1427f7917c00SJeff Kirsher wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx]; 1428f7917c00SJeff Kirsher inline_tx_skb(skb, &q->q, wr); 1429f7917c00SJeff Kirsher kfree_skb(skb); 1430f7917c00SJeff Kirsher 1431f7917c00SJeff Kirsher written += ndesc; 1432f7917c00SJeff Kirsher txq_advance(&q->q, ndesc); 1433f7917c00SJeff Kirsher if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) { 1434f7917c00SJeff Kirsher unsigned long old = q->q.stops; 1435f7917c00SJeff Kirsher 1436f7917c00SJeff Kirsher ctrlq_check_stop(q, wr); 1437f7917c00SJeff Kirsher if (q->q.stops != old) { /* suspended anew */ 1438f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1439f7917c00SJeff Kirsher goto ringdb; 1440f7917c00SJeff Kirsher } 1441f7917c00SJeff Kirsher } 1442f7917c00SJeff Kirsher if (written > 16) { 1443f7917c00SJeff Kirsher ring_tx_db(q->adap, &q->q, written); 1444f7917c00SJeff Kirsher written = 0; 1445f7917c00SJeff Kirsher } 1446f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1447f7917c00SJeff Kirsher } 1448f7917c00SJeff Kirsher q->full = 0; 1449f7917c00SJeff Kirsher ringdb: if (written) 1450f7917c00SJeff Kirsher ring_tx_db(q->adap, &q->q, written); 1451f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1452f7917c00SJeff Kirsher } 1453f7917c00SJeff Kirsher 1454f7917c00SJeff Kirsher /** 1455f7917c00SJeff Kirsher * t4_mgmt_tx - send a management message 1456f7917c00SJeff Kirsher * @adap: the adapter 1457f7917c00SJeff Kirsher * @skb: the packet containing the management message 1458f7917c00SJeff Kirsher * 1459f7917c00SJeff Kirsher * Send a management message through control queue 0. 1460f7917c00SJeff Kirsher */ 1461f7917c00SJeff Kirsher int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb) 1462f7917c00SJeff Kirsher { 1463f7917c00SJeff Kirsher int ret; 1464f7917c00SJeff Kirsher 1465f7917c00SJeff Kirsher local_bh_disable(); 1466f7917c00SJeff Kirsher ret = ctrl_xmit(&adap->sge.ctrlq[0], skb); 1467f7917c00SJeff Kirsher local_bh_enable(); 1468f7917c00SJeff Kirsher return ret; 1469f7917c00SJeff Kirsher } 1470f7917c00SJeff Kirsher 1471f7917c00SJeff Kirsher /** 1472f7917c00SJeff Kirsher * is_ofld_imm - check whether a packet can be sent as immediate data 1473f7917c00SJeff Kirsher * @skb: the packet 1474f7917c00SJeff Kirsher * 1475f7917c00SJeff Kirsher * Returns true if a packet can be sent as an offload WR with immediate 1476f7917c00SJeff Kirsher * data. We currently use the same limit as for Ethernet packets. 1477f7917c00SJeff Kirsher */ 1478f7917c00SJeff Kirsher static inline int is_ofld_imm(const struct sk_buff *skb) 1479f7917c00SJeff Kirsher { 1480f7917c00SJeff Kirsher return skb->len <= MAX_IMM_TX_PKT_LEN; 1481f7917c00SJeff Kirsher } 1482f7917c00SJeff Kirsher 1483f7917c00SJeff Kirsher /** 1484f7917c00SJeff Kirsher * calc_tx_flits_ofld - calculate # of flits for an offload packet 1485f7917c00SJeff Kirsher * @skb: the packet 1486f7917c00SJeff Kirsher * 1487f7917c00SJeff Kirsher * Returns the number of flits needed for the given offload packet. 1488f7917c00SJeff Kirsher * These packets are already fully constructed and no additional headers 1489f7917c00SJeff Kirsher * will be added. 1490f7917c00SJeff Kirsher */ 1491f7917c00SJeff Kirsher static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 1492f7917c00SJeff Kirsher { 1493f7917c00SJeff Kirsher unsigned int flits, cnt; 1494f7917c00SJeff Kirsher 1495f7917c00SJeff Kirsher if (is_ofld_imm(skb)) 1496f7917c00SJeff Kirsher return DIV_ROUND_UP(skb->len, 8); 1497f7917c00SJeff Kirsher 1498f7917c00SJeff Kirsher flits = skb_transport_offset(skb) / 8U; /* headers */ 1499f7917c00SJeff Kirsher cnt = skb_shinfo(skb)->nr_frags; 150015dd16c2SLi RongQing if (skb_tail_pointer(skb) != skb_transport_header(skb)) 1501f7917c00SJeff Kirsher cnt++; 1502f7917c00SJeff Kirsher return flits + sgl_len(cnt); 1503f7917c00SJeff Kirsher } 1504f7917c00SJeff Kirsher 1505f7917c00SJeff Kirsher /** 1506f7917c00SJeff Kirsher * txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion 1507f7917c00SJeff Kirsher * @adap: the adapter 1508f7917c00SJeff Kirsher * @q: the queue to stop 1509f7917c00SJeff Kirsher * 1510f7917c00SJeff Kirsher * Mark a Tx queue stopped due to I/O MMU exhaustion and resulting 1511f7917c00SJeff Kirsher * inability to map packets. A periodic timer attempts to restart 1512f7917c00SJeff Kirsher * queues so marked. 1513f7917c00SJeff Kirsher */ 1514f7917c00SJeff Kirsher static void txq_stop_maperr(struct sge_ofld_txq *q) 1515f7917c00SJeff Kirsher { 1516f7917c00SJeff Kirsher q->mapping_err++; 1517f7917c00SJeff Kirsher q->q.stops++; 1518f7917c00SJeff Kirsher set_bit(q->q.cntxt_id - q->adap->sge.egr_start, 1519f7917c00SJeff Kirsher q->adap->sge.txq_maperr); 1520f7917c00SJeff Kirsher } 1521f7917c00SJeff Kirsher 1522f7917c00SJeff Kirsher /** 1523f7917c00SJeff Kirsher * ofldtxq_stop - stop an offload Tx queue that has become full 1524f7917c00SJeff Kirsher * @q: the queue to stop 1525f7917c00SJeff Kirsher * @skb: the packet causing the queue to become full 1526f7917c00SJeff Kirsher * 1527f7917c00SJeff Kirsher * Stops an offload Tx queue that has become full and modifies the packet 1528f7917c00SJeff Kirsher * being written to request a wakeup. 1529f7917c00SJeff Kirsher */ 1530f7917c00SJeff Kirsher static void ofldtxq_stop(struct sge_ofld_txq *q, struct sk_buff *skb) 1531f7917c00SJeff Kirsher { 1532f7917c00SJeff Kirsher struct fw_wr_hdr *wr = (struct fw_wr_hdr *)skb->data; 1533f7917c00SJeff Kirsher 1534e2ac9628SHariprasad Shenai wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F); 1535f7917c00SJeff Kirsher q->q.stops++; 1536f7917c00SJeff Kirsher q->full = 1; 1537f7917c00SJeff Kirsher } 1538f7917c00SJeff Kirsher 1539f7917c00SJeff Kirsher /** 1540f7917c00SJeff Kirsher * service_ofldq - restart a suspended offload queue 1541f7917c00SJeff Kirsher * @q: the offload queue 1542f7917c00SJeff Kirsher * 1543f7917c00SJeff Kirsher * Services an offload Tx queue by moving packets from its packet queue 1544f7917c00SJeff Kirsher * to the HW Tx ring. The function starts and ends with the queue locked. 1545f7917c00SJeff Kirsher */ 1546f7917c00SJeff Kirsher static void service_ofldq(struct sge_ofld_txq *q) 1547f7917c00SJeff Kirsher { 1548f7917c00SJeff Kirsher u64 *pos; 1549f7917c00SJeff Kirsher int credits; 1550f7917c00SJeff Kirsher struct sk_buff *skb; 1551f7917c00SJeff Kirsher unsigned int written = 0; 1552f7917c00SJeff Kirsher unsigned int flits, ndesc; 1553f7917c00SJeff Kirsher 1554f7917c00SJeff Kirsher while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) { 1555f7917c00SJeff Kirsher /* 1556f7917c00SJeff Kirsher * We drop the lock but leave skb on sendq, thus retaining 1557f7917c00SJeff Kirsher * exclusive access to the state of the queue. 1558f7917c00SJeff Kirsher */ 1559f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1560f7917c00SJeff Kirsher 1561f7917c00SJeff Kirsher reclaim_completed_tx(q->adap, &q->q, false); 1562f7917c00SJeff Kirsher 1563f7917c00SJeff Kirsher flits = skb->priority; /* previously saved */ 1564f7917c00SJeff Kirsher ndesc = flits_to_desc(flits); 1565f7917c00SJeff Kirsher credits = txq_avail(&q->q) - ndesc; 1566f7917c00SJeff Kirsher BUG_ON(credits < 0); 1567f7917c00SJeff Kirsher if (unlikely(credits < TXQ_STOP_THRES)) 1568f7917c00SJeff Kirsher ofldtxq_stop(q, skb); 1569f7917c00SJeff Kirsher 1570f7917c00SJeff Kirsher pos = (u64 *)&q->q.desc[q->q.pidx]; 1571f7917c00SJeff Kirsher if (is_ofld_imm(skb)) 1572f7917c00SJeff Kirsher inline_tx_skb(skb, &q->q, pos); 1573f7917c00SJeff Kirsher else if (map_skb(q->adap->pdev_dev, skb, 1574f7917c00SJeff Kirsher (dma_addr_t *)skb->head)) { 1575f7917c00SJeff Kirsher txq_stop_maperr(q); 1576f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1577f7917c00SJeff Kirsher break; 1578f7917c00SJeff Kirsher } else { 1579f7917c00SJeff Kirsher int last_desc, hdr_len = skb_transport_offset(skb); 1580f7917c00SJeff Kirsher 1581f7917c00SJeff Kirsher memcpy(pos, skb->data, hdr_len); 1582f7917c00SJeff Kirsher write_sgl(skb, &q->q, (void *)pos + hdr_len, 1583f7917c00SJeff Kirsher pos + flits, hdr_len, 1584f7917c00SJeff Kirsher (dma_addr_t *)skb->head); 1585f7917c00SJeff Kirsher #ifdef CONFIG_NEED_DMA_MAP_STATE 1586f7917c00SJeff Kirsher skb->dev = q->adap->port[0]; 1587f7917c00SJeff Kirsher skb->destructor = deferred_unmap_destructor; 1588f7917c00SJeff Kirsher #endif 1589f7917c00SJeff Kirsher last_desc = q->q.pidx + ndesc - 1; 1590f7917c00SJeff Kirsher if (last_desc >= q->q.size) 1591f7917c00SJeff Kirsher last_desc -= q->q.size; 1592f7917c00SJeff Kirsher q->q.sdesc[last_desc].skb = skb; 1593f7917c00SJeff Kirsher } 1594f7917c00SJeff Kirsher 1595f7917c00SJeff Kirsher txq_advance(&q->q, ndesc); 1596f7917c00SJeff Kirsher written += ndesc; 1597f7917c00SJeff Kirsher if (unlikely(written > 32)) { 1598f7917c00SJeff Kirsher ring_tx_db(q->adap, &q->q, written); 1599f7917c00SJeff Kirsher written = 0; 1600f7917c00SJeff Kirsher } 1601f7917c00SJeff Kirsher 1602f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1603f7917c00SJeff Kirsher __skb_unlink(skb, &q->sendq); 1604f7917c00SJeff Kirsher if (is_ofld_imm(skb)) 1605f7917c00SJeff Kirsher kfree_skb(skb); 1606f7917c00SJeff Kirsher } 1607f7917c00SJeff Kirsher if (likely(written)) 1608f7917c00SJeff Kirsher ring_tx_db(q->adap, &q->q, written); 1609f7917c00SJeff Kirsher } 1610f7917c00SJeff Kirsher 1611f7917c00SJeff Kirsher /** 1612f7917c00SJeff Kirsher * ofld_xmit - send a packet through an offload queue 1613f7917c00SJeff Kirsher * @q: the Tx offload queue 1614f7917c00SJeff Kirsher * @skb: the packet 1615f7917c00SJeff Kirsher * 1616f7917c00SJeff Kirsher * Send an offload packet through an SGE offload queue. 1617f7917c00SJeff Kirsher */ 1618f7917c00SJeff Kirsher static int ofld_xmit(struct sge_ofld_txq *q, struct sk_buff *skb) 1619f7917c00SJeff Kirsher { 1620f7917c00SJeff Kirsher skb->priority = calc_tx_flits_ofld(skb); /* save for restart */ 1621f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1622f7917c00SJeff Kirsher __skb_queue_tail(&q->sendq, skb); 1623f7917c00SJeff Kirsher if (q->sendq.qlen == 1) 1624f7917c00SJeff Kirsher service_ofldq(q); 1625f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1626f7917c00SJeff Kirsher return NET_XMIT_SUCCESS; 1627f7917c00SJeff Kirsher } 1628f7917c00SJeff Kirsher 1629f7917c00SJeff Kirsher /** 1630f7917c00SJeff Kirsher * restart_ofldq - restart a suspended offload queue 1631f7917c00SJeff Kirsher * @data: the offload queue to restart 1632f7917c00SJeff Kirsher * 1633f7917c00SJeff Kirsher * Resumes transmission on a suspended Tx offload queue. 1634f7917c00SJeff Kirsher */ 1635f7917c00SJeff Kirsher static void restart_ofldq(unsigned long data) 1636f7917c00SJeff Kirsher { 1637f7917c00SJeff Kirsher struct sge_ofld_txq *q = (struct sge_ofld_txq *)data; 1638f7917c00SJeff Kirsher 1639f7917c00SJeff Kirsher spin_lock(&q->sendq.lock); 1640f7917c00SJeff Kirsher q->full = 0; /* the queue actually is completely empty now */ 1641f7917c00SJeff Kirsher service_ofldq(q); 1642f7917c00SJeff Kirsher spin_unlock(&q->sendq.lock); 1643f7917c00SJeff Kirsher } 1644f7917c00SJeff Kirsher 1645f7917c00SJeff Kirsher /** 1646f7917c00SJeff Kirsher * skb_txq - return the Tx queue an offload packet should use 1647f7917c00SJeff Kirsher * @skb: the packet 1648f7917c00SJeff Kirsher * 1649f7917c00SJeff Kirsher * Returns the Tx queue an offload packet should use as indicated by bits 1650f7917c00SJeff Kirsher * 1-15 in the packet's queue_mapping. 1651f7917c00SJeff Kirsher */ 1652f7917c00SJeff Kirsher static inline unsigned int skb_txq(const struct sk_buff *skb) 1653f7917c00SJeff Kirsher { 1654f7917c00SJeff Kirsher return skb->queue_mapping >> 1; 1655f7917c00SJeff Kirsher } 1656f7917c00SJeff Kirsher 1657f7917c00SJeff Kirsher /** 1658f7917c00SJeff Kirsher * is_ctrl_pkt - return whether an offload packet is a control packet 1659f7917c00SJeff Kirsher * @skb: the packet 1660f7917c00SJeff Kirsher * 1661f7917c00SJeff Kirsher * Returns whether an offload packet should use an OFLD or a CTRL 1662f7917c00SJeff Kirsher * Tx queue as indicated by bit 0 in the packet's queue_mapping. 1663f7917c00SJeff Kirsher */ 1664f7917c00SJeff Kirsher static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb) 1665f7917c00SJeff Kirsher { 1666f7917c00SJeff Kirsher return skb->queue_mapping & 1; 1667f7917c00SJeff Kirsher } 1668f7917c00SJeff Kirsher 1669f7917c00SJeff Kirsher static inline int ofld_send(struct adapter *adap, struct sk_buff *skb) 1670f7917c00SJeff Kirsher { 1671f7917c00SJeff Kirsher unsigned int idx = skb_txq(skb); 1672f7917c00SJeff Kirsher 16734fe44dd7SKumar Sanghvi if (unlikely(is_ctrl_pkt(skb))) { 16744fe44dd7SKumar Sanghvi /* Single ctrl queue is a requirement for LE workaround path */ 16754fe44dd7SKumar Sanghvi if (adap->tids.nsftids) 16764fe44dd7SKumar Sanghvi idx = 0; 1677f7917c00SJeff Kirsher return ctrl_xmit(&adap->sge.ctrlq[idx], skb); 16784fe44dd7SKumar Sanghvi } 1679f7917c00SJeff Kirsher return ofld_xmit(&adap->sge.ofldtxq[idx], skb); 1680f7917c00SJeff Kirsher } 1681f7917c00SJeff Kirsher 1682f7917c00SJeff Kirsher /** 1683f7917c00SJeff Kirsher * t4_ofld_send - send an offload packet 1684f7917c00SJeff Kirsher * @adap: the adapter 1685f7917c00SJeff Kirsher * @skb: the packet 1686f7917c00SJeff Kirsher * 1687f7917c00SJeff Kirsher * Sends an offload packet. We use the packet queue_mapping to select the 1688f7917c00SJeff Kirsher * appropriate Tx queue as follows: bit 0 indicates whether the packet 1689f7917c00SJeff Kirsher * should be sent as regular or control, bits 1-15 select the queue. 1690f7917c00SJeff Kirsher */ 1691f7917c00SJeff Kirsher int t4_ofld_send(struct adapter *adap, struct sk_buff *skb) 1692f7917c00SJeff Kirsher { 1693f7917c00SJeff Kirsher int ret; 1694f7917c00SJeff Kirsher 1695f7917c00SJeff Kirsher local_bh_disable(); 1696f7917c00SJeff Kirsher ret = ofld_send(adap, skb); 1697f7917c00SJeff Kirsher local_bh_enable(); 1698f7917c00SJeff Kirsher return ret; 1699f7917c00SJeff Kirsher } 1700f7917c00SJeff Kirsher 1701f7917c00SJeff Kirsher /** 1702f7917c00SJeff Kirsher * cxgb4_ofld_send - send an offload packet 1703f7917c00SJeff Kirsher * @dev: the net device 1704f7917c00SJeff Kirsher * @skb: the packet 1705f7917c00SJeff Kirsher * 1706f7917c00SJeff Kirsher * Sends an offload packet. This is an exported version of @t4_ofld_send, 1707f7917c00SJeff Kirsher * intended for ULDs. 1708f7917c00SJeff Kirsher */ 1709f7917c00SJeff Kirsher int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb) 1710f7917c00SJeff Kirsher { 1711f7917c00SJeff Kirsher return t4_ofld_send(netdev2adap(dev), skb); 1712f7917c00SJeff Kirsher } 1713f7917c00SJeff Kirsher EXPORT_SYMBOL(cxgb4_ofld_send); 1714f7917c00SJeff Kirsher 1715e91b0f24SIan Campbell static inline void copy_frags(struct sk_buff *skb, 1716f7917c00SJeff Kirsher const struct pkt_gl *gl, unsigned int offset) 1717f7917c00SJeff Kirsher { 1718e91b0f24SIan Campbell int i; 1719f7917c00SJeff Kirsher 1720f7917c00SJeff Kirsher /* usually there's just one frag */ 1721e91b0f24SIan Campbell __skb_fill_page_desc(skb, 0, gl->frags[0].page, 1722e91b0f24SIan Campbell gl->frags[0].offset + offset, 1723e91b0f24SIan Campbell gl->frags[0].size - offset); 1724e91b0f24SIan Campbell skb_shinfo(skb)->nr_frags = gl->nfrags; 1725e91b0f24SIan Campbell for (i = 1; i < gl->nfrags; i++) 1726e91b0f24SIan Campbell __skb_fill_page_desc(skb, i, gl->frags[i].page, 1727e91b0f24SIan Campbell gl->frags[i].offset, 1728e91b0f24SIan Campbell gl->frags[i].size); 1729f7917c00SJeff Kirsher 1730f7917c00SJeff Kirsher /* get a reference to the last page, we don't own it */ 1731e91b0f24SIan Campbell get_page(gl->frags[gl->nfrags - 1].page); 1732f7917c00SJeff Kirsher } 1733f7917c00SJeff Kirsher 1734f7917c00SJeff Kirsher /** 1735f7917c00SJeff Kirsher * cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list 1736f7917c00SJeff Kirsher * @gl: the gather list 1737f7917c00SJeff Kirsher * @skb_len: size of sk_buff main body if it carries fragments 1738f7917c00SJeff Kirsher * @pull_len: amount of data to move to the sk_buff's main body 1739f7917c00SJeff Kirsher * 1740f7917c00SJeff Kirsher * Builds an sk_buff from the given packet gather list. Returns the 1741f7917c00SJeff Kirsher * sk_buff or %NULL if sk_buff allocation failed. 1742f7917c00SJeff Kirsher */ 1743f7917c00SJeff Kirsher struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl, 1744f7917c00SJeff Kirsher unsigned int skb_len, unsigned int pull_len) 1745f7917c00SJeff Kirsher { 1746f7917c00SJeff Kirsher struct sk_buff *skb; 1747f7917c00SJeff Kirsher 1748f7917c00SJeff Kirsher /* 1749f7917c00SJeff Kirsher * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer 1750f7917c00SJeff Kirsher * size, which is expected since buffers are at least PAGE_SIZEd. 1751f7917c00SJeff Kirsher * In this case packets up to RX_COPY_THRES have only one fragment. 1752f7917c00SJeff Kirsher */ 1753f7917c00SJeff Kirsher if (gl->tot_len <= RX_COPY_THRES) { 1754f7917c00SJeff Kirsher skb = dev_alloc_skb(gl->tot_len); 1755f7917c00SJeff Kirsher if (unlikely(!skb)) 1756f7917c00SJeff Kirsher goto out; 1757f7917c00SJeff Kirsher __skb_put(skb, gl->tot_len); 1758f7917c00SJeff Kirsher skb_copy_to_linear_data(skb, gl->va, gl->tot_len); 1759f7917c00SJeff Kirsher } else { 1760f7917c00SJeff Kirsher skb = dev_alloc_skb(skb_len); 1761f7917c00SJeff Kirsher if (unlikely(!skb)) 1762f7917c00SJeff Kirsher goto out; 1763f7917c00SJeff Kirsher __skb_put(skb, pull_len); 1764f7917c00SJeff Kirsher skb_copy_to_linear_data(skb, gl->va, pull_len); 1765f7917c00SJeff Kirsher 1766e91b0f24SIan Campbell copy_frags(skb, gl, pull_len); 1767f7917c00SJeff Kirsher skb->len = gl->tot_len; 1768f7917c00SJeff Kirsher skb->data_len = skb->len - pull_len; 1769f7917c00SJeff Kirsher skb->truesize += skb->data_len; 1770f7917c00SJeff Kirsher } 1771f7917c00SJeff Kirsher out: return skb; 1772f7917c00SJeff Kirsher } 1773f7917c00SJeff Kirsher EXPORT_SYMBOL(cxgb4_pktgl_to_skb); 1774f7917c00SJeff Kirsher 1775f7917c00SJeff Kirsher /** 1776f7917c00SJeff Kirsher * t4_pktgl_free - free a packet gather list 1777f7917c00SJeff Kirsher * @gl: the gather list 1778f7917c00SJeff Kirsher * 1779f7917c00SJeff Kirsher * Releases the pages of a packet gather list. We do not own the last 1780f7917c00SJeff Kirsher * page on the list and do not free it. 1781f7917c00SJeff Kirsher */ 1782f7917c00SJeff Kirsher static void t4_pktgl_free(const struct pkt_gl *gl) 1783f7917c00SJeff Kirsher { 1784f7917c00SJeff Kirsher int n; 1785e91b0f24SIan Campbell const struct page_frag *p; 1786f7917c00SJeff Kirsher 1787f7917c00SJeff Kirsher for (p = gl->frags, n = gl->nfrags - 1; n--; p++) 1788f7917c00SJeff Kirsher put_page(p->page); 1789f7917c00SJeff Kirsher } 1790f7917c00SJeff Kirsher 1791f7917c00SJeff Kirsher /* 1792f7917c00SJeff Kirsher * Process an MPS trace packet. Give it an unused protocol number so it won't 1793f7917c00SJeff Kirsher * be delivered to anyone and send it to the stack for capture. 1794f7917c00SJeff Kirsher */ 1795f7917c00SJeff Kirsher static noinline int handle_trace_pkt(struct adapter *adap, 1796f7917c00SJeff Kirsher const struct pkt_gl *gl) 1797f7917c00SJeff Kirsher { 1798f7917c00SJeff Kirsher struct sk_buff *skb; 1799f7917c00SJeff Kirsher 1800f7917c00SJeff Kirsher skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); 1801f7917c00SJeff Kirsher if (unlikely(!skb)) { 1802f7917c00SJeff Kirsher t4_pktgl_free(gl); 1803f7917c00SJeff Kirsher return 0; 1804f7917c00SJeff Kirsher } 1805f7917c00SJeff Kirsher 1806d14807ddSHariprasad Shenai if (is_t4(adap->params.chip)) 18070a57a536SSantosh Rastapur __skb_pull(skb, sizeof(struct cpl_trace_pkt)); 18080a57a536SSantosh Rastapur else 18090a57a536SSantosh Rastapur __skb_pull(skb, sizeof(struct cpl_t5_trace_pkt)); 18100a57a536SSantosh Rastapur 1811f7917c00SJeff Kirsher skb_reset_mac_header(skb); 1812f7917c00SJeff Kirsher skb->protocol = htons(0xffff); 1813f7917c00SJeff Kirsher skb->dev = adap->port[0]; 1814f7917c00SJeff Kirsher netif_receive_skb(skb); 1815f7917c00SJeff Kirsher return 0; 1816f7917c00SJeff Kirsher } 1817f7917c00SJeff Kirsher 1818f7917c00SJeff Kirsher static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl, 1819f7917c00SJeff Kirsher const struct cpl_rx_pkt *pkt) 1820f7917c00SJeff Kirsher { 182152367a76SVipul Pandya struct adapter *adapter = rxq->rspq.adap; 182252367a76SVipul Pandya struct sge *s = &adapter->sge; 1823f7917c00SJeff Kirsher int ret; 1824f7917c00SJeff Kirsher struct sk_buff *skb; 1825f7917c00SJeff Kirsher 1826f7917c00SJeff Kirsher skb = napi_get_frags(&rxq->rspq.napi); 1827f7917c00SJeff Kirsher if (unlikely(!skb)) { 1828f7917c00SJeff Kirsher t4_pktgl_free(gl); 1829f7917c00SJeff Kirsher rxq->stats.rx_drops++; 1830f7917c00SJeff Kirsher return; 1831f7917c00SJeff Kirsher } 1832f7917c00SJeff Kirsher 183352367a76SVipul Pandya copy_frags(skb, gl, s->pktshift); 183452367a76SVipul Pandya skb->len = gl->tot_len - s->pktshift; 1835f7917c00SJeff Kirsher skb->data_len = skb->len; 1836f7917c00SJeff Kirsher skb->truesize += skb->data_len; 1837f7917c00SJeff Kirsher skb->ip_summed = CHECKSUM_UNNECESSARY; 1838f7917c00SJeff Kirsher skb_record_rx_queue(skb, rxq->rspq.idx); 18393a336cb1SHariprasad Shenai skb_mark_napi_id(skb, &rxq->rspq.napi); 1840f7917c00SJeff Kirsher if (rxq->rspq.netdev->features & NETIF_F_RXHASH) 18418264989cSTom Herbert skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val, 18428264989cSTom Herbert PKT_HASH_TYPE_L3); 1843f7917c00SJeff Kirsher 1844f7917c00SJeff Kirsher if (unlikely(pkt->vlan_ex)) { 184586a9bad3SPatrick McHardy __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan)); 1846f7917c00SJeff Kirsher rxq->stats.vlan_ex++; 1847f7917c00SJeff Kirsher } 1848f7917c00SJeff Kirsher ret = napi_gro_frags(&rxq->rspq.napi); 1849f7917c00SJeff Kirsher if (ret == GRO_HELD) 1850f7917c00SJeff Kirsher rxq->stats.lro_pkts++; 1851f7917c00SJeff Kirsher else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE) 1852f7917c00SJeff Kirsher rxq->stats.lro_merged++; 1853f7917c00SJeff Kirsher rxq->stats.pkts++; 1854f7917c00SJeff Kirsher rxq->stats.rx_cso++; 1855f7917c00SJeff Kirsher } 1856f7917c00SJeff Kirsher 1857f7917c00SJeff Kirsher /** 1858f7917c00SJeff Kirsher * t4_ethrx_handler - process an ingress ethernet packet 1859f7917c00SJeff Kirsher * @q: the response queue that received the packet 1860f7917c00SJeff Kirsher * @rsp: the response queue descriptor holding the RX_PKT message 1861f7917c00SJeff Kirsher * @si: the gather list of packet fragments 1862f7917c00SJeff Kirsher * 1863f7917c00SJeff Kirsher * Process an ingress ethernet packet and deliver it to the stack. 1864f7917c00SJeff Kirsher */ 1865f7917c00SJeff Kirsher int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, 1866f7917c00SJeff Kirsher const struct pkt_gl *si) 1867f7917c00SJeff Kirsher { 1868f7917c00SJeff Kirsher bool csum_ok; 1869f7917c00SJeff Kirsher struct sk_buff *skb; 1870f7917c00SJeff Kirsher const struct cpl_rx_pkt *pkt; 1871f7917c00SJeff Kirsher struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); 187252367a76SVipul Pandya struct sge *s = &q->adap->sge; 1873d14807ddSHariprasad Shenai int cpl_trace_pkt = is_t4(q->adap->params.chip) ? 18740a57a536SSantosh Rastapur CPL_TRACE_PKT : CPL_TRACE_PKT_T5; 187584a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 187684a200b3SVarun Prakash struct port_info *pi; 187784a200b3SVarun Prakash #endif 1878f7917c00SJeff Kirsher 18790a57a536SSantosh Rastapur if (unlikely(*(u8 *)rsp == cpl_trace_pkt)) 1880f7917c00SJeff Kirsher return handle_trace_pkt(q->adap, si); 1881f7917c00SJeff Kirsher 1882f7917c00SJeff Kirsher pkt = (const struct cpl_rx_pkt *)rsp; 1883cca2822dSHariprasad Shenai csum_ok = pkt->csum_calc && !pkt->err_vec && 1884cca2822dSHariprasad Shenai (q->netdev->features & NETIF_F_RXCSUM); 1885bdc590b9SHariprasad Shenai if ((pkt->l2info & htonl(RXF_TCP_F)) && 18863a336cb1SHariprasad Shenai !(cxgb_poll_busy_polling(q)) && 1887f7917c00SJeff Kirsher (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) { 1888f7917c00SJeff Kirsher do_gro(rxq, si, pkt); 1889f7917c00SJeff Kirsher return 0; 1890f7917c00SJeff Kirsher } 1891f7917c00SJeff Kirsher 1892f7917c00SJeff Kirsher skb = cxgb4_pktgl_to_skb(si, RX_PKT_SKB_LEN, RX_PULL_LEN); 1893f7917c00SJeff Kirsher if (unlikely(!skb)) { 1894f7917c00SJeff Kirsher t4_pktgl_free(si); 1895f7917c00SJeff Kirsher rxq->stats.rx_drops++; 1896f7917c00SJeff Kirsher return 0; 1897f7917c00SJeff Kirsher } 1898f7917c00SJeff Kirsher 189952367a76SVipul Pandya __skb_pull(skb, s->pktshift); /* remove ethernet header padding */ 1900f7917c00SJeff Kirsher skb->protocol = eth_type_trans(skb, q->netdev); 1901f7917c00SJeff Kirsher skb_record_rx_queue(skb, q->idx); 1902f7917c00SJeff Kirsher if (skb->dev->features & NETIF_F_RXHASH) 19038264989cSTom Herbert skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val, 19048264989cSTom Herbert PKT_HASH_TYPE_L3); 1905f7917c00SJeff Kirsher 1906f7917c00SJeff Kirsher rxq->stats.pkts++; 1907f7917c00SJeff Kirsher 1908bdc590b9SHariprasad Shenai if (csum_ok && (pkt->l2info & htonl(RXF_UDP_F | RXF_TCP_F))) { 1909f7917c00SJeff Kirsher if (!pkt->ip_frag) { 1910f7917c00SJeff Kirsher skb->ip_summed = CHECKSUM_UNNECESSARY; 1911f7917c00SJeff Kirsher rxq->stats.rx_cso++; 1912bdc590b9SHariprasad Shenai } else if (pkt->l2info & htonl(RXF_IP_F)) { 1913f7917c00SJeff Kirsher __sum16 c = (__force __sum16)pkt->csum; 1914f7917c00SJeff Kirsher skb->csum = csum_unfold(c); 1915f7917c00SJeff Kirsher skb->ip_summed = CHECKSUM_COMPLETE; 1916f7917c00SJeff Kirsher rxq->stats.rx_cso++; 1917f7917c00SJeff Kirsher } 191884a200b3SVarun Prakash } else { 1919f7917c00SJeff Kirsher skb_checksum_none_assert(skb); 192084a200b3SVarun Prakash #ifdef CONFIG_CHELSIO_T4_FCOE 192184a200b3SVarun Prakash #define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \ 192284a200b3SVarun Prakash RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F) 192384a200b3SVarun Prakash 192484a200b3SVarun Prakash pi = netdev_priv(skb->dev); 192584a200b3SVarun Prakash if (!(pkt->l2info & cpu_to_be32(CPL_RX_PKT_FLAGS))) { 192684a200b3SVarun Prakash if ((pkt->l2info & cpu_to_be32(RXF_FCOE_F)) && 192784a200b3SVarun Prakash (pi->fcoe.flags & CXGB_FCOE_ENABLED)) { 192884a200b3SVarun Prakash if (!(pkt->err_vec & cpu_to_be16(RXERR_CSUM_F))) 192984a200b3SVarun Prakash skb->ip_summed = CHECKSUM_UNNECESSARY; 193084a200b3SVarun Prakash } 193184a200b3SVarun Prakash } 193284a200b3SVarun Prakash 193384a200b3SVarun Prakash #undef CPL_RX_PKT_FLAGS 193484a200b3SVarun Prakash #endif /* CONFIG_CHELSIO_T4_FCOE */ 193584a200b3SVarun Prakash } 1936f7917c00SJeff Kirsher 1937f7917c00SJeff Kirsher if (unlikely(pkt->vlan_ex)) { 193886a9bad3SPatrick McHardy __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan)); 1939f7917c00SJeff Kirsher rxq->stats.vlan_ex++; 1940f7917c00SJeff Kirsher } 19413a336cb1SHariprasad Shenai skb_mark_napi_id(skb, &q->napi); 1942f7917c00SJeff Kirsher netif_receive_skb(skb); 1943f7917c00SJeff Kirsher return 0; 1944f7917c00SJeff Kirsher } 1945f7917c00SJeff Kirsher 1946f7917c00SJeff Kirsher /** 1947f7917c00SJeff Kirsher * restore_rx_bufs - put back a packet's Rx buffers 1948f7917c00SJeff Kirsher * @si: the packet gather list 1949f7917c00SJeff Kirsher * @q: the SGE free list 1950f7917c00SJeff Kirsher * @frags: number of FL buffers to restore 1951f7917c00SJeff Kirsher * 1952f7917c00SJeff Kirsher * Puts back on an FL the Rx buffers associated with @si. The buffers 1953f7917c00SJeff Kirsher * have already been unmapped and are left unmapped, we mark them so to 1954f7917c00SJeff Kirsher * prevent further unmapping attempts. 1955f7917c00SJeff Kirsher * 1956f7917c00SJeff Kirsher * This function undoes a series of @unmap_rx_buf calls when we find out 1957f7917c00SJeff Kirsher * that the current packet can't be processed right away afterall and we 1958f7917c00SJeff Kirsher * need to come back to it later. This is a very rare event and there's 1959f7917c00SJeff Kirsher * no effort to make this particularly efficient. 1960f7917c00SJeff Kirsher */ 1961f7917c00SJeff Kirsher static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q, 1962f7917c00SJeff Kirsher int frags) 1963f7917c00SJeff Kirsher { 1964f7917c00SJeff Kirsher struct rx_sw_desc *d; 1965f7917c00SJeff Kirsher 1966f7917c00SJeff Kirsher while (frags--) { 1967f7917c00SJeff Kirsher if (q->cidx == 0) 1968f7917c00SJeff Kirsher q->cidx = q->size - 1; 1969f7917c00SJeff Kirsher else 1970f7917c00SJeff Kirsher q->cidx--; 1971f7917c00SJeff Kirsher d = &q->sdesc[q->cidx]; 1972f7917c00SJeff Kirsher d->page = si->frags[frags].page; 1973f7917c00SJeff Kirsher d->dma_addr |= RX_UNMAPPED_BUF; 1974f7917c00SJeff Kirsher q->avail++; 1975f7917c00SJeff Kirsher } 1976f7917c00SJeff Kirsher } 1977f7917c00SJeff Kirsher 1978f7917c00SJeff Kirsher /** 1979f7917c00SJeff Kirsher * is_new_response - check if a response is newly written 1980f7917c00SJeff Kirsher * @r: the response descriptor 1981f7917c00SJeff Kirsher * @q: the response queue 1982f7917c00SJeff Kirsher * 1983f7917c00SJeff Kirsher * Returns true if a response descriptor contains a yet unprocessed 1984f7917c00SJeff Kirsher * response. 1985f7917c00SJeff Kirsher */ 1986f7917c00SJeff Kirsher static inline bool is_new_response(const struct rsp_ctrl *r, 1987f7917c00SJeff Kirsher const struct sge_rspq *q) 1988f7917c00SJeff Kirsher { 19891ecc7b7aSHariprasad Shenai return (r->type_gen >> RSPD_GEN_S) == q->gen; 1990f7917c00SJeff Kirsher } 1991f7917c00SJeff Kirsher 1992f7917c00SJeff Kirsher /** 1993f7917c00SJeff Kirsher * rspq_next - advance to the next entry in a response queue 1994f7917c00SJeff Kirsher * @q: the queue 1995f7917c00SJeff Kirsher * 1996f7917c00SJeff Kirsher * Updates the state of a response queue to advance it to the next entry. 1997f7917c00SJeff Kirsher */ 1998f7917c00SJeff Kirsher static inline void rspq_next(struct sge_rspq *q) 1999f7917c00SJeff Kirsher { 2000f7917c00SJeff Kirsher q->cur_desc = (void *)q->cur_desc + q->iqe_len; 2001f7917c00SJeff Kirsher if (unlikely(++q->cidx == q->size)) { 2002f7917c00SJeff Kirsher q->cidx = 0; 2003f7917c00SJeff Kirsher q->gen ^= 1; 2004f7917c00SJeff Kirsher q->cur_desc = q->desc; 2005f7917c00SJeff Kirsher } 2006f7917c00SJeff Kirsher } 2007f7917c00SJeff Kirsher 2008f7917c00SJeff Kirsher /** 2009f7917c00SJeff Kirsher * process_responses - process responses from an SGE response queue 2010f7917c00SJeff Kirsher * @q: the ingress queue to process 2011f7917c00SJeff Kirsher * @budget: how many responses can be processed in this round 2012f7917c00SJeff Kirsher * 2013f7917c00SJeff Kirsher * Process responses from an SGE response queue up to the supplied budget. 2014f7917c00SJeff Kirsher * Responses include received packets as well as control messages from FW 2015f7917c00SJeff Kirsher * or HW. 2016f7917c00SJeff Kirsher * 2017f7917c00SJeff Kirsher * Additionally choose the interrupt holdoff time for the next interrupt 2018f7917c00SJeff Kirsher * on this queue. If the system is under memory shortage use a fairly 2019f7917c00SJeff Kirsher * long delay to help recovery. 2020f7917c00SJeff Kirsher */ 2021f7917c00SJeff Kirsher static int process_responses(struct sge_rspq *q, int budget) 2022f7917c00SJeff Kirsher { 2023f7917c00SJeff Kirsher int ret, rsp_type; 2024f7917c00SJeff Kirsher int budget_left = budget; 2025f7917c00SJeff Kirsher const struct rsp_ctrl *rc; 2026f7917c00SJeff Kirsher struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); 202752367a76SVipul Pandya struct adapter *adapter = q->adap; 202852367a76SVipul Pandya struct sge *s = &adapter->sge; 2029f7917c00SJeff Kirsher 2030f7917c00SJeff Kirsher while (likely(budget_left)) { 2031f7917c00SJeff Kirsher rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc)); 2032f7917c00SJeff Kirsher if (!is_new_response(rc, q)) 2033f7917c00SJeff Kirsher break; 2034f7917c00SJeff Kirsher 2035019be1cfSAlexander Duyck dma_rmb(); 20361ecc7b7aSHariprasad Shenai rsp_type = RSPD_TYPE_G(rc->type_gen); 20371ecc7b7aSHariprasad Shenai if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) { 2038e91b0f24SIan Campbell struct page_frag *fp; 2039f7917c00SJeff Kirsher struct pkt_gl si; 2040f7917c00SJeff Kirsher const struct rx_sw_desc *rsd; 2041f7917c00SJeff Kirsher u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags; 2042f7917c00SJeff Kirsher 20431ecc7b7aSHariprasad Shenai if (len & RSPD_NEWBUF_F) { 2044f7917c00SJeff Kirsher if (likely(q->offset > 0)) { 2045f7917c00SJeff Kirsher free_rx_bufs(q->adap, &rxq->fl, 1); 2046f7917c00SJeff Kirsher q->offset = 0; 2047f7917c00SJeff Kirsher } 20481ecc7b7aSHariprasad Shenai len = RSPD_LEN_G(len); 2049f7917c00SJeff Kirsher } 2050f7917c00SJeff Kirsher si.tot_len = len; 2051f7917c00SJeff Kirsher 2052f7917c00SJeff Kirsher /* gather packet fragments */ 2053f7917c00SJeff Kirsher for (frags = 0, fp = si.frags; ; frags++, fp++) { 2054f7917c00SJeff Kirsher rsd = &rxq->fl.sdesc[rxq->fl.cidx]; 205552367a76SVipul Pandya bufsz = get_buf_size(adapter, rsd); 2056f7917c00SJeff Kirsher fp->page = rsd->page; 2057e91b0f24SIan Campbell fp->offset = q->offset; 2058e91b0f24SIan Campbell fp->size = min(bufsz, len); 2059e91b0f24SIan Campbell len -= fp->size; 2060f7917c00SJeff Kirsher if (!len) 2061f7917c00SJeff Kirsher break; 2062f7917c00SJeff Kirsher unmap_rx_buf(q->adap, &rxq->fl); 2063f7917c00SJeff Kirsher } 2064f7917c00SJeff Kirsher 2065f7917c00SJeff Kirsher /* 2066f7917c00SJeff Kirsher * Last buffer remains mapped so explicitly make it 2067f7917c00SJeff Kirsher * coherent for CPU access. 2068f7917c00SJeff Kirsher */ 2069f7917c00SJeff Kirsher dma_sync_single_for_cpu(q->adap->pdev_dev, 2070f7917c00SJeff Kirsher get_buf_addr(rsd), 2071e91b0f24SIan Campbell fp->size, DMA_FROM_DEVICE); 2072f7917c00SJeff Kirsher 2073f7917c00SJeff Kirsher si.va = page_address(si.frags[0].page) + 2074e91b0f24SIan Campbell si.frags[0].offset; 2075f7917c00SJeff Kirsher prefetch(si.va); 2076f7917c00SJeff Kirsher 2077f7917c00SJeff Kirsher si.nfrags = frags + 1; 2078f7917c00SJeff Kirsher ret = q->handler(q, q->cur_desc, &si); 2079f7917c00SJeff Kirsher if (likely(ret == 0)) 208052367a76SVipul Pandya q->offset += ALIGN(fp->size, s->fl_align); 2081f7917c00SJeff Kirsher else 2082f7917c00SJeff Kirsher restore_rx_bufs(&si, &rxq->fl, frags); 20831ecc7b7aSHariprasad Shenai } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) { 2084f7917c00SJeff Kirsher ret = q->handler(q, q->cur_desc, NULL); 2085f7917c00SJeff Kirsher } else { 2086f7917c00SJeff Kirsher ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN); 2087f7917c00SJeff Kirsher } 2088f7917c00SJeff Kirsher 2089f7917c00SJeff Kirsher if (unlikely(ret)) { 2090f7917c00SJeff Kirsher /* couldn't process descriptor, back off for recovery */ 20911ecc7b7aSHariprasad Shenai q->next_intr_params = QINTR_TIMER_IDX_V(NOMEM_TMR_IDX); 2092f7917c00SJeff Kirsher break; 2093f7917c00SJeff Kirsher } 2094f7917c00SJeff Kirsher 2095f7917c00SJeff Kirsher rspq_next(q); 2096f7917c00SJeff Kirsher budget_left--; 2097f7917c00SJeff Kirsher } 2098f7917c00SJeff Kirsher 2099f7917c00SJeff Kirsher if (q->offset >= 0 && rxq->fl.size - rxq->fl.avail >= 16) 2100f7917c00SJeff Kirsher __refill_fl(q->adap, &rxq->fl); 2101f7917c00SJeff Kirsher return budget - budget_left; 2102f7917c00SJeff Kirsher } 2103f7917c00SJeff Kirsher 21043a336cb1SHariprasad Shenai #ifdef CONFIG_NET_RX_BUSY_POLL 21053a336cb1SHariprasad Shenai int cxgb_busy_poll(struct napi_struct *napi) 21063a336cb1SHariprasad Shenai { 21073a336cb1SHariprasad Shenai struct sge_rspq *q = container_of(napi, struct sge_rspq, napi); 21083a336cb1SHariprasad Shenai unsigned int params, work_done; 21093a336cb1SHariprasad Shenai u32 val; 21103a336cb1SHariprasad Shenai 21113a336cb1SHariprasad Shenai if (!cxgb_poll_lock_poll(q)) 21123a336cb1SHariprasad Shenai return LL_FLUSH_BUSY; 21133a336cb1SHariprasad Shenai 21143a336cb1SHariprasad Shenai work_done = process_responses(q, 4); 21151ecc7b7aSHariprasad Shenai params = QINTR_TIMER_IDX_V(TIMERREG_COUNTER0_X) | QINTR_CNT_EN_V(1); 21163a336cb1SHariprasad Shenai q->next_intr_params = params; 21173a336cb1SHariprasad Shenai val = CIDXINC_V(work_done) | SEINTARM_V(params); 21183a336cb1SHariprasad Shenai 21193a336cb1SHariprasad Shenai /* If we don't have access to the new User GTS (T5+), use the old 21203a336cb1SHariprasad Shenai * doorbell mechanism; otherwise use the new BAR2 mechanism. 21213a336cb1SHariprasad Shenai */ 21223a336cb1SHariprasad Shenai if (unlikely(!q->bar2_addr)) 21233a336cb1SHariprasad Shenai t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A), 21243a336cb1SHariprasad Shenai val | INGRESSQID_V((u32)q->cntxt_id)); 21253a336cb1SHariprasad Shenai else { 21263a336cb1SHariprasad Shenai writel(val | INGRESSQID_V(q->bar2_qid), 21273a336cb1SHariprasad Shenai q->bar2_addr + SGE_UDB_GTS); 21283a336cb1SHariprasad Shenai wmb(); 21293a336cb1SHariprasad Shenai } 21303a336cb1SHariprasad Shenai 21313a336cb1SHariprasad Shenai cxgb_poll_unlock_poll(q); 21323a336cb1SHariprasad Shenai return work_done; 21333a336cb1SHariprasad Shenai } 21343a336cb1SHariprasad Shenai #endif /* CONFIG_NET_RX_BUSY_POLL */ 21353a336cb1SHariprasad Shenai 2136f7917c00SJeff Kirsher /** 2137f7917c00SJeff Kirsher * napi_rx_handler - the NAPI handler for Rx processing 2138f7917c00SJeff Kirsher * @napi: the napi instance 2139f7917c00SJeff Kirsher * @budget: how many packets we can process in this round 2140f7917c00SJeff Kirsher * 2141f7917c00SJeff Kirsher * Handler for new data events when using NAPI. This does not need any 2142f7917c00SJeff Kirsher * locking or protection from interrupts as data interrupts are off at 2143f7917c00SJeff Kirsher * this point and other adapter interrupts do not interfere (the latter 2144f7917c00SJeff Kirsher * in not a concern at all with MSI-X as non-data interrupts then have 2145f7917c00SJeff Kirsher * a separate handler). 2146f7917c00SJeff Kirsher */ 2147f7917c00SJeff Kirsher static int napi_rx_handler(struct napi_struct *napi, int budget) 2148f7917c00SJeff Kirsher { 2149f7917c00SJeff Kirsher unsigned int params; 2150f7917c00SJeff Kirsher struct sge_rspq *q = container_of(napi, struct sge_rspq, napi); 21513a336cb1SHariprasad Shenai int work_done; 2152d63a6dcfSHariprasad Shenai u32 val; 2153f7917c00SJeff Kirsher 21543a336cb1SHariprasad Shenai if (!cxgb_poll_lock_napi(q)) 21553a336cb1SHariprasad Shenai return budget; 21563a336cb1SHariprasad Shenai 21573a336cb1SHariprasad Shenai work_done = process_responses(q, budget); 2158f7917c00SJeff Kirsher if (likely(work_done < budget)) { 2159e553ec3fSHariprasad Shenai int timer_index; 2160e553ec3fSHariprasad Shenai 2161f7917c00SJeff Kirsher napi_complete(napi); 21621ecc7b7aSHariprasad Shenai timer_index = QINTR_TIMER_IDX_G(q->next_intr_params); 2163e553ec3fSHariprasad Shenai 2164e553ec3fSHariprasad Shenai if (q->adaptive_rx) { 2165e553ec3fSHariprasad Shenai if (work_done > max(timer_pkt_quota[timer_index], 2166e553ec3fSHariprasad Shenai MIN_NAPI_WORK)) 2167e553ec3fSHariprasad Shenai timer_index = (timer_index + 1); 2168e553ec3fSHariprasad Shenai else 2169e553ec3fSHariprasad Shenai timer_index = timer_index - 1; 2170e553ec3fSHariprasad Shenai 2171e553ec3fSHariprasad Shenai timer_index = clamp(timer_index, 0, SGE_TIMERREGS - 1); 21721ecc7b7aSHariprasad Shenai q->next_intr_params = 21731ecc7b7aSHariprasad Shenai QINTR_TIMER_IDX_V(timer_index) | 21741ecc7b7aSHariprasad Shenai QINTR_CNT_EN_V(0); 2175e553ec3fSHariprasad Shenai params = q->next_intr_params; 2176e553ec3fSHariprasad Shenai } else { 2177f7917c00SJeff Kirsher params = q->next_intr_params; 2178f7917c00SJeff Kirsher q->next_intr_params = q->intr_params; 2179e553ec3fSHariprasad Shenai } 2180f7917c00SJeff Kirsher } else 21811ecc7b7aSHariprasad Shenai params = QINTR_TIMER_IDX_V(7); 2182f7917c00SJeff Kirsher 2183f612b815SHariprasad Shenai val = CIDXINC_V(work_done) | SEINTARM_V(params); 2184df64e4d3SHariprasad Shenai 2185df64e4d3SHariprasad Shenai /* If we don't have access to the new User GTS (T5+), use the old 2186df64e4d3SHariprasad Shenai * doorbell mechanism; otherwise use the new BAR2 mechanism. 2187df64e4d3SHariprasad Shenai */ 2188df64e4d3SHariprasad Shenai if (unlikely(q->bar2_addr == NULL)) { 2189f612b815SHariprasad Shenai t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A), 2190f612b815SHariprasad Shenai val | INGRESSQID_V((u32)q->cntxt_id)); 2191d63a6dcfSHariprasad Shenai } else { 2192f612b815SHariprasad Shenai writel(val | INGRESSQID_V(q->bar2_qid), 2193df64e4d3SHariprasad Shenai q->bar2_addr + SGE_UDB_GTS); 2194d63a6dcfSHariprasad Shenai wmb(); 2195d63a6dcfSHariprasad Shenai } 21963a336cb1SHariprasad Shenai cxgb_poll_unlock_napi(q); 2197f7917c00SJeff Kirsher return work_done; 2198f7917c00SJeff Kirsher } 2199f7917c00SJeff Kirsher 2200f7917c00SJeff Kirsher /* 2201f7917c00SJeff Kirsher * The MSI-X interrupt handler for an SGE response queue. 2202f7917c00SJeff Kirsher */ 2203f7917c00SJeff Kirsher irqreturn_t t4_sge_intr_msix(int irq, void *cookie) 2204f7917c00SJeff Kirsher { 2205f7917c00SJeff Kirsher struct sge_rspq *q = cookie; 2206f7917c00SJeff Kirsher 2207f7917c00SJeff Kirsher napi_schedule(&q->napi); 2208f7917c00SJeff Kirsher return IRQ_HANDLED; 2209f7917c00SJeff Kirsher } 2210f7917c00SJeff Kirsher 2211f7917c00SJeff Kirsher /* 2212f7917c00SJeff Kirsher * Process the indirect interrupt entries in the interrupt queue and kick off 2213f7917c00SJeff Kirsher * NAPI for each queue that has generated an entry. 2214f7917c00SJeff Kirsher */ 2215f7917c00SJeff Kirsher static unsigned int process_intrq(struct adapter *adap) 2216f7917c00SJeff Kirsher { 2217f7917c00SJeff Kirsher unsigned int credits; 2218f7917c00SJeff Kirsher const struct rsp_ctrl *rc; 2219f7917c00SJeff Kirsher struct sge_rspq *q = &adap->sge.intrq; 2220d63a6dcfSHariprasad Shenai u32 val; 2221f7917c00SJeff Kirsher 2222f7917c00SJeff Kirsher spin_lock(&adap->sge.intrq_lock); 2223f7917c00SJeff Kirsher for (credits = 0; ; credits++) { 2224f7917c00SJeff Kirsher rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc)); 2225f7917c00SJeff Kirsher if (!is_new_response(rc, q)) 2226f7917c00SJeff Kirsher break; 2227f7917c00SJeff Kirsher 2228019be1cfSAlexander Duyck dma_rmb(); 22291ecc7b7aSHariprasad Shenai if (RSPD_TYPE_G(rc->type_gen) == RSPD_TYPE_INTR_X) { 2230f7917c00SJeff Kirsher unsigned int qid = ntohl(rc->pldbuflen_qid); 2231f7917c00SJeff Kirsher 2232f7917c00SJeff Kirsher qid -= adap->sge.ingr_start; 2233f7917c00SJeff Kirsher napi_schedule(&adap->sge.ingr_map[qid]->napi); 2234f7917c00SJeff Kirsher } 2235f7917c00SJeff Kirsher 2236f7917c00SJeff Kirsher rspq_next(q); 2237f7917c00SJeff Kirsher } 2238f7917c00SJeff Kirsher 2239f612b815SHariprasad Shenai val = CIDXINC_V(credits) | SEINTARM_V(q->intr_params); 2240df64e4d3SHariprasad Shenai 2241df64e4d3SHariprasad Shenai /* If we don't have access to the new User GTS (T5+), use the old 2242df64e4d3SHariprasad Shenai * doorbell mechanism; otherwise use the new BAR2 mechanism. 2243df64e4d3SHariprasad Shenai */ 2244df64e4d3SHariprasad Shenai if (unlikely(q->bar2_addr == NULL)) { 2245f612b815SHariprasad Shenai t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 2246f612b815SHariprasad Shenai val | INGRESSQID_V(q->cntxt_id)); 2247d63a6dcfSHariprasad Shenai } else { 2248f612b815SHariprasad Shenai writel(val | INGRESSQID_V(q->bar2_qid), 2249df64e4d3SHariprasad Shenai q->bar2_addr + SGE_UDB_GTS); 2250d63a6dcfSHariprasad Shenai wmb(); 2251d63a6dcfSHariprasad Shenai } 2252f7917c00SJeff Kirsher spin_unlock(&adap->sge.intrq_lock); 2253f7917c00SJeff Kirsher return credits; 2254f7917c00SJeff Kirsher } 2255f7917c00SJeff Kirsher 2256f7917c00SJeff Kirsher /* 2257f7917c00SJeff Kirsher * The MSI interrupt handler, which handles data events from SGE response queues 2258f7917c00SJeff Kirsher * as well as error and other async events as they all use the same MSI vector. 2259f7917c00SJeff Kirsher */ 2260f7917c00SJeff Kirsher static irqreturn_t t4_intr_msi(int irq, void *cookie) 2261f7917c00SJeff Kirsher { 2262f7917c00SJeff Kirsher struct adapter *adap = cookie; 2263f7917c00SJeff Kirsher 2264c3c7b121SHariprasad Shenai if (adap->flags & MASTER_PF) 2265f7917c00SJeff Kirsher t4_slow_intr_handler(adap); 2266f7917c00SJeff Kirsher process_intrq(adap); 2267f7917c00SJeff Kirsher return IRQ_HANDLED; 2268f7917c00SJeff Kirsher } 2269f7917c00SJeff Kirsher 2270f7917c00SJeff Kirsher /* 2271f7917c00SJeff Kirsher * Interrupt handler for legacy INTx interrupts. 2272f7917c00SJeff Kirsher * Handles data events from SGE response queues as well as error and other 2273f7917c00SJeff Kirsher * async events as they all use the same interrupt line. 2274f7917c00SJeff Kirsher */ 2275f7917c00SJeff Kirsher static irqreturn_t t4_intr_intx(int irq, void *cookie) 2276f7917c00SJeff Kirsher { 2277f7917c00SJeff Kirsher struct adapter *adap = cookie; 2278f7917c00SJeff Kirsher 2279f061de42SHariprasad Shenai t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0); 2280c3c7b121SHariprasad Shenai if (((adap->flags & MASTER_PF) && t4_slow_intr_handler(adap)) | 2281c3c7b121SHariprasad Shenai process_intrq(adap)) 2282f7917c00SJeff Kirsher return IRQ_HANDLED; 2283f7917c00SJeff Kirsher return IRQ_NONE; /* probably shared interrupt */ 2284f7917c00SJeff Kirsher } 2285f7917c00SJeff Kirsher 2286f7917c00SJeff Kirsher /** 2287f7917c00SJeff Kirsher * t4_intr_handler - select the top-level interrupt handler 2288f7917c00SJeff Kirsher * @adap: the adapter 2289f7917c00SJeff Kirsher * 2290f7917c00SJeff Kirsher * Selects the top-level interrupt handler based on the type of interrupts 2291f7917c00SJeff Kirsher * (MSI-X, MSI, or INTx). 2292f7917c00SJeff Kirsher */ 2293f7917c00SJeff Kirsher irq_handler_t t4_intr_handler(struct adapter *adap) 2294f7917c00SJeff Kirsher { 2295f7917c00SJeff Kirsher if (adap->flags & USING_MSIX) 2296f7917c00SJeff Kirsher return t4_sge_intr_msix; 2297f7917c00SJeff Kirsher if (adap->flags & USING_MSI) 2298f7917c00SJeff Kirsher return t4_intr_msi; 2299f7917c00SJeff Kirsher return t4_intr_intx; 2300f7917c00SJeff Kirsher } 2301f7917c00SJeff Kirsher 2302f7917c00SJeff Kirsher static void sge_rx_timer_cb(unsigned long data) 2303f7917c00SJeff Kirsher { 2304f7917c00SJeff Kirsher unsigned long m; 2305a3bfb617SHariprasad Shenai unsigned int i; 2306f7917c00SJeff Kirsher struct adapter *adap = (struct adapter *)data; 2307f7917c00SJeff Kirsher struct sge *s = &adap->sge; 2308f7917c00SJeff Kirsher 23094b8e27a8SHariprasad Shenai for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++) 2310f7917c00SJeff Kirsher for (m = s->starving_fl[i]; m; m &= m - 1) { 2311f7917c00SJeff Kirsher struct sge_eth_rxq *rxq; 2312f7917c00SJeff Kirsher unsigned int id = __ffs(m) + i * BITS_PER_LONG; 2313f7917c00SJeff Kirsher struct sge_fl *fl = s->egr_map[id]; 2314f7917c00SJeff Kirsher 2315f7917c00SJeff Kirsher clear_bit(id, s->starving_fl); 23164e857c58SPeter Zijlstra smp_mb__after_atomic(); 2317f7917c00SJeff Kirsher 2318c098b026SHariprasad Shenai if (fl_starving(adap, fl)) { 2319f7917c00SJeff Kirsher rxq = container_of(fl, struct sge_eth_rxq, fl); 2320f7917c00SJeff Kirsher if (napi_reschedule(&rxq->rspq.napi)) 2321f7917c00SJeff Kirsher fl->starving++; 2322f7917c00SJeff Kirsher else 2323f7917c00SJeff Kirsher set_bit(id, s->starving_fl); 2324f7917c00SJeff Kirsher } 2325f7917c00SJeff Kirsher } 2326a3bfb617SHariprasad Shenai /* The remainder of the SGE RX Timer Callback routine is dedicated to 2327a3bfb617SHariprasad Shenai * global Master PF activities like checking for chip ingress stalls, 2328a3bfb617SHariprasad Shenai * etc. 23290f4d201fSKumar Sanghvi */ 2330a3bfb617SHariprasad Shenai if (!(adap->flags & MASTER_PF)) 2331a3bfb617SHariprasad Shenai goto done; 23320f4d201fSKumar Sanghvi 2333a3bfb617SHariprasad Shenai t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD); 23340f4d201fSKumar Sanghvi 2335a3bfb617SHariprasad Shenai done: 2336f7917c00SJeff Kirsher mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD); 2337f7917c00SJeff Kirsher } 2338f7917c00SJeff Kirsher 2339f7917c00SJeff Kirsher static void sge_tx_timer_cb(unsigned long data) 2340f7917c00SJeff Kirsher { 2341f7917c00SJeff Kirsher unsigned long m; 2342f7917c00SJeff Kirsher unsigned int i, budget; 2343f7917c00SJeff Kirsher struct adapter *adap = (struct adapter *)data; 2344f7917c00SJeff Kirsher struct sge *s = &adap->sge; 2345f7917c00SJeff Kirsher 23464b8e27a8SHariprasad Shenai for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++) 2347f7917c00SJeff Kirsher for (m = s->txq_maperr[i]; m; m &= m - 1) { 2348f7917c00SJeff Kirsher unsigned long id = __ffs(m) + i * BITS_PER_LONG; 2349f7917c00SJeff Kirsher struct sge_ofld_txq *txq = s->egr_map[id]; 2350f7917c00SJeff Kirsher 2351f7917c00SJeff Kirsher clear_bit(id, s->txq_maperr); 2352f7917c00SJeff Kirsher tasklet_schedule(&txq->qresume_tsk); 2353f7917c00SJeff Kirsher } 2354f7917c00SJeff Kirsher 2355f7917c00SJeff Kirsher budget = MAX_TIMER_TX_RECLAIM; 2356f7917c00SJeff Kirsher i = s->ethtxq_rover; 2357f7917c00SJeff Kirsher do { 2358f7917c00SJeff Kirsher struct sge_eth_txq *q = &s->ethtxq[i]; 2359f7917c00SJeff Kirsher 2360f7917c00SJeff Kirsher if (q->q.in_use && 2361f7917c00SJeff Kirsher time_after_eq(jiffies, q->txq->trans_start + HZ / 100) && 2362f7917c00SJeff Kirsher __netif_tx_trylock(q->txq)) { 2363f7917c00SJeff Kirsher int avail = reclaimable(&q->q); 2364f7917c00SJeff Kirsher 2365f7917c00SJeff Kirsher if (avail) { 2366f7917c00SJeff Kirsher if (avail > budget) 2367f7917c00SJeff Kirsher avail = budget; 2368f7917c00SJeff Kirsher 2369f7917c00SJeff Kirsher free_tx_desc(adap, &q->q, avail, true); 2370f7917c00SJeff Kirsher q->q.in_use -= avail; 2371f7917c00SJeff Kirsher budget -= avail; 2372f7917c00SJeff Kirsher } 2373f7917c00SJeff Kirsher __netif_tx_unlock(q->txq); 2374f7917c00SJeff Kirsher } 2375f7917c00SJeff Kirsher 2376f7917c00SJeff Kirsher if (++i >= s->ethqsets) 2377f7917c00SJeff Kirsher i = 0; 2378f7917c00SJeff Kirsher } while (budget && i != s->ethtxq_rover); 2379f7917c00SJeff Kirsher s->ethtxq_rover = i; 2380f7917c00SJeff Kirsher mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2)); 2381f7917c00SJeff Kirsher } 2382f7917c00SJeff Kirsher 2383d63a6dcfSHariprasad Shenai /** 2384df64e4d3SHariprasad Shenai * bar2_address - return the BAR2 address for an SGE Queue's Registers 2385df64e4d3SHariprasad Shenai * @adapter: the adapter 2386df64e4d3SHariprasad Shenai * @qid: the SGE Queue ID 2387df64e4d3SHariprasad Shenai * @qtype: the SGE Queue Type (Egress or Ingress) 2388df64e4d3SHariprasad Shenai * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues 2389d63a6dcfSHariprasad Shenai * 2390df64e4d3SHariprasad Shenai * Returns the BAR2 address for the SGE Queue Registers associated with 2391df64e4d3SHariprasad Shenai * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also 2392df64e4d3SHariprasad Shenai * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE 2393df64e4d3SHariprasad Shenai * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID" 2394df64e4d3SHariprasad Shenai * Registers are supported (e.g. the Write Combining Doorbell Buffer). 2395d63a6dcfSHariprasad Shenai */ 2396df64e4d3SHariprasad Shenai static void __iomem *bar2_address(struct adapter *adapter, 2397df64e4d3SHariprasad Shenai unsigned int qid, 2398df64e4d3SHariprasad Shenai enum t4_bar2_qtype qtype, 2399df64e4d3SHariprasad Shenai unsigned int *pbar2_qid) 2400d63a6dcfSHariprasad Shenai { 2401df64e4d3SHariprasad Shenai u64 bar2_qoffset; 2402df64e4d3SHariprasad Shenai int ret; 2403d63a6dcfSHariprasad Shenai 2404e0456717SLinus Torvalds ret = t4_bar2_sge_qregs(adapter, qid, qtype, 0, 2405df64e4d3SHariprasad Shenai &bar2_qoffset, pbar2_qid); 2406df64e4d3SHariprasad Shenai if (ret) 2407df64e4d3SHariprasad Shenai return NULL; 2408d63a6dcfSHariprasad Shenai 2409df64e4d3SHariprasad Shenai return adapter->bar2 + bar2_qoffset; 2410d63a6dcfSHariprasad Shenai } 2411d63a6dcfSHariprasad Shenai 2412145ef8a5SHariprasad Shenai /* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0 2413145ef8a5SHariprasad Shenai * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map 2414145ef8a5SHariprasad Shenai */ 2415f7917c00SJeff Kirsher int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, 2416f7917c00SJeff Kirsher struct net_device *dev, int intr_idx, 2417145ef8a5SHariprasad Shenai struct sge_fl *fl, rspq_handler_t hnd, int cong) 2418f7917c00SJeff Kirsher { 2419f7917c00SJeff Kirsher int ret, flsz = 0; 2420f7917c00SJeff Kirsher struct fw_iq_cmd c; 242152367a76SVipul Pandya struct sge *s = &adap->sge; 2422f7917c00SJeff Kirsher struct port_info *pi = netdev_priv(dev); 2423f7917c00SJeff Kirsher 2424f7917c00SJeff Kirsher /* Size needs to be multiple of 16, including status entry. */ 2425f7917c00SJeff Kirsher iq->size = roundup(iq->size, 16); 2426f7917c00SJeff Kirsher 2427f7917c00SJeff Kirsher iq->desc = alloc_ring(adap->pdev_dev, iq->size, iq->iqe_len, 0, 2428f7917c00SJeff Kirsher &iq->phys_addr, NULL, 0, NUMA_NO_NODE); 2429f7917c00SJeff Kirsher if (!iq->desc) 2430f7917c00SJeff Kirsher return -ENOMEM; 2431f7917c00SJeff Kirsher 2432f7917c00SJeff Kirsher memset(&c, 0, sizeof(c)); 2433e2ac9628SHariprasad Shenai c.op_to_vfn = htonl(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F | 2434e2ac9628SHariprasad Shenai FW_CMD_WRITE_F | FW_CMD_EXEC_F | 2435b2612722SHariprasad Shenai FW_IQ_CMD_PFN_V(adap->pf) | FW_IQ_CMD_VFN_V(0)); 24366e4b51a6SHariprasad Shenai c.alloc_to_len16 = htonl(FW_IQ_CMD_ALLOC_F | FW_IQ_CMD_IQSTART_F | 2437f7917c00SJeff Kirsher FW_LEN16(c)); 24386e4b51a6SHariprasad Shenai c.type_to_iqandstindex = htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) | 24396e4b51a6SHariprasad Shenai FW_IQ_CMD_IQASYNCH_V(fwevtq) | FW_IQ_CMD_VIID_V(pi->viid) | 24401ecc7b7aSHariprasad Shenai FW_IQ_CMD_IQANDST_V(intr_idx < 0) | 24411ecc7b7aSHariprasad Shenai FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X) | 24426e4b51a6SHariprasad Shenai FW_IQ_CMD_IQANDSTINDEX_V(intr_idx >= 0 ? intr_idx : 2443f7917c00SJeff Kirsher -intr_idx - 1)); 24446e4b51a6SHariprasad Shenai c.iqdroprss_to_iqesize = htons(FW_IQ_CMD_IQPCIECH_V(pi->tx_chan) | 24456e4b51a6SHariprasad Shenai FW_IQ_CMD_IQGTSMODE_F | 24466e4b51a6SHariprasad Shenai FW_IQ_CMD_IQINTCNTTHRESH_V(iq->pktcnt_idx) | 24476e4b51a6SHariprasad Shenai FW_IQ_CMD_IQESIZE_V(ilog2(iq->iqe_len) - 4)); 2448f7917c00SJeff Kirsher c.iqsize = htons(iq->size); 2449f7917c00SJeff Kirsher c.iqaddr = cpu_to_be64(iq->phys_addr); 2450145ef8a5SHariprasad Shenai if (cong >= 0) 2451145ef8a5SHariprasad Shenai c.iqns_to_fl0congen = htonl(FW_IQ_CMD_IQFLINTCONGEN_F); 2452f7917c00SJeff Kirsher 2453f7917c00SJeff Kirsher if (fl) { 24543ccc6cf7SHariprasad Shenai enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip); 24553ccc6cf7SHariprasad Shenai 245613432997SHariprasad Shenai /* Allocate the ring for the hardware free list (with space 245713432997SHariprasad Shenai * for its status page) along with the associated software 245813432997SHariprasad Shenai * descriptor ring. The free list size needs to be a multiple 245913432997SHariprasad Shenai * of the Egress Queue Unit and at least 2 Egress Units larger 246013432997SHariprasad Shenai * than the SGE's Egress Congrestion Threshold 246113432997SHariprasad Shenai * (fl_starve_thres - 1). 246213432997SHariprasad Shenai */ 246313432997SHariprasad Shenai if (fl->size < s->fl_starve_thres - 1 + 2 * 8) 246413432997SHariprasad Shenai fl->size = s->fl_starve_thres - 1 + 2 * 8; 2465f7917c00SJeff Kirsher fl->size = roundup(fl->size, 8); 2466f7917c00SJeff Kirsher fl->desc = alloc_ring(adap->pdev_dev, fl->size, sizeof(__be64), 2467f7917c00SJeff Kirsher sizeof(struct rx_sw_desc), &fl->addr, 246852367a76SVipul Pandya &fl->sdesc, s->stat_len, NUMA_NO_NODE); 2469f7917c00SJeff Kirsher if (!fl->desc) 2470f7917c00SJeff Kirsher goto fl_nomem; 2471f7917c00SJeff Kirsher 247252367a76SVipul Pandya flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc); 2473145ef8a5SHariprasad Shenai c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F | 24746e4b51a6SHariprasad Shenai FW_IQ_CMD_FL0FETCHRO_F | 24756e4b51a6SHariprasad Shenai FW_IQ_CMD_FL0DATARO_F | 24766e4b51a6SHariprasad Shenai FW_IQ_CMD_FL0PADEN_F); 2477145ef8a5SHariprasad Shenai if (cong >= 0) 2478145ef8a5SHariprasad Shenai c.iqns_to_fl0congen |= 2479145ef8a5SHariprasad Shenai htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong) | 2480145ef8a5SHariprasad Shenai FW_IQ_CMD_FL0CONGCIF_F | 2481145ef8a5SHariprasad Shenai FW_IQ_CMD_FL0CONGEN_F); 24821ecc7b7aSHariprasad Shenai c.fl0dcaen_to_fl0cidxfthresh = 24831ecc7b7aSHariprasad Shenai htons(FW_IQ_CMD_FL0FBMIN_V(FETCHBURSTMIN_64B_X) | 24843ccc6cf7SHariprasad Shenai FW_IQ_CMD_FL0FBMAX_V((chip <= CHELSIO_T5) ? 24853ccc6cf7SHariprasad Shenai FETCHBURSTMAX_512B_X : 24863ccc6cf7SHariprasad Shenai FETCHBURSTMAX_256B_X)); 2487f7917c00SJeff Kirsher c.fl0size = htons(flsz); 2488f7917c00SJeff Kirsher c.fl0addr = cpu_to_be64(fl->addr); 2489f7917c00SJeff Kirsher } 2490f7917c00SJeff Kirsher 2491b2612722SHariprasad Shenai ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); 2492f7917c00SJeff Kirsher if (ret) 2493f7917c00SJeff Kirsher goto err; 2494f7917c00SJeff Kirsher 2495f7917c00SJeff Kirsher netif_napi_add(dev, &iq->napi, napi_rx_handler, 64); 24963a336cb1SHariprasad Shenai napi_hash_add(&iq->napi); 2497f7917c00SJeff Kirsher iq->cur_desc = iq->desc; 2498f7917c00SJeff Kirsher iq->cidx = 0; 2499f7917c00SJeff Kirsher iq->gen = 1; 2500f7917c00SJeff Kirsher iq->next_intr_params = iq->intr_params; 2501f7917c00SJeff Kirsher iq->cntxt_id = ntohs(c.iqid); 2502f7917c00SJeff Kirsher iq->abs_id = ntohs(c.physiqid); 2503df64e4d3SHariprasad Shenai iq->bar2_addr = bar2_address(adap, 2504df64e4d3SHariprasad Shenai iq->cntxt_id, 2505df64e4d3SHariprasad Shenai T4_BAR2_QTYPE_INGRESS, 2506df64e4d3SHariprasad Shenai &iq->bar2_qid); 2507f7917c00SJeff Kirsher iq->size--; /* subtract status entry */ 2508f7917c00SJeff Kirsher iq->netdev = dev; 2509f7917c00SJeff Kirsher iq->handler = hnd; 2510f7917c00SJeff Kirsher 2511f7917c00SJeff Kirsher /* set offset to -1 to distinguish ingress queues without FL */ 2512f7917c00SJeff Kirsher iq->offset = fl ? 0 : -1; 2513f7917c00SJeff Kirsher 2514f7917c00SJeff Kirsher adap->sge.ingr_map[iq->cntxt_id - adap->sge.ingr_start] = iq; 2515f7917c00SJeff Kirsher 2516f7917c00SJeff Kirsher if (fl) { 2517f7917c00SJeff Kirsher fl->cntxt_id = ntohs(c.fl0id); 2518f7917c00SJeff Kirsher fl->avail = fl->pend_cred = 0; 2519f7917c00SJeff Kirsher fl->pidx = fl->cidx = 0; 2520f7917c00SJeff Kirsher fl->alloc_failed = fl->large_alloc_failed = fl->starving = 0; 2521f7917c00SJeff Kirsher adap->sge.egr_map[fl->cntxt_id - adap->sge.egr_start] = fl; 2522d63a6dcfSHariprasad Shenai 2523df64e4d3SHariprasad Shenai /* Note, we must initialize the BAR2 Free List User Doorbell 2524df64e4d3SHariprasad Shenai * information before refilling the Free List! 2525d63a6dcfSHariprasad Shenai */ 2526df64e4d3SHariprasad Shenai fl->bar2_addr = bar2_address(adap, 2527df64e4d3SHariprasad Shenai fl->cntxt_id, 2528df64e4d3SHariprasad Shenai T4_BAR2_QTYPE_EGRESS, 2529df64e4d3SHariprasad Shenai &fl->bar2_qid); 2530f7917c00SJeff Kirsher refill_fl(adap, fl, fl_cap(fl), GFP_KERNEL); 2531f7917c00SJeff Kirsher } 2532b8b1ae99SHariprasad Shenai 2533b8b1ae99SHariprasad Shenai /* For T5 and later we attempt to set up the Congestion Manager values 2534b8b1ae99SHariprasad Shenai * of the new RX Ethernet Queue. This should really be handled by 2535b8b1ae99SHariprasad Shenai * firmware because it's more complex than any host driver wants to 2536b8b1ae99SHariprasad Shenai * get involved with and it's different per chip and this is almost 2537b8b1ae99SHariprasad Shenai * certainly wrong. Firmware would be wrong as well, but it would be 2538b8b1ae99SHariprasad Shenai * a lot easier to fix in one place ... For now we do something very 2539b8b1ae99SHariprasad Shenai * simple (and hopefully less wrong). 2540b8b1ae99SHariprasad Shenai */ 2541b8b1ae99SHariprasad Shenai if (!is_t4(adap->params.chip) && cong >= 0) { 2542b8b1ae99SHariprasad Shenai u32 param, val; 2543b8b1ae99SHariprasad Shenai int i; 2544b8b1ae99SHariprasad Shenai 2545b8b1ae99SHariprasad Shenai param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 2546b8b1ae99SHariprasad Shenai FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 2547b8b1ae99SHariprasad Shenai FW_PARAMS_PARAM_YZ_V(iq->cntxt_id)); 2548b8b1ae99SHariprasad Shenai if (cong == 0) { 2549b8b1ae99SHariprasad Shenai val = CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X); 2550b8b1ae99SHariprasad Shenai } else { 2551b8b1ae99SHariprasad Shenai val = 2552b8b1ae99SHariprasad Shenai CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X); 2553b8b1ae99SHariprasad Shenai for (i = 0; i < 4; i++) { 2554b8b1ae99SHariprasad Shenai if (cong & (1 << i)) 2555b8b1ae99SHariprasad Shenai val |= 2556b8b1ae99SHariprasad Shenai CONMCTXT_CNGCHMAP_V(1 << (i << 2)); 2557b8b1ae99SHariprasad Shenai } 2558b8b1ae99SHariprasad Shenai } 2559b2612722SHariprasad Shenai ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, 2560b8b1ae99SHariprasad Shenai ¶m, &val); 2561b8b1ae99SHariprasad Shenai if (ret) 2562b8b1ae99SHariprasad Shenai dev_warn(adap->pdev_dev, "Failed to set Congestion" 2563b8b1ae99SHariprasad Shenai " Manager Context for Ingress Queue %d: %d\n", 2564b8b1ae99SHariprasad Shenai iq->cntxt_id, -ret); 2565b8b1ae99SHariprasad Shenai } 2566b8b1ae99SHariprasad Shenai 2567f7917c00SJeff Kirsher return 0; 2568f7917c00SJeff Kirsher 2569f7917c00SJeff Kirsher fl_nomem: 2570f7917c00SJeff Kirsher ret = -ENOMEM; 2571f7917c00SJeff Kirsher err: 2572f7917c00SJeff Kirsher if (iq->desc) { 2573f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, iq->size * iq->iqe_len, 2574f7917c00SJeff Kirsher iq->desc, iq->phys_addr); 2575f7917c00SJeff Kirsher iq->desc = NULL; 2576f7917c00SJeff Kirsher } 2577f7917c00SJeff Kirsher if (fl && fl->desc) { 2578f7917c00SJeff Kirsher kfree(fl->sdesc); 2579f7917c00SJeff Kirsher fl->sdesc = NULL; 2580f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, flsz * sizeof(struct tx_desc), 2581f7917c00SJeff Kirsher fl->desc, fl->addr); 2582f7917c00SJeff Kirsher fl->desc = NULL; 2583f7917c00SJeff Kirsher } 2584f7917c00SJeff Kirsher return ret; 2585f7917c00SJeff Kirsher } 2586f7917c00SJeff Kirsher 2587f7917c00SJeff Kirsher static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id) 2588f7917c00SJeff Kirsher { 258922adfe0aSSantosh Rastapur q->cntxt_id = id; 2590df64e4d3SHariprasad Shenai q->bar2_addr = bar2_address(adap, 2591df64e4d3SHariprasad Shenai q->cntxt_id, 2592df64e4d3SHariprasad Shenai T4_BAR2_QTYPE_EGRESS, 2593df64e4d3SHariprasad Shenai &q->bar2_qid); 2594f7917c00SJeff Kirsher q->in_use = 0; 2595f7917c00SJeff Kirsher q->cidx = q->pidx = 0; 2596f7917c00SJeff Kirsher q->stops = q->restarts = 0; 2597f7917c00SJeff Kirsher q->stat = (void *)&q->desc[q->size]; 25983069ee9bSVipul Pandya spin_lock_init(&q->db_lock); 2599f7917c00SJeff Kirsher adap->sge.egr_map[id - adap->sge.egr_start] = q; 2600f7917c00SJeff Kirsher } 2601f7917c00SJeff Kirsher 2602f7917c00SJeff Kirsher int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, 2603f7917c00SJeff Kirsher struct net_device *dev, struct netdev_queue *netdevq, 2604f7917c00SJeff Kirsher unsigned int iqid) 2605f7917c00SJeff Kirsher { 2606f7917c00SJeff Kirsher int ret, nentries; 2607f7917c00SJeff Kirsher struct fw_eq_eth_cmd c; 260852367a76SVipul Pandya struct sge *s = &adap->sge; 2609f7917c00SJeff Kirsher struct port_info *pi = netdev_priv(dev); 2610f7917c00SJeff Kirsher 2611f7917c00SJeff Kirsher /* Add status entries */ 261252367a76SVipul Pandya nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); 2613f7917c00SJeff Kirsher 2614f7917c00SJeff Kirsher txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size, 2615f7917c00SJeff Kirsher sizeof(struct tx_desc), sizeof(struct tx_sw_desc), 261652367a76SVipul Pandya &txq->q.phys_addr, &txq->q.sdesc, s->stat_len, 2617f7917c00SJeff Kirsher netdev_queue_numa_node_read(netdevq)); 2618f7917c00SJeff Kirsher if (!txq->q.desc) 2619f7917c00SJeff Kirsher return -ENOMEM; 2620f7917c00SJeff Kirsher 2621f7917c00SJeff Kirsher memset(&c, 0, sizeof(c)); 2622e2ac9628SHariprasad Shenai c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD) | FW_CMD_REQUEST_F | 2623e2ac9628SHariprasad Shenai FW_CMD_WRITE_F | FW_CMD_EXEC_F | 2624b2612722SHariprasad Shenai FW_EQ_ETH_CMD_PFN_V(adap->pf) | 26256e4b51a6SHariprasad Shenai FW_EQ_ETH_CMD_VFN_V(0)); 26266e4b51a6SHariprasad Shenai c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F | 26276e4b51a6SHariprasad Shenai FW_EQ_ETH_CMD_EQSTART_F | FW_LEN16(c)); 26286e4b51a6SHariprasad Shenai c.viid_pkd = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F | 26296e4b51a6SHariprasad Shenai FW_EQ_ETH_CMD_VIID_V(pi->viid)); 26301ecc7b7aSHariprasad Shenai c.fetchszm_to_iqid = 26311ecc7b7aSHariprasad Shenai htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) | 26326e4b51a6SHariprasad Shenai FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) | 26331ecc7b7aSHariprasad Shenai FW_EQ_ETH_CMD_FETCHRO_F | FW_EQ_ETH_CMD_IQID_V(iqid)); 26341ecc7b7aSHariprasad Shenai c.dcaen_to_eqsize = 26351ecc7b7aSHariprasad Shenai htonl(FW_EQ_ETH_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) | 26361ecc7b7aSHariprasad Shenai FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | 26371ecc7b7aSHariprasad Shenai FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) | 26386e4b51a6SHariprasad Shenai FW_EQ_ETH_CMD_EQSIZE_V(nentries)); 2639f7917c00SJeff Kirsher c.eqaddr = cpu_to_be64(txq->q.phys_addr); 2640f7917c00SJeff Kirsher 2641b2612722SHariprasad Shenai ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); 2642f7917c00SJeff Kirsher if (ret) { 2643f7917c00SJeff Kirsher kfree(txq->q.sdesc); 2644f7917c00SJeff Kirsher txq->q.sdesc = NULL; 2645f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, 2646f7917c00SJeff Kirsher nentries * sizeof(struct tx_desc), 2647f7917c00SJeff Kirsher txq->q.desc, txq->q.phys_addr); 2648f7917c00SJeff Kirsher txq->q.desc = NULL; 2649f7917c00SJeff Kirsher return ret; 2650f7917c00SJeff Kirsher } 2651f7917c00SJeff Kirsher 26526e4b51a6SHariprasad Shenai init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd))); 2653f7917c00SJeff Kirsher txq->txq = netdevq; 2654f7917c00SJeff Kirsher txq->tso = txq->tx_cso = txq->vlan_ins = 0; 2655f7917c00SJeff Kirsher txq->mapping_err = 0; 2656f7917c00SJeff Kirsher return 0; 2657f7917c00SJeff Kirsher } 2658f7917c00SJeff Kirsher 2659f7917c00SJeff Kirsher int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq, 2660f7917c00SJeff Kirsher struct net_device *dev, unsigned int iqid, 2661f7917c00SJeff Kirsher unsigned int cmplqid) 2662f7917c00SJeff Kirsher { 2663f7917c00SJeff Kirsher int ret, nentries; 2664f7917c00SJeff Kirsher struct fw_eq_ctrl_cmd c; 266552367a76SVipul Pandya struct sge *s = &adap->sge; 2666f7917c00SJeff Kirsher struct port_info *pi = netdev_priv(dev); 2667f7917c00SJeff Kirsher 2668f7917c00SJeff Kirsher /* Add status entries */ 266952367a76SVipul Pandya nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); 2670f7917c00SJeff Kirsher 2671f7917c00SJeff Kirsher txq->q.desc = alloc_ring(adap->pdev_dev, nentries, 2672f7917c00SJeff Kirsher sizeof(struct tx_desc), 0, &txq->q.phys_addr, 2673982b81ebSHariprasad Shenai NULL, 0, dev_to_node(adap->pdev_dev)); 2674f7917c00SJeff Kirsher if (!txq->q.desc) 2675f7917c00SJeff Kirsher return -ENOMEM; 2676f7917c00SJeff Kirsher 2677e2ac9628SHariprasad Shenai c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | FW_CMD_REQUEST_F | 2678e2ac9628SHariprasad Shenai FW_CMD_WRITE_F | FW_CMD_EXEC_F | 2679b2612722SHariprasad Shenai FW_EQ_CTRL_CMD_PFN_V(adap->pf) | 26806e4b51a6SHariprasad Shenai FW_EQ_CTRL_CMD_VFN_V(0)); 26816e4b51a6SHariprasad Shenai c.alloc_to_len16 = htonl(FW_EQ_CTRL_CMD_ALLOC_F | 26826e4b51a6SHariprasad Shenai FW_EQ_CTRL_CMD_EQSTART_F | FW_LEN16(c)); 26836e4b51a6SHariprasad Shenai c.cmpliqid_eqid = htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid)); 2684f7917c00SJeff Kirsher c.physeqid_pkd = htonl(0); 26851ecc7b7aSHariprasad Shenai c.fetchszm_to_iqid = 26861ecc7b7aSHariprasad Shenai htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) | 26876e4b51a6SHariprasad Shenai FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) | 26881ecc7b7aSHariprasad Shenai FW_EQ_CTRL_CMD_FETCHRO_F | FW_EQ_CTRL_CMD_IQID_V(iqid)); 26891ecc7b7aSHariprasad Shenai c.dcaen_to_eqsize = 26901ecc7b7aSHariprasad Shenai htonl(FW_EQ_CTRL_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) | 26911ecc7b7aSHariprasad Shenai FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | 26921ecc7b7aSHariprasad Shenai FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) | 26936e4b51a6SHariprasad Shenai FW_EQ_CTRL_CMD_EQSIZE_V(nentries)); 2694f7917c00SJeff Kirsher c.eqaddr = cpu_to_be64(txq->q.phys_addr); 2695f7917c00SJeff Kirsher 2696b2612722SHariprasad Shenai ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); 2697f7917c00SJeff Kirsher if (ret) { 2698f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, 2699f7917c00SJeff Kirsher nentries * sizeof(struct tx_desc), 2700f7917c00SJeff Kirsher txq->q.desc, txq->q.phys_addr); 2701f7917c00SJeff Kirsher txq->q.desc = NULL; 2702f7917c00SJeff Kirsher return ret; 2703f7917c00SJeff Kirsher } 2704f7917c00SJeff Kirsher 27056e4b51a6SHariprasad Shenai init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid))); 2706f7917c00SJeff Kirsher txq->adap = adap; 2707f7917c00SJeff Kirsher skb_queue_head_init(&txq->sendq); 2708f7917c00SJeff Kirsher tasklet_init(&txq->qresume_tsk, restart_ctrlq, (unsigned long)txq); 2709f7917c00SJeff Kirsher txq->full = 0; 2710f7917c00SJeff Kirsher return 0; 2711f7917c00SJeff Kirsher } 2712f7917c00SJeff Kirsher 2713f7917c00SJeff Kirsher int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_ofld_txq *txq, 2714f7917c00SJeff Kirsher struct net_device *dev, unsigned int iqid) 2715f7917c00SJeff Kirsher { 2716f7917c00SJeff Kirsher int ret, nentries; 2717f7917c00SJeff Kirsher struct fw_eq_ofld_cmd c; 271852367a76SVipul Pandya struct sge *s = &adap->sge; 2719f7917c00SJeff Kirsher struct port_info *pi = netdev_priv(dev); 2720f7917c00SJeff Kirsher 2721f7917c00SJeff Kirsher /* Add status entries */ 272252367a76SVipul Pandya nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); 2723f7917c00SJeff Kirsher 2724f7917c00SJeff Kirsher txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size, 2725f7917c00SJeff Kirsher sizeof(struct tx_desc), sizeof(struct tx_sw_desc), 272652367a76SVipul Pandya &txq->q.phys_addr, &txq->q.sdesc, s->stat_len, 2727f7917c00SJeff Kirsher NUMA_NO_NODE); 2728f7917c00SJeff Kirsher if (!txq->q.desc) 2729f7917c00SJeff Kirsher return -ENOMEM; 2730f7917c00SJeff Kirsher 2731f7917c00SJeff Kirsher memset(&c, 0, sizeof(c)); 2732e2ac9628SHariprasad Shenai c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_OFLD_CMD) | FW_CMD_REQUEST_F | 2733e2ac9628SHariprasad Shenai FW_CMD_WRITE_F | FW_CMD_EXEC_F | 2734b2612722SHariprasad Shenai FW_EQ_OFLD_CMD_PFN_V(adap->pf) | 27356e4b51a6SHariprasad Shenai FW_EQ_OFLD_CMD_VFN_V(0)); 27366e4b51a6SHariprasad Shenai c.alloc_to_len16 = htonl(FW_EQ_OFLD_CMD_ALLOC_F | 27376e4b51a6SHariprasad Shenai FW_EQ_OFLD_CMD_EQSTART_F | FW_LEN16(c)); 27381ecc7b7aSHariprasad Shenai c.fetchszm_to_iqid = 27391ecc7b7aSHariprasad Shenai htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) | 27406e4b51a6SHariprasad Shenai FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) | 27411ecc7b7aSHariprasad Shenai FW_EQ_OFLD_CMD_FETCHRO_F | FW_EQ_OFLD_CMD_IQID_V(iqid)); 27421ecc7b7aSHariprasad Shenai c.dcaen_to_eqsize = 27431ecc7b7aSHariprasad Shenai htonl(FW_EQ_OFLD_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) | 27441ecc7b7aSHariprasad Shenai FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | 27451ecc7b7aSHariprasad Shenai FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) | 27466e4b51a6SHariprasad Shenai FW_EQ_OFLD_CMD_EQSIZE_V(nentries)); 2747f7917c00SJeff Kirsher c.eqaddr = cpu_to_be64(txq->q.phys_addr); 2748f7917c00SJeff Kirsher 2749b2612722SHariprasad Shenai ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); 2750f7917c00SJeff Kirsher if (ret) { 2751f7917c00SJeff Kirsher kfree(txq->q.sdesc); 2752f7917c00SJeff Kirsher txq->q.sdesc = NULL; 2753f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, 2754f7917c00SJeff Kirsher nentries * sizeof(struct tx_desc), 2755f7917c00SJeff Kirsher txq->q.desc, txq->q.phys_addr); 2756f7917c00SJeff Kirsher txq->q.desc = NULL; 2757f7917c00SJeff Kirsher return ret; 2758f7917c00SJeff Kirsher } 2759f7917c00SJeff Kirsher 27606e4b51a6SHariprasad Shenai init_txq(adap, &txq->q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd))); 2761f7917c00SJeff Kirsher txq->adap = adap; 2762f7917c00SJeff Kirsher skb_queue_head_init(&txq->sendq); 2763f7917c00SJeff Kirsher tasklet_init(&txq->qresume_tsk, restart_ofldq, (unsigned long)txq); 2764f7917c00SJeff Kirsher txq->full = 0; 2765f7917c00SJeff Kirsher txq->mapping_err = 0; 2766f7917c00SJeff Kirsher return 0; 2767f7917c00SJeff Kirsher } 2768f7917c00SJeff Kirsher 2769f7917c00SJeff Kirsher static void free_txq(struct adapter *adap, struct sge_txq *q) 2770f7917c00SJeff Kirsher { 277152367a76SVipul Pandya struct sge *s = &adap->sge; 277252367a76SVipul Pandya 2773f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, 277452367a76SVipul Pandya q->size * sizeof(struct tx_desc) + s->stat_len, 2775f7917c00SJeff Kirsher q->desc, q->phys_addr); 2776f7917c00SJeff Kirsher q->cntxt_id = 0; 2777f7917c00SJeff Kirsher q->sdesc = NULL; 2778f7917c00SJeff Kirsher q->desc = NULL; 2779f7917c00SJeff Kirsher } 2780f7917c00SJeff Kirsher 2781f7917c00SJeff Kirsher static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, 2782f7917c00SJeff Kirsher struct sge_fl *fl) 2783f7917c00SJeff Kirsher { 278452367a76SVipul Pandya struct sge *s = &adap->sge; 2785f7917c00SJeff Kirsher unsigned int fl_id = fl ? fl->cntxt_id : 0xffff; 2786f7917c00SJeff Kirsher 2787f7917c00SJeff Kirsher adap->sge.ingr_map[rq->cntxt_id - adap->sge.ingr_start] = NULL; 2788b2612722SHariprasad Shenai t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP, 2789f7917c00SJeff Kirsher rq->cntxt_id, fl_id, 0xffff); 2790f7917c00SJeff Kirsher dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len, 2791f7917c00SJeff Kirsher rq->desc, rq->phys_addr); 27923a336cb1SHariprasad Shenai napi_hash_del(&rq->napi); 2793f7917c00SJeff Kirsher netif_napi_del(&rq->napi); 2794f7917c00SJeff Kirsher rq->netdev = NULL; 2795f7917c00SJeff Kirsher rq->cntxt_id = rq->abs_id = 0; 2796f7917c00SJeff Kirsher rq->desc = NULL; 2797f7917c00SJeff Kirsher 2798f7917c00SJeff Kirsher if (fl) { 2799f7917c00SJeff Kirsher free_rx_bufs(adap, fl, fl->avail); 280052367a76SVipul Pandya dma_free_coherent(adap->pdev_dev, fl->size * 8 + s->stat_len, 2801f7917c00SJeff Kirsher fl->desc, fl->addr); 2802f7917c00SJeff Kirsher kfree(fl->sdesc); 2803f7917c00SJeff Kirsher fl->sdesc = NULL; 2804f7917c00SJeff Kirsher fl->cntxt_id = 0; 2805f7917c00SJeff Kirsher fl->desc = NULL; 2806f7917c00SJeff Kirsher } 2807f7917c00SJeff Kirsher } 2808f7917c00SJeff Kirsher 2809f7917c00SJeff Kirsher /** 28105fa76694SHariprasad Shenai * t4_free_ofld_rxqs - free a block of consecutive Rx queues 28115fa76694SHariprasad Shenai * @adap: the adapter 28125fa76694SHariprasad Shenai * @n: number of queues 28135fa76694SHariprasad Shenai * @q: pointer to first queue 28145fa76694SHariprasad Shenai * 28155fa76694SHariprasad Shenai * Release the resources of a consecutive block of offload Rx queues. 28165fa76694SHariprasad Shenai */ 28175fa76694SHariprasad Shenai void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q) 28185fa76694SHariprasad Shenai { 28195fa76694SHariprasad Shenai for ( ; n; n--, q++) 28205fa76694SHariprasad Shenai if (q->rspq.desc) 28215fa76694SHariprasad Shenai free_rspq_fl(adap, &q->rspq, 28225fa76694SHariprasad Shenai q->fl.size ? &q->fl : NULL); 28235fa76694SHariprasad Shenai } 28245fa76694SHariprasad Shenai 28255fa76694SHariprasad Shenai /** 2826f7917c00SJeff Kirsher * t4_free_sge_resources - free SGE resources 2827f7917c00SJeff Kirsher * @adap: the adapter 2828f7917c00SJeff Kirsher * 2829f7917c00SJeff Kirsher * Frees resources used by the SGE queue sets. 2830f7917c00SJeff Kirsher */ 2831f7917c00SJeff Kirsher void t4_free_sge_resources(struct adapter *adap) 2832f7917c00SJeff Kirsher { 2833f7917c00SJeff Kirsher int i; 2834f7917c00SJeff Kirsher struct sge_eth_rxq *eq = adap->sge.ethrxq; 2835f7917c00SJeff Kirsher struct sge_eth_txq *etq = adap->sge.ethtxq; 2836f7917c00SJeff Kirsher 2837f7917c00SJeff Kirsher /* clean up Ethernet Tx/Rx queues */ 2838f7917c00SJeff Kirsher for (i = 0; i < adap->sge.ethqsets; i++, eq++, etq++) { 2839f7917c00SJeff Kirsher if (eq->rspq.desc) 28405fa76694SHariprasad Shenai free_rspq_fl(adap, &eq->rspq, 28415fa76694SHariprasad Shenai eq->fl.size ? &eq->fl : NULL); 2842f7917c00SJeff Kirsher if (etq->q.desc) { 2843b2612722SHariprasad Shenai t4_eth_eq_free(adap, adap->mbox, adap->pf, 0, 2844f7917c00SJeff Kirsher etq->q.cntxt_id); 2845f7917c00SJeff Kirsher free_tx_desc(adap, &etq->q, etq->q.in_use, true); 2846f7917c00SJeff Kirsher kfree(etq->q.sdesc); 2847f7917c00SJeff Kirsher free_txq(adap, &etq->q); 2848f7917c00SJeff Kirsher } 2849f7917c00SJeff Kirsher } 2850f7917c00SJeff Kirsher 2851f7917c00SJeff Kirsher /* clean up RDMA and iSCSI Rx queues */ 28525fa76694SHariprasad Shenai t4_free_ofld_rxqs(adap, adap->sge.ofldqsets, adap->sge.ofldrxq); 28535fa76694SHariprasad Shenai t4_free_ofld_rxqs(adap, adap->sge.rdmaqs, adap->sge.rdmarxq); 28545fa76694SHariprasad Shenai t4_free_ofld_rxqs(adap, adap->sge.rdmaciqs, adap->sge.rdmaciq); 2855f7917c00SJeff Kirsher 2856f7917c00SJeff Kirsher /* clean up offload Tx queues */ 2857f7917c00SJeff Kirsher for (i = 0; i < ARRAY_SIZE(adap->sge.ofldtxq); i++) { 2858f7917c00SJeff Kirsher struct sge_ofld_txq *q = &adap->sge.ofldtxq[i]; 2859f7917c00SJeff Kirsher 2860f7917c00SJeff Kirsher if (q->q.desc) { 2861f7917c00SJeff Kirsher tasklet_kill(&q->qresume_tsk); 2862b2612722SHariprasad Shenai t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0, 2863f7917c00SJeff Kirsher q->q.cntxt_id); 2864f7917c00SJeff Kirsher free_tx_desc(adap, &q->q, q->q.in_use, false); 2865f7917c00SJeff Kirsher kfree(q->q.sdesc); 2866f7917c00SJeff Kirsher __skb_queue_purge(&q->sendq); 2867f7917c00SJeff Kirsher free_txq(adap, &q->q); 2868f7917c00SJeff Kirsher } 2869f7917c00SJeff Kirsher } 2870f7917c00SJeff Kirsher 2871f7917c00SJeff Kirsher /* clean up control Tx queues */ 2872f7917c00SJeff Kirsher for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) { 2873f7917c00SJeff Kirsher struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i]; 2874f7917c00SJeff Kirsher 2875f7917c00SJeff Kirsher if (cq->q.desc) { 2876f7917c00SJeff Kirsher tasklet_kill(&cq->qresume_tsk); 2877b2612722SHariprasad Shenai t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0, 2878f7917c00SJeff Kirsher cq->q.cntxt_id); 2879f7917c00SJeff Kirsher __skb_queue_purge(&cq->sendq); 2880f7917c00SJeff Kirsher free_txq(adap, &cq->q); 2881f7917c00SJeff Kirsher } 2882f7917c00SJeff Kirsher } 2883f7917c00SJeff Kirsher 2884f7917c00SJeff Kirsher if (adap->sge.fw_evtq.desc) 2885f7917c00SJeff Kirsher free_rspq_fl(adap, &adap->sge.fw_evtq, NULL); 2886f7917c00SJeff Kirsher 2887f7917c00SJeff Kirsher if (adap->sge.intrq.desc) 2888f7917c00SJeff Kirsher free_rspq_fl(adap, &adap->sge.intrq, NULL); 2889f7917c00SJeff Kirsher 2890f7917c00SJeff Kirsher /* clear the reverse egress queue map */ 28914b8e27a8SHariprasad Shenai memset(adap->sge.egr_map, 0, 28924b8e27a8SHariprasad Shenai adap->sge.egr_sz * sizeof(*adap->sge.egr_map)); 2893f7917c00SJeff Kirsher } 2894f7917c00SJeff Kirsher 2895f7917c00SJeff Kirsher void t4_sge_start(struct adapter *adap) 2896f7917c00SJeff Kirsher { 2897f7917c00SJeff Kirsher adap->sge.ethtxq_rover = 0; 2898f7917c00SJeff Kirsher mod_timer(&adap->sge.rx_timer, jiffies + RX_QCHECK_PERIOD); 2899f7917c00SJeff Kirsher mod_timer(&adap->sge.tx_timer, jiffies + TX_QCHECK_PERIOD); 2900f7917c00SJeff Kirsher } 2901f7917c00SJeff Kirsher 2902f7917c00SJeff Kirsher /** 2903f7917c00SJeff Kirsher * t4_sge_stop - disable SGE operation 2904f7917c00SJeff Kirsher * @adap: the adapter 2905f7917c00SJeff Kirsher * 2906f7917c00SJeff Kirsher * Stop tasklets and timers associated with the DMA engine. Note that 2907f7917c00SJeff Kirsher * this is effective only if measures have been taken to disable any HW 2908f7917c00SJeff Kirsher * events that may restart them. 2909f7917c00SJeff Kirsher */ 2910f7917c00SJeff Kirsher void t4_sge_stop(struct adapter *adap) 2911f7917c00SJeff Kirsher { 2912f7917c00SJeff Kirsher int i; 2913f7917c00SJeff Kirsher struct sge *s = &adap->sge; 2914f7917c00SJeff Kirsher 2915f7917c00SJeff Kirsher if (in_interrupt()) /* actions below require waiting */ 2916f7917c00SJeff Kirsher return; 2917f7917c00SJeff Kirsher 2918f7917c00SJeff Kirsher if (s->rx_timer.function) 2919f7917c00SJeff Kirsher del_timer_sync(&s->rx_timer); 2920f7917c00SJeff Kirsher if (s->tx_timer.function) 2921f7917c00SJeff Kirsher del_timer_sync(&s->tx_timer); 2922f7917c00SJeff Kirsher 2923f7917c00SJeff Kirsher for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++) { 2924f7917c00SJeff Kirsher struct sge_ofld_txq *q = &s->ofldtxq[i]; 2925f7917c00SJeff Kirsher 2926f7917c00SJeff Kirsher if (q->q.desc) 2927f7917c00SJeff Kirsher tasklet_kill(&q->qresume_tsk); 2928f7917c00SJeff Kirsher } 2929f7917c00SJeff Kirsher for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) { 2930f7917c00SJeff Kirsher struct sge_ctrl_txq *cq = &s->ctrlq[i]; 2931f7917c00SJeff Kirsher 2932f7917c00SJeff Kirsher if (cq->q.desc) 2933f7917c00SJeff Kirsher tasklet_kill(&cq->qresume_tsk); 2934f7917c00SJeff Kirsher } 2935f7917c00SJeff Kirsher } 2936f7917c00SJeff Kirsher 2937f7917c00SJeff Kirsher /** 293806640310SHariprasad Shenai * t4_sge_init_soft - grab core SGE values needed by SGE code 2939f7917c00SJeff Kirsher * @adap: the adapter 2940f7917c00SJeff Kirsher * 294106640310SHariprasad Shenai * We need to grab the SGE operating parameters that we need to have 294206640310SHariprasad Shenai * in order to do our job and make sure we can live with them. 2943f7917c00SJeff Kirsher */ 2944f7917c00SJeff Kirsher 294552367a76SVipul Pandya static int t4_sge_init_soft(struct adapter *adap) 294652367a76SVipul Pandya { 294752367a76SVipul Pandya struct sge *s = &adap->sge; 294852367a76SVipul Pandya u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu; 294952367a76SVipul Pandya u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5; 295052367a76SVipul Pandya u32 ingress_rx_threshold; 295152367a76SVipul Pandya 295252367a76SVipul Pandya /* 295352367a76SVipul Pandya * Verify that CPL messages are going to the Ingress Queue for 295452367a76SVipul Pandya * process_responses() and that only packet data is going to the 295552367a76SVipul Pandya * Free Lists. 295652367a76SVipul Pandya */ 2957f612b815SHariprasad Shenai if ((t4_read_reg(adap, SGE_CONTROL_A) & RXPKTCPLMODE_F) != 2958f612b815SHariprasad Shenai RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) { 295952367a76SVipul Pandya dev_err(adap->pdev_dev, "bad SGE CPL MODE\n"); 296052367a76SVipul Pandya return -EINVAL; 296152367a76SVipul Pandya } 296252367a76SVipul Pandya 296352367a76SVipul Pandya /* 296452367a76SVipul Pandya * Validate the Host Buffer Register Array indices that we want to 296552367a76SVipul Pandya * use ... 296652367a76SVipul Pandya * 296752367a76SVipul Pandya * XXX Note that we should really read through the Host Buffer Size 296852367a76SVipul Pandya * XXX register array and find the indices of the Buffer Sizes which 296952367a76SVipul Pandya * XXX meet our needs! 297052367a76SVipul Pandya */ 297152367a76SVipul Pandya #define READ_FL_BUF(x) \ 2972f612b815SHariprasad Shenai t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32)) 297352367a76SVipul Pandya 297452367a76SVipul Pandya fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF); 297552367a76SVipul Pandya fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF); 297652367a76SVipul Pandya fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF); 297752367a76SVipul Pandya fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF); 297852367a76SVipul Pandya 297992ddcc7bSKumar Sanghvi /* We only bother using the Large Page logic if the Large Page Buffer 298092ddcc7bSKumar Sanghvi * is larger than our Page Size Buffer. 298192ddcc7bSKumar Sanghvi */ 298292ddcc7bSKumar Sanghvi if (fl_large_pg <= fl_small_pg) 298392ddcc7bSKumar Sanghvi fl_large_pg = 0; 298492ddcc7bSKumar Sanghvi 298552367a76SVipul Pandya #undef READ_FL_BUF 298652367a76SVipul Pandya 298792ddcc7bSKumar Sanghvi /* The Page Size Buffer must be exactly equal to our Page Size and the 298892ddcc7bSKumar Sanghvi * Large Page Size Buffer should be 0 (per above) or a power of 2. 298992ddcc7bSKumar Sanghvi */ 299052367a76SVipul Pandya if (fl_small_pg != PAGE_SIZE || 299192ddcc7bSKumar Sanghvi (fl_large_pg & (fl_large_pg-1)) != 0) { 299252367a76SVipul Pandya dev_err(adap->pdev_dev, "bad SGE FL page buffer sizes [%d, %d]\n", 299352367a76SVipul Pandya fl_small_pg, fl_large_pg); 299452367a76SVipul Pandya return -EINVAL; 299552367a76SVipul Pandya } 299652367a76SVipul Pandya if (fl_large_pg) 299752367a76SVipul Pandya s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT; 299852367a76SVipul Pandya 299952367a76SVipul Pandya if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap) || 300052367a76SVipul Pandya fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) { 300152367a76SVipul Pandya dev_err(adap->pdev_dev, "bad SGE FL MTU sizes [%d, %d]\n", 300252367a76SVipul Pandya fl_small_mtu, fl_large_mtu); 300352367a76SVipul Pandya return -EINVAL; 300452367a76SVipul Pandya } 300552367a76SVipul Pandya 300652367a76SVipul Pandya /* 300752367a76SVipul Pandya * Retrieve our RX interrupt holdoff timer values and counter 300852367a76SVipul Pandya * threshold values from the SGE parameters. 300952367a76SVipul Pandya */ 3010f061de42SHariprasad Shenai timer_value_0_and_1 = t4_read_reg(adap, SGE_TIMER_VALUE_0_AND_1_A); 3011f061de42SHariprasad Shenai timer_value_2_and_3 = t4_read_reg(adap, SGE_TIMER_VALUE_2_AND_3_A); 3012f061de42SHariprasad Shenai timer_value_4_and_5 = t4_read_reg(adap, SGE_TIMER_VALUE_4_AND_5_A); 301352367a76SVipul Pandya s->timer_val[0] = core_ticks_to_us(adap, 3014f061de42SHariprasad Shenai TIMERVALUE0_G(timer_value_0_and_1)); 301552367a76SVipul Pandya s->timer_val[1] = core_ticks_to_us(adap, 3016f061de42SHariprasad Shenai TIMERVALUE1_G(timer_value_0_and_1)); 301752367a76SVipul Pandya s->timer_val[2] = core_ticks_to_us(adap, 3018f061de42SHariprasad Shenai TIMERVALUE2_G(timer_value_2_and_3)); 301952367a76SVipul Pandya s->timer_val[3] = core_ticks_to_us(adap, 3020f061de42SHariprasad Shenai TIMERVALUE3_G(timer_value_2_and_3)); 302152367a76SVipul Pandya s->timer_val[4] = core_ticks_to_us(adap, 3022f061de42SHariprasad Shenai TIMERVALUE4_G(timer_value_4_and_5)); 302352367a76SVipul Pandya s->timer_val[5] = core_ticks_to_us(adap, 3024f061de42SHariprasad Shenai TIMERVALUE5_G(timer_value_4_and_5)); 302552367a76SVipul Pandya 3026f612b815SHariprasad Shenai ingress_rx_threshold = t4_read_reg(adap, SGE_INGRESS_RX_THRESHOLD_A); 3027f612b815SHariprasad Shenai s->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold); 3028f612b815SHariprasad Shenai s->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold); 3029f612b815SHariprasad Shenai s->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold); 3030f612b815SHariprasad Shenai s->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold); 303152367a76SVipul Pandya 303252367a76SVipul Pandya return 0; 303352367a76SVipul Pandya } 303452367a76SVipul Pandya 303506640310SHariprasad Shenai /** 303606640310SHariprasad Shenai * t4_sge_init - initialize SGE 303706640310SHariprasad Shenai * @adap: the adapter 303806640310SHariprasad Shenai * 303906640310SHariprasad Shenai * Perform low-level SGE code initialization needed every time after a 304006640310SHariprasad Shenai * chip reset. 304152367a76SVipul Pandya */ 304252367a76SVipul Pandya int t4_sge_init(struct adapter *adap) 304352367a76SVipul Pandya { 304452367a76SVipul Pandya struct sge *s = &adap->sge; 3045ce8f407aSHariprasad Shenai u32 sge_control, sge_control2, sge_conm_ctrl; 3046ce8f407aSHariprasad Shenai unsigned int ingpadboundary, ingpackboundary; 3047c2b955e0SKumar Sanghvi int ret, egress_threshold; 304852367a76SVipul Pandya 304952367a76SVipul Pandya /* 305052367a76SVipul Pandya * Ingress Padding Boundary and Egress Status Page Size are set up by 305152367a76SVipul Pandya * t4_fixup_host_params(). 305252367a76SVipul Pandya */ 3053f612b815SHariprasad Shenai sge_control = t4_read_reg(adap, SGE_CONTROL_A); 3054f612b815SHariprasad Shenai s->pktshift = PKTSHIFT_G(sge_control); 3055f612b815SHariprasad Shenai s->stat_len = (sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64; 3056ce8f407aSHariprasad Shenai 3057ce8f407aSHariprasad Shenai /* T4 uses a single control field to specify both the PCIe Padding and 3058ce8f407aSHariprasad Shenai * Packing Boundary. T5 introduced the ability to specify these 3059ce8f407aSHariprasad Shenai * separately. The actual Ingress Packet Data alignment boundary 3060ce8f407aSHariprasad Shenai * within Packed Buffer Mode is the maximum of these two 30611ecc7b7aSHariprasad Shenai * specifications. (Note that it makes no real practical sense to 30621ecc7b7aSHariprasad Shenai * have the Pading Boudary be larger than the Packing Boundary but you 30631ecc7b7aSHariprasad Shenai * could set the chip up that way and, in fact, legacy T4 code would 30641ecc7b7aSHariprasad Shenai * end doing this because it would initialize the Padding Boundary and 30651ecc7b7aSHariprasad Shenai * leave the Packing Boundary initialized to 0 (16 bytes).) 3066ce8f407aSHariprasad Shenai */ 3067f612b815SHariprasad Shenai ingpadboundary = 1 << (INGPADBOUNDARY_G(sge_control) + 3068f612b815SHariprasad Shenai INGPADBOUNDARY_SHIFT_X); 3069ce8f407aSHariprasad Shenai if (is_t4(adap->params.chip)) { 3070ce8f407aSHariprasad Shenai s->fl_align = ingpadboundary; 3071ce8f407aSHariprasad Shenai } else { 3072ce8f407aSHariprasad Shenai /* T5 has a different interpretation of one of the PCIe Packing 3073ce8f407aSHariprasad Shenai * Boundary values. 3074ce8f407aSHariprasad Shenai */ 3075ce8f407aSHariprasad Shenai sge_control2 = t4_read_reg(adap, SGE_CONTROL2_A); 3076ce8f407aSHariprasad Shenai ingpackboundary = INGPACKBOUNDARY_G(sge_control2); 3077ce8f407aSHariprasad Shenai if (ingpackboundary == INGPACKBOUNDARY_16B_X) 3078ce8f407aSHariprasad Shenai ingpackboundary = 16; 3079ce8f407aSHariprasad Shenai else 3080ce8f407aSHariprasad Shenai ingpackboundary = 1 << (ingpackboundary + 3081ce8f407aSHariprasad Shenai INGPACKBOUNDARY_SHIFT_X); 3082ce8f407aSHariprasad Shenai 3083ce8f407aSHariprasad Shenai s->fl_align = max(ingpadboundary, ingpackboundary); 3084ce8f407aSHariprasad Shenai } 308552367a76SVipul Pandya 308652367a76SVipul Pandya ret = t4_sge_init_soft(adap); 308752367a76SVipul Pandya if (ret < 0) 308852367a76SVipul Pandya return ret; 308952367a76SVipul Pandya 309052367a76SVipul Pandya /* 309152367a76SVipul Pandya * A FL with <= fl_starve_thres buffers is starving and a periodic 309252367a76SVipul Pandya * timer will attempt to refill it. This needs to be larger than the 309352367a76SVipul Pandya * SGE's Egress Congestion Threshold. If it isn't, then we can get 309452367a76SVipul Pandya * stuck waiting for new packets while the SGE is waiting for us to 309552367a76SVipul Pandya * give it more Free List entries. (Note that the SGE's Egress 3096c2b955e0SKumar Sanghvi * Congestion Threshold is in units of 2 Free List pointers.) For T4, 3097c2b955e0SKumar Sanghvi * there was only a single field to control this. For T5 there's the 3098c2b955e0SKumar Sanghvi * original field which now only applies to Unpacked Mode Free List 3099c2b955e0SKumar Sanghvi * buffers and a new field which only applies to Packed Mode Free List 3100c2b955e0SKumar Sanghvi * buffers. 310152367a76SVipul Pandya */ 3102f612b815SHariprasad Shenai sge_conm_ctrl = t4_read_reg(adap, SGE_CONM_CTRL_A); 3103c2b955e0SKumar Sanghvi if (is_t4(adap->params.chip)) 3104f612b815SHariprasad Shenai egress_threshold = EGRTHRESHOLD_G(sge_conm_ctrl); 3105c2b955e0SKumar Sanghvi else 3106f612b815SHariprasad Shenai egress_threshold = EGRTHRESHOLDPACKING_G(sge_conm_ctrl); 3107c2b955e0SKumar Sanghvi s->fl_starve_thres = 2*egress_threshold + 1; 310852367a76SVipul Pandya 3109a3bfb617SHariprasad Shenai t4_idma_monitor_init(adap, &s->idma_monitor); 3110a3bfb617SHariprasad Shenai 31111ecc7b7aSHariprasad Shenai /* Set up timers used for recuring callbacks to process RX and TX 31121ecc7b7aSHariprasad Shenai * administrative tasks. 31131ecc7b7aSHariprasad Shenai */ 3114f7917c00SJeff Kirsher setup_timer(&s->rx_timer, sge_rx_timer_cb, (unsigned long)adap); 3115f7917c00SJeff Kirsher setup_timer(&s->tx_timer, sge_tx_timer_cb, (unsigned long)adap); 3116a3bfb617SHariprasad Shenai 3117f7917c00SJeff Kirsher spin_lock_init(&s->intrq_lock); 311852367a76SVipul Pandya 311952367a76SVipul Pandya return 0; 3120f7917c00SJeff Kirsher } 3121