196c93589SGidon Studinski /* 296c93589SGidon Studinski * Copyright (c) 2012-2018 The Linux Foundation. All rights reserved. 396c93589SGidon Studinski * 496c93589SGidon Studinski * Permission to use, copy, modify, and/or distribute this software for any 596c93589SGidon Studinski * purpose with or without fee is hereby granted, provided that the above 696c93589SGidon Studinski * copyright notice and this permission notice appear in all copies. 796c93589SGidon Studinski * 896c93589SGidon Studinski * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 996c93589SGidon Studinski * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1096c93589SGidon Studinski * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1196c93589SGidon Studinski * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1296c93589SGidon Studinski * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1396c93589SGidon Studinski * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1496c93589SGidon Studinski * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1596c93589SGidon Studinski */ 1696c93589SGidon Studinski 1796c93589SGidon Studinski #include <linux/etherdevice.h> 1896c93589SGidon Studinski #include <linux/moduleparam.h> 1996c93589SGidon Studinski #include <linux/prefetch.h> 2096c93589SGidon Studinski #include <linux/types.h> 2196c93589SGidon Studinski #include <linux/list.h> 2296c93589SGidon Studinski #include <linux/ip.h> 2396c93589SGidon Studinski #include <linux/ipv6.h> 2496c93589SGidon Studinski #include "wil6210.h" 2596c93589SGidon Studinski #include "txrx_edma.h" 2696c93589SGidon Studinski #include "txrx.h" 279202d7b6SMaya Erez #include "trace.h" 2896c93589SGidon Studinski 2996c93589SGidon Studinski #define WIL_EDMA_MAX_DATA_OFFSET (2) 3096c93589SGidon Studinski 3196c93589SGidon Studinski static void wil_tx_desc_unmap_edma(struct device *dev, 329202d7b6SMaya Erez union wil_tx_desc *desc, 3396c93589SGidon Studinski struct wil_ctx *ctx) 3496c93589SGidon Studinski { 359202d7b6SMaya Erez struct wil_tx_enhanced_desc *d = (struct wil_tx_enhanced_desc *)desc; 3696c93589SGidon Studinski dma_addr_t pa = wil_tx_desc_get_addr_edma(&d->dma); 3796c93589SGidon Studinski u16 dmalen = le16_to_cpu(d->dma.length); 3896c93589SGidon Studinski 3996c93589SGidon Studinski switch (ctx->mapped_as) { 4096c93589SGidon Studinski case wil_mapped_as_single: 4196c93589SGidon Studinski dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE); 4296c93589SGidon Studinski break; 4396c93589SGidon Studinski case wil_mapped_as_page: 4496c93589SGidon Studinski dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE); 4596c93589SGidon Studinski break; 4696c93589SGidon Studinski default: 4796c93589SGidon Studinski break; 4896c93589SGidon Studinski } 4996c93589SGidon Studinski } 5096c93589SGidon Studinski 5196c93589SGidon Studinski static int wil_find_free_sring(struct wil6210_priv *wil) 5296c93589SGidon Studinski { 5396c93589SGidon Studinski int i; 5496c93589SGidon Studinski 5596c93589SGidon Studinski for (i = 0; i < WIL6210_MAX_STATUS_RINGS; i++) { 5696c93589SGidon Studinski if (!wil->srings[i].va) 5796c93589SGidon Studinski return i; 5896c93589SGidon Studinski } 5996c93589SGidon Studinski 6096c93589SGidon Studinski return -EINVAL; 6196c93589SGidon Studinski } 6296c93589SGidon Studinski 6396c93589SGidon Studinski static void wil_sring_free(struct wil6210_priv *wil, 6496c93589SGidon Studinski struct wil_status_ring *sring) 6596c93589SGidon Studinski { 6696c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 6796c93589SGidon Studinski size_t sz; 6896c93589SGidon Studinski 6996c93589SGidon Studinski if (!sring || !sring->va) 7096c93589SGidon Studinski return; 7196c93589SGidon Studinski 7296c93589SGidon Studinski sz = sring->elem_size * sring->size; 7396c93589SGidon Studinski 7496c93589SGidon Studinski wil_dbg_misc(wil, "status_ring_free, size(bytes)=%zu, 0x%p:%pad\n", 7596c93589SGidon Studinski sz, sring->va, &sring->pa); 7696c93589SGidon Studinski 7796c93589SGidon Studinski dma_free_coherent(dev, sz, (void *)sring->va, sring->pa); 7896c93589SGidon Studinski sring->pa = 0; 7996c93589SGidon Studinski sring->va = NULL; 8096c93589SGidon Studinski } 8196c93589SGidon Studinski 8296c93589SGidon Studinski static int wil_sring_alloc(struct wil6210_priv *wil, 8396c93589SGidon Studinski struct wil_status_ring *sring) 8496c93589SGidon Studinski { 8596c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 8696c93589SGidon Studinski size_t sz = sring->elem_size * sring->size; 8796c93589SGidon Studinski 8896c93589SGidon Studinski wil_dbg_misc(wil, "status_ring_alloc: size=%zu\n", sz); 8996c93589SGidon Studinski 9096c93589SGidon Studinski if (sz == 0) { 9196c93589SGidon Studinski wil_err(wil, "Cannot allocate a zero size status ring\n"); 9296c93589SGidon Studinski return -EINVAL; 9396c93589SGidon Studinski } 9496c93589SGidon Studinski 9596c93589SGidon Studinski sring->swhead = 0; 9696c93589SGidon Studinski 9796c93589SGidon Studinski /* Status messages are allocated and initialized to 0. This is necessary 9896c93589SGidon Studinski * since DR bit should be initialized to 0. 9996c93589SGidon Studinski */ 10096c93589SGidon Studinski sring->va = dma_zalloc_coherent(dev, sz, &sring->pa, GFP_KERNEL); 10196c93589SGidon Studinski if (!sring->va) 10296c93589SGidon Studinski return -ENOMEM; 10396c93589SGidon Studinski 10496c93589SGidon Studinski wil_dbg_misc(wil, "status_ring[%d] 0x%p:%pad\n", sring->size, sring->va, 10596c93589SGidon Studinski &sring->pa); 10696c93589SGidon Studinski 10796c93589SGidon Studinski return 0; 10896c93589SGidon Studinski } 10996c93589SGidon Studinski 11096c93589SGidon Studinski static int wil_tx_init_edma(struct wil6210_priv *wil) 11196c93589SGidon Studinski { 11296c93589SGidon Studinski int ring_id = wil_find_free_sring(wil); 11396c93589SGidon Studinski struct wil_status_ring *sring; 11496c93589SGidon Studinski int rc; 11596c93589SGidon Studinski u16 status_ring_size; 11696c93589SGidon Studinski 11796c93589SGidon Studinski if (wil->tx_status_ring_order < WIL_SRING_SIZE_ORDER_MIN || 11896c93589SGidon Studinski wil->tx_status_ring_order > WIL_SRING_SIZE_ORDER_MAX) 11996c93589SGidon Studinski wil->tx_status_ring_order = WIL_TX_SRING_SIZE_ORDER_DEFAULT; 12096c93589SGidon Studinski 12196c93589SGidon Studinski status_ring_size = 1 << wil->tx_status_ring_order; 12296c93589SGidon Studinski 12396c93589SGidon Studinski wil_dbg_misc(wil, "init TX sring: size=%u, ring_id=%u\n", 12496c93589SGidon Studinski status_ring_size, ring_id); 12596c93589SGidon Studinski 12696c93589SGidon Studinski if (ring_id < 0) 12796c93589SGidon Studinski return ring_id; 12896c93589SGidon Studinski 12996c93589SGidon Studinski /* Allocate Tx status ring. Tx descriptor rings will be 13096c93589SGidon Studinski * allocated on WMI connect event 13196c93589SGidon Studinski */ 13296c93589SGidon Studinski sring = &wil->srings[ring_id]; 13396c93589SGidon Studinski 13496c93589SGidon Studinski sring->is_rx = false; 13596c93589SGidon Studinski sring->size = status_ring_size; 13696c93589SGidon Studinski sring->elem_size = sizeof(struct wil_ring_tx_status); 13796c93589SGidon Studinski rc = wil_sring_alloc(wil, sring); 13896c93589SGidon Studinski if (rc) 13996c93589SGidon Studinski return rc; 14096c93589SGidon Studinski 14196c93589SGidon Studinski rc = wil_wmi_tx_sring_cfg(wil, ring_id); 14296c93589SGidon Studinski if (rc) 14396c93589SGidon Studinski goto out_free; 14496c93589SGidon Studinski 14596c93589SGidon Studinski sring->desc_rdy_pol = 1; 14696c93589SGidon Studinski wil->tx_sring_idx = ring_id; 14796c93589SGidon Studinski 14896c93589SGidon Studinski return 0; 14996c93589SGidon Studinski out_free: 15096c93589SGidon Studinski wil_sring_free(wil, sring); 15196c93589SGidon Studinski return rc; 15296c93589SGidon Studinski } 15396c93589SGidon Studinski 15496c93589SGidon Studinski /** 15596c93589SGidon Studinski * Allocate one skb for Rx descriptor RING 15696c93589SGidon Studinski */ 15796c93589SGidon Studinski static int wil_ring_alloc_skb_edma(struct wil6210_priv *wil, 15896c93589SGidon Studinski struct wil_ring *ring, u32 i) 15996c93589SGidon Studinski { 16096c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 16196c93589SGidon Studinski unsigned int sz = wil->rx_buf_len + ETH_HLEN + 16296c93589SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 16396c93589SGidon Studinski dma_addr_t pa; 16496c93589SGidon Studinski u16 buff_id; 16596c93589SGidon Studinski struct list_head *active = &wil->rx_buff_mgmt.active; 16696c93589SGidon Studinski struct list_head *free = &wil->rx_buff_mgmt.free; 16796c93589SGidon Studinski struct wil_rx_buff *rx_buff; 16896c93589SGidon Studinski struct wil_rx_buff *buff_arr = wil->rx_buff_mgmt.buff_arr; 16996c93589SGidon Studinski struct sk_buff *skb; 17096c93589SGidon Studinski struct wil_rx_enhanced_desc dd, *d = ⅆ 17196c93589SGidon Studinski struct wil_rx_enhanced_desc *_d = (struct wil_rx_enhanced_desc *) 17296c93589SGidon Studinski &ring->va[i].rx.enhanced; 17396c93589SGidon Studinski 17496c93589SGidon Studinski if (unlikely(list_empty(free))) { 17596c93589SGidon Studinski wil->rx_buff_mgmt.free_list_empty_cnt++; 17696c93589SGidon Studinski return -EAGAIN; 17796c93589SGidon Studinski } 17896c93589SGidon Studinski 17996c93589SGidon Studinski skb = dev_alloc_skb(sz); 18096c93589SGidon Studinski if (unlikely(!skb)) 18196c93589SGidon Studinski return -ENOMEM; 18296c93589SGidon Studinski 18396c93589SGidon Studinski skb_put(skb, sz); 18496c93589SGidon Studinski 1850b853210SMaya Erez /** 1860b853210SMaya Erez * Make sure that the network stack calculates checksum for packets 1870b853210SMaya Erez * which failed the HW checksum calculation 1880b853210SMaya Erez */ 1890b853210SMaya Erez skb->ip_summed = CHECKSUM_NONE; 1900b853210SMaya Erez 19196c93589SGidon Studinski pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 19296c93589SGidon Studinski if (unlikely(dma_mapping_error(dev, pa))) { 19396c93589SGidon Studinski kfree_skb(skb); 19496c93589SGidon Studinski return -ENOMEM; 19596c93589SGidon Studinski } 19696c93589SGidon Studinski 19796c93589SGidon Studinski /* Get the buffer ID - the index of the rx buffer in the buff_arr */ 19896c93589SGidon Studinski rx_buff = list_first_entry(free, struct wil_rx_buff, list); 19996c93589SGidon Studinski buff_id = rx_buff->id; 20096c93589SGidon Studinski 20196c93589SGidon Studinski /* Move a buffer from the free list to the active list */ 20296c93589SGidon Studinski list_move(&rx_buff->list, active); 20396c93589SGidon Studinski 20496c93589SGidon Studinski buff_arr[buff_id].skb = skb; 20596c93589SGidon Studinski 20696c93589SGidon Studinski wil_desc_set_addr_edma(&d->dma.addr, &d->dma.addr_high_high, pa); 20796c93589SGidon Studinski d->dma.length = cpu_to_le16(sz); 20896c93589SGidon Studinski d->mac.buff_id = cpu_to_le16(buff_id); 20996c93589SGidon Studinski *_d = *d; 21096c93589SGidon Studinski 21196c93589SGidon Studinski /* Save the physical address in skb->cb for later use in dma_unmap */ 21296c93589SGidon Studinski memcpy(skb->cb, &pa, sizeof(pa)); 21396c93589SGidon Studinski 21496c93589SGidon Studinski return 0; 21596c93589SGidon Studinski } 21696c93589SGidon Studinski 2177be13fc3SGidon Studinski static inline 2187be13fc3SGidon Studinski void wil_get_next_rx_status_msg(struct wil_status_ring *sring, void *msg) 2197be13fc3SGidon Studinski { 2207be13fc3SGidon Studinski memcpy(msg, (void *)(sring->va + (sring->elem_size * sring->swhead)), 2217be13fc3SGidon Studinski sring->elem_size); 2227be13fc3SGidon Studinski } 2237be13fc3SGidon Studinski 2249202d7b6SMaya Erez static inline void wil_sring_advance_swhead(struct wil_status_ring *sring) 2259202d7b6SMaya Erez { 2269202d7b6SMaya Erez sring->swhead = (sring->swhead + 1) % sring->size; 2279202d7b6SMaya Erez if (sring->swhead == 0) 2289202d7b6SMaya Erez sring->desc_rdy_pol = 1 - sring->desc_rdy_pol; 2299202d7b6SMaya Erez } 2309202d7b6SMaya Erez 23196c93589SGidon Studinski static int wil_rx_refill_edma(struct wil6210_priv *wil) 23296c93589SGidon Studinski { 23396c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 23496c93589SGidon Studinski u32 next_head; 23596c93589SGidon Studinski int rc = 0; 23696c93589SGidon Studinski u32 swtail = *ring->edma_rx_swtail.va; 23796c93589SGidon Studinski 23896c93589SGidon Studinski for (; next_head = wil_ring_next_head(ring), (next_head != swtail); 23996c93589SGidon Studinski ring->swhead = next_head) { 24096c93589SGidon Studinski rc = wil_ring_alloc_skb_edma(wil, ring, ring->swhead); 24196c93589SGidon Studinski if (unlikely(rc)) { 24296c93589SGidon Studinski if (rc == -EAGAIN) 24396c93589SGidon Studinski wil_dbg_txrx(wil, "No free buffer ID found\n"); 24496c93589SGidon Studinski else 24596c93589SGidon Studinski wil_err_ratelimited(wil, 24696c93589SGidon Studinski "Error %d in refill desc[%d]\n", 24796c93589SGidon Studinski rc, ring->swhead); 24896c93589SGidon Studinski break; 24996c93589SGidon Studinski } 25096c93589SGidon Studinski } 25196c93589SGidon Studinski 25296c93589SGidon Studinski /* make sure all writes to descriptors (shared memory) are done before 25396c93589SGidon Studinski * committing them to HW 25496c93589SGidon Studinski */ 25596c93589SGidon Studinski wmb(); 25696c93589SGidon Studinski 25796c93589SGidon Studinski wil_w(wil, ring->hwtail, ring->swhead); 25896c93589SGidon Studinski 25996c93589SGidon Studinski return rc; 26096c93589SGidon Studinski } 26196c93589SGidon Studinski 26296c93589SGidon Studinski static void wil_move_all_rx_buff_to_free_list(struct wil6210_priv *wil, 26396c93589SGidon Studinski struct wil_ring *ring) 26496c93589SGidon Studinski { 26596c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 26696c93589SGidon Studinski u32 next_tail; 26796c93589SGidon Studinski u32 swhead = (ring->swhead + 1) % ring->size; 26896c93589SGidon Studinski dma_addr_t pa; 26996c93589SGidon Studinski u16 dmalen; 27096c93589SGidon Studinski 27196c93589SGidon Studinski for (; next_tail = wil_ring_next_tail(ring), (next_tail != swhead); 27296c93589SGidon Studinski ring->swtail = next_tail) { 27396c93589SGidon Studinski struct wil_rx_enhanced_desc dd, *d = ⅆ 27496c93589SGidon Studinski struct wil_rx_enhanced_desc *_d = 27596c93589SGidon Studinski (struct wil_rx_enhanced_desc *) 27696c93589SGidon Studinski &ring->va[ring->swtail].rx.enhanced; 27796c93589SGidon Studinski struct sk_buff *skb; 27896c93589SGidon Studinski u16 buff_id; 27996c93589SGidon Studinski 28096c93589SGidon Studinski *d = *_d; 28196c93589SGidon Studinski pa = wil_rx_desc_get_addr_edma(&d->dma); 28296c93589SGidon Studinski dmalen = le16_to_cpu(d->dma.length); 28396c93589SGidon Studinski dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE); 28496c93589SGidon Studinski 28596c93589SGidon Studinski /* Extract the SKB from the rx_buff management array */ 28696c93589SGidon Studinski buff_id = __le16_to_cpu(d->mac.buff_id); 28796c93589SGidon Studinski if (buff_id >= wil->rx_buff_mgmt.size) { 28896c93589SGidon Studinski wil_err(wil, "invalid buff_id %d\n", buff_id); 28996c93589SGidon Studinski continue; 29096c93589SGidon Studinski } 29196c93589SGidon Studinski skb = wil->rx_buff_mgmt.buff_arr[buff_id].skb; 29296c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr[buff_id].skb = NULL; 29396c93589SGidon Studinski if (unlikely(!skb)) 29496c93589SGidon Studinski wil_err(wil, "No Rx skb at buff_id %d\n", buff_id); 29596c93589SGidon Studinski else 29696c93589SGidon Studinski kfree_skb(skb); 29796c93589SGidon Studinski 29896c93589SGidon Studinski /* Move the buffer from the active to the free list */ 29996c93589SGidon Studinski list_move(&wil->rx_buff_mgmt.buff_arr[buff_id].list, 30096c93589SGidon Studinski &wil->rx_buff_mgmt.free); 30196c93589SGidon Studinski } 30296c93589SGidon Studinski } 30396c93589SGidon Studinski 30496c93589SGidon Studinski static void wil_free_rx_buff_arr(struct wil6210_priv *wil) 30596c93589SGidon Studinski { 30696c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 30796c93589SGidon Studinski 30896c93589SGidon Studinski if (!wil->rx_buff_mgmt.buff_arr) 30996c93589SGidon Studinski return; 31096c93589SGidon Studinski 31196c93589SGidon Studinski /* Move all the buffers to the free list in case active list is 31296c93589SGidon Studinski * not empty in order to release all SKBs before deleting the array 31396c93589SGidon Studinski */ 31496c93589SGidon Studinski wil_move_all_rx_buff_to_free_list(wil, ring); 31596c93589SGidon Studinski 31696c93589SGidon Studinski kfree(wil->rx_buff_mgmt.buff_arr); 31796c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr = NULL; 31896c93589SGidon Studinski } 31996c93589SGidon Studinski 32096c93589SGidon Studinski static int wil_init_rx_buff_arr(struct wil6210_priv *wil, 32196c93589SGidon Studinski size_t size) 32296c93589SGidon Studinski { 32396c93589SGidon Studinski struct wil_rx_buff *buff_arr; 32496c93589SGidon Studinski struct list_head *active = &wil->rx_buff_mgmt.active; 32596c93589SGidon Studinski struct list_head *free = &wil->rx_buff_mgmt.free; 32696c93589SGidon Studinski int i; 32796c93589SGidon Studinski 32896c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr = kcalloc(size, sizeof(struct wil_rx_buff), 32996c93589SGidon Studinski GFP_KERNEL); 33096c93589SGidon Studinski if (!wil->rx_buff_mgmt.buff_arr) 33196c93589SGidon Studinski return -ENOMEM; 33296c93589SGidon Studinski 33396c93589SGidon Studinski /* Set list heads */ 33496c93589SGidon Studinski INIT_LIST_HEAD(active); 33596c93589SGidon Studinski INIT_LIST_HEAD(free); 33696c93589SGidon Studinski 33796c93589SGidon Studinski /* Linkify the list */ 33896c93589SGidon Studinski buff_arr = wil->rx_buff_mgmt.buff_arr; 33996c93589SGidon Studinski for (i = 0; i < size; i++) { 34096c93589SGidon Studinski list_add(&buff_arr[i].list, free); 34196c93589SGidon Studinski buff_arr[i].id = i; 34296c93589SGidon Studinski } 34396c93589SGidon Studinski 34496c93589SGidon Studinski wil->rx_buff_mgmt.size = size; 34596c93589SGidon Studinski 34696c93589SGidon Studinski return 0; 34796c93589SGidon Studinski } 34896c93589SGidon Studinski 34996c93589SGidon Studinski static int wil_init_rx_sring(struct wil6210_priv *wil, 35096c93589SGidon Studinski u16 status_ring_size, 35196c93589SGidon Studinski size_t elem_size, 35296c93589SGidon Studinski u16 ring_id) 35396c93589SGidon Studinski { 35496c93589SGidon Studinski struct wil_status_ring *sring = &wil->srings[ring_id]; 35596c93589SGidon Studinski int rc; 35696c93589SGidon Studinski 35796c93589SGidon Studinski wil_dbg_misc(wil, "init RX sring: size=%u, ring_id=%u\n", sring->size, 35896c93589SGidon Studinski ring_id); 35996c93589SGidon Studinski 36096c93589SGidon Studinski memset(&sring->rx_data, 0, sizeof(sring->rx_data)); 36196c93589SGidon Studinski 36296c93589SGidon Studinski sring->is_rx = true; 36396c93589SGidon Studinski sring->size = status_ring_size; 36496c93589SGidon Studinski sring->elem_size = elem_size; 36596c93589SGidon Studinski rc = wil_sring_alloc(wil, sring); 36696c93589SGidon Studinski if (rc) 36796c93589SGidon Studinski return rc; 36896c93589SGidon Studinski 36996c93589SGidon Studinski rc = wil_wmi_rx_sring_add(wil, ring_id); 37096c93589SGidon Studinski if (rc) 37196c93589SGidon Studinski goto out_free; 37296c93589SGidon Studinski 37396c93589SGidon Studinski sring->desc_rdy_pol = 1; 37496c93589SGidon Studinski 37596c93589SGidon Studinski return 0; 37696c93589SGidon Studinski out_free: 37796c93589SGidon Studinski wil_sring_free(wil, sring); 37896c93589SGidon Studinski return rc; 37996c93589SGidon Studinski } 38096c93589SGidon Studinski 38196c93589SGidon Studinski static int wil_ring_alloc_desc_ring(struct wil6210_priv *wil, 38296c93589SGidon Studinski struct wil_ring *ring) 38396c93589SGidon Studinski { 38496c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 38596c93589SGidon Studinski size_t sz = ring->size * sizeof(ring->va[0]); 38696c93589SGidon Studinski 38796c93589SGidon Studinski wil_dbg_misc(wil, "alloc_desc_ring:\n"); 38896c93589SGidon Studinski 38996c93589SGidon Studinski BUILD_BUG_ON(sizeof(ring->va[0]) != 32); 39096c93589SGidon Studinski 39196c93589SGidon Studinski ring->swhead = 0; 39296c93589SGidon Studinski ring->swtail = 0; 39396c93589SGidon Studinski ring->ctx = kcalloc(ring->size, sizeof(ring->ctx[0]), GFP_KERNEL); 39496c93589SGidon Studinski if (!ring->ctx) 39596c93589SGidon Studinski goto err; 39696c93589SGidon Studinski 39796c93589SGidon Studinski ring->va = dma_zalloc_coherent(dev, sz, &ring->pa, GFP_KERNEL); 39896c93589SGidon Studinski if (!ring->va) 39996c93589SGidon Studinski goto err_free_ctx; 40096c93589SGidon Studinski 40196c93589SGidon Studinski if (ring->is_rx) { 40296c93589SGidon Studinski sz = sizeof(*ring->edma_rx_swtail.va); 40396c93589SGidon Studinski ring->edma_rx_swtail.va = 40496c93589SGidon Studinski dma_zalloc_coherent(dev, sz, &ring->edma_rx_swtail.pa, 40596c93589SGidon Studinski GFP_KERNEL); 40696c93589SGidon Studinski if (!ring->edma_rx_swtail.va) 40796c93589SGidon Studinski goto err_free_va; 40896c93589SGidon Studinski } 40996c93589SGidon Studinski 41096c93589SGidon Studinski wil_dbg_misc(wil, "%s ring[%d] 0x%p:%pad 0x%p\n", 41196c93589SGidon Studinski ring->is_rx ? "RX" : "TX", 41296c93589SGidon Studinski ring->size, ring->va, &ring->pa, ring->ctx); 41396c93589SGidon Studinski 41496c93589SGidon Studinski return 0; 41596c93589SGidon Studinski err_free_va: 41696c93589SGidon Studinski dma_free_coherent(dev, ring->size * sizeof(ring->va[0]), 41796c93589SGidon Studinski (void *)ring->va, ring->pa); 41896c93589SGidon Studinski ring->va = NULL; 41996c93589SGidon Studinski err_free_ctx: 42096c93589SGidon Studinski kfree(ring->ctx); 42196c93589SGidon Studinski ring->ctx = NULL; 42296c93589SGidon Studinski err: 42396c93589SGidon Studinski return -ENOMEM; 42496c93589SGidon Studinski } 42596c93589SGidon Studinski 42696c93589SGidon Studinski static void wil_ring_free_edma(struct wil6210_priv *wil, struct wil_ring *ring) 42796c93589SGidon Studinski { 42896c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 42996c93589SGidon Studinski size_t sz; 43096c93589SGidon Studinski int ring_index = 0; 43196c93589SGidon Studinski 43296c93589SGidon Studinski if (!ring->va) 43396c93589SGidon Studinski return; 43496c93589SGidon Studinski 43596c93589SGidon Studinski sz = ring->size * sizeof(ring->va[0]); 43696c93589SGidon Studinski 43796c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 43896c93589SGidon Studinski if (ring->is_rx) { 43996c93589SGidon Studinski wil_dbg_misc(wil, "free Rx ring [%d] 0x%p:%pad 0x%p\n", 44096c93589SGidon Studinski ring->size, ring->va, 44196c93589SGidon Studinski &ring->pa, ring->ctx); 44296c93589SGidon Studinski 44396c93589SGidon Studinski wil_move_all_rx_buff_to_free_list(wil, ring); 44496c93589SGidon Studinski goto out; 44596c93589SGidon Studinski } 44696c93589SGidon Studinski 44796c93589SGidon Studinski /* TX ring */ 44896c93589SGidon Studinski ring_index = ring - wil->ring_tx; 44996c93589SGidon Studinski 45096c93589SGidon Studinski wil_dbg_misc(wil, "free Tx ring %d [%d] 0x%p:%pad 0x%p\n", 45196c93589SGidon Studinski ring_index, ring->size, ring->va, 45296c93589SGidon Studinski &ring->pa, ring->ctx); 45396c93589SGidon Studinski 45496c93589SGidon Studinski while (!wil_ring_is_empty(ring)) { 45596c93589SGidon Studinski struct wil_ctx *ctx; 45696c93589SGidon Studinski 45796c93589SGidon Studinski struct wil_tx_enhanced_desc dd, *d = ⅆ 45896c93589SGidon Studinski struct wil_tx_enhanced_desc *_d = 45996c93589SGidon Studinski (struct wil_tx_enhanced_desc *) 46096c93589SGidon Studinski &ring->va[ring->swtail].tx.enhanced; 46196c93589SGidon Studinski 46296c93589SGidon Studinski ctx = &ring->ctx[ring->swtail]; 46396c93589SGidon Studinski if (!ctx) { 46496c93589SGidon Studinski wil_dbg_txrx(wil, 46596c93589SGidon Studinski "ctx(%d) was already completed\n", 46696c93589SGidon Studinski ring->swtail); 46796c93589SGidon Studinski ring->swtail = wil_ring_next_tail(ring); 46896c93589SGidon Studinski continue; 46996c93589SGidon Studinski } 47096c93589SGidon Studinski *d = *_d; 4719202d7b6SMaya Erez wil_tx_desc_unmap_edma(dev, (union wil_tx_desc *)d, ctx); 47296c93589SGidon Studinski if (ctx->skb) 47396c93589SGidon Studinski dev_kfree_skb_any(ctx->skb); 47496c93589SGidon Studinski ring->swtail = wil_ring_next_tail(ring); 47596c93589SGidon Studinski } 47696c93589SGidon Studinski 47796c93589SGidon Studinski out: 47896c93589SGidon Studinski dma_free_coherent(dev, sz, (void *)ring->va, ring->pa); 47996c93589SGidon Studinski kfree(ring->ctx); 48096c93589SGidon Studinski ring->pa = 0; 48196c93589SGidon Studinski ring->va = NULL; 48296c93589SGidon Studinski ring->ctx = NULL; 48396c93589SGidon Studinski } 48496c93589SGidon Studinski 48596c93589SGidon Studinski static int wil_init_rx_desc_ring(struct wil6210_priv *wil, u16 desc_ring_size, 48696c93589SGidon Studinski int status_ring_id) 48796c93589SGidon Studinski { 48896c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 48996c93589SGidon Studinski int rc; 49096c93589SGidon Studinski 49196c93589SGidon Studinski wil_dbg_misc(wil, "init RX desc ring\n"); 49296c93589SGidon Studinski 49396c93589SGidon Studinski ring->size = desc_ring_size; 49496c93589SGidon Studinski ring->is_rx = true; 49596c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 49696c93589SGidon Studinski if (rc) 49796c93589SGidon Studinski return rc; 49896c93589SGidon Studinski 49996c93589SGidon Studinski rc = wil_wmi_rx_desc_ring_add(wil, status_ring_id); 50096c93589SGidon Studinski if (rc) 50196c93589SGidon Studinski goto out_free; 50296c93589SGidon Studinski 50396c93589SGidon Studinski return 0; 50496c93589SGidon Studinski out_free: 50596c93589SGidon Studinski wil_ring_free_edma(wil, ring); 50696c93589SGidon Studinski return rc; 50796c93589SGidon Studinski } 50896c93589SGidon Studinski 5097be13fc3SGidon Studinski static void wil_get_reorder_params_edma(struct wil6210_priv *wil, 5107be13fc3SGidon Studinski struct sk_buff *skb, int *tid, 5117be13fc3SGidon Studinski int *cid, int *mid, u16 *seq, 5121bd82ee0SDedy Lansky int *mcast, int *retry) 5137be13fc3SGidon Studinski { 5147be13fc3SGidon Studinski struct wil_rx_status_extended *s = wil_skb_rxstatus(skb); 5157be13fc3SGidon Studinski 5167be13fc3SGidon Studinski *tid = wil_rx_status_get_tid(s); 5177be13fc3SGidon Studinski *cid = wil_rx_status_get_cid(s); 5187be13fc3SGidon Studinski *mid = wil_rx_status_get_mid(s); 5197be13fc3SGidon Studinski *seq = le16_to_cpu(wil_rx_status_get_seq(wil, s)); 5207be13fc3SGidon Studinski *mcast = wil_rx_status_get_mcast(s); 5211bd82ee0SDedy Lansky *retry = wil_rx_status_get_retry(s); 5227be13fc3SGidon Studinski } 5237be13fc3SGidon Studinski 5247be13fc3SGidon Studinski static void wil_get_netif_rx_params_edma(struct sk_buff *skb, int *cid, 5257be13fc3SGidon Studinski int *security) 5267be13fc3SGidon Studinski { 5277be13fc3SGidon Studinski struct wil_rx_status_extended *s = wil_skb_rxstatus(skb); 5287be13fc3SGidon Studinski 5297be13fc3SGidon Studinski *cid = wil_rx_status_get_cid(s); 5307be13fc3SGidon Studinski *security = wil_rx_status_get_security(s); 5317be13fc3SGidon Studinski } 5327be13fc3SGidon Studinski 5337be13fc3SGidon Studinski static int wil_rx_crypto_check_edma(struct wil6210_priv *wil, 5347be13fc3SGidon Studinski struct sk_buff *skb) 5357be13fc3SGidon Studinski { 5367be13fc3SGidon Studinski struct wil_rx_status_extended *st; 5377be13fc3SGidon Studinski int cid, tid, key_id, mc; 5387be13fc3SGidon Studinski struct wil_sta_info *s; 5397be13fc3SGidon Studinski struct wil_tid_crypto_rx *c; 5407be13fc3SGidon Studinski struct wil_tid_crypto_rx_single *cc; 5417be13fc3SGidon Studinski const u8 *pn; 5427be13fc3SGidon Studinski 5437be13fc3SGidon Studinski /* In HW reorder, HW is responsible for crypto check */ 5447be13fc3SGidon Studinski if (wil->use_rx_hw_reordering) 5457be13fc3SGidon Studinski return 0; 5467be13fc3SGidon Studinski 5477be13fc3SGidon Studinski st = wil_skb_rxstatus(skb); 5487be13fc3SGidon Studinski 5497be13fc3SGidon Studinski cid = wil_rx_status_get_cid(st); 5507be13fc3SGidon Studinski tid = wil_rx_status_get_tid(st); 5517be13fc3SGidon Studinski key_id = wil_rx_status_get_key_id(st); 5527be13fc3SGidon Studinski mc = wil_rx_status_get_mcast(st); 5537be13fc3SGidon Studinski s = &wil->sta[cid]; 5547be13fc3SGidon Studinski c = mc ? &s->group_crypto_rx : &s->tid_crypto_rx[tid]; 5557be13fc3SGidon Studinski cc = &c->key_id[key_id]; 5567be13fc3SGidon Studinski pn = (u8 *)&st->ext.pn_15_0; 5577be13fc3SGidon Studinski 5587be13fc3SGidon Studinski if (!cc->key_set) { 5597be13fc3SGidon Studinski wil_err_ratelimited(wil, 5607be13fc3SGidon Studinski "Key missing. CID %d TID %d MCast %d KEY_ID %d\n", 5617be13fc3SGidon Studinski cid, tid, mc, key_id); 5627be13fc3SGidon Studinski return -EINVAL; 5637be13fc3SGidon Studinski } 5647be13fc3SGidon Studinski 5657be13fc3SGidon Studinski if (reverse_memcmp(pn, cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) { 5667be13fc3SGidon Studinski wil_err_ratelimited(wil, 5677be13fc3SGidon Studinski "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n", 5687be13fc3SGidon Studinski cid, tid, mc, key_id, pn, cc->pn); 5697be13fc3SGidon Studinski return -EINVAL; 5707be13fc3SGidon Studinski } 5717be13fc3SGidon Studinski memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN); 5727be13fc3SGidon Studinski 5737be13fc3SGidon Studinski return 0; 5747be13fc3SGidon Studinski } 5757be13fc3SGidon Studinski 5767be13fc3SGidon Studinski static bool wil_is_rx_idle_edma(struct wil6210_priv *wil) 5777be13fc3SGidon Studinski { 5787be13fc3SGidon Studinski struct wil_status_ring *sring; 5797be13fc3SGidon Studinski struct wil_rx_status_extended msg1; 5807be13fc3SGidon Studinski void *msg = &msg1; 5817be13fc3SGidon Studinski u8 dr_bit; 5827be13fc3SGidon Studinski int i; 5837be13fc3SGidon Studinski 5847be13fc3SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 5857be13fc3SGidon Studinski sring = &wil->srings[i]; 5867be13fc3SGidon Studinski if (!sring->va) 5877be13fc3SGidon Studinski continue; 5887be13fc3SGidon Studinski 5897be13fc3SGidon Studinski wil_get_next_rx_status_msg(sring, msg); 5907be13fc3SGidon Studinski dr_bit = wil_rx_status_get_desc_rdy_bit(msg); 5917be13fc3SGidon Studinski 5927be13fc3SGidon Studinski /* Check if there are unhandled RX status messages */ 5937be13fc3SGidon Studinski if (dr_bit == sring->desc_rdy_pol) 5947be13fc3SGidon Studinski return false; 5957be13fc3SGidon Studinski } 5967be13fc3SGidon Studinski 5977be13fc3SGidon Studinski return true; 5987be13fc3SGidon Studinski } 5997be13fc3SGidon Studinski 60096c93589SGidon Studinski static void wil_rx_buf_len_init_edma(struct wil6210_priv *wil) 60196c93589SGidon Studinski { 60296c93589SGidon Studinski wil->rx_buf_len = rx_large_buf ? 60396c93589SGidon Studinski WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD; 60496c93589SGidon Studinski } 60596c93589SGidon Studinski 60696c93589SGidon Studinski static int wil_rx_init_edma(struct wil6210_priv *wil, u16 desc_ring_size) 60796c93589SGidon Studinski { 60896c93589SGidon Studinski u16 status_ring_size; 60996c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 61096c93589SGidon Studinski int rc; 61196c93589SGidon Studinski size_t elem_size = wil->use_compressed_rx_status ? 61296c93589SGidon Studinski sizeof(struct wil_rx_status_compressed) : 61396c93589SGidon Studinski sizeof(struct wil_rx_status_extended); 61496c93589SGidon Studinski int i; 61596c93589SGidon Studinski u16 max_rx_pl_per_desc; 61696c93589SGidon Studinski 6177be13fc3SGidon Studinski /* In SW reorder one must use extended status messages */ 6187be13fc3SGidon Studinski if (wil->use_compressed_rx_status && !wil->use_rx_hw_reordering) { 6197be13fc3SGidon Studinski wil_err(wil, 6207be13fc3SGidon Studinski "compressed RX status cannot be used with SW reorder\n"); 6217be13fc3SGidon Studinski return -EINVAL; 6227be13fc3SGidon Studinski } 6237be13fc3SGidon Studinski 62496c93589SGidon Studinski if (wil->rx_status_ring_order < WIL_SRING_SIZE_ORDER_MIN || 62596c93589SGidon Studinski wil->rx_status_ring_order > WIL_SRING_SIZE_ORDER_MAX) 62696c93589SGidon Studinski wil->rx_status_ring_order = WIL_RX_SRING_SIZE_ORDER_DEFAULT; 62796c93589SGidon Studinski 62896c93589SGidon Studinski status_ring_size = 1 << wil->rx_status_ring_order; 62996c93589SGidon Studinski 63096c93589SGidon Studinski wil_dbg_misc(wil, 63196c93589SGidon Studinski "rx_init, desc_ring_size=%u, status_ring_size=%u, elem_size=%zu\n", 63296c93589SGidon Studinski desc_ring_size, status_ring_size, elem_size); 63396c93589SGidon Studinski 63496c93589SGidon Studinski wil_rx_buf_len_init_edma(wil); 63596c93589SGidon Studinski 63696c93589SGidon Studinski max_rx_pl_per_desc = wil->rx_buf_len + ETH_HLEN + 63796c93589SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 63896c93589SGidon Studinski 63996c93589SGidon Studinski /* Use debugfs dbg_num_rx_srings if set, reserve one sring for TX */ 64096c93589SGidon Studinski if (wil->num_rx_status_rings > WIL6210_MAX_STATUS_RINGS - 1) 64196c93589SGidon Studinski wil->num_rx_status_rings = WIL6210_MAX_STATUS_RINGS - 1; 64296c93589SGidon Studinski 64396c93589SGidon Studinski wil_dbg_misc(wil, "rx_init: allocate %d status rings\n", 64496c93589SGidon Studinski wil->num_rx_status_rings); 64596c93589SGidon Studinski 64696c93589SGidon Studinski rc = wil_wmi_cfg_def_rx_offload(wil, max_rx_pl_per_desc); 64796c93589SGidon Studinski if (rc) 64896c93589SGidon Studinski return rc; 64996c93589SGidon Studinski 65096c93589SGidon Studinski /* Allocate status ring */ 65196c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 65296c93589SGidon Studinski int sring_id = wil_find_free_sring(wil); 65396c93589SGidon Studinski 65496c93589SGidon Studinski if (sring_id < 0) { 65596c93589SGidon Studinski rc = -EFAULT; 65696c93589SGidon Studinski goto err_free_status; 65796c93589SGidon Studinski } 65896c93589SGidon Studinski rc = wil_init_rx_sring(wil, status_ring_size, elem_size, 65996c93589SGidon Studinski sring_id); 66096c93589SGidon Studinski if (rc) 66196c93589SGidon Studinski goto err_free_status; 66296c93589SGidon Studinski } 66396c93589SGidon Studinski 66496c93589SGidon Studinski /* Allocate descriptor ring */ 66596c93589SGidon Studinski rc = wil_init_rx_desc_ring(wil, desc_ring_size, 66696c93589SGidon Studinski WIL_DEFAULT_RX_STATUS_RING_ID); 66796c93589SGidon Studinski if (rc) 66896c93589SGidon Studinski goto err_free_status; 66996c93589SGidon Studinski 67096c93589SGidon Studinski if (wil->rx_buff_id_count >= status_ring_size) { 67196c93589SGidon Studinski wil_info(wil, 67296c93589SGidon Studinski "rx_buff_id_count %d exceeds sring_size %d. set it to %d\n", 67396c93589SGidon Studinski wil->rx_buff_id_count, status_ring_size, 67496c93589SGidon Studinski status_ring_size - 1); 67596c93589SGidon Studinski wil->rx_buff_id_count = status_ring_size - 1; 67696c93589SGidon Studinski } 67796c93589SGidon Studinski 67896c93589SGidon Studinski /* Allocate Rx buffer array */ 67996c93589SGidon Studinski rc = wil_init_rx_buff_arr(wil, wil->rx_buff_id_count); 68096c93589SGidon Studinski if (rc) 68196c93589SGidon Studinski goto err_free_desc; 68296c93589SGidon Studinski 68396c93589SGidon Studinski /* Fill descriptor ring with credits */ 68496c93589SGidon Studinski rc = wil_rx_refill_edma(wil); 68596c93589SGidon Studinski if (rc) 68696c93589SGidon Studinski goto err_free_rx_buff_arr; 68796c93589SGidon Studinski 68896c93589SGidon Studinski return 0; 68996c93589SGidon Studinski err_free_rx_buff_arr: 69096c93589SGidon Studinski wil_free_rx_buff_arr(wil); 69196c93589SGidon Studinski err_free_desc: 69296c93589SGidon Studinski wil_ring_free_edma(wil, ring); 69396c93589SGidon Studinski err_free_status: 69496c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) 69596c93589SGidon Studinski wil_sring_free(wil, &wil->srings[i]); 69696c93589SGidon Studinski 69796c93589SGidon Studinski return rc; 69896c93589SGidon Studinski } 69996c93589SGidon Studinski 70096c93589SGidon Studinski static int wil_ring_init_tx_edma(struct wil6210_vif *vif, int ring_id, 70196c93589SGidon Studinski int size, int cid, int tid) 70296c93589SGidon Studinski { 70396c93589SGidon Studinski struct wil6210_priv *wil = vif_to_wil(vif); 70496c93589SGidon Studinski int rc; 70596c93589SGidon Studinski struct wil_ring *ring = &wil->ring_tx[ring_id]; 70696c93589SGidon Studinski struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 70796c93589SGidon Studinski 70896c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 70996c93589SGidon Studinski 71096c93589SGidon Studinski wil_dbg_misc(wil, 71196c93589SGidon Studinski "init TX ring: ring_id=%u, cid=%u, tid=%u, sring_id=%u\n", 71296c93589SGidon Studinski ring_id, cid, tid, wil->tx_sring_idx); 71396c93589SGidon Studinski 71496c93589SGidon Studinski wil_tx_data_init(txdata); 71596c93589SGidon Studinski ring->size = size; 71696c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 71796c93589SGidon Studinski if (rc) 71896c93589SGidon Studinski goto out; 71996c93589SGidon Studinski 72096c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = cid; 72196c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = tid; 72296c93589SGidon Studinski if (!vif->privacy) 72396c93589SGidon Studinski txdata->dot1x_open = true; 72496c93589SGidon Studinski 72596c93589SGidon Studinski rc = wil_wmi_tx_desc_ring_add(vif, ring_id, cid, tid); 72696c93589SGidon Studinski if (rc) { 72796c93589SGidon Studinski wil_err(wil, "WMI_TX_DESC_RING_ADD_CMD failed\n"); 72896c93589SGidon Studinski goto out_free; 72996c93589SGidon Studinski } 73096c93589SGidon Studinski 73196c93589SGidon Studinski if (txdata->dot1x_open && agg_wsize >= 0) 73296c93589SGidon Studinski wil_addba_tx_request(wil, ring_id, agg_wsize); 73396c93589SGidon Studinski 73496c93589SGidon Studinski return 0; 73596c93589SGidon Studinski out_free: 73696c93589SGidon Studinski spin_lock_bh(&txdata->lock); 73796c93589SGidon Studinski txdata->dot1x_open = false; 73896c93589SGidon Studinski txdata->enabled = 0; 73996c93589SGidon Studinski spin_unlock_bh(&txdata->lock); 74096c93589SGidon Studinski wil_ring_free_edma(wil, ring); 74196c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = WIL6210_MAX_CID; 74296c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = 0; 74396c93589SGidon Studinski 74496c93589SGidon Studinski out: 74596c93589SGidon Studinski return rc; 74696c93589SGidon Studinski } 74796c93589SGidon Studinski 7487be13fc3SGidon Studinski /* This function is used only for RX SW reorder */ 7497be13fc3SGidon Studinski static int wil_check_bar(struct wil6210_priv *wil, void *msg, int cid, 7507be13fc3SGidon Studinski struct sk_buff *skb, struct wil_net_stats *stats) 7517be13fc3SGidon Studinski { 7527be13fc3SGidon Studinski u8 ftype; 7537be13fc3SGidon Studinski u8 fc1; 7547be13fc3SGidon Studinski int mid; 7557be13fc3SGidon Studinski int tid; 7567be13fc3SGidon Studinski u16 seq; 7577be13fc3SGidon Studinski struct wil6210_vif *vif; 7587be13fc3SGidon Studinski 7597be13fc3SGidon Studinski ftype = wil_rx_status_get_frame_type(wil, msg); 7607be13fc3SGidon Studinski if (ftype == IEEE80211_FTYPE_DATA) 7617be13fc3SGidon Studinski return 0; 7627be13fc3SGidon Studinski 7637be13fc3SGidon Studinski fc1 = wil_rx_status_get_fc1(wil, msg); 7647be13fc3SGidon Studinski mid = wil_rx_status_get_mid(msg); 7657be13fc3SGidon Studinski tid = wil_rx_status_get_tid(msg); 7667be13fc3SGidon Studinski seq = le16_to_cpu(wil_rx_status_get_seq(wil, msg)); 7677be13fc3SGidon Studinski vif = wil->vifs[mid]; 7687be13fc3SGidon Studinski 7697be13fc3SGidon Studinski if (unlikely(!vif)) { 7707be13fc3SGidon Studinski wil_dbg_txrx(wil, "RX descriptor with invalid mid %d", mid); 7717be13fc3SGidon Studinski return -EAGAIN; 7727be13fc3SGidon Studinski } 7737be13fc3SGidon Studinski 7747be13fc3SGidon Studinski wil_dbg_txrx(wil, 7757be13fc3SGidon Studinski "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 7767be13fc3SGidon Studinski fc1, mid, cid, tid, seq); 7777be13fc3SGidon Studinski if (stats) 7787be13fc3SGidon Studinski stats->rx_non_data_frame++; 7797be13fc3SGidon Studinski if (wil_is_back_req(fc1)) { 7807be13fc3SGidon Studinski wil_dbg_txrx(wil, 7817be13fc3SGidon Studinski "BAR: MID %d CID %d TID %d Seq 0x%03x\n", 7827be13fc3SGidon Studinski mid, cid, tid, seq); 7837be13fc3SGidon Studinski wil_rx_bar(wil, vif, cid, tid, seq); 7847be13fc3SGidon Studinski } else { 7857be13fc3SGidon Studinski u32 sz = wil->use_compressed_rx_status ? 7867be13fc3SGidon Studinski sizeof(struct wil_rx_status_compressed) : 7877be13fc3SGidon Studinski sizeof(struct wil_rx_status_extended); 7887be13fc3SGidon Studinski 7897be13fc3SGidon Studinski /* print again all info. One can enable only this 7907be13fc3SGidon Studinski * without overhead for printing every Rx frame 7917be13fc3SGidon Studinski */ 7927be13fc3SGidon Studinski wil_dbg_txrx(wil, 7937be13fc3SGidon Studinski "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 7947be13fc3SGidon Studinski fc1, mid, cid, tid, seq); 7957be13fc3SGidon Studinski wil_hex_dump_txrx("RxS ", DUMP_PREFIX_NONE, 32, 4, 7967be13fc3SGidon Studinski (const void *)msg, sz, false); 7977be13fc3SGidon Studinski wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 7987be13fc3SGidon Studinski skb->data, skb_headlen(skb), false); 7997be13fc3SGidon Studinski } 8007be13fc3SGidon Studinski 8017be13fc3SGidon Studinski return -EAGAIN; 8027be13fc3SGidon Studinski } 8037be13fc3SGidon Studinski 804e15af41cSDedy Lansky static int wil_rx_error_check_edma(struct wil6210_priv *wil, 805e15af41cSDedy Lansky struct sk_buff *skb, 806e15af41cSDedy Lansky struct wil_net_stats *stats) 8077be13fc3SGidon Studinski { 8087be13fc3SGidon Studinski int error; 8097be13fc3SGidon Studinski int l2_rx_status; 8107be13fc3SGidon Studinski int l3_rx_status; 8117be13fc3SGidon Studinski int l4_rx_status; 812e15af41cSDedy Lansky void *msg = wil_skb_rxstatus(skb); 8137be13fc3SGidon Studinski 8147be13fc3SGidon Studinski error = wil_rx_status_get_error(msg); 8157be13fc3SGidon Studinski if (!error) { 8167be13fc3SGidon Studinski skb->ip_summed = CHECKSUM_UNNECESSARY; 8177be13fc3SGidon Studinski return 0; 8187be13fc3SGidon Studinski } 8197be13fc3SGidon Studinski 8207be13fc3SGidon Studinski l2_rx_status = wil_rx_status_get_l2_rx_status(msg); 8217be13fc3SGidon Studinski if (l2_rx_status != 0) { 8227be13fc3SGidon Studinski wil_dbg_txrx(wil, "L2 RX error, l2_rx_status=0x%x\n", 8237be13fc3SGidon Studinski l2_rx_status); 8247be13fc3SGidon Studinski /* Due to HW issue, KEY error will trigger a MIC error */ 8257be13fc3SGidon Studinski if (l2_rx_status & WIL_RX_EDMA_ERROR_MIC) { 8267be13fc3SGidon Studinski wil_dbg_txrx(wil, 8277be13fc3SGidon Studinski "L2 MIC/KEY error, dropping packet\n"); 8287be13fc3SGidon Studinski stats->rx_mic_error++; 8297be13fc3SGidon Studinski } 8307be13fc3SGidon Studinski if (l2_rx_status & WIL_RX_EDMA_ERROR_KEY) { 8317be13fc3SGidon Studinski wil_dbg_txrx(wil, "L2 KEY error, dropping packet\n"); 8327be13fc3SGidon Studinski stats->rx_key_error++; 8337be13fc3SGidon Studinski } 8347be13fc3SGidon Studinski if (l2_rx_status & WIL_RX_EDMA_ERROR_REPLAY) { 8357be13fc3SGidon Studinski wil_dbg_txrx(wil, 8367be13fc3SGidon Studinski "L2 REPLAY error, dropping packet\n"); 8377be13fc3SGidon Studinski stats->rx_replay++; 8387be13fc3SGidon Studinski } 8397be13fc3SGidon Studinski if (l2_rx_status & WIL_RX_EDMA_ERROR_AMSDU) { 8407be13fc3SGidon Studinski wil_dbg_txrx(wil, 8417be13fc3SGidon Studinski "L2 AMSDU error, dropping packet\n"); 8427be13fc3SGidon Studinski stats->rx_amsdu_error++; 8437be13fc3SGidon Studinski } 8447be13fc3SGidon Studinski return -EFAULT; 8457be13fc3SGidon Studinski } 8467be13fc3SGidon Studinski 8477be13fc3SGidon Studinski l3_rx_status = wil_rx_status_get_l3_rx_status(msg); 8487be13fc3SGidon Studinski l4_rx_status = wil_rx_status_get_l4_rx_status(msg); 8497be13fc3SGidon Studinski if (!l3_rx_status && !l4_rx_status) 8507be13fc3SGidon Studinski skb->ip_summed = CHECKSUM_UNNECESSARY; 8517be13fc3SGidon Studinski /* If HW reports bad checksum, let IP stack re-check it 8527be13fc3SGidon Studinski * For example, HW don't understand Microsoft IP stack that 8537be13fc3SGidon Studinski * mis-calculates TCP checksum - if it should be 0x0, 8547be13fc3SGidon Studinski * it writes 0xffff in violation of RFC 1624 8557be13fc3SGidon Studinski */ 8560b853210SMaya Erez else 8570b853210SMaya Erez stats->rx_csum_err++; 8587be13fc3SGidon Studinski 8597be13fc3SGidon Studinski return 0; 8607be13fc3SGidon Studinski } 8617be13fc3SGidon Studinski 8627be13fc3SGidon Studinski static struct sk_buff *wil_sring_reap_rx_edma(struct wil6210_priv *wil, 8637be13fc3SGidon Studinski struct wil_status_ring *sring) 8647be13fc3SGidon Studinski { 8657be13fc3SGidon Studinski struct device *dev = wil_to_dev(wil); 8667be13fc3SGidon Studinski struct wil_rx_status_extended msg1; 8677be13fc3SGidon Studinski void *msg = &msg1; 8687be13fc3SGidon Studinski u16 buff_id; 8697be13fc3SGidon Studinski struct sk_buff *skb; 8707be13fc3SGidon Studinski dma_addr_t pa; 8717be13fc3SGidon Studinski struct wil_ring_rx_data *rxdata = &sring->rx_data; 8727be13fc3SGidon Studinski unsigned int sz = wil->rx_buf_len + ETH_HLEN + 8737be13fc3SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 8747be13fc3SGidon Studinski struct wil_net_stats *stats = NULL; 8757be13fc3SGidon Studinski u16 dmalen; 8767be13fc3SGidon Studinski int cid; 8777be13fc3SGidon Studinski bool eop, headstolen; 8787be13fc3SGidon Studinski int delta; 8797be13fc3SGidon Studinski u8 dr_bit; 8807be13fc3SGidon Studinski u8 data_offset; 8817be13fc3SGidon Studinski struct wil_rx_status_extended *s; 8827be13fc3SGidon Studinski u16 sring_idx = sring - wil->srings; 8837be13fc3SGidon Studinski 8847be13fc3SGidon Studinski BUILD_BUG_ON(sizeof(struct wil_rx_status_extended) > sizeof(skb->cb)); 8857be13fc3SGidon Studinski 8867be13fc3SGidon Studinski again: 8877be13fc3SGidon Studinski wil_get_next_rx_status_msg(sring, msg); 8887be13fc3SGidon Studinski dr_bit = wil_rx_status_get_desc_rdy_bit(msg); 8897be13fc3SGidon Studinski 8907be13fc3SGidon Studinski /* Completed handling all the ready status messages */ 8917be13fc3SGidon Studinski if (dr_bit != sring->desc_rdy_pol) 8927be13fc3SGidon Studinski return NULL; 8937be13fc3SGidon Studinski 8947be13fc3SGidon Studinski /* Extract the buffer ID from the status message */ 8957be13fc3SGidon Studinski buff_id = le16_to_cpu(wil_rx_status_get_buff_id(msg)); 8967be13fc3SGidon Studinski if (unlikely(!wil_val_in_range(buff_id, 0, wil->rx_buff_mgmt.size))) { 8977be13fc3SGidon Studinski wil_err(wil, "Corrupt buff_id=%d, sring->swhead=%d\n", 8987be13fc3SGidon Studinski buff_id, sring->swhead); 8997be13fc3SGidon Studinski wil_sring_advance_swhead(sring); 9007be13fc3SGidon Studinski goto again; 9017be13fc3SGidon Studinski } 9027be13fc3SGidon Studinski 9037be13fc3SGidon Studinski wil_sring_advance_swhead(sring); 9047be13fc3SGidon Studinski 9057be13fc3SGidon Studinski /* Extract the SKB from the rx_buff management array */ 9067be13fc3SGidon Studinski skb = wil->rx_buff_mgmt.buff_arr[buff_id].skb; 9077be13fc3SGidon Studinski wil->rx_buff_mgmt.buff_arr[buff_id].skb = NULL; 9087be13fc3SGidon Studinski if (!skb) { 9097be13fc3SGidon Studinski wil_err(wil, "No Rx skb at buff_id %d\n", buff_id); 9107be13fc3SGidon Studinski goto again; 9117be13fc3SGidon Studinski } 9127be13fc3SGidon Studinski 9137be13fc3SGidon Studinski memcpy(&pa, skb->cb, sizeof(pa)); 9147be13fc3SGidon Studinski dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE); 9157be13fc3SGidon Studinski dmalen = le16_to_cpu(wil_rx_status_get_length(msg)); 9167be13fc3SGidon Studinski 9177be13fc3SGidon Studinski trace_wil6210_rx_status(wil, wil->use_compressed_rx_status, buff_id, 9187be13fc3SGidon Studinski msg); 9197be13fc3SGidon Studinski wil_dbg_txrx(wil, "Rx, buff_id=%u, sring_idx=%u, dmalen=%u bytes\n", 9207be13fc3SGidon Studinski buff_id, sring_idx, dmalen); 9217be13fc3SGidon Studinski wil_hex_dump_txrx("RxS ", DUMP_PREFIX_NONE, 32, 4, 9227be13fc3SGidon Studinski (const void *)msg, wil->use_compressed_rx_status ? 9237be13fc3SGidon Studinski sizeof(struct wil_rx_status_compressed) : 9247be13fc3SGidon Studinski sizeof(struct wil_rx_status_extended), false); 9257be13fc3SGidon Studinski 9267be13fc3SGidon Studinski /* Move the buffer from the active list to the free list */ 9277be13fc3SGidon Studinski list_move(&wil->rx_buff_mgmt.buff_arr[buff_id].list, 9287be13fc3SGidon Studinski &wil->rx_buff_mgmt.free); 9297be13fc3SGidon Studinski 9307be13fc3SGidon Studinski eop = wil_rx_status_get_eop(msg); 9317be13fc3SGidon Studinski 9327be13fc3SGidon Studinski cid = wil_rx_status_get_cid(msg); 9337be13fc3SGidon Studinski if (unlikely(!wil_val_in_range(cid, 0, WIL6210_MAX_CID))) { 9347be13fc3SGidon Studinski wil_err(wil, "Corrupt cid=%d, sring->swhead=%d\n", 9357be13fc3SGidon Studinski cid, sring->swhead); 9367be13fc3SGidon Studinski rxdata->skipping = true; 9377be13fc3SGidon Studinski goto skipping; 9387be13fc3SGidon Studinski } 9397be13fc3SGidon Studinski stats = &wil->sta[cid].stats; 9407be13fc3SGidon Studinski 9417be13fc3SGidon Studinski if (unlikely(skb->len < ETH_HLEN)) { 9427be13fc3SGidon Studinski wil_dbg_txrx(wil, "Short frame, len = %d\n", skb->len); 9437be13fc3SGidon Studinski stats->rx_short_frame++; 9447be13fc3SGidon Studinski rxdata->skipping = true; 9457be13fc3SGidon Studinski goto skipping; 9467be13fc3SGidon Studinski } 9477be13fc3SGidon Studinski 9487be13fc3SGidon Studinski if (unlikely(dmalen > sz)) { 9497be13fc3SGidon Studinski wil_err(wil, "Rx size too large: %d bytes!\n", dmalen); 9507be13fc3SGidon Studinski stats->rx_large_frame++; 9517be13fc3SGidon Studinski rxdata->skipping = true; 9527be13fc3SGidon Studinski } 9537be13fc3SGidon Studinski 9547be13fc3SGidon Studinski skipping: 9557be13fc3SGidon Studinski /* skipping indicates if a certain SKB should be dropped. 9567be13fc3SGidon Studinski * It is set in case there is an error on the current SKB or in case 9577be13fc3SGidon Studinski * of RX chaining: as long as we manage to merge the SKBs it will 9587be13fc3SGidon Studinski * be false. once we have a bad SKB or we don't manage to merge SKBs 9597be13fc3SGidon Studinski * it will be set to the !EOP value of the current SKB. 9607be13fc3SGidon Studinski * This guarantees that all the following SKBs until EOP will also 9617be13fc3SGidon Studinski * get dropped. 9627be13fc3SGidon Studinski */ 9637be13fc3SGidon Studinski if (unlikely(rxdata->skipping)) { 9647be13fc3SGidon Studinski kfree_skb(skb); 9657be13fc3SGidon Studinski if (rxdata->skb) { 9667be13fc3SGidon Studinski kfree_skb(rxdata->skb); 9677be13fc3SGidon Studinski rxdata->skb = NULL; 9687be13fc3SGidon Studinski } 9697be13fc3SGidon Studinski rxdata->skipping = !eop; 9707be13fc3SGidon Studinski goto again; 9717be13fc3SGidon Studinski } 9727be13fc3SGidon Studinski 9737be13fc3SGidon Studinski skb_trim(skb, dmalen); 9747be13fc3SGidon Studinski 9757be13fc3SGidon Studinski prefetch(skb->data); 9767be13fc3SGidon Studinski 9777be13fc3SGidon Studinski if (!rxdata->skb) { 9787be13fc3SGidon Studinski rxdata->skb = skb; 9797be13fc3SGidon Studinski } else { 9807be13fc3SGidon Studinski if (likely(skb_try_coalesce(rxdata->skb, skb, &headstolen, 9817be13fc3SGidon Studinski &delta))) { 9827be13fc3SGidon Studinski kfree_skb_partial(skb, headstolen); 9837be13fc3SGidon Studinski } else { 9847be13fc3SGidon Studinski wil_err(wil, "failed to merge skbs!\n"); 9857be13fc3SGidon Studinski kfree_skb(skb); 9867be13fc3SGidon Studinski kfree_skb(rxdata->skb); 9877be13fc3SGidon Studinski rxdata->skb = NULL; 9887be13fc3SGidon Studinski rxdata->skipping = !eop; 9897be13fc3SGidon Studinski goto again; 9907be13fc3SGidon Studinski } 9917be13fc3SGidon Studinski } 9927be13fc3SGidon Studinski 9937be13fc3SGidon Studinski if (!eop) 9947be13fc3SGidon Studinski goto again; 9957be13fc3SGidon Studinski 9967be13fc3SGidon Studinski /* reaching here rxdata->skb always contains a full packet */ 9977be13fc3SGidon Studinski skb = rxdata->skb; 9987be13fc3SGidon Studinski rxdata->skb = NULL; 9997be13fc3SGidon Studinski rxdata->skipping = false; 10007be13fc3SGidon Studinski 10017be13fc3SGidon Studinski if (stats) { 10027be13fc3SGidon Studinski stats->last_mcs_rx = wil_rx_status_get_mcs(msg); 10037be13fc3SGidon Studinski if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs)) 10047be13fc3SGidon Studinski stats->rx_per_mcs[stats->last_mcs_rx]++; 10057be13fc3SGidon Studinski } 10067be13fc3SGidon Studinski 10077be13fc3SGidon Studinski if (!wil->use_rx_hw_reordering && !wil->use_compressed_rx_status && 10087be13fc3SGidon Studinski wil_check_bar(wil, msg, cid, skb, stats) == -EAGAIN) { 10097be13fc3SGidon Studinski kfree_skb(skb); 10107be13fc3SGidon Studinski goto again; 10117be13fc3SGidon Studinski } 10127be13fc3SGidon Studinski 10137be13fc3SGidon Studinski /* Compensate for the HW data alignment according to the status 10147be13fc3SGidon Studinski * message 10157be13fc3SGidon Studinski */ 10167be13fc3SGidon Studinski data_offset = wil_rx_status_get_data_offset(msg); 10177be13fc3SGidon Studinski if (data_offset == 0xFF || 10187be13fc3SGidon Studinski data_offset > WIL_EDMA_MAX_DATA_OFFSET) { 10197be13fc3SGidon Studinski wil_err(wil, "Unexpected data offset %d\n", data_offset); 10207be13fc3SGidon Studinski kfree_skb(skb); 10217be13fc3SGidon Studinski goto again; 10227be13fc3SGidon Studinski } 10237be13fc3SGidon Studinski 10247be13fc3SGidon Studinski skb_pull(skb, data_offset); 10257be13fc3SGidon Studinski 10267be13fc3SGidon Studinski wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 10277be13fc3SGidon Studinski skb->data, skb_headlen(skb), false); 10287be13fc3SGidon Studinski 10297be13fc3SGidon Studinski /* Has to be done after dma_unmap_single as skb->cb is also 10307be13fc3SGidon Studinski * used for holding the pa 10317be13fc3SGidon Studinski */ 10327be13fc3SGidon Studinski s = wil_skb_rxstatus(skb); 10337be13fc3SGidon Studinski memcpy(s, msg, sring->elem_size); 10347be13fc3SGidon Studinski 10357be13fc3SGidon Studinski return skb; 10367be13fc3SGidon Studinski } 10377be13fc3SGidon Studinski 10387be13fc3SGidon Studinski void wil_rx_handle_edma(struct wil6210_priv *wil, int *quota) 10397be13fc3SGidon Studinski { 10407be13fc3SGidon Studinski struct net_device *ndev; 10417be13fc3SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 10427be13fc3SGidon Studinski struct wil_status_ring *sring; 10437be13fc3SGidon Studinski struct sk_buff *skb; 10447be13fc3SGidon Studinski int i; 10457be13fc3SGidon Studinski 10467be13fc3SGidon Studinski if (unlikely(!ring->va)) { 10477be13fc3SGidon Studinski wil_err(wil, "Rx IRQ while Rx not yet initialized\n"); 10487be13fc3SGidon Studinski return; 10497be13fc3SGidon Studinski } 10507be13fc3SGidon Studinski wil_dbg_txrx(wil, "rx_handle\n"); 10517be13fc3SGidon Studinski 10527be13fc3SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 10537be13fc3SGidon Studinski sring = &wil->srings[i]; 10547be13fc3SGidon Studinski if (unlikely(!sring->va)) { 10557be13fc3SGidon Studinski wil_err(wil, 10567be13fc3SGidon Studinski "Rx IRQ while Rx status ring %d not yet initialized\n", 10577be13fc3SGidon Studinski i); 10587be13fc3SGidon Studinski continue; 10597be13fc3SGidon Studinski } 10607be13fc3SGidon Studinski 10617be13fc3SGidon Studinski while ((*quota > 0) && 10627be13fc3SGidon Studinski (NULL != (skb = 10637be13fc3SGidon Studinski wil_sring_reap_rx_edma(wil, sring)))) { 10647be13fc3SGidon Studinski (*quota)--; 10657be13fc3SGidon Studinski if (wil->use_rx_hw_reordering) { 10667be13fc3SGidon Studinski void *msg = wil_skb_rxstatus(skb); 10677be13fc3SGidon Studinski int mid = wil_rx_status_get_mid(msg); 10687be13fc3SGidon Studinski struct wil6210_vif *vif = wil->vifs[mid]; 10697be13fc3SGidon Studinski 10707be13fc3SGidon Studinski if (unlikely(!vif)) { 10717be13fc3SGidon Studinski wil_dbg_txrx(wil, 10727be13fc3SGidon Studinski "RX desc invalid mid %d", 10737be13fc3SGidon Studinski mid); 10747be13fc3SGidon Studinski kfree_skb(skb); 10757be13fc3SGidon Studinski continue; 10767be13fc3SGidon Studinski } 10777be13fc3SGidon Studinski ndev = vif_to_ndev(vif); 10787be13fc3SGidon Studinski wil_netif_rx_any(skb, ndev); 10797be13fc3SGidon Studinski } else { 10807be13fc3SGidon Studinski wil_rx_reorder(wil, skb); 10817be13fc3SGidon Studinski } 10827be13fc3SGidon Studinski } 10837be13fc3SGidon Studinski 10847be13fc3SGidon Studinski wil_w(wil, sring->hwtail, (sring->swhead - 1) % sring->size); 10857be13fc3SGidon Studinski } 10867be13fc3SGidon Studinski 10877be13fc3SGidon Studinski wil_rx_refill_edma(wil); 10887be13fc3SGidon Studinski } 10897be13fc3SGidon Studinski 10909202d7b6SMaya Erez static int wil_tx_desc_map_edma(union wil_tx_desc *desc, 10919202d7b6SMaya Erez dma_addr_t pa, 10929202d7b6SMaya Erez u32 len, 10939202d7b6SMaya Erez int ring_index) 10949202d7b6SMaya Erez { 10959202d7b6SMaya Erez struct wil_tx_enhanced_desc *d = 10969202d7b6SMaya Erez (struct wil_tx_enhanced_desc *)&desc->enhanced; 10979202d7b6SMaya Erez 10989202d7b6SMaya Erez memset(d, 0, sizeof(struct wil_tx_enhanced_desc)); 10999202d7b6SMaya Erez 11009202d7b6SMaya Erez wil_desc_set_addr_edma(&d->dma.addr, &d->dma.addr_high_high, pa); 11019202d7b6SMaya Erez 11029202d7b6SMaya Erez /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/ 11039202d7b6SMaya Erez d->dma.length = cpu_to_le16((u16)len); 11049202d7b6SMaya Erez d->mac.d[0] = (ring_index << WIL_EDMA_DESC_TX_MAC_CFG_0_QID_POS); 11059202d7b6SMaya Erez /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi; 11069202d7b6SMaya Erez * 3 - eth mode 11079202d7b6SMaya Erez */ 11089202d7b6SMaya Erez d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) | 11099202d7b6SMaya Erez (0x3 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS); 11109202d7b6SMaya Erez 11119202d7b6SMaya Erez return 0; 11129202d7b6SMaya Erez } 11139202d7b6SMaya Erez 11149202d7b6SMaya Erez static inline void 11159202d7b6SMaya Erez wil_get_next_tx_status_msg(struct wil_status_ring *sring, 11169202d7b6SMaya Erez struct wil_ring_tx_status *msg) 11179202d7b6SMaya Erez { 11189202d7b6SMaya Erez struct wil_ring_tx_status *_msg = (struct wil_ring_tx_status *) 11199202d7b6SMaya Erez (sring->va + (sring->elem_size * sring->swhead)); 11209202d7b6SMaya Erez 11219202d7b6SMaya Erez *msg = *_msg; 11229202d7b6SMaya Erez } 11239202d7b6SMaya Erez 11249202d7b6SMaya Erez /** 11259202d7b6SMaya Erez * Clean up transmitted skb's from the Tx descriptor RING. 11269202d7b6SMaya Erez * Return number of descriptors cleared. 11279202d7b6SMaya Erez */ 11289202d7b6SMaya Erez int wil_tx_sring_handler(struct wil6210_priv *wil, 11299202d7b6SMaya Erez struct wil_status_ring *sring) 11309202d7b6SMaya Erez { 11319202d7b6SMaya Erez struct net_device *ndev; 11329202d7b6SMaya Erez struct device *dev = wil_to_dev(wil); 11339202d7b6SMaya Erez struct wil_ring *ring = NULL; 11349202d7b6SMaya Erez struct wil_ring_tx_data *txdata; 11359202d7b6SMaya Erez /* Total number of completed descriptors in all descriptor rings */ 11369202d7b6SMaya Erez int desc_cnt = 0; 11379202d7b6SMaya Erez int cid; 11389202d7b6SMaya Erez struct wil_net_stats *stats = NULL; 11399202d7b6SMaya Erez struct wil_tx_enhanced_desc *_d; 11409202d7b6SMaya Erez unsigned int ring_id; 11419202d7b6SMaya Erez unsigned int num_descs; 11429202d7b6SMaya Erez int i; 11439202d7b6SMaya Erez u8 dr_bit; /* Descriptor Ready bit */ 11449202d7b6SMaya Erez struct wil_ring_tx_status msg; 11459202d7b6SMaya Erez struct wil6210_vif *vif; 11469202d7b6SMaya Erez int used_before_complete; 11479202d7b6SMaya Erez int used_new; 11489202d7b6SMaya Erez 11499202d7b6SMaya Erez wil_get_next_tx_status_msg(sring, &msg); 11509202d7b6SMaya Erez dr_bit = msg.desc_ready >> TX_STATUS_DESC_READY_POS; 11519202d7b6SMaya Erez 11529202d7b6SMaya Erez /* Process completion messages while DR bit has the expected polarity */ 11539202d7b6SMaya Erez while (dr_bit == sring->desc_rdy_pol) { 11549202d7b6SMaya Erez num_descs = msg.num_descriptors; 11559202d7b6SMaya Erez if (!num_descs) { 11569202d7b6SMaya Erez wil_err(wil, "invalid num_descs 0\n"); 11579202d7b6SMaya Erez goto again; 11589202d7b6SMaya Erez } 11599202d7b6SMaya Erez 11609202d7b6SMaya Erez /* Find the corresponding descriptor ring */ 11619202d7b6SMaya Erez ring_id = msg.ring_id; 11629202d7b6SMaya Erez 11639202d7b6SMaya Erez if (unlikely(ring_id >= WIL6210_MAX_TX_RINGS)) { 11649202d7b6SMaya Erez wil_err(wil, "invalid ring id %d\n", ring_id); 11659202d7b6SMaya Erez goto again; 11669202d7b6SMaya Erez } 11679202d7b6SMaya Erez ring = &wil->ring_tx[ring_id]; 11689202d7b6SMaya Erez if (unlikely(!ring->va)) { 11699202d7b6SMaya Erez wil_err(wil, "Tx irq[%d]: ring not initialized\n", 11709202d7b6SMaya Erez ring_id); 11719202d7b6SMaya Erez goto again; 11729202d7b6SMaya Erez } 11739202d7b6SMaya Erez txdata = &wil->ring_tx_data[ring_id]; 11749202d7b6SMaya Erez if (unlikely(!txdata->enabled)) { 11759202d7b6SMaya Erez wil_info(wil, "Tx irq[%d]: ring disabled\n", ring_id); 11769202d7b6SMaya Erez goto again; 11779202d7b6SMaya Erez } 11789202d7b6SMaya Erez vif = wil->vifs[txdata->mid]; 11799202d7b6SMaya Erez if (unlikely(!vif)) { 11809202d7b6SMaya Erez wil_dbg_txrx(wil, "invalid MID %d for ring %d\n", 11819202d7b6SMaya Erez txdata->mid, ring_id); 11829202d7b6SMaya Erez goto again; 11839202d7b6SMaya Erez } 11849202d7b6SMaya Erez 11859202d7b6SMaya Erez ndev = vif_to_ndev(vif); 11869202d7b6SMaya Erez 11879202d7b6SMaya Erez cid = wil->ring2cid_tid[ring_id][0]; 11889202d7b6SMaya Erez if (cid < WIL6210_MAX_CID) 11899202d7b6SMaya Erez stats = &wil->sta[cid].stats; 11909202d7b6SMaya Erez 11919202d7b6SMaya Erez wil_dbg_txrx(wil, 11929202d7b6SMaya Erez "tx_status: completed desc_ring (%d), num_descs (%d)\n", 11939202d7b6SMaya Erez ring_id, num_descs); 11949202d7b6SMaya Erez 11959202d7b6SMaya Erez used_before_complete = wil_ring_used_tx(ring); 11969202d7b6SMaya Erez 11979202d7b6SMaya Erez for (i = 0 ; i < num_descs; ++i) { 11989202d7b6SMaya Erez struct wil_ctx *ctx = &ring->ctx[ring->swtail]; 11999202d7b6SMaya Erez struct wil_tx_enhanced_desc dd, *d = ⅆ 12009202d7b6SMaya Erez u16 dmalen; 12019202d7b6SMaya Erez struct sk_buff *skb = ctx->skb; 12029202d7b6SMaya Erez 12039202d7b6SMaya Erez _d = (struct wil_tx_enhanced_desc *) 12049202d7b6SMaya Erez &ring->va[ring->swtail].tx.enhanced; 12059202d7b6SMaya Erez *d = *_d; 12069202d7b6SMaya Erez 12079202d7b6SMaya Erez dmalen = le16_to_cpu(d->dma.length); 12089202d7b6SMaya Erez trace_wil6210_tx_status(&msg, ring->swtail, dmalen); 12099202d7b6SMaya Erez wil_dbg_txrx(wil, 12109202d7b6SMaya Erez "TxC[%2d][%3d] : %d bytes, status 0x%02x\n", 12119202d7b6SMaya Erez ring_id, ring->swtail, dmalen, 12129202d7b6SMaya Erez msg.status); 12139202d7b6SMaya Erez wil_hex_dump_txrx("TxS ", DUMP_PREFIX_NONE, 32, 4, 12149202d7b6SMaya Erez (const void *)&msg, sizeof(msg), 12159202d7b6SMaya Erez false); 12169202d7b6SMaya Erez 12179202d7b6SMaya Erez wil_tx_desc_unmap_edma(dev, 12189202d7b6SMaya Erez (union wil_tx_desc *)d, 12199202d7b6SMaya Erez ctx); 12209202d7b6SMaya Erez 12219202d7b6SMaya Erez if (skb) { 12229202d7b6SMaya Erez if (likely(msg.status == 0)) { 12239202d7b6SMaya Erez ndev->stats.tx_packets++; 12249202d7b6SMaya Erez ndev->stats.tx_bytes += skb->len; 12259202d7b6SMaya Erez if (stats) { 12269202d7b6SMaya Erez stats->tx_packets++; 12279202d7b6SMaya Erez stats->tx_bytes += skb->len; 1228a24a3d6aSDedy Lansky 1229a24a3d6aSDedy Lansky wil_tx_latency_calc(wil, skb, 1230a24a3d6aSDedy Lansky &wil->sta[cid]); 12319202d7b6SMaya Erez } 12329202d7b6SMaya Erez } else { 12339202d7b6SMaya Erez ndev->stats.tx_errors++; 12349202d7b6SMaya Erez if (stats) 12359202d7b6SMaya Erez stats->tx_errors++; 12369202d7b6SMaya Erez } 12379202d7b6SMaya Erez wil_consume_skb(skb, msg.status == 0); 12389202d7b6SMaya Erez } 12399202d7b6SMaya Erez memset(ctx, 0, sizeof(*ctx)); 12409202d7b6SMaya Erez /* Make sure the ctx is zeroed before updating the tail 12419202d7b6SMaya Erez * to prevent a case where wil_tx_ring will see 12429202d7b6SMaya Erez * this descriptor as used and handle it before ctx zero 12439202d7b6SMaya Erez * is completed. 12449202d7b6SMaya Erez */ 12459202d7b6SMaya Erez wmb(); 12469202d7b6SMaya Erez 12479202d7b6SMaya Erez ring->swtail = wil_ring_next_tail(ring); 12489202d7b6SMaya Erez 12499202d7b6SMaya Erez desc_cnt++; 12509202d7b6SMaya Erez } 12519202d7b6SMaya Erez 12529202d7b6SMaya Erez /* performance monitoring */ 12539202d7b6SMaya Erez used_new = wil_ring_used_tx(ring); 12549202d7b6SMaya Erez if (wil_val_in_range(wil->ring_idle_trsh, 12559202d7b6SMaya Erez used_new, used_before_complete)) { 12569202d7b6SMaya Erez wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n", 12579202d7b6SMaya Erez ring_id, used_before_complete, used_new); 12589202d7b6SMaya Erez txdata->last_idle = get_cycles(); 12599202d7b6SMaya Erez } 12609202d7b6SMaya Erez 12619202d7b6SMaya Erez again: 12629202d7b6SMaya Erez wil_sring_advance_swhead(sring); 12639202d7b6SMaya Erez 12649202d7b6SMaya Erez wil_get_next_tx_status_msg(sring, &msg); 12659202d7b6SMaya Erez dr_bit = msg.desc_ready >> TX_STATUS_DESC_READY_POS; 12669202d7b6SMaya Erez } 12679202d7b6SMaya Erez 12689202d7b6SMaya Erez /* shall we wake net queues? */ 12699202d7b6SMaya Erez if (desc_cnt) 12709202d7b6SMaya Erez wil_update_net_queues(wil, vif, NULL, false); 12719202d7b6SMaya Erez 12729202d7b6SMaya Erez /* Update the HW tail ptr (RD ptr) */ 12739202d7b6SMaya Erez wil_w(wil, sring->hwtail, (sring->swhead - 1) % sring->size); 12749202d7b6SMaya Erez 12759202d7b6SMaya Erez return desc_cnt; 12769202d7b6SMaya Erez } 12779202d7b6SMaya Erez 12789202d7b6SMaya Erez /** 12799202d7b6SMaya Erez * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding 12809202d7b6SMaya Erez * @skb is used to obtain the protocol and headers length. 12819202d7b6SMaya Erez * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data, 12829202d7b6SMaya Erez * 2 - middle, 3 - last descriptor. 12839202d7b6SMaya Erez */ 12849202d7b6SMaya Erez static void wil_tx_desc_offload_setup_tso_edma(struct wil_tx_enhanced_desc *d, 12859202d7b6SMaya Erez int tso_desc_type, bool is_ipv4, 12869202d7b6SMaya Erez int tcp_hdr_len, 12879202d7b6SMaya Erez int skb_net_hdr_len, 12889202d7b6SMaya Erez int mss) 12899202d7b6SMaya Erez { 12909202d7b6SMaya Erez /* Number of descriptors */ 12919202d7b6SMaya Erez d->mac.d[2] |= 1; 12929202d7b6SMaya Erez /* Maximum Segment Size */ 12939202d7b6SMaya Erez d->mac.tso_mss |= cpu_to_le16(mss >> 2); 12949202d7b6SMaya Erez /* L4 header len: TCP header length */ 12959202d7b6SMaya Erez d->dma.l4_hdr_len |= tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK; 12969202d7b6SMaya Erez /* EOP, TSO desc type, Segmentation enable, 12979202d7b6SMaya Erez * Insert IPv4 and TCP / UDP Checksum 12989202d7b6SMaya Erez */ 12999202d7b6SMaya Erez d->dma.cmd |= BIT(WIL_EDMA_DESC_TX_CFG_EOP_POS) | 13009202d7b6SMaya Erez tso_desc_type << WIL_EDMA_DESC_TX_CFG_TSO_DESC_TYPE_POS | 13019202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_SEG_EN_POS) | 13029202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_INSERT_IP_CHKSUM_POS) | 13039202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_INSERT_TCP_CHKSUM_POS); 13049202d7b6SMaya Erez /* Calculate pseudo-header */ 13059202d7b6SMaya Erez d->dma.w1 |= BIT(WIL_EDMA_DESC_TX_CFG_PSEUDO_HEADER_CALC_EN_POS) | 13069202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_L4_TYPE_POS); 13079202d7b6SMaya Erez /* IP Header Length */ 13089202d7b6SMaya Erez d->dma.ip_length |= skb_net_hdr_len; 13099202d7b6SMaya Erez /* MAC header length and IP address family*/ 13109202d7b6SMaya Erez d->dma.b11 |= ETH_HLEN | 13119202d7b6SMaya Erez is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS; 13129202d7b6SMaya Erez } 13139202d7b6SMaya Erez 13149202d7b6SMaya Erez static int wil_tx_tso_gen_desc(struct wil6210_priv *wil, void *buff_addr, 13159202d7b6SMaya Erez int len, uint i, int tso_desc_type, 13169202d7b6SMaya Erez skb_frag_t *frag, struct wil_ring *ring, 13179202d7b6SMaya Erez struct sk_buff *skb, bool is_ipv4, 13189202d7b6SMaya Erez int tcp_hdr_len, int skb_net_hdr_len, 13199202d7b6SMaya Erez int mss, int *descs_used) 13209202d7b6SMaya Erez { 13219202d7b6SMaya Erez struct device *dev = wil_to_dev(wil); 13229202d7b6SMaya Erez struct wil_tx_enhanced_desc *_desc = (struct wil_tx_enhanced_desc *) 13239202d7b6SMaya Erez &ring->va[i].tx.enhanced; 13249202d7b6SMaya Erez struct wil_tx_enhanced_desc desc_mem, *d = &desc_mem; 13259202d7b6SMaya Erez int ring_index = ring - wil->ring_tx; 13269202d7b6SMaya Erez dma_addr_t pa; 13279202d7b6SMaya Erez 13289202d7b6SMaya Erez if (len == 0) 13299202d7b6SMaya Erez return 0; 13309202d7b6SMaya Erez 13319202d7b6SMaya Erez if (!frag) { 13329202d7b6SMaya Erez pa = dma_map_single(dev, buff_addr, len, DMA_TO_DEVICE); 13339202d7b6SMaya Erez ring->ctx[i].mapped_as = wil_mapped_as_single; 13349202d7b6SMaya Erez } else { 13359202d7b6SMaya Erez pa = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 13369202d7b6SMaya Erez ring->ctx[i].mapped_as = wil_mapped_as_page; 13379202d7b6SMaya Erez } 13389202d7b6SMaya Erez if (unlikely(dma_mapping_error(dev, pa))) { 13399202d7b6SMaya Erez wil_err(wil, "TSO: Skb DMA map error\n"); 13409202d7b6SMaya Erez return -EINVAL; 13419202d7b6SMaya Erez } 13429202d7b6SMaya Erez 13439202d7b6SMaya Erez wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, 13449202d7b6SMaya Erez len, ring_index); 13459202d7b6SMaya Erez wil_tx_desc_offload_setup_tso_edma(d, tso_desc_type, is_ipv4, 13469202d7b6SMaya Erez tcp_hdr_len, 13479202d7b6SMaya Erez skb_net_hdr_len, mss); 13489202d7b6SMaya Erez 13499202d7b6SMaya Erez /* hold reference to skb 13509202d7b6SMaya Erez * to prevent skb release before accounting 13519202d7b6SMaya Erez * in case of immediate "tx done" 13529202d7b6SMaya Erez */ 13539202d7b6SMaya Erez if (tso_desc_type == wil_tso_type_lst) 13549202d7b6SMaya Erez ring->ctx[i].skb = skb_get(skb); 13559202d7b6SMaya Erez 13569202d7b6SMaya Erez wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 13579202d7b6SMaya Erez (const void *)d, sizeof(*d), false); 13589202d7b6SMaya Erez 13599202d7b6SMaya Erez *_desc = *d; 13609202d7b6SMaya Erez (*descs_used)++; 13619202d7b6SMaya Erez 13629202d7b6SMaya Erez return 0; 13639202d7b6SMaya Erez } 13649202d7b6SMaya Erez 13659202d7b6SMaya Erez static int __wil_tx_ring_tso_edma(struct wil6210_priv *wil, 13669202d7b6SMaya Erez struct wil6210_vif *vif, 13679202d7b6SMaya Erez struct wil_ring *ring, 13689202d7b6SMaya Erez struct sk_buff *skb) 13699202d7b6SMaya Erez { 13709202d7b6SMaya Erez int ring_index = ring - wil->ring_tx; 13719202d7b6SMaya Erez struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index]; 13729202d7b6SMaya Erez int nr_frags = skb_shinfo(skb)->nr_frags; 13739202d7b6SMaya Erez int min_desc_required = nr_frags + 2; /* Headers, Head, Fragments */ 13749202d7b6SMaya Erez int used, avail = wil_ring_avail_tx(ring); 13759202d7b6SMaya Erez int f, hdrlen, headlen; 13769202d7b6SMaya Erez int gso_type; 13779202d7b6SMaya Erez bool is_ipv4; 13789202d7b6SMaya Erez u32 swhead = ring->swhead; 13799202d7b6SMaya Erez int descs_used = 0; /* total number of used descriptors */ 13809202d7b6SMaya Erez int rc = -EINVAL; 13819202d7b6SMaya Erez int tcp_hdr_len; 13829202d7b6SMaya Erez int skb_net_hdr_len; 13839202d7b6SMaya Erez int mss = skb_shinfo(skb)->gso_size; 13849202d7b6SMaya Erez 13859202d7b6SMaya Erez wil_dbg_txrx(wil, "tx_ring_tso: %d bytes to ring %d\n", skb->len, 13869202d7b6SMaya Erez ring_index); 13879202d7b6SMaya Erez 13889202d7b6SMaya Erez if (unlikely(!txdata->enabled)) 13899202d7b6SMaya Erez return -EINVAL; 13909202d7b6SMaya Erez 13919202d7b6SMaya Erez if (unlikely(avail < min_desc_required)) { 13929202d7b6SMaya Erez wil_err_ratelimited(wil, 13939202d7b6SMaya Erez "TSO: Tx ring[%2d] full. No space for %d fragments\n", 13949202d7b6SMaya Erez ring_index, min_desc_required); 13959202d7b6SMaya Erez return -ENOMEM; 13969202d7b6SMaya Erez } 13979202d7b6SMaya Erez 13989202d7b6SMaya Erez gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4); 13999202d7b6SMaya Erez switch (gso_type) { 14009202d7b6SMaya Erez case SKB_GSO_TCPV4: 14019202d7b6SMaya Erez is_ipv4 = true; 14029202d7b6SMaya Erez break; 14039202d7b6SMaya Erez case SKB_GSO_TCPV6: 14049202d7b6SMaya Erez is_ipv4 = false; 14059202d7b6SMaya Erez break; 14069202d7b6SMaya Erez default: 14079202d7b6SMaya Erez return -EINVAL; 14089202d7b6SMaya Erez } 14099202d7b6SMaya Erez 14109202d7b6SMaya Erez if (skb->ip_summed != CHECKSUM_PARTIAL) 14119202d7b6SMaya Erez return -EINVAL; 14129202d7b6SMaya Erez 14139202d7b6SMaya Erez /* tcp header length and skb network header length are fixed for all 14149202d7b6SMaya Erez * packet's descriptors - read them once here 14159202d7b6SMaya Erez */ 14169202d7b6SMaya Erez tcp_hdr_len = tcp_hdrlen(skb); 14179202d7b6SMaya Erez skb_net_hdr_len = skb_network_header_len(skb); 14189202d7b6SMaya Erez 14199202d7b6SMaya Erez /* First descriptor must contain the header only 14209202d7b6SMaya Erez * Header Length = MAC header len + IP header len + TCP header len 14219202d7b6SMaya Erez */ 14229202d7b6SMaya Erez hdrlen = ETH_HLEN + tcp_hdr_len + skb_net_hdr_len; 14239202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: process header descriptor, hdrlen %u\n", 14249202d7b6SMaya Erez hdrlen); 14259202d7b6SMaya Erez rc = wil_tx_tso_gen_desc(wil, skb->data, hdrlen, swhead, 14269202d7b6SMaya Erez wil_tso_type_hdr, NULL, ring, skb, 14279202d7b6SMaya Erez is_ipv4, tcp_hdr_len, skb_net_hdr_len, 14289202d7b6SMaya Erez mss, &descs_used); 14299202d7b6SMaya Erez if (rc) 14309202d7b6SMaya Erez return -EINVAL; 14319202d7b6SMaya Erez 14329202d7b6SMaya Erez /* Second descriptor contains the head */ 14339202d7b6SMaya Erez headlen = skb_headlen(skb) - hdrlen; 14349202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: process skb head, headlen %u\n", headlen); 14359202d7b6SMaya Erez rc = wil_tx_tso_gen_desc(wil, skb->data + hdrlen, headlen, 14369202d7b6SMaya Erez (swhead + descs_used) % ring->size, 14379202d7b6SMaya Erez (nr_frags != 0) ? wil_tso_type_first : 14389202d7b6SMaya Erez wil_tso_type_lst, NULL, ring, skb, 14399202d7b6SMaya Erez is_ipv4, tcp_hdr_len, skb_net_hdr_len, 14409202d7b6SMaya Erez mss, &descs_used); 14419202d7b6SMaya Erez if (rc) 14429202d7b6SMaya Erez goto mem_error; 14439202d7b6SMaya Erez 14449202d7b6SMaya Erez /* Rest of the descriptors are from the SKB fragments */ 14459202d7b6SMaya Erez for (f = 0; f < nr_frags; f++) { 14469202d7b6SMaya Erez skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 14479202d7b6SMaya Erez int len = frag->size; 14489202d7b6SMaya Erez 14499202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: frag[%d]: len %u, descs_used %d\n", f, 14509202d7b6SMaya Erez len, descs_used); 14519202d7b6SMaya Erez 14529202d7b6SMaya Erez rc = wil_tx_tso_gen_desc(wil, NULL, len, 14539202d7b6SMaya Erez (swhead + descs_used) % ring->size, 14549202d7b6SMaya Erez (f != nr_frags - 1) ? 14559202d7b6SMaya Erez wil_tso_type_mid : wil_tso_type_lst, 14569202d7b6SMaya Erez frag, ring, skb, is_ipv4, 14579202d7b6SMaya Erez tcp_hdr_len, skb_net_hdr_len, 14589202d7b6SMaya Erez mss, &descs_used); 14599202d7b6SMaya Erez if (rc) 14609202d7b6SMaya Erez goto mem_error; 14619202d7b6SMaya Erez } 14629202d7b6SMaya Erez 14639202d7b6SMaya Erez /* performance monitoring */ 14649202d7b6SMaya Erez used = wil_ring_used_tx(ring); 14659202d7b6SMaya Erez if (wil_val_in_range(wil->ring_idle_trsh, 14669202d7b6SMaya Erez used, used + descs_used)) { 14679202d7b6SMaya Erez txdata->idle += get_cycles() - txdata->last_idle; 14689202d7b6SMaya Erez wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 14699202d7b6SMaya Erez ring_index, used, used + descs_used); 14709202d7b6SMaya Erez } 14719202d7b6SMaya Erez 14729202d7b6SMaya Erez /* advance swhead */ 14739202d7b6SMaya Erez wil_ring_advance_head(ring, descs_used); 14749202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, ring->swhead); 14759202d7b6SMaya Erez 14769202d7b6SMaya Erez /* make sure all writes to descriptors (shared memory) are done before 14779202d7b6SMaya Erez * committing them to HW 14789202d7b6SMaya Erez */ 14799202d7b6SMaya Erez wmb(); 14809202d7b6SMaya Erez 1481a24a3d6aSDedy Lansky if (wil->tx_latency) 1482a24a3d6aSDedy Lansky *(ktime_t *)&skb->cb = ktime_get(); 1483a24a3d6aSDedy Lansky else 1484a24a3d6aSDedy Lansky memset(skb->cb, 0, sizeof(ktime_t)); 1485a24a3d6aSDedy Lansky 14869202d7b6SMaya Erez wil_w(wil, ring->hwtail, ring->swhead); 14879202d7b6SMaya Erez 14889202d7b6SMaya Erez return 0; 14899202d7b6SMaya Erez 14909202d7b6SMaya Erez mem_error: 14919202d7b6SMaya Erez while (descs_used > 0) { 14929202d7b6SMaya Erez struct device *dev = wil_to_dev(wil); 14939202d7b6SMaya Erez struct wil_ctx *ctx; 14949202d7b6SMaya Erez int i = (swhead + descs_used - 1) % ring->size; 14959202d7b6SMaya Erez struct wil_tx_enhanced_desc dd, *d = ⅆ 14969202d7b6SMaya Erez struct wil_tx_enhanced_desc *_desc = 14979202d7b6SMaya Erez (struct wil_tx_enhanced_desc *) 14989202d7b6SMaya Erez &ring->va[i].tx.enhanced; 14999202d7b6SMaya Erez 15009202d7b6SMaya Erez *d = *_desc; 15019202d7b6SMaya Erez ctx = &ring->ctx[i]; 15029202d7b6SMaya Erez wil_tx_desc_unmap_edma(dev, (union wil_tx_desc *)d, ctx); 15039202d7b6SMaya Erez memset(ctx, 0, sizeof(*ctx)); 15049202d7b6SMaya Erez descs_used--; 15059202d7b6SMaya Erez } 15069202d7b6SMaya Erez return rc; 15079202d7b6SMaya Erez } 15089202d7b6SMaya Erez 150996c93589SGidon Studinski static int wil_ring_init_bcast_edma(struct wil6210_vif *vif, int ring_id, 151096c93589SGidon Studinski int size) 151196c93589SGidon Studinski { 151296c93589SGidon Studinski struct wil6210_priv *wil = vif_to_wil(vif); 151396c93589SGidon Studinski struct wil_ring *ring = &wil->ring_tx[ring_id]; 151496c93589SGidon Studinski int rc; 151596c93589SGidon Studinski struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 151696c93589SGidon Studinski 151796c93589SGidon Studinski wil_dbg_misc(wil, "init bcast: ring_id=%d, sring_id=%d\n", 151896c93589SGidon Studinski ring_id, wil->tx_sring_idx); 151996c93589SGidon Studinski 152096c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 152196c93589SGidon Studinski 152296c93589SGidon Studinski wil_tx_data_init(txdata); 152396c93589SGidon Studinski ring->size = size; 152496c93589SGidon Studinski ring->is_rx = false; 152596c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 152696c93589SGidon Studinski if (rc) 152796c93589SGidon Studinski goto out; 152896c93589SGidon Studinski 152996c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = WIL6210_MAX_CID; /* CID */ 153096c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = 0; /* TID */ 153196c93589SGidon Studinski if (!vif->privacy) 153296c93589SGidon Studinski txdata->dot1x_open = true; 153396c93589SGidon Studinski 153496c93589SGidon Studinski rc = wil_wmi_bcast_desc_ring_add(vif, ring_id); 153596c93589SGidon Studinski if (rc) 153696c93589SGidon Studinski goto out_free; 153796c93589SGidon Studinski 153896c93589SGidon Studinski return 0; 153996c93589SGidon Studinski 154096c93589SGidon Studinski out_free: 154196c93589SGidon Studinski spin_lock_bh(&txdata->lock); 154296c93589SGidon Studinski txdata->enabled = 0; 154396c93589SGidon Studinski txdata->dot1x_open = false; 154496c93589SGidon Studinski spin_unlock_bh(&txdata->lock); 154596c93589SGidon Studinski wil_ring_free_edma(wil, ring); 154696c93589SGidon Studinski 154796c93589SGidon Studinski out: 154896c93589SGidon Studinski return rc; 154996c93589SGidon Studinski } 155096c93589SGidon Studinski 155196c93589SGidon Studinski static void wil_tx_fini_edma(struct wil6210_priv *wil) 155296c93589SGidon Studinski { 155396c93589SGidon Studinski struct wil_status_ring *sring = &wil->srings[wil->tx_sring_idx]; 155496c93589SGidon Studinski 155596c93589SGidon Studinski wil_dbg_misc(wil, "free TX sring\n"); 155696c93589SGidon Studinski 155796c93589SGidon Studinski wil_sring_free(wil, sring); 155896c93589SGidon Studinski } 155996c93589SGidon Studinski 156096c93589SGidon Studinski static void wil_rx_data_free(struct wil_status_ring *sring) 156196c93589SGidon Studinski { 156296c93589SGidon Studinski if (!sring) 156396c93589SGidon Studinski return; 156496c93589SGidon Studinski 156596c93589SGidon Studinski kfree_skb(sring->rx_data.skb); 156696c93589SGidon Studinski sring->rx_data.skb = NULL; 156796c93589SGidon Studinski } 156896c93589SGidon Studinski 156996c93589SGidon Studinski static void wil_rx_fini_edma(struct wil6210_priv *wil) 157096c93589SGidon Studinski { 157196c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 157296c93589SGidon Studinski int i; 157396c93589SGidon Studinski 157496c93589SGidon Studinski wil_dbg_misc(wil, "rx_fini_edma\n"); 157596c93589SGidon Studinski 157696c93589SGidon Studinski wil_ring_free_edma(wil, ring); 157796c93589SGidon Studinski 157896c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 157996c93589SGidon Studinski wil_rx_data_free(&wil->srings[i]); 158096c93589SGidon Studinski wil_sring_free(wil, &wil->srings[i]); 158196c93589SGidon Studinski } 158296c93589SGidon Studinski 158396c93589SGidon Studinski wil_free_rx_buff_arr(wil); 158496c93589SGidon Studinski } 158596c93589SGidon Studinski 158696c93589SGidon Studinski void wil_init_txrx_ops_edma(struct wil6210_priv *wil) 158796c93589SGidon Studinski { 158896c93589SGidon Studinski wil->txrx_ops.configure_interrupt_moderation = 158996c93589SGidon Studinski wil_configure_interrupt_moderation_edma; 159096c93589SGidon Studinski /* TX ops */ 159196c93589SGidon Studinski wil->txrx_ops.ring_init_tx = wil_ring_init_tx_edma; 159296c93589SGidon Studinski wil->txrx_ops.ring_fini_tx = wil_ring_free_edma; 159396c93589SGidon Studinski wil->txrx_ops.ring_init_bcast = wil_ring_init_bcast_edma; 159496c93589SGidon Studinski wil->txrx_ops.tx_init = wil_tx_init_edma; 159596c93589SGidon Studinski wil->txrx_ops.tx_fini = wil_tx_fini_edma; 15969202d7b6SMaya Erez wil->txrx_ops.tx_desc_map = wil_tx_desc_map_edma; 15979202d7b6SMaya Erez wil->txrx_ops.tx_desc_unmap = wil_tx_desc_unmap_edma; 15989202d7b6SMaya Erez wil->txrx_ops.tx_ring_tso = __wil_tx_ring_tso_edma; 159996c93589SGidon Studinski /* RX ops */ 160096c93589SGidon Studinski wil->txrx_ops.rx_init = wil_rx_init_edma; 16017be13fc3SGidon Studinski wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp_edma; 16027be13fc3SGidon Studinski wil->txrx_ops.get_reorder_params = wil_get_reorder_params_edma; 16037be13fc3SGidon Studinski wil->txrx_ops.get_netif_rx_params = wil_get_netif_rx_params_edma; 16047be13fc3SGidon Studinski wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check_edma; 1605e15af41cSDedy Lansky wil->txrx_ops.rx_error_check = wil_rx_error_check_edma; 16067be13fc3SGidon Studinski wil->txrx_ops.is_rx_idle = wil_is_rx_idle_edma; 160796c93589SGidon Studinski wil->txrx_ops.rx_fini = wil_rx_fini_edma; 160896c93589SGidon Studinski } 160996c93589SGidon Studinski 1610