196c93589SGidon Studinski /* 296c93589SGidon Studinski * Copyright (c) 2012-2018 The Linux Foundation. All rights reserved. 396c93589SGidon Studinski * 496c93589SGidon Studinski * Permission to use, copy, modify, and/or distribute this software for any 596c93589SGidon Studinski * purpose with or without fee is hereby granted, provided that the above 696c93589SGidon Studinski * copyright notice and this permission notice appear in all copies. 796c93589SGidon Studinski * 896c93589SGidon Studinski * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 996c93589SGidon Studinski * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1096c93589SGidon Studinski * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1196c93589SGidon Studinski * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1296c93589SGidon Studinski * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1396c93589SGidon Studinski * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1496c93589SGidon Studinski * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1596c93589SGidon Studinski */ 1696c93589SGidon Studinski 1796c93589SGidon Studinski #include <linux/etherdevice.h> 1896c93589SGidon Studinski #include <linux/moduleparam.h> 1996c93589SGidon Studinski #include <linux/prefetch.h> 2096c93589SGidon Studinski #include <linux/types.h> 2196c93589SGidon Studinski #include <linux/list.h> 2296c93589SGidon Studinski #include <linux/ip.h> 2396c93589SGidon Studinski #include <linux/ipv6.h> 2496c93589SGidon Studinski #include "wil6210.h" 2596c93589SGidon Studinski #include "txrx_edma.h" 2696c93589SGidon Studinski #include "txrx.h" 279202d7b6SMaya Erez #include "trace.h" 2896c93589SGidon Studinski 2996c93589SGidon Studinski #define WIL_EDMA_MAX_DATA_OFFSET (2) 3096c93589SGidon Studinski 3196c93589SGidon Studinski static void wil_tx_desc_unmap_edma(struct device *dev, 329202d7b6SMaya Erez union wil_tx_desc *desc, 3396c93589SGidon Studinski struct wil_ctx *ctx) 3496c93589SGidon Studinski { 359202d7b6SMaya Erez struct wil_tx_enhanced_desc *d = (struct wil_tx_enhanced_desc *)desc; 3696c93589SGidon Studinski dma_addr_t pa = wil_tx_desc_get_addr_edma(&d->dma); 3796c93589SGidon Studinski u16 dmalen = le16_to_cpu(d->dma.length); 3896c93589SGidon Studinski 3996c93589SGidon Studinski switch (ctx->mapped_as) { 4096c93589SGidon Studinski case wil_mapped_as_single: 4196c93589SGidon Studinski dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE); 4296c93589SGidon Studinski break; 4396c93589SGidon Studinski case wil_mapped_as_page: 4496c93589SGidon Studinski dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE); 4596c93589SGidon Studinski break; 4696c93589SGidon Studinski default: 4796c93589SGidon Studinski break; 4896c93589SGidon Studinski } 4996c93589SGidon Studinski } 5096c93589SGidon Studinski 5196c93589SGidon Studinski static int wil_find_free_sring(struct wil6210_priv *wil) 5296c93589SGidon Studinski { 5396c93589SGidon Studinski int i; 5496c93589SGidon Studinski 5596c93589SGidon Studinski for (i = 0; i < WIL6210_MAX_STATUS_RINGS; i++) { 5696c93589SGidon Studinski if (!wil->srings[i].va) 5796c93589SGidon Studinski return i; 5896c93589SGidon Studinski } 5996c93589SGidon Studinski 6096c93589SGidon Studinski return -EINVAL; 6196c93589SGidon Studinski } 6296c93589SGidon Studinski 6396c93589SGidon Studinski static void wil_sring_free(struct wil6210_priv *wil, 6496c93589SGidon Studinski struct wil_status_ring *sring) 6596c93589SGidon Studinski { 6696c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 6796c93589SGidon Studinski size_t sz; 6896c93589SGidon Studinski 6996c93589SGidon Studinski if (!sring || !sring->va) 7096c93589SGidon Studinski return; 7196c93589SGidon Studinski 7296c93589SGidon Studinski sz = sring->elem_size * sring->size; 7396c93589SGidon Studinski 7496c93589SGidon Studinski wil_dbg_misc(wil, "status_ring_free, size(bytes)=%zu, 0x%p:%pad\n", 7596c93589SGidon Studinski sz, sring->va, &sring->pa); 7696c93589SGidon Studinski 7796c93589SGidon Studinski dma_free_coherent(dev, sz, (void *)sring->va, sring->pa); 7896c93589SGidon Studinski sring->pa = 0; 7996c93589SGidon Studinski sring->va = NULL; 8096c93589SGidon Studinski } 8196c93589SGidon Studinski 8296c93589SGidon Studinski static int wil_sring_alloc(struct wil6210_priv *wil, 8396c93589SGidon Studinski struct wil_status_ring *sring) 8496c93589SGidon Studinski { 8596c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 8696c93589SGidon Studinski size_t sz = sring->elem_size * sring->size; 8796c93589SGidon Studinski 8896c93589SGidon Studinski wil_dbg_misc(wil, "status_ring_alloc: size=%zu\n", sz); 8996c93589SGidon Studinski 9096c93589SGidon Studinski if (sz == 0) { 9196c93589SGidon Studinski wil_err(wil, "Cannot allocate a zero size status ring\n"); 9296c93589SGidon Studinski return -EINVAL; 9396c93589SGidon Studinski } 9496c93589SGidon Studinski 9596c93589SGidon Studinski sring->swhead = 0; 9696c93589SGidon Studinski 9796c93589SGidon Studinski /* Status messages are allocated and initialized to 0. This is necessary 9896c93589SGidon Studinski * since DR bit should be initialized to 0. 9996c93589SGidon Studinski */ 10096c93589SGidon Studinski sring->va = dma_zalloc_coherent(dev, sz, &sring->pa, GFP_KERNEL); 10196c93589SGidon Studinski if (!sring->va) 10296c93589SGidon Studinski return -ENOMEM; 10396c93589SGidon Studinski 10496c93589SGidon Studinski wil_dbg_misc(wil, "status_ring[%d] 0x%p:%pad\n", sring->size, sring->va, 10596c93589SGidon Studinski &sring->pa); 10696c93589SGidon Studinski 10796c93589SGidon Studinski return 0; 10896c93589SGidon Studinski } 10996c93589SGidon Studinski 11096c93589SGidon Studinski static int wil_tx_init_edma(struct wil6210_priv *wil) 11196c93589SGidon Studinski { 11296c93589SGidon Studinski int ring_id = wil_find_free_sring(wil); 11396c93589SGidon Studinski struct wil_status_ring *sring; 11496c93589SGidon Studinski int rc; 11596c93589SGidon Studinski u16 status_ring_size; 11696c93589SGidon Studinski 11796c93589SGidon Studinski if (wil->tx_status_ring_order < WIL_SRING_SIZE_ORDER_MIN || 11896c93589SGidon Studinski wil->tx_status_ring_order > WIL_SRING_SIZE_ORDER_MAX) 11996c93589SGidon Studinski wil->tx_status_ring_order = WIL_TX_SRING_SIZE_ORDER_DEFAULT; 12096c93589SGidon Studinski 12196c93589SGidon Studinski status_ring_size = 1 << wil->tx_status_ring_order; 12296c93589SGidon Studinski 12396c93589SGidon Studinski wil_dbg_misc(wil, "init TX sring: size=%u, ring_id=%u\n", 12496c93589SGidon Studinski status_ring_size, ring_id); 12596c93589SGidon Studinski 12696c93589SGidon Studinski if (ring_id < 0) 12796c93589SGidon Studinski return ring_id; 12896c93589SGidon Studinski 12996c93589SGidon Studinski /* Allocate Tx status ring. Tx descriptor rings will be 13096c93589SGidon Studinski * allocated on WMI connect event 13196c93589SGidon Studinski */ 13296c93589SGidon Studinski sring = &wil->srings[ring_id]; 13396c93589SGidon Studinski 13496c93589SGidon Studinski sring->is_rx = false; 13596c93589SGidon Studinski sring->size = status_ring_size; 13696c93589SGidon Studinski sring->elem_size = sizeof(struct wil_ring_tx_status); 13796c93589SGidon Studinski rc = wil_sring_alloc(wil, sring); 13896c93589SGidon Studinski if (rc) 13996c93589SGidon Studinski return rc; 14096c93589SGidon Studinski 14196c93589SGidon Studinski rc = wil_wmi_tx_sring_cfg(wil, ring_id); 14296c93589SGidon Studinski if (rc) 14396c93589SGidon Studinski goto out_free; 14496c93589SGidon Studinski 14596c93589SGidon Studinski sring->desc_rdy_pol = 1; 14696c93589SGidon Studinski wil->tx_sring_idx = ring_id; 14796c93589SGidon Studinski 14896c93589SGidon Studinski return 0; 14996c93589SGidon Studinski out_free: 15096c93589SGidon Studinski wil_sring_free(wil, sring); 15196c93589SGidon Studinski return rc; 15296c93589SGidon Studinski } 15396c93589SGidon Studinski 15496c93589SGidon Studinski /** 15596c93589SGidon Studinski * Allocate one skb for Rx descriptor RING 15696c93589SGidon Studinski */ 15796c93589SGidon Studinski static int wil_ring_alloc_skb_edma(struct wil6210_priv *wil, 15896c93589SGidon Studinski struct wil_ring *ring, u32 i) 15996c93589SGidon Studinski { 16096c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 16196c93589SGidon Studinski unsigned int sz = wil->rx_buf_len + ETH_HLEN + 16296c93589SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 16396c93589SGidon Studinski dma_addr_t pa; 16496c93589SGidon Studinski u16 buff_id; 16596c93589SGidon Studinski struct list_head *active = &wil->rx_buff_mgmt.active; 16696c93589SGidon Studinski struct list_head *free = &wil->rx_buff_mgmt.free; 16796c93589SGidon Studinski struct wil_rx_buff *rx_buff; 16896c93589SGidon Studinski struct wil_rx_buff *buff_arr = wil->rx_buff_mgmt.buff_arr; 16996c93589SGidon Studinski struct sk_buff *skb; 17096c93589SGidon Studinski struct wil_rx_enhanced_desc dd, *d = ⅆ 17196c93589SGidon Studinski struct wil_rx_enhanced_desc *_d = (struct wil_rx_enhanced_desc *) 17296c93589SGidon Studinski &ring->va[i].rx.enhanced; 17396c93589SGidon Studinski 17496c93589SGidon Studinski if (unlikely(list_empty(free))) { 17596c93589SGidon Studinski wil->rx_buff_mgmt.free_list_empty_cnt++; 17696c93589SGidon Studinski return -EAGAIN; 17796c93589SGidon Studinski } 17896c93589SGidon Studinski 17996c93589SGidon Studinski skb = dev_alloc_skb(sz); 18096c93589SGidon Studinski if (unlikely(!skb)) 18196c93589SGidon Studinski return -ENOMEM; 18296c93589SGidon Studinski 18396c93589SGidon Studinski skb_put(skb, sz); 18496c93589SGidon Studinski 18596c93589SGidon Studinski pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 18696c93589SGidon Studinski if (unlikely(dma_mapping_error(dev, pa))) { 18796c93589SGidon Studinski kfree_skb(skb); 18896c93589SGidon Studinski return -ENOMEM; 18996c93589SGidon Studinski } 19096c93589SGidon Studinski 19196c93589SGidon Studinski /* Get the buffer ID - the index of the rx buffer in the buff_arr */ 19296c93589SGidon Studinski rx_buff = list_first_entry(free, struct wil_rx_buff, list); 19396c93589SGidon Studinski buff_id = rx_buff->id; 19496c93589SGidon Studinski 19596c93589SGidon Studinski /* Move a buffer from the free list to the active list */ 19696c93589SGidon Studinski list_move(&rx_buff->list, active); 19796c93589SGidon Studinski 19896c93589SGidon Studinski buff_arr[buff_id].skb = skb; 19996c93589SGidon Studinski 20096c93589SGidon Studinski wil_desc_set_addr_edma(&d->dma.addr, &d->dma.addr_high_high, pa); 20196c93589SGidon Studinski d->dma.length = cpu_to_le16(sz); 20296c93589SGidon Studinski d->mac.buff_id = cpu_to_le16(buff_id); 20396c93589SGidon Studinski *_d = *d; 20496c93589SGidon Studinski 20596c93589SGidon Studinski /* Save the physical address in skb->cb for later use in dma_unmap */ 20696c93589SGidon Studinski memcpy(skb->cb, &pa, sizeof(pa)); 20796c93589SGidon Studinski 20896c93589SGidon Studinski return 0; 20996c93589SGidon Studinski } 21096c93589SGidon Studinski 2119202d7b6SMaya Erez static inline void wil_sring_advance_swhead(struct wil_status_ring *sring) 2129202d7b6SMaya Erez { 2139202d7b6SMaya Erez sring->swhead = (sring->swhead + 1) % sring->size; 2149202d7b6SMaya Erez if (sring->swhead == 0) 2159202d7b6SMaya Erez sring->desc_rdy_pol = 1 - sring->desc_rdy_pol; 2169202d7b6SMaya Erez } 2179202d7b6SMaya Erez 21896c93589SGidon Studinski static int wil_rx_refill_edma(struct wil6210_priv *wil) 21996c93589SGidon Studinski { 22096c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 22196c93589SGidon Studinski u32 next_head; 22296c93589SGidon Studinski int rc = 0; 22396c93589SGidon Studinski u32 swtail = *ring->edma_rx_swtail.va; 22496c93589SGidon Studinski 22596c93589SGidon Studinski for (; next_head = wil_ring_next_head(ring), (next_head != swtail); 22696c93589SGidon Studinski ring->swhead = next_head) { 22796c93589SGidon Studinski rc = wil_ring_alloc_skb_edma(wil, ring, ring->swhead); 22896c93589SGidon Studinski if (unlikely(rc)) { 22996c93589SGidon Studinski if (rc == -EAGAIN) 23096c93589SGidon Studinski wil_dbg_txrx(wil, "No free buffer ID found\n"); 23196c93589SGidon Studinski else 23296c93589SGidon Studinski wil_err_ratelimited(wil, 23396c93589SGidon Studinski "Error %d in refill desc[%d]\n", 23496c93589SGidon Studinski rc, ring->swhead); 23596c93589SGidon Studinski break; 23696c93589SGidon Studinski } 23796c93589SGidon Studinski } 23896c93589SGidon Studinski 23996c93589SGidon Studinski /* make sure all writes to descriptors (shared memory) are done before 24096c93589SGidon Studinski * committing them to HW 24196c93589SGidon Studinski */ 24296c93589SGidon Studinski wmb(); 24396c93589SGidon Studinski 24496c93589SGidon Studinski wil_w(wil, ring->hwtail, ring->swhead); 24596c93589SGidon Studinski 24696c93589SGidon Studinski return rc; 24796c93589SGidon Studinski } 24896c93589SGidon Studinski 24996c93589SGidon Studinski static void wil_move_all_rx_buff_to_free_list(struct wil6210_priv *wil, 25096c93589SGidon Studinski struct wil_ring *ring) 25196c93589SGidon Studinski { 25296c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 25396c93589SGidon Studinski u32 next_tail; 25496c93589SGidon Studinski u32 swhead = (ring->swhead + 1) % ring->size; 25596c93589SGidon Studinski dma_addr_t pa; 25696c93589SGidon Studinski u16 dmalen; 25796c93589SGidon Studinski 25896c93589SGidon Studinski for (; next_tail = wil_ring_next_tail(ring), (next_tail != swhead); 25996c93589SGidon Studinski ring->swtail = next_tail) { 26096c93589SGidon Studinski struct wil_rx_enhanced_desc dd, *d = ⅆ 26196c93589SGidon Studinski struct wil_rx_enhanced_desc *_d = 26296c93589SGidon Studinski (struct wil_rx_enhanced_desc *) 26396c93589SGidon Studinski &ring->va[ring->swtail].rx.enhanced; 26496c93589SGidon Studinski struct sk_buff *skb; 26596c93589SGidon Studinski u16 buff_id; 26696c93589SGidon Studinski 26796c93589SGidon Studinski *d = *_d; 26896c93589SGidon Studinski pa = wil_rx_desc_get_addr_edma(&d->dma); 26996c93589SGidon Studinski dmalen = le16_to_cpu(d->dma.length); 27096c93589SGidon Studinski dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE); 27196c93589SGidon Studinski 27296c93589SGidon Studinski /* Extract the SKB from the rx_buff management array */ 27396c93589SGidon Studinski buff_id = __le16_to_cpu(d->mac.buff_id); 27496c93589SGidon Studinski if (buff_id >= wil->rx_buff_mgmt.size) { 27596c93589SGidon Studinski wil_err(wil, "invalid buff_id %d\n", buff_id); 27696c93589SGidon Studinski continue; 27796c93589SGidon Studinski } 27896c93589SGidon Studinski skb = wil->rx_buff_mgmt.buff_arr[buff_id].skb; 27996c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr[buff_id].skb = NULL; 28096c93589SGidon Studinski if (unlikely(!skb)) 28196c93589SGidon Studinski wil_err(wil, "No Rx skb at buff_id %d\n", buff_id); 28296c93589SGidon Studinski else 28396c93589SGidon Studinski kfree_skb(skb); 28496c93589SGidon Studinski 28596c93589SGidon Studinski /* Move the buffer from the active to the free list */ 28696c93589SGidon Studinski list_move(&wil->rx_buff_mgmt.buff_arr[buff_id].list, 28796c93589SGidon Studinski &wil->rx_buff_mgmt.free); 28896c93589SGidon Studinski } 28996c93589SGidon Studinski } 29096c93589SGidon Studinski 29196c93589SGidon Studinski static void wil_free_rx_buff_arr(struct wil6210_priv *wil) 29296c93589SGidon Studinski { 29396c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 29496c93589SGidon Studinski 29596c93589SGidon Studinski if (!wil->rx_buff_mgmt.buff_arr) 29696c93589SGidon Studinski return; 29796c93589SGidon Studinski 29896c93589SGidon Studinski /* Move all the buffers to the free list in case active list is 29996c93589SGidon Studinski * not empty in order to release all SKBs before deleting the array 30096c93589SGidon Studinski */ 30196c93589SGidon Studinski wil_move_all_rx_buff_to_free_list(wil, ring); 30296c93589SGidon Studinski 30396c93589SGidon Studinski kfree(wil->rx_buff_mgmt.buff_arr); 30496c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr = NULL; 30596c93589SGidon Studinski } 30696c93589SGidon Studinski 30796c93589SGidon Studinski static int wil_init_rx_buff_arr(struct wil6210_priv *wil, 30896c93589SGidon Studinski size_t size) 30996c93589SGidon Studinski { 31096c93589SGidon Studinski struct wil_rx_buff *buff_arr; 31196c93589SGidon Studinski struct list_head *active = &wil->rx_buff_mgmt.active; 31296c93589SGidon Studinski struct list_head *free = &wil->rx_buff_mgmt.free; 31396c93589SGidon Studinski int i; 31496c93589SGidon Studinski 31596c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr = kcalloc(size, sizeof(struct wil_rx_buff), 31696c93589SGidon Studinski GFP_KERNEL); 31796c93589SGidon Studinski if (!wil->rx_buff_mgmt.buff_arr) 31896c93589SGidon Studinski return -ENOMEM; 31996c93589SGidon Studinski 32096c93589SGidon Studinski /* Set list heads */ 32196c93589SGidon Studinski INIT_LIST_HEAD(active); 32296c93589SGidon Studinski INIT_LIST_HEAD(free); 32396c93589SGidon Studinski 32496c93589SGidon Studinski /* Linkify the list */ 32596c93589SGidon Studinski buff_arr = wil->rx_buff_mgmt.buff_arr; 32696c93589SGidon Studinski for (i = 0; i < size; i++) { 32796c93589SGidon Studinski list_add(&buff_arr[i].list, free); 32896c93589SGidon Studinski buff_arr[i].id = i; 32996c93589SGidon Studinski } 33096c93589SGidon Studinski 33196c93589SGidon Studinski wil->rx_buff_mgmt.size = size; 33296c93589SGidon Studinski 33396c93589SGidon Studinski return 0; 33496c93589SGidon Studinski } 33596c93589SGidon Studinski 33696c93589SGidon Studinski static int wil_init_rx_sring(struct wil6210_priv *wil, 33796c93589SGidon Studinski u16 status_ring_size, 33896c93589SGidon Studinski size_t elem_size, 33996c93589SGidon Studinski u16 ring_id) 34096c93589SGidon Studinski { 34196c93589SGidon Studinski struct wil_status_ring *sring = &wil->srings[ring_id]; 34296c93589SGidon Studinski int rc; 34396c93589SGidon Studinski 34496c93589SGidon Studinski wil_dbg_misc(wil, "init RX sring: size=%u, ring_id=%u\n", sring->size, 34596c93589SGidon Studinski ring_id); 34696c93589SGidon Studinski 34796c93589SGidon Studinski memset(&sring->rx_data, 0, sizeof(sring->rx_data)); 34896c93589SGidon Studinski 34996c93589SGidon Studinski sring->is_rx = true; 35096c93589SGidon Studinski sring->size = status_ring_size; 35196c93589SGidon Studinski sring->elem_size = elem_size; 35296c93589SGidon Studinski rc = wil_sring_alloc(wil, sring); 35396c93589SGidon Studinski if (rc) 35496c93589SGidon Studinski return rc; 35596c93589SGidon Studinski 35696c93589SGidon Studinski rc = wil_wmi_rx_sring_add(wil, ring_id); 35796c93589SGidon Studinski if (rc) 35896c93589SGidon Studinski goto out_free; 35996c93589SGidon Studinski 36096c93589SGidon Studinski sring->desc_rdy_pol = 1; 36196c93589SGidon Studinski 36296c93589SGidon Studinski return 0; 36396c93589SGidon Studinski out_free: 36496c93589SGidon Studinski wil_sring_free(wil, sring); 36596c93589SGidon Studinski return rc; 36696c93589SGidon Studinski } 36796c93589SGidon Studinski 36896c93589SGidon Studinski static int wil_ring_alloc_desc_ring(struct wil6210_priv *wil, 36996c93589SGidon Studinski struct wil_ring *ring) 37096c93589SGidon Studinski { 37196c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 37296c93589SGidon Studinski size_t sz = ring->size * sizeof(ring->va[0]); 37396c93589SGidon Studinski 37496c93589SGidon Studinski wil_dbg_misc(wil, "alloc_desc_ring:\n"); 37596c93589SGidon Studinski 37696c93589SGidon Studinski BUILD_BUG_ON(sizeof(ring->va[0]) != 32); 37796c93589SGidon Studinski 37896c93589SGidon Studinski ring->swhead = 0; 37996c93589SGidon Studinski ring->swtail = 0; 38096c93589SGidon Studinski ring->ctx = kcalloc(ring->size, sizeof(ring->ctx[0]), GFP_KERNEL); 38196c93589SGidon Studinski if (!ring->ctx) 38296c93589SGidon Studinski goto err; 38396c93589SGidon Studinski 38496c93589SGidon Studinski ring->va = dma_zalloc_coherent(dev, sz, &ring->pa, GFP_KERNEL); 38596c93589SGidon Studinski if (!ring->va) 38696c93589SGidon Studinski goto err_free_ctx; 38796c93589SGidon Studinski 38896c93589SGidon Studinski if (ring->is_rx) { 38996c93589SGidon Studinski sz = sizeof(*ring->edma_rx_swtail.va); 39096c93589SGidon Studinski ring->edma_rx_swtail.va = 39196c93589SGidon Studinski dma_zalloc_coherent(dev, sz, &ring->edma_rx_swtail.pa, 39296c93589SGidon Studinski GFP_KERNEL); 39396c93589SGidon Studinski if (!ring->edma_rx_swtail.va) 39496c93589SGidon Studinski goto err_free_va; 39596c93589SGidon Studinski } 39696c93589SGidon Studinski 39796c93589SGidon Studinski wil_dbg_misc(wil, "%s ring[%d] 0x%p:%pad 0x%p\n", 39896c93589SGidon Studinski ring->is_rx ? "RX" : "TX", 39996c93589SGidon Studinski ring->size, ring->va, &ring->pa, ring->ctx); 40096c93589SGidon Studinski 40196c93589SGidon Studinski return 0; 40296c93589SGidon Studinski err_free_va: 40396c93589SGidon Studinski dma_free_coherent(dev, ring->size * sizeof(ring->va[0]), 40496c93589SGidon Studinski (void *)ring->va, ring->pa); 40596c93589SGidon Studinski ring->va = NULL; 40696c93589SGidon Studinski err_free_ctx: 40796c93589SGidon Studinski kfree(ring->ctx); 40896c93589SGidon Studinski ring->ctx = NULL; 40996c93589SGidon Studinski err: 41096c93589SGidon Studinski return -ENOMEM; 41196c93589SGidon Studinski } 41296c93589SGidon Studinski 41396c93589SGidon Studinski static void wil_ring_free_edma(struct wil6210_priv *wil, struct wil_ring *ring) 41496c93589SGidon Studinski { 41596c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 41696c93589SGidon Studinski size_t sz; 41796c93589SGidon Studinski int ring_index = 0; 41896c93589SGidon Studinski 41996c93589SGidon Studinski if (!ring->va) 42096c93589SGidon Studinski return; 42196c93589SGidon Studinski 42296c93589SGidon Studinski sz = ring->size * sizeof(ring->va[0]); 42396c93589SGidon Studinski 42496c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 42596c93589SGidon Studinski if (ring->is_rx) { 42696c93589SGidon Studinski wil_dbg_misc(wil, "free Rx ring [%d] 0x%p:%pad 0x%p\n", 42796c93589SGidon Studinski ring->size, ring->va, 42896c93589SGidon Studinski &ring->pa, ring->ctx); 42996c93589SGidon Studinski 43096c93589SGidon Studinski wil_move_all_rx_buff_to_free_list(wil, ring); 43196c93589SGidon Studinski goto out; 43296c93589SGidon Studinski } 43396c93589SGidon Studinski 43496c93589SGidon Studinski /* TX ring */ 43596c93589SGidon Studinski ring_index = ring - wil->ring_tx; 43696c93589SGidon Studinski 43796c93589SGidon Studinski wil_dbg_misc(wil, "free Tx ring %d [%d] 0x%p:%pad 0x%p\n", 43896c93589SGidon Studinski ring_index, ring->size, ring->va, 43996c93589SGidon Studinski &ring->pa, ring->ctx); 44096c93589SGidon Studinski 44196c93589SGidon Studinski while (!wil_ring_is_empty(ring)) { 44296c93589SGidon Studinski struct wil_ctx *ctx; 44396c93589SGidon Studinski 44496c93589SGidon Studinski struct wil_tx_enhanced_desc dd, *d = ⅆ 44596c93589SGidon Studinski struct wil_tx_enhanced_desc *_d = 44696c93589SGidon Studinski (struct wil_tx_enhanced_desc *) 44796c93589SGidon Studinski &ring->va[ring->swtail].tx.enhanced; 44896c93589SGidon Studinski 44996c93589SGidon Studinski ctx = &ring->ctx[ring->swtail]; 45096c93589SGidon Studinski if (!ctx) { 45196c93589SGidon Studinski wil_dbg_txrx(wil, 45296c93589SGidon Studinski "ctx(%d) was already completed\n", 45396c93589SGidon Studinski ring->swtail); 45496c93589SGidon Studinski ring->swtail = wil_ring_next_tail(ring); 45596c93589SGidon Studinski continue; 45696c93589SGidon Studinski } 45796c93589SGidon Studinski *d = *_d; 4589202d7b6SMaya Erez wil_tx_desc_unmap_edma(dev, (union wil_tx_desc *)d, ctx); 45996c93589SGidon Studinski if (ctx->skb) 46096c93589SGidon Studinski dev_kfree_skb_any(ctx->skb); 46196c93589SGidon Studinski ring->swtail = wil_ring_next_tail(ring); 46296c93589SGidon Studinski } 46396c93589SGidon Studinski 46496c93589SGidon Studinski out: 46596c93589SGidon Studinski dma_free_coherent(dev, sz, (void *)ring->va, ring->pa); 46696c93589SGidon Studinski kfree(ring->ctx); 46796c93589SGidon Studinski ring->pa = 0; 46896c93589SGidon Studinski ring->va = NULL; 46996c93589SGidon Studinski ring->ctx = NULL; 47096c93589SGidon Studinski } 47196c93589SGidon Studinski 47296c93589SGidon Studinski static int wil_init_rx_desc_ring(struct wil6210_priv *wil, u16 desc_ring_size, 47396c93589SGidon Studinski int status_ring_id) 47496c93589SGidon Studinski { 47596c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 47696c93589SGidon Studinski int rc; 47796c93589SGidon Studinski 47896c93589SGidon Studinski wil_dbg_misc(wil, "init RX desc ring\n"); 47996c93589SGidon Studinski 48096c93589SGidon Studinski ring->size = desc_ring_size; 48196c93589SGidon Studinski ring->is_rx = true; 48296c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 48396c93589SGidon Studinski if (rc) 48496c93589SGidon Studinski return rc; 48596c93589SGidon Studinski 48696c93589SGidon Studinski rc = wil_wmi_rx_desc_ring_add(wil, status_ring_id); 48796c93589SGidon Studinski if (rc) 48896c93589SGidon Studinski goto out_free; 48996c93589SGidon Studinski 49096c93589SGidon Studinski return 0; 49196c93589SGidon Studinski out_free: 49296c93589SGidon Studinski wil_ring_free_edma(wil, ring); 49396c93589SGidon Studinski return rc; 49496c93589SGidon Studinski } 49596c93589SGidon Studinski 49696c93589SGidon Studinski static void wil_rx_buf_len_init_edma(struct wil6210_priv *wil) 49796c93589SGidon Studinski { 49896c93589SGidon Studinski wil->rx_buf_len = rx_large_buf ? 49996c93589SGidon Studinski WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD; 50096c93589SGidon Studinski } 50196c93589SGidon Studinski 50296c93589SGidon Studinski static int wil_rx_init_edma(struct wil6210_priv *wil, u16 desc_ring_size) 50396c93589SGidon Studinski { 50496c93589SGidon Studinski u16 status_ring_size; 50596c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 50696c93589SGidon Studinski int rc; 50796c93589SGidon Studinski size_t elem_size = wil->use_compressed_rx_status ? 50896c93589SGidon Studinski sizeof(struct wil_rx_status_compressed) : 50996c93589SGidon Studinski sizeof(struct wil_rx_status_extended); 51096c93589SGidon Studinski int i; 51196c93589SGidon Studinski u16 max_rx_pl_per_desc; 51296c93589SGidon Studinski 51396c93589SGidon Studinski if (wil->rx_status_ring_order < WIL_SRING_SIZE_ORDER_MIN || 51496c93589SGidon Studinski wil->rx_status_ring_order > WIL_SRING_SIZE_ORDER_MAX) 51596c93589SGidon Studinski wil->rx_status_ring_order = WIL_RX_SRING_SIZE_ORDER_DEFAULT; 51696c93589SGidon Studinski 51796c93589SGidon Studinski status_ring_size = 1 << wil->rx_status_ring_order; 51896c93589SGidon Studinski 51996c93589SGidon Studinski wil_dbg_misc(wil, 52096c93589SGidon Studinski "rx_init, desc_ring_size=%u, status_ring_size=%u, elem_size=%zu\n", 52196c93589SGidon Studinski desc_ring_size, status_ring_size, elem_size); 52296c93589SGidon Studinski 52396c93589SGidon Studinski wil_rx_buf_len_init_edma(wil); 52496c93589SGidon Studinski 52596c93589SGidon Studinski max_rx_pl_per_desc = wil->rx_buf_len + ETH_HLEN + 52696c93589SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 52796c93589SGidon Studinski 52896c93589SGidon Studinski /* Use debugfs dbg_num_rx_srings if set, reserve one sring for TX */ 52996c93589SGidon Studinski if (wil->num_rx_status_rings > WIL6210_MAX_STATUS_RINGS - 1) 53096c93589SGidon Studinski wil->num_rx_status_rings = WIL6210_MAX_STATUS_RINGS - 1; 53196c93589SGidon Studinski 53296c93589SGidon Studinski wil_dbg_misc(wil, "rx_init: allocate %d status rings\n", 53396c93589SGidon Studinski wil->num_rx_status_rings); 53496c93589SGidon Studinski 53596c93589SGidon Studinski rc = wil_wmi_cfg_def_rx_offload(wil, max_rx_pl_per_desc); 53696c93589SGidon Studinski if (rc) 53796c93589SGidon Studinski return rc; 53896c93589SGidon Studinski 53996c93589SGidon Studinski /* Allocate status ring */ 54096c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 54196c93589SGidon Studinski int sring_id = wil_find_free_sring(wil); 54296c93589SGidon Studinski 54396c93589SGidon Studinski if (sring_id < 0) { 54496c93589SGidon Studinski rc = -EFAULT; 54596c93589SGidon Studinski goto err_free_status; 54696c93589SGidon Studinski } 54796c93589SGidon Studinski rc = wil_init_rx_sring(wil, status_ring_size, elem_size, 54896c93589SGidon Studinski sring_id); 54996c93589SGidon Studinski if (rc) 55096c93589SGidon Studinski goto err_free_status; 55196c93589SGidon Studinski } 55296c93589SGidon Studinski 55396c93589SGidon Studinski /* Allocate descriptor ring */ 55496c93589SGidon Studinski rc = wil_init_rx_desc_ring(wil, desc_ring_size, 55596c93589SGidon Studinski WIL_DEFAULT_RX_STATUS_RING_ID); 55696c93589SGidon Studinski if (rc) 55796c93589SGidon Studinski goto err_free_status; 55896c93589SGidon Studinski 55996c93589SGidon Studinski if (wil->rx_buff_id_count >= status_ring_size) { 56096c93589SGidon Studinski wil_info(wil, 56196c93589SGidon Studinski "rx_buff_id_count %d exceeds sring_size %d. set it to %d\n", 56296c93589SGidon Studinski wil->rx_buff_id_count, status_ring_size, 56396c93589SGidon Studinski status_ring_size - 1); 56496c93589SGidon Studinski wil->rx_buff_id_count = status_ring_size - 1; 56596c93589SGidon Studinski } 56696c93589SGidon Studinski 56796c93589SGidon Studinski /* Allocate Rx buffer array */ 56896c93589SGidon Studinski rc = wil_init_rx_buff_arr(wil, wil->rx_buff_id_count); 56996c93589SGidon Studinski if (rc) 57096c93589SGidon Studinski goto err_free_desc; 57196c93589SGidon Studinski 57296c93589SGidon Studinski /* Fill descriptor ring with credits */ 57396c93589SGidon Studinski rc = wil_rx_refill_edma(wil); 57496c93589SGidon Studinski if (rc) 57596c93589SGidon Studinski goto err_free_rx_buff_arr; 57696c93589SGidon Studinski 57796c93589SGidon Studinski return 0; 57896c93589SGidon Studinski err_free_rx_buff_arr: 57996c93589SGidon Studinski wil_free_rx_buff_arr(wil); 58096c93589SGidon Studinski err_free_desc: 58196c93589SGidon Studinski wil_ring_free_edma(wil, ring); 58296c93589SGidon Studinski err_free_status: 58396c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) 58496c93589SGidon Studinski wil_sring_free(wil, &wil->srings[i]); 58596c93589SGidon Studinski 58696c93589SGidon Studinski return rc; 58796c93589SGidon Studinski } 58896c93589SGidon Studinski 58996c93589SGidon Studinski static int wil_ring_init_tx_edma(struct wil6210_vif *vif, int ring_id, 59096c93589SGidon Studinski int size, int cid, int tid) 59196c93589SGidon Studinski { 59296c93589SGidon Studinski struct wil6210_priv *wil = vif_to_wil(vif); 59396c93589SGidon Studinski int rc; 59496c93589SGidon Studinski struct wil_ring *ring = &wil->ring_tx[ring_id]; 59596c93589SGidon Studinski struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 59696c93589SGidon Studinski 59796c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 59896c93589SGidon Studinski 59996c93589SGidon Studinski wil_dbg_misc(wil, 60096c93589SGidon Studinski "init TX ring: ring_id=%u, cid=%u, tid=%u, sring_id=%u\n", 60196c93589SGidon Studinski ring_id, cid, tid, wil->tx_sring_idx); 60296c93589SGidon Studinski 60396c93589SGidon Studinski wil_tx_data_init(txdata); 60496c93589SGidon Studinski ring->size = size; 60596c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 60696c93589SGidon Studinski if (rc) 60796c93589SGidon Studinski goto out; 60896c93589SGidon Studinski 60996c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = cid; 61096c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = tid; 61196c93589SGidon Studinski if (!vif->privacy) 61296c93589SGidon Studinski txdata->dot1x_open = true; 61396c93589SGidon Studinski 61496c93589SGidon Studinski rc = wil_wmi_tx_desc_ring_add(vif, ring_id, cid, tid); 61596c93589SGidon Studinski if (rc) { 61696c93589SGidon Studinski wil_err(wil, "WMI_TX_DESC_RING_ADD_CMD failed\n"); 61796c93589SGidon Studinski goto out_free; 61896c93589SGidon Studinski } 61996c93589SGidon Studinski 62096c93589SGidon Studinski if (txdata->dot1x_open && agg_wsize >= 0) 62196c93589SGidon Studinski wil_addba_tx_request(wil, ring_id, agg_wsize); 62296c93589SGidon Studinski 62396c93589SGidon Studinski return 0; 62496c93589SGidon Studinski out_free: 62596c93589SGidon Studinski spin_lock_bh(&txdata->lock); 62696c93589SGidon Studinski txdata->dot1x_open = false; 62796c93589SGidon Studinski txdata->enabled = 0; 62896c93589SGidon Studinski spin_unlock_bh(&txdata->lock); 62996c93589SGidon Studinski wil_ring_free_edma(wil, ring); 63096c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = WIL6210_MAX_CID; 63196c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = 0; 63296c93589SGidon Studinski 63396c93589SGidon Studinski out: 63496c93589SGidon Studinski return rc; 63596c93589SGidon Studinski } 63696c93589SGidon Studinski 6379202d7b6SMaya Erez static int wil_tx_desc_map_edma(union wil_tx_desc *desc, 6389202d7b6SMaya Erez dma_addr_t pa, 6399202d7b6SMaya Erez u32 len, 6409202d7b6SMaya Erez int ring_index) 6419202d7b6SMaya Erez { 6429202d7b6SMaya Erez struct wil_tx_enhanced_desc *d = 6439202d7b6SMaya Erez (struct wil_tx_enhanced_desc *)&desc->enhanced; 6449202d7b6SMaya Erez 6459202d7b6SMaya Erez memset(d, 0, sizeof(struct wil_tx_enhanced_desc)); 6469202d7b6SMaya Erez 6479202d7b6SMaya Erez wil_desc_set_addr_edma(&d->dma.addr, &d->dma.addr_high_high, pa); 6489202d7b6SMaya Erez 6499202d7b6SMaya Erez /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/ 6509202d7b6SMaya Erez d->dma.length = cpu_to_le16((u16)len); 6519202d7b6SMaya Erez d->mac.d[0] = (ring_index << WIL_EDMA_DESC_TX_MAC_CFG_0_QID_POS); 6529202d7b6SMaya Erez /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi; 6539202d7b6SMaya Erez * 3 - eth mode 6549202d7b6SMaya Erez */ 6559202d7b6SMaya Erez d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) | 6569202d7b6SMaya Erez (0x3 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS); 6579202d7b6SMaya Erez 6589202d7b6SMaya Erez return 0; 6599202d7b6SMaya Erez } 6609202d7b6SMaya Erez 6619202d7b6SMaya Erez static inline void 6629202d7b6SMaya Erez wil_get_next_tx_status_msg(struct wil_status_ring *sring, 6639202d7b6SMaya Erez struct wil_ring_tx_status *msg) 6649202d7b6SMaya Erez { 6659202d7b6SMaya Erez struct wil_ring_tx_status *_msg = (struct wil_ring_tx_status *) 6669202d7b6SMaya Erez (sring->va + (sring->elem_size * sring->swhead)); 6679202d7b6SMaya Erez 6689202d7b6SMaya Erez *msg = *_msg; 6699202d7b6SMaya Erez } 6709202d7b6SMaya Erez 6719202d7b6SMaya Erez /** 6729202d7b6SMaya Erez * Clean up transmitted skb's from the Tx descriptor RING. 6739202d7b6SMaya Erez * Return number of descriptors cleared. 6749202d7b6SMaya Erez */ 6759202d7b6SMaya Erez int wil_tx_sring_handler(struct wil6210_priv *wil, 6769202d7b6SMaya Erez struct wil_status_ring *sring) 6779202d7b6SMaya Erez { 6789202d7b6SMaya Erez struct net_device *ndev; 6799202d7b6SMaya Erez struct device *dev = wil_to_dev(wil); 6809202d7b6SMaya Erez struct wil_ring *ring = NULL; 6819202d7b6SMaya Erez struct wil_ring_tx_data *txdata; 6829202d7b6SMaya Erez /* Total number of completed descriptors in all descriptor rings */ 6839202d7b6SMaya Erez int desc_cnt = 0; 6849202d7b6SMaya Erez int cid; 6859202d7b6SMaya Erez struct wil_net_stats *stats = NULL; 6869202d7b6SMaya Erez struct wil_tx_enhanced_desc *_d; 6879202d7b6SMaya Erez unsigned int ring_id; 6889202d7b6SMaya Erez unsigned int num_descs; 6899202d7b6SMaya Erez int i; 6909202d7b6SMaya Erez u8 dr_bit; /* Descriptor Ready bit */ 6919202d7b6SMaya Erez struct wil_ring_tx_status msg; 6929202d7b6SMaya Erez struct wil6210_vif *vif; 6939202d7b6SMaya Erez int used_before_complete; 6949202d7b6SMaya Erez int used_new; 6959202d7b6SMaya Erez 6969202d7b6SMaya Erez wil_get_next_tx_status_msg(sring, &msg); 6979202d7b6SMaya Erez dr_bit = msg.desc_ready >> TX_STATUS_DESC_READY_POS; 6989202d7b6SMaya Erez 6999202d7b6SMaya Erez /* Process completion messages while DR bit has the expected polarity */ 7009202d7b6SMaya Erez while (dr_bit == sring->desc_rdy_pol) { 7019202d7b6SMaya Erez num_descs = msg.num_descriptors; 7029202d7b6SMaya Erez if (!num_descs) { 7039202d7b6SMaya Erez wil_err(wil, "invalid num_descs 0\n"); 7049202d7b6SMaya Erez goto again; 7059202d7b6SMaya Erez } 7069202d7b6SMaya Erez 7079202d7b6SMaya Erez /* Find the corresponding descriptor ring */ 7089202d7b6SMaya Erez ring_id = msg.ring_id; 7099202d7b6SMaya Erez 7109202d7b6SMaya Erez if (unlikely(ring_id >= WIL6210_MAX_TX_RINGS)) { 7119202d7b6SMaya Erez wil_err(wil, "invalid ring id %d\n", ring_id); 7129202d7b6SMaya Erez goto again; 7139202d7b6SMaya Erez } 7149202d7b6SMaya Erez ring = &wil->ring_tx[ring_id]; 7159202d7b6SMaya Erez if (unlikely(!ring->va)) { 7169202d7b6SMaya Erez wil_err(wil, "Tx irq[%d]: ring not initialized\n", 7179202d7b6SMaya Erez ring_id); 7189202d7b6SMaya Erez goto again; 7199202d7b6SMaya Erez } 7209202d7b6SMaya Erez txdata = &wil->ring_tx_data[ring_id]; 7219202d7b6SMaya Erez if (unlikely(!txdata->enabled)) { 7229202d7b6SMaya Erez wil_info(wil, "Tx irq[%d]: ring disabled\n", ring_id); 7239202d7b6SMaya Erez goto again; 7249202d7b6SMaya Erez } 7259202d7b6SMaya Erez vif = wil->vifs[txdata->mid]; 7269202d7b6SMaya Erez if (unlikely(!vif)) { 7279202d7b6SMaya Erez wil_dbg_txrx(wil, "invalid MID %d for ring %d\n", 7289202d7b6SMaya Erez txdata->mid, ring_id); 7299202d7b6SMaya Erez goto again; 7309202d7b6SMaya Erez } 7319202d7b6SMaya Erez 7329202d7b6SMaya Erez ndev = vif_to_ndev(vif); 7339202d7b6SMaya Erez 7349202d7b6SMaya Erez cid = wil->ring2cid_tid[ring_id][0]; 7359202d7b6SMaya Erez if (cid < WIL6210_MAX_CID) 7369202d7b6SMaya Erez stats = &wil->sta[cid].stats; 7379202d7b6SMaya Erez 7389202d7b6SMaya Erez wil_dbg_txrx(wil, 7399202d7b6SMaya Erez "tx_status: completed desc_ring (%d), num_descs (%d)\n", 7409202d7b6SMaya Erez ring_id, num_descs); 7419202d7b6SMaya Erez 7429202d7b6SMaya Erez used_before_complete = wil_ring_used_tx(ring); 7439202d7b6SMaya Erez 7449202d7b6SMaya Erez for (i = 0 ; i < num_descs; ++i) { 7459202d7b6SMaya Erez struct wil_ctx *ctx = &ring->ctx[ring->swtail]; 7469202d7b6SMaya Erez struct wil_tx_enhanced_desc dd, *d = ⅆ 7479202d7b6SMaya Erez u16 dmalen; 7489202d7b6SMaya Erez struct sk_buff *skb = ctx->skb; 7499202d7b6SMaya Erez 7509202d7b6SMaya Erez _d = (struct wil_tx_enhanced_desc *) 7519202d7b6SMaya Erez &ring->va[ring->swtail].tx.enhanced; 7529202d7b6SMaya Erez *d = *_d; 7539202d7b6SMaya Erez 7549202d7b6SMaya Erez dmalen = le16_to_cpu(d->dma.length); 7559202d7b6SMaya Erez trace_wil6210_tx_status(&msg, ring->swtail, dmalen); 7569202d7b6SMaya Erez wil_dbg_txrx(wil, 7579202d7b6SMaya Erez "TxC[%2d][%3d] : %d bytes, status 0x%02x\n", 7589202d7b6SMaya Erez ring_id, ring->swtail, dmalen, 7599202d7b6SMaya Erez msg.status); 7609202d7b6SMaya Erez wil_hex_dump_txrx("TxS ", DUMP_PREFIX_NONE, 32, 4, 7619202d7b6SMaya Erez (const void *)&msg, sizeof(msg), 7629202d7b6SMaya Erez false); 7639202d7b6SMaya Erez 7649202d7b6SMaya Erez wil_tx_desc_unmap_edma(dev, 7659202d7b6SMaya Erez (union wil_tx_desc *)d, 7669202d7b6SMaya Erez ctx); 7679202d7b6SMaya Erez 7689202d7b6SMaya Erez if (skb) { 7699202d7b6SMaya Erez if (likely(msg.status == 0)) { 7709202d7b6SMaya Erez ndev->stats.tx_packets++; 7719202d7b6SMaya Erez ndev->stats.tx_bytes += skb->len; 7729202d7b6SMaya Erez if (stats) { 7739202d7b6SMaya Erez stats->tx_packets++; 7749202d7b6SMaya Erez stats->tx_bytes += skb->len; 7759202d7b6SMaya Erez } 7769202d7b6SMaya Erez } else { 7779202d7b6SMaya Erez ndev->stats.tx_errors++; 7789202d7b6SMaya Erez if (stats) 7799202d7b6SMaya Erez stats->tx_errors++; 7809202d7b6SMaya Erez } 7819202d7b6SMaya Erez wil_consume_skb(skb, msg.status == 0); 7829202d7b6SMaya Erez } 7839202d7b6SMaya Erez memset(ctx, 0, sizeof(*ctx)); 7849202d7b6SMaya Erez /* Make sure the ctx is zeroed before updating the tail 7859202d7b6SMaya Erez * to prevent a case where wil_tx_ring will see 7869202d7b6SMaya Erez * this descriptor as used and handle it before ctx zero 7879202d7b6SMaya Erez * is completed. 7889202d7b6SMaya Erez */ 7899202d7b6SMaya Erez wmb(); 7909202d7b6SMaya Erez 7919202d7b6SMaya Erez ring->swtail = wil_ring_next_tail(ring); 7929202d7b6SMaya Erez 7939202d7b6SMaya Erez desc_cnt++; 7949202d7b6SMaya Erez } 7959202d7b6SMaya Erez 7969202d7b6SMaya Erez /* performance monitoring */ 7979202d7b6SMaya Erez used_new = wil_ring_used_tx(ring); 7989202d7b6SMaya Erez if (wil_val_in_range(wil->ring_idle_trsh, 7999202d7b6SMaya Erez used_new, used_before_complete)) { 8009202d7b6SMaya Erez wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n", 8019202d7b6SMaya Erez ring_id, used_before_complete, used_new); 8029202d7b6SMaya Erez txdata->last_idle = get_cycles(); 8039202d7b6SMaya Erez } 8049202d7b6SMaya Erez 8059202d7b6SMaya Erez again: 8069202d7b6SMaya Erez wil_sring_advance_swhead(sring); 8079202d7b6SMaya Erez 8089202d7b6SMaya Erez wil_get_next_tx_status_msg(sring, &msg); 8099202d7b6SMaya Erez dr_bit = msg.desc_ready >> TX_STATUS_DESC_READY_POS; 8109202d7b6SMaya Erez } 8119202d7b6SMaya Erez 8129202d7b6SMaya Erez /* shall we wake net queues? */ 8139202d7b6SMaya Erez if (desc_cnt) 8149202d7b6SMaya Erez wil_update_net_queues(wil, vif, NULL, false); 8159202d7b6SMaya Erez 8169202d7b6SMaya Erez /* Update the HW tail ptr (RD ptr) */ 8179202d7b6SMaya Erez wil_w(wil, sring->hwtail, (sring->swhead - 1) % sring->size); 8189202d7b6SMaya Erez 8199202d7b6SMaya Erez return desc_cnt; 8209202d7b6SMaya Erez } 8219202d7b6SMaya Erez 8229202d7b6SMaya Erez /** 8239202d7b6SMaya Erez * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding 8249202d7b6SMaya Erez * @skb is used to obtain the protocol and headers length. 8259202d7b6SMaya Erez * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data, 8269202d7b6SMaya Erez * 2 - middle, 3 - last descriptor. 8279202d7b6SMaya Erez */ 8289202d7b6SMaya Erez static void wil_tx_desc_offload_setup_tso_edma(struct wil_tx_enhanced_desc *d, 8299202d7b6SMaya Erez int tso_desc_type, bool is_ipv4, 8309202d7b6SMaya Erez int tcp_hdr_len, 8319202d7b6SMaya Erez int skb_net_hdr_len, 8329202d7b6SMaya Erez int mss) 8339202d7b6SMaya Erez { 8349202d7b6SMaya Erez /* Number of descriptors */ 8359202d7b6SMaya Erez d->mac.d[2] |= 1; 8369202d7b6SMaya Erez /* Maximum Segment Size */ 8379202d7b6SMaya Erez d->mac.tso_mss |= cpu_to_le16(mss >> 2); 8389202d7b6SMaya Erez /* L4 header len: TCP header length */ 8399202d7b6SMaya Erez d->dma.l4_hdr_len |= tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK; 8409202d7b6SMaya Erez /* EOP, TSO desc type, Segmentation enable, 8419202d7b6SMaya Erez * Insert IPv4 and TCP / UDP Checksum 8429202d7b6SMaya Erez */ 8439202d7b6SMaya Erez d->dma.cmd |= BIT(WIL_EDMA_DESC_TX_CFG_EOP_POS) | 8449202d7b6SMaya Erez tso_desc_type << WIL_EDMA_DESC_TX_CFG_TSO_DESC_TYPE_POS | 8459202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_SEG_EN_POS) | 8469202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_INSERT_IP_CHKSUM_POS) | 8479202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_INSERT_TCP_CHKSUM_POS); 8489202d7b6SMaya Erez /* Calculate pseudo-header */ 8499202d7b6SMaya Erez d->dma.w1 |= BIT(WIL_EDMA_DESC_TX_CFG_PSEUDO_HEADER_CALC_EN_POS) | 8509202d7b6SMaya Erez BIT(WIL_EDMA_DESC_TX_CFG_L4_TYPE_POS); 8519202d7b6SMaya Erez /* IP Header Length */ 8529202d7b6SMaya Erez d->dma.ip_length |= skb_net_hdr_len; 8539202d7b6SMaya Erez /* MAC header length and IP address family*/ 8549202d7b6SMaya Erez d->dma.b11 |= ETH_HLEN | 8559202d7b6SMaya Erez is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS; 8569202d7b6SMaya Erez } 8579202d7b6SMaya Erez 8589202d7b6SMaya Erez static int wil_tx_tso_gen_desc(struct wil6210_priv *wil, void *buff_addr, 8599202d7b6SMaya Erez int len, uint i, int tso_desc_type, 8609202d7b6SMaya Erez skb_frag_t *frag, struct wil_ring *ring, 8619202d7b6SMaya Erez struct sk_buff *skb, bool is_ipv4, 8629202d7b6SMaya Erez int tcp_hdr_len, int skb_net_hdr_len, 8639202d7b6SMaya Erez int mss, int *descs_used) 8649202d7b6SMaya Erez { 8659202d7b6SMaya Erez struct device *dev = wil_to_dev(wil); 8669202d7b6SMaya Erez struct wil_tx_enhanced_desc *_desc = (struct wil_tx_enhanced_desc *) 8679202d7b6SMaya Erez &ring->va[i].tx.enhanced; 8689202d7b6SMaya Erez struct wil_tx_enhanced_desc desc_mem, *d = &desc_mem; 8699202d7b6SMaya Erez int ring_index = ring - wil->ring_tx; 8709202d7b6SMaya Erez dma_addr_t pa; 8719202d7b6SMaya Erez 8729202d7b6SMaya Erez if (len == 0) 8739202d7b6SMaya Erez return 0; 8749202d7b6SMaya Erez 8759202d7b6SMaya Erez if (!frag) { 8769202d7b6SMaya Erez pa = dma_map_single(dev, buff_addr, len, DMA_TO_DEVICE); 8779202d7b6SMaya Erez ring->ctx[i].mapped_as = wil_mapped_as_single; 8789202d7b6SMaya Erez } else { 8799202d7b6SMaya Erez pa = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 8809202d7b6SMaya Erez ring->ctx[i].mapped_as = wil_mapped_as_page; 8819202d7b6SMaya Erez } 8829202d7b6SMaya Erez if (unlikely(dma_mapping_error(dev, pa))) { 8839202d7b6SMaya Erez wil_err(wil, "TSO: Skb DMA map error\n"); 8849202d7b6SMaya Erez return -EINVAL; 8859202d7b6SMaya Erez } 8869202d7b6SMaya Erez 8879202d7b6SMaya Erez wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, 8889202d7b6SMaya Erez len, ring_index); 8899202d7b6SMaya Erez wil_tx_desc_offload_setup_tso_edma(d, tso_desc_type, is_ipv4, 8909202d7b6SMaya Erez tcp_hdr_len, 8919202d7b6SMaya Erez skb_net_hdr_len, mss); 8929202d7b6SMaya Erez 8939202d7b6SMaya Erez /* hold reference to skb 8949202d7b6SMaya Erez * to prevent skb release before accounting 8959202d7b6SMaya Erez * in case of immediate "tx done" 8969202d7b6SMaya Erez */ 8979202d7b6SMaya Erez if (tso_desc_type == wil_tso_type_lst) 8989202d7b6SMaya Erez ring->ctx[i].skb = skb_get(skb); 8999202d7b6SMaya Erez 9009202d7b6SMaya Erez wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 9019202d7b6SMaya Erez (const void *)d, sizeof(*d), false); 9029202d7b6SMaya Erez 9039202d7b6SMaya Erez *_desc = *d; 9049202d7b6SMaya Erez (*descs_used)++; 9059202d7b6SMaya Erez 9069202d7b6SMaya Erez return 0; 9079202d7b6SMaya Erez } 9089202d7b6SMaya Erez 9099202d7b6SMaya Erez static int __wil_tx_ring_tso_edma(struct wil6210_priv *wil, 9109202d7b6SMaya Erez struct wil6210_vif *vif, 9119202d7b6SMaya Erez struct wil_ring *ring, 9129202d7b6SMaya Erez struct sk_buff *skb) 9139202d7b6SMaya Erez { 9149202d7b6SMaya Erez int ring_index = ring - wil->ring_tx; 9159202d7b6SMaya Erez struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index]; 9169202d7b6SMaya Erez int nr_frags = skb_shinfo(skb)->nr_frags; 9179202d7b6SMaya Erez int min_desc_required = nr_frags + 2; /* Headers, Head, Fragments */ 9189202d7b6SMaya Erez int used, avail = wil_ring_avail_tx(ring); 9199202d7b6SMaya Erez int f, hdrlen, headlen; 9209202d7b6SMaya Erez int gso_type; 9219202d7b6SMaya Erez bool is_ipv4; 9229202d7b6SMaya Erez u32 swhead = ring->swhead; 9239202d7b6SMaya Erez int descs_used = 0; /* total number of used descriptors */ 9249202d7b6SMaya Erez int rc = -EINVAL; 9259202d7b6SMaya Erez int tcp_hdr_len; 9269202d7b6SMaya Erez int skb_net_hdr_len; 9279202d7b6SMaya Erez int mss = skb_shinfo(skb)->gso_size; 9289202d7b6SMaya Erez 9299202d7b6SMaya Erez wil_dbg_txrx(wil, "tx_ring_tso: %d bytes to ring %d\n", skb->len, 9309202d7b6SMaya Erez ring_index); 9319202d7b6SMaya Erez 9329202d7b6SMaya Erez if (unlikely(!txdata->enabled)) 9339202d7b6SMaya Erez return -EINVAL; 9349202d7b6SMaya Erez 9359202d7b6SMaya Erez if (unlikely(avail < min_desc_required)) { 9369202d7b6SMaya Erez wil_err_ratelimited(wil, 9379202d7b6SMaya Erez "TSO: Tx ring[%2d] full. No space for %d fragments\n", 9389202d7b6SMaya Erez ring_index, min_desc_required); 9399202d7b6SMaya Erez return -ENOMEM; 9409202d7b6SMaya Erez } 9419202d7b6SMaya Erez 9429202d7b6SMaya Erez gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4); 9439202d7b6SMaya Erez switch (gso_type) { 9449202d7b6SMaya Erez case SKB_GSO_TCPV4: 9459202d7b6SMaya Erez is_ipv4 = true; 9469202d7b6SMaya Erez break; 9479202d7b6SMaya Erez case SKB_GSO_TCPV6: 9489202d7b6SMaya Erez is_ipv4 = false; 9499202d7b6SMaya Erez break; 9509202d7b6SMaya Erez default: 9519202d7b6SMaya Erez return -EINVAL; 9529202d7b6SMaya Erez } 9539202d7b6SMaya Erez 9549202d7b6SMaya Erez if (skb->ip_summed != CHECKSUM_PARTIAL) 9559202d7b6SMaya Erez return -EINVAL; 9569202d7b6SMaya Erez 9579202d7b6SMaya Erez /* tcp header length and skb network header length are fixed for all 9589202d7b6SMaya Erez * packet's descriptors - read them once here 9599202d7b6SMaya Erez */ 9609202d7b6SMaya Erez tcp_hdr_len = tcp_hdrlen(skb); 9619202d7b6SMaya Erez skb_net_hdr_len = skb_network_header_len(skb); 9629202d7b6SMaya Erez 9639202d7b6SMaya Erez /* First descriptor must contain the header only 9649202d7b6SMaya Erez * Header Length = MAC header len + IP header len + TCP header len 9659202d7b6SMaya Erez */ 9669202d7b6SMaya Erez hdrlen = ETH_HLEN + tcp_hdr_len + skb_net_hdr_len; 9679202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: process header descriptor, hdrlen %u\n", 9689202d7b6SMaya Erez hdrlen); 9699202d7b6SMaya Erez rc = wil_tx_tso_gen_desc(wil, skb->data, hdrlen, swhead, 9709202d7b6SMaya Erez wil_tso_type_hdr, NULL, ring, skb, 9719202d7b6SMaya Erez is_ipv4, tcp_hdr_len, skb_net_hdr_len, 9729202d7b6SMaya Erez mss, &descs_used); 9739202d7b6SMaya Erez if (rc) 9749202d7b6SMaya Erez return -EINVAL; 9759202d7b6SMaya Erez 9769202d7b6SMaya Erez /* Second descriptor contains the head */ 9779202d7b6SMaya Erez headlen = skb_headlen(skb) - hdrlen; 9789202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: process skb head, headlen %u\n", headlen); 9799202d7b6SMaya Erez rc = wil_tx_tso_gen_desc(wil, skb->data + hdrlen, headlen, 9809202d7b6SMaya Erez (swhead + descs_used) % ring->size, 9819202d7b6SMaya Erez (nr_frags != 0) ? wil_tso_type_first : 9829202d7b6SMaya Erez wil_tso_type_lst, NULL, ring, skb, 9839202d7b6SMaya Erez is_ipv4, tcp_hdr_len, skb_net_hdr_len, 9849202d7b6SMaya Erez mss, &descs_used); 9859202d7b6SMaya Erez if (rc) 9869202d7b6SMaya Erez goto mem_error; 9879202d7b6SMaya Erez 9889202d7b6SMaya Erez /* Rest of the descriptors are from the SKB fragments */ 9899202d7b6SMaya Erez for (f = 0; f < nr_frags; f++) { 9909202d7b6SMaya Erez skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 9919202d7b6SMaya Erez int len = frag->size; 9929202d7b6SMaya Erez 9939202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: frag[%d]: len %u, descs_used %d\n", f, 9949202d7b6SMaya Erez len, descs_used); 9959202d7b6SMaya Erez 9969202d7b6SMaya Erez rc = wil_tx_tso_gen_desc(wil, NULL, len, 9979202d7b6SMaya Erez (swhead + descs_used) % ring->size, 9989202d7b6SMaya Erez (f != nr_frags - 1) ? 9999202d7b6SMaya Erez wil_tso_type_mid : wil_tso_type_lst, 10009202d7b6SMaya Erez frag, ring, skb, is_ipv4, 10019202d7b6SMaya Erez tcp_hdr_len, skb_net_hdr_len, 10029202d7b6SMaya Erez mss, &descs_used); 10039202d7b6SMaya Erez if (rc) 10049202d7b6SMaya Erez goto mem_error; 10059202d7b6SMaya Erez } 10069202d7b6SMaya Erez 10079202d7b6SMaya Erez /* performance monitoring */ 10089202d7b6SMaya Erez used = wil_ring_used_tx(ring); 10099202d7b6SMaya Erez if (wil_val_in_range(wil->ring_idle_trsh, 10109202d7b6SMaya Erez used, used + descs_used)) { 10119202d7b6SMaya Erez txdata->idle += get_cycles() - txdata->last_idle; 10129202d7b6SMaya Erez wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 10139202d7b6SMaya Erez ring_index, used, used + descs_used); 10149202d7b6SMaya Erez } 10159202d7b6SMaya Erez 10169202d7b6SMaya Erez /* advance swhead */ 10179202d7b6SMaya Erez wil_ring_advance_head(ring, descs_used); 10189202d7b6SMaya Erez wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, ring->swhead); 10199202d7b6SMaya Erez 10209202d7b6SMaya Erez /* make sure all writes to descriptors (shared memory) are done before 10219202d7b6SMaya Erez * committing them to HW 10229202d7b6SMaya Erez */ 10239202d7b6SMaya Erez wmb(); 10249202d7b6SMaya Erez 10259202d7b6SMaya Erez wil_w(wil, ring->hwtail, ring->swhead); 10269202d7b6SMaya Erez 10279202d7b6SMaya Erez return 0; 10289202d7b6SMaya Erez 10299202d7b6SMaya Erez mem_error: 10309202d7b6SMaya Erez while (descs_used > 0) { 10319202d7b6SMaya Erez struct device *dev = wil_to_dev(wil); 10329202d7b6SMaya Erez struct wil_ctx *ctx; 10339202d7b6SMaya Erez int i = (swhead + descs_used - 1) % ring->size; 10349202d7b6SMaya Erez struct wil_tx_enhanced_desc dd, *d = ⅆ 10359202d7b6SMaya Erez struct wil_tx_enhanced_desc *_desc = 10369202d7b6SMaya Erez (struct wil_tx_enhanced_desc *) 10379202d7b6SMaya Erez &ring->va[i].tx.enhanced; 10389202d7b6SMaya Erez 10399202d7b6SMaya Erez *d = *_desc; 10409202d7b6SMaya Erez ctx = &ring->ctx[i]; 10419202d7b6SMaya Erez wil_tx_desc_unmap_edma(dev, (union wil_tx_desc *)d, ctx); 10429202d7b6SMaya Erez memset(ctx, 0, sizeof(*ctx)); 10439202d7b6SMaya Erez descs_used--; 10449202d7b6SMaya Erez } 10459202d7b6SMaya Erez return rc; 10469202d7b6SMaya Erez } 10479202d7b6SMaya Erez 104896c93589SGidon Studinski static int wil_ring_init_bcast_edma(struct wil6210_vif *vif, int ring_id, 104996c93589SGidon Studinski int size) 105096c93589SGidon Studinski { 105196c93589SGidon Studinski struct wil6210_priv *wil = vif_to_wil(vif); 105296c93589SGidon Studinski struct wil_ring *ring = &wil->ring_tx[ring_id]; 105396c93589SGidon Studinski int rc; 105496c93589SGidon Studinski struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 105596c93589SGidon Studinski 105696c93589SGidon Studinski wil_dbg_misc(wil, "init bcast: ring_id=%d, sring_id=%d\n", 105796c93589SGidon Studinski ring_id, wil->tx_sring_idx); 105896c93589SGidon Studinski 105996c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 106096c93589SGidon Studinski 106196c93589SGidon Studinski wil_tx_data_init(txdata); 106296c93589SGidon Studinski ring->size = size; 106396c93589SGidon Studinski ring->is_rx = false; 106496c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 106596c93589SGidon Studinski if (rc) 106696c93589SGidon Studinski goto out; 106796c93589SGidon Studinski 106896c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = WIL6210_MAX_CID; /* CID */ 106996c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = 0; /* TID */ 107096c93589SGidon Studinski if (!vif->privacy) 107196c93589SGidon Studinski txdata->dot1x_open = true; 107296c93589SGidon Studinski 107396c93589SGidon Studinski rc = wil_wmi_bcast_desc_ring_add(vif, ring_id); 107496c93589SGidon Studinski if (rc) 107596c93589SGidon Studinski goto out_free; 107696c93589SGidon Studinski 107796c93589SGidon Studinski return 0; 107896c93589SGidon Studinski 107996c93589SGidon Studinski out_free: 108096c93589SGidon Studinski spin_lock_bh(&txdata->lock); 108196c93589SGidon Studinski txdata->enabled = 0; 108296c93589SGidon Studinski txdata->dot1x_open = false; 108396c93589SGidon Studinski spin_unlock_bh(&txdata->lock); 108496c93589SGidon Studinski wil_ring_free_edma(wil, ring); 108596c93589SGidon Studinski 108696c93589SGidon Studinski out: 108796c93589SGidon Studinski return rc; 108896c93589SGidon Studinski } 108996c93589SGidon Studinski 109096c93589SGidon Studinski static void wil_tx_fini_edma(struct wil6210_priv *wil) 109196c93589SGidon Studinski { 109296c93589SGidon Studinski struct wil_status_ring *sring = &wil->srings[wil->tx_sring_idx]; 109396c93589SGidon Studinski 109496c93589SGidon Studinski wil_dbg_misc(wil, "free TX sring\n"); 109596c93589SGidon Studinski 109696c93589SGidon Studinski wil_sring_free(wil, sring); 109796c93589SGidon Studinski } 109896c93589SGidon Studinski 109996c93589SGidon Studinski static void wil_rx_data_free(struct wil_status_ring *sring) 110096c93589SGidon Studinski { 110196c93589SGidon Studinski if (!sring) 110296c93589SGidon Studinski return; 110396c93589SGidon Studinski 110496c93589SGidon Studinski kfree_skb(sring->rx_data.skb); 110596c93589SGidon Studinski sring->rx_data.skb = NULL; 110696c93589SGidon Studinski } 110796c93589SGidon Studinski 110896c93589SGidon Studinski static void wil_rx_fini_edma(struct wil6210_priv *wil) 110996c93589SGidon Studinski { 111096c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 111196c93589SGidon Studinski int i; 111296c93589SGidon Studinski 111396c93589SGidon Studinski wil_dbg_misc(wil, "rx_fini_edma\n"); 111496c93589SGidon Studinski 111596c93589SGidon Studinski wil_ring_free_edma(wil, ring); 111696c93589SGidon Studinski 111796c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 111896c93589SGidon Studinski wil_rx_data_free(&wil->srings[i]); 111996c93589SGidon Studinski wil_sring_free(wil, &wil->srings[i]); 112096c93589SGidon Studinski } 112196c93589SGidon Studinski 112296c93589SGidon Studinski wil_free_rx_buff_arr(wil); 112396c93589SGidon Studinski } 112496c93589SGidon Studinski 112596c93589SGidon Studinski void wil_init_txrx_ops_edma(struct wil6210_priv *wil) 112696c93589SGidon Studinski { 112796c93589SGidon Studinski wil->txrx_ops.configure_interrupt_moderation = 112896c93589SGidon Studinski wil_configure_interrupt_moderation_edma; 112996c93589SGidon Studinski /* TX ops */ 113096c93589SGidon Studinski wil->txrx_ops.ring_init_tx = wil_ring_init_tx_edma; 113196c93589SGidon Studinski wil->txrx_ops.ring_fini_tx = wil_ring_free_edma; 113296c93589SGidon Studinski wil->txrx_ops.ring_init_bcast = wil_ring_init_bcast_edma; 113396c93589SGidon Studinski wil->txrx_ops.tx_init = wil_tx_init_edma; 113496c93589SGidon Studinski wil->txrx_ops.tx_fini = wil_tx_fini_edma; 11359202d7b6SMaya Erez wil->txrx_ops.tx_desc_map = wil_tx_desc_map_edma; 11369202d7b6SMaya Erez wil->txrx_ops.tx_desc_unmap = wil_tx_desc_unmap_edma; 11379202d7b6SMaya Erez wil->txrx_ops.tx_ring_tso = __wil_tx_ring_tso_edma; 113896c93589SGidon Studinski /* RX ops */ 113996c93589SGidon Studinski wil->txrx_ops.rx_init = wil_rx_init_edma; 114096c93589SGidon Studinski wil->txrx_ops.rx_fini = wil_rx_fini_edma; 114196c93589SGidon Studinski } 114296c93589SGidon Studinski 1143