196c93589SGidon Studinski /* 296c93589SGidon Studinski * Copyright (c) 2012-2018 The Linux Foundation. All rights reserved. 396c93589SGidon Studinski * 496c93589SGidon Studinski * Permission to use, copy, modify, and/or distribute this software for any 596c93589SGidon Studinski * purpose with or without fee is hereby granted, provided that the above 696c93589SGidon Studinski * copyright notice and this permission notice appear in all copies. 796c93589SGidon Studinski * 896c93589SGidon Studinski * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 996c93589SGidon Studinski * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1096c93589SGidon Studinski * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1196c93589SGidon Studinski * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1296c93589SGidon Studinski * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1396c93589SGidon Studinski * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1496c93589SGidon Studinski * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1596c93589SGidon Studinski */ 1696c93589SGidon Studinski 1796c93589SGidon Studinski #include <linux/etherdevice.h> 1896c93589SGidon Studinski #include <linux/moduleparam.h> 1996c93589SGidon Studinski #include <linux/prefetch.h> 2096c93589SGidon Studinski #include <linux/types.h> 2196c93589SGidon Studinski #include <linux/list.h> 2296c93589SGidon Studinski #include <linux/ip.h> 2396c93589SGidon Studinski #include <linux/ipv6.h> 2496c93589SGidon Studinski #include "wil6210.h" 2596c93589SGidon Studinski #include "txrx_edma.h" 2696c93589SGidon Studinski #include "txrx.h" 2796c93589SGidon Studinski 2896c93589SGidon Studinski #define WIL_EDMA_MAX_DATA_OFFSET (2) 2996c93589SGidon Studinski 3096c93589SGidon Studinski static void wil_tx_desc_unmap_edma(struct device *dev, 3196c93589SGidon Studinski struct wil_tx_enhanced_desc *d, 3296c93589SGidon Studinski struct wil_ctx *ctx) 3396c93589SGidon Studinski { 3496c93589SGidon Studinski dma_addr_t pa = wil_tx_desc_get_addr_edma(&d->dma); 3596c93589SGidon Studinski u16 dmalen = le16_to_cpu(d->dma.length); 3696c93589SGidon Studinski 3796c93589SGidon Studinski switch (ctx->mapped_as) { 3896c93589SGidon Studinski case wil_mapped_as_single: 3996c93589SGidon Studinski dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE); 4096c93589SGidon Studinski break; 4196c93589SGidon Studinski case wil_mapped_as_page: 4296c93589SGidon Studinski dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE); 4396c93589SGidon Studinski break; 4496c93589SGidon Studinski default: 4596c93589SGidon Studinski break; 4696c93589SGidon Studinski } 4796c93589SGidon Studinski } 4896c93589SGidon Studinski 4996c93589SGidon Studinski static int wil_find_free_sring(struct wil6210_priv *wil) 5096c93589SGidon Studinski { 5196c93589SGidon Studinski int i; 5296c93589SGidon Studinski 5396c93589SGidon Studinski for (i = 0; i < WIL6210_MAX_STATUS_RINGS; i++) { 5496c93589SGidon Studinski if (!wil->srings[i].va) 5596c93589SGidon Studinski return i; 5696c93589SGidon Studinski } 5796c93589SGidon Studinski 5896c93589SGidon Studinski return -EINVAL; 5996c93589SGidon Studinski } 6096c93589SGidon Studinski 6196c93589SGidon Studinski static void wil_sring_free(struct wil6210_priv *wil, 6296c93589SGidon Studinski struct wil_status_ring *sring) 6396c93589SGidon Studinski { 6496c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 6596c93589SGidon Studinski size_t sz; 6696c93589SGidon Studinski 6796c93589SGidon Studinski if (!sring || !sring->va) 6896c93589SGidon Studinski return; 6996c93589SGidon Studinski 7096c93589SGidon Studinski sz = sring->elem_size * sring->size; 7196c93589SGidon Studinski 7296c93589SGidon Studinski wil_dbg_misc(wil, "status_ring_free, size(bytes)=%zu, 0x%p:%pad\n", 7396c93589SGidon Studinski sz, sring->va, &sring->pa); 7496c93589SGidon Studinski 7596c93589SGidon Studinski dma_free_coherent(dev, sz, (void *)sring->va, sring->pa); 7696c93589SGidon Studinski sring->pa = 0; 7796c93589SGidon Studinski sring->va = NULL; 7896c93589SGidon Studinski } 7996c93589SGidon Studinski 8096c93589SGidon Studinski static int wil_sring_alloc(struct wil6210_priv *wil, 8196c93589SGidon Studinski struct wil_status_ring *sring) 8296c93589SGidon Studinski { 8396c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 8496c93589SGidon Studinski size_t sz = sring->elem_size * sring->size; 8596c93589SGidon Studinski 8696c93589SGidon Studinski wil_dbg_misc(wil, "status_ring_alloc: size=%zu\n", sz); 8796c93589SGidon Studinski 8896c93589SGidon Studinski if (sz == 0) { 8996c93589SGidon Studinski wil_err(wil, "Cannot allocate a zero size status ring\n"); 9096c93589SGidon Studinski return -EINVAL; 9196c93589SGidon Studinski } 9296c93589SGidon Studinski 9396c93589SGidon Studinski sring->swhead = 0; 9496c93589SGidon Studinski 9596c93589SGidon Studinski /* Status messages are allocated and initialized to 0. This is necessary 9696c93589SGidon Studinski * since DR bit should be initialized to 0. 9796c93589SGidon Studinski */ 9896c93589SGidon Studinski sring->va = dma_zalloc_coherent(dev, sz, &sring->pa, GFP_KERNEL); 9996c93589SGidon Studinski if (!sring->va) 10096c93589SGidon Studinski return -ENOMEM; 10196c93589SGidon Studinski 10296c93589SGidon Studinski wil_dbg_misc(wil, "status_ring[%d] 0x%p:%pad\n", sring->size, sring->va, 10396c93589SGidon Studinski &sring->pa); 10496c93589SGidon Studinski 10596c93589SGidon Studinski return 0; 10696c93589SGidon Studinski } 10796c93589SGidon Studinski 10896c93589SGidon Studinski static int wil_tx_init_edma(struct wil6210_priv *wil) 10996c93589SGidon Studinski { 11096c93589SGidon Studinski int ring_id = wil_find_free_sring(wil); 11196c93589SGidon Studinski struct wil_status_ring *sring; 11296c93589SGidon Studinski int rc; 11396c93589SGidon Studinski u16 status_ring_size; 11496c93589SGidon Studinski 11596c93589SGidon Studinski if (wil->tx_status_ring_order < WIL_SRING_SIZE_ORDER_MIN || 11696c93589SGidon Studinski wil->tx_status_ring_order > WIL_SRING_SIZE_ORDER_MAX) 11796c93589SGidon Studinski wil->tx_status_ring_order = WIL_TX_SRING_SIZE_ORDER_DEFAULT; 11896c93589SGidon Studinski 11996c93589SGidon Studinski status_ring_size = 1 << wil->tx_status_ring_order; 12096c93589SGidon Studinski 12196c93589SGidon Studinski wil_dbg_misc(wil, "init TX sring: size=%u, ring_id=%u\n", 12296c93589SGidon Studinski status_ring_size, ring_id); 12396c93589SGidon Studinski 12496c93589SGidon Studinski if (ring_id < 0) 12596c93589SGidon Studinski return ring_id; 12696c93589SGidon Studinski 12796c93589SGidon Studinski /* Allocate Tx status ring. Tx descriptor rings will be 12896c93589SGidon Studinski * allocated on WMI connect event 12996c93589SGidon Studinski */ 13096c93589SGidon Studinski sring = &wil->srings[ring_id]; 13196c93589SGidon Studinski 13296c93589SGidon Studinski sring->is_rx = false; 13396c93589SGidon Studinski sring->size = status_ring_size; 13496c93589SGidon Studinski sring->elem_size = sizeof(struct wil_ring_tx_status); 13596c93589SGidon Studinski rc = wil_sring_alloc(wil, sring); 13696c93589SGidon Studinski if (rc) 13796c93589SGidon Studinski return rc; 13896c93589SGidon Studinski 13996c93589SGidon Studinski rc = wil_wmi_tx_sring_cfg(wil, ring_id); 14096c93589SGidon Studinski if (rc) 14196c93589SGidon Studinski goto out_free; 14296c93589SGidon Studinski 14396c93589SGidon Studinski sring->desc_rdy_pol = 1; 14496c93589SGidon Studinski wil->tx_sring_idx = ring_id; 14596c93589SGidon Studinski 14696c93589SGidon Studinski return 0; 14796c93589SGidon Studinski out_free: 14896c93589SGidon Studinski wil_sring_free(wil, sring); 14996c93589SGidon Studinski return rc; 15096c93589SGidon Studinski } 15196c93589SGidon Studinski 15296c93589SGidon Studinski /** 15396c93589SGidon Studinski * Allocate one skb for Rx descriptor RING 15496c93589SGidon Studinski */ 15596c93589SGidon Studinski static int wil_ring_alloc_skb_edma(struct wil6210_priv *wil, 15696c93589SGidon Studinski struct wil_ring *ring, u32 i) 15796c93589SGidon Studinski { 15896c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 15996c93589SGidon Studinski unsigned int sz = wil->rx_buf_len + ETH_HLEN + 16096c93589SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 16196c93589SGidon Studinski dma_addr_t pa; 16296c93589SGidon Studinski u16 buff_id; 16396c93589SGidon Studinski struct list_head *active = &wil->rx_buff_mgmt.active; 16496c93589SGidon Studinski struct list_head *free = &wil->rx_buff_mgmt.free; 16596c93589SGidon Studinski struct wil_rx_buff *rx_buff; 16696c93589SGidon Studinski struct wil_rx_buff *buff_arr = wil->rx_buff_mgmt.buff_arr; 16796c93589SGidon Studinski struct sk_buff *skb; 16896c93589SGidon Studinski struct wil_rx_enhanced_desc dd, *d = ⅆ 16996c93589SGidon Studinski struct wil_rx_enhanced_desc *_d = (struct wil_rx_enhanced_desc *) 17096c93589SGidon Studinski &ring->va[i].rx.enhanced; 17196c93589SGidon Studinski 17296c93589SGidon Studinski if (unlikely(list_empty(free))) { 17396c93589SGidon Studinski wil->rx_buff_mgmt.free_list_empty_cnt++; 17496c93589SGidon Studinski return -EAGAIN; 17596c93589SGidon Studinski } 17696c93589SGidon Studinski 17796c93589SGidon Studinski skb = dev_alloc_skb(sz); 17896c93589SGidon Studinski if (unlikely(!skb)) 17996c93589SGidon Studinski return -ENOMEM; 18096c93589SGidon Studinski 18196c93589SGidon Studinski skb_put(skb, sz); 18296c93589SGidon Studinski 18396c93589SGidon Studinski pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 18496c93589SGidon Studinski if (unlikely(dma_mapping_error(dev, pa))) { 18596c93589SGidon Studinski kfree_skb(skb); 18696c93589SGidon Studinski return -ENOMEM; 18796c93589SGidon Studinski } 18896c93589SGidon Studinski 18996c93589SGidon Studinski /* Get the buffer ID - the index of the rx buffer in the buff_arr */ 19096c93589SGidon Studinski rx_buff = list_first_entry(free, struct wil_rx_buff, list); 19196c93589SGidon Studinski buff_id = rx_buff->id; 19296c93589SGidon Studinski 19396c93589SGidon Studinski /* Move a buffer from the free list to the active list */ 19496c93589SGidon Studinski list_move(&rx_buff->list, active); 19596c93589SGidon Studinski 19696c93589SGidon Studinski buff_arr[buff_id].skb = skb; 19796c93589SGidon Studinski 19896c93589SGidon Studinski wil_desc_set_addr_edma(&d->dma.addr, &d->dma.addr_high_high, pa); 19996c93589SGidon Studinski d->dma.length = cpu_to_le16(sz); 20096c93589SGidon Studinski d->mac.buff_id = cpu_to_le16(buff_id); 20196c93589SGidon Studinski *_d = *d; 20296c93589SGidon Studinski 20396c93589SGidon Studinski /* Save the physical address in skb->cb for later use in dma_unmap */ 20496c93589SGidon Studinski memcpy(skb->cb, &pa, sizeof(pa)); 20596c93589SGidon Studinski 20696c93589SGidon Studinski return 0; 20796c93589SGidon Studinski } 20896c93589SGidon Studinski 20996c93589SGidon Studinski static int wil_rx_refill_edma(struct wil6210_priv *wil) 21096c93589SGidon Studinski { 21196c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 21296c93589SGidon Studinski u32 next_head; 21396c93589SGidon Studinski int rc = 0; 21496c93589SGidon Studinski u32 swtail = *ring->edma_rx_swtail.va; 21596c93589SGidon Studinski 21696c93589SGidon Studinski for (; next_head = wil_ring_next_head(ring), (next_head != swtail); 21796c93589SGidon Studinski ring->swhead = next_head) { 21896c93589SGidon Studinski rc = wil_ring_alloc_skb_edma(wil, ring, ring->swhead); 21996c93589SGidon Studinski if (unlikely(rc)) { 22096c93589SGidon Studinski if (rc == -EAGAIN) 22196c93589SGidon Studinski wil_dbg_txrx(wil, "No free buffer ID found\n"); 22296c93589SGidon Studinski else 22396c93589SGidon Studinski wil_err_ratelimited(wil, 22496c93589SGidon Studinski "Error %d in refill desc[%d]\n", 22596c93589SGidon Studinski rc, ring->swhead); 22696c93589SGidon Studinski break; 22796c93589SGidon Studinski } 22896c93589SGidon Studinski } 22996c93589SGidon Studinski 23096c93589SGidon Studinski /* make sure all writes to descriptors (shared memory) are done before 23196c93589SGidon Studinski * committing them to HW 23296c93589SGidon Studinski */ 23396c93589SGidon Studinski wmb(); 23496c93589SGidon Studinski 23596c93589SGidon Studinski wil_w(wil, ring->hwtail, ring->swhead); 23696c93589SGidon Studinski 23796c93589SGidon Studinski return rc; 23896c93589SGidon Studinski } 23996c93589SGidon Studinski 24096c93589SGidon Studinski static void wil_move_all_rx_buff_to_free_list(struct wil6210_priv *wil, 24196c93589SGidon Studinski struct wil_ring *ring) 24296c93589SGidon Studinski { 24396c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 24496c93589SGidon Studinski u32 next_tail; 24596c93589SGidon Studinski u32 swhead = (ring->swhead + 1) % ring->size; 24696c93589SGidon Studinski dma_addr_t pa; 24796c93589SGidon Studinski u16 dmalen; 24896c93589SGidon Studinski 24996c93589SGidon Studinski for (; next_tail = wil_ring_next_tail(ring), (next_tail != swhead); 25096c93589SGidon Studinski ring->swtail = next_tail) { 25196c93589SGidon Studinski struct wil_rx_enhanced_desc dd, *d = ⅆ 25296c93589SGidon Studinski struct wil_rx_enhanced_desc *_d = 25396c93589SGidon Studinski (struct wil_rx_enhanced_desc *) 25496c93589SGidon Studinski &ring->va[ring->swtail].rx.enhanced; 25596c93589SGidon Studinski struct sk_buff *skb; 25696c93589SGidon Studinski u16 buff_id; 25796c93589SGidon Studinski 25896c93589SGidon Studinski *d = *_d; 25996c93589SGidon Studinski pa = wil_rx_desc_get_addr_edma(&d->dma); 26096c93589SGidon Studinski dmalen = le16_to_cpu(d->dma.length); 26196c93589SGidon Studinski dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE); 26296c93589SGidon Studinski 26396c93589SGidon Studinski /* Extract the SKB from the rx_buff management array */ 26496c93589SGidon Studinski buff_id = __le16_to_cpu(d->mac.buff_id); 26596c93589SGidon Studinski if (buff_id >= wil->rx_buff_mgmt.size) { 26696c93589SGidon Studinski wil_err(wil, "invalid buff_id %d\n", buff_id); 26796c93589SGidon Studinski continue; 26896c93589SGidon Studinski } 26996c93589SGidon Studinski skb = wil->rx_buff_mgmt.buff_arr[buff_id].skb; 27096c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr[buff_id].skb = NULL; 27196c93589SGidon Studinski if (unlikely(!skb)) 27296c93589SGidon Studinski wil_err(wil, "No Rx skb at buff_id %d\n", buff_id); 27396c93589SGidon Studinski else 27496c93589SGidon Studinski kfree_skb(skb); 27596c93589SGidon Studinski 27696c93589SGidon Studinski /* Move the buffer from the active to the free list */ 27796c93589SGidon Studinski list_move(&wil->rx_buff_mgmt.buff_arr[buff_id].list, 27896c93589SGidon Studinski &wil->rx_buff_mgmt.free); 27996c93589SGidon Studinski } 28096c93589SGidon Studinski } 28196c93589SGidon Studinski 28296c93589SGidon Studinski static void wil_free_rx_buff_arr(struct wil6210_priv *wil) 28396c93589SGidon Studinski { 28496c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 28596c93589SGidon Studinski 28696c93589SGidon Studinski if (!wil->rx_buff_mgmt.buff_arr) 28796c93589SGidon Studinski return; 28896c93589SGidon Studinski 28996c93589SGidon Studinski /* Move all the buffers to the free list in case active list is 29096c93589SGidon Studinski * not empty in order to release all SKBs before deleting the array 29196c93589SGidon Studinski */ 29296c93589SGidon Studinski wil_move_all_rx_buff_to_free_list(wil, ring); 29396c93589SGidon Studinski 29496c93589SGidon Studinski kfree(wil->rx_buff_mgmt.buff_arr); 29596c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr = NULL; 29696c93589SGidon Studinski } 29796c93589SGidon Studinski 29896c93589SGidon Studinski static int wil_init_rx_buff_arr(struct wil6210_priv *wil, 29996c93589SGidon Studinski size_t size) 30096c93589SGidon Studinski { 30196c93589SGidon Studinski struct wil_rx_buff *buff_arr; 30296c93589SGidon Studinski struct list_head *active = &wil->rx_buff_mgmt.active; 30396c93589SGidon Studinski struct list_head *free = &wil->rx_buff_mgmt.free; 30496c93589SGidon Studinski int i; 30596c93589SGidon Studinski 30696c93589SGidon Studinski wil->rx_buff_mgmt.buff_arr = kcalloc(size, sizeof(struct wil_rx_buff), 30796c93589SGidon Studinski GFP_KERNEL); 30896c93589SGidon Studinski if (!wil->rx_buff_mgmt.buff_arr) 30996c93589SGidon Studinski return -ENOMEM; 31096c93589SGidon Studinski 31196c93589SGidon Studinski /* Set list heads */ 31296c93589SGidon Studinski INIT_LIST_HEAD(active); 31396c93589SGidon Studinski INIT_LIST_HEAD(free); 31496c93589SGidon Studinski 31596c93589SGidon Studinski /* Linkify the list */ 31696c93589SGidon Studinski buff_arr = wil->rx_buff_mgmt.buff_arr; 31796c93589SGidon Studinski for (i = 0; i < size; i++) { 31896c93589SGidon Studinski list_add(&buff_arr[i].list, free); 31996c93589SGidon Studinski buff_arr[i].id = i; 32096c93589SGidon Studinski } 32196c93589SGidon Studinski 32296c93589SGidon Studinski wil->rx_buff_mgmt.size = size; 32396c93589SGidon Studinski 32496c93589SGidon Studinski return 0; 32596c93589SGidon Studinski } 32696c93589SGidon Studinski 32796c93589SGidon Studinski static int wil_init_rx_sring(struct wil6210_priv *wil, 32896c93589SGidon Studinski u16 status_ring_size, 32996c93589SGidon Studinski size_t elem_size, 33096c93589SGidon Studinski u16 ring_id) 33196c93589SGidon Studinski { 33296c93589SGidon Studinski struct wil_status_ring *sring = &wil->srings[ring_id]; 33396c93589SGidon Studinski int rc; 33496c93589SGidon Studinski 33596c93589SGidon Studinski wil_dbg_misc(wil, "init RX sring: size=%u, ring_id=%u\n", sring->size, 33696c93589SGidon Studinski ring_id); 33796c93589SGidon Studinski 33896c93589SGidon Studinski memset(&sring->rx_data, 0, sizeof(sring->rx_data)); 33996c93589SGidon Studinski 34096c93589SGidon Studinski sring->is_rx = true; 34196c93589SGidon Studinski sring->size = status_ring_size; 34296c93589SGidon Studinski sring->elem_size = elem_size; 34396c93589SGidon Studinski rc = wil_sring_alloc(wil, sring); 34496c93589SGidon Studinski if (rc) 34596c93589SGidon Studinski return rc; 34696c93589SGidon Studinski 34796c93589SGidon Studinski rc = wil_wmi_rx_sring_add(wil, ring_id); 34896c93589SGidon Studinski if (rc) 34996c93589SGidon Studinski goto out_free; 35096c93589SGidon Studinski 35196c93589SGidon Studinski sring->desc_rdy_pol = 1; 35296c93589SGidon Studinski 35396c93589SGidon Studinski return 0; 35496c93589SGidon Studinski out_free: 35596c93589SGidon Studinski wil_sring_free(wil, sring); 35696c93589SGidon Studinski return rc; 35796c93589SGidon Studinski } 35896c93589SGidon Studinski 35996c93589SGidon Studinski static int wil_ring_alloc_desc_ring(struct wil6210_priv *wil, 36096c93589SGidon Studinski struct wil_ring *ring) 36196c93589SGidon Studinski { 36296c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 36396c93589SGidon Studinski size_t sz = ring->size * sizeof(ring->va[0]); 36496c93589SGidon Studinski 36596c93589SGidon Studinski wil_dbg_misc(wil, "alloc_desc_ring:\n"); 36696c93589SGidon Studinski 36796c93589SGidon Studinski BUILD_BUG_ON(sizeof(ring->va[0]) != 32); 36896c93589SGidon Studinski 36996c93589SGidon Studinski ring->swhead = 0; 37096c93589SGidon Studinski ring->swtail = 0; 37196c93589SGidon Studinski ring->ctx = kcalloc(ring->size, sizeof(ring->ctx[0]), GFP_KERNEL); 37296c93589SGidon Studinski if (!ring->ctx) 37396c93589SGidon Studinski goto err; 37496c93589SGidon Studinski 37596c93589SGidon Studinski ring->va = dma_zalloc_coherent(dev, sz, &ring->pa, GFP_KERNEL); 37696c93589SGidon Studinski if (!ring->va) 37796c93589SGidon Studinski goto err_free_ctx; 37896c93589SGidon Studinski 37996c93589SGidon Studinski if (ring->is_rx) { 38096c93589SGidon Studinski sz = sizeof(*ring->edma_rx_swtail.va); 38196c93589SGidon Studinski ring->edma_rx_swtail.va = 38296c93589SGidon Studinski dma_zalloc_coherent(dev, sz, &ring->edma_rx_swtail.pa, 38396c93589SGidon Studinski GFP_KERNEL); 38496c93589SGidon Studinski if (!ring->edma_rx_swtail.va) 38596c93589SGidon Studinski goto err_free_va; 38696c93589SGidon Studinski } 38796c93589SGidon Studinski 38896c93589SGidon Studinski wil_dbg_misc(wil, "%s ring[%d] 0x%p:%pad 0x%p\n", 38996c93589SGidon Studinski ring->is_rx ? "RX" : "TX", 39096c93589SGidon Studinski ring->size, ring->va, &ring->pa, ring->ctx); 39196c93589SGidon Studinski 39296c93589SGidon Studinski return 0; 39396c93589SGidon Studinski err_free_va: 39496c93589SGidon Studinski dma_free_coherent(dev, ring->size * sizeof(ring->va[0]), 39596c93589SGidon Studinski (void *)ring->va, ring->pa); 39696c93589SGidon Studinski ring->va = NULL; 39796c93589SGidon Studinski err_free_ctx: 39896c93589SGidon Studinski kfree(ring->ctx); 39996c93589SGidon Studinski ring->ctx = NULL; 40096c93589SGidon Studinski err: 40196c93589SGidon Studinski return -ENOMEM; 40296c93589SGidon Studinski } 40396c93589SGidon Studinski 40496c93589SGidon Studinski static void wil_ring_free_edma(struct wil6210_priv *wil, struct wil_ring *ring) 40596c93589SGidon Studinski { 40696c93589SGidon Studinski struct device *dev = wil_to_dev(wil); 40796c93589SGidon Studinski size_t sz; 40896c93589SGidon Studinski int ring_index = 0; 40996c93589SGidon Studinski 41096c93589SGidon Studinski if (!ring->va) 41196c93589SGidon Studinski return; 41296c93589SGidon Studinski 41396c93589SGidon Studinski sz = ring->size * sizeof(ring->va[0]); 41496c93589SGidon Studinski 41596c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 41696c93589SGidon Studinski if (ring->is_rx) { 41796c93589SGidon Studinski wil_dbg_misc(wil, "free Rx ring [%d] 0x%p:%pad 0x%p\n", 41896c93589SGidon Studinski ring->size, ring->va, 41996c93589SGidon Studinski &ring->pa, ring->ctx); 42096c93589SGidon Studinski 42196c93589SGidon Studinski wil_move_all_rx_buff_to_free_list(wil, ring); 42296c93589SGidon Studinski goto out; 42396c93589SGidon Studinski } 42496c93589SGidon Studinski 42596c93589SGidon Studinski /* TX ring */ 42696c93589SGidon Studinski ring_index = ring - wil->ring_tx; 42796c93589SGidon Studinski 42896c93589SGidon Studinski wil_dbg_misc(wil, "free Tx ring %d [%d] 0x%p:%pad 0x%p\n", 42996c93589SGidon Studinski ring_index, ring->size, ring->va, 43096c93589SGidon Studinski &ring->pa, ring->ctx); 43196c93589SGidon Studinski 43296c93589SGidon Studinski while (!wil_ring_is_empty(ring)) { 43396c93589SGidon Studinski struct wil_ctx *ctx; 43496c93589SGidon Studinski 43596c93589SGidon Studinski struct wil_tx_enhanced_desc dd, *d = ⅆ 43696c93589SGidon Studinski struct wil_tx_enhanced_desc *_d = 43796c93589SGidon Studinski (struct wil_tx_enhanced_desc *) 43896c93589SGidon Studinski &ring->va[ring->swtail].tx.enhanced; 43996c93589SGidon Studinski 44096c93589SGidon Studinski ctx = &ring->ctx[ring->swtail]; 44196c93589SGidon Studinski if (!ctx) { 44296c93589SGidon Studinski wil_dbg_txrx(wil, 44396c93589SGidon Studinski "ctx(%d) was already completed\n", 44496c93589SGidon Studinski ring->swtail); 44596c93589SGidon Studinski ring->swtail = wil_ring_next_tail(ring); 44696c93589SGidon Studinski continue; 44796c93589SGidon Studinski } 44896c93589SGidon Studinski *d = *_d; 44996c93589SGidon Studinski wil_tx_desc_unmap_edma(dev, d, ctx); 45096c93589SGidon Studinski if (ctx->skb) 45196c93589SGidon Studinski dev_kfree_skb_any(ctx->skb); 45296c93589SGidon Studinski ring->swtail = wil_ring_next_tail(ring); 45396c93589SGidon Studinski } 45496c93589SGidon Studinski 45596c93589SGidon Studinski out: 45696c93589SGidon Studinski dma_free_coherent(dev, sz, (void *)ring->va, ring->pa); 45796c93589SGidon Studinski kfree(ring->ctx); 45896c93589SGidon Studinski ring->pa = 0; 45996c93589SGidon Studinski ring->va = NULL; 46096c93589SGidon Studinski ring->ctx = NULL; 46196c93589SGidon Studinski } 46296c93589SGidon Studinski 46396c93589SGidon Studinski static int wil_init_rx_desc_ring(struct wil6210_priv *wil, u16 desc_ring_size, 46496c93589SGidon Studinski int status_ring_id) 46596c93589SGidon Studinski { 46696c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 46796c93589SGidon Studinski int rc; 46896c93589SGidon Studinski 46996c93589SGidon Studinski wil_dbg_misc(wil, "init RX desc ring\n"); 47096c93589SGidon Studinski 47196c93589SGidon Studinski ring->size = desc_ring_size; 47296c93589SGidon Studinski ring->is_rx = true; 47396c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 47496c93589SGidon Studinski if (rc) 47596c93589SGidon Studinski return rc; 47696c93589SGidon Studinski 47796c93589SGidon Studinski rc = wil_wmi_rx_desc_ring_add(wil, status_ring_id); 47896c93589SGidon Studinski if (rc) 47996c93589SGidon Studinski goto out_free; 48096c93589SGidon Studinski 48196c93589SGidon Studinski return 0; 48296c93589SGidon Studinski out_free: 48396c93589SGidon Studinski wil_ring_free_edma(wil, ring); 48496c93589SGidon Studinski return rc; 48596c93589SGidon Studinski } 48696c93589SGidon Studinski 48796c93589SGidon Studinski static void wil_rx_buf_len_init_edma(struct wil6210_priv *wil) 48896c93589SGidon Studinski { 48996c93589SGidon Studinski wil->rx_buf_len = rx_large_buf ? 49096c93589SGidon Studinski WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD; 49196c93589SGidon Studinski } 49296c93589SGidon Studinski 49396c93589SGidon Studinski static int wil_rx_init_edma(struct wil6210_priv *wil, u16 desc_ring_size) 49496c93589SGidon Studinski { 49596c93589SGidon Studinski u16 status_ring_size; 49696c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 49796c93589SGidon Studinski int rc; 49896c93589SGidon Studinski size_t elem_size = wil->use_compressed_rx_status ? 49996c93589SGidon Studinski sizeof(struct wil_rx_status_compressed) : 50096c93589SGidon Studinski sizeof(struct wil_rx_status_extended); 50196c93589SGidon Studinski int i; 50296c93589SGidon Studinski u16 max_rx_pl_per_desc; 50396c93589SGidon Studinski 50496c93589SGidon Studinski if (wil->rx_status_ring_order < WIL_SRING_SIZE_ORDER_MIN || 50596c93589SGidon Studinski wil->rx_status_ring_order > WIL_SRING_SIZE_ORDER_MAX) 50696c93589SGidon Studinski wil->rx_status_ring_order = WIL_RX_SRING_SIZE_ORDER_DEFAULT; 50796c93589SGidon Studinski 50896c93589SGidon Studinski status_ring_size = 1 << wil->rx_status_ring_order; 50996c93589SGidon Studinski 51096c93589SGidon Studinski wil_dbg_misc(wil, 51196c93589SGidon Studinski "rx_init, desc_ring_size=%u, status_ring_size=%u, elem_size=%zu\n", 51296c93589SGidon Studinski desc_ring_size, status_ring_size, elem_size); 51396c93589SGidon Studinski 51496c93589SGidon Studinski wil_rx_buf_len_init_edma(wil); 51596c93589SGidon Studinski 51696c93589SGidon Studinski max_rx_pl_per_desc = wil->rx_buf_len + ETH_HLEN + 51796c93589SGidon Studinski WIL_EDMA_MAX_DATA_OFFSET; 51896c93589SGidon Studinski 51996c93589SGidon Studinski /* Use debugfs dbg_num_rx_srings if set, reserve one sring for TX */ 52096c93589SGidon Studinski if (wil->num_rx_status_rings > WIL6210_MAX_STATUS_RINGS - 1) 52196c93589SGidon Studinski wil->num_rx_status_rings = WIL6210_MAX_STATUS_RINGS - 1; 52296c93589SGidon Studinski 52396c93589SGidon Studinski wil_dbg_misc(wil, "rx_init: allocate %d status rings\n", 52496c93589SGidon Studinski wil->num_rx_status_rings); 52596c93589SGidon Studinski 52696c93589SGidon Studinski rc = wil_wmi_cfg_def_rx_offload(wil, max_rx_pl_per_desc); 52796c93589SGidon Studinski if (rc) 52896c93589SGidon Studinski return rc; 52996c93589SGidon Studinski 53096c93589SGidon Studinski /* Allocate status ring */ 53196c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 53296c93589SGidon Studinski int sring_id = wil_find_free_sring(wil); 53396c93589SGidon Studinski 53496c93589SGidon Studinski if (sring_id < 0) { 53596c93589SGidon Studinski rc = -EFAULT; 53696c93589SGidon Studinski goto err_free_status; 53796c93589SGidon Studinski } 53896c93589SGidon Studinski rc = wil_init_rx_sring(wil, status_ring_size, elem_size, 53996c93589SGidon Studinski sring_id); 54096c93589SGidon Studinski if (rc) 54196c93589SGidon Studinski goto err_free_status; 54296c93589SGidon Studinski } 54396c93589SGidon Studinski 54496c93589SGidon Studinski /* Allocate descriptor ring */ 54596c93589SGidon Studinski rc = wil_init_rx_desc_ring(wil, desc_ring_size, 54696c93589SGidon Studinski WIL_DEFAULT_RX_STATUS_RING_ID); 54796c93589SGidon Studinski if (rc) 54896c93589SGidon Studinski goto err_free_status; 54996c93589SGidon Studinski 55096c93589SGidon Studinski if (wil->rx_buff_id_count >= status_ring_size) { 55196c93589SGidon Studinski wil_info(wil, 55296c93589SGidon Studinski "rx_buff_id_count %d exceeds sring_size %d. set it to %d\n", 55396c93589SGidon Studinski wil->rx_buff_id_count, status_ring_size, 55496c93589SGidon Studinski status_ring_size - 1); 55596c93589SGidon Studinski wil->rx_buff_id_count = status_ring_size - 1; 55696c93589SGidon Studinski } 55796c93589SGidon Studinski 55896c93589SGidon Studinski /* Allocate Rx buffer array */ 55996c93589SGidon Studinski rc = wil_init_rx_buff_arr(wil, wil->rx_buff_id_count); 56096c93589SGidon Studinski if (rc) 56196c93589SGidon Studinski goto err_free_desc; 56296c93589SGidon Studinski 56396c93589SGidon Studinski /* Fill descriptor ring with credits */ 56496c93589SGidon Studinski rc = wil_rx_refill_edma(wil); 56596c93589SGidon Studinski if (rc) 56696c93589SGidon Studinski goto err_free_rx_buff_arr; 56796c93589SGidon Studinski 56896c93589SGidon Studinski return 0; 56996c93589SGidon Studinski err_free_rx_buff_arr: 57096c93589SGidon Studinski wil_free_rx_buff_arr(wil); 57196c93589SGidon Studinski err_free_desc: 57296c93589SGidon Studinski wil_ring_free_edma(wil, ring); 57396c93589SGidon Studinski err_free_status: 57496c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) 57596c93589SGidon Studinski wil_sring_free(wil, &wil->srings[i]); 57696c93589SGidon Studinski 57796c93589SGidon Studinski return rc; 57896c93589SGidon Studinski } 57996c93589SGidon Studinski 58096c93589SGidon Studinski static int wil_ring_init_tx_edma(struct wil6210_vif *vif, int ring_id, 58196c93589SGidon Studinski int size, int cid, int tid) 58296c93589SGidon Studinski { 58396c93589SGidon Studinski struct wil6210_priv *wil = vif_to_wil(vif); 58496c93589SGidon Studinski int rc; 58596c93589SGidon Studinski struct wil_ring *ring = &wil->ring_tx[ring_id]; 58696c93589SGidon Studinski struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 58796c93589SGidon Studinski 58896c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 58996c93589SGidon Studinski 59096c93589SGidon Studinski wil_dbg_misc(wil, 59196c93589SGidon Studinski "init TX ring: ring_id=%u, cid=%u, tid=%u, sring_id=%u\n", 59296c93589SGidon Studinski ring_id, cid, tid, wil->tx_sring_idx); 59396c93589SGidon Studinski 59496c93589SGidon Studinski wil_tx_data_init(txdata); 59596c93589SGidon Studinski ring->size = size; 59696c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 59796c93589SGidon Studinski if (rc) 59896c93589SGidon Studinski goto out; 59996c93589SGidon Studinski 60096c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = cid; 60196c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = tid; 60296c93589SGidon Studinski if (!vif->privacy) 60396c93589SGidon Studinski txdata->dot1x_open = true; 60496c93589SGidon Studinski 60596c93589SGidon Studinski rc = wil_wmi_tx_desc_ring_add(vif, ring_id, cid, tid); 60696c93589SGidon Studinski if (rc) { 60796c93589SGidon Studinski wil_err(wil, "WMI_TX_DESC_RING_ADD_CMD failed\n"); 60896c93589SGidon Studinski goto out_free; 60996c93589SGidon Studinski } 61096c93589SGidon Studinski 61196c93589SGidon Studinski if (txdata->dot1x_open && agg_wsize >= 0) 61296c93589SGidon Studinski wil_addba_tx_request(wil, ring_id, agg_wsize); 61396c93589SGidon Studinski 61496c93589SGidon Studinski return 0; 61596c93589SGidon Studinski out_free: 61696c93589SGidon Studinski spin_lock_bh(&txdata->lock); 61796c93589SGidon Studinski txdata->dot1x_open = false; 61896c93589SGidon Studinski txdata->enabled = 0; 61996c93589SGidon Studinski spin_unlock_bh(&txdata->lock); 62096c93589SGidon Studinski wil_ring_free_edma(wil, ring); 62196c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = WIL6210_MAX_CID; 62296c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = 0; 62396c93589SGidon Studinski 62496c93589SGidon Studinski out: 62596c93589SGidon Studinski return rc; 62696c93589SGidon Studinski } 62796c93589SGidon Studinski 62896c93589SGidon Studinski static int wil_ring_init_bcast_edma(struct wil6210_vif *vif, int ring_id, 62996c93589SGidon Studinski int size) 63096c93589SGidon Studinski { 63196c93589SGidon Studinski struct wil6210_priv *wil = vif_to_wil(vif); 63296c93589SGidon Studinski struct wil_ring *ring = &wil->ring_tx[ring_id]; 63396c93589SGidon Studinski int rc; 63496c93589SGidon Studinski struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 63596c93589SGidon Studinski 63696c93589SGidon Studinski wil_dbg_misc(wil, "init bcast: ring_id=%d, sring_id=%d\n", 63796c93589SGidon Studinski ring_id, wil->tx_sring_idx); 63896c93589SGidon Studinski 63996c93589SGidon Studinski lockdep_assert_held(&wil->mutex); 64096c93589SGidon Studinski 64196c93589SGidon Studinski wil_tx_data_init(txdata); 64296c93589SGidon Studinski ring->size = size; 64396c93589SGidon Studinski ring->is_rx = false; 64496c93589SGidon Studinski rc = wil_ring_alloc_desc_ring(wil, ring); 64596c93589SGidon Studinski if (rc) 64696c93589SGidon Studinski goto out; 64796c93589SGidon Studinski 64896c93589SGidon Studinski wil->ring2cid_tid[ring_id][0] = WIL6210_MAX_CID; /* CID */ 64996c93589SGidon Studinski wil->ring2cid_tid[ring_id][1] = 0; /* TID */ 65096c93589SGidon Studinski if (!vif->privacy) 65196c93589SGidon Studinski txdata->dot1x_open = true; 65296c93589SGidon Studinski 65396c93589SGidon Studinski rc = wil_wmi_bcast_desc_ring_add(vif, ring_id); 65496c93589SGidon Studinski if (rc) 65596c93589SGidon Studinski goto out_free; 65696c93589SGidon Studinski 65796c93589SGidon Studinski return 0; 65896c93589SGidon Studinski 65996c93589SGidon Studinski out_free: 66096c93589SGidon Studinski spin_lock_bh(&txdata->lock); 66196c93589SGidon Studinski txdata->enabled = 0; 66296c93589SGidon Studinski txdata->dot1x_open = false; 66396c93589SGidon Studinski spin_unlock_bh(&txdata->lock); 66496c93589SGidon Studinski wil_ring_free_edma(wil, ring); 66596c93589SGidon Studinski 66696c93589SGidon Studinski out: 66796c93589SGidon Studinski return rc; 66896c93589SGidon Studinski } 66996c93589SGidon Studinski 67096c93589SGidon Studinski static void wil_tx_fini_edma(struct wil6210_priv *wil) 67196c93589SGidon Studinski { 67296c93589SGidon Studinski struct wil_status_ring *sring = &wil->srings[wil->tx_sring_idx]; 67396c93589SGidon Studinski 67496c93589SGidon Studinski wil_dbg_misc(wil, "free TX sring\n"); 67596c93589SGidon Studinski 67696c93589SGidon Studinski wil_sring_free(wil, sring); 67796c93589SGidon Studinski } 67896c93589SGidon Studinski 67996c93589SGidon Studinski static void wil_rx_data_free(struct wil_status_ring *sring) 68096c93589SGidon Studinski { 68196c93589SGidon Studinski if (!sring) 68296c93589SGidon Studinski return; 68396c93589SGidon Studinski 68496c93589SGidon Studinski kfree_skb(sring->rx_data.skb); 68596c93589SGidon Studinski sring->rx_data.skb = NULL; 68696c93589SGidon Studinski } 68796c93589SGidon Studinski 68896c93589SGidon Studinski static void wil_rx_fini_edma(struct wil6210_priv *wil) 68996c93589SGidon Studinski { 69096c93589SGidon Studinski struct wil_ring *ring = &wil->ring_rx; 69196c93589SGidon Studinski int i; 69296c93589SGidon Studinski 69396c93589SGidon Studinski wil_dbg_misc(wil, "rx_fini_edma\n"); 69496c93589SGidon Studinski 69596c93589SGidon Studinski wil_ring_free_edma(wil, ring); 69696c93589SGidon Studinski 69796c93589SGidon Studinski for (i = 0; i < wil->num_rx_status_rings; i++) { 69896c93589SGidon Studinski wil_rx_data_free(&wil->srings[i]); 69996c93589SGidon Studinski wil_sring_free(wil, &wil->srings[i]); 70096c93589SGidon Studinski } 70196c93589SGidon Studinski 70296c93589SGidon Studinski wil_free_rx_buff_arr(wil); 70396c93589SGidon Studinski } 70496c93589SGidon Studinski 70596c93589SGidon Studinski void wil_init_txrx_ops_edma(struct wil6210_priv *wil) 70696c93589SGidon Studinski { 70796c93589SGidon Studinski wil->txrx_ops.configure_interrupt_moderation = 70896c93589SGidon Studinski wil_configure_interrupt_moderation_edma; 70996c93589SGidon Studinski /* TX ops */ 71096c93589SGidon Studinski wil->txrx_ops.ring_init_tx = wil_ring_init_tx_edma; 71196c93589SGidon Studinski wil->txrx_ops.ring_fini_tx = wil_ring_free_edma; 71296c93589SGidon Studinski wil->txrx_ops.ring_init_bcast = wil_ring_init_bcast_edma; 71396c93589SGidon Studinski wil->txrx_ops.tx_init = wil_tx_init_edma; 71496c93589SGidon Studinski wil->txrx_ops.tx_fini = wil_tx_fini_edma; 71596c93589SGidon Studinski /* RX ops */ 71696c93589SGidon Studinski wil->txrx_ops.rx_init = wil_rx_init_edma; 71796c93589SGidon Studinski wil->txrx_ops.rx_fini = wil_rx_fini_edma; 71896c93589SGidon Studinski } 71996c93589SGidon Studinski 720