1e705c121SKalle Valo /****************************************************************************** 2e705c121SKalle Valo * 3cefec29eSJohannes Berg * This file is provided under a dual BSD/GPLv2 license. When using or 4cefec29eSJohannes Berg * redistributing this file, you may do so under either license. 5cefec29eSJohannes Berg * 6cefec29eSJohannes Berg * GPL LICENSE SUMMARY 7cefec29eSJohannes Berg * 8e705c121SKalle Valo * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 94cbb8e50SLuciano Coelho * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 10eda50cdeSSara Sharon * Copyright(c) 2016 - 2017 Intel Deutschland GmbH 11a8e82c36SJohannes Berg * Copyright(c) 2018 - 2020 Intel Corporation 12e705c121SKalle Valo * 13e705c121SKalle Valo * This program is free software; you can redistribute it and/or modify it 14e705c121SKalle Valo * under the terms of version 2 of the GNU General Public License as 15e705c121SKalle Valo * published by the Free Software Foundation. 16e705c121SKalle Valo * 17e705c121SKalle Valo * This program is distributed in the hope that it will be useful, but WITHOUT 18e705c121SKalle Valo * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19e705c121SKalle Valo * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 20e705c121SKalle Valo * more details. 21e705c121SKalle Valo * 22e705c121SKalle Valo * The full GNU General Public License is included in this distribution in the 23cefec29eSJohannes Berg * file called COPYING. 24e705c121SKalle Valo * 25e705c121SKalle Valo * Contact Information: 26cb2f8277SEmmanuel Grumbach * Intel Linux Wireless <linuxwifi@intel.com> 27e705c121SKalle Valo * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 28e705c121SKalle Valo * 29cefec29eSJohannes Berg * BSD LICENSE 30cefec29eSJohannes Berg * 31cefec29eSJohannes Berg * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 32cefec29eSJohannes Berg * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 33cefec29eSJohannes Berg * Copyright(c) 2016 - 2017 Intel Deutschland GmbH 34a8e82c36SJohannes Berg * Copyright(c) 2018 - 2020 Intel Corporation 35cefec29eSJohannes Berg * All rights reserved. 36cefec29eSJohannes Berg * 37cefec29eSJohannes Berg * Redistribution and use in source and binary forms, with or without 38cefec29eSJohannes Berg * modification, are permitted provided that the following conditions 39cefec29eSJohannes Berg * are met: 40cefec29eSJohannes Berg * 41cefec29eSJohannes Berg * * Redistributions of source code must retain the above copyright 42cefec29eSJohannes Berg * notice, this list of conditions and the following disclaimer. 43cefec29eSJohannes Berg * * Redistributions in binary form must reproduce the above copyright 44cefec29eSJohannes Berg * notice, this list of conditions and the following disclaimer in 45cefec29eSJohannes Berg * the documentation and/or other materials provided with the 46cefec29eSJohannes Berg * distribution. 47cefec29eSJohannes Berg * * Neither the name Intel Corporation nor the names of its 48cefec29eSJohannes Berg * contributors may be used to endorse or promote products derived 49cefec29eSJohannes Berg * from this software without specific prior written permission. 50cefec29eSJohannes Berg * 51cefec29eSJohannes Berg * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 52cefec29eSJohannes Berg * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 53cefec29eSJohannes Berg * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 54cefec29eSJohannes Berg * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 55cefec29eSJohannes Berg * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 56cefec29eSJohannes Berg * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 57cefec29eSJohannes Berg * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 58cefec29eSJohannes Berg * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 59cefec29eSJohannes Berg * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 60cefec29eSJohannes Berg * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 61cefec29eSJohannes Berg * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 62cefec29eSJohannes Berg * 63e705c121SKalle Valo *****************************************************************************/ 64e705c121SKalle Valo #include <linux/etherdevice.h> 656eb5e529SEmmanuel Grumbach #include <linux/ieee80211.h> 66e705c121SKalle Valo #include <linux/slab.h> 67e705c121SKalle Valo #include <linux/sched.h> 686eb5e529SEmmanuel Grumbach #include <net/ip6_checksum.h> 696eb5e529SEmmanuel Grumbach #include <net/tso.h> 70e705c121SKalle Valo 71e705c121SKalle Valo #include "iwl-debug.h" 72e705c121SKalle Valo #include "iwl-csr.h" 73e705c121SKalle Valo #include "iwl-prph.h" 74e705c121SKalle Valo #include "iwl-io.h" 75e705c121SKalle Valo #include "iwl-scd.h" 76e705c121SKalle Valo #include "iwl-op-mode.h" 77e705c121SKalle Valo #include "internal.h" 78d172a5efSJohannes Berg #include "fw/api/tx.h" 79e705c121SKalle Valo 80e705c121SKalle Valo /*************** DMA-QUEUE-GENERAL-FUNCTIONS ***** 81e705c121SKalle Valo * DMA services 82e705c121SKalle Valo * 83e705c121SKalle Valo * Theory of operation 84e705c121SKalle Valo * 85e705c121SKalle Valo * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer 86e705c121SKalle Valo * of buffer descriptors, each of which points to one or more data buffers for 87e705c121SKalle Valo * the device to read from or fill. Driver and device exchange status of each 88e705c121SKalle Valo * queue via "read" and "write" pointers. Driver keeps minimum of 2 empty 89e705c121SKalle Valo * entries in each circular buffer, to protect against confusing empty and full 90e705c121SKalle Valo * queue states. 91e705c121SKalle Valo * 92e705c121SKalle Valo * The device reads or writes the data in the queues via the device's several 93e705c121SKalle Valo * DMA/FIFO channels. Each queue is mapped to a single DMA channel. 94e705c121SKalle Valo * 95e705c121SKalle Valo * For Tx queue, there are low mark and high mark limits. If, after queuing 96e705c121SKalle Valo * the packet for Tx, free space become < low mark, Tx queue stopped. When 97e705c121SKalle Valo * reclaiming packets (on 'tx done IRQ), if free space become > high mark, 98e705c121SKalle Valo * Tx queue resumed. 99e705c121SKalle Valo * 100e705c121SKalle Valo ***************************************************/ 101e22744afSSara Sharon 102e705c121SKalle Valo 10313a3a390SSara Sharon int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans, 104e705c121SKalle Valo struct iwl_dma_ptr *ptr, size_t size) 105e705c121SKalle Valo { 106e705c121SKalle Valo if (WARN_ON(ptr->addr)) 107e705c121SKalle Valo return -EINVAL; 108e705c121SKalle Valo 109e705c121SKalle Valo ptr->addr = dma_alloc_coherent(trans->dev, size, 110e705c121SKalle Valo &ptr->dma, GFP_KERNEL); 111e705c121SKalle Valo if (!ptr->addr) 112e705c121SKalle Valo return -ENOMEM; 113e705c121SKalle Valo ptr->size = size; 114e705c121SKalle Valo return 0; 115e705c121SKalle Valo } 116e705c121SKalle Valo 11713a3a390SSara Sharon void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr) 118e705c121SKalle Valo { 119e705c121SKalle Valo if (unlikely(!ptr->addr)) 120e705c121SKalle Valo return; 121e705c121SKalle Valo 122e705c121SKalle Valo dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma); 123e705c121SKalle Valo memset(ptr, 0, sizeof(*ptr)); 124e705c121SKalle Valo } 125e705c121SKalle Valo 126e705c121SKalle Valo /* 127e705c121SKalle Valo * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware 128e705c121SKalle Valo */ 129e705c121SKalle Valo static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans, 130e705c121SKalle Valo struct iwl_txq *txq) 131e705c121SKalle Valo { 132e705c121SKalle Valo u32 reg = 0; 133bb98ecd4SSara Sharon int txq_id = txq->id; 134e705c121SKalle Valo 135e705c121SKalle Valo lockdep_assert_held(&txq->lock); 136e705c121SKalle Valo 137e705c121SKalle Valo /* 138e705c121SKalle Valo * explicitly wake up the NIC if: 139e705c121SKalle Valo * 1. shadow registers aren't enabled 140e705c121SKalle Valo * 2. NIC is woken up for CMD regardless of shadow outside this function 141e705c121SKalle Valo * 3. there is a chance that the NIC is asleep 142e705c121SKalle Valo */ 143286ca8ebSLuca Coelho if (!trans->trans_cfg->base_params->shadow_reg_enable && 1444f4822b7SMordechay Goodstein txq_id != trans->txqs.cmd.q_id && 145e705c121SKalle Valo test_bit(STATUS_TPOWER_PMI, &trans->status)) { 146e705c121SKalle Valo /* 147e705c121SKalle Valo * wake up nic if it's powered down ... 148e705c121SKalle Valo * uCode will wake up, and interrupt us again, so next 149e705c121SKalle Valo * time we'll skip this part. 150e705c121SKalle Valo */ 151e705c121SKalle Valo reg = iwl_read32(trans, CSR_UCODE_DRV_GP1); 152e705c121SKalle Valo 153e705c121SKalle Valo if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) { 154e705c121SKalle Valo IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n", 155e705c121SKalle Valo txq_id, reg); 156e705c121SKalle Valo iwl_set_bit(trans, CSR_GP_CNTRL, 1576dece0e9SLuca Coelho CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 158e705c121SKalle Valo txq->need_update = true; 159e705c121SKalle Valo return; 160e705c121SKalle Valo } 161e705c121SKalle Valo } 162e705c121SKalle Valo 163e705c121SKalle Valo /* 164e705c121SKalle Valo * if not in power-save mode, uCode will never sleep when we're 165e705c121SKalle Valo * trying to tx (during RFKILL, we're not trying to tx). 166e705c121SKalle Valo */ 167bb98ecd4SSara Sharon IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr); 1680cd58eaaSEmmanuel Grumbach if (!txq->block) 1690cd58eaaSEmmanuel Grumbach iwl_write32(trans, HBUS_TARG_WRPTR, 170bb98ecd4SSara Sharon txq->write_ptr | (txq_id << 8)); 171e705c121SKalle Valo } 172e705c121SKalle Valo 173e705c121SKalle Valo void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans) 174e705c121SKalle Valo { 175e705c121SKalle Valo int i; 176e705c121SKalle Valo 177286ca8ebSLuca Coelho for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) { 1784f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[i]; 179e705c121SKalle Valo 1804f4822b7SMordechay Goodstein if (!test_bit(i, trans->txqs.queue_used)) 181f6eac740SMordechai Goodstein continue; 182f6eac740SMordechai Goodstein 183e705c121SKalle Valo spin_lock_bh(&txq->lock); 184b2a3b1c1SSara Sharon if (txq->need_update) { 185e705c121SKalle Valo iwl_pcie_txq_inc_wr_ptr(trans, txq); 186b2a3b1c1SSara Sharon txq->need_update = false; 187e705c121SKalle Valo } 188e705c121SKalle Valo spin_unlock_bh(&txq->lock); 189e705c121SKalle Valo } 190e705c121SKalle Valo } 191e705c121SKalle Valo 1926983ba69SSara Sharon static inline void iwl_pcie_tfd_set_tb(struct iwl_trans *trans, void *tfd, 1936983ba69SSara Sharon u8 idx, dma_addr_t addr, u16 len) 194e705c121SKalle Valo { 1956983ba69SSara Sharon struct iwl_tfd *tfd_fh = (void *)tfd; 1966983ba69SSara Sharon struct iwl_tfd_tb *tb = &tfd_fh->tbs[idx]; 1976983ba69SSara Sharon 198e705c121SKalle Valo u16 hi_n_len = len << 4; 199e705c121SKalle Valo 200e705c121SKalle Valo put_unaligned_le32(addr, &tb->lo); 2017abf6fdeSJohannes Berg hi_n_len |= iwl_get_dma_hi_addr(addr); 202e705c121SKalle Valo 203e705c121SKalle Valo tb->hi_n_len = cpu_to_le16(hi_n_len); 204e705c121SKalle Valo 2056983ba69SSara Sharon tfd_fh->num_tbs = idx + 1; 2066983ba69SSara Sharon } 207e705c121SKalle Valo 208e705c121SKalle Valo static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq, 209e705c121SKalle Valo dma_addr_t addr, u16 len, bool reset) 210e705c121SKalle Valo { 2116983ba69SSara Sharon void *tfd; 212e705c121SKalle Valo u32 num_tbs; 213e705c121SKalle Valo 214885375d0SMordechay Goodstein tfd = txq->tfds + trans->txqs.tfd.size * txq->write_ptr; 215e705c121SKalle Valo 216e705c121SKalle Valo if (reset) 217885375d0SMordechay Goodstein memset(tfd, 0, trans->txqs.tfd.size); 218e705c121SKalle Valo 2190179bfffSMordechay Goodstein num_tbs = iwl_txq_gen1_tfd_get_num_tbs(trans, tfd); 220e705c121SKalle Valo 2216983ba69SSara Sharon /* Each TFD can point to a maximum max_tbs Tx buffers */ 222885375d0SMordechay Goodstein if (num_tbs >= trans->txqs.tfd.max_tbs) { 223e705c121SKalle Valo IWL_ERR(trans, "Error can not send more than %d chunks\n", 224885375d0SMordechay Goodstein trans->txqs.tfd.max_tbs); 225e705c121SKalle Valo return -EINVAL; 226e705c121SKalle Valo } 227e705c121SKalle Valo 228e705c121SKalle Valo if (WARN(addr & ~IWL_TX_DMA_MASK, 229e705c121SKalle Valo "Unaligned address = %llx\n", (unsigned long long)addr)) 230e705c121SKalle Valo return -EINVAL; 231e705c121SKalle Valo 2326983ba69SSara Sharon iwl_pcie_tfd_set_tb(trans, tfd, num_tbs, addr, len); 233e705c121SKalle Valo 234e705c121SKalle Valo return num_tbs; 235e705c121SKalle Valo } 236e705c121SKalle Valo 23701d11cd1SSara Sharon static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans) 23801d11cd1SSara Sharon { 23901d11cd1SSara Sharon struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 24001d11cd1SSara Sharon 24101d11cd1SSara Sharon lockdep_assert_held(&trans_pcie->reg_lock); 24201d11cd1SSara Sharon 243286ca8ebSLuca Coelho if (!trans->trans_cfg->base_params->apmg_wake_up_wa) 24401d11cd1SSara Sharon return; 24501d11cd1SSara Sharon if (WARN_ON(!trans_pcie->cmd_hold_nic_awake)) 24601d11cd1SSara Sharon return; 24701d11cd1SSara Sharon 24801d11cd1SSara Sharon trans_pcie->cmd_hold_nic_awake = false; 24901d11cd1SSara Sharon __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 2506dece0e9SLuca Coelho CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 25101d11cd1SSara Sharon } 25201d11cd1SSara Sharon 253e705c121SKalle Valo /* 254e705c121SKalle Valo * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's 255e705c121SKalle Valo */ 256e705c121SKalle Valo static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id) 257e705c121SKalle Valo { 258e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2594f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[txq_id]; 260e705c121SKalle Valo 261e705c121SKalle Valo spin_lock_bh(&txq->lock); 262bb98ecd4SSara Sharon while (txq->write_ptr != txq->read_ptr) { 263e705c121SKalle Valo IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 264bb98ecd4SSara Sharon txq_id, txq->read_ptr); 2656eb5e529SEmmanuel Grumbach 2664f4822b7SMordechay Goodstein if (txq_id != trans->txqs.cmd.q_id) { 267bb98ecd4SSara Sharon struct sk_buff *skb = txq->entries[txq->read_ptr].skb; 2686eb5e529SEmmanuel Grumbach 2696eb5e529SEmmanuel Grumbach if (WARN_ON_ONCE(!skb)) 2706eb5e529SEmmanuel Grumbach continue; 2716eb5e529SEmmanuel Grumbach 2720cd1ad2dSMordechay Goodstein iwl_txq_free_tso_page(trans, skb); 2736eb5e529SEmmanuel Grumbach } 274a4450980SMordechay Goodstein iwl_txq_free_tfd(trans, txq); 2750cd1ad2dSMordechay Goodstein txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 27601d11cd1SSara Sharon 277bb98ecd4SSara Sharon if (txq->read_ptr == txq->write_ptr) { 27801d11cd1SSara Sharon unsigned long flags; 27901d11cd1SSara Sharon 28001d11cd1SSara Sharon spin_lock_irqsave(&trans_pcie->reg_lock, flags); 2814f4822b7SMordechay Goodstein if (txq_id == trans->txqs.cmd.q_id) 28201d11cd1SSara Sharon iwl_pcie_clear_cmd_in_flight(trans); 28301d11cd1SSara Sharon spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 28401d11cd1SSara Sharon } 285e705c121SKalle Valo } 2863955525dSEmmanuel Grumbach 2873955525dSEmmanuel Grumbach while (!skb_queue_empty(&txq->overflow_q)) { 2883955525dSEmmanuel Grumbach struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 2893955525dSEmmanuel Grumbach 2903955525dSEmmanuel Grumbach iwl_op_mode_free_skb(trans->op_mode, skb); 2913955525dSEmmanuel Grumbach } 2923955525dSEmmanuel Grumbach 293e705c121SKalle Valo spin_unlock_bh(&txq->lock); 294e705c121SKalle Valo 295e705c121SKalle Valo /* just in case - this queue may have been stopped */ 296e705c121SKalle Valo iwl_wake_queue(trans, txq); 297e705c121SKalle Valo } 298e705c121SKalle Valo 299e705c121SKalle Valo /* 300e705c121SKalle Valo * iwl_pcie_txq_free - Deallocate DMA queue. 301e705c121SKalle Valo * @txq: Transmit queue to deallocate. 302e705c121SKalle Valo * 303e705c121SKalle Valo * Empty queue by removing and destroying all BD's. 304e705c121SKalle Valo * Free all buffers. 305e705c121SKalle Valo * 0-fill, but do not free "txq" descriptor structure. 306e705c121SKalle Valo */ 307e705c121SKalle Valo static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id) 308e705c121SKalle Valo { 3094f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[txq_id]; 310e705c121SKalle Valo struct device *dev = trans->dev; 311e705c121SKalle Valo int i; 312e705c121SKalle Valo 313e705c121SKalle Valo if (WARN_ON(!txq)) 314e705c121SKalle Valo return; 315e705c121SKalle Valo 316e705c121SKalle Valo iwl_pcie_txq_unmap(trans, txq_id); 317e705c121SKalle Valo 318e705c121SKalle Valo /* De-alloc array of command/tx buffers */ 3194f4822b7SMordechay Goodstein if (txq_id == trans->txqs.cmd.q_id) 320bb98ecd4SSara Sharon for (i = 0; i < txq->n_window; i++) { 321453431a5SWaiman Long kfree_sensitive(txq->entries[i].cmd); 322453431a5SWaiman Long kfree_sensitive(txq->entries[i].free_buf); 323e705c121SKalle Valo } 324e705c121SKalle Valo 325e705c121SKalle Valo /* De-alloc circular buffer of TFDs */ 326e705c121SKalle Valo if (txq->tfds) { 327e705c121SKalle Valo dma_free_coherent(dev, 328885375d0SMordechay Goodstein trans->txqs.tfd.size * 329286ca8ebSLuca Coelho trans->trans_cfg->base_params->max_tfd_queue_size, 330bb98ecd4SSara Sharon txq->tfds, txq->dma_addr); 331bb98ecd4SSara Sharon txq->dma_addr = 0; 332e705c121SKalle Valo txq->tfds = NULL; 333e705c121SKalle Valo 334e705c121SKalle Valo dma_free_coherent(dev, 335bb98ecd4SSara Sharon sizeof(*txq->first_tb_bufs) * txq->n_window, 3368de437c7SSara Sharon txq->first_tb_bufs, txq->first_tb_dma); 337e705c121SKalle Valo } 338e705c121SKalle Valo 339e705c121SKalle Valo kfree(txq->entries); 340e705c121SKalle Valo txq->entries = NULL; 341e705c121SKalle Valo 342e705c121SKalle Valo del_timer_sync(&txq->stuck_timer); 343e705c121SKalle Valo 344e705c121SKalle Valo /* 0-fill queue descriptor structure */ 345e705c121SKalle Valo memset(txq, 0, sizeof(*txq)); 346e705c121SKalle Valo } 347e705c121SKalle Valo 348e705c121SKalle Valo void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr) 349e705c121SKalle Valo { 350e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 351286ca8ebSLuca Coelho int nq = trans->trans_cfg->base_params->num_of_queues; 352e705c121SKalle Valo int chan; 353e705c121SKalle Valo u32 reg_val; 354e705c121SKalle Valo int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) - 355e705c121SKalle Valo SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32); 356e705c121SKalle Valo 357e705c121SKalle Valo /* make sure all queue are not stopped/used */ 3584f4822b7SMordechay Goodstein memset(trans->txqs.queue_stopped, 0, 3594f4822b7SMordechay Goodstein sizeof(trans->txqs.queue_stopped)); 3604f4822b7SMordechay Goodstein memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 361e705c121SKalle Valo 362e705c121SKalle Valo trans_pcie->scd_base_addr = 363e705c121SKalle Valo iwl_read_prph(trans, SCD_SRAM_BASE_ADDR); 364e705c121SKalle Valo 365e705c121SKalle Valo WARN_ON(scd_base_addr != 0 && 366e705c121SKalle Valo scd_base_addr != trans_pcie->scd_base_addr); 367e705c121SKalle Valo 368e705c121SKalle Valo /* reset context data, TX status and translation data */ 369e705c121SKalle Valo iwl_trans_write_mem(trans, trans_pcie->scd_base_addr + 370e705c121SKalle Valo SCD_CONTEXT_MEM_LOWER_BOUND, 371e705c121SKalle Valo NULL, clear_dwords); 372e705c121SKalle Valo 373e705c121SKalle Valo iwl_write_prph(trans, SCD_DRAM_BASE_ADDR, 3740179bfffSMordechay Goodstein trans->txqs.scd_bc_tbls.dma >> 10); 375e705c121SKalle Valo 376e705c121SKalle Valo /* The chain extension of the SCD doesn't work well. This feature is 377e705c121SKalle Valo * enabled by default by the HW, so we need to disable it manually. 378e705c121SKalle Valo */ 379286ca8ebSLuca Coelho if (trans->trans_cfg->base_params->scd_chain_ext_wa) 380e705c121SKalle Valo iwl_write_prph(trans, SCD_CHAINEXT_EN, 0); 381e705c121SKalle Valo 3824f4822b7SMordechay Goodstein iwl_trans_ac_txq_enable(trans, trans->txqs.cmd.q_id, 3834f4822b7SMordechay Goodstein trans->txqs.cmd.fifo, 3844f4822b7SMordechay Goodstein trans->txqs.cmd.wdg_timeout); 385e705c121SKalle Valo 386e705c121SKalle Valo /* Activate all Tx DMA/FIFO channels */ 387e705c121SKalle Valo iwl_scd_activate_fifos(trans); 388e705c121SKalle Valo 389e705c121SKalle Valo /* Enable DMA channel */ 390e705c121SKalle Valo for (chan = 0; chan < FH_TCSR_CHNL_NUM; chan++) 391e705c121SKalle Valo iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan), 392e705c121SKalle Valo FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | 393e705c121SKalle Valo FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE); 394e705c121SKalle Valo 395e705c121SKalle Valo /* Update FH chicken bits */ 396e705c121SKalle Valo reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG); 397e705c121SKalle Valo iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG, 398e705c121SKalle Valo reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); 399e705c121SKalle Valo 400e705c121SKalle Valo /* Enable L1-Active */ 401286ca8ebSLuca Coelho if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000) 402e705c121SKalle Valo iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG, 403e705c121SKalle Valo APMG_PCIDEV_STT_VAL_L1_ACT_DIS); 404e705c121SKalle Valo } 405e705c121SKalle Valo 406e705c121SKalle Valo void iwl_trans_pcie_tx_reset(struct iwl_trans *trans) 407e705c121SKalle Valo { 408e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 409e705c121SKalle Valo int txq_id; 410e705c121SKalle Valo 41113a3a390SSara Sharon /* 41213a3a390SSara Sharon * we should never get here in gen2 trans mode return early to avoid 41313a3a390SSara Sharon * having invalid accesses 41413a3a390SSara Sharon */ 415286ca8ebSLuca Coelho if (WARN_ON_ONCE(trans->trans_cfg->gen2)) 41613a3a390SSara Sharon return; 41713a3a390SSara Sharon 418286ca8ebSLuca Coelho for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 419e705c121SKalle Valo txq_id++) { 4204f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[txq_id]; 421286ca8ebSLuca Coelho if (trans->trans_cfg->use_tfh) 422e22744afSSara Sharon iwl_write_direct64(trans, 423e22744afSSara Sharon FH_MEM_CBBC_QUEUE(trans, txq_id), 424bb98ecd4SSara Sharon txq->dma_addr); 425e22744afSSara Sharon else 426e22744afSSara Sharon iwl_write_direct32(trans, 427e22744afSSara Sharon FH_MEM_CBBC_QUEUE(trans, txq_id), 428bb98ecd4SSara Sharon txq->dma_addr >> 8); 429e705c121SKalle Valo iwl_pcie_txq_unmap(trans, txq_id); 430bb98ecd4SSara Sharon txq->read_ptr = 0; 431bb98ecd4SSara Sharon txq->write_ptr = 0; 432e705c121SKalle Valo } 433e705c121SKalle Valo 434e705c121SKalle Valo /* Tell NIC where to find the "keep warm" buffer */ 435e705c121SKalle Valo iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 436e705c121SKalle Valo trans_pcie->kw.dma >> 4); 437e705c121SKalle Valo 438e705c121SKalle Valo /* 439e705c121SKalle Valo * Send 0 as the scd_base_addr since the device may have be reset 440e705c121SKalle Valo * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will 441e705c121SKalle Valo * contain garbage. 442e705c121SKalle Valo */ 443e705c121SKalle Valo iwl_pcie_tx_start(trans, 0); 444e705c121SKalle Valo } 445e705c121SKalle Valo 446e705c121SKalle Valo static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans) 447e705c121SKalle Valo { 448e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 449e705c121SKalle Valo unsigned long flags; 450e705c121SKalle Valo int ch, ret; 451e705c121SKalle Valo u32 mask = 0; 452e705c121SKalle Valo 453e705c121SKalle Valo spin_lock(&trans_pcie->irq_lock); 454e705c121SKalle Valo 45523ba9340SEmmanuel Grumbach if (!iwl_trans_grab_nic_access(trans, &flags)) 456e705c121SKalle Valo goto out; 457e705c121SKalle Valo 458e705c121SKalle Valo /* Stop each Tx DMA channel */ 459e705c121SKalle Valo for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) { 460e705c121SKalle Valo iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0); 461e705c121SKalle Valo mask |= FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch); 462e705c121SKalle Valo } 463e705c121SKalle Valo 464e705c121SKalle Valo /* Wait for DMA channels to be idle */ 465e705c121SKalle Valo ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000); 466e705c121SKalle Valo if (ret < 0) 467e705c121SKalle Valo IWL_ERR(trans, 468e705c121SKalle Valo "Failing on timeout while stopping DMA channel %d [0x%08x]\n", 469e705c121SKalle Valo ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG)); 470e705c121SKalle Valo 471e705c121SKalle Valo iwl_trans_release_nic_access(trans, &flags); 472e705c121SKalle Valo 473e705c121SKalle Valo out: 474e705c121SKalle Valo spin_unlock(&trans_pcie->irq_lock); 475e705c121SKalle Valo } 476e705c121SKalle Valo 477e705c121SKalle Valo /* 478e705c121SKalle Valo * iwl_pcie_tx_stop - Stop all Tx DMA channels 479e705c121SKalle Valo */ 480e705c121SKalle Valo int iwl_pcie_tx_stop(struct iwl_trans *trans) 481e705c121SKalle Valo { 482e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 483e705c121SKalle Valo int txq_id; 484e705c121SKalle Valo 485e705c121SKalle Valo /* Turn off all Tx DMA fifos */ 486e705c121SKalle Valo iwl_scd_deactivate_fifos(trans); 487e705c121SKalle Valo 488e705c121SKalle Valo /* Turn off all Tx DMA channels */ 489e705c121SKalle Valo iwl_pcie_tx_stop_fh(trans); 490e705c121SKalle Valo 491e705c121SKalle Valo /* 492e705c121SKalle Valo * This function can be called before the op_mode disabled the 493e705c121SKalle Valo * queues. This happens when we have an rfkill interrupt. 494e705c121SKalle Valo * Since we stop Tx altogether - mark the queues as stopped. 495e705c121SKalle Valo */ 4964f4822b7SMordechay Goodstein memset(trans->txqs.queue_stopped, 0, 4974f4822b7SMordechay Goodstein sizeof(trans->txqs.queue_stopped)); 4984f4822b7SMordechay Goodstein memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 499e705c121SKalle Valo 500e705c121SKalle Valo /* This can happen: start_hw, stop_device */ 501b2a3b1c1SSara Sharon if (!trans_pcie->txq_memory) 502e705c121SKalle Valo return 0; 503e705c121SKalle Valo 504e705c121SKalle Valo /* Unmap DMA from host system and free skb's */ 505286ca8ebSLuca Coelho for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 506e705c121SKalle Valo txq_id++) 507e705c121SKalle Valo iwl_pcie_txq_unmap(trans, txq_id); 508e705c121SKalle Valo 509e705c121SKalle Valo return 0; 510e705c121SKalle Valo } 511e705c121SKalle Valo 512e705c121SKalle Valo /* 513e705c121SKalle Valo * iwl_trans_tx_free - Free TXQ Context 514e705c121SKalle Valo * 515e705c121SKalle Valo * Destroy all TX DMA queues and structures 516e705c121SKalle Valo */ 517e705c121SKalle Valo void iwl_pcie_tx_free(struct iwl_trans *trans) 518e705c121SKalle Valo { 519e705c121SKalle Valo int txq_id; 520e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 521e705c121SKalle Valo 5224f4822b7SMordechay Goodstein memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 523de74c455SSara Sharon 524e705c121SKalle Valo /* Tx queues */ 525b2a3b1c1SSara Sharon if (trans_pcie->txq_memory) { 526e705c121SKalle Valo for (txq_id = 0; 527286ca8ebSLuca Coelho txq_id < trans->trans_cfg->base_params->num_of_queues; 528b2a3b1c1SSara Sharon txq_id++) { 529e705c121SKalle Valo iwl_pcie_txq_free(trans, txq_id); 5304f4822b7SMordechay Goodstein trans->txqs.txq[txq_id] = NULL; 531b2a3b1c1SSara Sharon } 532e705c121SKalle Valo } 533e705c121SKalle Valo 534b2a3b1c1SSara Sharon kfree(trans_pcie->txq_memory); 535b2a3b1c1SSara Sharon trans_pcie->txq_memory = NULL; 536e705c121SKalle Valo 537e705c121SKalle Valo iwl_pcie_free_dma_ptr(trans, &trans_pcie->kw); 538e705c121SKalle Valo 5390179bfffSMordechay Goodstein iwl_pcie_free_dma_ptr(trans, &trans->txqs.scd_bc_tbls); 540e705c121SKalle Valo } 541e705c121SKalle Valo 542e705c121SKalle Valo /* 543e705c121SKalle Valo * iwl_pcie_tx_alloc - allocate TX context 544e705c121SKalle Valo * Allocate all Tx DMA structures and initialize them 545e705c121SKalle Valo */ 546e705c121SKalle Valo static int iwl_pcie_tx_alloc(struct iwl_trans *trans) 547e705c121SKalle Valo { 548e705c121SKalle Valo int ret; 549e705c121SKalle Valo int txq_id, slots_num; 550e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 551286ca8ebSLuca Coelho u16 bc_tbls_size = trans->trans_cfg->base_params->num_of_queues; 552e705c121SKalle Valo 553a8e82c36SJohannes Berg if (WARN_ON(trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)) 554a8e82c36SJohannes Berg return -EINVAL; 555a8e82c36SJohannes Berg 556a8e82c36SJohannes Berg bc_tbls_size *= sizeof(struct iwlagn_scd_bc_tbl); 557e705c121SKalle Valo 558e705c121SKalle Valo /*It is not allowed to alloc twice, so warn when this happens. 559e705c121SKalle Valo * We cannot rely on the previous allocation, so free and fail */ 560b2a3b1c1SSara Sharon if (WARN_ON(trans_pcie->txq_memory)) { 561e705c121SKalle Valo ret = -EINVAL; 562e705c121SKalle Valo goto error; 563e705c121SKalle Valo } 564e705c121SKalle Valo 5650179bfffSMordechay Goodstein ret = iwl_pcie_alloc_dma_ptr(trans, &trans->txqs.scd_bc_tbls, 5667b3e42eaSGolan Ben Ami bc_tbls_size); 567e705c121SKalle Valo if (ret) { 568e705c121SKalle Valo IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 569e705c121SKalle Valo goto error; 570e705c121SKalle Valo } 571e705c121SKalle Valo 572e705c121SKalle Valo /* Alloc keep-warm buffer */ 573e705c121SKalle Valo ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE); 574e705c121SKalle Valo if (ret) { 575e705c121SKalle Valo IWL_ERR(trans, "Keep Warm allocation failed\n"); 576e705c121SKalle Valo goto error; 577e705c121SKalle Valo } 578e705c121SKalle Valo 57979b6c8feSLuca Coelho trans_pcie->txq_memory = 580286ca8ebSLuca Coelho kcalloc(trans->trans_cfg->base_params->num_of_queues, 581e705c121SKalle Valo sizeof(struct iwl_txq), GFP_KERNEL); 582b2a3b1c1SSara Sharon if (!trans_pcie->txq_memory) { 583e705c121SKalle Valo IWL_ERR(trans, "Not enough memory for txq\n"); 584e705c121SKalle Valo ret = -ENOMEM; 585e705c121SKalle Valo goto error; 586e705c121SKalle Valo } 587e705c121SKalle Valo 588e705c121SKalle Valo /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 589286ca8ebSLuca Coelho for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 590e705c121SKalle Valo txq_id++) { 5914f4822b7SMordechay Goodstein bool cmd_queue = (txq_id == trans->txqs.cmd.q_id); 592b8e8d7ceSSara Sharon 593ff911dcaSShaul Triebitz if (cmd_queue) 594718a8b23SShaul Triebitz slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE, 595ff911dcaSShaul Triebitz trans->cfg->min_txq_size); 596ff911dcaSShaul Triebitz else 597718a8b23SShaul Triebitz slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE, 598c30aef01SShaul Triebitz trans->cfg->min_256_ba_txq_size); 5994f4822b7SMordechay Goodstein trans->txqs.txq[txq_id] = &trans_pcie->txq_memory[txq_id]; 6000cd1ad2dSMordechay Goodstein ret = iwl_txq_alloc(trans, trans->txqs.txq[txq_id], slots_num, 6010cd1ad2dSMordechay Goodstein cmd_queue); 602e705c121SKalle Valo if (ret) { 603e705c121SKalle Valo IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 604e705c121SKalle Valo goto error; 605e705c121SKalle Valo } 6064f4822b7SMordechay Goodstein trans->txqs.txq[txq_id]->id = txq_id; 607e705c121SKalle Valo } 608e705c121SKalle Valo 609e705c121SKalle Valo return 0; 610e705c121SKalle Valo 611e705c121SKalle Valo error: 612e705c121SKalle Valo iwl_pcie_tx_free(trans); 613e705c121SKalle Valo 614e705c121SKalle Valo return ret; 615e705c121SKalle Valo } 616eda50cdeSSara Sharon 617e705c121SKalle Valo int iwl_pcie_tx_init(struct iwl_trans *trans) 618e705c121SKalle Valo { 619e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 620e705c121SKalle Valo int ret; 621e705c121SKalle Valo int txq_id, slots_num; 622e705c121SKalle Valo bool alloc = false; 623e705c121SKalle Valo 624b2a3b1c1SSara Sharon if (!trans_pcie->txq_memory) { 625e705c121SKalle Valo ret = iwl_pcie_tx_alloc(trans); 626e705c121SKalle Valo if (ret) 627e705c121SKalle Valo goto error; 628e705c121SKalle Valo alloc = true; 629e705c121SKalle Valo } 630e705c121SKalle Valo 631e705c121SKalle Valo spin_lock(&trans_pcie->irq_lock); 632e705c121SKalle Valo 633e705c121SKalle Valo /* Turn off all Tx DMA fifos */ 634e705c121SKalle Valo iwl_scd_deactivate_fifos(trans); 635e705c121SKalle Valo 636e705c121SKalle Valo /* Tell NIC where to find the "keep warm" buffer */ 637e705c121SKalle Valo iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 638e705c121SKalle Valo trans_pcie->kw.dma >> 4); 639e705c121SKalle Valo 640e705c121SKalle Valo spin_unlock(&trans_pcie->irq_lock); 641e705c121SKalle Valo 642e705c121SKalle Valo /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 643286ca8ebSLuca Coelho for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 644e705c121SKalle Valo txq_id++) { 6454f4822b7SMordechay Goodstein bool cmd_queue = (txq_id == trans->txqs.cmd.q_id); 646b8e8d7ceSSara Sharon 647ff911dcaSShaul Triebitz if (cmd_queue) 648718a8b23SShaul Triebitz slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE, 649ff911dcaSShaul Triebitz trans->cfg->min_txq_size); 650ff911dcaSShaul Triebitz else 651718a8b23SShaul Triebitz slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE, 652c30aef01SShaul Triebitz trans->cfg->min_256_ba_txq_size); 6530cd1ad2dSMordechay Goodstein ret = iwl_txq_init(trans, trans->txqs.txq[txq_id], slots_num, 6540cd1ad2dSMordechay Goodstein cmd_queue); 655e705c121SKalle Valo if (ret) { 656e705c121SKalle Valo IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 657e705c121SKalle Valo goto error; 658e705c121SKalle Valo } 659e705c121SKalle Valo 660eda50cdeSSara Sharon /* 661eda50cdeSSara Sharon * Tell nic where to find circular buffer of TFDs for a 662eda50cdeSSara Sharon * given Tx queue, and enable the DMA channel used for that 663eda50cdeSSara Sharon * queue. 664eda50cdeSSara Sharon * Circular buffer (TFD queue in DRAM) physical base address 665eda50cdeSSara Sharon */ 666eda50cdeSSara Sharon iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id), 6674f4822b7SMordechay Goodstein trans->txqs.txq[txq_id]->dma_addr >> 8); 668ae79785fSSara Sharon } 669e22744afSSara Sharon 670e705c121SKalle Valo iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE); 671286ca8ebSLuca Coelho if (trans->trans_cfg->base_params->num_of_queues > 20) 672e705c121SKalle Valo iwl_set_bits_prph(trans, SCD_GP_CTRL, 673e705c121SKalle Valo SCD_GP_CTRL_ENABLE_31_QUEUES); 674e705c121SKalle Valo 675e705c121SKalle Valo return 0; 676e705c121SKalle Valo error: 677e705c121SKalle Valo /*Upon error, free only if we allocated something */ 678e705c121SKalle Valo if (alloc) 679e705c121SKalle Valo iwl_pcie_tx_free(trans); 680e705c121SKalle Valo return ret; 681e705c121SKalle Valo } 682e705c121SKalle Valo 683e705c121SKalle Valo static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans, 684e705c121SKalle Valo const struct iwl_host_cmd *cmd) 685e705c121SKalle Valo { 686e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 687e705c121SKalle Valo int ret; 688e705c121SKalle Valo 689e705c121SKalle Valo lockdep_assert_held(&trans_pcie->reg_lock); 690e705c121SKalle Valo 6912b3fae66SMatt Chen /* Make sure the NIC is still alive in the bus */ 692f60c9e59SEmmanuel Grumbach if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 693f60c9e59SEmmanuel Grumbach return -ENODEV; 6942b3fae66SMatt Chen 695e705c121SKalle Valo /* 696e705c121SKalle Valo * wake up the NIC to make sure that the firmware will see the host 697e705c121SKalle Valo * command - we will let the NIC sleep once all the host commands 698e705c121SKalle Valo * returned. This needs to be done only on NICs that have 699e705c121SKalle Valo * apmg_wake_up_wa set. 700e705c121SKalle Valo */ 7017d34a7d7SLuca Coelho if (trans->trans_cfg->base_params->apmg_wake_up_wa && 702e705c121SKalle Valo !trans_pcie->cmd_hold_nic_awake) { 703e705c121SKalle Valo __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, 7046dece0e9SLuca Coelho CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 705e705c121SKalle Valo 706e705c121SKalle Valo ret = iwl_poll_bit(trans, CSR_GP_CNTRL, 7076dece0e9SLuca Coelho CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN, 7086dece0e9SLuca Coelho (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | 709e705c121SKalle Valo CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 710e705c121SKalle Valo 15000); 711e705c121SKalle Valo if (ret < 0) { 712e705c121SKalle Valo __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 7136dece0e9SLuca Coelho CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 714e705c121SKalle Valo IWL_ERR(trans, "Failed to wake NIC for hcmd\n"); 715e705c121SKalle Valo return -EIO; 716e705c121SKalle Valo } 717e705c121SKalle Valo trans_pcie->cmd_hold_nic_awake = true; 718e705c121SKalle Valo } 719e705c121SKalle Valo 720e705c121SKalle Valo return 0; 721e705c121SKalle Valo } 722e705c121SKalle Valo 723e705c121SKalle Valo /* 724e705c121SKalle Valo * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd 725e705c121SKalle Valo * 726e705c121SKalle Valo * When FW advances 'R' index, all entries between old and new 'R' index 727e705c121SKalle Valo * need to be reclaimed. As result, some free space forms. If there is 728e705c121SKalle Valo * enough free space (> low mark), wake the stack that feeds us. 729e705c121SKalle Valo */ 7307216dc99SJohannes Berg static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx) 731e705c121SKalle Valo { 732e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 7334f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[txq_id]; 734e705c121SKalle Valo unsigned long flags; 735e705c121SKalle Valo int nfreed = 0; 736f5955a6cSGolan Ben Ami u16 r; 737e705c121SKalle Valo 738e705c121SKalle Valo lockdep_assert_held(&txq->lock); 739e705c121SKalle Valo 7400cd1ad2dSMordechay Goodstein idx = iwl_txq_get_cmd_index(txq, idx); 7410cd1ad2dSMordechay Goodstein r = iwl_txq_get_cmd_index(txq, txq->read_ptr); 742f5955a6cSGolan Ben Ami 743286ca8ebSLuca Coelho if (idx >= trans->trans_cfg->base_params->max_tfd_queue_size || 7440cd1ad2dSMordechay Goodstein (!iwl_txq_used(txq, idx))) { 7454f4822b7SMordechay Goodstein WARN_ONCE(test_bit(txq_id, trans->txqs.queue_used), 746e705c121SKalle Valo "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n", 7477b3e42eaSGolan Ben Ami __func__, txq_id, idx, 748286ca8ebSLuca Coelho trans->trans_cfg->base_params->max_tfd_queue_size, 749bb98ecd4SSara Sharon txq->write_ptr, txq->read_ptr); 750e705c121SKalle Valo return; 751e705c121SKalle Valo } 752e705c121SKalle Valo 7530cd1ad2dSMordechay Goodstein for (idx = iwl_txq_inc_wrap(trans, idx); r != idx; 7540cd1ad2dSMordechay Goodstein r = iwl_txq_inc_wrap(trans, r)) { 7550cd1ad2dSMordechay Goodstein txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 756e705c121SKalle Valo 757e705c121SKalle Valo if (nfreed++ > 0) { 758e705c121SKalle Valo IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n", 759f5955a6cSGolan Ben Ami idx, txq->write_ptr, r); 760e705c121SKalle Valo iwl_force_nmi(trans); 761e705c121SKalle Valo } 762e705c121SKalle Valo } 763e705c121SKalle Valo 764bb98ecd4SSara Sharon if (txq->read_ptr == txq->write_ptr) { 765e705c121SKalle Valo spin_lock_irqsave(&trans_pcie->reg_lock, flags); 766e705c121SKalle Valo iwl_pcie_clear_cmd_in_flight(trans); 767e705c121SKalle Valo spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 768e705c121SKalle Valo } 769e705c121SKalle Valo 770a4450980SMordechay Goodstein iwl_txq_progress(txq); 771e705c121SKalle Valo } 772e705c121SKalle Valo 773e705c121SKalle Valo static int iwl_pcie_txq_set_ratid_map(struct iwl_trans *trans, u16 ra_tid, 774e705c121SKalle Valo u16 txq_id) 775e705c121SKalle Valo { 776e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 777e705c121SKalle Valo u32 tbl_dw_addr; 778e705c121SKalle Valo u32 tbl_dw; 779e705c121SKalle Valo u16 scd_q2ratid; 780e705c121SKalle Valo 781e705c121SKalle Valo scd_q2ratid = ra_tid & SCD_QUEUE_RA_TID_MAP_RATID_MSK; 782e705c121SKalle Valo 783e705c121SKalle Valo tbl_dw_addr = trans_pcie->scd_base_addr + 784e705c121SKalle Valo SCD_TRANS_TBL_OFFSET_QUEUE(txq_id); 785e705c121SKalle Valo 786e705c121SKalle Valo tbl_dw = iwl_trans_read_mem32(trans, tbl_dw_addr); 787e705c121SKalle Valo 788e705c121SKalle Valo if (txq_id & 0x1) 789e705c121SKalle Valo tbl_dw = (scd_q2ratid << 16) | (tbl_dw & 0x0000FFFF); 790e705c121SKalle Valo else 791e705c121SKalle Valo tbl_dw = scd_q2ratid | (tbl_dw & 0xFFFF0000); 792e705c121SKalle Valo 793e705c121SKalle Valo iwl_trans_write_mem32(trans, tbl_dw_addr, tbl_dw); 794e705c121SKalle Valo 795e705c121SKalle Valo return 0; 796e705c121SKalle Valo } 797e705c121SKalle Valo 798e705c121SKalle Valo /* Receiver address (actually, Rx station's index into station table), 799e705c121SKalle Valo * combined with Traffic ID (QOS priority), in format used by Tx Scheduler */ 800e705c121SKalle Valo #define BUILD_RAxTID(sta_id, tid) (((sta_id) << 4) + (tid)) 801e705c121SKalle Valo 802dcfbd67bSEmmanuel Grumbach bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn, 803e705c121SKalle Valo const struct iwl_trans_txq_scd_cfg *cfg, 804e705c121SKalle Valo unsigned int wdg_timeout) 805e705c121SKalle Valo { 806e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 8074f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[txq_id]; 808e705c121SKalle Valo int fifo = -1; 809dcfbd67bSEmmanuel Grumbach bool scd_bug = false; 810e705c121SKalle Valo 8114f4822b7SMordechay Goodstein if (test_and_set_bit(txq_id, trans->txqs.queue_used)) 812e705c121SKalle Valo WARN_ONCE(1, "queue %d already used - expect issues", txq_id); 813e705c121SKalle Valo 814e705c121SKalle Valo txq->wd_timeout = msecs_to_jiffies(wdg_timeout); 815e705c121SKalle Valo 816e705c121SKalle Valo if (cfg) { 817e705c121SKalle Valo fifo = cfg->fifo; 818e705c121SKalle Valo 819e705c121SKalle Valo /* Disable the scheduler prior configuring the cmd queue */ 8204f4822b7SMordechay Goodstein if (txq_id == trans->txqs.cmd.q_id && 821e705c121SKalle Valo trans_pcie->scd_set_active) 822e705c121SKalle Valo iwl_scd_enable_set_active(trans, 0); 823e705c121SKalle Valo 824e705c121SKalle Valo /* Stop this Tx queue before configuring it */ 825e705c121SKalle Valo iwl_scd_txq_set_inactive(trans, txq_id); 826e705c121SKalle Valo 827e705c121SKalle Valo /* Set this queue as a chain-building queue unless it is CMD */ 8284f4822b7SMordechay Goodstein if (txq_id != trans->txqs.cmd.q_id) 829e705c121SKalle Valo iwl_scd_txq_set_chain(trans, txq_id); 830e705c121SKalle Valo 831e705c121SKalle Valo if (cfg->aggregate) { 832e705c121SKalle Valo u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid); 833e705c121SKalle Valo 834e705c121SKalle Valo /* Map receiver-address / traffic-ID to this queue */ 835e705c121SKalle Valo iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id); 836e705c121SKalle Valo 837e705c121SKalle Valo /* enable aggregations for the queue */ 838e705c121SKalle Valo iwl_scd_txq_enable_agg(trans, txq_id); 839e705c121SKalle Valo txq->ampdu = true; 840e705c121SKalle Valo } else { 841e705c121SKalle Valo /* 842e705c121SKalle Valo * disable aggregations for the queue, this will also 843e705c121SKalle Valo * make the ra_tid mapping configuration irrelevant 844e705c121SKalle Valo * since it is now a non-AGG queue. 845e705c121SKalle Valo */ 846e705c121SKalle Valo iwl_scd_txq_disable_agg(trans, txq_id); 847e705c121SKalle Valo 848bb98ecd4SSara Sharon ssn = txq->read_ptr; 849e705c121SKalle Valo } 850dcfbd67bSEmmanuel Grumbach } else { 851dcfbd67bSEmmanuel Grumbach /* 852dcfbd67bSEmmanuel Grumbach * If we need to move the SCD write pointer by steps of 853dcfbd67bSEmmanuel Grumbach * 0x40, 0x80 or 0xc0, it gets stuck. Avoids this and let 854dcfbd67bSEmmanuel Grumbach * the op_mode know by returning true later. 855dcfbd67bSEmmanuel Grumbach * Do this only in case cfg is NULL since this trick can 856dcfbd67bSEmmanuel Grumbach * be done only if we have DQA enabled which is true for mvm 857dcfbd67bSEmmanuel Grumbach * only. And mvm never sets a cfg pointer. 858dcfbd67bSEmmanuel Grumbach * This is really ugly, but this is the easiest way out for 859dcfbd67bSEmmanuel Grumbach * this sad hardware issue. 860dcfbd67bSEmmanuel Grumbach * This bug has been fixed on devices 9000 and up. 861dcfbd67bSEmmanuel Grumbach */ 862286ca8ebSLuca Coelho scd_bug = !trans->trans_cfg->mq_rx_supported && 863dcfbd67bSEmmanuel Grumbach !((ssn - txq->write_ptr) & 0x3f) && 864dcfbd67bSEmmanuel Grumbach (ssn != txq->write_ptr); 865dcfbd67bSEmmanuel Grumbach if (scd_bug) 866dcfbd67bSEmmanuel Grumbach ssn++; 867e705c121SKalle Valo } 868e705c121SKalle Valo 869e705c121SKalle Valo /* Place first TFD at index corresponding to start sequence number. 870e705c121SKalle Valo * Assumes that ssn_idx is valid (!= 0xFFF) */ 871bb98ecd4SSara Sharon txq->read_ptr = (ssn & 0xff); 872bb98ecd4SSara Sharon txq->write_ptr = (ssn & 0xff); 873e705c121SKalle Valo iwl_write_direct32(trans, HBUS_TARG_WRPTR, 874e705c121SKalle Valo (ssn & 0xff) | (txq_id << 8)); 875e705c121SKalle Valo 876e705c121SKalle Valo if (cfg) { 877e705c121SKalle Valo u8 frame_limit = cfg->frame_limit; 878e705c121SKalle Valo 879e705c121SKalle Valo iwl_write_prph(trans, SCD_QUEUE_RDPTR(txq_id), ssn); 880e705c121SKalle Valo 881e705c121SKalle Valo /* Set up Tx window size and frame limit for this queue */ 882e705c121SKalle Valo iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr + 883e705c121SKalle Valo SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0); 884e705c121SKalle Valo iwl_trans_write_mem32(trans, 885e705c121SKalle Valo trans_pcie->scd_base_addr + 886e705c121SKalle Valo SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32), 887f3779f47SJohannes Berg SCD_QUEUE_CTX_REG2_VAL(WIN_SIZE, frame_limit) | 888f3779f47SJohannes Berg SCD_QUEUE_CTX_REG2_VAL(FRAME_LIMIT, frame_limit)); 889e705c121SKalle Valo 890e705c121SKalle Valo /* Set up status area in SRAM, map to Tx DMA/FIFO, activate */ 891e705c121SKalle Valo iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id), 892e705c121SKalle Valo (1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) | 893e705c121SKalle Valo (cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) | 894e705c121SKalle Valo (1 << SCD_QUEUE_STTS_REG_POS_WSL) | 895e705c121SKalle Valo SCD_QUEUE_STTS_REG_MSK); 896e705c121SKalle Valo 897e705c121SKalle Valo /* enable the scheduler for this queue (only) */ 8984f4822b7SMordechay Goodstein if (txq_id == trans->txqs.cmd.q_id && 899e705c121SKalle Valo trans_pcie->scd_set_active) 900e705c121SKalle Valo iwl_scd_enable_set_active(trans, BIT(txq_id)); 901e705c121SKalle Valo 902e705c121SKalle Valo IWL_DEBUG_TX_QUEUES(trans, 903e705c121SKalle Valo "Activate queue %d on FIFO %d WrPtr: %d\n", 904e705c121SKalle Valo txq_id, fifo, ssn & 0xff); 905e705c121SKalle Valo } else { 906e705c121SKalle Valo IWL_DEBUG_TX_QUEUES(trans, 907e705c121SKalle Valo "Activate queue %d WrPtr: %d\n", 908e705c121SKalle Valo txq_id, ssn & 0xff); 909e705c121SKalle Valo } 910dcfbd67bSEmmanuel Grumbach 911dcfbd67bSEmmanuel Grumbach return scd_bug; 912e705c121SKalle Valo } 913e705c121SKalle Valo 91442db09c1SLiad Kaufman void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id, 91542db09c1SLiad Kaufman bool shared_mode) 91642db09c1SLiad Kaufman { 9174f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[txq_id]; 91842db09c1SLiad Kaufman 91942db09c1SLiad Kaufman txq->ampdu = !shared_mode; 92042db09c1SLiad Kaufman } 92142db09c1SLiad Kaufman 922e705c121SKalle Valo void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id, 923e705c121SKalle Valo bool configure_scd) 924e705c121SKalle Valo { 925e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 926e705c121SKalle Valo u32 stts_addr = trans_pcie->scd_base_addr + 927e705c121SKalle Valo SCD_TX_STTS_QUEUE_OFFSET(txq_id); 928e705c121SKalle Valo static const u32 zero_val[4] = {}; 929e705c121SKalle Valo 9304f4822b7SMordechay Goodstein trans->txqs.txq[txq_id]->frozen_expiry_remainder = 0; 9314f4822b7SMordechay Goodstein trans->txqs.txq[txq_id]->frozen = false; 932e705c121SKalle Valo 933e705c121SKalle Valo /* 934e705c121SKalle Valo * Upon HW Rfkill - we stop the device, and then stop the queues 935e705c121SKalle Valo * in the op_mode. Just for the sake of the simplicity of the op_mode, 936e705c121SKalle Valo * allow the op_mode to call txq_disable after it already called 937e705c121SKalle Valo * stop_device. 938e705c121SKalle Valo */ 9394f4822b7SMordechay Goodstein if (!test_and_clear_bit(txq_id, trans->txqs.queue_used)) { 940e705c121SKalle Valo WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 941e705c121SKalle Valo "queue %d not used", txq_id); 942e705c121SKalle Valo return; 943e705c121SKalle Valo } 944e705c121SKalle Valo 945e705c121SKalle Valo if (configure_scd) { 946e705c121SKalle Valo iwl_scd_txq_set_inactive(trans, txq_id); 947e705c121SKalle Valo 948e705c121SKalle Valo iwl_trans_write_mem(trans, stts_addr, (void *)zero_val, 949e705c121SKalle Valo ARRAY_SIZE(zero_val)); 950e705c121SKalle Valo } 951e705c121SKalle Valo 952e705c121SKalle Valo iwl_pcie_txq_unmap(trans, txq_id); 9534f4822b7SMordechay Goodstein trans->txqs.txq[txq_id]->ampdu = false; 954e705c121SKalle Valo 955e705c121SKalle Valo IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", txq_id); 956e705c121SKalle Valo } 957e705c121SKalle Valo 958e705c121SKalle Valo /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 959e705c121SKalle Valo 960e705c121SKalle Valo /* 961e705c121SKalle Valo * iwl_pcie_enqueue_hcmd - enqueue a uCode command 962e705c121SKalle Valo * @priv: device private data point 963e705c121SKalle Valo * @cmd: a pointer to the ucode command structure 964e705c121SKalle Valo * 965e705c121SKalle Valo * The function returns < 0 values to indicate the operation 966e705c121SKalle Valo * failed. On success, it returns the index (>= 0) of command in the 967e705c121SKalle Valo * command queue. 968e705c121SKalle Valo */ 969e705c121SKalle Valo static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, 970e705c121SKalle Valo struct iwl_host_cmd *cmd) 971e705c121SKalle Valo { 972e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 9734f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 974e705c121SKalle Valo struct iwl_device_cmd *out_cmd; 975e705c121SKalle Valo struct iwl_cmd_meta *out_meta; 976e705c121SKalle Valo unsigned long flags; 977e705c121SKalle Valo void *dup_buf = NULL; 978e705c121SKalle Valo dma_addr_t phys_addr; 979e705c121SKalle Valo int idx; 9808de437c7SSara Sharon u16 copy_size, cmd_size, tb0_size; 981e705c121SKalle Valo bool had_nocopy = false; 982e705c121SKalle Valo u8 group_id = iwl_cmd_groupid(cmd->id); 983e705c121SKalle Valo int i, ret; 984e705c121SKalle Valo u32 cmd_pos; 985e705c121SKalle Valo const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; 986e705c121SKalle Valo u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; 987e705c121SKalle Valo 988b7d96bcaSLuca Coelho if (WARN(!trans->wide_cmd_header && 989b7d96bcaSLuca Coelho group_id > IWL_ALWAYS_LONG_GROUP, 990b7d96bcaSLuca Coelho "unsupported wide command %#x\n", cmd->id)) 991b7d96bcaSLuca Coelho return -EINVAL; 992b7d96bcaSLuca Coelho 993e705c121SKalle Valo if (group_id != 0) { 994e705c121SKalle Valo copy_size = sizeof(struct iwl_cmd_header_wide); 995e705c121SKalle Valo cmd_size = sizeof(struct iwl_cmd_header_wide); 996e705c121SKalle Valo } else { 997e705c121SKalle Valo copy_size = sizeof(struct iwl_cmd_header); 998e705c121SKalle Valo cmd_size = sizeof(struct iwl_cmd_header); 999e705c121SKalle Valo } 1000e705c121SKalle Valo 1001e705c121SKalle Valo /* need one for the header if the first is NOCOPY */ 1002e705c121SKalle Valo BUILD_BUG_ON(IWL_MAX_CMD_TBS_PER_TFD > IWL_NUM_OF_TBS - 1); 1003e705c121SKalle Valo 1004e705c121SKalle Valo for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1005e705c121SKalle Valo cmddata[i] = cmd->data[i]; 1006e705c121SKalle Valo cmdlen[i] = cmd->len[i]; 1007e705c121SKalle Valo 1008e705c121SKalle Valo if (!cmd->len[i]) 1009e705c121SKalle Valo continue; 1010e705c121SKalle Valo 10118de437c7SSara Sharon /* need at least IWL_FIRST_TB_SIZE copied */ 10128de437c7SSara Sharon if (copy_size < IWL_FIRST_TB_SIZE) { 10138de437c7SSara Sharon int copy = IWL_FIRST_TB_SIZE - copy_size; 1014e705c121SKalle Valo 1015e705c121SKalle Valo if (copy > cmdlen[i]) 1016e705c121SKalle Valo copy = cmdlen[i]; 1017e705c121SKalle Valo cmdlen[i] -= copy; 1018e705c121SKalle Valo cmddata[i] += copy; 1019e705c121SKalle Valo copy_size += copy; 1020e705c121SKalle Valo } 1021e705c121SKalle Valo 1022e705c121SKalle Valo if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { 1023e705c121SKalle Valo had_nocopy = true; 1024e705c121SKalle Valo if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { 1025e705c121SKalle Valo idx = -EINVAL; 1026e705c121SKalle Valo goto free_dup_buf; 1027e705c121SKalle Valo } 1028e705c121SKalle Valo } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { 1029e705c121SKalle Valo /* 1030e705c121SKalle Valo * This is also a chunk that isn't copied 1031e705c121SKalle Valo * to the static buffer so set had_nocopy. 1032e705c121SKalle Valo */ 1033e705c121SKalle Valo had_nocopy = true; 1034e705c121SKalle Valo 1035e705c121SKalle Valo /* only allowed once */ 1036e705c121SKalle Valo if (WARN_ON(dup_buf)) { 1037e705c121SKalle Valo idx = -EINVAL; 1038e705c121SKalle Valo goto free_dup_buf; 1039e705c121SKalle Valo } 1040e705c121SKalle Valo 1041e705c121SKalle Valo dup_buf = kmemdup(cmddata[i], cmdlen[i], 1042e705c121SKalle Valo GFP_ATOMIC); 1043e705c121SKalle Valo if (!dup_buf) 1044e705c121SKalle Valo return -ENOMEM; 1045e705c121SKalle Valo } else { 1046e705c121SKalle Valo /* NOCOPY must not be followed by normal! */ 1047e705c121SKalle Valo if (WARN_ON(had_nocopy)) { 1048e705c121SKalle Valo idx = -EINVAL; 1049e705c121SKalle Valo goto free_dup_buf; 1050e705c121SKalle Valo } 1051e705c121SKalle Valo copy_size += cmdlen[i]; 1052e705c121SKalle Valo } 1053e705c121SKalle Valo cmd_size += cmd->len[i]; 1054e705c121SKalle Valo } 1055e705c121SKalle Valo 1056e705c121SKalle Valo /* 1057e705c121SKalle Valo * If any of the command structures end up being larger than 1058e705c121SKalle Valo * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically 1059e705c121SKalle Valo * allocated into separate TFDs, then we will need to 1060e705c121SKalle Valo * increase the size of the buffers. 1061e705c121SKalle Valo */ 1062e705c121SKalle Valo if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, 1063e705c121SKalle Valo "Command %s (%#x) is too large (%d bytes)\n", 106439bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id), 106539bdb17eSSharon Dvir cmd->id, copy_size)) { 1066e705c121SKalle Valo idx = -EINVAL; 1067e705c121SKalle Valo goto free_dup_buf; 1068e705c121SKalle Valo } 1069e705c121SKalle Valo 1070e705c121SKalle Valo spin_lock_bh(&txq->lock); 1071e705c121SKalle Valo 10720cd1ad2dSMordechay Goodstein if (iwl_txq_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { 1073e705c121SKalle Valo spin_unlock_bh(&txq->lock); 1074e705c121SKalle Valo 1075e705c121SKalle Valo IWL_ERR(trans, "No space in command queue\n"); 1076e705c121SKalle Valo iwl_op_mode_cmd_queue_full(trans->op_mode); 1077e705c121SKalle Valo idx = -ENOSPC; 1078e705c121SKalle Valo goto free_dup_buf; 1079e705c121SKalle Valo } 1080e705c121SKalle Valo 10810cd1ad2dSMordechay Goodstein idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 1082e705c121SKalle Valo out_cmd = txq->entries[idx].cmd; 1083e705c121SKalle Valo out_meta = &txq->entries[idx].meta; 1084e705c121SKalle Valo 1085e705c121SKalle Valo memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ 1086e705c121SKalle Valo if (cmd->flags & CMD_WANT_SKB) 1087e705c121SKalle Valo out_meta->source = cmd; 1088e705c121SKalle Valo 1089e705c121SKalle Valo /* set up the header */ 1090e705c121SKalle Valo if (group_id != 0) { 1091e705c121SKalle Valo out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); 1092e705c121SKalle Valo out_cmd->hdr_wide.group_id = group_id; 1093e705c121SKalle Valo out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); 1094e705c121SKalle Valo out_cmd->hdr_wide.length = 1095e705c121SKalle Valo cpu_to_le16(cmd_size - 1096e705c121SKalle Valo sizeof(struct iwl_cmd_header_wide)); 1097e705c121SKalle Valo out_cmd->hdr_wide.reserved = 0; 1098e705c121SKalle Valo out_cmd->hdr_wide.sequence = 10994f4822b7SMordechay Goodstein cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) | 1100bb98ecd4SSara Sharon INDEX_TO_SEQ(txq->write_ptr)); 1101e705c121SKalle Valo 1102e705c121SKalle Valo cmd_pos = sizeof(struct iwl_cmd_header_wide); 1103e705c121SKalle Valo copy_size = sizeof(struct iwl_cmd_header_wide); 1104e705c121SKalle Valo } else { 1105e705c121SKalle Valo out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id); 1106e705c121SKalle Valo out_cmd->hdr.sequence = 11074f4822b7SMordechay Goodstein cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) | 1108bb98ecd4SSara Sharon INDEX_TO_SEQ(txq->write_ptr)); 1109e705c121SKalle Valo out_cmd->hdr.group_id = 0; 1110e705c121SKalle Valo 1111e705c121SKalle Valo cmd_pos = sizeof(struct iwl_cmd_header); 1112e705c121SKalle Valo copy_size = sizeof(struct iwl_cmd_header); 1113e705c121SKalle Valo } 1114e705c121SKalle Valo 1115e705c121SKalle Valo /* and copy the data that needs to be copied */ 1116e705c121SKalle Valo for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1117e705c121SKalle Valo int copy; 1118e705c121SKalle Valo 1119e705c121SKalle Valo if (!cmd->len[i]) 1120e705c121SKalle Valo continue; 1121e705c121SKalle Valo 1122e705c121SKalle Valo /* copy everything if not nocopy/dup */ 1123e705c121SKalle Valo if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1124e705c121SKalle Valo IWL_HCMD_DFL_DUP))) { 1125e705c121SKalle Valo copy = cmd->len[i]; 1126e705c121SKalle Valo 1127e705c121SKalle Valo memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1128e705c121SKalle Valo cmd_pos += copy; 1129e705c121SKalle Valo copy_size += copy; 1130e705c121SKalle Valo continue; 1131e705c121SKalle Valo } 1132e705c121SKalle Valo 1133e705c121SKalle Valo /* 11348de437c7SSara Sharon * Otherwise we need at least IWL_FIRST_TB_SIZE copied 11358de437c7SSara Sharon * in total (for bi-directional DMA), but copy up to what 1136e705c121SKalle Valo * we can fit into the payload for debug dump purposes. 1137e705c121SKalle Valo */ 1138e705c121SKalle Valo copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); 1139e705c121SKalle Valo 1140e705c121SKalle Valo memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1141e705c121SKalle Valo cmd_pos += copy; 1142e705c121SKalle Valo 1143e705c121SKalle Valo /* However, treat copy_size the proper way, we need it below */ 11448de437c7SSara Sharon if (copy_size < IWL_FIRST_TB_SIZE) { 11458de437c7SSara Sharon copy = IWL_FIRST_TB_SIZE - copy_size; 1146e705c121SKalle Valo 1147e705c121SKalle Valo if (copy > cmd->len[i]) 1148e705c121SKalle Valo copy = cmd->len[i]; 1149e705c121SKalle Valo copy_size += copy; 1150e705c121SKalle Valo } 1151e705c121SKalle Valo } 1152e705c121SKalle Valo 1153e705c121SKalle Valo IWL_DEBUG_HC(trans, 1154e705c121SKalle Valo "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", 115539bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id), 1156e705c121SKalle Valo group_id, out_cmd->hdr.cmd, 1157e705c121SKalle Valo le16_to_cpu(out_cmd->hdr.sequence), 11584f4822b7SMordechay Goodstein cmd_size, txq->write_ptr, idx, trans->txqs.cmd.q_id); 1159e705c121SKalle Valo 11608de437c7SSara Sharon /* start the TFD with the minimum copy bytes */ 11618de437c7SSara Sharon tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); 11628de437c7SSara Sharon memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size); 1163e705c121SKalle Valo iwl_pcie_txq_build_tfd(trans, txq, 11640cd1ad2dSMordechay Goodstein iwl_txq_get_first_tb_dma(txq, idx), 11658de437c7SSara Sharon tb0_size, true); 1166e705c121SKalle Valo 1167e705c121SKalle Valo /* map first command fragment, if any remains */ 11688de437c7SSara Sharon if (copy_size > tb0_size) { 1169e705c121SKalle Valo phys_addr = dma_map_single(trans->dev, 11708de437c7SSara Sharon ((u8 *)&out_cmd->hdr) + tb0_size, 11718de437c7SSara Sharon copy_size - tb0_size, 1172e705c121SKalle Valo DMA_TO_DEVICE); 1173e705c121SKalle Valo if (dma_mapping_error(trans->dev, phys_addr)) { 11740179bfffSMordechay Goodstein iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, 1175bb98ecd4SSara Sharon txq->write_ptr); 1176e705c121SKalle Valo idx = -ENOMEM; 1177e705c121SKalle Valo goto out; 1178e705c121SKalle Valo } 1179e705c121SKalle Valo 1180e705c121SKalle Valo iwl_pcie_txq_build_tfd(trans, txq, phys_addr, 11818de437c7SSara Sharon copy_size - tb0_size, false); 1182e705c121SKalle Valo } 1183e705c121SKalle Valo 1184e705c121SKalle Valo /* map the remaining (adjusted) nocopy/dup fragments */ 1185e705c121SKalle Valo for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1186e705c121SKalle Valo const void *data = cmddata[i]; 1187e705c121SKalle Valo 1188e705c121SKalle Valo if (!cmdlen[i]) 1189e705c121SKalle Valo continue; 1190e705c121SKalle Valo if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1191e705c121SKalle Valo IWL_HCMD_DFL_DUP))) 1192e705c121SKalle Valo continue; 1193e705c121SKalle Valo if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) 1194e705c121SKalle Valo data = dup_buf; 1195e705c121SKalle Valo phys_addr = dma_map_single(trans->dev, (void *)data, 1196e705c121SKalle Valo cmdlen[i], DMA_TO_DEVICE); 1197e705c121SKalle Valo if (dma_mapping_error(trans->dev, phys_addr)) { 11980179bfffSMordechay Goodstein iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, 1199bb98ecd4SSara Sharon txq->write_ptr); 1200e705c121SKalle Valo idx = -ENOMEM; 1201e705c121SKalle Valo goto out; 1202e705c121SKalle Valo } 1203e705c121SKalle Valo 1204e705c121SKalle Valo iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false); 1205e705c121SKalle Valo } 1206e705c121SKalle Valo 12073cd1980bSSara Sharon BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); 1208e705c121SKalle Valo out_meta->flags = cmd->flags; 1209e705c121SKalle Valo if (WARN_ON_ONCE(txq->entries[idx].free_buf)) 1210453431a5SWaiman Long kfree_sensitive(txq->entries[idx].free_buf); 1211e705c121SKalle Valo txq->entries[idx].free_buf = dup_buf; 1212e705c121SKalle Valo 1213e705c121SKalle Valo trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); 1214e705c121SKalle Valo 1215e705c121SKalle Valo /* start timer if queue currently empty */ 1216bb98ecd4SSara Sharon if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 1217e705c121SKalle Valo mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 1218e705c121SKalle Valo 1219e705c121SKalle Valo spin_lock_irqsave(&trans_pcie->reg_lock, flags); 1220e705c121SKalle Valo ret = iwl_pcie_set_cmd_in_flight(trans, cmd); 1221e705c121SKalle Valo if (ret < 0) { 1222e705c121SKalle Valo idx = ret; 1223e705c121SKalle Valo spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1224e705c121SKalle Valo goto out; 1225e705c121SKalle Valo } 1226e705c121SKalle Valo 1227e705c121SKalle Valo /* Increment and update queue's write index */ 12280cd1ad2dSMordechay Goodstein txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 1229e705c121SKalle Valo iwl_pcie_txq_inc_wr_ptr(trans, txq); 1230e705c121SKalle Valo 1231e705c121SKalle Valo spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1232e705c121SKalle Valo 1233e705c121SKalle Valo out: 1234e705c121SKalle Valo spin_unlock_bh(&txq->lock); 1235e705c121SKalle Valo free_dup_buf: 1236e705c121SKalle Valo if (idx < 0) 1237e705c121SKalle Valo kfree(dup_buf); 1238e705c121SKalle Valo return idx; 1239e705c121SKalle Valo } 1240e705c121SKalle Valo 1241e705c121SKalle Valo /* 1242e705c121SKalle Valo * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them 1243e705c121SKalle Valo * @rxb: Rx buffer to reclaim 1244e705c121SKalle Valo */ 1245e705c121SKalle Valo void iwl_pcie_hcmd_complete(struct iwl_trans *trans, 1246e705c121SKalle Valo struct iwl_rx_cmd_buffer *rxb) 1247e705c121SKalle Valo { 1248e705c121SKalle Valo struct iwl_rx_packet *pkt = rxb_addr(rxb); 1249e705c121SKalle Valo u16 sequence = le16_to_cpu(pkt->hdr.sequence); 1250d490e097SJohannes Berg u8 group_id; 125139bdb17eSSharon Dvir u32 cmd_id; 1252e705c121SKalle Valo int txq_id = SEQ_TO_QUEUE(sequence); 1253e705c121SKalle Valo int index = SEQ_TO_INDEX(sequence); 1254e705c121SKalle Valo int cmd_index; 1255e705c121SKalle Valo struct iwl_device_cmd *cmd; 1256e705c121SKalle Valo struct iwl_cmd_meta *meta; 1257e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 12584f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1259e705c121SKalle Valo 1260e705c121SKalle Valo /* If a Tx command is being handled and it isn't in the actual 1261e705c121SKalle Valo * command queue then there a command routing bug has been introduced 1262e705c121SKalle Valo * in the queue management code. */ 12634f4822b7SMordechay Goodstein if (WARN(txq_id != trans->txqs.cmd.q_id, 1264e705c121SKalle Valo "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n", 12654f4822b7SMordechay Goodstein txq_id, trans->txqs.cmd.q_id, sequence, txq->read_ptr, 1266b2a3b1c1SSara Sharon txq->write_ptr)) { 1267e705c121SKalle Valo iwl_print_hex_error(trans, pkt, 32); 1268e705c121SKalle Valo return; 1269e705c121SKalle Valo } 1270e705c121SKalle Valo 1271e705c121SKalle Valo spin_lock_bh(&txq->lock); 1272e705c121SKalle Valo 12730cd1ad2dSMordechay Goodstein cmd_index = iwl_txq_get_cmd_index(txq, index); 1274e705c121SKalle Valo cmd = txq->entries[cmd_index].cmd; 1275e705c121SKalle Valo meta = &txq->entries[cmd_index].meta; 1276d490e097SJohannes Berg group_id = cmd->hdr.group_id; 127739bdb17eSSharon Dvir cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0); 1278e705c121SKalle Valo 12790179bfffSMordechay Goodstein iwl_txq_gen1_tfd_unmap(trans, meta, txq, index); 1280e705c121SKalle Valo 1281e705c121SKalle Valo /* Input error checking is done when commands are added to queue. */ 1282e705c121SKalle Valo if (meta->flags & CMD_WANT_SKB) { 1283e705c121SKalle Valo struct page *p = rxb_steal_page(rxb); 1284e705c121SKalle Valo 1285e705c121SKalle Valo meta->source->resp_pkt = pkt; 1286e705c121SKalle Valo meta->source->_rx_page_addr = (unsigned long)page_address(p); 1287e705c121SKalle Valo meta->source->_rx_page_order = trans_pcie->rx_page_order; 1288e705c121SKalle Valo } 1289e705c121SKalle Valo 1290dcbb4746SEmmanuel Grumbach if (meta->flags & CMD_WANT_ASYNC_CALLBACK) 1291dcbb4746SEmmanuel Grumbach iwl_op_mode_async_cb(trans->op_mode, cmd); 1292dcbb4746SEmmanuel Grumbach 1293e705c121SKalle Valo iwl_pcie_cmdq_reclaim(trans, txq_id, index); 1294e705c121SKalle Valo 1295e705c121SKalle Valo if (!(meta->flags & CMD_ASYNC)) { 1296e705c121SKalle Valo if (!test_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) { 1297e705c121SKalle Valo IWL_WARN(trans, 1298e705c121SKalle Valo "HCMD_ACTIVE already clear for command %s\n", 129939bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd_id)); 1300e705c121SKalle Valo } 1301e705c121SKalle Valo clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1302e705c121SKalle Valo IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 130339bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd_id)); 1304e705c121SKalle Valo wake_up(&trans_pcie->wait_command_queue); 1305e705c121SKalle Valo } 1306e705c121SKalle Valo 1307e705c121SKalle Valo meta->flags = 0; 1308e705c121SKalle Valo 1309e705c121SKalle Valo spin_unlock_bh(&txq->lock); 1310e705c121SKalle Valo } 1311e705c121SKalle Valo 1312e705c121SKalle Valo #define HOST_COMPLETE_TIMEOUT (2 * HZ) 1313e705c121SKalle Valo 1314e705c121SKalle Valo static int iwl_pcie_send_hcmd_async(struct iwl_trans *trans, 1315e705c121SKalle Valo struct iwl_host_cmd *cmd) 1316e705c121SKalle Valo { 1317e705c121SKalle Valo int ret; 1318e705c121SKalle Valo 1319e705c121SKalle Valo /* An asynchronous command can not expect an SKB to be set. */ 1320e705c121SKalle Valo if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1321e705c121SKalle Valo return -EINVAL; 1322e705c121SKalle Valo 1323e705c121SKalle Valo ret = iwl_pcie_enqueue_hcmd(trans, cmd); 1324e705c121SKalle Valo if (ret < 0) { 1325e705c121SKalle Valo IWL_ERR(trans, 1326e705c121SKalle Valo "Error sending %s: enqueue_hcmd failed: %d\n", 132739bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id), ret); 1328e705c121SKalle Valo return ret; 1329e705c121SKalle Valo } 1330e705c121SKalle Valo return 0; 1331e705c121SKalle Valo } 1332e705c121SKalle Valo 1333e705c121SKalle Valo static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans, 1334e705c121SKalle Valo struct iwl_host_cmd *cmd) 1335e705c121SKalle Valo { 1336e705c121SKalle Valo struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 13374f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1338e705c121SKalle Valo int cmd_idx; 1339e705c121SKalle Valo int ret; 1340e705c121SKalle Valo 1341e705c121SKalle Valo IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", 134239bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id)); 1343e705c121SKalle Valo 1344e705c121SKalle Valo if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 1345e705c121SKalle Valo &trans->status), 1346e705c121SKalle Valo "Command %s: a command is already active!\n", 134739bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id))) 1348e705c121SKalle Valo return -EIO; 1349e705c121SKalle Valo 1350e705c121SKalle Valo IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", 135139bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id)); 1352e705c121SKalle Valo 1353e705c121SKalle Valo cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd); 1354e705c121SKalle Valo if (cmd_idx < 0) { 1355e705c121SKalle Valo ret = cmd_idx; 1356e705c121SKalle Valo clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1357e705c121SKalle Valo IWL_ERR(trans, 1358e705c121SKalle Valo "Error sending %s: enqueue_hcmd failed: %d\n", 135939bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id), ret); 1360e705c121SKalle Valo return ret; 1361e705c121SKalle Valo } 1362e705c121SKalle Valo 1363e705c121SKalle Valo ret = wait_event_timeout(trans_pcie->wait_command_queue, 1364e705c121SKalle Valo !test_bit(STATUS_SYNC_HCMD_ACTIVE, 1365e705c121SKalle Valo &trans->status), 1366e705c121SKalle Valo HOST_COMPLETE_TIMEOUT); 1367e705c121SKalle Valo if (!ret) { 1368e705c121SKalle Valo IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 136939bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id), 1370e705c121SKalle Valo jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 1371e705c121SKalle Valo 1372e705c121SKalle Valo IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 1373bb98ecd4SSara Sharon txq->read_ptr, txq->write_ptr); 1374e705c121SKalle Valo 1375e705c121SKalle Valo clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1376e705c121SKalle Valo IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 137739bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id)); 1378e705c121SKalle Valo ret = -ETIMEDOUT; 1379e705c121SKalle Valo 1380d1967ce6SShahar S Matityahu iwl_trans_pcie_sync_nmi(trans); 1381e705c121SKalle Valo goto cancel; 1382e705c121SKalle Valo } 1383e705c121SKalle Valo 1384e705c121SKalle Valo if (test_bit(STATUS_FW_ERROR, &trans->status)) { 13854290eaadSJohannes Berg iwl_trans_pcie_dump_regs(trans); 1386e705c121SKalle Valo IWL_ERR(trans, "FW error in SYNC CMD %s\n", 138739bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id)); 1388e705c121SKalle Valo dump_stack(); 1389e705c121SKalle Valo ret = -EIO; 1390e705c121SKalle Valo goto cancel; 1391e705c121SKalle Valo } 1392e705c121SKalle Valo 1393e705c121SKalle Valo if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1394326477e4SJohannes Berg test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1395e705c121SKalle Valo IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 1396e705c121SKalle Valo ret = -ERFKILL; 1397e705c121SKalle Valo goto cancel; 1398e705c121SKalle Valo } 1399e705c121SKalle Valo 1400e705c121SKalle Valo if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 1401e705c121SKalle Valo IWL_ERR(trans, "Error: Response NULL in '%s'\n", 140239bdb17eSSharon Dvir iwl_get_cmd_string(trans, cmd->id)); 1403e705c121SKalle Valo ret = -EIO; 1404e705c121SKalle Valo goto cancel; 1405e705c121SKalle Valo } 1406e705c121SKalle Valo 1407e705c121SKalle Valo return 0; 1408e705c121SKalle Valo 1409e705c121SKalle Valo cancel: 1410e705c121SKalle Valo if (cmd->flags & CMD_WANT_SKB) { 1411e705c121SKalle Valo /* 1412e705c121SKalle Valo * Cancel the CMD_WANT_SKB flag for the cmd in the 1413e705c121SKalle Valo * TX cmd queue. Otherwise in case the cmd comes 1414e705c121SKalle Valo * in later, it will possibly set an invalid 1415e705c121SKalle Valo * address (cmd->meta.source). 1416e705c121SKalle Valo */ 1417b2a3b1c1SSara Sharon txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 1418e705c121SKalle Valo } 1419e705c121SKalle Valo 1420e705c121SKalle Valo if (cmd->resp_pkt) { 1421e705c121SKalle Valo iwl_free_resp(cmd); 1422e705c121SKalle Valo cmd->resp_pkt = NULL; 1423e705c121SKalle Valo } 1424e705c121SKalle Valo 1425e705c121SKalle Valo return ret; 1426e705c121SKalle Valo } 1427e705c121SKalle Valo 1428e705c121SKalle Valo int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) 1429e705c121SKalle Valo { 14302b3fae66SMatt Chen /* Make sure the NIC is still alive in the bus */ 1431f60c9e59SEmmanuel Grumbach if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 1432f60c9e59SEmmanuel Grumbach return -ENODEV; 14332b3fae66SMatt Chen 1434e705c121SKalle Valo if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1435326477e4SJohannes Berg test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1436e705c121SKalle Valo IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1437e705c121SKalle Valo cmd->id); 1438e705c121SKalle Valo return -ERFKILL; 1439e705c121SKalle Valo } 1440e705c121SKalle Valo 1441e705c121SKalle Valo if (cmd->flags & CMD_ASYNC) 1442e705c121SKalle Valo return iwl_pcie_send_hcmd_async(trans, cmd); 1443e705c121SKalle Valo 1444e705c121SKalle Valo /* We still can fail on RFKILL that can be asserted while we wait */ 1445e705c121SKalle Valo return iwl_pcie_send_hcmd_sync(trans, cmd); 1446e705c121SKalle Valo } 1447e705c121SKalle Valo 14483a0b2a42SEmmanuel Grumbach static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb, 14493a0b2a42SEmmanuel Grumbach struct iwl_txq *txq, u8 hdr_len, 1450bb03927eSJohannes Berg struct iwl_cmd_meta *out_meta) 14513a0b2a42SEmmanuel Grumbach { 1452bb03927eSJohannes Berg u16 head_tb_len; 14533a0b2a42SEmmanuel Grumbach int i; 14543a0b2a42SEmmanuel Grumbach 14553a0b2a42SEmmanuel Grumbach /* 14563a0b2a42SEmmanuel Grumbach * Set up TFD's third entry to point directly to remainder 14573a0b2a42SEmmanuel Grumbach * of skb's head, if any 14583a0b2a42SEmmanuel Grumbach */ 1459bb03927eSJohannes Berg head_tb_len = skb_headlen(skb) - hdr_len; 14603a0b2a42SEmmanuel Grumbach 1461bb03927eSJohannes Berg if (head_tb_len > 0) { 1462bb03927eSJohannes Berg dma_addr_t tb_phys = dma_map_single(trans->dev, 14633a0b2a42SEmmanuel Grumbach skb->data + hdr_len, 1464bb03927eSJohannes Berg head_tb_len, DMA_TO_DEVICE); 1465bb03927eSJohannes Berg if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 14663a0b2a42SEmmanuel Grumbach return -EINVAL; 14679b08ae22SJohannes Berg trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb->data + hdr_len, 14689b08ae22SJohannes Berg tb_phys, head_tb_len); 1469bb03927eSJohannes Berg iwl_pcie_txq_build_tfd(trans, txq, tb_phys, head_tb_len, false); 14703a0b2a42SEmmanuel Grumbach } 14713a0b2a42SEmmanuel Grumbach 14723a0b2a42SEmmanuel Grumbach /* set up the remaining entries to point to the data */ 14733a0b2a42SEmmanuel Grumbach for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 14743a0b2a42SEmmanuel Grumbach const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 14753a0b2a42SEmmanuel Grumbach dma_addr_t tb_phys; 14763a0b2a42SEmmanuel Grumbach int tb_idx; 14773a0b2a42SEmmanuel Grumbach 14783a0b2a42SEmmanuel Grumbach if (!skb_frag_size(frag)) 14793a0b2a42SEmmanuel Grumbach continue; 14803a0b2a42SEmmanuel Grumbach 14813a0b2a42SEmmanuel Grumbach tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 14823a0b2a42SEmmanuel Grumbach skb_frag_size(frag), DMA_TO_DEVICE); 14833a0b2a42SEmmanuel Grumbach 14847d50d76eSJohannes Berg if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 14853a0b2a42SEmmanuel Grumbach return -EINVAL; 14869b08ae22SJohannes Berg trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb_frag_address(frag), 14879b08ae22SJohannes Berg tb_phys, skb_frag_size(frag)); 14883a0b2a42SEmmanuel Grumbach tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 14893a0b2a42SEmmanuel Grumbach skb_frag_size(frag), false); 14906e00a237SJohannes Berg if (tb_idx < 0) 14916e00a237SJohannes Berg return tb_idx; 14923a0b2a42SEmmanuel Grumbach 14933cd1980bSSara Sharon out_meta->tbs |= BIT(tb_idx); 14943a0b2a42SEmmanuel Grumbach } 14953a0b2a42SEmmanuel Grumbach 14963a0b2a42SEmmanuel Grumbach return 0; 14973a0b2a42SEmmanuel Grumbach } 14983a0b2a42SEmmanuel Grumbach 14996eb5e529SEmmanuel Grumbach #ifdef CONFIG_INET 1500066fd29aSSara Sharon static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 15016eb5e529SEmmanuel Grumbach struct iwl_txq *txq, u8 hdr_len, 15026eb5e529SEmmanuel Grumbach struct iwl_cmd_meta *out_meta, 1503a89c72ffSJohannes Berg struct iwl_device_tx_cmd *dev_cmd, 1504a89c72ffSJohannes Berg u16 tb1_len) 15056eb5e529SEmmanuel Grumbach { 150605e5a7e5SJohannes Berg struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 15076eb5e529SEmmanuel Grumbach struct ieee80211_hdr *hdr = (void *)skb->data; 15086eb5e529SEmmanuel Grumbach unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 15096eb5e529SEmmanuel Grumbach unsigned int mss = skb_shinfo(skb)->gso_size; 15106eb5e529SEmmanuel Grumbach u16 length, iv_len, amsdu_pad; 15116eb5e529SEmmanuel Grumbach u8 *start_hdr; 15126eb5e529SEmmanuel Grumbach struct iwl_tso_hdr_page *hdr_page; 15136eb5e529SEmmanuel Grumbach struct tso_t tso; 15146eb5e529SEmmanuel Grumbach 15156eb5e529SEmmanuel Grumbach /* if the packet is protected, then it must be CCMP or GCMP */ 15166eb5e529SEmmanuel Grumbach BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN); 15176eb5e529SEmmanuel Grumbach iv_len = ieee80211_has_protected(hdr->frame_control) ? 15186eb5e529SEmmanuel Grumbach IEEE80211_CCMP_HDR_LEN : 0; 15196eb5e529SEmmanuel Grumbach 15206eb5e529SEmmanuel Grumbach trace_iwlwifi_dev_tx(trans->dev, skb, 15210cd1ad2dSMordechay Goodstein iwl_txq_get_tfd(trans, txq, txq->write_ptr), 1522885375d0SMordechay Goodstein trans->txqs.tfd.size, 15238790fce4SJohannes Berg &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 0); 15246eb5e529SEmmanuel Grumbach 15256eb5e529SEmmanuel Grumbach ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 15266eb5e529SEmmanuel Grumbach snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 15276eb5e529SEmmanuel Grumbach total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; 15286eb5e529SEmmanuel Grumbach amsdu_pad = 0; 15296eb5e529SEmmanuel Grumbach 15306eb5e529SEmmanuel Grumbach /* total amount of header we may need for this A-MSDU */ 15316eb5e529SEmmanuel Grumbach hdr_room = DIV_ROUND_UP(total_len, mss) * 15326eb5e529SEmmanuel Grumbach (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len; 15336eb5e529SEmmanuel Grumbach 15346eb5e529SEmmanuel Grumbach /* Our device supports 9 segments at most, it will fit in 1 page */ 15357b02bf61SJohannes Berg hdr_page = get_page_hdr(trans, hdr_room, skb); 15366eb5e529SEmmanuel Grumbach if (!hdr_page) 15376eb5e529SEmmanuel Grumbach return -ENOMEM; 15386eb5e529SEmmanuel Grumbach 15396eb5e529SEmmanuel Grumbach start_hdr = hdr_page->pos; 15406eb5e529SEmmanuel Grumbach memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); 15416eb5e529SEmmanuel Grumbach hdr_page->pos += iv_len; 15426eb5e529SEmmanuel Grumbach 15436eb5e529SEmmanuel Grumbach /* 15446eb5e529SEmmanuel Grumbach * Pull the ieee80211 header + IV to be able to use TSO core, 15456eb5e529SEmmanuel Grumbach * we will restore it for the tx_status flow. 15466eb5e529SEmmanuel Grumbach */ 15476eb5e529SEmmanuel Grumbach skb_pull(skb, hdr_len + iv_len); 15486eb5e529SEmmanuel Grumbach 154905e5a7e5SJohannes Berg /* 155005e5a7e5SJohannes Berg * Remove the length of all the headers that we don't actually 155105e5a7e5SJohannes Berg * have in the MPDU by themselves, but that we duplicate into 155205e5a7e5SJohannes Berg * all the different MSDUs inside the A-MSDU. 155305e5a7e5SJohannes Berg */ 155405e5a7e5SJohannes Berg le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 155505e5a7e5SJohannes Berg 15566eb5e529SEmmanuel Grumbach tso_start(skb, &tso); 15576eb5e529SEmmanuel Grumbach 15586eb5e529SEmmanuel Grumbach while (total_len) { 15596eb5e529SEmmanuel Grumbach /* this is the data left for this subframe */ 15606eb5e529SEmmanuel Grumbach unsigned int data_left = 15616eb5e529SEmmanuel Grumbach min_t(unsigned int, mss, total_len); 15626eb5e529SEmmanuel Grumbach struct sk_buff *csum_skb = NULL; 15636eb5e529SEmmanuel Grumbach unsigned int hdr_tb_len; 15646eb5e529SEmmanuel Grumbach dma_addr_t hdr_tb_phys; 1565*59fa61f3SEmmanuel Grumbach u8 *subf_hdrs_start = hdr_page->pos; 15666eb5e529SEmmanuel Grumbach 15676eb5e529SEmmanuel Grumbach total_len -= data_left; 15686eb5e529SEmmanuel Grumbach 15696eb5e529SEmmanuel Grumbach memset(hdr_page->pos, 0, amsdu_pad); 15706eb5e529SEmmanuel Grumbach hdr_page->pos += amsdu_pad; 15716eb5e529SEmmanuel Grumbach amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 15726eb5e529SEmmanuel Grumbach data_left)) & 0x3; 15736eb5e529SEmmanuel Grumbach ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 15746eb5e529SEmmanuel Grumbach hdr_page->pos += ETH_ALEN; 15756eb5e529SEmmanuel Grumbach ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 15766eb5e529SEmmanuel Grumbach hdr_page->pos += ETH_ALEN; 15776eb5e529SEmmanuel Grumbach 15786eb5e529SEmmanuel Grumbach length = snap_ip_tcp_hdrlen + data_left; 15796eb5e529SEmmanuel Grumbach *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 15806eb5e529SEmmanuel Grumbach hdr_page->pos += sizeof(length); 15816eb5e529SEmmanuel Grumbach 15826eb5e529SEmmanuel Grumbach /* 15836eb5e529SEmmanuel Grumbach * This will copy the SNAP as well which will be considered 15846eb5e529SEmmanuel Grumbach * as MAC header. 15856eb5e529SEmmanuel Grumbach */ 15866eb5e529SEmmanuel Grumbach tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 15876eb5e529SEmmanuel Grumbach 15886eb5e529SEmmanuel Grumbach hdr_page->pos += snap_ip_tcp_hdrlen; 15896eb5e529SEmmanuel Grumbach 15906eb5e529SEmmanuel Grumbach hdr_tb_len = hdr_page->pos - start_hdr; 15916eb5e529SEmmanuel Grumbach hdr_tb_phys = dma_map_single(trans->dev, start_hdr, 15926eb5e529SEmmanuel Grumbach hdr_tb_len, DMA_TO_DEVICE); 15936eb5e529SEmmanuel Grumbach if (unlikely(dma_mapping_error(trans->dev, hdr_tb_phys))) { 15946eb5e529SEmmanuel Grumbach dev_kfree_skb(csum_skb); 15957d50d76eSJohannes Berg return -EINVAL; 15966eb5e529SEmmanuel Grumbach } 15976eb5e529SEmmanuel Grumbach iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys, 15986eb5e529SEmmanuel Grumbach hdr_tb_len, false); 1599bf77ee2eSSara Sharon trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, 16009b08ae22SJohannes Berg hdr_tb_phys, hdr_tb_len); 160105e5a7e5SJohannes Berg /* add this subframe's headers' length to the tx_cmd */ 160205e5a7e5SJohannes Berg le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 16036eb5e529SEmmanuel Grumbach 16046eb5e529SEmmanuel Grumbach /* prepare the start_hdr for the next subframe */ 16056eb5e529SEmmanuel Grumbach start_hdr = hdr_page->pos; 16066eb5e529SEmmanuel Grumbach 16076eb5e529SEmmanuel Grumbach /* put the payload */ 16086eb5e529SEmmanuel Grumbach while (data_left) { 16096eb5e529SEmmanuel Grumbach unsigned int size = min_t(unsigned int, tso.size, 16106eb5e529SEmmanuel Grumbach data_left); 16116eb5e529SEmmanuel Grumbach dma_addr_t tb_phys; 16126eb5e529SEmmanuel Grumbach 16136eb5e529SEmmanuel Grumbach tb_phys = dma_map_single(trans->dev, tso.data, 16146eb5e529SEmmanuel Grumbach size, DMA_TO_DEVICE); 16156eb5e529SEmmanuel Grumbach if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 16166eb5e529SEmmanuel Grumbach dev_kfree_skb(csum_skb); 16177d50d76eSJohannes Berg return -EINVAL; 16186eb5e529SEmmanuel Grumbach } 16196eb5e529SEmmanuel Grumbach 16206eb5e529SEmmanuel Grumbach iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 16216eb5e529SEmmanuel Grumbach size, false); 1622bf77ee2eSSara Sharon trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data, 16239b08ae22SJohannes Berg tb_phys, size); 16246eb5e529SEmmanuel Grumbach 16256eb5e529SEmmanuel Grumbach data_left -= size; 16266eb5e529SEmmanuel Grumbach tso_build_data(skb, &tso, size); 16276eb5e529SEmmanuel Grumbach } 16286eb5e529SEmmanuel Grumbach } 16296eb5e529SEmmanuel Grumbach 16306eb5e529SEmmanuel Grumbach /* re -add the WiFi header and IV */ 16316eb5e529SEmmanuel Grumbach skb_push(skb, hdr_len + iv_len); 16326eb5e529SEmmanuel Grumbach 16336eb5e529SEmmanuel Grumbach return 0; 16346eb5e529SEmmanuel Grumbach } 16356eb5e529SEmmanuel Grumbach #else /* CONFIG_INET */ 16366eb5e529SEmmanuel Grumbach static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 16376eb5e529SEmmanuel Grumbach struct iwl_txq *txq, u8 hdr_len, 16386eb5e529SEmmanuel Grumbach struct iwl_cmd_meta *out_meta, 1639a89c72ffSJohannes Berg struct iwl_device_tx_cmd *dev_cmd, 1640a89c72ffSJohannes Berg u16 tb1_len) 16416eb5e529SEmmanuel Grumbach { 16426eb5e529SEmmanuel Grumbach /* No A-MSDU without CONFIG_INET */ 16436eb5e529SEmmanuel Grumbach WARN_ON(1); 16446eb5e529SEmmanuel Grumbach 16456eb5e529SEmmanuel Grumbach return -1; 16466eb5e529SEmmanuel Grumbach } 16476eb5e529SEmmanuel Grumbach #endif /* CONFIG_INET */ 16486eb5e529SEmmanuel Grumbach 1649e705c121SKalle Valo int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, 1650a89c72ffSJohannes Berg struct iwl_device_tx_cmd *dev_cmd, int txq_id) 1651e705c121SKalle Valo { 1652e705c121SKalle Valo struct ieee80211_hdr *hdr; 1653e705c121SKalle Valo struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload; 1654e705c121SKalle Valo struct iwl_cmd_meta *out_meta; 1655e705c121SKalle Valo struct iwl_txq *txq; 1656e705c121SKalle Valo dma_addr_t tb0_phys, tb1_phys, scratch_phys; 1657e705c121SKalle Valo void *tb1_addr; 16584fe10bc6SSara Sharon void *tfd; 16593a0b2a42SEmmanuel Grumbach u16 len, tb1_len; 1660e705c121SKalle Valo bool wait_write_ptr; 1661e705c121SKalle Valo __le16 fc; 1662e705c121SKalle Valo u8 hdr_len; 1663e705c121SKalle Valo u16 wifi_seq; 1664c772a3d3SSara Sharon bool amsdu; 1665e705c121SKalle Valo 16664f4822b7SMordechay Goodstein txq = trans->txqs.txq[txq_id]; 1667e705c121SKalle Valo 16684f4822b7SMordechay Goodstein if (WARN_ONCE(!test_bit(txq_id, trans->txqs.queue_used), 1669e705c121SKalle Valo "TX on unused queue %d\n", txq_id)) 1670e705c121SKalle Valo return -EINVAL; 1671e705c121SKalle Valo 1672e705c121SKalle Valo if (skb_is_nonlinear(skb) && 1673885375d0SMordechay Goodstein skb_shinfo(skb)->nr_frags > IWL_TRANS_MAX_FRAGS(trans) && 1674e705c121SKalle Valo __skb_linearize(skb)) 1675e705c121SKalle Valo return -ENOMEM; 1676e705c121SKalle Valo 1677e705c121SKalle Valo /* mac80211 always puts the full header into the SKB's head, 1678e705c121SKalle Valo * so there's no need to check if it's readable there 1679e705c121SKalle Valo */ 1680e705c121SKalle Valo hdr = (struct ieee80211_hdr *)skb->data; 1681e705c121SKalle Valo fc = hdr->frame_control; 1682e705c121SKalle Valo hdr_len = ieee80211_hdrlen(fc); 1683e705c121SKalle Valo 1684e705c121SKalle Valo spin_lock(&txq->lock); 1685e705c121SKalle Valo 16860cd1ad2dSMordechay Goodstein if (iwl_txq_space(trans, txq) < txq->high_mark) { 16870cd1ad2dSMordechay Goodstein iwl_txq_stop(trans, txq); 16883955525dSEmmanuel Grumbach 16893955525dSEmmanuel Grumbach /* don't put the packet on the ring, if there is no room */ 16900cd1ad2dSMordechay Goodstein if (unlikely(iwl_txq_space(trans, txq) < 3)) { 1691a89c72ffSJohannes Berg struct iwl_device_tx_cmd **dev_cmd_ptr; 16923955525dSEmmanuel Grumbach 169321cb3222SJohannes Berg dev_cmd_ptr = (void *)((u8 *)skb->cb + 169422852fadSMordechay Goodstein trans->txqs.dev_cmd_offs); 169521cb3222SJohannes Berg 169621cb3222SJohannes Berg *dev_cmd_ptr = dev_cmd; 16973955525dSEmmanuel Grumbach __skb_queue_tail(&txq->overflow_q, skb); 16983955525dSEmmanuel Grumbach 16993955525dSEmmanuel Grumbach spin_unlock(&txq->lock); 17003955525dSEmmanuel Grumbach return 0; 17013955525dSEmmanuel Grumbach } 17023955525dSEmmanuel Grumbach } 17033955525dSEmmanuel Grumbach 1704e705c121SKalle Valo /* In AGG mode, the index in the ring must correspond to the WiFi 1705e705c121SKalle Valo * sequence number. This is a HW requirements to help the SCD to parse 1706e705c121SKalle Valo * the BA. 1707e705c121SKalle Valo * Check here that the packets are in the right place on the ring. 1708e705c121SKalle Valo */ 1709e705c121SKalle Valo wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl)); 1710e705c121SKalle Valo WARN_ONCE(txq->ampdu && 1711bb98ecd4SSara Sharon (wifi_seq & 0xff) != txq->write_ptr, 1712e705c121SKalle Valo "Q: %d WiFi Seq %d tfdNum %d", 1713bb98ecd4SSara Sharon txq_id, wifi_seq, txq->write_ptr); 1714e705c121SKalle Valo 1715e705c121SKalle Valo /* Set up driver data for this TFD */ 1716bb98ecd4SSara Sharon txq->entries[txq->write_ptr].skb = skb; 1717bb98ecd4SSara Sharon txq->entries[txq->write_ptr].cmd = dev_cmd; 1718e705c121SKalle Valo 1719e705c121SKalle Valo dev_cmd->hdr.sequence = 1720e705c121SKalle Valo cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 1721bb98ecd4SSara Sharon INDEX_TO_SEQ(txq->write_ptr))); 1722e705c121SKalle Valo 17230cd1ad2dSMordechay Goodstein tb0_phys = iwl_txq_get_first_tb_dma(txq, txq->write_ptr); 1724e705c121SKalle Valo scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) + 1725e705c121SKalle Valo offsetof(struct iwl_tx_cmd, scratch); 1726e705c121SKalle Valo 1727e705c121SKalle Valo tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys); 1728e705c121SKalle Valo tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys); 1729e705c121SKalle Valo 1730e705c121SKalle Valo /* Set up first empty entry in queue's array of Tx/cmd buffers */ 1731bb98ecd4SSara Sharon out_meta = &txq->entries[txq->write_ptr].meta; 1732e705c121SKalle Valo out_meta->flags = 0; 1733e705c121SKalle Valo 1734e705c121SKalle Valo /* 1735e705c121SKalle Valo * The second TB (tb1) points to the remainder of the TX command 1736e705c121SKalle Valo * and the 802.11 header - dword aligned size 1737e705c121SKalle Valo * (This calculation modifies the TX command, so do it before the 1738e705c121SKalle Valo * setup of the first TB) 1739e705c121SKalle Valo */ 1740e705c121SKalle Valo len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) + 17418de437c7SSara Sharon hdr_len - IWL_FIRST_TB_SIZE; 1742c772a3d3SSara Sharon /* do not align A-MSDU to dword as the subframe header aligns it */ 1743c772a3d3SSara Sharon amsdu = ieee80211_is_data_qos(fc) && 1744c772a3d3SSara Sharon (*ieee80211_get_qos_ctl(hdr) & 1745c772a3d3SSara Sharon IEEE80211_QOS_CTL_A_MSDU_PRESENT); 1746*59fa61f3SEmmanuel Grumbach if (!amsdu) { 1747e705c121SKalle Valo tb1_len = ALIGN(len, 4); 1748e705c121SKalle Valo /* Tell NIC about any 2-byte padding after MAC header */ 1749e705c121SKalle Valo if (tb1_len != len) 1750d172a5efSJohannes Berg tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD); 1751c772a3d3SSara Sharon } else { 1752c772a3d3SSara Sharon tb1_len = len; 1753c772a3d3SSara Sharon } 1754e705c121SKalle Valo 175505e5a7e5SJohannes Berg /* 175605e5a7e5SJohannes Berg * The first TB points to bi-directional DMA data, we'll 175705e5a7e5SJohannes Berg * memcpy the data into it later. 175805e5a7e5SJohannes Berg */ 1759e705c121SKalle Valo iwl_pcie_txq_build_tfd(trans, txq, tb0_phys, 17608de437c7SSara Sharon IWL_FIRST_TB_SIZE, true); 1761e705c121SKalle Valo 1762e705c121SKalle Valo /* there must be data left over for TB1 or this code must be changed */ 17638de437c7SSara Sharon BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE); 1764e705c121SKalle Valo 1765e705c121SKalle Valo /* map the data for TB1 */ 17668de437c7SSara Sharon tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 1767e705c121SKalle Valo tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 1768e705c121SKalle Valo if (unlikely(dma_mapping_error(trans->dev, tb1_phys))) 1769e705c121SKalle Valo goto out_err; 1770e705c121SKalle Valo iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false); 1771e705c121SKalle Valo 1772bf77ee2eSSara Sharon trace_iwlwifi_dev_tx(trans->dev, skb, 17730cd1ad2dSMordechay Goodstein iwl_txq_get_tfd(trans, txq, txq->write_ptr), 1774885375d0SMordechay Goodstein trans->txqs.tfd.size, 1775bf77ee2eSSara Sharon &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 1776bf77ee2eSSara Sharon hdr_len); 1777bf77ee2eSSara Sharon 1778bf1ad897SEliad Peller /* 1779bf1ad897SEliad Peller * If gso_size wasn't set, don't give the frame "amsdu treatment" 1780bf1ad897SEliad Peller * (adding subframes, etc.). 1781bf1ad897SEliad Peller * This can happen in some testing flows when the amsdu was already 1782bf1ad897SEliad Peller * pre-built, and we just need to send the resulting skb. 1783bf1ad897SEliad Peller */ 1784bf1ad897SEliad Peller if (amsdu && skb_shinfo(skb)->gso_size) { 17856eb5e529SEmmanuel Grumbach if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len, 17866eb5e529SEmmanuel Grumbach out_meta, dev_cmd, 17876eb5e529SEmmanuel Grumbach tb1_len))) 1788e705c121SKalle Valo goto out_err; 1789bb03927eSJohannes Berg } else { 17900044f171SJohannes Berg struct sk_buff *frag; 17910044f171SJohannes Berg 1792bb03927eSJohannes Berg if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len, 1793bb03927eSJohannes Berg out_meta))) 17946eb5e529SEmmanuel Grumbach goto out_err; 1795bb03927eSJohannes Berg 17960044f171SJohannes Berg skb_walk_frags(skb, frag) { 17970044f171SJohannes Berg if (unlikely(iwl_fill_data_tbs(trans, frag, txq, 0, 17980044f171SJohannes Berg out_meta))) 17990044f171SJohannes Berg goto out_err; 18000044f171SJohannes Berg } 18016eb5e529SEmmanuel Grumbach } 1802e705c121SKalle Valo 180305e5a7e5SJohannes Berg /* building the A-MSDU might have changed this data, so memcpy it now */ 1804c1f33442SLiad Kaufman memcpy(&txq->first_tb_bufs[txq->write_ptr], dev_cmd, IWL_FIRST_TB_SIZE); 180505e5a7e5SJohannes Berg 18060cd1ad2dSMordechay Goodstein tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr); 1807e705c121SKalle Valo /* Set up entry for this TFD in Tx byte-count array */ 18080179bfffSMordechay Goodstein iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len), 18090179bfffSMordechay Goodstein iwl_txq_gen1_tfd_get_num_tbs(trans, 18100179bfffSMordechay Goodstein tfd)); 1811e705c121SKalle Valo 1812e705c121SKalle Valo wait_write_ptr = ieee80211_has_morefrags(fc); 1813e705c121SKalle Valo 1814e705c121SKalle Valo /* start timer if queue currently empty */ 18150d52497aSEmmanuel Grumbach if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) { 1816e705c121SKalle Valo /* 1817e705c121SKalle Valo * If the TXQ is active, then set the timer, if not, 1818e705c121SKalle Valo * set the timer in remainder so that the timer will 1819e705c121SKalle Valo * be armed with the right value when the station will 1820e705c121SKalle Valo * wake up. 1821e705c121SKalle Valo */ 1822e705c121SKalle Valo if (!txq->frozen) 1823e705c121SKalle Valo mod_timer(&txq->stuck_timer, 1824e705c121SKalle Valo jiffies + txq->wd_timeout); 1825e705c121SKalle Valo else 1826e705c121SKalle Valo txq->frozen_expiry_remainder = txq->wd_timeout; 1827e705c121SKalle Valo } 1828e705c121SKalle Valo 1829e705c121SKalle Valo /* Tell device the write index *just past* this latest filled TFD */ 18300cd1ad2dSMordechay Goodstein txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 1831e705c121SKalle Valo if (!wait_write_ptr) 1832e705c121SKalle Valo iwl_pcie_txq_inc_wr_ptr(trans, txq); 1833e705c121SKalle Valo 1834e705c121SKalle Valo /* 1835e705c121SKalle Valo * At this point the frame is "transmitted" successfully 1836e705c121SKalle Valo * and we will get a TX status notification eventually. 1837e705c121SKalle Valo */ 1838e705c121SKalle Valo spin_unlock(&txq->lock); 1839e705c121SKalle Valo return 0; 1840e705c121SKalle Valo out_err: 18410179bfffSMordechay Goodstein iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, txq->write_ptr); 1842e705c121SKalle Valo spin_unlock(&txq->lock); 1843e705c121SKalle Valo return -1; 1844e705c121SKalle Valo } 1845