1 /****************************************************************************** 2 * 3 * This file is provided under a dual BSD/GPLv2 license. When using or 4 * redistributing this file, you may do so under either license. 5 * 6 * GPL LICENSE SUMMARY 7 * 8 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 9 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 10 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH 11 * Copyright(c) 2018 - 2020 Intel Corporation 12 * 13 * This program is free software; you can redistribute it and/or modify it 14 * under the terms of version 2 of the GNU General Public License as 15 * published by the Free Software Foundation. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 20 * more details. 21 * 22 * The full GNU General Public License is included in this distribution in the 23 * file called COPYING. 24 * 25 * Contact Information: 26 * Intel Linux Wireless <linuxwifi@intel.com> 27 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 28 * 29 * BSD LICENSE 30 * 31 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 32 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 33 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH 34 * Copyright(c) 2018 - 2020 Intel Corporation 35 * All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 41 * * Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * * Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in 45 * the documentation and/or other materials provided with the 46 * distribution. 47 * * Neither the name Intel Corporation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 52 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 53 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 54 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 55 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 56 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 57 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 58 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 59 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 60 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 61 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 62 * 63 *****************************************************************************/ 64 #include <linux/etherdevice.h> 65 #include <linux/ieee80211.h> 66 #include <linux/slab.h> 67 #include <linux/sched.h> 68 #include <net/ip6_checksum.h> 69 #include <net/tso.h> 70 71 #include "iwl-debug.h" 72 #include "iwl-csr.h" 73 #include "iwl-prph.h" 74 #include "iwl-io.h" 75 #include "iwl-scd.h" 76 #include "iwl-op-mode.h" 77 #include "internal.h" 78 #include "fw/api/tx.h" 79 80 /*************** DMA-QUEUE-GENERAL-FUNCTIONS ***** 81 * DMA services 82 * 83 * Theory of operation 84 * 85 * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer 86 * of buffer descriptors, each of which points to one or more data buffers for 87 * the device to read from or fill. Driver and device exchange status of each 88 * queue via "read" and "write" pointers. Driver keeps minimum of 2 empty 89 * entries in each circular buffer, to protect against confusing empty and full 90 * queue states. 91 * 92 * The device reads or writes the data in the queues via the device's several 93 * DMA/FIFO channels. Each queue is mapped to a single DMA channel. 94 * 95 * For Tx queue, there are low mark and high mark limits. If, after queuing 96 * the packet for Tx, free space become < low mark, Tx queue stopped. When 97 * reclaiming packets (on 'tx done IRQ), if free space become > high mark, 98 * Tx queue resumed. 99 * 100 ***************************************************/ 101 102 103 int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans, 104 struct iwl_dma_ptr *ptr, size_t size) 105 { 106 if (WARN_ON(ptr->addr)) 107 return -EINVAL; 108 109 ptr->addr = dma_alloc_coherent(trans->dev, size, 110 &ptr->dma, GFP_KERNEL); 111 if (!ptr->addr) 112 return -ENOMEM; 113 ptr->size = size; 114 return 0; 115 } 116 117 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr) 118 { 119 if (unlikely(!ptr->addr)) 120 return; 121 122 dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma); 123 memset(ptr, 0, sizeof(*ptr)); 124 } 125 126 /* 127 * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware 128 */ 129 static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans, 130 struct iwl_txq *txq) 131 { 132 u32 reg = 0; 133 int txq_id = txq->id; 134 135 lockdep_assert_held(&txq->lock); 136 137 /* 138 * explicitly wake up the NIC if: 139 * 1. shadow registers aren't enabled 140 * 2. NIC is woken up for CMD regardless of shadow outside this function 141 * 3. there is a chance that the NIC is asleep 142 */ 143 if (!trans->trans_cfg->base_params->shadow_reg_enable && 144 txq_id != trans->txqs.cmd.q_id && 145 test_bit(STATUS_TPOWER_PMI, &trans->status)) { 146 /* 147 * wake up nic if it's powered down ... 148 * uCode will wake up, and interrupt us again, so next 149 * time we'll skip this part. 150 */ 151 reg = iwl_read32(trans, CSR_UCODE_DRV_GP1); 152 153 if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) { 154 IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n", 155 txq_id, reg); 156 iwl_set_bit(trans, CSR_GP_CNTRL, 157 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 158 txq->need_update = true; 159 return; 160 } 161 } 162 163 /* 164 * if not in power-save mode, uCode will never sleep when we're 165 * trying to tx (during RFKILL, we're not trying to tx). 166 */ 167 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr); 168 if (!txq->block) 169 iwl_write32(trans, HBUS_TARG_WRPTR, 170 txq->write_ptr | (txq_id << 8)); 171 } 172 173 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans) 174 { 175 int i; 176 177 for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) { 178 struct iwl_txq *txq = trans->txqs.txq[i]; 179 180 if (!test_bit(i, trans->txqs.queue_used)) 181 continue; 182 183 spin_lock_bh(&txq->lock); 184 if (txq->need_update) { 185 iwl_pcie_txq_inc_wr_ptr(trans, txq); 186 txq->need_update = false; 187 } 188 spin_unlock_bh(&txq->lock); 189 } 190 } 191 192 static inline void iwl_pcie_tfd_set_tb(struct iwl_trans *trans, void *tfd, 193 u8 idx, dma_addr_t addr, u16 len) 194 { 195 struct iwl_tfd *tfd_fh = (void *)tfd; 196 struct iwl_tfd_tb *tb = &tfd_fh->tbs[idx]; 197 198 u16 hi_n_len = len << 4; 199 200 put_unaligned_le32(addr, &tb->lo); 201 hi_n_len |= iwl_get_dma_hi_addr(addr); 202 203 tb->hi_n_len = cpu_to_le16(hi_n_len); 204 205 tfd_fh->num_tbs = idx + 1; 206 } 207 208 /* 209 * iwl_pcie_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr] 210 * @trans - transport private data 211 * @txq - tx queue 212 * @dma_dir - the direction of the DMA mapping 213 * 214 * Does NOT advance any TFD circular buffer read/write indexes 215 * Does NOT free the TFD itself (which is within circular buffer) 216 */ 217 void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 218 { 219 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 220 * idx is bounded by n_window 221 */ 222 int rd_ptr = txq->read_ptr; 223 int idx = iwl_txq_get_cmd_index(txq, rd_ptr); 224 225 lockdep_assert_held(&txq->lock); 226 227 /* We have only q->n_window txq->entries, but we use 228 * TFD_QUEUE_SIZE_MAX tfds 229 */ 230 iwl_txq_gen1_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr); 231 232 /* free SKB */ 233 if (txq->entries) { 234 struct sk_buff *skb; 235 236 skb = txq->entries[idx].skb; 237 238 /* Can be called from irqs-disabled context 239 * If skb is not NULL, it means that the whole queue is being 240 * freed and that the queue is not empty - free the skb 241 */ 242 if (skb) { 243 iwl_op_mode_free_skb(trans->op_mode, skb); 244 txq->entries[idx].skb = NULL; 245 } 246 } 247 } 248 249 static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq, 250 dma_addr_t addr, u16 len, bool reset) 251 { 252 void *tfd; 253 u32 num_tbs; 254 255 tfd = txq->tfds + trans->txqs.tfd.size * txq->write_ptr; 256 257 if (reset) 258 memset(tfd, 0, trans->txqs.tfd.size); 259 260 num_tbs = iwl_txq_gen1_tfd_get_num_tbs(trans, tfd); 261 262 /* Each TFD can point to a maximum max_tbs Tx buffers */ 263 if (num_tbs >= trans->txqs.tfd.max_tbs) { 264 IWL_ERR(trans, "Error can not send more than %d chunks\n", 265 trans->txqs.tfd.max_tbs); 266 return -EINVAL; 267 } 268 269 if (WARN(addr & ~IWL_TX_DMA_MASK, 270 "Unaligned address = %llx\n", (unsigned long long)addr)) 271 return -EINVAL; 272 273 iwl_pcie_tfd_set_tb(trans, tfd, num_tbs, addr, len); 274 275 return num_tbs; 276 } 277 278 static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans) 279 { 280 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 281 282 lockdep_assert_held(&trans_pcie->reg_lock); 283 284 if (!trans->trans_cfg->base_params->apmg_wake_up_wa) 285 return; 286 if (WARN_ON(!trans_pcie->cmd_hold_nic_awake)) 287 return; 288 289 trans_pcie->cmd_hold_nic_awake = false; 290 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 291 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 292 } 293 294 /* 295 * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's 296 */ 297 static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id) 298 { 299 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 300 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 301 302 if (!txq) { 303 IWL_ERR(trans, "Trying to free a queue that wasn't allocated?\n"); 304 return; 305 } 306 307 spin_lock_bh(&txq->lock); 308 while (txq->write_ptr != txq->read_ptr) { 309 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 310 txq_id, txq->read_ptr); 311 312 if (txq_id != trans->txqs.cmd.q_id) { 313 struct sk_buff *skb = txq->entries[txq->read_ptr].skb; 314 315 if (WARN_ON_ONCE(!skb)) 316 continue; 317 318 iwl_txq_free_tso_page(trans, skb); 319 } 320 iwl_pcie_txq_free_tfd(trans, txq); 321 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 322 323 if (txq->read_ptr == txq->write_ptr) { 324 spin_lock(&trans_pcie->reg_lock); 325 if (txq_id == trans->txqs.cmd.q_id) 326 iwl_pcie_clear_cmd_in_flight(trans); 327 spin_unlock(&trans_pcie->reg_lock); 328 } 329 } 330 331 while (!skb_queue_empty(&txq->overflow_q)) { 332 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 333 334 iwl_op_mode_free_skb(trans->op_mode, skb); 335 } 336 337 spin_unlock_bh(&txq->lock); 338 339 /* just in case - this queue may have been stopped */ 340 iwl_wake_queue(trans, txq); 341 } 342 343 /* 344 * iwl_pcie_txq_free - Deallocate DMA queue. 345 * @txq: Transmit queue to deallocate. 346 * 347 * Empty queue by removing and destroying all BD's. 348 * Free all buffers. 349 * 0-fill, but do not free "txq" descriptor structure. 350 */ 351 static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id) 352 { 353 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 354 struct device *dev = trans->dev; 355 int i; 356 357 if (WARN_ON(!txq)) 358 return; 359 360 iwl_pcie_txq_unmap(trans, txq_id); 361 362 /* De-alloc array of command/tx buffers */ 363 if (txq_id == trans->txqs.cmd.q_id) 364 for (i = 0; i < txq->n_window; i++) { 365 kfree_sensitive(txq->entries[i].cmd); 366 kfree_sensitive(txq->entries[i].free_buf); 367 } 368 369 /* De-alloc circular buffer of TFDs */ 370 if (txq->tfds) { 371 dma_free_coherent(dev, 372 trans->txqs.tfd.size * 373 trans->trans_cfg->base_params->max_tfd_queue_size, 374 txq->tfds, txq->dma_addr); 375 txq->dma_addr = 0; 376 txq->tfds = NULL; 377 378 dma_free_coherent(dev, 379 sizeof(*txq->first_tb_bufs) * txq->n_window, 380 txq->first_tb_bufs, txq->first_tb_dma); 381 } 382 383 kfree(txq->entries); 384 txq->entries = NULL; 385 386 del_timer_sync(&txq->stuck_timer); 387 388 /* 0-fill queue descriptor structure */ 389 memset(txq, 0, sizeof(*txq)); 390 } 391 392 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr) 393 { 394 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 395 int nq = trans->trans_cfg->base_params->num_of_queues; 396 int chan; 397 u32 reg_val; 398 int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) - 399 SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32); 400 401 /* make sure all queue are not stopped/used */ 402 memset(trans->txqs.queue_stopped, 0, 403 sizeof(trans->txqs.queue_stopped)); 404 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 405 406 trans_pcie->scd_base_addr = 407 iwl_read_prph(trans, SCD_SRAM_BASE_ADDR); 408 409 WARN_ON(scd_base_addr != 0 && 410 scd_base_addr != trans_pcie->scd_base_addr); 411 412 /* reset context data, TX status and translation data */ 413 iwl_trans_write_mem(trans, trans_pcie->scd_base_addr + 414 SCD_CONTEXT_MEM_LOWER_BOUND, 415 NULL, clear_dwords); 416 417 iwl_write_prph(trans, SCD_DRAM_BASE_ADDR, 418 trans->txqs.scd_bc_tbls.dma >> 10); 419 420 /* The chain extension of the SCD doesn't work well. This feature is 421 * enabled by default by the HW, so we need to disable it manually. 422 */ 423 if (trans->trans_cfg->base_params->scd_chain_ext_wa) 424 iwl_write_prph(trans, SCD_CHAINEXT_EN, 0); 425 426 iwl_trans_ac_txq_enable(trans, trans->txqs.cmd.q_id, 427 trans->txqs.cmd.fifo, 428 trans->txqs.cmd.wdg_timeout); 429 430 /* Activate all Tx DMA/FIFO channels */ 431 iwl_scd_activate_fifos(trans); 432 433 /* Enable DMA channel */ 434 for (chan = 0; chan < FH_TCSR_CHNL_NUM; chan++) 435 iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan), 436 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | 437 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE); 438 439 /* Update FH chicken bits */ 440 reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG); 441 iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG, 442 reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); 443 444 /* Enable L1-Active */ 445 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000) 446 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG, 447 APMG_PCIDEV_STT_VAL_L1_ACT_DIS); 448 } 449 450 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans) 451 { 452 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 453 int txq_id; 454 455 /* 456 * we should never get here in gen2 trans mode return early to avoid 457 * having invalid accesses 458 */ 459 if (WARN_ON_ONCE(trans->trans_cfg->gen2)) 460 return; 461 462 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 463 txq_id++) { 464 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 465 if (trans->trans_cfg->use_tfh) 466 iwl_write_direct64(trans, 467 FH_MEM_CBBC_QUEUE(trans, txq_id), 468 txq->dma_addr); 469 else 470 iwl_write_direct32(trans, 471 FH_MEM_CBBC_QUEUE(trans, txq_id), 472 txq->dma_addr >> 8); 473 iwl_pcie_txq_unmap(trans, txq_id); 474 txq->read_ptr = 0; 475 txq->write_ptr = 0; 476 } 477 478 /* Tell NIC where to find the "keep warm" buffer */ 479 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 480 trans_pcie->kw.dma >> 4); 481 482 /* 483 * Send 0 as the scd_base_addr since the device may have be reset 484 * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will 485 * contain garbage. 486 */ 487 iwl_pcie_tx_start(trans, 0); 488 } 489 490 static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans) 491 { 492 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 493 unsigned long flags; 494 int ch, ret; 495 u32 mask = 0; 496 497 spin_lock(&trans_pcie->irq_lock); 498 499 if (!iwl_trans_grab_nic_access(trans, &flags)) 500 goto out; 501 502 /* Stop each Tx DMA channel */ 503 for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) { 504 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0); 505 mask |= FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch); 506 } 507 508 /* Wait for DMA channels to be idle */ 509 ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000); 510 if (ret < 0) 511 IWL_ERR(trans, 512 "Failing on timeout while stopping DMA channel %d [0x%08x]\n", 513 ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG)); 514 515 iwl_trans_release_nic_access(trans, &flags); 516 517 out: 518 spin_unlock(&trans_pcie->irq_lock); 519 } 520 521 /* 522 * iwl_pcie_tx_stop - Stop all Tx DMA channels 523 */ 524 int iwl_pcie_tx_stop(struct iwl_trans *trans) 525 { 526 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 527 int txq_id; 528 529 /* Turn off all Tx DMA fifos */ 530 iwl_scd_deactivate_fifos(trans); 531 532 /* Turn off all Tx DMA channels */ 533 iwl_pcie_tx_stop_fh(trans); 534 535 /* 536 * This function can be called before the op_mode disabled the 537 * queues. This happens when we have an rfkill interrupt. 538 * Since we stop Tx altogether - mark the queues as stopped. 539 */ 540 memset(trans->txqs.queue_stopped, 0, 541 sizeof(trans->txqs.queue_stopped)); 542 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 543 544 /* This can happen: start_hw, stop_device */ 545 if (!trans_pcie->txq_memory) 546 return 0; 547 548 /* Unmap DMA from host system and free skb's */ 549 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 550 txq_id++) 551 iwl_pcie_txq_unmap(trans, txq_id); 552 553 return 0; 554 } 555 556 /* 557 * iwl_trans_tx_free - Free TXQ Context 558 * 559 * Destroy all TX DMA queues and structures 560 */ 561 void iwl_pcie_tx_free(struct iwl_trans *trans) 562 { 563 int txq_id; 564 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 565 566 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 567 568 /* Tx queues */ 569 if (trans_pcie->txq_memory) { 570 for (txq_id = 0; 571 txq_id < trans->trans_cfg->base_params->num_of_queues; 572 txq_id++) { 573 iwl_pcie_txq_free(trans, txq_id); 574 trans->txqs.txq[txq_id] = NULL; 575 } 576 } 577 578 kfree(trans_pcie->txq_memory); 579 trans_pcie->txq_memory = NULL; 580 581 iwl_pcie_free_dma_ptr(trans, &trans_pcie->kw); 582 583 iwl_pcie_free_dma_ptr(trans, &trans->txqs.scd_bc_tbls); 584 } 585 586 /* 587 * iwl_pcie_tx_alloc - allocate TX context 588 * Allocate all Tx DMA structures and initialize them 589 */ 590 static int iwl_pcie_tx_alloc(struct iwl_trans *trans) 591 { 592 int ret; 593 int txq_id, slots_num; 594 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 595 u16 bc_tbls_size = trans->trans_cfg->base_params->num_of_queues; 596 597 if (WARN_ON(trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)) 598 return -EINVAL; 599 600 bc_tbls_size *= sizeof(struct iwlagn_scd_bc_tbl); 601 602 /*It is not allowed to alloc twice, so warn when this happens. 603 * We cannot rely on the previous allocation, so free and fail */ 604 if (WARN_ON(trans_pcie->txq_memory)) { 605 ret = -EINVAL; 606 goto error; 607 } 608 609 ret = iwl_pcie_alloc_dma_ptr(trans, &trans->txqs.scd_bc_tbls, 610 bc_tbls_size); 611 if (ret) { 612 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 613 goto error; 614 } 615 616 /* Alloc keep-warm buffer */ 617 ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE); 618 if (ret) { 619 IWL_ERR(trans, "Keep Warm allocation failed\n"); 620 goto error; 621 } 622 623 trans_pcie->txq_memory = 624 kcalloc(trans->trans_cfg->base_params->num_of_queues, 625 sizeof(struct iwl_txq), GFP_KERNEL); 626 if (!trans_pcie->txq_memory) { 627 IWL_ERR(trans, "Not enough memory for txq\n"); 628 ret = -ENOMEM; 629 goto error; 630 } 631 632 /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 633 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 634 txq_id++) { 635 bool cmd_queue = (txq_id == trans->txqs.cmd.q_id); 636 637 if (cmd_queue) 638 slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE, 639 trans->cfg->min_txq_size); 640 else 641 slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE, 642 trans->cfg->min_256_ba_txq_size); 643 trans->txqs.txq[txq_id] = &trans_pcie->txq_memory[txq_id]; 644 ret = iwl_txq_alloc(trans, trans->txqs.txq[txq_id], slots_num, 645 cmd_queue); 646 if (ret) { 647 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 648 goto error; 649 } 650 trans->txqs.txq[txq_id]->id = txq_id; 651 } 652 653 return 0; 654 655 error: 656 iwl_pcie_tx_free(trans); 657 658 return ret; 659 } 660 661 int iwl_pcie_tx_init(struct iwl_trans *trans) 662 { 663 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 664 int ret; 665 int txq_id, slots_num; 666 bool alloc = false; 667 668 if (!trans_pcie->txq_memory) { 669 ret = iwl_pcie_tx_alloc(trans); 670 if (ret) 671 goto error; 672 alloc = true; 673 } 674 675 spin_lock(&trans_pcie->irq_lock); 676 677 /* Turn off all Tx DMA fifos */ 678 iwl_scd_deactivate_fifos(trans); 679 680 /* Tell NIC where to find the "keep warm" buffer */ 681 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 682 trans_pcie->kw.dma >> 4); 683 684 spin_unlock(&trans_pcie->irq_lock); 685 686 /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 687 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 688 txq_id++) { 689 bool cmd_queue = (txq_id == trans->txqs.cmd.q_id); 690 691 if (cmd_queue) 692 slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE, 693 trans->cfg->min_txq_size); 694 else 695 slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE, 696 trans->cfg->min_256_ba_txq_size); 697 ret = iwl_txq_init(trans, trans->txqs.txq[txq_id], slots_num, 698 cmd_queue); 699 if (ret) { 700 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 701 goto error; 702 } 703 704 /* 705 * Tell nic where to find circular buffer of TFDs for a 706 * given Tx queue, and enable the DMA channel used for that 707 * queue. 708 * Circular buffer (TFD queue in DRAM) physical base address 709 */ 710 iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id), 711 trans->txqs.txq[txq_id]->dma_addr >> 8); 712 } 713 714 iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE); 715 if (trans->trans_cfg->base_params->num_of_queues > 20) 716 iwl_set_bits_prph(trans, SCD_GP_CTRL, 717 SCD_GP_CTRL_ENABLE_31_QUEUES); 718 719 return 0; 720 error: 721 /*Upon error, free only if we allocated something */ 722 if (alloc) 723 iwl_pcie_tx_free(trans); 724 return ret; 725 } 726 727 static inline void iwl_pcie_txq_progress(struct iwl_txq *txq) 728 { 729 lockdep_assert_held(&txq->lock); 730 731 if (!txq->wd_timeout) 732 return; 733 734 /* 735 * station is asleep and we send data - that must 736 * be uAPSD or PS-Poll. Don't rearm the timer. 737 */ 738 if (txq->frozen) 739 return; 740 741 /* 742 * if empty delete timer, otherwise move timer forward 743 * since we're making progress on this queue 744 */ 745 if (txq->read_ptr == txq->write_ptr) 746 del_timer(&txq->stuck_timer); 747 else 748 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 749 } 750 751 /* Frees buffers until index _not_ inclusive */ 752 void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, 753 struct sk_buff_head *skbs) 754 { 755 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 756 int tfd_num = iwl_txq_get_cmd_index(txq, ssn); 757 int read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr); 758 int last_to_free; 759 760 /* This function is not meant to release cmd queue*/ 761 if (WARN_ON(txq_id == trans->txqs.cmd.q_id)) 762 return; 763 764 spin_lock_bh(&txq->lock); 765 766 if (!test_bit(txq_id, trans->txqs.queue_used)) { 767 IWL_DEBUG_TX_QUEUES(trans, "Q %d inactive - ignoring idx %d\n", 768 txq_id, ssn); 769 goto out; 770 } 771 772 if (read_ptr == tfd_num) 773 goto out; 774 775 IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n", 776 txq_id, txq->read_ptr, tfd_num, ssn); 777 778 /*Since we free until index _not_ inclusive, the one before index is 779 * the last we will free. This one must be used */ 780 last_to_free = iwl_txq_dec_wrap(trans, tfd_num); 781 782 if (!iwl_txq_used(txq, last_to_free)) { 783 IWL_ERR(trans, 784 "%s: Read index for txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n", 785 __func__, txq_id, last_to_free, 786 trans->trans_cfg->base_params->max_tfd_queue_size, 787 txq->write_ptr, txq->read_ptr); 788 goto out; 789 } 790 791 if (WARN_ON(!skb_queue_empty(skbs))) 792 goto out; 793 794 for (; 795 read_ptr != tfd_num; 796 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr), 797 read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr)) { 798 struct sk_buff *skb = txq->entries[read_ptr].skb; 799 800 if (WARN_ON_ONCE(!skb)) 801 continue; 802 803 iwl_txq_free_tso_page(trans, skb); 804 805 __skb_queue_tail(skbs, skb); 806 807 txq->entries[read_ptr].skb = NULL; 808 809 if (!trans->trans_cfg->use_tfh) 810 iwl_txq_gen1_inval_byte_cnt_tbl(trans, txq); 811 812 iwl_pcie_txq_free_tfd(trans, txq); 813 } 814 815 iwl_pcie_txq_progress(txq); 816 817 if (iwl_txq_space(trans, txq) > txq->low_mark && 818 test_bit(txq_id, trans->txqs.queue_stopped)) { 819 struct sk_buff_head overflow_skbs; 820 821 __skb_queue_head_init(&overflow_skbs); 822 skb_queue_splice_init(&txq->overflow_q, &overflow_skbs); 823 824 /* 825 * We are going to transmit from the overflow queue. 826 * Remember this state so that wait_for_txq_empty will know we 827 * are adding more packets to the TFD queue. It cannot rely on 828 * the state of &txq->overflow_q, as we just emptied it, but 829 * haven't TXed the content yet. 830 */ 831 txq->overflow_tx = true; 832 833 /* 834 * This is tricky: we are in reclaim path which is non 835 * re-entrant, so noone will try to take the access the 836 * txq data from that path. We stopped tx, so we can't 837 * have tx as well. Bottom line, we can unlock and re-lock 838 * later. 839 */ 840 spin_unlock_bh(&txq->lock); 841 842 while (!skb_queue_empty(&overflow_skbs)) { 843 struct sk_buff *skb = __skb_dequeue(&overflow_skbs); 844 struct iwl_device_tx_cmd *dev_cmd_ptr; 845 846 dev_cmd_ptr = *(void **)((u8 *)skb->cb + 847 trans->txqs.dev_cmd_offs); 848 849 /* 850 * Note that we can very well be overflowing again. 851 * In that case, iwl_txq_space will be small again 852 * and we won't wake mac80211's queue. 853 */ 854 iwl_trans_tx(trans, skb, dev_cmd_ptr, txq_id); 855 } 856 857 if (iwl_txq_space(trans, txq) > txq->low_mark) 858 iwl_wake_queue(trans, txq); 859 860 spin_lock_bh(&txq->lock); 861 txq->overflow_tx = false; 862 } 863 864 out: 865 spin_unlock_bh(&txq->lock); 866 } 867 868 /* Set wr_ptr of specific device and txq */ 869 void iwl_trans_pcie_set_q_ptrs(struct iwl_trans *trans, int txq_id, int ptr) 870 { 871 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 872 873 spin_lock_bh(&txq->lock); 874 875 txq->write_ptr = ptr; 876 txq->read_ptr = txq->write_ptr; 877 878 spin_unlock_bh(&txq->lock); 879 } 880 881 static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans, 882 const struct iwl_host_cmd *cmd) 883 { 884 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 885 int ret; 886 887 lockdep_assert_held(&trans_pcie->reg_lock); 888 889 /* Make sure the NIC is still alive in the bus */ 890 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 891 return -ENODEV; 892 893 /* 894 * wake up the NIC to make sure that the firmware will see the host 895 * command - we will let the NIC sleep once all the host commands 896 * returned. This needs to be done only on NICs that have 897 * apmg_wake_up_wa set. 898 */ 899 if (trans->trans_cfg->base_params->apmg_wake_up_wa && 900 !trans_pcie->cmd_hold_nic_awake) { 901 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, 902 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 903 904 ret = iwl_poll_bit(trans, CSR_GP_CNTRL, 905 CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN, 906 (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | 907 CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 908 15000); 909 if (ret < 0) { 910 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 911 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 912 IWL_ERR(trans, "Failed to wake NIC for hcmd\n"); 913 return -EIO; 914 } 915 trans_pcie->cmd_hold_nic_awake = true; 916 } 917 918 return 0; 919 } 920 921 /* 922 * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd 923 * 924 * When FW advances 'R' index, all entries between old and new 'R' index 925 * need to be reclaimed. As result, some free space forms. If there is 926 * enough free space (> low mark), wake the stack that feeds us. 927 */ 928 static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx) 929 { 930 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 931 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 932 int nfreed = 0; 933 u16 r; 934 935 lockdep_assert_held(&txq->lock); 936 937 idx = iwl_txq_get_cmd_index(txq, idx); 938 r = iwl_txq_get_cmd_index(txq, txq->read_ptr); 939 940 if (idx >= trans->trans_cfg->base_params->max_tfd_queue_size || 941 (!iwl_txq_used(txq, idx))) { 942 WARN_ONCE(test_bit(txq_id, trans->txqs.queue_used), 943 "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n", 944 __func__, txq_id, idx, 945 trans->trans_cfg->base_params->max_tfd_queue_size, 946 txq->write_ptr, txq->read_ptr); 947 return; 948 } 949 950 for (idx = iwl_txq_inc_wrap(trans, idx); r != idx; 951 r = iwl_txq_inc_wrap(trans, r)) { 952 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 953 954 if (nfreed++ > 0) { 955 IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n", 956 idx, txq->write_ptr, r); 957 iwl_force_nmi(trans); 958 } 959 } 960 961 if (txq->read_ptr == txq->write_ptr) { 962 /* BHs are also disabled due to txq->lock */ 963 spin_lock(&trans_pcie->reg_lock); 964 iwl_pcie_clear_cmd_in_flight(trans); 965 spin_unlock(&trans_pcie->reg_lock); 966 } 967 968 iwl_pcie_txq_progress(txq); 969 } 970 971 static int iwl_pcie_txq_set_ratid_map(struct iwl_trans *trans, u16 ra_tid, 972 u16 txq_id) 973 { 974 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 975 u32 tbl_dw_addr; 976 u32 tbl_dw; 977 u16 scd_q2ratid; 978 979 scd_q2ratid = ra_tid & SCD_QUEUE_RA_TID_MAP_RATID_MSK; 980 981 tbl_dw_addr = trans_pcie->scd_base_addr + 982 SCD_TRANS_TBL_OFFSET_QUEUE(txq_id); 983 984 tbl_dw = iwl_trans_read_mem32(trans, tbl_dw_addr); 985 986 if (txq_id & 0x1) 987 tbl_dw = (scd_q2ratid << 16) | (tbl_dw & 0x0000FFFF); 988 else 989 tbl_dw = scd_q2ratid | (tbl_dw & 0xFFFF0000); 990 991 iwl_trans_write_mem32(trans, tbl_dw_addr, tbl_dw); 992 993 return 0; 994 } 995 996 /* Receiver address (actually, Rx station's index into station table), 997 * combined with Traffic ID (QOS priority), in format used by Tx Scheduler */ 998 #define BUILD_RAxTID(sta_id, tid) (((sta_id) << 4) + (tid)) 999 1000 bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn, 1001 const struct iwl_trans_txq_scd_cfg *cfg, 1002 unsigned int wdg_timeout) 1003 { 1004 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1005 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 1006 int fifo = -1; 1007 bool scd_bug = false; 1008 1009 if (test_and_set_bit(txq_id, trans->txqs.queue_used)) 1010 WARN_ONCE(1, "queue %d already used - expect issues", txq_id); 1011 1012 txq->wd_timeout = msecs_to_jiffies(wdg_timeout); 1013 1014 if (cfg) { 1015 fifo = cfg->fifo; 1016 1017 /* Disable the scheduler prior configuring the cmd queue */ 1018 if (txq_id == trans->txqs.cmd.q_id && 1019 trans_pcie->scd_set_active) 1020 iwl_scd_enable_set_active(trans, 0); 1021 1022 /* Stop this Tx queue before configuring it */ 1023 iwl_scd_txq_set_inactive(trans, txq_id); 1024 1025 /* Set this queue as a chain-building queue unless it is CMD */ 1026 if (txq_id != trans->txqs.cmd.q_id) 1027 iwl_scd_txq_set_chain(trans, txq_id); 1028 1029 if (cfg->aggregate) { 1030 u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid); 1031 1032 /* Map receiver-address / traffic-ID to this queue */ 1033 iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id); 1034 1035 /* enable aggregations for the queue */ 1036 iwl_scd_txq_enable_agg(trans, txq_id); 1037 txq->ampdu = true; 1038 } else { 1039 /* 1040 * disable aggregations for the queue, this will also 1041 * make the ra_tid mapping configuration irrelevant 1042 * since it is now a non-AGG queue. 1043 */ 1044 iwl_scd_txq_disable_agg(trans, txq_id); 1045 1046 ssn = txq->read_ptr; 1047 } 1048 } else { 1049 /* 1050 * If we need to move the SCD write pointer by steps of 1051 * 0x40, 0x80 or 0xc0, it gets stuck. Avoids this and let 1052 * the op_mode know by returning true later. 1053 * Do this only in case cfg is NULL since this trick can 1054 * be done only if we have DQA enabled which is true for mvm 1055 * only. And mvm never sets a cfg pointer. 1056 * This is really ugly, but this is the easiest way out for 1057 * this sad hardware issue. 1058 * This bug has been fixed on devices 9000 and up. 1059 */ 1060 scd_bug = !trans->trans_cfg->mq_rx_supported && 1061 !((ssn - txq->write_ptr) & 0x3f) && 1062 (ssn != txq->write_ptr); 1063 if (scd_bug) 1064 ssn++; 1065 } 1066 1067 /* Place first TFD at index corresponding to start sequence number. 1068 * Assumes that ssn_idx is valid (!= 0xFFF) */ 1069 txq->read_ptr = (ssn & 0xff); 1070 txq->write_ptr = (ssn & 0xff); 1071 iwl_write_direct32(trans, HBUS_TARG_WRPTR, 1072 (ssn & 0xff) | (txq_id << 8)); 1073 1074 if (cfg) { 1075 u8 frame_limit = cfg->frame_limit; 1076 1077 iwl_write_prph(trans, SCD_QUEUE_RDPTR(txq_id), ssn); 1078 1079 /* Set up Tx window size and frame limit for this queue */ 1080 iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr + 1081 SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0); 1082 iwl_trans_write_mem32(trans, 1083 trans_pcie->scd_base_addr + 1084 SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32), 1085 SCD_QUEUE_CTX_REG2_VAL(WIN_SIZE, frame_limit) | 1086 SCD_QUEUE_CTX_REG2_VAL(FRAME_LIMIT, frame_limit)); 1087 1088 /* Set up status area in SRAM, map to Tx DMA/FIFO, activate */ 1089 iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id), 1090 (1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) | 1091 (cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) | 1092 (1 << SCD_QUEUE_STTS_REG_POS_WSL) | 1093 SCD_QUEUE_STTS_REG_MSK); 1094 1095 /* enable the scheduler for this queue (only) */ 1096 if (txq_id == trans->txqs.cmd.q_id && 1097 trans_pcie->scd_set_active) 1098 iwl_scd_enable_set_active(trans, BIT(txq_id)); 1099 1100 IWL_DEBUG_TX_QUEUES(trans, 1101 "Activate queue %d on FIFO %d WrPtr: %d\n", 1102 txq_id, fifo, ssn & 0xff); 1103 } else { 1104 IWL_DEBUG_TX_QUEUES(trans, 1105 "Activate queue %d WrPtr: %d\n", 1106 txq_id, ssn & 0xff); 1107 } 1108 1109 return scd_bug; 1110 } 1111 1112 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id, 1113 bool shared_mode) 1114 { 1115 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 1116 1117 txq->ampdu = !shared_mode; 1118 } 1119 1120 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id, 1121 bool configure_scd) 1122 { 1123 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1124 u32 stts_addr = trans_pcie->scd_base_addr + 1125 SCD_TX_STTS_QUEUE_OFFSET(txq_id); 1126 static const u32 zero_val[4] = {}; 1127 1128 trans->txqs.txq[txq_id]->frozen_expiry_remainder = 0; 1129 trans->txqs.txq[txq_id]->frozen = false; 1130 1131 /* 1132 * Upon HW Rfkill - we stop the device, and then stop the queues 1133 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1134 * allow the op_mode to call txq_disable after it already called 1135 * stop_device. 1136 */ 1137 if (!test_and_clear_bit(txq_id, trans->txqs.queue_used)) { 1138 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1139 "queue %d not used", txq_id); 1140 return; 1141 } 1142 1143 if (configure_scd) { 1144 iwl_scd_txq_set_inactive(trans, txq_id); 1145 1146 iwl_trans_write_mem(trans, stts_addr, (void *)zero_val, 1147 ARRAY_SIZE(zero_val)); 1148 } 1149 1150 iwl_pcie_txq_unmap(trans, txq_id); 1151 trans->txqs.txq[txq_id]->ampdu = false; 1152 1153 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", txq_id); 1154 } 1155 1156 /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 1157 1158 /* 1159 * iwl_pcie_enqueue_hcmd - enqueue a uCode command 1160 * @priv: device private data point 1161 * @cmd: a pointer to the ucode command structure 1162 * 1163 * The function returns < 0 values to indicate the operation 1164 * failed. On success, it returns the index (>= 0) of command in the 1165 * command queue. 1166 */ 1167 static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, 1168 struct iwl_host_cmd *cmd) 1169 { 1170 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1171 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1172 struct iwl_device_cmd *out_cmd; 1173 struct iwl_cmd_meta *out_meta; 1174 void *dup_buf = NULL; 1175 dma_addr_t phys_addr; 1176 int idx; 1177 u16 copy_size, cmd_size, tb0_size; 1178 bool had_nocopy = false; 1179 u8 group_id = iwl_cmd_groupid(cmd->id); 1180 int i, ret; 1181 u32 cmd_pos; 1182 const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; 1183 u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; 1184 unsigned long flags; 1185 1186 if (WARN(!trans->wide_cmd_header && 1187 group_id > IWL_ALWAYS_LONG_GROUP, 1188 "unsupported wide command %#x\n", cmd->id)) 1189 return -EINVAL; 1190 1191 if (group_id != 0) { 1192 copy_size = sizeof(struct iwl_cmd_header_wide); 1193 cmd_size = sizeof(struct iwl_cmd_header_wide); 1194 } else { 1195 copy_size = sizeof(struct iwl_cmd_header); 1196 cmd_size = sizeof(struct iwl_cmd_header); 1197 } 1198 1199 /* need one for the header if the first is NOCOPY */ 1200 BUILD_BUG_ON(IWL_MAX_CMD_TBS_PER_TFD > IWL_NUM_OF_TBS - 1); 1201 1202 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1203 cmddata[i] = cmd->data[i]; 1204 cmdlen[i] = cmd->len[i]; 1205 1206 if (!cmd->len[i]) 1207 continue; 1208 1209 /* need at least IWL_FIRST_TB_SIZE copied */ 1210 if (copy_size < IWL_FIRST_TB_SIZE) { 1211 int copy = IWL_FIRST_TB_SIZE - copy_size; 1212 1213 if (copy > cmdlen[i]) 1214 copy = cmdlen[i]; 1215 cmdlen[i] -= copy; 1216 cmddata[i] += copy; 1217 copy_size += copy; 1218 } 1219 1220 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { 1221 had_nocopy = true; 1222 if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { 1223 idx = -EINVAL; 1224 goto free_dup_buf; 1225 } 1226 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { 1227 /* 1228 * This is also a chunk that isn't copied 1229 * to the static buffer so set had_nocopy. 1230 */ 1231 had_nocopy = true; 1232 1233 /* only allowed once */ 1234 if (WARN_ON(dup_buf)) { 1235 idx = -EINVAL; 1236 goto free_dup_buf; 1237 } 1238 1239 dup_buf = kmemdup(cmddata[i], cmdlen[i], 1240 GFP_ATOMIC); 1241 if (!dup_buf) 1242 return -ENOMEM; 1243 } else { 1244 /* NOCOPY must not be followed by normal! */ 1245 if (WARN_ON(had_nocopy)) { 1246 idx = -EINVAL; 1247 goto free_dup_buf; 1248 } 1249 copy_size += cmdlen[i]; 1250 } 1251 cmd_size += cmd->len[i]; 1252 } 1253 1254 /* 1255 * If any of the command structures end up being larger than 1256 * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically 1257 * allocated into separate TFDs, then we will need to 1258 * increase the size of the buffers. 1259 */ 1260 if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, 1261 "Command %s (%#x) is too large (%d bytes)\n", 1262 iwl_get_cmd_string(trans, cmd->id), 1263 cmd->id, copy_size)) { 1264 idx = -EINVAL; 1265 goto free_dup_buf; 1266 } 1267 1268 spin_lock_irqsave(&txq->lock, flags); 1269 1270 if (iwl_txq_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { 1271 spin_unlock_irqrestore(&txq->lock, flags); 1272 1273 IWL_ERR(trans, "No space in command queue\n"); 1274 iwl_op_mode_cmd_queue_full(trans->op_mode); 1275 idx = -ENOSPC; 1276 goto free_dup_buf; 1277 } 1278 1279 idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 1280 out_cmd = txq->entries[idx].cmd; 1281 out_meta = &txq->entries[idx].meta; 1282 1283 memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ 1284 if (cmd->flags & CMD_WANT_SKB) 1285 out_meta->source = cmd; 1286 1287 /* set up the header */ 1288 if (group_id != 0) { 1289 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); 1290 out_cmd->hdr_wide.group_id = group_id; 1291 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); 1292 out_cmd->hdr_wide.length = 1293 cpu_to_le16(cmd_size - 1294 sizeof(struct iwl_cmd_header_wide)); 1295 out_cmd->hdr_wide.reserved = 0; 1296 out_cmd->hdr_wide.sequence = 1297 cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) | 1298 INDEX_TO_SEQ(txq->write_ptr)); 1299 1300 cmd_pos = sizeof(struct iwl_cmd_header_wide); 1301 copy_size = sizeof(struct iwl_cmd_header_wide); 1302 } else { 1303 out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id); 1304 out_cmd->hdr.sequence = 1305 cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) | 1306 INDEX_TO_SEQ(txq->write_ptr)); 1307 out_cmd->hdr.group_id = 0; 1308 1309 cmd_pos = sizeof(struct iwl_cmd_header); 1310 copy_size = sizeof(struct iwl_cmd_header); 1311 } 1312 1313 /* and copy the data that needs to be copied */ 1314 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1315 int copy; 1316 1317 if (!cmd->len[i]) 1318 continue; 1319 1320 /* copy everything if not nocopy/dup */ 1321 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1322 IWL_HCMD_DFL_DUP))) { 1323 copy = cmd->len[i]; 1324 1325 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1326 cmd_pos += copy; 1327 copy_size += copy; 1328 continue; 1329 } 1330 1331 /* 1332 * Otherwise we need at least IWL_FIRST_TB_SIZE copied 1333 * in total (for bi-directional DMA), but copy up to what 1334 * we can fit into the payload for debug dump purposes. 1335 */ 1336 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); 1337 1338 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1339 cmd_pos += copy; 1340 1341 /* However, treat copy_size the proper way, we need it below */ 1342 if (copy_size < IWL_FIRST_TB_SIZE) { 1343 copy = IWL_FIRST_TB_SIZE - copy_size; 1344 1345 if (copy > cmd->len[i]) 1346 copy = cmd->len[i]; 1347 copy_size += copy; 1348 } 1349 } 1350 1351 IWL_DEBUG_HC(trans, 1352 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", 1353 iwl_get_cmd_string(trans, cmd->id), 1354 group_id, out_cmd->hdr.cmd, 1355 le16_to_cpu(out_cmd->hdr.sequence), 1356 cmd_size, txq->write_ptr, idx, trans->txqs.cmd.q_id); 1357 1358 /* start the TFD with the minimum copy bytes */ 1359 tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); 1360 memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size); 1361 iwl_pcie_txq_build_tfd(trans, txq, 1362 iwl_txq_get_first_tb_dma(txq, idx), 1363 tb0_size, true); 1364 1365 /* map first command fragment, if any remains */ 1366 if (copy_size > tb0_size) { 1367 phys_addr = dma_map_single(trans->dev, 1368 ((u8 *)&out_cmd->hdr) + tb0_size, 1369 copy_size - tb0_size, 1370 DMA_TO_DEVICE); 1371 if (dma_mapping_error(trans->dev, phys_addr)) { 1372 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, 1373 txq->write_ptr); 1374 idx = -ENOMEM; 1375 goto out; 1376 } 1377 1378 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, 1379 copy_size - tb0_size, false); 1380 } 1381 1382 /* map the remaining (adjusted) nocopy/dup fragments */ 1383 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1384 const void *data = cmddata[i]; 1385 1386 if (!cmdlen[i]) 1387 continue; 1388 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1389 IWL_HCMD_DFL_DUP))) 1390 continue; 1391 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) 1392 data = dup_buf; 1393 phys_addr = dma_map_single(trans->dev, (void *)data, 1394 cmdlen[i], DMA_TO_DEVICE); 1395 if (dma_mapping_error(trans->dev, phys_addr)) { 1396 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, 1397 txq->write_ptr); 1398 idx = -ENOMEM; 1399 goto out; 1400 } 1401 1402 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false); 1403 } 1404 1405 BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); 1406 out_meta->flags = cmd->flags; 1407 if (WARN_ON_ONCE(txq->entries[idx].free_buf)) 1408 kfree_sensitive(txq->entries[idx].free_buf); 1409 txq->entries[idx].free_buf = dup_buf; 1410 1411 trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); 1412 1413 /* start timer if queue currently empty */ 1414 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 1415 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 1416 1417 spin_lock(&trans_pcie->reg_lock); 1418 ret = iwl_pcie_set_cmd_in_flight(trans, cmd); 1419 if (ret < 0) { 1420 idx = ret; 1421 goto unlock_reg; 1422 } 1423 1424 /* Increment and update queue's write index */ 1425 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 1426 iwl_pcie_txq_inc_wr_ptr(trans, txq); 1427 1428 unlock_reg: 1429 spin_unlock(&trans_pcie->reg_lock); 1430 out: 1431 spin_unlock_irqrestore(&txq->lock, flags); 1432 free_dup_buf: 1433 if (idx < 0) 1434 kfree(dup_buf); 1435 return idx; 1436 } 1437 1438 /* 1439 * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them 1440 * @rxb: Rx buffer to reclaim 1441 */ 1442 void iwl_pcie_hcmd_complete(struct iwl_trans *trans, 1443 struct iwl_rx_cmd_buffer *rxb) 1444 { 1445 struct iwl_rx_packet *pkt = rxb_addr(rxb); 1446 u16 sequence = le16_to_cpu(pkt->hdr.sequence); 1447 u8 group_id; 1448 u32 cmd_id; 1449 int txq_id = SEQ_TO_QUEUE(sequence); 1450 int index = SEQ_TO_INDEX(sequence); 1451 int cmd_index; 1452 struct iwl_device_cmd *cmd; 1453 struct iwl_cmd_meta *meta; 1454 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1455 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1456 1457 /* If a Tx command is being handled and it isn't in the actual 1458 * command queue then there a command routing bug has been introduced 1459 * in the queue management code. */ 1460 if (WARN(txq_id != trans->txqs.cmd.q_id, 1461 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n", 1462 txq_id, trans->txqs.cmd.q_id, sequence, txq->read_ptr, 1463 txq->write_ptr)) { 1464 iwl_print_hex_error(trans, pkt, 32); 1465 return; 1466 } 1467 1468 spin_lock_bh(&txq->lock); 1469 1470 cmd_index = iwl_txq_get_cmd_index(txq, index); 1471 cmd = txq->entries[cmd_index].cmd; 1472 meta = &txq->entries[cmd_index].meta; 1473 group_id = cmd->hdr.group_id; 1474 cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0); 1475 1476 iwl_txq_gen1_tfd_unmap(trans, meta, txq, index); 1477 1478 /* Input error checking is done when commands are added to queue. */ 1479 if (meta->flags & CMD_WANT_SKB) { 1480 struct page *p = rxb_steal_page(rxb); 1481 1482 meta->source->resp_pkt = pkt; 1483 meta->source->_rx_page_addr = (unsigned long)page_address(p); 1484 meta->source->_rx_page_order = trans_pcie->rx_page_order; 1485 } 1486 1487 if (meta->flags & CMD_WANT_ASYNC_CALLBACK) 1488 iwl_op_mode_async_cb(trans->op_mode, cmd); 1489 1490 iwl_pcie_cmdq_reclaim(trans, txq_id, index); 1491 1492 if (!(meta->flags & CMD_ASYNC)) { 1493 if (!test_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) { 1494 IWL_WARN(trans, 1495 "HCMD_ACTIVE already clear for command %s\n", 1496 iwl_get_cmd_string(trans, cmd_id)); 1497 } 1498 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1499 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1500 iwl_get_cmd_string(trans, cmd_id)); 1501 wake_up(&trans_pcie->wait_command_queue); 1502 } 1503 1504 meta->flags = 0; 1505 1506 spin_unlock_bh(&txq->lock); 1507 } 1508 1509 #define HOST_COMPLETE_TIMEOUT (2 * HZ) 1510 1511 static int iwl_pcie_send_hcmd_async(struct iwl_trans *trans, 1512 struct iwl_host_cmd *cmd) 1513 { 1514 int ret; 1515 1516 /* An asynchronous command can not expect an SKB to be set. */ 1517 if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1518 return -EINVAL; 1519 1520 ret = iwl_pcie_enqueue_hcmd(trans, cmd); 1521 if (ret < 0) { 1522 IWL_ERR(trans, 1523 "Error sending %s: enqueue_hcmd failed: %d\n", 1524 iwl_get_cmd_string(trans, cmd->id), ret); 1525 return ret; 1526 } 1527 return 0; 1528 } 1529 1530 static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans, 1531 struct iwl_host_cmd *cmd) 1532 { 1533 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1534 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1535 int cmd_idx; 1536 int ret; 1537 1538 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", 1539 iwl_get_cmd_string(trans, cmd->id)); 1540 1541 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 1542 &trans->status), 1543 "Command %s: a command is already active!\n", 1544 iwl_get_cmd_string(trans, cmd->id))) 1545 return -EIO; 1546 1547 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", 1548 iwl_get_cmd_string(trans, cmd->id)); 1549 1550 cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd); 1551 if (cmd_idx < 0) { 1552 ret = cmd_idx; 1553 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1554 IWL_ERR(trans, 1555 "Error sending %s: enqueue_hcmd failed: %d\n", 1556 iwl_get_cmd_string(trans, cmd->id), ret); 1557 return ret; 1558 } 1559 1560 ret = wait_event_timeout(trans_pcie->wait_command_queue, 1561 !test_bit(STATUS_SYNC_HCMD_ACTIVE, 1562 &trans->status), 1563 HOST_COMPLETE_TIMEOUT); 1564 if (!ret) { 1565 IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 1566 iwl_get_cmd_string(trans, cmd->id), 1567 jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 1568 1569 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 1570 txq->read_ptr, txq->write_ptr); 1571 1572 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1573 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1574 iwl_get_cmd_string(trans, cmd->id)); 1575 ret = -ETIMEDOUT; 1576 1577 iwl_trans_pcie_sync_nmi(trans); 1578 goto cancel; 1579 } 1580 1581 if (test_bit(STATUS_FW_ERROR, &trans->status)) { 1582 iwl_trans_pcie_dump_regs(trans); 1583 IWL_ERR(trans, "FW error in SYNC CMD %s\n", 1584 iwl_get_cmd_string(trans, cmd->id)); 1585 dump_stack(); 1586 ret = -EIO; 1587 goto cancel; 1588 } 1589 1590 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1591 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1592 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 1593 ret = -ERFKILL; 1594 goto cancel; 1595 } 1596 1597 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 1598 IWL_ERR(trans, "Error: Response NULL in '%s'\n", 1599 iwl_get_cmd_string(trans, cmd->id)); 1600 ret = -EIO; 1601 goto cancel; 1602 } 1603 1604 return 0; 1605 1606 cancel: 1607 if (cmd->flags & CMD_WANT_SKB) { 1608 /* 1609 * Cancel the CMD_WANT_SKB flag for the cmd in the 1610 * TX cmd queue. Otherwise in case the cmd comes 1611 * in later, it will possibly set an invalid 1612 * address (cmd->meta.source). 1613 */ 1614 txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 1615 } 1616 1617 if (cmd->resp_pkt) { 1618 iwl_free_resp(cmd); 1619 cmd->resp_pkt = NULL; 1620 } 1621 1622 return ret; 1623 } 1624 1625 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) 1626 { 1627 /* Make sure the NIC is still alive in the bus */ 1628 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 1629 return -ENODEV; 1630 1631 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1632 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1633 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1634 cmd->id); 1635 return -ERFKILL; 1636 } 1637 1638 if (cmd->flags & CMD_ASYNC) 1639 return iwl_pcie_send_hcmd_async(trans, cmd); 1640 1641 /* We still can fail on RFKILL that can be asserted while we wait */ 1642 return iwl_pcie_send_hcmd_sync(trans, cmd); 1643 } 1644 1645 static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb, 1646 struct iwl_txq *txq, u8 hdr_len, 1647 struct iwl_cmd_meta *out_meta) 1648 { 1649 u16 head_tb_len; 1650 int i; 1651 1652 /* 1653 * Set up TFD's third entry to point directly to remainder 1654 * of skb's head, if any 1655 */ 1656 head_tb_len = skb_headlen(skb) - hdr_len; 1657 1658 if (head_tb_len > 0) { 1659 dma_addr_t tb_phys = dma_map_single(trans->dev, 1660 skb->data + hdr_len, 1661 head_tb_len, DMA_TO_DEVICE); 1662 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 1663 return -EINVAL; 1664 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb->data + hdr_len, 1665 tb_phys, head_tb_len); 1666 iwl_pcie_txq_build_tfd(trans, txq, tb_phys, head_tb_len, false); 1667 } 1668 1669 /* set up the remaining entries to point to the data */ 1670 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1671 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1672 dma_addr_t tb_phys; 1673 int tb_idx; 1674 1675 if (!skb_frag_size(frag)) 1676 continue; 1677 1678 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 1679 skb_frag_size(frag), DMA_TO_DEVICE); 1680 1681 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 1682 return -EINVAL; 1683 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb_frag_address(frag), 1684 tb_phys, skb_frag_size(frag)); 1685 tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 1686 skb_frag_size(frag), false); 1687 if (tb_idx < 0) 1688 return tb_idx; 1689 1690 out_meta->tbs |= BIT(tb_idx); 1691 } 1692 1693 return 0; 1694 } 1695 1696 #ifdef CONFIG_INET 1697 static void iwl_compute_pseudo_hdr_csum(void *iph, struct tcphdr *tcph, 1698 bool ipv6, unsigned int len) 1699 { 1700 if (ipv6) { 1701 struct ipv6hdr *iphv6 = iph; 1702 1703 tcph->check = ~csum_ipv6_magic(&iphv6->saddr, &iphv6->daddr, 1704 len + tcph->doff * 4, 1705 IPPROTO_TCP, 0); 1706 } else { 1707 struct iphdr *iphv4 = iph; 1708 1709 ip_send_check(iphv4); 1710 tcph->check = ~csum_tcpudp_magic(iphv4->saddr, iphv4->daddr, 1711 len + tcph->doff * 4, 1712 IPPROTO_TCP, 0); 1713 } 1714 } 1715 1716 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 1717 struct iwl_txq *txq, u8 hdr_len, 1718 struct iwl_cmd_meta *out_meta, 1719 struct iwl_device_tx_cmd *dev_cmd, 1720 u16 tb1_len) 1721 { 1722 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 1723 struct iwl_trans_pcie *trans_pcie = 1724 IWL_TRANS_GET_PCIE_TRANS(txq->trans); 1725 struct ieee80211_hdr *hdr = (void *)skb->data; 1726 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 1727 unsigned int mss = skb_shinfo(skb)->gso_size; 1728 u16 length, iv_len, amsdu_pad; 1729 u8 *start_hdr; 1730 struct iwl_tso_hdr_page *hdr_page; 1731 struct tso_t tso; 1732 1733 /* if the packet is protected, then it must be CCMP or GCMP */ 1734 BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN); 1735 iv_len = ieee80211_has_protected(hdr->frame_control) ? 1736 IEEE80211_CCMP_HDR_LEN : 0; 1737 1738 trace_iwlwifi_dev_tx(trans->dev, skb, 1739 iwl_txq_get_tfd(trans, txq, txq->write_ptr), 1740 trans->txqs.tfd.size, 1741 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 0); 1742 1743 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 1744 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 1745 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; 1746 amsdu_pad = 0; 1747 1748 /* total amount of header we may need for this A-MSDU */ 1749 hdr_room = DIV_ROUND_UP(total_len, mss) * 1750 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len; 1751 1752 /* Our device supports 9 segments at most, it will fit in 1 page */ 1753 hdr_page = get_page_hdr(trans, hdr_room, skb); 1754 if (!hdr_page) 1755 return -ENOMEM; 1756 1757 start_hdr = hdr_page->pos; 1758 memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); 1759 hdr_page->pos += iv_len; 1760 1761 /* 1762 * Pull the ieee80211 header + IV to be able to use TSO core, 1763 * we will restore it for the tx_status flow. 1764 */ 1765 skb_pull(skb, hdr_len + iv_len); 1766 1767 /* 1768 * Remove the length of all the headers that we don't actually 1769 * have in the MPDU by themselves, but that we duplicate into 1770 * all the different MSDUs inside the A-MSDU. 1771 */ 1772 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 1773 1774 tso_start(skb, &tso); 1775 1776 while (total_len) { 1777 /* this is the data left for this subframe */ 1778 unsigned int data_left = 1779 min_t(unsigned int, mss, total_len); 1780 struct sk_buff *csum_skb = NULL; 1781 unsigned int hdr_tb_len; 1782 dma_addr_t hdr_tb_phys; 1783 struct tcphdr *tcph; 1784 u8 *iph, *subf_hdrs_start = hdr_page->pos; 1785 1786 total_len -= data_left; 1787 1788 memset(hdr_page->pos, 0, amsdu_pad); 1789 hdr_page->pos += amsdu_pad; 1790 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 1791 data_left)) & 0x3; 1792 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 1793 hdr_page->pos += ETH_ALEN; 1794 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 1795 hdr_page->pos += ETH_ALEN; 1796 1797 length = snap_ip_tcp_hdrlen + data_left; 1798 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 1799 hdr_page->pos += sizeof(length); 1800 1801 /* 1802 * This will copy the SNAP as well which will be considered 1803 * as MAC header. 1804 */ 1805 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 1806 iph = hdr_page->pos + 8; 1807 tcph = (void *)(iph + ip_hdrlen); 1808 1809 /* For testing on current hardware only */ 1810 if (trans_pcie->sw_csum_tx) { 1811 csum_skb = alloc_skb(data_left + tcp_hdrlen(skb), 1812 GFP_ATOMIC); 1813 if (!csum_skb) 1814 return -ENOMEM; 1815 1816 iwl_compute_pseudo_hdr_csum(iph, tcph, 1817 skb->protocol == 1818 htons(ETH_P_IPV6), 1819 data_left); 1820 1821 skb_put_data(csum_skb, tcph, tcp_hdrlen(skb)); 1822 skb_reset_transport_header(csum_skb); 1823 csum_skb->csum_start = 1824 (unsigned char *)tcp_hdr(csum_skb) - 1825 csum_skb->head; 1826 } 1827 1828 hdr_page->pos += snap_ip_tcp_hdrlen; 1829 1830 hdr_tb_len = hdr_page->pos - start_hdr; 1831 hdr_tb_phys = dma_map_single(trans->dev, start_hdr, 1832 hdr_tb_len, DMA_TO_DEVICE); 1833 if (unlikely(dma_mapping_error(trans->dev, hdr_tb_phys))) { 1834 dev_kfree_skb(csum_skb); 1835 return -EINVAL; 1836 } 1837 iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys, 1838 hdr_tb_len, false); 1839 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, 1840 hdr_tb_phys, hdr_tb_len); 1841 /* add this subframe's headers' length to the tx_cmd */ 1842 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 1843 1844 /* prepare the start_hdr for the next subframe */ 1845 start_hdr = hdr_page->pos; 1846 1847 /* put the payload */ 1848 while (data_left) { 1849 unsigned int size = min_t(unsigned int, tso.size, 1850 data_left); 1851 dma_addr_t tb_phys; 1852 1853 if (trans_pcie->sw_csum_tx) 1854 skb_put_data(csum_skb, tso.data, size); 1855 1856 tb_phys = dma_map_single(trans->dev, tso.data, 1857 size, DMA_TO_DEVICE); 1858 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 1859 dev_kfree_skb(csum_skb); 1860 return -EINVAL; 1861 } 1862 1863 iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 1864 size, false); 1865 trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data, 1866 tb_phys, size); 1867 1868 data_left -= size; 1869 tso_build_data(skb, &tso, size); 1870 } 1871 1872 /* For testing on early hardware only */ 1873 if (trans_pcie->sw_csum_tx) { 1874 __wsum csum; 1875 1876 csum = skb_checksum(csum_skb, 1877 skb_checksum_start_offset(csum_skb), 1878 csum_skb->len - 1879 skb_checksum_start_offset(csum_skb), 1880 0); 1881 dev_kfree_skb(csum_skb); 1882 dma_sync_single_for_cpu(trans->dev, hdr_tb_phys, 1883 hdr_tb_len, DMA_TO_DEVICE); 1884 tcph->check = csum_fold(csum); 1885 dma_sync_single_for_device(trans->dev, hdr_tb_phys, 1886 hdr_tb_len, DMA_TO_DEVICE); 1887 } 1888 } 1889 1890 /* re -add the WiFi header and IV */ 1891 skb_push(skb, hdr_len + iv_len); 1892 1893 return 0; 1894 } 1895 #else /* CONFIG_INET */ 1896 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 1897 struct iwl_txq *txq, u8 hdr_len, 1898 struct iwl_cmd_meta *out_meta, 1899 struct iwl_device_tx_cmd *dev_cmd, 1900 u16 tb1_len) 1901 { 1902 /* No A-MSDU without CONFIG_INET */ 1903 WARN_ON(1); 1904 1905 return -1; 1906 } 1907 #endif /* CONFIG_INET */ 1908 1909 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, 1910 struct iwl_device_tx_cmd *dev_cmd, int txq_id) 1911 { 1912 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1913 struct ieee80211_hdr *hdr; 1914 struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload; 1915 struct iwl_cmd_meta *out_meta; 1916 struct iwl_txq *txq; 1917 dma_addr_t tb0_phys, tb1_phys, scratch_phys; 1918 void *tb1_addr; 1919 void *tfd; 1920 u16 len, tb1_len; 1921 bool wait_write_ptr; 1922 __le16 fc; 1923 u8 hdr_len; 1924 u16 wifi_seq; 1925 bool amsdu; 1926 1927 txq = trans->txqs.txq[txq_id]; 1928 1929 if (WARN_ONCE(!test_bit(txq_id, trans->txqs.queue_used), 1930 "TX on unused queue %d\n", txq_id)) 1931 return -EINVAL; 1932 1933 if (unlikely(trans_pcie->sw_csum_tx && 1934 skb->ip_summed == CHECKSUM_PARTIAL)) { 1935 int offs = skb_checksum_start_offset(skb); 1936 int csum_offs = offs + skb->csum_offset; 1937 __wsum csum; 1938 1939 if (skb_ensure_writable(skb, csum_offs + sizeof(__sum16))) 1940 return -1; 1941 1942 csum = skb_checksum(skb, offs, skb->len - offs, 0); 1943 *(__sum16 *)(skb->data + csum_offs) = csum_fold(csum); 1944 1945 skb->ip_summed = CHECKSUM_UNNECESSARY; 1946 } 1947 1948 if (skb_is_nonlinear(skb) && 1949 skb_shinfo(skb)->nr_frags > IWL_TRANS_MAX_FRAGS(trans) && 1950 __skb_linearize(skb)) 1951 return -ENOMEM; 1952 1953 /* mac80211 always puts the full header into the SKB's head, 1954 * so there's no need to check if it's readable there 1955 */ 1956 hdr = (struct ieee80211_hdr *)skb->data; 1957 fc = hdr->frame_control; 1958 hdr_len = ieee80211_hdrlen(fc); 1959 1960 spin_lock(&txq->lock); 1961 1962 if (iwl_txq_space(trans, txq) < txq->high_mark) { 1963 iwl_txq_stop(trans, txq); 1964 1965 /* don't put the packet on the ring, if there is no room */ 1966 if (unlikely(iwl_txq_space(trans, txq) < 3)) { 1967 struct iwl_device_tx_cmd **dev_cmd_ptr; 1968 1969 dev_cmd_ptr = (void *)((u8 *)skb->cb + 1970 trans->txqs.dev_cmd_offs); 1971 1972 *dev_cmd_ptr = dev_cmd; 1973 __skb_queue_tail(&txq->overflow_q, skb); 1974 1975 spin_unlock(&txq->lock); 1976 return 0; 1977 } 1978 } 1979 1980 /* In AGG mode, the index in the ring must correspond to the WiFi 1981 * sequence number. This is a HW requirements to help the SCD to parse 1982 * the BA. 1983 * Check here that the packets are in the right place on the ring. 1984 */ 1985 wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl)); 1986 WARN_ONCE(txq->ampdu && 1987 (wifi_seq & 0xff) != txq->write_ptr, 1988 "Q: %d WiFi Seq %d tfdNum %d", 1989 txq_id, wifi_seq, txq->write_ptr); 1990 1991 /* Set up driver data for this TFD */ 1992 txq->entries[txq->write_ptr].skb = skb; 1993 txq->entries[txq->write_ptr].cmd = dev_cmd; 1994 1995 dev_cmd->hdr.sequence = 1996 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 1997 INDEX_TO_SEQ(txq->write_ptr))); 1998 1999 tb0_phys = iwl_txq_get_first_tb_dma(txq, txq->write_ptr); 2000 scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) + 2001 offsetof(struct iwl_tx_cmd, scratch); 2002 2003 tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys); 2004 tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys); 2005 2006 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 2007 out_meta = &txq->entries[txq->write_ptr].meta; 2008 out_meta->flags = 0; 2009 2010 /* 2011 * The second TB (tb1) points to the remainder of the TX command 2012 * and the 802.11 header - dword aligned size 2013 * (This calculation modifies the TX command, so do it before the 2014 * setup of the first TB) 2015 */ 2016 len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) + 2017 hdr_len - IWL_FIRST_TB_SIZE; 2018 /* do not align A-MSDU to dword as the subframe header aligns it */ 2019 amsdu = ieee80211_is_data_qos(fc) && 2020 (*ieee80211_get_qos_ctl(hdr) & 2021 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 2022 if (trans_pcie->sw_csum_tx || !amsdu) { 2023 tb1_len = ALIGN(len, 4); 2024 /* Tell NIC about any 2-byte padding after MAC header */ 2025 if (tb1_len != len) 2026 tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD); 2027 } else { 2028 tb1_len = len; 2029 } 2030 2031 /* 2032 * The first TB points to bi-directional DMA data, we'll 2033 * memcpy the data into it later. 2034 */ 2035 iwl_pcie_txq_build_tfd(trans, txq, tb0_phys, 2036 IWL_FIRST_TB_SIZE, true); 2037 2038 /* there must be data left over for TB1 or this code must be changed */ 2039 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE); 2040 2041 /* map the data for TB1 */ 2042 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 2043 tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 2044 if (unlikely(dma_mapping_error(trans->dev, tb1_phys))) 2045 goto out_err; 2046 iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false); 2047 2048 trace_iwlwifi_dev_tx(trans->dev, skb, 2049 iwl_txq_get_tfd(trans, txq, txq->write_ptr), 2050 trans->txqs.tfd.size, 2051 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 2052 hdr_len); 2053 2054 /* 2055 * If gso_size wasn't set, don't give the frame "amsdu treatment" 2056 * (adding subframes, etc.). 2057 * This can happen in some testing flows when the amsdu was already 2058 * pre-built, and we just need to send the resulting skb. 2059 */ 2060 if (amsdu && skb_shinfo(skb)->gso_size) { 2061 if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len, 2062 out_meta, dev_cmd, 2063 tb1_len))) 2064 goto out_err; 2065 } else { 2066 struct sk_buff *frag; 2067 2068 if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len, 2069 out_meta))) 2070 goto out_err; 2071 2072 skb_walk_frags(skb, frag) { 2073 if (unlikely(iwl_fill_data_tbs(trans, frag, txq, 0, 2074 out_meta))) 2075 goto out_err; 2076 } 2077 } 2078 2079 /* building the A-MSDU might have changed this data, so memcpy it now */ 2080 memcpy(&txq->first_tb_bufs[txq->write_ptr], dev_cmd, IWL_FIRST_TB_SIZE); 2081 2082 tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr); 2083 /* Set up entry for this TFD in Tx byte-count array */ 2084 iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len), 2085 iwl_txq_gen1_tfd_get_num_tbs(trans, 2086 tfd)); 2087 2088 wait_write_ptr = ieee80211_has_morefrags(fc); 2089 2090 /* start timer if queue currently empty */ 2091 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) { 2092 /* 2093 * If the TXQ is active, then set the timer, if not, 2094 * set the timer in remainder so that the timer will 2095 * be armed with the right value when the station will 2096 * wake up. 2097 */ 2098 if (!txq->frozen) 2099 mod_timer(&txq->stuck_timer, 2100 jiffies + txq->wd_timeout); 2101 else 2102 txq->frozen_expiry_remainder = txq->wd_timeout; 2103 } 2104 2105 /* Tell device the write index *just past* this latest filled TFD */ 2106 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 2107 if (!wait_write_ptr) 2108 iwl_pcie_txq_inc_wr_ptr(trans, txq); 2109 2110 /* 2111 * At this point the frame is "transmitted" successfully 2112 * and we will get a TX status notification eventually. 2113 */ 2114 spin_unlock(&txq->lock); 2115 return 0; 2116 out_err: 2117 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, txq->write_ptr); 2118 spin_unlock(&txq->lock); 2119 return -1; 2120 } 2121