1 /****************************************************************************** 2 * 3 * This file is provided under a dual BSD/GPLv2 license. When using or 4 * redistributing this file, you may do so under either license. 5 * 6 * GPL LICENSE SUMMARY 7 * 8 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 9 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 10 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH 11 * Copyright(c) 2018 - 2020 Intel Corporation 12 * 13 * This program is free software; you can redistribute it and/or modify it 14 * under the terms of version 2 of the GNU General Public License as 15 * published by the Free Software Foundation. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 20 * more details. 21 * 22 * The full GNU General Public License is included in this distribution in the 23 * file called COPYING. 24 * 25 * Contact Information: 26 * Intel Linux Wireless <linuxwifi@intel.com> 27 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 28 * 29 * BSD LICENSE 30 * 31 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 32 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 33 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH 34 * Copyright(c) 2018 - 2020 Intel Corporation 35 * All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 41 * * Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * * Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in 45 * the documentation and/or other materials provided with the 46 * distribution. 47 * * Neither the name Intel Corporation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 52 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 53 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 54 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 55 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 56 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 57 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 58 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 59 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 60 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 61 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 62 * 63 *****************************************************************************/ 64 #include <linux/etherdevice.h> 65 #include <linux/ieee80211.h> 66 #include <linux/slab.h> 67 #include <linux/sched.h> 68 #include <net/ip6_checksum.h> 69 #include <net/tso.h> 70 71 #include "iwl-debug.h" 72 #include "iwl-csr.h" 73 #include "iwl-prph.h" 74 #include "iwl-io.h" 75 #include "iwl-scd.h" 76 #include "iwl-op-mode.h" 77 #include "internal.h" 78 #include "fw/api/tx.h" 79 80 /*************** DMA-QUEUE-GENERAL-FUNCTIONS ***** 81 * DMA services 82 * 83 * Theory of operation 84 * 85 * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer 86 * of buffer descriptors, each of which points to one or more data buffers for 87 * the device to read from or fill. Driver and device exchange status of each 88 * queue via "read" and "write" pointers. Driver keeps minimum of 2 empty 89 * entries in each circular buffer, to protect against confusing empty and full 90 * queue states. 91 * 92 * The device reads or writes the data in the queues via the device's several 93 * DMA/FIFO channels. Each queue is mapped to a single DMA channel. 94 * 95 * For Tx queue, there are low mark and high mark limits. If, after queuing 96 * the packet for Tx, free space become < low mark, Tx queue stopped. When 97 * reclaiming packets (on 'tx done IRQ), if free space become > high mark, 98 * Tx queue resumed. 99 * 100 ***************************************************/ 101 102 103 int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans, 104 struct iwl_dma_ptr *ptr, size_t size) 105 { 106 if (WARN_ON(ptr->addr)) 107 return -EINVAL; 108 109 ptr->addr = dma_alloc_coherent(trans->dev, size, 110 &ptr->dma, GFP_KERNEL); 111 if (!ptr->addr) 112 return -ENOMEM; 113 ptr->size = size; 114 return 0; 115 } 116 117 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr) 118 { 119 if (unlikely(!ptr->addr)) 120 return; 121 122 dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma); 123 memset(ptr, 0, sizeof(*ptr)); 124 } 125 126 /* 127 * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware 128 */ 129 static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans, 130 struct iwl_txq *txq) 131 { 132 u32 reg = 0; 133 int txq_id = txq->id; 134 135 lockdep_assert_held(&txq->lock); 136 137 /* 138 * explicitly wake up the NIC if: 139 * 1. shadow registers aren't enabled 140 * 2. NIC is woken up for CMD regardless of shadow outside this function 141 * 3. there is a chance that the NIC is asleep 142 */ 143 if (!trans->trans_cfg->base_params->shadow_reg_enable && 144 txq_id != trans->txqs.cmd.q_id && 145 test_bit(STATUS_TPOWER_PMI, &trans->status)) { 146 /* 147 * wake up nic if it's powered down ... 148 * uCode will wake up, and interrupt us again, so next 149 * time we'll skip this part. 150 */ 151 reg = iwl_read32(trans, CSR_UCODE_DRV_GP1); 152 153 if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) { 154 IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n", 155 txq_id, reg); 156 iwl_set_bit(trans, CSR_GP_CNTRL, 157 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 158 txq->need_update = true; 159 return; 160 } 161 } 162 163 /* 164 * if not in power-save mode, uCode will never sleep when we're 165 * trying to tx (during RFKILL, we're not trying to tx). 166 */ 167 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr); 168 if (!txq->block) 169 iwl_write32(trans, HBUS_TARG_WRPTR, 170 txq->write_ptr | (txq_id << 8)); 171 } 172 173 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans) 174 { 175 int i; 176 177 for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) { 178 struct iwl_txq *txq = trans->txqs.txq[i]; 179 180 if (!test_bit(i, trans->txqs.queue_used)) 181 continue; 182 183 spin_lock_bh(&txq->lock); 184 if (txq->need_update) { 185 iwl_pcie_txq_inc_wr_ptr(trans, txq); 186 txq->need_update = false; 187 } 188 spin_unlock_bh(&txq->lock); 189 } 190 } 191 192 static inline void iwl_pcie_tfd_set_tb(struct iwl_trans *trans, void *tfd, 193 u8 idx, dma_addr_t addr, u16 len) 194 { 195 struct iwl_tfd *tfd_fh = (void *)tfd; 196 struct iwl_tfd_tb *tb = &tfd_fh->tbs[idx]; 197 198 u16 hi_n_len = len << 4; 199 200 put_unaligned_le32(addr, &tb->lo); 201 hi_n_len |= iwl_get_dma_hi_addr(addr); 202 203 tb->hi_n_len = cpu_to_le16(hi_n_len); 204 205 tfd_fh->num_tbs = idx + 1; 206 } 207 208 /* 209 * iwl_pcie_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr] 210 * @trans - transport private data 211 * @txq - tx queue 212 * @dma_dir - the direction of the DMA mapping 213 * 214 * Does NOT advance any TFD circular buffer read/write indexes 215 * Does NOT free the TFD itself (which is within circular buffer) 216 */ 217 void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 218 { 219 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 220 * idx is bounded by n_window 221 */ 222 int rd_ptr = txq->read_ptr; 223 int idx = iwl_txq_get_cmd_index(txq, rd_ptr); 224 225 lockdep_assert_held(&txq->lock); 226 227 /* We have only q->n_window txq->entries, but we use 228 * TFD_QUEUE_SIZE_MAX tfds 229 */ 230 iwl_txq_gen1_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr); 231 232 /* free SKB */ 233 if (txq->entries) { 234 struct sk_buff *skb; 235 236 skb = txq->entries[idx].skb; 237 238 /* Can be called from irqs-disabled context 239 * If skb is not NULL, it means that the whole queue is being 240 * freed and that the queue is not empty - free the skb 241 */ 242 if (skb) { 243 iwl_op_mode_free_skb(trans->op_mode, skb); 244 txq->entries[idx].skb = NULL; 245 } 246 } 247 } 248 249 static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq, 250 dma_addr_t addr, u16 len, bool reset) 251 { 252 void *tfd; 253 u32 num_tbs; 254 255 tfd = txq->tfds + trans->txqs.tfd.size * txq->write_ptr; 256 257 if (reset) 258 memset(tfd, 0, trans->txqs.tfd.size); 259 260 num_tbs = iwl_txq_gen1_tfd_get_num_tbs(trans, tfd); 261 262 /* Each TFD can point to a maximum max_tbs Tx buffers */ 263 if (num_tbs >= trans->txqs.tfd.max_tbs) { 264 IWL_ERR(trans, "Error can not send more than %d chunks\n", 265 trans->txqs.tfd.max_tbs); 266 return -EINVAL; 267 } 268 269 if (WARN(addr & ~IWL_TX_DMA_MASK, 270 "Unaligned address = %llx\n", (unsigned long long)addr)) 271 return -EINVAL; 272 273 iwl_pcie_tfd_set_tb(trans, tfd, num_tbs, addr, len); 274 275 return num_tbs; 276 } 277 278 static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans) 279 { 280 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 281 282 lockdep_assert_held(&trans_pcie->reg_lock); 283 284 if (!trans->trans_cfg->base_params->apmg_wake_up_wa) 285 return; 286 if (WARN_ON(!trans_pcie->cmd_hold_nic_awake)) 287 return; 288 289 trans_pcie->cmd_hold_nic_awake = false; 290 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 291 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 292 } 293 294 /* 295 * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's 296 */ 297 static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id) 298 { 299 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 300 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 301 302 spin_lock_bh(&txq->lock); 303 while (txq->write_ptr != txq->read_ptr) { 304 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 305 txq_id, txq->read_ptr); 306 307 if (txq_id != trans->txqs.cmd.q_id) { 308 struct sk_buff *skb = txq->entries[txq->read_ptr].skb; 309 310 if (WARN_ON_ONCE(!skb)) 311 continue; 312 313 iwl_txq_free_tso_page(trans, skb); 314 } 315 iwl_pcie_txq_free_tfd(trans, txq); 316 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 317 318 if (txq->read_ptr == txq->write_ptr) { 319 unsigned long flags; 320 321 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 322 if (txq_id == trans->txqs.cmd.q_id) 323 iwl_pcie_clear_cmd_in_flight(trans); 324 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 325 } 326 } 327 328 while (!skb_queue_empty(&txq->overflow_q)) { 329 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 330 331 iwl_op_mode_free_skb(trans->op_mode, skb); 332 } 333 334 spin_unlock_bh(&txq->lock); 335 336 /* just in case - this queue may have been stopped */ 337 iwl_wake_queue(trans, txq); 338 } 339 340 /* 341 * iwl_pcie_txq_free - Deallocate DMA queue. 342 * @txq: Transmit queue to deallocate. 343 * 344 * Empty queue by removing and destroying all BD's. 345 * Free all buffers. 346 * 0-fill, but do not free "txq" descriptor structure. 347 */ 348 static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id) 349 { 350 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 351 struct device *dev = trans->dev; 352 int i; 353 354 if (WARN_ON(!txq)) 355 return; 356 357 iwl_pcie_txq_unmap(trans, txq_id); 358 359 /* De-alloc array of command/tx buffers */ 360 if (txq_id == trans->txqs.cmd.q_id) 361 for (i = 0; i < txq->n_window; i++) { 362 kfree_sensitive(txq->entries[i].cmd); 363 kfree_sensitive(txq->entries[i].free_buf); 364 } 365 366 /* De-alloc circular buffer of TFDs */ 367 if (txq->tfds) { 368 dma_free_coherent(dev, 369 trans->txqs.tfd.size * 370 trans->trans_cfg->base_params->max_tfd_queue_size, 371 txq->tfds, txq->dma_addr); 372 txq->dma_addr = 0; 373 txq->tfds = NULL; 374 375 dma_free_coherent(dev, 376 sizeof(*txq->first_tb_bufs) * txq->n_window, 377 txq->first_tb_bufs, txq->first_tb_dma); 378 } 379 380 kfree(txq->entries); 381 txq->entries = NULL; 382 383 del_timer_sync(&txq->stuck_timer); 384 385 /* 0-fill queue descriptor structure */ 386 memset(txq, 0, sizeof(*txq)); 387 } 388 389 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr) 390 { 391 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 392 int nq = trans->trans_cfg->base_params->num_of_queues; 393 int chan; 394 u32 reg_val; 395 int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) - 396 SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32); 397 398 /* make sure all queue are not stopped/used */ 399 memset(trans->txqs.queue_stopped, 0, 400 sizeof(trans->txqs.queue_stopped)); 401 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 402 403 trans_pcie->scd_base_addr = 404 iwl_read_prph(trans, SCD_SRAM_BASE_ADDR); 405 406 WARN_ON(scd_base_addr != 0 && 407 scd_base_addr != trans_pcie->scd_base_addr); 408 409 /* reset context data, TX status and translation data */ 410 iwl_trans_write_mem(trans, trans_pcie->scd_base_addr + 411 SCD_CONTEXT_MEM_LOWER_BOUND, 412 NULL, clear_dwords); 413 414 iwl_write_prph(trans, SCD_DRAM_BASE_ADDR, 415 trans->txqs.scd_bc_tbls.dma >> 10); 416 417 /* The chain extension of the SCD doesn't work well. This feature is 418 * enabled by default by the HW, so we need to disable it manually. 419 */ 420 if (trans->trans_cfg->base_params->scd_chain_ext_wa) 421 iwl_write_prph(trans, SCD_CHAINEXT_EN, 0); 422 423 iwl_trans_ac_txq_enable(trans, trans->txqs.cmd.q_id, 424 trans->txqs.cmd.fifo, 425 trans->txqs.cmd.wdg_timeout); 426 427 /* Activate all Tx DMA/FIFO channels */ 428 iwl_scd_activate_fifos(trans); 429 430 /* Enable DMA channel */ 431 for (chan = 0; chan < FH_TCSR_CHNL_NUM; chan++) 432 iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan), 433 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | 434 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE); 435 436 /* Update FH chicken bits */ 437 reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG); 438 iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG, 439 reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); 440 441 /* Enable L1-Active */ 442 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000) 443 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG, 444 APMG_PCIDEV_STT_VAL_L1_ACT_DIS); 445 } 446 447 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans) 448 { 449 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 450 int txq_id; 451 452 /* 453 * we should never get here in gen2 trans mode return early to avoid 454 * having invalid accesses 455 */ 456 if (WARN_ON_ONCE(trans->trans_cfg->gen2)) 457 return; 458 459 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 460 txq_id++) { 461 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 462 if (trans->trans_cfg->use_tfh) 463 iwl_write_direct64(trans, 464 FH_MEM_CBBC_QUEUE(trans, txq_id), 465 txq->dma_addr); 466 else 467 iwl_write_direct32(trans, 468 FH_MEM_CBBC_QUEUE(trans, txq_id), 469 txq->dma_addr >> 8); 470 iwl_pcie_txq_unmap(trans, txq_id); 471 txq->read_ptr = 0; 472 txq->write_ptr = 0; 473 } 474 475 /* Tell NIC where to find the "keep warm" buffer */ 476 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 477 trans_pcie->kw.dma >> 4); 478 479 /* 480 * Send 0 as the scd_base_addr since the device may have be reset 481 * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will 482 * contain garbage. 483 */ 484 iwl_pcie_tx_start(trans, 0); 485 } 486 487 static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans) 488 { 489 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 490 unsigned long flags; 491 int ch, ret; 492 u32 mask = 0; 493 494 spin_lock(&trans_pcie->irq_lock); 495 496 if (!iwl_trans_grab_nic_access(trans, &flags)) 497 goto out; 498 499 /* Stop each Tx DMA channel */ 500 for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) { 501 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0); 502 mask |= FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch); 503 } 504 505 /* Wait for DMA channels to be idle */ 506 ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000); 507 if (ret < 0) 508 IWL_ERR(trans, 509 "Failing on timeout while stopping DMA channel %d [0x%08x]\n", 510 ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG)); 511 512 iwl_trans_release_nic_access(trans, &flags); 513 514 out: 515 spin_unlock(&trans_pcie->irq_lock); 516 } 517 518 /* 519 * iwl_pcie_tx_stop - Stop all Tx DMA channels 520 */ 521 int iwl_pcie_tx_stop(struct iwl_trans *trans) 522 { 523 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 524 int txq_id; 525 526 /* Turn off all Tx DMA fifos */ 527 iwl_scd_deactivate_fifos(trans); 528 529 /* Turn off all Tx DMA channels */ 530 iwl_pcie_tx_stop_fh(trans); 531 532 /* 533 * This function can be called before the op_mode disabled the 534 * queues. This happens when we have an rfkill interrupt. 535 * Since we stop Tx altogether - mark the queues as stopped. 536 */ 537 memset(trans->txqs.queue_stopped, 0, 538 sizeof(trans->txqs.queue_stopped)); 539 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 540 541 /* This can happen: start_hw, stop_device */ 542 if (!trans_pcie->txq_memory) 543 return 0; 544 545 /* Unmap DMA from host system and free skb's */ 546 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 547 txq_id++) 548 iwl_pcie_txq_unmap(trans, txq_id); 549 550 return 0; 551 } 552 553 /* 554 * iwl_trans_tx_free - Free TXQ Context 555 * 556 * Destroy all TX DMA queues and structures 557 */ 558 void iwl_pcie_tx_free(struct iwl_trans *trans) 559 { 560 int txq_id; 561 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 562 563 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 564 565 /* Tx queues */ 566 if (trans_pcie->txq_memory) { 567 for (txq_id = 0; 568 txq_id < trans->trans_cfg->base_params->num_of_queues; 569 txq_id++) { 570 iwl_pcie_txq_free(trans, txq_id); 571 trans->txqs.txq[txq_id] = NULL; 572 } 573 } 574 575 kfree(trans_pcie->txq_memory); 576 trans_pcie->txq_memory = NULL; 577 578 iwl_pcie_free_dma_ptr(trans, &trans_pcie->kw); 579 580 iwl_pcie_free_dma_ptr(trans, &trans->txqs.scd_bc_tbls); 581 } 582 583 /* 584 * iwl_pcie_tx_alloc - allocate TX context 585 * Allocate all Tx DMA structures and initialize them 586 */ 587 static int iwl_pcie_tx_alloc(struct iwl_trans *trans) 588 { 589 int ret; 590 int txq_id, slots_num; 591 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 592 u16 bc_tbls_size = trans->trans_cfg->base_params->num_of_queues; 593 594 if (WARN_ON(trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)) 595 return -EINVAL; 596 597 bc_tbls_size *= sizeof(struct iwlagn_scd_bc_tbl); 598 599 /*It is not allowed to alloc twice, so warn when this happens. 600 * We cannot rely on the previous allocation, so free and fail */ 601 if (WARN_ON(trans_pcie->txq_memory)) { 602 ret = -EINVAL; 603 goto error; 604 } 605 606 ret = iwl_pcie_alloc_dma_ptr(trans, &trans->txqs.scd_bc_tbls, 607 bc_tbls_size); 608 if (ret) { 609 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 610 goto error; 611 } 612 613 /* Alloc keep-warm buffer */ 614 ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE); 615 if (ret) { 616 IWL_ERR(trans, "Keep Warm allocation failed\n"); 617 goto error; 618 } 619 620 trans_pcie->txq_memory = 621 kcalloc(trans->trans_cfg->base_params->num_of_queues, 622 sizeof(struct iwl_txq), GFP_KERNEL); 623 if (!trans_pcie->txq_memory) { 624 IWL_ERR(trans, "Not enough memory for txq\n"); 625 ret = -ENOMEM; 626 goto error; 627 } 628 629 /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 630 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 631 txq_id++) { 632 bool cmd_queue = (txq_id == trans->txqs.cmd.q_id); 633 634 if (cmd_queue) 635 slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE, 636 trans->cfg->min_txq_size); 637 else 638 slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE, 639 trans->cfg->min_256_ba_txq_size); 640 trans->txqs.txq[txq_id] = &trans_pcie->txq_memory[txq_id]; 641 ret = iwl_txq_alloc(trans, trans->txqs.txq[txq_id], slots_num, 642 cmd_queue); 643 if (ret) { 644 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 645 goto error; 646 } 647 trans->txqs.txq[txq_id]->id = txq_id; 648 } 649 650 return 0; 651 652 error: 653 iwl_pcie_tx_free(trans); 654 655 return ret; 656 } 657 658 int iwl_pcie_tx_init(struct iwl_trans *trans) 659 { 660 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 661 int ret; 662 int txq_id, slots_num; 663 bool alloc = false; 664 665 if (!trans_pcie->txq_memory) { 666 ret = iwl_pcie_tx_alloc(trans); 667 if (ret) 668 goto error; 669 alloc = true; 670 } 671 672 spin_lock(&trans_pcie->irq_lock); 673 674 /* Turn off all Tx DMA fifos */ 675 iwl_scd_deactivate_fifos(trans); 676 677 /* Tell NIC where to find the "keep warm" buffer */ 678 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 679 trans_pcie->kw.dma >> 4); 680 681 spin_unlock(&trans_pcie->irq_lock); 682 683 /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 684 for (txq_id = 0; txq_id < trans->trans_cfg->base_params->num_of_queues; 685 txq_id++) { 686 bool cmd_queue = (txq_id == trans->txqs.cmd.q_id); 687 688 if (cmd_queue) 689 slots_num = max_t(u32, IWL_CMD_QUEUE_SIZE, 690 trans->cfg->min_txq_size); 691 else 692 slots_num = max_t(u32, IWL_DEFAULT_QUEUE_SIZE, 693 trans->cfg->min_256_ba_txq_size); 694 ret = iwl_txq_init(trans, trans->txqs.txq[txq_id], slots_num, 695 cmd_queue); 696 if (ret) { 697 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 698 goto error; 699 } 700 701 /* 702 * Tell nic where to find circular buffer of TFDs for a 703 * given Tx queue, and enable the DMA channel used for that 704 * queue. 705 * Circular buffer (TFD queue in DRAM) physical base address 706 */ 707 iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id), 708 trans->txqs.txq[txq_id]->dma_addr >> 8); 709 } 710 711 iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE); 712 if (trans->trans_cfg->base_params->num_of_queues > 20) 713 iwl_set_bits_prph(trans, SCD_GP_CTRL, 714 SCD_GP_CTRL_ENABLE_31_QUEUES); 715 716 return 0; 717 error: 718 /*Upon error, free only if we allocated something */ 719 if (alloc) 720 iwl_pcie_tx_free(trans); 721 return ret; 722 } 723 724 static inline void iwl_pcie_txq_progress(struct iwl_txq *txq) 725 { 726 lockdep_assert_held(&txq->lock); 727 728 if (!txq->wd_timeout) 729 return; 730 731 /* 732 * station is asleep and we send data - that must 733 * be uAPSD or PS-Poll. Don't rearm the timer. 734 */ 735 if (txq->frozen) 736 return; 737 738 /* 739 * if empty delete timer, otherwise move timer forward 740 * since we're making progress on this queue 741 */ 742 if (txq->read_ptr == txq->write_ptr) 743 del_timer(&txq->stuck_timer); 744 else 745 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 746 } 747 748 /* Frees buffers until index _not_ inclusive */ 749 void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, 750 struct sk_buff_head *skbs) 751 { 752 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 753 int tfd_num = iwl_txq_get_cmd_index(txq, ssn); 754 int read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr); 755 int last_to_free; 756 757 /* This function is not meant to release cmd queue*/ 758 if (WARN_ON(txq_id == trans->txqs.cmd.q_id)) 759 return; 760 761 spin_lock_bh(&txq->lock); 762 763 if (!test_bit(txq_id, trans->txqs.queue_used)) { 764 IWL_DEBUG_TX_QUEUES(trans, "Q %d inactive - ignoring idx %d\n", 765 txq_id, ssn); 766 goto out; 767 } 768 769 if (read_ptr == tfd_num) 770 goto out; 771 772 IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n", 773 txq_id, txq->read_ptr, tfd_num, ssn); 774 775 /*Since we free until index _not_ inclusive, the one before index is 776 * the last we will free. This one must be used */ 777 last_to_free = iwl_txq_dec_wrap(trans, tfd_num); 778 779 if (!iwl_txq_used(txq, last_to_free)) { 780 IWL_ERR(trans, 781 "%s: Read index for txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n", 782 __func__, txq_id, last_to_free, 783 trans->trans_cfg->base_params->max_tfd_queue_size, 784 txq->write_ptr, txq->read_ptr); 785 goto out; 786 } 787 788 if (WARN_ON(!skb_queue_empty(skbs))) 789 goto out; 790 791 for (; 792 read_ptr != tfd_num; 793 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr), 794 read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr)) { 795 struct sk_buff *skb = txq->entries[read_ptr].skb; 796 797 if (WARN_ON_ONCE(!skb)) 798 continue; 799 800 iwl_txq_free_tso_page(trans, skb); 801 802 __skb_queue_tail(skbs, skb); 803 804 txq->entries[read_ptr].skb = NULL; 805 806 if (!trans->trans_cfg->use_tfh) 807 iwl_txq_gen1_inval_byte_cnt_tbl(trans, txq); 808 809 iwl_pcie_txq_free_tfd(trans, txq); 810 } 811 812 iwl_pcie_txq_progress(txq); 813 814 if (iwl_txq_space(trans, txq) > txq->low_mark && 815 test_bit(txq_id, trans->txqs.queue_stopped)) { 816 struct sk_buff_head overflow_skbs; 817 818 __skb_queue_head_init(&overflow_skbs); 819 skb_queue_splice_init(&txq->overflow_q, &overflow_skbs); 820 821 /* 822 * We are going to transmit from the overflow queue. 823 * Remember this state so that wait_for_txq_empty will know we 824 * are adding more packets to the TFD queue. It cannot rely on 825 * the state of &txq->overflow_q, as we just emptied it, but 826 * haven't TXed the content yet. 827 */ 828 txq->overflow_tx = true; 829 830 /* 831 * This is tricky: we are in reclaim path which is non 832 * re-entrant, so noone will try to take the access the 833 * txq data from that path. We stopped tx, so we can't 834 * have tx as well. Bottom line, we can unlock and re-lock 835 * later. 836 */ 837 spin_unlock_bh(&txq->lock); 838 839 while (!skb_queue_empty(&overflow_skbs)) { 840 struct sk_buff *skb = __skb_dequeue(&overflow_skbs); 841 struct iwl_device_tx_cmd *dev_cmd_ptr; 842 843 dev_cmd_ptr = *(void **)((u8 *)skb->cb + 844 trans->txqs.dev_cmd_offs); 845 846 /* 847 * Note that we can very well be overflowing again. 848 * In that case, iwl_txq_space will be small again 849 * and we won't wake mac80211's queue. 850 */ 851 iwl_trans_tx(trans, skb, dev_cmd_ptr, txq_id); 852 } 853 854 if (iwl_txq_space(trans, txq) > txq->low_mark) 855 iwl_wake_queue(trans, txq); 856 857 spin_lock_bh(&txq->lock); 858 txq->overflow_tx = false; 859 } 860 861 out: 862 spin_unlock_bh(&txq->lock); 863 } 864 865 /* Set wr_ptr of specific device and txq */ 866 void iwl_trans_pcie_set_q_ptrs(struct iwl_trans *trans, int txq_id, int ptr) 867 { 868 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 869 870 spin_lock_bh(&txq->lock); 871 872 txq->write_ptr = ptr; 873 txq->read_ptr = txq->write_ptr; 874 875 spin_unlock_bh(&txq->lock); 876 } 877 878 static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans, 879 const struct iwl_host_cmd *cmd) 880 { 881 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 882 int ret; 883 884 lockdep_assert_held(&trans_pcie->reg_lock); 885 886 /* Make sure the NIC is still alive in the bus */ 887 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 888 return -ENODEV; 889 890 /* 891 * wake up the NIC to make sure that the firmware will see the host 892 * command - we will let the NIC sleep once all the host commands 893 * returned. This needs to be done only on NICs that have 894 * apmg_wake_up_wa set. 895 */ 896 if (trans->trans_cfg->base_params->apmg_wake_up_wa && 897 !trans_pcie->cmd_hold_nic_awake) { 898 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, 899 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 900 901 ret = iwl_poll_bit(trans, CSR_GP_CNTRL, 902 CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN, 903 (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | 904 CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 905 15000); 906 if (ret < 0) { 907 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 908 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 909 IWL_ERR(trans, "Failed to wake NIC for hcmd\n"); 910 return -EIO; 911 } 912 trans_pcie->cmd_hold_nic_awake = true; 913 } 914 915 return 0; 916 } 917 918 /* 919 * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd 920 * 921 * When FW advances 'R' index, all entries between old and new 'R' index 922 * need to be reclaimed. As result, some free space forms. If there is 923 * enough free space (> low mark), wake the stack that feeds us. 924 */ 925 static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx) 926 { 927 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 928 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 929 unsigned long flags; 930 int nfreed = 0; 931 u16 r; 932 933 lockdep_assert_held(&txq->lock); 934 935 idx = iwl_txq_get_cmd_index(txq, idx); 936 r = iwl_txq_get_cmd_index(txq, txq->read_ptr); 937 938 if (idx >= trans->trans_cfg->base_params->max_tfd_queue_size || 939 (!iwl_txq_used(txq, idx))) { 940 WARN_ONCE(test_bit(txq_id, trans->txqs.queue_used), 941 "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n", 942 __func__, txq_id, idx, 943 trans->trans_cfg->base_params->max_tfd_queue_size, 944 txq->write_ptr, txq->read_ptr); 945 return; 946 } 947 948 for (idx = iwl_txq_inc_wrap(trans, idx); r != idx; 949 r = iwl_txq_inc_wrap(trans, r)) { 950 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 951 952 if (nfreed++ > 0) { 953 IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n", 954 idx, txq->write_ptr, r); 955 iwl_force_nmi(trans); 956 } 957 } 958 959 if (txq->read_ptr == txq->write_ptr) { 960 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 961 iwl_pcie_clear_cmd_in_flight(trans); 962 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 963 } 964 965 iwl_pcie_txq_progress(txq); 966 } 967 968 static int iwl_pcie_txq_set_ratid_map(struct iwl_trans *trans, u16 ra_tid, 969 u16 txq_id) 970 { 971 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 972 u32 tbl_dw_addr; 973 u32 tbl_dw; 974 u16 scd_q2ratid; 975 976 scd_q2ratid = ra_tid & SCD_QUEUE_RA_TID_MAP_RATID_MSK; 977 978 tbl_dw_addr = trans_pcie->scd_base_addr + 979 SCD_TRANS_TBL_OFFSET_QUEUE(txq_id); 980 981 tbl_dw = iwl_trans_read_mem32(trans, tbl_dw_addr); 982 983 if (txq_id & 0x1) 984 tbl_dw = (scd_q2ratid << 16) | (tbl_dw & 0x0000FFFF); 985 else 986 tbl_dw = scd_q2ratid | (tbl_dw & 0xFFFF0000); 987 988 iwl_trans_write_mem32(trans, tbl_dw_addr, tbl_dw); 989 990 return 0; 991 } 992 993 /* Receiver address (actually, Rx station's index into station table), 994 * combined with Traffic ID (QOS priority), in format used by Tx Scheduler */ 995 #define BUILD_RAxTID(sta_id, tid) (((sta_id) << 4) + (tid)) 996 997 bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn, 998 const struct iwl_trans_txq_scd_cfg *cfg, 999 unsigned int wdg_timeout) 1000 { 1001 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1002 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 1003 int fifo = -1; 1004 bool scd_bug = false; 1005 1006 if (test_and_set_bit(txq_id, trans->txqs.queue_used)) 1007 WARN_ONCE(1, "queue %d already used - expect issues", txq_id); 1008 1009 txq->wd_timeout = msecs_to_jiffies(wdg_timeout); 1010 1011 if (cfg) { 1012 fifo = cfg->fifo; 1013 1014 /* Disable the scheduler prior configuring the cmd queue */ 1015 if (txq_id == trans->txqs.cmd.q_id && 1016 trans_pcie->scd_set_active) 1017 iwl_scd_enable_set_active(trans, 0); 1018 1019 /* Stop this Tx queue before configuring it */ 1020 iwl_scd_txq_set_inactive(trans, txq_id); 1021 1022 /* Set this queue as a chain-building queue unless it is CMD */ 1023 if (txq_id != trans->txqs.cmd.q_id) 1024 iwl_scd_txq_set_chain(trans, txq_id); 1025 1026 if (cfg->aggregate) { 1027 u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid); 1028 1029 /* Map receiver-address / traffic-ID to this queue */ 1030 iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id); 1031 1032 /* enable aggregations for the queue */ 1033 iwl_scd_txq_enable_agg(trans, txq_id); 1034 txq->ampdu = true; 1035 } else { 1036 /* 1037 * disable aggregations for the queue, this will also 1038 * make the ra_tid mapping configuration irrelevant 1039 * since it is now a non-AGG queue. 1040 */ 1041 iwl_scd_txq_disable_agg(trans, txq_id); 1042 1043 ssn = txq->read_ptr; 1044 } 1045 } else { 1046 /* 1047 * If we need to move the SCD write pointer by steps of 1048 * 0x40, 0x80 or 0xc0, it gets stuck. Avoids this and let 1049 * the op_mode know by returning true later. 1050 * Do this only in case cfg is NULL since this trick can 1051 * be done only if we have DQA enabled which is true for mvm 1052 * only. And mvm never sets a cfg pointer. 1053 * This is really ugly, but this is the easiest way out for 1054 * this sad hardware issue. 1055 * This bug has been fixed on devices 9000 and up. 1056 */ 1057 scd_bug = !trans->trans_cfg->mq_rx_supported && 1058 !((ssn - txq->write_ptr) & 0x3f) && 1059 (ssn != txq->write_ptr); 1060 if (scd_bug) 1061 ssn++; 1062 } 1063 1064 /* Place first TFD at index corresponding to start sequence number. 1065 * Assumes that ssn_idx is valid (!= 0xFFF) */ 1066 txq->read_ptr = (ssn & 0xff); 1067 txq->write_ptr = (ssn & 0xff); 1068 iwl_write_direct32(trans, HBUS_TARG_WRPTR, 1069 (ssn & 0xff) | (txq_id << 8)); 1070 1071 if (cfg) { 1072 u8 frame_limit = cfg->frame_limit; 1073 1074 iwl_write_prph(trans, SCD_QUEUE_RDPTR(txq_id), ssn); 1075 1076 /* Set up Tx window size and frame limit for this queue */ 1077 iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr + 1078 SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0); 1079 iwl_trans_write_mem32(trans, 1080 trans_pcie->scd_base_addr + 1081 SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32), 1082 SCD_QUEUE_CTX_REG2_VAL(WIN_SIZE, frame_limit) | 1083 SCD_QUEUE_CTX_REG2_VAL(FRAME_LIMIT, frame_limit)); 1084 1085 /* Set up status area in SRAM, map to Tx DMA/FIFO, activate */ 1086 iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id), 1087 (1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) | 1088 (cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) | 1089 (1 << SCD_QUEUE_STTS_REG_POS_WSL) | 1090 SCD_QUEUE_STTS_REG_MSK); 1091 1092 /* enable the scheduler for this queue (only) */ 1093 if (txq_id == trans->txqs.cmd.q_id && 1094 trans_pcie->scd_set_active) 1095 iwl_scd_enable_set_active(trans, BIT(txq_id)); 1096 1097 IWL_DEBUG_TX_QUEUES(trans, 1098 "Activate queue %d on FIFO %d WrPtr: %d\n", 1099 txq_id, fifo, ssn & 0xff); 1100 } else { 1101 IWL_DEBUG_TX_QUEUES(trans, 1102 "Activate queue %d WrPtr: %d\n", 1103 txq_id, ssn & 0xff); 1104 } 1105 1106 return scd_bug; 1107 } 1108 1109 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id, 1110 bool shared_mode) 1111 { 1112 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 1113 1114 txq->ampdu = !shared_mode; 1115 } 1116 1117 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id, 1118 bool configure_scd) 1119 { 1120 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1121 u32 stts_addr = trans_pcie->scd_base_addr + 1122 SCD_TX_STTS_QUEUE_OFFSET(txq_id); 1123 static const u32 zero_val[4] = {}; 1124 1125 trans->txqs.txq[txq_id]->frozen_expiry_remainder = 0; 1126 trans->txqs.txq[txq_id]->frozen = false; 1127 1128 /* 1129 * Upon HW Rfkill - we stop the device, and then stop the queues 1130 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1131 * allow the op_mode to call txq_disable after it already called 1132 * stop_device. 1133 */ 1134 if (!test_and_clear_bit(txq_id, trans->txqs.queue_used)) { 1135 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1136 "queue %d not used", txq_id); 1137 return; 1138 } 1139 1140 if (configure_scd) { 1141 iwl_scd_txq_set_inactive(trans, txq_id); 1142 1143 iwl_trans_write_mem(trans, stts_addr, (void *)zero_val, 1144 ARRAY_SIZE(zero_val)); 1145 } 1146 1147 iwl_pcie_txq_unmap(trans, txq_id); 1148 trans->txqs.txq[txq_id]->ampdu = false; 1149 1150 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", txq_id); 1151 } 1152 1153 /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 1154 1155 /* 1156 * iwl_pcie_enqueue_hcmd - enqueue a uCode command 1157 * @priv: device private data point 1158 * @cmd: a pointer to the ucode command structure 1159 * 1160 * The function returns < 0 values to indicate the operation 1161 * failed. On success, it returns the index (>= 0) of command in the 1162 * command queue. 1163 */ 1164 static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, 1165 struct iwl_host_cmd *cmd) 1166 { 1167 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1168 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1169 struct iwl_device_cmd *out_cmd; 1170 struct iwl_cmd_meta *out_meta; 1171 unsigned long flags; 1172 void *dup_buf = NULL; 1173 dma_addr_t phys_addr; 1174 int idx; 1175 u16 copy_size, cmd_size, tb0_size; 1176 bool had_nocopy = false; 1177 u8 group_id = iwl_cmd_groupid(cmd->id); 1178 int i, ret; 1179 u32 cmd_pos; 1180 const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; 1181 u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; 1182 1183 if (WARN(!trans->wide_cmd_header && 1184 group_id > IWL_ALWAYS_LONG_GROUP, 1185 "unsupported wide command %#x\n", cmd->id)) 1186 return -EINVAL; 1187 1188 if (group_id != 0) { 1189 copy_size = sizeof(struct iwl_cmd_header_wide); 1190 cmd_size = sizeof(struct iwl_cmd_header_wide); 1191 } else { 1192 copy_size = sizeof(struct iwl_cmd_header); 1193 cmd_size = sizeof(struct iwl_cmd_header); 1194 } 1195 1196 /* need one for the header if the first is NOCOPY */ 1197 BUILD_BUG_ON(IWL_MAX_CMD_TBS_PER_TFD > IWL_NUM_OF_TBS - 1); 1198 1199 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1200 cmddata[i] = cmd->data[i]; 1201 cmdlen[i] = cmd->len[i]; 1202 1203 if (!cmd->len[i]) 1204 continue; 1205 1206 /* need at least IWL_FIRST_TB_SIZE copied */ 1207 if (copy_size < IWL_FIRST_TB_SIZE) { 1208 int copy = IWL_FIRST_TB_SIZE - copy_size; 1209 1210 if (copy > cmdlen[i]) 1211 copy = cmdlen[i]; 1212 cmdlen[i] -= copy; 1213 cmddata[i] += copy; 1214 copy_size += copy; 1215 } 1216 1217 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { 1218 had_nocopy = true; 1219 if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { 1220 idx = -EINVAL; 1221 goto free_dup_buf; 1222 } 1223 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { 1224 /* 1225 * This is also a chunk that isn't copied 1226 * to the static buffer so set had_nocopy. 1227 */ 1228 had_nocopy = true; 1229 1230 /* only allowed once */ 1231 if (WARN_ON(dup_buf)) { 1232 idx = -EINVAL; 1233 goto free_dup_buf; 1234 } 1235 1236 dup_buf = kmemdup(cmddata[i], cmdlen[i], 1237 GFP_ATOMIC); 1238 if (!dup_buf) 1239 return -ENOMEM; 1240 } else { 1241 /* NOCOPY must not be followed by normal! */ 1242 if (WARN_ON(had_nocopy)) { 1243 idx = -EINVAL; 1244 goto free_dup_buf; 1245 } 1246 copy_size += cmdlen[i]; 1247 } 1248 cmd_size += cmd->len[i]; 1249 } 1250 1251 /* 1252 * If any of the command structures end up being larger than 1253 * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically 1254 * allocated into separate TFDs, then we will need to 1255 * increase the size of the buffers. 1256 */ 1257 if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, 1258 "Command %s (%#x) is too large (%d bytes)\n", 1259 iwl_get_cmd_string(trans, cmd->id), 1260 cmd->id, copy_size)) { 1261 idx = -EINVAL; 1262 goto free_dup_buf; 1263 } 1264 1265 spin_lock_bh(&txq->lock); 1266 1267 if (iwl_txq_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { 1268 spin_unlock_bh(&txq->lock); 1269 1270 IWL_ERR(trans, "No space in command queue\n"); 1271 iwl_op_mode_cmd_queue_full(trans->op_mode); 1272 idx = -ENOSPC; 1273 goto free_dup_buf; 1274 } 1275 1276 idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 1277 out_cmd = txq->entries[idx].cmd; 1278 out_meta = &txq->entries[idx].meta; 1279 1280 memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ 1281 if (cmd->flags & CMD_WANT_SKB) 1282 out_meta->source = cmd; 1283 1284 /* set up the header */ 1285 if (group_id != 0) { 1286 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); 1287 out_cmd->hdr_wide.group_id = group_id; 1288 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); 1289 out_cmd->hdr_wide.length = 1290 cpu_to_le16(cmd_size - 1291 sizeof(struct iwl_cmd_header_wide)); 1292 out_cmd->hdr_wide.reserved = 0; 1293 out_cmd->hdr_wide.sequence = 1294 cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) | 1295 INDEX_TO_SEQ(txq->write_ptr)); 1296 1297 cmd_pos = sizeof(struct iwl_cmd_header_wide); 1298 copy_size = sizeof(struct iwl_cmd_header_wide); 1299 } else { 1300 out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id); 1301 out_cmd->hdr.sequence = 1302 cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) | 1303 INDEX_TO_SEQ(txq->write_ptr)); 1304 out_cmd->hdr.group_id = 0; 1305 1306 cmd_pos = sizeof(struct iwl_cmd_header); 1307 copy_size = sizeof(struct iwl_cmd_header); 1308 } 1309 1310 /* and copy the data that needs to be copied */ 1311 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1312 int copy; 1313 1314 if (!cmd->len[i]) 1315 continue; 1316 1317 /* copy everything if not nocopy/dup */ 1318 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1319 IWL_HCMD_DFL_DUP))) { 1320 copy = cmd->len[i]; 1321 1322 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1323 cmd_pos += copy; 1324 copy_size += copy; 1325 continue; 1326 } 1327 1328 /* 1329 * Otherwise we need at least IWL_FIRST_TB_SIZE copied 1330 * in total (for bi-directional DMA), but copy up to what 1331 * we can fit into the payload for debug dump purposes. 1332 */ 1333 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); 1334 1335 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1336 cmd_pos += copy; 1337 1338 /* However, treat copy_size the proper way, we need it below */ 1339 if (copy_size < IWL_FIRST_TB_SIZE) { 1340 copy = IWL_FIRST_TB_SIZE - copy_size; 1341 1342 if (copy > cmd->len[i]) 1343 copy = cmd->len[i]; 1344 copy_size += copy; 1345 } 1346 } 1347 1348 IWL_DEBUG_HC(trans, 1349 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", 1350 iwl_get_cmd_string(trans, cmd->id), 1351 group_id, out_cmd->hdr.cmd, 1352 le16_to_cpu(out_cmd->hdr.sequence), 1353 cmd_size, txq->write_ptr, idx, trans->txqs.cmd.q_id); 1354 1355 /* start the TFD with the minimum copy bytes */ 1356 tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); 1357 memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size); 1358 iwl_pcie_txq_build_tfd(trans, txq, 1359 iwl_txq_get_first_tb_dma(txq, idx), 1360 tb0_size, true); 1361 1362 /* map first command fragment, if any remains */ 1363 if (copy_size > tb0_size) { 1364 phys_addr = dma_map_single(trans->dev, 1365 ((u8 *)&out_cmd->hdr) + tb0_size, 1366 copy_size - tb0_size, 1367 DMA_TO_DEVICE); 1368 if (dma_mapping_error(trans->dev, phys_addr)) { 1369 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, 1370 txq->write_ptr); 1371 idx = -ENOMEM; 1372 goto out; 1373 } 1374 1375 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, 1376 copy_size - tb0_size, false); 1377 } 1378 1379 /* map the remaining (adjusted) nocopy/dup fragments */ 1380 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1381 const void *data = cmddata[i]; 1382 1383 if (!cmdlen[i]) 1384 continue; 1385 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1386 IWL_HCMD_DFL_DUP))) 1387 continue; 1388 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) 1389 data = dup_buf; 1390 phys_addr = dma_map_single(trans->dev, (void *)data, 1391 cmdlen[i], DMA_TO_DEVICE); 1392 if (dma_mapping_error(trans->dev, phys_addr)) { 1393 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, 1394 txq->write_ptr); 1395 idx = -ENOMEM; 1396 goto out; 1397 } 1398 1399 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false); 1400 } 1401 1402 BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); 1403 out_meta->flags = cmd->flags; 1404 if (WARN_ON_ONCE(txq->entries[idx].free_buf)) 1405 kfree_sensitive(txq->entries[idx].free_buf); 1406 txq->entries[idx].free_buf = dup_buf; 1407 1408 trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); 1409 1410 /* start timer if queue currently empty */ 1411 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 1412 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 1413 1414 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 1415 ret = iwl_pcie_set_cmd_in_flight(trans, cmd); 1416 if (ret < 0) { 1417 idx = ret; 1418 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1419 goto out; 1420 } 1421 1422 /* Increment and update queue's write index */ 1423 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 1424 iwl_pcie_txq_inc_wr_ptr(trans, txq); 1425 1426 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1427 1428 out: 1429 spin_unlock_bh(&txq->lock); 1430 free_dup_buf: 1431 if (idx < 0) 1432 kfree(dup_buf); 1433 return idx; 1434 } 1435 1436 /* 1437 * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them 1438 * @rxb: Rx buffer to reclaim 1439 */ 1440 void iwl_pcie_hcmd_complete(struct iwl_trans *trans, 1441 struct iwl_rx_cmd_buffer *rxb) 1442 { 1443 struct iwl_rx_packet *pkt = rxb_addr(rxb); 1444 u16 sequence = le16_to_cpu(pkt->hdr.sequence); 1445 u8 group_id; 1446 u32 cmd_id; 1447 int txq_id = SEQ_TO_QUEUE(sequence); 1448 int index = SEQ_TO_INDEX(sequence); 1449 int cmd_index; 1450 struct iwl_device_cmd *cmd; 1451 struct iwl_cmd_meta *meta; 1452 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1453 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1454 1455 /* If a Tx command is being handled and it isn't in the actual 1456 * command queue then there a command routing bug has been introduced 1457 * in the queue management code. */ 1458 if (WARN(txq_id != trans->txqs.cmd.q_id, 1459 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n", 1460 txq_id, trans->txqs.cmd.q_id, sequence, txq->read_ptr, 1461 txq->write_ptr)) { 1462 iwl_print_hex_error(trans, pkt, 32); 1463 return; 1464 } 1465 1466 spin_lock_bh(&txq->lock); 1467 1468 cmd_index = iwl_txq_get_cmd_index(txq, index); 1469 cmd = txq->entries[cmd_index].cmd; 1470 meta = &txq->entries[cmd_index].meta; 1471 group_id = cmd->hdr.group_id; 1472 cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0); 1473 1474 iwl_txq_gen1_tfd_unmap(trans, meta, txq, index); 1475 1476 /* Input error checking is done when commands are added to queue. */ 1477 if (meta->flags & CMD_WANT_SKB) { 1478 struct page *p = rxb_steal_page(rxb); 1479 1480 meta->source->resp_pkt = pkt; 1481 meta->source->_rx_page_addr = (unsigned long)page_address(p); 1482 meta->source->_rx_page_order = trans_pcie->rx_page_order; 1483 } 1484 1485 if (meta->flags & CMD_WANT_ASYNC_CALLBACK) 1486 iwl_op_mode_async_cb(trans->op_mode, cmd); 1487 1488 iwl_pcie_cmdq_reclaim(trans, txq_id, index); 1489 1490 if (!(meta->flags & CMD_ASYNC)) { 1491 if (!test_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) { 1492 IWL_WARN(trans, 1493 "HCMD_ACTIVE already clear for command %s\n", 1494 iwl_get_cmd_string(trans, cmd_id)); 1495 } 1496 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1497 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1498 iwl_get_cmd_string(trans, cmd_id)); 1499 wake_up(&trans_pcie->wait_command_queue); 1500 } 1501 1502 meta->flags = 0; 1503 1504 spin_unlock_bh(&txq->lock); 1505 } 1506 1507 #define HOST_COMPLETE_TIMEOUT (2 * HZ) 1508 1509 static int iwl_pcie_send_hcmd_async(struct iwl_trans *trans, 1510 struct iwl_host_cmd *cmd) 1511 { 1512 int ret; 1513 1514 /* An asynchronous command can not expect an SKB to be set. */ 1515 if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1516 return -EINVAL; 1517 1518 ret = iwl_pcie_enqueue_hcmd(trans, cmd); 1519 if (ret < 0) { 1520 IWL_ERR(trans, 1521 "Error sending %s: enqueue_hcmd failed: %d\n", 1522 iwl_get_cmd_string(trans, cmd->id), ret); 1523 return ret; 1524 } 1525 return 0; 1526 } 1527 1528 static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans, 1529 struct iwl_host_cmd *cmd) 1530 { 1531 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1532 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1533 int cmd_idx; 1534 int ret; 1535 1536 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", 1537 iwl_get_cmd_string(trans, cmd->id)); 1538 1539 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 1540 &trans->status), 1541 "Command %s: a command is already active!\n", 1542 iwl_get_cmd_string(trans, cmd->id))) 1543 return -EIO; 1544 1545 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", 1546 iwl_get_cmd_string(trans, cmd->id)); 1547 1548 cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd); 1549 if (cmd_idx < 0) { 1550 ret = cmd_idx; 1551 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1552 IWL_ERR(trans, 1553 "Error sending %s: enqueue_hcmd failed: %d\n", 1554 iwl_get_cmd_string(trans, cmd->id), ret); 1555 return ret; 1556 } 1557 1558 ret = wait_event_timeout(trans_pcie->wait_command_queue, 1559 !test_bit(STATUS_SYNC_HCMD_ACTIVE, 1560 &trans->status), 1561 HOST_COMPLETE_TIMEOUT); 1562 if (!ret) { 1563 IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 1564 iwl_get_cmd_string(trans, cmd->id), 1565 jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 1566 1567 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 1568 txq->read_ptr, txq->write_ptr); 1569 1570 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1571 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1572 iwl_get_cmd_string(trans, cmd->id)); 1573 ret = -ETIMEDOUT; 1574 1575 iwl_trans_pcie_sync_nmi(trans); 1576 goto cancel; 1577 } 1578 1579 if (test_bit(STATUS_FW_ERROR, &trans->status)) { 1580 iwl_trans_pcie_dump_regs(trans); 1581 IWL_ERR(trans, "FW error in SYNC CMD %s\n", 1582 iwl_get_cmd_string(trans, cmd->id)); 1583 dump_stack(); 1584 ret = -EIO; 1585 goto cancel; 1586 } 1587 1588 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1589 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1590 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 1591 ret = -ERFKILL; 1592 goto cancel; 1593 } 1594 1595 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 1596 IWL_ERR(trans, "Error: Response NULL in '%s'\n", 1597 iwl_get_cmd_string(trans, cmd->id)); 1598 ret = -EIO; 1599 goto cancel; 1600 } 1601 1602 return 0; 1603 1604 cancel: 1605 if (cmd->flags & CMD_WANT_SKB) { 1606 /* 1607 * Cancel the CMD_WANT_SKB flag for the cmd in the 1608 * TX cmd queue. Otherwise in case the cmd comes 1609 * in later, it will possibly set an invalid 1610 * address (cmd->meta.source). 1611 */ 1612 txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 1613 } 1614 1615 if (cmd->resp_pkt) { 1616 iwl_free_resp(cmd); 1617 cmd->resp_pkt = NULL; 1618 } 1619 1620 return ret; 1621 } 1622 1623 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) 1624 { 1625 /* Make sure the NIC is still alive in the bus */ 1626 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 1627 return -ENODEV; 1628 1629 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1630 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1631 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1632 cmd->id); 1633 return -ERFKILL; 1634 } 1635 1636 if (cmd->flags & CMD_ASYNC) 1637 return iwl_pcie_send_hcmd_async(trans, cmd); 1638 1639 /* We still can fail on RFKILL that can be asserted while we wait */ 1640 return iwl_pcie_send_hcmd_sync(trans, cmd); 1641 } 1642 1643 static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb, 1644 struct iwl_txq *txq, u8 hdr_len, 1645 struct iwl_cmd_meta *out_meta) 1646 { 1647 u16 head_tb_len; 1648 int i; 1649 1650 /* 1651 * Set up TFD's third entry to point directly to remainder 1652 * of skb's head, if any 1653 */ 1654 head_tb_len = skb_headlen(skb) - hdr_len; 1655 1656 if (head_tb_len > 0) { 1657 dma_addr_t tb_phys = dma_map_single(trans->dev, 1658 skb->data + hdr_len, 1659 head_tb_len, DMA_TO_DEVICE); 1660 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 1661 return -EINVAL; 1662 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb->data + hdr_len, 1663 tb_phys, head_tb_len); 1664 iwl_pcie_txq_build_tfd(trans, txq, tb_phys, head_tb_len, false); 1665 } 1666 1667 /* set up the remaining entries to point to the data */ 1668 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1669 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1670 dma_addr_t tb_phys; 1671 int tb_idx; 1672 1673 if (!skb_frag_size(frag)) 1674 continue; 1675 1676 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 1677 skb_frag_size(frag), DMA_TO_DEVICE); 1678 1679 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 1680 return -EINVAL; 1681 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb_frag_address(frag), 1682 tb_phys, skb_frag_size(frag)); 1683 tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 1684 skb_frag_size(frag), false); 1685 if (tb_idx < 0) 1686 return tb_idx; 1687 1688 out_meta->tbs |= BIT(tb_idx); 1689 } 1690 1691 return 0; 1692 } 1693 1694 #ifdef CONFIG_INET 1695 static void iwl_compute_pseudo_hdr_csum(void *iph, struct tcphdr *tcph, 1696 bool ipv6, unsigned int len) 1697 { 1698 if (ipv6) { 1699 struct ipv6hdr *iphv6 = iph; 1700 1701 tcph->check = ~csum_ipv6_magic(&iphv6->saddr, &iphv6->daddr, 1702 len + tcph->doff * 4, 1703 IPPROTO_TCP, 0); 1704 } else { 1705 struct iphdr *iphv4 = iph; 1706 1707 ip_send_check(iphv4); 1708 tcph->check = ~csum_tcpudp_magic(iphv4->saddr, iphv4->daddr, 1709 len + tcph->doff * 4, 1710 IPPROTO_TCP, 0); 1711 } 1712 } 1713 1714 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 1715 struct iwl_txq *txq, u8 hdr_len, 1716 struct iwl_cmd_meta *out_meta, 1717 struct iwl_device_tx_cmd *dev_cmd, 1718 u16 tb1_len) 1719 { 1720 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 1721 struct iwl_trans_pcie *trans_pcie = 1722 IWL_TRANS_GET_PCIE_TRANS(txq->trans); 1723 struct ieee80211_hdr *hdr = (void *)skb->data; 1724 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 1725 unsigned int mss = skb_shinfo(skb)->gso_size; 1726 u16 length, iv_len, amsdu_pad; 1727 u8 *start_hdr; 1728 struct iwl_tso_hdr_page *hdr_page; 1729 struct tso_t tso; 1730 1731 /* if the packet is protected, then it must be CCMP or GCMP */ 1732 BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN); 1733 iv_len = ieee80211_has_protected(hdr->frame_control) ? 1734 IEEE80211_CCMP_HDR_LEN : 0; 1735 1736 trace_iwlwifi_dev_tx(trans->dev, skb, 1737 iwl_txq_get_tfd(trans, txq, txq->write_ptr), 1738 trans->txqs.tfd.size, 1739 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 0); 1740 1741 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 1742 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 1743 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; 1744 amsdu_pad = 0; 1745 1746 /* total amount of header we may need for this A-MSDU */ 1747 hdr_room = DIV_ROUND_UP(total_len, mss) * 1748 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len; 1749 1750 /* Our device supports 9 segments at most, it will fit in 1 page */ 1751 hdr_page = get_page_hdr(trans, hdr_room, skb); 1752 if (!hdr_page) 1753 return -ENOMEM; 1754 1755 start_hdr = hdr_page->pos; 1756 memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); 1757 hdr_page->pos += iv_len; 1758 1759 /* 1760 * Pull the ieee80211 header + IV to be able to use TSO core, 1761 * we will restore it for the tx_status flow. 1762 */ 1763 skb_pull(skb, hdr_len + iv_len); 1764 1765 /* 1766 * Remove the length of all the headers that we don't actually 1767 * have in the MPDU by themselves, but that we duplicate into 1768 * all the different MSDUs inside the A-MSDU. 1769 */ 1770 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 1771 1772 tso_start(skb, &tso); 1773 1774 while (total_len) { 1775 /* this is the data left for this subframe */ 1776 unsigned int data_left = 1777 min_t(unsigned int, mss, total_len); 1778 struct sk_buff *csum_skb = NULL; 1779 unsigned int hdr_tb_len; 1780 dma_addr_t hdr_tb_phys; 1781 struct tcphdr *tcph; 1782 u8 *iph, *subf_hdrs_start = hdr_page->pos; 1783 1784 total_len -= data_left; 1785 1786 memset(hdr_page->pos, 0, amsdu_pad); 1787 hdr_page->pos += amsdu_pad; 1788 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 1789 data_left)) & 0x3; 1790 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 1791 hdr_page->pos += ETH_ALEN; 1792 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 1793 hdr_page->pos += ETH_ALEN; 1794 1795 length = snap_ip_tcp_hdrlen + data_left; 1796 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 1797 hdr_page->pos += sizeof(length); 1798 1799 /* 1800 * This will copy the SNAP as well which will be considered 1801 * as MAC header. 1802 */ 1803 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 1804 iph = hdr_page->pos + 8; 1805 tcph = (void *)(iph + ip_hdrlen); 1806 1807 /* For testing on current hardware only */ 1808 if (trans_pcie->sw_csum_tx) { 1809 csum_skb = alloc_skb(data_left + tcp_hdrlen(skb), 1810 GFP_ATOMIC); 1811 if (!csum_skb) 1812 return -ENOMEM; 1813 1814 iwl_compute_pseudo_hdr_csum(iph, tcph, 1815 skb->protocol == 1816 htons(ETH_P_IPV6), 1817 data_left); 1818 1819 skb_put_data(csum_skb, tcph, tcp_hdrlen(skb)); 1820 skb_reset_transport_header(csum_skb); 1821 csum_skb->csum_start = 1822 (unsigned char *)tcp_hdr(csum_skb) - 1823 csum_skb->head; 1824 } 1825 1826 hdr_page->pos += snap_ip_tcp_hdrlen; 1827 1828 hdr_tb_len = hdr_page->pos - start_hdr; 1829 hdr_tb_phys = dma_map_single(trans->dev, start_hdr, 1830 hdr_tb_len, DMA_TO_DEVICE); 1831 if (unlikely(dma_mapping_error(trans->dev, hdr_tb_phys))) { 1832 dev_kfree_skb(csum_skb); 1833 return -EINVAL; 1834 } 1835 iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys, 1836 hdr_tb_len, false); 1837 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, 1838 hdr_tb_phys, hdr_tb_len); 1839 /* add this subframe's headers' length to the tx_cmd */ 1840 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 1841 1842 /* prepare the start_hdr for the next subframe */ 1843 start_hdr = hdr_page->pos; 1844 1845 /* put the payload */ 1846 while (data_left) { 1847 unsigned int size = min_t(unsigned int, tso.size, 1848 data_left); 1849 dma_addr_t tb_phys; 1850 1851 if (trans_pcie->sw_csum_tx) 1852 skb_put_data(csum_skb, tso.data, size); 1853 1854 tb_phys = dma_map_single(trans->dev, tso.data, 1855 size, DMA_TO_DEVICE); 1856 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 1857 dev_kfree_skb(csum_skb); 1858 return -EINVAL; 1859 } 1860 1861 iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 1862 size, false); 1863 trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data, 1864 tb_phys, size); 1865 1866 data_left -= size; 1867 tso_build_data(skb, &tso, size); 1868 } 1869 1870 /* For testing on early hardware only */ 1871 if (trans_pcie->sw_csum_tx) { 1872 __wsum csum; 1873 1874 csum = skb_checksum(csum_skb, 1875 skb_checksum_start_offset(csum_skb), 1876 csum_skb->len - 1877 skb_checksum_start_offset(csum_skb), 1878 0); 1879 dev_kfree_skb(csum_skb); 1880 dma_sync_single_for_cpu(trans->dev, hdr_tb_phys, 1881 hdr_tb_len, DMA_TO_DEVICE); 1882 tcph->check = csum_fold(csum); 1883 dma_sync_single_for_device(trans->dev, hdr_tb_phys, 1884 hdr_tb_len, DMA_TO_DEVICE); 1885 } 1886 } 1887 1888 /* re -add the WiFi header and IV */ 1889 skb_push(skb, hdr_len + iv_len); 1890 1891 return 0; 1892 } 1893 #else /* CONFIG_INET */ 1894 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 1895 struct iwl_txq *txq, u8 hdr_len, 1896 struct iwl_cmd_meta *out_meta, 1897 struct iwl_device_tx_cmd *dev_cmd, 1898 u16 tb1_len) 1899 { 1900 /* No A-MSDU without CONFIG_INET */ 1901 WARN_ON(1); 1902 1903 return -1; 1904 } 1905 #endif /* CONFIG_INET */ 1906 1907 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, 1908 struct iwl_device_tx_cmd *dev_cmd, int txq_id) 1909 { 1910 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1911 struct ieee80211_hdr *hdr; 1912 struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload; 1913 struct iwl_cmd_meta *out_meta; 1914 struct iwl_txq *txq; 1915 dma_addr_t tb0_phys, tb1_phys, scratch_phys; 1916 void *tb1_addr; 1917 void *tfd; 1918 u16 len, tb1_len; 1919 bool wait_write_ptr; 1920 __le16 fc; 1921 u8 hdr_len; 1922 u16 wifi_seq; 1923 bool amsdu; 1924 1925 txq = trans->txqs.txq[txq_id]; 1926 1927 if (WARN_ONCE(!test_bit(txq_id, trans->txqs.queue_used), 1928 "TX on unused queue %d\n", txq_id)) 1929 return -EINVAL; 1930 1931 if (unlikely(trans_pcie->sw_csum_tx && 1932 skb->ip_summed == CHECKSUM_PARTIAL)) { 1933 int offs = skb_checksum_start_offset(skb); 1934 int csum_offs = offs + skb->csum_offset; 1935 __wsum csum; 1936 1937 if (skb_ensure_writable(skb, csum_offs + sizeof(__sum16))) 1938 return -1; 1939 1940 csum = skb_checksum(skb, offs, skb->len - offs, 0); 1941 *(__sum16 *)(skb->data + csum_offs) = csum_fold(csum); 1942 1943 skb->ip_summed = CHECKSUM_UNNECESSARY; 1944 } 1945 1946 if (skb_is_nonlinear(skb) && 1947 skb_shinfo(skb)->nr_frags > IWL_TRANS_MAX_FRAGS(trans) && 1948 __skb_linearize(skb)) 1949 return -ENOMEM; 1950 1951 /* mac80211 always puts the full header into the SKB's head, 1952 * so there's no need to check if it's readable there 1953 */ 1954 hdr = (struct ieee80211_hdr *)skb->data; 1955 fc = hdr->frame_control; 1956 hdr_len = ieee80211_hdrlen(fc); 1957 1958 spin_lock(&txq->lock); 1959 1960 if (iwl_txq_space(trans, txq) < txq->high_mark) { 1961 iwl_txq_stop(trans, txq); 1962 1963 /* don't put the packet on the ring, if there is no room */ 1964 if (unlikely(iwl_txq_space(trans, txq) < 3)) { 1965 struct iwl_device_tx_cmd **dev_cmd_ptr; 1966 1967 dev_cmd_ptr = (void *)((u8 *)skb->cb + 1968 trans->txqs.dev_cmd_offs); 1969 1970 *dev_cmd_ptr = dev_cmd; 1971 __skb_queue_tail(&txq->overflow_q, skb); 1972 1973 spin_unlock(&txq->lock); 1974 return 0; 1975 } 1976 } 1977 1978 /* In AGG mode, the index in the ring must correspond to the WiFi 1979 * sequence number. This is a HW requirements to help the SCD to parse 1980 * the BA. 1981 * Check here that the packets are in the right place on the ring. 1982 */ 1983 wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl)); 1984 WARN_ONCE(txq->ampdu && 1985 (wifi_seq & 0xff) != txq->write_ptr, 1986 "Q: %d WiFi Seq %d tfdNum %d", 1987 txq_id, wifi_seq, txq->write_ptr); 1988 1989 /* Set up driver data for this TFD */ 1990 txq->entries[txq->write_ptr].skb = skb; 1991 txq->entries[txq->write_ptr].cmd = dev_cmd; 1992 1993 dev_cmd->hdr.sequence = 1994 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 1995 INDEX_TO_SEQ(txq->write_ptr))); 1996 1997 tb0_phys = iwl_txq_get_first_tb_dma(txq, txq->write_ptr); 1998 scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) + 1999 offsetof(struct iwl_tx_cmd, scratch); 2000 2001 tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys); 2002 tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys); 2003 2004 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 2005 out_meta = &txq->entries[txq->write_ptr].meta; 2006 out_meta->flags = 0; 2007 2008 /* 2009 * The second TB (tb1) points to the remainder of the TX command 2010 * and the 802.11 header - dword aligned size 2011 * (This calculation modifies the TX command, so do it before the 2012 * setup of the first TB) 2013 */ 2014 len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) + 2015 hdr_len - IWL_FIRST_TB_SIZE; 2016 /* do not align A-MSDU to dword as the subframe header aligns it */ 2017 amsdu = ieee80211_is_data_qos(fc) && 2018 (*ieee80211_get_qos_ctl(hdr) & 2019 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 2020 if (trans_pcie->sw_csum_tx || !amsdu) { 2021 tb1_len = ALIGN(len, 4); 2022 /* Tell NIC about any 2-byte padding after MAC header */ 2023 if (tb1_len != len) 2024 tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD); 2025 } else { 2026 tb1_len = len; 2027 } 2028 2029 /* 2030 * The first TB points to bi-directional DMA data, we'll 2031 * memcpy the data into it later. 2032 */ 2033 iwl_pcie_txq_build_tfd(trans, txq, tb0_phys, 2034 IWL_FIRST_TB_SIZE, true); 2035 2036 /* there must be data left over for TB1 or this code must be changed */ 2037 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE); 2038 2039 /* map the data for TB1 */ 2040 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 2041 tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 2042 if (unlikely(dma_mapping_error(trans->dev, tb1_phys))) 2043 goto out_err; 2044 iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false); 2045 2046 trace_iwlwifi_dev_tx(trans->dev, skb, 2047 iwl_txq_get_tfd(trans, txq, txq->write_ptr), 2048 trans->txqs.tfd.size, 2049 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 2050 hdr_len); 2051 2052 /* 2053 * If gso_size wasn't set, don't give the frame "amsdu treatment" 2054 * (adding subframes, etc.). 2055 * This can happen in some testing flows when the amsdu was already 2056 * pre-built, and we just need to send the resulting skb. 2057 */ 2058 if (amsdu && skb_shinfo(skb)->gso_size) { 2059 if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len, 2060 out_meta, dev_cmd, 2061 tb1_len))) 2062 goto out_err; 2063 } else { 2064 struct sk_buff *frag; 2065 2066 if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len, 2067 out_meta))) 2068 goto out_err; 2069 2070 skb_walk_frags(skb, frag) { 2071 if (unlikely(iwl_fill_data_tbs(trans, frag, txq, 0, 2072 out_meta))) 2073 goto out_err; 2074 } 2075 } 2076 2077 /* building the A-MSDU might have changed this data, so memcpy it now */ 2078 memcpy(&txq->first_tb_bufs[txq->write_ptr], dev_cmd, IWL_FIRST_TB_SIZE); 2079 2080 tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr); 2081 /* Set up entry for this TFD in Tx byte-count array */ 2082 iwl_txq_gen1_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len), 2083 iwl_txq_gen1_tfd_get_num_tbs(trans, 2084 tfd)); 2085 2086 wait_write_ptr = ieee80211_has_morefrags(fc); 2087 2088 /* start timer if queue currently empty */ 2089 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) { 2090 /* 2091 * If the TXQ is active, then set the timer, if not, 2092 * set the timer in remainder so that the timer will 2093 * be armed with the right value when the station will 2094 * wake up. 2095 */ 2096 if (!txq->frozen) 2097 mod_timer(&txq->stuck_timer, 2098 jiffies + txq->wd_timeout); 2099 else 2100 txq->frozen_expiry_remainder = txq->wd_timeout; 2101 } 2102 2103 /* Tell device the write index *just past* this latest filled TFD */ 2104 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 2105 if (!wait_write_ptr) 2106 iwl_pcie_txq_inc_wr_ptr(trans, txq); 2107 2108 /* 2109 * At this point the frame is "transmitted" successfully 2110 * and we will get a TX status notification eventually. 2111 */ 2112 spin_unlock(&txq->lock); 2113 return 0; 2114 out_err: 2115 iwl_txq_gen1_tfd_unmap(trans, out_meta, txq, txq->write_ptr); 2116 spin_unlock(&txq->lock); 2117 return -1; 2118 } 2119