1 /****************************************************************************** 2 * 3 * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. 4 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH 5 * Copyright(c) 2016 Intel Deutschland GmbH 6 * 7 * Portions of this file are derived from the ipw3945 project, as well 8 * as portions of the ieee80211 subsystem header files. 9 * 10 * This program is free software; you can redistribute it and/or modify it 11 * under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, but WITHOUT 15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 17 * more details. 18 * 19 * You should have received a copy of the GNU General Public License along with 20 * this program; if not, write to the Free Software Foundation, Inc., 21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA 22 * 23 * The full GNU General Public License is included in this distribution in the 24 * file called LICENSE. 25 * 26 * Contact Information: 27 * Intel Linux Wireless <linuxwifi@intel.com> 28 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 29 * 30 *****************************************************************************/ 31 #include <linux/etherdevice.h> 32 #include <linux/ieee80211.h> 33 #include <linux/slab.h> 34 #include <linux/sched.h> 35 #include <linux/pm_runtime.h> 36 #include <net/ip6_checksum.h> 37 #include <net/tso.h> 38 39 #include "iwl-debug.h" 40 #include "iwl-csr.h" 41 #include "iwl-prph.h" 42 #include "iwl-io.h" 43 #include "iwl-scd.h" 44 #include "iwl-op-mode.h" 45 #include "internal.h" 46 /* FIXME: need to abstract out TX command (once we know what it looks like) */ 47 #include "dvm/commands.h" 48 49 #define IWL_TX_CRC_SIZE 4 50 #define IWL_TX_DELIMITER_SIZE 4 51 52 /*************** DMA-QUEUE-GENERAL-FUNCTIONS ***** 53 * DMA services 54 * 55 * Theory of operation 56 * 57 * A Tx or Rx queue resides in host DRAM, and is comprised of a circular buffer 58 * of buffer descriptors, each of which points to one or more data buffers for 59 * the device to read from or fill. Driver and device exchange status of each 60 * queue via "read" and "write" pointers. Driver keeps minimum of 2 empty 61 * entries in each circular buffer, to protect against confusing empty and full 62 * queue states. 63 * 64 * The device reads or writes the data in the queues via the device's several 65 * DMA/FIFO channels. Each queue is mapped to a single DMA channel. 66 * 67 * For Tx queue, there are low mark and high mark limits. If, after queuing 68 * the packet for Tx, free space become < low mark, Tx queue stopped. When 69 * reclaiming packets (on 'tx done IRQ), if free space become > high mark, 70 * Tx queue resumed. 71 * 72 ***************************************************/ 73 74 static int iwl_queue_space(const struct iwl_txq *q) 75 { 76 unsigned int max; 77 unsigned int used; 78 79 /* 80 * To avoid ambiguity between empty and completely full queues, there 81 * should always be less than TFD_QUEUE_SIZE_MAX elements in the queue. 82 * If q->n_window is smaller than TFD_QUEUE_SIZE_MAX, there is no need 83 * to reserve any queue entries for this purpose. 84 */ 85 if (q->n_window < TFD_QUEUE_SIZE_MAX) 86 max = q->n_window; 87 else 88 max = TFD_QUEUE_SIZE_MAX - 1; 89 90 /* 91 * TFD_QUEUE_SIZE_MAX is a power of 2, so the following is equivalent to 92 * modulo by TFD_QUEUE_SIZE_MAX and is well defined. 93 */ 94 used = (q->write_ptr - q->read_ptr) & (TFD_QUEUE_SIZE_MAX - 1); 95 96 if (WARN_ON(used > max)) 97 return 0; 98 99 return max - used; 100 } 101 102 /* 103 * iwl_queue_init - Initialize queue's high/low-water and read/write indexes 104 */ 105 static int iwl_queue_init(struct iwl_txq *q, int slots_num, u32 id) 106 { 107 q->n_window = slots_num; 108 q->id = id; 109 110 /* slots_num must be power-of-two size, otherwise 111 * get_cmd_index is broken. */ 112 if (WARN_ON(!is_power_of_2(slots_num))) 113 return -EINVAL; 114 115 q->low_mark = q->n_window / 4; 116 if (q->low_mark < 4) 117 q->low_mark = 4; 118 119 q->high_mark = q->n_window / 8; 120 if (q->high_mark < 2) 121 q->high_mark = 2; 122 123 q->write_ptr = 0; 124 q->read_ptr = 0; 125 126 return 0; 127 } 128 129 static int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans, 130 struct iwl_dma_ptr *ptr, size_t size) 131 { 132 if (WARN_ON(ptr->addr)) 133 return -EINVAL; 134 135 ptr->addr = dma_alloc_coherent(trans->dev, size, 136 &ptr->dma, GFP_KERNEL); 137 if (!ptr->addr) 138 return -ENOMEM; 139 ptr->size = size; 140 return 0; 141 } 142 143 static void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, 144 struct iwl_dma_ptr *ptr) 145 { 146 if (unlikely(!ptr->addr)) 147 return; 148 149 dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma); 150 memset(ptr, 0, sizeof(*ptr)); 151 } 152 153 static void iwl_pcie_txq_stuck_timer(unsigned long data) 154 { 155 struct iwl_txq *txq = (void *)data; 156 struct iwl_trans_pcie *trans_pcie = txq->trans_pcie; 157 struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie); 158 159 spin_lock(&txq->lock); 160 /* check if triggered erroneously */ 161 if (txq->read_ptr == txq->write_ptr) { 162 spin_unlock(&txq->lock); 163 return; 164 } 165 spin_unlock(&txq->lock); 166 167 IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->id, 168 jiffies_to_msecs(txq->wd_timeout)); 169 170 iwl_trans_pcie_log_scd_error(trans, txq); 171 172 iwl_force_nmi(trans); 173 } 174 175 /* 176 * iwl_pcie_txq_update_byte_cnt_tbl - Set up entry in Tx byte-count array 177 */ 178 static void iwl_pcie_txq_update_byte_cnt_tbl(struct iwl_trans *trans, 179 struct iwl_txq *txq, u16 byte_cnt, 180 int num_tbs) 181 { 182 struct iwlagn_scd_bc_tbl *scd_bc_tbl; 183 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 184 int write_ptr = txq->write_ptr; 185 int txq_id = txq->id; 186 u8 sec_ctl = 0; 187 u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE; 188 __le16 bc_ent; 189 struct iwl_tx_cmd *tx_cmd = 190 (void *)txq->entries[txq->write_ptr].cmd->payload; 191 192 scd_bc_tbl = trans_pcie->scd_bc_tbls.addr; 193 194 sec_ctl = tx_cmd->sec_ctl; 195 196 switch (sec_ctl & TX_CMD_SEC_MSK) { 197 case TX_CMD_SEC_CCM: 198 len += IEEE80211_CCMP_MIC_LEN; 199 break; 200 case TX_CMD_SEC_TKIP: 201 len += IEEE80211_TKIP_ICV_LEN; 202 break; 203 case TX_CMD_SEC_WEP: 204 len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN; 205 break; 206 } 207 if (trans_pcie->bc_table_dword) 208 len = DIV_ROUND_UP(len, 4); 209 210 if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX)) 211 return; 212 213 if (trans->cfg->use_tfh) { 214 u8 filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) + 215 num_tbs * sizeof(struct iwl_tfh_tb); 216 /* 217 * filled_tfd_size contains the number of filled bytes in the 218 * TFD. 219 * Dividing it by 64 will give the number of chunks to fetch 220 * to SRAM- 0 for one chunk, 1 for 2 and so on. 221 * If, for example, TFD contains only 3 TBs then 32 bytes 222 * of the TFD are used, and only one chunk of 64 bytes should 223 * be fetched 224 */ 225 u8 num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1; 226 227 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12)); 228 } else { 229 u8 sta_id = tx_cmd->sta_id; 230 231 bc_ent = cpu_to_le16(len | (sta_id << 12)); 232 } 233 234 scd_bc_tbl[txq_id].tfd_offset[write_ptr] = bc_ent; 235 236 if (write_ptr < TFD_QUEUE_SIZE_BC_DUP) 237 scd_bc_tbl[txq_id]. 238 tfd_offset[TFD_QUEUE_SIZE_MAX + write_ptr] = bc_ent; 239 } 240 241 static void iwl_pcie_txq_inval_byte_cnt_tbl(struct iwl_trans *trans, 242 struct iwl_txq *txq) 243 { 244 struct iwl_trans_pcie *trans_pcie = 245 IWL_TRANS_GET_PCIE_TRANS(trans); 246 struct iwlagn_scd_bc_tbl *scd_bc_tbl = trans_pcie->scd_bc_tbls.addr; 247 int txq_id = txq->id; 248 int read_ptr = txq->read_ptr; 249 u8 sta_id = 0; 250 __le16 bc_ent; 251 struct iwl_tx_cmd *tx_cmd = 252 (void *)txq->entries[read_ptr].cmd->payload; 253 254 WARN_ON(read_ptr >= TFD_QUEUE_SIZE_MAX); 255 256 if (txq_id != trans_pcie->cmd_queue) 257 sta_id = tx_cmd->sta_id; 258 259 bc_ent = cpu_to_le16(1 | (sta_id << 12)); 260 261 scd_bc_tbl[txq_id].tfd_offset[read_ptr] = bc_ent; 262 263 if (read_ptr < TFD_QUEUE_SIZE_BC_DUP) 264 scd_bc_tbl[txq_id]. 265 tfd_offset[TFD_QUEUE_SIZE_MAX + read_ptr] = bc_ent; 266 } 267 268 /* 269 * iwl_pcie_txq_inc_wr_ptr - Send new write index to hardware 270 */ 271 static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans, 272 struct iwl_txq *txq) 273 { 274 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 275 u32 reg = 0; 276 int txq_id = txq->id; 277 278 lockdep_assert_held(&txq->lock); 279 280 /* 281 * explicitly wake up the NIC if: 282 * 1. shadow registers aren't enabled 283 * 2. NIC is woken up for CMD regardless of shadow outside this function 284 * 3. there is a chance that the NIC is asleep 285 */ 286 if (!trans->cfg->base_params->shadow_reg_enable && 287 txq_id != trans_pcie->cmd_queue && 288 test_bit(STATUS_TPOWER_PMI, &trans->status)) { 289 /* 290 * wake up nic if it's powered down ... 291 * uCode will wake up, and interrupt us again, so next 292 * time we'll skip this part. 293 */ 294 reg = iwl_read32(trans, CSR_UCODE_DRV_GP1); 295 296 if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) { 297 IWL_DEBUG_INFO(trans, "Tx queue %d requesting wakeup, GP1 = 0x%x\n", 298 txq_id, reg); 299 iwl_set_bit(trans, CSR_GP_CNTRL, 300 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 301 txq->need_update = true; 302 return; 303 } 304 } 305 306 /* 307 * if not in power-save mode, uCode will never sleep when we're 308 * trying to tx (during RFKILL, we're not trying to tx). 309 */ 310 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr); 311 if (!txq->block) 312 iwl_write32(trans, HBUS_TARG_WRPTR, 313 txq->write_ptr | (txq_id << 8)); 314 } 315 316 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans) 317 { 318 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 319 int i; 320 321 for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) { 322 struct iwl_txq *txq = &trans_pcie->txq[i]; 323 324 spin_lock_bh(&txq->lock); 325 if (trans_pcie->txq[i].need_update) { 326 iwl_pcie_txq_inc_wr_ptr(trans, txq); 327 trans_pcie->txq[i].need_update = false; 328 } 329 spin_unlock_bh(&txq->lock); 330 } 331 } 332 333 static inline void *iwl_pcie_get_tfd(struct iwl_trans_pcie *trans_pcie, 334 struct iwl_txq *txq, int idx) 335 { 336 return txq->tfds + trans_pcie->tfd_size * idx; 337 } 338 339 static inline dma_addr_t iwl_pcie_tfd_tb_get_addr(struct iwl_trans *trans, 340 void *_tfd, u8 idx) 341 { 342 343 if (trans->cfg->use_tfh) { 344 struct iwl_tfh_tfd *tfd = _tfd; 345 struct iwl_tfh_tb *tb = &tfd->tbs[idx]; 346 347 return (dma_addr_t)(le64_to_cpu(tb->addr)); 348 } else { 349 struct iwl_tfd *tfd = _tfd; 350 struct iwl_tfd_tb *tb = &tfd->tbs[idx]; 351 dma_addr_t addr = get_unaligned_le32(&tb->lo); 352 dma_addr_t hi_len; 353 354 if (sizeof(dma_addr_t) <= sizeof(u32)) 355 return addr; 356 357 hi_len = le16_to_cpu(tb->hi_n_len) & 0xF; 358 359 /* 360 * shift by 16 twice to avoid warnings on 32-bit 361 * (where this code never runs anyway due to the 362 * if statement above) 363 */ 364 return addr | ((hi_len << 16) << 16); 365 } 366 } 367 368 static inline void iwl_pcie_tfd_set_tb(struct iwl_trans *trans, void *tfd, 369 u8 idx, dma_addr_t addr, u16 len) 370 { 371 if (trans->cfg->use_tfh) { 372 struct iwl_tfh_tfd *tfd_fh = (void *)tfd; 373 struct iwl_tfh_tb *tb = &tfd_fh->tbs[idx]; 374 375 put_unaligned_le64(addr, &tb->addr); 376 tb->tb_len = cpu_to_le16(len); 377 378 tfd_fh->num_tbs = cpu_to_le16(idx + 1); 379 } else { 380 struct iwl_tfd *tfd_fh = (void *)tfd; 381 struct iwl_tfd_tb *tb = &tfd_fh->tbs[idx]; 382 383 u16 hi_n_len = len << 4; 384 385 put_unaligned_le32(addr, &tb->lo); 386 if (sizeof(dma_addr_t) > sizeof(u32)) 387 hi_n_len |= ((addr >> 16) >> 16) & 0xF; 388 389 tb->hi_n_len = cpu_to_le16(hi_n_len); 390 391 tfd_fh->num_tbs = idx + 1; 392 } 393 } 394 395 static inline u8 iwl_pcie_tfd_get_num_tbs(struct iwl_trans *trans, void *_tfd) 396 { 397 if (trans->cfg->use_tfh) { 398 struct iwl_tfh_tfd *tfd = _tfd; 399 400 return le16_to_cpu(tfd->num_tbs) & 0x1f; 401 } else { 402 struct iwl_tfd *tfd = _tfd; 403 404 return tfd->num_tbs & 0x1f; 405 } 406 } 407 408 static void iwl_pcie_tfd_unmap(struct iwl_trans *trans, 409 struct iwl_cmd_meta *meta, 410 struct iwl_txq *txq, int index) 411 { 412 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 413 int i, num_tbs; 414 void *tfd = iwl_pcie_get_tfd(trans_pcie, txq, index); 415 416 /* Sanity check on number of chunks */ 417 num_tbs = iwl_pcie_tfd_get_num_tbs(trans, tfd); 418 419 if (num_tbs >= trans_pcie->max_tbs) { 420 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 421 /* @todo issue fatal error, it is quite serious situation */ 422 return; 423 } 424 425 /* first TB is never freed - it's the bidirectional DMA data */ 426 427 for (i = 1; i < num_tbs; i++) { 428 if (meta->tbs & BIT(i)) 429 dma_unmap_page(trans->dev, 430 iwl_pcie_tfd_tb_get_addr(trans, tfd, i), 431 iwl_pcie_tfd_tb_get_len(trans, tfd, i), 432 DMA_TO_DEVICE); 433 else 434 dma_unmap_single(trans->dev, 435 iwl_pcie_tfd_tb_get_addr(trans, tfd, 436 i), 437 iwl_pcie_tfd_tb_get_len(trans, tfd, 438 i), 439 DMA_TO_DEVICE); 440 } 441 442 if (trans->cfg->use_tfh) { 443 struct iwl_tfh_tfd *tfd_fh = (void *)tfd; 444 445 tfd_fh->num_tbs = 0; 446 } else { 447 struct iwl_tfd *tfd_fh = (void *)tfd; 448 449 tfd_fh->num_tbs = 0; 450 } 451 452 } 453 454 /* 455 * iwl_pcie_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr] 456 * @trans - transport private data 457 * @txq - tx queue 458 * @dma_dir - the direction of the DMA mapping 459 * 460 * Does NOT advance any TFD circular buffer read/write indexes 461 * Does NOT free the TFD itself (which is within circular buffer) 462 */ 463 static void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 464 { 465 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 466 * idx is bounded by n_window 467 */ 468 int rd_ptr = txq->read_ptr; 469 int idx = get_cmd_index(txq, rd_ptr); 470 471 lockdep_assert_held(&txq->lock); 472 473 /* We have only q->n_window txq->entries, but we use 474 * TFD_QUEUE_SIZE_MAX tfds 475 */ 476 iwl_pcie_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr); 477 478 /* free SKB */ 479 if (txq->entries) { 480 struct sk_buff *skb; 481 482 skb = txq->entries[idx].skb; 483 484 /* Can be called from irqs-disabled context 485 * If skb is not NULL, it means that the whole queue is being 486 * freed and that the queue is not empty - free the skb 487 */ 488 if (skb) { 489 iwl_op_mode_free_skb(trans->op_mode, skb); 490 txq->entries[idx].skb = NULL; 491 } 492 } 493 } 494 495 static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq, 496 dma_addr_t addr, u16 len, bool reset) 497 { 498 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 499 void *tfd; 500 u32 num_tbs; 501 502 tfd = txq->tfds + trans_pcie->tfd_size * txq->write_ptr; 503 504 if (reset) 505 memset(tfd, 0, trans_pcie->tfd_size); 506 507 num_tbs = iwl_pcie_tfd_get_num_tbs(trans, tfd); 508 509 /* Each TFD can point to a maximum max_tbs Tx buffers */ 510 if (num_tbs >= trans_pcie->max_tbs) { 511 IWL_ERR(trans, "Error can not send more than %d chunks\n", 512 trans_pcie->max_tbs); 513 return -EINVAL; 514 } 515 516 if (WARN(addr & ~IWL_TX_DMA_MASK, 517 "Unaligned address = %llx\n", (unsigned long long)addr)) 518 return -EINVAL; 519 520 iwl_pcie_tfd_set_tb(trans, tfd, num_tbs, addr, len); 521 522 return num_tbs; 523 } 524 525 static int iwl_pcie_txq_alloc(struct iwl_trans *trans, 526 struct iwl_txq *txq, int slots_num, 527 u32 txq_id) 528 { 529 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 530 size_t tfd_sz = trans_pcie->tfd_size * TFD_QUEUE_SIZE_MAX; 531 size_t tb0_buf_sz; 532 int i; 533 534 if (WARN_ON(txq->entries || txq->tfds)) 535 return -EINVAL; 536 537 setup_timer(&txq->stuck_timer, iwl_pcie_txq_stuck_timer, 538 (unsigned long)txq); 539 txq->trans_pcie = trans_pcie; 540 541 txq->n_window = slots_num; 542 543 txq->entries = kcalloc(slots_num, 544 sizeof(struct iwl_pcie_txq_entry), 545 GFP_KERNEL); 546 547 if (!txq->entries) 548 goto error; 549 550 if (txq_id == trans_pcie->cmd_queue) 551 for (i = 0; i < slots_num; i++) { 552 txq->entries[i].cmd = 553 kmalloc(sizeof(struct iwl_device_cmd), 554 GFP_KERNEL); 555 if (!txq->entries[i].cmd) 556 goto error; 557 } 558 559 /* Circular buffer of transmit frame descriptors (TFDs), 560 * shared with device */ 561 txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz, 562 &txq->dma_addr, GFP_KERNEL); 563 if (!txq->tfds) 564 goto error; 565 566 BUILD_BUG_ON(IWL_FIRST_TB_SIZE_ALIGN != sizeof(*txq->first_tb_bufs)); 567 568 tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num; 569 570 txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz, 571 &txq->first_tb_dma, 572 GFP_KERNEL); 573 if (!txq->first_tb_bufs) 574 goto err_free_tfds; 575 576 txq->id = txq_id; 577 578 return 0; 579 err_free_tfds: 580 dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr); 581 error: 582 if (txq->entries && txq_id == trans_pcie->cmd_queue) 583 for (i = 0; i < slots_num; i++) 584 kfree(txq->entries[i].cmd); 585 kfree(txq->entries); 586 txq->entries = NULL; 587 588 return -ENOMEM; 589 590 } 591 592 static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, 593 int slots_num, u32 txq_id) 594 { 595 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 596 int ret; 597 598 txq->need_update = false; 599 600 /* TFD_QUEUE_SIZE_MAX must be power-of-two size, otherwise 601 * iwl_queue_inc_wrap and iwl_queue_dec_wrap are broken. */ 602 BUILD_BUG_ON(TFD_QUEUE_SIZE_MAX & (TFD_QUEUE_SIZE_MAX - 1)); 603 604 /* Initialize queue's high/low-water marks, and head/tail indexes */ 605 ret = iwl_queue_init(txq, slots_num, txq_id); 606 if (ret) 607 return ret; 608 609 spin_lock_init(&txq->lock); 610 611 if (txq_id == trans_pcie->cmd_queue) { 612 static struct lock_class_key iwl_pcie_cmd_queue_lock_class; 613 614 lockdep_set_class(&txq->lock, &iwl_pcie_cmd_queue_lock_class); 615 } 616 617 __skb_queue_head_init(&txq->overflow_q); 618 619 /* 620 * Tell nic where to find circular buffer of Tx Frame Descriptors for 621 * given Tx queue, and enable the DMA channel used for that queue. 622 * Circular buffer (TFD queue in DRAM) physical base address */ 623 if (trans->cfg->use_tfh) 624 iwl_write_direct64(trans, 625 FH_MEM_CBBC_QUEUE(trans, txq_id), 626 txq->dma_addr); 627 else 628 iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id), 629 txq->dma_addr >> 8); 630 631 return 0; 632 } 633 634 static void iwl_pcie_free_tso_page(struct iwl_trans_pcie *trans_pcie, 635 struct sk_buff *skb) 636 { 637 struct page **page_ptr; 638 639 page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); 640 641 if (*page_ptr) { 642 __free_page(*page_ptr); 643 *page_ptr = NULL; 644 } 645 } 646 647 static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans) 648 { 649 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 650 651 lockdep_assert_held(&trans_pcie->reg_lock); 652 653 if (trans_pcie->ref_cmd_in_flight) { 654 trans_pcie->ref_cmd_in_flight = false; 655 IWL_DEBUG_RPM(trans, "clear ref_cmd_in_flight - unref\n"); 656 iwl_trans_unref(trans); 657 } 658 659 if (!trans->cfg->base_params->apmg_wake_up_wa) 660 return; 661 if (WARN_ON(!trans_pcie->cmd_hold_nic_awake)) 662 return; 663 664 trans_pcie->cmd_hold_nic_awake = false; 665 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 666 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 667 } 668 669 /* 670 * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's 671 */ 672 static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id) 673 { 674 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 675 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 676 677 spin_lock_bh(&txq->lock); 678 while (txq->write_ptr != txq->read_ptr) { 679 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 680 txq_id, txq->read_ptr); 681 682 if (txq_id != trans_pcie->cmd_queue) { 683 struct sk_buff *skb = txq->entries[txq->read_ptr].skb; 684 685 if (WARN_ON_ONCE(!skb)) 686 continue; 687 688 iwl_pcie_free_tso_page(trans_pcie, skb); 689 } 690 iwl_pcie_txq_free_tfd(trans, txq); 691 txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr); 692 693 if (txq->read_ptr == txq->write_ptr) { 694 unsigned long flags; 695 696 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 697 if (txq_id != trans_pcie->cmd_queue) { 698 IWL_DEBUG_RPM(trans, "Q %d - last tx freed\n", 699 txq->id); 700 iwl_trans_unref(trans); 701 } else { 702 iwl_pcie_clear_cmd_in_flight(trans); 703 } 704 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 705 } 706 } 707 txq->active = false; 708 709 while (!skb_queue_empty(&txq->overflow_q)) { 710 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 711 712 iwl_op_mode_free_skb(trans->op_mode, skb); 713 } 714 715 spin_unlock_bh(&txq->lock); 716 717 /* just in case - this queue may have been stopped */ 718 iwl_wake_queue(trans, txq); 719 } 720 721 /* 722 * iwl_pcie_txq_free - Deallocate DMA queue. 723 * @txq: Transmit queue to deallocate. 724 * 725 * Empty queue by removing and destroying all BD's. 726 * Free all buffers. 727 * 0-fill, but do not free "txq" descriptor structure. 728 */ 729 static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id) 730 { 731 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 732 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 733 struct device *dev = trans->dev; 734 int i; 735 736 if (WARN_ON(!txq)) 737 return; 738 739 iwl_pcie_txq_unmap(trans, txq_id); 740 741 /* De-alloc array of command/tx buffers */ 742 if (txq_id == trans_pcie->cmd_queue) 743 for (i = 0; i < txq->n_window; i++) { 744 kzfree(txq->entries[i].cmd); 745 kzfree(txq->entries[i].free_buf); 746 } 747 748 /* De-alloc circular buffer of TFDs */ 749 if (txq->tfds) { 750 dma_free_coherent(dev, 751 trans_pcie->tfd_size * TFD_QUEUE_SIZE_MAX, 752 txq->tfds, txq->dma_addr); 753 txq->dma_addr = 0; 754 txq->tfds = NULL; 755 756 dma_free_coherent(dev, 757 sizeof(*txq->first_tb_bufs) * txq->n_window, 758 txq->first_tb_bufs, txq->first_tb_dma); 759 } 760 761 kfree(txq->entries); 762 txq->entries = NULL; 763 764 del_timer_sync(&txq->stuck_timer); 765 766 /* 0-fill queue descriptor structure */ 767 memset(txq, 0, sizeof(*txq)); 768 } 769 770 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr) 771 { 772 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 773 int nq = trans->cfg->base_params->num_of_queues; 774 int chan; 775 u32 reg_val; 776 int clear_dwords = (SCD_TRANS_TBL_OFFSET_QUEUE(nq) - 777 SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(u32); 778 779 /* make sure all queue are not stopped/used */ 780 memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped)); 781 memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); 782 783 if (trans->cfg->use_tfh) 784 return; 785 786 trans_pcie->scd_base_addr = 787 iwl_read_prph(trans, SCD_SRAM_BASE_ADDR); 788 789 WARN_ON(scd_base_addr != 0 && 790 scd_base_addr != trans_pcie->scd_base_addr); 791 792 /* reset context data, TX status and translation data */ 793 iwl_trans_write_mem(trans, trans_pcie->scd_base_addr + 794 SCD_CONTEXT_MEM_LOWER_BOUND, 795 NULL, clear_dwords); 796 797 iwl_write_prph(trans, SCD_DRAM_BASE_ADDR, 798 trans_pcie->scd_bc_tbls.dma >> 10); 799 800 /* The chain extension of the SCD doesn't work well. This feature is 801 * enabled by default by the HW, so we need to disable it manually. 802 */ 803 if (trans->cfg->base_params->scd_chain_ext_wa) 804 iwl_write_prph(trans, SCD_CHAINEXT_EN, 0); 805 806 iwl_trans_ac_txq_enable(trans, trans_pcie->cmd_queue, 807 trans_pcie->cmd_fifo, 808 trans_pcie->cmd_q_wdg_timeout); 809 810 /* Activate all Tx DMA/FIFO channels */ 811 iwl_scd_activate_fifos(trans); 812 813 /* Enable DMA channel */ 814 for (chan = 0; chan < FH_TCSR_CHNL_NUM; chan++) 815 iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan), 816 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | 817 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE); 818 819 /* Update FH chicken bits */ 820 reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG); 821 iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG, 822 reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); 823 824 /* Enable L1-Active */ 825 if (trans->cfg->device_family != IWL_DEVICE_FAMILY_8000) 826 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG, 827 APMG_PCIDEV_STT_VAL_L1_ACT_DIS); 828 } 829 830 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans) 831 { 832 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 833 int txq_id; 834 835 for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues; 836 txq_id++) { 837 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 838 if (trans->cfg->use_tfh) 839 iwl_write_direct64(trans, 840 FH_MEM_CBBC_QUEUE(trans, txq_id), 841 txq->dma_addr); 842 else 843 iwl_write_direct32(trans, 844 FH_MEM_CBBC_QUEUE(trans, txq_id), 845 txq->dma_addr >> 8); 846 iwl_pcie_txq_unmap(trans, txq_id); 847 txq->read_ptr = 0; 848 txq->write_ptr = 0; 849 } 850 851 /* Tell NIC where to find the "keep warm" buffer */ 852 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 853 trans_pcie->kw.dma >> 4); 854 855 /* 856 * Send 0 as the scd_base_addr since the device may have be reset 857 * while we were in WoWLAN in which case SCD_SRAM_BASE_ADDR will 858 * contain garbage. 859 */ 860 iwl_pcie_tx_start(trans, 0); 861 } 862 863 static void iwl_pcie_tx_stop_fh(struct iwl_trans *trans) 864 { 865 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 866 unsigned long flags; 867 int ch, ret; 868 u32 mask = 0; 869 870 spin_lock(&trans_pcie->irq_lock); 871 872 if (!iwl_trans_grab_nic_access(trans, &flags)) 873 goto out; 874 875 /* Stop each Tx DMA channel */ 876 for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) { 877 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0); 878 mask |= FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch); 879 } 880 881 /* Wait for DMA channels to be idle */ 882 ret = iwl_poll_bit(trans, FH_TSSR_TX_STATUS_REG, mask, mask, 5000); 883 if (ret < 0) 884 IWL_ERR(trans, 885 "Failing on timeout while stopping DMA channel %d [0x%08x]\n", 886 ch, iwl_read32(trans, FH_TSSR_TX_STATUS_REG)); 887 888 iwl_trans_release_nic_access(trans, &flags); 889 890 out: 891 spin_unlock(&trans_pcie->irq_lock); 892 } 893 894 /* 895 * iwl_pcie_tx_stop - Stop all Tx DMA channels 896 */ 897 int iwl_pcie_tx_stop(struct iwl_trans *trans) 898 { 899 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 900 int txq_id; 901 902 /* Turn off all Tx DMA fifos */ 903 iwl_scd_deactivate_fifos(trans); 904 905 /* Turn off all Tx DMA channels */ 906 iwl_pcie_tx_stop_fh(trans); 907 908 /* 909 * This function can be called before the op_mode disabled the 910 * queues. This happens when we have an rfkill interrupt. 911 * Since we stop Tx altogether - mark the queues as stopped. 912 */ 913 memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped)); 914 memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); 915 916 /* This can happen: start_hw, stop_device */ 917 if (!trans_pcie->txq) 918 return 0; 919 920 /* Unmap DMA from host system and free skb's */ 921 for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues; 922 txq_id++) 923 iwl_pcie_txq_unmap(trans, txq_id); 924 925 return 0; 926 } 927 928 /* 929 * iwl_trans_tx_free - Free TXQ Context 930 * 931 * Destroy all TX DMA queues and structures 932 */ 933 void iwl_pcie_tx_free(struct iwl_trans *trans) 934 { 935 int txq_id; 936 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 937 938 /* Tx queues */ 939 if (trans_pcie->txq) { 940 for (txq_id = 0; 941 txq_id < trans->cfg->base_params->num_of_queues; txq_id++) 942 iwl_pcie_txq_free(trans, txq_id); 943 } 944 945 kfree(trans_pcie->txq); 946 trans_pcie->txq = NULL; 947 948 iwl_pcie_free_dma_ptr(trans, &trans_pcie->kw); 949 950 iwl_pcie_free_dma_ptr(trans, &trans_pcie->scd_bc_tbls); 951 } 952 953 /* 954 * iwl_pcie_tx_alloc - allocate TX context 955 * Allocate all Tx DMA structures and initialize them 956 */ 957 static int iwl_pcie_tx_alloc(struct iwl_trans *trans) 958 { 959 int ret; 960 int txq_id, slots_num; 961 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 962 963 u16 scd_bc_tbls_size = trans->cfg->base_params->num_of_queues * 964 sizeof(struct iwlagn_scd_bc_tbl); 965 966 /*It is not allowed to alloc twice, so warn when this happens. 967 * We cannot rely on the previous allocation, so free and fail */ 968 if (WARN_ON(trans_pcie->txq)) { 969 ret = -EINVAL; 970 goto error; 971 } 972 973 ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->scd_bc_tbls, 974 scd_bc_tbls_size); 975 if (ret) { 976 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 977 goto error; 978 } 979 980 /* Alloc keep-warm buffer */ 981 ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE); 982 if (ret) { 983 IWL_ERR(trans, "Keep Warm allocation failed\n"); 984 goto error; 985 } 986 987 trans_pcie->txq = kcalloc(trans->cfg->base_params->num_of_queues, 988 sizeof(struct iwl_txq), GFP_KERNEL); 989 if (!trans_pcie->txq) { 990 IWL_ERR(trans, "Not enough memory for txq\n"); 991 ret = -ENOMEM; 992 goto error; 993 } 994 995 /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 996 for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues; 997 txq_id++) { 998 slots_num = (txq_id == trans_pcie->cmd_queue) ? 999 TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS; 1000 ret = iwl_pcie_txq_alloc(trans, &trans_pcie->txq[txq_id], 1001 slots_num, txq_id); 1002 if (ret) { 1003 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 1004 goto error; 1005 } 1006 } 1007 1008 return 0; 1009 1010 error: 1011 iwl_pcie_tx_free(trans); 1012 1013 return ret; 1014 } 1015 int iwl_pcie_tx_init(struct iwl_trans *trans) 1016 { 1017 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1018 int ret; 1019 int txq_id, slots_num; 1020 bool alloc = false; 1021 1022 if (!trans_pcie->txq) { 1023 ret = iwl_pcie_tx_alloc(trans); 1024 if (ret) 1025 goto error; 1026 alloc = true; 1027 } 1028 1029 spin_lock(&trans_pcie->irq_lock); 1030 1031 /* Turn off all Tx DMA fifos */ 1032 iwl_scd_deactivate_fifos(trans); 1033 1034 /* Tell NIC where to find the "keep warm" buffer */ 1035 iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG, 1036 trans_pcie->kw.dma >> 4); 1037 1038 spin_unlock(&trans_pcie->irq_lock); 1039 1040 /* Alloc and init all Tx queues, including the command queue (#4/#9) */ 1041 for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues; 1042 txq_id++) { 1043 slots_num = (txq_id == trans_pcie->cmd_queue) ? 1044 TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS; 1045 ret = iwl_pcie_txq_init(trans, &trans_pcie->txq[txq_id], 1046 slots_num, txq_id); 1047 if (ret) { 1048 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 1049 goto error; 1050 } 1051 } 1052 1053 if (trans->cfg->use_tfh) { 1054 iwl_write_direct32(trans, TFH_TRANSFER_MODE, 1055 TFH_TRANSFER_MAX_PENDING_REQ | 1056 TFH_CHUNK_SIZE_128 | 1057 TFH_CHUNK_SPLIT_MODE); 1058 return 0; 1059 } 1060 1061 iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE); 1062 if (trans->cfg->base_params->num_of_queues > 20) 1063 iwl_set_bits_prph(trans, SCD_GP_CTRL, 1064 SCD_GP_CTRL_ENABLE_31_QUEUES); 1065 1066 return 0; 1067 error: 1068 /*Upon error, free only if we allocated something */ 1069 if (alloc) 1070 iwl_pcie_tx_free(trans); 1071 return ret; 1072 } 1073 1074 static inline void iwl_pcie_txq_progress(struct iwl_txq *txq) 1075 { 1076 lockdep_assert_held(&txq->lock); 1077 1078 if (!txq->wd_timeout) 1079 return; 1080 1081 /* 1082 * station is asleep and we send data - that must 1083 * be uAPSD or PS-Poll. Don't rearm the timer. 1084 */ 1085 if (txq->frozen) 1086 return; 1087 1088 /* 1089 * if empty delete timer, otherwise move timer forward 1090 * since we're making progress on this queue 1091 */ 1092 if (txq->read_ptr == txq->write_ptr) 1093 del_timer(&txq->stuck_timer); 1094 else 1095 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 1096 } 1097 1098 /* Frees buffers until index _not_ inclusive */ 1099 void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, 1100 struct sk_buff_head *skbs) 1101 { 1102 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1103 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 1104 int tfd_num = ssn & (TFD_QUEUE_SIZE_MAX - 1); 1105 int last_to_free; 1106 1107 /* This function is not meant to release cmd queue*/ 1108 if (WARN_ON(txq_id == trans_pcie->cmd_queue)) 1109 return; 1110 1111 spin_lock_bh(&txq->lock); 1112 1113 if (!txq->active) { 1114 IWL_DEBUG_TX_QUEUES(trans, "Q %d inactive - ignoring idx %d\n", 1115 txq_id, ssn); 1116 goto out; 1117 } 1118 1119 if (txq->read_ptr == tfd_num) 1120 goto out; 1121 1122 IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n", 1123 txq_id, txq->read_ptr, tfd_num, ssn); 1124 1125 /*Since we free until index _not_ inclusive, the one before index is 1126 * the last we will free. This one must be used */ 1127 last_to_free = iwl_queue_dec_wrap(tfd_num); 1128 1129 if (!iwl_queue_used(txq, last_to_free)) { 1130 IWL_ERR(trans, 1131 "%s: Read index for DMA queue txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n", 1132 __func__, txq_id, last_to_free, TFD_QUEUE_SIZE_MAX, 1133 txq->write_ptr, txq->read_ptr); 1134 goto out; 1135 } 1136 1137 if (WARN_ON(!skb_queue_empty(skbs))) 1138 goto out; 1139 1140 for (; 1141 txq->read_ptr != tfd_num; 1142 txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr)) { 1143 struct sk_buff *skb = txq->entries[txq->read_ptr].skb; 1144 1145 if (WARN_ON_ONCE(!skb)) 1146 continue; 1147 1148 iwl_pcie_free_tso_page(trans_pcie, skb); 1149 1150 __skb_queue_tail(skbs, skb); 1151 1152 txq->entries[txq->read_ptr].skb = NULL; 1153 1154 if (!trans->cfg->use_tfh) 1155 iwl_pcie_txq_inval_byte_cnt_tbl(trans, txq); 1156 1157 iwl_pcie_txq_free_tfd(trans, txq); 1158 } 1159 1160 iwl_pcie_txq_progress(txq); 1161 1162 if (iwl_queue_space(txq) > txq->low_mark && 1163 test_bit(txq_id, trans_pcie->queue_stopped)) { 1164 struct sk_buff_head overflow_skbs; 1165 1166 __skb_queue_head_init(&overflow_skbs); 1167 skb_queue_splice_init(&txq->overflow_q, &overflow_skbs); 1168 1169 /* 1170 * This is tricky: we are in reclaim path which is non 1171 * re-entrant, so noone will try to take the access the 1172 * txq data from that path. We stopped tx, so we can't 1173 * have tx as well. Bottom line, we can unlock and re-lock 1174 * later. 1175 */ 1176 spin_unlock_bh(&txq->lock); 1177 1178 while (!skb_queue_empty(&overflow_skbs)) { 1179 struct sk_buff *skb = __skb_dequeue(&overflow_skbs); 1180 struct iwl_device_cmd *dev_cmd_ptr; 1181 1182 dev_cmd_ptr = *(void **)((u8 *)skb->cb + 1183 trans_pcie->dev_cmd_offs); 1184 1185 /* 1186 * Note that we can very well be overflowing again. 1187 * In that case, iwl_queue_space will be small again 1188 * and we won't wake mac80211's queue. 1189 */ 1190 iwl_trans_pcie_tx(trans, skb, dev_cmd_ptr, txq_id); 1191 } 1192 spin_lock_bh(&txq->lock); 1193 1194 if (iwl_queue_space(txq) > txq->low_mark) 1195 iwl_wake_queue(trans, txq); 1196 } 1197 1198 if (txq->read_ptr == txq->write_ptr) { 1199 IWL_DEBUG_RPM(trans, "Q %d - last tx reclaimed\n", txq->id); 1200 iwl_trans_unref(trans); 1201 } 1202 1203 out: 1204 spin_unlock_bh(&txq->lock); 1205 } 1206 1207 static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans, 1208 const struct iwl_host_cmd *cmd) 1209 { 1210 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1211 int ret; 1212 1213 lockdep_assert_held(&trans_pcie->reg_lock); 1214 1215 if (!(cmd->flags & CMD_SEND_IN_IDLE) && 1216 !trans_pcie->ref_cmd_in_flight) { 1217 trans_pcie->ref_cmd_in_flight = true; 1218 IWL_DEBUG_RPM(trans, "set ref_cmd_in_flight - ref\n"); 1219 iwl_trans_ref(trans); 1220 } 1221 1222 /* 1223 * wake up the NIC to make sure that the firmware will see the host 1224 * command - we will let the NIC sleep once all the host commands 1225 * returned. This needs to be done only on NICs that have 1226 * apmg_wake_up_wa set. 1227 */ 1228 if (trans->cfg->base_params->apmg_wake_up_wa && 1229 !trans_pcie->cmd_hold_nic_awake) { 1230 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, 1231 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 1232 1233 ret = iwl_poll_bit(trans, CSR_GP_CNTRL, 1234 CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN, 1235 (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | 1236 CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 1237 15000); 1238 if (ret < 0) { 1239 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, 1240 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 1241 IWL_ERR(trans, "Failed to wake NIC for hcmd\n"); 1242 return -EIO; 1243 } 1244 trans_pcie->cmd_hold_nic_awake = true; 1245 } 1246 1247 return 0; 1248 } 1249 1250 /* 1251 * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd 1252 * 1253 * When FW advances 'R' index, all entries between old and new 'R' index 1254 * need to be reclaimed. As result, some free space forms. If there is 1255 * enough free space (> low mark), wake the stack that feeds us. 1256 */ 1257 static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx) 1258 { 1259 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1260 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 1261 unsigned long flags; 1262 int nfreed = 0; 1263 1264 lockdep_assert_held(&txq->lock); 1265 1266 if ((idx >= TFD_QUEUE_SIZE_MAX) || (!iwl_queue_used(txq, idx))) { 1267 IWL_ERR(trans, 1268 "%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n", 1269 __func__, txq_id, idx, TFD_QUEUE_SIZE_MAX, 1270 txq->write_ptr, txq->read_ptr); 1271 return; 1272 } 1273 1274 for (idx = iwl_queue_inc_wrap(idx); txq->read_ptr != idx; 1275 txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr)) { 1276 1277 if (nfreed++ > 0) { 1278 IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n", 1279 idx, txq->write_ptr, txq->read_ptr); 1280 iwl_force_nmi(trans); 1281 } 1282 } 1283 1284 if (txq->read_ptr == txq->write_ptr) { 1285 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 1286 iwl_pcie_clear_cmd_in_flight(trans); 1287 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1288 } 1289 1290 iwl_pcie_txq_progress(txq); 1291 } 1292 1293 static int iwl_pcie_txq_set_ratid_map(struct iwl_trans *trans, u16 ra_tid, 1294 u16 txq_id) 1295 { 1296 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1297 u32 tbl_dw_addr; 1298 u32 tbl_dw; 1299 u16 scd_q2ratid; 1300 1301 scd_q2ratid = ra_tid & SCD_QUEUE_RA_TID_MAP_RATID_MSK; 1302 1303 tbl_dw_addr = trans_pcie->scd_base_addr + 1304 SCD_TRANS_TBL_OFFSET_QUEUE(txq_id); 1305 1306 tbl_dw = iwl_trans_read_mem32(trans, tbl_dw_addr); 1307 1308 if (txq_id & 0x1) 1309 tbl_dw = (scd_q2ratid << 16) | (tbl_dw & 0x0000FFFF); 1310 else 1311 tbl_dw = scd_q2ratid | (tbl_dw & 0xFFFF0000); 1312 1313 iwl_trans_write_mem32(trans, tbl_dw_addr, tbl_dw); 1314 1315 return 0; 1316 } 1317 1318 /* Receiver address (actually, Rx station's index into station table), 1319 * combined with Traffic ID (QOS priority), in format used by Tx Scheduler */ 1320 #define BUILD_RAxTID(sta_id, tid) (((sta_id) << 4) + (tid)) 1321 1322 void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn, 1323 const struct iwl_trans_txq_scd_cfg *cfg, 1324 unsigned int wdg_timeout) 1325 { 1326 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1327 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 1328 int fifo = -1; 1329 1330 if (test_and_set_bit(txq_id, trans_pcie->queue_used)) 1331 WARN_ONCE(1, "queue %d already used - expect issues", txq_id); 1332 1333 if (cfg && trans->cfg->use_tfh) 1334 WARN_ONCE(1, "Expected no calls to SCD configuration"); 1335 1336 txq->wd_timeout = msecs_to_jiffies(wdg_timeout); 1337 1338 if (cfg) { 1339 fifo = cfg->fifo; 1340 1341 /* Disable the scheduler prior configuring the cmd queue */ 1342 if (txq_id == trans_pcie->cmd_queue && 1343 trans_pcie->scd_set_active) 1344 iwl_scd_enable_set_active(trans, 0); 1345 1346 /* Stop this Tx queue before configuring it */ 1347 iwl_scd_txq_set_inactive(trans, txq_id); 1348 1349 /* Set this queue as a chain-building queue unless it is CMD */ 1350 if (txq_id != trans_pcie->cmd_queue) 1351 iwl_scd_txq_set_chain(trans, txq_id); 1352 1353 if (cfg->aggregate) { 1354 u16 ra_tid = BUILD_RAxTID(cfg->sta_id, cfg->tid); 1355 1356 /* Map receiver-address / traffic-ID to this queue */ 1357 iwl_pcie_txq_set_ratid_map(trans, ra_tid, txq_id); 1358 1359 /* enable aggregations for the queue */ 1360 iwl_scd_txq_enable_agg(trans, txq_id); 1361 txq->ampdu = true; 1362 } else { 1363 /* 1364 * disable aggregations for the queue, this will also 1365 * make the ra_tid mapping configuration irrelevant 1366 * since it is now a non-AGG queue. 1367 */ 1368 iwl_scd_txq_disable_agg(trans, txq_id); 1369 1370 ssn = txq->read_ptr; 1371 } 1372 } 1373 1374 /* Place first TFD at index corresponding to start sequence number. 1375 * Assumes that ssn_idx is valid (!= 0xFFF) */ 1376 txq->read_ptr = (ssn & 0xff); 1377 txq->write_ptr = (ssn & 0xff); 1378 iwl_write_direct32(trans, HBUS_TARG_WRPTR, 1379 (ssn & 0xff) | (txq_id << 8)); 1380 1381 if (cfg) { 1382 u8 frame_limit = cfg->frame_limit; 1383 1384 iwl_write_prph(trans, SCD_QUEUE_RDPTR(txq_id), ssn); 1385 1386 /* Set up Tx window size and frame limit for this queue */ 1387 iwl_trans_write_mem32(trans, trans_pcie->scd_base_addr + 1388 SCD_CONTEXT_QUEUE_OFFSET(txq_id), 0); 1389 iwl_trans_write_mem32(trans, 1390 trans_pcie->scd_base_addr + 1391 SCD_CONTEXT_QUEUE_OFFSET(txq_id) + sizeof(u32), 1392 ((frame_limit << SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) & 1393 SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) | 1394 ((frame_limit << SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) & 1395 SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK)); 1396 1397 /* Set up status area in SRAM, map to Tx DMA/FIFO, activate */ 1398 iwl_write_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id), 1399 (1 << SCD_QUEUE_STTS_REG_POS_ACTIVE) | 1400 (cfg->fifo << SCD_QUEUE_STTS_REG_POS_TXF) | 1401 (1 << SCD_QUEUE_STTS_REG_POS_WSL) | 1402 SCD_QUEUE_STTS_REG_MSK); 1403 1404 /* enable the scheduler for this queue (only) */ 1405 if (txq_id == trans_pcie->cmd_queue && 1406 trans_pcie->scd_set_active) 1407 iwl_scd_enable_set_active(trans, BIT(txq_id)); 1408 1409 IWL_DEBUG_TX_QUEUES(trans, 1410 "Activate queue %d on FIFO %d WrPtr: %d\n", 1411 txq_id, fifo, ssn & 0xff); 1412 } else { 1413 IWL_DEBUG_TX_QUEUES(trans, 1414 "Activate queue %d WrPtr: %d\n", 1415 txq_id, ssn & 0xff); 1416 } 1417 1418 txq->active = true; 1419 } 1420 1421 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id, 1422 bool shared_mode) 1423 { 1424 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1425 struct iwl_txq *txq = &trans_pcie->txq[txq_id]; 1426 1427 txq->ampdu = !shared_mode; 1428 } 1429 1430 dma_addr_t iwl_trans_pcie_get_txq_byte_table(struct iwl_trans *trans, int txq) 1431 { 1432 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1433 1434 return trans_pcie->scd_bc_tbls.dma + 1435 txq * sizeof(struct iwlagn_scd_bc_tbl); 1436 } 1437 1438 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id, 1439 bool configure_scd) 1440 { 1441 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1442 u32 stts_addr = trans_pcie->scd_base_addr + 1443 SCD_TX_STTS_QUEUE_OFFSET(txq_id); 1444 static const u32 zero_val[4] = {}; 1445 1446 trans_pcie->txq[txq_id].frozen_expiry_remainder = 0; 1447 trans_pcie->txq[txq_id].frozen = false; 1448 1449 /* 1450 * Upon HW Rfkill - we stop the device, and then stop the queues 1451 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1452 * allow the op_mode to call txq_disable after it already called 1453 * stop_device. 1454 */ 1455 if (!test_and_clear_bit(txq_id, trans_pcie->queue_used)) { 1456 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1457 "queue %d not used", txq_id); 1458 return; 1459 } 1460 1461 if (configure_scd && trans->cfg->use_tfh) 1462 WARN_ONCE(1, "Expected no calls to SCD configuration"); 1463 1464 if (configure_scd) { 1465 iwl_scd_txq_set_inactive(trans, txq_id); 1466 1467 iwl_trans_write_mem(trans, stts_addr, (void *)zero_val, 1468 ARRAY_SIZE(zero_val)); 1469 } 1470 1471 iwl_pcie_txq_unmap(trans, txq_id); 1472 trans_pcie->txq[txq_id].ampdu = false; 1473 1474 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", txq_id); 1475 } 1476 1477 /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 1478 1479 /* 1480 * iwl_pcie_enqueue_hcmd - enqueue a uCode command 1481 * @priv: device private data point 1482 * @cmd: a pointer to the ucode command structure 1483 * 1484 * The function returns < 0 values to indicate the operation 1485 * failed. On success, it returns the index (>= 0) of command in the 1486 * command queue. 1487 */ 1488 static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, 1489 struct iwl_host_cmd *cmd) 1490 { 1491 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1492 struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; 1493 struct iwl_device_cmd *out_cmd; 1494 struct iwl_cmd_meta *out_meta; 1495 unsigned long flags; 1496 void *dup_buf = NULL; 1497 dma_addr_t phys_addr; 1498 int idx; 1499 u16 copy_size, cmd_size, tb0_size; 1500 bool had_nocopy = false; 1501 u8 group_id = iwl_cmd_groupid(cmd->id); 1502 int i, ret; 1503 u32 cmd_pos; 1504 const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; 1505 u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; 1506 1507 if (WARN(!trans->wide_cmd_header && 1508 group_id > IWL_ALWAYS_LONG_GROUP, 1509 "unsupported wide command %#x\n", cmd->id)) 1510 return -EINVAL; 1511 1512 if (group_id != 0) { 1513 copy_size = sizeof(struct iwl_cmd_header_wide); 1514 cmd_size = sizeof(struct iwl_cmd_header_wide); 1515 } else { 1516 copy_size = sizeof(struct iwl_cmd_header); 1517 cmd_size = sizeof(struct iwl_cmd_header); 1518 } 1519 1520 /* need one for the header if the first is NOCOPY */ 1521 BUILD_BUG_ON(IWL_MAX_CMD_TBS_PER_TFD > IWL_NUM_OF_TBS - 1); 1522 1523 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1524 cmddata[i] = cmd->data[i]; 1525 cmdlen[i] = cmd->len[i]; 1526 1527 if (!cmd->len[i]) 1528 continue; 1529 1530 /* need at least IWL_FIRST_TB_SIZE copied */ 1531 if (copy_size < IWL_FIRST_TB_SIZE) { 1532 int copy = IWL_FIRST_TB_SIZE - copy_size; 1533 1534 if (copy > cmdlen[i]) 1535 copy = cmdlen[i]; 1536 cmdlen[i] -= copy; 1537 cmddata[i] += copy; 1538 copy_size += copy; 1539 } 1540 1541 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { 1542 had_nocopy = true; 1543 if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { 1544 idx = -EINVAL; 1545 goto free_dup_buf; 1546 } 1547 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { 1548 /* 1549 * This is also a chunk that isn't copied 1550 * to the static buffer so set had_nocopy. 1551 */ 1552 had_nocopy = true; 1553 1554 /* only allowed once */ 1555 if (WARN_ON(dup_buf)) { 1556 idx = -EINVAL; 1557 goto free_dup_buf; 1558 } 1559 1560 dup_buf = kmemdup(cmddata[i], cmdlen[i], 1561 GFP_ATOMIC); 1562 if (!dup_buf) 1563 return -ENOMEM; 1564 } else { 1565 /* NOCOPY must not be followed by normal! */ 1566 if (WARN_ON(had_nocopy)) { 1567 idx = -EINVAL; 1568 goto free_dup_buf; 1569 } 1570 copy_size += cmdlen[i]; 1571 } 1572 cmd_size += cmd->len[i]; 1573 } 1574 1575 /* 1576 * If any of the command structures end up being larger than 1577 * the TFD_MAX_PAYLOAD_SIZE and they aren't dynamically 1578 * allocated into separate TFDs, then we will need to 1579 * increase the size of the buffers. 1580 */ 1581 if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, 1582 "Command %s (%#x) is too large (%d bytes)\n", 1583 iwl_get_cmd_string(trans, cmd->id), 1584 cmd->id, copy_size)) { 1585 idx = -EINVAL; 1586 goto free_dup_buf; 1587 } 1588 1589 spin_lock_bh(&txq->lock); 1590 1591 if (iwl_queue_space(txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { 1592 spin_unlock_bh(&txq->lock); 1593 1594 IWL_ERR(trans, "No space in command queue\n"); 1595 iwl_op_mode_cmd_queue_full(trans->op_mode); 1596 idx = -ENOSPC; 1597 goto free_dup_buf; 1598 } 1599 1600 idx = get_cmd_index(txq, txq->write_ptr); 1601 out_cmd = txq->entries[idx].cmd; 1602 out_meta = &txq->entries[idx].meta; 1603 1604 memset(out_meta, 0, sizeof(*out_meta)); /* re-initialize to NULL */ 1605 if (cmd->flags & CMD_WANT_SKB) 1606 out_meta->source = cmd; 1607 1608 /* set up the header */ 1609 if (group_id != 0) { 1610 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); 1611 out_cmd->hdr_wide.group_id = group_id; 1612 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); 1613 out_cmd->hdr_wide.length = 1614 cpu_to_le16(cmd_size - 1615 sizeof(struct iwl_cmd_header_wide)); 1616 out_cmd->hdr_wide.reserved = 0; 1617 out_cmd->hdr_wide.sequence = 1618 cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | 1619 INDEX_TO_SEQ(txq->write_ptr)); 1620 1621 cmd_pos = sizeof(struct iwl_cmd_header_wide); 1622 copy_size = sizeof(struct iwl_cmd_header_wide); 1623 } else { 1624 out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id); 1625 out_cmd->hdr.sequence = 1626 cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | 1627 INDEX_TO_SEQ(txq->write_ptr)); 1628 out_cmd->hdr.group_id = 0; 1629 1630 cmd_pos = sizeof(struct iwl_cmd_header); 1631 copy_size = sizeof(struct iwl_cmd_header); 1632 } 1633 1634 /* and copy the data that needs to be copied */ 1635 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1636 int copy; 1637 1638 if (!cmd->len[i]) 1639 continue; 1640 1641 /* copy everything if not nocopy/dup */ 1642 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1643 IWL_HCMD_DFL_DUP))) { 1644 copy = cmd->len[i]; 1645 1646 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1647 cmd_pos += copy; 1648 copy_size += copy; 1649 continue; 1650 } 1651 1652 /* 1653 * Otherwise we need at least IWL_FIRST_TB_SIZE copied 1654 * in total (for bi-directional DMA), but copy up to what 1655 * we can fit into the payload for debug dump purposes. 1656 */ 1657 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); 1658 1659 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 1660 cmd_pos += copy; 1661 1662 /* However, treat copy_size the proper way, we need it below */ 1663 if (copy_size < IWL_FIRST_TB_SIZE) { 1664 copy = IWL_FIRST_TB_SIZE - copy_size; 1665 1666 if (copy > cmd->len[i]) 1667 copy = cmd->len[i]; 1668 copy_size += copy; 1669 } 1670 } 1671 1672 IWL_DEBUG_HC(trans, 1673 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", 1674 iwl_get_cmd_string(trans, cmd->id), 1675 group_id, out_cmd->hdr.cmd, 1676 le16_to_cpu(out_cmd->hdr.sequence), 1677 cmd_size, txq->write_ptr, idx, trans_pcie->cmd_queue); 1678 1679 /* start the TFD with the minimum copy bytes */ 1680 tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); 1681 memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size); 1682 iwl_pcie_txq_build_tfd(trans, txq, 1683 iwl_pcie_get_first_tb_dma(txq, idx), 1684 tb0_size, true); 1685 1686 /* map first command fragment, if any remains */ 1687 if (copy_size > tb0_size) { 1688 phys_addr = dma_map_single(trans->dev, 1689 ((u8 *)&out_cmd->hdr) + tb0_size, 1690 copy_size - tb0_size, 1691 DMA_TO_DEVICE); 1692 if (dma_mapping_error(trans->dev, phys_addr)) { 1693 iwl_pcie_tfd_unmap(trans, out_meta, txq, 1694 txq->write_ptr); 1695 idx = -ENOMEM; 1696 goto out; 1697 } 1698 1699 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, 1700 copy_size - tb0_size, false); 1701 } 1702 1703 /* map the remaining (adjusted) nocopy/dup fragments */ 1704 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 1705 const void *data = cmddata[i]; 1706 1707 if (!cmdlen[i]) 1708 continue; 1709 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 1710 IWL_HCMD_DFL_DUP))) 1711 continue; 1712 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) 1713 data = dup_buf; 1714 phys_addr = dma_map_single(trans->dev, (void *)data, 1715 cmdlen[i], DMA_TO_DEVICE); 1716 if (dma_mapping_error(trans->dev, phys_addr)) { 1717 iwl_pcie_tfd_unmap(trans, out_meta, txq, 1718 txq->write_ptr); 1719 idx = -ENOMEM; 1720 goto out; 1721 } 1722 1723 iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false); 1724 } 1725 1726 BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); 1727 out_meta->flags = cmd->flags; 1728 if (WARN_ON_ONCE(txq->entries[idx].free_buf)) 1729 kzfree(txq->entries[idx].free_buf); 1730 txq->entries[idx].free_buf = dup_buf; 1731 1732 trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); 1733 1734 /* start timer if queue currently empty */ 1735 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 1736 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 1737 1738 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 1739 ret = iwl_pcie_set_cmd_in_flight(trans, cmd); 1740 if (ret < 0) { 1741 idx = ret; 1742 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1743 goto out; 1744 } 1745 1746 /* Increment and update queue's write index */ 1747 txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr); 1748 iwl_pcie_txq_inc_wr_ptr(trans, txq); 1749 1750 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1751 1752 out: 1753 spin_unlock_bh(&txq->lock); 1754 free_dup_buf: 1755 if (idx < 0) 1756 kfree(dup_buf); 1757 return idx; 1758 } 1759 1760 /* 1761 * iwl_pcie_hcmd_complete - Pull unused buffers off the queue and reclaim them 1762 * @rxb: Rx buffer to reclaim 1763 */ 1764 void iwl_pcie_hcmd_complete(struct iwl_trans *trans, 1765 struct iwl_rx_cmd_buffer *rxb) 1766 { 1767 struct iwl_rx_packet *pkt = rxb_addr(rxb); 1768 u16 sequence = le16_to_cpu(pkt->hdr.sequence); 1769 u8 group_id = iwl_cmd_groupid(pkt->hdr.group_id); 1770 u32 cmd_id; 1771 int txq_id = SEQ_TO_QUEUE(sequence); 1772 int index = SEQ_TO_INDEX(sequence); 1773 int cmd_index; 1774 struct iwl_device_cmd *cmd; 1775 struct iwl_cmd_meta *meta; 1776 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1777 struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; 1778 1779 /* If a Tx command is being handled and it isn't in the actual 1780 * command queue then there a command routing bug has been introduced 1781 * in the queue management code. */ 1782 if (WARN(txq_id != trans_pcie->cmd_queue, 1783 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n", 1784 txq_id, trans_pcie->cmd_queue, sequence, 1785 trans_pcie->txq[trans_pcie->cmd_queue].read_ptr, 1786 trans_pcie->txq[trans_pcie->cmd_queue].write_ptr)) { 1787 iwl_print_hex_error(trans, pkt, 32); 1788 return; 1789 } 1790 1791 spin_lock_bh(&txq->lock); 1792 1793 cmd_index = get_cmd_index(txq, index); 1794 cmd = txq->entries[cmd_index].cmd; 1795 meta = &txq->entries[cmd_index].meta; 1796 cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0); 1797 1798 iwl_pcie_tfd_unmap(trans, meta, txq, index); 1799 1800 /* Input error checking is done when commands are added to queue. */ 1801 if (meta->flags & CMD_WANT_SKB) { 1802 struct page *p = rxb_steal_page(rxb); 1803 1804 meta->source->resp_pkt = pkt; 1805 meta->source->_rx_page_addr = (unsigned long)page_address(p); 1806 meta->source->_rx_page_order = trans_pcie->rx_page_order; 1807 } 1808 1809 if (meta->flags & CMD_WANT_ASYNC_CALLBACK) 1810 iwl_op_mode_async_cb(trans->op_mode, cmd); 1811 1812 iwl_pcie_cmdq_reclaim(trans, txq_id, index); 1813 1814 if (!(meta->flags & CMD_ASYNC)) { 1815 if (!test_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) { 1816 IWL_WARN(trans, 1817 "HCMD_ACTIVE already clear for command %s\n", 1818 iwl_get_cmd_string(trans, cmd_id)); 1819 } 1820 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1821 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1822 iwl_get_cmd_string(trans, cmd_id)); 1823 wake_up(&trans_pcie->wait_command_queue); 1824 } 1825 1826 if (meta->flags & CMD_MAKE_TRANS_IDLE) { 1827 IWL_DEBUG_INFO(trans, "complete %s - mark trans as idle\n", 1828 iwl_get_cmd_string(trans, cmd->hdr.cmd)); 1829 set_bit(STATUS_TRANS_IDLE, &trans->status); 1830 wake_up(&trans_pcie->d0i3_waitq); 1831 } 1832 1833 if (meta->flags & CMD_WAKE_UP_TRANS) { 1834 IWL_DEBUG_INFO(trans, "complete %s - clear trans idle flag\n", 1835 iwl_get_cmd_string(trans, cmd->hdr.cmd)); 1836 clear_bit(STATUS_TRANS_IDLE, &trans->status); 1837 wake_up(&trans_pcie->d0i3_waitq); 1838 } 1839 1840 meta->flags = 0; 1841 1842 spin_unlock_bh(&txq->lock); 1843 } 1844 1845 #define HOST_COMPLETE_TIMEOUT (2 * HZ) 1846 1847 static int iwl_pcie_send_hcmd_async(struct iwl_trans *trans, 1848 struct iwl_host_cmd *cmd) 1849 { 1850 int ret; 1851 1852 /* An asynchronous command can not expect an SKB to be set. */ 1853 if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1854 return -EINVAL; 1855 1856 ret = iwl_pcie_enqueue_hcmd(trans, cmd); 1857 if (ret < 0) { 1858 IWL_ERR(trans, 1859 "Error sending %s: enqueue_hcmd failed: %d\n", 1860 iwl_get_cmd_string(trans, cmd->id), ret); 1861 return ret; 1862 } 1863 return 0; 1864 } 1865 1866 static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans, 1867 struct iwl_host_cmd *cmd) 1868 { 1869 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1870 int cmd_idx; 1871 int ret; 1872 1873 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", 1874 iwl_get_cmd_string(trans, cmd->id)); 1875 1876 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 1877 &trans->status), 1878 "Command %s: a command is already active!\n", 1879 iwl_get_cmd_string(trans, cmd->id))) 1880 return -EIO; 1881 1882 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", 1883 iwl_get_cmd_string(trans, cmd->id)); 1884 1885 if (pm_runtime_suspended(&trans_pcie->pci_dev->dev)) { 1886 ret = wait_event_timeout(trans_pcie->d0i3_waitq, 1887 pm_runtime_active(&trans_pcie->pci_dev->dev), 1888 msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT)); 1889 if (!ret) { 1890 IWL_ERR(trans, "Timeout exiting D0i3 before hcmd\n"); 1891 return -ETIMEDOUT; 1892 } 1893 } 1894 1895 cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd); 1896 if (cmd_idx < 0) { 1897 ret = cmd_idx; 1898 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1899 IWL_ERR(trans, 1900 "Error sending %s: enqueue_hcmd failed: %d\n", 1901 iwl_get_cmd_string(trans, cmd->id), ret); 1902 return ret; 1903 } 1904 1905 ret = wait_event_timeout(trans_pcie->wait_command_queue, 1906 !test_bit(STATUS_SYNC_HCMD_ACTIVE, 1907 &trans->status), 1908 HOST_COMPLETE_TIMEOUT); 1909 if (!ret) { 1910 struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue]; 1911 1912 IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 1913 iwl_get_cmd_string(trans, cmd->id), 1914 jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 1915 1916 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 1917 txq->read_ptr, txq->write_ptr); 1918 1919 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1920 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1921 iwl_get_cmd_string(trans, cmd->id)); 1922 ret = -ETIMEDOUT; 1923 1924 iwl_force_nmi(trans); 1925 iwl_trans_fw_error(trans); 1926 1927 goto cancel; 1928 } 1929 1930 if (test_bit(STATUS_FW_ERROR, &trans->status)) { 1931 IWL_ERR(trans, "FW error in SYNC CMD %s\n", 1932 iwl_get_cmd_string(trans, cmd->id)); 1933 dump_stack(); 1934 ret = -EIO; 1935 goto cancel; 1936 } 1937 1938 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1939 test_bit(STATUS_RFKILL, &trans->status)) { 1940 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 1941 ret = -ERFKILL; 1942 goto cancel; 1943 } 1944 1945 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 1946 IWL_ERR(trans, "Error: Response NULL in '%s'\n", 1947 iwl_get_cmd_string(trans, cmd->id)); 1948 ret = -EIO; 1949 goto cancel; 1950 } 1951 1952 return 0; 1953 1954 cancel: 1955 if (cmd->flags & CMD_WANT_SKB) { 1956 /* 1957 * Cancel the CMD_WANT_SKB flag for the cmd in the 1958 * TX cmd queue. Otherwise in case the cmd comes 1959 * in later, it will possibly set an invalid 1960 * address (cmd->meta.source). 1961 */ 1962 trans_pcie->txq[trans_pcie->cmd_queue]. 1963 entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 1964 } 1965 1966 if (cmd->resp_pkt) { 1967 iwl_free_resp(cmd); 1968 cmd->resp_pkt = NULL; 1969 } 1970 1971 return ret; 1972 } 1973 1974 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) 1975 { 1976 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1977 test_bit(STATUS_RFKILL, &trans->status)) { 1978 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1979 cmd->id); 1980 return -ERFKILL; 1981 } 1982 1983 if (cmd->flags & CMD_ASYNC) 1984 return iwl_pcie_send_hcmd_async(trans, cmd); 1985 1986 /* We still can fail on RFKILL that can be asserted while we wait */ 1987 return iwl_pcie_send_hcmd_sync(trans, cmd); 1988 } 1989 1990 static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb, 1991 struct iwl_txq *txq, u8 hdr_len, 1992 struct iwl_cmd_meta *out_meta, 1993 struct iwl_device_cmd *dev_cmd, u16 tb1_len) 1994 { 1995 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1996 u16 tb2_len; 1997 int i; 1998 1999 /* 2000 * Set up TFD's third entry to point directly to remainder 2001 * of skb's head, if any 2002 */ 2003 tb2_len = skb_headlen(skb) - hdr_len; 2004 2005 if (tb2_len > 0) { 2006 dma_addr_t tb2_phys = dma_map_single(trans->dev, 2007 skb->data + hdr_len, 2008 tb2_len, DMA_TO_DEVICE); 2009 if (unlikely(dma_mapping_error(trans->dev, tb2_phys))) { 2010 iwl_pcie_tfd_unmap(trans, out_meta, txq, 2011 txq->write_ptr); 2012 return -EINVAL; 2013 } 2014 iwl_pcie_txq_build_tfd(trans, txq, tb2_phys, tb2_len, false); 2015 } 2016 2017 /* set up the remaining entries to point to the data */ 2018 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2019 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2020 dma_addr_t tb_phys; 2021 int tb_idx; 2022 2023 if (!skb_frag_size(frag)) 2024 continue; 2025 2026 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 2027 skb_frag_size(frag), DMA_TO_DEVICE); 2028 2029 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 2030 iwl_pcie_tfd_unmap(trans, out_meta, txq, 2031 txq->write_ptr); 2032 return -EINVAL; 2033 } 2034 tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 2035 skb_frag_size(frag), false); 2036 2037 out_meta->tbs |= BIT(tb_idx); 2038 } 2039 2040 trace_iwlwifi_dev_tx(trans->dev, skb, 2041 iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr), 2042 trans_pcie->tfd_size, 2043 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 2044 skb->data + hdr_len, tb2_len); 2045 trace_iwlwifi_dev_tx_data(trans->dev, skb, 2046 hdr_len, skb->len - hdr_len); 2047 return 0; 2048 } 2049 2050 #ifdef CONFIG_INET 2051 static struct iwl_tso_hdr_page * 2052 get_page_hdr(struct iwl_trans *trans, size_t len) 2053 { 2054 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2055 struct iwl_tso_hdr_page *p = this_cpu_ptr(trans_pcie->tso_hdr_page); 2056 2057 if (!p->page) 2058 goto alloc; 2059 2060 /* enough room on this page */ 2061 if (p->pos + len < (u8 *)page_address(p->page) + PAGE_SIZE) 2062 return p; 2063 2064 /* We don't have enough room on this page, get a new one. */ 2065 __free_page(p->page); 2066 2067 alloc: 2068 p->page = alloc_page(GFP_ATOMIC); 2069 if (!p->page) 2070 return NULL; 2071 p->pos = page_address(p->page); 2072 return p; 2073 } 2074 2075 static void iwl_compute_pseudo_hdr_csum(void *iph, struct tcphdr *tcph, 2076 bool ipv6, unsigned int len) 2077 { 2078 if (ipv6) { 2079 struct ipv6hdr *iphv6 = iph; 2080 2081 tcph->check = ~csum_ipv6_magic(&iphv6->saddr, &iphv6->daddr, 2082 len + tcph->doff * 4, 2083 IPPROTO_TCP, 0); 2084 } else { 2085 struct iphdr *iphv4 = iph; 2086 2087 ip_send_check(iphv4); 2088 tcph->check = ~csum_tcpudp_magic(iphv4->saddr, iphv4->daddr, 2089 len + tcph->doff * 4, 2090 IPPROTO_TCP, 0); 2091 } 2092 } 2093 2094 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 2095 struct iwl_txq *txq, u8 hdr_len, 2096 struct iwl_cmd_meta *out_meta, 2097 struct iwl_device_cmd *dev_cmd, u16 tb1_len) 2098 { 2099 struct iwl_trans_pcie *trans_pcie = txq->trans_pcie; 2100 struct ieee80211_hdr *hdr = (void *)skb->data; 2101 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 2102 unsigned int mss = skb_shinfo(skb)->gso_size; 2103 u16 length, iv_len, amsdu_pad; 2104 u8 *start_hdr; 2105 struct iwl_tso_hdr_page *hdr_page; 2106 struct page **page_ptr; 2107 int ret; 2108 struct tso_t tso; 2109 2110 /* if the packet is protected, then it must be CCMP or GCMP */ 2111 BUILD_BUG_ON(IEEE80211_CCMP_HDR_LEN != IEEE80211_GCMP_HDR_LEN); 2112 iv_len = ieee80211_has_protected(hdr->frame_control) ? 2113 IEEE80211_CCMP_HDR_LEN : 0; 2114 2115 trace_iwlwifi_dev_tx(trans->dev, skb, 2116 iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr), 2117 trans_pcie->tfd_size, 2118 &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 2119 NULL, 0); 2120 2121 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 2122 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 2123 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; 2124 amsdu_pad = 0; 2125 2126 /* total amount of header we may need for this A-MSDU */ 2127 hdr_room = DIV_ROUND_UP(total_len, mss) * 2128 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len; 2129 2130 /* Our device supports 9 segments at most, it will fit in 1 page */ 2131 hdr_page = get_page_hdr(trans, hdr_room); 2132 if (!hdr_page) 2133 return -ENOMEM; 2134 2135 get_page(hdr_page->page); 2136 start_hdr = hdr_page->pos; 2137 page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); 2138 *page_ptr = hdr_page->page; 2139 memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); 2140 hdr_page->pos += iv_len; 2141 2142 /* 2143 * Pull the ieee80211 header + IV to be able to use TSO core, 2144 * we will restore it for the tx_status flow. 2145 */ 2146 skb_pull(skb, hdr_len + iv_len); 2147 2148 tso_start(skb, &tso); 2149 2150 while (total_len) { 2151 /* this is the data left for this subframe */ 2152 unsigned int data_left = 2153 min_t(unsigned int, mss, total_len); 2154 struct sk_buff *csum_skb = NULL; 2155 unsigned int hdr_tb_len; 2156 dma_addr_t hdr_tb_phys; 2157 struct tcphdr *tcph; 2158 u8 *iph; 2159 2160 total_len -= data_left; 2161 2162 memset(hdr_page->pos, 0, amsdu_pad); 2163 hdr_page->pos += amsdu_pad; 2164 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 2165 data_left)) & 0x3; 2166 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 2167 hdr_page->pos += ETH_ALEN; 2168 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 2169 hdr_page->pos += ETH_ALEN; 2170 2171 length = snap_ip_tcp_hdrlen + data_left; 2172 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 2173 hdr_page->pos += sizeof(length); 2174 2175 /* 2176 * This will copy the SNAP as well which will be considered 2177 * as MAC header. 2178 */ 2179 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 2180 iph = hdr_page->pos + 8; 2181 tcph = (void *)(iph + ip_hdrlen); 2182 2183 /* For testing on current hardware only */ 2184 if (trans_pcie->sw_csum_tx) { 2185 csum_skb = alloc_skb(data_left + tcp_hdrlen(skb), 2186 GFP_ATOMIC); 2187 if (!csum_skb) { 2188 ret = -ENOMEM; 2189 goto out_unmap; 2190 } 2191 2192 iwl_compute_pseudo_hdr_csum(iph, tcph, 2193 skb->protocol == 2194 htons(ETH_P_IPV6), 2195 data_left); 2196 2197 memcpy(skb_put(csum_skb, tcp_hdrlen(skb)), 2198 tcph, tcp_hdrlen(skb)); 2199 skb_set_transport_header(csum_skb, 0); 2200 csum_skb->csum_start = 2201 (unsigned char *)tcp_hdr(csum_skb) - 2202 csum_skb->head; 2203 } 2204 2205 hdr_page->pos += snap_ip_tcp_hdrlen; 2206 2207 hdr_tb_len = hdr_page->pos - start_hdr; 2208 hdr_tb_phys = dma_map_single(trans->dev, start_hdr, 2209 hdr_tb_len, DMA_TO_DEVICE); 2210 if (unlikely(dma_mapping_error(trans->dev, hdr_tb_phys))) { 2211 dev_kfree_skb(csum_skb); 2212 ret = -EINVAL; 2213 goto out_unmap; 2214 } 2215 iwl_pcie_txq_build_tfd(trans, txq, hdr_tb_phys, 2216 hdr_tb_len, false); 2217 trace_iwlwifi_dev_tx_tso_chunk(trans->dev, start_hdr, 2218 hdr_tb_len); 2219 2220 /* prepare the start_hdr for the next subframe */ 2221 start_hdr = hdr_page->pos; 2222 2223 /* put the payload */ 2224 while (data_left) { 2225 unsigned int size = min_t(unsigned int, tso.size, 2226 data_left); 2227 dma_addr_t tb_phys; 2228 2229 if (trans_pcie->sw_csum_tx) 2230 memcpy(skb_put(csum_skb, size), tso.data, size); 2231 2232 tb_phys = dma_map_single(trans->dev, tso.data, 2233 size, DMA_TO_DEVICE); 2234 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 2235 dev_kfree_skb(csum_skb); 2236 ret = -EINVAL; 2237 goto out_unmap; 2238 } 2239 2240 iwl_pcie_txq_build_tfd(trans, txq, tb_phys, 2241 size, false); 2242 trace_iwlwifi_dev_tx_tso_chunk(trans->dev, tso.data, 2243 size); 2244 2245 data_left -= size; 2246 tso_build_data(skb, &tso, size); 2247 } 2248 2249 /* For testing on early hardware only */ 2250 if (trans_pcie->sw_csum_tx) { 2251 __wsum csum; 2252 2253 csum = skb_checksum(csum_skb, 2254 skb_checksum_start_offset(csum_skb), 2255 csum_skb->len - 2256 skb_checksum_start_offset(csum_skb), 2257 0); 2258 dev_kfree_skb(csum_skb); 2259 dma_sync_single_for_cpu(trans->dev, hdr_tb_phys, 2260 hdr_tb_len, DMA_TO_DEVICE); 2261 tcph->check = csum_fold(csum); 2262 dma_sync_single_for_device(trans->dev, hdr_tb_phys, 2263 hdr_tb_len, DMA_TO_DEVICE); 2264 } 2265 } 2266 2267 /* re -add the WiFi header and IV */ 2268 skb_push(skb, hdr_len + iv_len); 2269 2270 return 0; 2271 2272 out_unmap: 2273 iwl_pcie_tfd_unmap(trans, out_meta, txq, txq->write_ptr); 2274 return ret; 2275 } 2276 #else /* CONFIG_INET */ 2277 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, 2278 struct iwl_txq *txq, u8 hdr_len, 2279 struct iwl_cmd_meta *out_meta, 2280 struct iwl_device_cmd *dev_cmd, u16 tb1_len) 2281 { 2282 /* No A-MSDU without CONFIG_INET */ 2283 WARN_ON(1); 2284 2285 return -1; 2286 } 2287 #endif /* CONFIG_INET */ 2288 2289 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, 2290 struct iwl_device_cmd *dev_cmd, int txq_id) 2291 { 2292 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2293 struct ieee80211_hdr *hdr; 2294 struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload; 2295 struct iwl_cmd_meta *out_meta; 2296 struct iwl_txq *txq; 2297 dma_addr_t tb0_phys, tb1_phys, scratch_phys; 2298 void *tb1_addr; 2299 void *tfd; 2300 u16 len, tb1_len; 2301 bool wait_write_ptr; 2302 __le16 fc; 2303 u8 hdr_len; 2304 u16 wifi_seq; 2305 bool amsdu; 2306 2307 txq = &trans_pcie->txq[txq_id]; 2308 2309 if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used), 2310 "TX on unused queue %d\n", txq_id)) 2311 return -EINVAL; 2312 2313 if (unlikely(trans_pcie->sw_csum_tx && 2314 skb->ip_summed == CHECKSUM_PARTIAL)) { 2315 int offs = skb_checksum_start_offset(skb); 2316 int csum_offs = offs + skb->csum_offset; 2317 __wsum csum; 2318 2319 if (skb_ensure_writable(skb, csum_offs + sizeof(__sum16))) 2320 return -1; 2321 2322 csum = skb_checksum(skb, offs, skb->len - offs, 0); 2323 *(__sum16 *)(skb->data + csum_offs) = csum_fold(csum); 2324 2325 skb->ip_summed = CHECKSUM_UNNECESSARY; 2326 } 2327 2328 if (skb_is_nonlinear(skb) && 2329 skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS(trans_pcie) && 2330 __skb_linearize(skb)) 2331 return -ENOMEM; 2332 2333 /* mac80211 always puts the full header into the SKB's head, 2334 * so there's no need to check if it's readable there 2335 */ 2336 hdr = (struct ieee80211_hdr *)skb->data; 2337 fc = hdr->frame_control; 2338 hdr_len = ieee80211_hdrlen(fc); 2339 2340 spin_lock(&txq->lock); 2341 2342 if (iwl_queue_space(txq) < txq->high_mark) { 2343 iwl_stop_queue(trans, txq); 2344 2345 /* don't put the packet on the ring, if there is no room */ 2346 if (unlikely(iwl_queue_space(txq) < 3)) { 2347 struct iwl_device_cmd **dev_cmd_ptr; 2348 2349 dev_cmd_ptr = (void *)((u8 *)skb->cb + 2350 trans_pcie->dev_cmd_offs); 2351 2352 *dev_cmd_ptr = dev_cmd; 2353 __skb_queue_tail(&txq->overflow_q, skb); 2354 2355 spin_unlock(&txq->lock); 2356 return 0; 2357 } 2358 } 2359 2360 /* In AGG mode, the index in the ring must correspond to the WiFi 2361 * sequence number. This is a HW requirements to help the SCD to parse 2362 * the BA. 2363 * Check here that the packets are in the right place on the ring. 2364 */ 2365 wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl)); 2366 WARN_ONCE(txq->ampdu && 2367 (wifi_seq & 0xff) != txq->write_ptr, 2368 "Q: %d WiFi Seq %d tfdNum %d", 2369 txq_id, wifi_seq, txq->write_ptr); 2370 2371 /* Set up driver data for this TFD */ 2372 txq->entries[txq->write_ptr].skb = skb; 2373 txq->entries[txq->write_ptr].cmd = dev_cmd; 2374 2375 dev_cmd->hdr.sequence = 2376 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 2377 INDEX_TO_SEQ(txq->write_ptr))); 2378 2379 tb0_phys = iwl_pcie_get_first_tb_dma(txq, txq->write_ptr); 2380 scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) + 2381 offsetof(struct iwl_tx_cmd, scratch); 2382 2383 tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys); 2384 tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys); 2385 2386 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 2387 out_meta = &txq->entries[txq->write_ptr].meta; 2388 out_meta->flags = 0; 2389 2390 /* 2391 * The second TB (tb1) points to the remainder of the TX command 2392 * and the 802.11 header - dword aligned size 2393 * (This calculation modifies the TX command, so do it before the 2394 * setup of the first TB) 2395 */ 2396 len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) + 2397 hdr_len - IWL_FIRST_TB_SIZE; 2398 /* do not align A-MSDU to dword as the subframe header aligns it */ 2399 amsdu = ieee80211_is_data_qos(fc) && 2400 (*ieee80211_get_qos_ctl(hdr) & 2401 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 2402 if (trans_pcie->sw_csum_tx || !amsdu) { 2403 tb1_len = ALIGN(len, 4); 2404 /* Tell NIC about any 2-byte padding after MAC header */ 2405 if (tb1_len != len) 2406 tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK; 2407 } else { 2408 tb1_len = len; 2409 } 2410 2411 /* The first TB points to bi-directional DMA data */ 2412 memcpy(&txq->first_tb_bufs[txq->write_ptr], &dev_cmd->hdr, 2413 IWL_FIRST_TB_SIZE); 2414 iwl_pcie_txq_build_tfd(trans, txq, tb0_phys, 2415 IWL_FIRST_TB_SIZE, true); 2416 2417 /* there must be data left over for TB1 or this code must be changed */ 2418 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE); 2419 2420 /* map the data for TB1 */ 2421 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 2422 tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 2423 if (unlikely(dma_mapping_error(trans->dev, tb1_phys))) 2424 goto out_err; 2425 iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false); 2426 2427 if (amsdu) { 2428 if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len, 2429 out_meta, dev_cmd, 2430 tb1_len))) 2431 goto out_err; 2432 } else if (unlikely(iwl_fill_data_tbs(trans, skb, txq, hdr_len, 2433 out_meta, dev_cmd, tb1_len))) { 2434 goto out_err; 2435 } 2436 2437 tfd = iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr); 2438 /* Set up entry for this TFD in Tx byte-count array */ 2439 iwl_pcie_txq_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len), 2440 iwl_pcie_tfd_get_num_tbs(trans, tfd)); 2441 2442 wait_write_ptr = ieee80211_has_morefrags(fc); 2443 2444 /* start timer if queue currently empty */ 2445 if (txq->read_ptr == txq->write_ptr) { 2446 if (txq->wd_timeout) { 2447 /* 2448 * If the TXQ is active, then set the timer, if not, 2449 * set the timer in remainder so that the timer will 2450 * be armed with the right value when the station will 2451 * wake up. 2452 */ 2453 if (!txq->frozen) 2454 mod_timer(&txq->stuck_timer, 2455 jiffies + txq->wd_timeout); 2456 else 2457 txq->frozen_expiry_remainder = txq->wd_timeout; 2458 } 2459 IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", txq->id); 2460 iwl_trans_ref(trans); 2461 } 2462 2463 /* Tell device the write index *just past* this latest filled TFD */ 2464 txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr); 2465 if (!wait_write_ptr) 2466 iwl_pcie_txq_inc_wr_ptr(trans, txq); 2467 2468 /* 2469 * At this point the frame is "transmitted" successfully 2470 * and we will get a TX status notification eventually. 2471 */ 2472 spin_unlock(&txq->lock); 2473 return 0; 2474 out_err: 2475 spin_unlock(&txq->lock); 2476 return -1; 2477 } 2478