1 /****************************************************************************** 2 * 3 * This file is provided under a dual BSD/GPLv2 license. When using or 4 * redistributing this file, you may do so under either license. 5 * 6 * GPL LICENSE SUMMARY 7 * 8 * Copyright(c) 2017 Intel Deutschland GmbH 9 * Copyright(c) 2018 - 2019 Intel Corporation 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * BSD LICENSE 21 * 22 * Copyright(c) 2017 Intel Deutschland GmbH 23 * Copyright(c) 2018 - 2019 Intel Corporation 24 * All rights reserved. 25 * 26 * Redistribution and use in source and binary forms, with or without 27 * modification, are permitted provided that the following conditions 28 * are met: 29 * 30 * * Redistributions of source code must retain the above copyright 31 * notice, this list of conditions and the following disclaimer. 32 * * Redistributions in binary form must reproduce the above copyright 33 * notice, this list of conditions and the following disclaimer in 34 * the documentation and/or other materials provided with the 35 * distribution. 36 * * Neither the name Intel Corporation nor the names of its 37 * contributors may be used to endorse or promote products derived 38 * from this software without specific prior written permission. 39 * 40 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 41 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 42 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 43 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 44 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 45 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 46 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 47 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 48 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 49 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 50 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 51 * 52 *****************************************************************************/ 53 #include <linux/pm_runtime.h> 54 #include <net/tso.h> 55 #include <linux/tcp.h> 56 57 #include "iwl-debug.h" 58 #include "iwl-csr.h" 59 #include "iwl-io.h" 60 #include "internal.h" 61 #include "fw/api/tx.h" 62 63 /* 64 * iwl_pcie_gen2_tx_stop - Stop all Tx DMA channels 65 */ 66 void iwl_pcie_gen2_tx_stop(struct iwl_trans *trans) 67 { 68 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 69 int txq_id; 70 71 /* 72 * This function can be called before the op_mode disabled the 73 * queues. This happens when we have an rfkill interrupt. 74 * Since we stop Tx altogether - mark the queues as stopped. 75 */ 76 memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped)); 77 memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); 78 79 /* Unmap DMA from host system and free skb's */ 80 for (txq_id = 0; txq_id < ARRAY_SIZE(trans_pcie->txq); txq_id++) { 81 if (!trans_pcie->txq[txq_id]) 82 continue; 83 iwl_pcie_gen2_txq_unmap(trans, txq_id); 84 } 85 } 86 87 /* 88 * iwl_pcie_txq_update_byte_tbl - Set up entry in Tx byte-count array 89 */ 90 void iwl_pcie_gen2_update_byte_tbl(struct iwl_trans_pcie *trans_pcie, 91 struct iwl_txq *txq, u16 byte_cnt, 92 int num_tbs) 93 { 94 struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr; 95 struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie); 96 struct iwl_gen3_bc_tbl *scd_bc_tbl_gen3 = txq->bc_tbl.addr; 97 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 98 u8 filled_tfd_size, num_fetch_chunks; 99 u16 len = byte_cnt; 100 __le16 bc_ent; 101 102 if (trans_pcie->bc_table_dword) 103 len = DIV_ROUND_UP(len, 4); 104 105 if (WARN_ON(len > 0xFFF || idx >= txq->n_window)) 106 return; 107 108 filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) + 109 num_tbs * sizeof(struct iwl_tfh_tb); 110 /* 111 * filled_tfd_size contains the number of filled bytes in the TFD. 112 * Dividing it by 64 will give the number of chunks to fetch 113 * to SRAM- 0 for one chunk, 1 for 2 and so on. 114 * If, for example, TFD contains only 3 TBs then 32 bytes 115 * of the TFD are used, and only one chunk of 64 bytes should 116 * be fetched 117 */ 118 num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1; 119 120 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12)); 121 if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_22560) 122 scd_bc_tbl_gen3->tfd_offset[idx] = bc_ent; 123 else 124 scd_bc_tbl->tfd_offset[idx] = bc_ent; 125 } 126 127 /* 128 * iwl_pcie_gen2_txq_inc_wr_ptr - Send new write index to hardware 129 */ 130 void iwl_pcie_gen2_txq_inc_wr_ptr(struct iwl_trans *trans, 131 struct iwl_txq *txq) 132 { 133 lockdep_assert_held(&txq->lock); 134 135 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr); 136 137 /* 138 * if not in power-save mode, uCode will never sleep when we're 139 * trying to tx (during RFKILL, we're not trying to tx). 140 */ 141 iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16)); 142 } 143 144 static u8 iwl_pcie_gen2_get_num_tbs(struct iwl_trans *trans, 145 struct iwl_tfh_tfd *tfd) 146 { 147 return le16_to_cpu(tfd->num_tbs) & 0x1f; 148 } 149 150 static void iwl_pcie_gen2_tfd_unmap(struct iwl_trans *trans, 151 struct iwl_cmd_meta *meta, 152 struct iwl_tfh_tfd *tfd) 153 { 154 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 155 int i, num_tbs; 156 157 /* Sanity check on number of chunks */ 158 num_tbs = iwl_pcie_gen2_get_num_tbs(trans, tfd); 159 160 if (num_tbs > trans_pcie->max_tbs) { 161 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 162 return; 163 } 164 165 /* first TB is never freed - it's the bidirectional DMA data */ 166 for (i = 1; i < num_tbs; i++) { 167 if (meta->tbs & BIT(i)) 168 dma_unmap_page(trans->dev, 169 le64_to_cpu(tfd->tbs[i].addr), 170 le16_to_cpu(tfd->tbs[i].tb_len), 171 DMA_TO_DEVICE); 172 else 173 dma_unmap_single(trans->dev, 174 le64_to_cpu(tfd->tbs[i].addr), 175 le16_to_cpu(tfd->tbs[i].tb_len), 176 DMA_TO_DEVICE); 177 } 178 179 tfd->num_tbs = 0; 180 } 181 182 static void iwl_pcie_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 183 { 184 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 185 * idx is bounded by n_window 186 */ 187 int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr); 188 189 lockdep_assert_held(&txq->lock); 190 191 iwl_pcie_gen2_tfd_unmap(trans, &txq->entries[idx].meta, 192 iwl_pcie_get_tfd(trans, txq, idx)); 193 194 /* free SKB */ 195 if (txq->entries) { 196 struct sk_buff *skb; 197 198 skb = txq->entries[idx].skb; 199 200 /* Can be called from irqs-disabled context 201 * If skb is not NULL, it means that the whole queue is being 202 * freed and that the queue is not empty - free the skb 203 */ 204 if (skb) { 205 iwl_op_mode_free_skb(trans->op_mode, skb); 206 txq->entries[idx].skb = NULL; 207 } 208 } 209 } 210 211 static int iwl_pcie_gen2_set_tb(struct iwl_trans *trans, 212 struct iwl_tfh_tfd *tfd, dma_addr_t addr, 213 u16 len) 214 { 215 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 216 int idx = iwl_pcie_gen2_get_num_tbs(trans, tfd); 217 struct iwl_tfh_tb *tb; 218 219 if (WARN_ON(idx >= IWL_TFH_NUM_TBS)) 220 return -EINVAL; 221 tb = &tfd->tbs[idx]; 222 223 /* Each TFD can point to a maximum max_tbs Tx buffers */ 224 if (le16_to_cpu(tfd->num_tbs) >= trans_pcie->max_tbs) { 225 IWL_ERR(trans, "Error can not send more than %d chunks\n", 226 trans_pcie->max_tbs); 227 return -EINVAL; 228 } 229 230 put_unaligned_le64(addr, &tb->addr); 231 tb->tb_len = cpu_to_le16(len); 232 233 tfd->num_tbs = cpu_to_le16(idx + 1); 234 235 return idx; 236 } 237 238 static int iwl_pcie_gen2_build_amsdu(struct iwl_trans *trans, 239 struct sk_buff *skb, 240 struct iwl_tfh_tfd *tfd, int start_len, 241 u8 hdr_len, struct iwl_device_cmd *dev_cmd) 242 { 243 #ifdef CONFIG_INET 244 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 245 struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload; 246 struct ieee80211_hdr *hdr = (void *)skb->data; 247 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 248 unsigned int mss = skb_shinfo(skb)->gso_size; 249 u16 length, iv_len, amsdu_pad; 250 u8 *start_hdr; 251 struct iwl_tso_hdr_page *hdr_page; 252 struct page **page_ptr; 253 struct tso_t tso; 254 255 /* if the packet is protected, then it must be CCMP or GCMP */ 256 iv_len = ieee80211_has_protected(hdr->frame_control) ? 257 IEEE80211_CCMP_HDR_LEN : 0; 258 259 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), 260 &dev_cmd->hdr, start_len, 0); 261 262 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 263 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 264 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; 265 amsdu_pad = 0; 266 267 /* total amount of header we may need for this A-MSDU */ 268 hdr_room = DIV_ROUND_UP(total_len, mss) * 269 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len; 270 271 /* Our device supports 9 segments at most, it will fit in 1 page */ 272 hdr_page = get_page_hdr(trans, hdr_room); 273 if (!hdr_page) 274 return -ENOMEM; 275 276 get_page(hdr_page->page); 277 start_hdr = hdr_page->pos; 278 page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); 279 *page_ptr = hdr_page->page; 280 memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); 281 hdr_page->pos += iv_len; 282 283 /* 284 * Pull the ieee80211 header + IV to be able to use TSO core, 285 * we will restore it for the tx_status flow. 286 */ 287 skb_pull(skb, hdr_len + iv_len); 288 289 /* 290 * Remove the length of all the headers that we don't actually 291 * have in the MPDU by themselves, but that we duplicate into 292 * all the different MSDUs inside the A-MSDU. 293 */ 294 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 295 296 tso_start(skb, &tso); 297 298 while (total_len) { 299 /* this is the data left for this subframe */ 300 unsigned int data_left = min_t(unsigned int, mss, total_len); 301 struct sk_buff *csum_skb = NULL; 302 unsigned int tb_len; 303 dma_addr_t tb_phys; 304 u8 *subf_hdrs_start = hdr_page->pos; 305 306 total_len -= data_left; 307 308 memset(hdr_page->pos, 0, amsdu_pad); 309 hdr_page->pos += amsdu_pad; 310 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 311 data_left)) & 0x3; 312 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 313 hdr_page->pos += ETH_ALEN; 314 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 315 hdr_page->pos += ETH_ALEN; 316 317 length = snap_ip_tcp_hdrlen + data_left; 318 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 319 hdr_page->pos += sizeof(length); 320 321 /* 322 * This will copy the SNAP as well which will be considered 323 * as MAC header. 324 */ 325 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 326 327 hdr_page->pos += snap_ip_tcp_hdrlen; 328 329 tb_len = hdr_page->pos - start_hdr; 330 tb_phys = dma_map_single(trans->dev, start_hdr, 331 tb_len, DMA_TO_DEVICE); 332 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 333 dev_kfree_skb(csum_skb); 334 goto out_err; 335 } 336 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len); 337 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, tb_len); 338 /* add this subframe's headers' length to the tx_cmd */ 339 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 340 341 /* prepare the start_hdr for the next subframe */ 342 start_hdr = hdr_page->pos; 343 344 /* put the payload */ 345 while (data_left) { 346 tb_len = min_t(unsigned int, tso.size, data_left); 347 tb_phys = dma_map_single(trans->dev, tso.data, 348 tb_len, DMA_TO_DEVICE); 349 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 350 dev_kfree_skb(csum_skb); 351 goto out_err; 352 } 353 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len); 354 trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data, 355 tb_len); 356 357 data_left -= tb_len; 358 tso_build_data(skb, &tso, tb_len); 359 } 360 } 361 362 /* re -add the WiFi header and IV */ 363 skb_push(skb, hdr_len + iv_len); 364 365 return 0; 366 367 out_err: 368 #endif 369 return -EINVAL; 370 } 371 372 static struct 373 iwl_tfh_tfd *iwl_pcie_gen2_build_tx_amsdu(struct iwl_trans *trans, 374 struct iwl_txq *txq, 375 struct iwl_device_cmd *dev_cmd, 376 struct sk_buff *skb, 377 struct iwl_cmd_meta *out_meta, 378 int hdr_len, 379 int tx_cmd_len) 380 { 381 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 382 struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); 383 dma_addr_t tb_phys; 384 int len; 385 void *tb1_addr; 386 387 tb_phys = iwl_pcie_get_first_tb_dma(txq, idx); 388 389 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 390 391 /* 392 * The second TB (tb1) points to the remainder of the TX command 393 * and the 802.11 header - dword aligned size 394 * (This calculation modifies the TX command, so do it before the 395 * setup of the first TB) 396 */ 397 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 398 IWL_FIRST_TB_SIZE; 399 400 /* do not align A-MSDU to dword as the subframe header aligns it */ 401 402 /* map the data for TB1 */ 403 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 404 tb_phys = dma_map_single(trans->dev, tb1_addr, len, DMA_TO_DEVICE); 405 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 406 goto out_err; 407 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, len); 408 409 if (iwl_pcie_gen2_build_amsdu(trans, skb, tfd, 410 len + IWL_FIRST_TB_SIZE, 411 hdr_len, dev_cmd)) 412 goto out_err; 413 414 /* building the A-MSDU might have changed this data, memcpy it now */ 415 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 416 return tfd; 417 418 out_err: 419 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 420 return NULL; 421 } 422 423 static int iwl_pcie_gen2_tx_add_frags(struct iwl_trans *trans, 424 struct sk_buff *skb, 425 struct iwl_tfh_tfd *tfd, 426 struct iwl_cmd_meta *out_meta) 427 { 428 int i; 429 430 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 431 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 432 dma_addr_t tb_phys; 433 int tb_idx; 434 435 if (!skb_frag_size(frag)) 436 continue; 437 438 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 439 skb_frag_size(frag), DMA_TO_DEVICE); 440 441 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 442 return -ENOMEM; 443 tb_idx = iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, 444 skb_frag_size(frag)); 445 trace_iwlwifi_dev_tx_tb(trans->dev, skb, 446 skb_frag_address(frag), 447 skb_frag_size(frag)); 448 if (tb_idx < 0) 449 return tb_idx; 450 451 out_meta->tbs |= BIT(tb_idx); 452 } 453 454 return 0; 455 } 456 457 static struct 458 iwl_tfh_tfd *iwl_pcie_gen2_build_tx(struct iwl_trans *trans, 459 struct iwl_txq *txq, 460 struct iwl_device_cmd *dev_cmd, 461 struct sk_buff *skb, 462 struct iwl_cmd_meta *out_meta, 463 int hdr_len, 464 int tx_cmd_len, 465 bool pad) 466 { 467 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 468 struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); 469 dma_addr_t tb_phys; 470 int len, tb1_len, tb2_len; 471 void *tb1_addr; 472 473 tb_phys = iwl_pcie_get_first_tb_dma(txq, idx); 474 475 /* The first TB points to bi-directional DMA data */ 476 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 477 478 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 479 480 /* 481 * The second TB (tb1) points to the remainder of the TX command 482 * and the 802.11 header - dword aligned size 483 * (This calculation modifies the TX command, so do it before the 484 * setup of the first TB) 485 */ 486 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 487 IWL_FIRST_TB_SIZE; 488 489 if (pad) 490 tb1_len = ALIGN(len, 4); 491 else 492 tb1_len = len; 493 494 /* map the data for TB1 */ 495 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 496 tb_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 497 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 498 goto out_err; 499 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb1_len); 500 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), &dev_cmd->hdr, 501 IWL_FIRST_TB_SIZE + tb1_len, hdr_len); 502 503 /* set up TFD's third entry to point to remainder of skb's head */ 504 tb2_len = skb_headlen(skb) - hdr_len; 505 506 if (tb2_len > 0) { 507 tb_phys = dma_map_single(trans->dev, skb->data + hdr_len, 508 tb2_len, DMA_TO_DEVICE); 509 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 510 goto out_err; 511 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb2_len); 512 trace_iwlwifi_dev_tx_tb(trans->dev, skb, 513 skb->data + hdr_len, 514 tb2_len); 515 } 516 517 if (iwl_pcie_gen2_tx_add_frags(trans, skb, tfd, out_meta)) 518 goto out_err; 519 520 return tfd; 521 522 out_err: 523 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 524 return NULL; 525 } 526 527 static 528 struct iwl_tfh_tfd *iwl_pcie_gen2_build_tfd(struct iwl_trans *trans, 529 struct iwl_txq *txq, 530 struct iwl_device_cmd *dev_cmd, 531 struct sk_buff *skb, 532 struct iwl_cmd_meta *out_meta) 533 { 534 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 535 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 536 struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); 537 int len, hdr_len; 538 bool amsdu; 539 540 /* There must be data left over for TB1 or this code must be changed */ 541 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2) < IWL_FIRST_TB_SIZE); 542 543 memset(tfd, 0, sizeof(*tfd)); 544 545 if (trans->cfg->device_family < IWL_DEVICE_FAMILY_22560) 546 len = sizeof(struct iwl_tx_cmd_gen2); 547 else 548 len = sizeof(struct iwl_tx_cmd_gen3); 549 550 amsdu = ieee80211_is_data_qos(hdr->frame_control) && 551 (*ieee80211_get_qos_ctl(hdr) & 552 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 553 554 hdr_len = ieee80211_hdrlen(hdr->frame_control); 555 556 /* 557 * Only build A-MSDUs here if doing so by GSO, otherwise it may be 558 * an A-MSDU for other reasons, e.g. NAN or an A-MSDU having been 559 * built in the higher layers already. 560 */ 561 if (amsdu && skb_shinfo(skb)->gso_size) 562 return iwl_pcie_gen2_build_tx_amsdu(trans, txq, dev_cmd, skb, 563 out_meta, hdr_len, len); 564 565 return iwl_pcie_gen2_build_tx(trans, txq, dev_cmd, skb, out_meta, 566 hdr_len, len, !amsdu); 567 } 568 569 int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb, 570 struct iwl_device_cmd *dev_cmd, int txq_id) 571 { 572 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 573 struct iwl_cmd_meta *out_meta; 574 struct iwl_txq *txq = trans_pcie->txq[txq_id]; 575 u16 cmd_len; 576 int idx; 577 void *tfd; 578 579 if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used), 580 "TX on unused queue %d\n", txq_id)) 581 return -EINVAL; 582 583 if (skb_is_nonlinear(skb) && 584 skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS(trans_pcie) && 585 __skb_linearize(skb)) 586 return -ENOMEM; 587 588 spin_lock(&txq->lock); 589 590 if (iwl_queue_space(trans, txq) < txq->high_mark) { 591 iwl_stop_queue(trans, txq); 592 593 /* don't put the packet on the ring, if there is no room */ 594 if (unlikely(iwl_queue_space(trans, txq) < 3)) { 595 struct iwl_device_cmd **dev_cmd_ptr; 596 597 dev_cmd_ptr = (void *)((u8 *)skb->cb + 598 trans_pcie->dev_cmd_offs); 599 600 *dev_cmd_ptr = dev_cmd; 601 __skb_queue_tail(&txq->overflow_q, skb); 602 spin_unlock(&txq->lock); 603 return 0; 604 } 605 } 606 607 idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 608 609 /* Set up driver data for this TFD */ 610 txq->entries[idx].skb = skb; 611 txq->entries[idx].cmd = dev_cmd; 612 613 dev_cmd->hdr.sequence = 614 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 615 INDEX_TO_SEQ(idx))); 616 617 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 618 out_meta = &txq->entries[idx].meta; 619 out_meta->flags = 0; 620 621 tfd = iwl_pcie_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta); 622 if (!tfd) { 623 spin_unlock(&txq->lock); 624 return -1; 625 } 626 627 if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_22560) { 628 struct iwl_tx_cmd_gen3 *tx_cmd_gen3 = 629 (void *)dev_cmd->payload; 630 631 cmd_len = le16_to_cpu(tx_cmd_gen3->len); 632 } else { 633 struct iwl_tx_cmd_gen2 *tx_cmd_gen2 = 634 (void *)dev_cmd->payload; 635 636 cmd_len = le16_to_cpu(tx_cmd_gen2->len); 637 } 638 639 /* Set up entry for this TFD in Tx byte-count array */ 640 iwl_pcie_gen2_update_byte_tbl(trans_pcie, txq, cmd_len, 641 iwl_pcie_gen2_get_num_tbs(trans, tfd)); 642 643 /* start timer if queue currently empty */ 644 if (txq->read_ptr == txq->write_ptr) { 645 if (txq->wd_timeout) 646 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 647 IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", txq->id); 648 iwl_trans_ref(trans); 649 } 650 651 /* Tell device the write index *just past* this latest filled TFD */ 652 txq->write_ptr = iwl_queue_inc_wrap(trans, txq->write_ptr); 653 iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq); 654 /* 655 * At this point the frame is "transmitted" successfully 656 * and we will get a TX status notification eventually. 657 */ 658 spin_unlock(&txq->lock); 659 return 0; 660 } 661 662 /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 663 664 /* 665 * iwl_pcie_gen2_enqueue_hcmd - enqueue a uCode command 666 * @priv: device private data point 667 * @cmd: a pointer to the ucode command structure 668 * 669 * The function returns < 0 values to indicate the operation 670 * failed. On success, it returns the index (>= 0) of command in the 671 * command queue. 672 */ 673 static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, 674 struct iwl_host_cmd *cmd) 675 { 676 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 677 struct iwl_txq *txq = trans_pcie->txq[trans_pcie->cmd_queue]; 678 struct iwl_device_cmd *out_cmd; 679 struct iwl_cmd_meta *out_meta; 680 unsigned long flags; 681 void *dup_buf = NULL; 682 dma_addr_t phys_addr; 683 int i, cmd_pos, idx; 684 u16 copy_size, cmd_size, tb0_size; 685 bool had_nocopy = false; 686 u8 group_id = iwl_cmd_groupid(cmd->id); 687 const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; 688 u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; 689 struct iwl_tfh_tfd *tfd; 690 691 copy_size = sizeof(struct iwl_cmd_header_wide); 692 cmd_size = sizeof(struct iwl_cmd_header_wide); 693 694 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 695 cmddata[i] = cmd->data[i]; 696 cmdlen[i] = cmd->len[i]; 697 698 if (!cmd->len[i]) 699 continue; 700 701 /* need at least IWL_FIRST_TB_SIZE copied */ 702 if (copy_size < IWL_FIRST_TB_SIZE) { 703 int copy = IWL_FIRST_TB_SIZE - copy_size; 704 705 if (copy > cmdlen[i]) 706 copy = cmdlen[i]; 707 cmdlen[i] -= copy; 708 cmddata[i] += copy; 709 copy_size += copy; 710 } 711 712 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { 713 had_nocopy = true; 714 if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { 715 idx = -EINVAL; 716 goto free_dup_buf; 717 } 718 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { 719 /* 720 * This is also a chunk that isn't copied 721 * to the static buffer so set had_nocopy. 722 */ 723 had_nocopy = true; 724 725 /* only allowed once */ 726 if (WARN_ON(dup_buf)) { 727 idx = -EINVAL; 728 goto free_dup_buf; 729 } 730 731 dup_buf = kmemdup(cmddata[i], cmdlen[i], 732 GFP_ATOMIC); 733 if (!dup_buf) 734 return -ENOMEM; 735 } else { 736 /* NOCOPY must not be followed by normal! */ 737 if (WARN_ON(had_nocopy)) { 738 idx = -EINVAL; 739 goto free_dup_buf; 740 } 741 copy_size += cmdlen[i]; 742 } 743 cmd_size += cmd->len[i]; 744 } 745 746 /* 747 * If any of the command structures end up being larger than the 748 * TFD_MAX_PAYLOAD_SIZE and they aren't dynamically allocated into 749 * separate TFDs, then we will need to increase the size of the buffers 750 */ 751 if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, 752 "Command %s (%#x) is too large (%d bytes)\n", 753 iwl_get_cmd_string(trans, cmd->id), cmd->id, copy_size)) { 754 idx = -EINVAL; 755 goto free_dup_buf; 756 } 757 758 spin_lock_bh(&txq->lock); 759 760 idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 761 tfd = iwl_pcie_get_tfd(trans, txq, txq->write_ptr); 762 memset(tfd, 0, sizeof(*tfd)); 763 764 if (iwl_queue_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { 765 spin_unlock_bh(&txq->lock); 766 767 IWL_ERR(trans, "No space in command queue\n"); 768 iwl_op_mode_cmd_queue_full(trans->op_mode); 769 idx = -ENOSPC; 770 goto free_dup_buf; 771 } 772 773 out_cmd = txq->entries[idx].cmd; 774 out_meta = &txq->entries[idx].meta; 775 776 /* re-initialize to NULL */ 777 memset(out_meta, 0, sizeof(*out_meta)); 778 if (cmd->flags & CMD_WANT_SKB) 779 out_meta->source = cmd; 780 781 /* set up the header */ 782 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); 783 out_cmd->hdr_wide.group_id = group_id; 784 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); 785 out_cmd->hdr_wide.length = 786 cpu_to_le16(cmd_size - sizeof(struct iwl_cmd_header_wide)); 787 out_cmd->hdr_wide.reserved = 0; 788 out_cmd->hdr_wide.sequence = 789 cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | 790 INDEX_TO_SEQ(txq->write_ptr)); 791 792 cmd_pos = sizeof(struct iwl_cmd_header_wide); 793 copy_size = sizeof(struct iwl_cmd_header_wide); 794 795 /* and copy the data that needs to be copied */ 796 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 797 int copy; 798 799 if (!cmd->len[i]) 800 continue; 801 802 /* copy everything if not nocopy/dup */ 803 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 804 IWL_HCMD_DFL_DUP))) { 805 copy = cmd->len[i]; 806 807 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 808 cmd_pos += copy; 809 copy_size += copy; 810 continue; 811 } 812 813 /* 814 * Otherwise we need at least IWL_FIRST_TB_SIZE copied 815 * in total (for bi-directional DMA), but copy up to what 816 * we can fit into the payload for debug dump purposes. 817 */ 818 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); 819 820 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 821 cmd_pos += copy; 822 823 /* However, treat copy_size the proper way, we need it below */ 824 if (copy_size < IWL_FIRST_TB_SIZE) { 825 copy = IWL_FIRST_TB_SIZE - copy_size; 826 827 if (copy > cmd->len[i]) 828 copy = cmd->len[i]; 829 copy_size += copy; 830 } 831 } 832 833 IWL_DEBUG_HC(trans, 834 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", 835 iwl_get_cmd_string(trans, cmd->id), group_id, 836 out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), 837 cmd_size, txq->write_ptr, idx, trans_pcie->cmd_queue); 838 839 /* start the TFD with the minimum copy bytes */ 840 tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); 841 memcpy(&txq->first_tb_bufs[idx], out_cmd, tb0_size); 842 iwl_pcie_gen2_set_tb(trans, tfd, iwl_pcie_get_first_tb_dma(txq, idx), 843 tb0_size); 844 845 /* map first command fragment, if any remains */ 846 if (copy_size > tb0_size) { 847 phys_addr = dma_map_single(trans->dev, 848 (u8 *)out_cmd + tb0_size, 849 copy_size - tb0_size, 850 DMA_TO_DEVICE); 851 if (dma_mapping_error(trans->dev, phys_addr)) { 852 idx = -ENOMEM; 853 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 854 goto out; 855 } 856 iwl_pcie_gen2_set_tb(trans, tfd, phys_addr, 857 copy_size - tb0_size); 858 } 859 860 /* map the remaining (adjusted) nocopy/dup fragments */ 861 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 862 const void *data = cmddata[i]; 863 864 if (!cmdlen[i]) 865 continue; 866 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 867 IWL_HCMD_DFL_DUP))) 868 continue; 869 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) 870 data = dup_buf; 871 phys_addr = dma_map_single(trans->dev, (void *)data, 872 cmdlen[i], DMA_TO_DEVICE); 873 if (dma_mapping_error(trans->dev, phys_addr)) { 874 idx = -ENOMEM; 875 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 876 goto out; 877 } 878 iwl_pcie_gen2_set_tb(trans, tfd, phys_addr, cmdlen[i]); 879 } 880 881 BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); 882 out_meta->flags = cmd->flags; 883 if (WARN_ON_ONCE(txq->entries[idx].free_buf)) 884 kzfree(txq->entries[idx].free_buf); 885 txq->entries[idx].free_buf = dup_buf; 886 887 trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); 888 889 /* start timer if queue currently empty */ 890 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 891 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 892 893 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 894 if (!(cmd->flags & CMD_SEND_IN_IDLE) && 895 !trans_pcie->ref_cmd_in_flight) { 896 trans_pcie->ref_cmd_in_flight = true; 897 IWL_DEBUG_RPM(trans, "set ref_cmd_in_flight - ref\n"); 898 iwl_trans_ref(trans); 899 } 900 /* Increment and update queue's write index */ 901 txq->write_ptr = iwl_queue_inc_wrap(trans, txq->write_ptr); 902 iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq); 903 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 904 905 out: 906 spin_unlock_bh(&txq->lock); 907 free_dup_buf: 908 if (idx < 0) 909 kfree(dup_buf); 910 return idx; 911 } 912 913 #define HOST_COMPLETE_TIMEOUT (2 * HZ) 914 915 static int iwl_pcie_gen2_send_hcmd_sync(struct iwl_trans *trans, 916 struct iwl_host_cmd *cmd) 917 { 918 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 919 const char *cmd_str = iwl_get_cmd_string(trans, cmd->id); 920 struct iwl_txq *txq = trans_pcie->txq[trans_pcie->cmd_queue]; 921 int cmd_idx; 922 int ret; 923 924 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", cmd_str); 925 926 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 927 &trans->status), 928 "Command %s: a command is already active!\n", cmd_str)) 929 return -EIO; 930 931 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", cmd_str); 932 933 if (pm_runtime_suspended(&trans_pcie->pci_dev->dev)) { 934 ret = wait_event_timeout(trans_pcie->d0i3_waitq, 935 pm_runtime_active(&trans_pcie->pci_dev->dev), 936 msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT)); 937 if (!ret) { 938 IWL_ERR(trans, "Timeout exiting D0i3 before hcmd\n"); 939 return -ETIMEDOUT; 940 } 941 } 942 943 cmd_idx = iwl_pcie_gen2_enqueue_hcmd(trans, cmd); 944 if (cmd_idx < 0) { 945 ret = cmd_idx; 946 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 947 IWL_ERR(trans, "Error sending %s: enqueue_hcmd failed: %d\n", 948 cmd_str, ret); 949 return ret; 950 } 951 952 ret = wait_event_timeout(trans_pcie->wait_command_queue, 953 !test_bit(STATUS_SYNC_HCMD_ACTIVE, 954 &trans->status), 955 HOST_COMPLETE_TIMEOUT); 956 if (!ret) { 957 IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 958 cmd_str, jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 959 960 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 961 txq->read_ptr, txq->write_ptr); 962 963 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 964 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 965 cmd_str); 966 ret = -ETIMEDOUT; 967 968 iwl_trans_pcie_sync_nmi(trans); 969 goto cancel; 970 } 971 972 if (test_bit(STATUS_FW_ERROR, &trans->status)) { 973 IWL_ERR(trans, "FW error in SYNC CMD %s\n", cmd_str); 974 dump_stack(); 975 ret = -EIO; 976 goto cancel; 977 } 978 979 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 980 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 981 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 982 ret = -ERFKILL; 983 goto cancel; 984 } 985 986 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 987 IWL_ERR(trans, "Error: Response NULL in '%s'\n", cmd_str); 988 ret = -EIO; 989 goto cancel; 990 } 991 992 return 0; 993 994 cancel: 995 if (cmd->flags & CMD_WANT_SKB) { 996 /* 997 * Cancel the CMD_WANT_SKB flag for the cmd in the 998 * TX cmd queue. Otherwise in case the cmd comes 999 * in later, it will possibly set an invalid 1000 * address (cmd->meta.source). 1001 */ 1002 txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 1003 } 1004 1005 if (cmd->resp_pkt) { 1006 iwl_free_resp(cmd); 1007 cmd->resp_pkt = NULL; 1008 } 1009 1010 return ret; 1011 } 1012 1013 int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans *trans, 1014 struct iwl_host_cmd *cmd) 1015 { 1016 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1017 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1018 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1019 cmd->id); 1020 return -ERFKILL; 1021 } 1022 1023 if (cmd->flags & CMD_ASYNC) { 1024 int ret; 1025 1026 /* An asynchronous command can not expect an SKB to be set. */ 1027 if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1028 return -EINVAL; 1029 1030 ret = iwl_pcie_gen2_enqueue_hcmd(trans, cmd); 1031 if (ret < 0) { 1032 IWL_ERR(trans, 1033 "Error sending %s: enqueue_hcmd failed: %d\n", 1034 iwl_get_cmd_string(trans, cmd->id), ret); 1035 return ret; 1036 } 1037 return 0; 1038 } 1039 1040 return iwl_pcie_gen2_send_hcmd_sync(trans, cmd); 1041 } 1042 1043 /* 1044 * iwl_pcie_gen2_txq_unmap - Unmap any remaining DMA mappings and free skb's 1045 */ 1046 void iwl_pcie_gen2_txq_unmap(struct iwl_trans *trans, int txq_id) 1047 { 1048 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1049 struct iwl_txq *txq = trans_pcie->txq[txq_id]; 1050 1051 spin_lock_bh(&txq->lock); 1052 while (txq->write_ptr != txq->read_ptr) { 1053 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 1054 txq_id, txq->read_ptr); 1055 1056 if (txq_id != trans_pcie->cmd_queue) { 1057 int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr); 1058 struct sk_buff *skb = txq->entries[idx].skb; 1059 1060 if (WARN_ON_ONCE(!skb)) 1061 continue; 1062 1063 iwl_pcie_free_tso_page(trans_pcie, skb); 1064 } 1065 iwl_pcie_gen2_free_tfd(trans, txq); 1066 txq->read_ptr = iwl_queue_inc_wrap(trans, txq->read_ptr); 1067 1068 if (txq->read_ptr == txq->write_ptr) { 1069 unsigned long flags; 1070 1071 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 1072 if (txq_id != trans_pcie->cmd_queue) { 1073 IWL_DEBUG_RPM(trans, "Q %d - last tx freed\n", 1074 txq->id); 1075 iwl_trans_unref(trans); 1076 } else if (trans_pcie->ref_cmd_in_flight) { 1077 trans_pcie->ref_cmd_in_flight = false; 1078 IWL_DEBUG_RPM(trans, 1079 "clear ref_cmd_in_flight\n"); 1080 iwl_trans_unref(trans); 1081 } 1082 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 1083 } 1084 } 1085 1086 while (!skb_queue_empty(&txq->overflow_q)) { 1087 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 1088 1089 iwl_op_mode_free_skb(trans->op_mode, skb); 1090 } 1091 1092 spin_unlock_bh(&txq->lock); 1093 1094 /* just in case - this queue may have been stopped */ 1095 iwl_wake_queue(trans, txq); 1096 } 1097 1098 void iwl_pcie_gen2_txq_free_memory(struct iwl_trans *trans, 1099 struct iwl_txq *txq) 1100 { 1101 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1102 struct device *dev = trans->dev; 1103 1104 /* De-alloc circular buffer of TFDs */ 1105 if (txq->tfds) { 1106 dma_free_coherent(dev, 1107 trans_pcie->tfd_size * txq->n_window, 1108 txq->tfds, txq->dma_addr); 1109 dma_free_coherent(dev, 1110 sizeof(*txq->first_tb_bufs) * txq->n_window, 1111 txq->first_tb_bufs, txq->first_tb_dma); 1112 } 1113 1114 kfree(txq->entries); 1115 iwl_pcie_free_dma_ptr(trans, &txq->bc_tbl); 1116 kfree(txq); 1117 } 1118 1119 /* 1120 * iwl_pcie_txq_free - Deallocate DMA queue. 1121 * @txq: Transmit queue to deallocate. 1122 * 1123 * Empty queue by removing and destroying all BD's. 1124 * Free all buffers. 1125 * 0-fill, but do not free "txq" descriptor structure. 1126 */ 1127 static void iwl_pcie_gen2_txq_free(struct iwl_trans *trans, int txq_id) 1128 { 1129 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1130 struct iwl_txq *txq = trans_pcie->txq[txq_id]; 1131 int i; 1132 1133 if (WARN_ON(!txq)) 1134 return; 1135 1136 iwl_pcie_gen2_txq_unmap(trans, txq_id); 1137 1138 /* De-alloc array of command/tx buffers */ 1139 if (txq_id == trans_pcie->cmd_queue) 1140 for (i = 0; i < txq->n_window; i++) { 1141 kzfree(txq->entries[i].cmd); 1142 kzfree(txq->entries[i].free_buf); 1143 } 1144 del_timer_sync(&txq->stuck_timer); 1145 1146 iwl_pcie_gen2_txq_free_memory(trans, txq); 1147 1148 trans_pcie->txq[txq_id] = NULL; 1149 1150 clear_bit(txq_id, trans_pcie->queue_used); 1151 } 1152 1153 int iwl_trans_pcie_dyn_txq_alloc_dma(struct iwl_trans *trans, 1154 struct iwl_txq **intxq, int size, 1155 unsigned int timeout) 1156 { 1157 int ret; 1158 1159 struct iwl_txq *txq; 1160 txq = kzalloc(sizeof(*txq), GFP_KERNEL); 1161 if (!txq) 1162 return -ENOMEM; 1163 ret = iwl_pcie_alloc_dma_ptr(trans, &txq->bc_tbl, 1164 (trans->cfg->device_family >= 1165 IWL_DEVICE_FAMILY_22560) ? 1166 sizeof(struct iwl_gen3_bc_tbl) : 1167 sizeof(struct iwlagn_scd_bc_tbl)); 1168 if (ret) { 1169 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 1170 kfree(txq); 1171 return -ENOMEM; 1172 } 1173 1174 ret = iwl_pcie_txq_alloc(trans, txq, size, false); 1175 if (ret) { 1176 IWL_ERR(trans, "Tx queue alloc failed\n"); 1177 goto error; 1178 } 1179 ret = iwl_pcie_txq_init(trans, txq, size, false); 1180 if (ret) { 1181 IWL_ERR(trans, "Tx queue init failed\n"); 1182 goto error; 1183 } 1184 1185 txq->wd_timeout = msecs_to_jiffies(timeout); 1186 1187 *intxq = txq; 1188 return 0; 1189 1190 error: 1191 iwl_pcie_gen2_txq_free_memory(trans, txq); 1192 return ret; 1193 } 1194 1195 int iwl_trans_pcie_txq_alloc_response(struct iwl_trans *trans, 1196 struct iwl_txq *txq, 1197 struct iwl_host_cmd *hcmd) 1198 { 1199 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1200 struct iwl_tx_queue_cfg_rsp *rsp; 1201 int ret, qid; 1202 u32 wr_ptr; 1203 1204 if (WARN_ON(iwl_rx_packet_payload_len(hcmd->resp_pkt) != 1205 sizeof(*rsp))) { 1206 ret = -EINVAL; 1207 goto error_free_resp; 1208 } 1209 1210 rsp = (void *)hcmd->resp_pkt->data; 1211 qid = le16_to_cpu(rsp->queue_number); 1212 wr_ptr = le16_to_cpu(rsp->write_pointer); 1213 1214 if (qid >= ARRAY_SIZE(trans_pcie->txq)) { 1215 WARN_ONCE(1, "queue index %d unsupported", qid); 1216 ret = -EIO; 1217 goto error_free_resp; 1218 } 1219 1220 if (test_and_set_bit(qid, trans_pcie->queue_used)) { 1221 WARN_ONCE(1, "queue %d already used", qid); 1222 ret = -EIO; 1223 goto error_free_resp; 1224 } 1225 1226 txq->id = qid; 1227 trans_pcie->txq[qid] = txq; 1228 wr_ptr &= (trans->cfg->base_params->max_tfd_queue_size - 1); 1229 1230 /* Place first TFD at index corresponding to start sequence number */ 1231 txq->read_ptr = wr_ptr; 1232 txq->write_ptr = wr_ptr; 1233 1234 IWL_DEBUG_TX_QUEUES(trans, "Activate queue %d\n", qid); 1235 1236 iwl_free_resp(hcmd); 1237 return qid; 1238 1239 error_free_resp: 1240 iwl_free_resp(hcmd); 1241 iwl_pcie_gen2_txq_free_memory(trans, txq); 1242 return ret; 1243 } 1244 1245 int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans, 1246 __le16 flags, u8 sta_id, u8 tid, 1247 int cmd_id, int size, 1248 unsigned int timeout) 1249 { 1250 struct iwl_txq *txq = NULL; 1251 struct iwl_tx_queue_cfg_cmd cmd = { 1252 .flags = flags, 1253 .sta_id = sta_id, 1254 .tid = tid, 1255 }; 1256 struct iwl_host_cmd hcmd = { 1257 .id = cmd_id, 1258 .len = { sizeof(cmd) }, 1259 .data = { &cmd, }, 1260 .flags = CMD_WANT_SKB, 1261 }; 1262 int ret; 1263 1264 ret = iwl_trans_pcie_dyn_txq_alloc_dma(trans, &txq, size, timeout); 1265 if (ret) 1266 return ret; 1267 1268 cmd.tfdq_addr = cpu_to_le64(txq->dma_addr); 1269 cmd.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma); 1270 cmd.cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(size)); 1271 1272 ret = iwl_trans_send_cmd(trans, &hcmd); 1273 if (ret) 1274 goto error; 1275 1276 return iwl_trans_pcie_txq_alloc_response(trans, txq, &hcmd); 1277 1278 error: 1279 iwl_pcie_gen2_txq_free_memory(trans, txq); 1280 return ret; 1281 } 1282 1283 void iwl_trans_pcie_dyn_txq_free(struct iwl_trans *trans, int queue) 1284 { 1285 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1286 1287 /* 1288 * Upon HW Rfkill - we stop the device, and then stop the queues 1289 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1290 * allow the op_mode to call txq_disable after it already called 1291 * stop_device. 1292 */ 1293 if (!test_and_clear_bit(queue, trans_pcie->queue_used)) { 1294 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1295 "queue %d not used", queue); 1296 return; 1297 } 1298 1299 iwl_pcie_gen2_txq_unmap(trans, queue); 1300 1301 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", queue); 1302 } 1303 1304 void iwl_pcie_gen2_tx_free(struct iwl_trans *trans) 1305 { 1306 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1307 int i; 1308 1309 memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); 1310 1311 /* Free all TX queues */ 1312 for (i = 0; i < ARRAY_SIZE(trans_pcie->txq); i++) { 1313 if (!trans_pcie->txq[i]) 1314 continue; 1315 1316 iwl_pcie_gen2_txq_free(trans, i); 1317 } 1318 } 1319 1320 int iwl_pcie_gen2_tx_init(struct iwl_trans *trans, int txq_id, int queue_size) 1321 { 1322 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1323 struct iwl_txq *queue; 1324 int ret; 1325 1326 /* alloc and init the tx queue */ 1327 if (!trans_pcie->txq[txq_id]) { 1328 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 1329 if (!queue) { 1330 IWL_ERR(trans, "Not enough memory for tx queue\n"); 1331 return -ENOMEM; 1332 } 1333 trans_pcie->txq[txq_id] = queue; 1334 ret = iwl_pcie_txq_alloc(trans, queue, queue_size, true); 1335 if (ret) { 1336 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 1337 goto error; 1338 } 1339 } else { 1340 queue = trans_pcie->txq[txq_id]; 1341 } 1342 1343 ret = iwl_pcie_txq_init(trans, queue, queue_size, 1344 (txq_id == trans_pcie->cmd_queue)); 1345 if (ret) { 1346 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 1347 goto error; 1348 } 1349 trans_pcie->txq[txq_id]->id = txq_id; 1350 set_bit(txq_id, trans_pcie->queue_used); 1351 1352 return 0; 1353 1354 error: 1355 iwl_pcie_gen2_tx_free(trans); 1356 return ret; 1357 } 1358 1359