1 /****************************************************************************** 2 * 3 * This file is provided under a dual BSD/GPLv2 license. When using or 4 * redistributing this file, you may do so under either license. 5 * 6 * GPL LICENSE SUMMARY 7 * 8 * Copyright(c) 2017 Intel Deutschland GmbH 9 * Copyright(c) 2018 - 2019 Intel Corporation 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * BSD LICENSE 21 * 22 * Copyright(c) 2017 Intel Deutschland GmbH 23 * Copyright(c) 2018 - 2019 Intel Corporation 24 * All rights reserved. 25 * 26 * Redistribution and use in source and binary forms, with or without 27 * modification, are permitted provided that the following conditions 28 * are met: 29 * 30 * * Redistributions of source code must retain the above copyright 31 * notice, this list of conditions and the following disclaimer. 32 * * Redistributions in binary form must reproduce the above copyright 33 * notice, this list of conditions and the following disclaimer in 34 * the documentation and/or other materials provided with the 35 * distribution. 36 * * Neither the name Intel Corporation nor the names of its 37 * contributors may be used to endorse or promote products derived 38 * from this software without specific prior written permission. 39 * 40 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 41 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 42 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 43 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 44 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 45 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 46 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 47 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 48 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 49 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 50 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 51 * 52 *****************************************************************************/ 53 #include <net/tso.h> 54 #include <linux/tcp.h> 55 56 #include "iwl-debug.h" 57 #include "iwl-csr.h" 58 #include "iwl-io.h" 59 #include "internal.h" 60 #include "fw/api/tx.h" 61 62 /* 63 * iwl_pcie_gen2_tx_stop - Stop all Tx DMA channels 64 */ 65 void iwl_pcie_gen2_tx_stop(struct iwl_trans *trans) 66 { 67 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 68 int txq_id; 69 70 /* 71 * This function can be called before the op_mode disabled the 72 * queues. This happens when we have an rfkill interrupt. 73 * Since we stop Tx altogether - mark the queues as stopped. 74 */ 75 memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped)); 76 memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); 77 78 /* Unmap DMA from host system and free skb's */ 79 for (txq_id = 0; txq_id < ARRAY_SIZE(trans_pcie->txq); txq_id++) { 80 if (!trans_pcie->txq[txq_id]) 81 continue; 82 iwl_pcie_gen2_txq_unmap(trans, txq_id); 83 } 84 } 85 86 /* 87 * iwl_pcie_txq_update_byte_tbl - Set up entry in Tx byte-count array 88 */ 89 static void iwl_pcie_gen2_update_byte_tbl(struct iwl_trans_pcie *trans_pcie, 90 struct iwl_txq *txq, u16 byte_cnt, 91 int num_tbs) 92 { 93 struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr; 94 struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie); 95 struct iwl_gen3_bc_tbl *scd_bc_tbl_gen3 = txq->bc_tbl.addr; 96 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 97 u8 filled_tfd_size, num_fetch_chunks; 98 u16 len = byte_cnt; 99 __le16 bc_ent; 100 101 if (WARN(idx >= txq->n_window, "%d >= %d\n", idx, txq->n_window)) 102 return; 103 104 filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) + 105 num_tbs * sizeof(struct iwl_tfh_tb); 106 /* 107 * filled_tfd_size contains the number of filled bytes in the TFD. 108 * Dividing it by 64 will give the number of chunks to fetch 109 * to SRAM- 0 for one chunk, 1 for 2 and so on. 110 * If, for example, TFD contains only 3 TBs then 32 bytes 111 * of the TFD are used, and only one chunk of 64 bytes should 112 * be fetched 113 */ 114 num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1; 115 116 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 117 /* Starting from AX210, the HW expects bytes */ 118 WARN_ON(trans_pcie->bc_table_dword); 119 WARN_ON(len > 0x3FFF); 120 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 14)); 121 scd_bc_tbl_gen3->tfd_offset[idx] = bc_ent; 122 } else { 123 /* Before AX210, the HW expects DW */ 124 WARN_ON(!trans_pcie->bc_table_dword); 125 len = DIV_ROUND_UP(len, 4); 126 WARN_ON(len > 0xFFF); 127 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12)); 128 scd_bc_tbl->tfd_offset[idx] = bc_ent; 129 } 130 } 131 132 /* 133 * iwl_pcie_gen2_txq_inc_wr_ptr - Send new write index to hardware 134 */ 135 void iwl_pcie_gen2_txq_inc_wr_ptr(struct iwl_trans *trans, 136 struct iwl_txq *txq) 137 { 138 lockdep_assert_held(&txq->lock); 139 140 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr); 141 142 /* 143 * if not in power-save mode, uCode will never sleep when we're 144 * trying to tx (during RFKILL, we're not trying to tx). 145 */ 146 iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16)); 147 } 148 149 static u8 iwl_pcie_gen2_get_num_tbs(struct iwl_trans *trans, 150 struct iwl_tfh_tfd *tfd) 151 { 152 return le16_to_cpu(tfd->num_tbs) & 0x1f; 153 } 154 155 static void iwl_pcie_gen2_tfd_unmap(struct iwl_trans *trans, 156 struct iwl_cmd_meta *meta, 157 struct iwl_tfh_tfd *tfd) 158 { 159 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 160 int i, num_tbs; 161 162 /* Sanity check on number of chunks */ 163 num_tbs = iwl_pcie_gen2_get_num_tbs(trans, tfd); 164 165 if (num_tbs > trans_pcie->max_tbs) { 166 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 167 return; 168 } 169 170 /* first TB is never freed - it's the bidirectional DMA data */ 171 for (i = 1; i < num_tbs; i++) { 172 if (meta->tbs & BIT(i)) 173 dma_unmap_page(trans->dev, 174 le64_to_cpu(tfd->tbs[i].addr), 175 le16_to_cpu(tfd->tbs[i].tb_len), 176 DMA_TO_DEVICE); 177 else 178 dma_unmap_single(trans->dev, 179 le64_to_cpu(tfd->tbs[i].addr), 180 le16_to_cpu(tfd->tbs[i].tb_len), 181 DMA_TO_DEVICE); 182 } 183 184 tfd->num_tbs = 0; 185 } 186 187 static void iwl_pcie_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 188 { 189 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 190 * idx is bounded by n_window 191 */ 192 int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr); 193 194 lockdep_assert_held(&txq->lock); 195 196 iwl_pcie_gen2_tfd_unmap(trans, &txq->entries[idx].meta, 197 iwl_pcie_get_tfd(trans, txq, idx)); 198 199 /* free SKB */ 200 if (txq->entries) { 201 struct sk_buff *skb; 202 203 skb = txq->entries[idx].skb; 204 205 /* Can be called from irqs-disabled context 206 * If skb is not NULL, it means that the whole queue is being 207 * freed and that the queue is not empty - free the skb 208 */ 209 if (skb) { 210 iwl_op_mode_free_skb(trans->op_mode, skb); 211 txq->entries[idx].skb = NULL; 212 } 213 } 214 } 215 216 static int iwl_pcie_gen2_set_tb(struct iwl_trans *trans, 217 struct iwl_tfh_tfd *tfd, dma_addr_t addr, 218 u16 len) 219 { 220 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 221 int idx = iwl_pcie_gen2_get_num_tbs(trans, tfd); 222 struct iwl_tfh_tb *tb; 223 224 if (WARN_ON(idx >= IWL_TFH_NUM_TBS)) 225 return -EINVAL; 226 tb = &tfd->tbs[idx]; 227 228 /* Each TFD can point to a maximum max_tbs Tx buffers */ 229 if (le16_to_cpu(tfd->num_tbs) >= trans_pcie->max_tbs) { 230 IWL_ERR(trans, "Error can not send more than %d chunks\n", 231 trans_pcie->max_tbs); 232 return -EINVAL; 233 } 234 235 put_unaligned_le64(addr, &tb->addr); 236 tb->tb_len = cpu_to_le16(len); 237 238 tfd->num_tbs = cpu_to_le16(idx + 1); 239 240 return idx; 241 } 242 243 static int iwl_pcie_gen2_build_amsdu(struct iwl_trans *trans, 244 struct sk_buff *skb, 245 struct iwl_tfh_tfd *tfd, int start_len, 246 u8 hdr_len, struct iwl_device_cmd *dev_cmd) 247 { 248 #ifdef CONFIG_INET 249 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 250 struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload; 251 struct ieee80211_hdr *hdr = (void *)skb->data; 252 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 253 unsigned int mss = skb_shinfo(skb)->gso_size; 254 u16 length, amsdu_pad; 255 u8 *start_hdr; 256 struct iwl_tso_hdr_page *hdr_page; 257 struct page **page_ptr; 258 struct tso_t tso; 259 260 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), 261 &dev_cmd->hdr, start_len, 0); 262 263 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 264 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 265 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len; 266 amsdu_pad = 0; 267 268 /* total amount of header we may need for this A-MSDU */ 269 hdr_room = DIV_ROUND_UP(total_len, mss) * 270 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)); 271 272 /* Our device supports 9 segments at most, it will fit in 1 page */ 273 hdr_page = get_page_hdr(trans, hdr_room); 274 if (!hdr_page) 275 return -ENOMEM; 276 277 get_page(hdr_page->page); 278 start_hdr = hdr_page->pos; 279 page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); 280 *page_ptr = hdr_page->page; 281 282 /* 283 * Pull the ieee80211 header to be able to use TSO core, 284 * we will restore it for the tx_status flow. 285 */ 286 skb_pull(skb, hdr_len); 287 288 /* 289 * Remove the length of all the headers that we don't actually 290 * have in the MPDU by themselves, but that we duplicate into 291 * all the different MSDUs inside the A-MSDU. 292 */ 293 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 294 295 tso_start(skb, &tso); 296 297 while (total_len) { 298 /* this is the data left for this subframe */ 299 unsigned int data_left = min_t(unsigned int, mss, total_len); 300 struct sk_buff *csum_skb = NULL; 301 unsigned int tb_len; 302 dma_addr_t tb_phys; 303 u8 *subf_hdrs_start = hdr_page->pos; 304 305 total_len -= data_left; 306 307 memset(hdr_page->pos, 0, amsdu_pad); 308 hdr_page->pos += amsdu_pad; 309 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 310 data_left)) & 0x3; 311 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 312 hdr_page->pos += ETH_ALEN; 313 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 314 hdr_page->pos += ETH_ALEN; 315 316 length = snap_ip_tcp_hdrlen + data_left; 317 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 318 hdr_page->pos += sizeof(length); 319 320 /* 321 * This will copy the SNAP as well which will be considered 322 * as MAC header. 323 */ 324 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 325 326 hdr_page->pos += snap_ip_tcp_hdrlen; 327 328 tb_len = hdr_page->pos - start_hdr; 329 tb_phys = dma_map_single(trans->dev, start_hdr, 330 tb_len, DMA_TO_DEVICE); 331 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 332 dev_kfree_skb(csum_skb); 333 goto out_err; 334 } 335 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len); 336 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, 337 tb_phys, tb_len); 338 /* add this subframe's headers' length to the tx_cmd */ 339 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 340 341 /* prepare the start_hdr for the next subframe */ 342 start_hdr = hdr_page->pos; 343 344 /* put the payload */ 345 while (data_left) { 346 tb_len = min_t(unsigned int, tso.size, data_left); 347 tb_phys = dma_map_single(trans->dev, tso.data, 348 tb_len, DMA_TO_DEVICE); 349 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { 350 dev_kfree_skb(csum_skb); 351 goto out_err; 352 } 353 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len); 354 trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data, 355 tb_phys, tb_len); 356 357 data_left -= tb_len; 358 tso_build_data(skb, &tso, tb_len); 359 } 360 } 361 362 /* re -add the WiFi header */ 363 skb_push(skb, hdr_len); 364 365 return 0; 366 367 out_err: 368 #endif 369 return -EINVAL; 370 } 371 372 static struct 373 iwl_tfh_tfd *iwl_pcie_gen2_build_tx_amsdu(struct iwl_trans *trans, 374 struct iwl_txq *txq, 375 struct iwl_device_cmd *dev_cmd, 376 struct sk_buff *skb, 377 struct iwl_cmd_meta *out_meta, 378 int hdr_len, 379 int tx_cmd_len) 380 { 381 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 382 struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); 383 dma_addr_t tb_phys; 384 int len; 385 void *tb1_addr; 386 387 tb_phys = iwl_pcie_get_first_tb_dma(txq, idx); 388 389 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 390 391 /* 392 * The second TB (tb1) points to the remainder of the TX command 393 * and the 802.11 header - dword aligned size 394 * (This calculation modifies the TX command, so do it before the 395 * setup of the first TB) 396 */ 397 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 398 IWL_FIRST_TB_SIZE; 399 400 /* do not align A-MSDU to dword as the subframe header aligns it */ 401 402 /* map the data for TB1 */ 403 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 404 tb_phys = dma_map_single(trans->dev, tb1_addr, len, DMA_TO_DEVICE); 405 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 406 goto out_err; 407 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, len); 408 409 if (iwl_pcie_gen2_build_amsdu(trans, skb, tfd, 410 len + IWL_FIRST_TB_SIZE, 411 hdr_len, dev_cmd)) 412 goto out_err; 413 414 /* building the A-MSDU might have changed this data, memcpy it now */ 415 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 416 return tfd; 417 418 out_err: 419 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 420 return NULL; 421 } 422 423 static int iwl_pcie_gen2_tx_add_frags(struct iwl_trans *trans, 424 struct sk_buff *skb, 425 struct iwl_tfh_tfd *tfd, 426 struct iwl_cmd_meta *out_meta) 427 { 428 int i; 429 430 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 431 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 432 dma_addr_t tb_phys; 433 int tb_idx; 434 435 if (!skb_frag_size(frag)) 436 continue; 437 438 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 439 skb_frag_size(frag), DMA_TO_DEVICE); 440 441 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 442 return -ENOMEM; 443 tb_idx = iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, 444 skb_frag_size(frag)); 445 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb_frag_address(frag), 446 tb_phys, skb_frag_size(frag)); 447 if (tb_idx < 0) 448 return tb_idx; 449 450 out_meta->tbs |= BIT(tb_idx); 451 } 452 453 return 0; 454 } 455 456 static struct 457 iwl_tfh_tfd *iwl_pcie_gen2_build_tx(struct iwl_trans *trans, 458 struct iwl_txq *txq, 459 struct iwl_device_cmd *dev_cmd, 460 struct sk_buff *skb, 461 struct iwl_cmd_meta *out_meta, 462 int hdr_len, 463 int tx_cmd_len, 464 bool pad) 465 { 466 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 467 struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); 468 dma_addr_t tb_phys; 469 int len, tb1_len, tb2_len; 470 void *tb1_addr; 471 struct sk_buff *frag; 472 473 tb_phys = iwl_pcie_get_first_tb_dma(txq, idx); 474 475 /* The first TB points to bi-directional DMA data */ 476 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 477 478 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 479 480 /* 481 * The second TB (tb1) points to the remainder of the TX command 482 * and the 802.11 header - dword aligned size 483 * (This calculation modifies the TX command, so do it before the 484 * setup of the first TB) 485 */ 486 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 487 IWL_FIRST_TB_SIZE; 488 489 if (pad) 490 tb1_len = ALIGN(len, 4); 491 else 492 tb1_len = len; 493 494 /* map the data for TB1 */ 495 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 496 tb_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 497 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 498 goto out_err; 499 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb1_len); 500 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), &dev_cmd->hdr, 501 IWL_FIRST_TB_SIZE + tb1_len, hdr_len); 502 503 /* set up TFD's third entry to point to remainder of skb's head */ 504 tb2_len = skb_headlen(skb) - hdr_len; 505 506 if (tb2_len > 0) { 507 tb_phys = dma_map_single(trans->dev, skb->data + hdr_len, 508 tb2_len, DMA_TO_DEVICE); 509 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 510 goto out_err; 511 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb2_len); 512 trace_iwlwifi_dev_tx_tb(trans->dev, skb, skb->data + hdr_len, 513 tb_phys, tb2_len); 514 } 515 516 if (iwl_pcie_gen2_tx_add_frags(trans, skb, tfd, out_meta)) 517 goto out_err; 518 519 skb_walk_frags(skb, frag) { 520 tb_phys = dma_map_single(trans->dev, frag->data, 521 skb_headlen(frag), DMA_TO_DEVICE); 522 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 523 goto out_err; 524 iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, skb_headlen(frag)); 525 trace_iwlwifi_dev_tx_tb(trans->dev, skb, frag->data, 526 tb_phys, skb_headlen(frag)); 527 if (iwl_pcie_gen2_tx_add_frags(trans, frag, tfd, out_meta)) 528 goto out_err; 529 } 530 531 return tfd; 532 533 out_err: 534 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 535 return NULL; 536 } 537 538 static 539 struct iwl_tfh_tfd *iwl_pcie_gen2_build_tfd(struct iwl_trans *trans, 540 struct iwl_txq *txq, 541 struct iwl_device_cmd *dev_cmd, 542 struct sk_buff *skb, 543 struct iwl_cmd_meta *out_meta) 544 { 545 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 546 int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 547 struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); 548 int len, hdr_len; 549 bool amsdu; 550 551 /* There must be data left over for TB1 or this code must be changed */ 552 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2) < IWL_FIRST_TB_SIZE); 553 554 memset(tfd, 0, sizeof(*tfd)); 555 556 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) 557 len = sizeof(struct iwl_tx_cmd_gen2); 558 else 559 len = sizeof(struct iwl_tx_cmd_gen3); 560 561 amsdu = ieee80211_is_data_qos(hdr->frame_control) && 562 (*ieee80211_get_qos_ctl(hdr) & 563 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 564 565 hdr_len = ieee80211_hdrlen(hdr->frame_control); 566 567 /* 568 * Only build A-MSDUs here if doing so by GSO, otherwise it may be 569 * an A-MSDU for other reasons, e.g. NAN or an A-MSDU having been 570 * built in the higher layers already. 571 */ 572 if (amsdu && skb_shinfo(skb)->gso_size) 573 return iwl_pcie_gen2_build_tx_amsdu(trans, txq, dev_cmd, skb, 574 out_meta, hdr_len, len); 575 576 return iwl_pcie_gen2_build_tx(trans, txq, dev_cmd, skb, out_meta, 577 hdr_len, len, !amsdu); 578 } 579 580 int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb, 581 struct iwl_device_cmd *dev_cmd, int txq_id) 582 { 583 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 584 struct iwl_cmd_meta *out_meta; 585 struct iwl_txq *txq = trans_pcie->txq[txq_id]; 586 u16 cmd_len; 587 int idx; 588 void *tfd; 589 590 if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used), 591 "TX on unused queue %d\n", txq_id)) 592 return -EINVAL; 593 594 if (skb_is_nonlinear(skb) && 595 skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS(trans_pcie) && 596 __skb_linearize(skb)) 597 return -ENOMEM; 598 599 spin_lock(&txq->lock); 600 601 if (iwl_queue_space(trans, txq) < txq->high_mark) { 602 iwl_stop_queue(trans, txq); 603 604 /* don't put the packet on the ring, if there is no room */ 605 if (unlikely(iwl_queue_space(trans, txq) < 3)) { 606 struct iwl_device_cmd **dev_cmd_ptr; 607 608 dev_cmd_ptr = (void *)((u8 *)skb->cb + 609 trans_pcie->dev_cmd_offs); 610 611 *dev_cmd_ptr = dev_cmd; 612 __skb_queue_tail(&txq->overflow_q, skb); 613 spin_unlock(&txq->lock); 614 return 0; 615 } 616 } 617 618 idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 619 620 /* Set up driver data for this TFD */ 621 txq->entries[idx].skb = skb; 622 txq->entries[idx].cmd = dev_cmd; 623 624 dev_cmd->hdr.sequence = 625 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 626 INDEX_TO_SEQ(idx))); 627 628 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 629 out_meta = &txq->entries[idx].meta; 630 out_meta->flags = 0; 631 632 tfd = iwl_pcie_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta); 633 if (!tfd) { 634 spin_unlock(&txq->lock); 635 return -1; 636 } 637 638 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 639 struct iwl_tx_cmd_gen3 *tx_cmd_gen3 = 640 (void *)dev_cmd->payload; 641 642 cmd_len = le16_to_cpu(tx_cmd_gen3->len); 643 } else { 644 struct iwl_tx_cmd_gen2 *tx_cmd_gen2 = 645 (void *)dev_cmd->payload; 646 647 cmd_len = le16_to_cpu(tx_cmd_gen2->len); 648 } 649 650 /* Set up entry for this TFD in Tx byte-count array */ 651 iwl_pcie_gen2_update_byte_tbl(trans_pcie, txq, cmd_len, 652 iwl_pcie_gen2_get_num_tbs(trans, tfd)); 653 654 /* start timer if queue currently empty */ 655 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 656 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 657 658 /* Tell device the write index *just past* this latest filled TFD */ 659 txq->write_ptr = iwl_queue_inc_wrap(trans, txq->write_ptr); 660 iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq); 661 /* 662 * At this point the frame is "transmitted" successfully 663 * and we will get a TX status notification eventually. 664 */ 665 spin_unlock(&txq->lock); 666 return 0; 667 } 668 669 /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 670 671 /* 672 * iwl_pcie_gen2_enqueue_hcmd - enqueue a uCode command 673 * @priv: device private data point 674 * @cmd: a pointer to the ucode command structure 675 * 676 * The function returns < 0 values to indicate the operation 677 * failed. On success, it returns the index (>= 0) of command in the 678 * command queue. 679 */ 680 static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, 681 struct iwl_host_cmd *cmd) 682 { 683 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 684 struct iwl_txq *txq = trans_pcie->txq[trans_pcie->cmd_queue]; 685 struct iwl_device_cmd *out_cmd; 686 struct iwl_cmd_meta *out_meta; 687 unsigned long flags; 688 void *dup_buf = NULL; 689 dma_addr_t phys_addr; 690 int i, cmd_pos, idx; 691 u16 copy_size, cmd_size, tb0_size; 692 bool had_nocopy = false; 693 u8 group_id = iwl_cmd_groupid(cmd->id); 694 const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; 695 u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; 696 struct iwl_tfh_tfd *tfd; 697 698 copy_size = sizeof(struct iwl_cmd_header_wide); 699 cmd_size = sizeof(struct iwl_cmd_header_wide); 700 701 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 702 cmddata[i] = cmd->data[i]; 703 cmdlen[i] = cmd->len[i]; 704 705 if (!cmd->len[i]) 706 continue; 707 708 /* need at least IWL_FIRST_TB_SIZE copied */ 709 if (copy_size < IWL_FIRST_TB_SIZE) { 710 int copy = IWL_FIRST_TB_SIZE - copy_size; 711 712 if (copy > cmdlen[i]) 713 copy = cmdlen[i]; 714 cmdlen[i] -= copy; 715 cmddata[i] += copy; 716 copy_size += copy; 717 } 718 719 if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { 720 had_nocopy = true; 721 if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { 722 idx = -EINVAL; 723 goto free_dup_buf; 724 } 725 } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { 726 /* 727 * This is also a chunk that isn't copied 728 * to the static buffer so set had_nocopy. 729 */ 730 had_nocopy = true; 731 732 /* only allowed once */ 733 if (WARN_ON(dup_buf)) { 734 idx = -EINVAL; 735 goto free_dup_buf; 736 } 737 738 dup_buf = kmemdup(cmddata[i], cmdlen[i], 739 GFP_ATOMIC); 740 if (!dup_buf) 741 return -ENOMEM; 742 } else { 743 /* NOCOPY must not be followed by normal! */ 744 if (WARN_ON(had_nocopy)) { 745 idx = -EINVAL; 746 goto free_dup_buf; 747 } 748 copy_size += cmdlen[i]; 749 } 750 cmd_size += cmd->len[i]; 751 } 752 753 /* 754 * If any of the command structures end up being larger than the 755 * TFD_MAX_PAYLOAD_SIZE and they aren't dynamically allocated into 756 * separate TFDs, then we will need to increase the size of the buffers 757 */ 758 if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, 759 "Command %s (%#x) is too large (%d bytes)\n", 760 iwl_get_cmd_string(trans, cmd->id), cmd->id, copy_size)) { 761 idx = -EINVAL; 762 goto free_dup_buf; 763 } 764 765 spin_lock_bh(&txq->lock); 766 767 idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); 768 tfd = iwl_pcie_get_tfd(trans, txq, txq->write_ptr); 769 memset(tfd, 0, sizeof(*tfd)); 770 771 if (iwl_queue_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { 772 spin_unlock_bh(&txq->lock); 773 774 IWL_ERR(trans, "No space in command queue\n"); 775 iwl_op_mode_cmd_queue_full(trans->op_mode); 776 idx = -ENOSPC; 777 goto free_dup_buf; 778 } 779 780 out_cmd = txq->entries[idx].cmd; 781 out_meta = &txq->entries[idx].meta; 782 783 /* re-initialize to NULL */ 784 memset(out_meta, 0, sizeof(*out_meta)); 785 if (cmd->flags & CMD_WANT_SKB) 786 out_meta->source = cmd; 787 788 /* set up the header */ 789 out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); 790 out_cmd->hdr_wide.group_id = group_id; 791 out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); 792 out_cmd->hdr_wide.length = 793 cpu_to_le16(cmd_size - sizeof(struct iwl_cmd_header_wide)); 794 out_cmd->hdr_wide.reserved = 0; 795 out_cmd->hdr_wide.sequence = 796 cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | 797 INDEX_TO_SEQ(txq->write_ptr)); 798 799 cmd_pos = sizeof(struct iwl_cmd_header_wide); 800 copy_size = sizeof(struct iwl_cmd_header_wide); 801 802 /* and copy the data that needs to be copied */ 803 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 804 int copy; 805 806 if (!cmd->len[i]) 807 continue; 808 809 /* copy everything if not nocopy/dup */ 810 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 811 IWL_HCMD_DFL_DUP))) { 812 copy = cmd->len[i]; 813 814 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 815 cmd_pos += copy; 816 copy_size += copy; 817 continue; 818 } 819 820 /* 821 * Otherwise we need at least IWL_FIRST_TB_SIZE copied 822 * in total (for bi-directional DMA), but copy up to what 823 * we can fit into the payload for debug dump purposes. 824 */ 825 copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); 826 827 memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); 828 cmd_pos += copy; 829 830 /* However, treat copy_size the proper way, we need it below */ 831 if (copy_size < IWL_FIRST_TB_SIZE) { 832 copy = IWL_FIRST_TB_SIZE - copy_size; 833 834 if (copy > cmd->len[i]) 835 copy = cmd->len[i]; 836 copy_size += copy; 837 } 838 } 839 840 IWL_DEBUG_HC(trans, 841 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", 842 iwl_get_cmd_string(trans, cmd->id), group_id, 843 out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), 844 cmd_size, txq->write_ptr, idx, trans_pcie->cmd_queue); 845 846 /* start the TFD with the minimum copy bytes */ 847 tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); 848 memcpy(&txq->first_tb_bufs[idx], out_cmd, tb0_size); 849 iwl_pcie_gen2_set_tb(trans, tfd, iwl_pcie_get_first_tb_dma(txq, idx), 850 tb0_size); 851 852 /* map first command fragment, if any remains */ 853 if (copy_size > tb0_size) { 854 phys_addr = dma_map_single(trans->dev, 855 (u8 *)out_cmd + tb0_size, 856 copy_size - tb0_size, 857 DMA_TO_DEVICE); 858 if (dma_mapping_error(trans->dev, phys_addr)) { 859 idx = -ENOMEM; 860 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 861 goto out; 862 } 863 iwl_pcie_gen2_set_tb(trans, tfd, phys_addr, 864 copy_size - tb0_size); 865 } 866 867 /* map the remaining (adjusted) nocopy/dup fragments */ 868 for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { 869 const void *data = cmddata[i]; 870 871 if (!cmdlen[i]) 872 continue; 873 if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | 874 IWL_HCMD_DFL_DUP))) 875 continue; 876 if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) 877 data = dup_buf; 878 phys_addr = dma_map_single(trans->dev, (void *)data, 879 cmdlen[i], DMA_TO_DEVICE); 880 if (dma_mapping_error(trans->dev, phys_addr)) { 881 idx = -ENOMEM; 882 iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); 883 goto out; 884 } 885 iwl_pcie_gen2_set_tb(trans, tfd, phys_addr, cmdlen[i]); 886 } 887 888 BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); 889 out_meta->flags = cmd->flags; 890 if (WARN_ON_ONCE(txq->entries[idx].free_buf)) 891 kzfree(txq->entries[idx].free_buf); 892 txq->entries[idx].free_buf = dup_buf; 893 894 trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); 895 896 /* start timer if queue currently empty */ 897 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 898 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 899 900 spin_lock_irqsave(&trans_pcie->reg_lock, flags); 901 /* Increment and update queue's write index */ 902 txq->write_ptr = iwl_queue_inc_wrap(trans, txq->write_ptr); 903 iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq); 904 spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); 905 906 out: 907 spin_unlock_bh(&txq->lock); 908 free_dup_buf: 909 if (idx < 0) 910 kfree(dup_buf); 911 return idx; 912 } 913 914 #define HOST_COMPLETE_TIMEOUT (2 * HZ) 915 916 static int iwl_pcie_gen2_send_hcmd_sync(struct iwl_trans *trans, 917 struct iwl_host_cmd *cmd) 918 { 919 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 920 const char *cmd_str = iwl_get_cmd_string(trans, cmd->id); 921 struct iwl_txq *txq = trans_pcie->txq[trans_pcie->cmd_queue]; 922 int cmd_idx; 923 int ret; 924 925 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", cmd_str); 926 927 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 928 &trans->status), 929 "Command %s: a command is already active!\n", cmd_str)) 930 return -EIO; 931 932 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", cmd_str); 933 934 cmd_idx = iwl_pcie_gen2_enqueue_hcmd(trans, cmd); 935 if (cmd_idx < 0) { 936 ret = cmd_idx; 937 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 938 IWL_ERR(trans, "Error sending %s: enqueue_hcmd failed: %d\n", 939 cmd_str, ret); 940 return ret; 941 } 942 943 ret = wait_event_timeout(trans_pcie->wait_command_queue, 944 !test_bit(STATUS_SYNC_HCMD_ACTIVE, 945 &trans->status), 946 HOST_COMPLETE_TIMEOUT); 947 if (!ret) { 948 IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 949 cmd_str, jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 950 951 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 952 txq->read_ptr, txq->write_ptr); 953 954 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 955 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 956 cmd_str); 957 ret = -ETIMEDOUT; 958 959 iwl_trans_pcie_sync_nmi(trans); 960 goto cancel; 961 } 962 963 if (test_bit(STATUS_FW_ERROR, &trans->status)) { 964 IWL_ERR(trans, "FW error in SYNC CMD %s\n", cmd_str); 965 dump_stack(); 966 ret = -EIO; 967 goto cancel; 968 } 969 970 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 971 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 972 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 973 ret = -ERFKILL; 974 goto cancel; 975 } 976 977 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 978 IWL_ERR(trans, "Error: Response NULL in '%s'\n", cmd_str); 979 ret = -EIO; 980 goto cancel; 981 } 982 983 return 0; 984 985 cancel: 986 if (cmd->flags & CMD_WANT_SKB) { 987 /* 988 * Cancel the CMD_WANT_SKB flag for the cmd in the 989 * TX cmd queue. Otherwise in case the cmd comes 990 * in later, it will possibly set an invalid 991 * address (cmd->meta.source). 992 */ 993 txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 994 } 995 996 if (cmd->resp_pkt) { 997 iwl_free_resp(cmd); 998 cmd->resp_pkt = NULL; 999 } 1000 1001 return ret; 1002 } 1003 1004 int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans *trans, 1005 struct iwl_host_cmd *cmd) 1006 { 1007 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1008 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1009 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1010 cmd->id); 1011 return -ERFKILL; 1012 } 1013 1014 if (cmd->flags & CMD_ASYNC) { 1015 int ret; 1016 1017 /* An asynchronous command can not expect an SKB to be set. */ 1018 if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1019 return -EINVAL; 1020 1021 ret = iwl_pcie_gen2_enqueue_hcmd(trans, cmd); 1022 if (ret < 0) { 1023 IWL_ERR(trans, 1024 "Error sending %s: enqueue_hcmd failed: %d\n", 1025 iwl_get_cmd_string(trans, cmd->id), ret); 1026 return ret; 1027 } 1028 return 0; 1029 } 1030 1031 return iwl_pcie_gen2_send_hcmd_sync(trans, cmd); 1032 } 1033 1034 /* 1035 * iwl_pcie_gen2_txq_unmap - Unmap any remaining DMA mappings and free skb's 1036 */ 1037 void iwl_pcie_gen2_txq_unmap(struct iwl_trans *trans, int txq_id) 1038 { 1039 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1040 struct iwl_txq *txq = trans_pcie->txq[txq_id]; 1041 1042 spin_lock_bh(&txq->lock); 1043 while (txq->write_ptr != txq->read_ptr) { 1044 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 1045 txq_id, txq->read_ptr); 1046 1047 if (txq_id != trans_pcie->cmd_queue) { 1048 int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr); 1049 struct sk_buff *skb = txq->entries[idx].skb; 1050 1051 if (WARN_ON_ONCE(!skb)) 1052 continue; 1053 1054 iwl_pcie_free_tso_page(trans_pcie, skb); 1055 } 1056 iwl_pcie_gen2_free_tfd(trans, txq); 1057 txq->read_ptr = iwl_queue_inc_wrap(trans, txq->read_ptr); 1058 } 1059 1060 while (!skb_queue_empty(&txq->overflow_q)) { 1061 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 1062 1063 iwl_op_mode_free_skb(trans->op_mode, skb); 1064 } 1065 1066 spin_unlock_bh(&txq->lock); 1067 1068 /* just in case - this queue may have been stopped */ 1069 iwl_wake_queue(trans, txq); 1070 } 1071 1072 void iwl_pcie_gen2_txq_free_memory(struct iwl_trans *trans, 1073 struct iwl_txq *txq) 1074 { 1075 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1076 struct device *dev = trans->dev; 1077 1078 /* De-alloc circular buffer of TFDs */ 1079 if (txq->tfds) { 1080 dma_free_coherent(dev, 1081 trans_pcie->tfd_size * txq->n_window, 1082 txq->tfds, txq->dma_addr); 1083 dma_free_coherent(dev, 1084 sizeof(*txq->first_tb_bufs) * txq->n_window, 1085 txq->first_tb_bufs, txq->first_tb_dma); 1086 } 1087 1088 kfree(txq->entries); 1089 iwl_pcie_free_dma_ptr(trans, &txq->bc_tbl); 1090 kfree(txq); 1091 } 1092 1093 /* 1094 * iwl_pcie_txq_free - Deallocate DMA queue. 1095 * @txq: Transmit queue to deallocate. 1096 * 1097 * Empty queue by removing and destroying all BD's. 1098 * Free all buffers. 1099 * 0-fill, but do not free "txq" descriptor structure. 1100 */ 1101 static void iwl_pcie_gen2_txq_free(struct iwl_trans *trans, int txq_id) 1102 { 1103 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1104 struct iwl_txq *txq = trans_pcie->txq[txq_id]; 1105 int i; 1106 1107 if (WARN_ON(!txq)) 1108 return; 1109 1110 iwl_pcie_gen2_txq_unmap(trans, txq_id); 1111 1112 /* De-alloc array of command/tx buffers */ 1113 if (txq_id == trans_pcie->cmd_queue) 1114 for (i = 0; i < txq->n_window; i++) { 1115 kzfree(txq->entries[i].cmd); 1116 kzfree(txq->entries[i].free_buf); 1117 } 1118 del_timer_sync(&txq->stuck_timer); 1119 1120 iwl_pcie_gen2_txq_free_memory(trans, txq); 1121 1122 trans_pcie->txq[txq_id] = NULL; 1123 1124 clear_bit(txq_id, trans_pcie->queue_used); 1125 } 1126 1127 int iwl_trans_pcie_dyn_txq_alloc_dma(struct iwl_trans *trans, 1128 struct iwl_txq **intxq, int size, 1129 unsigned int timeout) 1130 { 1131 int ret; 1132 1133 struct iwl_txq *txq; 1134 txq = kzalloc(sizeof(*txq), GFP_KERNEL); 1135 if (!txq) 1136 return -ENOMEM; 1137 ret = iwl_pcie_alloc_dma_ptr(trans, &txq->bc_tbl, 1138 (trans->trans_cfg->device_family >= 1139 IWL_DEVICE_FAMILY_AX210) ? 1140 sizeof(struct iwl_gen3_bc_tbl) : 1141 sizeof(struct iwlagn_scd_bc_tbl)); 1142 if (ret) { 1143 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 1144 kfree(txq); 1145 return -ENOMEM; 1146 } 1147 1148 ret = iwl_pcie_txq_alloc(trans, txq, size, false); 1149 if (ret) { 1150 IWL_ERR(trans, "Tx queue alloc failed\n"); 1151 goto error; 1152 } 1153 ret = iwl_pcie_txq_init(trans, txq, size, false); 1154 if (ret) { 1155 IWL_ERR(trans, "Tx queue init failed\n"); 1156 goto error; 1157 } 1158 1159 txq->wd_timeout = msecs_to_jiffies(timeout); 1160 1161 *intxq = txq; 1162 return 0; 1163 1164 error: 1165 iwl_pcie_gen2_txq_free_memory(trans, txq); 1166 return ret; 1167 } 1168 1169 int iwl_trans_pcie_txq_alloc_response(struct iwl_trans *trans, 1170 struct iwl_txq *txq, 1171 struct iwl_host_cmd *hcmd) 1172 { 1173 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1174 struct iwl_tx_queue_cfg_rsp *rsp; 1175 int ret, qid; 1176 u32 wr_ptr; 1177 1178 if (WARN_ON(iwl_rx_packet_payload_len(hcmd->resp_pkt) != 1179 sizeof(*rsp))) { 1180 ret = -EINVAL; 1181 goto error_free_resp; 1182 } 1183 1184 rsp = (void *)hcmd->resp_pkt->data; 1185 qid = le16_to_cpu(rsp->queue_number); 1186 wr_ptr = le16_to_cpu(rsp->write_pointer); 1187 1188 if (qid >= ARRAY_SIZE(trans_pcie->txq)) { 1189 WARN_ONCE(1, "queue index %d unsupported", qid); 1190 ret = -EIO; 1191 goto error_free_resp; 1192 } 1193 1194 if (test_and_set_bit(qid, trans_pcie->queue_used)) { 1195 WARN_ONCE(1, "queue %d already used", qid); 1196 ret = -EIO; 1197 goto error_free_resp; 1198 } 1199 1200 txq->id = qid; 1201 trans_pcie->txq[qid] = txq; 1202 wr_ptr &= (trans->trans_cfg->base_params->max_tfd_queue_size - 1); 1203 1204 /* Place first TFD at index corresponding to start sequence number */ 1205 txq->read_ptr = wr_ptr; 1206 txq->write_ptr = wr_ptr; 1207 1208 IWL_DEBUG_TX_QUEUES(trans, "Activate queue %d\n", qid); 1209 1210 iwl_free_resp(hcmd); 1211 return qid; 1212 1213 error_free_resp: 1214 iwl_free_resp(hcmd); 1215 iwl_pcie_gen2_txq_free_memory(trans, txq); 1216 return ret; 1217 } 1218 1219 int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans, 1220 __le16 flags, u8 sta_id, u8 tid, 1221 int cmd_id, int size, 1222 unsigned int timeout) 1223 { 1224 struct iwl_txq *txq = NULL; 1225 struct iwl_tx_queue_cfg_cmd cmd = { 1226 .flags = flags, 1227 .sta_id = sta_id, 1228 .tid = tid, 1229 }; 1230 struct iwl_host_cmd hcmd = { 1231 .id = cmd_id, 1232 .len = { sizeof(cmd) }, 1233 .data = { &cmd, }, 1234 .flags = CMD_WANT_SKB, 1235 }; 1236 int ret; 1237 1238 ret = iwl_trans_pcie_dyn_txq_alloc_dma(trans, &txq, size, timeout); 1239 if (ret) 1240 return ret; 1241 1242 cmd.tfdq_addr = cpu_to_le64(txq->dma_addr); 1243 cmd.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma); 1244 cmd.cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(size)); 1245 1246 ret = iwl_trans_send_cmd(trans, &hcmd); 1247 if (ret) 1248 goto error; 1249 1250 return iwl_trans_pcie_txq_alloc_response(trans, txq, &hcmd); 1251 1252 error: 1253 iwl_pcie_gen2_txq_free_memory(trans, txq); 1254 return ret; 1255 } 1256 1257 void iwl_trans_pcie_dyn_txq_free(struct iwl_trans *trans, int queue) 1258 { 1259 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1260 1261 /* 1262 * Upon HW Rfkill - we stop the device, and then stop the queues 1263 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1264 * allow the op_mode to call txq_disable after it already called 1265 * stop_device. 1266 */ 1267 if (!test_and_clear_bit(queue, trans_pcie->queue_used)) { 1268 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1269 "queue %d not used", queue); 1270 return; 1271 } 1272 1273 iwl_pcie_gen2_txq_unmap(trans, queue); 1274 1275 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", queue); 1276 } 1277 1278 void iwl_pcie_gen2_tx_free(struct iwl_trans *trans) 1279 { 1280 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1281 int i; 1282 1283 memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); 1284 1285 /* Free all TX queues */ 1286 for (i = 0; i < ARRAY_SIZE(trans_pcie->txq); i++) { 1287 if (!trans_pcie->txq[i]) 1288 continue; 1289 1290 iwl_pcie_gen2_txq_free(trans, i); 1291 } 1292 } 1293 1294 int iwl_pcie_gen2_tx_init(struct iwl_trans *trans, int txq_id, int queue_size) 1295 { 1296 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1297 struct iwl_txq *queue; 1298 int ret; 1299 1300 /* alloc and init the tx queue */ 1301 if (!trans_pcie->txq[txq_id]) { 1302 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 1303 if (!queue) { 1304 IWL_ERR(trans, "Not enough memory for tx queue\n"); 1305 return -ENOMEM; 1306 } 1307 trans_pcie->txq[txq_id] = queue; 1308 ret = iwl_pcie_txq_alloc(trans, queue, queue_size, true); 1309 if (ret) { 1310 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 1311 goto error; 1312 } 1313 } else { 1314 queue = trans_pcie->txq[txq_id]; 1315 } 1316 1317 ret = iwl_pcie_txq_init(trans, queue, queue_size, 1318 (txq_id == trans_pcie->cmd_queue)); 1319 if (ret) { 1320 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 1321 goto error; 1322 } 1323 trans_pcie->txq[txq_id]->id = txq_id; 1324 set_bit(txq_id, trans_pcie->queue_used); 1325 1326 return 0; 1327 1328 error: 1329 iwl_pcie_gen2_tx_free(trans); 1330 return ret; 1331 } 1332 1333