1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 #include <net/tso.h> 6 #include <linux/tcp.h> 7 8 #include "iwl-debug.h" 9 #include "iwl-io.h" 10 #include "fw/api/commands.h" 11 #include "fw/api/tx.h" 12 #include "fw/api/datapath.h" 13 #include "queue/tx.h" 14 #include "iwl-fh.h" 15 #include "iwl-scd.h" 16 #include <linux/dmapool.h> 17 18 /* 19 * iwl_txq_update_byte_tbl - Set up entry in Tx byte-count array 20 */ 21 static void iwl_pcie_gen2_update_byte_tbl(struct iwl_trans *trans, 22 struct iwl_txq *txq, u16 byte_cnt, 23 int num_tbs) 24 { 25 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 26 u8 filled_tfd_size, num_fetch_chunks; 27 u16 len = byte_cnt; 28 __le16 bc_ent; 29 30 if (WARN(idx >= txq->n_window, "%d >= %d\n", idx, txq->n_window)) 31 return; 32 33 filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) + 34 num_tbs * sizeof(struct iwl_tfh_tb); 35 /* 36 * filled_tfd_size contains the number of filled bytes in the TFD. 37 * Dividing it by 64 will give the number of chunks to fetch 38 * to SRAM- 0 for one chunk, 1 for 2 and so on. 39 * If, for example, TFD contains only 3 TBs then 32 bytes 40 * of the TFD are used, and only one chunk of 64 bytes should 41 * be fetched 42 */ 43 num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1; 44 45 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 46 struct iwl_gen3_bc_tbl_entry *scd_bc_tbl_gen3 = txq->bc_tbl.addr; 47 48 /* Starting from AX210, the HW expects bytes */ 49 WARN_ON(trans->txqs.bc_table_dword); 50 WARN_ON(len > 0x3FFF); 51 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 14)); 52 scd_bc_tbl_gen3[idx].tfd_offset = bc_ent; 53 } else { 54 struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr; 55 56 /* Before AX210, the HW expects DW */ 57 WARN_ON(!trans->txqs.bc_table_dword); 58 len = DIV_ROUND_UP(len, 4); 59 WARN_ON(len > 0xFFF); 60 bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12)); 61 scd_bc_tbl->tfd_offset[idx] = bc_ent; 62 } 63 } 64 65 /* 66 * iwl_txq_inc_wr_ptr - Send new write index to hardware 67 */ 68 void iwl_txq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_txq *txq) 69 { 70 lockdep_assert_held(&txq->lock); 71 72 IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr); 73 74 /* 75 * if not in power-save mode, uCode will never sleep when we're 76 * trying to tx (during RFKILL, we're not trying to tx). 77 */ 78 iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16)); 79 } 80 81 static u8 iwl_txq_gen2_get_num_tbs(struct iwl_trans *trans, 82 struct iwl_tfh_tfd *tfd) 83 { 84 return le16_to_cpu(tfd->num_tbs) & 0x1f; 85 } 86 87 void iwl_txq_gen2_tfd_unmap(struct iwl_trans *trans, struct iwl_cmd_meta *meta, 88 struct iwl_tfh_tfd *tfd) 89 { 90 int i, num_tbs; 91 92 /* Sanity check on number of chunks */ 93 num_tbs = iwl_txq_gen2_get_num_tbs(trans, tfd); 94 95 if (num_tbs > trans->txqs.tfd.max_tbs) { 96 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 97 return; 98 } 99 100 /* first TB is never freed - it's the bidirectional DMA data */ 101 for (i = 1; i < num_tbs; i++) { 102 if (meta->tbs & BIT(i)) 103 dma_unmap_page(trans->dev, 104 le64_to_cpu(tfd->tbs[i].addr), 105 le16_to_cpu(tfd->tbs[i].tb_len), 106 DMA_TO_DEVICE); 107 else 108 dma_unmap_single(trans->dev, 109 le64_to_cpu(tfd->tbs[i].addr), 110 le16_to_cpu(tfd->tbs[i].tb_len), 111 DMA_TO_DEVICE); 112 } 113 114 tfd->num_tbs = 0; 115 } 116 117 void iwl_txq_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 118 { 119 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 120 * idx is bounded by n_window 121 */ 122 int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr); 123 struct sk_buff *skb; 124 125 lockdep_assert_held(&txq->lock); 126 127 if (!txq->entries) 128 return; 129 130 iwl_txq_gen2_tfd_unmap(trans, &txq->entries[idx].meta, 131 iwl_txq_get_tfd(trans, txq, idx)); 132 133 skb = txq->entries[idx].skb; 134 135 /* Can be called from irqs-disabled context 136 * If skb is not NULL, it means that the whole queue is being 137 * freed and that the queue is not empty - free the skb 138 */ 139 if (skb) { 140 iwl_op_mode_free_skb(trans->op_mode, skb); 141 txq->entries[idx].skb = NULL; 142 } 143 } 144 145 int iwl_txq_gen2_set_tb(struct iwl_trans *trans, struct iwl_tfh_tfd *tfd, 146 dma_addr_t addr, u16 len) 147 { 148 int idx = iwl_txq_gen2_get_num_tbs(trans, tfd); 149 struct iwl_tfh_tb *tb; 150 151 /* 152 * Only WARN here so we know about the issue, but we mess up our 153 * unmap path because not every place currently checks for errors 154 * returned from this function - it can only return an error if 155 * there's no more space, and so when we know there is enough we 156 * don't always check ... 157 */ 158 WARN(iwl_txq_crosses_4g_boundary(addr, len), 159 "possible DMA problem with iova:0x%llx, len:%d\n", 160 (unsigned long long)addr, len); 161 162 if (WARN_ON(idx >= IWL_TFH_NUM_TBS)) 163 return -EINVAL; 164 tb = &tfd->tbs[idx]; 165 166 /* Each TFD can point to a maximum max_tbs Tx buffers */ 167 if (le16_to_cpu(tfd->num_tbs) >= trans->txqs.tfd.max_tbs) { 168 IWL_ERR(trans, "Error can not send more than %d chunks\n", 169 trans->txqs.tfd.max_tbs); 170 return -EINVAL; 171 } 172 173 put_unaligned_le64(addr, &tb->addr); 174 tb->tb_len = cpu_to_le16(len); 175 176 tfd->num_tbs = cpu_to_le16(idx + 1); 177 178 return idx; 179 } 180 181 static struct page *get_workaround_page(struct iwl_trans *trans, 182 struct sk_buff *skb) 183 { 184 struct page **page_ptr; 185 struct page *ret; 186 187 page_ptr = (void *)((u8 *)skb->cb + trans->txqs.page_offs); 188 189 ret = alloc_page(GFP_ATOMIC); 190 if (!ret) 191 return NULL; 192 193 /* set the chaining pointer to the previous page if there */ 194 *(void **)((u8 *)page_address(ret) + PAGE_SIZE - sizeof(void *)) = *page_ptr; 195 *page_ptr = ret; 196 197 return ret; 198 } 199 200 /* 201 * Add a TB and if needed apply the FH HW bug workaround; 202 * meta != NULL indicates that it's a page mapping and we 203 * need to dma_unmap_page() and set the meta->tbs bit in 204 * this case. 205 */ 206 static int iwl_txq_gen2_set_tb_with_wa(struct iwl_trans *trans, 207 struct sk_buff *skb, 208 struct iwl_tfh_tfd *tfd, 209 dma_addr_t phys, void *virt, 210 u16 len, struct iwl_cmd_meta *meta) 211 { 212 dma_addr_t oldphys = phys; 213 struct page *page; 214 int ret; 215 216 if (unlikely(dma_mapping_error(trans->dev, phys))) 217 return -ENOMEM; 218 219 if (likely(!iwl_txq_crosses_4g_boundary(phys, len))) { 220 ret = iwl_txq_gen2_set_tb(trans, tfd, phys, len); 221 222 if (ret < 0) 223 goto unmap; 224 225 if (meta) 226 meta->tbs |= BIT(ret); 227 228 ret = 0; 229 goto trace; 230 } 231 232 /* 233 * Work around a hardware bug. If (as expressed in the 234 * condition above) the TB ends on a 32-bit boundary, 235 * then the next TB may be accessed with the wrong 236 * address. 237 * To work around it, copy the data elsewhere and make 238 * a new mapping for it so the device will not fail. 239 */ 240 241 if (WARN_ON(len > PAGE_SIZE - sizeof(void *))) { 242 ret = -ENOBUFS; 243 goto unmap; 244 } 245 246 page = get_workaround_page(trans, skb); 247 if (!page) { 248 ret = -ENOMEM; 249 goto unmap; 250 } 251 252 memcpy(page_address(page), virt, len); 253 254 phys = dma_map_single(trans->dev, page_address(page), len, 255 DMA_TO_DEVICE); 256 if (unlikely(dma_mapping_error(trans->dev, phys))) 257 return -ENOMEM; 258 ret = iwl_txq_gen2_set_tb(trans, tfd, phys, len); 259 if (ret < 0) { 260 /* unmap the new allocation as single */ 261 oldphys = phys; 262 meta = NULL; 263 goto unmap; 264 } 265 IWL_WARN(trans, 266 "TB bug workaround: copied %d bytes from 0x%llx to 0x%llx\n", 267 len, (unsigned long long)oldphys, (unsigned long long)phys); 268 269 ret = 0; 270 unmap: 271 if (meta) 272 dma_unmap_page(trans->dev, oldphys, len, DMA_TO_DEVICE); 273 else 274 dma_unmap_single(trans->dev, oldphys, len, DMA_TO_DEVICE); 275 trace: 276 trace_iwlwifi_dev_tx_tb(trans->dev, skb, virt, phys, len); 277 278 return ret; 279 } 280 281 #ifdef CONFIG_INET 282 struct iwl_tso_hdr_page *get_page_hdr(struct iwl_trans *trans, size_t len, 283 struct sk_buff *skb) 284 { 285 struct iwl_tso_hdr_page *p = this_cpu_ptr(trans->txqs.tso_hdr_page); 286 struct page **page_ptr; 287 288 page_ptr = (void *)((u8 *)skb->cb + trans->txqs.page_offs); 289 290 if (WARN_ON(*page_ptr)) 291 return NULL; 292 293 if (!p->page) 294 goto alloc; 295 296 /* 297 * Check if there's enough room on this page 298 * 299 * Note that we put a page chaining pointer *last* in the 300 * page - we need it somewhere, and if it's there then we 301 * avoid DMA mapping the last bits of the page which may 302 * trigger the 32-bit boundary hardware bug. 303 * 304 * (see also get_workaround_page() in tx-gen2.c) 305 */ 306 if (p->pos + len < (u8 *)page_address(p->page) + PAGE_SIZE - 307 sizeof(void *)) 308 goto out; 309 310 /* We don't have enough room on this page, get a new one. */ 311 __free_page(p->page); 312 313 alloc: 314 p->page = alloc_page(GFP_ATOMIC); 315 if (!p->page) 316 return NULL; 317 p->pos = page_address(p->page); 318 /* set the chaining pointer to NULL */ 319 *(void **)((u8 *)page_address(p->page) + PAGE_SIZE - sizeof(void *)) = NULL; 320 out: 321 *page_ptr = p->page; 322 get_page(p->page); 323 return p; 324 } 325 #endif 326 327 static int iwl_txq_gen2_build_amsdu(struct iwl_trans *trans, 328 struct sk_buff *skb, 329 struct iwl_tfh_tfd *tfd, int start_len, 330 u8 hdr_len, 331 struct iwl_device_tx_cmd *dev_cmd) 332 { 333 #ifdef CONFIG_INET 334 struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload; 335 struct ieee80211_hdr *hdr = (void *)skb->data; 336 unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; 337 unsigned int mss = skb_shinfo(skb)->gso_size; 338 u16 length, amsdu_pad; 339 u8 *start_hdr; 340 struct iwl_tso_hdr_page *hdr_page; 341 struct tso_t tso; 342 343 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), 344 &dev_cmd->hdr, start_len, 0); 345 346 ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); 347 snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); 348 total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len; 349 amsdu_pad = 0; 350 351 /* total amount of header we may need for this A-MSDU */ 352 hdr_room = DIV_ROUND_UP(total_len, mss) * 353 (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)); 354 355 /* Our device supports 9 segments at most, it will fit in 1 page */ 356 hdr_page = get_page_hdr(trans, hdr_room, skb); 357 if (!hdr_page) 358 return -ENOMEM; 359 360 start_hdr = hdr_page->pos; 361 362 /* 363 * Pull the ieee80211 header to be able to use TSO core, 364 * we will restore it for the tx_status flow. 365 */ 366 skb_pull(skb, hdr_len); 367 368 /* 369 * Remove the length of all the headers that we don't actually 370 * have in the MPDU by themselves, but that we duplicate into 371 * all the different MSDUs inside the A-MSDU. 372 */ 373 le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); 374 375 tso_start(skb, &tso); 376 377 while (total_len) { 378 /* this is the data left for this subframe */ 379 unsigned int data_left = min_t(unsigned int, mss, total_len); 380 unsigned int tb_len; 381 dma_addr_t tb_phys; 382 u8 *subf_hdrs_start = hdr_page->pos; 383 384 total_len -= data_left; 385 386 memset(hdr_page->pos, 0, amsdu_pad); 387 hdr_page->pos += amsdu_pad; 388 amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + 389 data_left)) & 0x3; 390 ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); 391 hdr_page->pos += ETH_ALEN; 392 ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); 393 hdr_page->pos += ETH_ALEN; 394 395 length = snap_ip_tcp_hdrlen + data_left; 396 *((__be16 *)hdr_page->pos) = cpu_to_be16(length); 397 hdr_page->pos += sizeof(length); 398 399 /* 400 * This will copy the SNAP as well which will be considered 401 * as MAC header. 402 */ 403 tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); 404 405 hdr_page->pos += snap_ip_tcp_hdrlen; 406 407 tb_len = hdr_page->pos - start_hdr; 408 tb_phys = dma_map_single(trans->dev, start_hdr, 409 tb_len, DMA_TO_DEVICE); 410 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 411 goto out_err; 412 /* 413 * No need for _with_wa, this is from the TSO page and 414 * we leave some space at the end of it so can't hit 415 * the buggy scenario. 416 */ 417 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, tb_len); 418 trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, 419 tb_phys, tb_len); 420 /* add this subframe's headers' length to the tx_cmd */ 421 le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); 422 423 /* prepare the start_hdr for the next subframe */ 424 start_hdr = hdr_page->pos; 425 426 /* put the payload */ 427 while (data_left) { 428 int ret; 429 430 tb_len = min_t(unsigned int, tso.size, data_left); 431 tb_phys = dma_map_single(trans->dev, tso.data, 432 tb_len, DMA_TO_DEVICE); 433 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, 434 tb_phys, tso.data, 435 tb_len, NULL); 436 if (ret) 437 goto out_err; 438 439 data_left -= tb_len; 440 tso_build_data(skb, &tso, tb_len); 441 } 442 } 443 444 /* re -add the WiFi header */ 445 skb_push(skb, hdr_len); 446 447 return 0; 448 449 out_err: 450 #endif 451 return -EINVAL; 452 } 453 454 static struct 455 iwl_tfh_tfd *iwl_txq_gen2_build_tx_amsdu(struct iwl_trans *trans, 456 struct iwl_txq *txq, 457 struct iwl_device_tx_cmd *dev_cmd, 458 struct sk_buff *skb, 459 struct iwl_cmd_meta *out_meta, 460 int hdr_len, 461 int tx_cmd_len) 462 { 463 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 464 struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx); 465 dma_addr_t tb_phys; 466 int len; 467 void *tb1_addr; 468 469 tb_phys = iwl_txq_get_first_tb_dma(txq, idx); 470 471 /* 472 * No need for _with_wa, the first TB allocation is aligned up 473 * to a 64-byte boundary and thus can't be at the end or cross 474 * a page boundary (much less a 2^32 boundary). 475 */ 476 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 477 478 /* 479 * The second TB (tb1) points to the remainder of the TX command 480 * and the 802.11 header - dword aligned size 481 * (This calculation modifies the TX command, so do it before the 482 * setup of the first TB) 483 */ 484 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 485 IWL_FIRST_TB_SIZE; 486 487 /* do not align A-MSDU to dword as the subframe header aligns it */ 488 489 /* map the data for TB1 */ 490 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 491 tb_phys = dma_map_single(trans->dev, tb1_addr, len, DMA_TO_DEVICE); 492 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 493 goto out_err; 494 /* 495 * No need for _with_wa(), we ensure (via alignment) that the data 496 * here can never cross or end at a page boundary. 497 */ 498 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, len); 499 500 if (iwl_txq_gen2_build_amsdu(trans, skb, tfd, len + IWL_FIRST_TB_SIZE, 501 hdr_len, dev_cmd)) 502 goto out_err; 503 504 /* building the A-MSDU might have changed this data, memcpy it now */ 505 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 506 return tfd; 507 508 out_err: 509 iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd); 510 return NULL; 511 } 512 513 static int iwl_txq_gen2_tx_add_frags(struct iwl_trans *trans, 514 struct sk_buff *skb, 515 struct iwl_tfh_tfd *tfd, 516 struct iwl_cmd_meta *out_meta) 517 { 518 int i; 519 520 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 521 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 522 dma_addr_t tb_phys; 523 unsigned int fragsz = skb_frag_size(frag); 524 int ret; 525 526 if (!fragsz) 527 continue; 528 529 tb_phys = skb_frag_dma_map(trans->dev, frag, 0, 530 fragsz, DMA_TO_DEVICE); 531 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, tb_phys, 532 skb_frag_address(frag), 533 fragsz, out_meta); 534 if (ret) 535 return ret; 536 } 537 538 return 0; 539 } 540 541 static struct 542 iwl_tfh_tfd *iwl_txq_gen2_build_tx(struct iwl_trans *trans, 543 struct iwl_txq *txq, 544 struct iwl_device_tx_cmd *dev_cmd, 545 struct sk_buff *skb, 546 struct iwl_cmd_meta *out_meta, 547 int hdr_len, 548 int tx_cmd_len, 549 bool pad) 550 { 551 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 552 struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx); 553 dma_addr_t tb_phys; 554 int len, tb1_len, tb2_len; 555 void *tb1_addr; 556 struct sk_buff *frag; 557 558 tb_phys = iwl_txq_get_first_tb_dma(txq, idx); 559 560 /* The first TB points to bi-directional DMA data */ 561 memcpy(&txq->first_tb_bufs[idx], dev_cmd, IWL_FIRST_TB_SIZE); 562 563 /* 564 * No need for _with_wa, the first TB allocation is aligned up 565 * to a 64-byte boundary and thus can't be at the end or cross 566 * a page boundary (much less a 2^32 boundary). 567 */ 568 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); 569 570 /* 571 * The second TB (tb1) points to the remainder of the TX command 572 * and the 802.11 header - dword aligned size 573 * (This calculation modifies the TX command, so do it before the 574 * setup of the first TB) 575 */ 576 len = tx_cmd_len + sizeof(struct iwl_cmd_header) + hdr_len - 577 IWL_FIRST_TB_SIZE; 578 579 if (pad) 580 tb1_len = ALIGN(len, 4); 581 else 582 tb1_len = len; 583 584 /* map the data for TB1 */ 585 tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; 586 tb_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); 587 if (unlikely(dma_mapping_error(trans->dev, tb_phys))) 588 goto out_err; 589 /* 590 * No need for _with_wa(), we ensure (via alignment) that the data 591 * here can never cross or end at a page boundary. 592 */ 593 iwl_txq_gen2_set_tb(trans, tfd, tb_phys, tb1_len); 594 trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), &dev_cmd->hdr, 595 IWL_FIRST_TB_SIZE + tb1_len, hdr_len); 596 597 /* set up TFD's third entry to point to remainder of skb's head */ 598 tb2_len = skb_headlen(skb) - hdr_len; 599 600 if (tb2_len > 0) { 601 int ret; 602 603 tb_phys = dma_map_single(trans->dev, skb->data + hdr_len, 604 tb2_len, DMA_TO_DEVICE); 605 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, tb_phys, 606 skb->data + hdr_len, tb2_len, 607 NULL); 608 if (ret) 609 goto out_err; 610 } 611 612 if (iwl_txq_gen2_tx_add_frags(trans, skb, tfd, out_meta)) 613 goto out_err; 614 615 skb_walk_frags(skb, frag) { 616 int ret; 617 618 tb_phys = dma_map_single(trans->dev, frag->data, 619 skb_headlen(frag), DMA_TO_DEVICE); 620 ret = iwl_txq_gen2_set_tb_with_wa(trans, skb, tfd, tb_phys, 621 frag->data, 622 skb_headlen(frag), NULL); 623 if (ret) 624 goto out_err; 625 if (iwl_txq_gen2_tx_add_frags(trans, frag, tfd, out_meta)) 626 goto out_err; 627 } 628 629 return tfd; 630 631 out_err: 632 iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd); 633 return NULL; 634 } 635 636 static 637 struct iwl_tfh_tfd *iwl_txq_gen2_build_tfd(struct iwl_trans *trans, 638 struct iwl_txq *txq, 639 struct iwl_device_tx_cmd *dev_cmd, 640 struct sk_buff *skb, 641 struct iwl_cmd_meta *out_meta) 642 { 643 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 644 int idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 645 struct iwl_tfh_tfd *tfd = iwl_txq_get_tfd(trans, txq, idx); 646 int len, hdr_len; 647 bool amsdu; 648 649 /* There must be data left over for TB1 or this code must be changed */ 650 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2) < IWL_FIRST_TB_SIZE); 651 652 memset(tfd, 0, sizeof(*tfd)); 653 654 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) 655 len = sizeof(struct iwl_tx_cmd_gen2); 656 else 657 len = sizeof(struct iwl_tx_cmd_gen3); 658 659 amsdu = ieee80211_is_data_qos(hdr->frame_control) && 660 (*ieee80211_get_qos_ctl(hdr) & 661 IEEE80211_QOS_CTL_A_MSDU_PRESENT); 662 663 hdr_len = ieee80211_hdrlen(hdr->frame_control); 664 665 /* 666 * Only build A-MSDUs here if doing so by GSO, otherwise it may be 667 * an A-MSDU for other reasons, e.g. NAN or an A-MSDU having been 668 * built in the higher layers already. 669 */ 670 if (amsdu && skb_shinfo(skb)->gso_size) 671 return iwl_txq_gen2_build_tx_amsdu(trans, txq, dev_cmd, skb, 672 out_meta, hdr_len, len); 673 return iwl_txq_gen2_build_tx(trans, txq, dev_cmd, skb, out_meta, 674 hdr_len, len, !amsdu); 675 } 676 677 int iwl_txq_space(struct iwl_trans *trans, const struct iwl_txq *q) 678 { 679 unsigned int max; 680 unsigned int used; 681 682 /* 683 * To avoid ambiguity between empty and completely full queues, there 684 * should always be less than max_tfd_queue_size elements in the queue. 685 * If q->n_window is smaller than max_tfd_queue_size, there is no need 686 * to reserve any queue entries for this purpose. 687 */ 688 if (q->n_window < trans->trans_cfg->base_params->max_tfd_queue_size) 689 max = q->n_window; 690 else 691 max = trans->trans_cfg->base_params->max_tfd_queue_size - 1; 692 693 /* 694 * max_tfd_queue_size is a power of 2, so the following is equivalent to 695 * modulo by max_tfd_queue_size and is well defined. 696 */ 697 used = (q->write_ptr - q->read_ptr) & 698 (trans->trans_cfg->base_params->max_tfd_queue_size - 1); 699 700 if (WARN_ON(used > max)) 701 return 0; 702 703 return max - used; 704 } 705 706 int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb, 707 struct iwl_device_tx_cmd *dev_cmd, int txq_id) 708 { 709 struct iwl_cmd_meta *out_meta; 710 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 711 u16 cmd_len; 712 int idx; 713 void *tfd; 714 715 if (WARN_ONCE(txq_id >= IWL_MAX_TVQM_QUEUES, 716 "queue %d out of range", txq_id)) 717 return -EINVAL; 718 719 if (WARN_ONCE(!test_bit(txq_id, trans->txqs.queue_used), 720 "TX on unused queue %d\n", txq_id)) 721 return -EINVAL; 722 723 if (skb_is_nonlinear(skb) && 724 skb_shinfo(skb)->nr_frags > IWL_TRANS_MAX_FRAGS(trans) && 725 __skb_linearize(skb)) 726 return -ENOMEM; 727 728 spin_lock(&txq->lock); 729 730 if (iwl_txq_space(trans, txq) < txq->high_mark) { 731 iwl_txq_stop(trans, txq); 732 733 /* don't put the packet on the ring, if there is no room */ 734 if (unlikely(iwl_txq_space(trans, txq) < 3)) { 735 struct iwl_device_tx_cmd **dev_cmd_ptr; 736 737 dev_cmd_ptr = (void *)((u8 *)skb->cb + 738 trans->txqs.dev_cmd_offs); 739 740 *dev_cmd_ptr = dev_cmd; 741 __skb_queue_tail(&txq->overflow_q, skb); 742 spin_unlock(&txq->lock); 743 return 0; 744 } 745 } 746 747 idx = iwl_txq_get_cmd_index(txq, txq->write_ptr); 748 749 /* Set up driver data for this TFD */ 750 txq->entries[idx].skb = skb; 751 txq->entries[idx].cmd = dev_cmd; 752 753 dev_cmd->hdr.sequence = 754 cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | 755 INDEX_TO_SEQ(idx))); 756 757 /* Set up first empty entry in queue's array of Tx/cmd buffers */ 758 out_meta = &txq->entries[idx].meta; 759 out_meta->flags = 0; 760 761 tfd = iwl_txq_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta); 762 if (!tfd) { 763 spin_unlock(&txq->lock); 764 return -1; 765 } 766 767 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 768 struct iwl_tx_cmd_gen3 *tx_cmd_gen3 = 769 (void *)dev_cmd->payload; 770 771 cmd_len = le16_to_cpu(tx_cmd_gen3->len); 772 } else { 773 struct iwl_tx_cmd_gen2 *tx_cmd_gen2 = 774 (void *)dev_cmd->payload; 775 776 cmd_len = le16_to_cpu(tx_cmd_gen2->len); 777 } 778 779 /* Set up entry for this TFD in Tx byte-count array */ 780 iwl_pcie_gen2_update_byte_tbl(trans, txq, cmd_len, 781 iwl_txq_gen2_get_num_tbs(trans, tfd)); 782 783 /* start timer if queue currently empty */ 784 if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) 785 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 786 787 /* Tell device the write index *just past* this latest filled TFD */ 788 txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); 789 iwl_txq_inc_wr_ptr(trans, txq); 790 /* 791 * At this point the frame is "transmitted" successfully 792 * and we will get a TX status notification eventually. 793 */ 794 spin_unlock(&txq->lock); 795 return 0; 796 } 797 798 /*************** HOST COMMAND QUEUE FUNCTIONS *****/ 799 800 /* 801 * iwl_txq_gen2_unmap - Unmap any remaining DMA mappings and free skb's 802 */ 803 void iwl_txq_gen2_unmap(struct iwl_trans *trans, int txq_id) 804 { 805 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 806 807 spin_lock_bh(&txq->lock); 808 while (txq->write_ptr != txq->read_ptr) { 809 IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", 810 txq_id, txq->read_ptr); 811 812 if (txq_id != trans->txqs.cmd.q_id) { 813 int idx = iwl_txq_get_cmd_index(txq, txq->read_ptr); 814 struct sk_buff *skb = txq->entries[idx].skb; 815 816 if (!WARN_ON_ONCE(!skb)) 817 iwl_txq_free_tso_page(trans, skb); 818 } 819 iwl_txq_gen2_free_tfd(trans, txq); 820 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr); 821 } 822 823 while (!skb_queue_empty(&txq->overflow_q)) { 824 struct sk_buff *skb = __skb_dequeue(&txq->overflow_q); 825 826 iwl_op_mode_free_skb(trans->op_mode, skb); 827 } 828 829 spin_unlock_bh(&txq->lock); 830 831 /* just in case - this queue may have been stopped */ 832 iwl_wake_queue(trans, txq); 833 } 834 835 static void iwl_txq_gen2_free_memory(struct iwl_trans *trans, 836 struct iwl_txq *txq) 837 { 838 struct device *dev = trans->dev; 839 840 /* De-alloc circular buffer of TFDs */ 841 if (txq->tfds) { 842 dma_free_coherent(dev, 843 trans->txqs.tfd.size * txq->n_window, 844 txq->tfds, txq->dma_addr); 845 dma_free_coherent(dev, 846 sizeof(*txq->first_tb_bufs) * txq->n_window, 847 txq->first_tb_bufs, txq->first_tb_dma); 848 } 849 850 kfree(txq->entries); 851 if (txq->bc_tbl.addr) 852 dma_pool_free(trans->txqs.bc_pool, 853 txq->bc_tbl.addr, txq->bc_tbl.dma); 854 kfree(txq); 855 } 856 857 /* 858 * iwl_pcie_txq_free - Deallocate DMA queue. 859 * @txq: Transmit queue to deallocate. 860 * 861 * Empty queue by removing and destroying all BD's. 862 * Free all buffers. 863 * 0-fill, but do not free "txq" descriptor structure. 864 */ 865 static void iwl_txq_gen2_free(struct iwl_trans *trans, int txq_id) 866 { 867 struct iwl_txq *txq; 868 int i; 869 870 if (WARN_ONCE(txq_id >= IWL_MAX_TVQM_QUEUES, 871 "queue %d out of range", txq_id)) 872 return; 873 874 txq = trans->txqs.txq[txq_id]; 875 876 if (WARN_ON(!txq)) 877 return; 878 879 iwl_txq_gen2_unmap(trans, txq_id); 880 881 /* De-alloc array of command/tx buffers */ 882 if (txq_id == trans->txqs.cmd.q_id) 883 for (i = 0; i < txq->n_window; i++) { 884 kfree_sensitive(txq->entries[i].cmd); 885 kfree_sensitive(txq->entries[i].free_buf); 886 } 887 del_timer_sync(&txq->stuck_timer); 888 889 iwl_txq_gen2_free_memory(trans, txq); 890 891 trans->txqs.txq[txq_id] = NULL; 892 893 clear_bit(txq_id, trans->txqs.queue_used); 894 } 895 896 /* 897 * iwl_queue_init - Initialize queue's high/low-water and read/write indexes 898 */ 899 static int iwl_queue_init(struct iwl_txq *q, int slots_num) 900 { 901 q->n_window = slots_num; 902 903 /* slots_num must be power-of-two size, otherwise 904 * iwl_txq_get_cmd_index is broken. */ 905 if (WARN_ON(!is_power_of_2(slots_num))) 906 return -EINVAL; 907 908 q->low_mark = q->n_window / 4; 909 if (q->low_mark < 4) 910 q->low_mark = 4; 911 912 q->high_mark = q->n_window / 8; 913 if (q->high_mark < 2) 914 q->high_mark = 2; 915 916 q->write_ptr = 0; 917 q->read_ptr = 0; 918 919 return 0; 920 } 921 922 int iwl_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, 923 bool cmd_queue) 924 { 925 int ret; 926 u32 tfd_queue_max_size = 927 trans->trans_cfg->base_params->max_tfd_queue_size; 928 929 txq->need_update = false; 930 931 /* max_tfd_queue_size must be power-of-two size, otherwise 932 * iwl_txq_inc_wrap and iwl_txq_dec_wrap are broken. */ 933 if (WARN_ONCE(tfd_queue_max_size & (tfd_queue_max_size - 1), 934 "Max tfd queue size must be a power of two, but is %d", 935 tfd_queue_max_size)) 936 return -EINVAL; 937 938 /* Initialize queue's high/low-water marks, and head/tail indexes */ 939 ret = iwl_queue_init(txq, slots_num); 940 if (ret) 941 return ret; 942 943 spin_lock_init(&txq->lock); 944 945 if (cmd_queue) { 946 static struct lock_class_key iwl_txq_cmd_queue_lock_class; 947 948 lockdep_set_class(&txq->lock, &iwl_txq_cmd_queue_lock_class); 949 } 950 951 __skb_queue_head_init(&txq->overflow_q); 952 953 return 0; 954 } 955 956 void iwl_txq_free_tso_page(struct iwl_trans *trans, struct sk_buff *skb) 957 { 958 struct page **page_ptr; 959 struct page *next; 960 961 page_ptr = (void *)((u8 *)skb->cb + trans->txqs.page_offs); 962 next = *page_ptr; 963 *page_ptr = NULL; 964 965 while (next) { 966 struct page *tmp = next; 967 968 next = *(void **)((u8 *)page_address(next) + PAGE_SIZE - 969 sizeof(void *)); 970 __free_page(tmp); 971 } 972 } 973 974 void iwl_txq_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq) 975 { 976 u32 txq_id = txq->id; 977 u32 status; 978 bool active; 979 u8 fifo; 980 981 if (trans->trans_cfg->use_tfh) { 982 IWL_ERR(trans, "Queue %d is stuck %d %d\n", txq_id, 983 txq->read_ptr, txq->write_ptr); 984 /* TODO: access new SCD registers and dump them */ 985 return; 986 } 987 988 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id)); 989 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7; 990 active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE)); 991 992 IWL_ERR(trans, 993 "Queue %d is %sactive on fifo %d and stuck for %u ms. SW [%d, %d] HW [%d, %d] FH TRB=0x0%x\n", 994 txq_id, active ? "" : "in", fifo, 995 jiffies_to_msecs(txq->wd_timeout), 996 txq->read_ptr, txq->write_ptr, 997 iwl_read_prph(trans, SCD_QUEUE_RDPTR(txq_id)) & 998 (trans->trans_cfg->base_params->max_tfd_queue_size - 1), 999 iwl_read_prph(trans, SCD_QUEUE_WRPTR(txq_id)) & 1000 (trans->trans_cfg->base_params->max_tfd_queue_size - 1), 1001 iwl_read_direct32(trans, FH_TX_TRB_REG(fifo))); 1002 } 1003 1004 static void iwl_txq_stuck_timer(struct timer_list *t) 1005 { 1006 struct iwl_txq *txq = from_timer(txq, t, stuck_timer); 1007 struct iwl_trans *trans = txq->trans; 1008 1009 spin_lock(&txq->lock); 1010 /* check if triggered erroneously */ 1011 if (txq->read_ptr == txq->write_ptr) { 1012 spin_unlock(&txq->lock); 1013 return; 1014 } 1015 spin_unlock(&txq->lock); 1016 1017 iwl_txq_log_scd_error(trans, txq); 1018 1019 iwl_force_nmi(trans); 1020 } 1021 1022 int iwl_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, 1023 bool cmd_queue) 1024 { 1025 size_t tfd_sz = trans->txqs.tfd.size * 1026 trans->trans_cfg->base_params->max_tfd_queue_size; 1027 size_t tb0_buf_sz; 1028 int i; 1029 1030 if (WARN_ONCE(slots_num <= 0, "Invalid slots num:%d\n", slots_num)) 1031 return -EINVAL; 1032 1033 if (WARN_ON(txq->entries || txq->tfds)) 1034 return -EINVAL; 1035 1036 if (trans->trans_cfg->use_tfh) 1037 tfd_sz = trans->txqs.tfd.size * slots_num; 1038 1039 timer_setup(&txq->stuck_timer, iwl_txq_stuck_timer, 0); 1040 txq->trans = trans; 1041 1042 txq->n_window = slots_num; 1043 1044 txq->entries = kcalloc(slots_num, 1045 sizeof(struct iwl_pcie_txq_entry), 1046 GFP_KERNEL); 1047 1048 if (!txq->entries) 1049 goto error; 1050 1051 if (cmd_queue) 1052 for (i = 0; i < slots_num; i++) { 1053 txq->entries[i].cmd = 1054 kmalloc(sizeof(struct iwl_device_cmd), 1055 GFP_KERNEL); 1056 if (!txq->entries[i].cmd) 1057 goto error; 1058 } 1059 1060 /* Circular buffer of transmit frame descriptors (TFDs), 1061 * shared with device */ 1062 txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz, 1063 &txq->dma_addr, GFP_KERNEL); 1064 if (!txq->tfds) 1065 goto error; 1066 1067 BUILD_BUG_ON(sizeof(*txq->first_tb_bufs) != IWL_FIRST_TB_SIZE_ALIGN); 1068 1069 tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num; 1070 1071 txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz, 1072 &txq->first_tb_dma, 1073 GFP_KERNEL); 1074 if (!txq->first_tb_bufs) 1075 goto err_free_tfds; 1076 1077 return 0; 1078 err_free_tfds: 1079 dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr); 1080 txq->tfds = NULL; 1081 error: 1082 if (txq->entries && cmd_queue) 1083 for (i = 0; i < slots_num; i++) 1084 kfree(txq->entries[i].cmd); 1085 kfree(txq->entries); 1086 txq->entries = NULL; 1087 1088 return -ENOMEM; 1089 } 1090 1091 static struct iwl_txq * 1092 iwl_txq_dyn_alloc_dma(struct iwl_trans *trans, int size, unsigned int timeout) 1093 { 1094 size_t bc_tbl_size, bc_tbl_entries; 1095 struct iwl_txq *txq; 1096 int ret; 1097 1098 WARN_ON(!trans->txqs.bc_tbl_size); 1099 1100 bc_tbl_size = trans->txqs.bc_tbl_size; 1101 bc_tbl_entries = bc_tbl_size / sizeof(u16); 1102 1103 if (WARN_ON(size > bc_tbl_entries)) 1104 return ERR_PTR(-EINVAL); 1105 1106 txq = kzalloc(sizeof(*txq), GFP_KERNEL); 1107 if (!txq) 1108 return ERR_PTR(-ENOMEM); 1109 1110 txq->bc_tbl.addr = dma_pool_alloc(trans->txqs.bc_pool, GFP_KERNEL, 1111 &txq->bc_tbl.dma); 1112 if (!txq->bc_tbl.addr) { 1113 IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); 1114 kfree(txq); 1115 return ERR_PTR(-ENOMEM); 1116 } 1117 1118 ret = iwl_txq_alloc(trans, txq, size, false); 1119 if (ret) { 1120 IWL_ERR(trans, "Tx queue alloc failed\n"); 1121 goto error; 1122 } 1123 ret = iwl_txq_init(trans, txq, size, false); 1124 if (ret) { 1125 IWL_ERR(trans, "Tx queue init failed\n"); 1126 goto error; 1127 } 1128 1129 txq->wd_timeout = msecs_to_jiffies(timeout); 1130 1131 return txq; 1132 1133 error: 1134 iwl_txq_gen2_free_memory(trans, txq); 1135 return ERR_PTR(ret); 1136 } 1137 1138 static int iwl_txq_alloc_response(struct iwl_trans *trans, struct iwl_txq *txq, 1139 struct iwl_host_cmd *hcmd) 1140 { 1141 struct iwl_tx_queue_cfg_rsp *rsp; 1142 int ret, qid; 1143 u32 wr_ptr; 1144 1145 if (WARN_ON(iwl_rx_packet_payload_len(hcmd->resp_pkt) != 1146 sizeof(*rsp))) { 1147 ret = -EINVAL; 1148 goto error_free_resp; 1149 } 1150 1151 rsp = (void *)hcmd->resp_pkt->data; 1152 qid = le16_to_cpu(rsp->queue_number); 1153 wr_ptr = le16_to_cpu(rsp->write_pointer); 1154 1155 if (qid >= ARRAY_SIZE(trans->txqs.txq)) { 1156 WARN_ONCE(1, "queue index %d unsupported", qid); 1157 ret = -EIO; 1158 goto error_free_resp; 1159 } 1160 1161 if (test_and_set_bit(qid, trans->txqs.queue_used)) { 1162 WARN_ONCE(1, "queue %d already used", qid); 1163 ret = -EIO; 1164 goto error_free_resp; 1165 } 1166 1167 if (WARN_ONCE(trans->txqs.txq[qid], 1168 "queue %d already allocated\n", qid)) { 1169 ret = -EIO; 1170 goto error_free_resp; 1171 } 1172 1173 txq->id = qid; 1174 trans->txqs.txq[qid] = txq; 1175 wr_ptr &= (trans->trans_cfg->base_params->max_tfd_queue_size - 1); 1176 1177 /* Place first TFD at index corresponding to start sequence number */ 1178 txq->read_ptr = wr_ptr; 1179 txq->write_ptr = wr_ptr; 1180 1181 IWL_DEBUG_TX_QUEUES(trans, "Activate queue %d\n", qid); 1182 1183 iwl_free_resp(hcmd); 1184 return qid; 1185 1186 error_free_resp: 1187 iwl_free_resp(hcmd); 1188 iwl_txq_gen2_free_memory(trans, txq); 1189 return ret; 1190 } 1191 1192 int iwl_txq_dyn_alloc(struct iwl_trans *trans, u32 flags, u32 sta_mask, 1193 u8 tid, int size, unsigned int timeout) 1194 { 1195 struct iwl_txq *txq; 1196 union { 1197 struct iwl_tx_queue_cfg_cmd old; 1198 struct iwl_scd_queue_cfg_cmd new; 1199 } cmd; 1200 struct iwl_host_cmd hcmd = { 1201 .flags = CMD_WANT_SKB, 1202 }; 1203 int ret; 1204 1205 if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_BZ && 1206 trans->hw_rev_step == SILICON_A_STEP) 1207 size = 4096; 1208 1209 txq = iwl_txq_dyn_alloc_dma(trans, size, timeout); 1210 if (IS_ERR(txq)) 1211 return PTR_ERR(txq); 1212 1213 if (trans->txqs.queue_alloc_cmd_ver == 0) { 1214 memset(&cmd.old, 0, sizeof(cmd.old)); 1215 cmd.old.tfdq_addr = cpu_to_le64(txq->dma_addr); 1216 cmd.old.byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma); 1217 cmd.old.cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(size)); 1218 cmd.old.flags = cpu_to_le16(flags | TX_QUEUE_CFG_ENABLE_QUEUE); 1219 cmd.old.tid = tid; 1220 1221 if (hweight32(sta_mask) != 1) { 1222 ret = -EINVAL; 1223 goto error; 1224 } 1225 cmd.old.sta_id = ffs(sta_mask) - 1; 1226 1227 hcmd.id = SCD_QUEUE_CFG; 1228 hcmd.len[0] = sizeof(cmd.old); 1229 hcmd.data[0] = &cmd.old; 1230 } else if (trans->txqs.queue_alloc_cmd_ver == 3) { 1231 memset(&cmd.new, 0, sizeof(cmd.new)); 1232 cmd.new.operation = cpu_to_le32(IWL_SCD_QUEUE_ADD); 1233 cmd.new.u.add.tfdq_dram_addr = cpu_to_le64(txq->dma_addr); 1234 cmd.new.u.add.bc_dram_addr = cpu_to_le64(txq->bc_tbl.dma); 1235 cmd.new.u.add.cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(size)); 1236 cmd.new.u.add.flags = cpu_to_le32(flags); 1237 cmd.new.u.add.sta_mask = cpu_to_le32(sta_mask); 1238 cmd.new.u.add.tid = tid; 1239 1240 hcmd.id = WIDE_ID(DATA_PATH_GROUP, SCD_QUEUE_CONFIG_CMD); 1241 hcmd.len[0] = sizeof(cmd.new); 1242 hcmd.data[0] = &cmd.new; 1243 } else { 1244 ret = -EOPNOTSUPP; 1245 goto error; 1246 } 1247 1248 ret = iwl_trans_send_cmd(trans, &hcmd); 1249 if (ret) 1250 goto error; 1251 1252 return iwl_txq_alloc_response(trans, txq, &hcmd); 1253 1254 error: 1255 iwl_txq_gen2_free_memory(trans, txq); 1256 return ret; 1257 } 1258 1259 void iwl_txq_dyn_free(struct iwl_trans *trans, int queue) 1260 { 1261 if (WARN(queue >= IWL_MAX_TVQM_QUEUES, 1262 "queue %d out of range", queue)) 1263 return; 1264 1265 /* 1266 * Upon HW Rfkill - we stop the device, and then stop the queues 1267 * in the op_mode. Just for the sake of the simplicity of the op_mode, 1268 * allow the op_mode to call txq_disable after it already called 1269 * stop_device. 1270 */ 1271 if (!test_and_clear_bit(queue, trans->txqs.queue_used)) { 1272 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), 1273 "queue %d not used", queue); 1274 return; 1275 } 1276 1277 iwl_txq_gen2_free(trans, queue); 1278 1279 IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", queue); 1280 } 1281 1282 void iwl_txq_gen2_tx_free(struct iwl_trans *trans) 1283 { 1284 int i; 1285 1286 memset(trans->txqs.queue_used, 0, sizeof(trans->txqs.queue_used)); 1287 1288 /* Free all TX queues */ 1289 for (i = 0; i < ARRAY_SIZE(trans->txqs.txq); i++) { 1290 if (!trans->txqs.txq[i]) 1291 continue; 1292 1293 iwl_txq_gen2_free(trans, i); 1294 } 1295 } 1296 1297 int iwl_txq_gen2_init(struct iwl_trans *trans, int txq_id, int queue_size) 1298 { 1299 struct iwl_txq *queue; 1300 int ret; 1301 1302 /* alloc and init the tx queue */ 1303 if (!trans->txqs.txq[txq_id]) { 1304 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 1305 if (!queue) { 1306 IWL_ERR(trans, "Not enough memory for tx queue\n"); 1307 return -ENOMEM; 1308 } 1309 trans->txqs.txq[txq_id] = queue; 1310 ret = iwl_txq_alloc(trans, queue, queue_size, true); 1311 if (ret) { 1312 IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); 1313 goto error; 1314 } 1315 } else { 1316 queue = trans->txqs.txq[txq_id]; 1317 } 1318 1319 ret = iwl_txq_init(trans, queue, queue_size, 1320 (txq_id == trans->txqs.cmd.q_id)); 1321 if (ret) { 1322 IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); 1323 goto error; 1324 } 1325 trans->txqs.txq[txq_id]->id = txq_id; 1326 set_bit(txq_id, trans->txqs.queue_used); 1327 1328 return 0; 1329 1330 error: 1331 iwl_txq_gen2_tx_free(trans); 1332 return ret; 1333 } 1334 1335 static inline dma_addr_t iwl_txq_gen1_tfd_tb_get_addr(struct iwl_trans *trans, 1336 void *_tfd, u8 idx) 1337 { 1338 struct iwl_tfd *tfd; 1339 struct iwl_tfd_tb *tb; 1340 dma_addr_t addr; 1341 dma_addr_t hi_len; 1342 1343 if (trans->trans_cfg->use_tfh) { 1344 struct iwl_tfh_tfd *tfh_tfd = _tfd; 1345 struct iwl_tfh_tb *tfh_tb = &tfh_tfd->tbs[idx]; 1346 1347 return (dma_addr_t)(le64_to_cpu(tfh_tb->addr)); 1348 } 1349 1350 tfd = _tfd; 1351 tb = &tfd->tbs[idx]; 1352 addr = get_unaligned_le32(&tb->lo); 1353 1354 if (sizeof(dma_addr_t) <= sizeof(u32)) 1355 return addr; 1356 1357 hi_len = le16_to_cpu(tb->hi_n_len) & 0xF; 1358 1359 /* 1360 * shift by 16 twice to avoid warnings on 32-bit 1361 * (where this code never runs anyway due to the 1362 * if statement above) 1363 */ 1364 return addr | ((hi_len << 16) << 16); 1365 } 1366 1367 void iwl_txq_gen1_tfd_unmap(struct iwl_trans *trans, 1368 struct iwl_cmd_meta *meta, 1369 struct iwl_txq *txq, int index) 1370 { 1371 int i, num_tbs; 1372 void *tfd = iwl_txq_get_tfd(trans, txq, index); 1373 1374 /* Sanity check on number of chunks */ 1375 num_tbs = iwl_txq_gen1_tfd_get_num_tbs(trans, tfd); 1376 1377 if (num_tbs > trans->txqs.tfd.max_tbs) { 1378 IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); 1379 /* @todo issue fatal error, it is quite serious situation */ 1380 return; 1381 } 1382 1383 /* first TB is never freed - it's the bidirectional DMA data */ 1384 1385 for (i = 1; i < num_tbs; i++) { 1386 if (meta->tbs & BIT(i)) 1387 dma_unmap_page(trans->dev, 1388 iwl_txq_gen1_tfd_tb_get_addr(trans, 1389 tfd, i), 1390 iwl_txq_gen1_tfd_tb_get_len(trans, 1391 tfd, i), 1392 DMA_TO_DEVICE); 1393 else 1394 dma_unmap_single(trans->dev, 1395 iwl_txq_gen1_tfd_tb_get_addr(trans, 1396 tfd, i), 1397 iwl_txq_gen1_tfd_tb_get_len(trans, 1398 tfd, i), 1399 DMA_TO_DEVICE); 1400 } 1401 1402 meta->tbs = 0; 1403 1404 if (trans->trans_cfg->use_tfh) { 1405 struct iwl_tfh_tfd *tfd_fh = (void *)tfd; 1406 1407 tfd_fh->num_tbs = 0; 1408 } else { 1409 struct iwl_tfd *tfd_fh = (void *)tfd; 1410 1411 tfd_fh->num_tbs = 0; 1412 } 1413 } 1414 1415 #define IWL_TX_CRC_SIZE 4 1416 #define IWL_TX_DELIMITER_SIZE 4 1417 1418 /* 1419 * iwl_txq_gen1_update_byte_cnt_tbl - Set up entry in Tx byte-count array 1420 */ 1421 void iwl_txq_gen1_update_byte_cnt_tbl(struct iwl_trans *trans, 1422 struct iwl_txq *txq, u16 byte_cnt, 1423 int num_tbs) 1424 { 1425 struct iwlagn_scd_bc_tbl *scd_bc_tbl; 1426 int write_ptr = txq->write_ptr; 1427 int txq_id = txq->id; 1428 u8 sec_ctl = 0; 1429 u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE; 1430 __le16 bc_ent; 1431 struct iwl_device_tx_cmd *dev_cmd = txq->entries[txq->write_ptr].cmd; 1432 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 1433 u8 sta_id = tx_cmd->sta_id; 1434 1435 scd_bc_tbl = trans->txqs.scd_bc_tbls.addr; 1436 1437 sec_ctl = tx_cmd->sec_ctl; 1438 1439 switch (sec_ctl & TX_CMD_SEC_MSK) { 1440 case TX_CMD_SEC_CCM: 1441 len += IEEE80211_CCMP_MIC_LEN; 1442 break; 1443 case TX_CMD_SEC_TKIP: 1444 len += IEEE80211_TKIP_ICV_LEN; 1445 break; 1446 case TX_CMD_SEC_WEP: 1447 len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN; 1448 break; 1449 } 1450 if (trans->txqs.bc_table_dword) 1451 len = DIV_ROUND_UP(len, 4); 1452 1453 if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX)) 1454 return; 1455 1456 bc_ent = cpu_to_le16(len | (sta_id << 12)); 1457 1458 scd_bc_tbl[txq_id].tfd_offset[write_ptr] = bc_ent; 1459 1460 if (write_ptr < TFD_QUEUE_SIZE_BC_DUP) 1461 scd_bc_tbl[txq_id].tfd_offset[TFD_QUEUE_SIZE_MAX + write_ptr] = 1462 bc_ent; 1463 } 1464 1465 void iwl_txq_gen1_inval_byte_cnt_tbl(struct iwl_trans *trans, 1466 struct iwl_txq *txq) 1467 { 1468 struct iwlagn_scd_bc_tbl *scd_bc_tbl = trans->txqs.scd_bc_tbls.addr; 1469 int txq_id = txq->id; 1470 int read_ptr = txq->read_ptr; 1471 u8 sta_id = 0; 1472 __le16 bc_ent; 1473 struct iwl_device_tx_cmd *dev_cmd = txq->entries[read_ptr].cmd; 1474 struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; 1475 1476 WARN_ON(read_ptr >= TFD_QUEUE_SIZE_MAX); 1477 1478 if (txq_id != trans->txqs.cmd.q_id) 1479 sta_id = tx_cmd->sta_id; 1480 1481 bc_ent = cpu_to_le16(1 | (sta_id << 12)); 1482 1483 scd_bc_tbl[txq_id].tfd_offset[read_ptr] = bc_ent; 1484 1485 if (read_ptr < TFD_QUEUE_SIZE_BC_DUP) 1486 scd_bc_tbl[txq_id].tfd_offset[TFD_QUEUE_SIZE_MAX + read_ptr] = 1487 bc_ent; 1488 } 1489 1490 /* 1491 * iwl_txq_free_tfd - Free all chunks referenced by TFD [txq->q.read_ptr] 1492 * @trans - transport private data 1493 * @txq - tx queue 1494 * @dma_dir - the direction of the DMA mapping 1495 * 1496 * Does NOT advance any TFD circular buffer read/write indexes 1497 * Does NOT free the TFD itself (which is within circular buffer) 1498 */ 1499 void iwl_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) 1500 { 1501 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and 1502 * idx is bounded by n_window 1503 */ 1504 int rd_ptr = txq->read_ptr; 1505 int idx = iwl_txq_get_cmd_index(txq, rd_ptr); 1506 struct sk_buff *skb; 1507 1508 lockdep_assert_held(&txq->lock); 1509 1510 if (!txq->entries) 1511 return; 1512 1513 /* We have only q->n_window txq->entries, but we use 1514 * TFD_QUEUE_SIZE_MAX tfds 1515 */ 1516 iwl_txq_gen1_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr); 1517 1518 /* free SKB */ 1519 skb = txq->entries[idx].skb; 1520 1521 /* Can be called from irqs-disabled context 1522 * If skb is not NULL, it means that the whole queue is being 1523 * freed and that the queue is not empty - free the skb 1524 */ 1525 if (skb) { 1526 iwl_op_mode_free_skb(trans->op_mode, skb); 1527 txq->entries[idx].skb = NULL; 1528 } 1529 } 1530 1531 void iwl_txq_progress(struct iwl_txq *txq) 1532 { 1533 lockdep_assert_held(&txq->lock); 1534 1535 if (!txq->wd_timeout) 1536 return; 1537 1538 /* 1539 * station is asleep and we send data - that must 1540 * be uAPSD or PS-Poll. Don't rearm the timer. 1541 */ 1542 if (txq->frozen) 1543 return; 1544 1545 /* 1546 * if empty delete timer, otherwise move timer forward 1547 * since we're making progress on this queue 1548 */ 1549 if (txq->read_ptr == txq->write_ptr) 1550 del_timer(&txq->stuck_timer); 1551 else 1552 mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); 1553 } 1554 1555 /* Frees buffers until index _not_ inclusive */ 1556 void iwl_txq_reclaim(struct iwl_trans *trans, int txq_id, int ssn, 1557 struct sk_buff_head *skbs) 1558 { 1559 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 1560 int tfd_num, read_ptr, last_to_free; 1561 1562 /* This function is not meant to release cmd queue*/ 1563 if (WARN_ON(txq_id == trans->txqs.cmd.q_id)) 1564 return; 1565 1566 if (WARN_ON(!txq)) 1567 return; 1568 1569 tfd_num = iwl_txq_get_cmd_index(txq, ssn); 1570 read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr); 1571 1572 spin_lock_bh(&txq->lock); 1573 1574 if (!test_bit(txq_id, trans->txqs.queue_used)) { 1575 IWL_DEBUG_TX_QUEUES(trans, "Q %d inactive - ignoring idx %d\n", 1576 txq_id, ssn); 1577 goto out; 1578 } 1579 1580 if (read_ptr == tfd_num) 1581 goto out; 1582 1583 IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n", 1584 txq_id, txq->read_ptr, tfd_num, ssn); 1585 1586 /*Since we free until index _not_ inclusive, the one before index is 1587 * the last we will free. This one must be used */ 1588 last_to_free = iwl_txq_dec_wrap(trans, tfd_num); 1589 1590 if (!iwl_txq_used(txq, last_to_free)) { 1591 IWL_ERR(trans, 1592 "%s: Read index for txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n", 1593 __func__, txq_id, last_to_free, 1594 trans->trans_cfg->base_params->max_tfd_queue_size, 1595 txq->write_ptr, txq->read_ptr); 1596 1597 iwl_op_mode_time_point(trans->op_mode, 1598 IWL_FW_INI_TIME_POINT_FAKE_TX, 1599 NULL); 1600 goto out; 1601 } 1602 1603 if (WARN_ON(!skb_queue_empty(skbs))) 1604 goto out; 1605 1606 for (; 1607 read_ptr != tfd_num; 1608 txq->read_ptr = iwl_txq_inc_wrap(trans, txq->read_ptr), 1609 read_ptr = iwl_txq_get_cmd_index(txq, txq->read_ptr)) { 1610 struct sk_buff *skb = txq->entries[read_ptr].skb; 1611 1612 if (WARN_ON_ONCE(!skb)) 1613 continue; 1614 1615 iwl_txq_free_tso_page(trans, skb); 1616 1617 __skb_queue_tail(skbs, skb); 1618 1619 txq->entries[read_ptr].skb = NULL; 1620 1621 if (!trans->trans_cfg->use_tfh) 1622 iwl_txq_gen1_inval_byte_cnt_tbl(trans, txq); 1623 1624 iwl_txq_free_tfd(trans, txq); 1625 } 1626 1627 iwl_txq_progress(txq); 1628 1629 if (iwl_txq_space(trans, txq) > txq->low_mark && 1630 test_bit(txq_id, trans->txqs.queue_stopped)) { 1631 struct sk_buff_head overflow_skbs; 1632 1633 __skb_queue_head_init(&overflow_skbs); 1634 skb_queue_splice_init(&txq->overflow_q, &overflow_skbs); 1635 1636 /* 1637 * We are going to transmit from the overflow queue. 1638 * Remember this state so that wait_for_txq_empty will know we 1639 * are adding more packets to the TFD queue. It cannot rely on 1640 * the state of &txq->overflow_q, as we just emptied it, but 1641 * haven't TXed the content yet. 1642 */ 1643 txq->overflow_tx = true; 1644 1645 /* 1646 * This is tricky: we are in reclaim path which is non 1647 * re-entrant, so noone will try to take the access the 1648 * txq data from that path. We stopped tx, so we can't 1649 * have tx as well. Bottom line, we can unlock and re-lock 1650 * later. 1651 */ 1652 spin_unlock_bh(&txq->lock); 1653 1654 while (!skb_queue_empty(&overflow_skbs)) { 1655 struct sk_buff *skb = __skb_dequeue(&overflow_skbs); 1656 struct iwl_device_tx_cmd *dev_cmd_ptr; 1657 1658 dev_cmd_ptr = *(void **)((u8 *)skb->cb + 1659 trans->txqs.dev_cmd_offs); 1660 1661 /* 1662 * Note that we can very well be overflowing again. 1663 * In that case, iwl_txq_space will be small again 1664 * and we won't wake mac80211's queue. 1665 */ 1666 iwl_trans_tx(trans, skb, dev_cmd_ptr, txq_id); 1667 } 1668 1669 if (iwl_txq_space(trans, txq) > txq->low_mark) 1670 iwl_wake_queue(trans, txq); 1671 1672 spin_lock_bh(&txq->lock); 1673 txq->overflow_tx = false; 1674 } 1675 1676 out: 1677 spin_unlock_bh(&txq->lock); 1678 } 1679 1680 /* Set wr_ptr of specific device and txq */ 1681 void iwl_txq_set_q_ptrs(struct iwl_trans *trans, int txq_id, int ptr) 1682 { 1683 struct iwl_txq *txq = trans->txqs.txq[txq_id]; 1684 1685 spin_lock_bh(&txq->lock); 1686 1687 txq->write_ptr = ptr; 1688 txq->read_ptr = txq->write_ptr; 1689 1690 spin_unlock_bh(&txq->lock); 1691 } 1692 1693 void iwl_trans_txq_freeze_timer(struct iwl_trans *trans, unsigned long txqs, 1694 bool freeze) 1695 { 1696 int queue; 1697 1698 for_each_set_bit(queue, &txqs, BITS_PER_LONG) { 1699 struct iwl_txq *txq = trans->txqs.txq[queue]; 1700 unsigned long now; 1701 1702 spin_lock_bh(&txq->lock); 1703 1704 now = jiffies; 1705 1706 if (txq->frozen == freeze) 1707 goto next_queue; 1708 1709 IWL_DEBUG_TX_QUEUES(trans, "%s TXQ %d\n", 1710 freeze ? "Freezing" : "Waking", queue); 1711 1712 txq->frozen = freeze; 1713 1714 if (txq->read_ptr == txq->write_ptr) 1715 goto next_queue; 1716 1717 if (freeze) { 1718 if (unlikely(time_after(now, 1719 txq->stuck_timer.expires))) { 1720 /* 1721 * The timer should have fired, maybe it is 1722 * spinning right now on the lock. 1723 */ 1724 goto next_queue; 1725 } 1726 /* remember how long until the timer fires */ 1727 txq->frozen_expiry_remainder = 1728 txq->stuck_timer.expires - now; 1729 del_timer(&txq->stuck_timer); 1730 goto next_queue; 1731 } 1732 1733 /* 1734 * Wake a non-empty queue -> arm timer with the 1735 * remainder before it froze 1736 */ 1737 mod_timer(&txq->stuck_timer, 1738 now + txq->frozen_expiry_remainder); 1739 1740 next_queue: 1741 spin_unlock_bh(&txq->lock); 1742 } 1743 } 1744 1745 #define HOST_COMPLETE_TIMEOUT (2 * HZ) 1746 1747 static int iwl_trans_txq_send_hcmd_sync(struct iwl_trans *trans, 1748 struct iwl_host_cmd *cmd) 1749 { 1750 const char *cmd_str = iwl_get_cmd_string(trans, cmd->id); 1751 struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; 1752 int cmd_idx; 1753 int ret; 1754 1755 IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", cmd_str); 1756 1757 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, 1758 &trans->status), 1759 "Command %s: a command is already active!\n", cmd_str)) 1760 return -EIO; 1761 1762 IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", cmd_str); 1763 1764 cmd_idx = trans->ops->send_cmd(trans, cmd); 1765 if (cmd_idx < 0) { 1766 ret = cmd_idx; 1767 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1768 IWL_ERR(trans, "Error sending %s: enqueue_hcmd failed: %d\n", 1769 cmd_str, ret); 1770 return ret; 1771 } 1772 1773 ret = wait_event_timeout(trans->wait_command_queue, 1774 !test_bit(STATUS_SYNC_HCMD_ACTIVE, 1775 &trans->status), 1776 HOST_COMPLETE_TIMEOUT); 1777 if (!ret) { 1778 IWL_ERR(trans, "Error sending %s: time out after %dms.\n", 1779 cmd_str, jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); 1780 1781 IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", 1782 txq->read_ptr, txq->write_ptr); 1783 1784 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1785 IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", 1786 cmd_str); 1787 ret = -ETIMEDOUT; 1788 1789 iwl_trans_sync_nmi(trans); 1790 goto cancel; 1791 } 1792 1793 if (test_bit(STATUS_FW_ERROR, &trans->status)) { 1794 if (!test_and_clear_bit(STATUS_SUPPRESS_CMD_ERROR_ONCE, 1795 &trans->status)) { 1796 IWL_ERR(trans, "FW error in SYNC CMD %s\n", cmd_str); 1797 dump_stack(); 1798 } 1799 ret = -EIO; 1800 goto cancel; 1801 } 1802 1803 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1804 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1805 IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); 1806 ret = -ERFKILL; 1807 goto cancel; 1808 } 1809 1810 if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { 1811 IWL_ERR(trans, "Error: Response NULL in '%s'\n", cmd_str); 1812 ret = -EIO; 1813 goto cancel; 1814 } 1815 1816 return 0; 1817 1818 cancel: 1819 if (cmd->flags & CMD_WANT_SKB) { 1820 /* 1821 * Cancel the CMD_WANT_SKB flag for the cmd in the 1822 * TX cmd queue. Otherwise in case the cmd comes 1823 * in later, it will possibly set an invalid 1824 * address (cmd->meta.source). 1825 */ 1826 txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; 1827 } 1828 1829 if (cmd->resp_pkt) { 1830 iwl_free_resp(cmd); 1831 cmd->resp_pkt = NULL; 1832 } 1833 1834 return ret; 1835 } 1836 1837 int iwl_trans_txq_send_hcmd(struct iwl_trans *trans, 1838 struct iwl_host_cmd *cmd) 1839 { 1840 /* Make sure the NIC is still alive in the bus */ 1841 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 1842 return -ENODEV; 1843 1844 if (!(cmd->flags & CMD_SEND_IN_RFKILL) && 1845 test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { 1846 IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", 1847 cmd->id); 1848 return -ERFKILL; 1849 } 1850 1851 if (unlikely(trans->system_pm_mode == IWL_PLAT_PM_MODE_D3 && 1852 !(cmd->flags & CMD_SEND_IN_D3))) { 1853 IWL_DEBUG_WOWLAN(trans, "Dropping CMD 0x%x: D3\n", cmd->id); 1854 return -EHOSTDOWN; 1855 } 1856 1857 if (cmd->flags & CMD_ASYNC) { 1858 int ret; 1859 1860 /* An asynchronous command can not expect an SKB to be set. */ 1861 if (WARN_ON(cmd->flags & CMD_WANT_SKB)) 1862 return -EINVAL; 1863 1864 ret = trans->ops->send_cmd(trans, cmd); 1865 if (ret < 0) { 1866 IWL_ERR(trans, 1867 "Error sending %s: enqueue_hcmd failed: %d\n", 1868 iwl_get_cmd_string(trans, cmd->id), ret); 1869 return ret; 1870 } 1871 return 0; 1872 } 1873 1874 return iwl_trans_txq_send_hcmd_sync(trans, cmd); 1875 } 1876 1877