1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include "hif.h" 19 #include "pci.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * recieves an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 63 u32 ce_ctrl_addr, 64 unsigned int n) 65 { 66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); 67 } 68 69 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 70 u32 ce_ctrl_addr) 71 { 72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); 73 } 74 75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 76 u32 ce_ctrl_addr, 77 unsigned int n) 78 { 79 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 80 void __iomem *indicator_addr; 81 82 if (!test_bit(ATH10K_PCI_FEATURE_HW_1_0_WARKAROUND, ar_pci->features)) { 83 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 84 return; 85 } 86 87 /* workaround for QCA988x_1.0 HW CE */ 88 indicator_addr = ar_pci->mem + ce_ctrl_addr + DST_WATERMARK_ADDRESS; 89 90 if (ce_ctrl_addr == ath10k_ce_base_address(CDC_WAR_DATA_CE)) { 91 iowrite32((CDC_WAR_MAGIC_STR | n), indicator_addr); 92 } else { 93 unsigned long irq_flags; 94 local_irq_save(irq_flags); 95 iowrite32(1, indicator_addr); 96 97 /* 98 * PCIE write waits for ACK in IPQ8K, there is no 99 * need to read back value. 100 */ 101 (void)ioread32(indicator_addr); 102 (void)ioread32(indicator_addr); /* conservative */ 103 104 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 105 106 iowrite32(0, indicator_addr); 107 local_irq_restore(irq_flags); 108 } 109 } 110 111 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 112 u32 ce_ctrl_addr) 113 { 114 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); 115 } 116 117 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 118 u32 ce_ctrl_addr) 119 { 120 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); 121 } 122 123 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 124 u32 ce_ctrl_addr, 125 unsigned int addr) 126 { 127 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); 128 } 129 130 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 131 u32 ce_ctrl_addr, 132 unsigned int n) 133 { 134 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); 135 } 136 137 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 138 u32 ce_ctrl_addr, 139 unsigned int n) 140 { 141 u32 ctrl1_addr = ath10k_pci_read32((ar), 142 (ce_ctrl_addr) + CE_CTRL1_ADDRESS); 143 144 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 145 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | 146 CE_CTRL1_DMAX_LENGTH_SET(n)); 147 } 148 149 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 150 u32 ce_ctrl_addr, 151 unsigned int n) 152 { 153 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 154 155 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 156 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | 157 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); 158 } 159 160 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 161 u32 ce_ctrl_addr, 162 unsigned int n) 163 { 164 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 165 166 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 167 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | 168 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); 169 } 170 171 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 172 u32 ce_ctrl_addr) 173 { 174 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); 175 } 176 177 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 178 u32 ce_ctrl_addr, 179 u32 addr) 180 { 181 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); 182 } 183 184 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 unsigned int n) 187 { 188 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); 189 } 190 191 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 192 u32 ce_ctrl_addr, 193 unsigned int n) 194 { 195 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 196 197 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 198 (addr & ~SRC_WATERMARK_HIGH_MASK) | 199 SRC_WATERMARK_HIGH_SET(n)); 200 } 201 202 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 203 u32 ce_ctrl_addr, 204 unsigned int n) 205 { 206 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 207 208 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 209 (addr & ~SRC_WATERMARK_LOW_MASK) | 210 SRC_WATERMARK_LOW_SET(n)); 211 } 212 213 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 214 u32 ce_ctrl_addr, 215 unsigned int n) 216 { 217 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 218 219 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 220 (addr & ~DST_WATERMARK_HIGH_MASK) | 221 DST_WATERMARK_HIGH_SET(n)); 222 } 223 224 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 225 u32 ce_ctrl_addr, 226 unsigned int n) 227 { 228 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 229 230 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 231 (addr & ~DST_WATERMARK_LOW_MASK) | 232 DST_WATERMARK_LOW_SET(n)); 233 } 234 235 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 236 u32 ce_ctrl_addr) 237 { 238 u32 host_ie_addr = ath10k_pci_read32(ar, 239 ce_ctrl_addr + HOST_IE_ADDRESS); 240 241 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 242 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); 243 } 244 245 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 246 u32 ce_ctrl_addr) 247 { 248 u32 host_ie_addr = ath10k_pci_read32(ar, 249 ce_ctrl_addr + HOST_IE_ADDRESS); 250 251 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 252 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); 253 } 254 255 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 256 u32 ce_ctrl_addr) 257 { 258 u32 host_ie_addr = ath10k_pci_read32(ar, 259 ce_ctrl_addr + HOST_IE_ADDRESS); 260 261 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 262 host_ie_addr & ~CE_WATERMARK_MASK); 263 } 264 265 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 266 u32 ce_ctrl_addr) 267 { 268 u32 misc_ie_addr = ath10k_pci_read32(ar, 269 ce_ctrl_addr + MISC_IE_ADDRESS); 270 271 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 272 misc_ie_addr | CE_ERROR_MASK); 273 } 274 275 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 276 u32 ce_ctrl_addr, 277 unsigned int mask) 278 { 279 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); 280 } 281 282 283 /* 284 * Guts of ath10k_ce_send, used by both ath10k_ce_send and 285 * ath10k_ce_sendlist_send. 286 * The caller takes responsibility for any needed locking. 287 */ 288 static int ath10k_ce_send_nolock(struct ce_state *ce_state, 289 void *per_transfer_context, 290 u32 buffer, 291 unsigned int nbytes, 292 unsigned int transfer_id, 293 unsigned int flags) 294 { 295 struct ath10k *ar = ce_state->ar; 296 struct ce_ring_state *src_ring = ce_state->src_ring; 297 struct ce_desc *desc, *sdesc; 298 unsigned int nentries_mask = src_ring->nentries_mask; 299 unsigned int sw_index = src_ring->sw_index; 300 unsigned int write_index = src_ring->write_index; 301 u32 ctrl_addr = ce_state->ctrl_addr; 302 u32 desc_flags = 0; 303 int ret = 0; 304 305 if (nbytes > ce_state->src_sz_max) 306 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n", 307 __func__, nbytes, ce_state->src_sz_max); 308 309 ath10k_pci_wake(ar); 310 311 if (unlikely(CE_RING_DELTA(nentries_mask, 312 write_index, sw_index - 1) <= 0)) { 313 ret = -EIO; 314 goto exit; 315 } 316 317 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 318 write_index); 319 sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index); 320 321 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 322 323 if (flags & CE_SEND_FLAG_GATHER) 324 desc_flags |= CE_DESC_FLAGS_GATHER; 325 if (flags & CE_SEND_FLAG_BYTE_SWAP) 326 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 327 328 sdesc->addr = __cpu_to_le32(buffer); 329 sdesc->nbytes = __cpu_to_le16(nbytes); 330 sdesc->flags = __cpu_to_le16(desc_flags); 331 332 *desc = *sdesc; 333 334 src_ring->per_transfer_context[write_index] = per_transfer_context; 335 336 /* Update Source Ring Write Index */ 337 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 338 339 /* WORKAROUND */ 340 if (!(flags & CE_SEND_FLAG_GATHER)) 341 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 342 343 src_ring->write_index = write_index; 344 exit: 345 ath10k_pci_sleep(ar); 346 return ret; 347 } 348 349 int ath10k_ce_send(struct ce_state *ce_state, 350 void *per_transfer_context, 351 u32 buffer, 352 unsigned int nbytes, 353 unsigned int transfer_id, 354 unsigned int flags) 355 { 356 struct ath10k *ar = ce_state->ar; 357 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 358 int ret; 359 360 spin_lock_bh(&ar_pci->ce_lock); 361 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 362 buffer, nbytes, transfer_id, flags); 363 spin_unlock_bh(&ar_pci->ce_lock); 364 365 return ret; 366 } 367 368 void ath10k_ce_sendlist_buf_add(struct ce_sendlist *sendlist, u32 buffer, 369 unsigned int nbytes, u32 flags) 370 { 371 unsigned int num_items = sendlist->num_items; 372 struct ce_sendlist_item *item; 373 374 item = &sendlist->item[num_items]; 375 item->data = buffer; 376 item->u.nbytes = nbytes; 377 item->flags = flags; 378 sendlist->num_items++; 379 } 380 381 int ath10k_ce_sendlist_send(struct ce_state *ce_state, 382 void *per_transfer_context, 383 struct ce_sendlist *sendlist, 384 unsigned int transfer_id) 385 { 386 struct ce_ring_state *src_ring = ce_state->src_ring; 387 struct ce_sendlist_item *item; 388 struct ath10k *ar = ce_state->ar; 389 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 390 unsigned int nentries_mask = src_ring->nentries_mask; 391 unsigned int num_items = sendlist->num_items; 392 unsigned int sw_index; 393 unsigned int write_index; 394 int i, delta, ret = -ENOMEM; 395 396 spin_lock_bh(&ar_pci->ce_lock); 397 398 sw_index = src_ring->sw_index; 399 write_index = src_ring->write_index; 400 401 delta = CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 402 403 if (delta >= num_items) { 404 /* 405 * Handle all but the last item uniformly. 406 */ 407 for (i = 0; i < num_items - 1; i++) { 408 item = &sendlist->item[i]; 409 ret = ath10k_ce_send_nolock(ce_state, 410 CE_SENDLIST_ITEM_CTXT, 411 (u32) item->data, 412 item->u.nbytes, transfer_id, 413 item->flags | 414 CE_SEND_FLAG_GATHER); 415 if (ret) 416 ath10k_warn("CE send failed for item: %d\n", i); 417 } 418 /* 419 * Provide valid context pointer for final item. 420 */ 421 item = &sendlist->item[i]; 422 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 423 (u32) item->data, item->u.nbytes, 424 transfer_id, item->flags); 425 if (ret) 426 ath10k_warn("CE send failed for last item: %d\n", i); 427 } 428 429 spin_unlock_bh(&ar_pci->ce_lock); 430 431 return ret; 432 } 433 434 int ath10k_ce_recv_buf_enqueue(struct ce_state *ce_state, 435 void *per_recv_context, 436 u32 buffer) 437 { 438 struct ce_ring_state *dest_ring = ce_state->dest_ring; 439 u32 ctrl_addr = ce_state->ctrl_addr; 440 struct ath10k *ar = ce_state->ar; 441 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 442 unsigned int nentries_mask = dest_ring->nentries_mask; 443 unsigned int write_index; 444 unsigned int sw_index; 445 int ret; 446 447 spin_lock_bh(&ar_pci->ce_lock); 448 write_index = dest_ring->write_index; 449 sw_index = dest_ring->sw_index; 450 451 ath10k_pci_wake(ar); 452 453 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) { 454 struct ce_desc *base = dest_ring->base_addr_owner_space; 455 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 456 457 /* Update destination descriptor */ 458 desc->addr = __cpu_to_le32(buffer); 459 desc->nbytes = 0; 460 461 dest_ring->per_transfer_context[write_index] = 462 per_recv_context; 463 464 /* Update Destination Ring Write Index */ 465 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 466 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 467 dest_ring->write_index = write_index; 468 ret = 0; 469 } else { 470 ret = -EIO; 471 } 472 ath10k_pci_sleep(ar); 473 spin_unlock_bh(&ar_pci->ce_lock); 474 475 return ret; 476 } 477 478 /* 479 * Guts of ath10k_ce_completed_recv_next. 480 * The caller takes responsibility for any necessary locking. 481 */ 482 static int ath10k_ce_completed_recv_next_nolock(struct ce_state *ce_state, 483 void **per_transfer_contextp, 484 u32 *bufferp, 485 unsigned int *nbytesp, 486 unsigned int *transfer_idp, 487 unsigned int *flagsp) 488 { 489 struct ce_ring_state *dest_ring = ce_state->dest_ring; 490 unsigned int nentries_mask = dest_ring->nentries_mask; 491 unsigned int sw_index = dest_ring->sw_index; 492 493 struct ce_desc *base = dest_ring->base_addr_owner_space; 494 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 495 struct ce_desc sdesc; 496 u16 nbytes; 497 498 /* Copy in one go for performance reasons */ 499 sdesc = *desc; 500 501 nbytes = __le16_to_cpu(sdesc.nbytes); 502 if (nbytes == 0) { 503 /* 504 * This closes a relatively unusual race where the Host 505 * sees the updated DRRI before the update to the 506 * corresponding descriptor has completed. We treat this 507 * as a descriptor that is not yet done. 508 */ 509 return -EIO; 510 } 511 512 desc->nbytes = 0; 513 514 /* Return data from completed destination descriptor */ 515 *bufferp = __le32_to_cpu(sdesc.addr); 516 *nbytesp = nbytes; 517 *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA); 518 519 if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP) 520 *flagsp = CE_RECV_FLAG_SWAPPED; 521 else 522 *flagsp = 0; 523 524 if (per_transfer_contextp) 525 *per_transfer_contextp = 526 dest_ring->per_transfer_context[sw_index]; 527 528 /* sanity */ 529 dest_ring->per_transfer_context[sw_index] = NULL; 530 531 /* Update sw_index */ 532 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 533 dest_ring->sw_index = sw_index; 534 535 return 0; 536 } 537 538 int ath10k_ce_completed_recv_next(struct ce_state *ce_state, 539 void **per_transfer_contextp, 540 u32 *bufferp, 541 unsigned int *nbytesp, 542 unsigned int *transfer_idp, 543 unsigned int *flagsp) 544 { 545 struct ath10k *ar = ce_state->ar; 546 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 547 int ret; 548 549 spin_lock_bh(&ar_pci->ce_lock); 550 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 551 per_transfer_contextp, 552 bufferp, nbytesp, 553 transfer_idp, flagsp); 554 spin_unlock_bh(&ar_pci->ce_lock); 555 556 return ret; 557 } 558 559 int ath10k_ce_revoke_recv_next(struct ce_state *ce_state, 560 void **per_transfer_contextp, 561 u32 *bufferp) 562 { 563 struct ce_ring_state *dest_ring; 564 unsigned int nentries_mask; 565 unsigned int sw_index; 566 unsigned int write_index; 567 int ret; 568 struct ath10k *ar; 569 struct ath10k_pci *ar_pci; 570 571 dest_ring = ce_state->dest_ring; 572 573 if (!dest_ring) 574 return -EIO; 575 576 ar = ce_state->ar; 577 ar_pci = ath10k_pci_priv(ar); 578 579 spin_lock_bh(&ar_pci->ce_lock); 580 581 nentries_mask = dest_ring->nentries_mask; 582 sw_index = dest_ring->sw_index; 583 write_index = dest_ring->write_index; 584 if (write_index != sw_index) { 585 struct ce_desc *base = dest_ring->base_addr_owner_space; 586 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 587 588 /* Return data from completed destination descriptor */ 589 *bufferp = __le32_to_cpu(desc->addr); 590 591 if (per_transfer_contextp) 592 *per_transfer_contextp = 593 dest_ring->per_transfer_context[sw_index]; 594 595 /* sanity */ 596 dest_ring->per_transfer_context[sw_index] = NULL; 597 598 /* Update sw_index */ 599 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 600 dest_ring->sw_index = sw_index; 601 ret = 0; 602 } else { 603 ret = -EIO; 604 } 605 606 spin_unlock_bh(&ar_pci->ce_lock); 607 608 return ret; 609 } 610 611 /* 612 * Guts of ath10k_ce_completed_send_next. 613 * The caller takes responsibility for any necessary locking. 614 */ 615 static int ath10k_ce_completed_send_next_nolock(struct ce_state *ce_state, 616 void **per_transfer_contextp, 617 u32 *bufferp, 618 unsigned int *nbytesp, 619 unsigned int *transfer_idp) 620 { 621 struct ce_ring_state *src_ring = ce_state->src_ring; 622 u32 ctrl_addr = ce_state->ctrl_addr; 623 struct ath10k *ar = ce_state->ar; 624 unsigned int nentries_mask = src_ring->nentries_mask; 625 unsigned int sw_index = src_ring->sw_index; 626 unsigned int read_index; 627 int ret = -EIO; 628 629 if (src_ring->hw_index == sw_index) { 630 /* 631 * The SW completion index has caught up with the cached 632 * version of the HW completion index. 633 * Update the cached HW completion index to see whether 634 * the SW has really caught up to the HW, or if the cached 635 * value of the HW index has become stale. 636 */ 637 ath10k_pci_wake(ar); 638 src_ring->hw_index = 639 ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 640 ath10k_pci_sleep(ar); 641 } 642 read_index = src_ring->hw_index; 643 644 if ((read_index != sw_index) && (read_index != 0xffffffff)) { 645 struct ce_desc *sbase = src_ring->shadow_base; 646 struct ce_desc *sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index); 647 648 /* Return data from completed source descriptor */ 649 *bufferp = __le32_to_cpu(sdesc->addr); 650 *nbytesp = __le16_to_cpu(sdesc->nbytes); 651 *transfer_idp = MS(__le16_to_cpu(sdesc->flags), 652 CE_DESC_FLAGS_META_DATA); 653 654 if (per_transfer_contextp) 655 *per_transfer_contextp = 656 src_ring->per_transfer_context[sw_index]; 657 658 /* sanity */ 659 src_ring->per_transfer_context[sw_index] = NULL; 660 661 /* Update sw_index */ 662 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 663 src_ring->sw_index = sw_index; 664 ret = 0; 665 } 666 667 return ret; 668 } 669 670 /* NB: Modeled after ath10k_ce_completed_send_next */ 671 int ath10k_ce_cancel_send_next(struct ce_state *ce_state, 672 void **per_transfer_contextp, 673 u32 *bufferp, 674 unsigned int *nbytesp, 675 unsigned int *transfer_idp) 676 { 677 struct ce_ring_state *src_ring; 678 unsigned int nentries_mask; 679 unsigned int sw_index; 680 unsigned int write_index; 681 int ret; 682 struct ath10k *ar; 683 struct ath10k_pci *ar_pci; 684 685 src_ring = ce_state->src_ring; 686 687 if (!src_ring) 688 return -EIO; 689 690 ar = ce_state->ar; 691 ar_pci = ath10k_pci_priv(ar); 692 693 spin_lock_bh(&ar_pci->ce_lock); 694 695 nentries_mask = src_ring->nentries_mask; 696 sw_index = src_ring->sw_index; 697 write_index = src_ring->write_index; 698 699 if (write_index != sw_index) { 700 struct ce_desc *base = src_ring->base_addr_owner_space; 701 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 702 703 /* Return data from completed source descriptor */ 704 *bufferp = __le32_to_cpu(desc->addr); 705 *nbytesp = __le16_to_cpu(desc->nbytes); 706 *transfer_idp = MS(__le16_to_cpu(desc->flags), 707 CE_DESC_FLAGS_META_DATA); 708 709 if (per_transfer_contextp) 710 *per_transfer_contextp = 711 src_ring->per_transfer_context[sw_index]; 712 713 /* sanity */ 714 src_ring->per_transfer_context[sw_index] = NULL; 715 716 /* Update sw_index */ 717 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 718 src_ring->sw_index = sw_index; 719 ret = 0; 720 } else { 721 ret = -EIO; 722 } 723 724 spin_unlock_bh(&ar_pci->ce_lock); 725 726 return ret; 727 } 728 729 int ath10k_ce_completed_send_next(struct ce_state *ce_state, 730 void **per_transfer_contextp, 731 u32 *bufferp, 732 unsigned int *nbytesp, 733 unsigned int *transfer_idp) 734 { 735 struct ath10k *ar = ce_state->ar; 736 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 737 int ret; 738 739 spin_lock_bh(&ar_pci->ce_lock); 740 ret = ath10k_ce_completed_send_next_nolock(ce_state, 741 per_transfer_contextp, 742 bufferp, nbytesp, 743 transfer_idp); 744 spin_unlock_bh(&ar_pci->ce_lock); 745 746 return ret; 747 } 748 749 /* 750 * Guts of interrupt handler for per-engine interrupts on a particular CE. 751 * 752 * Invokes registered callbacks for recv_complete, 753 * send_complete, and watermarks. 754 */ 755 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 756 { 757 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 758 struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id]; 759 u32 ctrl_addr = ce_state->ctrl_addr; 760 void *transfer_context; 761 u32 buf; 762 unsigned int nbytes; 763 unsigned int id; 764 unsigned int flags; 765 766 ath10k_pci_wake(ar); 767 spin_lock_bh(&ar_pci->ce_lock); 768 769 /* Clear the copy-complete interrupts that will be handled here. */ 770 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 771 HOST_IS_COPY_COMPLETE_MASK); 772 773 if (ce_state->recv_cb) { 774 /* 775 * Pop completed recv buffers and call the registered 776 * recv callback for each 777 */ 778 while (ath10k_ce_completed_recv_next_nolock(ce_state, 779 &transfer_context, 780 &buf, &nbytes, 781 &id, &flags) == 0) { 782 spin_unlock_bh(&ar_pci->ce_lock); 783 ce_state->recv_cb(ce_state, transfer_context, buf, 784 nbytes, id, flags); 785 spin_lock_bh(&ar_pci->ce_lock); 786 } 787 } 788 789 if (ce_state->send_cb) { 790 /* 791 * Pop completed send buffers and call the registered 792 * send callback for each 793 */ 794 while (ath10k_ce_completed_send_next_nolock(ce_state, 795 &transfer_context, 796 &buf, 797 &nbytes, 798 &id) == 0) { 799 spin_unlock_bh(&ar_pci->ce_lock); 800 ce_state->send_cb(ce_state, transfer_context, 801 buf, nbytes, id); 802 spin_lock_bh(&ar_pci->ce_lock); 803 } 804 } 805 806 /* 807 * Misc CE interrupts are not being handled, but still need 808 * to be cleared. 809 */ 810 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); 811 812 spin_unlock_bh(&ar_pci->ce_lock); 813 ath10k_pci_sleep(ar); 814 } 815 816 /* 817 * Handler for per-engine interrupts on ALL active CEs. 818 * This is used in cases where the system is sharing a 819 * single interrput for all CEs 820 */ 821 822 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 823 { 824 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 825 int ce_id; 826 u32 intr_summary; 827 828 ath10k_pci_wake(ar); 829 intr_summary = CE_INTERRUPT_SUMMARY(ar); 830 831 for (ce_id = 0; intr_summary && (ce_id < ar_pci->ce_count); ce_id++) { 832 if (intr_summary & (1 << ce_id)) 833 intr_summary &= ~(1 << ce_id); 834 else 835 /* no intr pending on this CE */ 836 continue; 837 838 ath10k_ce_per_engine_service(ar, ce_id); 839 } 840 841 ath10k_pci_sleep(ar); 842 } 843 844 /* 845 * Adjust interrupts for the copy complete handler. 846 * If it's needed for either send or recv, then unmask 847 * this interrupt; otherwise, mask it. 848 * 849 * Called with ce_lock held. 850 */ 851 static void ath10k_ce_per_engine_handler_adjust(struct ce_state *ce_state, 852 int disable_copy_compl_intr) 853 { 854 u32 ctrl_addr = ce_state->ctrl_addr; 855 struct ath10k *ar = ce_state->ar; 856 857 ath10k_pci_wake(ar); 858 859 if ((!disable_copy_compl_intr) && 860 (ce_state->send_cb || ce_state->recv_cb)) 861 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 862 else 863 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 864 865 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 866 867 ath10k_pci_sleep(ar); 868 } 869 870 void ath10k_ce_disable_interrupts(struct ath10k *ar) 871 { 872 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 873 int ce_id; 874 875 ath10k_pci_wake(ar); 876 for (ce_id = 0; ce_id < ar_pci->ce_count; ce_id++) { 877 struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id]; 878 u32 ctrl_addr = ce_state->ctrl_addr; 879 880 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 881 } 882 ath10k_pci_sleep(ar); 883 } 884 885 void ath10k_ce_send_cb_register(struct ce_state *ce_state, 886 void (*send_cb) (struct ce_state *ce_state, 887 void *transfer_context, 888 u32 buffer, 889 unsigned int nbytes, 890 unsigned int transfer_id), 891 int disable_interrupts) 892 { 893 struct ath10k *ar = ce_state->ar; 894 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 895 896 spin_lock_bh(&ar_pci->ce_lock); 897 ce_state->send_cb = send_cb; 898 ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts); 899 spin_unlock_bh(&ar_pci->ce_lock); 900 } 901 902 void ath10k_ce_recv_cb_register(struct ce_state *ce_state, 903 void (*recv_cb) (struct ce_state *ce_state, 904 void *transfer_context, 905 u32 buffer, 906 unsigned int nbytes, 907 unsigned int transfer_id, 908 unsigned int flags)) 909 { 910 struct ath10k *ar = ce_state->ar; 911 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 912 913 spin_lock_bh(&ar_pci->ce_lock); 914 ce_state->recv_cb = recv_cb; 915 ath10k_ce_per_engine_handler_adjust(ce_state, 0); 916 spin_unlock_bh(&ar_pci->ce_lock); 917 } 918 919 static int ath10k_ce_init_src_ring(struct ath10k *ar, 920 unsigned int ce_id, 921 struct ce_state *ce_state, 922 const struct ce_attr *attr) 923 { 924 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 925 struct ce_ring_state *src_ring; 926 unsigned int nentries = attr->src_nentries; 927 unsigned int ce_nbytes; 928 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 929 dma_addr_t base_addr; 930 char *ptr; 931 932 nentries = roundup_pow_of_two(nentries); 933 934 if (ce_state->src_ring) { 935 WARN_ON(ce_state->src_ring->nentries != nentries); 936 return 0; 937 } 938 939 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); 940 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 941 if (ptr == NULL) 942 return -ENOMEM; 943 944 ce_state->src_ring = (struct ce_ring_state *)ptr; 945 src_ring = ce_state->src_ring; 946 947 ptr += sizeof(struct ce_ring_state); 948 src_ring->nentries = nentries; 949 src_ring->nentries_mask = nentries - 1; 950 951 ath10k_pci_wake(ar); 952 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 953 src_ring->hw_index = src_ring->sw_index; 954 955 src_ring->write_index = 956 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 957 ath10k_pci_sleep(ar); 958 959 src_ring->per_transfer_context = (void **)ptr; 960 961 /* 962 * Legacy platforms that do not support cache 963 * coherent DMA are unsupported 964 */ 965 src_ring->base_addr_owner_space_unaligned = 966 pci_alloc_consistent(ar_pci->pdev, 967 (nentries * sizeof(struct ce_desc) + 968 CE_DESC_RING_ALIGN), 969 &base_addr); 970 src_ring->base_addr_ce_space_unaligned = base_addr; 971 972 src_ring->base_addr_owner_space = PTR_ALIGN( 973 src_ring->base_addr_owner_space_unaligned, 974 CE_DESC_RING_ALIGN); 975 src_ring->base_addr_ce_space = ALIGN( 976 src_ring->base_addr_ce_space_unaligned, 977 CE_DESC_RING_ALIGN); 978 979 /* 980 * Also allocate a shadow src ring in regular 981 * mem to use for faster access. 982 */ 983 src_ring->shadow_base_unaligned = 984 kmalloc((nentries * sizeof(struct ce_desc) + 985 CE_DESC_RING_ALIGN), GFP_KERNEL); 986 987 src_ring->shadow_base = PTR_ALIGN( 988 src_ring->shadow_base_unaligned, 989 CE_DESC_RING_ALIGN); 990 991 ath10k_pci_wake(ar); 992 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 993 src_ring->base_addr_ce_space); 994 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 995 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 996 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 997 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 998 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 999 ath10k_pci_sleep(ar); 1000 1001 return 0; 1002 } 1003 1004 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1005 unsigned int ce_id, 1006 struct ce_state *ce_state, 1007 const struct ce_attr *attr) 1008 { 1009 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1010 struct ce_ring_state *dest_ring; 1011 unsigned int nentries = attr->dest_nentries; 1012 unsigned int ce_nbytes; 1013 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1014 dma_addr_t base_addr; 1015 char *ptr; 1016 1017 nentries = roundup_pow_of_two(nentries); 1018 1019 if (ce_state->dest_ring) { 1020 WARN_ON(ce_state->dest_ring->nentries != nentries); 1021 return 0; 1022 } 1023 1024 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); 1025 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 1026 if (ptr == NULL) 1027 return -ENOMEM; 1028 1029 ce_state->dest_ring = (struct ce_ring_state *)ptr; 1030 dest_ring = ce_state->dest_ring; 1031 1032 ptr += sizeof(struct ce_ring_state); 1033 dest_ring->nentries = nentries; 1034 dest_ring->nentries_mask = nentries - 1; 1035 1036 ath10k_pci_wake(ar); 1037 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1038 dest_ring->write_index = 1039 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1040 ath10k_pci_sleep(ar); 1041 1042 dest_ring->per_transfer_context = (void **)ptr; 1043 1044 /* 1045 * Legacy platforms that do not support cache 1046 * coherent DMA are unsupported 1047 */ 1048 dest_ring->base_addr_owner_space_unaligned = 1049 pci_alloc_consistent(ar_pci->pdev, 1050 (nentries * sizeof(struct ce_desc) + 1051 CE_DESC_RING_ALIGN), 1052 &base_addr); 1053 dest_ring->base_addr_ce_space_unaligned = base_addr; 1054 1055 /* 1056 * Correctly initialize memory to 0 to prevent garbage 1057 * data crashing system when download firmware 1058 */ 1059 memset(dest_ring->base_addr_owner_space_unaligned, 0, 1060 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN); 1061 1062 dest_ring->base_addr_owner_space = PTR_ALIGN( 1063 dest_ring->base_addr_owner_space_unaligned, 1064 CE_DESC_RING_ALIGN); 1065 dest_ring->base_addr_ce_space = ALIGN( 1066 dest_ring->base_addr_ce_space_unaligned, 1067 CE_DESC_RING_ALIGN); 1068 1069 ath10k_pci_wake(ar); 1070 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 1071 dest_ring->base_addr_ce_space); 1072 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1073 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1074 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1075 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1076 ath10k_pci_sleep(ar); 1077 1078 return 0; 1079 } 1080 1081 static struct ce_state *ath10k_ce_init_state(struct ath10k *ar, 1082 unsigned int ce_id, 1083 const struct ce_attr *attr) 1084 { 1085 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1086 struct ce_state *ce_state = NULL; 1087 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1088 1089 spin_lock_bh(&ar_pci->ce_lock); 1090 1091 if (!ar_pci->ce_id_to_state[ce_id]) { 1092 ce_state = kzalloc(sizeof(*ce_state), GFP_ATOMIC); 1093 if (ce_state == NULL) { 1094 spin_unlock_bh(&ar_pci->ce_lock); 1095 return NULL; 1096 } 1097 1098 ar_pci->ce_id_to_state[ce_id] = ce_state; 1099 ce_state->ar = ar; 1100 ce_state->id = ce_id; 1101 ce_state->ctrl_addr = ctrl_addr; 1102 ce_state->state = CE_RUNNING; 1103 /* Save attribute flags */ 1104 ce_state->attr_flags = attr->flags; 1105 ce_state->src_sz_max = attr->src_sz_max; 1106 } 1107 1108 spin_unlock_bh(&ar_pci->ce_lock); 1109 1110 return ce_state; 1111 } 1112 1113 /* 1114 * Initialize a Copy Engine based on caller-supplied attributes. 1115 * This may be called once to initialize both source and destination 1116 * rings or it may be called twice for separate source and destination 1117 * initialization. It may be that only one side or the other is 1118 * initialized by software/firmware. 1119 */ 1120 struct ce_state *ath10k_ce_init(struct ath10k *ar, 1121 unsigned int ce_id, 1122 const struct ce_attr *attr) 1123 { 1124 struct ce_state *ce_state; 1125 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1126 1127 ce_state = ath10k_ce_init_state(ar, ce_id, attr); 1128 if (!ce_state) { 1129 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id); 1130 return NULL; 1131 } 1132 1133 if (attr->src_nentries) { 1134 if (ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr)) { 1135 ath10k_err("Failed to initialize CE src ring for ID: %d\n", 1136 ce_id); 1137 ath10k_ce_deinit(ce_state); 1138 return NULL; 1139 } 1140 } 1141 1142 if (attr->dest_nentries) { 1143 if (ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr)) { 1144 ath10k_err("Failed to initialize CE dest ring for ID: %d\n", 1145 ce_id); 1146 ath10k_ce_deinit(ce_state); 1147 return NULL; 1148 } 1149 } 1150 1151 /* Enable CE error interrupts */ 1152 ath10k_pci_wake(ar); 1153 ath10k_ce_error_intr_enable(ar, ctrl_addr); 1154 ath10k_pci_sleep(ar); 1155 1156 return ce_state; 1157 } 1158 1159 void ath10k_ce_deinit(struct ce_state *ce_state) 1160 { 1161 unsigned int ce_id = ce_state->id; 1162 struct ath10k *ar = ce_state->ar; 1163 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1164 1165 ce_state->state = CE_UNUSED; 1166 ar_pci->ce_id_to_state[ce_id] = NULL; 1167 1168 if (ce_state->src_ring) { 1169 kfree(ce_state->src_ring->shadow_base_unaligned); 1170 pci_free_consistent(ar_pci->pdev, 1171 (ce_state->src_ring->nentries * 1172 sizeof(struct ce_desc) + 1173 CE_DESC_RING_ALIGN), 1174 ce_state->src_ring->base_addr_owner_space, 1175 ce_state->src_ring->base_addr_ce_space); 1176 kfree(ce_state->src_ring); 1177 } 1178 1179 if (ce_state->dest_ring) { 1180 pci_free_consistent(ar_pci->pdev, 1181 (ce_state->dest_ring->nentries * 1182 sizeof(struct ce_desc) + 1183 CE_DESC_RING_ALIGN), 1184 ce_state->dest_ring->base_addr_owner_space, 1185 ce_state->dest_ring->base_addr_ce_space); 1186 kfree(ce_state->dest_ring); 1187 } 1188 kfree(ce_state); 1189 } 1190