1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include "hif.h" 19 #include "pci.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * recieves an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 63 u32 ce_ctrl_addr, 64 unsigned int n) 65 { 66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); 67 } 68 69 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 70 u32 ce_ctrl_addr) 71 { 72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); 73 } 74 75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 76 u32 ce_ctrl_addr, 77 unsigned int n) 78 { 79 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 80 void __iomem *indicator_addr; 81 82 if (!test_bit(ATH10K_PCI_FEATURE_HW_1_0_WORKAROUND, ar_pci->features)) { 83 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 84 return; 85 } 86 87 /* workaround for QCA988x_1.0 HW CE */ 88 indicator_addr = ar_pci->mem + ce_ctrl_addr + DST_WATERMARK_ADDRESS; 89 90 if (ce_ctrl_addr == ath10k_ce_base_address(CDC_WAR_DATA_CE)) { 91 iowrite32((CDC_WAR_MAGIC_STR | n), indicator_addr); 92 } else { 93 unsigned long irq_flags; 94 local_irq_save(irq_flags); 95 iowrite32(1, indicator_addr); 96 97 /* 98 * PCIE write waits for ACK in IPQ8K, there is no 99 * need to read back value. 100 */ 101 (void)ioread32(indicator_addr); 102 (void)ioread32(indicator_addr); /* conservative */ 103 104 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 105 106 iowrite32(0, indicator_addr); 107 local_irq_restore(irq_flags); 108 } 109 } 110 111 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 112 u32 ce_ctrl_addr) 113 { 114 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); 115 } 116 117 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 118 u32 ce_ctrl_addr) 119 { 120 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); 121 } 122 123 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 124 u32 ce_ctrl_addr, 125 unsigned int addr) 126 { 127 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); 128 } 129 130 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 131 u32 ce_ctrl_addr, 132 unsigned int n) 133 { 134 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); 135 } 136 137 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 138 u32 ce_ctrl_addr, 139 unsigned int n) 140 { 141 u32 ctrl1_addr = ath10k_pci_read32((ar), 142 (ce_ctrl_addr) + CE_CTRL1_ADDRESS); 143 144 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 145 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | 146 CE_CTRL1_DMAX_LENGTH_SET(n)); 147 } 148 149 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 150 u32 ce_ctrl_addr, 151 unsigned int n) 152 { 153 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 154 155 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 156 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | 157 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); 158 } 159 160 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 161 u32 ce_ctrl_addr, 162 unsigned int n) 163 { 164 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 165 166 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 167 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | 168 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); 169 } 170 171 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 172 u32 ce_ctrl_addr) 173 { 174 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); 175 } 176 177 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 178 u32 ce_ctrl_addr, 179 u32 addr) 180 { 181 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); 182 } 183 184 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 unsigned int n) 187 { 188 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); 189 } 190 191 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 192 u32 ce_ctrl_addr, 193 unsigned int n) 194 { 195 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 196 197 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 198 (addr & ~SRC_WATERMARK_HIGH_MASK) | 199 SRC_WATERMARK_HIGH_SET(n)); 200 } 201 202 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 203 u32 ce_ctrl_addr, 204 unsigned int n) 205 { 206 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 207 208 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 209 (addr & ~SRC_WATERMARK_LOW_MASK) | 210 SRC_WATERMARK_LOW_SET(n)); 211 } 212 213 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 214 u32 ce_ctrl_addr, 215 unsigned int n) 216 { 217 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 218 219 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 220 (addr & ~DST_WATERMARK_HIGH_MASK) | 221 DST_WATERMARK_HIGH_SET(n)); 222 } 223 224 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 225 u32 ce_ctrl_addr, 226 unsigned int n) 227 { 228 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 229 230 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 231 (addr & ~DST_WATERMARK_LOW_MASK) | 232 DST_WATERMARK_LOW_SET(n)); 233 } 234 235 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 236 u32 ce_ctrl_addr) 237 { 238 u32 host_ie_addr = ath10k_pci_read32(ar, 239 ce_ctrl_addr + HOST_IE_ADDRESS); 240 241 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 242 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); 243 } 244 245 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 246 u32 ce_ctrl_addr) 247 { 248 u32 host_ie_addr = ath10k_pci_read32(ar, 249 ce_ctrl_addr + HOST_IE_ADDRESS); 250 251 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 252 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); 253 } 254 255 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 256 u32 ce_ctrl_addr) 257 { 258 u32 host_ie_addr = ath10k_pci_read32(ar, 259 ce_ctrl_addr + HOST_IE_ADDRESS); 260 261 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 262 host_ie_addr & ~CE_WATERMARK_MASK); 263 } 264 265 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 266 u32 ce_ctrl_addr) 267 { 268 u32 misc_ie_addr = ath10k_pci_read32(ar, 269 ce_ctrl_addr + MISC_IE_ADDRESS); 270 271 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 272 misc_ie_addr | CE_ERROR_MASK); 273 } 274 275 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 276 u32 ce_ctrl_addr, 277 unsigned int mask) 278 { 279 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); 280 } 281 282 283 /* 284 * Guts of ath10k_ce_send, used by both ath10k_ce_send and 285 * ath10k_ce_sendlist_send. 286 * The caller takes responsibility for any needed locking. 287 */ 288 static int ath10k_ce_send_nolock(struct ce_state *ce_state, 289 void *per_transfer_context, 290 u32 buffer, 291 unsigned int nbytes, 292 unsigned int transfer_id, 293 unsigned int flags) 294 { 295 struct ath10k *ar = ce_state->ar; 296 struct ce_ring_state *src_ring = ce_state->src_ring; 297 struct ce_desc *desc, *sdesc; 298 unsigned int nentries_mask = src_ring->nentries_mask; 299 unsigned int sw_index = src_ring->sw_index; 300 unsigned int write_index = src_ring->write_index; 301 u32 ctrl_addr = ce_state->ctrl_addr; 302 u32 desc_flags = 0; 303 int ret = 0; 304 305 if (nbytes > ce_state->src_sz_max) 306 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n", 307 __func__, nbytes, ce_state->src_sz_max); 308 309 ath10k_pci_wake(ar); 310 311 if (unlikely(CE_RING_DELTA(nentries_mask, 312 write_index, sw_index - 1) <= 0)) { 313 ret = -EIO; 314 goto exit; 315 } 316 317 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 318 write_index); 319 sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index); 320 321 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 322 323 if (flags & CE_SEND_FLAG_GATHER) 324 desc_flags |= CE_DESC_FLAGS_GATHER; 325 if (flags & CE_SEND_FLAG_BYTE_SWAP) 326 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 327 328 sdesc->addr = __cpu_to_le32(buffer); 329 sdesc->nbytes = __cpu_to_le16(nbytes); 330 sdesc->flags = __cpu_to_le16(desc_flags); 331 332 *desc = *sdesc; 333 334 src_ring->per_transfer_context[write_index] = per_transfer_context; 335 336 /* Update Source Ring Write Index */ 337 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 338 339 /* WORKAROUND */ 340 if (!(flags & CE_SEND_FLAG_GATHER)) 341 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 342 343 src_ring->write_index = write_index; 344 exit: 345 ath10k_pci_sleep(ar); 346 return ret; 347 } 348 349 int ath10k_ce_send(struct ce_state *ce_state, 350 void *per_transfer_context, 351 u32 buffer, 352 unsigned int nbytes, 353 unsigned int transfer_id, 354 unsigned int flags) 355 { 356 struct ath10k *ar = ce_state->ar; 357 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 358 int ret; 359 360 spin_lock_bh(&ar_pci->ce_lock); 361 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 362 buffer, nbytes, transfer_id, flags); 363 spin_unlock_bh(&ar_pci->ce_lock); 364 365 return ret; 366 } 367 368 void ath10k_ce_sendlist_buf_add(struct ce_sendlist *sendlist, u32 buffer, 369 unsigned int nbytes, u32 flags) 370 { 371 unsigned int num_items = sendlist->num_items; 372 struct ce_sendlist_item *item; 373 374 item = &sendlist->item[num_items]; 375 item->data = buffer; 376 item->u.nbytes = nbytes; 377 item->flags = flags; 378 sendlist->num_items++; 379 } 380 381 int ath10k_ce_sendlist_send(struct ce_state *ce_state, 382 void *per_transfer_context, 383 struct ce_sendlist *sendlist, 384 unsigned int transfer_id) 385 { 386 struct ce_ring_state *src_ring = ce_state->src_ring; 387 struct ce_sendlist_item *item; 388 struct ath10k *ar = ce_state->ar; 389 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 390 unsigned int nentries_mask = src_ring->nentries_mask; 391 unsigned int num_items = sendlist->num_items; 392 unsigned int sw_index; 393 unsigned int write_index; 394 int i, delta, ret = -ENOMEM; 395 396 spin_lock_bh(&ar_pci->ce_lock); 397 398 sw_index = src_ring->sw_index; 399 write_index = src_ring->write_index; 400 401 delta = CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 402 403 if (delta >= num_items) { 404 /* 405 * Handle all but the last item uniformly. 406 */ 407 for (i = 0; i < num_items - 1; i++) { 408 item = &sendlist->item[i]; 409 ret = ath10k_ce_send_nolock(ce_state, 410 CE_SENDLIST_ITEM_CTXT, 411 (u32) item->data, 412 item->u.nbytes, transfer_id, 413 item->flags | 414 CE_SEND_FLAG_GATHER); 415 if (ret) 416 ath10k_warn("CE send failed for item: %d\n", i); 417 } 418 /* 419 * Provide valid context pointer for final item. 420 */ 421 item = &sendlist->item[i]; 422 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 423 (u32) item->data, item->u.nbytes, 424 transfer_id, item->flags); 425 if (ret) 426 ath10k_warn("CE send failed for last item: %d\n", i); 427 } 428 429 spin_unlock_bh(&ar_pci->ce_lock); 430 431 return ret; 432 } 433 434 int ath10k_ce_recv_buf_enqueue(struct ce_state *ce_state, 435 void *per_recv_context, 436 u32 buffer) 437 { 438 struct ce_ring_state *dest_ring = ce_state->dest_ring; 439 u32 ctrl_addr = ce_state->ctrl_addr; 440 struct ath10k *ar = ce_state->ar; 441 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 442 unsigned int nentries_mask = dest_ring->nentries_mask; 443 unsigned int write_index; 444 unsigned int sw_index; 445 int ret; 446 447 spin_lock_bh(&ar_pci->ce_lock); 448 write_index = dest_ring->write_index; 449 sw_index = dest_ring->sw_index; 450 451 ath10k_pci_wake(ar); 452 453 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) { 454 struct ce_desc *base = dest_ring->base_addr_owner_space; 455 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 456 457 /* Update destination descriptor */ 458 desc->addr = __cpu_to_le32(buffer); 459 desc->nbytes = 0; 460 461 dest_ring->per_transfer_context[write_index] = 462 per_recv_context; 463 464 /* Update Destination Ring Write Index */ 465 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 466 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 467 dest_ring->write_index = write_index; 468 ret = 0; 469 } else { 470 ret = -EIO; 471 } 472 ath10k_pci_sleep(ar); 473 spin_unlock_bh(&ar_pci->ce_lock); 474 475 return ret; 476 } 477 478 /* 479 * Guts of ath10k_ce_completed_recv_next. 480 * The caller takes responsibility for any necessary locking. 481 */ 482 static int ath10k_ce_completed_recv_next_nolock(struct ce_state *ce_state, 483 void **per_transfer_contextp, 484 u32 *bufferp, 485 unsigned int *nbytesp, 486 unsigned int *transfer_idp, 487 unsigned int *flagsp) 488 { 489 struct ce_ring_state *dest_ring = ce_state->dest_ring; 490 unsigned int nentries_mask = dest_ring->nentries_mask; 491 unsigned int sw_index = dest_ring->sw_index; 492 493 struct ce_desc *base = dest_ring->base_addr_owner_space; 494 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 495 struct ce_desc sdesc; 496 u16 nbytes; 497 498 /* Copy in one go for performance reasons */ 499 sdesc = *desc; 500 501 nbytes = __le16_to_cpu(sdesc.nbytes); 502 if (nbytes == 0) { 503 /* 504 * This closes a relatively unusual race where the Host 505 * sees the updated DRRI before the update to the 506 * corresponding descriptor has completed. We treat this 507 * as a descriptor that is not yet done. 508 */ 509 return -EIO; 510 } 511 512 desc->nbytes = 0; 513 514 /* Return data from completed destination descriptor */ 515 *bufferp = __le32_to_cpu(sdesc.addr); 516 *nbytesp = nbytes; 517 *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA); 518 519 if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP) 520 *flagsp = CE_RECV_FLAG_SWAPPED; 521 else 522 *flagsp = 0; 523 524 if (per_transfer_contextp) 525 *per_transfer_contextp = 526 dest_ring->per_transfer_context[sw_index]; 527 528 /* sanity */ 529 dest_ring->per_transfer_context[sw_index] = NULL; 530 531 /* Update sw_index */ 532 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 533 dest_ring->sw_index = sw_index; 534 535 return 0; 536 } 537 538 int ath10k_ce_completed_recv_next(struct ce_state *ce_state, 539 void **per_transfer_contextp, 540 u32 *bufferp, 541 unsigned int *nbytesp, 542 unsigned int *transfer_idp, 543 unsigned int *flagsp) 544 { 545 struct ath10k *ar = ce_state->ar; 546 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 547 int ret; 548 549 spin_lock_bh(&ar_pci->ce_lock); 550 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 551 per_transfer_contextp, 552 bufferp, nbytesp, 553 transfer_idp, flagsp); 554 spin_unlock_bh(&ar_pci->ce_lock); 555 556 return ret; 557 } 558 559 int ath10k_ce_revoke_recv_next(struct ce_state *ce_state, 560 void **per_transfer_contextp, 561 u32 *bufferp) 562 { 563 struct ce_ring_state *dest_ring; 564 unsigned int nentries_mask; 565 unsigned int sw_index; 566 unsigned int write_index; 567 int ret; 568 struct ath10k *ar; 569 struct ath10k_pci *ar_pci; 570 571 dest_ring = ce_state->dest_ring; 572 573 if (!dest_ring) 574 return -EIO; 575 576 ar = ce_state->ar; 577 ar_pci = ath10k_pci_priv(ar); 578 579 spin_lock_bh(&ar_pci->ce_lock); 580 581 nentries_mask = dest_ring->nentries_mask; 582 sw_index = dest_ring->sw_index; 583 write_index = dest_ring->write_index; 584 if (write_index != sw_index) { 585 struct ce_desc *base = dest_ring->base_addr_owner_space; 586 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 587 588 /* Return data from completed destination descriptor */ 589 *bufferp = __le32_to_cpu(desc->addr); 590 591 if (per_transfer_contextp) 592 *per_transfer_contextp = 593 dest_ring->per_transfer_context[sw_index]; 594 595 /* sanity */ 596 dest_ring->per_transfer_context[sw_index] = NULL; 597 598 /* Update sw_index */ 599 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 600 dest_ring->sw_index = sw_index; 601 ret = 0; 602 } else { 603 ret = -EIO; 604 } 605 606 spin_unlock_bh(&ar_pci->ce_lock); 607 608 return ret; 609 } 610 611 /* 612 * Guts of ath10k_ce_completed_send_next. 613 * The caller takes responsibility for any necessary locking. 614 */ 615 static int ath10k_ce_completed_send_next_nolock(struct ce_state *ce_state, 616 void **per_transfer_contextp, 617 u32 *bufferp, 618 unsigned int *nbytesp, 619 unsigned int *transfer_idp) 620 { 621 struct ce_ring_state *src_ring = ce_state->src_ring; 622 u32 ctrl_addr = ce_state->ctrl_addr; 623 struct ath10k *ar = ce_state->ar; 624 unsigned int nentries_mask = src_ring->nentries_mask; 625 unsigned int sw_index = src_ring->sw_index; 626 unsigned int read_index; 627 int ret = -EIO; 628 629 if (src_ring->hw_index == sw_index) { 630 /* 631 * The SW completion index has caught up with the cached 632 * version of the HW completion index. 633 * Update the cached HW completion index to see whether 634 * the SW has really caught up to the HW, or if the cached 635 * value of the HW index has become stale. 636 */ 637 ath10k_pci_wake(ar); 638 src_ring->hw_index = 639 ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 640 src_ring->hw_index &= nentries_mask; 641 ath10k_pci_sleep(ar); 642 } 643 read_index = src_ring->hw_index; 644 645 if ((read_index != sw_index) && (read_index != 0xffffffff)) { 646 struct ce_desc *sbase = src_ring->shadow_base; 647 struct ce_desc *sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index); 648 649 /* Return data from completed source descriptor */ 650 *bufferp = __le32_to_cpu(sdesc->addr); 651 *nbytesp = __le16_to_cpu(sdesc->nbytes); 652 *transfer_idp = MS(__le16_to_cpu(sdesc->flags), 653 CE_DESC_FLAGS_META_DATA); 654 655 if (per_transfer_contextp) 656 *per_transfer_contextp = 657 src_ring->per_transfer_context[sw_index]; 658 659 /* sanity */ 660 src_ring->per_transfer_context[sw_index] = NULL; 661 662 /* Update sw_index */ 663 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 664 src_ring->sw_index = sw_index; 665 ret = 0; 666 } 667 668 return ret; 669 } 670 671 /* NB: Modeled after ath10k_ce_completed_send_next */ 672 int ath10k_ce_cancel_send_next(struct ce_state *ce_state, 673 void **per_transfer_contextp, 674 u32 *bufferp, 675 unsigned int *nbytesp, 676 unsigned int *transfer_idp) 677 { 678 struct ce_ring_state *src_ring; 679 unsigned int nentries_mask; 680 unsigned int sw_index; 681 unsigned int write_index; 682 int ret; 683 struct ath10k *ar; 684 struct ath10k_pci *ar_pci; 685 686 src_ring = ce_state->src_ring; 687 688 if (!src_ring) 689 return -EIO; 690 691 ar = ce_state->ar; 692 ar_pci = ath10k_pci_priv(ar); 693 694 spin_lock_bh(&ar_pci->ce_lock); 695 696 nentries_mask = src_ring->nentries_mask; 697 sw_index = src_ring->sw_index; 698 write_index = src_ring->write_index; 699 700 if (write_index != sw_index) { 701 struct ce_desc *base = src_ring->base_addr_owner_space; 702 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 703 704 /* Return data from completed source descriptor */ 705 *bufferp = __le32_to_cpu(desc->addr); 706 *nbytesp = __le16_to_cpu(desc->nbytes); 707 *transfer_idp = MS(__le16_to_cpu(desc->flags), 708 CE_DESC_FLAGS_META_DATA); 709 710 if (per_transfer_contextp) 711 *per_transfer_contextp = 712 src_ring->per_transfer_context[sw_index]; 713 714 /* sanity */ 715 src_ring->per_transfer_context[sw_index] = NULL; 716 717 /* Update sw_index */ 718 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 719 src_ring->sw_index = sw_index; 720 ret = 0; 721 } else { 722 ret = -EIO; 723 } 724 725 spin_unlock_bh(&ar_pci->ce_lock); 726 727 return ret; 728 } 729 730 int ath10k_ce_completed_send_next(struct ce_state *ce_state, 731 void **per_transfer_contextp, 732 u32 *bufferp, 733 unsigned int *nbytesp, 734 unsigned int *transfer_idp) 735 { 736 struct ath10k *ar = ce_state->ar; 737 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 738 int ret; 739 740 spin_lock_bh(&ar_pci->ce_lock); 741 ret = ath10k_ce_completed_send_next_nolock(ce_state, 742 per_transfer_contextp, 743 bufferp, nbytesp, 744 transfer_idp); 745 spin_unlock_bh(&ar_pci->ce_lock); 746 747 return ret; 748 } 749 750 /* 751 * Guts of interrupt handler for per-engine interrupts on a particular CE. 752 * 753 * Invokes registered callbacks for recv_complete, 754 * send_complete, and watermarks. 755 */ 756 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 757 { 758 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 759 struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id]; 760 u32 ctrl_addr = ce_state->ctrl_addr; 761 void *transfer_context; 762 u32 buf; 763 unsigned int nbytes; 764 unsigned int id; 765 unsigned int flags; 766 767 ath10k_pci_wake(ar); 768 spin_lock_bh(&ar_pci->ce_lock); 769 770 /* Clear the copy-complete interrupts that will be handled here. */ 771 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 772 HOST_IS_COPY_COMPLETE_MASK); 773 774 if (ce_state->recv_cb) { 775 /* 776 * Pop completed recv buffers and call the registered 777 * recv callback for each 778 */ 779 while (ath10k_ce_completed_recv_next_nolock(ce_state, 780 &transfer_context, 781 &buf, &nbytes, 782 &id, &flags) == 0) { 783 spin_unlock_bh(&ar_pci->ce_lock); 784 ce_state->recv_cb(ce_state, transfer_context, buf, 785 nbytes, id, flags); 786 spin_lock_bh(&ar_pci->ce_lock); 787 } 788 } 789 790 if (ce_state->send_cb) { 791 /* 792 * Pop completed send buffers and call the registered 793 * send callback for each 794 */ 795 while (ath10k_ce_completed_send_next_nolock(ce_state, 796 &transfer_context, 797 &buf, 798 &nbytes, 799 &id) == 0) { 800 spin_unlock_bh(&ar_pci->ce_lock); 801 ce_state->send_cb(ce_state, transfer_context, 802 buf, nbytes, id); 803 spin_lock_bh(&ar_pci->ce_lock); 804 } 805 } 806 807 /* 808 * Misc CE interrupts are not being handled, but still need 809 * to be cleared. 810 */ 811 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); 812 813 spin_unlock_bh(&ar_pci->ce_lock); 814 ath10k_pci_sleep(ar); 815 } 816 817 /* 818 * Handler for per-engine interrupts on ALL active CEs. 819 * This is used in cases where the system is sharing a 820 * single interrput for all CEs 821 */ 822 823 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 824 { 825 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 826 int ce_id; 827 u32 intr_summary; 828 829 ath10k_pci_wake(ar); 830 intr_summary = CE_INTERRUPT_SUMMARY(ar); 831 832 for (ce_id = 0; intr_summary && (ce_id < ar_pci->ce_count); ce_id++) { 833 if (intr_summary & (1 << ce_id)) 834 intr_summary &= ~(1 << ce_id); 835 else 836 /* no intr pending on this CE */ 837 continue; 838 839 ath10k_ce_per_engine_service(ar, ce_id); 840 } 841 842 ath10k_pci_sleep(ar); 843 } 844 845 /* 846 * Adjust interrupts for the copy complete handler. 847 * If it's needed for either send or recv, then unmask 848 * this interrupt; otherwise, mask it. 849 * 850 * Called with ce_lock held. 851 */ 852 static void ath10k_ce_per_engine_handler_adjust(struct ce_state *ce_state, 853 int disable_copy_compl_intr) 854 { 855 u32 ctrl_addr = ce_state->ctrl_addr; 856 struct ath10k *ar = ce_state->ar; 857 858 ath10k_pci_wake(ar); 859 860 if ((!disable_copy_compl_intr) && 861 (ce_state->send_cb || ce_state->recv_cb)) 862 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 863 else 864 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 865 866 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 867 868 ath10k_pci_sleep(ar); 869 } 870 871 void ath10k_ce_disable_interrupts(struct ath10k *ar) 872 { 873 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 874 int ce_id; 875 876 ath10k_pci_wake(ar); 877 for (ce_id = 0; ce_id < ar_pci->ce_count; ce_id++) { 878 struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id]; 879 u32 ctrl_addr = ce_state->ctrl_addr; 880 881 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 882 } 883 ath10k_pci_sleep(ar); 884 } 885 886 void ath10k_ce_send_cb_register(struct ce_state *ce_state, 887 void (*send_cb) (struct ce_state *ce_state, 888 void *transfer_context, 889 u32 buffer, 890 unsigned int nbytes, 891 unsigned int transfer_id), 892 int disable_interrupts) 893 { 894 struct ath10k *ar = ce_state->ar; 895 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 896 897 spin_lock_bh(&ar_pci->ce_lock); 898 ce_state->send_cb = send_cb; 899 ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts); 900 spin_unlock_bh(&ar_pci->ce_lock); 901 } 902 903 void ath10k_ce_recv_cb_register(struct ce_state *ce_state, 904 void (*recv_cb) (struct ce_state *ce_state, 905 void *transfer_context, 906 u32 buffer, 907 unsigned int nbytes, 908 unsigned int transfer_id, 909 unsigned int flags)) 910 { 911 struct ath10k *ar = ce_state->ar; 912 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 913 914 spin_lock_bh(&ar_pci->ce_lock); 915 ce_state->recv_cb = recv_cb; 916 ath10k_ce_per_engine_handler_adjust(ce_state, 0); 917 spin_unlock_bh(&ar_pci->ce_lock); 918 } 919 920 static int ath10k_ce_init_src_ring(struct ath10k *ar, 921 unsigned int ce_id, 922 struct ce_state *ce_state, 923 const struct ce_attr *attr) 924 { 925 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 926 struct ce_ring_state *src_ring; 927 unsigned int nentries = attr->src_nentries; 928 unsigned int ce_nbytes; 929 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 930 dma_addr_t base_addr; 931 char *ptr; 932 933 nentries = roundup_pow_of_two(nentries); 934 935 if (ce_state->src_ring) { 936 WARN_ON(ce_state->src_ring->nentries != nentries); 937 return 0; 938 } 939 940 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); 941 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 942 if (ptr == NULL) 943 return -ENOMEM; 944 945 ce_state->src_ring = (struct ce_ring_state *)ptr; 946 src_ring = ce_state->src_ring; 947 948 ptr += sizeof(struct ce_ring_state); 949 src_ring->nentries = nentries; 950 src_ring->nentries_mask = nentries - 1; 951 952 ath10k_pci_wake(ar); 953 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 954 src_ring->sw_index &= src_ring->nentries_mask; 955 src_ring->hw_index = src_ring->sw_index; 956 957 src_ring->write_index = 958 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 959 src_ring->write_index &= src_ring->nentries_mask; 960 ath10k_pci_sleep(ar); 961 962 src_ring->per_transfer_context = (void **)ptr; 963 964 /* 965 * Legacy platforms that do not support cache 966 * coherent DMA are unsupported 967 */ 968 src_ring->base_addr_owner_space_unaligned = 969 pci_alloc_consistent(ar_pci->pdev, 970 (nentries * sizeof(struct ce_desc) + 971 CE_DESC_RING_ALIGN), 972 &base_addr); 973 src_ring->base_addr_ce_space_unaligned = base_addr; 974 975 src_ring->base_addr_owner_space = PTR_ALIGN( 976 src_ring->base_addr_owner_space_unaligned, 977 CE_DESC_RING_ALIGN); 978 src_ring->base_addr_ce_space = ALIGN( 979 src_ring->base_addr_ce_space_unaligned, 980 CE_DESC_RING_ALIGN); 981 982 /* 983 * Also allocate a shadow src ring in regular 984 * mem to use for faster access. 985 */ 986 src_ring->shadow_base_unaligned = 987 kmalloc((nentries * sizeof(struct ce_desc) + 988 CE_DESC_RING_ALIGN), GFP_KERNEL); 989 990 src_ring->shadow_base = PTR_ALIGN( 991 src_ring->shadow_base_unaligned, 992 CE_DESC_RING_ALIGN); 993 994 ath10k_pci_wake(ar); 995 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 996 src_ring->base_addr_ce_space); 997 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 998 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 999 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 1000 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 1001 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 1002 ath10k_pci_sleep(ar); 1003 1004 return 0; 1005 } 1006 1007 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1008 unsigned int ce_id, 1009 struct ce_state *ce_state, 1010 const struct ce_attr *attr) 1011 { 1012 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1013 struct ce_ring_state *dest_ring; 1014 unsigned int nentries = attr->dest_nentries; 1015 unsigned int ce_nbytes; 1016 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1017 dma_addr_t base_addr; 1018 char *ptr; 1019 1020 nentries = roundup_pow_of_two(nentries); 1021 1022 if (ce_state->dest_ring) { 1023 WARN_ON(ce_state->dest_ring->nentries != nentries); 1024 return 0; 1025 } 1026 1027 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); 1028 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 1029 if (ptr == NULL) 1030 return -ENOMEM; 1031 1032 ce_state->dest_ring = (struct ce_ring_state *)ptr; 1033 dest_ring = ce_state->dest_ring; 1034 1035 ptr += sizeof(struct ce_ring_state); 1036 dest_ring->nentries = nentries; 1037 dest_ring->nentries_mask = nentries - 1; 1038 1039 ath10k_pci_wake(ar); 1040 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1041 dest_ring->sw_index &= dest_ring->nentries_mask; 1042 dest_ring->write_index = 1043 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1044 dest_ring->write_index &= dest_ring->nentries_mask; 1045 ath10k_pci_sleep(ar); 1046 1047 dest_ring->per_transfer_context = (void **)ptr; 1048 1049 /* 1050 * Legacy platforms that do not support cache 1051 * coherent DMA are unsupported 1052 */ 1053 dest_ring->base_addr_owner_space_unaligned = 1054 pci_alloc_consistent(ar_pci->pdev, 1055 (nentries * sizeof(struct ce_desc) + 1056 CE_DESC_RING_ALIGN), 1057 &base_addr); 1058 dest_ring->base_addr_ce_space_unaligned = base_addr; 1059 1060 /* 1061 * Correctly initialize memory to 0 to prevent garbage 1062 * data crashing system when download firmware 1063 */ 1064 memset(dest_ring->base_addr_owner_space_unaligned, 0, 1065 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN); 1066 1067 dest_ring->base_addr_owner_space = PTR_ALIGN( 1068 dest_ring->base_addr_owner_space_unaligned, 1069 CE_DESC_RING_ALIGN); 1070 dest_ring->base_addr_ce_space = ALIGN( 1071 dest_ring->base_addr_ce_space_unaligned, 1072 CE_DESC_RING_ALIGN); 1073 1074 ath10k_pci_wake(ar); 1075 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 1076 dest_ring->base_addr_ce_space); 1077 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1078 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1079 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1080 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1081 ath10k_pci_sleep(ar); 1082 1083 return 0; 1084 } 1085 1086 static struct ce_state *ath10k_ce_init_state(struct ath10k *ar, 1087 unsigned int ce_id, 1088 const struct ce_attr *attr) 1089 { 1090 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1091 struct ce_state *ce_state = NULL; 1092 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1093 1094 spin_lock_bh(&ar_pci->ce_lock); 1095 1096 if (!ar_pci->ce_id_to_state[ce_id]) { 1097 ce_state = kzalloc(sizeof(*ce_state), GFP_ATOMIC); 1098 if (ce_state == NULL) { 1099 spin_unlock_bh(&ar_pci->ce_lock); 1100 return NULL; 1101 } 1102 1103 ar_pci->ce_id_to_state[ce_id] = ce_state; 1104 ce_state->ar = ar; 1105 ce_state->id = ce_id; 1106 ce_state->ctrl_addr = ctrl_addr; 1107 ce_state->state = CE_RUNNING; 1108 /* Save attribute flags */ 1109 ce_state->attr_flags = attr->flags; 1110 ce_state->src_sz_max = attr->src_sz_max; 1111 } 1112 1113 spin_unlock_bh(&ar_pci->ce_lock); 1114 1115 return ce_state; 1116 } 1117 1118 /* 1119 * Initialize a Copy Engine based on caller-supplied attributes. 1120 * This may be called once to initialize both source and destination 1121 * rings or it may be called twice for separate source and destination 1122 * initialization. It may be that only one side or the other is 1123 * initialized by software/firmware. 1124 */ 1125 struct ce_state *ath10k_ce_init(struct ath10k *ar, 1126 unsigned int ce_id, 1127 const struct ce_attr *attr) 1128 { 1129 struct ce_state *ce_state; 1130 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1131 1132 ce_state = ath10k_ce_init_state(ar, ce_id, attr); 1133 if (!ce_state) { 1134 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id); 1135 return NULL; 1136 } 1137 1138 if (attr->src_nentries) { 1139 if (ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr)) { 1140 ath10k_err("Failed to initialize CE src ring for ID: %d\n", 1141 ce_id); 1142 ath10k_ce_deinit(ce_state); 1143 return NULL; 1144 } 1145 } 1146 1147 if (attr->dest_nentries) { 1148 if (ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr)) { 1149 ath10k_err("Failed to initialize CE dest ring for ID: %d\n", 1150 ce_id); 1151 ath10k_ce_deinit(ce_state); 1152 return NULL; 1153 } 1154 } 1155 1156 /* Enable CE error interrupts */ 1157 ath10k_pci_wake(ar); 1158 ath10k_ce_error_intr_enable(ar, ctrl_addr); 1159 ath10k_pci_sleep(ar); 1160 1161 return ce_state; 1162 } 1163 1164 void ath10k_ce_deinit(struct ce_state *ce_state) 1165 { 1166 unsigned int ce_id = ce_state->id; 1167 struct ath10k *ar = ce_state->ar; 1168 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1169 1170 ce_state->state = CE_UNUSED; 1171 ar_pci->ce_id_to_state[ce_id] = NULL; 1172 1173 if (ce_state->src_ring) { 1174 kfree(ce_state->src_ring->shadow_base_unaligned); 1175 pci_free_consistent(ar_pci->pdev, 1176 (ce_state->src_ring->nentries * 1177 sizeof(struct ce_desc) + 1178 CE_DESC_RING_ALIGN), 1179 ce_state->src_ring->base_addr_owner_space, 1180 ce_state->src_ring->base_addr_ce_space); 1181 kfree(ce_state->src_ring); 1182 } 1183 1184 if (ce_state->dest_ring) { 1185 pci_free_consistent(ar_pci->pdev, 1186 (ce_state->dest_ring->nentries * 1187 sizeof(struct ce_desc) + 1188 CE_DESC_RING_ALIGN), 1189 ce_state->dest_ring->base_addr_owner_space, 1190 ce_state->dest_ring->base_addr_ce_space); 1191 kfree(ce_state->dest_ring); 1192 } 1193 kfree(ce_state); 1194 } 1195