1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include "hif.h" 19 #include "pci.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * receives an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 63 u32 ce_ctrl_addr, 64 unsigned int n) 65 { 66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); 67 } 68 69 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 70 u32 ce_ctrl_addr) 71 { 72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); 73 } 74 75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 76 u32 ce_ctrl_addr, 77 unsigned int n) 78 { 79 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 80 } 81 82 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 83 u32 ce_ctrl_addr) 84 { 85 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); 86 } 87 88 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 89 u32 ce_ctrl_addr) 90 { 91 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); 92 } 93 94 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 95 u32 ce_ctrl_addr, 96 unsigned int addr) 97 { 98 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); 99 } 100 101 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 102 u32 ce_ctrl_addr, 103 unsigned int n) 104 { 105 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); 106 } 107 108 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 109 u32 ce_ctrl_addr, 110 unsigned int n) 111 { 112 u32 ctrl1_addr = ath10k_pci_read32((ar), 113 (ce_ctrl_addr) + CE_CTRL1_ADDRESS); 114 115 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 116 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | 117 CE_CTRL1_DMAX_LENGTH_SET(n)); 118 } 119 120 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 121 u32 ce_ctrl_addr, 122 unsigned int n) 123 { 124 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 125 126 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 127 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | 128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); 129 } 130 131 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 132 u32 ce_ctrl_addr, 133 unsigned int n) 134 { 135 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 136 137 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 138 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | 139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); 140 } 141 142 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 143 u32 ce_ctrl_addr) 144 { 145 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); 146 } 147 148 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 149 u32 ce_ctrl_addr, 150 u32 addr) 151 { 152 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); 153 } 154 155 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 156 u32 ce_ctrl_addr, 157 unsigned int n) 158 { 159 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); 160 } 161 162 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 163 u32 ce_ctrl_addr, 164 unsigned int n) 165 { 166 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 167 168 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 169 (addr & ~SRC_WATERMARK_HIGH_MASK) | 170 SRC_WATERMARK_HIGH_SET(n)); 171 } 172 173 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 unsigned int n) 176 { 177 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 178 179 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 180 (addr & ~SRC_WATERMARK_LOW_MASK) | 181 SRC_WATERMARK_LOW_SET(n)); 182 } 183 184 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 unsigned int n) 187 { 188 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 189 190 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 191 (addr & ~DST_WATERMARK_HIGH_MASK) | 192 DST_WATERMARK_HIGH_SET(n)); 193 } 194 195 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 196 u32 ce_ctrl_addr, 197 unsigned int n) 198 { 199 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 200 201 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 202 (addr & ~DST_WATERMARK_LOW_MASK) | 203 DST_WATERMARK_LOW_SET(n)); 204 } 205 206 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 207 u32 ce_ctrl_addr) 208 { 209 u32 host_ie_addr = ath10k_pci_read32(ar, 210 ce_ctrl_addr + HOST_IE_ADDRESS); 211 212 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 213 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); 214 } 215 216 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 217 u32 ce_ctrl_addr) 218 { 219 u32 host_ie_addr = ath10k_pci_read32(ar, 220 ce_ctrl_addr + HOST_IE_ADDRESS); 221 222 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 223 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); 224 } 225 226 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 227 u32 ce_ctrl_addr) 228 { 229 u32 host_ie_addr = ath10k_pci_read32(ar, 230 ce_ctrl_addr + HOST_IE_ADDRESS); 231 232 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 233 host_ie_addr & ~CE_WATERMARK_MASK); 234 } 235 236 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 237 u32 ce_ctrl_addr) 238 { 239 u32 misc_ie_addr = ath10k_pci_read32(ar, 240 ce_ctrl_addr + MISC_IE_ADDRESS); 241 242 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 243 misc_ie_addr | CE_ERROR_MASK); 244 } 245 246 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 247 u32 ce_ctrl_addr) 248 { 249 u32 misc_ie_addr = ath10k_pci_read32(ar, 250 ce_ctrl_addr + MISC_IE_ADDRESS); 251 252 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 253 misc_ie_addr & ~CE_ERROR_MASK); 254 } 255 256 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 257 u32 ce_ctrl_addr, 258 unsigned int mask) 259 { 260 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); 261 } 262 263 /* 264 * Guts of ath10k_ce_send, used by both ath10k_ce_send and 265 * ath10k_ce_sendlist_send. 266 * The caller takes responsibility for any needed locking. 267 */ 268 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 269 void *per_transfer_context, 270 u32 buffer, 271 unsigned int nbytes, 272 unsigned int transfer_id, 273 unsigned int flags) 274 { 275 struct ath10k *ar = ce_state->ar; 276 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 277 struct ce_desc *desc, sdesc; 278 unsigned int nentries_mask = src_ring->nentries_mask; 279 unsigned int sw_index = src_ring->sw_index; 280 unsigned int write_index = src_ring->write_index; 281 u32 ctrl_addr = ce_state->ctrl_addr; 282 u32 desc_flags = 0; 283 int ret = 0; 284 285 if (nbytes > ce_state->src_sz_max) 286 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 287 __func__, nbytes, ce_state->src_sz_max); 288 289 if (unlikely(CE_RING_DELTA(nentries_mask, 290 write_index, sw_index - 1) <= 0)) { 291 ret = -ENOSR; 292 goto exit; 293 } 294 295 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 296 write_index); 297 298 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 299 300 if (flags & CE_SEND_FLAG_GATHER) 301 desc_flags |= CE_DESC_FLAGS_GATHER; 302 if (flags & CE_SEND_FLAG_BYTE_SWAP) 303 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 304 305 sdesc.addr = __cpu_to_le32(buffer); 306 sdesc.nbytes = __cpu_to_le16(nbytes); 307 sdesc.flags = __cpu_to_le16(desc_flags); 308 309 *desc = sdesc; 310 311 src_ring->per_transfer_context[write_index] = per_transfer_context; 312 313 /* Update Source Ring Write Index */ 314 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 315 316 /* WORKAROUND */ 317 if (!(flags & CE_SEND_FLAG_GATHER)) 318 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 319 320 src_ring->write_index = write_index; 321 exit: 322 return ret; 323 } 324 325 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 326 { 327 struct ath10k *ar = pipe->ar; 328 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 329 struct ath10k_ce_ring *src_ring = pipe->src_ring; 330 u32 ctrl_addr = pipe->ctrl_addr; 331 332 lockdep_assert_held(&ar_pci->ce_lock); 333 334 /* 335 * This function must be called only if there is an incomplete 336 * scatter-gather transfer (before index register is updated) 337 * that needs to be cleaned up. 338 */ 339 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 340 return; 341 342 if (WARN_ON_ONCE(src_ring->write_index == 343 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 344 return; 345 346 src_ring->write_index--; 347 src_ring->write_index &= src_ring->nentries_mask; 348 349 src_ring->per_transfer_context[src_ring->write_index] = NULL; 350 } 351 352 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 353 void *per_transfer_context, 354 u32 buffer, 355 unsigned int nbytes, 356 unsigned int transfer_id, 357 unsigned int flags) 358 { 359 struct ath10k *ar = ce_state->ar; 360 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 361 int ret; 362 363 spin_lock_bh(&ar_pci->ce_lock); 364 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 365 buffer, nbytes, transfer_id, flags); 366 spin_unlock_bh(&ar_pci->ce_lock); 367 368 return ret; 369 } 370 371 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 372 { 373 struct ath10k *ar = pipe->ar; 374 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 375 int delta; 376 377 spin_lock_bh(&ar_pci->ce_lock); 378 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 379 pipe->src_ring->write_index, 380 pipe->src_ring->sw_index - 1); 381 spin_unlock_bh(&ar_pci->ce_lock); 382 383 return delta; 384 } 385 386 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 387 { 388 struct ath10k *ar = pipe->ar; 389 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 390 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 391 unsigned int nentries_mask = dest_ring->nentries_mask; 392 unsigned int write_index = dest_ring->write_index; 393 unsigned int sw_index = dest_ring->sw_index; 394 395 lockdep_assert_held(&ar_pci->ce_lock); 396 397 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 398 } 399 400 int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr) 401 { 402 struct ath10k *ar = pipe->ar; 403 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 404 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 405 unsigned int nentries_mask = dest_ring->nentries_mask; 406 unsigned int write_index = dest_ring->write_index; 407 unsigned int sw_index = dest_ring->sw_index; 408 struct ce_desc *base = dest_ring->base_addr_owner_space; 409 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 410 u32 ctrl_addr = pipe->ctrl_addr; 411 412 lockdep_assert_held(&ar_pci->ce_lock); 413 414 if ((pipe->id != 5) && 415 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 416 return -ENOSPC; 417 418 desc->addr = __cpu_to_le32(paddr); 419 desc->nbytes = 0; 420 421 dest_ring->per_transfer_context[write_index] = ctx; 422 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 423 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 424 dest_ring->write_index = write_index; 425 426 return 0; 427 } 428 429 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 430 { 431 struct ath10k *ar = pipe->ar; 432 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 433 unsigned int nentries_mask = dest_ring->nentries_mask; 434 unsigned int write_index = dest_ring->write_index; 435 u32 ctrl_addr = pipe->ctrl_addr; 436 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 437 438 /* Prevent CE ring stuck issue that will occur when ring is full. 439 * Make sure that write index is 1 less than read index. 440 */ 441 if ((cur_write_idx + nentries) == dest_ring->sw_index) 442 nentries -= 1; 443 444 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 445 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 446 dest_ring->write_index = write_index; 447 } 448 449 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr) 450 { 451 struct ath10k *ar = pipe->ar; 452 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 453 int ret; 454 455 spin_lock_bh(&ar_pci->ce_lock); 456 ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr); 457 spin_unlock_bh(&ar_pci->ce_lock); 458 459 return ret; 460 } 461 462 /* 463 * Guts of ath10k_ce_completed_recv_next. 464 * The caller takes responsibility for any necessary locking. 465 */ 466 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 467 void **per_transfer_contextp, 468 unsigned int *nbytesp) 469 { 470 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 471 unsigned int nentries_mask = dest_ring->nentries_mask; 472 unsigned int sw_index = dest_ring->sw_index; 473 474 struct ce_desc *base = dest_ring->base_addr_owner_space; 475 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 476 struct ce_desc sdesc; 477 u16 nbytes; 478 479 /* Copy in one go for performance reasons */ 480 sdesc = *desc; 481 482 nbytes = __le16_to_cpu(sdesc.nbytes); 483 if (nbytes == 0) { 484 /* 485 * This closes a relatively unusual race where the Host 486 * sees the updated DRRI before the update to the 487 * corresponding descriptor has completed. We treat this 488 * as a descriptor that is not yet done. 489 */ 490 return -EIO; 491 } 492 493 desc->nbytes = 0; 494 495 /* Return data from completed destination descriptor */ 496 *nbytesp = nbytes; 497 498 if (per_transfer_contextp) 499 *per_transfer_contextp = 500 dest_ring->per_transfer_context[sw_index]; 501 502 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 503 * So update transfer context all CEs except CE5. 504 */ 505 if (ce_state->id != 5) 506 dest_ring->per_transfer_context[sw_index] = NULL; 507 508 /* Update sw_index */ 509 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 510 dest_ring->sw_index = sw_index; 511 512 return 0; 513 } 514 515 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 516 void **per_transfer_contextp, 517 unsigned int *nbytesp) 518 { 519 struct ath10k *ar = ce_state->ar; 520 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 521 int ret; 522 523 spin_lock_bh(&ar_pci->ce_lock); 524 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 525 per_transfer_contextp, 526 nbytesp); 527 spin_unlock_bh(&ar_pci->ce_lock); 528 529 return ret; 530 } 531 532 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 533 void **per_transfer_contextp, 534 u32 *bufferp) 535 { 536 struct ath10k_ce_ring *dest_ring; 537 unsigned int nentries_mask; 538 unsigned int sw_index; 539 unsigned int write_index; 540 int ret; 541 struct ath10k *ar; 542 struct ath10k_pci *ar_pci; 543 544 dest_ring = ce_state->dest_ring; 545 546 if (!dest_ring) 547 return -EIO; 548 549 ar = ce_state->ar; 550 ar_pci = ath10k_pci_priv(ar); 551 552 spin_lock_bh(&ar_pci->ce_lock); 553 554 nentries_mask = dest_ring->nentries_mask; 555 sw_index = dest_ring->sw_index; 556 write_index = dest_ring->write_index; 557 if (write_index != sw_index) { 558 struct ce_desc *base = dest_ring->base_addr_owner_space; 559 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 560 561 /* Return data from completed destination descriptor */ 562 *bufferp = __le32_to_cpu(desc->addr); 563 564 if (per_transfer_contextp) 565 *per_transfer_contextp = 566 dest_ring->per_transfer_context[sw_index]; 567 568 /* sanity */ 569 dest_ring->per_transfer_context[sw_index] = NULL; 570 desc->nbytes = 0; 571 572 /* Update sw_index */ 573 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 574 dest_ring->sw_index = sw_index; 575 ret = 0; 576 } else { 577 ret = -EIO; 578 } 579 580 spin_unlock_bh(&ar_pci->ce_lock); 581 582 return ret; 583 } 584 585 /* 586 * Guts of ath10k_ce_completed_send_next. 587 * The caller takes responsibility for any necessary locking. 588 */ 589 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 590 void **per_transfer_contextp) 591 { 592 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 593 u32 ctrl_addr = ce_state->ctrl_addr; 594 struct ath10k *ar = ce_state->ar; 595 unsigned int nentries_mask = src_ring->nentries_mask; 596 unsigned int sw_index = src_ring->sw_index; 597 unsigned int read_index; 598 599 if (src_ring->hw_index == sw_index) { 600 /* 601 * The SW completion index has caught up with the cached 602 * version of the HW completion index. 603 * Update the cached HW completion index to see whether 604 * the SW has really caught up to the HW, or if the cached 605 * value of the HW index has become stale. 606 */ 607 608 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 609 if (read_index == 0xffffffff) 610 return -ENODEV; 611 612 read_index &= nentries_mask; 613 src_ring->hw_index = read_index; 614 } 615 616 read_index = src_ring->hw_index; 617 618 if (read_index == sw_index) 619 return -EIO; 620 621 if (per_transfer_contextp) 622 *per_transfer_contextp = 623 src_ring->per_transfer_context[sw_index]; 624 625 /* sanity */ 626 src_ring->per_transfer_context[sw_index] = NULL; 627 628 /* Update sw_index */ 629 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 630 src_ring->sw_index = sw_index; 631 632 return 0; 633 } 634 635 /* NB: Modeled after ath10k_ce_completed_send_next */ 636 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 637 void **per_transfer_contextp, 638 u32 *bufferp, 639 unsigned int *nbytesp, 640 unsigned int *transfer_idp) 641 { 642 struct ath10k_ce_ring *src_ring; 643 unsigned int nentries_mask; 644 unsigned int sw_index; 645 unsigned int write_index; 646 int ret; 647 struct ath10k *ar; 648 struct ath10k_pci *ar_pci; 649 650 src_ring = ce_state->src_ring; 651 652 if (!src_ring) 653 return -EIO; 654 655 ar = ce_state->ar; 656 ar_pci = ath10k_pci_priv(ar); 657 658 spin_lock_bh(&ar_pci->ce_lock); 659 660 nentries_mask = src_ring->nentries_mask; 661 sw_index = src_ring->sw_index; 662 write_index = src_ring->write_index; 663 664 if (write_index != sw_index) { 665 struct ce_desc *base = src_ring->base_addr_owner_space; 666 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 667 668 /* Return data from completed source descriptor */ 669 *bufferp = __le32_to_cpu(desc->addr); 670 *nbytesp = __le16_to_cpu(desc->nbytes); 671 *transfer_idp = MS(__le16_to_cpu(desc->flags), 672 CE_DESC_FLAGS_META_DATA); 673 674 if (per_transfer_contextp) 675 *per_transfer_contextp = 676 src_ring->per_transfer_context[sw_index]; 677 678 /* sanity */ 679 src_ring->per_transfer_context[sw_index] = NULL; 680 681 /* Update sw_index */ 682 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 683 src_ring->sw_index = sw_index; 684 ret = 0; 685 } else { 686 ret = -EIO; 687 } 688 689 spin_unlock_bh(&ar_pci->ce_lock); 690 691 return ret; 692 } 693 694 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 695 void **per_transfer_contextp) 696 { 697 struct ath10k *ar = ce_state->ar; 698 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 699 int ret; 700 701 spin_lock_bh(&ar_pci->ce_lock); 702 ret = ath10k_ce_completed_send_next_nolock(ce_state, 703 per_transfer_contextp); 704 spin_unlock_bh(&ar_pci->ce_lock); 705 706 return ret; 707 } 708 709 /* 710 * Guts of interrupt handler for per-engine interrupts on a particular CE. 711 * 712 * Invokes registered callbacks for recv_complete, 713 * send_complete, and watermarks. 714 */ 715 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 716 { 717 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 718 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 719 u32 ctrl_addr = ce_state->ctrl_addr; 720 721 spin_lock_bh(&ar_pci->ce_lock); 722 723 /* Clear the copy-complete interrupts that will be handled here. */ 724 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 725 HOST_IS_COPY_COMPLETE_MASK); 726 727 spin_unlock_bh(&ar_pci->ce_lock); 728 729 if (ce_state->recv_cb) 730 ce_state->recv_cb(ce_state); 731 732 if (ce_state->send_cb) 733 ce_state->send_cb(ce_state); 734 735 spin_lock_bh(&ar_pci->ce_lock); 736 737 /* 738 * Misc CE interrupts are not being handled, but still need 739 * to be cleared. 740 */ 741 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); 742 743 spin_unlock_bh(&ar_pci->ce_lock); 744 } 745 746 /* 747 * Handler for per-engine interrupts on ALL active CEs. 748 * This is used in cases where the system is sharing a 749 * single interrput for all CEs 750 */ 751 752 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 753 { 754 int ce_id; 755 u32 intr_summary; 756 757 intr_summary = CE_INTERRUPT_SUMMARY(ar); 758 759 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 760 if (intr_summary & (1 << ce_id)) 761 intr_summary &= ~(1 << ce_id); 762 else 763 /* no intr pending on this CE */ 764 continue; 765 766 ath10k_ce_per_engine_service(ar, ce_id); 767 } 768 } 769 770 /* 771 * Adjust interrupts for the copy complete handler. 772 * If it's needed for either send or recv, then unmask 773 * this interrupt; otherwise, mask it. 774 * 775 * Called with ce_lock held. 776 */ 777 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 778 { 779 u32 ctrl_addr = ce_state->ctrl_addr; 780 struct ath10k *ar = ce_state->ar; 781 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 782 783 if ((!disable_copy_compl_intr) && 784 (ce_state->send_cb || ce_state->recv_cb)) 785 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 786 else 787 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 788 789 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 790 } 791 792 int ath10k_ce_disable_interrupts(struct ath10k *ar) 793 { 794 int ce_id; 795 796 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 797 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 798 799 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 800 ath10k_ce_error_intr_disable(ar, ctrl_addr); 801 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 802 } 803 804 return 0; 805 } 806 807 void ath10k_ce_enable_interrupts(struct ath10k *ar) 808 { 809 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 810 int ce_id; 811 812 /* Skip the last copy engine, CE7 the diagnostic window, as that 813 * uses polling and isn't initialized for interrupts. 814 */ 815 for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++) 816 ath10k_ce_per_engine_handler_adjust(&ar_pci->ce_states[ce_id]); 817 } 818 819 static int ath10k_ce_init_src_ring(struct ath10k *ar, 820 unsigned int ce_id, 821 const struct ce_attr *attr) 822 { 823 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 824 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 825 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 826 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 827 828 nentries = roundup_pow_of_two(attr->src_nentries); 829 830 memset(src_ring->base_addr_owner_space, 0, 831 nentries * sizeof(struct ce_desc)); 832 833 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 834 src_ring->sw_index &= src_ring->nentries_mask; 835 src_ring->hw_index = src_ring->sw_index; 836 837 src_ring->write_index = 838 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 839 src_ring->write_index &= src_ring->nentries_mask; 840 841 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 842 src_ring->base_addr_ce_space); 843 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 844 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 845 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 846 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 847 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 848 849 ath10k_dbg(ar, ATH10K_DBG_BOOT, 850 "boot init ce src ring id %d entries %d base_addr %pK\n", 851 ce_id, nentries, src_ring->base_addr_owner_space); 852 853 return 0; 854 } 855 856 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 857 unsigned int ce_id, 858 const struct ce_attr *attr) 859 { 860 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 861 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 862 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 863 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 864 865 nentries = roundup_pow_of_two(attr->dest_nentries); 866 867 memset(dest_ring->base_addr_owner_space, 0, 868 nentries * sizeof(struct ce_desc)); 869 870 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 871 dest_ring->sw_index &= dest_ring->nentries_mask; 872 dest_ring->write_index = 873 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 874 dest_ring->write_index &= dest_ring->nentries_mask; 875 876 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 877 dest_ring->base_addr_ce_space); 878 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 879 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 880 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 881 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 882 883 ath10k_dbg(ar, ATH10K_DBG_BOOT, 884 "boot ce dest ring id %d entries %d base_addr %pK\n", 885 ce_id, nentries, dest_ring->base_addr_owner_space); 886 887 return 0; 888 } 889 890 static struct ath10k_ce_ring * 891 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 892 const struct ce_attr *attr) 893 { 894 struct ath10k_ce_ring *src_ring; 895 u32 nentries = attr->src_nentries; 896 dma_addr_t base_addr; 897 898 nentries = roundup_pow_of_two(nentries); 899 900 src_ring = kzalloc(sizeof(*src_ring) + 901 (nentries * 902 sizeof(*src_ring->per_transfer_context)), 903 GFP_KERNEL); 904 if (src_ring == NULL) 905 return ERR_PTR(-ENOMEM); 906 907 src_ring->nentries = nentries; 908 src_ring->nentries_mask = nentries - 1; 909 910 /* 911 * Legacy platforms that do not support cache 912 * coherent DMA are unsupported 913 */ 914 src_ring->base_addr_owner_space_unaligned = 915 dma_alloc_coherent(ar->dev, 916 (nentries * sizeof(struct ce_desc) + 917 CE_DESC_RING_ALIGN), 918 &base_addr, GFP_KERNEL); 919 if (!src_ring->base_addr_owner_space_unaligned) { 920 kfree(src_ring); 921 return ERR_PTR(-ENOMEM); 922 } 923 924 src_ring->base_addr_ce_space_unaligned = base_addr; 925 926 src_ring->base_addr_owner_space = PTR_ALIGN( 927 src_ring->base_addr_owner_space_unaligned, 928 CE_DESC_RING_ALIGN); 929 src_ring->base_addr_ce_space = ALIGN( 930 src_ring->base_addr_ce_space_unaligned, 931 CE_DESC_RING_ALIGN); 932 933 return src_ring; 934 } 935 936 static struct ath10k_ce_ring * 937 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 938 const struct ce_attr *attr) 939 { 940 struct ath10k_ce_ring *dest_ring; 941 u32 nentries; 942 dma_addr_t base_addr; 943 944 nentries = roundup_pow_of_two(attr->dest_nentries); 945 946 dest_ring = kzalloc(sizeof(*dest_ring) + 947 (nentries * 948 sizeof(*dest_ring->per_transfer_context)), 949 GFP_KERNEL); 950 if (dest_ring == NULL) 951 return ERR_PTR(-ENOMEM); 952 953 dest_ring->nentries = nentries; 954 dest_ring->nentries_mask = nentries - 1; 955 956 /* 957 * Legacy platforms that do not support cache 958 * coherent DMA are unsupported 959 */ 960 dest_ring->base_addr_owner_space_unaligned = 961 dma_alloc_coherent(ar->dev, 962 (nentries * sizeof(struct ce_desc) + 963 CE_DESC_RING_ALIGN), 964 &base_addr, GFP_KERNEL); 965 if (!dest_ring->base_addr_owner_space_unaligned) { 966 kfree(dest_ring); 967 return ERR_PTR(-ENOMEM); 968 } 969 970 dest_ring->base_addr_ce_space_unaligned = base_addr; 971 972 /* 973 * Correctly initialize memory to 0 to prevent garbage 974 * data crashing system when download firmware 975 */ 976 memset(dest_ring->base_addr_owner_space_unaligned, 0, 977 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN); 978 979 dest_ring->base_addr_owner_space = PTR_ALIGN( 980 dest_ring->base_addr_owner_space_unaligned, 981 CE_DESC_RING_ALIGN); 982 dest_ring->base_addr_ce_space = ALIGN( 983 dest_ring->base_addr_ce_space_unaligned, 984 CE_DESC_RING_ALIGN); 985 986 return dest_ring; 987 } 988 989 /* 990 * Initialize a Copy Engine based on caller-supplied attributes. 991 * This may be called once to initialize both source and destination 992 * rings or it may be called twice for separate source and destination 993 * initialization. It may be that only one side or the other is 994 * initialized by software/firmware. 995 */ 996 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 997 const struct ce_attr *attr) 998 { 999 int ret; 1000 1001 if (attr->src_nentries) { 1002 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1003 if (ret) { 1004 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1005 ce_id, ret); 1006 return ret; 1007 } 1008 } 1009 1010 if (attr->dest_nentries) { 1011 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1012 if (ret) { 1013 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1014 ce_id, ret); 1015 return ret; 1016 } 1017 } 1018 1019 return 0; 1020 } 1021 1022 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1023 { 1024 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1025 1026 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0); 1027 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1028 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1029 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1030 } 1031 1032 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1033 { 1034 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1035 1036 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0); 1037 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1038 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1039 } 1040 1041 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1042 { 1043 ath10k_ce_deinit_src_ring(ar, ce_id); 1044 ath10k_ce_deinit_dest_ring(ar, ce_id); 1045 } 1046 1047 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1048 const struct ce_attr *attr) 1049 { 1050 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1051 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1052 int ret; 1053 1054 /* 1055 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1056 * additional TX locking checks. 1057 * 1058 * For the lack of a better place do the check here. 1059 */ 1060 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1061 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1062 BUILD_BUG_ON(2 * TARGET_10X_NUM_MSDU_DESC > 1063 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1064 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1065 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1066 1067 ce_state->ar = ar; 1068 ce_state->id = ce_id; 1069 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1070 ce_state->attr_flags = attr->flags; 1071 ce_state->src_sz_max = attr->src_sz_max; 1072 1073 if (attr->src_nentries) 1074 ce_state->send_cb = attr->send_cb; 1075 1076 if (attr->dest_nentries) 1077 ce_state->recv_cb = attr->recv_cb; 1078 1079 if (attr->src_nentries) { 1080 ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr); 1081 if (IS_ERR(ce_state->src_ring)) { 1082 ret = PTR_ERR(ce_state->src_ring); 1083 ath10k_err(ar, "failed to allocate copy engine source ring %d: %d\n", 1084 ce_id, ret); 1085 ce_state->src_ring = NULL; 1086 return ret; 1087 } 1088 } 1089 1090 if (attr->dest_nentries) { 1091 ce_state->dest_ring = ath10k_ce_alloc_dest_ring(ar, ce_id, 1092 attr); 1093 if (IS_ERR(ce_state->dest_ring)) { 1094 ret = PTR_ERR(ce_state->dest_ring); 1095 ath10k_err(ar, "failed to allocate copy engine destination ring %d: %d\n", 1096 ce_id, ret); 1097 ce_state->dest_ring = NULL; 1098 return ret; 1099 } 1100 } 1101 1102 return 0; 1103 } 1104 1105 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1106 { 1107 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1108 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1109 1110 if (ce_state->src_ring) { 1111 dma_free_coherent(ar->dev, 1112 (ce_state->src_ring->nentries * 1113 sizeof(struct ce_desc) + 1114 CE_DESC_RING_ALIGN), 1115 ce_state->src_ring->base_addr_owner_space, 1116 ce_state->src_ring->base_addr_ce_space); 1117 kfree(ce_state->src_ring); 1118 } 1119 1120 if (ce_state->dest_ring) { 1121 dma_free_coherent(ar->dev, 1122 (ce_state->dest_ring->nentries * 1123 sizeof(struct ce_desc) + 1124 CE_DESC_RING_ALIGN), 1125 ce_state->dest_ring->base_addr_owner_space, 1126 ce_state->dest_ring->base_addr_ce_space); 1127 kfree(ce_state->dest_ring); 1128 } 1129 1130 ce_state->src_ring = NULL; 1131 ce_state->dest_ring = NULL; 1132 } 1133