1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include "hif.h" 19 #include "ce.h" 20 #include "debug.h" 21 22 /* 23 * Support for Copy Engine hardware, which is mainly used for 24 * communication between Host and Target over a PCIe interconnect. 25 */ 26 27 /* 28 * A single CopyEngine (CE) comprises two "rings": 29 * a source ring 30 * a destination ring 31 * 32 * Each ring consists of a number of descriptors which specify 33 * an address, length, and meta-data. 34 * 35 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target) 36 * controls one ring and the other side controls the other ring. 37 * The source side chooses when to initiate a transfer and it 38 * chooses what to send (buffer address, length). The destination 39 * side keeps a supply of "anonymous receive buffers" available and 40 * it handles incoming data as it arrives (when the destination 41 * receives an interrupt). 42 * 43 * The sender may send a simple buffer (address/length) or it may 44 * send a small list of buffers. When a small list is sent, hardware 45 * "gathers" these and they end up in a single destination buffer 46 * with a single interrupt. 47 * 48 * There are several "contexts" managed by this layer -- more, it 49 * may seem -- than should be needed. These are provided mainly for 50 * maximum flexibility and especially to facilitate a simpler HIF 51 * implementation. There are per-CopyEngine recv, send, and watermark 52 * contexts. These are supplied by the caller when a recv, send, 53 * or watermark handler is established and they are echoed back to 54 * the caller when the respective callbacks are invoked. There is 55 * also a per-transfer context supplied by the caller when a buffer 56 * (or sendlist) is sent and when a buffer is enqueued for recv. 57 * These per-transfer contexts are echoed back to the caller when 58 * the buffer is sent/received. 59 */ 60 61 static inline unsigned int 62 ath10k_set_ring_byte(unsigned int offset, 63 struct ath10k_hw_ce_regs_addr_map *addr_map) 64 { 65 return ((offset << addr_map->lsb) & addr_map->mask); 66 } 67 68 static inline unsigned int 69 ath10k_get_ring_byte(unsigned int offset, 70 struct ath10k_hw_ce_regs_addr_map *addr_map) 71 { 72 return ((offset & addr_map->mask) >> (addr_map->lsb)); 73 } 74 75 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) 76 { 77 struct ath10k_ce *ce = ath10k_ce_priv(ar); 78 79 return ce->bus_ops->read32(ar, offset); 80 } 81 82 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) 83 { 84 struct ath10k_ce *ce = ath10k_ce_priv(ar); 85 86 ce->bus_ops->write32(ar, offset, value); 87 } 88 89 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 90 u32 ce_ctrl_addr, 91 unsigned int n) 92 { 93 ath10k_ce_write32(ar, ce_ctrl_addr + 94 ar->hw_ce_regs->dst_wr_index_addr, n); 95 } 96 97 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 98 u32 ce_ctrl_addr) 99 { 100 return ath10k_ce_read32(ar, ce_ctrl_addr + 101 ar->hw_ce_regs->dst_wr_index_addr); 102 } 103 104 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 105 u32 ce_ctrl_addr, 106 unsigned int n) 107 { 108 ath10k_ce_write32(ar, ce_ctrl_addr + 109 ar->hw_ce_regs->sr_wr_index_addr, n); 110 } 111 112 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 113 u32 ce_ctrl_addr) 114 { 115 return ath10k_ce_read32(ar, ce_ctrl_addr + 116 ar->hw_ce_regs->sr_wr_index_addr); 117 } 118 119 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 120 u32 ce_ctrl_addr) 121 { 122 return ath10k_ce_read32(ar, ce_ctrl_addr + 123 ar->hw_ce_regs->current_srri_addr); 124 } 125 126 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 127 u32 ce_ctrl_addr, 128 unsigned int addr) 129 { 130 ath10k_ce_write32(ar, ce_ctrl_addr + 131 ar->hw_ce_regs->sr_base_addr, addr); 132 } 133 134 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 135 u32 ce_ctrl_addr, 136 unsigned int n) 137 { 138 ath10k_ce_write32(ar, ce_ctrl_addr + 139 ar->hw_ce_regs->sr_size_addr, n); 140 } 141 142 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 143 u32 ce_ctrl_addr, 144 unsigned int n) 145 { 146 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 147 148 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 149 ctrl_regs->addr); 150 151 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 152 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | 153 ath10k_set_ring_byte(n, ctrl_regs->dmax)); 154 } 155 156 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 157 u32 ce_ctrl_addr, 158 unsigned int n) 159 { 160 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 161 162 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 163 ctrl_regs->addr); 164 165 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 166 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | 167 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); 168 } 169 170 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 171 u32 ce_ctrl_addr, 172 unsigned int n) 173 { 174 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 175 176 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 177 ctrl_regs->addr); 178 179 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 180 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | 181 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); 182 } 183 184 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 185 u32 ce_ctrl_addr) 186 { 187 return ath10k_ce_read32(ar, ce_ctrl_addr + 188 ar->hw_ce_regs->current_drri_addr); 189 } 190 191 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 192 u32 ce_ctrl_addr, 193 u32 addr) 194 { 195 ath10k_ce_write32(ar, ce_ctrl_addr + 196 ar->hw_ce_regs->dr_base_addr, addr); 197 } 198 199 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 200 u32 ce_ctrl_addr, 201 unsigned int n) 202 { 203 ath10k_ce_write32(ar, ce_ctrl_addr + 204 ar->hw_ce_regs->dr_size_addr, n); 205 } 206 207 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 208 u32 ce_ctrl_addr, 209 unsigned int n) 210 { 211 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 212 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 213 214 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 215 (addr & ~(srcr_wm->wm_high->mask)) | 216 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); 217 } 218 219 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 220 u32 ce_ctrl_addr, 221 unsigned int n) 222 { 223 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 224 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 225 226 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 227 (addr & ~(srcr_wm->wm_low->mask)) | 228 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); 229 } 230 231 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 232 u32 ce_ctrl_addr, 233 unsigned int n) 234 { 235 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 236 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 237 238 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 239 (addr & ~(dstr_wm->wm_high->mask)) | 240 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); 241 } 242 243 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 244 u32 ce_ctrl_addr, 245 unsigned int n) 246 { 247 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 248 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 249 250 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 251 (addr & ~(dstr_wm->wm_low->mask)) | 252 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); 253 } 254 255 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 256 u32 ce_ctrl_addr) 257 { 258 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 259 260 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 261 ar->hw_ce_regs->host_ie_addr); 262 263 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 264 host_ie_addr | host_ie->copy_complete->mask); 265 } 266 267 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 268 u32 ce_ctrl_addr) 269 { 270 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 271 272 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 273 ar->hw_ce_regs->host_ie_addr); 274 275 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 276 host_ie_addr & ~(host_ie->copy_complete->mask)); 277 } 278 279 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 280 u32 ce_ctrl_addr) 281 { 282 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 283 284 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 285 ar->hw_ce_regs->host_ie_addr); 286 287 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 288 host_ie_addr & ~(wm_regs->wm_mask)); 289 } 290 291 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 292 u32 ce_ctrl_addr) 293 { 294 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 295 296 u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 297 ar->hw_ce_regs->misc_ie_addr); 298 299 ath10k_ce_write32(ar, 300 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 301 misc_ie_addr | misc_regs->err_mask); 302 } 303 304 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 305 u32 ce_ctrl_addr) 306 { 307 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 308 309 u32 misc_ie_addr = ath10k_ce_read32(ar, 310 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); 311 312 ath10k_ce_write32(ar, 313 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 314 misc_ie_addr & ~(misc_regs->err_mask)); 315 } 316 317 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 318 u32 ce_ctrl_addr, 319 unsigned int mask) 320 { 321 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 322 323 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); 324 } 325 326 /* 327 * Guts of ath10k_ce_send. 328 * The caller takes responsibility for any needed locking. 329 */ 330 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 331 void *per_transfer_context, 332 dma_addr_t buffer, 333 unsigned int nbytes, 334 unsigned int transfer_id, 335 unsigned int flags) 336 { 337 struct ath10k *ar = ce_state->ar; 338 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 339 struct ce_desc *desc, sdesc; 340 unsigned int nentries_mask = src_ring->nentries_mask; 341 unsigned int sw_index = src_ring->sw_index; 342 unsigned int write_index = src_ring->write_index; 343 u32 ctrl_addr = ce_state->ctrl_addr; 344 u32 desc_flags = 0; 345 int ret = 0; 346 347 if (nbytes > ce_state->src_sz_max) 348 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 349 __func__, nbytes, ce_state->src_sz_max); 350 351 if (unlikely(CE_RING_DELTA(nentries_mask, 352 write_index, sw_index - 1) <= 0)) { 353 ret = -ENOSR; 354 goto exit; 355 } 356 357 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 358 write_index); 359 360 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 361 362 if (flags & CE_SEND_FLAG_GATHER) 363 desc_flags |= CE_DESC_FLAGS_GATHER; 364 if (flags & CE_SEND_FLAG_BYTE_SWAP) 365 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 366 367 sdesc.addr = __cpu_to_le32(buffer); 368 sdesc.nbytes = __cpu_to_le16(nbytes); 369 sdesc.flags = __cpu_to_le16(desc_flags); 370 371 *desc = sdesc; 372 373 src_ring->per_transfer_context[write_index] = per_transfer_context; 374 375 /* Update Source Ring Write Index */ 376 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 377 378 /* WORKAROUND */ 379 if (!(flags & CE_SEND_FLAG_GATHER)) 380 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 381 382 src_ring->write_index = write_index; 383 exit: 384 return ret; 385 } 386 387 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state, 388 void *per_transfer_context, 389 dma_addr_t buffer, 390 unsigned int nbytes, 391 unsigned int transfer_id, 392 unsigned int flags) 393 { 394 struct ath10k *ar = ce_state->ar; 395 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 396 struct ce_desc_64 *desc, sdesc; 397 unsigned int nentries_mask = src_ring->nentries_mask; 398 unsigned int sw_index = src_ring->sw_index; 399 unsigned int write_index = src_ring->write_index; 400 u32 ctrl_addr = ce_state->ctrl_addr; 401 __le32 *addr; 402 u32 desc_flags = 0; 403 int ret = 0; 404 405 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) 406 return -ESHUTDOWN; 407 408 if (nbytes > ce_state->src_sz_max) 409 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 410 __func__, nbytes, ce_state->src_sz_max); 411 412 if (unlikely(CE_RING_DELTA(nentries_mask, 413 write_index, sw_index - 1) <= 0)) { 414 ret = -ENOSR; 415 goto exit; 416 } 417 418 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 419 write_index); 420 421 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 422 423 if (flags & CE_SEND_FLAG_GATHER) 424 desc_flags |= CE_DESC_FLAGS_GATHER; 425 426 if (flags & CE_SEND_FLAG_BYTE_SWAP) 427 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 428 429 addr = (__le32 *)&sdesc.addr; 430 431 flags |= upper_32_bits(buffer) & CE_DESC_FLAGS_GET_MASK; 432 addr[0] = __cpu_to_le32(buffer); 433 addr[1] = __cpu_to_le32(flags); 434 if (flags & CE_SEND_FLAG_GATHER) 435 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER); 436 else 437 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER)); 438 439 sdesc.nbytes = __cpu_to_le16(nbytes); 440 sdesc.flags = __cpu_to_le16(desc_flags); 441 442 *desc = sdesc; 443 444 src_ring->per_transfer_context[write_index] = per_transfer_context; 445 446 /* Update Source Ring Write Index */ 447 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 448 449 if (!(flags & CE_SEND_FLAG_GATHER)) 450 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 451 452 src_ring->write_index = write_index; 453 exit: 454 return ret; 455 } 456 457 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 458 void *per_transfer_context, 459 dma_addr_t buffer, 460 unsigned int nbytes, 461 unsigned int transfer_id, 462 unsigned int flags) 463 { 464 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, 465 buffer, nbytes, transfer_id, flags); 466 } 467 468 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 469 { 470 struct ath10k *ar = pipe->ar; 471 struct ath10k_ce *ce = ath10k_ce_priv(ar); 472 struct ath10k_ce_ring *src_ring = pipe->src_ring; 473 u32 ctrl_addr = pipe->ctrl_addr; 474 475 lockdep_assert_held(&ce->ce_lock); 476 477 /* 478 * This function must be called only if there is an incomplete 479 * scatter-gather transfer (before index register is updated) 480 * that needs to be cleaned up. 481 */ 482 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 483 return; 484 485 if (WARN_ON_ONCE(src_ring->write_index == 486 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 487 return; 488 489 src_ring->write_index--; 490 src_ring->write_index &= src_ring->nentries_mask; 491 492 src_ring->per_transfer_context[src_ring->write_index] = NULL; 493 } 494 495 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 496 void *per_transfer_context, 497 dma_addr_t buffer, 498 unsigned int nbytes, 499 unsigned int transfer_id, 500 unsigned int flags) 501 { 502 struct ath10k *ar = ce_state->ar; 503 struct ath10k_ce *ce = ath10k_ce_priv(ar); 504 int ret; 505 506 spin_lock_bh(&ce->ce_lock); 507 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 508 buffer, nbytes, transfer_id, flags); 509 spin_unlock_bh(&ce->ce_lock); 510 511 return ret; 512 } 513 514 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 515 { 516 struct ath10k *ar = pipe->ar; 517 struct ath10k_ce *ce = ath10k_ce_priv(ar); 518 int delta; 519 520 spin_lock_bh(&ce->ce_lock); 521 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 522 pipe->src_ring->write_index, 523 pipe->src_ring->sw_index - 1); 524 spin_unlock_bh(&ce->ce_lock); 525 526 return delta; 527 } 528 529 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 530 { 531 struct ath10k *ar = pipe->ar; 532 struct ath10k_ce *ce = ath10k_ce_priv(ar); 533 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 534 unsigned int nentries_mask = dest_ring->nentries_mask; 535 unsigned int write_index = dest_ring->write_index; 536 unsigned int sw_index = dest_ring->sw_index; 537 538 lockdep_assert_held(&ce->ce_lock); 539 540 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 541 } 542 543 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 544 dma_addr_t paddr) 545 { 546 struct ath10k *ar = pipe->ar; 547 struct ath10k_ce *ce = ath10k_ce_priv(ar); 548 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 549 unsigned int nentries_mask = dest_ring->nentries_mask; 550 unsigned int write_index = dest_ring->write_index; 551 unsigned int sw_index = dest_ring->sw_index; 552 struct ce_desc *base = dest_ring->base_addr_owner_space; 553 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 554 u32 ctrl_addr = pipe->ctrl_addr; 555 556 lockdep_assert_held(&ce->ce_lock); 557 558 if ((pipe->id != 5) && 559 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 560 return -ENOSPC; 561 562 desc->addr = __cpu_to_le32(paddr); 563 desc->nbytes = 0; 564 565 dest_ring->per_transfer_context[write_index] = ctx; 566 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 567 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 568 dest_ring->write_index = write_index; 569 570 return 0; 571 } 572 573 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe, 574 void *ctx, 575 dma_addr_t paddr) 576 { 577 struct ath10k *ar = pipe->ar; 578 struct ath10k_ce *ce = ath10k_ce_priv(ar); 579 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 580 unsigned int nentries_mask = dest_ring->nentries_mask; 581 unsigned int write_index = dest_ring->write_index; 582 unsigned int sw_index = dest_ring->sw_index; 583 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 584 struct ce_desc_64 *desc = 585 CE_DEST_RING_TO_DESC_64(base, write_index); 586 u32 ctrl_addr = pipe->ctrl_addr; 587 588 lockdep_assert_held(&ce->ce_lock); 589 590 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 591 return -ENOSPC; 592 593 desc->addr = __cpu_to_le64(paddr); 594 desc->addr &= __cpu_to_le64(CE_DESC_37BIT_ADDR_MASK); 595 596 desc->nbytes = 0; 597 598 dest_ring->per_transfer_context[write_index] = ctx; 599 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 600 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 601 dest_ring->write_index = write_index; 602 603 return 0; 604 } 605 606 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 607 { 608 struct ath10k *ar = pipe->ar; 609 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 610 unsigned int nentries_mask = dest_ring->nentries_mask; 611 unsigned int write_index = dest_ring->write_index; 612 u32 ctrl_addr = pipe->ctrl_addr; 613 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 614 615 /* Prevent CE ring stuck issue that will occur when ring is full. 616 * Make sure that write index is 1 less than read index. 617 */ 618 if ((cur_write_idx + nentries) == dest_ring->sw_index) 619 nentries -= 1; 620 621 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 622 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 623 dest_ring->write_index = write_index; 624 } 625 626 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 627 dma_addr_t paddr) 628 { 629 struct ath10k *ar = pipe->ar; 630 struct ath10k_ce *ce = ath10k_ce_priv(ar); 631 int ret; 632 633 spin_lock_bh(&ce->ce_lock); 634 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); 635 spin_unlock_bh(&ce->ce_lock); 636 637 return ret; 638 } 639 640 /* 641 * Guts of ath10k_ce_completed_recv_next. 642 * The caller takes responsibility for any necessary locking. 643 */ 644 static int 645 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 646 void **per_transfer_contextp, 647 unsigned int *nbytesp) 648 { 649 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 650 unsigned int nentries_mask = dest_ring->nentries_mask; 651 unsigned int sw_index = dest_ring->sw_index; 652 653 struct ce_desc *base = dest_ring->base_addr_owner_space; 654 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 655 struct ce_desc sdesc; 656 u16 nbytes; 657 658 /* Copy in one go for performance reasons */ 659 sdesc = *desc; 660 661 nbytes = __le16_to_cpu(sdesc.nbytes); 662 if (nbytes == 0) { 663 /* 664 * This closes a relatively unusual race where the Host 665 * sees the updated DRRI before the update to the 666 * corresponding descriptor has completed. We treat this 667 * as a descriptor that is not yet done. 668 */ 669 return -EIO; 670 } 671 672 desc->nbytes = 0; 673 674 /* Return data from completed destination descriptor */ 675 *nbytesp = nbytes; 676 677 if (per_transfer_contextp) 678 *per_transfer_contextp = 679 dest_ring->per_transfer_context[sw_index]; 680 681 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 682 * So update transfer context all CEs except CE5. 683 */ 684 if (ce_state->id != 5) 685 dest_ring->per_transfer_context[sw_index] = NULL; 686 687 /* Update sw_index */ 688 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 689 dest_ring->sw_index = sw_index; 690 691 return 0; 692 } 693 694 static int 695 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state, 696 void **per_transfer_contextp, 697 unsigned int *nbytesp) 698 { 699 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 700 unsigned int nentries_mask = dest_ring->nentries_mask; 701 unsigned int sw_index = dest_ring->sw_index; 702 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 703 struct ce_desc_64 *desc = 704 CE_DEST_RING_TO_DESC_64(base, sw_index); 705 struct ce_desc_64 sdesc; 706 u16 nbytes; 707 708 /* Copy in one go for performance reasons */ 709 sdesc = *desc; 710 711 nbytes = __le16_to_cpu(sdesc.nbytes); 712 if (nbytes == 0) { 713 /* This closes a relatively unusual race where the Host 714 * sees the updated DRRI before the update to the 715 * corresponding descriptor has completed. We treat this 716 * as a descriptor that is not yet done. 717 */ 718 return -EIO; 719 } 720 721 desc->nbytes = 0; 722 723 /* Return data from completed destination descriptor */ 724 *nbytesp = nbytes; 725 726 if (per_transfer_contextp) 727 *per_transfer_contextp = 728 dest_ring->per_transfer_context[sw_index]; 729 730 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 731 * So update transfer context all CEs except CE5. 732 */ 733 if (ce_state->id != 5) 734 dest_ring->per_transfer_context[sw_index] = NULL; 735 736 /* Update sw_index */ 737 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 738 dest_ring->sw_index = sw_index; 739 740 return 0; 741 } 742 743 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 744 void **per_transfer_ctx, 745 unsigned int *nbytesp) 746 { 747 return ce_state->ops->ce_completed_recv_next_nolock(ce_state, 748 per_transfer_ctx, 749 nbytesp); 750 } 751 752 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 753 void **per_transfer_contextp, 754 unsigned int *nbytesp) 755 { 756 struct ath10k *ar = ce_state->ar; 757 struct ath10k_ce *ce = ath10k_ce_priv(ar); 758 int ret; 759 760 spin_lock_bh(&ce->ce_lock); 761 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, 762 per_transfer_contextp, 763 nbytesp); 764 765 spin_unlock_bh(&ce->ce_lock); 766 767 return ret; 768 } 769 770 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 771 void **per_transfer_contextp, 772 dma_addr_t *bufferp) 773 { 774 struct ath10k_ce_ring *dest_ring; 775 unsigned int nentries_mask; 776 unsigned int sw_index; 777 unsigned int write_index; 778 int ret; 779 struct ath10k *ar; 780 struct ath10k_ce *ce; 781 782 dest_ring = ce_state->dest_ring; 783 784 if (!dest_ring) 785 return -EIO; 786 787 ar = ce_state->ar; 788 ce = ath10k_ce_priv(ar); 789 790 spin_lock_bh(&ce->ce_lock); 791 792 nentries_mask = dest_ring->nentries_mask; 793 sw_index = dest_ring->sw_index; 794 write_index = dest_ring->write_index; 795 if (write_index != sw_index) { 796 struct ce_desc *base = dest_ring->base_addr_owner_space; 797 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 798 799 /* Return data from completed destination descriptor */ 800 *bufferp = __le32_to_cpu(desc->addr); 801 802 if (per_transfer_contextp) 803 *per_transfer_contextp = 804 dest_ring->per_transfer_context[sw_index]; 805 806 /* sanity */ 807 dest_ring->per_transfer_context[sw_index] = NULL; 808 desc->nbytes = 0; 809 810 /* Update sw_index */ 811 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 812 dest_ring->sw_index = sw_index; 813 ret = 0; 814 } else { 815 ret = -EIO; 816 } 817 818 spin_unlock_bh(&ce->ce_lock); 819 820 return ret; 821 } 822 823 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state, 824 void **per_transfer_contextp, 825 dma_addr_t *bufferp) 826 { 827 struct ath10k_ce_ring *dest_ring; 828 unsigned int nentries_mask; 829 unsigned int sw_index; 830 unsigned int write_index; 831 int ret; 832 struct ath10k *ar; 833 struct ath10k_ce *ce; 834 835 dest_ring = ce_state->dest_ring; 836 837 if (!dest_ring) 838 return -EIO; 839 840 ar = ce_state->ar; 841 ce = ath10k_ce_priv(ar); 842 843 spin_lock_bh(&ce->ce_lock); 844 845 nentries_mask = dest_ring->nentries_mask; 846 sw_index = dest_ring->sw_index; 847 write_index = dest_ring->write_index; 848 if (write_index != sw_index) { 849 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 850 struct ce_desc_64 *desc = 851 CE_DEST_RING_TO_DESC_64(base, sw_index); 852 853 /* Return data from completed destination descriptor */ 854 *bufferp = __le64_to_cpu(desc->addr); 855 856 if (per_transfer_contextp) 857 *per_transfer_contextp = 858 dest_ring->per_transfer_context[sw_index]; 859 860 /* sanity */ 861 dest_ring->per_transfer_context[sw_index] = NULL; 862 desc->nbytes = 0; 863 864 /* Update sw_index */ 865 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 866 dest_ring->sw_index = sw_index; 867 ret = 0; 868 } else { 869 ret = -EIO; 870 } 871 872 spin_unlock_bh(&ce->ce_lock); 873 874 return ret; 875 } 876 877 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 878 void **per_transfer_contextp, 879 dma_addr_t *bufferp) 880 { 881 return ce_state->ops->ce_revoke_recv_next(ce_state, 882 per_transfer_contextp, 883 bufferp); 884 } 885 886 /* 887 * Guts of ath10k_ce_completed_send_next. 888 * The caller takes responsibility for any necessary locking. 889 */ 890 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 891 void **per_transfer_contextp) 892 { 893 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 894 u32 ctrl_addr = ce_state->ctrl_addr; 895 struct ath10k *ar = ce_state->ar; 896 unsigned int nentries_mask = src_ring->nentries_mask; 897 unsigned int sw_index = src_ring->sw_index; 898 unsigned int read_index; 899 struct ce_desc *desc; 900 901 if (src_ring->hw_index == sw_index) { 902 /* 903 * The SW completion index has caught up with the cached 904 * version of the HW completion index. 905 * Update the cached HW completion index to see whether 906 * the SW has really caught up to the HW, or if the cached 907 * value of the HW index has become stale. 908 */ 909 910 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 911 if (read_index == 0xffffffff) 912 return -ENODEV; 913 914 read_index &= nentries_mask; 915 src_ring->hw_index = read_index; 916 } 917 918 read_index = src_ring->hw_index; 919 920 if (read_index == sw_index) 921 return -EIO; 922 923 if (per_transfer_contextp) 924 *per_transfer_contextp = 925 src_ring->per_transfer_context[sw_index]; 926 927 /* sanity */ 928 src_ring->per_transfer_context[sw_index] = NULL; 929 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 930 sw_index); 931 desc->nbytes = 0; 932 933 /* Update sw_index */ 934 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 935 src_ring->sw_index = sw_index; 936 937 return 0; 938 } 939 940 static void ath10k_ce_extract_desc_data(struct ath10k *ar, 941 struct ath10k_ce_ring *src_ring, 942 u32 sw_index, 943 dma_addr_t *bufferp, 944 u32 *nbytesp, 945 u32 *transfer_idp) 946 { 947 struct ce_desc *base = src_ring->base_addr_owner_space; 948 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 949 950 /* Return data from completed source descriptor */ 951 *bufferp = __le32_to_cpu(desc->addr); 952 *nbytesp = __le16_to_cpu(desc->nbytes); 953 *transfer_idp = MS(__le16_to_cpu(desc->flags), 954 CE_DESC_FLAGS_META_DATA); 955 } 956 957 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar, 958 struct ath10k_ce_ring *src_ring, 959 u32 sw_index, 960 dma_addr_t *bufferp, 961 u32 *nbytesp, 962 u32 *transfer_idp) 963 { 964 struct ce_desc_64 *base = src_ring->base_addr_owner_space; 965 struct ce_desc_64 *desc = 966 CE_SRC_RING_TO_DESC_64(base, sw_index); 967 968 /* Return data from completed source descriptor */ 969 *bufferp = __le64_to_cpu(desc->addr); 970 *nbytesp = __le16_to_cpu(desc->nbytes); 971 *transfer_idp = MS(__le16_to_cpu(desc->flags), 972 CE_DESC_FLAGS_META_DATA); 973 } 974 975 /* NB: Modeled after ath10k_ce_completed_send_next */ 976 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 977 void **per_transfer_contextp, 978 dma_addr_t *bufferp, 979 unsigned int *nbytesp, 980 unsigned int *transfer_idp) 981 { 982 struct ath10k_ce_ring *src_ring; 983 unsigned int nentries_mask; 984 unsigned int sw_index; 985 unsigned int write_index; 986 int ret; 987 struct ath10k *ar; 988 struct ath10k_ce *ce; 989 990 src_ring = ce_state->src_ring; 991 992 if (!src_ring) 993 return -EIO; 994 995 ar = ce_state->ar; 996 ce = ath10k_ce_priv(ar); 997 998 spin_lock_bh(&ce->ce_lock); 999 1000 nentries_mask = src_ring->nentries_mask; 1001 sw_index = src_ring->sw_index; 1002 write_index = src_ring->write_index; 1003 1004 if (write_index != sw_index) { 1005 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, 1006 bufferp, nbytesp, 1007 transfer_idp); 1008 1009 if (per_transfer_contextp) 1010 *per_transfer_contextp = 1011 src_ring->per_transfer_context[sw_index]; 1012 1013 /* sanity */ 1014 src_ring->per_transfer_context[sw_index] = NULL; 1015 1016 /* Update sw_index */ 1017 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1018 src_ring->sw_index = sw_index; 1019 ret = 0; 1020 } else { 1021 ret = -EIO; 1022 } 1023 1024 spin_unlock_bh(&ce->ce_lock); 1025 1026 return ret; 1027 } 1028 1029 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 1030 void **per_transfer_contextp) 1031 { 1032 struct ath10k *ar = ce_state->ar; 1033 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1034 int ret; 1035 1036 spin_lock_bh(&ce->ce_lock); 1037 ret = ath10k_ce_completed_send_next_nolock(ce_state, 1038 per_transfer_contextp); 1039 spin_unlock_bh(&ce->ce_lock); 1040 1041 return ret; 1042 } 1043 1044 /* 1045 * Guts of interrupt handler for per-engine interrupts on a particular CE. 1046 * 1047 * Invokes registered callbacks for recv_complete, 1048 * send_complete, and watermarks. 1049 */ 1050 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 1051 { 1052 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1053 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1054 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 1055 u32 ctrl_addr = ce_state->ctrl_addr; 1056 1057 spin_lock_bh(&ce->ce_lock); 1058 1059 /* Clear the copy-complete interrupts that will be handled here. */ 1060 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 1061 wm_regs->cc_mask); 1062 1063 spin_unlock_bh(&ce->ce_lock); 1064 1065 if (ce_state->recv_cb) 1066 ce_state->recv_cb(ce_state); 1067 1068 if (ce_state->send_cb) 1069 ce_state->send_cb(ce_state); 1070 1071 spin_lock_bh(&ce->ce_lock); 1072 1073 /* 1074 * Misc CE interrupts are not being handled, but still need 1075 * to be cleared. 1076 */ 1077 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask); 1078 1079 spin_unlock_bh(&ce->ce_lock); 1080 } 1081 1082 /* 1083 * Handler for per-engine interrupts on ALL active CEs. 1084 * This is used in cases where the system is sharing a 1085 * single interrput for all CEs 1086 */ 1087 1088 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 1089 { 1090 int ce_id; 1091 u32 intr_summary; 1092 1093 intr_summary = ath10k_ce_interrupt_summary(ar); 1094 1095 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 1096 if (intr_summary & (1 << ce_id)) 1097 intr_summary &= ~(1 << ce_id); 1098 else 1099 /* no intr pending on this CE */ 1100 continue; 1101 1102 ath10k_ce_per_engine_service(ar, ce_id); 1103 } 1104 } 1105 1106 /* 1107 * Adjust interrupts for the copy complete handler. 1108 * If it's needed for either send or recv, then unmask 1109 * this interrupt; otherwise, mask it. 1110 * 1111 * Called with ce_lock held. 1112 */ 1113 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 1114 { 1115 u32 ctrl_addr = ce_state->ctrl_addr; 1116 struct ath10k *ar = ce_state->ar; 1117 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 1118 1119 if ((!disable_copy_compl_intr) && 1120 (ce_state->send_cb || ce_state->recv_cb)) 1121 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 1122 else 1123 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1124 1125 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1126 } 1127 1128 int ath10k_ce_disable_interrupts(struct ath10k *ar) 1129 { 1130 int ce_id; 1131 1132 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 1133 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1134 1135 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1136 ath10k_ce_error_intr_disable(ar, ctrl_addr); 1137 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1138 } 1139 1140 return 0; 1141 } 1142 1143 void ath10k_ce_enable_interrupts(struct ath10k *ar) 1144 { 1145 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1146 int ce_id; 1147 struct ath10k_ce_pipe *ce_state; 1148 1149 /* Skip the last copy engine, CE7 the diagnostic window, as that 1150 * uses polling and isn't initialized for interrupts. 1151 */ 1152 for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++) { 1153 ce_state = &ce->ce_states[ce_id]; 1154 ath10k_ce_per_engine_handler_adjust(ce_state); 1155 } 1156 } 1157 1158 static int ath10k_ce_init_src_ring(struct ath10k *ar, 1159 unsigned int ce_id, 1160 const struct ce_attr *attr) 1161 { 1162 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1163 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1164 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1165 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1166 1167 nentries = roundup_pow_of_two(attr->src_nentries); 1168 1169 if (ar->hw_params.target_64bit) 1170 memset(src_ring->base_addr_owner_space, 0, 1171 nentries * sizeof(struct ce_desc_64)); 1172 else 1173 memset(src_ring->base_addr_owner_space, 0, 1174 nentries * sizeof(struct ce_desc)); 1175 1176 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1177 src_ring->sw_index &= src_ring->nentries_mask; 1178 src_ring->hw_index = src_ring->sw_index; 1179 1180 src_ring->write_index = 1181 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 1182 src_ring->write_index &= src_ring->nentries_mask; 1183 1184 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 1185 src_ring->base_addr_ce_space); 1186 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 1187 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 1188 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 1189 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 1190 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 1191 1192 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1193 "boot init ce src ring id %d entries %d base_addr %pK\n", 1194 ce_id, nentries, src_ring->base_addr_owner_space); 1195 1196 return 0; 1197 } 1198 1199 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1200 unsigned int ce_id, 1201 const struct ce_attr *attr) 1202 { 1203 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1204 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1205 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 1206 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1207 1208 nentries = roundup_pow_of_two(attr->dest_nentries); 1209 1210 if (ar->hw_params.target_64bit) 1211 memset(dest_ring->base_addr_owner_space, 0, 1212 nentries * sizeof(struct ce_desc_64)); 1213 else 1214 memset(dest_ring->base_addr_owner_space, 0, 1215 nentries * sizeof(struct ce_desc)); 1216 1217 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1218 dest_ring->sw_index &= dest_ring->nentries_mask; 1219 dest_ring->write_index = 1220 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1221 dest_ring->write_index &= dest_ring->nentries_mask; 1222 1223 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 1224 dest_ring->base_addr_ce_space); 1225 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1226 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1227 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1228 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1229 1230 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1231 "boot ce dest ring id %d entries %d base_addr %pK\n", 1232 ce_id, nentries, dest_ring->base_addr_owner_space); 1233 1234 return 0; 1235 } 1236 1237 static struct ath10k_ce_ring * 1238 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 1239 const struct ce_attr *attr) 1240 { 1241 struct ath10k_ce_ring *src_ring; 1242 u32 nentries = attr->src_nentries; 1243 dma_addr_t base_addr; 1244 1245 nentries = roundup_pow_of_two(nentries); 1246 1247 src_ring = kzalloc(sizeof(*src_ring) + 1248 (nentries * 1249 sizeof(*src_ring->per_transfer_context)), 1250 GFP_KERNEL); 1251 if (src_ring == NULL) 1252 return ERR_PTR(-ENOMEM); 1253 1254 src_ring->nentries = nentries; 1255 src_ring->nentries_mask = nentries - 1; 1256 1257 /* 1258 * Legacy platforms that do not support cache 1259 * coherent DMA are unsupported 1260 */ 1261 src_ring->base_addr_owner_space_unaligned = 1262 dma_alloc_coherent(ar->dev, 1263 (nentries * sizeof(struct ce_desc) + 1264 CE_DESC_RING_ALIGN), 1265 &base_addr, GFP_KERNEL); 1266 if (!src_ring->base_addr_owner_space_unaligned) { 1267 kfree(src_ring); 1268 return ERR_PTR(-ENOMEM); 1269 } 1270 1271 src_ring->base_addr_ce_space_unaligned = base_addr; 1272 1273 src_ring->base_addr_owner_space = 1274 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1275 CE_DESC_RING_ALIGN); 1276 src_ring->base_addr_ce_space = 1277 ALIGN(src_ring->base_addr_ce_space_unaligned, 1278 CE_DESC_RING_ALIGN); 1279 1280 return src_ring; 1281 } 1282 1283 static struct ath10k_ce_ring * 1284 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, 1285 const struct ce_attr *attr) 1286 { 1287 struct ath10k_ce_ring *src_ring; 1288 u32 nentries = attr->src_nentries; 1289 dma_addr_t base_addr; 1290 1291 nentries = roundup_pow_of_two(nentries); 1292 1293 src_ring = kzalloc(sizeof(*src_ring) + 1294 (nentries * 1295 sizeof(*src_ring->per_transfer_context)), 1296 GFP_KERNEL); 1297 if (!src_ring) 1298 return ERR_PTR(-ENOMEM); 1299 1300 src_ring->nentries = nentries; 1301 src_ring->nentries_mask = nentries - 1; 1302 1303 /* Legacy platforms that do not support cache 1304 * coherent DMA are unsupported 1305 */ 1306 src_ring->base_addr_owner_space_unaligned = 1307 dma_alloc_coherent(ar->dev, 1308 (nentries * sizeof(struct ce_desc_64) + 1309 CE_DESC_RING_ALIGN), 1310 &base_addr, GFP_KERNEL); 1311 if (!src_ring->base_addr_owner_space_unaligned) { 1312 kfree(src_ring); 1313 return ERR_PTR(-ENOMEM); 1314 } 1315 1316 src_ring->base_addr_ce_space_unaligned = base_addr; 1317 1318 src_ring->base_addr_owner_space = 1319 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1320 CE_DESC_RING_ALIGN); 1321 src_ring->base_addr_ce_space = 1322 ALIGN(src_ring->base_addr_ce_space_unaligned, 1323 CE_DESC_RING_ALIGN); 1324 1325 return src_ring; 1326 } 1327 1328 static struct ath10k_ce_ring * 1329 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 1330 const struct ce_attr *attr) 1331 { 1332 struct ath10k_ce_ring *dest_ring; 1333 u32 nentries; 1334 dma_addr_t base_addr; 1335 1336 nentries = roundup_pow_of_two(attr->dest_nentries); 1337 1338 dest_ring = kzalloc(sizeof(*dest_ring) + 1339 (nentries * 1340 sizeof(*dest_ring->per_transfer_context)), 1341 GFP_KERNEL); 1342 if (dest_ring == NULL) 1343 return ERR_PTR(-ENOMEM); 1344 1345 dest_ring->nentries = nentries; 1346 dest_ring->nentries_mask = nentries - 1; 1347 1348 /* 1349 * Legacy platforms that do not support cache 1350 * coherent DMA are unsupported 1351 */ 1352 dest_ring->base_addr_owner_space_unaligned = 1353 dma_zalloc_coherent(ar->dev, 1354 (nentries * sizeof(struct ce_desc) + 1355 CE_DESC_RING_ALIGN), 1356 &base_addr, GFP_KERNEL); 1357 if (!dest_ring->base_addr_owner_space_unaligned) { 1358 kfree(dest_ring); 1359 return ERR_PTR(-ENOMEM); 1360 } 1361 1362 dest_ring->base_addr_ce_space_unaligned = base_addr; 1363 1364 dest_ring->base_addr_owner_space = 1365 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1366 CE_DESC_RING_ALIGN); 1367 dest_ring->base_addr_ce_space = 1368 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1369 CE_DESC_RING_ALIGN); 1370 1371 return dest_ring; 1372 } 1373 1374 static struct ath10k_ce_ring * 1375 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, 1376 const struct ce_attr *attr) 1377 { 1378 struct ath10k_ce_ring *dest_ring; 1379 u32 nentries; 1380 dma_addr_t base_addr; 1381 1382 nentries = roundup_pow_of_two(attr->dest_nentries); 1383 1384 dest_ring = kzalloc(sizeof(*dest_ring) + 1385 (nentries * 1386 sizeof(*dest_ring->per_transfer_context)), 1387 GFP_KERNEL); 1388 if (!dest_ring) 1389 return ERR_PTR(-ENOMEM); 1390 1391 dest_ring->nentries = nentries; 1392 dest_ring->nentries_mask = nentries - 1; 1393 1394 /* Legacy platforms that do not support cache 1395 * coherent DMA are unsupported 1396 */ 1397 dest_ring->base_addr_owner_space_unaligned = 1398 dma_alloc_coherent(ar->dev, 1399 (nentries * sizeof(struct ce_desc_64) + 1400 CE_DESC_RING_ALIGN), 1401 &base_addr, GFP_KERNEL); 1402 if (!dest_ring->base_addr_owner_space_unaligned) { 1403 kfree(dest_ring); 1404 return ERR_PTR(-ENOMEM); 1405 } 1406 1407 dest_ring->base_addr_ce_space_unaligned = base_addr; 1408 1409 /* Correctly initialize memory to 0 to prevent garbage 1410 * data crashing system when download firmware 1411 */ 1412 memset(dest_ring->base_addr_owner_space_unaligned, 0, 1413 nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN); 1414 1415 dest_ring->base_addr_owner_space = 1416 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1417 CE_DESC_RING_ALIGN); 1418 dest_ring->base_addr_ce_space = 1419 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1420 CE_DESC_RING_ALIGN); 1421 1422 return dest_ring; 1423 } 1424 1425 /* 1426 * Initialize a Copy Engine based on caller-supplied attributes. 1427 * This may be called once to initialize both source and destination 1428 * rings or it may be called twice for separate source and destination 1429 * initialization. It may be that only one side or the other is 1430 * initialized by software/firmware. 1431 */ 1432 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 1433 const struct ce_attr *attr) 1434 { 1435 int ret; 1436 1437 if (attr->src_nentries) { 1438 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1439 if (ret) { 1440 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1441 ce_id, ret); 1442 return ret; 1443 } 1444 } 1445 1446 if (attr->dest_nentries) { 1447 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1448 if (ret) { 1449 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1450 ce_id, ret); 1451 return ret; 1452 } 1453 } 1454 1455 return 0; 1456 } 1457 1458 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1459 { 1460 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1461 1462 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0); 1463 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1464 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1465 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1466 } 1467 1468 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1469 { 1470 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1471 1472 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0); 1473 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1474 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1475 } 1476 1477 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1478 { 1479 ath10k_ce_deinit_src_ring(ar, ce_id); 1480 ath10k_ce_deinit_dest_ring(ar, ce_id); 1481 } 1482 1483 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1484 { 1485 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1486 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1487 1488 if (ce_state->src_ring) { 1489 dma_free_coherent(ar->dev, 1490 (ce_state->src_ring->nentries * 1491 sizeof(struct ce_desc) + 1492 CE_DESC_RING_ALIGN), 1493 ce_state->src_ring->base_addr_owner_space, 1494 ce_state->src_ring->base_addr_ce_space); 1495 kfree(ce_state->src_ring); 1496 } 1497 1498 if (ce_state->dest_ring) { 1499 dma_free_coherent(ar->dev, 1500 (ce_state->dest_ring->nentries * 1501 sizeof(struct ce_desc) + 1502 CE_DESC_RING_ALIGN), 1503 ce_state->dest_ring->base_addr_owner_space, 1504 ce_state->dest_ring->base_addr_ce_space); 1505 kfree(ce_state->dest_ring); 1506 } 1507 1508 ce_state->src_ring = NULL; 1509 ce_state->dest_ring = NULL; 1510 } 1511 1512 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) 1513 { 1514 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1515 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1516 1517 if (ce_state->src_ring) { 1518 dma_free_coherent(ar->dev, 1519 (ce_state->src_ring->nentries * 1520 sizeof(struct ce_desc_64) + 1521 CE_DESC_RING_ALIGN), 1522 ce_state->src_ring->base_addr_owner_space, 1523 ce_state->src_ring->base_addr_ce_space); 1524 kfree(ce_state->src_ring); 1525 } 1526 1527 if (ce_state->dest_ring) { 1528 dma_free_coherent(ar->dev, 1529 (ce_state->dest_ring->nentries * 1530 sizeof(struct ce_desc_64) + 1531 CE_DESC_RING_ALIGN), 1532 ce_state->dest_ring->base_addr_owner_space, 1533 ce_state->dest_ring->base_addr_ce_space); 1534 kfree(ce_state->dest_ring); 1535 } 1536 1537 ce_state->src_ring = NULL; 1538 ce_state->dest_ring = NULL; 1539 } 1540 1541 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1542 { 1543 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1544 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1545 1546 ce_state->ops->ce_free_pipe(ar, ce_id); 1547 } 1548 1549 void ath10k_ce_dump_registers(struct ath10k *ar, 1550 struct ath10k_fw_crash_data *crash_data) 1551 { 1552 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1553 struct ath10k_ce_crash_data ce_data; 1554 u32 addr, id; 1555 1556 lockdep_assert_held(&ar->data_lock); 1557 1558 ath10k_err(ar, "Copy Engine register dump:\n"); 1559 1560 spin_lock_bh(&ce->ce_lock); 1561 for (id = 0; id < CE_COUNT; id++) { 1562 addr = ath10k_ce_base_address(ar, id); 1563 ce_data.base_addr = cpu_to_le32(addr); 1564 1565 ce_data.src_wr_idx = 1566 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1567 ce_data.src_r_idx = 1568 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1569 ce_data.dst_wr_idx = 1570 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1571 ce_data.dst_r_idx = 1572 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1573 1574 if (crash_data) 1575 crash_data->ce_crash_data[id] = ce_data; 1576 1577 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1578 le32_to_cpu(ce_data.base_addr), 1579 le32_to_cpu(ce_data.src_wr_idx), 1580 le32_to_cpu(ce_data.src_r_idx), 1581 le32_to_cpu(ce_data.dst_wr_idx), 1582 le32_to_cpu(ce_data.dst_r_idx)); 1583 } 1584 1585 spin_unlock_bh(&ce->ce_lock); 1586 } 1587 1588 static const struct ath10k_ce_ops ce_ops = { 1589 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring, 1590 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring, 1591 .ce_rx_post_buf = __ath10k_ce_rx_post_buf, 1592 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock, 1593 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next, 1594 .ce_extract_desc_data = ath10k_ce_extract_desc_data, 1595 .ce_free_pipe = _ath10k_ce_free_pipe, 1596 .ce_send_nolock = _ath10k_ce_send_nolock, 1597 }; 1598 1599 static const struct ath10k_ce_ops ce_64_ops = { 1600 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64, 1601 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64, 1602 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64, 1603 .ce_completed_recv_next_nolock = 1604 _ath10k_ce_completed_recv_next_nolock_64, 1605 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64, 1606 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64, 1607 .ce_free_pipe = _ath10k_ce_free_pipe_64, 1608 .ce_send_nolock = _ath10k_ce_send_nolock_64, 1609 }; 1610 1611 static void ath10k_ce_set_ops(struct ath10k *ar, 1612 struct ath10k_ce_pipe *ce_state) 1613 { 1614 switch (ar->hw_rev) { 1615 case ATH10K_HW_WCN3990: 1616 ce_state->ops = &ce_64_ops; 1617 break; 1618 default: 1619 ce_state->ops = &ce_ops; 1620 break; 1621 } 1622 } 1623 1624 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1625 const struct ce_attr *attr) 1626 { 1627 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1628 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1629 int ret; 1630 1631 ath10k_ce_set_ops(ar, ce_state); 1632 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1633 * additional TX locking checks. 1634 * 1635 * For the lack of a better place do the check here. 1636 */ 1637 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1638 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1639 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1640 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1641 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1642 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1643 1644 ce_state->ar = ar; 1645 ce_state->id = ce_id; 1646 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1647 ce_state->attr_flags = attr->flags; 1648 ce_state->src_sz_max = attr->src_sz_max; 1649 1650 if (attr->src_nentries) 1651 ce_state->send_cb = attr->send_cb; 1652 1653 if (attr->dest_nentries) 1654 ce_state->recv_cb = attr->recv_cb; 1655 1656 if (attr->src_nentries) { 1657 ce_state->src_ring = 1658 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); 1659 if (IS_ERR(ce_state->src_ring)) { 1660 ret = PTR_ERR(ce_state->src_ring); 1661 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n", 1662 ce_id, ret); 1663 ce_state->src_ring = NULL; 1664 return ret; 1665 } 1666 } 1667 1668 if (attr->dest_nentries) { 1669 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, 1670 ce_id, 1671 attr); 1672 if (IS_ERR(ce_state->dest_ring)) { 1673 ret = PTR_ERR(ce_state->dest_ring); 1674 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n", 1675 ce_id, ret); 1676 ce_state->dest_ring = NULL; 1677 return ret; 1678 } 1679 } 1680 1681 return 0; 1682 } 1683