1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. 4 * Copyright (c) 2018 The Linux Foundation. All rights reserved. 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "hif.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * receives an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar, 63 struct ath10k_ce_pipe *ce_state) 64 { 65 u32 ce_id = ce_state->id; 66 u32 addr = 0; 67 68 switch (ce_id) { 69 case 0: 70 addr = 0x00032000; 71 break; 72 case 3: 73 addr = 0x0003200C; 74 break; 75 case 4: 76 addr = 0x00032010; 77 break; 78 case 5: 79 addr = 0x00032014; 80 break; 81 case 7: 82 addr = 0x0003201C; 83 break; 84 default: 85 ath10k_warn(ar, "invalid CE id: %d", ce_id); 86 break; 87 } 88 return addr; 89 } 90 91 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar, 92 struct ath10k_ce_pipe *ce_state) 93 { 94 u32 ce_id = ce_state->id; 95 u32 addr = 0; 96 97 switch (ce_id) { 98 case 1: 99 addr = 0x00032034; 100 break; 101 case 2: 102 addr = 0x00032038; 103 break; 104 case 5: 105 addr = 0x00032044; 106 break; 107 case 7: 108 addr = 0x0003204C; 109 break; 110 case 8: 111 addr = 0x00032050; 112 break; 113 case 9: 114 addr = 0x00032054; 115 break; 116 case 10: 117 addr = 0x00032058; 118 break; 119 case 11: 120 addr = 0x0003205C; 121 break; 122 default: 123 ath10k_warn(ar, "invalid CE id: %d", ce_id); 124 break; 125 } 126 127 return addr; 128 } 129 130 static inline unsigned int 131 ath10k_set_ring_byte(unsigned int offset, 132 struct ath10k_hw_ce_regs_addr_map *addr_map) 133 { 134 return ((offset << addr_map->lsb) & addr_map->mask); 135 } 136 137 static inline unsigned int 138 ath10k_get_ring_byte(unsigned int offset, 139 struct ath10k_hw_ce_regs_addr_map *addr_map) 140 { 141 return ((offset & addr_map->mask) >> (addr_map->lsb)); 142 } 143 144 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) 145 { 146 struct ath10k_ce *ce = ath10k_ce_priv(ar); 147 148 return ce->bus_ops->read32(ar, offset); 149 } 150 151 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) 152 { 153 struct ath10k_ce *ce = ath10k_ce_priv(ar); 154 155 ce->bus_ops->write32(ar, offset, value); 156 } 157 158 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 159 u32 ce_ctrl_addr, 160 unsigned int n) 161 { 162 ath10k_ce_write32(ar, ce_ctrl_addr + 163 ar->hw_ce_regs->dst_wr_index_addr, n); 164 } 165 166 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 167 u32 ce_ctrl_addr) 168 { 169 return ath10k_ce_read32(ar, ce_ctrl_addr + 170 ar->hw_ce_regs->dst_wr_index_addr); 171 } 172 173 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 unsigned int n) 176 { 177 ath10k_ce_write32(ar, ce_ctrl_addr + 178 ar->hw_ce_regs->sr_wr_index_addr, n); 179 } 180 181 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 182 u32 ce_ctrl_addr) 183 { 184 return ath10k_ce_read32(ar, ce_ctrl_addr + 185 ar->hw_ce_regs->sr_wr_index_addr); 186 } 187 188 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar, 189 u32 ce_id) 190 { 191 struct ath10k_ce *ce = ath10k_ce_priv(ar); 192 193 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK; 194 } 195 196 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 197 u32 ce_ctrl_addr) 198 { 199 struct ath10k_ce *ce = ath10k_ce_priv(ar); 200 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 201 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 202 u32 index; 203 204 if (ar->hw_params.rri_on_ddr && 205 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 206 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id); 207 else 208 index = ath10k_ce_read32(ar, ce_ctrl_addr + 209 ar->hw_ce_regs->current_srri_addr); 210 211 return index; 212 } 213 214 static inline void 215 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar, 216 struct ath10k_ce_pipe *ce_state, 217 unsigned int value) 218 { 219 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value); 220 } 221 222 static inline void 223 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar, 224 struct ath10k_ce_pipe *ce_state, 225 unsigned int value) 226 { 227 ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value); 228 } 229 230 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 231 u32 ce_ctrl_addr, 232 unsigned int addr) 233 { 234 ath10k_ce_write32(ar, ce_ctrl_addr + 235 ar->hw_ce_regs->sr_base_addr, addr); 236 } 237 238 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 239 u32 ce_ctrl_addr, 240 unsigned int n) 241 { 242 ath10k_ce_write32(ar, ce_ctrl_addr + 243 ar->hw_ce_regs->sr_size_addr, n); 244 } 245 246 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 247 u32 ce_ctrl_addr, 248 unsigned int n) 249 { 250 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 251 252 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 253 ctrl_regs->addr); 254 255 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 256 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | 257 ath10k_set_ring_byte(n, ctrl_regs->dmax)); 258 } 259 260 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 261 u32 ce_ctrl_addr, 262 unsigned int n) 263 { 264 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 265 266 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 267 ctrl_regs->addr); 268 269 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 270 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | 271 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); 272 } 273 274 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 275 u32 ce_ctrl_addr, 276 unsigned int n) 277 { 278 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 279 280 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 281 ctrl_regs->addr); 282 283 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 284 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | 285 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); 286 } 287 288 static inline 289 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id) 290 { 291 struct ath10k_ce *ce = ath10k_ce_priv(ar); 292 293 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) & 294 CE_DDR_RRI_MASK; 295 } 296 297 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 298 u32 ce_ctrl_addr) 299 { 300 struct ath10k_ce *ce = ath10k_ce_priv(ar); 301 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 302 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 303 u32 index; 304 305 if (ar->hw_params.rri_on_ddr && 306 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 307 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id); 308 else 309 index = ath10k_ce_read32(ar, ce_ctrl_addr + 310 ar->hw_ce_regs->current_drri_addr); 311 312 return index; 313 } 314 315 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 316 u32 ce_ctrl_addr, 317 u32 addr) 318 { 319 ath10k_ce_write32(ar, ce_ctrl_addr + 320 ar->hw_ce_regs->dr_base_addr, addr); 321 } 322 323 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 324 u32 ce_ctrl_addr, 325 unsigned int n) 326 { 327 ath10k_ce_write32(ar, ce_ctrl_addr + 328 ar->hw_ce_regs->dr_size_addr, n); 329 } 330 331 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 332 u32 ce_ctrl_addr, 333 unsigned int n) 334 { 335 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 336 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 337 338 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 339 (addr & ~(srcr_wm->wm_high->mask)) | 340 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); 341 } 342 343 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 344 u32 ce_ctrl_addr, 345 unsigned int n) 346 { 347 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 348 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 349 350 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 351 (addr & ~(srcr_wm->wm_low->mask)) | 352 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); 353 } 354 355 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 356 u32 ce_ctrl_addr, 357 unsigned int n) 358 { 359 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 360 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 361 362 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 363 (addr & ~(dstr_wm->wm_high->mask)) | 364 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); 365 } 366 367 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 368 u32 ce_ctrl_addr, 369 unsigned int n) 370 { 371 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 372 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 373 374 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 375 (addr & ~(dstr_wm->wm_low->mask)) | 376 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); 377 } 378 379 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 380 u32 ce_ctrl_addr) 381 { 382 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 383 384 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 385 ar->hw_ce_regs->host_ie_addr); 386 387 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 388 host_ie_addr | host_ie->copy_complete->mask); 389 } 390 391 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 392 u32 ce_ctrl_addr) 393 { 394 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 395 396 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 397 ar->hw_ce_regs->host_ie_addr); 398 399 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 400 host_ie_addr & ~(host_ie->copy_complete->mask)); 401 } 402 403 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 404 u32 ce_ctrl_addr) 405 { 406 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 407 408 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 409 ar->hw_ce_regs->host_ie_addr); 410 411 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 412 host_ie_addr & ~(wm_regs->wm_mask)); 413 } 414 415 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 416 u32 ce_ctrl_addr) 417 { 418 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 419 420 u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 421 ar->hw_ce_regs->misc_ie_addr); 422 423 ath10k_ce_write32(ar, 424 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 425 misc_ie_addr | misc_regs->err_mask); 426 } 427 428 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 429 u32 ce_ctrl_addr) 430 { 431 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 432 433 u32 misc_ie_addr = ath10k_ce_read32(ar, 434 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); 435 436 ath10k_ce_write32(ar, 437 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 438 misc_ie_addr & ~(misc_regs->err_mask)); 439 } 440 441 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 442 u32 ce_ctrl_addr, 443 unsigned int mask) 444 { 445 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 446 447 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); 448 } 449 450 /* 451 * Guts of ath10k_ce_send. 452 * The caller takes responsibility for any needed locking. 453 */ 454 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 455 void *per_transfer_context, 456 dma_addr_t buffer, 457 unsigned int nbytes, 458 unsigned int transfer_id, 459 unsigned int flags) 460 { 461 struct ath10k *ar = ce_state->ar; 462 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 463 struct ce_desc *desc, sdesc; 464 unsigned int nentries_mask = src_ring->nentries_mask; 465 unsigned int sw_index = src_ring->sw_index; 466 unsigned int write_index = src_ring->write_index; 467 u32 ctrl_addr = ce_state->ctrl_addr; 468 u32 desc_flags = 0; 469 int ret = 0; 470 471 if (nbytes > ce_state->src_sz_max) 472 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 473 __func__, nbytes, ce_state->src_sz_max); 474 475 if (unlikely(CE_RING_DELTA(nentries_mask, 476 write_index, sw_index - 1) <= 0)) { 477 ret = -ENOSR; 478 goto exit; 479 } 480 481 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 482 write_index); 483 484 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 485 486 if (flags & CE_SEND_FLAG_GATHER) 487 desc_flags |= CE_DESC_FLAGS_GATHER; 488 if (flags & CE_SEND_FLAG_BYTE_SWAP) 489 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 490 491 sdesc.addr = __cpu_to_le32(buffer); 492 sdesc.nbytes = __cpu_to_le16(nbytes); 493 sdesc.flags = __cpu_to_le16(desc_flags); 494 495 *desc = sdesc; 496 497 src_ring->per_transfer_context[write_index] = per_transfer_context; 498 499 /* Update Source Ring Write Index */ 500 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 501 502 /* WORKAROUND */ 503 if (!(flags & CE_SEND_FLAG_GATHER)) { 504 if (ar->hw_params.shadow_reg_support) 505 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state, 506 write_index); 507 else 508 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, 509 write_index); 510 } 511 512 src_ring->write_index = write_index; 513 exit: 514 return ret; 515 } 516 517 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state, 518 void *per_transfer_context, 519 dma_addr_t buffer, 520 unsigned int nbytes, 521 unsigned int transfer_id, 522 unsigned int flags) 523 { 524 struct ath10k *ar = ce_state->ar; 525 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 526 struct ce_desc_64 *desc, sdesc; 527 unsigned int nentries_mask = src_ring->nentries_mask; 528 unsigned int sw_index; 529 unsigned int write_index = src_ring->write_index; 530 u32 ctrl_addr = ce_state->ctrl_addr; 531 __le32 *addr; 532 u32 desc_flags = 0; 533 int ret = 0; 534 535 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) 536 return -ESHUTDOWN; 537 538 if (nbytes > ce_state->src_sz_max) 539 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 540 __func__, nbytes, ce_state->src_sz_max); 541 542 if (ar->hw_params.rri_on_ddr) 543 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id); 544 else 545 sw_index = src_ring->sw_index; 546 547 if (unlikely(CE_RING_DELTA(nentries_mask, 548 write_index, sw_index - 1) <= 0)) { 549 ret = -ENOSR; 550 goto exit; 551 } 552 553 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 554 write_index); 555 556 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 557 558 if (flags & CE_SEND_FLAG_GATHER) 559 desc_flags |= CE_DESC_FLAGS_GATHER; 560 561 if (flags & CE_SEND_FLAG_BYTE_SWAP) 562 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 563 564 addr = (__le32 *)&sdesc.addr; 565 566 flags |= upper_32_bits(buffer) & CE_DESC_FLAGS_GET_MASK; 567 addr[0] = __cpu_to_le32(buffer); 568 addr[1] = __cpu_to_le32(flags); 569 if (flags & CE_SEND_FLAG_GATHER) 570 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER); 571 else 572 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER)); 573 574 sdesc.nbytes = __cpu_to_le16(nbytes); 575 sdesc.flags = __cpu_to_le16(desc_flags); 576 577 *desc = sdesc; 578 579 src_ring->per_transfer_context[write_index] = per_transfer_context; 580 581 /* Update Source Ring Write Index */ 582 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 583 584 if (!(flags & CE_SEND_FLAG_GATHER)) 585 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 586 587 src_ring->write_index = write_index; 588 exit: 589 return ret; 590 } 591 592 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 593 void *per_transfer_context, 594 dma_addr_t buffer, 595 unsigned int nbytes, 596 unsigned int transfer_id, 597 unsigned int flags) 598 { 599 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, 600 buffer, nbytes, transfer_id, flags); 601 } 602 EXPORT_SYMBOL(ath10k_ce_send_nolock); 603 604 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 605 { 606 struct ath10k *ar = pipe->ar; 607 struct ath10k_ce *ce = ath10k_ce_priv(ar); 608 struct ath10k_ce_ring *src_ring = pipe->src_ring; 609 u32 ctrl_addr = pipe->ctrl_addr; 610 611 lockdep_assert_held(&ce->ce_lock); 612 613 /* 614 * This function must be called only if there is an incomplete 615 * scatter-gather transfer (before index register is updated) 616 * that needs to be cleaned up. 617 */ 618 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 619 return; 620 621 if (WARN_ON_ONCE(src_ring->write_index == 622 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 623 return; 624 625 src_ring->write_index--; 626 src_ring->write_index &= src_ring->nentries_mask; 627 628 src_ring->per_transfer_context[src_ring->write_index] = NULL; 629 } 630 EXPORT_SYMBOL(__ath10k_ce_send_revert); 631 632 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 633 void *per_transfer_context, 634 dma_addr_t buffer, 635 unsigned int nbytes, 636 unsigned int transfer_id, 637 unsigned int flags) 638 { 639 struct ath10k *ar = ce_state->ar; 640 struct ath10k_ce *ce = ath10k_ce_priv(ar); 641 int ret; 642 643 spin_lock_bh(&ce->ce_lock); 644 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 645 buffer, nbytes, transfer_id, flags); 646 spin_unlock_bh(&ce->ce_lock); 647 648 return ret; 649 } 650 EXPORT_SYMBOL(ath10k_ce_send); 651 652 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 653 { 654 struct ath10k *ar = pipe->ar; 655 struct ath10k_ce *ce = ath10k_ce_priv(ar); 656 int delta; 657 658 spin_lock_bh(&ce->ce_lock); 659 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 660 pipe->src_ring->write_index, 661 pipe->src_ring->sw_index - 1); 662 spin_unlock_bh(&ce->ce_lock); 663 664 return delta; 665 } 666 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries); 667 668 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 669 { 670 struct ath10k *ar = pipe->ar; 671 struct ath10k_ce *ce = ath10k_ce_priv(ar); 672 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 673 unsigned int nentries_mask = dest_ring->nentries_mask; 674 unsigned int write_index = dest_ring->write_index; 675 unsigned int sw_index = dest_ring->sw_index; 676 677 lockdep_assert_held(&ce->ce_lock); 678 679 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 680 } 681 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs); 682 683 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 684 dma_addr_t paddr) 685 { 686 struct ath10k *ar = pipe->ar; 687 struct ath10k_ce *ce = ath10k_ce_priv(ar); 688 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 689 unsigned int nentries_mask = dest_ring->nentries_mask; 690 unsigned int write_index = dest_ring->write_index; 691 unsigned int sw_index = dest_ring->sw_index; 692 struct ce_desc *base = dest_ring->base_addr_owner_space; 693 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 694 u32 ctrl_addr = pipe->ctrl_addr; 695 696 lockdep_assert_held(&ce->ce_lock); 697 698 if ((pipe->id != 5) && 699 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 700 return -ENOSPC; 701 702 desc->addr = __cpu_to_le32(paddr); 703 desc->nbytes = 0; 704 705 dest_ring->per_transfer_context[write_index] = ctx; 706 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 707 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 708 dest_ring->write_index = write_index; 709 710 return 0; 711 } 712 713 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe, 714 void *ctx, 715 dma_addr_t paddr) 716 { 717 struct ath10k *ar = pipe->ar; 718 struct ath10k_ce *ce = ath10k_ce_priv(ar); 719 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 720 unsigned int nentries_mask = dest_ring->nentries_mask; 721 unsigned int write_index = dest_ring->write_index; 722 unsigned int sw_index = dest_ring->sw_index; 723 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 724 struct ce_desc_64 *desc = 725 CE_DEST_RING_TO_DESC_64(base, write_index); 726 u32 ctrl_addr = pipe->ctrl_addr; 727 728 lockdep_assert_held(&ce->ce_lock); 729 730 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 731 return -ENOSPC; 732 733 desc->addr = __cpu_to_le64(paddr); 734 desc->addr &= __cpu_to_le64(CE_DESC_37BIT_ADDR_MASK); 735 736 desc->nbytes = 0; 737 738 dest_ring->per_transfer_context[write_index] = ctx; 739 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 740 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 741 dest_ring->write_index = write_index; 742 743 return 0; 744 } 745 746 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 747 { 748 struct ath10k *ar = pipe->ar; 749 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 750 unsigned int nentries_mask = dest_ring->nentries_mask; 751 unsigned int write_index = dest_ring->write_index; 752 u32 ctrl_addr = pipe->ctrl_addr; 753 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 754 755 /* Prevent CE ring stuck issue that will occur when ring is full. 756 * Make sure that write index is 1 less than read index. 757 */ 758 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index) 759 nentries -= 1; 760 761 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 762 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 763 dest_ring->write_index = write_index; 764 } 765 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx); 766 767 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 768 dma_addr_t paddr) 769 { 770 struct ath10k *ar = pipe->ar; 771 struct ath10k_ce *ce = ath10k_ce_priv(ar); 772 int ret; 773 774 spin_lock_bh(&ce->ce_lock); 775 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); 776 spin_unlock_bh(&ce->ce_lock); 777 778 return ret; 779 } 780 EXPORT_SYMBOL(ath10k_ce_rx_post_buf); 781 782 /* 783 * Guts of ath10k_ce_completed_recv_next. 784 * The caller takes responsibility for any necessary locking. 785 */ 786 static int 787 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 788 void **per_transfer_contextp, 789 unsigned int *nbytesp) 790 { 791 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 792 unsigned int nentries_mask = dest_ring->nentries_mask; 793 unsigned int sw_index = dest_ring->sw_index; 794 795 struct ce_desc *base = dest_ring->base_addr_owner_space; 796 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 797 struct ce_desc sdesc; 798 u16 nbytes; 799 800 /* Copy in one go for performance reasons */ 801 sdesc = *desc; 802 803 nbytes = __le16_to_cpu(sdesc.nbytes); 804 if (nbytes == 0) { 805 /* 806 * This closes a relatively unusual race where the Host 807 * sees the updated DRRI before the update to the 808 * corresponding descriptor has completed. We treat this 809 * as a descriptor that is not yet done. 810 */ 811 return -EIO; 812 } 813 814 desc->nbytes = 0; 815 816 /* Return data from completed destination descriptor */ 817 *nbytesp = nbytes; 818 819 if (per_transfer_contextp) 820 *per_transfer_contextp = 821 dest_ring->per_transfer_context[sw_index]; 822 823 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 824 * So update transfer context all CEs except CE5. 825 */ 826 if (ce_state->id != 5) 827 dest_ring->per_transfer_context[sw_index] = NULL; 828 829 /* Update sw_index */ 830 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 831 dest_ring->sw_index = sw_index; 832 833 return 0; 834 } 835 836 static int 837 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state, 838 void **per_transfer_contextp, 839 unsigned int *nbytesp) 840 { 841 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 842 unsigned int nentries_mask = dest_ring->nentries_mask; 843 unsigned int sw_index = dest_ring->sw_index; 844 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 845 struct ce_desc_64 *desc = 846 CE_DEST_RING_TO_DESC_64(base, sw_index); 847 struct ce_desc_64 sdesc; 848 u16 nbytes; 849 850 /* Copy in one go for performance reasons */ 851 sdesc = *desc; 852 853 nbytes = __le16_to_cpu(sdesc.nbytes); 854 if (nbytes == 0) { 855 /* This closes a relatively unusual race where the Host 856 * sees the updated DRRI before the update to the 857 * corresponding descriptor has completed. We treat this 858 * as a descriptor that is not yet done. 859 */ 860 return -EIO; 861 } 862 863 desc->nbytes = 0; 864 865 /* Return data from completed destination descriptor */ 866 *nbytesp = nbytes; 867 868 if (per_transfer_contextp) 869 *per_transfer_contextp = 870 dest_ring->per_transfer_context[sw_index]; 871 872 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 873 * So update transfer context all CEs except CE5. 874 */ 875 if (ce_state->id != 5) 876 dest_ring->per_transfer_context[sw_index] = NULL; 877 878 /* Update sw_index */ 879 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 880 dest_ring->sw_index = sw_index; 881 882 return 0; 883 } 884 885 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 886 void **per_transfer_ctx, 887 unsigned int *nbytesp) 888 { 889 return ce_state->ops->ce_completed_recv_next_nolock(ce_state, 890 per_transfer_ctx, 891 nbytesp); 892 } 893 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock); 894 895 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 896 void **per_transfer_contextp, 897 unsigned int *nbytesp) 898 { 899 struct ath10k *ar = ce_state->ar; 900 struct ath10k_ce *ce = ath10k_ce_priv(ar); 901 int ret; 902 903 spin_lock_bh(&ce->ce_lock); 904 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, 905 per_transfer_contextp, 906 nbytesp); 907 908 spin_unlock_bh(&ce->ce_lock); 909 910 return ret; 911 } 912 EXPORT_SYMBOL(ath10k_ce_completed_recv_next); 913 914 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 915 void **per_transfer_contextp, 916 dma_addr_t *bufferp) 917 { 918 struct ath10k_ce_ring *dest_ring; 919 unsigned int nentries_mask; 920 unsigned int sw_index; 921 unsigned int write_index; 922 int ret; 923 struct ath10k *ar; 924 struct ath10k_ce *ce; 925 926 dest_ring = ce_state->dest_ring; 927 928 if (!dest_ring) 929 return -EIO; 930 931 ar = ce_state->ar; 932 ce = ath10k_ce_priv(ar); 933 934 spin_lock_bh(&ce->ce_lock); 935 936 nentries_mask = dest_ring->nentries_mask; 937 sw_index = dest_ring->sw_index; 938 write_index = dest_ring->write_index; 939 if (write_index != sw_index) { 940 struct ce_desc *base = dest_ring->base_addr_owner_space; 941 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 942 943 /* Return data from completed destination descriptor */ 944 *bufferp = __le32_to_cpu(desc->addr); 945 946 if (per_transfer_contextp) 947 *per_transfer_contextp = 948 dest_ring->per_transfer_context[sw_index]; 949 950 /* sanity */ 951 dest_ring->per_transfer_context[sw_index] = NULL; 952 desc->nbytes = 0; 953 954 /* Update sw_index */ 955 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 956 dest_ring->sw_index = sw_index; 957 ret = 0; 958 } else { 959 ret = -EIO; 960 } 961 962 spin_unlock_bh(&ce->ce_lock); 963 964 return ret; 965 } 966 967 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state, 968 void **per_transfer_contextp, 969 dma_addr_t *bufferp) 970 { 971 struct ath10k_ce_ring *dest_ring; 972 unsigned int nentries_mask; 973 unsigned int sw_index; 974 unsigned int write_index; 975 int ret; 976 struct ath10k *ar; 977 struct ath10k_ce *ce; 978 979 dest_ring = ce_state->dest_ring; 980 981 if (!dest_ring) 982 return -EIO; 983 984 ar = ce_state->ar; 985 ce = ath10k_ce_priv(ar); 986 987 spin_lock_bh(&ce->ce_lock); 988 989 nentries_mask = dest_ring->nentries_mask; 990 sw_index = dest_ring->sw_index; 991 write_index = dest_ring->write_index; 992 if (write_index != sw_index) { 993 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 994 struct ce_desc_64 *desc = 995 CE_DEST_RING_TO_DESC_64(base, sw_index); 996 997 /* Return data from completed destination descriptor */ 998 *bufferp = __le64_to_cpu(desc->addr); 999 1000 if (per_transfer_contextp) 1001 *per_transfer_contextp = 1002 dest_ring->per_transfer_context[sw_index]; 1003 1004 /* sanity */ 1005 dest_ring->per_transfer_context[sw_index] = NULL; 1006 desc->nbytes = 0; 1007 1008 /* Update sw_index */ 1009 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1010 dest_ring->sw_index = sw_index; 1011 ret = 0; 1012 } else { 1013 ret = -EIO; 1014 } 1015 1016 spin_unlock_bh(&ce->ce_lock); 1017 1018 return ret; 1019 } 1020 1021 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 1022 void **per_transfer_contextp, 1023 dma_addr_t *bufferp) 1024 { 1025 return ce_state->ops->ce_revoke_recv_next(ce_state, 1026 per_transfer_contextp, 1027 bufferp); 1028 } 1029 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next); 1030 1031 /* 1032 * Guts of ath10k_ce_completed_send_next. 1033 * The caller takes responsibility for any necessary locking. 1034 */ 1035 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 1036 void **per_transfer_contextp) 1037 { 1038 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1039 u32 ctrl_addr = ce_state->ctrl_addr; 1040 struct ath10k *ar = ce_state->ar; 1041 unsigned int nentries_mask = src_ring->nentries_mask; 1042 unsigned int sw_index = src_ring->sw_index; 1043 unsigned int read_index; 1044 struct ce_desc *desc; 1045 1046 if (src_ring->hw_index == sw_index) { 1047 /* 1048 * The SW completion index has caught up with the cached 1049 * version of the HW completion index. 1050 * Update the cached HW completion index to see whether 1051 * the SW has really caught up to the HW, or if the cached 1052 * value of the HW index has become stale. 1053 */ 1054 1055 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1056 if (read_index == 0xffffffff) 1057 return -ENODEV; 1058 1059 read_index &= nentries_mask; 1060 src_ring->hw_index = read_index; 1061 } 1062 1063 if (ar->hw_params.rri_on_ddr) 1064 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1065 else 1066 read_index = src_ring->hw_index; 1067 1068 if (read_index == sw_index) 1069 return -EIO; 1070 1071 if (per_transfer_contextp) 1072 *per_transfer_contextp = 1073 src_ring->per_transfer_context[sw_index]; 1074 1075 /* sanity */ 1076 src_ring->per_transfer_context[sw_index] = NULL; 1077 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 1078 sw_index); 1079 desc->nbytes = 0; 1080 1081 /* Update sw_index */ 1082 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1083 src_ring->sw_index = sw_index; 1084 1085 return 0; 1086 } 1087 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock); 1088 1089 static void ath10k_ce_extract_desc_data(struct ath10k *ar, 1090 struct ath10k_ce_ring *src_ring, 1091 u32 sw_index, 1092 dma_addr_t *bufferp, 1093 u32 *nbytesp, 1094 u32 *transfer_idp) 1095 { 1096 struct ce_desc *base = src_ring->base_addr_owner_space; 1097 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 1098 1099 /* Return data from completed source descriptor */ 1100 *bufferp = __le32_to_cpu(desc->addr); 1101 *nbytesp = __le16_to_cpu(desc->nbytes); 1102 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1103 CE_DESC_FLAGS_META_DATA); 1104 } 1105 1106 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar, 1107 struct ath10k_ce_ring *src_ring, 1108 u32 sw_index, 1109 dma_addr_t *bufferp, 1110 u32 *nbytesp, 1111 u32 *transfer_idp) 1112 { 1113 struct ce_desc_64 *base = src_ring->base_addr_owner_space; 1114 struct ce_desc_64 *desc = 1115 CE_SRC_RING_TO_DESC_64(base, sw_index); 1116 1117 /* Return data from completed source descriptor */ 1118 *bufferp = __le64_to_cpu(desc->addr); 1119 *nbytesp = __le16_to_cpu(desc->nbytes); 1120 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1121 CE_DESC_FLAGS_META_DATA); 1122 } 1123 1124 /* NB: Modeled after ath10k_ce_completed_send_next */ 1125 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 1126 void **per_transfer_contextp, 1127 dma_addr_t *bufferp, 1128 unsigned int *nbytesp, 1129 unsigned int *transfer_idp) 1130 { 1131 struct ath10k_ce_ring *src_ring; 1132 unsigned int nentries_mask; 1133 unsigned int sw_index; 1134 unsigned int write_index; 1135 int ret; 1136 struct ath10k *ar; 1137 struct ath10k_ce *ce; 1138 1139 src_ring = ce_state->src_ring; 1140 1141 if (!src_ring) 1142 return -EIO; 1143 1144 ar = ce_state->ar; 1145 ce = ath10k_ce_priv(ar); 1146 1147 spin_lock_bh(&ce->ce_lock); 1148 1149 nentries_mask = src_ring->nentries_mask; 1150 sw_index = src_ring->sw_index; 1151 write_index = src_ring->write_index; 1152 1153 if (write_index != sw_index) { 1154 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, 1155 bufferp, nbytesp, 1156 transfer_idp); 1157 1158 if (per_transfer_contextp) 1159 *per_transfer_contextp = 1160 src_ring->per_transfer_context[sw_index]; 1161 1162 /* sanity */ 1163 src_ring->per_transfer_context[sw_index] = NULL; 1164 1165 /* Update sw_index */ 1166 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1167 src_ring->sw_index = sw_index; 1168 ret = 0; 1169 } else { 1170 ret = -EIO; 1171 } 1172 1173 spin_unlock_bh(&ce->ce_lock); 1174 1175 return ret; 1176 } 1177 EXPORT_SYMBOL(ath10k_ce_cancel_send_next); 1178 1179 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 1180 void **per_transfer_contextp) 1181 { 1182 struct ath10k *ar = ce_state->ar; 1183 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1184 int ret; 1185 1186 spin_lock_bh(&ce->ce_lock); 1187 ret = ath10k_ce_completed_send_next_nolock(ce_state, 1188 per_transfer_contextp); 1189 spin_unlock_bh(&ce->ce_lock); 1190 1191 return ret; 1192 } 1193 EXPORT_SYMBOL(ath10k_ce_completed_send_next); 1194 1195 /* 1196 * Guts of interrupt handler for per-engine interrupts on a particular CE. 1197 * 1198 * Invokes registered callbacks for recv_complete, 1199 * send_complete, and watermarks. 1200 */ 1201 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 1202 { 1203 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1204 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1205 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 1206 u32 ctrl_addr = ce_state->ctrl_addr; 1207 1208 spin_lock_bh(&ce->ce_lock); 1209 1210 /* Clear the copy-complete interrupts that will be handled here. */ 1211 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 1212 wm_regs->cc_mask); 1213 1214 spin_unlock_bh(&ce->ce_lock); 1215 1216 if (ce_state->recv_cb) 1217 ce_state->recv_cb(ce_state); 1218 1219 if (ce_state->send_cb) 1220 ce_state->send_cb(ce_state); 1221 1222 spin_lock_bh(&ce->ce_lock); 1223 1224 /* 1225 * Misc CE interrupts are not being handled, but still need 1226 * to be cleared. 1227 */ 1228 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask); 1229 1230 spin_unlock_bh(&ce->ce_lock); 1231 } 1232 EXPORT_SYMBOL(ath10k_ce_per_engine_service); 1233 1234 /* 1235 * Handler for per-engine interrupts on ALL active CEs. 1236 * This is used in cases where the system is sharing a 1237 * single interrput for all CEs 1238 */ 1239 1240 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 1241 { 1242 int ce_id; 1243 u32 intr_summary; 1244 1245 intr_summary = ath10k_ce_interrupt_summary(ar); 1246 1247 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 1248 if (intr_summary & (1 << ce_id)) 1249 intr_summary &= ~(1 << ce_id); 1250 else 1251 /* no intr pending on this CE */ 1252 continue; 1253 1254 ath10k_ce_per_engine_service(ar, ce_id); 1255 } 1256 } 1257 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any); 1258 1259 /* 1260 * Adjust interrupts for the copy complete handler. 1261 * If it's needed for either send or recv, then unmask 1262 * this interrupt; otherwise, mask it. 1263 * 1264 * Called with ce_lock held. 1265 */ 1266 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 1267 { 1268 u32 ctrl_addr = ce_state->ctrl_addr; 1269 struct ath10k *ar = ce_state->ar; 1270 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 1271 1272 if ((!disable_copy_compl_intr) && 1273 (ce_state->send_cb || ce_state->recv_cb)) 1274 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 1275 else 1276 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1277 1278 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1279 } 1280 1281 int ath10k_ce_disable_interrupts(struct ath10k *ar) 1282 { 1283 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1284 struct ath10k_ce_pipe *ce_state; 1285 u32 ctrl_addr; 1286 int ce_id; 1287 1288 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 1289 ce_state = &ce->ce_states[ce_id]; 1290 if (ce_state->attr_flags & CE_ATTR_POLL) 1291 continue; 1292 1293 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1294 1295 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1296 ath10k_ce_error_intr_disable(ar, ctrl_addr); 1297 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1298 } 1299 1300 return 0; 1301 } 1302 EXPORT_SYMBOL(ath10k_ce_disable_interrupts); 1303 1304 void ath10k_ce_enable_interrupts(struct ath10k *ar) 1305 { 1306 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1307 int ce_id; 1308 struct ath10k_ce_pipe *ce_state; 1309 1310 /* Enable interrupts for copy engine that 1311 * are not using polling mode. 1312 */ 1313 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 1314 ce_state = &ce->ce_states[ce_id]; 1315 if (ce_state->attr_flags & CE_ATTR_POLL) 1316 continue; 1317 1318 ath10k_ce_per_engine_handler_adjust(ce_state); 1319 } 1320 } 1321 EXPORT_SYMBOL(ath10k_ce_enable_interrupts); 1322 1323 static int ath10k_ce_init_src_ring(struct ath10k *ar, 1324 unsigned int ce_id, 1325 const struct ce_attr *attr) 1326 { 1327 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1328 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1329 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1330 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1331 1332 nentries = roundup_pow_of_two(attr->src_nentries); 1333 1334 if (ar->hw_params.target_64bit) 1335 memset(src_ring->base_addr_owner_space, 0, 1336 nentries * sizeof(struct ce_desc_64)); 1337 else 1338 memset(src_ring->base_addr_owner_space, 0, 1339 nentries * sizeof(struct ce_desc)); 1340 1341 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1342 src_ring->sw_index &= src_ring->nentries_mask; 1343 src_ring->hw_index = src_ring->sw_index; 1344 1345 src_ring->write_index = 1346 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 1347 src_ring->write_index &= src_ring->nentries_mask; 1348 1349 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 1350 src_ring->base_addr_ce_space); 1351 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 1352 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 1353 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 1354 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 1355 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 1356 1357 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1358 "boot init ce src ring id %d entries %d base_addr %pK\n", 1359 ce_id, nentries, src_ring->base_addr_owner_space); 1360 1361 return 0; 1362 } 1363 1364 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1365 unsigned int ce_id, 1366 const struct ce_attr *attr) 1367 { 1368 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1369 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1370 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 1371 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1372 1373 nentries = roundup_pow_of_two(attr->dest_nentries); 1374 1375 if (ar->hw_params.target_64bit) 1376 memset(dest_ring->base_addr_owner_space, 0, 1377 nentries * sizeof(struct ce_desc_64)); 1378 else 1379 memset(dest_ring->base_addr_owner_space, 0, 1380 nentries * sizeof(struct ce_desc)); 1381 1382 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1383 dest_ring->sw_index &= dest_ring->nentries_mask; 1384 dest_ring->write_index = 1385 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1386 dest_ring->write_index &= dest_ring->nentries_mask; 1387 1388 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 1389 dest_ring->base_addr_ce_space); 1390 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1391 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1392 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1393 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1394 1395 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1396 "boot ce dest ring id %d entries %d base_addr %pK\n", 1397 ce_id, nentries, dest_ring->base_addr_owner_space); 1398 1399 return 0; 1400 } 1401 1402 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar, 1403 struct ath10k_ce_ring *src_ring, 1404 u32 nentries) 1405 { 1406 src_ring->shadow_base_unaligned = kcalloc(nentries, 1407 sizeof(struct ce_desc), 1408 GFP_KERNEL); 1409 if (!src_ring->shadow_base_unaligned) 1410 return -ENOMEM; 1411 1412 src_ring->shadow_base = (struct ce_desc *) 1413 PTR_ALIGN(src_ring->shadow_base_unaligned, 1414 CE_DESC_RING_ALIGN); 1415 return 0; 1416 } 1417 1418 static struct ath10k_ce_ring * 1419 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 1420 const struct ce_attr *attr) 1421 { 1422 struct ath10k_ce_ring *src_ring; 1423 u32 nentries = attr->src_nentries; 1424 dma_addr_t base_addr; 1425 int ret; 1426 1427 nentries = roundup_pow_of_two(nentries); 1428 1429 src_ring = kzalloc(struct_size(src_ring, per_transfer_context, 1430 nentries), GFP_KERNEL); 1431 if (src_ring == NULL) 1432 return ERR_PTR(-ENOMEM); 1433 1434 src_ring->nentries = nentries; 1435 src_ring->nentries_mask = nentries - 1; 1436 1437 /* 1438 * Legacy platforms that do not support cache 1439 * coherent DMA are unsupported 1440 */ 1441 src_ring->base_addr_owner_space_unaligned = 1442 dma_alloc_coherent(ar->dev, 1443 (nentries * sizeof(struct ce_desc) + 1444 CE_DESC_RING_ALIGN), 1445 &base_addr, GFP_KERNEL); 1446 if (!src_ring->base_addr_owner_space_unaligned) { 1447 kfree(src_ring); 1448 return ERR_PTR(-ENOMEM); 1449 } 1450 1451 src_ring->base_addr_ce_space_unaligned = base_addr; 1452 1453 src_ring->base_addr_owner_space = 1454 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1455 CE_DESC_RING_ALIGN); 1456 src_ring->base_addr_ce_space = 1457 ALIGN(src_ring->base_addr_ce_space_unaligned, 1458 CE_DESC_RING_ALIGN); 1459 1460 if (ar->hw_params.shadow_reg_support) { 1461 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1462 if (ret) { 1463 dma_free_coherent(ar->dev, 1464 (nentries * sizeof(struct ce_desc) + 1465 CE_DESC_RING_ALIGN), 1466 src_ring->base_addr_owner_space_unaligned, 1467 base_addr); 1468 kfree(src_ring); 1469 return ERR_PTR(ret); 1470 } 1471 } 1472 1473 return src_ring; 1474 } 1475 1476 static struct ath10k_ce_ring * 1477 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, 1478 const struct ce_attr *attr) 1479 { 1480 struct ath10k_ce_ring *src_ring; 1481 u32 nentries = attr->src_nentries; 1482 dma_addr_t base_addr; 1483 int ret; 1484 1485 nentries = roundup_pow_of_two(nentries); 1486 1487 src_ring = kzalloc(struct_size(src_ring, per_transfer_context, 1488 nentries), GFP_KERNEL); 1489 if (!src_ring) 1490 return ERR_PTR(-ENOMEM); 1491 1492 src_ring->nentries = nentries; 1493 src_ring->nentries_mask = nentries - 1; 1494 1495 /* Legacy platforms that do not support cache 1496 * coherent DMA are unsupported 1497 */ 1498 src_ring->base_addr_owner_space_unaligned = 1499 dma_alloc_coherent(ar->dev, 1500 (nentries * sizeof(struct ce_desc_64) + 1501 CE_DESC_RING_ALIGN), 1502 &base_addr, GFP_KERNEL); 1503 if (!src_ring->base_addr_owner_space_unaligned) { 1504 kfree(src_ring); 1505 return ERR_PTR(-ENOMEM); 1506 } 1507 1508 src_ring->base_addr_ce_space_unaligned = base_addr; 1509 1510 src_ring->base_addr_owner_space = 1511 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1512 CE_DESC_RING_ALIGN); 1513 src_ring->base_addr_ce_space = 1514 ALIGN(src_ring->base_addr_ce_space_unaligned, 1515 CE_DESC_RING_ALIGN); 1516 1517 if (ar->hw_params.shadow_reg_support) { 1518 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1519 if (ret) { 1520 dma_free_coherent(ar->dev, 1521 (nentries * sizeof(struct ce_desc_64) + 1522 CE_DESC_RING_ALIGN), 1523 src_ring->base_addr_owner_space_unaligned, 1524 base_addr); 1525 kfree(src_ring); 1526 return ERR_PTR(ret); 1527 } 1528 } 1529 1530 return src_ring; 1531 } 1532 1533 static struct ath10k_ce_ring * 1534 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 1535 const struct ce_attr *attr) 1536 { 1537 struct ath10k_ce_ring *dest_ring; 1538 u32 nentries; 1539 dma_addr_t base_addr; 1540 1541 nentries = roundup_pow_of_two(attr->dest_nentries); 1542 1543 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, 1544 nentries), GFP_KERNEL); 1545 if (dest_ring == NULL) 1546 return ERR_PTR(-ENOMEM); 1547 1548 dest_ring->nentries = nentries; 1549 dest_ring->nentries_mask = nentries - 1; 1550 1551 /* 1552 * Legacy platforms that do not support cache 1553 * coherent DMA are unsupported 1554 */ 1555 dest_ring->base_addr_owner_space_unaligned = 1556 dma_zalloc_coherent(ar->dev, 1557 (nentries * sizeof(struct ce_desc) + 1558 CE_DESC_RING_ALIGN), 1559 &base_addr, GFP_KERNEL); 1560 if (!dest_ring->base_addr_owner_space_unaligned) { 1561 kfree(dest_ring); 1562 return ERR_PTR(-ENOMEM); 1563 } 1564 1565 dest_ring->base_addr_ce_space_unaligned = base_addr; 1566 1567 dest_ring->base_addr_owner_space = 1568 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1569 CE_DESC_RING_ALIGN); 1570 dest_ring->base_addr_ce_space = 1571 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1572 CE_DESC_RING_ALIGN); 1573 1574 return dest_ring; 1575 } 1576 1577 static struct ath10k_ce_ring * 1578 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, 1579 const struct ce_attr *attr) 1580 { 1581 struct ath10k_ce_ring *dest_ring; 1582 u32 nentries; 1583 dma_addr_t base_addr; 1584 1585 nentries = roundup_pow_of_two(attr->dest_nentries); 1586 1587 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, 1588 nentries), GFP_KERNEL); 1589 if (!dest_ring) 1590 return ERR_PTR(-ENOMEM); 1591 1592 dest_ring->nentries = nentries; 1593 dest_ring->nentries_mask = nentries - 1; 1594 1595 /* Legacy platforms that do not support cache 1596 * coherent DMA are unsupported 1597 */ 1598 dest_ring->base_addr_owner_space_unaligned = 1599 dma_alloc_coherent(ar->dev, 1600 (nentries * sizeof(struct ce_desc_64) + 1601 CE_DESC_RING_ALIGN), 1602 &base_addr, GFP_KERNEL); 1603 if (!dest_ring->base_addr_owner_space_unaligned) { 1604 kfree(dest_ring); 1605 return ERR_PTR(-ENOMEM); 1606 } 1607 1608 dest_ring->base_addr_ce_space_unaligned = base_addr; 1609 1610 /* Correctly initialize memory to 0 to prevent garbage 1611 * data crashing system when download firmware 1612 */ 1613 memset(dest_ring->base_addr_owner_space_unaligned, 0, 1614 nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN); 1615 1616 dest_ring->base_addr_owner_space = 1617 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1618 CE_DESC_RING_ALIGN); 1619 dest_ring->base_addr_ce_space = 1620 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1621 CE_DESC_RING_ALIGN); 1622 1623 return dest_ring; 1624 } 1625 1626 /* 1627 * Initialize a Copy Engine based on caller-supplied attributes. 1628 * This may be called once to initialize both source and destination 1629 * rings or it may be called twice for separate source and destination 1630 * initialization. It may be that only one side or the other is 1631 * initialized by software/firmware. 1632 */ 1633 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 1634 const struct ce_attr *attr) 1635 { 1636 int ret; 1637 1638 if (attr->src_nentries) { 1639 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1640 if (ret) { 1641 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1642 ce_id, ret); 1643 return ret; 1644 } 1645 } 1646 1647 if (attr->dest_nentries) { 1648 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1649 if (ret) { 1650 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1651 ce_id, ret); 1652 return ret; 1653 } 1654 } 1655 1656 return 0; 1657 } 1658 EXPORT_SYMBOL(ath10k_ce_init_pipe); 1659 1660 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1661 { 1662 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1663 1664 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0); 1665 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1666 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1667 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1668 } 1669 1670 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1671 { 1672 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1673 1674 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0); 1675 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1676 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1677 } 1678 1679 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1680 { 1681 ath10k_ce_deinit_src_ring(ar, ce_id); 1682 ath10k_ce_deinit_dest_ring(ar, ce_id); 1683 } 1684 EXPORT_SYMBOL(ath10k_ce_deinit_pipe); 1685 1686 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1687 { 1688 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1689 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1690 1691 if (ce_state->src_ring) { 1692 if (ar->hw_params.shadow_reg_support) 1693 kfree(ce_state->src_ring->shadow_base_unaligned); 1694 dma_free_coherent(ar->dev, 1695 (ce_state->src_ring->nentries * 1696 sizeof(struct ce_desc) + 1697 CE_DESC_RING_ALIGN), 1698 ce_state->src_ring->base_addr_owner_space, 1699 ce_state->src_ring->base_addr_ce_space); 1700 kfree(ce_state->src_ring); 1701 } 1702 1703 if (ce_state->dest_ring) { 1704 dma_free_coherent(ar->dev, 1705 (ce_state->dest_ring->nentries * 1706 sizeof(struct ce_desc) + 1707 CE_DESC_RING_ALIGN), 1708 ce_state->dest_ring->base_addr_owner_space, 1709 ce_state->dest_ring->base_addr_ce_space); 1710 kfree(ce_state->dest_ring); 1711 } 1712 1713 ce_state->src_ring = NULL; 1714 ce_state->dest_ring = NULL; 1715 } 1716 1717 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) 1718 { 1719 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1720 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1721 1722 if (ce_state->src_ring) { 1723 if (ar->hw_params.shadow_reg_support) 1724 kfree(ce_state->src_ring->shadow_base_unaligned); 1725 dma_free_coherent(ar->dev, 1726 (ce_state->src_ring->nentries * 1727 sizeof(struct ce_desc_64) + 1728 CE_DESC_RING_ALIGN), 1729 ce_state->src_ring->base_addr_owner_space, 1730 ce_state->src_ring->base_addr_ce_space); 1731 kfree(ce_state->src_ring); 1732 } 1733 1734 if (ce_state->dest_ring) { 1735 dma_free_coherent(ar->dev, 1736 (ce_state->dest_ring->nentries * 1737 sizeof(struct ce_desc_64) + 1738 CE_DESC_RING_ALIGN), 1739 ce_state->dest_ring->base_addr_owner_space, 1740 ce_state->dest_ring->base_addr_ce_space); 1741 kfree(ce_state->dest_ring); 1742 } 1743 1744 ce_state->src_ring = NULL; 1745 ce_state->dest_ring = NULL; 1746 } 1747 1748 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1749 { 1750 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1751 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1752 1753 ce_state->ops->ce_free_pipe(ar, ce_id); 1754 } 1755 EXPORT_SYMBOL(ath10k_ce_free_pipe); 1756 1757 void ath10k_ce_dump_registers(struct ath10k *ar, 1758 struct ath10k_fw_crash_data *crash_data) 1759 { 1760 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1761 struct ath10k_ce_crash_data ce_data; 1762 u32 addr, id; 1763 1764 lockdep_assert_held(&ar->data_lock); 1765 1766 ath10k_err(ar, "Copy Engine register dump:\n"); 1767 1768 spin_lock_bh(&ce->ce_lock); 1769 for (id = 0; id < CE_COUNT; id++) { 1770 addr = ath10k_ce_base_address(ar, id); 1771 ce_data.base_addr = cpu_to_le32(addr); 1772 1773 ce_data.src_wr_idx = 1774 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1775 ce_data.src_r_idx = 1776 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1777 ce_data.dst_wr_idx = 1778 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1779 ce_data.dst_r_idx = 1780 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1781 1782 if (crash_data) 1783 crash_data->ce_crash_data[id] = ce_data; 1784 1785 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1786 le32_to_cpu(ce_data.base_addr), 1787 le32_to_cpu(ce_data.src_wr_idx), 1788 le32_to_cpu(ce_data.src_r_idx), 1789 le32_to_cpu(ce_data.dst_wr_idx), 1790 le32_to_cpu(ce_data.dst_r_idx)); 1791 } 1792 1793 spin_unlock_bh(&ce->ce_lock); 1794 } 1795 EXPORT_SYMBOL(ath10k_ce_dump_registers); 1796 1797 static const struct ath10k_ce_ops ce_ops = { 1798 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring, 1799 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring, 1800 .ce_rx_post_buf = __ath10k_ce_rx_post_buf, 1801 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock, 1802 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next, 1803 .ce_extract_desc_data = ath10k_ce_extract_desc_data, 1804 .ce_free_pipe = _ath10k_ce_free_pipe, 1805 .ce_send_nolock = _ath10k_ce_send_nolock, 1806 }; 1807 1808 static const struct ath10k_ce_ops ce_64_ops = { 1809 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64, 1810 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64, 1811 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64, 1812 .ce_completed_recv_next_nolock = 1813 _ath10k_ce_completed_recv_next_nolock_64, 1814 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64, 1815 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64, 1816 .ce_free_pipe = _ath10k_ce_free_pipe_64, 1817 .ce_send_nolock = _ath10k_ce_send_nolock_64, 1818 }; 1819 1820 static void ath10k_ce_set_ops(struct ath10k *ar, 1821 struct ath10k_ce_pipe *ce_state) 1822 { 1823 switch (ar->hw_rev) { 1824 case ATH10K_HW_WCN3990: 1825 ce_state->ops = &ce_64_ops; 1826 break; 1827 default: 1828 ce_state->ops = &ce_ops; 1829 break; 1830 } 1831 } 1832 1833 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1834 const struct ce_attr *attr) 1835 { 1836 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1837 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1838 int ret; 1839 1840 ath10k_ce_set_ops(ar, ce_state); 1841 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1842 * additional TX locking checks. 1843 * 1844 * For the lack of a better place do the check here. 1845 */ 1846 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1847 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1848 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1849 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1850 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1851 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1852 1853 ce_state->ar = ar; 1854 ce_state->id = ce_id; 1855 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1856 ce_state->attr_flags = attr->flags; 1857 ce_state->src_sz_max = attr->src_sz_max; 1858 1859 if (attr->src_nentries) 1860 ce_state->send_cb = attr->send_cb; 1861 1862 if (attr->dest_nentries) 1863 ce_state->recv_cb = attr->recv_cb; 1864 1865 if (attr->src_nentries) { 1866 ce_state->src_ring = 1867 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); 1868 if (IS_ERR(ce_state->src_ring)) { 1869 ret = PTR_ERR(ce_state->src_ring); 1870 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n", 1871 ce_id, ret); 1872 ce_state->src_ring = NULL; 1873 return ret; 1874 } 1875 } 1876 1877 if (attr->dest_nentries) { 1878 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, 1879 ce_id, 1880 attr); 1881 if (IS_ERR(ce_state->dest_ring)) { 1882 ret = PTR_ERR(ce_state->dest_ring); 1883 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n", 1884 ce_id, ret); 1885 ce_state->dest_ring = NULL; 1886 return ret; 1887 } 1888 } 1889 1890 return 0; 1891 } 1892 EXPORT_SYMBOL(ath10k_ce_alloc_pipe); 1893 1894 void ath10k_ce_alloc_rri(struct ath10k *ar) 1895 { 1896 int i; 1897 u32 value; 1898 u32 ctrl1_regs; 1899 u32 ce_base_addr; 1900 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1901 1902 ce->vaddr_rri = dma_alloc_coherent(ar->dev, 1903 (CE_COUNT * sizeof(u32)), 1904 &ce->paddr_rri, GFP_KERNEL); 1905 1906 if (!ce->vaddr_rri) 1907 return; 1908 1909 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low, 1910 lower_32_bits(ce->paddr_rri)); 1911 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high, 1912 (upper_32_bits(ce->paddr_rri) & 1913 CE_DESC_FLAGS_GET_MASK)); 1914 1915 for (i = 0; i < CE_COUNT; i++) { 1916 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr; 1917 ce_base_addr = ath10k_ce_base_address(ar, i); 1918 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs); 1919 value |= ar->hw_ce_regs->upd->mask; 1920 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value); 1921 } 1922 1923 memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32)); 1924 } 1925 EXPORT_SYMBOL(ath10k_ce_alloc_rri); 1926 1927 void ath10k_ce_free_rri(struct ath10k *ar) 1928 { 1929 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1930 1931 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)), 1932 ce->vaddr_rri, 1933 ce->paddr_rri); 1934 } 1935 EXPORT_SYMBOL(ath10k_ce_free_rri); 1936