1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. 4 * Copyright (c) 2018 The Linux Foundation. All rights reserved. 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "hif.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * receives an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar, 63 struct ath10k_ce_pipe *ce_state) 64 { 65 u32 ce_id = ce_state->id; 66 u32 addr = 0; 67 68 switch (ce_id) { 69 case 0: 70 addr = 0x00032000; 71 break; 72 case 3: 73 addr = 0x0003200C; 74 break; 75 case 4: 76 addr = 0x00032010; 77 break; 78 case 5: 79 addr = 0x00032014; 80 break; 81 case 7: 82 addr = 0x0003201C; 83 break; 84 default: 85 ath10k_warn(ar, "invalid CE id: %d", ce_id); 86 break; 87 } 88 return addr; 89 } 90 91 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar, 92 struct ath10k_ce_pipe *ce_state) 93 { 94 u32 ce_id = ce_state->id; 95 u32 addr = 0; 96 97 switch (ce_id) { 98 case 1: 99 addr = 0x00032034; 100 break; 101 case 2: 102 addr = 0x00032038; 103 break; 104 case 5: 105 addr = 0x00032044; 106 break; 107 case 7: 108 addr = 0x0003204C; 109 break; 110 case 8: 111 addr = 0x00032050; 112 break; 113 case 9: 114 addr = 0x00032054; 115 break; 116 case 10: 117 addr = 0x00032058; 118 break; 119 case 11: 120 addr = 0x0003205C; 121 break; 122 default: 123 ath10k_warn(ar, "invalid CE id: %d", ce_id); 124 break; 125 } 126 127 return addr; 128 } 129 130 static inline unsigned int 131 ath10k_set_ring_byte(unsigned int offset, 132 struct ath10k_hw_ce_regs_addr_map *addr_map) 133 { 134 return ((offset << addr_map->lsb) & addr_map->mask); 135 } 136 137 static inline unsigned int 138 ath10k_get_ring_byte(unsigned int offset, 139 struct ath10k_hw_ce_regs_addr_map *addr_map) 140 { 141 return ((offset & addr_map->mask) >> (addr_map->lsb)); 142 } 143 144 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) 145 { 146 struct ath10k_ce *ce = ath10k_ce_priv(ar); 147 148 return ce->bus_ops->read32(ar, offset); 149 } 150 151 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) 152 { 153 struct ath10k_ce *ce = ath10k_ce_priv(ar); 154 155 ce->bus_ops->write32(ar, offset, value); 156 } 157 158 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 159 u32 ce_ctrl_addr, 160 unsigned int n) 161 { 162 ath10k_ce_write32(ar, ce_ctrl_addr + 163 ar->hw_ce_regs->dst_wr_index_addr, n); 164 } 165 166 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 167 u32 ce_ctrl_addr) 168 { 169 return ath10k_ce_read32(ar, ce_ctrl_addr + 170 ar->hw_ce_regs->dst_wr_index_addr); 171 } 172 173 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 unsigned int n) 176 { 177 ath10k_ce_write32(ar, ce_ctrl_addr + 178 ar->hw_ce_regs->sr_wr_index_addr, n); 179 } 180 181 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 182 u32 ce_ctrl_addr) 183 { 184 return ath10k_ce_read32(ar, ce_ctrl_addr + 185 ar->hw_ce_regs->sr_wr_index_addr); 186 } 187 188 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar, 189 u32 ce_id) 190 { 191 struct ath10k_ce *ce = ath10k_ce_priv(ar); 192 193 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK; 194 } 195 196 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 197 u32 ce_ctrl_addr) 198 { 199 struct ath10k_ce *ce = ath10k_ce_priv(ar); 200 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 201 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 202 u32 index; 203 204 if (ar->hw_params.rri_on_ddr && 205 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 206 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id); 207 else 208 index = ath10k_ce_read32(ar, ce_ctrl_addr + 209 ar->hw_ce_regs->current_srri_addr); 210 211 return index; 212 } 213 214 static inline void 215 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar, 216 struct ath10k_ce_pipe *ce_state, 217 unsigned int value) 218 { 219 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value); 220 } 221 222 static inline void 223 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar, 224 struct ath10k_ce_pipe *ce_state, 225 unsigned int value) 226 { 227 ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value); 228 } 229 230 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 231 u32 ce_ctrl_addr, 232 unsigned int addr) 233 { 234 ath10k_ce_write32(ar, ce_ctrl_addr + 235 ar->hw_ce_regs->sr_base_addr, addr); 236 } 237 238 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 239 u32 ce_ctrl_addr, 240 unsigned int n) 241 { 242 ath10k_ce_write32(ar, ce_ctrl_addr + 243 ar->hw_ce_regs->sr_size_addr, n); 244 } 245 246 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 247 u32 ce_ctrl_addr, 248 unsigned int n) 249 { 250 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 251 252 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 253 ctrl_regs->addr); 254 255 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 256 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | 257 ath10k_set_ring_byte(n, ctrl_regs->dmax)); 258 } 259 260 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 261 u32 ce_ctrl_addr, 262 unsigned int n) 263 { 264 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 265 266 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 267 ctrl_regs->addr); 268 269 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 270 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | 271 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); 272 } 273 274 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 275 u32 ce_ctrl_addr, 276 unsigned int n) 277 { 278 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 279 280 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 281 ctrl_regs->addr); 282 283 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 284 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | 285 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); 286 } 287 288 static inline 289 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id) 290 { 291 struct ath10k_ce *ce = ath10k_ce_priv(ar); 292 293 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) & 294 CE_DDR_RRI_MASK; 295 } 296 297 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 298 u32 ce_ctrl_addr) 299 { 300 struct ath10k_ce *ce = ath10k_ce_priv(ar); 301 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 302 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 303 u32 index; 304 305 if (ar->hw_params.rri_on_ddr && 306 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 307 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id); 308 else 309 index = ath10k_ce_read32(ar, ce_ctrl_addr + 310 ar->hw_ce_regs->current_drri_addr); 311 312 return index; 313 } 314 315 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 316 u32 ce_ctrl_addr, 317 u32 addr) 318 { 319 ath10k_ce_write32(ar, ce_ctrl_addr + 320 ar->hw_ce_regs->dr_base_addr, addr); 321 } 322 323 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 324 u32 ce_ctrl_addr, 325 unsigned int n) 326 { 327 ath10k_ce_write32(ar, ce_ctrl_addr + 328 ar->hw_ce_regs->dr_size_addr, n); 329 } 330 331 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 332 u32 ce_ctrl_addr, 333 unsigned int n) 334 { 335 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 336 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 337 338 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 339 (addr & ~(srcr_wm->wm_high->mask)) | 340 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); 341 } 342 343 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 344 u32 ce_ctrl_addr, 345 unsigned int n) 346 { 347 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 348 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 349 350 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 351 (addr & ~(srcr_wm->wm_low->mask)) | 352 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); 353 } 354 355 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 356 u32 ce_ctrl_addr, 357 unsigned int n) 358 { 359 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 360 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 361 362 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 363 (addr & ~(dstr_wm->wm_high->mask)) | 364 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); 365 } 366 367 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 368 u32 ce_ctrl_addr, 369 unsigned int n) 370 { 371 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 372 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 373 374 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 375 (addr & ~(dstr_wm->wm_low->mask)) | 376 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); 377 } 378 379 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 380 u32 ce_ctrl_addr) 381 { 382 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 383 384 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 385 ar->hw_ce_regs->host_ie_addr); 386 387 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 388 host_ie_addr | host_ie->copy_complete->mask); 389 } 390 391 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 392 u32 ce_ctrl_addr) 393 { 394 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 395 396 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 397 ar->hw_ce_regs->host_ie_addr); 398 399 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 400 host_ie_addr & ~(host_ie->copy_complete->mask)); 401 } 402 403 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 404 u32 ce_ctrl_addr) 405 { 406 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 407 408 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 409 ar->hw_ce_regs->host_ie_addr); 410 411 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 412 host_ie_addr & ~(wm_regs->wm_mask)); 413 } 414 415 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 416 u32 ce_ctrl_addr) 417 { 418 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 419 420 u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 421 ar->hw_ce_regs->misc_ie_addr); 422 423 ath10k_ce_write32(ar, 424 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 425 misc_ie_addr | misc_regs->err_mask); 426 } 427 428 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 429 u32 ce_ctrl_addr) 430 { 431 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 432 433 u32 misc_ie_addr = ath10k_ce_read32(ar, 434 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); 435 436 ath10k_ce_write32(ar, 437 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 438 misc_ie_addr & ~(misc_regs->err_mask)); 439 } 440 441 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 442 u32 ce_ctrl_addr, 443 unsigned int mask) 444 { 445 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 446 447 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); 448 } 449 450 /* 451 * Guts of ath10k_ce_send. 452 * The caller takes responsibility for any needed locking. 453 */ 454 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 455 void *per_transfer_context, 456 dma_addr_t buffer, 457 unsigned int nbytes, 458 unsigned int transfer_id, 459 unsigned int flags) 460 { 461 struct ath10k *ar = ce_state->ar; 462 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 463 struct ce_desc *desc, sdesc; 464 unsigned int nentries_mask = src_ring->nentries_mask; 465 unsigned int sw_index = src_ring->sw_index; 466 unsigned int write_index = src_ring->write_index; 467 u32 ctrl_addr = ce_state->ctrl_addr; 468 u32 desc_flags = 0; 469 int ret = 0; 470 471 if (nbytes > ce_state->src_sz_max) 472 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 473 __func__, nbytes, ce_state->src_sz_max); 474 475 if (unlikely(CE_RING_DELTA(nentries_mask, 476 write_index, sw_index - 1) <= 0)) { 477 ret = -ENOSR; 478 goto exit; 479 } 480 481 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 482 write_index); 483 484 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 485 486 if (flags & CE_SEND_FLAG_GATHER) 487 desc_flags |= CE_DESC_FLAGS_GATHER; 488 if (flags & CE_SEND_FLAG_BYTE_SWAP) 489 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 490 491 sdesc.addr = __cpu_to_le32(buffer); 492 sdesc.nbytes = __cpu_to_le16(nbytes); 493 sdesc.flags = __cpu_to_le16(desc_flags); 494 495 *desc = sdesc; 496 497 src_ring->per_transfer_context[write_index] = per_transfer_context; 498 499 /* Update Source Ring Write Index */ 500 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 501 502 /* WORKAROUND */ 503 if (!(flags & CE_SEND_FLAG_GATHER)) { 504 if (ar->hw_params.shadow_reg_support) 505 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state, 506 write_index); 507 else 508 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, 509 write_index); 510 } 511 512 src_ring->write_index = write_index; 513 exit: 514 return ret; 515 } 516 517 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state, 518 void *per_transfer_context, 519 dma_addr_t buffer, 520 unsigned int nbytes, 521 unsigned int transfer_id, 522 unsigned int flags) 523 { 524 struct ath10k *ar = ce_state->ar; 525 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 526 struct ce_desc_64 *desc, sdesc; 527 unsigned int nentries_mask = src_ring->nentries_mask; 528 unsigned int sw_index; 529 unsigned int write_index = src_ring->write_index; 530 u32 ctrl_addr = ce_state->ctrl_addr; 531 __le32 *addr; 532 u32 desc_flags = 0; 533 int ret = 0; 534 535 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) 536 return -ESHUTDOWN; 537 538 if (nbytes > ce_state->src_sz_max) 539 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 540 __func__, nbytes, ce_state->src_sz_max); 541 542 if (ar->hw_params.rri_on_ddr) 543 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id); 544 else 545 sw_index = src_ring->sw_index; 546 547 if (unlikely(CE_RING_DELTA(nentries_mask, 548 write_index, sw_index - 1) <= 0)) { 549 ret = -ENOSR; 550 goto exit; 551 } 552 553 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 554 write_index); 555 556 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 557 558 if (flags & CE_SEND_FLAG_GATHER) 559 desc_flags |= CE_DESC_FLAGS_GATHER; 560 561 if (flags & CE_SEND_FLAG_BYTE_SWAP) 562 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 563 564 addr = (__le32 *)&sdesc.addr; 565 566 flags |= upper_32_bits(buffer) & CE_DESC_FLAGS_GET_MASK; 567 addr[0] = __cpu_to_le32(buffer); 568 addr[1] = __cpu_to_le32(flags); 569 if (flags & CE_SEND_FLAG_GATHER) 570 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER); 571 else 572 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER)); 573 574 sdesc.nbytes = __cpu_to_le16(nbytes); 575 sdesc.flags = __cpu_to_le16(desc_flags); 576 577 *desc = sdesc; 578 579 src_ring->per_transfer_context[write_index] = per_transfer_context; 580 581 /* Update Source Ring Write Index */ 582 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 583 584 if (!(flags & CE_SEND_FLAG_GATHER)) 585 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 586 587 src_ring->write_index = write_index; 588 exit: 589 return ret; 590 } 591 592 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 593 void *per_transfer_context, 594 dma_addr_t buffer, 595 unsigned int nbytes, 596 unsigned int transfer_id, 597 unsigned int flags) 598 { 599 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, 600 buffer, nbytes, transfer_id, flags); 601 } 602 EXPORT_SYMBOL(ath10k_ce_send_nolock); 603 604 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 605 { 606 struct ath10k *ar = pipe->ar; 607 struct ath10k_ce *ce = ath10k_ce_priv(ar); 608 struct ath10k_ce_ring *src_ring = pipe->src_ring; 609 u32 ctrl_addr = pipe->ctrl_addr; 610 611 lockdep_assert_held(&ce->ce_lock); 612 613 /* 614 * This function must be called only if there is an incomplete 615 * scatter-gather transfer (before index register is updated) 616 * that needs to be cleaned up. 617 */ 618 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 619 return; 620 621 if (WARN_ON_ONCE(src_ring->write_index == 622 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 623 return; 624 625 src_ring->write_index--; 626 src_ring->write_index &= src_ring->nentries_mask; 627 628 src_ring->per_transfer_context[src_ring->write_index] = NULL; 629 } 630 EXPORT_SYMBOL(__ath10k_ce_send_revert); 631 632 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 633 void *per_transfer_context, 634 dma_addr_t buffer, 635 unsigned int nbytes, 636 unsigned int transfer_id, 637 unsigned int flags) 638 { 639 struct ath10k *ar = ce_state->ar; 640 struct ath10k_ce *ce = ath10k_ce_priv(ar); 641 int ret; 642 643 spin_lock_bh(&ce->ce_lock); 644 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 645 buffer, nbytes, transfer_id, flags); 646 spin_unlock_bh(&ce->ce_lock); 647 648 return ret; 649 } 650 EXPORT_SYMBOL(ath10k_ce_send); 651 652 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 653 { 654 struct ath10k *ar = pipe->ar; 655 struct ath10k_ce *ce = ath10k_ce_priv(ar); 656 int delta; 657 658 spin_lock_bh(&ce->ce_lock); 659 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 660 pipe->src_ring->write_index, 661 pipe->src_ring->sw_index - 1); 662 spin_unlock_bh(&ce->ce_lock); 663 664 return delta; 665 } 666 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries); 667 668 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 669 { 670 struct ath10k *ar = pipe->ar; 671 struct ath10k_ce *ce = ath10k_ce_priv(ar); 672 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 673 unsigned int nentries_mask = dest_ring->nentries_mask; 674 unsigned int write_index = dest_ring->write_index; 675 unsigned int sw_index = dest_ring->sw_index; 676 677 lockdep_assert_held(&ce->ce_lock); 678 679 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 680 } 681 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs); 682 683 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 684 dma_addr_t paddr) 685 { 686 struct ath10k *ar = pipe->ar; 687 struct ath10k_ce *ce = ath10k_ce_priv(ar); 688 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 689 unsigned int nentries_mask = dest_ring->nentries_mask; 690 unsigned int write_index = dest_ring->write_index; 691 unsigned int sw_index = dest_ring->sw_index; 692 struct ce_desc *base = dest_ring->base_addr_owner_space; 693 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 694 u32 ctrl_addr = pipe->ctrl_addr; 695 696 lockdep_assert_held(&ce->ce_lock); 697 698 if ((pipe->id != 5) && 699 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 700 return -ENOSPC; 701 702 desc->addr = __cpu_to_le32(paddr); 703 desc->nbytes = 0; 704 705 dest_ring->per_transfer_context[write_index] = ctx; 706 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 707 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 708 dest_ring->write_index = write_index; 709 710 return 0; 711 } 712 713 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe, 714 void *ctx, 715 dma_addr_t paddr) 716 { 717 struct ath10k *ar = pipe->ar; 718 struct ath10k_ce *ce = ath10k_ce_priv(ar); 719 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 720 unsigned int nentries_mask = dest_ring->nentries_mask; 721 unsigned int write_index = dest_ring->write_index; 722 unsigned int sw_index = dest_ring->sw_index; 723 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 724 struct ce_desc_64 *desc = 725 CE_DEST_RING_TO_DESC_64(base, write_index); 726 u32 ctrl_addr = pipe->ctrl_addr; 727 728 lockdep_assert_held(&ce->ce_lock); 729 730 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 731 return -ENOSPC; 732 733 desc->addr = __cpu_to_le64(paddr); 734 desc->addr &= __cpu_to_le64(CE_DESC_37BIT_ADDR_MASK); 735 736 desc->nbytes = 0; 737 738 dest_ring->per_transfer_context[write_index] = ctx; 739 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 740 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 741 dest_ring->write_index = write_index; 742 743 return 0; 744 } 745 746 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 747 { 748 struct ath10k *ar = pipe->ar; 749 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 750 unsigned int nentries_mask = dest_ring->nentries_mask; 751 unsigned int write_index = dest_ring->write_index; 752 u32 ctrl_addr = pipe->ctrl_addr; 753 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 754 755 /* Prevent CE ring stuck issue that will occur when ring is full. 756 * Make sure that write index is 1 less than read index. 757 */ 758 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index) 759 nentries -= 1; 760 761 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 762 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 763 dest_ring->write_index = write_index; 764 } 765 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx); 766 767 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 768 dma_addr_t paddr) 769 { 770 struct ath10k *ar = pipe->ar; 771 struct ath10k_ce *ce = ath10k_ce_priv(ar); 772 int ret; 773 774 spin_lock_bh(&ce->ce_lock); 775 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); 776 spin_unlock_bh(&ce->ce_lock); 777 778 return ret; 779 } 780 EXPORT_SYMBOL(ath10k_ce_rx_post_buf); 781 782 /* 783 * Guts of ath10k_ce_completed_recv_next. 784 * The caller takes responsibility for any necessary locking. 785 */ 786 static int 787 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 788 void **per_transfer_contextp, 789 unsigned int *nbytesp) 790 { 791 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 792 unsigned int nentries_mask = dest_ring->nentries_mask; 793 unsigned int sw_index = dest_ring->sw_index; 794 795 struct ce_desc *base = dest_ring->base_addr_owner_space; 796 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 797 struct ce_desc sdesc; 798 u16 nbytes; 799 800 /* Copy in one go for performance reasons */ 801 sdesc = *desc; 802 803 nbytes = __le16_to_cpu(sdesc.nbytes); 804 if (nbytes == 0) { 805 /* 806 * This closes a relatively unusual race where the Host 807 * sees the updated DRRI before the update to the 808 * corresponding descriptor has completed. We treat this 809 * as a descriptor that is not yet done. 810 */ 811 return -EIO; 812 } 813 814 desc->nbytes = 0; 815 816 /* Return data from completed destination descriptor */ 817 *nbytesp = nbytes; 818 819 if (per_transfer_contextp) 820 *per_transfer_contextp = 821 dest_ring->per_transfer_context[sw_index]; 822 823 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 824 * So update transfer context all CEs except CE5. 825 */ 826 if (ce_state->id != 5) 827 dest_ring->per_transfer_context[sw_index] = NULL; 828 829 /* Update sw_index */ 830 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 831 dest_ring->sw_index = sw_index; 832 833 return 0; 834 } 835 836 static int 837 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state, 838 void **per_transfer_contextp, 839 unsigned int *nbytesp) 840 { 841 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 842 unsigned int nentries_mask = dest_ring->nentries_mask; 843 unsigned int sw_index = dest_ring->sw_index; 844 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 845 struct ce_desc_64 *desc = 846 CE_DEST_RING_TO_DESC_64(base, sw_index); 847 struct ce_desc_64 sdesc; 848 u16 nbytes; 849 850 /* Copy in one go for performance reasons */ 851 sdesc = *desc; 852 853 nbytes = __le16_to_cpu(sdesc.nbytes); 854 if (nbytes == 0) { 855 /* This closes a relatively unusual race where the Host 856 * sees the updated DRRI before the update to the 857 * corresponding descriptor has completed. We treat this 858 * as a descriptor that is not yet done. 859 */ 860 return -EIO; 861 } 862 863 desc->nbytes = 0; 864 865 /* Return data from completed destination descriptor */ 866 *nbytesp = nbytes; 867 868 if (per_transfer_contextp) 869 *per_transfer_contextp = 870 dest_ring->per_transfer_context[sw_index]; 871 872 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 873 * So update transfer context all CEs except CE5. 874 */ 875 if (ce_state->id != 5) 876 dest_ring->per_transfer_context[sw_index] = NULL; 877 878 /* Update sw_index */ 879 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 880 dest_ring->sw_index = sw_index; 881 882 return 0; 883 } 884 885 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 886 void **per_transfer_ctx, 887 unsigned int *nbytesp) 888 { 889 return ce_state->ops->ce_completed_recv_next_nolock(ce_state, 890 per_transfer_ctx, 891 nbytesp); 892 } 893 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock); 894 895 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 896 void **per_transfer_contextp, 897 unsigned int *nbytesp) 898 { 899 struct ath10k *ar = ce_state->ar; 900 struct ath10k_ce *ce = ath10k_ce_priv(ar); 901 int ret; 902 903 spin_lock_bh(&ce->ce_lock); 904 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, 905 per_transfer_contextp, 906 nbytesp); 907 908 spin_unlock_bh(&ce->ce_lock); 909 910 return ret; 911 } 912 EXPORT_SYMBOL(ath10k_ce_completed_recv_next); 913 914 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 915 void **per_transfer_contextp, 916 dma_addr_t *bufferp) 917 { 918 struct ath10k_ce_ring *dest_ring; 919 unsigned int nentries_mask; 920 unsigned int sw_index; 921 unsigned int write_index; 922 int ret; 923 struct ath10k *ar; 924 struct ath10k_ce *ce; 925 926 dest_ring = ce_state->dest_ring; 927 928 if (!dest_ring) 929 return -EIO; 930 931 ar = ce_state->ar; 932 ce = ath10k_ce_priv(ar); 933 934 spin_lock_bh(&ce->ce_lock); 935 936 nentries_mask = dest_ring->nentries_mask; 937 sw_index = dest_ring->sw_index; 938 write_index = dest_ring->write_index; 939 if (write_index != sw_index) { 940 struct ce_desc *base = dest_ring->base_addr_owner_space; 941 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 942 943 /* Return data from completed destination descriptor */ 944 *bufferp = __le32_to_cpu(desc->addr); 945 946 if (per_transfer_contextp) 947 *per_transfer_contextp = 948 dest_ring->per_transfer_context[sw_index]; 949 950 /* sanity */ 951 dest_ring->per_transfer_context[sw_index] = NULL; 952 desc->nbytes = 0; 953 954 /* Update sw_index */ 955 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 956 dest_ring->sw_index = sw_index; 957 ret = 0; 958 } else { 959 ret = -EIO; 960 } 961 962 spin_unlock_bh(&ce->ce_lock); 963 964 return ret; 965 } 966 967 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state, 968 void **per_transfer_contextp, 969 dma_addr_t *bufferp) 970 { 971 struct ath10k_ce_ring *dest_ring; 972 unsigned int nentries_mask; 973 unsigned int sw_index; 974 unsigned int write_index; 975 int ret; 976 struct ath10k *ar; 977 struct ath10k_ce *ce; 978 979 dest_ring = ce_state->dest_ring; 980 981 if (!dest_ring) 982 return -EIO; 983 984 ar = ce_state->ar; 985 ce = ath10k_ce_priv(ar); 986 987 spin_lock_bh(&ce->ce_lock); 988 989 nentries_mask = dest_ring->nentries_mask; 990 sw_index = dest_ring->sw_index; 991 write_index = dest_ring->write_index; 992 if (write_index != sw_index) { 993 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 994 struct ce_desc_64 *desc = 995 CE_DEST_RING_TO_DESC_64(base, sw_index); 996 997 /* Return data from completed destination descriptor */ 998 *bufferp = __le64_to_cpu(desc->addr); 999 1000 if (per_transfer_contextp) 1001 *per_transfer_contextp = 1002 dest_ring->per_transfer_context[sw_index]; 1003 1004 /* sanity */ 1005 dest_ring->per_transfer_context[sw_index] = NULL; 1006 desc->nbytes = 0; 1007 1008 /* Update sw_index */ 1009 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1010 dest_ring->sw_index = sw_index; 1011 ret = 0; 1012 } else { 1013 ret = -EIO; 1014 } 1015 1016 spin_unlock_bh(&ce->ce_lock); 1017 1018 return ret; 1019 } 1020 1021 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 1022 void **per_transfer_contextp, 1023 dma_addr_t *bufferp) 1024 { 1025 return ce_state->ops->ce_revoke_recv_next(ce_state, 1026 per_transfer_contextp, 1027 bufferp); 1028 } 1029 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next); 1030 1031 /* 1032 * Guts of ath10k_ce_completed_send_next. 1033 * The caller takes responsibility for any necessary locking. 1034 */ 1035 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 1036 void **per_transfer_contextp) 1037 { 1038 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1039 u32 ctrl_addr = ce_state->ctrl_addr; 1040 struct ath10k *ar = ce_state->ar; 1041 unsigned int nentries_mask = src_ring->nentries_mask; 1042 unsigned int sw_index = src_ring->sw_index; 1043 unsigned int read_index; 1044 struct ce_desc *desc; 1045 1046 if (src_ring->hw_index == sw_index) { 1047 /* 1048 * The SW completion index has caught up with the cached 1049 * version of the HW completion index. 1050 * Update the cached HW completion index to see whether 1051 * the SW has really caught up to the HW, or if the cached 1052 * value of the HW index has become stale. 1053 */ 1054 1055 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1056 if (read_index == 0xffffffff) 1057 return -ENODEV; 1058 1059 read_index &= nentries_mask; 1060 src_ring->hw_index = read_index; 1061 } 1062 1063 if (ar->hw_params.rri_on_ddr) 1064 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1065 else 1066 read_index = src_ring->hw_index; 1067 1068 if (read_index == sw_index) 1069 return -EIO; 1070 1071 if (per_transfer_contextp) 1072 *per_transfer_contextp = 1073 src_ring->per_transfer_context[sw_index]; 1074 1075 /* sanity */ 1076 src_ring->per_transfer_context[sw_index] = NULL; 1077 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 1078 sw_index); 1079 desc->nbytes = 0; 1080 1081 /* Update sw_index */ 1082 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1083 src_ring->sw_index = sw_index; 1084 1085 return 0; 1086 } 1087 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock); 1088 1089 static void ath10k_ce_extract_desc_data(struct ath10k *ar, 1090 struct ath10k_ce_ring *src_ring, 1091 u32 sw_index, 1092 dma_addr_t *bufferp, 1093 u32 *nbytesp, 1094 u32 *transfer_idp) 1095 { 1096 struct ce_desc *base = src_ring->base_addr_owner_space; 1097 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 1098 1099 /* Return data from completed source descriptor */ 1100 *bufferp = __le32_to_cpu(desc->addr); 1101 *nbytesp = __le16_to_cpu(desc->nbytes); 1102 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1103 CE_DESC_FLAGS_META_DATA); 1104 } 1105 1106 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar, 1107 struct ath10k_ce_ring *src_ring, 1108 u32 sw_index, 1109 dma_addr_t *bufferp, 1110 u32 *nbytesp, 1111 u32 *transfer_idp) 1112 { 1113 struct ce_desc_64 *base = src_ring->base_addr_owner_space; 1114 struct ce_desc_64 *desc = 1115 CE_SRC_RING_TO_DESC_64(base, sw_index); 1116 1117 /* Return data from completed source descriptor */ 1118 *bufferp = __le64_to_cpu(desc->addr); 1119 *nbytesp = __le16_to_cpu(desc->nbytes); 1120 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1121 CE_DESC_FLAGS_META_DATA); 1122 } 1123 1124 /* NB: Modeled after ath10k_ce_completed_send_next */ 1125 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 1126 void **per_transfer_contextp, 1127 dma_addr_t *bufferp, 1128 unsigned int *nbytesp, 1129 unsigned int *transfer_idp) 1130 { 1131 struct ath10k_ce_ring *src_ring; 1132 unsigned int nentries_mask; 1133 unsigned int sw_index; 1134 unsigned int write_index; 1135 int ret; 1136 struct ath10k *ar; 1137 struct ath10k_ce *ce; 1138 1139 src_ring = ce_state->src_ring; 1140 1141 if (!src_ring) 1142 return -EIO; 1143 1144 ar = ce_state->ar; 1145 ce = ath10k_ce_priv(ar); 1146 1147 spin_lock_bh(&ce->ce_lock); 1148 1149 nentries_mask = src_ring->nentries_mask; 1150 sw_index = src_ring->sw_index; 1151 write_index = src_ring->write_index; 1152 1153 if (write_index != sw_index) { 1154 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, 1155 bufferp, nbytesp, 1156 transfer_idp); 1157 1158 if (per_transfer_contextp) 1159 *per_transfer_contextp = 1160 src_ring->per_transfer_context[sw_index]; 1161 1162 /* sanity */ 1163 src_ring->per_transfer_context[sw_index] = NULL; 1164 1165 /* Update sw_index */ 1166 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1167 src_ring->sw_index = sw_index; 1168 ret = 0; 1169 } else { 1170 ret = -EIO; 1171 } 1172 1173 spin_unlock_bh(&ce->ce_lock); 1174 1175 return ret; 1176 } 1177 EXPORT_SYMBOL(ath10k_ce_cancel_send_next); 1178 1179 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 1180 void **per_transfer_contextp) 1181 { 1182 struct ath10k *ar = ce_state->ar; 1183 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1184 int ret; 1185 1186 spin_lock_bh(&ce->ce_lock); 1187 ret = ath10k_ce_completed_send_next_nolock(ce_state, 1188 per_transfer_contextp); 1189 spin_unlock_bh(&ce->ce_lock); 1190 1191 return ret; 1192 } 1193 EXPORT_SYMBOL(ath10k_ce_completed_send_next); 1194 1195 /* 1196 * Guts of interrupt handler for per-engine interrupts on a particular CE. 1197 * 1198 * Invokes registered callbacks for recv_complete, 1199 * send_complete, and watermarks. 1200 */ 1201 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 1202 { 1203 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1204 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1205 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 1206 u32 ctrl_addr = ce_state->ctrl_addr; 1207 1208 spin_lock_bh(&ce->ce_lock); 1209 1210 /* Clear the copy-complete interrupts that will be handled here. */ 1211 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 1212 wm_regs->cc_mask); 1213 1214 spin_unlock_bh(&ce->ce_lock); 1215 1216 if (ce_state->recv_cb) 1217 ce_state->recv_cb(ce_state); 1218 1219 if (ce_state->send_cb) 1220 ce_state->send_cb(ce_state); 1221 1222 spin_lock_bh(&ce->ce_lock); 1223 1224 /* 1225 * Misc CE interrupts are not being handled, but still need 1226 * to be cleared. 1227 */ 1228 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask); 1229 1230 spin_unlock_bh(&ce->ce_lock); 1231 } 1232 EXPORT_SYMBOL(ath10k_ce_per_engine_service); 1233 1234 /* 1235 * Handler for per-engine interrupts on ALL active CEs. 1236 * This is used in cases where the system is sharing a 1237 * single interrput for all CEs 1238 */ 1239 1240 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 1241 { 1242 int ce_id; 1243 u32 intr_summary; 1244 1245 intr_summary = ath10k_ce_interrupt_summary(ar); 1246 1247 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 1248 if (intr_summary & (1 << ce_id)) 1249 intr_summary &= ~(1 << ce_id); 1250 else 1251 /* no intr pending on this CE */ 1252 continue; 1253 1254 ath10k_ce_per_engine_service(ar, ce_id); 1255 } 1256 } 1257 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any); 1258 1259 /* 1260 * Adjust interrupts for the copy complete handler. 1261 * If it's needed for either send or recv, then unmask 1262 * this interrupt; otherwise, mask it. 1263 * 1264 * Called with ce_lock held. 1265 */ 1266 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 1267 { 1268 u32 ctrl_addr = ce_state->ctrl_addr; 1269 struct ath10k *ar = ce_state->ar; 1270 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 1271 1272 if ((!disable_copy_compl_intr) && 1273 (ce_state->send_cb || ce_state->recv_cb)) 1274 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 1275 else 1276 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1277 1278 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1279 } 1280 1281 int ath10k_ce_disable_interrupts(struct ath10k *ar) 1282 { 1283 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1284 struct ath10k_ce_pipe *ce_state; 1285 u32 ctrl_addr; 1286 int ce_id; 1287 1288 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 1289 ce_state = &ce->ce_states[ce_id]; 1290 if (ce_state->attr_flags & CE_ATTR_POLL) 1291 continue; 1292 1293 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1294 1295 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1296 ath10k_ce_error_intr_disable(ar, ctrl_addr); 1297 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1298 } 1299 1300 return 0; 1301 } 1302 EXPORT_SYMBOL(ath10k_ce_disable_interrupts); 1303 1304 void ath10k_ce_enable_interrupts(struct ath10k *ar) 1305 { 1306 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1307 int ce_id; 1308 struct ath10k_ce_pipe *ce_state; 1309 1310 /* Enable interrupts for copy engine that 1311 * are not using polling mode. 1312 */ 1313 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 1314 ce_state = &ce->ce_states[ce_id]; 1315 if (ce_state->attr_flags & CE_ATTR_POLL) 1316 continue; 1317 1318 ath10k_ce_per_engine_handler_adjust(ce_state); 1319 } 1320 } 1321 EXPORT_SYMBOL(ath10k_ce_enable_interrupts); 1322 1323 static int ath10k_ce_init_src_ring(struct ath10k *ar, 1324 unsigned int ce_id, 1325 const struct ce_attr *attr) 1326 { 1327 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1328 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1329 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1330 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1331 1332 nentries = roundup_pow_of_two(attr->src_nentries); 1333 1334 if (ar->hw_params.target_64bit) 1335 memset(src_ring->base_addr_owner_space, 0, 1336 nentries * sizeof(struct ce_desc_64)); 1337 else 1338 memset(src_ring->base_addr_owner_space, 0, 1339 nentries * sizeof(struct ce_desc)); 1340 1341 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1342 src_ring->sw_index &= src_ring->nentries_mask; 1343 src_ring->hw_index = src_ring->sw_index; 1344 1345 src_ring->write_index = 1346 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 1347 src_ring->write_index &= src_ring->nentries_mask; 1348 1349 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 1350 src_ring->base_addr_ce_space); 1351 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 1352 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 1353 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 1354 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 1355 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 1356 1357 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1358 "boot init ce src ring id %d entries %d base_addr %pK\n", 1359 ce_id, nentries, src_ring->base_addr_owner_space); 1360 1361 return 0; 1362 } 1363 1364 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1365 unsigned int ce_id, 1366 const struct ce_attr *attr) 1367 { 1368 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1369 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1370 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 1371 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1372 1373 nentries = roundup_pow_of_two(attr->dest_nentries); 1374 1375 if (ar->hw_params.target_64bit) 1376 memset(dest_ring->base_addr_owner_space, 0, 1377 nentries * sizeof(struct ce_desc_64)); 1378 else 1379 memset(dest_ring->base_addr_owner_space, 0, 1380 nentries * sizeof(struct ce_desc)); 1381 1382 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1383 dest_ring->sw_index &= dest_ring->nentries_mask; 1384 dest_ring->write_index = 1385 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1386 dest_ring->write_index &= dest_ring->nentries_mask; 1387 1388 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 1389 dest_ring->base_addr_ce_space); 1390 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1391 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1392 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1393 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1394 1395 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1396 "boot ce dest ring id %d entries %d base_addr %pK\n", 1397 ce_id, nentries, dest_ring->base_addr_owner_space); 1398 1399 return 0; 1400 } 1401 1402 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar, 1403 struct ath10k_ce_ring *src_ring, 1404 u32 nentries) 1405 { 1406 src_ring->shadow_base_unaligned = kcalloc(nentries, 1407 sizeof(struct ce_desc), 1408 GFP_KERNEL); 1409 if (!src_ring->shadow_base_unaligned) 1410 return -ENOMEM; 1411 1412 src_ring->shadow_base = (struct ce_desc *) 1413 PTR_ALIGN(src_ring->shadow_base_unaligned, 1414 CE_DESC_RING_ALIGN); 1415 return 0; 1416 } 1417 1418 static struct ath10k_ce_ring * 1419 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 1420 const struct ce_attr *attr) 1421 { 1422 struct ath10k_ce_ring *src_ring; 1423 u32 nentries = attr->src_nentries; 1424 dma_addr_t base_addr; 1425 int ret; 1426 1427 nentries = roundup_pow_of_two(nentries); 1428 1429 src_ring = kzalloc(struct_size(src_ring, per_transfer_context, 1430 nentries), GFP_KERNEL); 1431 if (src_ring == NULL) 1432 return ERR_PTR(-ENOMEM); 1433 1434 src_ring->nentries = nentries; 1435 src_ring->nentries_mask = nentries - 1; 1436 1437 /* 1438 * Legacy platforms that do not support cache 1439 * coherent DMA are unsupported 1440 */ 1441 src_ring->base_addr_owner_space_unaligned = 1442 dma_alloc_coherent(ar->dev, 1443 (nentries * sizeof(struct ce_desc) + 1444 CE_DESC_RING_ALIGN), 1445 &base_addr, GFP_KERNEL); 1446 if (!src_ring->base_addr_owner_space_unaligned) { 1447 kfree(src_ring); 1448 return ERR_PTR(-ENOMEM); 1449 } 1450 1451 src_ring->base_addr_ce_space_unaligned = base_addr; 1452 1453 src_ring->base_addr_owner_space = 1454 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1455 CE_DESC_RING_ALIGN); 1456 src_ring->base_addr_ce_space = 1457 ALIGN(src_ring->base_addr_ce_space_unaligned, 1458 CE_DESC_RING_ALIGN); 1459 1460 if (ar->hw_params.shadow_reg_support) { 1461 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1462 if (ret) { 1463 dma_free_coherent(ar->dev, 1464 (nentries * sizeof(struct ce_desc) + 1465 CE_DESC_RING_ALIGN), 1466 src_ring->base_addr_owner_space_unaligned, 1467 base_addr); 1468 kfree(src_ring); 1469 return ERR_PTR(ret); 1470 } 1471 } 1472 1473 return src_ring; 1474 } 1475 1476 static struct ath10k_ce_ring * 1477 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, 1478 const struct ce_attr *attr) 1479 { 1480 struct ath10k_ce_ring *src_ring; 1481 u32 nentries = attr->src_nentries; 1482 dma_addr_t base_addr; 1483 int ret; 1484 1485 nentries = roundup_pow_of_two(nentries); 1486 1487 src_ring = kzalloc(struct_size(src_ring, per_transfer_context, 1488 nentries), GFP_KERNEL); 1489 if (!src_ring) 1490 return ERR_PTR(-ENOMEM); 1491 1492 src_ring->nentries = nentries; 1493 src_ring->nentries_mask = nentries - 1; 1494 1495 /* Legacy platforms that do not support cache 1496 * coherent DMA are unsupported 1497 */ 1498 src_ring->base_addr_owner_space_unaligned = 1499 dma_alloc_coherent(ar->dev, 1500 (nentries * sizeof(struct ce_desc_64) + 1501 CE_DESC_RING_ALIGN), 1502 &base_addr, GFP_KERNEL); 1503 if (!src_ring->base_addr_owner_space_unaligned) { 1504 kfree(src_ring); 1505 return ERR_PTR(-ENOMEM); 1506 } 1507 1508 src_ring->base_addr_ce_space_unaligned = base_addr; 1509 1510 src_ring->base_addr_owner_space = 1511 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1512 CE_DESC_RING_ALIGN); 1513 src_ring->base_addr_ce_space = 1514 ALIGN(src_ring->base_addr_ce_space_unaligned, 1515 CE_DESC_RING_ALIGN); 1516 1517 if (ar->hw_params.shadow_reg_support) { 1518 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1519 if (ret) { 1520 dma_free_coherent(ar->dev, 1521 (nentries * sizeof(struct ce_desc_64) + 1522 CE_DESC_RING_ALIGN), 1523 src_ring->base_addr_owner_space_unaligned, 1524 base_addr); 1525 kfree(src_ring); 1526 return ERR_PTR(ret); 1527 } 1528 } 1529 1530 return src_ring; 1531 } 1532 1533 static struct ath10k_ce_ring * 1534 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 1535 const struct ce_attr *attr) 1536 { 1537 struct ath10k_ce_ring *dest_ring; 1538 u32 nentries; 1539 dma_addr_t base_addr; 1540 1541 nentries = roundup_pow_of_two(attr->dest_nentries); 1542 1543 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, 1544 nentries), GFP_KERNEL); 1545 if (dest_ring == NULL) 1546 return ERR_PTR(-ENOMEM); 1547 1548 dest_ring->nentries = nentries; 1549 dest_ring->nentries_mask = nentries - 1; 1550 1551 /* 1552 * Legacy platforms that do not support cache 1553 * coherent DMA are unsupported 1554 */ 1555 dest_ring->base_addr_owner_space_unaligned = 1556 dma_alloc_coherent(ar->dev, 1557 (nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN), 1558 &base_addr, GFP_KERNEL); 1559 if (!dest_ring->base_addr_owner_space_unaligned) { 1560 kfree(dest_ring); 1561 return ERR_PTR(-ENOMEM); 1562 } 1563 1564 dest_ring->base_addr_ce_space_unaligned = base_addr; 1565 1566 dest_ring->base_addr_owner_space = 1567 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1568 CE_DESC_RING_ALIGN); 1569 dest_ring->base_addr_ce_space = 1570 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1571 CE_DESC_RING_ALIGN); 1572 1573 return dest_ring; 1574 } 1575 1576 static struct ath10k_ce_ring * 1577 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, 1578 const struct ce_attr *attr) 1579 { 1580 struct ath10k_ce_ring *dest_ring; 1581 u32 nentries; 1582 dma_addr_t base_addr; 1583 1584 nentries = roundup_pow_of_two(attr->dest_nentries); 1585 1586 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, 1587 nentries), GFP_KERNEL); 1588 if (!dest_ring) 1589 return ERR_PTR(-ENOMEM); 1590 1591 dest_ring->nentries = nentries; 1592 dest_ring->nentries_mask = nentries - 1; 1593 1594 /* Legacy platforms that do not support cache 1595 * coherent DMA are unsupported 1596 */ 1597 dest_ring->base_addr_owner_space_unaligned = 1598 dma_alloc_coherent(ar->dev, 1599 (nentries * sizeof(struct ce_desc_64) + 1600 CE_DESC_RING_ALIGN), 1601 &base_addr, GFP_KERNEL); 1602 if (!dest_ring->base_addr_owner_space_unaligned) { 1603 kfree(dest_ring); 1604 return ERR_PTR(-ENOMEM); 1605 } 1606 1607 dest_ring->base_addr_ce_space_unaligned = base_addr; 1608 1609 /* Correctly initialize memory to 0 to prevent garbage 1610 * data crashing system when download firmware 1611 */ 1612 memset(dest_ring->base_addr_owner_space_unaligned, 0, 1613 nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN); 1614 1615 dest_ring->base_addr_owner_space = 1616 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1617 CE_DESC_RING_ALIGN); 1618 dest_ring->base_addr_ce_space = 1619 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1620 CE_DESC_RING_ALIGN); 1621 1622 return dest_ring; 1623 } 1624 1625 /* 1626 * Initialize a Copy Engine based on caller-supplied attributes. 1627 * This may be called once to initialize both source and destination 1628 * rings or it may be called twice for separate source and destination 1629 * initialization. It may be that only one side or the other is 1630 * initialized by software/firmware. 1631 */ 1632 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 1633 const struct ce_attr *attr) 1634 { 1635 int ret; 1636 1637 if (attr->src_nentries) { 1638 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1639 if (ret) { 1640 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1641 ce_id, ret); 1642 return ret; 1643 } 1644 } 1645 1646 if (attr->dest_nentries) { 1647 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1648 if (ret) { 1649 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1650 ce_id, ret); 1651 return ret; 1652 } 1653 } 1654 1655 return 0; 1656 } 1657 EXPORT_SYMBOL(ath10k_ce_init_pipe); 1658 1659 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1660 { 1661 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1662 1663 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0); 1664 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1665 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1666 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1667 } 1668 1669 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1670 { 1671 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1672 1673 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0); 1674 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1675 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1676 } 1677 1678 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1679 { 1680 ath10k_ce_deinit_src_ring(ar, ce_id); 1681 ath10k_ce_deinit_dest_ring(ar, ce_id); 1682 } 1683 EXPORT_SYMBOL(ath10k_ce_deinit_pipe); 1684 1685 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1686 { 1687 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1688 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1689 1690 if (ce_state->src_ring) { 1691 if (ar->hw_params.shadow_reg_support) 1692 kfree(ce_state->src_ring->shadow_base_unaligned); 1693 dma_free_coherent(ar->dev, 1694 (ce_state->src_ring->nentries * 1695 sizeof(struct ce_desc) + 1696 CE_DESC_RING_ALIGN), 1697 ce_state->src_ring->base_addr_owner_space, 1698 ce_state->src_ring->base_addr_ce_space); 1699 kfree(ce_state->src_ring); 1700 } 1701 1702 if (ce_state->dest_ring) { 1703 dma_free_coherent(ar->dev, 1704 (ce_state->dest_ring->nentries * 1705 sizeof(struct ce_desc) + 1706 CE_DESC_RING_ALIGN), 1707 ce_state->dest_ring->base_addr_owner_space, 1708 ce_state->dest_ring->base_addr_ce_space); 1709 kfree(ce_state->dest_ring); 1710 } 1711 1712 ce_state->src_ring = NULL; 1713 ce_state->dest_ring = NULL; 1714 } 1715 1716 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) 1717 { 1718 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1719 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1720 1721 if (ce_state->src_ring) { 1722 if (ar->hw_params.shadow_reg_support) 1723 kfree(ce_state->src_ring->shadow_base_unaligned); 1724 dma_free_coherent(ar->dev, 1725 (ce_state->src_ring->nentries * 1726 sizeof(struct ce_desc_64) + 1727 CE_DESC_RING_ALIGN), 1728 ce_state->src_ring->base_addr_owner_space, 1729 ce_state->src_ring->base_addr_ce_space); 1730 kfree(ce_state->src_ring); 1731 } 1732 1733 if (ce_state->dest_ring) { 1734 dma_free_coherent(ar->dev, 1735 (ce_state->dest_ring->nentries * 1736 sizeof(struct ce_desc_64) + 1737 CE_DESC_RING_ALIGN), 1738 ce_state->dest_ring->base_addr_owner_space, 1739 ce_state->dest_ring->base_addr_ce_space); 1740 kfree(ce_state->dest_ring); 1741 } 1742 1743 ce_state->src_ring = NULL; 1744 ce_state->dest_ring = NULL; 1745 } 1746 1747 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1748 { 1749 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1750 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1751 1752 ce_state->ops->ce_free_pipe(ar, ce_id); 1753 } 1754 EXPORT_SYMBOL(ath10k_ce_free_pipe); 1755 1756 void ath10k_ce_dump_registers(struct ath10k *ar, 1757 struct ath10k_fw_crash_data *crash_data) 1758 { 1759 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1760 struct ath10k_ce_crash_data ce_data; 1761 u32 addr, id; 1762 1763 lockdep_assert_held(&ar->data_lock); 1764 1765 ath10k_err(ar, "Copy Engine register dump:\n"); 1766 1767 spin_lock_bh(&ce->ce_lock); 1768 for (id = 0; id < CE_COUNT; id++) { 1769 addr = ath10k_ce_base_address(ar, id); 1770 ce_data.base_addr = cpu_to_le32(addr); 1771 1772 ce_data.src_wr_idx = 1773 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1774 ce_data.src_r_idx = 1775 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1776 ce_data.dst_wr_idx = 1777 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1778 ce_data.dst_r_idx = 1779 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1780 1781 if (crash_data) 1782 crash_data->ce_crash_data[id] = ce_data; 1783 1784 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1785 le32_to_cpu(ce_data.base_addr), 1786 le32_to_cpu(ce_data.src_wr_idx), 1787 le32_to_cpu(ce_data.src_r_idx), 1788 le32_to_cpu(ce_data.dst_wr_idx), 1789 le32_to_cpu(ce_data.dst_r_idx)); 1790 } 1791 1792 spin_unlock_bh(&ce->ce_lock); 1793 } 1794 EXPORT_SYMBOL(ath10k_ce_dump_registers); 1795 1796 static const struct ath10k_ce_ops ce_ops = { 1797 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring, 1798 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring, 1799 .ce_rx_post_buf = __ath10k_ce_rx_post_buf, 1800 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock, 1801 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next, 1802 .ce_extract_desc_data = ath10k_ce_extract_desc_data, 1803 .ce_free_pipe = _ath10k_ce_free_pipe, 1804 .ce_send_nolock = _ath10k_ce_send_nolock, 1805 }; 1806 1807 static const struct ath10k_ce_ops ce_64_ops = { 1808 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64, 1809 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64, 1810 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64, 1811 .ce_completed_recv_next_nolock = 1812 _ath10k_ce_completed_recv_next_nolock_64, 1813 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64, 1814 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64, 1815 .ce_free_pipe = _ath10k_ce_free_pipe_64, 1816 .ce_send_nolock = _ath10k_ce_send_nolock_64, 1817 }; 1818 1819 static void ath10k_ce_set_ops(struct ath10k *ar, 1820 struct ath10k_ce_pipe *ce_state) 1821 { 1822 switch (ar->hw_rev) { 1823 case ATH10K_HW_WCN3990: 1824 ce_state->ops = &ce_64_ops; 1825 break; 1826 default: 1827 ce_state->ops = &ce_ops; 1828 break; 1829 } 1830 } 1831 1832 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1833 const struct ce_attr *attr) 1834 { 1835 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1836 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1837 int ret; 1838 1839 ath10k_ce_set_ops(ar, ce_state); 1840 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1841 * additional TX locking checks. 1842 * 1843 * For the lack of a better place do the check here. 1844 */ 1845 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1846 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1847 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1848 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1849 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1850 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1851 1852 ce_state->ar = ar; 1853 ce_state->id = ce_id; 1854 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1855 ce_state->attr_flags = attr->flags; 1856 ce_state->src_sz_max = attr->src_sz_max; 1857 1858 if (attr->src_nentries) 1859 ce_state->send_cb = attr->send_cb; 1860 1861 if (attr->dest_nentries) 1862 ce_state->recv_cb = attr->recv_cb; 1863 1864 if (attr->src_nentries) { 1865 ce_state->src_ring = 1866 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); 1867 if (IS_ERR(ce_state->src_ring)) { 1868 ret = PTR_ERR(ce_state->src_ring); 1869 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n", 1870 ce_id, ret); 1871 ce_state->src_ring = NULL; 1872 return ret; 1873 } 1874 } 1875 1876 if (attr->dest_nentries) { 1877 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, 1878 ce_id, 1879 attr); 1880 if (IS_ERR(ce_state->dest_ring)) { 1881 ret = PTR_ERR(ce_state->dest_ring); 1882 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n", 1883 ce_id, ret); 1884 ce_state->dest_ring = NULL; 1885 return ret; 1886 } 1887 } 1888 1889 return 0; 1890 } 1891 EXPORT_SYMBOL(ath10k_ce_alloc_pipe); 1892 1893 void ath10k_ce_alloc_rri(struct ath10k *ar) 1894 { 1895 int i; 1896 u32 value; 1897 u32 ctrl1_regs; 1898 u32 ce_base_addr; 1899 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1900 1901 ce->vaddr_rri = dma_alloc_coherent(ar->dev, 1902 (CE_COUNT * sizeof(u32)), 1903 &ce->paddr_rri, GFP_KERNEL); 1904 1905 if (!ce->vaddr_rri) 1906 return; 1907 1908 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low, 1909 lower_32_bits(ce->paddr_rri)); 1910 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high, 1911 (upper_32_bits(ce->paddr_rri) & 1912 CE_DESC_FLAGS_GET_MASK)); 1913 1914 for (i = 0; i < CE_COUNT; i++) { 1915 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr; 1916 ce_base_addr = ath10k_ce_base_address(ar, i); 1917 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs); 1918 value |= ar->hw_ce_regs->upd->mask; 1919 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value); 1920 } 1921 1922 memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32)); 1923 } 1924 EXPORT_SYMBOL(ath10k_ce_alloc_rri); 1925 1926 void ath10k_ce_free_rri(struct ath10k *ar) 1927 { 1928 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1929 1930 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)), 1931 ce->vaddr_rri, 1932 ce->paddr_rri); 1933 } 1934 EXPORT_SYMBOL(ath10k_ce_free_rri); 1935