xref: /openbmc/linux/drivers/net/wireless/ath/ath10k/ce.c (revision 2fa49589)
1 /*
2  * Copyright (c) 2005-2011 Atheros Communications Inc.
3  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
4  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
5  *
6  * Permission to use, copy, modify, and/or distribute this software for any
7  * purpose with or without fee is hereby granted, provided that the above
8  * copyright notice and this permission notice appear in all copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  */
18 
19 #include "hif.h"
20 #include "ce.h"
21 #include "debug.h"
22 
23 /*
24  * Support for Copy Engine hardware, which is mainly used for
25  * communication between Host and Target over a PCIe interconnect.
26  */
27 
28 /*
29  * A single CopyEngine (CE) comprises two "rings":
30  *   a source ring
31  *   a destination ring
32  *
33  * Each ring consists of a number of descriptors which specify
34  * an address, length, and meta-data.
35  *
36  * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
37  * controls one ring and the other side controls the other ring.
38  * The source side chooses when to initiate a transfer and it
39  * chooses what to send (buffer address, length). The destination
40  * side keeps a supply of "anonymous receive buffers" available and
41  * it handles incoming data as it arrives (when the destination
42  * receives an interrupt).
43  *
44  * The sender may send a simple buffer (address/length) or it may
45  * send a small list of buffers.  When a small list is sent, hardware
46  * "gathers" these and they end up in a single destination buffer
47  * with a single interrupt.
48  *
49  * There are several "contexts" managed by this layer -- more, it
50  * may seem -- than should be needed. These are provided mainly for
51  * maximum flexibility and especially to facilitate a simpler HIF
52  * implementation. There are per-CopyEngine recv, send, and watermark
53  * contexts. These are supplied by the caller when a recv, send,
54  * or watermark handler is established and they are echoed back to
55  * the caller when the respective callbacks are invoked. There is
56  * also a per-transfer context supplied by the caller when a buffer
57  * (or sendlist) is sent and when a buffer is enqueued for recv.
58  * These per-transfer contexts are echoed back to the caller when
59  * the buffer is sent/received.
60  */
61 
62 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
63 					struct ath10k_ce_pipe *ce_state)
64 {
65 	u32 ce_id = ce_state->id;
66 	u32 addr = 0;
67 
68 	switch (ce_id) {
69 	case 0:
70 		addr = 0x00032000;
71 		break;
72 	case 3:
73 		addr = 0x0003200C;
74 		break;
75 	case 4:
76 		addr = 0x00032010;
77 		break;
78 	case 5:
79 		addr = 0x00032014;
80 		break;
81 	case 7:
82 		addr = 0x0003201C;
83 		break;
84 	default:
85 		ath10k_warn(ar, "invalid CE id: %d", ce_id);
86 		break;
87 	}
88 	return addr;
89 }
90 
91 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
92 					 struct ath10k_ce_pipe *ce_state)
93 {
94 	u32 ce_id = ce_state->id;
95 	u32 addr = 0;
96 
97 	switch (ce_id) {
98 	case 1:
99 		addr = 0x00032034;
100 		break;
101 	case 2:
102 		addr = 0x00032038;
103 		break;
104 	case 5:
105 		addr = 0x00032044;
106 		break;
107 	case 7:
108 		addr = 0x0003204C;
109 		break;
110 	case 8:
111 		addr = 0x00032050;
112 		break;
113 	case 9:
114 		addr = 0x00032054;
115 		break;
116 	case 10:
117 		addr = 0x00032058;
118 		break;
119 	case 11:
120 		addr = 0x0003205C;
121 		break;
122 	default:
123 		ath10k_warn(ar, "invalid CE id: %d", ce_id);
124 		break;
125 	}
126 
127 	return addr;
128 }
129 
130 static inline unsigned int
131 ath10k_set_ring_byte(unsigned int offset,
132 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
133 {
134 	return ((offset << addr_map->lsb) & addr_map->mask);
135 }
136 
137 static inline unsigned int
138 ath10k_get_ring_byte(unsigned int offset,
139 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
140 {
141 	return ((offset & addr_map->mask) >> (addr_map->lsb));
142 }
143 
144 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
145 {
146 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
147 
148 	return ce->bus_ops->read32(ar, offset);
149 }
150 
151 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
152 {
153 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
154 
155 	ce->bus_ops->write32(ar, offset, value);
156 }
157 
158 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
159 						       u32 ce_ctrl_addr,
160 						       unsigned int n)
161 {
162 	ath10k_ce_write32(ar, ce_ctrl_addr +
163 			  ar->hw_ce_regs->dst_wr_index_addr, n);
164 }
165 
166 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
167 						      u32 ce_ctrl_addr)
168 {
169 	return ath10k_ce_read32(ar, ce_ctrl_addr +
170 				ar->hw_ce_regs->dst_wr_index_addr);
171 }
172 
173 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
174 						      u32 ce_ctrl_addr,
175 						      unsigned int n)
176 {
177 	ath10k_ce_write32(ar, ce_ctrl_addr +
178 			  ar->hw_ce_regs->sr_wr_index_addr, n);
179 }
180 
181 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
182 						     u32 ce_ctrl_addr)
183 {
184 	return ath10k_ce_read32(ar, ce_ctrl_addr +
185 				ar->hw_ce_regs->sr_wr_index_addr);
186 }
187 
188 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
189 							 u32 ce_id)
190 {
191 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
192 
193 	return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
194 }
195 
196 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
197 						    u32 ce_ctrl_addr)
198 {
199 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
200 	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
201 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
202 	u32 index;
203 
204 	if (ar->hw_params.rri_on_ddr &&
205 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
206 		index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
207 	else
208 		index = ath10k_ce_read32(ar, ce_ctrl_addr +
209 					 ar->hw_ce_regs->current_srri_addr);
210 
211 	return index;
212 }
213 
214 static inline void
215 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
216 					  struct ath10k_ce_pipe *ce_state,
217 					  unsigned int value)
218 {
219 	ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
220 }
221 
222 static inline void
223 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
224 					   struct ath10k_ce_pipe *ce_state,
225 					   unsigned int value)
226 {
227 	ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
228 }
229 
230 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
231 						    u32 ce_ctrl_addr,
232 						    unsigned int addr)
233 {
234 	ath10k_ce_write32(ar, ce_ctrl_addr +
235 			  ar->hw_ce_regs->sr_base_addr, addr);
236 }
237 
238 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
239 					       u32 ce_ctrl_addr,
240 					       unsigned int n)
241 {
242 	ath10k_ce_write32(ar, ce_ctrl_addr +
243 			  ar->hw_ce_regs->sr_size_addr, n);
244 }
245 
246 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
247 					       u32 ce_ctrl_addr,
248 					       unsigned int n)
249 {
250 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
251 
252 	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
253 					  ctrl_regs->addr);
254 
255 	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
256 			  (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
257 			  ath10k_set_ring_byte(n, ctrl_regs->dmax));
258 }
259 
260 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
261 						    u32 ce_ctrl_addr,
262 						    unsigned int n)
263 {
264 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
265 
266 	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
267 					  ctrl_regs->addr);
268 
269 	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
270 			  (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
271 			  ath10k_set_ring_byte(n, ctrl_regs->src_ring));
272 }
273 
274 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
275 						     u32 ce_ctrl_addr,
276 						     unsigned int n)
277 {
278 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
279 
280 	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
281 					  ctrl_regs->addr);
282 
283 	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
284 			  (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
285 			  ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
286 }
287 
288 static inline
289 	u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
290 {
291 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
292 
293 	return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
294 		CE_DDR_RRI_MASK;
295 }
296 
297 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
298 						     u32 ce_ctrl_addr)
299 {
300 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
301 	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
302 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
303 	u32 index;
304 
305 	if (ar->hw_params.rri_on_ddr &&
306 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
307 		index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
308 	else
309 		index = ath10k_ce_read32(ar, ce_ctrl_addr +
310 					 ar->hw_ce_regs->current_drri_addr);
311 
312 	return index;
313 }
314 
315 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
316 						     u32 ce_ctrl_addr,
317 						     u32 addr)
318 {
319 	ath10k_ce_write32(ar, ce_ctrl_addr +
320 			  ar->hw_ce_regs->dr_base_addr, addr);
321 }
322 
323 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
324 						u32 ce_ctrl_addr,
325 						unsigned int n)
326 {
327 	ath10k_ce_write32(ar, ce_ctrl_addr +
328 			  ar->hw_ce_regs->dr_size_addr, n);
329 }
330 
331 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
332 						   u32 ce_ctrl_addr,
333 						   unsigned int n)
334 {
335 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
336 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
337 
338 	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
339 			  (addr & ~(srcr_wm->wm_high->mask)) |
340 			  (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
341 }
342 
343 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
344 						  u32 ce_ctrl_addr,
345 						  unsigned int n)
346 {
347 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
348 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
349 
350 	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
351 			  (addr & ~(srcr_wm->wm_low->mask)) |
352 			  (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
353 }
354 
355 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
356 						    u32 ce_ctrl_addr,
357 						    unsigned int n)
358 {
359 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
360 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
361 
362 	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
363 			  (addr & ~(dstr_wm->wm_high->mask)) |
364 			  (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
365 }
366 
367 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
368 						   u32 ce_ctrl_addr,
369 						   unsigned int n)
370 {
371 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
372 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
373 
374 	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
375 			  (addr & ~(dstr_wm->wm_low->mask)) |
376 			  (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
377 }
378 
379 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
380 							u32 ce_ctrl_addr)
381 {
382 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
383 
384 	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
385 					    ar->hw_ce_regs->host_ie_addr);
386 
387 	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
388 			  host_ie_addr | host_ie->copy_complete->mask);
389 }
390 
391 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
392 							u32 ce_ctrl_addr)
393 {
394 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
395 
396 	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
397 					    ar->hw_ce_regs->host_ie_addr);
398 
399 	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
400 			  host_ie_addr & ~(host_ie->copy_complete->mask));
401 }
402 
403 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
404 						    u32 ce_ctrl_addr)
405 {
406 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
407 
408 	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
409 					    ar->hw_ce_regs->host_ie_addr);
410 
411 	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
412 			  host_ie_addr & ~(wm_regs->wm_mask));
413 }
414 
415 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
416 					       u32 ce_ctrl_addr)
417 {
418 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
419 
420 	u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
421 					    ar->hw_ce_regs->misc_ie_addr);
422 
423 	ath10k_ce_write32(ar,
424 			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
425 			  misc_ie_addr | misc_regs->err_mask);
426 }
427 
428 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
429 						u32 ce_ctrl_addr)
430 {
431 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
432 
433 	u32 misc_ie_addr = ath10k_ce_read32(ar,
434 			ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
435 
436 	ath10k_ce_write32(ar,
437 			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
438 			  misc_ie_addr & ~(misc_regs->err_mask));
439 }
440 
441 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
442 						     u32 ce_ctrl_addr,
443 						     unsigned int mask)
444 {
445 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
446 
447 	ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
448 }
449 
450 /*
451  * Guts of ath10k_ce_send.
452  * The caller takes responsibility for any needed locking.
453  */
454 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
455 				  void *per_transfer_context,
456 				  dma_addr_t buffer,
457 				  unsigned int nbytes,
458 				  unsigned int transfer_id,
459 				  unsigned int flags)
460 {
461 	struct ath10k *ar = ce_state->ar;
462 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
463 	struct ce_desc *desc, sdesc;
464 	unsigned int nentries_mask = src_ring->nentries_mask;
465 	unsigned int sw_index = src_ring->sw_index;
466 	unsigned int write_index = src_ring->write_index;
467 	u32 ctrl_addr = ce_state->ctrl_addr;
468 	u32 desc_flags = 0;
469 	int ret = 0;
470 
471 	if (nbytes > ce_state->src_sz_max)
472 		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
473 			    __func__, nbytes, ce_state->src_sz_max);
474 
475 	if (unlikely(CE_RING_DELTA(nentries_mask,
476 				   write_index, sw_index - 1) <= 0)) {
477 		ret = -ENOSR;
478 		goto exit;
479 	}
480 
481 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
482 				   write_index);
483 
484 	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
485 
486 	if (flags & CE_SEND_FLAG_GATHER)
487 		desc_flags |= CE_DESC_FLAGS_GATHER;
488 	if (flags & CE_SEND_FLAG_BYTE_SWAP)
489 		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
490 
491 	sdesc.addr   = __cpu_to_le32(buffer);
492 	sdesc.nbytes = __cpu_to_le16(nbytes);
493 	sdesc.flags  = __cpu_to_le16(desc_flags);
494 
495 	*desc = sdesc;
496 
497 	src_ring->per_transfer_context[write_index] = per_transfer_context;
498 
499 	/* Update Source Ring Write Index */
500 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
501 
502 	/* WORKAROUND */
503 	if (!(flags & CE_SEND_FLAG_GATHER)) {
504 		if (ar->hw_params.shadow_reg_support)
505 			ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
506 								  write_index);
507 		else
508 			ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
509 							   write_index);
510 	}
511 
512 	src_ring->write_index = write_index;
513 exit:
514 	return ret;
515 }
516 
517 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
518 				     void *per_transfer_context,
519 				     dma_addr_t buffer,
520 				     unsigned int nbytes,
521 				     unsigned int transfer_id,
522 				     unsigned int flags)
523 {
524 	struct ath10k *ar = ce_state->ar;
525 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
526 	struct ce_desc_64 *desc, sdesc;
527 	unsigned int nentries_mask = src_ring->nentries_mask;
528 	unsigned int sw_index;
529 	unsigned int write_index = src_ring->write_index;
530 	u32 ctrl_addr = ce_state->ctrl_addr;
531 	__le32 *addr;
532 	u32 desc_flags = 0;
533 	int ret = 0;
534 
535 	if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
536 		return -ESHUTDOWN;
537 
538 	if (nbytes > ce_state->src_sz_max)
539 		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
540 			    __func__, nbytes, ce_state->src_sz_max);
541 
542 	if (ar->hw_params.rri_on_ddr)
543 		sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
544 	else
545 		sw_index = src_ring->sw_index;
546 
547 	if (unlikely(CE_RING_DELTA(nentries_mask,
548 				   write_index, sw_index - 1) <= 0)) {
549 		ret = -ENOSR;
550 		goto exit;
551 	}
552 
553 	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
554 				      write_index);
555 
556 	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
557 
558 	if (flags & CE_SEND_FLAG_GATHER)
559 		desc_flags |= CE_DESC_FLAGS_GATHER;
560 
561 	if (flags & CE_SEND_FLAG_BYTE_SWAP)
562 		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
563 
564 	addr = (__le32 *)&sdesc.addr;
565 
566 	flags |= upper_32_bits(buffer) & CE_DESC_FLAGS_GET_MASK;
567 	addr[0] = __cpu_to_le32(buffer);
568 	addr[1] = __cpu_to_le32(flags);
569 	if (flags & CE_SEND_FLAG_GATHER)
570 		addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
571 	else
572 		addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
573 
574 	sdesc.nbytes = __cpu_to_le16(nbytes);
575 	sdesc.flags  = __cpu_to_le16(desc_flags);
576 
577 	*desc = sdesc;
578 
579 	src_ring->per_transfer_context[write_index] = per_transfer_context;
580 
581 	/* Update Source Ring Write Index */
582 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
583 
584 	if (!(flags & CE_SEND_FLAG_GATHER))
585 		ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
586 
587 	src_ring->write_index = write_index;
588 exit:
589 	return ret;
590 }
591 
592 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
593 			  void *per_transfer_context,
594 			  dma_addr_t buffer,
595 			  unsigned int nbytes,
596 			  unsigned int transfer_id,
597 			  unsigned int flags)
598 {
599 	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
600 				    buffer, nbytes, transfer_id, flags);
601 }
602 EXPORT_SYMBOL(ath10k_ce_send_nolock);
603 
604 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
605 {
606 	struct ath10k *ar = pipe->ar;
607 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
608 	struct ath10k_ce_ring *src_ring = pipe->src_ring;
609 	u32 ctrl_addr = pipe->ctrl_addr;
610 
611 	lockdep_assert_held(&ce->ce_lock);
612 
613 	/*
614 	 * This function must be called only if there is an incomplete
615 	 * scatter-gather transfer (before index register is updated)
616 	 * that needs to be cleaned up.
617 	 */
618 	if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
619 		return;
620 
621 	if (WARN_ON_ONCE(src_ring->write_index ==
622 			 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
623 		return;
624 
625 	src_ring->write_index--;
626 	src_ring->write_index &= src_ring->nentries_mask;
627 
628 	src_ring->per_transfer_context[src_ring->write_index] = NULL;
629 }
630 EXPORT_SYMBOL(__ath10k_ce_send_revert);
631 
632 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
633 		   void *per_transfer_context,
634 		   dma_addr_t buffer,
635 		   unsigned int nbytes,
636 		   unsigned int transfer_id,
637 		   unsigned int flags)
638 {
639 	struct ath10k *ar = ce_state->ar;
640 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
641 	int ret;
642 
643 	spin_lock_bh(&ce->ce_lock);
644 	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
645 				    buffer, nbytes, transfer_id, flags);
646 	spin_unlock_bh(&ce->ce_lock);
647 
648 	return ret;
649 }
650 EXPORT_SYMBOL(ath10k_ce_send);
651 
652 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
653 {
654 	struct ath10k *ar = pipe->ar;
655 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
656 	int delta;
657 
658 	spin_lock_bh(&ce->ce_lock);
659 	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
660 			      pipe->src_ring->write_index,
661 			      pipe->src_ring->sw_index - 1);
662 	spin_unlock_bh(&ce->ce_lock);
663 
664 	return delta;
665 }
666 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
667 
668 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
669 {
670 	struct ath10k *ar = pipe->ar;
671 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
672 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
673 	unsigned int nentries_mask = dest_ring->nentries_mask;
674 	unsigned int write_index = dest_ring->write_index;
675 	unsigned int sw_index = dest_ring->sw_index;
676 
677 	lockdep_assert_held(&ce->ce_lock);
678 
679 	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
680 }
681 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
682 
683 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
684 				   dma_addr_t paddr)
685 {
686 	struct ath10k *ar = pipe->ar;
687 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
688 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
689 	unsigned int nentries_mask = dest_ring->nentries_mask;
690 	unsigned int write_index = dest_ring->write_index;
691 	unsigned int sw_index = dest_ring->sw_index;
692 	struct ce_desc *base = dest_ring->base_addr_owner_space;
693 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
694 	u32 ctrl_addr = pipe->ctrl_addr;
695 
696 	lockdep_assert_held(&ce->ce_lock);
697 
698 	if ((pipe->id != 5) &&
699 	    CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
700 		return -ENOSPC;
701 
702 	desc->addr = __cpu_to_le32(paddr);
703 	desc->nbytes = 0;
704 
705 	dest_ring->per_transfer_context[write_index] = ctx;
706 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
707 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
708 	dest_ring->write_index = write_index;
709 
710 	return 0;
711 }
712 
713 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
714 				      void *ctx,
715 				      dma_addr_t paddr)
716 {
717 	struct ath10k *ar = pipe->ar;
718 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
719 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
720 	unsigned int nentries_mask = dest_ring->nentries_mask;
721 	unsigned int write_index = dest_ring->write_index;
722 	unsigned int sw_index = dest_ring->sw_index;
723 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
724 	struct ce_desc_64 *desc =
725 			CE_DEST_RING_TO_DESC_64(base, write_index);
726 	u32 ctrl_addr = pipe->ctrl_addr;
727 
728 	lockdep_assert_held(&ce->ce_lock);
729 
730 	if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
731 		return -ENOSPC;
732 
733 	desc->addr = __cpu_to_le64(paddr);
734 	desc->addr &= __cpu_to_le64(CE_DESC_37BIT_ADDR_MASK);
735 
736 	desc->nbytes = 0;
737 
738 	dest_ring->per_transfer_context[write_index] = ctx;
739 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
740 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
741 	dest_ring->write_index = write_index;
742 
743 	return 0;
744 }
745 
746 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
747 {
748 	struct ath10k *ar = pipe->ar;
749 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
750 	unsigned int nentries_mask = dest_ring->nentries_mask;
751 	unsigned int write_index = dest_ring->write_index;
752 	u32 ctrl_addr = pipe->ctrl_addr;
753 	u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
754 
755 	/* Prevent CE ring stuck issue that will occur when ring is full.
756 	 * Make sure that write index is 1 less than read index.
757 	 */
758 	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
759 		nentries -= 1;
760 
761 	write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
762 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
763 	dest_ring->write_index = write_index;
764 }
765 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
766 
767 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
768 			  dma_addr_t paddr)
769 {
770 	struct ath10k *ar = pipe->ar;
771 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
772 	int ret;
773 
774 	spin_lock_bh(&ce->ce_lock);
775 	ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
776 	spin_unlock_bh(&ce->ce_lock);
777 
778 	return ret;
779 }
780 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
781 
782 /*
783  * Guts of ath10k_ce_completed_recv_next.
784  * The caller takes responsibility for any necessary locking.
785  */
786 static int
787 	 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
788 					       void **per_transfer_contextp,
789 					       unsigned int *nbytesp)
790 {
791 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
792 	unsigned int nentries_mask = dest_ring->nentries_mask;
793 	unsigned int sw_index = dest_ring->sw_index;
794 
795 	struct ce_desc *base = dest_ring->base_addr_owner_space;
796 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
797 	struct ce_desc sdesc;
798 	u16 nbytes;
799 
800 	/* Copy in one go for performance reasons */
801 	sdesc = *desc;
802 
803 	nbytes = __le16_to_cpu(sdesc.nbytes);
804 	if (nbytes == 0) {
805 		/*
806 		 * This closes a relatively unusual race where the Host
807 		 * sees the updated DRRI before the update to the
808 		 * corresponding descriptor has completed. We treat this
809 		 * as a descriptor that is not yet done.
810 		 */
811 		return -EIO;
812 	}
813 
814 	desc->nbytes = 0;
815 
816 	/* Return data from completed destination descriptor */
817 	*nbytesp = nbytes;
818 
819 	if (per_transfer_contextp)
820 		*per_transfer_contextp =
821 			dest_ring->per_transfer_context[sw_index];
822 
823 	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
824 	 * So update transfer context all CEs except CE5.
825 	 */
826 	if (ce_state->id != 5)
827 		dest_ring->per_transfer_context[sw_index] = NULL;
828 
829 	/* Update sw_index */
830 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
831 	dest_ring->sw_index = sw_index;
832 
833 	return 0;
834 }
835 
836 static int
837 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
838 					 void **per_transfer_contextp,
839 					 unsigned int *nbytesp)
840 {
841 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
842 	unsigned int nentries_mask = dest_ring->nentries_mask;
843 	unsigned int sw_index = dest_ring->sw_index;
844 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
845 	struct ce_desc_64 *desc =
846 		CE_DEST_RING_TO_DESC_64(base, sw_index);
847 	struct ce_desc_64 sdesc;
848 	u16 nbytes;
849 
850 	/* Copy in one go for performance reasons */
851 	sdesc = *desc;
852 
853 	nbytes = __le16_to_cpu(sdesc.nbytes);
854 	if (nbytes == 0) {
855 		/* This closes a relatively unusual race where the Host
856 		 * sees the updated DRRI before the update to the
857 		 * corresponding descriptor has completed. We treat this
858 		 * as a descriptor that is not yet done.
859 		 */
860 		return -EIO;
861 	}
862 
863 	desc->nbytes = 0;
864 
865 	/* Return data from completed destination descriptor */
866 	*nbytesp = nbytes;
867 
868 	if (per_transfer_contextp)
869 		*per_transfer_contextp =
870 			dest_ring->per_transfer_context[sw_index];
871 
872 	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
873 	 * So update transfer context all CEs except CE5.
874 	 */
875 	if (ce_state->id != 5)
876 		dest_ring->per_transfer_context[sw_index] = NULL;
877 
878 	/* Update sw_index */
879 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
880 	dest_ring->sw_index = sw_index;
881 
882 	return 0;
883 }
884 
885 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
886 					 void **per_transfer_ctx,
887 					 unsigned int *nbytesp)
888 {
889 	return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
890 							    per_transfer_ctx,
891 							    nbytesp);
892 }
893 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
894 
895 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
896 				  void **per_transfer_contextp,
897 				  unsigned int *nbytesp)
898 {
899 	struct ath10k *ar = ce_state->ar;
900 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
901 	int ret;
902 
903 	spin_lock_bh(&ce->ce_lock);
904 	ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
905 						   per_transfer_contextp,
906 						   nbytesp);
907 
908 	spin_unlock_bh(&ce->ce_lock);
909 
910 	return ret;
911 }
912 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
913 
914 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
915 				       void **per_transfer_contextp,
916 				       dma_addr_t *bufferp)
917 {
918 	struct ath10k_ce_ring *dest_ring;
919 	unsigned int nentries_mask;
920 	unsigned int sw_index;
921 	unsigned int write_index;
922 	int ret;
923 	struct ath10k *ar;
924 	struct ath10k_ce *ce;
925 
926 	dest_ring = ce_state->dest_ring;
927 
928 	if (!dest_ring)
929 		return -EIO;
930 
931 	ar = ce_state->ar;
932 	ce = ath10k_ce_priv(ar);
933 
934 	spin_lock_bh(&ce->ce_lock);
935 
936 	nentries_mask = dest_ring->nentries_mask;
937 	sw_index = dest_ring->sw_index;
938 	write_index = dest_ring->write_index;
939 	if (write_index != sw_index) {
940 		struct ce_desc *base = dest_ring->base_addr_owner_space;
941 		struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
942 
943 		/* Return data from completed destination descriptor */
944 		*bufferp = __le32_to_cpu(desc->addr);
945 
946 		if (per_transfer_contextp)
947 			*per_transfer_contextp =
948 				dest_ring->per_transfer_context[sw_index];
949 
950 		/* sanity */
951 		dest_ring->per_transfer_context[sw_index] = NULL;
952 		desc->nbytes = 0;
953 
954 		/* Update sw_index */
955 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
956 		dest_ring->sw_index = sw_index;
957 		ret = 0;
958 	} else {
959 		ret = -EIO;
960 	}
961 
962 	spin_unlock_bh(&ce->ce_lock);
963 
964 	return ret;
965 }
966 
967 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
968 					  void **per_transfer_contextp,
969 					  dma_addr_t *bufferp)
970 {
971 	struct ath10k_ce_ring *dest_ring;
972 	unsigned int nentries_mask;
973 	unsigned int sw_index;
974 	unsigned int write_index;
975 	int ret;
976 	struct ath10k *ar;
977 	struct ath10k_ce *ce;
978 
979 	dest_ring = ce_state->dest_ring;
980 
981 	if (!dest_ring)
982 		return -EIO;
983 
984 	ar = ce_state->ar;
985 	ce = ath10k_ce_priv(ar);
986 
987 	spin_lock_bh(&ce->ce_lock);
988 
989 	nentries_mask = dest_ring->nentries_mask;
990 	sw_index = dest_ring->sw_index;
991 	write_index = dest_ring->write_index;
992 	if (write_index != sw_index) {
993 		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
994 		struct ce_desc_64 *desc =
995 			CE_DEST_RING_TO_DESC_64(base, sw_index);
996 
997 		/* Return data from completed destination descriptor */
998 		*bufferp = __le64_to_cpu(desc->addr);
999 
1000 		if (per_transfer_contextp)
1001 			*per_transfer_contextp =
1002 				dest_ring->per_transfer_context[sw_index];
1003 
1004 		/* sanity */
1005 		dest_ring->per_transfer_context[sw_index] = NULL;
1006 		desc->nbytes = 0;
1007 
1008 		/* Update sw_index */
1009 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1010 		dest_ring->sw_index = sw_index;
1011 		ret = 0;
1012 	} else {
1013 		ret = -EIO;
1014 	}
1015 
1016 	spin_unlock_bh(&ce->ce_lock);
1017 
1018 	return ret;
1019 }
1020 
1021 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1022 			       void **per_transfer_contextp,
1023 			       dma_addr_t *bufferp)
1024 {
1025 	return ce_state->ops->ce_revoke_recv_next(ce_state,
1026 						  per_transfer_contextp,
1027 						  bufferp);
1028 }
1029 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1030 
1031 /*
1032  * Guts of ath10k_ce_completed_send_next.
1033  * The caller takes responsibility for any necessary locking.
1034  */
1035 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1036 					 void **per_transfer_contextp)
1037 {
1038 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1039 	u32 ctrl_addr = ce_state->ctrl_addr;
1040 	struct ath10k *ar = ce_state->ar;
1041 	unsigned int nentries_mask = src_ring->nentries_mask;
1042 	unsigned int sw_index = src_ring->sw_index;
1043 	unsigned int read_index;
1044 	struct ce_desc *desc;
1045 
1046 	if (src_ring->hw_index == sw_index) {
1047 		/*
1048 		 * The SW completion index has caught up with the cached
1049 		 * version of the HW completion index.
1050 		 * Update the cached HW completion index to see whether
1051 		 * the SW has really caught up to the HW, or if the cached
1052 		 * value of the HW index has become stale.
1053 		 */
1054 
1055 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1056 		if (read_index == 0xffffffff)
1057 			return -ENODEV;
1058 
1059 		read_index &= nentries_mask;
1060 		src_ring->hw_index = read_index;
1061 	}
1062 
1063 	if (ar->hw_params.rri_on_ddr)
1064 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1065 	else
1066 		read_index = src_ring->hw_index;
1067 
1068 	if (read_index == sw_index)
1069 		return -EIO;
1070 
1071 	if (per_transfer_contextp)
1072 		*per_transfer_contextp =
1073 			src_ring->per_transfer_context[sw_index];
1074 
1075 	/* sanity */
1076 	src_ring->per_transfer_context[sw_index] = NULL;
1077 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1078 				   sw_index);
1079 	desc->nbytes = 0;
1080 
1081 	/* Update sw_index */
1082 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1083 	src_ring->sw_index = sw_index;
1084 
1085 	return 0;
1086 }
1087 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1088 
1089 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1090 					struct ath10k_ce_ring *src_ring,
1091 					u32 sw_index,
1092 					dma_addr_t *bufferp,
1093 					u32 *nbytesp,
1094 					u32 *transfer_idp)
1095 {
1096 		struct ce_desc *base = src_ring->base_addr_owner_space;
1097 		struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1098 
1099 		/* Return data from completed source descriptor */
1100 		*bufferp = __le32_to_cpu(desc->addr);
1101 		*nbytesp = __le16_to_cpu(desc->nbytes);
1102 		*transfer_idp = MS(__le16_to_cpu(desc->flags),
1103 				   CE_DESC_FLAGS_META_DATA);
1104 }
1105 
1106 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1107 					   struct ath10k_ce_ring *src_ring,
1108 					   u32 sw_index,
1109 					   dma_addr_t *bufferp,
1110 					   u32 *nbytesp,
1111 					   u32 *transfer_idp)
1112 {
1113 		struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1114 		struct ce_desc_64 *desc =
1115 			CE_SRC_RING_TO_DESC_64(base, sw_index);
1116 
1117 		/* Return data from completed source descriptor */
1118 		*bufferp = __le64_to_cpu(desc->addr);
1119 		*nbytesp = __le16_to_cpu(desc->nbytes);
1120 		*transfer_idp = MS(__le16_to_cpu(desc->flags),
1121 				   CE_DESC_FLAGS_META_DATA);
1122 }
1123 
1124 /* NB: Modeled after ath10k_ce_completed_send_next */
1125 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1126 			       void **per_transfer_contextp,
1127 			       dma_addr_t *bufferp,
1128 			       unsigned int *nbytesp,
1129 			       unsigned int *transfer_idp)
1130 {
1131 	struct ath10k_ce_ring *src_ring;
1132 	unsigned int nentries_mask;
1133 	unsigned int sw_index;
1134 	unsigned int write_index;
1135 	int ret;
1136 	struct ath10k *ar;
1137 	struct ath10k_ce *ce;
1138 
1139 	src_ring = ce_state->src_ring;
1140 
1141 	if (!src_ring)
1142 		return -EIO;
1143 
1144 	ar = ce_state->ar;
1145 	ce = ath10k_ce_priv(ar);
1146 
1147 	spin_lock_bh(&ce->ce_lock);
1148 
1149 	nentries_mask = src_ring->nentries_mask;
1150 	sw_index = src_ring->sw_index;
1151 	write_index = src_ring->write_index;
1152 
1153 	if (write_index != sw_index) {
1154 		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1155 						    bufferp, nbytesp,
1156 						    transfer_idp);
1157 
1158 		if (per_transfer_contextp)
1159 			*per_transfer_contextp =
1160 				src_ring->per_transfer_context[sw_index];
1161 
1162 		/* sanity */
1163 		src_ring->per_transfer_context[sw_index] = NULL;
1164 
1165 		/* Update sw_index */
1166 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1167 		src_ring->sw_index = sw_index;
1168 		ret = 0;
1169 	} else {
1170 		ret = -EIO;
1171 	}
1172 
1173 	spin_unlock_bh(&ce->ce_lock);
1174 
1175 	return ret;
1176 }
1177 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1178 
1179 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1180 				  void **per_transfer_contextp)
1181 {
1182 	struct ath10k *ar = ce_state->ar;
1183 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1184 	int ret;
1185 
1186 	spin_lock_bh(&ce->ce_lock);
1187 	ret = ath10k_ce_completed_send_next_nolock(ce_state,
1188 						   per_transfer_contextp);
1189 	spin_unlock_bh(&ce->ce_lock);
1190 
1191 	return ret;
1192 }
1193 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1194 
1195 /*
1196  * Guts of interrupt handler for per-engine interrupts on a particular CE.
1197  *
1198  * Invokes registered callbacks for recv_complete,
1199  * send_complete, and watermarks.
1200  */
1201 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1202 {
1203 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1204 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1205 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1206 	u32 ctrl_addr = ce_state->ctrl_addr;
1207 
1208 	spin_lock_bh(&ce->ce_lock);
1209 
1210 	/* Clear the copy-complete interrupts that will be handled here. */
1211 	ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1212 					  wm_regs->cc_mask);
1213 
1214 	spin_unlock_bh(&ce->ce_lock);
1215 
1216 	if (ce_state->recv_cb)
1217 		ce_state->recv_cb(ce_state);
1218 
1219 	if (ce_state->send_cb)
1220 		ce_state->send_cb(ce_state);
1221 
1222 	spin_lock_bh(&ce->ce_lock);
1223 
1224 	/*
1225 	 * Misc CE interrupts are not being handled, but still need
1226 	 * to be cleared.
1227 	 */
1228 	ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask);
1229 
1230 	spin_unlock_bh(&ce->ce_lock);
1231 }
1232 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1233 
1234 /*
1235  * Handler for per-engine interrupts on ALL active CEs.
1236  * This is used in cases where the system is sharing a
1237  * single interrput for all CEs
1238  */
1239 
1240 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1241 {
1242 	int ce_id;
1243 	u32 intr_summary;
1244 
1245 	intr_summary = ath10k_ce_interrupt_summary(ar);
1246 
1247 	for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1248 		if (intr_summary & (1 << ce_id))
1249 			intr_summary &= ~(1 << ce_id);
1250 		else
1251 			/* no intr pending on this CE */
1252 			continue;
1253 
1254 		ath10k_ce_per_engine_service(ar, ce_id);
1255 	}
1256 }
1257 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1258 
1259 /*
1260  * Adjust interrupts for the copy complete handler.
1261  * If it's needed for either send or recv, then unmask
1262  * this interrupt; otherwise, mask it.
1263  *
1264  * Called with ce_lock held.
1265  */
1266 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1267 {
1268 	u32 ctrl_addr = ce_state->ctrl_addr;
1269 	struct ath10k *ar = ce_state->ar;
1270 	bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1271 
1272 	if ((!disable_copy_compl_intr) &&
1273 	    (ce_state->send_cb || ce_state->recv_cb))
1274 		ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1275 	else
1276 		ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1277 
1278 	ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1279 }
1280 
1281 int ath10k_ce_disable_interrupts(struct ath10k *ar)
1282 {
1283 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1284 	struct ath10k_ce_pipe *ce_state;
1285 	u32 ctrl_addr;
1286 	int ce_id;
1287 
1288 	for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1289 		ce_state  = &ce->ce_states[ce_id];
1290 		if (ce_state->attr_flags & CE_ATTR_POLL)
1291 			continue;
1292 
1293 		ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1294 
1295 		ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1296 		ath10k_ce_error_intr_disable(ar, ctrl_addr);
1297 		ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1298 	}
1299 
1300 	return 0;
1301 }
1302 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1303 
1304 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1305 {
1306 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307 	int ce_id;
1308 	struct ath10k_ce_pipe *ce_state;
1309 
1310 	/* Enable interrupts for copy engine that
1311 	 * are not using polling mode.
1312 	 */
1313 	for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1314 		ce_state  = &ce->ce_states[ce_id];
1315 		if (ce_state->attr_flags & CE_ATTR_POLL)
1316 			continue;
1317 
1318 		ath10k_ce_per_engine_handler_adjust(ce_state);
1319 	}
1320 }
1321 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1322 
1323 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1324 				   unsigned int ce_id,
1325 				   const struct ce_attr *attr)
1326 {
1327 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1328 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1329 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1330 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1331 
1332 	nentries = roundup_pow_of_two(attr->src_nentries);
1333 
1334 	if (ar->hw_params.target_64bit)
1335 		memset(src_ring->base_addr_owner_space, 0,
1336 		       nentries * sizeof(struct ce_desc_64));
1337 	else
1338 		memset(src_ring->base_addr_owner_space, 0,
1339 		       nentries * sizeof(struct ce_desc));
1340 
1341 	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1342 	src_ring->sw_index &= src_ring->nentries_mask;
1343 	src_ring->hw_index = src_ring->sw_index;
1344 
1345 	src_ring->write_index =
1346 		ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1347 	src_ring->write_index &= src_ring->nentries_mask;
1348 
1349 	ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr,
1350 					 src_ring->base_addr_ce_space);
1351 	ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1352 	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1353 	ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1354 	ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1355 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1356 
1357 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
1358 		   "boot init ce src ring id %d entries %d base_addr %pK\n",
1359 		   ce_id, nentries, src_ring->base_addr_owner_space);
1360 
1361 	return 0;
1362 }
1363 
1364 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1365 				    unsigned int ce_id,
1366 				    const struct ce_attr *attr)
1367 {
1368 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1369 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1370 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1371 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1372 
1373 	nentries = roundup_pow_of_two(attr->dest_nentries);
1374 
1375 	if (ar->hw_params.target_64bit)
1376 		memset(dest_ring->base_addr_owner_space, 0,
1377 		       nentries * sizeof(struct ce_desc_64));
1378 	else
1379 		memset(dest_ring->base_addr_owner_space, 0,
1380 		       nentries * sizeof(struct ce_desc));
1381 
1382 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1383 	dest_ring->sw_index &= dest_ring->nentries_mask;
1384 	dest_ring->write_index =
1385 		ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1386 	dest_ring->write_index &= dest_ring->nentries_mask;
1387 
1388 	ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr,
1389 					  dest_ring->base_addr_ce_space);
1390 	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1391 	ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1392 	ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1393 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1394 
1395 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
1396 		   "boot ce dest ring id %d entries %d base_addr %pK\n",
1397 		   ce_id, nentries, dest_ring->base_addr_owner_space);
1398 
1399 	return 0;
1400 }
1401 
1402 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1403 				       struct ath10k_ce_ring *src_ring,
1404 				       u32 nentries)
1405 {
1406 	src_ring->shadow_base_unaligned = kcalloc(nentries,
1407 						  sizeof(struct ce_desc),
1408 						  GFP_KERNEL);
1409 	if (!src_ring->shadow_base_unaligned)
1410 		return -ENOMEM;
1411 
1412 	src_ring->shadow_base = (struct ce_desc *)
1413 			PTR_ALIGN(src_ring->shadow_base_unaligned,
1414 				  CE_DESC_RING_ALIGN);
1415 	return 0;
1416 }
1417 
1418 static struct ath10k_ce_ring *
1419 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1420 			 const struct ce_attr *attr)
1421 {
1422 	struct ath10k_ce_ring *src_ring;
1423 	u32 nentries = attr->src_nentries;
1424 	dma_addr_t base_addr;
1425 	int ret;
1426 
1427 	nentries = roundup_pow_of_two(nentries);
1428 
1429 	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1430 				       nentries), GFP_KERNEL);
1431 	if (src_ring == NULL)
1432 		return ERR_PTR(-ENOMEM);
1433 
1434 	src_ring->nentries = nentries;
1435 	src_ring->nentries_mask = nentries - 1;
1436 
1437 	/*
1438 	 * Legacy platforms that do not support cache
1439 	 * coherent DMA are unsupported
1440 	 */
1441 	src_ring->base_addr_owner_space_unaligned =
1442 		dma_alloc_coherent(ar->dev,
1443 				   (nentries * sizeof(struct ce_desc) +
1444 				    CE_DESC_RING_ALIGN),
1445 				   &base_addr, GFP_KERNEL);
1446 	if (!src_ring->base_addr_owner_space_unaligned) {
1447 		kfree(src_ring);
1448 		return ERR_PTR(-ENOMEM);
1449 	}
1450 
1451 	src_ring->base_addr_ce_space_unaligned = base_addr;
1452 
1453 	src_ring->base_addr_owner_space =
1454 			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1455 				  CE_DESC_RING_ALIGN);
1456 	src_ring->base_addr_ce_space =
1457 			ALIGN(src_ring->base_addr_ce_space_unaligned,
1458 			      CE_DESC_RING_ALIGN);
1459 
1460 	if (ar->hw_params.shadow_reg_support) {
1461 		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1462 		if (ret) {
1463 			dma_free_coherent(ar->dev,
1464 					  (nentries * sizeof(struct ce_desc) +
1465 					   CE_DESC_RING_ALIGN),
1466 					  src_ring->base_addr_owner_space_unaligned,
1467 					  base_addr);
1468 			kfree(src_ring);
1469 			return ERR_PTR(ret);
1470 		}
1471 	}
1472 
1473 	return src_ring;
1474 }
1475 
1476 static struct ath10k_ce_ring *
1477 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1478 			    const struct ce_attr *attr)
1479 {
1480 	struct ath10k_ce_ring *src_ring;
1481 	u32 nentries = attr->src_nentries;
1482 	dma_addr_t base_addr;
1483 	int ret;
1484 
1485 	nentries = roundup_pow_of_two(nentries);
1486 
1487 	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1488 				       nentries), GFP_KERNEL);
1489 	if (!src_ring)
1490 		return ERR_PTR(-ENOMEM);
1491 
1492 	src_ring->nentries = nentries;
1493 	src_ring->nentries_mask = nentries - 1;
1494 
1495 	/* Legacy platforms that do not support cache
1496 	 * coherent DMA are unsupported
1497 	 */
1498 	src_ring->base_addr_owner_space_unaligned =
1499 		dma_alloc_coherent(ar->dev,
1500 				   (nentries * sizeof(struct ce_desc_64) +
1501 				    CE_DESC_RING_ALIGN),
1502 				   &base_addr, GFP_KERNEL);
1503 	if (!src_ring->base_addr_owner_space_unaligned) {
1504 		kfree(src_ring);
1505 		return ERR_PTR(-ENOMEM);
1506 	}
1507 
1508 	src_ring->base_addr_ce_space_unaligned = base_addr;
1509 
1510 	src_ring->base_addr_owner_space =
1511 			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1512 				  CE_DESC_RING_ALIGN);
1513 	src_ring->base_addr_ce_space =
1514 			ALIGN(src_ring->base_addr_ce_space_unaligned,
1515 			      CE_DESC_RING_ALIGN);
1516 
1517 	if (ar->hw_params.shadow_reg_support) {
1518 		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1519 		if (ret) {
1520 			dma_free_coherent(ar->dev,
1521 					  (nentries * sizeof(struct ce_desc_64) +
1522 					   CE_DESC_RING_ALIGN),
1523 					  src_ring->base_addr_owner_space_unaligned,
1524 					  base_addr);
1525 			kfree(src_ring);
1526 			return ERR_PTR(ret);
1527 		}
1528 	}
1529 
1530 	return src_ring;
1531 }
1532 
1533 static struct ath10k_ce_ring *
1534 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1535 			  const struct ce_attr *attr)
1536 {
1537 	struct ath10k_ce_ring *dest_ring;
1538 	u32 nentries;
1539 	dma_addr_t base_addr;
1540 
1541 	nentries = roundup_pow_of_two(attr->dest_nentries);
1542 
1543 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1544 					nentries), GFP_KERNEL);
1545 	if (dest_ring == NULL)
1546 		return ERR_PTR(-ENOMEM);
1547 
1548 	dest_ring->nentries = nentries;
1549 	dest_ring->nentries_mask = nentries - 1;
1550 
1551 	/*
1552 	 * Legacy platforms that do not support cache
1553 	 * coherent DMA are unsupported
1554 	 */
1555 	dest_ring->base_addr_owner_space_unaligned =
1556 		dma_alloc_coherent(ar->dev,
1557 				   (nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN),
1558 				   &base_addr, GFP_KERNEL);
1559 	if (!dest_ring->base_addr_owner_space_unaligned) {
1560 		kfree(dest_ring);
1561 		return ERR_PTR(-ENOMEM);
1562 	}
1563 
1564 	dest_ring->base_addr_ce_space_unaligned = base_addr;
1565 
1566 	dest_ring->base_addr_owner_space =
1567 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1568 				  CE_DESC_RING_ALIGN);
1569 	dest_ring->base_addr_ce_space =
1570 				ALIGN(dest_ring->base_addr_ce_space_unaligned,
1571 				      CE_DESC_RING_ALIGN);
1572 
1573 	return dest_ring;
1574 }
1575 
1576 static struct ath10k_ce_ring *
1577 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1578 			     const struct ce_attr *attr)
1579 {
1580 	struct ath10k_ce_ring *dest_ring;
1581 	u32 nentries;
1582 	dma_addr_t base_addr;
1583 
1584 	nentries = roundup_pow_of_two(attr->dest_nentries);
1585 
1586 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1587 					nentries), GFP_KERNEL);
1588 	if (!dest_ring)
1589 		return ERR_PTR(-ENOMEM);
1590 
1591 	dest_ring->nentries = nentries;
1592 	dest_ring->nentries_mask = nentries - 1;
1593 
1594 	/* Legacy platforms that do not support cache
1595 	 * coherent DMA are unsupported
1596 	 */
1597 	dest_ring->base_addr_owner_space_unaligned =
1598 		dma_alloc_coherent(ar->dev,
1599 				   (nentries * sizeof(struct ce_desc_64) +
1600 				    CE_DESC_RING_ALIGN),
1601 				   &base_addr, GFP_KERNEL);
1602 	if (!dest_ring->base_addr_owner_space_unaligned) {
1603 		kfree(dest_ring);
1604 		return ERR_PTR(-ENOMEM);
1605 	}
1606 
1607 	dest_ring->base_addr_ce_space_unaligned = base_addr;
1608 
1609 	/* Correctly initialize memory to 0 to prevent garbage
1610 	 * data crashing system when download firmware
1611 	 */
1612 	memset(dest_ring->base_addr_owner_space_unaligned, 0,
1613 	       nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN);
1614 
1615 	dest_ring->base_addr_owner_space =
1616 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1617 				  CE_DESC_RING_ALIGN);
1618 	dest_ring->base_addr_ce_space =
1619 			ALIGN(dest_ring->base_addr_ce_space_unaligned,
1620 			      CE_DESC_RING_ALIGN);
1621 
1622 	return dest_ring;
1623 }
1624 
1625 /*
1626  * Initialize a Copy Engine based on caller-supplied attributes.
1627  * This may be called once to initialize both source and destination
1628  * rings or it may be called twice for separate source and destination
1629  * initialization. It may be that only one side or the other is
1630  * initialized by software/firmware.
1631  */
1632 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1633 			const struct ce_attr *attr)
1634 {
1635 	int ret;
1636 
1637 	if (attr->src_nentries) {
1638 		ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1639 		if (ret) {
1640 			ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1641 				   ce_id, ret);
1642 			return ret;
1643 		}
1644 	}
1645 
1646 	if (attr->dest_nentries) {
1647 		ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1648 		if (ret) {
1649 			ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1650 				   ce_id, ret);
1651 			return ret;
1652 		}
1653 	}
1654 
1655 	return 0;
1656 }
1657 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1658 
1659 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1660 {
1661 	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1662 
1663 	ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0);
1664 	ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1665 	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1666 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1667 }
1668 
1669 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1670 {
1671 	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1672 
1673 	ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0);
1674 	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1675 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1676 }
1677 
1678 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1679 {
1680 	ath10k_ce_deinit_src_ring(ar, ce_id);
1681 	ath10k_ce_deinit_dest_ring(ar, ce_id);
1682 }
1683 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1684 
1685 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1686 {
1687 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1688 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1689 
1690 	if (ce_state->src_ring) {
1691 		if (ar->hw_params.shadow_reg_support)
1692 			kfree(ce_state->src_ring->shadow_base_unaligned);
1693 		dma_free_coherent(ar->dev,
1694 				  (ce_state->src_ring->nentries *
1695 				   sizeof(struct ce_desc) +
1696 				   CE_DESC_RING_ALIGN),
1697 				  ce_state->src_ring->base_addr_owner_space,
1698 				  ce_state->src_ring->base_addr_ce_space);
1699 		kfree(ce_state->src_ring);
1700 	}
1701 
1702 	if (ce_state->dest_ring) {
1703 		dma_free_coherent(ar->dev,
1704 				  (ce_state->dest_ring->nentries *
1705 				   sizeof(struct ce_desc) +
1706 				   CE_DESC_RING_ALIGN),
1707 				  ce_state->dest_ring->base_addr_owner_space,
1708 				  ce_state->dest_ring->base_addr_ce_space);
1709 		kfree(ce_state->dest_ring);
1710 	}
1711 
1712 	ce_state->src_ring = NULL;
1713 	ce_state->dest_ring = NULL;
1714 }
1715 
1716 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1717 {
1718 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1719 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1720 
1721 	if (ce_state->src_ring) {
1722 		if (ar->hw_params.shadow_reg_support)
1723 			kfree(ce_state->src_ring->shadow_base_unaligned);
1724 		dma_free_coherent(ar->dev,
1725 				  (ce_state->src_ring->nentries *
1726 				   sizeof(struct ce_desc_64) +
1727 				   CE_DESC_RING_ALIGN),
1728 				  ce_state->src_ring->base_addr_owner_space,
1729 				  ce_state->src_ring->base_addr_ce_space);
1730 		kfree(ce_state->src_ring);
1731 	}
1732 
1733 	if (ce_state->dest_ring) {
1734 		dma_free_coherent(ar->dev,
1735 				  (ce_state->dest_ring->nentries *
1736 				   sizeof(struct ce_desc_64) +
1737 				   CE_DESC_RING_ALIGN),
1738 				  ce_state->dest_ring->base_addr_owner_space,
1739 				  ce_state->dest_ring->base_addr_ce_space);
1740 		kfree(ce_state->dest_ring);
1741 	}
1742 
1743 	ce_state->src_ring = NULL;
1744 	ce_state->dest_ring = NULL;
1745 }
1746 
1747 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1748 {
1749 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1750 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1751 
1752 	ce_state->ops->ce_free_pipe(ar, ce_id);
1753 }
1754 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1755 
1756 void ath10k_ce_dump_registers(struct ath10k *ar,
1757 			      struct ath10k_fw_crash_data *crash_data)
1758 {
1759 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1760 	struct ath10k_ce_crash_data ce_data;
1761 	u32 addr, id;
1762 
1763 	lockdep_assert_held(&ar->data_lock);
1764 
1765 	ath10k_err(ar, "Copy Engine register dump:\n");
1766 
1767 	spin_lock_bh(&ce->ce_lock);
1768 	for (id = 0; id < CE_COUNT; id++) {
1769 		addr = ath10k_ce_base_address(ar, id);
1770 		ce_data.base_addr = cpu_to_le32(addr);
1771 
1772 		ce_data.src_wr_idx =
1773 			cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1774 		ce_data.src_r_idx =
1775 			cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1776 		ce_data.dst_wr_idx =
1777 			cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1778 		ce_data.dst_r_idx =
1779 			cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1780 
1781 		if (crash_data)
1782 			crash_data->ce_crash_data[id] = ce_data;
1783 
1784 		ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1785 			   le32_to_cpu(ce_data.base_addr),
1786 			   le32_to_cpu(ce_data.src_wr_idx),
1787 			   le32_to_cpu(ce_data.src_r_idx),
1788 			   le32_to_cpu(ce_data.dst_wr_idx),
1789 			   le32_to_cpu(ce_data.dst_r_idx));
1790 	}
1791 
1792 	spin_unlock_bh(&ce->ce_lock);
1793 }
1794 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1795 
1796 static const struct ath10k_ce_ops ce_ops = {
1797 	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1798 	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1799 	.ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1800 	.ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1801 	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1802 	.ce_extract_desc_data = ath10k_ce_extract_desc_data,
1803 	.ce_free_pipe = _ath10k_ce_free_pipe,
1804 	.ce_send_nolock = _ath10k_ce_send_nolock,
1805 };
1806 
1807 static const struct ath10k_ce_ops ce_64_ops = {
1808 	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1809 	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1810 	.ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1811 	.ce_completed_recv_next_nolock =
1812 				_ath10k_ce_completed_recv_next_nolock_64,
1813 	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1814 	.ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1815 	.ce_free_pipe = _ath10k_ce_free_pipe_64,
1816 	.ce_send_nolock = _ath10k_ce_send_nolock_64,
1817 };
1818 
1819 static void ath10k_ce_set_ops(struct ath10k *ar,
1820 			      struct ath10k_ce_pipe *ce_state)
1821 {
1822 	switch (ar->hw_rev) {
1823 	case ATH10K_HW_WCN3990:
1824 		ce_state->ops = &ce_64_ops;
1825 		break;
1826 	default:
1827 		ce_state->ops = &ce_ops;
1828 		break;
1829 	}
1830 }
1831 
1832 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1833 			 const struct ce_attr *attr)
1834 {
1835 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1836 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1837 	int ret;
1838 
1839 	ath10k_ce_set_ops(ar, ce_state);
1840 	/* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1841 	 * additional TX locking checks.
1842 	 *
1843 	 * For the lack of a better place do the check here.
1844 	 */
1845 	BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1846 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1847 	BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1848 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1849 	BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1850 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1851 
1852 	ce_state->ar = ar;
1853 	ce_state->id = ce_id;
1854 	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1855 	ce_state->attr_flags = attr->flags;
1856 	ce_state->src_sz_max = attr->src_sz_max;
1857 
1858 	if (attr->src_nentries)
1859 		ce_state->send_cb = attr->send_cb;
1860 
1861 	if (attr->dest_nentries)
1862 		ce_state->recv_cb = attr->recv_cb;
1863 
1864 	if (attr->src_nentries) {
1865 		ce_state->src_ring =
1866 			ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1867 		if (IS_ERR(ce_state->src_ring)) {
1868 			ret = PTR_ERR(ce_state->src_ring);
1869 			ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1870 				   ce_id, ret);
1871 			ce_state->src_ring = NULL;
1872 			return ret;
1873 		}
1874 	}
1875 
1876 	if (attr->dest_nentries) {
1877 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1878 									ce_id,
1879 									attr);
1880 		if (IS_ERR(ce_state->dest_ring)) {
1881 			ret = PTR_ERR(ce_state->dest_ring);
1882 			ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1883 				   ce_id, ret);
1884 			ce_state->dest_ring = NULL;
1885 			return ret;
1886 		}
1887 	}
1888 
1889 	return 0;
1890 }
1891 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1892 
1893 void ath10k_ce_alloc_rri(struct ath10k *ar)
1894 {
1895 	int i;
1896 	u32 value;
1897 	u32 ctrl1_regs;
1898 	u32 ce_base_addr;
1899 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1900 
1901 	ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1902 					   (CE_COUNT * sizeof(u32)),
1903 					   &ce->paddr_rri, GFP_KERNEL);
1904 
1905 	if (!ce->vaddr_rri)
1906 		return;
1907 
1908 	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1909 			  lower_32_bits(ce->paddr_rri));
1910 	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1911 			  (upper_32_bits(ce->paddr_rri) &
1912 			  CE_DESC_FLAGS_GET_MASK));
1913 
1914 	for (i = 0; i < CE_COUNT; i++) {
1915 		ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1916 		ce_base_addr = ath10k_ce_base_address(ar, i);
1917 		value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1918 		value |= ar->hw_ce_regs->upd->mask;
1919 		ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1920 	}
1921 
1922 	memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32));
1923 }
1924 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1925 
1926 void ath10k_ce_free_rri(struct ath10k *ar)
1927 {
1928 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1929 
1930 	dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1931 			  ce->vaddr_rri,
1932 			  ce->paddr_rri);
1933 }
1934 EXPORT_SYMBOL(ath10k_ce_free_rri);
1935