xref: /openbmc/linux/drivers/net/wireless/ath/ath10k/ce.c (revision 9fc093b7)
1 // SPDX-License-Identifier: ISC
2 /*
3  * Copyright (c) 2005-2011 Atheros Communications Inc.
4  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
5  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
6  */
7 
8 #include "hif.h"
9 #include "ce.h"
10 #include "debug.h"
11 
12 /*
13  * Support for Copy Engine hardware, which is mainly used for
14  * communication between Host and Target over a PCIe interconnect.
15  */
16 
17 /*
18  * A single CopyEngine (CE) comprises two "rings":
19  *   a source ring
20  *   a destination ring
21  *
22  * Each ring consists of a number of descriptors which specify
23  * an address, length, and meta-data.
24  *
25  * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
26  * controls one ring and the other side controls the other ring.
27  * The source side chooses when to initiate a transfer and it
28  * chooses what to send (buffer address, length). The destination
29  * side keeps a supply of "anonymous receive buffers" available and
30  * it handles incoming data as it arrives (when the destination
31  * receives an interrupt).
32  *
33  * The sender may send a simple buffer (address/length) or it may
34  * send a small list of buffers.  When a small list is sent, hardware
35  * "gathers" these and they end up in a single destination buffer
36  * with a single interrupt.
37  *
38  * There are several "contexts" managed by this layer -- more, it
39  * may seem -- than should be needed. These are provided mainly for
40  * maximum flexibility and especially to facilitate a simpler HIF
41  * implementation. There are per-CopyEngine recv, send, and watermark
42  * contexts. These are supplied by the caller when a recv, send,
43  * or watermark handler is established and they are echoed back to
44  * the caller when the respective callbacks are invoked. There is
45  * also a per-transfer context supplied by the caller when a buffer
46  * (or sendlist) is sent and when a buffer is enqueued for recv.
47  * These per-transfer contexts are echoed back to the caller when
48  * the buffer is sent/received.
49  */
50 
shadow_sr_wr_ind_addr(struct ath10k * ar,struct ath10k_ce_pipe * ce_state)51 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
52 					struct ath10k_ce_pipe *ce_state)
53 {
54 	u32 ce_id = ce_state->id;
55 	u32 addr = 0;
56 
57 	switch (ce_id) {
58 	case 0:
59 		addr = 0x00032000;
60 		break;
61 	case 3:
62 		addr = 0x0003200C;
63 		break;
64 	case 4:
65 		addr = 0x00032010;
66 		break;
67 	case 5:
68 		addr = 0x00032014;
69 		break;
70 	case 7:
71 		addr = 0x0003201C;
72 		break;
73 	default:
74 		ath10k_warn(ar, "invalid CE id: %d", ce_id);
75 		break;
76 	}
77 	return addr;
78 }
79 
80 static inline unsigned int
ath10k_set_ring_byte(unsigned int offset,struct ath10k_hw_ce_regs_addr_map * addr_map)81 ath10k_set_ring_byte(unsigned int offset,
82 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
83 {
84 	return ((offset << addr_map->lsb) & addr_map->mask);
85 }
86 
ath10k_ce_read32(struct ath10k * ar,u32 offset)87 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
88 {
89 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
90 
91 	return ce->bus_ops->read32(ar, offset);
92 }
93 
ath10k_ce_write32(struct ath10k * ar,u32 offset,u32 value)94 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
95 {
96 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
97 
98 	ce->bus_ops->write32(ar, offset, value);
99 }
100 
ath10k_ce_dest_ring_write_index_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)101 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
102 						       u32 ce_ctrl_addr,
103 						       unsigned int n)
104 {
105 	ath10k_ce_write32(ar, ce_ctrl_addr +
106 			  ar->hw_ce_regs->dst_wr_index_addr, n);
107 }
108 
ath10k_ce_dest_ring_write_index_get(struct ath10k * ar,u32 ce_ctrl_addr)109 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
110 						      u32 ce_ctrl_addr)
111 {
112 	return ath10k_ce_read32(ar, ce_ctrl_addr +
113 				ar->hw_ce_regs->dst_wr_index_addr);
114 }
115 
ath10k_ce_src_ring_write_index_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)116 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
117 						      u32 ce_ctrl_addr,
118 						      unsigned int n)
119 {
120 	ath10k_ce_write32(ar, ce_ctrl_addr +
121 			  ar->hw_ce_regs->sr_wr_index_addr, n);
122 }
123 
ath10k_ce_src_ring_write_index_get(struct ath10k * ar,u32 ce_ctrl_addr)124 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
125 						     u32 ce_ctrl_addr)
126 {
127 	return ath10k_ce_read32(ar, ce_ctrl_addr +
128 				ar->hw_ce_regs->sr_wr_index_addr);
129 }
130 
ath10k_ce_src_ring_read_index_from_ddr(struct ath10k * ar,u32 ce_id)131 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
132 							 u32 ce_id)
133 {
134 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
135 
136 	return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
137 }
138 
ath10k_ce_src_ring_read_index_get(struct ath10k * ar,u32 ce_ctrl_addr)139 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
140 						    u32 ce_ctrl_addr)
141 {
142 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
143 	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
144 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
145 	u32 index;
146 
147 	if (ar->hw_params.rri_on_ddr &&
148 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
149 		index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
150 	else
151 		index = ath10k_ce_read32(ar, ce_ctrl_addr +
152 					 ar->hw_ce_regs->current_srri_addr);
153 
154 	return index;
155 }
156 
157 static inline void
ath10k_ce_shadow_src_ring_write_index_set(struct ath10k * ar,struct ath10k_ce_pipe * ce_state,unsigned int value)158 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
159 					  struct ath10k_ce_pipe *ce_state,
160 					  unsigned int value)
161 {
162 	ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
163 }
164 
ath10k_ce_src_ring_base_addr_set(struct ath10k * ar,u32 ce_id,u64 addr)165 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
166 						    u32 ce_id,
167 						    u64 addr)
168 {
169 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
170 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
171 	u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
172 	u32 addr_lo = lower_32_bits(addr);
173 
174 	ath10k_ce_write32(ar, ce_ctrl_addr +
175 			  ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
176 
177 	if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
178 		ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
179 							    addr);
180 	}
181 }
182 
ath10k_ce_set_src_ring_base_addr_hi(struct ath10k * ar,u32 ce_ctrl_addr,u64 addr)183 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
184 						u32 ce_ctrl_addr,
185 						u64 addr)
186 {
187 	u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
188 
189 	ath10k_ce_write32(ar, ce_ctrl_addr +
190 			  ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
191 }
192 
ath10k_ce_src_ring_size_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)193 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
194 					       u32 ce_ctrl_addr,
195 					       unsigned int n)
196 {
197 	ath10k_ce_write32(ar, ce_ctrl_addr +
198 			  ar->hw_ce_regs->sr_size_addr, n);
199 }
200 
ath10k_ce_src_ring_dmax_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)201 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
202 					       u32 ce_ctrl_addr,
203 					       unsigned int n)
204 {
205 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
206 
207 	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
208 					  ctrl_regs->addr);
209 
210 	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
211 			  (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
212 			  ath10k_set_ring_byte(n, ctrl_regs->dmax));
213 }
214 
ath10k_ce_src_ring_byte_swap_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)215 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
216 						    u32 ce_ctrl_addr,
217 						    unsigned int n)
218 {
219 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
220 
221 	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
222 					  ctrl_regs->addr);
223 
224 	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
225 			  (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
226 			  ath10k_set_ring_byte(n, ctrl_regs->src_ring));
227 }
228 
ath10k_ce_dest_ring_byte_swap_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)229 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
230 						     u32 ce_ctrl_addr,
231 						     unsigned int n)
232 {
233 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
234 
235 	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
236 					  ctrl_regs->addr);
237 
238 	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
239 			  (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
240 			  ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
241 }
242 
243 static inline
ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k * ar,u32 ce_id)244 	u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
245 {
246 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
247 
248 	return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
249 		CE_DDR_RRI_MASK;
250 }
251 
ath10k_ce_dest_ring_read_index_get(struct ath10k * ar,u32 ce_ctrl_addr)252 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
253 						     u32 ce_ctrl_addr)
254 {
255 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
256 	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
257 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
258 	u32 index;
259 
260 	if (ar->hw_params.rri_on_ddr &&
261 	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
262 		index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
263 	else
264 		index = ath10k_ce_read32(ar, ce_ctrl_addr +
265 					 ar->hw_ce_regs->current_drri_addr);
266 
267 	return index;
268 }
269 
ath10k_ce_dest_ring_base_addr_set(struct ath10k * ar,u32 ce_id,u64 addr)270 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
271 						     u32 ce_id,
272 						     u64 addr)
273 {
274 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
275 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
276 	u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
277 	u32 addr_lo = lower_32_bits(addr);
278 
279 	ath10k_ce_write32(ar, ce_ctrl_addr +
280 			  ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
281 
282 	if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
283 		ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
284 							     addr);
285 	}
286 }
287 
ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k * ar,u32 ce_ctrl_addr,u64 addr)288 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
289 						 u32 ce_ctrl_addr,
290 						 u64 addr)
291 {
292 	u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
293 	u32 reg_value;
294 
295 	reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
296 				     ar->hw_ce_regs->dr_base_addr_hi);
297 	reg_value &= ~CE_DESC_ADDR_HI_MASK;
298 	reg_value |= addr_hi;
299 	ath10k_ce_write32(ar, ce_ctrl_addr +
300 			  ar->hw_ce_regs->dr_base_addr_hi, reg_value);
301 }
302 
ath10k_ce_dest_ring_size_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)303 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
304 						u32 ce_ctrl_addr,
305 						unsigned int n)
306 {
307 	ath10k_ce_write32(ar, ce_ctrl_addr +
308 			  ar->hw_ce_regs->dr_size_addr, n);
309 }
310 
ath10k_ce_src_ring_highmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)311 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
312 						   u32 ce_ctrl_addr,
313 						   unsigned int n)
314 {
315 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
316 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
317 
318 	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
319 			  (addr & ~(srcr_wm->wm_high->mask)) |
320 			  (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
321 }
322 
ath10k_ce_src_ring_lowmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)323 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
324 						  u32 ce_ctrl_addr,
325 						  unsigned int n)
326 {
327 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
328 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
329 
330 	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
331 			  (addr & ~(srcr_wm->wm_low->mask)) |
332 			  (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
333 }
334 
ath10k_ce_dest_ring_highmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)335 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
336 						    u32 ce_ctrl_addr,
337 						    unsigned int n)
338 {
339 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
340 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
341 
342 	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
343 			  (addr & ~(dstr_wm->wm_high->mask)) |
344 			  (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
345 }
346 
ath10k_ce_dest_ring_lowmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)347 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
348 						   u32 ce_ctrl_addr,
349 						   unsigned int n)
350 {
351 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
352 	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
353 
354 	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
355 			  (addr & ~(dstr_wm->wm_low->mask)) |
356 			  (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
357 }
358 
ath10k_ce_copy_complete_inter_enable(struct ath10k * ar,u32 ce_ctrl_addr)359 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
360 							u32 ce_ctrl_addr)
361 {
362 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
363 
364 	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
365 					    ar->hw_ce_regs->host_ie_addr);
366 
367 	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
368 			  host_ie_addr | host_ie->copy_complete->mask);
369 }
370 
ath10k_ce_copy_complete_intr_disable(struct ath10k * ar,u32 ce_ctrl_addr)371 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
372 							u32 ce_ctrl_addr)
373 {
374 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
375 
376 	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
377 					    ar->hw_ce_regs->host_ie_addr);
378 
379 	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
380 			  host_ie_addr & ~(host_ie->copy_complete->mask));
381 }
382 
ath10k_ce_watermark_intr_disable(struct ath10k * ar,u32 ce_ctrl_addr)383 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
384 						    u32 ce_ctrl_addr)
385 {
386 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
387 
388 	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
389 					    ar->hw_ce_regs->host_ie_addr);
390 
391 	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
392 			  host_ie_addr & ~(wm_regs->wm_mask));
393 }
394 
ath10k_ce_error_intr_disable(struct ath10k * ar,u32 ce_ctrl_addr)395 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
396 						u32 ce_ctrl_addr)
397 {
398 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
399 
400 	u32 misc_ie_addr = ath10k_ce_read32(ar,
401 			ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
402 
403 	ath10k_ce_write32(ar,
404 			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
405 			  misc_ie_addr & ~(misc_regs->err_mask));
406 }
407 
ath10k_ce_engine_int_status_clear(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int mask)408 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
409 						     u32 ce_ctrl_addr,
410 						     unsigned int mask)
411 {
412 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
413 
414 	ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
415 }
416 
417 /*
418  * Guts of ath10k_ce_send.
419  * The caller takes responsibility for any needed locking.
420  */
_ath10k_ce_send_nolock(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)421 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
422 				  void *per_transfer_context,
423 				  dma_addr_t buffer,
424 				  unsigned int nbytes,
425 				  unsigned int transfer_id,
426 				  unsigned int flags)
427 {
428 	struct ath10k *ar = ce_state->ar;
429 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
430 	struct ce_desc *desc, sdesc;
431 	unsigned int nentries_mask = src_ring->nentries_mask;
432 	unsigned int sw_index = src_ring->sw_index;
433 	unsigned int write_index = src_ring->write_index;
434 	u32 ctrl_addr = ce_state->ctrl_addr;
435 	u32 desc_flags = 0;
436 	int ret = 0;
437 
438 	if (nbytes > ce_state->src_sz_max)
439 		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
440 			    __func__, nbytes, ce_state->src_sz_max);
441 
442 	if (unlikely(CE_RING_DELTA(nentries_mask,
443 				   write_index, sw_index - 1) <= 0)) {
444 		ret = -ENOSR;
445 		goto exit;
446 	}
447 
448 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
449 				   write_index);
450 
451 	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
452 
453 	if (flags & CE_SEND_FLAG_GATHER)
454 		desc_flags |= CE_DESC_FLAGS_GATHER;
455 	if (flags & CE_SEND_FLAG_BYTE_SWAP)
456 		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
457 
458 	sdesc.addr   = __cpu_to_le32(buffer);
459 	sdesc.nbytes = __cpu_to_le16(nbytes);
460 	sdesc.flags  = __cpu_to_le16(desc_flags);
461 
462 	*desc = sdesc;
463 
464 	src_ring->per_transfer_context[write_index] = per_transfer_context;
465 
466 	/* Update Source Ring Write Index */
467 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
468 
469 	/* WORKAROUND */
470 	if (!(flags & CE_SEND_FLAG_GATHER))
471 		ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
472 
473 	src_ring->write_index = write_index;
474 exit:
475 	return ret;
476 }
477 
_ath10k_ce_send_nolock_64(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)478 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
479 				     void *per_transfer_context,
480 				     dma_addr_t buffer,
481 				     unsigned int nbytes,
482 				     unsigned int transfer_id,
483 				     unsigned int flags)
484 {
485 	struct ath10k *ar = ce_state->ar;
486 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
487 	struct ce_desc_64 *desc, sdesc;
488 	unsigned int nentries_mask = src_ring->nentries_mask;
489 	unsigned int sw_index;
490 	unsigned int write_index = src_ring->write_index;
491 	u32 ctrl_addr = ce_state->ctrl_addr;
492 	__le32 *addr;
493 	u32 desc_flags = 0;
494 	int ret = 0;
495 
496 	if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
497 		return -ESHUTDOWN;
498 
499 	if (nbytes > ce_state->src_sz_max)
500 		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
501 			    __func__, nbytes, ce_state->src_sz_max);
502 
503 	if (ar->hw_params.rri_on_ddr)
504 		sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
505 	else
506 		sw_index = src_ring->sw_index;
507 
508 	if (unlikely(CE_RING_DELTA(nentries_mask,
509 				   write_index, sw_index - 1) <= 0)) {
510 		ret = -ENOSR;
511 		goto exit;
512 	}
513 
514 	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
515 				      write_index);
516 
517 	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
518 
519 	if (flags & CE_SEND_FLAG_GATHER)
520 		desc_flags |= CE_DESC_FLAGS_GATHER;
521 
522 	if (flags & CE_SEND_FLAG_BYTE_SWAP)
523 		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
524 
525 	addr = (__le32 *)&sdesc.addr;
526 
527 	flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
528 	addr[0] = __cpu_to_le32(buffer);
529 	addr[1] = __cpu_to_le32(flags);
530 	if (flags & CE_SEND_FLAG_GATHER)
531 		addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
532 	else
533 		addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
534 
535 	sdesc.nbytes = __cpu_to_le16(nbytes);
536 	sdesc.flags  = __cpu_to_le16(desc_flags);
537 
538 	*desc = sdesc;
539 
540 	src_ring->per_transfer_context[write_index] = per_transfer_context;
541 
542 	/* Update Source Ring Write Index */
543 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
544 
545 	if (!(flags & CE_SEND_FLAG_GATHER)) {
546 		if (ar->hw_params.shadow_reg_support)
547 			ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
548 								  write_index);
549 		else
550 			ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
551 							   write_index);
552 	}
553 
554 	src_ring->write_index = write_index;
555 exit:
556 	return ret;
557 }
558 
ath10k_ce_send_nolock(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)559 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
560 			  void *per_transfer_context,
561 			  dma_addr_t buffer,
562 			  unsigned int nbytes,
563 			  unsigned int transfer_id,
564 			  unsigned int flags)
565 {
566 	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
567 				    buffer, nbytes, transfer_id, flags);
568 }
569 EXPORT_SYMBOL(ath10k_ce_send_nolock);
570 
__ath10k_ce_send_revert(struct ath10k_ce_pipe * pipe)571 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
572 {
573 	struct ath10k *ar = pipe->ar;
574 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
575 	struct ath10k_ce_ring *src_ring = pipe->src_ring;
576 	u32 ctrl_addr = pipe->ctrl_addr;
577 
578 	lockdep_assert_held(&ce->ce_lock);
579 
580 	/*
581 	 * This function must be called only if there is an incomplete
582 	 * scatter-gather transfer (before index register is updated)
583 	 * that needs to be cleaned up.
584 	 */
585 	if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
586 		return;
587 
588 	if (WARN_ON_ONCE(src_ring->write_index ==
589 			 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
590 		return;
591 
592 	src_ring->write_index--;
593 	src_ring->write_index &= src_ring->nentries_mask;
594 
595 	src_ring->per_transfer_context[src_ring->write_index] = NULL;
596 }
597 EXPORT_SYMBOL(__ath10k_ce_send_revert);
598 
ath10k_ce_send(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)599 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
600 		   void *per_transfer_context,
601 		   dma_addr_t buffer,
602 		   unsigned int nbytes,
603 		   unsigned int transfer_id,
604 		   unsigned int flags)
605 {
606 	struct ath10k *ar = ce_state->ar;
607 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
608 	int ret;
609 
610 	spin_lock_bh(&ce->ce_lock);
611 	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
612 				    buffer, nbytes, transfer_id, flags);
613 	spin_unlock_bh(&ce->ce_lock);
614 
615 	return ret;
616 }
617 EXPORT_SYMBOL(ath10k_ce_send);
618 
ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe * pipe)619 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
620 {
621 	struct ath10k *ar = pipe->ar;
622 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
623 	int delta;
624 
625 	spin_lock_bh(&ce->ce_lock);
626 	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
627 			      pipe->src_ring->write_index,
628 			      pipe->src_ring->sw_index - 1);
629 	spin_unlock_bh(&ce->ce_lock);
630 
631 	return delta;
632 }
633 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
634 
__ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe * pipe)635 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
636 {
637 	struct ath10k *ar = pipe->ar;
638 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
639 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
640 	unsigned int nentries_mask = dest_ring->nentries_mask;
641 	unsigned int write_index = dest_ring->write_index;
642 	unsigned int sw_index = dest_ring->sw_index;
643 
644 	lockdep_assert_held(&ce->ce_lock);
645 
646 	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
647 }
648 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
649 
__ath10k_ce_rx_post_buf(struct ath10k_ce_pipe * pipe,void * ctx,dma_addr_t paddr)650 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
651 				   dma_addr_t paddr)
652 {
653 	struct ath10k *ar = pipe->ar;
654 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
655 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
656 	unsigned int nentries_mask = dest_ring->nentries_mask;
657 	unsigned int write_index = dest_ring->write_index;
658 	unsigned int sw_index = dest_ring->sw_index;
659 	struct ce_desc *base = dest_ring->base_addr_owner_space;
660 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
661 	u32 ctrl_addr = pipe->ctrl_addr;
662 
663 	lockdep_assert_held(&ce->ce_lock);
664 
665 	if ((pipe->id != 5) &&
666 	    CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
667 		return -ENOSPC;
668 
669 	desc->addr = __cpu_to_le32(paddr);
670 	desc->nbytes = 0;
671 
672 	dest_ring->per_transfer_context[write_index] = ctx;
673 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
674 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
675 	dest_ring->write_index = write_index;
676 
677 	return 0;
678 }
679 
__ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe * pipe,void * ctx,dma_addr_t paddr)680 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
681 				      void *ctx,
682 				      dma_addr_t paddr)
683 {
684 	struct ath10k *ar = pipe->ar;
685 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
686 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
687 	unsigned int nentries_mask = dest_ring->nentries_mask;
688 	unsigned int write_index = dest_ring->write_index;
689 	unsigned int sw_index = dest_ring->sw_index;
690 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
691 	struct ce_desc_64 *desc =
692 			CE_DEST_RING_TO_DESC_64(base, write_index);
693 	u32 ctrl_addr = pipe->ctrl_addr;
694 
695 	lockdep_assert_held(&ce->ce_lock);
696 
697 	if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
698 		return -ENOSPC;
699 
700 	desc->addr = __cpu_to_le64(paddr);
701 	desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
702 
703 	desc->nbytes = 0;
704 
705 	dest_ring->per_transfer_context[write_index] = ctx;
706 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
707 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
708 	dest_ring->write_index = write_index;
709 
710 	return 0;
711 }
712 
ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe * pipe,u32 nentries)713 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
714 {
715 	struct ath10k *ar = pipe->ar;
716 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
717 	unsigned int nentries_mask = dest_ring->nentries_mask;
718 	unsigned int write_index = dest_ring->write_index;
719 	u32 ctrl_addr = pipe->ctrl_addr;
720 	u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
721 
722 	/* Prevent CE ring stuck issue that will occur when ring is full.
723 	 * Make sure that write index is 1 less than read index.
724 	 */
725 	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
726 		nentries -= 1;
727 
728 	write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
729 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
730 	dest_ring->write_index = write_index;
731 }
732 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
733 
ath10k_ce_rx_post_buf(struct ath10k_ce_pipe * pipe,void * ctx,dma_addr_t paddr)734 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
735 			  dma_addr_t paddr)
736 {
737 	struct ath10k *ar = pipe->ar;
738 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
739 	int ret;
740 
741 	spin_lock_bh(&ce->ce_lock);
742 	ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
743 	spin_unlock_bh(&ce->ce_lock);
744 
745 	return ret;
746 }
747 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
748 
749 /*
750  * Guts of ath10k_ce_completed_recv_next.
751  * The caller takes responsibility for any necessary locking.
752  */
753 static int
_ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,unsigned int * nbytesp)754 	 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
755 					       void **per_transfer_contextp,
756 					       unsigned int *nbytesp)
757 {
758 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
759 	unsigned int nentries_mask = dest_ring->nentries_mask;
760 	unsigned int sw_index = dest_ring->sw_index;
761 
762 	struct ce_desc *base = dest_ring->base_addr_owner_space;
763 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
764 	struct ce_desc sdesc;
765 	u16 nbytes;
766 
767 	/* Copy in one go for performance reasons */
768 	sdesc = *desc;
769 
770 	nbytes = __le16_to_cpu(sdesc.nbytes);
771 	if (nbytes == 0) {
772 		/*
773 		 * This closes a relatively unusual race where the Host
774 		 * sees the updated DRRI before the update to the
775 		 * corresponding descriptor has completed. We treat this
776 		 * as a descriptor that is not yet done.
777 		 */
778 		return -EIO;
779 	}
780 
781 	desc->nbytes = 0;
782 
783 	/* Return data from completed destination descriptor */
784 	*nbytesp = nbytes;
785 
786 	if (per_transfer_contextp)
787 		*per_transfer_contextp =
788 			dest_ring->per_transfer_context[sw_index];
789 
790 	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
791 	 * So update transfer context all CEs except CE5.
792 	 */
793 	if (ce_state->id != 5)
794 		dest_ring->per_transfer_context[sw_index] = NULL;
795 
796 	/* Update sw_index */
797 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
798 	dest_ring->sw_index = sw_index;
799 
800 	return 0;
801 }
802 
803 static int
_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,unsigned int * nbytesp)804 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
805 					 void **per_transfer_contextp,
806 					 unsigned int *nbytesp)
807 {
808 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
809 	unsigned int nentries_mask = dest_ring->nentries_mask;
810 	unsigned int sw_index = dest_ring->sw_index;
811 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
812 	struct ce_desc_64 *desc =
813 		CE_DEST_RING_TO_DESC_64(base, sw_index);
814 	struct ce_desc_64 sdesc;
815 	u16 nbytes;
816 
817 	/* Copy in one go for performance reasons */
818 	sdesc = *desc;
819 
820 	nbytes = __le16_to_cpu(sdesc.nbytes);
821 	if (nbytes == 0) {
822 		/* This closes a relatively unusual race where the Host
823 		 * sees the updated DRRI before the update to the
824 		 * corresponding descriptor has completed. We treat this
825 		 * as a descriptor that is not yet done.
826 		 */
827 		return -EIO;
828 	}
829 
830 	desc->nbytes = 0;
831 
832 	/* Return data from completed destination descriptor */
833 	*nbytesp = nbytes;
834 
835 	if (per_transfer_contextp)
836 		*per_transfer_contextp =
837 			dest_ring->per_transfer_context[sw_index];
838 
839 	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
840 	 * So update transfer context all CEs except CE5.
841 	 */
842 	if (ce_state->id != 5)
843 		dest_ring->per_transfer_context[sw_index] = NULL;
844 
845 	/* Update sw_index */
846 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
847 	dest_ring->sw_index = sw_index;
848 
849 	return 0;
850 }
851 
ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_ctx,unsigned int * nbytesp)852 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
853 					 void **per_transfer_ctx,
854 					 unsigned int *nbytesp)
855 {
856 	return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
857 							    per_transfer_ctx,
858 							    nbytesp);
859 }
860 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
861 
ath10k_ce_completed_recv_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,unsigned int * nbytesp)862 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
863 				  void **per_transfer_contextp,
864 				  unsigned int *nbytesp)
865 {
866 	struct ath10k *ar = ce_state->ar;
867 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
868 	int ret;
869 
870 	spin_lock_bh(&ce->ce_lock);
871 	ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
872 						   per_transfer_contextp,
873 						   nbytesp);
874 
875 	spin_unlock_bh(&ce->ce_lock);
876 
877 	return ret;
878 }
879 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
880 
_ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp)881 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
882 				       void **per_transfer_contextp,
883 				       dma_addr_t *bufferp)
884 {
885 	struct ath10k_ce_ring *dest_ring;
886 	unsigned int nentries_mask;
887 	unsigned int sw_index;
888 	unsigned int write_index;
889 	int ret;
890 	struct ath10k *ar;
891 	struct ath10k_ce *ce;
892 
893 	dest_ring = ce_state->dest_ring;
894 
895 	if (!dest_ring)
896 		return -EIO;
897 
898 	ar = ce_state->ar;
899 	ce = ath10k_ce_priv(ar);
900 
901 	spin_lock_bh(&ce->ce_lock);
902 
903 	nentries_mask = dest_ring->nentries_mask;
904 	sw_index = dest_ring->sw_index;
905 	write_index = dest_ring->write_index;
906 	if (write_index != sw_index) {
907 		struct ce_desc *base = dest_ring->base_addr_owner_space;
908 		struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
909 
910 		/* Return data from completed destination descriptor */
911 		*bufferp = __le32_to_cpu(desc->addr);
912 
913 		if (per_transfer_contextp)
914 			*per_transfer_contextp =
915 				dest_ring->per_transfer_context[sw_index];
916 
917 		/* sanity */
918 		dest_ring->per_transfer_context[sw_index] = NULL;
919 		desc->nbytes = 0;
920 
921 		/* Update sw_index */
922 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
923 		dest_ring->sw_index = sw_index;
924 		ret = 0;
925 	} else {
926 		ret = -EIO;
927 	}
928 
929 	spin_unlock_bh(&ce->ce_lock);
930 
931 	return ret;
932 }
933 
_ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp)934 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
935 					  void **per_transfer_contextp,
936 					  dma_addr_t *bufferp)
937 {
938 	struct ath10k_ce_ring *dest_ring;
939 	unsigned int nentries_mask;
940 	unsigned int sw_index;
941 	unsigned int write_index;
942 	int ret;
943 	struct ath10k *ar;
944 	struct ath10k_ce *ce;
945 
946 	dest_ring = ce_state->dest_ring;
947 
948 	if (!dest_ring)
949 		return -EIO;
950 
951 	ar = ce_state->ar;
952 	ce = ath10k_ce_priv(ar);
953 
954 	spin_lock_bh(&ce->ce_lock);
955 
956 	nentries_mask = dest_ring->nentries_mask;
957 	sw_index = dest_ring->sw_index;
958 	write_index = dest_ring->write_index;
959 	if (write_index != sw_index) {
960 		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
961 		struct ce_desc_64 *desc =
962 			CE_DEST_RING_TO_DESC_64(base, sw_index);
963 
964 		/* Return data from completed destination descriptor */
965 		*bufferp = __le64_to_cpu(desc->addr);
966 
967 		if (per_transfer_contextp)
968 			*per_transfer_contextp =
969 				dest_ring->per_transfer_context[sw_index];
970 
971 		/* sanity */
972 		dest_ring->per_transfer_context[sw_index] = NULL;
973 		desc->nbytes = 0;
974 
975 		/* Update sw_index */
976 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
977 		dest_ring->sw_index = sw_index;
978 		ret = 0;
979 	} else {
980 		ret = -EIO;
981 	}
982 
983 	spin_unlock_bh(&ce->ce_lock);
984 
985 	return ret;
986 }
987 
ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp)988 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
989 			       void **per_transfer_contextp,
990 			       dma_addr_t *bufferp)
991 {
992 	return ce_state->ops->ce_revoke_recv_next(ce_state,
993 						  per_transfer_contextp,
994 						  bufferp);
995 }
996 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
997 
998 /*
999  * Guts of ath10k_ce_completed_send_next.
1000  * The caller takes responsibility for any necessary locking.
1001  */
_ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1002 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1003 						 void **per_transfer_contextp)
1004 {
1005 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1006 	u32 ctrl_addr = ce_state->ctrl_addr;
1007 	struct ath10k *ar = ce_state->ar;
1008 	unsigned int nentries_mask = src_ring->nentries_mask;
1009 	unsigned int sw_index = src_ring->sw_index;
1010 	unsigned int read_index;
1011 	struct ce_desc *desc;
1012 
1013 	if (src_ring->hw_index == sw_index) {
1014 		/*
1015 		 * The SW completion index has caught up with the cached
1016 		 * version of the HW completion index.
1017 		 * Update the cached HW completion index to see whether
1018 		 * the SW has really caught up to the HW, or if the cached
1019 		 * value of the HW index has become stale.
1020 		 */
1021 
1022 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1023 		if (read_index == 0xffffffff)
1024 			return -ENODEV;
1025 
1026 		read_index &= nentries_mask;
1027 		src_ring->hw_index = read_index;
1028 	}
1029 
1030 	if (ar->hw_params.rri_on_ddr)
1031 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1032 	else
1033 		read_index = src_ring->hw_index;
1034 
1035 	if (read_index == sw_index)
1036 		return -EIO;
1037 
1038 	if (per_transfer_contextp)
1039 		*per_transfer_contextp =
1040 			src_ring->per_transfer_context[sw_index];
1041 
1042 	/* sanity */
1043 	src_ring->per_transfer_context[sw_index] = NULL;
1044 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1045 				   sw_index);
1046 	desc->nbytes = 0;
1047 
1048 	/* Update sw_index */
1049 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1050 	src_ring->sw_index = sw_index;
1051 
1052 	return 0;
1053 }
1054 
_ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1055 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1056 						    void **per_transfer_contextp)
1057 {
1058 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1059 	u32 ctrl_addr = ce_state->ctrl_addr;
1060 	struct ath10k *ar = ce_state->ar;
1061 	unsigned int nentries_mask = src_ring->nentries_mask;
1062 	unsigned int sw_index = src_ring->sw_index;
1063 	unsigned int read_index;
1064 	struct ce_desc_64 *desc;
1065 
1066 	if (src_ring->hw_index == sw_index) {
1067 		/*
1068 		 * The SW completion index has caught up with the cached
1069 		 * version of the HW completion index.
1070 		 * Update the cached HW completion index to see whether
1071 		 * the SW has really caught up to the HW, or if the cached
1072 		 * value of the HW index has become stale.
1073 		 */
1074 
1075 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1076 		if (read_index == 0xffffffff)
1077 			return -ENODEV;
1078 
1079 		read_index &= nentries_mask;
1080 		src_ring->hw_index = read_index;
1081 	}
1082 
1083 	if (ar->hw_params.rri_on_ddr)
1084 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1085 	else
1086 		read_index = src_ring->hw_index;
1087 
1088 	if (read_index == sw_index)
1089 		return -EIO;
1090 
1091 	if (per_transfer_contextp)
1092 		*per_transfer_contextp =
1093 			src_ring->per_transfer_context[sw_index];
1094 
1095 	/* sanity */
1096 	src_ring->per_transfer_context[sw_index] = NULL;
1097 	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1098 				      sw_index);
1099 	desc->nbytes = 0;
1100 
1101 	/* Update sw_index */
1102 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1103 	src_ring->sw_index = sw_index;
1104 
1105 	return 0;
1106 }
1107 
ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1108 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1109 					 void **per_transfer_contextp)
1110 {
1111 	return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1112 							    per_transfer_contextp);
1113 }
1114 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1115 
ath10k_ce_extract_desc_data(struct ath10k * ar,struct ath10k_ce_ring * src_ring,u32 sw_index,dma_addr_t * bufferp,u32 * nbytesp,u32 * transfer_idp)1116 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1117 					struct ath10k_ce_ring *src_ring,
1118 					u32 sw_index,
1119 					dma_addr_t *bufferp,
1120 					u32 *nbytesp,
1121 					u32 *transfer_idp)
1122 {
1123 		struct ce_desc *base = src_ring->base_addr_owner_space;
1124 		struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1125 
1126 		/* Return data from completed source descriptor */
1127 		*bufferp = __le32_to_cpu(desc->addr);
1128 		*nbytesp = __le16_to_cpu(desc->nbytes);
1129 		*transfer_idp = MS(__le16_to_cpu(desc->flags),
1130 				   CE_DESC_FLAGS_META_DATA);
1131 }
1132 
ath10k_ce_extract_desc_data_64(struct ath10k * ar,struct ath10k_ce_ring * src_ring,u32 sw_index,dma_addr_t * bufferp,u32 * nbytesp,u32 * transfer_idp)1133 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1134 					   struct ath10k_ce_ring *src_ring,
1135 					   u32 sw_index,
1136 					   dma_addr_t *bufferp,
1137 					   u32 *nbytesp,
1138 					   u32 *transfer_idp)
1139 {
1140 		struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1141 		struct ce_desc_64 *desc =
1142 			CE_SRC_RING_TO_DESC_64(base, sw_index);
1143 
1144 		/* Return data from completed source descriptor */
1145 		*bufferp = __le64_to_cpu(desc->addr);
1146 		*nbytesp = __le16_to_cpu(desc->nbytes);
1147 		*transfer_idp = MS(__le16_to_cpu(desc->flags),
1148 				   CE_DESC_FLAGS_META_DATA);
1149 }
1150 
1151 /* NB: Modeled after ath10k_ce_completed_send_next */
ath10k_ce_cancel_send_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp,unsigned int * nbytesp,unsigned int * transfer_idp)1152 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1153 			       void **per_transfer_contextp,
1154 			       dma_addr_t *bufferp,
1155 			       unsigned int *nbytesp,
1156 			       unsigned int *transfer_idp)
1157 {
1158 	struct ath10k_ce_ring *src_ring;
1159 	unsigned int nentries_mask;
1160 	unsigned int sw_index;
1161 	unsigned int write_index;
1162 	int ret;
1163 	struct ath10k *ar;
1164 	struct ath10k_ce *ce;
1165 
1166 	src_ring = ce_state->src_ring;
1167 
1168 	if (!src_ring)
1169 		return -EIO;
1170 
1171 	ar = ce_state->ar;
1172 	ce = ath10k_ce_priv(ar);
1173 
1174 	spin_lock_bh(&ce->ce_lock);
1175 
1176 	nentries_mask = src_ring->nentries_mask;
1177 	sw_index = src_ring->sw_index;
1178 	write_index = src_ring->write_index;
1179 
1180 	if (write_index != sw_index) {
1181 		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1182 						    bufferp, nbytesp,
1183 						    transfer_idp);
1184 
1185 		if (per_transfer_contextp)
1186 			*per_transfer_contextp =
1187 				src_ring->per_transfer_context[sw_index];
1188 
1189 		/* sanity */
1190 		src_ring->per_transfer_context[sw_index] = NULL;
1191 
1192 		/* Update sw_index */
1193 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1194 		src_ring->sw_index = sw_index;
1195 		ret = 0;
1196 	} else {
1197 		ret = -EIO;
1198 	}
1199 
1200 	spin_unlock_bh(&ce->ce_lock);
1201 
1202 	return ret;
1203 }
1204 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1205 
ath10k_ce_completed_send_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1206 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1207 				  void **per_transfer_contextp)
1208 {
1209 	struct ath10k *ar = ce_state->ar;
1210 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1211 	int ret;
1212 
1213 	spin_lock_bh(&ce->ce_lock);
1214 	ret = ath10k_ce_completed_send_next_nolock(ce_state,
1215 						   per_transfer_contextp);
1216 	spin_unlock_bh(&ce->ce_lock);
1217 
1218 	return ret;
1219 }
1220 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1221 
1222 /*
1223  * Guts of interrupt handler for per-engine interrupts on a particular CE.
1224  *
1225  * Invokes registered callbacks for recv_complete,
1226  * send_complete, and watermarks.
1227  */
ath10k_ce_per_engine_service(struct ath10k * ar,unsigned int ce_id)1228 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1229 {
1230 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1231 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1232 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1233 	u32 ctrl_addr = ce_state->ctrl_addr;
1234 
1235 	/*
1236 	 * Clear before handling
1237 	 *
1238 	 * Misc CE interrupts are not being handled, but still need
1239 	 * to be cleared.
1240 	 *
1241 	 * NOTE: When the last copy engine interrupt is cleared the
1242 	 * hardware will go to sleep.  Once this happens any access to
1243 	 * the CE registers can cause a hardware fault.
1244 	 */
1245 	ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1246 					  wm_regs->cc_mask | wm_regs->wm_mask);
1247 
1248 	if (ce_state->recv_cb)
1249 		ce_state->recv_cb(ce_state);
1250 
1251 	if (ce_state->send_cb)
1252 		ce_state->send_cb(ce_state);
1253 }
1254 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1255 
1256 /*
1257  * Handler for per-engine interrupts on ALL active CEs.
1258  * This is used in cases where the system is sharing a
1259  * single interrupt for all CEs
1260  */
1261 
ath10k_ce_per_engine_service_any(struct ath10k * ar)1262 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1263 {
1264 	int ce_id;
1265 	u32 intr_summary;
1266 
1267 	intr_summary = ath10k_ce_interrupt_summary(ar);
1268 
1269 	for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1270 		if (intr_summary & (1 << ce_id))
1271 			intr_summary &= ~(1 << ce_id);
1272 		else
1273 			/* no intr pending on this CE */
1274 			continue;
1275 
1276 		ath10k_ce_per_engine_service(ar, ce_id);
1277 	}
1278 }
1279 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1280 
1281 /*
1282  * Adjust interrupts for the copy complete handler.
1283  * If it's needed for either send or recv, then unmask
1284  * this interrupt; otherwise, mask it.
1285  *
1286  * Called with ce_lock held.
1287  */
ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe * ce_state)1288 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1289 {
1290 	u32 ctrl_addr = ce_state->ctrl_addr;
1291 	struct ath10k *ar = ce_state->ar;
1292 	bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1293 
1294 	if ((!disable_copy_compl_intr) &&
1295 	    (ce_state->send_cb || ce_state->recv_cb))
1296 		ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1297 	else
1298 		ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1299 
1300 	ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1301 }
1302 
ath10k_ce_disable_interrupt(struct ath10k * ar,int ce_id)1303 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1304 {
1305 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1306 	struct ath10k_ce_pipe *ce_state;
1307 	u32 ctrl_addr;
1308 
1309 	ce_state  = &ce->ce_states[ce_id];
1310 	if (ce_state->attr_flags & CE_ATTR_POLL)
1311 		return;
1312 
1313 	ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1314 
1315 	ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1316 	ath10k_ce_error_intr_disable(ar, ctrl_addr);
1317 	ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1318 }
1319 EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1320 
ath10k_ce_disable_interrupts(struct ath10k * ar)1321 void ath10k_ce_disable_interrupts(struct ath10k *ar)
1322 {
1323 	int ce_id;
1324 
1325 	for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1326 		ath10k_ce_disable_interrupt(ar, ce_id);
1327 }
1328 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1329 
ath10k_ce_enable_interrupt(struct ath10k * ar,int ce_id)1330 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1331 {
1332 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1333 	struct ath10k_ce_pipe *ce_state;
1334 
1335 	ce_state  = &ce->ce_states[ce_id];
1336 	if (ce_state->attr_flags & CE_ATTR_POLL)
1337 		return;
1338 
1339 	ath10k_ce_per_engine_handler_adjust(ce_state);
1340 }
1341 EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1342 
ath10k_ce_enable_interrupts(struct ath10k * ar)1343 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1344 {
1345 	int ce_id;
1346 
1347 	/* Enable interrupts for copy engine that
1348 	 * are not using polling mode.
1349 	 */
1350 	for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1351 		ath10k_ce_enable_interrupt(ar, ce_id);
1352 }
1353 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1354 
ath10k_ce_init_src_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1355 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1356 				   unsigned int ce_id,
1357 				   const struct ce_attr *attr)
1358 {
1359 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1360 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1361 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1362 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1363 
1364 	nentries = roundup_pow_of_two(attr->src_nentries);
1365 
1366 	if (ar->hw_params.target_64bit)
1367 		memset(src_ring->base_addr_owner_space, 0,
1368 		       nentries * sizeof(struct ce_desc_64));
1369 	else
1370 		memset(src_ring->base_addr_owner_space, 0,
1371 		       nentries * sizeof(struct ce_desc));
1372 
1373 	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1374 	src_ring->sw_index &= src_ring->nentries_mask;
1375 	src_ring->hw_index = src_ring->sw_index;
1376 
1377 	src_ring->write_index =
1378 		ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1379 	src_ring->write_index &= src_ring->nentries_mask;
1380 
1381 	ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1382 					 src_ring->base_addr_ce_space);
1383 	ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1384 	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1385 	ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1386 	ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1387 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1388 
1389 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
1390 		   "boot init ce src ring id %d entries %d base_addr %pK\n",
1391 		   ce_id, nentries, src_ring->base_addr_owner_space);
1392 
1393 	return 0;
1394 }
1395 
ath10k_ce_init_dest_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1396 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1397 				    unsigned int ce_id,
1398 				    const struct ce_attr *attr)
1399 {
1400 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1401 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1402 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1403 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1404 
1405 	nentries = roundup_pow_of_two(attr->dest_nentries);
1406 
1407 	if (ar->hw_params.target_64bit)
1408 		memset(dest_ring->base_addr_owner_space, 0,
1409 		       nentries * sizeof(struct ce_desc_64));
1410 	else
1411 		memset(dest_ring->base_addr_owner_space, 0,
1412 		       nentries * sizeof(struct ce_desc));
1413 
1414 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1415 	dest_ring->sw_index &= dest_ring->nentries_mask;
1416 	dest_ring->write_index =
1417 		ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1418 	dest_ring->write_index &= dest_ring->nentries_mask;
1419 
1420 	ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1421 					  dest_ring->base_addr_ce_space);
1422 	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1423 	ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1424 	ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1425 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1426 
1427 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
1428 		   "boot ce dest ring id %d entries %d base_addr %pK\n",
1429 		   ce_id, nentries, dest_ring->base_addr_owner_space);
1430 
1431 	return 0;
1432 }
1433 
ath10k_ce_alloc_shadow_base(struct ath10k * ar,struct ath10k_ce_ring * src_ring,u32 nentries)1434 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1435 				       struct ath10k_ce_ring *src_ring,
1436 				       u32 nentries)
1437 {
1438 	src_ring->shadow_base_unaligned = kcalloc(nentries,
1439 						  sizeof(struct ce_desc_64),
1440 						  GFP_KERNEL);
1441 	if (!src_ring->shadow_base_unaligned)
1442 		return -ENOMEM;
1443 
1444 	src_ring->shadow_base = (struct ce_desc_64 *)
1445 			PTR_ALIGN(src_ring->shadow_base_unaligned,
1446 				  CE_DESC_RING_ALIGN);
1447 	return 0;
1448 }
1449 
1450 static struct ath10k_ce_ring *
ath10k_ce_alloc_src_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1451 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1452 			 const struct ce_attr *attr)
1453 {
1454 	struct ath10k_ce_ring *src_ring;
1455 	u32 nentries = attr->src_nentries;
1456 	dma_addr_t base_addr;
1457 	int ret;
1458 
1459 	nentries = roundup_pow_of_two(nentries);
1460 
1461 	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1462 				       nentries), GFP_KERNEL);
1463 	if (src_ring == NULL)
1464 		return ERR_PTR(-ENOMEM);
1465 
1466 	src_ring->nentries = nentries;
1467 	src_ring->nentries_mask = nentries - 1;
1468 
1469 	/*
1470 	 * Legacy platforms that do not support cache
1471 	 * coherent DMA are unsupported
1472 	 */
1473 	src_ring->base_addr_owner_space_unaligned =
1474 		dma_alloc_coherent(ar->dev,
1475 				   (nentries * sizeof(struct ce_desc) +
1476 				    CE_DESC_RING_ALIGN),
1477 				   &base_addr, GFP_KERNEL);
1478 	if (!src_ring->base_addr_owner_space_unaligned) {
1479 		kfree(src_ring);
1480 		return ERR_PTR(-ENOMEM);
1481 	}
1482 
1483 	src_ring->base_addr_ce_space_unaligned = base_addr;
1484 
1485 	src_ring->base_addr_owner_space =
1486 			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1487 				  CE_DESC_RING_ALIGN);
1488 	src_ring->base_addr_ce_space =
1489 			ALIGN(src_ring->base_addr_ce_space_unaligned,
1490 			      CE_DESC_RING_ALIGN);
1491 
1492 	if (ar->hw_params.shadow_reg_support) {
1493 		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1494 		if (ret) {
1495 			dma_free_coherent(ar->dev,
1496 					  (nentries * sizeof(struct ce_desc) +
1497 					   CE_DESC_RING_ALIGN),
1498 					  src_ring->base_addr_owner_space_unaligned,
1499 					  base_addr);
1500 			kfree(src_ring);
1501 			return ERR_PTR(ret);
1502 		}
1503 	}
1504 
1505 	return src_ring;
1506 }
1507 
1508 static struct ath10k_ce_ring *
ath10k_ce_alloc_src_ring_64(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1509 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1510 			    const struct ce_attr *attr)
1511 {
1512 	struct ath10k_ce_ring *src_ring;
1513 	u32 nentries = attr->src_nentries;
1514 	dma_addr_t base_addr;
1515 	int ret;
1516 
1517 	nentries = roundup_pow_of_two(nentries);
1518 
1519 	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1520 				       nentries), GFP_KERNEL);
1521 	if (!src_ring)
1522 		return ERR_PTR(-ENOMEM);
1523 
1524 	src_ring->nentries = nentries;
1525 	src_ring->nentries_mask = nentries - 1;
1526 
1527 	/* Legacy platforms that do not support cache
1528 	 * coherent DMA are unsupported
1529 	 */
1530 	src_ring->base_addr_owner_space_unaligned =
1531 		dma_alloc_coherent(ar->dev,
1532 				   (nentries * sizeof(struct ce_desc_64) +
1533 				    CE_DESC_RING_ALIGN),
1534 				   &base_addr, GFP_KERNEL);
1535 	if (!src_ring->base_addr_owner_space_unaligned) {
1536 		kfree(src_ring);
1537 		return ERR_PTR(-ENOMEM);
1538 	}
1539 
1540 	src_ring->base_addr_ce_space_unaligned = base_addr;
1541 
1542 	src_ring->base_addr_owner_space =
1543 			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1544 				  CE_DESC_RING_ALIGN);
1545 	src_ring->base_addr_ce_space =
1546 			ALIGN(src_ring->base_addr_ce_space_unaligned,
1547 			      CE_DESC_RING_ALIGN);
1548 
1549 	if (ar->hw_params.shadow_reg_support) {
1550 		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1551 		if (ret) {
1552 			dma_free_coherent(ar->dev,
1553 					  (nentries * sizeof(struct ce_desc_64) +
1554 					   CE_DESC_RING_ALIGN),
1555 					  src_ring->base_addr_owner_space_unaligned,
1556 					  base_addr);
1557 			kfree(src_ring);
1558 			return ERR_PTR(ret);
1559 		}
1560 	}
1561 
1562 	return src_ring;
1563 }
1564 
1565 static struct ath10k_ce_ring *
ath10k_ce_alloc_dest_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1566 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1567 			  const struct ce_attr *attr)
1568 {
1569 	struct ath10k_ce_ring *dest_ring;
1570 	u32 nentries;
1571 	dma_addr_t base_addr;
1572 
1573 	nentries = roundup_pow_of_two(attr->dest_nentries);
1574 
1575 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1576 					nentries), GFP_KERNEL);
1577 	if (dest_ring == NULL)
1578 		return ERR_PTR(-ENOMEM);
1579 
1580 	dest_ring->nentries = nentries;
1581 	dest_ring->nentries_mask = nentries - 1;
1582 
1583 	/*
1584 	 * Legacy platforms that do not support cache
1585 	 * coherent DMA are unsupported
1586 	 */
1587 	dest_ring->base_addr_owner_space_unaligned =
1588 		dma_alloc_coherent(ar->dev,
1589 				   (nentries * sizeof(struct ce_desc) +
1590 				    CE_DESC_RING_ALIGN),
1591 				   &base_addr, GFP_KERNEL);
1592 	if (!dest_ring->base_addr_owner_space_unaligned) {
1593 		kfree(dest_ring);
1594 		return ERR_PTR(-ENOMEM);
1595 	}
1596 
1597 	dest_ring->base_addr_ce_space_unaligned = base_addr;
1598 
1599 	dest_ring->base_addr_owner_space =
1600 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1601 				  CE_DESC_RING_ALIGN);
1602 	dest_ring->base_addr_ce_space =
1603 				ALIGN(dest_ring->base_addr_ce_space_unaligned,
1604 				      CE_DESC_RING_ALIGN);
1605 
1606 	return dest_ring;
1607 }
1608 
1609 static struct ath10k_ce_ring *
ath10k_ce_alloc_dest_ring_64(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1610 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1611 			     const struct ce_attr *attr)
1612 {
1613 	struct ath10k_ce_ring *dest_ring;
1614 	u32 nentries;
1615 	dma_addr_t base_addr;
1616 
1617 	nentries = roundup_pow_of_two(attr->dest_nentries);
1618 
1619 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1620 					nentries), GFP_KERNEL);
1621 	if (!dest_ring)
1622 		return ERR_PTR(-ENOMEM);
1623 
1624 	dest_ring->nentries = nentries;
1625 	dest_ring->nentries_mask = nentries - 1;
1626 
1627 	/* Legacy platforms that do not support cache
1628 	 * coherent DMA are unsupported
1629 	 */
1630 	dest_ring->base_addr_owner_space_unaligned =
1631 		dma_alloc_coherent(ar->dev,
1632 				   (nentries * sizeof(struct ce_desc_64) +
1633 				    CE_DESC_RING_ALIGN),
1634 				   &base_addr, GFP_KERNEL);
1635 	if (!dest_ring->base_addr_owner_space_unaligned) {
1636 		kfree(dest_ring);
1637 		return ERR_PTR(-ENOMEM);
1638 	}
1639 
1640 	dest_ring->base_addr_ce_space_unaligned = base_addr;
1641 
1642 	/* Correctly initialize memory to 0 to prevent garbage
1643 	 * data crashing system when download firmware
1644 	 */
1645 	dest_ring->base_addr_owner_space =
1646 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1647 				  CE_DESC_RING_ALIGN);
1648 	dest_ring->base_addr_ce_space =
1649 			ALIGN(dest_ring->base_addr_ce_space_unaligned,
1650 			      CE_DESC_RING_ALIGN);
1651 
1652 	return dest_ring;
1653 }
1654 
1655 /*
1656  * Initialize a Copy Engine based on caller-supplied attributes.
1657  * This may be called once to initialize both source and destination
1658  * rings or it may be called twice for separate source and destination
1659  * initialization. It may be that only one side or the other is
1660  * initialized by software/firmware.
1661  */
ath10k_ce_init_pipe(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1662 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1663 			const struct ce_attr *attr)
1664 {
1665 	int ret;
1666 
1667 	if (attr->src_nentries) {
1668 		ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1669 		if (ret) {
1670 			ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1671 				   ce_id, ret);
1672 			return ret;
1673 		}
1674 	}
1675 
1676 	if (attr->dest_nentries) {
1677 		ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1678 		if (ret) {
1679 			ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1680 				   ce_id, ret);
1681 			return ret;
1682 		}
1683 	}
1684 
1685 	return 0;
1686 }
1687 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1688 
ath10k_ce_deinit_src_ring(struct ath10k * ar,unsigned int ce_id)1689 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1690 {
1691 	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1692 
1693 	ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1694 	ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1695 	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1696 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1697 }
1698 
ath10k_ce_deinit_dest_ring(struct ath10k * ar,unsigned int ce_id)1699 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1700 {
1701 	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1702 
1703 	ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1704 	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1705 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1706 }
1707 
ath10k_ce_deinit_pipe(struct ath10k * ar,unsigned int ce_id)1708 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1709 {
1710 	ath10k_ce_deinit_src_ring(ar, ce_id);
1711 	ath10k_ce_deinit_dest_ring(ar, ce_id);
1712 }
1713 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1714 
_ath10k_ce_free_pipe(struct ath10k * ar,int ce_id)1715 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1716 {
1717 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1718 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1719 
1720 	if (ce_state->src_ring) {
1721 		if (ar->hw_params.shadow_reg_support)
1722 			kfree(ce_state->src_ring->shadow_base_unaligned);
1723 		dma_free_coherent(ar->dev,
1724 				  (ce_state->src_ring->nentries *
1725 				   sizeof(struct ce_desc) +
1726 				   CE_DESC_RING_ALIGN),
1727 				  ce_state->src_ring->base_addr_owner_space,
1728 				  ce_state->src_ring->base_addr_ce_space);
1729 		kfree(ce_state->src_ring);
1730 	}
1731 
1732 	if (ce_state->dest_ring) {
1733 		dma_free_coherent(ar->dev,
1734 				  (ce_state->dest_ring->nentries *
1735 				   sizeof(struct ce_desc) +
1736 				   CE_DESC_RING_ALIGN),
1737 				  ce_state->dest_ring->base_addr_owner_space,
1738 				  ce_state->dest_ring->base_addr_ce_space);
1739 		kfree(ce_state->dest_ring);
1740 	}
1741 
1742 	ce_state->src_ring = NULL;
1743 	ce_state->dest_ring = NULL;
1744 }
1745 
_ath10k_ce_free_pipe_64(struct ath10k * ar,int ce_id)1746 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1747 {
1748 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1749 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1750 
1751 	if (ce_state->src_ring) {
1752 		if (ar->hw_params.shadow_reg_support)
1753 			kfree(ce_state->src_ring->shadow_base_unaligned);
1754 		dma_free_coherent(ar->dev,
1755 				  (ce_state->src_ring->nentries *
1756 				   sizeof(struct ce_desc_64) +
1757 				   CE_DESC_RING_ALIGN),
1758 				  ce_state->src_ring->base_addr_owner_space,
1759 				  ce_state->src_ring->base_addr_ce_space);
1760 		kfree(ce_state->src_ring);
1761 	}
1762 
1763 	if (ce_state->dest_ring) {
1764 		dma_free_coherent(ar->dev,
1765 				  (ce_state->dest_ring->nentries *
1766 				   sizeof(struct ce_desc_64) +
1767 				   CE_DESC_RING_ALIGN),
1768 				  ce_state->dest_ring->base_addr_owner_space,
1769 				  ce_state->dest_ring->base_addr_ce_space);
1770 		kfree(ce_state->dest_ring);
1771 	}
1772 
1773 	ce_state->src_ring = NULL;
1774 	ce_state->dest_ring = NULL;
1775 }
1776 
ath10k_ce_free_pipe(struct ath10k * ar,int ce_id)1777 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1778 {
1779 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1780 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1781 
1782 	ce_state->ops->ce_free_pipe(ar, ce_id);
1783 }
1784 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1785 
ath10k_ce_dump_registers(struct ath10k * ar,struct ath10k_fw_crash_data * crash_data)1786 void ath10k_ce_dump_registers(struct ath10k *ar,
1787 			      struct ath10k_fw_crash_data *crash_data)
1788 {
1789 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1790 	struct ath10k_ce_crash_data ce_data;
1791 	u32 addr, id;
1792 
1793 	lockdep_assert_held(&ar->dump_mutex);
1794 
1795 	ath10k_err(ar, "Copy Engine register dump:\n");
1796 
1797 	spin_lock_bh(&ce->ce_lock);
1798 	for (id = 0; id < CE_COUNT; id++) {
1799 		addr = ath10k_ce_base_address(ar, id);
1800 		ce_data.base_addr = cpu_to_le32(addr);
1801 
1802 		ce_data.src_wr_idx =
1803 			cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1804 		ce_data.src_r_idx =
1805 			cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1806 		ce_data.dst_wr_idx =
1807 			cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1808 		ce_data.dst_r_idx =
1809 			cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1810 
1811 		if (crash_data)
1812 			crash_data->ce_crash_data[id] = ce_data;
1813 
1814 		ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1815 			   le32_to_cpu(ce_data.base_addr),
1816 			   le32_to_cpu(ce_data.src_wr_idx),
1817 			   le32_to_cpu(ce_data.src_r_idx),
1818 			   le32_to_cpu(ce_data.dst_wr_idx),
1819 			   le32_to_cpu(ce_data.dst_r_idx));
1820 	}
1821 
1822 	spin_unlock_bh(&ce->ce_lock);
1823 }
1824 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1825 
1826 static const struct ath10k_ce_ops ce_ops = {
1827 	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1828 	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1829 	.ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1830 	.ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1831 	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1832 	.ce_extract_desc_data = ath10k_ce_extract_desc_data,
1833 	.ce_free_pipe = _ath10k_ce_free_pipe,
1834 	.ce_send_nolock = _ath10k_ce_send_nolock,
1835 	.ce_set_src_ring_base_addr_hi = NULL,
1836 	.ce_set_dest_ring_base_addr_hi = NULL,
1837 	.ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1838 };
1839 
1840 static const struct ath10k_ce_ops ce_64_ops = {
1841 	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1842 	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1843 	.ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1844 	.ce_completed_recv_next_nolock =
1845 				_ath10k_ce_completed_recv_next_nolock_64,
1846 	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1847 	.ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1848 	.ce_free_pipe = _ath10k_ce_free_pipe_64,
1849 	.ce_send_nolock = _ath10k_ce_send_nolock_64,
1850 	.ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1851 	.ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1852 	.ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1853 };
1854 
ath10k_ce_set_ops(struct ath10k * ar,struct ath10k_ce_pipe * ce_state)1855 static void ath10k_ce_set_ops(struct ath10k *ar,
1856 			      struct ath10k_ce_pipe *ce_state)
1857 {
1858 	switch (ar->hw_rev) {
1859 	case ATH10K_HW_WCN3990:
1860 		ce_state->ops = &ce_64_ops;
1861 		break;
1862 	default:
1863 		ce_state->ops = &ce_ops;
1864 		break;
1865 	}
1866 }
1867 
ath10k_ce_alloc_pipe(struct ath10k * ar,int ce_id,const struct ce_attr * attr)1868 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1869 			 const struct ce_attr *attr)
1870 {
1871 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1872 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1873 	int ret;
1874 
1875 	ath10k_ce_set_ops(ar, ce_state);
1876 	/* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1877 	 * additional TX locking checks.
1878 	 *
1879 	 * For the lack of a better place do the check here.
1880 	 */
1881 	BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1882 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1883 	BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1884 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1885 	BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1886 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1887 
1888 	ce_state->ar = ar;
1889 	ce_state->id = ce_id;
1890 	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1891 	ce_state->attr_flags = attr->flags;
1892 	ce_state->src_sz_max = attr->src_sz_max;
1893 
1894 	if (attr->src_nentries)
1895 		ce_state->send_cb = attr->send_cb;
1896 
1897 	if (attr->dest_nentries)
1898 		ce_state->recv_cb = attr->recv_cb;
1899 
1900 	if (attr->src_nentries) {
1901 		ce_state->src_ring =
1902 			ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1903 		if (IS_ERR(ce_state->src_ring)) {
1904 			ret = PTR_ERR(ce_state->src_ring);
1905 			ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1906 				   ce_id, ret);
1907 			ce_state->src_ring = NULL;
1908 			return ret;
1909 		}
1910 	}
1911 
1912 	if (attr->dest_nentries) {
1913 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1914 									ce_id,
1915 									attr);
1916 		if (IS_ERR(ce_state->dest_ring)) {
1917 			ret = PTR_ERR(ce_state->dest_ring);
1918 			ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1919 				   ce_id, ret);
1920 			ce_state->dest_ring = NULL;
1921 			return ret;
1922 		}
1923 	}
1924 
1925 	return 0;
1926 }
1927 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1928 
ath10k_ce_alloc_rri(struct ath10k * ar)1929 void ath10k_ce_alloc_rri(struct ath10k *ar)
1930 {
1931 	int i;
1932 	u32 value;
1933 	u32 ctrl1_regs;
1934 	u32 ce_base_addr;
1935 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1936 
1937 	ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1938 					   (CE_COUNT * sizeof(u32)),
1939 					   &ce->paddr_rri, GFP_KERNEL);
1940 
1941 	if (!ce->vaddr_rri)
1942 		return;
1943 
1944 	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1945 			  lower_32_bits(ce->paddr_rri));
1946 	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1947 			  (upper_32_bits(ce->paddr_rri) &
1948 			  CE_DESC_ADDR_HI_MASK));
1949 
1950 	for (i = 0; i < CE_COUNT; i++) {
1951 		ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1952 		ce_base_addr = ath10k_ce_base_address(ar, i);
1953 		value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1954 		value |= ar->hw_ce_regs->upd->mask;
1955 		ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1956 	}
1957 }
1958 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1959 
ath10k_ce_free_rri(struct ath10k * ar)1960 void ath10k_ce_free_rri(struct ath10k *ar)
1961 {
1962 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1963 
1964 	dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1965 			  ce->vaddr_rri,
1966 			  ce->paddr_rri);
1967 }
1968 EXPORT_SYMBOL(ath10k_ce_free_rri);
1969