1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018, Intel Corporation. */
3 
4 #include "ice_common.h"
5 
6 #define ICE_CQ_INIT_REGS(qinfo, prefix)				\
7 do {								\
8 	(qinfo)->sq.head = prefix##_ATQH;			\
9 	(qinfo)->sq.tail = prefix##_ATQT;			\
10 	(qinfo)->sq.len = prefix##_ATQLEN;			\
11 	(qinfo)->sq.bah = prefix##_ATQBAH;			\
12 	(qinfo)->sq.bal = prefix##_ATQBAL;			\
13 	(qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;	\
14 	(qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M;	\
15 	(qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M;	\
16 	(qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;		\
17 	(qinfo)->rq.head = prefix##_ARQH;			\
18 	(qinfo)->rq.tail = prefix##_ARQT;			\
19 	(qinfo)->rq.len = prefix##_ARQLEN;			\
20 	(qinfo)->rq.bah = prefix##_ARQBAH;			\
21 	(qinfo)->rq.bal = prefix##_ARQBAL;			\
22 	(qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;	\
23 	(qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M;	\
24 	(qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M;	\
25 	(qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;		\
26 } while (0)
27 
28 /**
29  * ice_adminq_init_regs - Initialize AdminQ registers
30  * @hw: pointer to the hardware structure
31  *
32  * This assumes the alloc_sq and alloc_rq functions have already been called
33  */
34 static void ice_adminq_init_regs(struct ice_hw *hw)
35 {
36 	struct ice_ctl_q_info *cq = &hw->adminq;
37 
38 	ICE_CQ_INIT_REGS(cq, PF_FW);
39 }
40 
41 /**
42  * ice_mailbox_init_regs - Initialize Mailbox registers
43  * @hw: pointer to the hardware structure
44  *
45  * This assumes the alloc_sq and alloc_rq functions have already been called
46  */
47 static void ice_mailbox_init_regs(struct ice_hw *hw)
48 {
49 	struct ice_ctl_q_info *cq = &hw->mailboxq;
50 
51 	ICE_CQ_INIT_REGS(cq, PF_MBX);
52 }
53 
54 /**
55  * ice_sb_init_regs - Initialize Sideband registers
56  * @hw: pointer to the hardware structure
57  *
58  * This assumes the alloc_sq and alloc_rq functions have already been called
59  */
60 static void ice_sb_init_regs(struct ice_hw *hw)
61 {
62 	struct ice_ctl_q_info *cq = &hw->sbq;
63 
64 	ICE_CQ_INIT_REGS(cq, PF_SB);
65 }
66 
67 /**
68  * ice_check_sq_alive
69  * @hw: pointer to the HW struct
70  * @cq: pointer to the specific Control queue
71  *
72  * Returns true if Queue is enabled else false.
73  */
74 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)
75 {
76 	/* check both queue-length and queue-enable fields */
77 	if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)
78 		return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |
79 						cq->sq.len_ena_mask)) ==
80 			(cq->num_sq_entries | cq->sq.len_ena_mask);
81 
82 	return false;
83 }
84 
85 /**
86  * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings
87  * @hw: pointer to the hardware structure
88  * @cq: pointer to the specific Control queue
89  */
90 static enum ice_status
91 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
92 {
93 	size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);
94 
95 	cq->sq.desc_buf.va = dmam_alloc_coherent(ice_hw_to_dev(hw), size,
96 						 &cq->sq.desc_buf.pa,
97 						 GFP_KERNEL | __GFP_ZERO);
98 	if (!cq->sq.desc_buf.va)
99 		return ICE_ERR_NO_MEMORY;
100 	cq->sq.desc_buf.size = size;
101 
102 	cq->sq.cmd_buf = devm_kcalloc(ice_hw_to_dev(hw), cq->num_sq_entries,
103 				      sizeof(struct ice_sq_cd), GFP_KERNEL);
104 	if (!cq->sq.cmd_buf) {
105 		dmam_free_coherent(ice_hw_to_dev(hw), cq->sq.desc_buf.size,
106 				   cq->sq.desc_buf.va, cq->sq.desc_buf.pa);
107 		cq->sq.desc_buf.va = NULL;
108 		cq->sq.desc_buf.pa = 0;
109 		cq->sq.desc_buf.size = 0;
110 		return ICE_ERR_NO_MEMORY;
111 	}
112 
113 	return 0;
114 }
115 
116 /**
117  * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings
118  * @hw: pointer to the hardware structure
119  * @cq: pointer to the specific Control queue
120  */
121 static enum ice_status
122 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
123 {
124 	size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);
125 
126 	cq->rq.desc_buf.va = dmam_alloc_coherent(ice_hw_to_dev(hw), size,
127 						 &cq->rq.desc_buf.pa,
128 						 GFP_KERNEL | __GFP_ZERO);
129 	if (!cq->rq.desc_buf.va)
130 		return ICE_ERR_NO_MEMORY;
131 	cq->rq.desc_buf.size = size;
132 	return 0;
133 }
134 
135 /**
136  * ice_free_cq_ring - Free control queue ring
137  * @hw: pointer to the hardware structure
138  * @ring: pointer to the specific control queue ring
139  *
140  * This assumes the posted buffers have already been cleaned
141  * and de-allocated
142  */
143 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
144 {
145 	dmam_free_coherent(ice_hw_to_dev(hw), ring->desc_buf.size,
146 			   ring->desc_buf.va, ring->desc_buf.pa);
147 	ring->desc_buf.va = NULL;
148 	ring->desc_buf.pa = 0;
149 	ring->desc_buf.size = 0;
150 }
151 
152 /**
153  * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ
154  * @hw: pointer to the hardware structure
155  * @cq: pointer to the specific Control queue
156  */
157 static enum ice_status
158 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
159 {
160 	int i;
161 
162 	/* We'll be allocating the buffer info memory first, then we can
163 	 * allocate the mapped buffers for the event processing
164 	 */
165 	cq->rq.dma_head = devm_kcalloc(ice_hw_to_dev(hw), cq->num_rq_entries,
166 				       sizeof(cq->rq.desc_buf), GFP_KERNEL);
167 	if (!cq->rq.dma_head)
168 		return ICE_ERR_NO_MEMORY;
169 	cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;
170 
171 	/* allocate the mapped buffers */
172 	for (i = 0; i < cq->num_rq_entries; i++) {
173 		struct ice_aq_desc *desc;
174 		struct ice_dma_mem *bi;
175 
176 		bi = &cq->rq.r.rq_bi[i];
177 		bi->va = dmam_alloc_coherent(ice_hw_to_dev(hw),
178 					     cq->rq_buf_size, &bi->pa,
179 					     GFP_KERNEL | __GFP_ZERO);
180 		if (!bi->va)
181 			goto unwind_alloc_rq_bufs;
182 		bi->size = cq->rq_buf_size;
183 
184 		/* now configure the descriptors for use */
185 		desc = ICE_CTL_Q_DESC(cq->rq, i);
186 
187 		desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF);
188 		if (cq->rq_buf_size > ICE_AQ_LG_BUF)
189 			desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB);
190 		desc->opcode = 0;
191 		/* This is in accordance with Admin queue design, there is no
192 		 * register for buffer size configuration
193 		 */
194 		desc->datalen = cpu_to_le16(bi->size);
195 		desc->retval = 0;
196 		desc->cookie_high = 0;
197 		desc->cookie_low = 0;
198 		desc->params.generic.addr_high =
199 			cpu_to_le32(upper_32_bits(bi->pa));
200 		desc->params.generic.addr_low =
201 			cpu_to_le32(lower_32_bits(bi->pa));
202 		desc->params.generic.param0 = 0;
203 		desc->params.generic.param1 = 0;
204 	}
205 	return 0;
206 
207 unwind_alloc_rq_bufs:
208 	/* don't try to free the one that failed... */
209 	i--;
210 	for (; i >= 0; i--) {
211 		dmam_free_coherent(ice_hw_to_dev(hw), cq->rq.r.rq_bi[i].size,
212 				   cq->rq.r.rq_bi[i].va, cq->rq.r.rq_bi[i].pa);
213 		cq->rq.r.rq_bi[i].va = NULL;
214 		cq->rq.r.rq_bi[i].pa = 0;
215 		cq->rq.r.rq_bi[i].size = 0;
216 	}
217 	cq->rq.r.rq_bi = NULL;
218 	devm_kfree(ice_hw_to_dev(hw), cq->rq.dma_head);
219 	cq->rq.dma_head = NULL;
220 
221 	return ICE_ERR_NO_MEMORY;
222 }
223 
224 /**
225  * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ
226  * @hw: pointer to the hardware structure
227  * @cq: pointer to the specific Control queue
228  */
229 static enum ice_status
230 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
231 {
232 	int i;
233 
234 	/* No mapped memory needed yet, just the buffer info structures */
235 	cq->sq.dma_head = devm_kcalloc(ice_hw_to_dev(hw), cq->num_sq_entries,
236 				       sizeof(cq->sq.desc_buf), GFP_KERNEL);
237 	if (!cq->sq.dma_head)
238 		return ICE_ERR_NO_MEMORY;
239 	cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;
240 
241 	/* allocate the mapped buffers */
242 	for (i = 0; i < cq->num_sq_entries; i++) {
243 		struct ice_dma_mem *bi;
244 
245 		bi = &cq->sq.r.sq_bi[i];
246 		bi->va = dmam_alloc_coherent(ice_hw_to_dev(hw),
247 					     cq->sq_buf_size, &bi->pa,
248 					     GFP_KERNEL | __GFP_ZERO);
249 		if (!bi->va)
250 			goto unwind_alloc_sq_bufs;
251 		bi->size = cq->sq_buf_size;
252 	}
253 	return 0;
254 
255 unwind_alloc_sq_bufs:
256 	/* don't try to free the one that failed... */
257 	i--;
258 	for (; i >= 0; i--) {
259 		dmam_free_coherent(ice_hw_to_dev(hw), cq->sq.r.sq_bi[i].size,
260 				   cq->sq.r.sq_bi[i].va, cq->sq.r.sq_bi[i].pa);
261 		cq->sq.r.sq_bi[i].va = NULL;
262 		cq->sq.r.sq_bi[i].pa = 0;
263 		cq->sq.r.sq_bi[i].size = 0;
264 	}
265 	cq->sq.r.sq_bi = NULL;
266 	devm_kfree(ice_hw_to_dev(hw), cq->sq.dma_head);
267 	cq->sq.dma_head = NULL;
268 
269 	return ICE_ERR_NO_MEMORY;
270 }
271 
272 static enum ice_status
273 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
274 {
275 	/* Clear Head and Tail */
276 	wr32(hw, ring->head, 0);
277 	wr32(hw, ring->tail, 0);
278 
279 	/* set starting point */
280 	wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
281 	wr32(hw, ring->bal, lower_32_bits(ring->desc_buf.pa));
282 	wr32(hw, ring->bah, upper_32_bits(ring->desc_buf.pa));
283 
284 	/* Check one register to verify that config was applied */
285 	if (rd32(hw, ring->bal) != lower_32_bits(ring->desc_buf.pa))
286 		return ICE_ERR_AQ_ERROR;
287 
288 	return 0;
289 }
290 
291 /**
292  * ice_cfg_sq_regs - configure Control ATQ registers
293  * @hw: pointer to the hardware structure
294  * @cq: pointer to the specific Control queue
295  *
296  * Configure base address and length registers for the transmit queue
297  */
298 static enum ice_status
299 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
300 {
301 	return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
302 }
303 
304 /**
305  * ice_cfg_rq_regs - configure Control ARQ register
306  * @hw: pointer to the hardware structure
307  * @cq: pointer to the specific Control queue
308  *
309  * Configure base address and length registers for the receive (event queue)
310  */
311 static enum ice_status
312 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
313 {
314 	enum ice_status status;
315 
316 	status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
317 	if (status)
318 		return status;
319 
320 	/* Update tail in the HW to post pre-allocated buffers */
321 	wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
322 
323 	return 0;
324 }
325 
326 #define ICE_FREE_CQ_BUFS(hw, qi, ring)					\
327 do {									\
328 	/* free descriptors */						\
329 	if ((qi)->ring.r.ring##_bi) {					\
330 		int i;							\
331 									\
332 		for (i = 0; i < (qi)->num_##ring##_entries; i++)	\
333 			if ((qi)->ring.r.ring##_bi[i].pa) {		\
334 				dmam_free_coherent(ice_hw_to_dev(hw),	\
335 					(qi)->ring.r.ring##_bi[i].size,	\
336 					(qi)->ring.r.ring##_bi[i].va,	\
337 					(qi)->ring.r.ring##_bi[i].pa);	\
338 					(qi)->ring.r.ring##_bi[i].va = NULL;\
339 					(qi)->ring.r.ring##_bi[i].pa = 0;\
340 					(qi)->ring.r.ring##_bi[i].size = 0;\
341 		}							\
342 	}								\
343 	/* free the buffer info list */					\
344 	if ((qi)->ring.cmd_buf)						\
345 		devm_kfree(ice_hw_to_dev(hw), (qi)->ring.cmd_buf);	\
346 	/* free DMA head */						\
347 	devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head);		\
348 } while (0)
349 
350 /**
351  * ice_init_sq - main initialization routine for Control ATQ
352  * @hw: pointer to the hardware structure
353  * @cq: pointer to the specific Control queue
354  *
355  * This is the main initialization routine for the Control Send Queue
356  * Prior to calling this function, the driver *MUST* set the following fields
357  * in the cq->structure:
358  *     - cq->num_sq_entries
359  *     - cq->sq_buf_size
360  *
361  * Do *NOT* hold the lock when calling this as the memory allocation routines
362  * called are not going to be atomic context safe
363  */
364 static enum ice_status ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
365 {
366 	enum ice_status ret_code;
367 
368 	if (cq->sq.count > 0) {
369 		/* queue already initialized */
370 		ret_code = ICE_ERR_NOT_READY;
371 		goto init_ctrlq_exit;
372 	}
373 
374 	/* verify input for valid configuration */
375 	if (!cq->num_sq_entries || !cq->sq_buf_size) {
376 		ret_code = ICE_ERR_CFG;
377 		goto init_ctrlq_exit;
378 	}
379 
380 	cq->sq.next_to_use = 0;
381 	cq->sq.next_to_clean = 0;
382 
383 	/* allocate the ring memory */
384 	ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);
385 	if (ret_code)
386 		goto init_ctrlq_exit;
387 
388 	/* allocate buffers in the rings */
389 	ret_code = ice_alloc_sq_bufs(hw, cq);
390 	if (ret_code)
391 		goto init_ctrlq_free_rings;
392 
393 	/* initialize base registers */
394 	ret_code = ice_cfg_sq_regs(hw, cq);
395 	if (ret_code)
396 		goto init_ctrlq_free_rings;
397 
398 	/* success! */
399 	cq->sq.count = cq->num_sq_entries;
400 	goto init_ctrlq_exit;
401 
402 init_ctrlq_free_rings:
403 	ICE_FREE_CQ_BUFS(hw, cq, sq);
404 	ice_free_cq_ring(hw, &cq->sq);
405 
406 init_ctrlq_exit:
407 	return ret_code;
408 }
409 
410 /**
411  * ice_init_rq - initialize ARQ
412  * @hw: pointer to the hardware structure
413  * @cq: pointer to the specific Control queue
414  *
415  * The main initialization routine for the Admin Receive (Event) Queue.
416  * Prior to calling this function, the driver *MUST* set the following fields
417  * in the cq->structure:
418  *     - cq->num_rq_entries
419  *     - cq->rq_buf_size
420  *
421  * Do *NOT* hold the lock when calling this as the memory allocation routines
422  * called are not going to be atomic context safe
423  */
424 static enum ice_status ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
425 {
426 	enum ice_status ret_code;
427 
428 	if (cq->rq.count > 0) {
429 		/* queue already initialized */
430 		ret_code = ICE_ERR_NOT_READY;
431 		goto init_ctrlq_exit;
432 	}
433 
434 	/* verify input for valid configuration */
435 	if (!cq->num_rq_entries || !cq->rq_buf_size) {
436 		ret_code = ICE_ERR_CFG;
437 		goto init_ctrlq_exit;
438 	}
439 
440 	cq->rq.next_to_use = 0;
441 	cq->rq.next_to_clean = 0;
442 
443 	/* allocate the ring memory */
444 	ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);
445 	if (ret_code)
446 		goto init_ctrlq_exit;
447 
448 	/* allocate buffers in the rings */
449 	ret_code = ice_alloc_rq_bufs(hw, cq);
450 	if (ret_code)
451 		goto init_ctrlq_free_rings;
452 
453 	/* initialize base registers */
454 	ret_code = ice_cfg_rq_regs(hw, cq);
455 	if (ret_code)
456 		goto init_ctrlq_free_rings;
457 
458 	/* success! */
459 	cq->rq.count = cq->num_rq_entries;
460 	goto init_ctrlq_exit;
461 
462 init_ctrlq_free_rings:
463 	ICE_FREE_CQ_BUFS(hw, cq, rq);
464 	ice_free_cq_ring(hw, &cq->rq);
465 
466 init_ctrlq_exit:
467 	return ret_code;
468 }
469 
470 /**
471  * ice_shutdown_sq - shutdown the Control ATQ
472  * @hw: pointer to the hardware structure
473  * @cq: pointer to the specific Control queue
474  *
475  * The main shutdown routine for the Control Transmit Queue
476  */
477 static enum ice_status
478 ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
479 {
480 	enum ice_status ret_code = 0;
481 
482 	mutex_lock(&cq->sq_lock);
483 
484 	if (!cq->sq.count) {
485 		ret_code = ICE_ERR_NOT_READY;
486 		goto shutdown_sq_out;
487 	}
488 
489 	/* Stop firmware AdminQ processing */
490 	wr32(hw, cq->sq.head, 0);
491 	wr32(hw, cq->sq.tail, 0);
492 	wr32(hw, cq->sq.len, 0);
493 	wr32(hw, cq->sq.bal, 0);
494 	wr32(hw, cq->sq.bah, 0);
495 
496 	cq->sq.count = 0;	/* to indicate uninitialized queue */
497 
498 	/* free ring buffers and the ring itself */
499 	ICE_FREE_CQ_BUFS(hw, cq, sq);
500 	ice_free_cq_ring(hw, &cq->sq);
501 
502 shutdown_sq_out:
503 	mutex_unlock(&cq->sq_lock);
504 	return ret_code;
505 }
506 
507 /**
508  * ice_aq_ver_check - Check the reported AQ API version.
509  * @hw: pointer to the hardware structure
510  *
511  * Checks if the driver should load on a given AQ API version.
512  *
513  * Return: 'true' iff the driver should attempt to load. 'false' otherwise.
514  */
515 static bool ice_aq_ver_check(struct ice_hw *hw)
516 {
517 	if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) {
518 		/* Major API version is newer than expected, don't load */
519 		dev_warn(ice_hw_to_dev(hw),
520 			 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
521 		return false;
522 	} else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) {
523 		if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2))
524 			dev_info(ice_hw_to_dev(hw),
525 				 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
526 		else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR)
527 			dev_info(ice_hw_to_dev(hw),
528 				 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
529 	} else {
530 		/* Major API version is older than expected, log a warning */
531 		dev_info(ice_hw_to_dev(hw),
532 			 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
533 	}
534 	return true;
535 }
536 
537 /**
538  * ice_shutdown_rq - shutdown Control ARQ
539  * @hw: pointer to the hardware structure
540  * @cq: pointer to the specific Control queue
541  *
542  * The main shutdown routine for the Control Receive Queue
543  */
544 static enum ice_status
545 ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
546 {
547 	enum ice_status ret_code = 0;
548 
549 	mutex_lock(&cq->rq_lock);
550 
551 	if (!cq->rq.count) {
552 		ret_code = ICE_ERR_NOT_READY;
553 		goto shutdown_rq_out;
554 	}
555 
556 	/* Stop Control Queue processing */
557 	wr32(hw, cq->rq.head, 0);
558 	wr32(hw, cq->rq.tail, 0);
559 	wr32(hw, cq->rq.len, 0);
560 	wr32(hw, cq->rq.bal, 0);
561 	wr32(hw, cq->rq.bah, 0);
562 
563 	/* set rq.count to 0 to indicate uninitialized queue */
564 	cq->rq.count = 0;
565 
566 	/* free ring buffers and the ring itself */
567 	ICE_FREE_CQ_BUFS(hw, cq, rq);
568 	ice_free_cq_ring(hw, &cq->rq);
569 
570 shutdown_rq_out:
571 	mutex_unlock(&cq->rq_lock);
572 	return ret_code;
573 }
574 
575 /**
576  * ice_init_check_adminq - Check version for Admin Queue to know if its alive
577  * @hw: pointer to the hardware structure
578  */
579 static enum ice_status ice_init_check_adminq(struct ice_hw *hw)
580 {
581 	struct ice_ctl_q_info *cq = &hw->adminq;
582 	enum ice_status status;
583 
584 	status = ice_aq_get_fw_ver(hw, NULL);
585 	if (status)
586 		goto init_ctrlq_free_rq;
587 
588 	if (!ice_aq_ver_check(hw)) {
589 		status = ICE_ERR_FW_API_VER;
590 		goto init_ctrlq_free_rq;
591 	}
592 
593 	return 0;
594 
595 init_ctrlq_free_rq:
596 	ice_shutdown_rq(hw, cq);
597 	ice_shutdown_sq(hw, cq);
598 	return status;
599 }
600 
601 /**
602  * ice_init_ctrlq - main initialization routine for any control Queue
603  * @hw: pointer to the hardware structure
604  * @q_type: specific Control queue type
605  *
606  * Prior to calling this function, the driver *MUST* set the following fields
607  * in the cq->structure:
608  *     - cq->num_sq_entries
609  *     - cq->num_rq_entries
610  *     - cq->rq_buf_size
611  *     - cq->sq_buf_size
612  *
613  * NOTE: this function does not initialize the controlq locks
614  */
615 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
616 {
617 	struct ice_ctl_q_info *cq;
618 	enum ice_status ret_code;
619 
620 	switch (q_type) {
621 	case ICE_CTL_Q_ADMIN:
622 		ice_adminq_init_regs(hw);
623 		cq = &hw->adminq;
624 		break;
625 	case ICE_CTL_Q_SB:
626 		ice_sb_init_regs(hw);
627 		cq = &hw->sbq;
628 		break;
629 	case ICE_CTL_Q_MAILBOX:
630 		ice_mailbox_init_regs(hw);
631 		cq = &hw->mailboxq;
632 		break;
633 	default:
634 		return ICE_ERR_PARAM;
635 	}
636 	cq->qtype = q_type;
637 
638 	/* verify input for valid configuration */
639 	if (!cq->num_rq_entries || !cq->num_sq_entries ||
640 	    !cq->rq_buf_size || !cq->sq_buf_size) {
641 		return ICE_ERR_CFG;
642 	}
643 
644 	/* setup SQ command write back timeout */
645 	cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
646 
647 	/* allocate the ATQ */
648 	ret_code = ice_init_sq(hw, cq);
649 	if (ret_code)
650 		return ret_code;
651 
652 	/* allocate the ARQ */
653 	ret_code = ice_init_rq(hw, cq);
654 	if (ret_code)
655 		goto init_ctrlq_free_sq;
656 
657 	/* success! */
658 	return 0;
659 
660 init_ctrlq_free_sq:
661 	ice_shutdown_sq(hw, cq);
662 	return ret_code;
663 }
664 
665 /**
666  * ice_is_sbq_supported - is the sideband queue supported
667  * @hw: pointer to the hardware structure
668  *
669  * Returns true if the sideband control queue interface is
670  * supported for the device, false otherwise
671  */
672 bool ice_is_sbq_supported(struct ice_hw *hw)
673 {
674 	/* The device sideband queue is only supported on devices with the
675 	 * generic MAC type.
676 	 */
677 	return hw->mac_type == ICE_MAC_GENERIC;
678 }
679 
680 /**
681  * ice_get_sbq - returns the right control queue to use for sideband
682  * @hw: pointer to the hardware structure
683  */
684 struct ice_ctl_q_info *ice_get_sbq(struct ice_hw *hw)
685 {
686 	if (ice_is_sbq_supported(hw))
687 		return &hw->sbq;
688 	return &hw->adminq;
689 }
690 
691 /**
692  * ice_shutdown_ctrlq - shutdown routine for any control queue
693  * @hw: pointer to the hardware structure
694  * @q_type: specific Control queue type
695  *
696  * NOTE: this function does not destroy the control queue locks.
697  */
698 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
699 {
700 	struct ice_ctl_q_info *cq;
701 
702 	switch (q_type) {
703 	case ICE_CTL_Q_ADMIN:
704 		cq = &hw->adminq;
705 		if (ice_check_sq_alive(hw, cq))
706 			ice_aq_q_shutdown(hw, true);
707 		break;
708 	case ICE_CTL_Q_SB:
709 		cq = &hw->sbq;
710 		break;
711 	case ICE_CTL_Q_MAILBOX:
712 		cq = &hw->mailboxq;
713 		break;
714 	default:
715 		return;
716 	}
717 
718 	ice_shutdown_sq(hw, cq);
719 	ice_shutdown_rq(hw, cq);
720 }
721 
722 /**
723  * ice_shutdown_all_ctrlq - shutdown routine for all control queues
724  * @hw: pointer to the hardware structure
725  *
726  * NOTE: this function does not destroy the control queue locks. The driver
727  * may call this at runtime to shutdown and later restart control queues, such
728  * as in response to a reset event.
729  */
730 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
731 {
732 	/* Shutdown FW admin queue */
733 	ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
734 	/* Shutdown PHY Sideband */
735 	if (ice_is_sbq_supported(hw))
736 		ice_shutdown_ctrlq(hw, ICE_CTL_Q_SB);
737 	/* Shutdown PF-VF Mailbox */
738 	ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
739 }
740 
741 /**
742  * ice_init_all_ctrlq - main initialization routine for all control queues
743  * @hw: pointer to the hardware structure
744  *
745  * Prior to calling this function, the driver MUST* set the following fields
746  * in the cq->structure for all control queues:
747  *     - cq->num_sq_entries
748  *     - cq->num_rq_entries
749  *     - cq->rq_buf_size
750  *     - cq->sq_buf_size
751  *
752  * NOTE: this function does not initialize the controlq locks.
753  */
754 enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
755 {
756 	enum ice_status status;
757 	u32 retry = 0;
758 
759 	/* Init FW admin queue */
760 	do {
761 		status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
762 		if (status)
763 			return status;
764 
765 		status = ice_init_check_adminq(hw);
766 		if (status != ICE_ERR_AQ_FW_CRITICAL)
767 			break;
768 
769 		ice_debug(hw, ICE_DBG_AQ_MSG, "Retry Admin Queue init due to FW critical error\n");
770 		ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
771 		msleep(ICE_CTL_Q_ADMIN_INIT_MSEC);
772 	} while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT);
773 
774 	if (status)
775 		return status;
776 	/* sideband control queue (SBQ) interface is not supported on some
777 	 * devices. Initialize if supported, else fallback to the admin queue
778 	 * interface
779 	 */
780 	if (ice_is_sbq_supported(hw)) {
781 		status = ice_init_ctrlq(hw, ICE_CTL_Q_SB);
782 		if (status)
783 			return status;
784 	}
785 	/* Init Mailbox queue */
786 	return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
787 }
788 
789 /**
790  * ice_init_ctrlq_locks - Initialize locks for a control queue
791  * @cq: pointer to the control queue
792  *
793  * Initializes the send and receive queue locks for a given control queue.
794  */
795 static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)
796 {
797 	mutex_init(&cq->sq_lock);
798 	mutex_init(&cq->rq_lock);
799 }
800 
801 /**
802  * ice_create_all_ctrlq - main initialization routine for all control queues
803  * @hw: pointer to the hardware structure
804  *
805  * Prior to calling this function, the driver *MUST* set the following fields
806  * in the cq->structure for all control queues:
807  *     - cq->num_sq_entries
808  *     - cq->num_rq_entries
809  *     - cq->rq_buf_size
810  *     - cq->sq_buf_size
811  *
812  * This function creates all the control queue locks and then calls
813  * ice_init_all_ctrlq. It should be called once during driver load. If the
814  * driver needs to re-initialize control queues at run time it should call
815  * ice_init_all_ctrlq instead.
816  */
817 enum ice_status ice_create_all_ctrlq(struct ice_hw *hw)
818 {
819 	ice_init_ctrlq_locks(&hw->adminq);
820 	if (ice_is_sbq_supported(hw))
821 		ice_init_ctrlq_locks(&hw->sbq);
822 	ice_init_ctrlq_locks(&hw->mailboxq);
823 
824 	return ice_init_all_ctrlq(hw);
825 }
826 
827 /**
828  * ice_destroy_ctrlq_locks - Destroy locks for a control queue
829  * @cq: pointer to the control queue
830  *
831  * Destroys the send and receive queue locks for a given control queue.
832  */
833 static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)
834 {
835 	mutex_destroy(&cq->sq_lock);
836 	mutex_destroy(&cq->rq_lock);
837 }
838 
839 /**
840  * ice_destroy_all_ctrlq - exit routine for all control queues
841  * @hw: pointer to the hardware structure
842  *
843  * This function shuts down all the control queues and then destroys the
844  * control queue locks. It should be called once during driver unload. The
845  * driver should call ice_shutdown_all_ctrlq if it needs to shut down and
846  * reinitialize control queues, such as in response to a reset event.
847  */
848 void ice_destroy_all_ctrlq(struct ice_hw *hw)
849 {
850 	/* shut down all the control queues first */
851 	ice_shutdown_all_ctrlq(hw);
852 
853 	ice_destroy_ctrlq_locks(&hw->adminq);
854 	if (ice_is_sbq_supported(hw))
855 		ice_destroy_ctrlq_locks(&hw->sbq);
856 	ice_destroy_ctrlq_locks(&hw->mailboxq);
857 }
858 
859 /**
860  * ice_clean_sq - cleans Admin send queue (ATQ)
861  * @hw: pointer to the hardware structure
862  * @cq: pointer to the specific Control queue
863  *
864  * returns the number of free desc
865  */
866 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
867 {
868 	struct ice_ctl_q_ring *sq = &cq->sq;
869 	u16 ntc = sq->next_to_clean;
870 	struct ice_sq_cd *details;
871 	struct ice_aq_desc *desc;
872 
873 	desc = ICE_CTL_Q_DESC(*sq, ntc);
874 	details = ICE_CTL_Q_DETAILS(*sq, ntc);
875 
876 	while (rd32(hw, cq->sq.head) != ntc) {
877 		ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
878 		memset(desc, 0, sizeof(*desc));
879 		memset(details, 0, sizeof(*details));
880 		ntc++;
881 		if (ntc == sq->count)
882 			ntc = 0;
883 		desc = ICE_CTL_Q_DESC(*sq, ntc);
884 		details = ICE_CTL_Q_DETAILS(*sq, ntc);
885 	}
886 
887 	sq->next_to_clean = ntc;
888 
889 	return ICE_CTL_Q_DESC_UNUSED(sq);
890 }
891 
892 /**
893  * ice_debug_cq
894  * @hw: pointer to the hardware structure
895  * @desc: pointer to control queue descriptor
896  * @buf: pointer to command buffer
897  * @buf_len: max length of buf
898  *
899  * Dumps debug log about control command with descriptor contents.
900  */
901 static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len)
902 {
903 	struct ice_aq_desc *cq_desc = desc;
904 	u16 len;
905 
906 	if (!IS_ENABLED(CONFIG_DYNAMIC_DEBUG) &&
907 	    !((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
908 		return;
909 
910 	if (!desc)
911 		return;
912 
913 	len = le16_to_cpu(cq_desc->datalen);
914 
915 	ice_debug(hw, ICE_DBG_AQ_DESC, "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
916 		  le16_to_cpu(cq_desc->opcode),
917 		  le16_to_cpu(cq_desc->flags),
918 		  le16_to_cpu(cq_desc->datalen), le16_to_cpu(cq_desc->retval));
919 	ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
920 		  le32_to_cpu(cq_desc->cookie_high),
921 		  le32_to_cpu(cq_desc->cookie_low));
922 	ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1)  0x%08X 0x%08X\n",
923 		  le32_to_cpu(cq_desc->params.generic.param0),
924 		  le32_to_cpu(cq_desc->params.generic.param1));
925 	ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l)   0x%08X 0x%08X\n",
926 		  le32_to_cpu(cq_desc->params.generic.addr_high),
927 		  le32_to_cpu(cq_desc->params.generic.addr_low));
928 	if (buf && cq_desc->datalen != 0) {
929 		ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
930 		if (buf_len < len)
931 			len = buf_len;
932 
933 		ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, buf, len);
934 	}
935 }
936 
937 /**
938  * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
939  * @hw: pointer to the HW struct
940  * @cq: pointer to the specific Control queue
941  *
942  * Returns true if the firmware has processed all descriptors on the
943  * admin send queue. Returns false if there are still requests pending.
944  */
945 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)
946 {
947 	/* AQ designers suggest use of head for better
948 	 * timing reliability than DD bit
949 	 */
950 	return rd32(hw, cq->sq.head) == cq->sq.next_to_use;
951 }
952 
953 /**
954  * ice_sq_send_cmd - send command to Control Queue (ATQ)
955  * @hw: pointer to the HW struct
956  * @cq: pointer to the specific Control queue
957  * @desc: prefilled descriptor describing the command
958  * @buf: buffer to use for indirect commands (or NULL for direct commands)
959  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
960  * @cd: pointer to command details structure
961  *
962  * This is the main send command routine for the ATQ. It runs the queue,
963  * cleans the queue, etc.
964  */
965 enum ice_status
966 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,
967 		struct ice_aq_desc *desc, void *buf, u16 buf_size,
968 		struct ice_sq_cd *cd)
969 {
970 	struct ice_dma_mem *dma_buf = NULL;
971 	struct ice_aq_desc *desc_on_ring;
972 	bool cmd_completed = false;
973 	enum ice_status status = 0;
974 	struct ice_sq_cd *details;
975 	u32 total_delay = 0;
976 	u16 retval = 0;
977 	u32 val = 0;
978 
979 	/* if reset is in progress return a soft error */
980 	if (hw->reset_ongoing)
981 		return ICE_ERR_RESET_ONGOING;
982 	mutex_lock(&cq->sq_lock);
983 
984 	cq->sq_last_status = ICE_AQ_RC_OK;
985 
986 	if (!cq->sq.count) {
987 		ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send queue not initialized.\n");
988 		status = ICE_ERR_AQ_EMPTY;
989 		goto sq_send_command_error;
990 	}
991 
992 	if ((buf && !buf_size) || (!buf && buf_size)) {
993 		status = ICE_ERR_PARAM;
994 		goto sq_send_command_error;
995 	}
996 
997 	if (buf) {
998 		if (buf_size > cq->sq_buf_size) {
999 			ice_debug(hw, ICE_DBG_AQ_MSG, "Invalid buffer size for Control Send queue: %d.\n",
1000 				  buf_size);
1001 			status = ICE_ERR_INVAL_SIZE;
1002 			goto sq_send_command_error;
1003 		}
1004 
1005 		desc->flags |= cpu_to_le16(ICE_AQ_FLAG_BUF);
1006 		if (buf_size > ICE_AQ_LG_BUF)
1007 			desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB);
1008 	}
1009 
1010 	val = rd32(hw, cq->sq.head);
1011 	if (val >= cq->num_sq_entries) {
1012 		ice_debug(hw, ICE_DBG_AQ_MSG, "head overrun at %d in the Control Send Queue ring\n",
1013 			  val);
1014 		status = ICE_ERR_AQ_EMPTY;
1015 		goto sq_send_command_error;
1016 	}
1017 
1018 	details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
1019 	if (cd)
1020 		*details = *cd;
1021 	else
1022 		memset(details, 0, sizeof(*details));
1023 
1024 	/* Call clean and check queue available function to reclaim the
1025 	 * descriptors that were processed by FW/MBX; the function returns the
1026 	 * number of desc available. The clean function called here could be
1027 	 * called in a separate thread in case of asynchronous completions.
1028 	 */
1029 	if (ice_clean_sq(hw, cq) == 0) {
1030 		ice_debug(hw, ICE_DBG_AQ_MSG, "Error: Control Send Queue is full.\n");
1031 		status = ICE_ERR_AQ_FULL;
1032 		goto sq_send_command_error;
1033 	}
1034 
1035 	/* initialize the temp desc pointer with the right desc */
1036 	desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);
1037 
1038 	/* if the desc is available copy the temp desc to the right place */
1039 	memcpy(desc_on_ring, desc, sizeof(*desc_on_ring));
1040 
1041 	/* if buf is not NULL assume indirect command */
1042 	if (buf) {
1043 		dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];
1044 		/* copy the user buf into the respective DMA buf */
1045 		memcpy(dma_buf->va, buf, buf_size);
1046 		desc_on_ring->datalen = cpu_to_le16(buf_size);
1047 
1048 		/* Update the address values in the desc with the pa value
1049 		 * for respective buffer
1050 		 */
1051 		desc_on_ring->params.generic.addr_high =
1052 			cpu_to_le32(upper_32_bits(dma_buf->pa));
1053 		desc_on_ring->params.generic.addr_low =
1054 			cpu_to_le32(lower_32_bits(dma_buf->pa));
1055 	}
1056 
1057 	/* Debug desc and buffer */
1058 	ice_debug(hw, ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n");
1059 
1060 	ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size);
1061 
1062 	(cq->sq.next_to_use)++;
1063 	if (cq->sq.next_to_use == cq->sq.count)
1064 		cq->sq.next_to_use = 0;
1065 	wr32(hw, cq->sq.tail, cq->sq.next_to_use);
1066 
1067 	do {
1068 		if (ice_sq_done(hw, cq))
1069 			break;
1070 
1071 		udelay(ICE_CTL_Q_SQ_CMD_USEC);
1072 		total_delay++;
1073 	} while (total_delay < cq->sq_cmd_timeout);
1074 
1075 	/* if ready, copy the desc back to temp */
1076 	if (ice_sq_done(hw, cq)) {
1077 		memcpy(desc, desc_on_ring, sizeof(*desc));
1078 		if (buf) {
1079 			/* get returned length to copy */
1080 			u16 copy_size = le16_to_cpu(desc->datalen);
1081 
1082 			if (copy_size > buf_size) {
1083 				ice_debug(hw, ICE_DBG_AQ_MSG, "Return len %d > than buf len %d\n",
1084 					  copy_size, buf_size);
1085 				status = ICE_ERR_AQ_ERROR;
1086 			} else {
1087 				memcpy(buf, dma_buf->va, copy_size);
1088 			}
1089 		}
1090 		retval = le16_to_cpu(desc->retval);
1091 		if (retval) {
1092 			ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue command 0x%04X completed with error 0x%X\n",
1093 				  le16_to_cpu(desc->opcode),
1094 				  retval);
1095 
1096 			/* strip off FW internal code */
1097 			retval &= 0xff;
1098 		}
1099 		cmd_completed = true;
1100 		if (!status && retval != ICE_AQ_RC_OK)
1101 			status = ICE_ERR_AQ_ERROR;
1102 		cq->sq_last_status = (enum ice_aq_err)retval;
1103 	}
1104 
1105 	ice_debug(hw, ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n");
1106 
1107 	ice_debug_cq(hw, (void *)desc, buf, buf_size);
1108 
1109 	/* save writeback AQ if requested */
1110 	if (details->wb_desc)
1111 		memcpy(details->wb_desc, desc_on_ring,
1112 		       sizeof(*details->wb_desc));
1113 
1114 	/* update the error if time out occurred */
1115 	if (!cmd_completed) {
1116 		if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask ||
1117 		    rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) {
1118 			ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n");
1119 			status = ICE_ERR_AQ_FW_CRITICAL;
1120 		} else {
1121 			ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue Writeback timeout.\n");
1122 			status = ICE_ERR_AQ_TIMEOUT;
1123 		}
1124 	}
1125 
1126 sq_send_command_error:
1127 	mutex_unlock(&cq->sq_lock);
1128 	return status;
1129 }
1130 
1131 /**
1132  * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function
1133  * @desc: pointer to the temp descriptor (non DMA mem)
1134  * @opcode: the opcode can be used to decide which flags to turn off or on
1135  *
1136  * Fill the desc with default values
1137  */
1138 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)
1139 {
1140 	/* zero out the desc */
1141 	memset(desc, 0, sizeof(*desc));
1142 	desc->opcode = cpu_to_le16(opcode);
1143 	desc->flags = cpu_to_le16(ICE_AQ_FLAG_SI);
1144 }
1145 
1146 /**
1147  * ice_clean_rq_elem
1148  * @hw: pointer to the HW struct
1149  * @cq: pointer to the specific Control queue
1150  * @e: event info from the receive descriptor, includes any buffers
1151  * @pending: number of events that could be left to process
1152  *
1153  * This function cleans one Admin Receive Queue element and returns
1154  * the contents through e. It can also return how many events are
1155  * left to process through 'pending'.
1156  */
1157 enum ice_status
1158 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,
1159 		  struct ice_rq_event_info *e, u16 *pending)
1160 {
1161 	u16 ntc = cq->rq.next_to_clean;
1162 	enum ice_aq_err rq_last_status;
1163 	enum ice_status ret_code = 0;
1164 	struct ice_aq_desc *desc;
1165 	struct ice_dma_mem *bi;
1166 	u16 desc_idx;
1167 	u16 datalen;
1168 	u16 flags;
1169 	u16 ntu;
1170 
1171 	/* pre-clean the event info */
1172 	memset(&e->desc, 0, sizeof(e->desc));
1173 
1174 	/* take the lock before we start messing with the ring */
1175 	mutex_lock(&cq->rq_lock);
1176 
1177 	if (!cq->rq.count) {
1178 		ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive queue not initialized.\n");
1179 		ret_code = ICE_ERR_AQ_EMPTY;
1180 		goto clean_rq_elem_err;
1181 	}
1182 
1183 	/* set next_to_use to head */
1184 	ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1185 
1186 	if (ntu == ntc) {
1187 		/* nothing to do - shouldn't need to update ring's values */
1188 		ret_code = ICE_ERR_AQ_NO_WORK;
1189 		goto clean_rq_elem_out;
1190 	}
1191 
1192 	/* now clean the next descriptor */
1193 	desc = ICE_CTL_Q_DESC(cq->rq, ntc);
1194 	desc_idx = ntc;
1195 
1196 	rq_last_status = (enum ice_aq_err)le16_to_cpu(desc->retval);
1197 	flags = le16_to_cpu(desc->flags);
1198 	if (flags & ICE_AQ_FLAG_ERR) {
1199 		ret_code = ICE_ERR_AQ_ERROR;
1200 		ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive Queue Event 0x%04X received with error 0x%X\n",
1201 			  le16_to_cpu(desc->opcode), rq_last_status);
1202 	}
1203 	memcpy(&e->desc, desc, sizeof(e->desc));
1204 	datalen = le16_to_cpu(desc->datalen);
1205 	e->msg_len = min_t(u16, datalen, e->buf_len);
1206 	if (e->msg_buf && e->msg_len)
1207 		memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len);
1208 
1209 	ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n");
1210 
1211 	ice_debug_cq(hw, (void *)desc, e->msg_buf, cq->rq_buf_size);
1212 
1213 	/* Restore the original datalen and buffer address in the desc,
1214 	 * FW updates datalen to indicate the event message size
1215 	 */
1216 	bi = &cq->rq.r.rq_bi[ntc];
1217 	memset(desc, 0, sizeof(*desc));
1218 
1219 	desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF);
1220 	if (cq->rq_buf_size > ICE_AQ_LG_BUF)
1221 		desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB);
1222 	desc->datalen = cpu_to_le16(bi->size);
1223 	desc->params.generic.addr_high = cpu_to_le32(upper_32_bits(bi->pa));
1224 	desc->params.generic.addr_low = cpu_to_le32(lower_32_bits(bi->pa));
1225 
1226 	/* set tail = the last cleaned desc index. */
1227 	wr32(hw, cq->rq.tail, ntc);
1228 	/* ntc is updated to tail + 1 */
1229 	ntc++;
1230 	if (ntc == cq->num_rq_entries)
1231 		ntc = 0;
1232 	cq->rq.next_to_clean = ntc;
1233 	cq->rq.next_to_use = ntu;
1234 
1235 clean_rq_elem_out:
1236 	/* Set pending if needed, unlock and return */
1237 	if (pending) {
1238 		/* re-read HW head to calculate actual pending messages */
1239 		ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1240 		*pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));
1241 	}
1242 clean_rq_elem_err:
1243 	mutex_unlock(&cq->rq_lock);
1244 
1245 	return ret_code;
1246 }
1247