1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
2 /*
3  * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #include "efa_com.h"
7 #include "efa_regs_defs.h"
8 
9 #define ADMIN_CMD_TIMEOUT_US 30000000 /* usecs */
10 
11 #define EFA_REG_READ_TIMEOUT_US 50000 /* usecs */
12 #define EFA_MMIO_READ_INVALID 0xffffffff
13 
14 #define EFA_POLL_INTERVAL_MS 100 /* msecs */
15 
16 #define EFA_ASYNC_QUEUE_DEPTH 16
17 #define EFA_ADMIN_QUEUE_DEPTH 32
18 
19 #define EFA_CTRL_MAJOR          0
20 #define EFA_CTRL_MINOR          0
21 #define EFA_CTRL_SUB_MINOR      1
22 
23 enum efa_cmd_status {
24 	EFA_CMD_SUBMITTED,
25 	EFA_CMD_COMPLETED,
26 };
27 
28 struct efa_comp_ctx {
29 	struct completion wait_event;
30 	struct efa_admin_acq_entry *user_cqe;
31 	u32 comp_size;
32 	enum efa_cmd_status status;
33 	u8 cmd_opcode;
34 	u8 occupied;
35 };
36 
37 static const char *efa_com_cmd_str(u8 cmd)
38 {
39 #define EFA_CMD_STR_CASE(_cmd) case EFA_ADMIN_##_cmd: return #_cmd
40 
41 	switch (cmd) {
42 	EFA_CMD_STR_CASE(CREATE_QP);
43 	EFA_CMD_STR_CASE(MODIFY_QP);
44 	EFA_CMD_STR_CASE(QUERY_QP);
45 	EFA_CMD_STR_CASE(DESTROY_QP);
46 	EFA_CMD_STR_CASE(CREATE_AH);
47 	EFA_CMD_STR_CASE(DESTROY_AH);
48 	EFA_CMD_STR_CASE(REG_MR);
49 	EFA_CMD_STR_CASE(DEREG_MR);
50 	EFA_CMD_STR_CASE(CREATE_CQ);
51 	EFA_CMD_STR_CASE(DESTROY_CQ);
52 	EFA_CMD_STR_CASE(GET_FEATURE);
53 	EFA_CMD_STR_CASE(SET_FEATURE);
54 	EFA_CMD_STR_CASE(GET_STATS);
55 	EFA_CMD_STR_CASE(ALLOC_PD);
56 	EFA_CMD_STR_CASE(DEALLOC_PD);
57 	EFA_CMD_STR_CASE(ALLOC_UAR);
58 	EFA_CMD_STR_CASE(DEALLOC_UAR);
59 	EFA_CMD_STR_CASE(CREATE_EQ);
60 	EFA_CMD_STR_CASE(DESTROY_EQ);
61 	default: return "unknown command opcode";
62 	}
63 #undef EFA_CMD_STR_CASE
64 }
65 
66 void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low)
67 {
68 	*addr_low = lower_32_bits(addr);
69 	*addr_high = upper_32_bits(addr);
70 }
71 
72 static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)
73 {
74 	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
75 	struct efa_admin_mmio_req_read_less_resp *read_resp;
76 	unsigned long exp_time;
77 	u32 mmio_read_reg = 0;
78 	u32 err;
79 
80 	read_resp = mmio_read->read_resp;
81 
82 	spin_lock(&mmio_read->lock);
83 	mmio_read->seq_num++;
84 
85 	/* trash DMA req_id to identify when hardware is done */
86 	read_resp->req_id = mmio_read->seq_num + 0x9aL;
87 	EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset);
88 	EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID,
89 		mmio_read->seq_num);
90 
91 	writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF);
92 
93 	exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout);
94 	do {
95 		if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num)
96 			break;
97 		udelay(1);
98 	} while (time_is_after_jiffies(exp_time));
99 
100 	if (read_resp->req_id != mmio_read->seq_num) {
101 		ibdev_err_ratelimited(
102 			edev->efa_dev,
103 			"Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n",
104 			mmio_read->seq_num, offset, read_resp->req_id,
105 			read_resp->reg_off);
106 		err = EFA_MMIO_READ_INVALID;
107 		goto out;
108 	}
109 
110 	if (read_resp->reg_off != offset) {
111 		ibdev_err_ratelimited(
112 			edev->efa_dev,
113 			"Reading register failed: wrong offset provided\n");
114 		err = EFA_MMIO_READ_INVALID;
115 		goto out;
116 	}
117 
118 	err = read_resp->reg_val;
119 out:
120 	spin_unlock(&mmio_read->lock);
121 	return err;
122 }
123 
124 static int efa_com_admin_init_sq(struct efa_com_dev *edev)
125 {
126 	struct efa_com_admin_queue *aq = &edev->aq;
127 	struct efa_com_admin_sq *sq = &aq->sq;
128 	u16 size = aq->depth * sizeof(*sq->entries);
129 	u32 aq_caps = 0;
130 	u32 addr_high;
131 	u32 addr_low;
132 
133 	sq->entries =
134 		dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL);
135 	if (!sq->entries)
136 		return -ENOMEM;
137 
138 	spin_lock_init(&sq->lock);
139 
140 	sq->cc = 0;
141 	sq->pc = 0;
142 	sq->phase = 1;
143 
144 	sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF);
145 
146 	addr_high = upper_32_bits(sq->dma_addr);
147 	addr_low = lower_32_bits(sq->dma_addr);
148 
149 	writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF);
150 	writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF);
151 
152 	EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth);
153 	EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE,
154 		sizeof(struct efa_admin_aq_entry));
155 
156 	writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF);
157 
158 	return 0;
159 }
160 
161 static int efa_com_admin_init_cq(struct efa_com_dev *edev)
162 {
163 	struct efa_com_admin_queue *aq = &edev->aq;
164 	struct efa_com_admin_cq *cq = &aq->cq;
165 	u16 size = aq->depth * sizeof(*cq->entries);
166 	u32 acq_caps = 0;
167 	u32 addr_high;
168 	u32 addr_low;
169 
170 	cq->entries =
171 		dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);
172 	if (!cq->entries)
173 		return -ENOMEM;
174 
175 	spin_lock_init(&cq->lock);
176 
177 	cq->cc = 0;
178 	cq->phase = 1;
179 
180 	addr_high = upper_32_bits(cq->dma_addr);
181 	addr_low = lower_32_bits(cq->dma_addr);
182 
183 	writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF);
184 	writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF);
185 
186 	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth);
187 	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE,
188 		sizeof(struct efa_admin_acq_entry));
189 	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR,
190 		aq->msix_vector_idx);
191 
192 	writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF);
193 
194 	return 0;
195 }
196 
197 static int efa_com_admin_init_aenq(struct efa_com_dev *edev,
198 				   struct efa_aenq_handlers *aenq_handlers)
199 {
200 	struct efa_com_aenq *aenq = &edev->aenq;
201 	u32 addr_low, addr_high;
202 	u32 aenq_caps = 0;
203 	u16 size;
204 
205 	if (!aenq_handlers) {
206 		ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n");
207 		return -EINVAL;
208 	}
209 
210 	size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries);
211 	aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr,
212 					   GFP_KERNEL);
213 	if (!aenq->entries)
214 		return -ENOMEM;
215 
216 	aenq->aenq_handlers = aenq_handlers;
217 	aenq->depth = EFA_ASYNC_QUEUE_DEPTH;
218 	aenq->cc = 0;
219 	aenq->phase = 1;
220 
221 	addr_low = lower_32_bits(aenq->dma_addr);
222 	addr_high = upper_32_bits(aenq->dma_addr);
223 
224 	writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF);
225 	writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF);
226 
227 	EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth);
228 	EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE,
229 		sizeof(struct efa_admin_aenq_entry));
230 	EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR,
231 		aenq->msix_vector_idx);
232 	writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF);
233 
234 	/*
235 	 * Init cons_db to mark that all entries in the queue
236 	 * are initially available
237 	 */
238 	writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF);
239 
240 	return 0;
241 }
242 
243 /* ID to be used with efa_com_get_comp_ctx */
244 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq)
245 {
246 	u16 ctx_id;
247 
248 	spin_lock(&aq->comp_ctx_lock);
249 	ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next];
250 	aq->comp_ctx_pool_next++;
251 	spin_unlock(&aq->comp_ctx_lock);
252 
253 	return ctx_id;
254 }
255 
256 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,
257 				   u16 ctx_id)
258 {
259 	spin_lock(&aq->comp_ctx_lock);
260 	aq->comp_ctx_pool_next--;
261 	aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id;
262 	spin_unlock(&aq->comp_ctx_lock);
263 }
264 
265 static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,
266 					struct efa_comp_ctx *comp_ctx)
267 {
268 	u16 cmd_id = EFA_GET(&comp_ctx->user_cqe->acq_common_descriptor.command,
269 			     EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
270 	u16 ctx_id = cmd_id & (aq->depth - 1);
271 
272 	ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id);
273 	comp_ctx->occupied = 0;
274 	efa_com_dealloc_ctx_id(aq, ctx_id);
275 }
276 
277 static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq,
278 						 u16 cmd_id, bool capture)
279 {
280 	u16 ctx_id = cmd_id & (aq->depth - 1);
281 
282 	if (aq->comp_ctx[ctx_id].occupied && capture) {
283 		ibdev_err_ratelimited(
284 			aq->efa_dev,
285 			"Completion context for command_id %#x is occupied\n",
286 			cmd_id);
287 		return NULL;
288 	}
289 
290 	if (capture) {
291 		aq->comp_ctx[ctx_id].occupied = 1;
292 		ibdev_dbg(aq->efa_dev,
293 			  "Take completion ctxt for command_id %#x\n", cmd_id);
294 	}
295 
296 	return &aq->comp_ctx[ctx_id];
297 }
298 
299 static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
300 						       struct efa_admin_aq_entry *cmd,
301 						       size_t cmd_size_in_bytes,
302 						       struct efa_admin_acq_entry *comp,
303 						       size_t comp_size_in_bytes)
304 {
305 	struct efa_admin_aq_entry *aqe;
306 	struct efa_comp_ctx *comp_ctx;
307 	u16 queue_size_mask;
308 	u16 cmd_id;
309 	u16 ctx_id;
310 	u16 pi;
311 
312 	queue_size_mask = aq->depth - 1;
313 	pi = aq->sq.pc & queue_size_mask;
314 
315 	ctx_id = efa_com_alloc_ctx_id(aq);
316 
317 	/* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */
318 	cmd_id = ctx_id & queue_size_mask;
319 	cmd_id |= aq->sq.pc & ~queue_size_mask;
320 	cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
321 
322 	cmd->aq_common_descriptor.command_id = cmd_id;
323 	EFA_SET(&cmd->aq_common_descriptor.flags,
324 		EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase);
325 
326 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true);
327 	if (!comp_ctx) {
328 		efa_com_dealloc_ctx_id(aq, ctx_id);
329 		return ERR_PTR(-EINVAL);
330 	}
331 
332 	comp_ctx->status = EFA_CMD_SUBMITTED;
333 	comp_ctx->comp_size = comp_size_in_bytes;
334 	comp_ctx->user_cqe = comp;
335 	comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
336 
337 	reinit_completion(&comp_ctx->wait_event);
338 
339 	aqe = &aq->sq.entries[pi];
340 	memset(aqe, 0, sizeof(*aqe));
341 	memcpy(aqe, cmd, cmd_size_in_bytes);
342 
343 	aq->sq.pc++;
344 	atomic64_inc(&aq->stats.submitted_cmd);
345 
346 	if ((aq->sq.pc & queue_size_mask) == 0)
347 		aq->sq.phase = !aq->sq.phase;
348 
349 	/* barrier not needed in case of writel */
350 	writel(aq->sq.pc, aq->sq.db_addr);
351 
352 	return comp_ctx;
353 }
354 
355 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)
356 {
357 	size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool);
358 	size_t size = aq->depth * sizeof(struct efa_comp_ctx);
359 	struct efa_comp_ctx *comp_ctx;
360 	u16 i;
361 
362 	aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL);
363 	aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL);
364 	if (!aq->comp_ctx || !aq->comp_ctx_pool) {
365 		devm_kfree(aq->dmadev, aq->comp_ctx_pool);
366 		devm_kfree(aq->dmadev, aq->comp_ctx);
367 		return -ENOMEM;
368 	}
369 
370 	for (i = 0; i < aq->depth; i++) {
371 		comp_ctx = efa_com_get_comp_ctx(aq, i, false);
372 		if (comp_ctx)
373 			init_completion(&comp_ctx->wait_event);
374 
375 		aq->comp_ctx_pool[i] = i;
376 	}
377 
378 	spin_lock_init(&aq->comp_ctx_lock);
379 
380 	aq->comp_ctx_pool_next = 0;
381 
382 	return 0;
383 }
384 
385 static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
386 						     struct efa_admin_aq_entry *cmd,
387 						     size_t cmd_size_in_bytes,
388 						     struct efa_admin_acq_entry *comp,
389 						     size_t comp_size_in_bytes)
390 {
391 	struct efa_comp_ctx *comp_ctx;
392 
393 	spin_lock(&aq->sq.lock);
394 	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) {
395 		ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n");
396 		spin_unlock(&aq->sq.lock);
397 		return ERR_PTR(-ENODEV);
398 	}
399 
400 	comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp,
401 					      comp_size_in_bytes);
402 	spin_unlock(&aq->sq.lock);
403 	if (IS_ERR(comp_ctx))
404 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
405 
406 	return comp_ctx;
407 }
408 
409 static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,
410 						   struct efa_admin_acq_entry *cqe)
411 {
412 	struct efa_comp_ctx *comp_ctx;
413 	u16 cmd_id;
414 
415 	cmd_id = EFA_GET(&cqe->acq_common_descriptor.command,
416 			 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
417 
418 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);
419 	if (!comp_ctx) {
420 		ibdev_err(aq->efa_dev,
421 			  "comp_ctx is NULL. Changing the admin queue running state\n");
422 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
423 		return;
424 	}
425 
426 	comp_ctx->status = EFA_CMD_COMPLETED;
427 	memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size);
428 
429 	if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))
430 		complete(&comp_ctx->wait_event);
431 }
432 
433 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)
434 {
435 	struct efa_admin_acq_entry *cqe;
436 	u16 queue_size_mask;
437 	u16 comp_num = 0;
438 	u8 phase;
439 	u16 ci;
440 
441 	queue_size_mask = aq->depth - 1;
442 
443 	ci = aq->cq.cc & queue_size_mask;
444 	phase = aq->cq.phase;
445 
446 	cqe = &aq->cq.entries[ci];
447 
448 	/* Go over all the completions */
449 	while ((READ_ONCE(cqe->acq_common_descriptor.flags) &
450 		EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
451 		/*
452 		 * Do not read the rest of the completion entry before the
453 		 * phase bit was validated
454 		 */
455 		dma_rmb();
456 		efa_com_handle_single_admin_completion(aq, cqe);
457 
458 		ci++;
459 		comp_num++;
460 		if (ci == aq->depth) {
461 			ci = 0;
462 			phase = !phase;
463 		}
464 
465 		cqe = &aq->cq.entries[ci];
466 	}
467 
468 	aq->cq.cc += comp_num;
469 	aq->cq.phase = phase;
470 	aq->sq.cc += comp_num;
471 	atomic64_add(comp_num, &aq->stats.completed_cmd);
472 }
473 
474 static int efa_com_comp_status_to_errno(u8 comp_status)
475 {
476 	switch (comp_status) {
477 	case EFA_ADMIN_SUCCESS:
478 		return 0;
479 	case EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
480 		return -ENOMEM;
481 	case EFA_ADMIN_UNSUPPORTED_OPCODE:
482 		return -EOPNOTSUPP;
483 	case EFA_ADMIN_BAD_OPCODE:
484 	case EFA_ADMIN_MALFORMED_REQUEST:
485 	case EFA_ADMIN_ILLEGAL_PARAMETER:
486 	case EFA_ADMIN_UNKNOWN_ERROR:
487 		return -EINVAL;
488 	default:
489 		return -EINVAL;
490 	}
491 }
492 
493 static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx,
494 						     struct efa_com_admin_queue *aq)
495 {
496 	unsigned long timeout;
497 	unsigned long flags;
498 	int err;
499 
500 	timeout = jiffies + usecs_to_jiffies(aq->completion_timeout);
501 
502 	while (1) {
503 		spin_lock_irqsave(&aq->cq.lock, flags);
504 		efa_com_handle_admin_completion(aq);
505 		spin_unlock_irqrestore(&aq->cq.lock, flags);
506 
507 		if (comp_ctx->status != EFA_CMD_SUBMITTED)
508 			break;
509 
510 		if (time_is_before_jiffies(timeout)) {
511 			ibdev_err_ratelimited(
512 				aq->efa_dev,
513 				"Wait for completion (polling) timeout\n");
514 			/* EFA didn't have any completion */
515 			atomic64_inc(&aq->stats.no_completion);
516 
517 			clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
518 			err = -ETIME;
519 			goto out;
520 		}
521 
522 		msleep(aq->poll_interval);
523 	}
524 
525 	err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status);
526 out:
527 	efa_com_put_comp_ctx(aq, comp_ctx);
528 	return err;
529 }
530 
531 static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx,
532 							struct efa_com_admin_queue *aq)
533 {
534 	unsigned long flags;
535 	int err;
536 
537 	wait_for_completion_timeout(&comp_ctx->wait_event,
538 				    usecs_to_jiffies(aq->completion_timeout));
539 
540 	/*
541 	 * In case the command wasn't completed find out the root cause.
542 	 * There might be 2 kinds of errors
543 	 * 1) No completion (timeout reached)
544 	 * 2) There is completion but the device didn't get any msi-x interrupt.
545 	 */
546 	if (comp_ctx->status == EFA_CMD_SUBMITTED) {
547 		spin_lock_irqsave(&aq->cq.lock, flags);
548 		efa_com_handle_admin_completion(aq);
549 		spin_unlock_irqrestore(&aq->cq.lock, flags);
550 
551 		atomic64_inc(&aq->stats.no_completion);
552 
553 		if (comp_ctx->status == EFA_CMD_COMPLETED)
554 			ibdev_err_ratelimited(
555 				aq->efa_dev,
556 				"The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (ctx: 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",
557 				efa_com_cmd_str(comp_ctx->cmd_opcode),
558 				comp_ctx->cmd_opcode, comp_ctx->status,
559 				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);
560 		else
561 			ibdev_err_ratelimited(
562 				aq->efa_dev,
563 				"The device didn't send any completion for admin cmd %s(%d) status %d (ctx 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",
564 				efa_com_cmd_str(comp_ctx->cmd_opcode),
565 				comp_ctx->cmd_opcode, comp_ctx->status,
566 				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);
567 
568 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
569 		err = -ETIME;
570 		goto out;
571 	}
572 
573 	err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status);
574 out:
575 	efa_com_put_comp_ctx(aq, comp_ctx);
576 	return err;
577 }
578 
579 /*
580  * There are two types to wait for completion.
581  * Polling mode - wait until the completion is available.
582  * Async mode - wait on wait queue until the completion is ready
583  * (or the timeout expired).
584  * It is expected that the IRQ called efa_com_handle_admin_completion
585  * to mark the completions.
586  */
587 static int efa_com_wait_and_process_admin_cq(struct efa_comp_ctx *comp_ctx,
588 					     struct efa_com_admin_queue *aq)
589 {
590 	if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))
591 		return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq);
592 
593 	return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq);
594 }
595 
596 /**
597  * efa_com_cmd_exec - Execute admin command
598  * @aq: admin queue.
599  * @cmd: the admin command to execute.
600  * @cmd_size: the command size.
601  * @comp: command completion return entry.
602  * @comp_size: command completion size.
603  * Submit an admin command and then wait until the device will return a
604  * completion.
605  * The completion will be copied into comp.
606  *
607  * @return - 0 on success, negative value on failure.
608  */
609 int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
610 		     struct efa_admin_aq_entry *cmd,
611 		     size_t cmd_size,
612 		     struct efa_admin_acq_entry *comp,
613 		     size_t comp_size)
614 {
615 	struct efa_comp_ctx *comp_ctx;
616 	int err;
617 
618 	might_sleep();
619 
620 	/* In case of queue FULL */
621 	down(&aq->avail_cmds);
622 
623 	ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n",
624 		  efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
625 		  cmd->aq_common_descriptor.opcode);
626 	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);
627 	if (IS_ERR(comp_ctx)) {
628 		ibdev_err_ratelimited(
629 			aq->efa_dev,
630 			"Failed to submit command %s (opcode %u) err %ld\n",
631 			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
632 			cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx));
633 
634 		up(&aq->avail_cmds);
635 		atomic64_inc(&aq->stats.cmd_err);
636 		return PTR_ERR(comp_ctx);
637 	}
638 
639 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);
640 	if (err) {
641 		ibdev_err_ratelimited(
642 			aq->efa_dev,
643 			"Failed to process command %s (opcode %u) comp_status %d err %d\n",
644 			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
645 			cmd->aq_common_descriptor.opcode,
646 			comp_ctx->user_cqe->acq_common_descriptor.status, err);
647 		atomic64_inc(&aq->stats.cmd_err);
648 	}
649 
650 	up(&aq->avail_cmds);
651 
652 	return err;
653 }
654 
655 /**
656  * efa_com_admin_destroy - Destroy the admin and the async events queues.
657  * @edev: EFA communication layer struct
658  */
659 void efa_com_admin_destroy(struct efa_com_dev *edev)
660 {
661 	struct efa_com_admin_queue *aq = &edev->aq;
662 	struct efa_com_aenq *aenq = &edev->aenq;
663 	struct efa_com_admin_cq *cq = &aq->cq;
664 	struct efa_com_admin_sq *sq = &aq->sq;
665 	u16 size;
666 
667 	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
668 
669 	devm_kfree(edev->dmadev, aq->comp_ctx_pool);
670 	devm_kfree(edev->dmadev, aq->comp_ctx);
671 
672 	size = aq->depth * sizeof(*sq->entries);
673 	dma_free_coherent(edev->dmadev, size, sq->entries, sq->dma_addr);
674 
675 	size = aq->depth * sizeof(*cq->entries);
676 	dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr);
677 
678 	size = aenq->depth * sizeof(*aenq->entries);
679 	dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr);
680 }
681 
682 /**
683  * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode
684  * @edev: EFA communication layer struct
685  * @polling: Enable/Disable polling mode
686  *
687  * Set the admin completion mode.
688  */
689 void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling)
690 {
691 	u32 mask_value = 0;
692 
693 	if (polling)
694 		EFA_SET(&mask_value, EFA_REGS_INTR_MASK_EN, 1);
695 
696 	writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF);
697 	if (polling)
698 		set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);
699 	else
700 		clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);
701 }
702 
703 static void efa_com_stats_init(struct efa_com_dev *edev)
704 {
705 	atomic64_t *s = (atomic64_t *)&edev->aq.stats;
706 	int i;
707 
708 	for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++)
709 		atomic64_set(s, 0);
710 }
711 
712 /**
713  * efa_com_admin_init - Init the admin and the async queues
714  * @edev: EFA communication layer struct
715  * @aenq_handlers: Those handlers to be called upon event.
716  *
717  * Initialize the admin submission and completion queues.
718  * Initialize the asynchronous events notification queues.
719  *
720  * @return - 0 on success, negative value on failure.
721  */
722 int efa_com_admin_init(struct efa_com_dev *edev,
723 		       struct efa_aenq_handlers *aenq_handlers)
724 {
725 	struct efa_com_admin_queue *aq = &edev->aq;
726 	u32 timeout;
727 	u32 dev_sts;
728 	u32 cap;
729 	int err;
730 
731 	dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
732 	if (!EFA_GET(&dev_sts, EFA_REGS_DEV_STS_READY)) {
733 		ibdev_err(edev->efa_dev,
734 			  "Device isn't ready, abort com init %#x\n", dev_sts);
735 		return -ENODEV;
736 	}
737 
738 	aq->depth = EFA_ADMIN_QUEUE_DEPTH;
739 
740 	aq->dmadev = edev->dmadev;
741 	aq->efa_dev = edev->efa_dev;
742 	set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state);
743 
744 	sema_init(&aq->avail_cmds, aq->depth);
745 
746 	efa_com_stats_init(edev);
747 
748 	err = efa_com_init_comp_ctxt(aq);
749 	if (err)
750 		return err;
751 
752 	err = efa_com_admin_init_sq(edev);
753 	if (err)
754 		goto err_destroy_comp_ctxt;
755 
756 	err = efa_com_admin_init_cq(edev);
757 	if (err)
758 		goto err_destroy_sq;
759 
760 	efa_com_set_admin_polling_mode(edev, false);
761 
762 	err = efa_com_admin_init_aenq(edev, aenq_handlers);
763 	if (err)
764 		goto err_destroy_cq;
765 
766 	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
767 	timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO);
768 	if (timeout)
769 		/* the resolution of timeout reg is 100ms */
770 		aq->completion_timeout = timeout * 100000;
771 	else
772 		aq->completion_timeout = ADMIN_CMD_TIMEOUT_US;
773 
774 	aq->poll_interval = EFA_POLL_INTERVAL_MS;
775 
776 	set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
777 
778 	return 0;
779 
780 err_destroy_cq:
781 	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries),
782 			  aq->cq.entries, aq->cq.dma_addr);
783 err_destroy_sq:
784 	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries),
785 			  aq->sq.entries, aq->sq.dma_addr);
786 err_destroy_comp_ctxt:
787 	devm_kfree(edev->dmadev, aq->comp_ctx);
788 
789 	return err;
790 }
791 
792 /**
793  * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler
794  * @edev: EFA communication layer struct
795  *
796  * This method goes over the admin completion queue and wakes up
797  * all the pending threads that wait on the commands wait event.
798  *
799  * Note: Should be called after MSI-X interrupt.
800  */
801 void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev)
802 {
803 	unsigned long flags;
804 
805 	spin_lock_irqsave(&edev->aq.cq.lock, flags);
806 	efa_com_handle_admin_completion(&edev->aq);
807 	spin_unlock_irqrestore(&edev->aq.cq.lock, flags);
808 }
809 
810 /*
811  * efa_handle_specific_aenq_event:
812  * return the handler that is relevant to the specific event group
813  */
814 static efa_aenq_handler efa_com_get_specific_aenq_cb(struct efa_com_dev *edev,
815 						     u16 group)
816 {
817 	struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers;
818 
819 	if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group])
820 		return aenq_handlers->handlers[group];
821 
822 	return aenq_handlers->unimplemented_handler;
823 }
824 
825 /**
826  * efa_com_aenq_intr_handler - AENQ interrupt handler
827  * @edev: EFA communication layer struct
828  * @data: Data of interrupt handler.
829  *
830  * Go over the async event notification queue and call the proper aenq handler.
831  */
832 void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data)
833 {
834 	struct efa_admin_aenq_common_desc *aenq_common;
835 	struct efa_com_aenq *aenq = &edev->aenq;
836 	struct efa_admin_aenq_entry *aenq_e;
837 	efa_aenq_handler handler_cb;
838 	u32 processed = 0;
839 	u8 phase;
840 	u32 ci;
841 
842 	ci = aenq->cc & (aenq->depth - 1);
843 	phase = aenq->phase;
844 	aenq_e = &aenq->entries[ci]; /* Get first entry */
845 	aenq_common = &aenq_e->aenq_common_desc;
846 
847 	/* Go over all the events */
848 	while ((READ_ONCE(aenq_common->flags) &
849 		EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) {
850 		/*
851 		 * Do not read the rest of the completion entry before the
852 		 * phase bit was validated
853 		 */
854 		dma_rmb();
855 
856 		/* Handle specific event*/
857 		handler_cb = efa_com_get_specific_aenq_cb(edev,
858 							  aenq_common->group);
859 		handler_cb(data, aenq_e); /* call the actual event handler*/
860 
861 		/* Get next event entry */
862 		ci++;
863 		processed++;
864 
865 		if (ci == aenq->depth) {
866 			ci = 0;
867 			phase = !phase;
868 		}
869 		aenq_e = &aenq->entries[ci];
870 		aenq_common = &aenq_e->aenq_common_desc;
871 	}
872 
873 	aenq->cc += processed;
874 	aenq->phase = phase;
875 
876 	/* Don't update aenq doorbell if there weren't any processed events */
877 	if (!processed)
878 		return;
879 
880 	/* barrier not needed in case of writel */
881 	writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF);
882 }
883 
884 static void efa_com_mmio_reg_read_resp_addr_init(struct efa_com_dev *edev)
885 {
886 	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
887 	u32 addr_high;
888 	u32 addr_low;
889 
890 	/* dma_addr_bits is unknown at this point */
891 	addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0);
892 	addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0);
893 
894 	writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF);
895 	writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF);
896 }
897 
898 int efa_com_mmio_reg_read_init(struct efa_com_dev *edev)
899 {
900 	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
901 
902 	spin_lock_init(&mmio_read->lock);
903 	mmio_read->read_resp =
904 		dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp),
905 				   &mmio_read->read_resp_dma_addr, GFP_KERNEL);
906 	if (!mmio_read->read_resp)
907 		return -ENOMEM;
908 
909 	efa_com_mmio_reg_read_resp_addr_init(edev);
910 
911 	mmio_read->read_resp->req_id = 0;
912 	mmio_read->seq_num = 0;
913 	mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US;
914 
915 	return 0;
916 }
917 
918 void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev)
919 {
920 	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
921 
922 	dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp),
923 			  mmio_read->read_resp, mmio_read->read_resp_dma_addr);
924 }
925 
926 int efa_com_validate_version(struct efa_com_dev *edev)
927 {
928 	u32 min_ctrl_ver = 0;
929 	u32 ctrl_ver_masked;
930 	u32 min_ver = 0;
931 	u32 ctrl_ver;
932 	u32 ver;
933 
934 	/*
935 	 * Make sure the EFA version and the controller version are at least
936 	 * as the driver expects
937 	 */
938 	ver = efa_com_reg_read32(edev, EFA_REGS_VERSION_OFF);
939 	ctrl_ver = efa_com_reg_read32(edev,
940 				      EFA_REGS_CONTROLLER_VERSION_OFF);
941 
942 	ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n",
943 		  EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION),
944 		  EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION));
945 
946 	EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION,
947 		EFA_ADMIN_API_VERSION_MAJOR);
948 	EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION,
949 		EFA_ADMIN_API_VERSION_MINOR);
950 	if (ver < min_ver) {
951 		ibdev_err(edev->efa_dev,
952 			  "EFA version is lower than the minimal version the driver supports\n");
953 		return -EOPNOTSUPP;
954 	}
955 
956 	ibdev_dbg(
957 		edev->efa_dev,
958 		"efa controller version: %d.%d.%d implementation version %d\n",
959 		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION),
960 		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION),
961 		EFA_GET(&ctrl_ver,
962 			EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION),
963 		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID));
964 
965 	ctrl_ver_masked =
966 		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) |
967 		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) |
968 		EFA_GET(&ctrl_ver,
969 			EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION);
970 
971 	EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION,
972 		EFA_CTRL_MAJOR);
973 	EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION,
974 		EFA_CTRL_MINOR);
975 	EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION,
976 		EFA_CTRL_SUB_MINOR);
977 	/* Validate the ctrl version without the implementation ID */
978 	if (ctrl_ver_masked < min_ctrl_ver) {
979 		ibdev_err(edev->efa_dev,
980 			  "EFA ctrl version is lower than the minimal ctrl version the driver supports\n");
981 		return -EOPNOTSUPP;
982 	}
983 
984 	return 0;
985 }
986 
987 /**
988  * efa_com_get_dma_width - Retrieve physical dma address width the device
989  * supports.
990  * @edev: EFA communication layer struct
991  *
992  * Retrieve the maximum physical address bits the device can handle.
993  *
994  * @return: > 0 on Success and negative value otherwise.
995  */
996 int efa_com_get_dma_width(struct efa_com_dev *edev)
997 {
998 	u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
999 	int width;
1000 
1001 	width = EFA_GET(&caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH);
1002 
1003 	ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width);
1004 
1005 	if (width < 32 || width > 64) {
1006 		ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width);
1007 		return -EINVAL;
1008 	}
1009 
1010 	edev->dma_addr_bits = width;
1011 
1012 	return width;
1013 }
1014 
1015 static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on)
1016 {
1017 	u32 val, i;
1018 
1019 	for (i = 0; i < timeout; i++) {
1020 		val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
1021 
1022 		if (EFA_GET(&val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on)
1023 			return 0;
1024 
1025 		ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val);
1026 		msleep(EFA_POLL_INTERVAL_MS);
1027 	}
1028 
1029 	return -ETIME;
1030 }
1031 
1032 /**
1033  * efa_com_dev_reset - Perform device FLR to the device.
1034  * @edev: EFA communication layer struct
1035  * @reset_reason: Specify what is the trigger for the reset in case of an error.
1036  *
1037  * @return - 0 on success, negative value on failure.
1038  */
1039 int efa_com_dev_reset(struct efa_com_dev *edev,
1040 		      enum efa_regs_reset_reason_types reset_reason)
1041 {
1042 	u32 stat, timeout, cap;
1043 	u32 reset_val = 0;
1044 	int err;
1045 
1046 	stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
1047 	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
1048 
1049 	if (!EFA_GET(&stat, EFA_REGS_DEV_STS_READY)) {
1050 		ibdev_err(edev->efa_dev,
1051 			  "Device isn't ready, can't reset device\n");
1052 		return -EINVAL;
1053 	}
1054 
1055 	timeout = EFA_GET(&cap, EFA_REGS_CAPS_RESET_TIMEOUT);
1056 	if (!timeout) {
1057 		ibdev_err(edev->efa_dev, "Invalid timeout value\n");
1058 		return -EINVAL;
1059 	}
1060 
1061 	/* start reset */
1062 	EFA_SET(&reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1);
1063 	EFA_SET(&reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason);
1064 	writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
1065 
1066 	/* reset clears the mmio readless address, restore it */
1067 	efa_com_mmio_reg_read_resp_addr_init(edev);
1068 
1069 	err = wait_for_reset_state(edev, timeout, 1);
1070 	if (err) {
1071 		ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n");
1072 		return err;
1073 	}
1074 
1075 	/* reset done */
1076 	writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
1077 	err = wait_for_reset_state(edev, timeout, 0);
1078 	if (err) {
1079 		ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n");
1080 		return err;
1081 	}
1082 
1083 	timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO);
1084 	if (timeout)
1085 		/* the resolution of timeout reg is 100ms */
1086 		edev->aq.completion_timeout = timeout * 100000;
1087 	else
1088 		edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US;
1089 
1090 	return 0;
1091 }
1092 
1093 static int efa_com_create_eq(struct efa_com_dev *edev,
1094 			     struct efa_com_create_eq_params *params,
1095 			     struct efa_com_create_eq_result *result)
1096 {
1097 	struct efa_com_admin_queue *aq = &edev->aq;
1098 	struct efa_admin_create_eq_resp resp = {};
1099 	struct efa_admin_create_eq_cmd cmd = {};
1100 	int err;
1101 
1102 	cmd.aq_common_descriptor.opcode = EFA_ADMIN_CREATE_EQ;
1103 	EFA_SET(&cmd.caps, EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS,
1104 		params->entry_size_in_bytes / 4);
1105 	cmd.depth = params->depth;
1106 	cmd.event_bitmask = params->event_bitmask;
1107 	cmd.msix_vec = params->msix_vec;
1108 
1109 	efa_com_set_dma_addr(params->dma_addr, &cmd.ba.mem_addr_high,
1110 			     &cmd.ba.mem_addr_low);
1111 
1112 	err = efa_com_cmd_exec(aq,
1113 			       (struct efa_admin_aq_entry *)&cmd,
1114 			       sizeof(cmd),
1115 			       (struct efa_admin_acq_entry *)&resp,
1116 			       sizeof(resp));
1117 	if (err) {
1118 		ibdev_err_ratelimited(edev->efa_dev,
1119 				      "Failed to create eq[%d]\n", err);
1120 		return err;
1121 	}
1122 
1123 	result->eqn = resp.eqn;
1124 
1125 	return 0;
1126 }
1127 
1128 static void efa_com_destroy_eq(struct efa_com_dev *edev,
1129 			       struct efa_com_destroy_eq_params *params)
1130 {
1131 	struct efa_com_admin_queue *aq = &edev->aq;
1132 	struct efa_admin_destroy_eq_resp resp = {};
1133 	struct efa_admin_destroy_eq_cmd cmd = {};
1134 	int err;
1135 
1136 	cmd.aq_common_descriptor.opcode = EFA_ADMIN_DESTROY_EQ;
1137 	cmd.eqn = params->eqn;
1138 
1139 	err = efa_com_cmd_exec(aq,
1140 			       (struct efa_admin_aq_entry *)&cmd,
1141 			       sizeof(cmd),
1142 			       (struct efa_admin_acq_entry *)&resp,
1143 			       sizeof(resp));
1144 	if (err)
1145 		ibdev_err_ratelimited(edev->efa_dev,
1146 				      "Failed to destroy EQ-%u [%d]\n", cmd.eqn,
1147 				      err);
1148 }
1149 
1150 static void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq)
1151 {
1152 	u32 val = 0;
1153 
1154 	EFA_SET(&val, EFA_REGS_EQ_DB_EQN, eeq->eqn);
1155 	EFA_SET(&val, EFA_REGS_EQ_DB_ARM, 1);
1156 
1157 	writel(val, edev->reg_bar + EFA_REGS_EQ_DB_OFF);
1158 }
1159 
1160 void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev,
1161 				  struct efa_com_eq *eeq)
1162 {
1163 	struct efa_admin_eqe *eqe;
1164 	u32 processed = 0;
1165 	u8 phase;
1166 	u32 ci;
1167 
1168 	ci = eeq->cc & (eeq->depth - 1);
1169 	phase = eeq->phase;
1170 	eqe = &eeq->eqes[ci];
1171 
1172 	/* Go over all the events */
1173 	while ((READ_ONCE(eqe->common) & EFA_ADMIN_EQE_PHASE_MASK) == phase) {
1174 		/*
1175 		 * Do not read the rest of the completion entry before the
1176 		 * phase bit was validated
1177 		 */
1178 		dma_rmb();
1179 
1180 		eeq->cb(eeq, eqe);
1181 
1182 		/* Get next event entry */
1183 		ci++;
1184 		processed++;
1185 
1186 		if (ci == eeq->depth) {
1187 			ci = 0;
1188 			phase = !phase;
1189 		}
1190 
1191 		eqe = &eeq->eqes[ci];
1192 	}
1193 
1194 	eeq->cc += processed;
1195 	eeq->phase = phase;
1196 	efa_com_arm_eq(eeq->edev, eeq);
1197 }
1198 
1199 void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq)
1200 {
1201 	struct efa_com_destroy_eq_params params = {
1202 		.eqn = eeq->eqn,
1203 	};
1204 
1205 	efa_com_destroy_eq(edev, &params);
1206 	dma_free_coherent(edev->dmadev, eeq->depth * sizeof(*eeq->eqes),
1207 			  eeq->eqes, eeq->dma_addr);
1208 }
1209 
1210 int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq,
1211 		    efa_eqe_handler cb, u16 depth, u8 msix_vec)
1212 {
1213 	struct efa_com_create_eq_params params = {};
1214 	struct efa_com_create_eq_result result = {};
1215 	int err;
1216 
1217 	params.depth = depth;
1218 	params.entry_size_in_bytes = sizeof(*eeq->eqes);
1219 	EFA_SET(&params.event_bitmask,
1220 		EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS, 1);
1221 	params.msix_vec = msix_vec;
1222 
1223 	eeq->eqes = dma_alloc_coherent(edev->dmadev,
1224 				       params.depth * sizeof(*eeq->eqes),
1225 				       &params.dma_addr, GFP_KERNEL);
1226 	if (!eeq->eqes)
1227 		return -ENOMEM;
1228 
1229 	err = efa_com_create_eq(edev, &params, &result);
1230 	if (err)
1231 		goto err_free_coherent;
1232 
1233 	eeq->eqn = result.eqn;
1234 	eeq->edev = edev;
1235 	eeq->dma_addr = params.dma_addr;
1236 	eeq->phase = 1;
1237 	eeq->depth = params.depth;
1238 	eeq->cb = cb;
1239 	efa_com_arm_eq(edev, eeq);
1240 
1241 	return 0;
1242 
1243 err_free_coherent:
1244 	dma_free_coherent(edev->dmadev, params.depth * sizeof(*eeq->eqes),
1245 			  eeq->eqes, params.dma_addr);
1246 	return err;
1247 }
1248