Lines Matching +full:supports +full:- +full:cqe
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
3 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
74 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; in efa_com_reg_read32()
80 read_resp = mmio_read->read_resp; in efa_com_reg_read32()
82 spin_lock(&mmio_read->lock); in efa_com_reg_read32()
83 mmio_read->seq_num++; in efa_com_reg_read32()
86 read_resp->req_id = mmio_read->seq_num + 0x9aL; in efa_com_reg_read32()
89 mmio_read->seq_num); in efa_com_reg_read32()
91 writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF); in efa_com_reg_read32()
93 exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout); in efa_com_reg_read32()
95 if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num) in efa_com_reg_read32()
100 if (read_resp->req_id != mmio_read->seq_num) { in efa_com_reg_read32()
102 edev->efa_dev, in efa_com_reg_read32()
104 mmio_read->seq_num, offset, read_resp->req_id, in efa_com_reg_read32()
105 read_resp->reg_off); in efa_com_reg_read32()
110 if (read_resp->reg_off != offset) { in efa_com_reg_read32()
112 edev->efa_dev, in efa_com_reg_read32()
118 err = read_resp->reg_val; in efa_com_reg_read32()
120 spin_unlock(&mmio_read->lock); in efa_com_reg_read32()
126 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_init_sq()
127 struct efa_com_admin_sq *sq = &aq->sq; in efa_com_admin_init_sq()
128 u16 size = aq->depth * sizeof(*sq->entries); in efa_com_admin_init_sq()
133 sq->entries = in efa_com_admin_init_sq()
134 dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL); in efa_com_admin_init_sq()
135 if (!sq->entries) in efa_com_admin_init_sq()
136 return -ENOMEM; in efa_com_admin_init_sq()
138 spin_lock_init(&sq->lock); in efa_com_admin_init_sq()
140 sq->cc = 0; in efa_com_admin_init_sq()
141 sq->pc = 0; in efa_com_admin_init_sq()
142 sq->phase = 1; in efa_com_admin_init_sq()
144 sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF); in efa_com_admin_init_sq()
146 addr_high = upper_32_bits(sq->dma_addr); in efa_com_admin_init_sq()
147 addr_low = lower_32_bits(sq->dma_addr); in efa_com_admin_init_sq()
149 writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF); in efa_com_admin_init_sq()
150 writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF); in efa_com_admin_init_sq()
152 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth); in efa_com_admin_init_sq()
156 writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF); in efa_com_admin_init_sq()
163 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_init_cq()
164 struct efa_com_admin_cq *cq = &aq->cq; in efa_com_admin_init_cq()
165 u16 size = aq->depth * sizeof(*cq->entries); in efa_com_admin_init_cq()
170 cq->entries = in efa_com_admin_init_cq()
171 dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); in efa_com_admin_init_cq()
172 if (!cq->entries) in efa_com_admin_init_cq()
173 return -ENOMEM; in efa_com_admin_init_cq()
175 spin_lock_init(&cq->lock); in efa_com_admin_init_cq()
177 cq->cc = 0; in efa_com_admin_init_cq()
178 cq->phase = 1; in efa_com_admin_init_cq()
180 addr_high = upper_32_bits(cq->dma_addr); in efa_com_admin_init_cq()
181 addr_low = lower_32_bits(cq->dma_addr); in efa_com_admin_init_cq()
183 writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF); in efa_com_admin_init_cq()
184 writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF); in efa_com_admin_init_cq()
186 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth); in efa_com_admin_init_cq()
190 aq->msix_vector_idx); in efa_com_admin_init_cq()
192 writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF); in efa_com_admin_init_cq()
200 struct efa_com_aenq *aenq = &edev->aenq; in efa_com_admin_init_aenq()
206 ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n"); in efa_com_admin_init_aenq()
207 return -EINVAL; in efa_com_admin_init_aenq()
210 size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries); in efa_com_admin_init_aenq()
211 aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr, in efa_com_admin_init_aenq()
213 if (!aenq->entries) in efa_com_admin_init_aenq()
214 return -ENOMEM; in efa_com_admin_init_aenq()
216 aenq->aenq_handlers = aenq_handlers; in efa_com_admin_init_aenq()
217 aenq->depth = EFA_ASYNC_QUEUE_DEPTH; in efa_com_admin_init_aenq()
218 aenq->cc = 0; in efa_com_admin_init_aenq()
219 aenq->phase = 1; in efa_com_admin_init_aenq()
221 addr_low = lower_32_bits(aenq->dma_addr); in efa_com_admin_init_aenq()
222 addr_high = upper_32_bits(aenq->dma_addr); in efa_com_admin_init_aenq()
224 writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF); in efa_com_admin_init_aenq()
225 writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF); in efa_com_admin_init_aenq()
227 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth); in efa_com_admin_init_aenq()
231 aenq->msix_vector_idx); in efa_com_admin_init_aenq()
232 writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF); in efa_com_admin_init_aenq()
238 writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); in efa_com_admin_init_aenq()
248 spin_lock(&aq->comp_ctx_lock); in efa_com_alloc_ctx_id()
249 ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next]; in efa_com_alloc_ctx_id()
250 aq->comp_ctx_pool_next++; in efa_com_alloc_ctx_id()
251 spin_unlock(&aq->comp_ctx_lock); in efa_com_alloc_ctx_id()
259 spin_lock(&aq->comp_ctx_lock); in efa_com_dealloc_ctx_id()
260 aq->comp_ctx_pool_next--; in efa_com_dealloc_ctx_id()
261 aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id; in efa_com_dealloc_ctx_id()
262 spin_unlock(&aq->comp_ctx_lock); in efa_com_dealloc_ctx_id()
268 u16 cmd_id = EFA_GET(&comp_ctx->user_cqe->acq_common_descriptor.command, in efa_com_put_comp_ctx()
270 u16 ctx_id = cmd_id & (aq->depth - 1); in efa_com_put_comp_ctx()
272 ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id); in efa_com_put_comp_ctx()
273 comp_ctx->occupied = 0; in efa_com_put_comp_ctx()
280 u16 ctx_id = cmd_id & (aq->depth - 1); in efa_com_get_comp_ctx()
282 if (aq->comp_ctx[ctx_id].occupied && capture) { in efa_com_get_comp_ctx()
284 aq->efa_dev, in efa_com_get_comp_ctx()
291 aq->comp_ctx[ctx_id].occupied = 1; in efa_com_get_comp_ctx()
292 ibdev_dbg(aq->efa_dev, in efa_com_get_comp_ctx()
296 return &aq->comp_ctx[ctx_id]; in efa_com_get_comp_ctx()
312 queue_size_mask = aq->depth - 1; in __efa_com_submit_admin_cmd()
313 pi = aq->sq.pc & queue_size_mask; in __efa_com_submit_admin_cmd()
319 cmd_id |= aq->sq.pc & ~queue_size_mask; in __efa_com_submit_admin_cmd()
322 cmd->aq_common_descriptor.command_id = cmd_id; in __efa_com_submit_admin_cmd()
323 EFA_SET(&cmd->aq_common_descriptor.flags, in __efa_com_submit_admin_cmd()
324 EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase); in __efa_com_submit_admin_cmd()
329 return ERR_PTR(-EINVAL); in __efa_com_submit_admin_cmd()
332 comp_ctx->status = EFA_CMD_SUBMITTED; in __efa_com_submit_admin_cmd()
333 comp_ctx->comp_size = comp_size_in_bytes; in __efa_com_submit_admin_cmd()
334 comp_ctx->user_cqe = comp; in __efa_com_submit_admin_cmd()
335 comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode; in __efa_com_submit_admin_cmd()
337 reinit_completion(&comp_ctx->wait_event); in __efa_com_submit_admin_cmd()
339 aqe = &aq->sq.entries[pi]; in __efa_com_submit_admin_cmd()
343 aq->sq.pc++; in __efa_com_submit_admin_cmd()
344 atomic64_inc(&aq->stats.submitted_cmd); in __efa_com_submit_admin_cmd()
346 if ((aq->sq.pc & queue_size_mask) == 0) in __efa_com_submit_admin_cmd()
347 aq->sq.phase = !aq->sq.phase; in __efa_com_submit_admin_cmd()
350 writel(aq->sq.pc, aq->sq.db_addr); in __efa_com_submit_admin_cmd()
357 size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool); in efa_com_init_comp_ctxt()
358 size_t size = aq->depth * sizeof(struct efa_comp_ctx); in efa_com_init_comp_ctxt()
362 aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL); in efa_com_init_comp_ctxt()
363 aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL); in efa_com_init_comp_ctxt()
364 if (!aq->comp_ctx || !aq->comp_ctx_pool) { in efa_com_init_comp_ctxt()
365 devm_kfree(aq->dmadev, aq->comp_ctx_pool); in efa_com_init_comp_ctxt()
366 devm_kfree(aq->dmadev, aq->comp_ctx); in efa_com_init_comp_ctxt()
367 return -ENOMEM; in efa_com_init_comp_ctxt()
370 for (i = 0; i < aq->depth; i++) { in efa_com_init_comp_ctxt()
373 init_completion(&comp_ctx->wait_event); in efa_com_init_comp_ctxt()
375 aq->comp_ctx_pool[i] = i; in efa_com_init_comp_ctxt()
378 spin_lock_init(&aq->comp_ctx_lock); in efa_com_init_comp_ctxt()
380 aq->comp_ctx_pool_next = 0; in efa_com_init_comp_ctxt()
393 spin_lock(&aq->sq.lock); in efa_com_submit_admin_cmd()
394 if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { in efa_com_submit_admin_cmd()
395 ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); in efa_com_submit_admin_cmd()
396 spin_unlock(&aq->sq.lock); in efa_com_submit_admin_cmd()
397 return ERR_PTR(-ENODEV); in efa_com_submit_admin_cmd()
402 spin_unlock(&aq->sq.lock); in efa_com_submit_admin_cmd()
404 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_submit_admin_cmd()
410 struct efa_admin_acq_entry *cqe) in efa_com_handle_single_admin_completion() argument
415 cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, in efa_com_handle_single_admin_completion()
419 if (comp_ctx->status != EFA_CMD_SUBMITTED) { in efa_com_handle_single_admin_completion()
420 ibdev_err(aq->efa_dev, in efa_com_handle_single_admin_completion()
422 cmd_id, aq->sq.pc, aq->sq.cc, aq->cq.cc); in efa_com_handle_single_admin_completion()
423 return -EINVAL; in efa_com_handle_single_admin_completion()
426 comp_ctx->status = EFA_CMD_COMPLETED; in efa_com_handle_single_admin_completion()
427 memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size); in efa_com_handle_single_admin_completion()
429 if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) in efa_com_handle_single_admin_completion()
430 complete(&comp_ctx->wait_event); in efa_com_handle_single_admin_completion()
437 struct efa_admin_acq_entry *cqe; in efa_com_handle_admin_completion() local
444 queue_size_mask = aq->depth - 1; in efa_com_handle_admin_completion()
446 ci = aq->cq.cc & queue_size_mask; in efa_com_handle_admin_completion()
447 phase = aq->cq.phase; in efa_com_handle_admin_completion()
449 cqe = &aq->cq.entries[ci]; in efa_com_handle_admin_completion()
452 while ((READ_ONCE(cqe->acq_common_descriptor.flags) & in efa_com_handle_admin_completion()
459 err = efa_com_handle_single_admin_completion(aq, cqe); in efa_com_handle_admin_completion()
463 aq->cq.cc++; in efa_com_handle_admin_completion()
465 if (ci == aq->depth) { in efa_com_handle_admin_completion()
470 cqe = &aq->cq.entries[ci]; in efa_com_handle_admin_completion()
473 aq->cq.phase = phase; in efa_com_handle_admin_completion()
474 aq->sq.cc += comp_cmds; in efa_com_handle_admin_completion()
475 atomic64_add(comp_cmds, &aq->stats.completed_cmd); in efa_com_handle_admin_completion()
484 return -ENOMEM; in efa_com_comp_status_to_errno()
486 return -EOPNOTSUPP; in efa_com_comp_status_to_errno()
491 return -EINVAL; in efa_com_comp_status_to_errno()
493 return -EINVAL; in efa_com_comp_status_to_errno()
504 timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); in efa_com_wait_and_process_admin_cq_polling()
507 spin_lock_irqsave(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_polling()
509 spin_unlock_irqrestore(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_polling()
511 if (comp_ctx->status != EFA_CMD_SUBMITTED) in efa_com_wait_and_process_admin_cq_polling()
516 aq->efa_dev, in efa_com_wait_and_process_admin_cq_polling()
519 atomic64_inc(&aq->stats.no_completion); in efa_com_wait_and_process_admin_cq_polling()
521 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_wait_and_process_admin_cq_polling()
522 err = -ETIME; in efa_com_wait_and_process_admin_cq_polling()
526 msleep(aq->poll_interval); in efa_com_wait_and_process_admin_cq_polling()
529 err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); in efa_com_wait_and_process_admin_cq_polling()
541 wait_for_completion_timeout(&comp_ctx->wait_event, in efa_com_wait_and_process_admin_cq_interrupts()
542 usecs_to_jiffies(aq->completion_timeout)); in efa_com_wait_and_process_admin_cq_interrupts()
548 * 2) There is completion but the device didn't get any msi-x interrupt. in efa_com_wait_and_process_admin_cq_interrupts()
550 if (comp_ctx->status == EFA_CMD_SUBMITTED) { in efa_com_wait_and_process_admin_cq_interrupts()
551 spin_lock_irqsave(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_interrupts()
553 spin_unlock_irqrestore(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_interrupts()
555 atomic64_inc(&aq->stats.no_completion); in efa_com_wait_and_process_admin_cq_interrupts()
557 if (comp_ctx->status == EFA_CMD_COMPLETED) in efa_com_wait_and_process_admin_cq_interrupts()
559 aq->efa_dev, in efa_com_wait_and_process_admin_cq_interrupts()
560 …"The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%… in efa_com_wait_and_process_admin_cq_interrupts()
561 efa_com_cmd_str(comp_ctx->cmd_opcode), in efa_com_wait_and_process_admin_cq_interrupts()
562 comp_ctx->cmd_opcode, comp_ctx->status, in efa_com_wait_and_process_admin_cq_interrupts()
563 comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc); in efa_com_wait_and_process_admin_cq_interrupts()
566 aq->efa_dev, in efa_com_wait_and_process_admin_cq_interrupts()
568 efa_com_cmd_str(comp_ctx->cmd_opcode), in efa_com_wait_and_process_admin_cq_interrupts()
569 comp_ctx->cmd_opcode, comp_ctx->status, in efa_com_wait_and_process_admin_cq_interrupts()
570 comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc); in efa_com_wait_and_process_admin_cq_interrupts()
572 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_wait_and_process_admin_cq_interrupts()
573 err = -ETIME; in efa_com_wait_and_process_admin_cq_interrupts()
577 err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); in efa_com_wait_and_process_admin_cq_interrupts()
585 * Polling mode - wait until the completion is available.
586 * Async mode - wait on wait queue until the completion is ready
594 if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) in efa_com_wait_and_process_admin_cq()
601 * efa_com_cmd_exec - Execute admin command
611 * @return - 0 on success, negative value on failure.
625 down(&aq->avail_cmds); in efa_com_cmd_exec()
627 ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", in efa_com_cmd_exec()
628 efa_com_cmd_str(cmd->aq_common_descriptor.opcode), in efa_com_cmd_exec()
629 cmd->aq_common_descriptor.opcode); in efa_com_cmd_exec()
633 aq->efa_dev, in efa_com_cmd_exec()
635 efa_com_cmd_str(cmd->aq_common_descriptor.opcode), in efa_com_cmd_exec()
636 cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx)); in efa_com_cmd_exec()
638 up(&aq->avail_cmds); in efa_com_cmd_exec()
639 atomic64_inc(&aq->stats.cmd_err); in efa_com_cmd_exec()
646 aq->efa_dev, in efa_com_cmd_exec()
648 efa_com_cmd_str(cmd->aq_common_descriptor.opcode), in efa_com_cmd_exec()
649 cmd->aq_common_descriptor.opcode, in efa_com_cmd_exec()
650 comp_ctx->user_cqe->acq_common_descriptor.status, err); in efa_com_cmd_exec()
651 atomic64_inc(&aq->stats.cmd_err); in efa_com_cmd_exec()
654 up(&aq->avail_cmds); in efa_com_cmd_exec()
660 * efa_com_admin_destroy - Destroy the admin and the async events queues.
665 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_destroy()
666 struct efa_com_aenq *aenq = &edev->aenq; in efa_com_admin_destroy()
667 struct efa_com_admin_cq *cq = &aq->cq; in efa_com_admin_destroy()
668 struct efa_com_admin_sq *sq = &aq->sq; in efa_com_admin_destroy()
671 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_admin_destroy()
673 devm_kfree(edev->dmadev, aq->comp_ctx_pool); in efa_com_admin_destroy()
674 devm_kfree(edev->dmadev, aq->comp_ctx); in efa_com_admin_destroy()
676 size = aq->depth * sizeof(*sq->entries); in efa_com_admin_destroy()
677 dma_free_coherent(edev->dmadev, size, sq->entries, sq->dma_addr); in efa_com_admin_destroy()
679 size = aq->depth * sizeof(*cq->entries); in efa_com_admin_destroy()
680 dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr); in efa_com_admin_destroy()
682 size = aenq->depth * sizeof(*aenq->entries); in efa_com_admin_destroy()
683 dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr); in efa_com_admin_destroy()
687 * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode
700 writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF); in efa_com_set_admin_polling_mode()
702 set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); in efa_com_set_admin_polling_mode()
704 clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); in efa_com_set_admin_polling_mode()
709 atomic64_t *s = (atomic64_t *)&edev->aq.stats; in efa_com_stats_init()
712 for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++) in efa_com_stats_init()
717 * efa_com_admin_init - Init the admin and the async queues
724 * @return - 0 on success, negative value on failure.
729 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_init()
737 ibdev_err(edev->efa_dev, in efa_com_admin_init()
739 return -ENODEV; in efa_com_admin_init()
742 aq->depth = EFA_ADMIN_QUEUE_DEPTH; in efa_com_admin_init()
744 aq->dmadev = edev->dmadev; in efa_com_admin_init()
745 aq->efa_dev = edev->efa_dev; in efa_com_admin_init()
746 set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state); in efa_com_admin_init()
748 sema_init(&aq->avail_cmds, aq->depth); in efa_com_admin_init()
774 aq->completion_timeout = timeout * 100000; in efa_com_admin_init()
776 aq->completion_timeout = ADMIN_CMD_TIMEOUT_US; in efa_com_admin_init()
778 aq->poll_interval = EFA_POLL_INTERVAL_MS; in efa_com_admin_init()
780 set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_admin_init()
785 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries), in efa_com_admin_init()
786 aq->cq.entries, aq->cq.dma_addr); in efa_com_admin_init()
788 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries), in efa_com_admin_init()
789 aq->sq.entries, aq->sq.dma_addr); in efa_com_admin_init()
791 devm_kfree(edev->dmadev, aq->comp_ctx); in efa_com_admin_init()
797 * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler
803 * Note: Should be called after MSI-X interrupt.
809 spin_lock_irqsave(&edev->aq.cq.lock, flags); in efa_com_admin_q_comp_intr_handler()
810 efa_com_handle_admin_completion(&edev->aq); in efa_com_admin_q_comp_intr_handler()
811 spin_unlock_irqrestore(&edev->aq.cq.lock, flags); in efa_com_admin_q_comp_intr_handler()
821 struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers; in efa_com_get_specific_aenq_cb()
823 if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group]) in efa_com_get_specific_aenq_cb()
824 return aenq_handlers->handlers[group]; in efa_com_get_specific_aenq_cb()
826 return aenq_handlers->unimplemented_handler; in efa_com_get_specific_aenq_cb()
830 * efa_com_aenq_intr_handler - AENQ interrupt handler
839 struct efa_com_aenq *aenq = &edev->aenq; in efa_com_aenq_intr_handler()
846 ci = aenq->cc & (aenq->depth - 1); in efa_com_aenq_intr_handler()
847 phase = aenq->phase; in efa_com_aenq_intr_handler()
848 aenq_e = &aenq->entries[ci]; /* Get first entry */ in efa_com_aenq_intr_handler()
849 aenq_common = &aenq_e->aenq_common_desc; in efa_com_aenq_intr_handler()
852 while ((READ_ONCE(aenq_common->flags) & in efa_com_aenq_intr_handler()
862 aenq_common->group); in efa_com_aenq_intr_handler()
869 if (ci == aenq->depth) { in efa_com_aenq_intr_handler()
873 aenq_e = &aenq->entries[ci]; in efa_com_aenq_intr_handler()
874 aenq_common = &aenq_e->aenq_common_desc; in efa_com_aenq_intr_handler()
877 aenq->cc += processed; in efa_com_aenq_intr_handler()
878 aenq->phase = phase; in efa_com_aenq_intr_handler()
885 writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); in efa_com_aenq_intr_handler()
890 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; in efa_com_mmio_reg_read_resp_addr_init()
895 addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0); in efa_com_mmio_reg_read_resp_addr_init()
896 addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0); in efa_com_mmio_reg_read_resp_addr_init()
898 writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF); in efa_com_mmio_reg_read_resp_addr_init()
899 writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF); in efa_com_mmio_reg_read_resp_addr_init()
904 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; in efa_com_mmio_reg_read_init()
906 spin_lock_init(&mmio_read->lock); in efa_com_mmio_reg_read_init()
907 mmio_read->read_resp = in efa_com_mmio_reg_read_init()
908 dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), in efa_com_mmio_reg_read_init()
909 &mmio_read->read_resp_dma_addr, GFP_KERNEL); in efa_com_mmio_reg_read_init()
910 if (!mmio_read->read_resp) in efa_com_mmio_reg_read_init()
911 return -ENOMEM; in efa_com_mmio_reg_read_init()
915 mmio_read->read_resp->req_id = 0; in efa_com_mmio_reg_read_init()
916 mmio_read->seq_num = 0; in efa_com_mmio_reg_read_init()
917 mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US; in efa_com_mmio_reg_read_init()
924 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; in efa_com_mmio_reg_read_destroy()
926 dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), in efa_com_mmio_reg_read_destroy()
927 mmio_read->read_resp, mmio_read->read_resp_dma_addr); in efa_com_mmio_reg_read_destroy()
946 ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n", in efa_com_validate_version()
955 ibdev_err(edev->efa_dev, in efa_com_validate_version()
956 "EFA version is lower than the minimal version the driver supports\n"); in efa_com_validate_version()
957 return -EOPNOTSUPP; in efa_com_validate_version()
961 edev->efa_dev, in efa_com_validate_version()
983 ibdev_err(edev->efa_dev, in efa_com_validate_version()
984 "EFA ctrl version is lower than the minimal ctrl version the driver supports\n"); in efa_com_validate_version()
985 return -EOPNOTSUPP; in efa_com_validate_version()
992 * efa_com_get_dma_width - Retrieve physical dma address width the device
993 * supports.
1007 ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width); in efa_com_get_dma_width()
1010 ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width); in efa_com_get_dma_width()
1011 return -EINVAL; in efa_com_get_dma_width()
1014 edev->dma_addr_bits = width; in efa_com_get_dma_width()
1029 ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val); in wait_for_reset_state()
1033 return -ETIME; in wait_for_reset_state()
1037 * efa_com_dev_reset - Perform device FLR to the device.
1041 * @return - 0 on success, negative value on failure.
1054 ibdev_err(edev->efa_dev, in efa_com_dev_reset()
1056 return -EINVAL; in efa_com_dev_reset()
1061 ibdev_err(edev->efa_dev, "Invalid timeout value\n"); in efa_com_dev_reset()
1062 return -EINVAL; in efa_com_dev_reset()
1068 writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); in efa_com_dev_reset()
1075 ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n"); in efa_com_dev_reset()
1080 writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); in efa_com_dev_reset()
1083 ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n"); in efa_com_dev_reset()
1090 edev->aq.completion_timeout = timeout * 100000; in efa_com_dev_reset()
1092 edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US; in efa_com_dev_reset()
1101 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_create_eq()
1108 params->entry_size_in_bytes / 4); in efa_com_create_eq()
1109 cmd.depth = params->depth; in efa_com_create_eq()
1110 cmd.event_bitmask = params->event_bitmask; in efa_com_create_eq()
1111 cmd.msix_vec = params->msix_vec; in efa_com_create_eq()
1113 efa_com_set_dma_addr(params->dma_addr, &cmd.ba.mem_addr_high, in efa_com_create_eq()
1122 ibdev_err_ratelimited(edev->efa_dev, in efa_com_create_eq()
1127 result->eqn = resp.eqn; in efa_com_create_eq()
1135 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_destroy_eq()
1141 cmd.eqn = params->eqn; in efa_com_destroy_eq()
1149 ibdev_err_ratelimited(edev->efa_dev, in efa_com_destroy_eq()
1150 "Failed to destroy EQ-%u [%d]\n", cmd.eqn, in efa_com_destroy_eq()
1158 EFA_SET(&val, EFA_REGS_EQ_DB_EQN, eeq->eqn); in efa_com_arm_eq()
1161 writel(val, edev->reg_bar + EFA_REGS_EQ_DB_OFF); in efa_com_arm_eq()
1172 ci = eeq->cc & (eeq->depth - 1); in efa_com_eq_comp_intr_handler()
1173 phase = eeq->phase; in efa_com_eq_comp_intr_handler()
1174 eqe = &eeq->eqes[ci]; in efa_com_eq_comp_intr_handler()
1177 while ((READ_ONCE(eqe->common) & EFA_ADMIN_EQE_PHASE_MASK) == phase) { in efa_com_eq_comp_intr_handler()
1184 eeq->cb(eeq, eqe); in efa_com_eq_comp_intr_handler()
1190 if (ci == eeq->depth) { in efa_com_eq_comp_intr_handler()
1195 eqe = &eeq->eqes[ci]; in efa_com_eq_comp_intr_handler()
1198 eeq->cc += processed; in efa_com_eq_comp_intr_handler()
1199 eeq->phase = phase; in efa_com_eq_comp_intr_handler()
1200 efa_com_arm_eq(eeq->edev, eeq); in efa_com_eq_comp_intr_handler()
1206 .eqn = eeq->eqn, in efa_com_eq_destroy()
1210 dma_free_coherent(edev->dmadev, eeq->depth * sizeof(*eeq->eqes), in efa_com_eq_destroy()
1211 eeq->eqes, eeq->dma_addr); in efa_com_eq_destroy()
1222 params.entry_size_in_bytes = sizeof(*eeq->eqes); in efa_com_eq_init()
1227 eeq->eqes = dma_alloc_coherent(edev->dmadev, in efa_com_eq_init()
1228 params.depth * sizeof(*eeq->eqes), in efa_com_eq_init()
1230 if (!eeq->eqes) in efa_com_eq_init()
1231 return -ENOMEM; in efa_com_eq_init()
1237 eeq->eqn = result.eqn; in efa_com_eq_init()
1238 eeq->edev = edev; in efa_com_eq_init()
1239 eeq->dma_addr = params.dma_addr; in efa_com_eq_init()
1240 eeq->phase = 1; in efa_com_eq_init()
1241 eeq->depth = params.depth; in efa_com_eq_init()
1242 eeq->cb = cb; in efa_com_eq_init()
1248 dma_free_coherent(edev->dmadev, params.depth * sizeof(*eeq->eqes), in efa_com_eq_init()
1249 eeq->eqes, params.dma_addr); in efa_com_eq_init()