Lines Matching refs:cq
169 static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) in get_cqe() argument
171 return get_cqe_from_buf(&cq->buf, entry); in get_cqe()
179 static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) in next_cqe_sw() argument
181 return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe)); in next_cqe_sw()
204 static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq, in update_cons_index() argument
208 *cq->set_ci_db = cpu_to_be32(cq->cons_index); in update_cons_index()
211 mthca_write64(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn, incr - 1, in update_cons_index()
219 struct mthca_cq *cq; in mthca_cq_completion() local
221 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); in mthca_cq_completion()
223 if (!cq) { in mthca_cq_completion()
228 ++cq->arm_sn; in mthca_cq_completion()
230 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in mthca_cq_completion()
236 struct mthca_cq *cq; in mthca_cq_event() local
241 cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); in mthca_cq_event()
242 if (cq) in mthca_cq_event()
243 ++cq->refcount; in mthca_cq_event()
247 if (!cq) { in mthca_cq_event()
254 event.element.cq = &cq->ibcq; in mthca_cq_event()
255 if (cq->ibcq.event_handler) in mthca_cq_event()
256 cq->ibcq.event_handler(&event, cq->ibcq.cq_context); in mthca_cq_event()
259 if (!--cq->refcount) in mthca_cq_event()
260 wake_up(&cq->wait); in mthca_cq_event()
273 void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, in mthca_cq_clean() argument
280 spin_lock_irq(&cq->lock); in mthca_cq_clean()
289 for (prod_index = cq->cons_index; in mthca_cq_clean()
290 cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe)); in mthca_cq_clean()
292 if (prod_index == cq->cons_index + cq->ibcq.cqe) in mthca_cq_clean()
297 qpn, cq->cqn, cq->cons_index, prod_index); in mthca_cq_clean()
303 while ((int) --prod_index - (int) cq->cons_index >= 0) { in mthca_cq_clean()
304 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in mthca_cq_clean()
310 memcpy(get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe), in mthca_cq_clean()
316 set_cqe_hw(get_cqe(cq, (cq->cons_index + i) & cq->ibcq.cqe)); in mthca_cq_clean()
318 cq->cons_index += nfreed; in mthca_cq_clean()
319 update_cons_index(dev, cq, nfreed); in mthca_cq_clean()
322 spin_unlock_irq(&cq->lock); in mthca_cq_clean()
325 void mthca_cq_resize_copy_cqes(struct mthca_cq *cq) in mthca_cq_resize_copy_cqes() argument
335 if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) && in mthca_cq_resize_copy_cqes()
336 cq->ibcq.cqe < cq->resize_buf->cqe) { in mthca_cq_resize_copy_cqes()
337 cq->cons_index &= cq->ibcq.cqe; in mthca_cq_resize_copy_cqes()
338 if (cqe_sw(get_cqe(cq, cq->ibcq.cqe))) in mthca_cq_resize_copy_cqes()
339 cq->cons_index -= cq->ibcq.cqe + 1; in mthca_cq_resize_copy_cqes()
342 for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i) in mthca_cq_resize_copy_cqes()
343 memcpy(get_cqe_from_buf(&cq->resize_buf->buf, in mthca_cq_resize_copy_cqes()
344 i & cq->resize_buf->cqe), in mthca_cq_resize_copy_cqes()
345 get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE); in mthca_cq_resize_copy_cqes()
372 static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, in handle_error_cqe() argument
384 cq->cqn, cq->cons_index); in handle_error_cqe()
479 struct mthca_cq *cq, in mthca_poll_one() argument
493 cqe = next_cqe_sw(cq); in mthca_poll_one()
505 cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn), in mthca_poll_one()
571 handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send, in mthca_poll_one()
648 ++cq->cons_index; in mthca_poll_one()
658 struct mthca_cq *cq = to_mcq(ibcq); in mthca_poll_cq() local
665 spin_lock_irqsave(&cq->lock, flags); in mthca_poll_cq()
670 err = mthca_poll_one(dev, cq, &qp, in mthca_poll_cq()
679 update_cons_index(dev, cq, freed); in mthca_poll_cq()
688 if (unlikely(err == -EAGAIN && cq->resize_buf && in mthca_poll_cq()
689 cq->resize_buf->state == CQ_RESIZE_READY)) { in mthca_poll_cq()
698 cq->cons_index &= cq->ibcq.cqe; in mthca_poll_cq()
700 if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf, in mthca_poll_cq()
701 cq->cons_index & cq->resize_buf->cqe))) { in mthca_poll_cq()
705 tbuf = cq->buf; in mthca_poll_cq()
706 tcqe = cq->ibcq.cqe; in mthca_poll_cq()
707 cq->buf = cq->resize_buf->buf; in mthca_poll_cq()
708 cq->ibcq.cqe = cq->resize_buf->cqe; in mthca_poll_cq()
710 cq->resize_buf->buf = tbuf; in mthca_poll_cq()
711 cq->resize_buf->cqe = tcqe; in mthca_poll_cq()
712 cq->resize_buf->state = CQ_RESIZE_SWAPPED; in mthca_poll_cq()
718 spin_unlock_irqrestore(&cq->lock, flags); in mthca_poll_cq()
723 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) in mthca_tavor_arm_cq() argument
728 to_mcq(cq)->cqn; in mthca_tavor_arm_cq()
730 mthca_write64(dbhi, 0xffffffff, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL, in mthca_tavor_arm_cq()
731 MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock)); in mthca_tavor_arm_cq()
738 struct mthca_cq *cq = to_mcq(ibcq); in mthca_arbel_arm_cq() local
741 u32 sn = cq->arm_sn & 3; in mthca_arbel_arm_cq()
743 db_rec[0] = cpu_to_be32(cq->cons_index); in mthca_arbel_arm_cq()
744 db_rec[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) | in mthca_arbel_arm_cq()
748 mthca_write_db_rec(db_rec, cq->arm_db); in mthca_arbel_arm_cq()
759 MTHCA_ARBEL_CQ_DB_REQ_NOT) | cq->cqn; in mthca_arbel_arm_cq()
761 mthca_write64(dbhi, cq->cons_index, in mthca_arbel_arm_cq()
770 struct mthca_cq *cq) in mthca_init_cq() argument
776 cq->ibcq.cqe = nent - 1; in mthca_init_cq()
777 cq->is_kernel = !ctx; in mthca_init_cq()
779 cq->cqn = mthca_alloc(&dev->cq_table.alloc); in mthca_init_cq()
780 if (cq->cqn == -1) in mthca_init_cq()
784 err = mthca_table_get(dev, dev->cq_table.table, cq->cqn); in mthca_init_cq()
788 if (cq->is_kernel) { in mthca_init_cq()
789 cq->arm_sn = 1; in mthca_init_cq()
793 cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, in mthca_init_cq()
794 cq->cqn, &cq->set_ci_db); in mthca_init_cq()
795 if (cq->set_ci_db_index < 0) in mthca_init_cq()
798 cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM, in mthca_init_cq()
799 cq->cqn, &cq->arm_db); in mthca_init_cq()
800 if (cq->arm_db_index < 0) in mthca_init_cq()
813 if (cq->is_kernel) { in mthca_init_cq()
814 err = mthca_alloc_cq_buf(dev, &cq->buf, nent); in mthca_init_cq()
819 spin_lock_init(&cq->lock); in mthca_init_cq()
820 cq->refcount = 1; in mthca_init_cq()
821 init_waitqueue_head(&cq->wait); in mthca_init_cq()
822 mutex_init(&cq->mutex); in mthca_init_cq()
836 cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey); in mthca_init_cq()
837 cq_context->cqn = cpu_to_be32(cq->cqn); in mthca_init_cq()
840 cq_context->ci_db = cpu_to_be32(cq->set_ci_db_index); in mthca_init_cq()
841 cq_context->state_db = cpu_to_be32(cq->arm_db_index); in mthca_init_cq()
844 err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn); in mthca_init_cq()
851 err = mthca_array_set(&dev->cq_table.cq, in mthca_init_cq()
852 cq->cqn & (dev->limits.num_cqs - 1), cq); in mthca_init_cq()
859 cq->cons_index = 0; in mthca_init_cq()
866 if (cq->is_kernel) in mthca_init_cq()
867 mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mthca_init_cq()
873 if (cq->is_kernel && mthca_is_memfree(dev)) in mthca_init_cq()
874 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); in mthca_init_cq()
877 if (cq->is_kernel && mthca_is_memfree(dev)) in mthca_init_cq()
878 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); in mthca_init_cq()
881 mthca_table_put(dev, dev->cq_table.table, cq->cqn); in mthca_init_cq()
884 mthca_free(&dev->cq_table.alloc, cq->cqn); in mthca_init_cq()
889 static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq) in get_cq_refcount() argument
894 c = cq->refcount; in get_cq_refcount()
901 struct mthca_cq *cq) in mthca_free_cq() argument
912 err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn); in mthca_free_cq()
921 cq->cqn, cq->cons_index, in mthca_free_cq()
922 cq->is_kernel ? !!next_cqe_sw(cq) : 0); in mthca_free_cq()
928 mthca_array_clear(&dev->cq_table.cq, in mthca_free_cq()
929 cq->cqn & (dev->limits.num_cqs - 1)); in mthca_free_cq()
930 --cq->refcount; in mthca_free_cq()
938 wait_event(cq->wait, !get_cq_refcount(dev, cq)); in mthca_free_cq()
940 if (cq->is_kernel) { in mthca_free_cq()
941 mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mthca_free_cq()
943 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); in mthca_free_cq()
944 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); in mthca_free_cq()
948 mthca_table_put(dev, dev->cq_table.table, cq->cqn); in mthca_free_cq()
949 mthca_free(&dev->cq_table.alloc, cq->cqn); in mthca_free_cq()
966 err = mthca_array_init(&dev->cq_table.cq, in mthca_init_cq_table()
976 mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs); in mthca_cleanup_cq_table()