Lines Matching refs:cq
43 static void mlx4_ib_cq_comp(struct mlx4_cq *cq) in mlx4_ib_cq_comp() argument
45 struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; in mlx4_ib_cq_comp()
49 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) in mlx4_ib_cq_event() argument
56 "on CQ %06x\n", type, cq->cqn); in mlx4_ib_cq_event()
60 ibcq = &to_mibcq(cq)->ibcq; in mlx4_ib_cq_event()
64 event.element.cq = ibcq; in mlx4_ib_cq_event()
74 static void *get_cqe(struct mlx4_ib_cq *cq, int n) in get_cqe() argument
76 return get_cqe_from_buf(&cq->buf, n); in get_cqe()
79 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) in get_sw_cqe() argument
81 struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); in get_sw_cqe()
82 struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe); in get_sw_cqe()
85 !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; in get_sw_cqe()
88 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) in next_cqe_sw() argument
90 return get_sw_cqe(cq, cq->mcq.cons_index); in next_cqe_sw()
93 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) in mlx4_ib_modify_cq() argument
95 struct mlx4_ib_cq *mcq = to_mcq(cq); in mlx4_ib_modify_cq()
96 struct mlx4_ib_dev *dev = to_mdev(cq->device); in mlx4_ib_modify_cq()
181 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_create_cq() local
195 cq->ibcq.cqe = entries - 1; in mlx4_ib_create_cq()
196 mutex_init(&cq->resize_mutex); in mlx4_ib_create_cq()
197 spin_lock_init(&cq->lock); in mlx4_ib_create_cq()
198 cq->resize_buf = NULL; in mlx4_ib_create_cq()
199 cq->resize_umem = NULL; in mlx4_ib_create_cq()
200 cq->create_flags = attr->flags; in mlx4_ib_create_cq()
201 INIT_LIST_HEAD(&cq->send_qp_list); in mlx4_ib_create_cq()
202 INIT_LIST_HEAD(&cq->recv_qp_list); in mlx4_ib_create_cq()
213 err = mlx4_ib_get_cq_umem(dev, &cq->buf, &cq->umem, in mlx4_ib_create_cq()
218 err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &cq->db); in mlx4_ib_create_cq()
223 cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS; in mlx4_ib_create_cq()
225 err = mlx4_db_alloc(dev->dev, &cq->db, 1); in mlx4_ib_create_cq()
229 cq->mcq.set_ci_db = cq->db.db; in mlx4_ib_create_cq()
230 cq->mcq.arm_db = cq->db.db + 1; in mlx4_ib_create_cq()
231 *cq->mcq.set_ci_db = 0; in mlx4_ib_create_cq()
232 *cq->mcq.arm_db = 0; in mlx4_ib_create_cq()
234 err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries); in mlx4_ib_create_cq()
238 buf_addr = &cq->buf.buf; in mlx4_ib_create_cq()
241 cq->mcq.usage = MLX4_RES_USAGE_DRIVER; in mlx4_ib_create_cq()
247 err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, cq->db.dma, in mlx4_ib_create_cq()
248 &cq->mcq, vector, 0, in mlx4_ib_create_cq()
249 !!(cq->create_flags & in mlx4_ib_create_cq()
256 cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp; in mlx4_ib_create_cq()
258 cq->mcq.comp = mlx4_ib_cq_comp; in mlx4_ib_create_cq()
259 cq->mcq.event = mlx4_ib_cq_event; in mlx4_ib_create_cq()
262 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { in mlx4_ib_create_cq()
270 mlx4_cq_free(dev->dev, &cq->mcq); in mlx4_ib_create_cq()
274 mlx4_ib_db_unmap_user(context, &cq->db); in mlx4_ib_create_cq()
277 mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); in mlx4_ib_create_cq()
279 ib_umem_release(cq->umem); in mlx4_ib_create_cq()
281 mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mlx4_ib_create_cq()
285 mlx4_db_free(dev->dev, &cq->db); in mlx4_ib_create_cq()
290 static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, in mlx4_alloc_resize_buf() argument
295 if (cq->resize_buf) in mlx4_alloc_resize_buf()
298 cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL); in mlx4_alloc_resize_buf()
299 if (!cq->resize_buf) in mlx4_alloc_resize_buf()
302 err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); in mlx4_alloc_resize_buf()
304 kfree(cq->resize_buf); in mlx4_alloc_resize_buf()
305 cq->resize_buf = NULL; in mlx4_alloc_resize_buf()
309 cq->resize_buf->cqe = entries - 1; in mlx4_alloc_resize_buf()
314 static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, in mlx4_alloc_resize_umem() argument
320 if (cq->resize_umem) in mlx4_alloc_resize_umem()
326 cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL); in mlx4_alloc_resize_umem()
327 if (!cq->resize_buf) in mlx4_alloc_resize_umem()
330 err = mlx4_ib_get_cq_umem(dev, &cq->resize_buf->buf, &cq->resize_umem, in mlx4_alloc_resize_umem()
333 kfree(cq->resize_buf); in mlx4_alloc_resize_umem()
334 cq->resize_buf = NULL; in mlx4_alloc_resize_umem()
338 cq->resize_buf->cqe = entries - 1; in mlx4_alloc_resize_umem()
343 static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq) in mlx4_ib_get_outstanding_cqes() argument
347 i = cq->mcq.cons_index; in mlx4_ib_get_outstanding_cqes()
348 while (get_sw_cqe(cq, i)) in mlx4_ib_get_outstanding_cqes()
351 return i - cq->mcq.cons_index; in mlx4_ib_get_outstanding_cqes()
354 static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq) in mlx4_ib_cq_resize_copy_cqes() argument
358 int cqe_size = cq->buf.entry_size; in mlx4_ib_cq_resize_copy_cqes()
361 i = cq->mcq.cons_index; in mlx4_ib_cq_resize_copy_cqes()
362 cqe = get_cqe(cq, i & cq->ibcq.cqe); in mlx4_ib_cq_resize_copy_cqes()
366 new_cqe = get_cqe_from_buf(&cq->resize_buf->buf, in mlx4_ib_cq_resize_copy_cqes()
367 (i + 1) & cq->resize_buf->cqe); in mlx4_ib_cq_resize_copy_cqes()
368 memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size); in mlx4_ib_cq_resize_copy_cqes()
372 (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); in mlx4_ib_cq_resize_copy_cqes()
373 cqe = get_cqe(cq, ++i & cq->ibcq.cqe); in mlx4_ib_cq_resize_copy_cqes()
376 ++cq->mcq.cons_index; in mlx4_ib_cq_resize_copy_cqes()
382 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_resize_cq() local
387 mutex_lock(&cq->resize_mutex); in mlx4_ib_resize_cq()
405 err = mlx4_alloc_resize_umem(dev, cq, entries, udata); in mlx4_ib_resize_cq()
410 outst_cqe = mlx4_ib_get_outstanding_cqes(cq); in mlx4_ib_resize_cq()
416 err = mlx4_alloc_resize_buf(dev, cq, entries); in mlx4_ib_resize_cq()
421 mtt = cq->buf.mtt; in mlx4_ib_resize_cq()
423 err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt); in mlx4_ib_resize_cq()
429 cq->buf = cq->resize_buf->buf; in mlx4_ib_resize_cq()
430 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_resize_cq()
431 ib_umem_release(cq->umem); in mlx4_ib_resize_cq()
432 cq->umem = cq->resize_umem; in mlx4_ib_resize_cq()
434 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
435 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
436 cq->resize_umem = NULL; in mlx4_ib_resize_cq()
441 spin_lock_irq(&cq->lock); in mlx4_ib_resize_cq()
442 if (cq->resize_buf) { in mlx4_ib_resize_cq()
443 mlx4_ib_cq_resize_copy_cqes(cq); in mlx4_ib_resize_cq()
444 tmp_buf = cq->buf; in mlx4_ib_resize_cq()
445 tmp_cqe = cq->ibcq.cqe; in mlx4_ib_resize_cq()
446 cq->buf = cq->resize_buf->buf; in mlx4_ib_resize_cq()
447 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_resize_cq()
449 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
450 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
452 spin_unlock_irq(&cq->lock); in mlx4_ib_resize_cq()
461 mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt); in mlx4_ib_resize_cq()
463 mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf, in mlx4_ib_resize_cq()
464 cq->resize_buf->cqe); in mlx4_ib_resize_cq()
466 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
467 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
469 ib_umem_release(cq->resize_umem); in mlx4_ib_resize_cq()
470 cq->resize_umem = NULL; in mlx4_ib_resize_cq()
472 mutex_unlock(&cq->resize_mutex); in mlx4_ib_resize_cq()
477 int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) in mlx4_ib_destroy_cq() argument
479 struct mlx4_ib_dev *dev = to_mdev(cq->device); in mlx4_ib_destroy_cq()
480 struct mlx4_ib_cq *mcq = to_mcq(cq); in mlx4_ib_destroy_cq()
493 mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); in mlx4_ib_destroy_cq()
580 static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, in use_tunnel_data() argument
631 static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries, in mlx4_ib_poll_sw_comp() argument
640 list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) { in mlx4_ib_poll_sw_comp()
646 list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) { in mlx4_ib_poll_sw_comp()
656 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, in mlx4_ib_poll_one() argument
673 cqe = next_cqe_sw(cq); in mlx4_ib_poll_one()
677 if (cq->buf.entry_size == 64) in mlx4_ib_poll_one()
680 ++cq->mcq.cons_index; in mlx4_ib_poll_one()
694 if (cq->resize_buf) { in mlx4_ib_poll_one()
695 struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device); in mlx4_ib_poll_one()
697 mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mlx4_ib_poll_one()
698 cq->buf = cq->resize_buf->buf; in mlx4_ib_poll_one()
699 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_poll_one()
701 kfree(cq->resize_buf); in mlx4_ib_poll_one()
702 cq->resize_buf = NULL; in mlx4_ib_poll_one()
715 mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev, in mlx4_ib_poll_one()
727 msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev, in mlx4_ib_poll_one()
837 if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) { in mlx4_ib_poll_one()
841 use_tunnel_data(*cur_qp, cq, wc, tail, cqe, in mlx4_ib_poll_one()
879 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_poll_cq() local
883 struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device); in mlx4_ib_poll_cq()
885 spin_lock_irqsave(&cq->lock, flags); in mlx4_ib_poll_cq()
887 mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled); in mlx4_ib_poll_cq()
892 if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled)) in mlx4_ib_poll_cq()
896 mlx4_cq_set_ci(&cq->mcq); in mlx4_ib_poll_cq()
899 spin_unlock_irqrestore(&cq->lock, flags); in mlx4_ib_poll_cq()
915 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) in __mlx4_ib_cq_clean() argument
921 int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0; in __mlx4_ib_cq_clean()
930 for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) in __mlx4_ib_cq_clean()
931 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) in __mlx4_ib_cq_clean()
938 while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { in __mlx4_ib_cq_clean()
939 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in __mlx4_ib_cq_clean()
947 dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); in __mlx4_ib_cq_clean()
958 cq->mcq.cons_index += nfreed; in __mlx4_ib_cq_clean()
964 mlx4_cq_set_ci(&cq->mcq); in __mlx4_ib_cq_clean()
968 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) in mlx4_ib_cq_clean() argument
970 spin_lock_irq(&cq->lock); in mlx4_ib_cq_clean()
971 __mlx4_ib_cq_clean(cq, qpn, srq); in mlx4_ib_cq_clean()
972 spin_unlock_irq(&cq->lock); in mlx4_ib_cq_clean()