186 	struct mlx5_ib_wq *wq = &qp->sq;  in mlx5_ib_read_user_wqe_sq()  local
196 					   wq->offset, wq->wqe_cnt,  in mlx5_ib_read_user_wqe_sq()
197 					   wq->wqe_shift, buflen,  in mlx5_ib_read_user_wqe_sq()
221 					   buflen - bytes_copied, 0, wq->offset,  in mlx5_ib_read_user_wqe_sq()
222 					   wq->wqe_cnt, wq->wqe_shift,  in mlx5_ib_read_user_wqe_sq()
253 	struct mlx5_ib_wq *wq = &qp->rq;  in mlx5_ib_read_user_wqe_rq()  local
258 					   wq->offset, wq->wqe_cnt,  in mlx5_ib_read_user_wqe_rq()
259 					   wq->wqe_shift, buflen,  in mlx5_ib_read_user_wqe_rq()
273 	struct mlx5_ib_wq *wq = &qp->rq;  in mlx5_ib_read_wqe_rq()  local
274 	size_t wqe_size = 1 << wq->wqe_shift;  in mlx5_ib_read_wqe_rq()
901 		rwq->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,  in create_user_rq()
1347 	void *wq;  in create_raw_packet_qp_sq()  local
1363 		ubuffer->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,  in create_raw_packet_qp_sq()
1394 	wq = MLX5_ADDR_OF(sqc, sqc, wq);  in create_raw_packet_qp_sq()
1395 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);  in create_raw_packet_qp_sq()
1396 	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));  in create_raw_packet_qp_sq()
1397 	MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));  in create_raw_packet_qp_sq()
1398 	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));  in create_raw_packet_qp_sq()
1399 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));  in create_raw_packet_qp_sq()
1400 	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));  in create_raw_packet_qp_sq()
1401 	MLX5_SET(wq, wq, log_wq_pg_sz,  in create_raw_packet_qp_sq()
1403 	MLX5_SET(wq, wq, page_offset, page_offset_quantized);  in create_raw_packet_qp_sq()
1405 	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);  in create_raw_packet_qp_sq()
1440 	void *wq;  in create_raw_packet_qp_rq()  local
1453 	page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,  in create_raw_packet_qp_rq()
1480 	wq = MLX5_ADDR_OF(rqc, rqc, wq);  in create_raw_packet_qp_rq()
1481 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);  in create_raw_packet_qp_rq()
1483 		MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);  in create_raw_packet_qp_rq()
1484 	MLX5_SET(wq, wq, page_offset, page_offset_quantized);  in create_raw_packet_qp_rq()
1485 	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));  in create_raw_packet_qp_rq()
1486 	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));  in create_raw_packet_qp_rq()
1487 	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);  in create_raw_packet_qp_rq()
1488 	MLX5_SET(wq, wq, log_wq_pg_sz,  in create_raw_packet_qp_rq()
1490 	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));  in create_raw_packet_qp_rq()
1492 	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);  in create_raw_packet_qp_rq()
5159 		event.element.wq = &rwq->ibwq;  in mlx5_ib_wq_event()
5203 	void *wq;  in create_rq()  local
5227 	wq = MLX5_ADDR_OF(rqc, rqc, wq);  in create_rq()
5228 	MLX5_SET(wq, wq, wq_type,  in create_rq()
5237 			MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);  in create_rq()
5240 	MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);  in create_rq()
5250 		MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);  in create_rq()
5251 		MLX5_SET(wq, wq, log_wqe_stride_size,  in create_rq()
5254 		MLX5_SET(wq, wq, log_wqe_num_of_strides,  in create_rq()
5258 	MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);  in create_rq()
5259 	MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);  in create_rq()
5260 	MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);  in create_rq()
5261 	MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);  in create_rq()
5262 	MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);  in create_rq()
5263 	MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);  in create_rq()
5291 	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);  in create_rq()
5489 int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)  in mlx5_ib_destroy_wq()  argument
5491 	struct mlx5_ib_dev *dev = to_mdev(wq->device);  in mlx5_ib_destroy_wq()
5492 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);  in mlx5_ib_destroy_wq()
5498 	destroy_user_rq(dev, wq->pd, rwq, udata);  in mlx5_ib_destroy_wq()
5583 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,  in mlx5_ib_modify_wq()  argument
5586 	struct mlx5_ib_dev *dev = to_mdev(wq->device);  in mlx5_ib_modify_wq()
5587 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);  in mlx5_ib_modify_wq()
5626 	MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);  in mlx5_ib_modify_wq()