956 	void *qpc;  in _create_user_qp()  local
1010 			ubuffer->umem, qpc, log_page_size,  in _create_user_qp()
1032 	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);  in _create_user_qp()
1036 		MLX5_SET(qpc, qpc, log_page_size,  in _create_user_qp()
1038 		MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);  in _create_user_qp()
1040 	MLX5_SET(qpc, qpc, uar_page, uar_index);  in _create_user_qp()
1105 	void *qpc;  in _create_kernel_qp()  local
1162 	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);  in _create_kernel_qp()
1163 	MLX5_SET(qpc, qpc, uar_page, uar_index);  in _create_kernel_qp()
1164 	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));  in _create_kernel_qp()
1165 	MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);  in _create_kernel_qp()
1168 	MLX5_SET(qpc, qpc, fre, 1);  in _create_kernel_qp()
1169 	MLX5_SET(qpc, qpc, rlky, 1);  in _create_kernel_qp()
1172 		MLX5_SET(qpc, qpc, deth_sqpn, 1);  in _create_kernel_qp()
1346 	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);  in create_raw_packet_qp_sq()  local
1386 	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));  in create_raw_packet_qp_sq()
1387 	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));  in create_raw_packet_qp_sq()
1396 	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));  in create_raw_packet_qp_sq()
1397 	MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));  in create_raw_packet_qp_sq()
1398 	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));  in create_raw_packet_qp_sq()
1400 	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));  in create_raw_packet_qp_sq()
1441 	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);  in create_raw_packet_qp_rq()  local
1474 	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));  in create_raw_packet_qp_rq()
1475 	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));  in create_raw_packet_qp_rq()
1485 	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));  in create_raw_packet_qp_rq()
1486 	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));  in create_raw_packet_qp_rq()
1487 	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);  in create_raw_packet_qp_rq()
1490 	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));  in create_raw_packet_qp_rq()
1924 					 void *qpc)  in configure_requester_scat_cqe()  argument
1936 		MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);  in configure_requester_scat_cqe()
1942 		MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);  in configure_requester_scat_cqe()
2003 	void *qpc;  in create_xrc_tgt_qp()  local
2014 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);  in create_xrc_tgt_qp()
2016 	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);  in create_xrc_tgt_qp()
2017 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);  in create_xrc_tgt_qp()
2018 	MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);  in create_xrc_tgt_qp()
2021 		MLX5_SET(qpc, qpc, block_lb_mc, 1);  in create_xrc_tgt_qp()
2023 		MLX5_SET(qpc, qpc, cd_master, 1);  in create_xrc_tgt_qp()
2025 		MLX5_SET(qpc, qpc, cd_slave_send, 1);  in create_xrc_tgt_qp()
2027 		MLX5_SET(qpc, qpc, cd_slave_receive, 1);  in create_xrc_tgt_qp()
2029 	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));  in create_xrc_tgt_qp()
2030 	MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);  in create_xrc_tgt_qp()
2031 	MLX5_SET(qpc, qpc, no_sq, 1);  in create_xrc_tgt_qp()
2032 	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);  in create_xrc_tgt_qp()
2033 	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);  in create_xrc_tgt_qp()
2034 	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);  in create_xrc_tgt_qp()
2035 	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);  in create_xrc_tgt_qp()
2036 	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);  in create_xrc_tgt_qp()
2040 		MLX5_SET(qpc, qpc, user_index, uidx);  in create_xrc_tgt_qp()
2043 		MLX5_SET(qpc, qpc, end_padding_mode,  in create_xrc_tgt_qp()
2086 	void *qpc;  in create_dci()  local
2129 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);  in create_dci()
2131 	MLX5_SET(qpc, qpc, st, mlx5_st);  in create_dci()
2132 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);  in create_dci()
2133 	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);  in create_dci()
2136 		MLX5_SET(qpc, qpc, wq_signature, 1);  in create_dci()
2139 		MLX5_SET(qpc, qpc, cd_master, 1);  in create_dci()
2141 		MLX5_SET(qpc, qpc, cd_slave_send, 1);  in create_dci()
2143 		configure_requester_scat_cqe(dev, qp, init_attr, qpc);  in create_dci()
2146 		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);  in create_dci()
2147 		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));  in create_dci()
2151 		MLX5_SET(qpc, qpc, log_num_dci_stream_channels,  in create_dci()
2153 		MLX5_SET(qpc, qpc, log_num_dci_errored_streams,  in create_dci()
2157 	MLX5_SET(qpc, qpc, ts_format, ts_format);  in create_dci()
2158 	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));  in create_dci()
2160 	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));  in create_dci()
2164 		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);  in create_dci()
2165 		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,  in create_dci()
2168 		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);  in create_dci()
2169 		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,  in create_dci()
2174 		MLX5_SET(qpc, qpc, cqn_snd,  in create_dci()
2178 		MLX5_SET(qpc, qpc, cqn_rcv,  in create_dci()
2181 	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);  in create_dci()
2185 		MLX5_SET(qpc, qpc, user_index, uidx);  in create_dci()
2188 		MLX5_SET(qpc, qpc, end_padding_mode,  in create_dci()
2247 	void *qpc;  in create_user_qp()  local
2300 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);  in create_user_qp()
2302 	MLX5_SET(qpc, qpc, st, mlx5_st);  in create_user_qp()
2303 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);  in create_user_qp()
2304 	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);  in create_user_qp()
2307 		MLX5_SET(qpc, qpc, wq_signature, 1);  in create_user_qp()
2310 		MLX5_SET(qpc, qpc, block_lb_mc, 1);  in create_user_qp()
2313 		MLX5_SET(qpc, qpc, cd_master, 1);  in create_user_qp()
2315 		MLX5_SET(qpc, qpc, cd_slave_send, 1);  in create_user_qp()
2317 		MLX5_SET(qpc, qpc, cd_slave_receive, 1);  in create_user_qp()
2319 		MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);  in create_user_qp()
2325 		MLX5_SET(qpc, qpc, cs_res,  in create_user_qp()
2331 		configure_requester_scat_cqe(dev, qp, init_attr, qpc);  in create_user_qp()
2334 		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);  in create_user_qp()
2335 		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));  in create_user_qp()
2339 		MLX5_SET(qpc, qpc, ts_format, ts_format);  in create_user_qp()
2341 	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));  in create_user_qp()
2344 		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));  in create_user_qp()
2346 		MLX5_SET(qpc, qpc, no_sq, 1);  in create_user_qp()
2349 			MLX5_SET(qpc, qpc, offload_type,  in create_user_qp()
2356 		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);  in create_user_qp()
2357 		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);  in create_user_qp()
2358 		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);  in create_user_qp()
2362 			MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);  in create_user_qp()
2363 			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);  in create_user_qp()
2365 			MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);  in create_user_qp()
2366 			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);  in create_user_qp()
2371 		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);  in create_user_qp()
2374 		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);  in create_user_qp()
2376 	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);  in create_user_qp()
2380 		MLX5_SET(qpc, qpc, user_index, uidx);  in create_user_qp()
2384 		MLX5_SET(qpc, qpc, end_padding_mode,  in create_user_qp()
2447 	void *qpc;  in create_kernel_qp()  local
2477 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);  in create_kernel_qp()
2479 	MLX5_SET(qpc, qpc, st, mlx5_st);  in create_kernel_qp()
2480 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);  in create_kernel_qp()
2483 		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);  in create_kernel_qp()
2485 		MLX5_SET(qpc, qpc, latency_sensitive, 1);  in create_kernel_qp()
2489 		MLX5_SET(qpc, qpc, block_lb_mc, 1);  in create_kernel_qp()
2492 		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);  in create_kernel_qp()
2493 		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));  in create_kernel_qp()
2496 	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));  in create_kernel_qp()
2499 		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));  in create_kernel_qp()
2501 		MLX5_SET(qpc, qpc, no_sq, 1);  in create_kernel_qp()
2504 		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);  in create_kernel_qp()
2505 		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,  in create_kernel_qp()
2508 		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);  in create_kernel_qp()
2509 		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,  in create_kernel_qp()
2514 		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);  in create_kernel_qp()
2517 		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);  in create_kernel_qp()
2519 	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);  in create_kernel_qp()
2523 		MLX5_SET(qpc, qpc, user_index, uidx);  in create_kernel_qp()
2527 		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);  in create_kernel_qp()
2531 		MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);  in create_kernel_qp()
3353 				void *qpc)  in set_qpc_atomic_flags()  argument
3372 	MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));  in set_qpc_atomic_flags()
3381 		MLX5_SET(qpc, qpc, rae, 1);  in set_qpc_atomic_flags()
3382 		MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);  in set_qpc_atomic_flags()
3385 	MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));  in set_qpc_atomic_flags()
4080 	u32 *qpc;  in __mlx5_ib_qp_set_counter()  local
4097 	qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);  in __mlx5_ib_qp_set_counter()
4098 	MLX5_SET(qpc, qpc, counter_set_id, set_id);  in __mlx5_ib_qp_set_counter()
4154 	void *qpc, *pri_path, *alt_path;  in __mlx5_ib_modify_qp()  local
4166 	qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);  in __mlx5_ib_modify_qp()
4167 	if (!qpc)  in __mlx5_ib_modify_qp()
4171 	MLX5_SET(qpc, qpc, st, mlx5_st);  in __mlx5_ib_modify_qp()
4174 		MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);  in __mlx5_ib_modify_qp()
4178 			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);  in __mlx5_ib_modify_qp()
4181 			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);  in __mlx5_ib_modify_qp()
4184 			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);  in __mlx5_ib_modify_qp()
4193 	MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);  in __mlx5_ib_modify_qp()
4199 		MLX5_SET(qpc, qpc, mtu, IB_MTU_256);  in __mlx5_ib_modify_qp()
4200 		MLX5_SET(qpc, qpc, log_msg_max, 8);  in __mlx5_ib_modify_qp()
4204 		MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);  in __mlx5_ib_modify_qp()
4205 		MLX5_SET(qpc, qpc, log_msg_max, 12);  in __mlx5_ib_modify_qp()
4213 		MLX5_SET(qpc, qpc, mtu, attr->path_mtu);  in __mlx5_ib_modify_qp()
4214 		MLX5_SET(qpc, qpc, log_msg_max,  in __mlx5_ib_modify_qp()
4219 		MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);  in __mlx5_ib_modify_qp()
4221 	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);  in __mlx5_ib_modify_qp()
4222 	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);  in __mlx5_ib_modify_qp()
4260 	MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);  in __mlx5_ib_modify_qp()
4262 		MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);  in __mlx5_ib_modify_qp()
4264 		MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);  in __mlx5_ib_modify_qp()
4266 	MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);  in __mlx5_ib_modify_qp()
4269 		MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);  in __mlx5_ib_modify_qp()
4272 		MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);  in __mlx5_ib_modify_qp()
4275 		MLX5_SET(qpc, qpc, log_sra_max, ilog2(attr->max_rd_atomic));  in __mlx5_ib_modify_qp()
4278 		MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);  in __mlx5_ib_modify_qp()
4281 		MLX5_SET(qpc, qpc, log_rra_max,  in __mlx5_ib_modify_qp()
4285 		err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);  in __mlx5_ib_modify_qp()
4291 		MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);  in __mlx5_ib_modify_qp()
4294 		MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);  in __mlx5_ib_modify_qp()
4297 		MLX5_SET(qpc, qpc, q_key, attr->qkey);  in __mlx5_ib_modify_qp()
4300 		MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);  in __mlx5_ib_modify_qp()
4314 		MLX5_SET(qpc, qpc, counter_set_id, set_id);  in __mlx5_ib_modify_qp()
4318 		MLX5_SET(qpc, qpc, rlky, 1);  in __mlx5_ib_modify_qp()
4321 		MLX5_SET(qpc, qpc, deth_sqpn, 1);  in __mlx5_ib_modify_qp()
4386 		err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,  in __mlx5_ib_modify_qp()
4434 	kfree(qpc);  in __mlx5_ib_modify_qp()
4937 	void *qpc, *pri_path, *alt_path;  in query_qp_attr()  local
4950 	qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);  in query_qp_attr()
4952 	qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));  in query_qp_attr()
4953 	if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)  in query_qp_attr()
4956 	qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);  in query_qp_attr()
4957 	qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));  in query_qp_attr()
4958 	qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);  in query_qp_attr()
4959 	qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);  in query_qp_attr()
4960 	qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);  in query_qp_attr()
4961 	qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);  in query_qp_attr()
4963 	if (MLX5_GET(qpc, qpc, rre))  in query_qp_attr()
4965 	if (MLX5_GET(qpc, qpc, rwe))  in query_qp_attr()
4967 	if (MLX5_GET(qpc, qpc, rae))  in query_qp_attr()
4970 	qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);  in query_qp_attr()
4971 	qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);  in query_qp_attr()
4972 	qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);  in query_qp_attr()
4973 	qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);  in query_qp_attr()
4974 	qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);  in query_qp_attr()
4976 	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);  in query_qp_attr()
4977 	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);  in query_qp_attr()