1297cccebSAlex Vesker // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2297cccebSAlex Vesker /* Copyright (c) 2019 Mellanox Technologies. */ 3297cccebSAlex Vesker 4c0702a4bSErez Shitrit #include <linux/smp.h> 5297cccebSAlex Vesker #include "dr_types.h" 6297cccebSAlex Vesker 7297cccebSAlex Vesker #define QUEUE_SIZE 128 8297cccebSAlex Vesker #define SIGNAL_PER_DIV_QUEUE 16 9297cccebSAlex Vesker #define TH_NUMS_TO_DRAIN 2 1017b56073SYevgeny Kliteynik #define DR_SEND_INFO_POOL_SIZE 1000 11297cccebSAlex Vesker 12297cccebSAlex Vesker enum { CQ_OK = 0, CQ_EMPTY = -1, CQ_POLL_ERR = -2 }; 13297cccebSAlex Vesker 14297cccebSAlex Vesker struct dr_data_seg { 15297cccebSAlex Vesker u64 addr; 16297cccebSAlex Vesker u32 length; 17297cccebSAlex Vesker u32 lkey; 18297cccebSAlex Vesker unsigned int send_flags; 19297cccebSAlex Vesker }; 20297cccebSAlex Vesker 21297cccebSAlex Vesker struct postsend_info { 22297cccebSAlex Vesker struct dr_data_seg write; 23297cccebSAlex Vesker struct dr_data_seg read; 24297cccebSAlex Vesker u64 remote_addr; 25297cccebSAlex Vesker u32 rkey; 26297cccebSAlex Vesker }; 27297cccebSAlex Vesker 28297cccebSAlex Vesker struct dr_qp_rtr_attr { 29297cccebSAlex Vesker struct mlx5dr_cmd_gid_attr dgid_attr; 30297cccebSAlex Vesker enum ib_mtu mtu; 31297cccebSAlex Vesker u32 qp_num; 32297cccebSAlex Vesker u16 port_num; 33297cccebSAlex Vesker u8 min_rnr_timer; 34297cccebSAlex Vesker u8 sgid_index; 35297cccebSAlex Vesker u16 udp_src_port; 367304d603SYevgeny Kliteynik u8 fl:1; 37297cccebSAlex Vesker }; 38297cccebSAlex Vesker 39297cccebSAlex Vesker struct dr_qp_rts_attr { 40297cccebSAlex Vesker u8 timeout; 41297cccebSAlex Vesker u8 retry_cnt; 42297cccebSAlex Vesker u8 rnr_retry; 43297cccebSAlex Vesker }; 44297cccebSAlex Vesker 45297cccebSAlex Vesker struct dr_qp_init_attr { 46297cccebSAlex Vesker u32 cqn; 47297cccebSAlex Vesker u32 pdn; 48297cccebSAlex Vesker u32 max_send_wr; 49297cccebSAlex Vesker struct mlx5_uars_page *uar; 50aeacb52aSYevgeny Kliteynik u8 isolate_vl_tc:1; 51297cccebSAlex Vesker }; 52297cccebSAlex Vesker 5317b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj { 5417b56073SYevgeny Kliteynik struct mlx5dr_ste_send_info ste_send_info; 5517b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool *pool; 5617b56073SYevgeny Kliteynik struct list_head list_node; 5717b56073SYevgeny Kliteynik }; 5817b56073SYevgeny Kliteynik 5917b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool { 6017b56073SYevgeny Kliteynik struct list_head free_list; 6117b56073SYevgeny Kliteynik }; 6217b56073SYevgeny Kliteynik 6317b56073SYevgeny Kliteynik static int dr_send_info_pool_fill(struct mlx5dr_send_info_pool *pool) 6417b56073SYevgeny Kliteynik { 6517b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj *pool_obj, *tmp_pool_obj; 6617b56073SYevgeny Kliteynik int i; 6717b56073SYevgeny Kliteynik 6817b56073SYevgeny Kliteynik for (i = 0; i < DR_SEND_INFO_POOL_SIZE; i++) { 6917b56073SYevgeny Kliteynik pool_obj = kzalloc(sizeof(*pool_obj), GFP_KERNEL); 7017b56073SYevgeny Kliteynik if (!pool_obj) 7117b56073SYevgeny Kliteynik goto clean_pool; 7217b56073SYevgeny Kliteynik 7317b56073SYevgeny Kliteynik pool_obj->pool = pool; 7417b56073SYevgeny Kliteynik list_add_tail(&pool_obj->list_node, &pool->free_list); 7517b56073SYevgeny Kliteynik } 7617b56073SYevgeny Kliteynik 7717b56073SYevgeny Kliteynik return 0; 7817b56073SYevgeny Kliteynik 7917b56073SYevgeny Kliteynik clean_pool: 8017b56073SYevgeny Kliteynik list_for_each_entry_safe(pool_obj, tmp_pool_obj, &pool->free_list, list_node) { 8117b56073SYevgeny Kliteynik list_del(&pool_obj->list_node); 8217b56073SYevgeny Kliteynik kfree(pool_obj); 8317b56073SYevgeny Kliteynik } 8417b56073SYevgeny Kliteynik 8517b56073SYevgeny Kliteynik return -ENOMEM; 8617b56073SYevgeny Kliteynik } 8717b56073SYevgeny Kliteynik 8817b56073SYevgeny Kliteynik static void dr_send_info_pool_destroy(struct mlx5dr_send_info_pool *pool) 8917b56073SYevgeny Kliteynik { 9017b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj *pool_obj, *tmp_pool_obj; 9117b56073SYevgeny Kliteynik 9217b56073SYevgeny Kliteynik list_for_each_entry_safe(pool_obj, tmp_pool_obj, &pool->free_list, list_node) { 9317b56073SYevgeny Kliteynik list_del(&pool_obj->list_node); 9417b56073SYevgeny Kliteynik kfree(pool_obj); 9517b56073SYevgeny Kliteynik } 9617b56073SYevgeny Kliteynik 9717b56073SYevgeny Kliteynik kfree(pool); 9817b56073SYevgeny Kliteynik } 9917b56073SYevgeny Kliteynik 10017b56073SYevgeny Kliteynik void mlx5dr_send_info_pool_destroy(struct mlx5dr_domain *dmn) 10117b56073SYevgeny Kliteynik { 10217b56073SYevgeny Kliteynik dr_send_info_pool_destroy(dmn->send_info_pool_tx); 10317b56073SYevgeny Kliteynik dr_send_info_pool_destroy(dmn->send_info_pool_rx); 10417b56073SYevgeny Kliteynik } 10517b56073SYevgeny Kliteynik 10617b56073SYevgeny Kliteynik static struct mlx5dr_send_info_pool *dr_send_info_pool_create(void) 10717b56073SYevgeny Kliteynik { 10817b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool *pool; 10917b56073SYevgeny Kliteynik int ret; 11017b56073SYevgeny Kliteynik 11117b56073SYevgeny Kliteynik pool = kzalloc(sizeof(*pool), GFP_KERNEL); 11217b56073SYevgeny Kliteynik if (!pool) 11317b56073SYevgeny Kliteynik return NULL; 11417b56073SYevgeny Kliteynik 11517b56073SYevgeny Kliteynik INIT_LIST_HEAD(&pool->free_list); 11617b56073SYevgeny Kliteynik 11717b56073SYevgeny Kliteynik ret = dr_send_info_pool_fill(pool); 11817b56073SYevgeny Kliteynik if (ret) { 11917b56073SYevgeny Kliteynik kfree(pool); 12017b56073SYevgeny Kliteynik return NULL; 12117b56073SYevgeny Kliteynik } 12217b56073SYevgeny Kliteynik 12317b56073SYevgeny Kliteynik return pool; 12417b56073SYevgeny Kliteynik } 12517b56073SYevgeny Kliteynik 12617b56073SYevgeny Kliteynik int mlx5dr_send_info_pool_create(struct mlx5dr_domain *dmn) 12717b56073SYevgeny Kliteynik { 12817b56073SYevgeny Kliteynik dmn->send_info_pool_rx = dr_send_info_pool_create(); 12917b56073SYevgeny Kliteynik if (!dmn->send_info_pool_rx) 13017b56073SYevgeny Kliteynik return -ENOMEM; 13117b56073SYevgeny Kliteynik 13217b56073SYevgeny Kliteynik dmn->send_info_pool_tx = dr_send_info_pool_create(); 13317b56073SYevgeny Kliteynik if (!dmn->send_info_pool_tx) { 13417b56073SYevgeny Kliteynik dr_send_info_pool_destroy(dmn->send_info_pool_rx); 13517b56073SYevgeny Kliteynik return -ENOMEM; 13617b56073SYevgeny Kliteynik } 13717b56073SYevgeny Kliteynik 13817b56073SYevgeny Kliteynik return 0; 13917b56073SYevgeny Kliteynik } 14017b56073SYevgeny Kliteynik 14117b56073SYevgeny Kliteynik struct mlx5dr_ste_send_info 14217b56073SYevgeny Kliteynik *mlx5dr_send_info_alloc(struct mlx5dr_domain *dmn, 14317b56073SYevgeny Kliteynik enum mlx5dr_domain_nic_type nic_type) 14417b56073SYevgeny Kliteynik { 14517b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj *pool_obj; 14617b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool *pool; 14717b56073SYevgeny Kliteynik int ret; 14817b56073SYevgeny Kliteynik 14917b56073SYevgeny Kliteynik pool = nic_type == DR_DOMAIN_NIC_TYPE_RX ? dmn->send_info_pool_rx : 15017b56073SYevgeny Kliteynik dmn->send_info_pool_tx; 15117b56073SYevgeny Kliteynik 15217b56073SYevgeny Kliteynik if (unlikely(list_empty(&pool->free_list))) { 15317b56073SYevgeny Kliteynik ret = dr_send_info_pool_fill(pool); 15417b56073SYevgeny Kliteynik if (ret) 15517b56073SYevgeny Kliteynik return NULL; 15617b56073SYevgeny Kliteynik } 15717b56073SYevgeny Kliteynik 15817b56073SYevgeny Kliteynik pool_obj = list_first_entry_or_null(&pool->free_list, 15917b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj, 16017b56073SYevgeny Kliteynik list_node); 16117b56073SYevgeny Kliteynik 16217b56073SYevgeny Kliteynik if (likely(pool_obj)) { 16317b56073SYevgeny Kliteynik list_del_init(&pool_obj->list_node); 16417b56073SYevgeny Kliteynik } else { 16517b56073SYevgeny Kliteynik WARN_ONCE(!pool_obj, "Failed getting ste send info obj from pool"); 16617b56073SYevgeny Kliteynik return NULL; 16717b56073SYevgeny Kliteynik } 16817b56073SYevgeny Kliteynik 16917b56073SYevgeny Kliteynik return &pool_obj->ste_send_info; 17017b56073SYevgeny Kliteynik } 17117b56073SYevgeny Kliteynik 17217b56073SYevgeny Kliteynik void mlx5dr_send_info_free(struct mlx5dr_ste_send_info *ste_send_info) 17317b56073SYevgeny Kliteynik { 17417b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj *pool_obj; 17517b56073SYevgeny Kliteynik 17617b56073SYevgeny Kliteynik pool_obj = container_of(ste_send_info, 17717b56073SYevgeny Kliteynik struct mlx5dr_send_info_pool_obj, 17817b56073SYevgeny Kliteynik ste_send_info); 17917b56073SYevgeny Kliteynik 18017b56073SYevgeny Kliteynik list_add(&pool_obj->list_node, &pool_obj->pool->free_list); 18117b56073SYevgeny Kliteynik } 18217b56073SYevgeny Kliteynik 183297cccebSAlex Vesker static int dr_parse_cqe(struct mlx5dr_cq *dr_cq, struct mlx5_cqe64 *cqe64) 184297cccebSAlex Vesker { 185297cccebSAlex Vesker unsigned int idx; 186297cccebSAlex Vesker u8 opcode; 187297cccebSAlex Vesker 188297cccebSAlex Vesker opcode = get_cqe_opcode(cqe64); 189297cccebSAlex Vesker if (opcode == MLX5_CQE_REQ_ERR) { 190297cccebSAlex Vesker idx = be16_to_cpu(cqe64->wqe_counter) & 191297cccebSAlex Vesker (dr_cq->qp->sq.wqe_cnt - 1); 192297cccebSAlex Vesker dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1; 193297cccebSAlex Vesker } else if (opcode == MLX5_CQE_RESP_ERR) { 194297cccebSAlex Vesker ++dr_cq->qp->sq.cc; 195297cccebSAlex Vesker } else { 196297cccebSAlex Vesker idx = be16_to_cpu(cqe64->wqe_counter) & 197297cccebSAlex Vesker (dr_cq->qp->sq.wqe_cnt - 1); 198297cccebSAlex Vesker dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1; 199297cccebSAlex Vesker 200297cccebSAlex Vesker return CQ_OK; 201297cccebSAlex Vesker } 202297cccebSAlex Vesker 203297cccebSAlex Vesker return CQ_POLL_ERR; 204297cccebSAlex Vesker } 205297cccebSAlex Vesker 206297cccebSAlex Vesker static int dr_cq_poll_one(struct mlx5dr_cq *dr_cq) 207297cccebSAlex Vesker { 208297cccebSAlex Vesker struct mlx5_cqe64 *cqe64; 209297cccebSAlex Vesker int err; 210297cccebSAlex Vesker 211297cccebSAlex Vesker cqe64 = mlx5_cqwq_get_cqe(&dr_cq->wq); 2125fd08f65SYevgeny Kliteynik if (!cqe64) { 2135fd08f65SYevgeny Kliteynik if (unlikely(dr_cq->mdev->state == 2145fd08f65SYevgeny Kliteynik MLX5_DEVICE_STATE_INTERNAL_ERROR)) { 2155fd08f65SYevgeny Kliteynik mlx5_core_dbg_once(dr_cq->mdev, 2165fd08f65SYevgeny Kliteynik "Polling CQ while device is shutting down\n"); 2175fd08f65SYevgeny Kliteynik return CQ_POLL_ERR; 2185fd08f65SYevgeny Kliteynik } 219297cccebSAlex Vesker return CQ_EMPTY; 2205fd08f65SYevgeny Kliteynik } 221297cccebSAlex Vesker 222297cccebSAlex Vesker mlx5_cqwq_pop(&dr_cq->wq); 223297cccebSAlex Vesker err = dr_parse_cqe(dr_cq, cqe64); 224297cccebSAlex Vesker mlx5_cqwq_update_db_record(&dr_cq->wq); 225297cccebSAlex Vesker 226297cccebSAlex Vesker return err; 227297cccebSAlex Vesker } 228297cccebSAlex Vesker 229297cccebSAlex Vesker static int dr_poll_cq(struct mlx5dr_cq *dr_cq, int ne) 230297cccebSAlex Vesker { 231297cccebSAlex Vesker int npolled; 232297cccebSAlex Vesker int err = 0; 233297cccebSAlex Vesker 234297cccebSAlex Vesker for (npolled = 0; npolled < ne; ++npolled) { 235297cccebSAlex Vesker err = dr_cq_poll_one(dr_cq); 236297cccebSAlex Vesker if (err != CQ_OK) 237297cccebSAlex Vesker break; 238297cccebSAlex Vesker } 239297cccebSAlex Vesker 240297cccebSAlex Vesker return err == CQ_POLL_ERR ? err : npolled; 241297cccebSAlex Vesker } 242297cccebSAlex Vesker 243297cccebSAlex Vesker static struct mlx5dr_qp *dr_create_rc_qp(struct mlx5_core_dev *mdev, 244297cccebSAlex Vesker struct dr_qp_init_attr *attr) 245297cccebSAlex Vesker { 246ec44e72bSLeon Romanovsky u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 247297cccebSAlex Vesker u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {}; 248297cccebSAlex Vesker struct mlx5_wq_param wqp; 249297cccebSAlex Vesker struct mlx5dr_qp *dr_qp; 250297cccebSAlex Vesker int inlen; 251297cccebSAlex Vesker void *qpc; 252297cccebSAlex Vesker void *in; 253297cccebSAlex Vesker int err; 254297cccebSAlex Vesker 255297cccebSAlex Vesker dr_qp = kzalloc(sizeof(*dr_qp), GFP_KERNEL); 256297cccebSAlex Vesker if (!dr_qp) 257297cccebSAlex Vesker return NULL; 258297cccebSAlex Vesker 259297cccebSAlex Vesker wqp.buf_numa_node = mdev->priv.numa_node; 260297cccebSAlex Vesker wqp.db_numa_node = mdev->priv.numa_node; 261297cccebSAlex Vesker 262297cccebSAlex Vesker dr_qp->rq.pc = 0; 263297cccebSAlex Vesker dr_qp->rq.cc = 0; 264297cccebSAlex Vesker dr_qp->rq.wqe_cnt = 4; 265297cccebSAlex Vesker dr_qp->sq.pc = 0; 266297cccebSAlex Vesker dr_qp->sq.cc = 0; 267297cccebSAlex Vesker dr_qp->sq.wqe_cnt = roundup_pow_of_two(attr->max_send_wr); 268297cccebSAlex Vesker 269297cccebSAlex Vesker MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 270297cccebSAlex Vesker MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(dr_qp->rq.wqe_cnt)); 271297cccebSAlex Vesker MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(dr_qp->sq.wqe_cnt)); 272297cccebSAlex Vesker err = mlx5_wq_qp_create(mdev, &wqp, temp_qpc, &dr_qp->wq, 273297cccebSAlex Vesker &dr_qp->wq_ctrl); 274297cccebSAlex Vesker if (err) { 275b7d0db55SErez Shitrit mlx5_core_warn(mdev, "Can't create QP WQ\n"); 276297cccebSAlex Vesker goto err_wq; 277297cccebSAlex Vesker } 278297cccebSAlex Vesker 279297cccebSAlex Vesker dr_qp->sq.wqe_head = kcalloc(dr_qp->sq.wqe_cnt, 280297cccebSAlex Vesker sizeof(dr_qp->sq.wqe_head[0]), 281297cccebSAlex Vesker GFP_KERNEL); 282297cccebSAlex Vesker 283297cccebSAlex Vesker if (!dr_qp->sq.wqe_head) { 284297cccebSAlex Vesker mlx5_core_warn(mdev, "Can't allocate wqe head\n"); 285297cccebSAlex Vesker goto err_wqe_head; 286297cccebSAlex Vesker } 287297cccebSAlex Vesker 288297cccebSAlex Vesker inlen = MLX5_ST_SZ_BYTES(create_qp_in) + 289297cccebSAlex Vesker MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * 290297cccebSAlex Vesker dr_qp->wq_ctrl.buf.npages; 291297cccebSAlex Vesker in = kvzalloc(inlen, GFP_KERNEL); 292297cccebSAlex Vesker if (!in) { 293297cccebSAlex Vesker err = -ENOMEM; 294297cccebSAlex Vesker goto err_in; 295297cccebSAlex Vesker } 296297cccebSAlex Vesker 297297cccebSAlex Vesker qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 298297cccebSAlex Vesker MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 299297cccebSAlex Vesker MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 300aeacb52aSYevgeny Kliteynik MLX5_SET(qpc, qpc, isolate_vl_tc, attr->isolate_vl_tc); 301297cccebSAlex Vesker MLX5_SET(qpc, qpc, pd, attr->pdn); 302297cccebSAlex Vesker MLX5_SET(qpc, qpc, uar_page, attr->uar->index); 303297cccebSAlex Vesker MLX5_SET(qpc, qpc, log_page_size, 304297cccebSAlex Vesker dr_qp->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 305297cccebSAlex Vesker MLX5_SET(qpc, qpc, fre, 1); 306297cccebSAlex Vesker MLX5_SET(qpc, qpc, rlky, 1); 307297cccebSAlex Vesker MLX5_SET(qpc, qpc, cqn_snd, attr->cqn); 308297cccebSAlex Vesker MLX5_SET(qpc, qpc, cqn_rcv, attr->cqn); 309297cccebSAlex Vesker MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 310297cccebSAlex Vesker MLX5_SET(qpc, qpc, log_rq_size, ilog2(dr_qp->rq.wqe_cnt)); 311297cccebSAlex Vesker MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 312297cccebSAlex Vesker MLX5_SET(qpc, qpc, log_sq_size, ilog2(dr_qp->sq.wqe_cnt)); 3134806f1e2SMaor Gottlieb MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(mdev)); 314297cccebSAlex Vesker MLX5_SET64(qpc, qpc, dbr_addr, dr_qp->wq_ctrl.db.dma); 315297cccebSAlex Vesker if (MLX5_CAP_GEN(mdev, cqe_version) == 1) 316297cccebSAlex Vesker MLX5_SET(qpc, qpc, user_index, 0xFFFFFF); 317297cccebSAlex Vesker mlx5_fill_page_frag_array(&dr_qp->wq_ctrl.buf, 318297cccebSAlex Vesker (__be64 *)MLX5_ADDR_OF(create_qp_in, 319297cccebSAlex Vesker in, pas)); 320297cccebSAlex Vesker 321ec44e72bSLeon Romanovsky MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 322ec44e72bSLeon Romanovsky err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 323f93f4f4fSLeon Romanovsky dr_qp->qpn = MLX5_GET(create_qp_out, out, qpn); 32447a357deSDenis Efremov kvfree(in); 325ec44e72bSLeon Romanovsky if (err) 326297cccebSAlex Vesker goto err_in; 327297cccebSAlex Vesker dr_qp->uar = attr->uar; 328297cccebSAlex Vesker 329297cccebSAlex Vesker return dr_qp; 330297cccebSAlex Vesker 331297cccebSAlex Vesker err_in: 332297cccebSAlex Vesker kfree(dr_qp->sq.wqe_head); 333297cccebSAlex Vesker err_wqe_head: 334297cccebSAlex Vesker mlx5_wq_destroy(&dr_qp->wq_ctrl); 335297cccebSAlex Vesker err_wq: 336297cccebSAlex Vesker kfree(dr_qp); 337297cccebSAlex Vesker return NULL; 338297cccebSAlex Vesker } 339297cccebSAlex Vesker 340297cccebSAlex Vesker static void dr_destroy_qp(struct mlx5_core_dev *mdev, 341297cccebSAlex Vesker struct mlx5dr_qp *dr_qp) 342297cccebSAlex Vesker { 343ec44e72bSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 344ec44e72bSLeon Romanovsky 345ec44e72bSLeon Romanovsky MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 346f93f4f4fSLeon Romanovsky MLX5_SET(destroy_qp_in, in, qpn, dr_qp->qpn); 347ec44e72bSLeon Romanovsky mlx5_cmd_exec_in(mdev, destroy_qp, in); 348ec44e72bSLeon Romanovsky 349297cccebSAlex Vesker kfree(dr_qp->sq.wqe_head); 350297cccebSAlex Vesker mlx5_wq_destroy(&dr_qp->wq_ctrl); 351297cccebSAlex Vesker kfree(dr_qp); 352297cccebSAlex Vesker } 353297cccebSAlex Vesker 354297cccebSAlex Vesker static void dr_cmd_notify_hw(struct mlx5dr_qp *dr_qp, void *ctrl) 355297cccebSAlex Vesker { 356297cccebSAlex Vesker dma_wmb(); 357ff1925bbSYevgeny Kliteynik *dr_qp->wq.sq.db = cpu_to_be32(dr_qp->sq.pc & 0xffff); 358297cccebSAlex Vesker 359297cccebSAlex Vesker /* After wmb() the hw aware of new work */ 360297cccebSAlex Vesker wmb(); 361297cccebSAlex Vesker 362297cccebSAlex Vesker mlx5_write64(ctrl, dr_qp->uar->map + MLX5_BF_OFFSET); 363297cccebSAlex Vesker } 364297cccebSAlex Vesker 365297cccebSAlex Vesker static void dr_rdma_segments(struct mlx5dr_qp *dr_qp, u64 remote_addr, 366297cccebSAlex Vesker u32 rkey, struct dr_data_seg *data_seg, 3677d22ad73SYevgeny Kliteynik u32 opcode, bool notify_hw) 368297cccebSAlex Vesker { 369297cccebSAlex Vesker struct mlx5_wqe_raddr_seg *wq_raddr; 370297cccebSAlex Vesker struct mlx5_wqe_ctrl_seg *wq_ctrl; 371297cccebSAlex Vesker struct mlx5_wqe_data_seg *wq_dseg; 372297cccebSAlex Vesker unsigned int size; 373297cccebSAlex Vesker unsigned int idx; 374297cccebSAlex Vesker 375297cccebSAlex Vesker size = sizeof(*wq_ctrl) / 16 + sizeof(*wq_dseg) / 16 + 376297cccebSAlex Vesker sizeof(*wq_raddr) / 16; 377297cccebSAlex Vesker 378297cccebSAlex Vesker idx = dr_qp->sq.pc & (dr_qp->sq.wqe_cnt - 1); 379297cccebSAlex Vesker 380297cccebSAlex Vesker wq_ctrl = mlx5_wq_cyc_get_wqe(&dr_qp->wq.sq, idx); 381297cccebSAlex Vesker wq_ctrl->imm = 0; 382297cccebSAlex Vesker wq_ctrl->fm_ce_se = (data_seg->send_flags) ? 383297cccebSAlex Vesker MLX5_WQE_CTRL_CQ_UPDATE : 0; 384297cccebSAlex Vesker wq_ctrl->opmod_idx_opcode = cpu_to_be32(((dr_qp->sq.pc & 0xffff) << 8) | 385297cccebSAlex Vesker opcode); 386f93f4f4fSLeon Romanovsky wq_ctrl->qpn_ds = cpu_to_be32(size | dr_qp->qpn << 8); 387297cccebSAlex Vesker wq_raddr = (void *)(wq_ctrl + 1); 388297cccebSAlex Vesker wq_raddr->raddr = cpu_to_be64(remote_addr); 389297cccebSAlex Vesker wq_raddr->rkey = cpu_to_be32(rkey); 390297cccebSAlex Vesker wq_raddr->reserved = 0; 391297cccebSAlex Vesker 392297cccebSAlex Vesker wq_dseg = (void *)(wq_raddr + 1); 393297cccebSAlex Vesker wq_dseg->byte_count = cpu_to_be32(data_seg->length); 394297cccebSAlex Vesker wq_dseg->lkey = cpu_to_be32(data_seg->lkey); 395297cccebSAlex Vesker wq_dseg->addr = cpu_to_be64(data_seg->addr); 396297cccebSAlex Vesker 397297cccebSAlex Vesker dr_qp->sq.wqe_head[idx] = dr_qp->sq.pc++; 398297cccebSAlex Vesker 3997d22ad73SYevgeny Kliteynik if (notify_hw) 400297cccebSAlex Vesker dr_cmd_notify_hw(dr_qp, wq_ctrl); 401297cccebSAlex Vesker } 402297cccebSAlex Vesker 403297cccebSAlex Vesker static void dr_post_send(struct mlx5dr_qp *dr_qp, struct postsend_info *send_info) 404297cccebSAlex Vesker { 405297cccebSAlex Vesker dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey, 4067d22ad73SYevgeny Kliteynik &send_info->write, MLX5_OPCODE_RDMA_WRITE, false); 407297cccebSAlex Vesker dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey, 4087d22ad73SYevgeny Kliteynik &send_info->read, MLX5_OPCODE_RDMA_READ, true); 409297cccebSAlex Vesker } 410297cccebSAlex Vesker 411297cccebSAlex Vesker /** 412297cccebSAlex Vesker * mlx5dr_send_fill_and_append_ste_send_info: Add data to be sent 413297cccebSAlex Vesker * with send_list parameters: 414297cccebSAlex Vesker * 415297cccebSAlex Vesker * @ste: The data that attached to this specific ste 416297cccebSAlex Vesker * @size: of data to write 417297cccebSAlex Vesker * @offset: of the data from start of the hw_ste entry 418297cccebSAlex Vesker * @data: data 419297cccebSAlex Vesker * @ste_info: ste to be sent with send_list 420297cccebSAlex Vesker * @send_list: to append into it 421297cccebSAlex Vesker * @copy_data: if true indicates that the data should be kept because 422297cccebSAlex Vesker * it's not backuped any where (like in re-hash). 423297cccebSAlex Vesker * if false, it lets the data to be updated after 424297cccebSAlex Vesker * it was added to the list. 425297cccebSAlex Vesker */ 426297cccebSAlex Vesker void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size, 427297cccebSAlex Vesker u16 offset, u8 *data, 428297cccebSAlex Vesker struct mlx5dr_ste_send_info *ste_info, 429297cccebSAlex Vesker struct list_head *send_list, 430297cccebSAlex Vesker bool copy_data) 431297cccebSAlex Vesker { 432297cccebSAlex Vesker ste_info->size = size; 433297cccebSAlex Vesker ste_info->ste = ste; 434297cccebSAlex Vesker ste_info->offset = offset; 435297cccebSAlex Vesker 436297cccebSAlex Vesker if (copy_data) { 437297cccebSAlex Vesker memcpy(ste_info->data_cont, data, size); 438297cccebSAlex Vesker ste_info->data = ste_info->data_cont; 439297cccebSAlex Vesker } else { 440297cccebSAlex Vesker ste_info->data = data; 441297cccebSAlex Vesker } 442297cccebSAlex Vesker 443297cccebSAlex Vesker list_add_tail(&ste_info->send_list, send_list); 444297cccebSAlex Vesker } 445297cccebSAlex Vesker 446297cccebSAlex Vesker /* The function tries to consume one wc each time, unless the queue is full, in 447297cccebSAlex Vesker * that case, which means that the hw is behind the sw in a full queue len 448297cccebSAlex Vesker * the function will drain the cq till it empty. 449297cccebSAlex Vesker */ 450297cccebSAlex Vesker static int dr_handle_pending_wc(struct mlx5dr_domain *dmn, 451297cccebSAlex Vesker struct mlx5dr_send_ring *send_ring) 452297cccebSAlex Vesker { 453297cccebSAlex Vesker bool is_drain = false; 454297cccebSAlex Vesker int ne; 455297cccebSAlex Vesker 456297cccebSAlex Vesker if (send_ring->pending_wqe < send_ring->signal_th) 457297cccebSAlex Vesker return 0; 458297cccebSAlex Vesker 459297cccebSAlex Vesker /* Queue is full start drain it */ 460297cccebSAlex Vesker if (send_ring->pending_wqe >= 461297cccebSAlex Vesker dmn->send_ring->signal_th * TH_NUMS_TO_DRAIN) 462297cccebSAlex Vesker is_drain = true; 463297cccebSAlex Vesker 464297cccebSAlex Vesker do { 465297cccebSAlex Vesker ne = dr_poll_cq(send_ring->cq, 1); 466d5a84e96SYevgeny Kliteynik if (unlikely(ne < 0)) { 467d5a84e96SYevgeny Kliteynik mlx5_core_warn_once(dmn->mdev, "SMFS QPN 0x%x is disabled/limited", 468d5a84e96SYevgeny Kliteynik send_ring->qp->qpn); 469d5a84e96SYevgeny Kliteynik send_ring->err_state = true; 470297cccebSAlex Vesker return ne; 471d5a84e96SYevgeny Kliteynik } else if (ne == 1) { 472297cccebSAlex Vesker send_ring->pending_wqe -= send_ring->signal_th; 473d5a84e96SYevgeny Kliteynik } 474297cccebSAlex Vesker } while (is_drain && send_ring->pending_wqe); 475297cccebSAlex Vesker 476297cccebSAlex Vesker return 0; 477297cccebSAlex Vesker } 478297cccebSAlex Vesker 479297cccebSAlex Vesker static void dr_fill_data_segs(struct mlx5dr_send_ring *send_ring, 480297cccebSAlex Vesker struct postsend_info *send_info) 481297cccebSAlex Vesker { 482297cccebSAlex Vesker send_ring->pending_wqe++; 483297cccebSAlex Vesker 484297cccebSAlex Vesker if (send_ring->pending_wqe % send_ring->signal_th == 0) 485297cccebSAlex Vesker send_info->write.send_flags |= IB_SEND_SIGNALED; 486297cccebSAlex Vesker 487297cccebSAlex Vesker send_ring->pending_wqe++; 488297cccebSAlex Vesker send_info->read.length = send_info->write.length; 489297cccebSAlex Vesker /* Read into the same write area */ 490297cccebSAlex Vesker send_info->read.addr = (uintptr_t)send_info->write.addr; 49183fec3f1SAharon Landau send_info->read.lkey = send_ring->mr->mkey; 492297cccebSAlex Vesker 493297cccebSAlex Vesker if (send_ring->pending_wqe % send_ring->signal_th == 0) 494297cccebSAlex Vesker send_info->read.send_flags = IB_SEND_SIGNALED; 495297cccebSAlex Vesker else 496297cccebSAlex Vesker send_info->read.send_flags = 0; 497297cccebSAlex Vesker } 498297cccebSAlex Vesker 499297cccebSAlex Vesker static int dr_postsend_icm_data(struct mlx5dr_domain *dmn, 500297cccebSAlex Vesker struct postsend_info *send_info) 501297cccebSAlex Vesker { 502297cccebSAlex Vesker struct mlx5dr_send_ring *send_ring = dmn->send_ring; 503297cccebSAlex Vesker u32 buff_offset; 504297cccebSAlex Vesker int ret; 505297cccebSAlex Vesker 506d5a84e96SYevgeny Kliteynik if (unlikely(dmn->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || 507d5a84e96SYevgeny Kliteynik send_ring->err_state)) { 508d5a84e96SYevgeny Kliteynik mlx5_core_dbg_once(dmn->mdev, 509d5a84e96SYevgeny Kliteynik "Skipping post send: QP err state: %d, device state: %d\n", 510d5a84e96SYevgeny Kliteynik send_ring->err_state, dmn->mdev->state); 511d5a84e96SYevgeny Kliteynik return 0; 512d5a84e96SYevgeny Kliteynik } 513d5a84e96SYevgeny Kliteynik 514cedb2819SAlex Vesker spin_lock(&send_ring->lock); 515cedb2819SAlex Vesker 516297cccebSAlex Vesker ret = dr_handle_pending_wc(dmn, send_ring); 517297cccebSAlex Vesker if (ret) 518cedb2819SAlex Vesker goto out_unlock; 519297cccebSAlex Vesker 520297cccebSAlex Vesker if (send_info->write.length > dmn->info.max_inline_size) { 521297cccebSAlex Vesker buff_offset = (send_ring->tx_head & 522297cccebSAlex Vesker (dmn->send_ring->signal_th - 1)) * 523297cccebSAlex Vesker send_ring->max_post_send_size; 524297cccebSAlex Vesker /* Copy to ring mr */ 525297cccebSAlex Vesker memcpy(send_ring->buf + buff_offset, 526297cccebSAlex Vesker (void *)(uintptr_t)send_info->write.addr, 527297cccebSAlex Vesker send_info->write.length); 528297cccebSAlex Vesker send_info->write.addr = (uintptr_t)send_ring->mr->dma_addr + buff_offset; 52983fec3f1SAharon Landau send_info->write.lkey = send_ring->mr->mkey; 530297cccebSAlex Vesker } 531297cccebSAlex Vesker 532297cccebSAlex Vesker send_ring->tx_head++; 533297cccebSAlex Vesker dr_fill_data_segs(send_ring, send_info); 534297cccebSAlex Vesker dr_post_send(send_ring->qp, send_info); 535297cccebSAlex Vesker 536cedb2819SAlex Vesker out_unlock: 537cedb2819SAlex Vesker spin_unlock(&send_ring->lock); 538cedb2819SAlex Vesker return ret; 539297cccebSAlex Vesker } 540297cccebSAlex Vesker 541297cccebSAlex Vesker static int dr_get_tbl_copy_details(struct mlx5dr_domain *dmn, 542297cccebSAlex Vesker struct mlx5dr_ste_htbl *htbl, 543297cccebSAlex Vesker u8 **data, 544297cccebSAlex Vesker u32 *byte_size, 545297cccebSAlex Vesker int *iterations, 546297cccebSAlex Vesker int *num_stes) 547297cccebSAlex Vesker { 548f51bb517SRongwei Liu u32 chunk_byte_size = mlx5dr_icm_pool_get_chunk_byte_size(htbl->chunk); 549297cccebSAlex Vesker int alloc_size; 550297cccebSAlex Vesker 551f51bb517SRongwei Liu if (chunk_byte_size > dmn->send_ring->max_post_send_size) { 552f51bb517SRongwei Liu *iterations = chunk_byte_size / dmn->send_ring->max_post_send_size; 553297cccebSAlex Vesker *byte_size = dmn->send_ring->max_post_send_size; 554297cccebSAlex Vesker alloc_size = *byte_size; 555297cccebSAlex Vesker *num_stes = *byte_size / DR_STE_SIZE; 556297cccebSAlex Vesker } else { 557297cccebSAlex Vesker *iterations = 1; 558f51bb517SRongwei Liu *num_stes = mlx5dr_icm_pool_get_chunk_num_of_entries(htbl->chunk); 559297cccebSAlex Vesker alloc_size = *num_stes * DR_STE_SIZE; 560297cccebSAlex Vesker } 561297cccebSAlex Vesker 562b7f86258SRoi Dayan *data = kvzalloc(alloc_size, GFP_KERNEL); 563297cccebSAlex Vesker if (!*data) 564297cccebSAlex Vesker return -ENOMEM; 565297cccebSAlex Vesker 566297cccebSAlex Vesker return 0; 567297cccebSAlex Vesker } 568297cccebSAlex Vesker 569297cccebSAlex Vesker /** 570297cccebSAlex Vesker * mlx5dr_send_postsend_ste: write size bytes into offset from the hw cm. 571297cccebSAlex Vesker * 572297cccebSAlex Vesker * @dmn: Domain 573297cccebSAlex Vesker * @ste: The ste struct that contains the data (at 574297cccebSAlex Vesker * least part of it) 575297cccebSAlex Vesker * @data: The real data to send size data 576297cccebSAlex Vesker * @size: for writing. 577297cccebSAlex Vesker * @offset: The offset from the icm mapped data to 578297cccebSAlex Vesker * start write to this for write only part of the 579297cccebSAlex Vesker * buffer. 580297cccebSAlex Vesker * 581297cccebSAlex Vesker * Return: 0 on success. 582297cccebSAlex Vesker */ 583297cccebSAlex Vesker int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn, struct mlx5dr_ste *ste, 584297cccebSAlex Vesker u8 *data, u16 size, u16 offset) 585297cccebSAlex Vesker { 586297cccebSAlex Vesker struct postsend_info send_info = {}; 587297cccebSAlex Vesker 5884fe45e1dSYevgeny Kliteynik mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, data, size); 5894fe45e1dSYevgeny Kliteynik 590297cccebSAlex Vesker send_info.write.addr = (uintptr_t)data; 591297cccebSAlex Vesker send_info.write.length = size; 592297cccebSAlex Vesker send_info.write.lkey = 0; 593297cccebSAlex Vesker send_info.remote_addr = mlx5dr_ste_get_mr_addr(ste) + offset; 594003f4f9aSRongwei Liu send_info.rkey = mlx5dr_icm_pool_get_chunk_rkey(ste->htbl->chunk); 595297cccebSAlex Vesker 596297cccebSAlex Vesker return dr_postsend_icm_data(dmn, &send_info); 597297cccebSAlex Vesker } 598297cccebSAlex Vesker 599297cccebSAlex Vesker int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, 600297cccebSAlex Vesker struct mlx5dr_ste_htbl *htbl, 601297cccebSAlex Vesker u8 *formatted_ste, u8 *mask) 602297cccebSAlex Vesker { 603f51bb517SRongwei Liu u32 byte_size = mlx5dr_icm_pool_get_chunk_byte_size(htbl->chunk); 604297cccebSAlex Vesker int num_stes_per_iter; 605297cccebSAlex Vesker int iterations; 606297cccebSAlex Vesker u8 *data; 607297cccebSAlex Vesker int ret; 608297cccebSAlex Vesker int i; 609297cccebSAlex Vesker int j; 610297cccebSAlex Vesker 611297cccebSAlex Vesker ret = dr_get_tbl_copy_details(dmn, htbl, &data, &byte_size, 612297cccebSAlex Vesker &iterations, &num_stes_per_iter); 613297cccebSAlex Vesker if (ret) 614297cccebSAlex Vesker return ret; 615297cccebSAlex Vesker 6164fe45e1dSYevgeny Kliteynik mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, formatted_ste, DR_STE_SIZE); 6174fe45e1dSYevgeny Kliteynik 618297cccebSAlex Vesker /* Send the data iteration times */ 619297cccebSAlex Vesker for (i = 0; i < iterations; i++) { 620297cccebSAlex Vesker u32 ste_index = i * (byte_size / DR_STE_SIZE); 621297cccebSAlex Vesker struct postsend_info send_info = {}; 622297cccebSAlex Vesker 623297cccebSAlex Vesker /* Copy all ste's on the data buffer 624297cccebSAlex Vesker * need to add the bit_mask 625297cccebSAlex Vesker */ 626297cccebSAlex Vesker for (j = 0; j < num_stes_per_iter; j++) { 627597534bdSRongwei Liu struct mlx5dr_ste *ste = &htbl->chunk->ste_arr[ste_index + j]; 628297cccebSAlex Vesker u32 ste_off = j * DR_STE_SIZE; 629297cccebSAlex Vesker 63097ffd895SYevgeny Kliteynik if (mlx5dr_ste_is_not_used(ste)) { 631297cccebSAlex Vesker memcpy(data + ste_off, 632297cccebSAlex Vesker formatted_ste, DR_STE_SIZE); 633297cccebSAlex Vesker } else { 634297cccebSAlex Vesker /* Copy data */ 635297cccebSAlex Vesker memcpy(data + ste_off, 6360d7f1595SRongwei Liu htbl->chunk->hw_ste_arr + 6370d7f1595SRongwei Liu DR_STE_SIZE_REDUCED * (ste_index + j), 638297cccebSAlex Vesker DR_STE_SIZE_REDUCED); 639297cccebSAlex Vesker /* Copy bit_mask */ 640297cccebSAlex Vesker memcpy(data + ste_off + DR_STE_SIZE_REDUCED, 641297cccebSAlex Vesker mask, DR_STE_SIZE_MASK); 6424fe45e1dSYevgeny Kliteynik /* Only when we have mask we need to re-arrange the STE */ 6434fe45e1dSYevgeny Kliteynik mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, 6444fe45e1dSYevgeny Kliteynik data + (j * DR_STE_SIZE), 6454fe45e1dSYevgeny Kliteynik DR_STE_SIZE); 646297cccebSAlex Vesker } 647297cccebSAlex Vesker } 648297cccebSAlex Vesker 649297cccebSAlex Vesker send_info.write.addr = (uintptr_t)data; 650297cccebSAlex Vesker send_info.write.length = byte_size; 651297cccebSAlex Vesker send_info.write.lkey = 0; 652297cccebSAlex Vesker send_info.remote_addr = 653597534bdSRongwei Liu mlx5dr_ste_get_mr_addr(htbl->chunk->ste_arr + ste_index); 654003f4f9aSRongwei Liu send_info.rkey = mlx5dr_icm_pool_get_chunk_rkey(htbl->chunk); 655297cccebSAlex Vesker 656297cccebSAlex Vesker ret = dr_postsend_icm_data(dmn, &send_info); 657297cccebSAlex Vesker if (ret) 658297cccebSAlex Vesker goto out_free; 659297cccebSAlex Vesker } 660297cccebSAlex Vesker 661297cccebSAlex Vesker out_free: 662b7f86258SRoi Dayan kvfree(data); 663297cccebSAlex Vesker return ret; 664297cccebSAlex Vesker } 665297cccebSAlex Vesker 666297cccebSAlex Vesker /* Initialize htble with default STEs */ 667297cccebSAlex Vesker int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn, 668297cccebSAlex Vesker struct mlx5dr_ste_htbl *htbl, 669297cccebSAlex Vesker u8 *ste_init_data, 670297cccebSAlex Vesker bool update_hw_ste) 671297cccebSAlex Vesker { 672f51bb517SRongwei Liu u32 byte_size = mlx5dr_icm_pool_get_chunk_byte_size(htbl->chunk); 673297cccebSAlex Vesker int iterations; 674297cccebSAlex Vesker int num_stes; 6754fe45e1dSYevgeny Kliteynik u8 *copy_dst; 676297cccebSAlex Vesker u8 *data; 677297cccebSAlex Vesker int ret; 678297cccebSAlex Vesker int i; 679297cccebSAlex Vesker 680297cccebSAlex Vesker ret = dr_get_tbl_copy_details(dmn, htbl, &data, &byte_size, 681297cccebSAlex Vesker &iterations, &num_stes); 682297cccebSAlex Vesker if (ret) 683297cccebSAlex Vesker return ret; 684297cccebSAlex Vesker 685297cccebSAlex Vesker if (update_hw_ste) { 6864fe45e1dSYevgeny Kliteynik /* Copy the reduced STE to hash table ste_arr */ 6874fe45e1dSYevgeny Kliteynik for (i = 0; i < num_stes; i++) { 688597534bdSRongwei Liu copy_dst = htbl->chunk->hw_ste_arr + i * DR_STE_SIZE_REDUCED; 689297cccebSAlex Vesker memcpy(copy_dst, ste_init_data, DR_STE_SIZE_REDUCED); 690297cccebSAlex Vesker } 691297cccebSAlex Vesker } 692297cccebSAlex Vesker 6934fe45e1dSYevgeny Kliteynik mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, ste_init_data, DR_STE_SIZE); 6944fe45e1dSYevgeny Kliteynik 6954fe45e1dSYevgeny Kliteynik /* Copy the same STE on the data buffer */ 6964fe45e1dSYevgeny Kliteynik for (i = 0; i < num_stes; i++) { 6974fe45e1dSYevgeny Kliteynik copy_dst = data + i * DR_STE_SIZE; 6984fe45e1dSYevgeny Kliteynik memcpy(copy_dst, ste_init_data, DR_STE_SIZE); 6994fe45e1dSYevgeny Kliteynik } 7004fe45e1dSYevgeny Kliteynik 701297cccebSAlex Vesker /* Send the data iteration times */ 702297cccebSAlex Vesker for (i = 0; i < iterations; i++) { 703297cccebSAlex Vesker u8 ste_index = i * (byte_size / DR_STE_SIZE); 704297cccebSAlex Vesker struct postsend_info send_info = {}; 705297cccebSAlex Vesker 706297cccebSAlex Vesker send_info.write.addr = (uintptr_t)data; 707297cccebSAlex Vesker send_info.write.length = byte_size; 708297cccebSAlex Vesker send_info.write.lkey = 0; 709297cccebSAlex Vesker send_info.remote_addr = 710597534bdSRongwei Liu mlx5dr_ste_get_mr_addr(htbl->chunk->ste_arr + ste_index); 711003f4f9aSRongwei Liu send_info.rkey = mlx5dr_icm_pool_get_chunk_rkey(htbl->chunk); 712297cccebSAlex Vesker 713297cccebSAlex Vesker ret = dr_postsend_icm_data(dmn, &send_info); 714297cccebSAlex Vesker if (ret) 715297cccebSAlex Vesker goto out_free; 716297cccebSAlex Vesker } 717297cccebSAlex Vesker 718297cccebSAlex Vesker out_free: 719b7f86258SRoi Dayan kvfree(data); 720297cccebSAlex Vesker return ret; 721297cccebSAlex Vesker } 722297cccebSAlex Vesker 723297cccebSAlex Vesker int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn, 724297cccebSAlex Vesker struct mlx5dr_action *action) 725297cccebSAlex Vesker { 726297cccebSAlex Vesker struct postsend_info send_info = {}; 727297cccebSAlex Vesker 7289dac2966SJianbo Liu send_info.write.addr = (uintptr_t)action->rewrite->data; 7299dac2966SJianbo Liu send_info.write.length = action->rewrite->num_of_actions * 730692b0399SHamdan Igbaria DR_MODIFY_ACTION_SIZE; 731297cccebSAlex Vesker send_info.write.lkey = 0; 732003f4f9aSRongwei Liu send_info.remote_addr = 733003f4f9aSRongwei Liu mlx5dr_icm_pool_get_chunk_mr_addr(action->rewrite->chunk); 734003f4f9aSRongwei Liu send_info.rkey = mlx5dr_icm_pool_get_chunk_rkey(action->rewrite->chunk); 735297cccebSAlex Vesker 736*4238654cSzhang songyi return dr_postsend_icm_data(dmn, &send_info); 737297cccebSAlex Vesker } 738297cccebSAlex Vesker 739297cccebSAlex Vesker static int dr_modify_qp_rst2init(struct mlx5_core_dev *mdev, 740297cccebSAlex Vesker struct mlx5dr_qp *dr_qp, 741297cccebSAlex Vesker int port) 742297cccebSAlex Vesker { 743297cccebSAlex Vesker u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; 744297cccebSAlex Vesker void *qpc; 745297cccebSAlex Vesker 746297cccebSAlex Vesker qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc); 747297cccebSAlex Vesker 748297cccebSAlex Vesker MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, port); 749297cccebSAlex Vesker MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); 750297cccebSAlex Vesker MLX5_SET(qpc, qpc, rre, 1); 751297cccebSAlex Vesker MLX5_SET(qpc, qpc, rwe, 1); 752297cccebSAlex Vesker 753acab4b88SLeon Romanovsky MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP); 754f93f4f4fSLeon Romanovsky MLX5_SET(rst2init_qp_in, in, qpn, dr_qp->qpn); 755acab4b88SLeon Romanovsky 756acab4b88SLeon Romanovsky return mlx5_cmd_exec_in(mdev, rst2init_qp, in); 757297cccebSAlex Vesker } 758297cccebSAlex Vesker 759297cccebSAlex Vesker static int dr_cmd_modify_qp_rtr2rts(struct mlx5_core_dev *mdev, 760297cccebSAlex Vesker struct mlx5dr_qp *dr_qp, 761297cccebSAlex Vesker struct dr_qp_rts_attr *attr) 762297cccebSAlex Vesker { 763297cccebSAlex Vesker u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; 764297cccebSAlex Vesker void *qpc; 765297cccebSAlex Vesker 766297cccebSAlex Vesker qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc); 767297cccebSAlex Vesker 768f93f4f4fSLeon Romanovsky MLX5_SET(rtr2rts_qp_in, in, qpn, dr_qp->qpn); 769297cccebSAlex Vesker 770297cccebSAlex Vesker MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt); 771297cccebSAlex Vesker MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry); 772ec449ed8SYevgeny Kliteynik MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x8); /* ~1ms */ 773297cccebSAlex Vesker 774acab4b88SLeon Romanovsky MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP); 775f93f4f4fSLeon Romanovsky MLX5_SET(rtr2rts_qp_in, in, qpn, dr_qp->qpn); 776acab4b88SLeon Romanovsky 777acab4b88SLeon Romanovsky return mlx5_cmd_exec_in(mdev, rtr2rts_qp, in); 778297cccebSAlex Vesker } 779297cccebSAlex Vesker 780297cccebSAlex Vesker static int dr_cmd_modify_qp_init2rtr(struct mlx5_core_dev *mdev, 781297cccebSAlex Vesker struct mlx5dr_qp *dr_qp, 782297cccebSAlex Vesker struct dr_qp_rtr_attr *attr) 783297cccebSAlex Vesker { 784297cccebSAlex Vesker u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; 785297cccebSAlex Vesker void *qpc; 786297cccebSAlex Vesker 787297cccebSAlex Vesker qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc); 788297cccebSAlex Vesker 789f93f4f4fSLeon Romanovsky MLX5_SET(init2rtr_qp_in, in, qpn, dr_qp->qpn); 790297cccebSAlex Vesker 791297cccebSAlex Vesker MLX5_SET(qpc, qpc, mtu, attr->mtu); 792297cccebSAlex Vesker MLX5_SET(qpc, qpc, log_msg_max, DR_CHUNK_SIZE_MAX - 1); 793297cccebSAlex Vesker MLX5_SET(qpc, qpc, remote_qpn, attr->qp_num); 794297cccebSAlex Vesker memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32), 795297cccebSAlex Vesker attr->dgid_attr.mac, sizeof(attr->dgid_attr.mac)); 796297cccebSAlex Vesker memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), 797297cccebSAlex Vesker attr->dgid_attr.gid, sizeof(attr->dgid_attr.gid)); 798297cccebSAlex Vesker MLX5_SET(qpc, qpc, primary_address_path.src_addr_index, 799297cccebSAlex Vesker attr->sgid_index); 800297cccebSAlex Vesker 801297cccebSAlex Vesker if (attr->dgid_attr.roce_ver == MLX5_ROCE_VERSION_2) 802297cccebSAlex Vesker MLX5_SET(qpc, qpc, primary_address_path.udp_sport, 803297cccebSAlex Vesker attr->udp_src_port); 804297cccebSAlex Vesker 805297cccebSAlex Vesker MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, attr->port_num); 8067304d603SYevgeny Kliteynik MLX5_SET(qpc, qpc, primary_address_path.fl, attr->fl); 807297cccebSAlex Vesker MLX5_SET(qpc, qpc, min_rnr_nak, 1); 808297cccebSAlex Vesker 809acab4b88SLeon Romanovsky MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP); 810f93f4f4fSLeon Romanovsky MLX5_SET(init2rtr_qp_in, in, qpn, dr_qp->qpn); 811acab4b88SLeon Romanovsky 812acab4b88SLeon Romanovsky return mlx5_cmd_exec_in(mdev, init2rtr_qp, in); 813297cccebSAlex Vesker } 814297cccebSAlex Vesker 8157304d603SYevgeny Kliteynik static bool dr_send_allow_fl(struct mlx5dr_cmd_caps *caps) 8167304d603SYevgeny Kliteynik { 8177304d603SYevgeny Kliteynik /* Check whether RC RoCE QP creation with force loopback is allowed. 8187304d603SYevgeny Kliteynik * There are two separate capability bits for this: 8197304d603SYevgeny Kliteynik * - force loopback when RoCE is enabled 8207304d603SYevgeny Kliteynik * - force loopback when RoCE is disabled 8217304d603SYevgeny Kliteynik */ 8227304d603SYevgeny Kliteynik return ((caps->roce_caps.roce_en && 8237304d603SYevgeny Kliteynik caps->roce_caps.fl_rc_qp_when_roce_enabled) || 8247304d603SYevgeny Kliteynik (!caps->roce_caps.roce_en && 8257304d603SYevgeny Kliteynik caps->roce_caps.fl_rc_qp_when_roce_disabled)); 8267304d603SYevgeny Kliteynik } 8277304d603SYevgeny Kliteynik 828297cccebSAlex Vesker static int dr_prepare_qp_to_rts(struct mlx5dr_domain *dmn) 829297cccebSAlex Vesker { 830297cccebSAlex Vesker struct mlx5dr_qp *dr_qp = dmn->send_ring->qp; 831297cccebSAlex Vesker struct dr_qp_rts_attr rts_attr = {}; 832297cccebSAlex Vesker struct dr_qp_rtr_attr rtr_attr = {}; 833297cccebSAlex Vesker enum ib_mtu mtu = IB_MTU_1024; 834297cccebSAlex Vesker u16 gid_index = 0; 835297cccebSAlex Vesker int port = 1; 836297cccebSAlex Vesker int ret; 837297cccebSAlex Vesker 838297cccebSAlex Vesker /* Init */ 839297cccebSAlex Vesker ret = dr_modify_qp_rst2init(dmn->mdev, dr_qp, port); 840b7d0db55SErez Shitrit if (ret) { 841b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed modify QP rst2init\n"); 842297cccebSAlex Vesker return ret; 843b7d0db55SErez Shitrit } 844297cccebSAlex Vesker 845297cccebSAlex Vesker /* RTR */ 846297cccebSAlex Vesker rtr_attr.mtu = mtu; 847f93f4f4fSLeon Romanovsky rtr_attr.qp_num = dr_qp->qpn; 848297cccebSAlex Vesker rtr_attr.min_rnr_timer = 12; 849297cccebSAlex Vesker rtr_attr.port_num = port; 850297cccebSAlex Vesker rtr_attr.udp_src_port = dmn->info.caps.roce_min_src_udp; 851297cccebSAlex Vesker 8527304d603SYevgeny Kliteynik /* If QP creation with force loopback is allowed, then there 8537304d603SYevgeny Kliteynik * is no need for GID index when creating the QP. 8547304d603SYevgeny Kliteynik * Otherwise we query GID attributes and use GID index. 8557304d603SYevgeny Kliteynik */ 8567304d603SYevgeny Kliteynik rtr_attr.fl = dr_send_allow_fl(&dmn->info.caps); 8577304d603SYevgeny Kliteynik if (!rtr_attr.fl) { 8587304d603SYevgeny Kliteynik ret = mlx5dr_cmd_query_gid(dmn->mdev, port, gid_index, 8597304d603SYevgeny Kliteynik &rtr_attr.dgid_attr); 8607304d603SYevgeny Kliteynik if (ret) 8617304d603SYevgeny Kliteynik return ret; 8627304d603SYevgeny Kliteynik 8637304d603SYevgeny Kliteynik rtr_attr.sgid_index = gid_index; 8647304d603SYevgeny Kliteynik } 8657304d603SYevgeny Kliteynik 866297cccebSAlex Vesker ret = dr_cmd_modify_qp_init2rtr(dmn->mdev, dr_qp, &rtr_attr); 867b7d0db55SErez Shitrit if (ret) { 868b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed modify QP init2rtr\n"); 869297cccebSAlex Vesker return ret; 870b7d0db55SErez Shitrit } 871297cccebSAlex Vesker 872297cccebSAlex Vesker /* RTS */ 873297cccebSAlex Vesker rts_attr.timeout = 14; 874297cccebSAlex Vesker rts_attr.retry_cnt = 7; 875297cccebSAlex Vesker rts_attr.rnr_retry = 7; 876297cccebSAlex Vesker 877297cccebSAlex Vesker ret = dr_cmd_modify_qp_rtr2rts(dmn->mdev, dr_qp, &rts_attr); 878b7d0db55SErez Shitrit if (ret) { 879b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed modify QP rtr2rts\n"); 880297cccebSAlex Vesker return ret; 881b7d0db55SErez Shitrit } 882297cccebSAlex Vesker 883297cccebSAlex Vesker return 0; 884297cccebSAlex Vesker } 885297cccebSAlex Vesker 8868075411dSErez Shitrit static void dr_cq_complete(struct mlx5_core_cq *mcq, 8878075411dSErez Shitrit struct mlx5_eqe *eqe) 8888075411dSErez Shitrit { 8898075411dSErez Shitrit pr_err("CQ completion CQ: #%u\n", mcq->cqn); 8908075411dSErez Shitrit } 8918075411dSErez Shitrit 892297cccebSAlex Vesker static struct mlx5dr_cq *dr_create_cq(struct mlx5_core_dev *mdev, 893297cccebSAlex Vesker struct mlx5_uars_page *uar, 894297cccebSAlex Vesker size_t ncqe) 895297cccebSAlex Vesker { 896297cccebSAlex Vesker u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {}; 897297cccebSAlex Vesker u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 898297cccebSAlex Vesker struct mlx5_wq_param wqp; 899297cccebSAlex Vesker struct mlx5_cqe64 *cqe; 900297cccebSAlex Vesker struct mlx5dr_cq *cq; 901297cccebSAlex Vesker int inlen, err, eqn; 902297cccebSAlex Vesker void *cqc, *in; 903297cccebSAlex Vesker __be64 *pas; 90482996995SAlex Vesker int vector; 905297cccebSAlex Vesker u32 i; 906297cccebSAlex Vesker 907297cccebSAlex Vesker cq = kzalloc(sizeof(*cq), GFP_KERNEL); 908297cccebSAlex Vesker if (!cq) 909297cccebSAlex Vesker return NULL; 910297cccebSAlex Vesker 911297cccebSAlex Vesker ncqe = roundup_pow_of_two(ncqe); 912297cccebSAlex Vesker MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(ncqe)); 913297cccebSAlex Vesker 914297cccebSAlex Vesker wqp.buf_numa_node = mdev->priv.numa_node; 915297cccebSAlex Vesker wqp.db_numa_node = mdev->priv.numa_node; 916297cccebSAlex Vesker 917297cccebSAlex Vesker err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &cq->wq, 918297cccebSAlex Vesker &cq->wq_ctrl); 919297cccebSAlex Vesker if (err) 920297cccebSAlex Vesker goto out; 921297cccebSAlex Vesker 922297cccebSAlex Vesker for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 923297cccebSAlex Vesker cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 924297cccebSAlex Vesker cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK; 925297cccebSAlex Vesker } 926297cccebSAlex Vesker 927297cccebSAlex Vesker inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 928297cccebSAlex Vesker sizeof(u64) * cq->wq_ctrl.buf.npages; 929297cccebSAlex Vesker in = kvzalloc(inlen, GFP_KERNEL); 930297cccebSAlex Vesker if (!in) 931297cccebSAlex Vesker goto err_cqwq; 932297cccebSAlex Vesker 933c0702a4bSErez Shitrit vector = raw_smp_processor_id() % mlx5_comp_vectors_count(mdev); 934563476aeSShay Drory err = mlx5_vector2eqn(mdev, vector, &eqn); 935297cccebSAlex Vesker if (err) { 936297cccebSAlex Vesker kvfree(in); 937297cccebSAlex Vesker goto err_cqwq; 938297cccebSAlex Vesker } 939297cccebSAlex Vesker 940297cccebSAlex Vesker cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 941297cccebSAlex Vesker MLX5_SET(cqc, cqc, log_cq_size, ilog2(ncqe)); 942616d5769STal Gilboa MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 943297cccebSAlex Vesker MLX5_SET(cqc, cqc, uar_page, uar->index); 944297cccebSAlex Vesker MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 945297cccebSAlex Vesker MLX5_ADAPTER_PAGE_SHIFT); 946297cccebSAlex Vesker MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 947297cccebSAlex Vesker 948297cccebSAlex Vesker pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 949297cccebSAlex Vesker mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, pas); 950297cccebSAlex Vesker 9518075411dSErez Shitrit cq->mcq.comp = dr_cq_complete; 952297cccebSAlex Vesker 953297cccebSAlex Vesker err = mlx5_core_create_cq(mdev, &cq->mcq, in, inlen, out, sizeof(out)); 954297cccebSAlex Vesker kvfree(in); 955297cccebSAlex Vesker 956297cccebSAlex Vesker if (err) 957297cccebSAlex Vesker goto err_cqwq; 958297cccebSAlex Vesker 959297cccebSAlex Vesker cq->mcq.cqe_sz = 64; 960297cccebSAlex Vesker cq->mcq.set_ci_db = cq->wq_ctrl.db.db; 961297cccebSAlex Vesker cq->mcq.arm_db = cq->wq_ctrl.db.db + 1; 962297cccebSAlex Vesker *cq->mcq.set_ci_db = 0; 9638075411dSErez Shitrit 9648075411dSErez Shitrit /* set no-zero value, in order to avoid the HW to run db-recovery on 9658075411dSErez Shitrit * CQ that used in polling mode. 9668075411dSErez Shitrit */ 9678075411dSErez Shitrit *cq->mcq.arm_db = cpu_to_be32(2 << 28); 9688075411dSErez Shitrit 969297cccebSAlex Vesker cq->mcq.vector = 0; 970297cccebSAlex Vesker cq->mcq.uar = uar; 9715fd08f65SYevgeny Kliteynik cq->mdev = mdev; 972297cccebSAlex Vesker 973297cccebSAlex Vesker return cq; 974297cccebSAlex Vesker 975297cccebSAlex Vesker err_cqwq: 976297cccebSAlex Vesker mlx5_wq_destroy(&cq->wq_ctrl); 977297cccebSAlex Vesker out: 978297cccebSAlex Vesker kfree(cq); 979297cccebSAlex Vesker return NULL; 980297cccebSAlex Vesker } 981297cccebSAlex Vesker 982297cccebSAlex Vesker static void dr_destroy_cq(struct mlx5_core_dev *mdev, struct mlx5dr_cq *cq) 983297cccebSAlex Vesker { 984297cccebSAlex Vesker mlx5_core_destroy_cq(mdev, &cq->mcq); 985297cccebSAlex Vesker mlx5_wq_destroy(&cq->wq_ctrl); 986297cccebSAlex Vesker kfree(cq); 987297cccebSAlex Vesker } 988297cccebSAlex Vesker 98983fec3f1SAharon Landau static int dr_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey) 990297cccebSAlex Vesker { 991297cccebSAlex Vesker u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; 992297cccebSAlex Vesker void *mkc; 993297cccebSAlex Vesker 994297cccebSAlex Vesker mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 995297cccebSAlex Vesker MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 996297cccebSAlex Vesker MLX5_SET(mkc, mkc, a, 1); 997297cccebSAlex Vesker MLX5_SET(mkc, mkc, rw, 1); 998297cccebSAlex Vesker MLX5_SET(mkc, mkc, rr, 1); 999297cccebSAlex Vesker MLX5_SET(mkc, mkc, lw, 1); 1000297cccebSAlex Vesker MLX5_SET(mkc, mkc, lr, 1); 1001297cccebSAlex Vesker 1002297cccebSAlex Vesker MLX5_SET(mkc, mkc, pd, pdn); 1003297cccebSAlex Vesker MLX5_SET(mkc, mkc, length64, 1); 1004297cccebSAlex Vesker MLX5_SET(mkc, mkc, qpn, 0xffffff); 1005297cccebSAlex Vesker 1006297cccebSAlex Vesker return mlx5_core_create_mkey(mdev, mkey, in, sizeof(in)); 1007297cccebSAlex Vesker } 1008297cccebSAlex Vesker 1009297cccebSAlex Vesker static struct mlx5dr_mr *dr_reg_mr(struct mlx5_core_dev *mdev, 1010297cccebSAlex Vesker u32 pdn, void *buf, size_t size) 1011297cccebSAlex Vesker { 1012297cccebSAlex Vesker struct mlx5dr_mr *mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1013297cccebSAlex Vesker struct device *dma_device; 1014297cccebSAlex Vesker dma_addr_t dma_addr; 1015297cccebSAlex Vesker int err; 1016297cccebSAlex Vesker 1017297cccebSAlex Vesker if (!mr) 1018297cccebSAlex Vesker return NULL; 1019297cccebSAlex Vesker 10207be3412aSParav Pandit dma_device = mlx5_core_dma_dev(mdev); 1021297cccebSAlex Vesker dma_addr = dma_map_single(dma_device, buf, size, 1022297cccebSAlex Vesker DMA_BIDIRECTIONAL); 1023297cccebSAlex Vesker err = dma_mapping_error(dma_device, dma_addr); 1024297cccebSAlex Vesker if (err) { 1025297cccebSAlex Vesker mlx5_core_warn(mdev, "Can't dma buf\n"); 1026297cccebSAlex Vesker kfree(mr); 1027297cccebSAlex Vesker return NULL; 1028297cccebSAlex Vesker } 1029297cccebSAlex Vesker 1030297cccebSAlex Vesker err = dr_create_mkey(mdev, pdn, &mr->mkey); 1031297cccebSAlex Vesker if (err) { 1032297cccebSAlex Vesker mlx5_core_warn(mdev, "Can't create mkey\n"); 1033297cccebSAlex Vesker dma_unmap_single(dma_device, dma_addr, size, 1034297cccebSAlex Vesker DMA_BIDIRECTIONAL); 1035297cccebSAlex Vesker kfree(mr); 1036297cccebSAlex Vesker return NULL; 1037297cccebSAlex Vesker } 1038297cccebSAlex Vesker 1039297cccebSAlex Vesker mr->dma_addr = dma_addr; 1040297cccebSAlex Vesker mr->size = size; 1041297cccebSAlex Vesker mr->addr = buf; 1042297cccebSAlex Vesker 1043297cccebSAlex Vesker return mr; 1044297cccebSAlex Vesker } 1045297cccebSAlex Vesker 1046297cccebSAlex Vesker static void dr_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5dr_mr *mr) 1047297cccebSAlex Vesker { 104883fec3f1SAharon Landau mlx5_core_destroy_mkey(mdev, mr->mkey); 10497be3412aSParav Pandit dma_unmap_single(mlx5_core_dma_dev(mdev), mr->dma_addr, mr->size, 1050297cccebSAlex Vesker DMA_BIDIRECTIONAL); 1051297cccebSAlex Vesker kfree(mr); 1052297cccebSAlex Vesker } 1053297cccebSAlex Vesker 1054297cccebSAlex Vesker int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn) 1055297cccebSAlex Vesker { 1056297cccebSAlex Vesker struct dr_qp_init_attr init_attr = {}; 1057297cccebSAlex Vesker int cq_size; 1058297cccebSAlex Vesker int size; 1059297cccebSAlex Vesker int ret; 1060297cccebSAlex Vesker 1061297cccebSAlex Vesker dmn->send_ring = kzalloc(sizeof(*dmn->send_ring), GFP_KERNEL); 1062297cccebSAlex Vesker if (!dmn->send_ring) 1063297cccebSAlex Vesker return -ENOMEM; 1064297cccebSAlex Vesker 1065297cccebSAlex Vesker cq_size = QUEUE_SIZE + 1; 1066297cccebSAlex Vesker dmn->send_ring->cq = dr_create_cq(dmn->mdev, dmn->uar, cq_size); 1067297cccebSAlex Vesker if (!dmn->send_ring->cq) { 1068b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed creating CQ\n"); 1069297cccebSAlex Vesker ret = -ENOMEM; 1070297cccebSAlex Vesker goto free_send_ring; 1071297cccebSAlex Vesker } 1072297cccebSAlex Vesker 1073297cccebSAlex Vesker init_attr.cqn = dmn->send_ring->cq->mcq.cqn; 1074297cccebSAlex Vesker init_attr.pdn = dmn->pdn; 1075297cccebSAlex Vesker init_attr.uar = dmn->uar; 1076297cccebSAlex Vesker init_attr.max_send_wr = QUEUE_SIZE; 1077aeacb52aSYevgeny Kliteynik 1078aeacb52aSYevgeny Kliteynik /* Isolated VL is applicable only if force loopback is supported */ 1079aeacb52aSYevgeny Kliteynik if (dr_send_allow_fl(&dmn->info.caps)) 1080aeacb52aSYevgeny Kliteynik init_attr.isolate_vl_tc = dmn->info.caps.isolate_vl_tc; 1081aeacb52aSYevgeny Kliteynik 1082cedb2819SAlex Vesker spin_lock_init(&dmn->send_ring->lock); 1083297cccebSAlex Vesker 1084297cccebSAlex Vesker dmn->send_ring->qp = dr_create_rc_qp(dmn->mdev, &init_attr); 1085297cccebSAlex Vesker if (!dmn->send_ring->qp) { 1086b7d0db55SErez Shitrit mlx5dr_err(dmn, "Failed creating QP\n"); 1087297cccebSAlex Vesker ret = -ENOMEM; 1088297cccebSAlex Vesker goto clean_cq; 1089297cccebSAlex Vesker } 1090297cccebSAlex Vesker 1091297cccebSAlex Vesker dmn->send_ring->cq->qp = dmn->send_ring->qp; 1092297cccebSAlex Vesker 1093297cccebSAlex Vesker dmn->info.max_send_wr = QUEUE_SIZE; 1094297cccebSAlex Vesker dmn->info.max_inline_size = min(dmn->send_ring->qp->max_inline_data, 1095297cccebSAlex Vesker DR_STE_SIZE); 1096297cccebSAlex Vesker 1097297cccebSAlex Vesker dmn->send_ring->signal_th = dmn->info.max_send_wr / 1098297cccebSAlex Vesker SIGNAL_PER_DIV_QUEUE; 1099297cccebSAlex Vesker 1100297cccebSAlex Vesker /* Prepare qp to be used */ 1101297cccebSAlex Vesker ret = dr_prepare_qp_to_rts(dmn); 1102297cccebSAlex Vesker if (ret) 1103297cccebSAlex Vesker goto clean_qp; 1104297cccebSAlex Vesker 1105297cccebSAlex Vesker dmn->send_ring->max_post_send_size = 1106297cccebSAlex Vesker mlx5dr_icm_pool_chunk_size_to_byte(DR_CHUNK_SIZE_1K, 1107297cccebSAlex Vesker DR_ICM_TYPE_STE); 1108297cccebSAlex Vesker 1109297cccebSAlex Vesker /* Allocating the max size as a buffer for writing */ 1110297cccebSAlex Vesker size = dmn->send_ring->signal_th * dmn->send_ring->max_post_send_size; 1111297cccebSAlex Vesker dmn->send_ring->buf = kzalloc(size, GFP_KERNEL); 1112297cccebSAlex Vesker if (!dmn->send_ring->buf) { 1113297cccebSAlex Vesker ret = -ENOMEM; 1114297cccebSAlex Vesker goto clean_qp; 1115297cccebSAlex Vesker } 1116297cccebSAlex Vesker 1117297cccebSAlex Vesker dmn->send_ring->buf_size = size; 1118297cccebSAlex Vesker 1119297cccebSAlex Vesker dmn->send_ring->mr = dr_reg_mr(dmn->mdev, 1120297cccebSAlex Vesker dmn->pdn, dmn->send_ring->buf, size); 1121297cccebSAlex Vesker if (!dmn->send_ring->mr) { 1122297cccebSAlex Vesker ret = -ENOMEM; 1123297cccebSAlex Vesker goto free_mem; 1124297cccebSAlex Vesker } 1125297cccebSAlex Vesker 1126297cccebSAlex Vesker dmn->send_ring->sync_mr = dr_reg_mr(dmn->mdev, 1127297cccebSAlex Vesker dmn->pdn, dmn->send_ring->sync_buff, 1128297cccebSAlex Vesker MIN_READ_SYNC); 1129297cccebSAlex Vesker if (!dmn->send_ring->sync_mr) { 1130297cccebSAlex Vesker ret = -ENOMEM; 1131297cccebSAlex Vesker goto clean_mr; 1132297cccebSAlex Vesker } 1133297cccebSAlex Vesker 1134297cccebSAlex Vesker return 0; 1135297cccebSAlex Vesker 1136297cccebSAlex Vesker clean_mr: 1137297cccebSAlex Vesker dr_dereg_mr(dmn->mdev, dmn->send_ring->mr); 1138297cccebSAlex Vesker free_mem: 1139297cccebSAlex Vesker kfree(dmn->send_ring->buf); 1140297cccebSAlex Vesker clean_qp: 1141297cccebSAlex Vesker dr_destroy_qp(dmn->mdev, dmn->send_ring->qp); 1142297cccebSAlex Vesker clean_cq: 1143297cccebSAlex Vesker dr_destroy_cq(dmn->mdev, dmn->send_ring->cq); 1144297cccebSAlex Vesker free_send_ring: 1145297cccebSAlex Vesker kfree(dmn->send_ring); 1146297cccebSAlex Vesker 1147297cccebSAlex Vesker return ret; 1148297cccebSAlex Vesker } 1149297cccebSAlex Vesker 1150297cccebSAlex Vesker void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn, 1151297cccebSAlex Vesker struct mlx5dr_send_ring *send_ring) 1152297cccebSAlex Vesker { 1153297cccebSAlex Vesker dr_destroy_qp(dmn->mdev, send_ring->qp); 1154297cccebSAlex Vesker dr_destroy_cq(dmn->mdev, send_ring->cq); 1155297cccebSAlex Vesker dr_dereg_mr(dmn->mdev, send_ring->sync_mr); 1156297cccebSAlex Vesker dr_dereg_mr(dmn->mdev, send_ring->mr); 1157297cccebSAlex Vesker kfree(send_ring->buf); 1158297cccebSAlex Vesker kfree(send_ring); 1159297cccebSAlex Vesker } 1160297cccebSAlex Vesker 1161297cccebSAlex Vesker int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn) 1162297cccebSAlex Vesker { 1163297cccebSAlex Vesker struct mlx5dr_send_ring *send_ring = dmn->send_ring; 1164297cccebSAlex Vesker struct postsend_info send_info = {}; 1165297cccebSAlex Vesker u8 data[DR_STE_SIZE]; 1166297cccebSAlex Vesker int num_of_sends_req; 1167297cccebSAlex Vesker int ret; 1168297cccebSAlex Vesker int i; 1169297cccebSAlex Vesker 1170297cccebSAlex Vesker /* Sending this amount of requests makes sure we will get drain */ 1171297cccebSAlex Vesker num_of_sends_req = send_ring->signal_th * TH_NUMS_TO_DRAIN / 2; 1172297cccebSAlex Vesker 1173297cccebSAlex Vesker /* Send fake requests forcing the last to be signaled */ 1174297cccebSAlex Vesker send_info.write.addr = (uintptr_t)data; 1175297cccebSAlex Vesker send_info.write.length = DR_STE_SIZE; 1176297cccebSAlex Vesker send_info.write.lkey = 0; 1177297cccebSAlex Vesker /* Using the sync_mr in order to write/read */ 1178297cccebSAlex Vesker send_info.remote_addr = (uintptr_t)send_ring->sync_mr->addr; 117983fec3f1SAharon Landau send_info.rkey = send_ring->sync_mr->mkey; 1180297cccebSAlex Vesker 1181297cccebSAlex Vesker for (i = 0; i < num_of_sends_req; i++) { 1182297cccebSAlex Vesker ret = dr_postsend_icm_data(dmn, &send_info); 1183297cccebSAlex Vesker if (ret) 1184297cccebSAlex Vesker return ret; 1185297cccebSAlex Vesker } 1186297cccebSAlex Vesker 1187cedb2819SAlex Vesker spin_lock(&send_ring->lock); 1188297cccebSAlex Vesker ret = dr_handle_pending_wc(dmn, send_ring); 1189cedb2819SAlex Vesker spin_unlock(&send_ring->lock); 1190297cccebSAlex Vesker 1191297cccebSAlex Vesker return ret; 1192297cccebSAlex Vesker } 1193