1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include <linux/smp.h> 5 #include "dr_types.h" 6 7 #define QUEUE_SIZE 128 8 #define SIGNAL_PER_DIV_QUEUE 16 9 #define TH_NUMS_TO_DRAIN 2 10 11 enum { CQ_OK = 0, CQ_EMPTY = -1, CQ_POLL_ERR = -2 }; 12 13 struct dr_data_seg { 14 u64 addr; 15 u32 length; 16 u32 lkey; 17 unsigned int send_flags; 18 }; 19 20 struct postsend_info { 21 struct dr_data_seg write; 22 struct dr_data_seg read; 23 u64 remote_addr; 24 u32 rkey; 25 }; 26 27 struct dr_qp_rtr_attr { 28 struct mlx5dr_cmd_gid_attr dgid_attr; 29 enum ib_mtu mtu; 30 u32 qp_num; 31 u16 port_num; 32 u8 min_rnr_timer; 33 u8 sgid_index; 34 u16 udp_src_port; 35 u8 fl:1; 36 }; 37 38 struct dr_qp_rts_attr { 39 u8 timeout; 40 u8 retry_cnt; 41 u8 rnr_retry; 42 }; 43 44 struct dr_qp_init_attr { 45 u32 cqn; 46 u32 pdn; 47 u32 max_send_wr; 48 struct mlx5_uars_page *uar; 49 u8 isolate_vl_tc:1; 50 }; 51 52 static int dr_parse_cqe(struct mlx5dr_cq *dr_cq, struct mlx5_cqe64 *cqe64) 53 { 54 unsigned int idx; 55 u8 opcode; 56 57 opcode = get_cqe_opcode(cqe64); 58 if (opcode == MLX5_CQE_REQ_ERR) { 59 idx = be16_to_cpu(cqe64->wqe_counter) & 60 (dr_cq->qp->sq.wqe_cnt - 1); 61 dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1; 62 } else if (opcode == MLX5_CQE_RESP_ERR) { 63 ++dr_cq->qp->sq.cc; 64 } else { 65 idx = be16_to_cpu(cqe64->wqe_counter) & 66 (dr_cq->qp->sq.wqe_cnt - 1); 67 dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1; 68 69 return CQ_OK; 70 } 71 72 return CQ_POLL_ERR; 73 } 74 75 static int dr_cq_poll_one(struct mlx5dr_cq *dr_cq) 76 { 77 struct mlx5_cqe64 *cqe64; 78 int err; 79 80 cqe64 = mlx5_cqwq_get_cqe(&dr_cq->wq); 81 if (!cqe64) 82 return CQ_EMPTY; 83 84 mlx5_cqwq_pop(&dr_cq->wq); 85 err = dr_parse_cqe(dr_cq, cqe64); 86 mlx5_cqwq_update_db_record(&dr_cq->wq); 87 88 return err; 89 } 90 91 static int dr_poll_cq(struct mlx5dr_cq *dr_cq, int ne) 92 { 93 int npolled; 94 int err = 0; 95 96 for (npolled = 0; npolled < ne; ++npolled) { 97 err = dr_cq_poll_one(dr_cq); 98 if (err != CQ_OK) 99 break; 100 } 101 102 return err == CQ_POLL_ERR ? err : npolled; 103 } 104 105 static struct mlx5dr_qp *dr_create_rc_qp(struct mlx5_core_dev *mdev, 106 struct dr_qp_init_attr *attr) 107 { 108 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 109 u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {}; 110 struct mlx5_wq_param wqp; 111 struct mlx5dr_qp *dr_qp; 112 int inlen; 113 void *qpc; 114 void *in; 115 int err; 116 117 dr_qp = kzalloc(sizeof(*dr_qp), GFP_KERNEL); 118 if (!dr_qp) 119 return NULL; 120 121 wqp.buf_numa_node = mdev->priv.numa_node; 122 wqp.db_numa_node = mdev->priv.numa_node; 123 124 dr_qp->rq.pc = 0; 125 dr_qp->rq.cc = 0; 126 dr_qp->rq.wqe_cnt = 4; 127 dr_qp->sq.pc = 0; 128 dr_qp->sq.cc = 0; 129 dr_qp->sq.wqe_cnt = roundup_pow_of_two(attr->max_send_wr); 130 131 MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 132 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(dr_qp->rq.wqe_cnt)); 133 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(dr_qp->sq.wqe_cnt)); 134 err = mlx5_wq_qp_create(mdev, &wqp, temp_qpc, &dr_qp->wq, 135 &dr_qp->wq_ctrl); 136 if (err) { 137 mlx5_core_warn(mdev, "Can't create QP WQ\n"); 138 goto err_wq; 139 } 140 141 dr_qp->sq.wqe_head = kcalloc(dr_qp->sq.wqe_cnt, 142 sizeof(dr_qp->sq.wqe_head[0]), 143 GFP_KERNEL); 144 145 if (!dr_qp->sq.wqe_head) { 146 mlx5_core_warn(mdev, "Can't allocate wqe head\n"); 147 goto err_wqe_head; 148 } 149 150 inlen = MLX5_ST_SZ_BYTES(create_qp_in) + 151 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * 152 dr_qp->wq_ctrl.buf.npages; 153 in = kvzalloc(inlen, GFP_KERNEL); 154 if (!in) { 155 err = -ENOMEM; 156 goto err_in; 157 } 158 159 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 160 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 161 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 162 MLX5_SET(qpc, qpc, isolate_vl_tc, attr->isolate_vl_tc); 163 MLX5_SET(qpc, qpc, pd, attr->pdn); 164 MLX5_SET(qpc, qpc, uar_page, attr->uar->index); 165 MLX5_SET(qpc, qpc, log_page_size, 166 dr_qp->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 167 MLX5_SET(qpc, qpc, fre, 1); 168 MLX5_SET(qpc, qpc, rlky, 1); 169 MLX5_SET(qpc, qpc, cqn_snd, attr->cqn); 170 MLX5_SET(qpc, qpc, cqn_rcv, attr->cqn); 171 MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 172 MLX5_SET(qpc, qpc, log_rq_size, ilog2(dr_qp->rq.wqe_cnt)); 173 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 174 MLX5_SET(qpc, qpc, log_sq_size, ilog2(dr_qp->sq.wqe_cnt)); 175 MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(mdev)); 176 MLX5_SET64(qpc, qpc, dbr_addr, dr_qp->wq_ctrl.db.dma); 177 if (MLX5_CAP_GEN(mdev, cqe_version) == 1) 178 MLX5_SET(qpc, qpc, user_index, 0xFFFFFF); 179 mlx5_fill_page_frag_array(&dr_qp->wq_ctrl.buf, 180 (__be64 *)MLX5_ADDR_OF(create_qp_in, 181 in, pas)); 182 183 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 184 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 185 dr_qp->qpn = MLX5_GET(create_qp_out, out, qpn); 186 kvfree(in); 187 if (err) 188 goto err_in; 189 dr_qp->uar = attr->uar; 190 191 return dr_qp; 192 193 err_in: 194 kfree(dr_qp->sq.wqe_head); 195 err_wqe_head: 196 mlx5_wq_destroy(&dr_qp->wq_ctrl); 197 err_wq: 198 kfree(dr_qp); 199 return NULL; 200 } 201 202 static void dr_destroy_qp(struct mlx5_core_dev *mdev, 203 struct mlx5dr_qp *dr_qp) 204 { 205 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 206 207 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 208 MLX5_SET(destroy_qp_in, in, qpn, dr_qp->qpn); 209 mlx5_cmd_exec_in(mdev, destroy_qp, in); 210 211 kfree(dr_qp->sq.wqe_head); 212 mlx5_wq_destroy(&dr_qp->wq_ctrl); 213 kfree(dr_qp); 214 } 215 216 static void dr_cmd_notify_hw(struct mlx5dr_qp *dr_qp, void *ctrl) 217 { 218 dma_wmb(); 219 *dr_qp->wq.sq.db = cpu_to_be32(dr_qp->sq.pc & 0xffff); 220 221 /* After wmb() the hw aware of new work */ 222 wmb(); 223 224 mlx5_write64(ctrl, dr_qp->uar->map + MLX5_BF_OFFSET); 225 } 226 227 static void dr_rdma_segments(struct mlx5dr_qp *dr_qp, u64 remote_addr, 228 u32 rkey, struct dr_data_seg *data_seg, 229 u32 opcode, bool notify_hw) 230 { 231 struct mlx5_wqe_raddr_seg *wq_raddr; 232 struct mlx5_wqe_ctrl_seg *wq_ctrl; 233 struct mlx5_wqe_data_seg *wq_dseg; 234 unsigned int size; 235 unsigned int idx; 236 237 size = sizeof(*wq_ctrl) / 16 + sizeof(*wq_dseg) / 16 + 238 sizeof(*wq_raddr) / 16; 239 240 idx = dr_qp->sq.pc & (dr_qp->sq.wqe_cnt - 1); 241 242 wq_ctrl = mlx5_wq_cyc_get_wqe(&dr_qp->wq.sq, idx); 243 wq_ctrl->imm = 0; 244 wq_ctrl->fm_ce_se = (data_seg->send_flags) ? 245 MLX5_WQE_CTRL_CQ_UPDATE : 0; 246 wq_ctrl->opmod_idx_opcode = cpu_to_be32(((dr_qp->sq.pc & 0xffff) << 8) | 247 opcode); 248 wq_ctrl->qpn_ds = cpu_to_be32(size | dr_qp->qpn << 8); 249 wq_raddr = (void *)(wq_ctrl + 1); 250 wq_raddr->raddr = cpu_to_be64(remote_addr); 251 wq_raddr->rkey = cpu_to_be32(rkey); 252 wq_raddr->reserved = 0; 253 254 wq_dseg = (void *)(wq_raddr + 1); 255 wq_dseg->byte_count = cpu_to_be32(data_seg->length); 256 wq_dseg->lkey = cpu_to_be32(data_seg->lkey); 257 wq_dseg->addr = cpu_to_be64(data_seg->addr); 258 259 dr_qp->sq.wqe_head[idx] = dr_qp->sq.pc++; 260 261 if (notify_hw) 262 dr_cmd_notify_hw(dr_qp, wq_ctrl); 263 } 264 265 static void dr_post_send(struct mlx5dr_qp *dr_qp, struct postsend_info *send_info) 266 { 267 dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey, 268 &send_info->write, MLX5_OPCODE_RDMA_WRITE, false); 269 dr_rdma_segments(dr_qp, send_info->remote_addr, send_info->rkey, 270 &send_info->read, MLX5_OPCODE_RDMA_READ, true); 271 } 272 273 /** 274 * mlx5dr_send_fill_and_append_ste_send_info: Add data to be sent 275 * with send_list parameters: 276 * 277 * @ste: The data that attached to this specific ste 278 * @size: of data to write 279 * @offset: of the data from start of the hw_ste entry 280 * @data: data 281 * @ste_info: ste to be sent with send_list 282 * @send_list: to append into it 283 * @copy_data: if true indicates that the data should be kept because 284 * it's not backuped any where (like in re-hash). 285 * if false, it lets the data to be updated after 286 * it was added to the list. 287 */ 288 void mlx5dr_send_fill_and_append_ste_send_info(struct mlx5dr_ste *ste, u16 size, 289 u16 offset, u8 *data, 290 struct mlx5dr_ste_send_info *ste_info, 291 struct list_head *send_list, 292 bool copy_data) 293 { 294 ste_info->size = size; 295 ste_info->ste = ste; 296 ste_info->offset = offset; 297 298 if (copy_data) { 299 memcpy(ste_info->data_cont, data, size); 300 ste_info->data = ste_info->data_cont; 301 } else { 302 ste_info->data = data; 303 } 304 305 list_add_tail(&ste_info->send_list, send_list); 306 } 307 308 /* The function tries to consume one wc each time, unless the queue is full, in 309 * that case, which means that the hw is behind the sw in a full queue len 310 * the function will drain the cq till it empty. 311 */ 312 static int dr_handle_pending_wc(struct mlx5dr_domain *dmn, 313 struct mlx5dr_send_ring *send_ring) 314 { 315 bool is_drain = false; 316 int ne; 317 318 if (send_ring->pending_wqe < send_ring->signal_th) 319 return 0; 320 321 /* Queue is full start drain it */ 322 if (send_ring->pending_wqe >= 323 dmn->send_ring->signal_th * TH_NUMS_TO_DRAIN) 324 is_drain = true; 325 326 do { 327 ne = dr_poll_cq(send_ring->cq, 1); 328 if (unlikely(ne < 0)) { 329 mlx5_core_warn_once(dmn->mdev, "SMFS QPN 0x%x is disabled/limited", 330 send_ring->qp->qpn); 331 send_ring->err_state = true; 332 return ne; 333 } else if (ne == 1) { 334 send_ring->pending_wqe -= send_ring->signal_th; 335 } 336 } while (is_drain && send_ring->pending_wqe); 337 338 return 0; 339 } 340 341 static void dr_fill_data_segs(struct mlx5dr_send_ring *send_ring, 342 struct postsend_info *send_info) 343 { 344 send_ring->pending_wqe++; 345 346 if (send_ring->pending_wqe % send_ring->signal_th == 0) 347 send_info->write.send_flags |= IB_SEND_SIGNALED; 348 349 send_ring->pending_wqe++; 350 send_info->read.length = send_info->write.length; 351 /* Read into the same write area */ 352 send_info->read.addr = (uintptr_t)send_info->write.addr; 353 send_info->read.lkey = send_ring->mr->mkey.key; 354 355 if (send_ring->pending_wqe % send_ring->signal_th == 0) 356 send_info->read.send_flags = IB_SEND_SIGNALED; 357 else 358 send_info->read.send_flags = 0; 359 } 360 361 static int dr_postsend_icm_data(struct mlx5dr_domain *dmn, 362 struct postsend_info *send_info) 363 { 364 struct mlx5dr_send_ring *send_ring = dmn->send_ring; 365 u32 buff_offset; 366 int ret; 367 368 if (unlikely(dmn->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || 369 send_ring->err_state)) { 370 mlx5_core_dbg_once(dmn->mdev, 371 "Skipping post send: QP err state: %d, device state: %d\n", 372 send_ring->err_state, dmn->mdev->state); 373 return 0; 374 } 375 376 spin_lock(&send_ring->lock); 377 378 ret = dr_handle_pending_wc(dmn, send_ring); 379 if (ret) 380 goto out_unlock; 381 382 if (send_info->write.length > dmn->info.max_inline_size) { 383 buff_offset = (send_ring->tx_head & 384 (dmn->send_ring->signal_th - 1)) * 385 send_ring->max_post_send_size; 386 /* Copy to ring mr */ 387 memcpy(send_ring->buf + buff_offset, 388 (void *)(uintptr_t)send_info->write.addr, 389 send_info->write.length); 390 send_info->write.addr = (uintptr_t)send_ring->mr->dma_addr + buff_offset; 391 send_info->write.lkey = send_ring->mr->mkey.key; 392 } 393 394 send_ring->tx_head++; 395 dr_fill_data_segs(send_ring, send_info); 396 dr_post_send(send_ring->qp, send_info); 397 398 out_unlock: 399 spin_unlock(&send_ring->lock); 400 return ret; 401 } 402 403 static int dr_get_tbl_copy_details(struct mlx5dr_domain *dmn, 404 struct mlx5dr_ste_htbl *htbl, 405 u8 **data, 406 u32 *byte_size, 407 int *iterations, 408 int *num_stes) 409 { 410 int alloc_size; 411 412 if (htbl->chunk->byte_size > dmn->send_ring->max_post_send_size) { 413 *iterations = htbl->chunk->byte_size / 414 dmn->send_ring->max_post_send_size; 415 *byte_size = dmn->send_ring->max_post_send_size; 416 alloc_size = *byte_size; 417 *num_stes = *byte_size / DR_STE_SIZE; 418 } else { 419 *iterations = 1; 420 *num_stes = htbl->chunk->num_of_entries; 421 alloc_size = *num_stes * DR_STE_SIZE; 422 } 423 424 *data = kvzalloc(alloc_size, GFP_KERNEL); 425 if (!*data) 426 return -ENOMEM; 427 428 return 0; 429 } 430 431 /** 432 * mlx5dr_send_postsend_ste: write size bytes into offset from the hw cm. 433 * 434 * @dmn: Domain 435 * @ste: The ste struct that contains the data (at 436 * least part of it) 437 * @data: The real data to send size data 438 * @size: for writing. 439 * @offset: The offset from the icm mapped data to 440 * start write to this for write only part of the 441 * buffer. 442 * 443 * Return: 0 on success. 444 */ 445 int mlx5dr_send_postsend_ste(struct mlx5dr_domain *dmn, struct mlx5dr_ste *ste, 446 u8 *data, u16 size, u16 offset) 447 { 448 struct postsend_info send_info = {}; 449 450 mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, data, size); 451 452 send_info.write.addr = (uintptr_t)data; 453 send_info.write.length = size; 454 send_info.write.lkey = 0; 455 send_info.remote_addr = mlx5dr_ste_get_mr_addr(ste) + offset; 456 send_info.rkey = ste->htbl->chunk->rkey; 457 458 return dr_postsend_icm_data(dmn, &send_info); 459 } 460 461 int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, 462 struct mlx5dr_ste_htbl *htbl, 463 u8 *formatted_ste, u8 *mask) 464 { 465 u32 byte_size = htbl->chunk->byte_size; 466 int num_stes_per_iter; 467 int iterations; 468 u8 *data; 469 int ret; 470 int i; 471 int j; 472 473 ret = dr_get_tbl_copy_details(dmn, htbl, &data, &byte_size, 474 &iterations, &num_stes_per_iter); 475 if (ret) 476 return ret; 477 478 mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, formatted_ste, DR_STE_SIZE); 479 480 /* Send the data iteration times */ 481 for (i = 0; i < iterations; i++) { 482 u32 ste_index = i * (byte_size / DR_STE_SIZE); 483 struct postsend_info send_info = {}; 484 485 /* Copy all ste's on the data buffer 486 * need to add the bit_mask 487 */ 488 for (j = 0; j < num_stes_per_iter; j++) { 489 struct mlx5dr_ste *ste = &htbl->ste_arr[ste_index + j]; 490 u32 ste_off = j * DR_STE_SIZE; 491 492 if (mlx5dr_ste_is_not_used(ste)) { 493 memcpy(data + ste_off, 494 formatted_ste, DR_STE_SIZE); 495 } else { 496 /* Copy data */ 497 memcpy(data + ste_off, 498 htbl->ste_arr[ste_index + j].hw_ste, 499 DR_STE_SIZE_REDUCED); 500 /* Copy bit_mask */ 501 memcpy(data + ste_off + DR_STE_SIZE_REDUCED, 502 mask, DR_STE_SIZE_MASK); 503 /* Only when we have mask we need to re-arrange the STE */ 504 mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, 505 data + (j * DR_STE_SIZE), 506 DR_STE_SIZE); 507 } 508 } 509 510 send_info.write.addr = (uintptr_t)data; 511 send_info.write.length = byte_size; 512 send_info.write.lkey = 0; 513 send_info.remote_addr = 514 mlx5dr_ste_get_mr_addr(htbl->ste_arr + ste_index); 515 send_info.rkey = htbl->chunk->rkey; 516 517 ret = dr_postsend_icm_data(dmn, &send_info); 518 if (ret) 519 goto out_free; 520 } 521 522 out_free: 523 kvfree(data); 524 return ret; 525 } 526 527 /* Initialize htble with default STEs */ 528 int mlx5dr_send_postsend_formatted_htbl(struct mlx5dr_domain *dmn, 529 struct mlx5dr_ste_htbl *htbl, 530 u8 *ste_init_data, 531 bool update_hw_ste) 532 { 533 u32 byte_size = htbl->chunk->byte_size; 534 int iterations; 535 int num_stes; 536 u8 *copy_dst; 537 u8 *data; 538 int ret; 539 int i; 540 541 ret = dr_get_tbl_copy_details(dmn, htbl, &data, &byte_size, 542 &iterations, &num_stes); 543 if (ret) 544 return ret; 545 546 if (update_hw_ste) { 547 /* Copy the reduced STE to hash table ste_arr */ 548 for (i = 0; i < num_stes; i++) { 549 copy_dst = htbl->hw_ste_arr + i * DR_STE_SIZE_REDUCED; 550 memcpy(copy_dst, ste_init_data, DR_STE_SIZE_REDUCED); 551 } 552 } 553 554 mlx5dr_ste_prepare_for_postsend(dmn->ste_ctx, ste_init_data, DR_STE_SIZE); 555 556 /* Copy the same STE on the data buffer */ 557 for (i = 0; i < num_stes; i++) { 558 copy_dst = data + i * DR_STE_SIZE; 559 memcpy(copy_dst, ste_init_data, DR_STE_SIZE); 560 } 561 562 /* Send the data iteration times */ 563 for (i = 0; i < iterations; i++) { 564 u8 ste_index = i * (byte_size / DR_STE_SIZE); 565 struct postsend_info send_info = {}; 566 567 send_info.write.addr = (uintptr_t)data; 568 send_info.write.length = byte_size; 569 send_info.write.lkey = 0; 570 send_info.remote_addr = 571 mlx5dr_ste_get_mr_addr(htbl->ste_arr + ste_index); 572 send_info.rkey = htbl->chunk->rkey; 573 574 ret = dr_postsend_icm_data(dmn, &send_info); 575 if (ret) 576 goto out_free; 577 } 578 579 out_free: 580 kvfree(data); 581 return ret; 582 } 583 584 int mlx5dr_send_postsend_action(struct mlx5dr_domain *dmn, 585 struct mlx5dr_action *action) 586 { 587 struct postsend_info send_info = {}; 588 int ret; 589 590 send_info.write.addr = (uintptr_t)action->rewrite->data; 591 send_info.write.length = action->rewrite->num_of_actions * 592 DR_MODIFY_ACTION_SIZE; 593 send_info.write.lkey = 0; 594 send_info.remote_addr = action->rewrite->chunk->mr_addr; 595 send_info.rkey = action->rewrite->chunk->rkey; 596 597 ret = dr_postsend_icm_data(dmn, &send_info); 598 599 return ret; 600 } 601 602 static int dr_modify_qp_rst2init(struct mlx5_core_dev *mdev, 603 struct mlx5dr_qp *dr_qp, 604 int port) 605 { 606 u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; 607 void *qpc; 608 609 qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc); 610 611 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, port); 612 MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); 613 MLX5_SET(qpc, qpc, rre, 1); 614 MLX5_SET(qpc, qpc, rwe, 1); 615 616 MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP); 617 MLX5_SET(rst2init_qp_in, in, qpn, dr_qp->qpn); 618 619 return mlx5_cmd_exec_in(mdev, rst2init_qp, in); 620 } 621 622 static int dr_cmd_modify_qp_rtr2rts(struct mlx5_core_dev *mdev, 623 struct mlx5dr_qp *dr_qp, 624 struct dr_qp_rts_attr *attr) 625 { 626 u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; 627 void *qpc; 628 629 qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc); 630 631 MLX5_SET(rtr2rts_qp_in, in, qpn, dr_qp->qpn); 632 633 MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt); 634 MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry); 635 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x8); /* ~1ms */ 636 637 MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP); 638 MLX5_SET(rtr2rts_qp_in, in, qpn, dr_qp->qpn); 639 640 return mlx5_cmd_exec_in(mdev, rtr2rts_qp, in); 641 } 642 643 static int dr_cmd_modify_qp_init2rtr(struct mlx5_core_dev *mdev, 644 struct mlx5dr_qp *dr_qp, 645 struct dr_qp_rtr_attr *attr) 646 { 647 u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; 648 void *qpc; 649 650 qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc); 651 652 MLX5_SET(init2rtr_qp_in, in, qpn, dr_qp->qpn); 653 654 MLX5_SET(qpc, qpc, mtu, attr->mtu); 655 MLX5_SET(qpc, qpc, log_msg_max, DR_CHUNK_SIZE_MAX - 1); 656 MLX5_SET(qpc, qpc, remote_qpn, attr->qp_num); 657 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32), 658 attr->dgid_attr.mac, sizeof(attr->dgid_attr.mac)); 659 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), 660 attr->dgid_attr.gid, sizeof(attr->dgid_attr.gid)); 661 MLX5_SET(qpc, qpc, primary_address_path.src_addr_index, 662 attr->sgid_index); 663 664 if (attr->dgid_attr.roce_ver == MLX5_ROCE_VERSION_2) 665 MLX5_SET(qpc, qpc, primary_address_path.udp_sport, 666 attr->udp_src_port); 667 668 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, attr->port_num); 669 MLX5_SET(qpc, qpc, primary_address_path.fl, attr->fl); 670 MLX5_SET(qpc, qpc, min_rnr_nak, 1); 671 672 MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP); 673 MLX5_SET(init2rtr_qp_in, in, qpn, dr_qp->qpn); 674 675 return mlx5_cmd_exec_in(mdev, init2rtr_qp, in); 676 } 677 678 static bool dr_send_allow_fl(struct mlx5dr_cmd_caps *caps) 679 { 680 /* Check whether RC RoCE QP creation with force loopback is allowed. 681 * There are two separate capability bits for this: 682 * - force loopback when RoCE is enabled 683 * - force loopback when RoCE is disabled 684 */ 685 return ((caps->roce_caps.roce_en && 686 caps->roce_caps.fl_rc_qp_when_roce_enabled) || 687 (!caps->roce_caps.roce_en && 688 caps->roce_caps.fl_rc_qp_when_roce_disabled)); 689 } 690 691 static int dr_prepare_qp_to_rts(struct mlx5dr_domain *dmn) 692 { 693 struct mlx5dr_qp *dr_qp = dmn->send_ring->qp; 694 struct dr_qp_rts_attr rts_attr = {}; 695 struct dr_qp_rtr_attr rtr_attr = {}; 696 enum ib_mtu mtu = IB_MTU_1024; 697 u16 gid_index = 0; 698 int port = 1; 699 int ret; 700 701 /* Init */ 702 ret = dr_modify_qp_rst2init(dmn->mdev, dr_qp, port); 703 if (ret) { 704 mlx5dr_err(dmn, "Failed modify QP rst2init\n"); 705 return ret; 706 } 707 708 /* RTR */ 709 rtr_attr.mtu = mtu; 710 rtr_attr.qp_num = dr_qp->qpn; 711 rtr_attr.min_rnr_timer = 12; 712 rtr_attr.port_num = port; 713 rtr_attr.udp_src_port = dmn->info.caps.roce_min_src_udp; 714 715 /* If QP creation with force loopback is allowed, then there 716 * is no need for GID index when creating the QP. 717 * Otherwise we query GID attributes and use GID index. 718 */ 719 rtr_attr.fl = dr_send_allow_fl(&dmn->info.caps); 720 if (!rtr_attr.fl) { 721 ret = mlx5dr_cmd_query_gid(dmn->mdev, port, gid_index, 722 &rtr_attr.dgid_attr); 723 if (ret) 724 return ret; 725 726 rtr_attr.sgid_index = gid_index; 727 } 728 729 ret = dr_cmd_modify_qp_init2rtr(dmn->mdev, dr_qp, &rtr_attr); 730 if (ret) { 731 mlx5dr_err(dmn, "Failed modify QP init2rtr\n"); 732 return ret; 733 } 734 735 /* RTS */ 736 rts_attr.timeout = 14; 737 rts_attr.retry_cnt = 7; 738 rts_attr.rnr_retry = 7; 739 740 ret = dr_cmd_modify_qp_rtr2rts(dmn->mdev, dr_qp, &rts_attr); 741 if (ret) { 742 mlx5dr_err(dmn, "Failed modify QP rtr2rts\n"); 743 return ret; 744 } 745 746 return 0; 747 } 748 749 static void dr_cq_complete(struct mlx5_core_cq *mcq, 750 struct mlx5_eqe *eqe) 751 { 752 pr_err("CQ completion CQ: #%u\n", mcq->cqn); 753 } 754 755 static struct mlx5dr_cq *dr_create_cq(struct mlx5_core_dev *mdev, 756 struct mlx5_uars_page *uar, 757 size_t ncqe) 758 { 759 u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {}; 760 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 761 struct mlx5_wq_param wqp; 762 struct mlx5_cqe64 *cqe; 763 struct mlx5dr_cq *cq; 764 int inlen, err, eqn; 765 void *cqc, *in; 766 __be64 *pas; 767 int vector; 768 u32 i; 769 770 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 771 if (!cq) 772 return NULL; 773 774 ncqe = roundup_pow_of_two(ncqe); 775 MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(ncqe)); 776 777 wqp.buf_numa_node = mdev->priv.numa_node; 778 wqp.db_numa_node = mdev->priv.numa_node; 779 780 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &cq->wq, 781 &cq->wq_ctrl); 782 if (err) 783 goto out; 784 785 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 786 cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 787 cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK; 788 } 789 790 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 791 sizeof(u64) * cq->wq_ctrl.buf.npages; 792 in = kvzalloc(inlen, GFP_KERNEL); 793 if (!in) 794 goto err_cqwq; 795 796 vector = raw_smp_processor_id() % mlx5_comp_vectors_count(mdev); 797 err = mlx5_vector2eqn(mdev, vector, &eqn); 798 if (err) { 799 kvfree(in); 800 goto err_cqwq; 801 } 802 803 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 804 MLX5_SET(cqc, cqc, log_cq_size, ilog2(ncqe)); 805 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 806 MLX5_SET(cqc, cqc, uar_page, uar->index); 807 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 808 MLX5_ADAPTER_PAGE_SHIFT); 809 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 810 811 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 812 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, pas); 813 814 cq->mcq.comp = dr_cq_complete; 815 816 err = mlx5_core_create_cq(mdev, &cq->mcq, in, inlen, out, sizeof(out)); 817 kvfree(in); 818 819 if (err) 820 goto err_cqwq; 821 822 cq->mcq.cqe_sz = 64; 823 cq->mcq.set_ci_db = cq->wq_ctrl.db.db; 824 cq->mcq.arm_db = cq->wq_ctrl.db.db + 1; 825 *cq->mcq.set_ci_db = 0; 826 827 /* set no-zero value, in order to avoid the HW to run db-recovery on 828 * CQ that used in polling mode. 829 */ 830 *cq->mcq.arm_db = cpu_to_be32(2 << 28); 831 832 cq->mcq.vector = 0; 833 cq->mcq.uar = uar; 834 835 return cq; 836 837 err_cqwq: 838 mlx5_wq_destroy(&cq->wq_ctrl); 839 out: 840 kfree(cq); 841 return NULL; 842 } 843 844 static void dr_destroy_cq(struct mlx5_core_dev *mdev, struct mlx5dr_cq *cq) 845 { 846 mlx5_core_destroy_cq(mdev, &cq->mcq); 847 mlx5_wq_destroy(&cq->wq_ctrl); 848 kfree(cq); 849 } 850 851 static int 852 dr_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, struct mlx5_core_mkey *mkey) 853 { 854 u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; 855 void *mkc; 856 857 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 858 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 859 MLX5_SET(mkc, mkc, a, 1); 860 MLX5_SET(mkc, mkc, rw, 1); 861 MLX5_SET(mkc, mkc, rr, 1); 862 MLX5_SET(mkc, mkc, lw, 1); 863 MLX5_SET(mkc, mkc, lr, 1); 864 865 MLX5_SET(mkc, mkc, pd, pdn); 866 MLX5_SET(mkc, mkc, length64, 1); 867 MLX5_SET(mkc, mkc, qpn, 0xffffff); 868 869 return mlx5_core_create_mkey(mdev, mkey, in, sizeof(in)); 870 } 871 872 static struct mlx5dr_mr *dr_reg_mr(struct mlx5_core_dev *mdev, 873 u32 pdn, void *buf, size_t size) 874 { 875 struct mlx5dr_mr *mr = kzalloc(sizeof(*mr), GFP_KERNEL); 876 struct device *dma_device; 877 dma_addr_t dma_addr; 878 int err; 879 880 if (!mr) 881 return NULL; 882 883 dma_device = mlx5_core_dma_dev(mdev); 884 dma_addr = dma_map_single(dma_device, buf, size, 885 DMA_BIDIRECTIONAL); 886 err = dma_mapping_error(dma_device, dma_addr); 887 if (err) { 888 mlx5_core_warn(mdev, "Can't dma buf\n"); 889 kfree(mr); 890 return NULL; 891 } 892 893 err = dr_create_mkey(mdev, pdn, &mr->mkey); 894 if (err) { 895 mlx5_core_warn(mdev, "Can't create mkey\n"); 896 dma_unmap_single(dma_device, dma_addr, size, 897 DMA_BIDIRECTIONAL); 898 kfree(mr); 899 return NULL; 900 } 901 902 mr->dma_addr = dma_addr; 903 mr->size = size; 904 mr->addr = buf; 905 906 return mr; 907 } 908 909 static void dr_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5dr_mr *mr) 910 { 911 mlx5_core_destroy_mkey(mdev, &mr->mkey); 912 dma_unmap_single(mlx5_core_dma_dev(mdev), mr->dma_addr, mr->size, 913 DMA_BIDIRECTIONAL); 914 kfree(mr); 915 } 916 917 int mlx5dr_send_ring_alloc(struct mlx5dr_domain *dmn) 918 { 919 struct dr_qp_init_attr init_attr = {}; 920 int cq_size; 921 int size; 922 int ret; 923 924 dmn->send_ring = kzalloc(sizeof(*dmn->send_ring), GFP_KERNEL); 925 if (!dmn->send_ring) 926 return -ENOMEM; 927 928 cq_size = QUEUE_SIZE + 1; 929 dmn->send_ring->cq = dr_create_cq(dmn->mdev, dmn->uar, cq_size); 930 if (!dmn->send_ring->cq) { 931 mlx5dr_err(dmn, "Failed creating CQ\n"); 932 ret = -ENOMEM; 933 goto free_send_ring; 934 } 935 936 init_attr.cqn = dmn->send_ring->cq->mcq.cqn; 937 init_attr.pdn = dmn->pdn; 938 init_attr.uar = dmn->uar; 939 init_attr.max_send_wr = QUEUE_SIZE; 940 941 /* Isolated VL is applicable only if force loopback is supported */ 942 if (dr_send_allow_fl(&dmn->info.caps)) 943 init_attr.isolate_vl_tc = dmn->info.caps.isolate_vl_tc; 944 945 spin_lock_init(&dmn->send_ring->lock); 946 947 dmn->send_ring->qp = dr_create_rc_qp(dmn->mdev, &init_attr); 948 if (!dmn->send_ring->qp) { 949 mlx5dr_err(dmn, "Failed creating QP\n"); 950 ret = -ENOMEM; 951 goto clean_cq; 952 } 953 954 dmn->send_ring->cq->qp = dmn->send_ring->qp; 955 956 dmn->info.max_send_wr = QUEUE_SIZE; 957 dmn->info.max_inline_size = min(dmn->send_ring->qp->max_inline_data, 958 DR_STE_SIZE); 959 960 dmn->send_ring->signal_th = dmn->info.max_send_wr / 961 SIGNAL_PER_DIV_QUEUE; 962 963 /* Prepare qp to be used */ 964 ret = dr_prepare_qp_to_rts(dmn); 965 if (ret) 966 goto clean_qp; 967 968 dmn->send_ring->max_post_send_size = 969 mlx5dr_icm_pool_chunk_size_to_byte(DR_CHUNK_SIZE_1K, 970 DR_ICM_TYPE_STE); 971 972 /* Allocating the max size as a buffer for writing */ 973 size = dmn->send_ring->signal_th * dmn->send_ring->max_post_send_size; 974 dmn->send_ring->buf = kzalloc(size, GFP_KERNEL); 975 if (!dmn->send_ring->buf) { 976 ret = -ENOMEM; 977 goto clean_qp; 978 } 979 980 dmn->send_ring->buf_size = size; 981 982 dmn->send_ring->mr = dr_reg_mr(dmn->mdev, 983 dmn->pdn, dmn->send_ring->buf, size); 984 if (!dmn->send_ring->mr) { 985 ret = -ENOMEM; 986 goto free_mem; 987 } 988 989 dmn->send_ring->sync_mr = dr_reg_mr(dmn->mdev, 990 dmn->pdn, dmn->send_ring->sync_buff, 991 MIN_READ_SYNC); 992 if (!dmn->send_ring->sync_mr) { 993 ret = -ENOMEM; 994 goto clean_mr; 995 } 996 997 return 0; 998 999 clean_mr: 1000 dr_dereg_mr(dmn->mdev, dmn->send_ring->mr); 1001 free_mem: 1002 kfree(dmn->send_ring->buf); 1003 clean_qp: 1004 dr_destroy_qp(dmn->mdev, dmn->send_ring->qp); 1005 clean_cq: 1006 dr_destroy_cq(dmn->mdev, dmn->send_ring->cq); 1007 free_send_ring: 1008 kfree(dmn->send_ring); 1009 1010 return ret; 1011 } 1012 1013 void mlx5dr_send_ring_free(struct mlx5dr_domain *dmn, 1014 struct mlx5dr_send_ring *send_ring) 1015 { 1016 dr_destroy_qp(dmn->mdev, send_ring->qp); 1017 dr_destroy_cq(dmn->mdev, send_ring->cq); 1018 dr_dereg_mr(dmn->mdev, send_ring->sync_mr); 1019 dr_dereg_mr(dmn->mdev, send_ring->mr); 1020 kfree(send_ring->buf); 1021 kfree(send_ring); 1022 } 1023 1024 int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn) 1025 { 1026 struct mlx5dr_send_ring *send_ring = dmn->send_ring; 1027 struct postsend_info send_info = {}; 1028 u8 data[DR_STE_SIZE]; 1029 int num_of_sends_req; 1030 int ret; 1031 int i; 1032 1033 /* Sending this amount of requests makes sure we will get drain */ 1034 num_of_sends_req = send_ring->signal_th * TH_NUMS_TO_DRAIN / 2; 1035 1036 /* Send fake requests forcing the last to be signaled */ 1037 send_info.write.addr = (uintptr_t)data; 1038 send_info.write.length = DR_STE_SIZE; 1039 send_info.write.lkey = 0; 1040 /* Using the sync_mr in order to write/read */ 1041 send_info.remote_addr = (uintptr_t)send_ring->sync_mr->addr; 1042 send_info.rkey = send_ring->sync_mr->mkey.key; 1043 1044 for (i = 0; i < num_of_sends_req; i++) { 1045 ret = dr_postsend_icm_data(dmn, &send_info); 1046 if (ret) 1047 return ret; 1048 } 1049 1050 spin_lock(&send_ring->lock); 1051 ret = dr_handle_pending_wc(dmn, send_ring); 1052 spin_unlock(&send_ring->lock); 1053 1054 return ret; 1055 } 1056