1 /* 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <net/addrconf.h> 35 #include <linux/etherdevice.h> 36 #include <linux/mlx5/vport.h> 37 38 #include "mlx5_core.h" 39 #include "lib/mlx5.h" 40 #include "fpga/conn.h" 41 42 #define MLX5_FPGA_PKEY 0xFFFF 43 #define MLX5_FPGA_PKEY_INDEX 0 /* RoCE PKEY 0xFFFF is always at index 0 */ 44 #define MLX5_FPGA_RECV_SIZE 2048 45 #define MLX5_FPGA_PORT_NUM 1 46 #define MLX5_FPGA_CQ_BUDGET 64 47 48 static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn, 49 struct mlx5_fpga_dma_buf *buf) 50 { 51 struct device *dma_device; 52 int err = 0; 53 54 if (unlikely(!buf->sg[0].data)) 55 goto out; 56 57 dma_device = mlx5_core_dma_dev(conn->fdev->mdev); 58 buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data, 59 buf->sg[0].size, buf->dma_dir); 60 err = dma_mapping_error(dma_device, buf->sg[0].dma_addr); 61 if (unlikely(err)) { 62 mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err); 63 err = -ENOMEM; 64 goto out; 65 } 66 67 if (!buf->sg[1].data) 68 goto out; 69 70 buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data, 71 buf->sg[1].size, buf->dma_dir); 72 err = dma_mapping_error(dma_device, buf->sg[1].dma_addr); 73 if (unlikely(err)) { 74 mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err); 75 dma_unmap_single(dma_device, buf->sg[0].dma_addr, 76 buf->sg[0].size, buf->dma_dir); 77 err = -ENOMEM; 78 } 79 80 out: 81 return err; 82 } 83 84 static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn, 85 struct mlx5_fpga_dma_buf *buf) 86 { 87 struct device *dma_device; 88 89 dma_device = mlx5_core_dma_dev(conn->fdev->mdev); 90 if (buf->sg[1].data) 91 dma_unmap_single(dma_device, buf->sg[1].dma_addr, 92 buf->sg[1].size, buf->dma_dir); 93 94 if (likely(buf->sg[0].data)) 95 dma_unmap_single(dma_device, buf->sg[0].dma_addr, 96 buf->sg[0].size, buf->dma_dir); 97 } 98 99 static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn, 100 struct mlx5_fpga_dma_buf *buf) 101 { 102 struct mlx5_wqe_data_seg *data; 103 unsigned int ix; 104 int err = 0; 105 106 err = mlx5_fpga_conn_map_buf(conn, buf); 107 if (unlikely(err)) 108 goto out; 109 110 if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) { 111 mlx5_fpga_conn_unmap_buf(conn, buf); 112 return -EBUSY; 113 } 114 115 ix = conn->qp.rq.pc & (conn->qp.rq.size - 1); 116 data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix); 117 data->byte_count = cpu_to_be32(buf->sg[0].size); 118 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); 119 data->addr = cpu_to_be64(buf->sg[0].dma_addr); 120 121 conn->qp.rq.pc++; 122 conn->qp.rq.bufs[ix] = buf; 123 124 /* Make sure that descriptors are written before doorbell record. */ 125 dma_wmb(); 126 *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff); 127 out: 128 return err; 129 } 130 131 static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe) 132 { 133 /* ensure wqe is visible to device before updating doorbell record */ 134 dma_wmb(); 135 *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc); 136 /* Make sure that doorbell record is visible before ringing */ 137 wmb(); 138 mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET); 139 } 140 141 static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn, 142 struct mlx5_fpga_dma_buf *buf) 143 { 144 struct mlx5_wqe_ctrl_seg *ctrl; 145 struct mlx5_wqe_data_seg *data; 146 unsigned int ix, sgi; 147 int size = 1; 148 149 ix = conn->qp.sq.pc & (conn->qp.sq.size - 1); 150 151 ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix); 152 data = (void *)(ctrl + 1); 153 154 for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) { 155 if (!buf->sg[sgi].data) 156 break; 157 data->byte_count = cpu_to_be32(buf->sg[sgi].size); 158 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); 159 data->addr = cpu_to_be64(buf->sg[sgi].dma_addr); 160 data++; 161 size++; 162 } 163 164 ctrl->imm = 0; 165 ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 166 ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) | 167 MLX5_OPCODE_SEND); 168 ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.qpn << 8)); 169 170 conn->qp.sq.pc++; 171 conn->qp.sq.bufs[ix] = buf; 172 mlx5_fpga_conn_notify_hw(conn, ctrl); 173 } 174 175 int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn, 176 struct mlx5_fpga_dma_buf *buf) 177 { 178 unsigned long flags; 179 int err; 180 181 if (!conn->qp.active) 182 return -ENOTCONN; 183 184 buf->dma_dir = DMA_TO_DEVICE; 185 err = mlx5_fpga_conn_map_buf(conn, buf); 186 if (err) 187 return err; 188 189 spin_lock_irqsave(&conn->qp.sq.lock, flags); 190 191 if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) { 192 list_add_tail(&buf->list, &conn->qp.sq.backlog); 193 goto out_unlock; 194 } 195 196 mlx5_fpga_conn_post_send(conn, buf); 197 198 out_unlock: 199 spin_unlock_irqrestore(&conn->qp.sq.lock, flags); 200 return err; 201 } 202 203 static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn) 204 { 205 struct mlx5_fpga_dma_buf *buf; 206 int err; 207 208 buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0); 209 if (!buf) 210 return -ENOMEM; 211 212 buf->sg[0].data = (void *)(buf + 1); 213 buf->sg[0].size = MLX5_FPGA_RECV_SIZE; 214 buf->dma_dir = DMA_FROM_DEVICE; 215 216 err = mlx5_fpga_conn_post_recv(conn, buf); 217 if (err) 218 kfree(buf); 219 220 return err; 221 } 222 223 static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, 224 struct mlx5_core_mkey *mkey) 225 { 226 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 227 void *mkc; 228 u32 *in; 229 int err; 230 231 in = kvzalloc(inlen, GFP_KERNEL); 232 if (!in) 233 return -ENOMEM; 234 235 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 236 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 237 MLX5_SET(mkc, mkc, lw, 1); 238 MLX5_SET(mkc, mkc, lr, 1); 239 240 MLX5_SET(mkc, mkc, pd, pdn); 241 MLX5_SET(mkc, mkc, length64, 1); 242 MLX5_SET(mkc, mkc, qpn, 0xffffff); 243 244 err = mlx5_core_create_mkey(mdev, mkey, in, inlen); 245 246 kvfree(in); 247 return err; 248 } 249 250 static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn, 251 struct mlx5_cqe64 *cqe, u8 status) 252 { 253 struct mlx5_fpga_dma_buf *buf; 254 int ix, err; 255 256 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1); 257 buf = conn->qp.rq.bufs[ix]; 258 conn->qp.rq.bufs[ix] = NULL; 259 conn->qp.rq.cc++; 260 261 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR))) 262 mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n", 263 buf, conn->fpga_qpn, status); 264 else 265 mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n", 266 buf, conn->fpga_qpn, status); 267 268 mlx5_fpga_conn_unmap_buf(conn, buf); 269 270 if (unlikely(status || !conn->qp.active)) { 271 conn->qp.active = false; 272 kfree(buf); 273 return; 274 } 275 276 buf->sg[0].size = be32_to_cpu(cqe->byte_cnt); 277 mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n", 278 buf->sg[0].size); 279 conn->recv_cb(conn->cb_arg, buf); 280 281 buf->sg[0].size = MLX5_FPGA_RECV_SIZE; 282 err = mlx5_fpga_conn_post_recv(conn, buf); 283 if (unlikely(err)) { 284 mlx5_fpga_warn(conn->fdev, 285 "Failed to re-post recv buf: %d\n", err); 286 kfree(buf); 287 } 288 } 289 290 static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn, 291 struct mlx5_cqe64 *cqe, u8 status) 292 { 293 struct mlx5_fpga_dma_buf *buf, *nextbuf; 294 unsigned long flags; 295 int ix; 296 297 spin_lock_irqsave(&conn->qp.sq.lock, flags); 298 299 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1); 300 buf = conn->qp.sq.bufs[ix]; 301 conn->qp.sq.bufs[ix] = NULL; 302 conn->qp.sq.cc++; 303 304 /* Handle backlog still under the spinlock to ensure message post order */ 305 if (unlikely(!list_empty(&conn->qp.sq.backlog))) { 306 if (likely(conn->qp.active)) { 307 nextbuf = list_first_entry(&conn->qp.sq.backlog, 308 struct mlx5_fpga_dma_buf, list); 309 list_del(&nextbuf->list); 310 mlx5_fpga_conn_post_send(conn, nextbuf); 311 } 312 } 313 314 spin_unlock_irqrestore(&conn->qp.sq.lock, flags); 315 316 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR))) 317 mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n", 318 buf, conn->fpga_qpn, status); 319 else 320 mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n", 321 buf, conn->fpga_qpn, status); 322 323 mlx5_fpga_conn_unmap_buf(conn, buf); 324 325 if (likely(buf->complete)) 326 buf->complete(conn, conn->fdev, buf, status); 327 328 if (unlikely(status)) 329 conn->qp.active = false; 330 } 331 332 static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn, 333 struct mlx5_cqe64 *cqe) 334 { 335 u8 opcode, status = 0; 336 337 opcode = get_cqe_opcode(cqe); 338 339 switch (opcode) { 340 case MLX5_CQE_REQ_ERR: 341 status = ((struct mlx5_err_cqe *)cqe)->syndrome; 342 fallthrough; 343 case MLX5_CQE_REQ: 344 mlx5_fpga_conn_sq_cqe(conn, cqe, status); 345 break; 346 347 case MLX5_CQE_RESP_ERR: 348 status = ((struct mlx5_err_cqe *)cqe)->syndrome; 349 fallthrough; 350 case MLX5_CQE_RESP_SEND: 351 mlx5_fpga_conn_rq_cqe(conn, cqe, status); 352 break; 353 default: 354 mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n", 355 opcode); 356 } 357 } 358 359 static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn) 360 { 361 mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT, 362 conn->fdev->conn_res.uar->map, conn->cq.wq.cc); 363 } 364 365 static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn, 366 unsigned int budget) 367 { 368 struct mlx5_cqe64 *cqe; 369 370 while (budget) { 371 cqe = mlx5_cqwq_get_cqe(&conn->cq.wq); 372 if (!cqe) 373 break; 374 375 budget--; 376 mlx5_cqwq_pop(&conn->cq.wq); 377 mlx5_fpga_conn_handle_cqe(conn, cqe); 378 mlx5_cqwq_update_db_record(&conn->cq.wq); 379 } 380 if (!budget) { 381 tasklet_schedule(&conn->cq.tasklet); 382 return; 383 } 384 385 mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc); 386 /* ensure cq space is freed before enabling more cqes */ 387 wmb(); 388 mlx5_fpga_conn_arm_cq(conn); 389 } 390 391 static void mlx5_fpga_conn_cq_tasklet(struct tasklet_struct *t) 392 { 393 struct mlx5_fpga_conn *conn = from_tasklet(conn, t, cq.tasklet); 394 395 if (unlikely(!conn->qp.active)) 396 return; 397 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET); 398 } 399 400 static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq, 401 struct mlx5_eqe *eqe) 402 { 403 struct mlx5_fpga_conn *conn; 404 405 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq); 406 if (unlikely(!conn->qp.active)) 407 return; 408 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET); 409 } 410 411 static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size) 412 { 413 struct mlx5_fpga_device *fdev = conn->fdev; 414 struct mlx5_core_dev *mdev = fdev->mdev; 415 u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0}; 416 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 417 struct mlx5_wq_param wqp; 418 struct mlx5_cqe64 *cqe; 419 int inlen, err, eqn; 420 unsigned int irqn; 421 void *cqc, *in; 422 __be64 *pas; 423 u32 i; 424 425 cq_size = roundup_pow_of_two(cq_size); 426 MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size)); 427 428 wqp.buf_numa_node = mdev->priv.numa_node; 429 wqp.db_numa_node = mdev->priv.numa_node; 430 431 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq, 432 &conn->cq.wq_ctrl); 433 if (err) 434 return err; 435 436 for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) { 437 cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i); 438 cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK; 439 } 440 441 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 442 sizeof(u64) * conn->cq.wq_ctrl.buf.npages; 443 in = kvzalloc(inlen, GFP_KERNEL); 444 if (!in) { 445 err = -ENOMEM; 446 goto err_cqwq; 447 } 448 449 err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn); 450 if (err) { 451 kvfree(in); 452 goto err_cqwq; 453 } 454 455 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 456 MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size)); 457 MLX5_SET(cqc, cqc, c_eqn, eqn); 458 MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index); 459 MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.buf.page_shift - 460 MLX5_ADAPTER_PAGE_SHIFT); 461 MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma); 462 463 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 464 mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.buf, pas); 465 466 err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen, out, sizeof(out)); 467 kvfree(in); 468 469 if (err) 470 goto err_cqwq; 471 472 conn->cq.mcq.cqe_sz = 64; 473 conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db; 474 conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1; 475 *conn->cq.mcq.set_ci_db = 0; 476 *conn->cq.mcq.arm_db = 0; 477 conn->cq.mcq.vector = 0; 478 conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete; 479 conn->cq.mcq.irqn = irqn; 480 conn->cq.mcq.uar = fdev->conn_res.uar; 481 tasklet_setup(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet); 482 483 mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn); 484 485 goto out; 486 487 err_cqwq: 488 mlx5_wq_destroy(&conn->cq.wq_ctrl); 489 out: 490 return err; 491 } 492 493 static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn) 494 { 495 tasklet_disable(&conn->cq.tasklet); 496 tasklet_kill(&conn->cq.tasklet); 497 mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq); 498 mlx5_wq_destroy(&conn->cq.wq_ctrl); 499 } 500 501 static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc) 502 { 503 struct mlx5_fpga_device *fdev = conn->fdev; 504 struct mlx5_core_dev *mdev = fdev->mdev; 505 struct mlx5_wq_param wqp; 506 507 wqp.buf_numa_node = mdev->priv.numa_node; 508 wqp.db_numa_node = mdev->priv.numa_node; 509 510 return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq, 511 &conn->qp.wq_ctrl); 512 } 513 514 static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn, 515 unsigned int tx_size, unsigned int rx_size) 516 { 517 struct mlx5_fpga_device *fdev = conn->fdev; 518 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 519 struct mlx5_core_dev *mdev = fdev->mdev; 520 u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {}; 521 void *in = NULL, *qpc; 522 int err, inlen; 523 524 conn->qp.rq.pc = 0; 525 conn->qp.rq.cc = 0; 526 conn->qp.rq.size = roundup_pow_of_two(rx_size); 527 conn->qp.sq.pc = 0; 528 conn->qp.sq.cc = 0; 529 conn->qp.sq.size = roundup_pow_of_two(tx_size); 530 531 MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 532 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size)); 533 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size)); 534 err = mlx5_fpga_conn_create_wq(conn, temp_qpc); 535 if (err) 536 goto out; 537 538 conn->qp.rq.bufs = kvcalloc(conn->qp.rq.size, 539 sizeof(conn->qp.rq.bufs[0]), 540 GFP_KERNEL); 541 if (!conn->qp.rq.bufs) { 542 err = -ENOMEM; 543 goto err_wq; 544 } 545 546 conn->qp.sq.bufs = kvcalloc(conn->qp.sq.size, 547 sizeof(conn->qp.sq.bufs[0]), 548 GFP_KERNEL); 549 if (!conn->qp.sq.bufs) { 550 err = -ENOMEM; 551 goto err_rq_bufs; 552 } 553 554 inlen = MLX5_ST_SZ_BYTES(create_qp_in) + 555 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * 556 conn->qp.wq_ctrl.buf.npages; 557 in = kvzalloc(inlen, GFP_KERNEL); 558 if (!in) { 559 err = -ENOMEM; 560 goto err_sq_bufs; 561 } 562 563 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 564 MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index); 565 MLX5_SET(qpc, qpc, log_page_size, 566 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 567 MLX5_SET(qpc, qpc, fre, 1); 568 MLX5_SET(qpc, qpc, rlky, 1); 569 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 570 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 571 MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn); 572 MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 573 MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size)); 574 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 575 MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size)); 576 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn); 577 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn); 578 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); 579 if (MLX5_CAP_GEN(mdev, cqe_version) == 1) 580 MLX5_SET(qpc, qpc, user_index, 0xFFFFFF); 581 582 mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf, 583 (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas)); 584 585 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 586 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 587 if (err) 588 goto err_sq_bufs; 589 590 conn->qp.qpn = MLX5_GET(create_qp_out, out, qpn); 591 mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.qpn); 592 593 goto out; 594 595 err_sq_bufs: 596 kvfree(conn->qp.sq.bufs); 597 err_rq_bufs: 598 kvfree(conn->qp.rq.bufs); 599 err_wq: 600 mlx5_wq_destroy(&conn->qp.wq_ctrl); 601 out: 602 kvfree(in); 603 return err; 604 } 605 606 static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn) 607 { 608 int ix; 609 610 for (ix = 0; ix < conn->qp.rq.size; ix++) { 611 if (!conn->qp.rq.bufs[ix]) 612 continue; 613 mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]); 614 kfree(conn->qp.rq.bufs[ix]); 615 conn->qp.rq.bufs[ix] = NULL; 616 } 617 } 618 619 static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn) 620 { 621 struct mlx5_fpga_dma_buf *buf, *temp; 622 int ix; 623 624 for (ix = 0; ix < conn->qp.sq.size; ix++) { 625 buf = conn->qp.sq.bufs[ix]; 626 if (!buf) 627 continue; 628 conn->qp.sq.bufs[ix] = NULL; 629 mlx5_fpga_conn_unmap_buf(conn, buf); 630 if (!buf->complete) 631 continue; 632 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR); 633 } 634 list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) { 635 mlx5_fpga_conn_unmap_buf(conn, buf); 636 if (!buf->complete) 637 continue; 638 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR); 639 } 640 } 641 642 static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn) 643 { 644 struct mlx5_core_dev *dev = conn->fdev->mdev; 645 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 646 647 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 648 MLX5_SET(destroy_qp_in, in, qpn, conn->qp.qpn); 649 mlx5_cmd_exec_in(dev, destroy_qp, in); 650 651 mlx5_fpga_conn_free_recv_bufs(conn); 652 mlx5_fpga_conn_flush_send_bufs(conn); 653 kvfree(conn->qp.sq.bufs); 654 kvfree(conn->qp.rq.bufs); 655 mlx5_wq_destroy(&conn->qp.wq_ctrl); 656 } 657 658 static int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn) 659 { 660 struct mlx5_core_dev *mdev = conn->fdev->mdev; 661 u32 in[MLX5_ST_SZ_DW(qp_2rst_in)] = {}; 662 663 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.qpn); 664 665 MLX5_SET(qp_2rst_in, in, opcode, MLX5_CMD_OP_2RST_QP); 666 MLX5_SET(qp_2rst_in, in, qpn, conn->qp.qpn); 667 668 return mlx5_cmd_exec_in(mdev, qp_2rst, in); 669 } 670 671 static int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn) 672 { 673 u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; 674 struct mlx5_fpga_device *fdev = conn->fdev; 675 struct mlx5_core_dev *mdev = fdev->mdev; 676 u32 *qpc; 677 678 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.qpn); 679 680 qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc); 681 682 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 683 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 684 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX); 685 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM); 686 MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn); 687 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn); 688 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn); 689 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); 690 691 MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP); 692 MLX5_SET(rst2init_qp_in, in, qpn, conn->qp.qpn); 693 694 return mlx5_cmd_exec_in(mdev, rst2init_qp, in); 695 } 696 697 static int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn) 698 { 699 u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; 700 struct mlx5_fpga_device *fdev = conn->fdev; 701 struct mlx5_core_dev *mdev = fdev->mdev; 702 u32 *qpc; 703 704 mlx5_fpga_dbg(conn->fdev, "QP RTR\n"); 705 706 qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc); 707 708 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES); 709 MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg)); 710 MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn); 711 MLX5_SET(qpc, qpc, next_rcv_psn, 712 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn)); 713 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX); 714 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM); 715 ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32), 716 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32)); 717 MLX5_SET(qpc, qpc, primary_address_path.udp_sport, 718 MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port)); 719 MLX5_SET(qpc, qpc, primary_address_path.src_addr_index, 720 conn->qp.sgid_index); 721 MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0); 722 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), 723 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip), 724 MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip)); 725 726 MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP); 727 MLX5_SET(init2rtr_qp_in, in, qpn, conn->qp.qpn); 728 729 return mlx5_cmd_exec_in(mdev, init2rtr_qp, in); 730 } 731 732 static int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn) 733 { 734 struct mlx5_fpga_device *fdev = conn->fdev; 735 u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; 736 struct mlx5_core_dev *mdev = fdev->mdev; 737 u32 *qpc; 738 739 mlx5_fpga_dbg(conn->fdev, "QP RTS\n"); 740 741 qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc); 742 743 MLX5_SET(qpc, qpc, log_ack_req_freq, 8); 744 MLX5_SET(qpc, qpc, min_rnr_nak, 0x12); 745 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12); /* ~1.07s */ 746 MLX5_SET(qpc, qpc, next_send_psn, 747 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn)); 748 MLX5_SET(qpc, qpc, retry_count, 7); 749 MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */ 750 751 MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP); 752 MLX5_SET(rtr2rts_qp_in, in, qpn, conn->qp.qpn); 753 MLX5_SET(rtr2rts_qp_in, in, opt_param_mask, MLX5_QP_OPTPAR_RNR_TIMEOUT); 754 755 return mlx5_cmd_exec_in(mdev, rtr2rts_qp, in); 756 } 757 758 static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn) 759 { 760 struct mlx5_fpga_device *fdev = conn->fdev; 761 int err; 762 763 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE); 764 err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn, 765 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc); 766 if (err) { 767 mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err); 768 goto out; 769 } 770 771 err = mlx5_fpga_conn_reset_qp(conn); 772 if (err) { 773 mlx5_fpga_err(fdev, "Failed to change QP state to reset\n"); 774 goto err_fpga_qp; 775 } 776 777 err = mlx5_fpga_conn_init_qp(conn); 778 if (err) { 779 mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n"); 780 goto err_fpga_qp; 781 } 782 conn->qp.active = true; 783 784 while (!mlx5_fpga_conn_post_recv_buf(conn)) 785 ; 786 787 err = mlx5_fpga_conn_rtr_qp(conn); 788 if (err) { 789 mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n"); 790 goto err_recv_bufs; 791 } 792 793 err = mlx5_fpga_conn_rts_qp(conn); 794 if (err) { 795 mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n"); 796 goto err_recv_bufs; 797 } 798 goto out; 799 800 err_recv_bufs: 801 mlx5_fpga_conn_free_recv_bufs(conn); 802 err_fpga_qp: 803 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT); 804 if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn, 805 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc)) 806 mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n"); 807 out: 808 return err; 809 } 810 811 struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev, 812 struct mlx5_fpga_conn_attr *attr, 813 enum mlx5_ifc_fpga_qp_type qp_type) 814 { 815 struct mlx5_fpga_conn *ret, *conn; 816 u8 *remote_mac, *remote_ip; 817 int err; 818 819 if (!attr->recv_cb) 820 return ERR_PTR(-EINVAL); 821 822 conn = kzalloc(sizeof(*conn), GFP_KERNEL); 823 if (!conn) 824 return ERR_PTR(-ENOMEM); 825 826 conn->fdev = fdev; 827 INIT_LIST_HEAD(&conn->qp.sq.backlog); 828 829 spin_lock_init(&conn->qp.sq.lock); 830 831 conn->recv_cb = attr->recv_cb; 832 conn->cb_arg = attr->cb_arg; 833 834 remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32); 835 err = mlx5_query_mac_address(fdev->mdev, remote_mac); 836 if (err) { 837 mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err); 838 ret = ERR_PTR(err); 839 goto err; 840 } 841 842 /* Build Modified EUI-64 IPv6 address from the MAC address */ 843 remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip); 844 remote_ip[0] = 0xfe; 845 remote_ip[1] = 0x80; 846 addrconf_addr_eui48(&remote_ip[8], remote_mac); 847 848 err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index); 849 if (err) { 850 mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err); 851 ret = ERR_PTR(err); 852 goto err; 853 } 854 855 err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 856 MLX5_ROCE_VERSION_2, 857 MLX5_ROCE_L3_TYPE_IPV6, 858 remote_ip, remote_mac, true, 0, 859 MLX5_FPGA_PORT_NUM); 860 if (err) { 861 mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err); 862 ret = ERR_PTR(err); 863 goto err_rsvd_gid; 864 } 865 mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index); 866 867 /* Allow for one cqe per rx/tx wqe, plus one cqe for the next wqe, 868 * created during processing of the cqe 869 */ 870 err = mlx5_fpga_conn_create_cq(conn, 871 (attr->tx_size + attr->rx_size) * 2); 872 if (err) { 873 mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err); 874 ret = ERR_PTR(err); 875 goto err_gid; 876 } 877 878 mlx5_fpga_conn_arm_cq(conn); 879 880 err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size); 881 if (err) { 882 mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err); 883 ret = ERR_PTR(err); 884 goto err_cq; 885 } 886 887 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT); 888 MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type); 889 MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC); 890 MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q); 891 MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0); 892 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1); 893 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0); 894 MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY); 895 MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.qpn); 896 MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7); 897 MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7); 898 899 err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc, 900 &conn->fpga_qpn); 901 if (err) { 902 mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err); 903 ret = ERR_PTR(err); 904 goto err_qp; 905 } 906 907 err = mlx5_fpga_conn_connect(conn); 908 if (err) { 909 ret = ERR_PTR(err); 910 goto err_conn; 911 } 912 913 mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn); 914 ret = conn; 915 goto out; 916 917 err_conn: 918 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn); 919 err_qp: 920 mlx5_fpga_conn_destroy_qp(conn); 921 err_cq: 922 mlx5_fpga_conn_destroy_cq(conn); 923 err_gid: 924 mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL, 925 NULL, false, 0, MLX5_FPGA_PORT_NUM); 926 err_rsvd_gid: 927 mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index); 928 err: 929 kfree(conn); 930 out: 931 return ret; 932 } 933 934 void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn) 935 { 936 conn->qp.active = false; 937 tasklet_disable(&conn->cq.tasklet); 938 synchronize_irq(conn->cq.mcq.irqn); 939 940 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn); 941 mlx5_fpga_conn_destroy_qp(conn); 942 mlx5_fpga_conn_destroy_cq(conn); 943 944 mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0, 945 NULL, NULL, false, 0, MLX5_FPGA_PORT_NUM); 946 mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index); 947 kfree(conn); 948 } 949 950 int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev) 951 { 952 int err; 953 954 err = mlx5_nic_vport_enable_roce(fdev->mdev); 955 if (err) { 956 mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err); 957 goto out; 958 } 959 960 fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev); 961 if (IS_ERR(fdev->conn_res.uar)) { 962 err = PTR_ERR(fdev->conn_res.uar); 963 mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err); 964 goto err_roce; 965 } 966 mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n", 967 fdev->conn_res.uar->index); 968 969 err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn); 970 if (err) { 971 mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err); 972 goto err_uar; 973 } 974 mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn); 975 976 err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn, 977 &fdev->conn_res.mkey); 978 if (err) { 979 mlx5_fpga_err(fdev, "create mkey failed, %d\n", err); 980 goto err_dealloc_pd; 981 } 982 mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key); 983 984 return 0; 985 986 err_dealloc_pd: 987 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); 988 err_uar: 989 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); 990 err_roce: 991 mlx5_nic_vport_disable_roce(fdev->mdev); 992 out: 993 return err; 994 } 995 996 void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev) 997 { 998 mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey); 999 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); 1000 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); 1001 mlx5_nic_vport_disable_roce(fdev->mdev); 1002 } 1003