1 /* 2 * QEMU paravirtual RDMA - Command channel 3 * 4 * Copyright (C) 2018 Oracle 5 * Copyright (C) 2018 Red Hat Inc 6 * 7 * Authors: 8 * Yuval Shaia <yuval.shaia@oracle.com> 9 * Marcel Apfelbaum <marcel@redhat.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2 or later. 12 * See the COPYING file in the top-level directory. 13 * 14 */ 15 16 #include "qemu/osdep.h" 17 #include "cpu.h" 18 #include "hw/hw.h" 19 #include "hw/pci/pci.h" 20 #include "hw/pci/pci_ids.h" 21 22 #include "../rdma_backend.h" 23 #include "../rdma_rm.h" 24 #include "../rdma_utils.h" 25 26 #include "trace.h" 27 #include "pvrdma.h" 28 #include "standard-headers/rdma/vmw_pvrdma-abi.h" 29 30 static void *pvrdma_map_to_pdir(PCIDevice *pdev, uint64_t pdir_dma, 31 uint32_t nchunks, size_t length) 32 { 33 uint64_t *dir, *tbl; 34 int tbl_idx, dir_idx, addr_idx; 35 void *host_virt = NULL, *curr_page; 36 37 if (!nchunks) { 38 rdma_error_report("Got nchunks=0"); 39 return NULL; 40 } 41 42 dir = rdma_pci_dma_map(pdev, pdir_dma, TARGET_PAGE_SIZE); 43 if (!dir) { 44 rdma_error_report("Failed to map to page directory"); 45 return NULL; 46 } 47 48 tbl = rdma_pci_dma_map(pdev, dir[0], TARGET_PAGE_SIZE); 49 if (!tbl) { 50 rdma_error_report("Failed to map to page table 0"); 51 goto out_unmap_dir; 52 } 53 54 curr_page = rdma_pci_dma_map(pdev, (dma_addr_t)tbl[0], TARGET_PAGE_SIZE); 55 if (!curr_page) { 56 rdma_error_report("Failed to map the page 0"); 57 goto out_unmap_tbl; 58 } 59 60 host_virt = mremap(curr_page, 0, length, MREMAP_MAYMOVE); 61 if (host_virt == MAP_FAILED) { 62 host_virt = NULL; 63 rdma_error_report("Failed to remap memory for host_virt"); 64 goto out_unmap_tbl; 65 } 66 trace_pvrdma_map_to_pdir_host_virt(curr_page, host_virt); 67 68 rdma_pci_dma_unmap(pdev, curr_page, TARGET_PAGE_SIZE); 69 70 dir_idx = 0; 71 tbl_idx = 1; 72 addr_idx = 1; 73 while (addr_idx < nchunks) { 74 if (tbl_idx == TARGET_PAGE_SIZE / sizeof(uint64_t)) { 75 tbl_idx = 0; 76 dir_idx++; 77 rdma_pci_dma_unmap(pdev, tbl, TARGET_PAGE_SIZE); 78 tbl = rdma_pci_dma_map(pdev, dir[dir_idx], TARGET_PAGE_SIZE); 79 if (!tbl) { 80 rdma_error_report("Failed to map to page table %d", dir_idx); 81 goto out_unmap_host_virt; 82 } 83 } 84 85 curr_page = rdma_pci_dma_map(pdev, (dma_addr_t)tbl[tbl_idx], 86 TARGET_PAGE_SIZE); 87 if (!curr_page) { 88 rdma_error_report("Failed to map to page %d, dir %d", tbl_idx, 89 dir_idx); 90 goto out_unmap_host_virt; 91 } 92 93 mremap(curr_page, 0, TARGET_PAGE_SIZE, MREMAP_MAYMOVE | MREMAP_FIXED, 94 host_virt + TARGET_PAGE_SIZE * addr_idx); 95 96 trace_pvrdma_map_to_pdir_next_page(addr_idx, curr_page, host_virt + 97 TARGET_PAGE_SIZE * addr_idx); 98 99 rdma_pci_dma_unmap(pdev, curr_page, TARGET_PAGE_SIZE); 100 101 addr_idx++; 102 103 tbl_idx++; 104 } 105 106 goto out_unmap_tbl; 107 108 out_unmap_host_virt: 109 munmap(host_virt, length); 110 host_virt = NULL; 111 112 out_unmap_tbl: 113 rdma_pci_dma_unmap(pdev, tbl, TARGET_PAGE_SIZE); 114 115 out_unmap_dir: 116 rdma_pci_dma_unmap(pdev, dir, TARGET_PAGE_SIZE); 117 118 return host_virt; 119 } 120 121 static int query_port(PVRDMADev *dev, union pvrdma_cmd_req *req, 122 union pvrdma_cmd_resp *rsp) 123 { 124 struct pvrdma_cmd_query_port *cmd = &req->query_port; 125 struct pvrdma_cmd_query_port_resp *resp = &rsp->query_port_resp; 126 struct pvrdma_port_attr attrs = {}; 127 128 if (cmd->port_num > MAX_PORTS) { 129 return -EINVAL; 130 } 131 132 if (rdma_backend_query_port(&dev->backend_dev, 133 (struct ibv_port_attr *)&attrs)) { 134 return -ENOMEM; 135 } 136 137 memset(resp, 0, sizeof(*resp)); 138 139 resp->attrs.state = dev->func0->device_active ? attrs.state : 140 PVRDMA_PORT_DOWN; 141 resp->attrs.max_mtu = attrs.max_mtu; 142 resp->attrs.active_mtu = attrs.active_mtu; 143 resp->attrs.phys_state = attrs.phys_state; 144 resp->attrs.gid_tbl_len = MIN(MAX_PORT_GIDS, attrs.gid_tbl_len); 145 resp->attrs.max_msg_sz = 1024; 146 resp->attrs.pkey_tbl_len = MIN(MAX_PORT_PKEYS, attrs.pkey_tbl_len); 147 resp->attrs.active_width = 1; 148 resp->attrs.active_speed = 1; 149 150 return 0; 151 } 152 153 static int query_pkey(PVRDMADev *dev, union pvrdma_cmd_req *req, 154 union pvrdma_cmd_resp *rsp) 155 { 156 struct pvrdma_cmd_query_pkey *cmd = &req->query_pkey; 157 struct pvrdma_cmd_query_pkey_resp *resp = &rsp->query_pkey_resp; 158 159 if (cmd->port_num > MAX_PORTS) { 160 return -EINVAL; 161 } 162 163 if (cmd->index > MAX_PKEYS) { 164 return -EINVAL; 165 } 166 167 memset(resp, 0, sizeof(*resp)); 168 169 resp->pkey = PVRDMA_PKEY; 170 171 return 0; 172 } 173 174 static int create_pd(PVRDMADev *dev, union pvrdma_cmd_req *req, 175 union pvrdma_cmd_resp *rsp) 176 { 177 struct pvrdma_cmd_create_pd *cmd = &req->create_pd; 178 struct pvrdma_cmd_create_pd_resp *resp = &rsp->create_pd_resp; 179 int rc; 180 181 memset(resp, 0, sizeof(*resp)); 182 rc = rdma_rm_alloc_pd(&dev->rdma_dev_res, &dev->backend_dev, 183 &resp->pd_handle, cmd->ctx_handle); 184 185 return rc; 186 } 187 188 static int destroy_pd(PVRDMADev *dev, union pvrdma_cmd_req *req, 189 union pvrdma_cmd_resp *rsp) 190 { 191 struct pvrdma_cmd_destroy_pd *cmd = &req->destroy_pd; 192 193 rdma_rm_dealloc_pd(&dev->rdma_dev_res, cmd->pd_handle); 194 195 return 0; 196 } 197 198 static int create_mr(PVRDMADev *dev, union pvrdma_cmd_req *req, 199 union pvrdma_cmd_resp *rsp) 200 { 201 struct pvrdma_cmd_create_mr *cmd = &req->create_mr; 202 struct pvrdma_cmd_create_mr_resp *resp = &rsp->create_mr_resp; 203 PCIDevice *pci_dev = PCI_DEVICE(dev); 204 void *host_virt = NULL; 205 int rc = 0; 206 207 memset(resp, 0, sizeof(*resp)); 208 209 if (!(cmd->flags & PVRDMA_MR_FLAG_DMA)) { 210 host_virt = pvrdma_map_to_pdir(pci_dev, cmd->pdir_dma, cmd->nchunks, 211 cmd->length); 212 if (!host_virt) { 213 rdma_error_report("Failed to map to pdir"); 214 return -EINVAL; 215 } 216 } 217 218 rc = rdma_rm_alloc_mr(&dev->rdma_dev_res, cmd->pd_handle, cmd->start, 219 cmd->length, host_virt, cmd->access_flags, 220 &resp->mr_handle, &resp->lkey, &resp->rkey); 221 if (rc && host_virt) { 222 munmap(host_virt, cmd->length); 223 } 224 225 return rc; 226 } 227 228 static int destroy_mr(PVRDMADev *dev, union pvrdma_cmd_req *req, 229 union pvrdma_cmd_resp *rsp) 230 { 231 struct pvrdma_cmd_destroy_mr *cmd = &req->destroy_mr; 232 233 rdma_rm_dealloc_mr(&dev->rdma_dev_res, cmd->mr_handle); 234 235 return 0; 236 } 237 238 static int create_cq_ring(PCIDevice *pci_dev , PvrdmaRing **ring, 239 uint64_t pdir_dma, uint32_t nchunks, uint32_t cqe) 240 { 241 uint64_t *dir = NULL, *tbl = NULL; 242 PvrdmaRing *r; 243 int rc = -EINVAL; 244 char ring_name[MAX_RING_NAME_SZ]; 245 246 if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) { 247 rdma_error_report("Got invalid nchunks: %d", nchunks); 248 return rc; 249 } 250 251 dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE); 252 if (!dir) { 253 rdma_error_report("Failed to map to CQ page directory"); 254 goto out; 255 } 256 257 tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE); 258 if (!tbl) { 259 rdma_error_report("Failed to map to CQ page table"); 260 goto out; 261 } 262 263 r = g_malloc(sizeof(*r)); 264 *ring = r; 265 266 r->ring_state = (struct pvrdma_ring *) 267 rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE); 268 269 if (!r->ring_state) { 270 rdma_error_report("Failed to map to CQ ring state"); 271 goto out_free_ring; 272 } 273 274 sprintf(ring_name, "cq_ring_%" PRIx64, pdir_dma); 275 rc = pvrdma_ring_init(r, ring_name, pci_dev, &r->ring_state[1], 276 cqe, sizeof(struct pvrdma_cqe), 277 /* first page is ring state */ 278 (dma_addr_t *)&tbl[1], nchunks - 1); 279 if (rc) { 280 goto out_unmap_ring_state; 281 } 282 283 goto out; 284 285 out_unmap_ring_state: 286 /* ring_state was in slot 1, not 0 so need to jump back */ 287 rdma_pci_dma_unmap(pci_dev, --r->ring_state, TARGET_PAGE_SIZE); 288 289 out_free_ring: 290 g_free(r); 291 292 out: 293 rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE); 294 rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE); 295 296 return rc; 297 } 298 299 static void destroy_cq_ring(PvrdmaRing *ring) 300 { 301 pvrdma_ring_free(ring); 302 /* ring_state was in slot 1, not 0 so need to jump back */ 303 rdma_pci_dma_unmap(ring->dev, --ring->ring_state, TARGET_PAGE_SIZE); 304 g_free(ring); 305 } 306 307 static int create_cq(PVRDMADev *dev, union pvrdma_cmd_req *req, 308 union pvrdma_cmd_resp *rsp) 309 { 310 struct pvrdma_cmd_create_cq *cmd = &req->create_cq; 311 struct pvrdma_cmd_create_cq_resp *resp = &rsp->create_cq_resp; 312 PvrdmaRing *ring = NULL; 313 int rc; 314 315 memset(resp, 0, sizeof(*resp)); 316 317 resp->cqe = cmd->cqe; 318 319 rc = create_cq_ring(PCI_DEVICE(dev), &ring, cmd->pdir_dma, cmd->nchunks, 320 cmd->cqe); 321 if (rc) { 322 return rc; 323 } 324 325 rc = rdma_rm_alloc_cq(&dev->rdma_dev_res, &dev->backend_dev, cmd->cqe, 326 &resp->cq_handle, ring); 327 if (rc) { 328 destroy_cq_ring(ring); 329 } 330 331 resp->cqe = cmd->cqe; 332 333 return rc; 334 } 335 336 static int destroy_cq(PVRDMADev *dev, union pvrdma_cmd_req *req, 337 union pvrdma_cmd_resp *rsp) 338 { 339 struct pvrdma_cmd_destroy_cq *cmd = &req->destroy_cq; 340 RdmaRmCQ *cq; 341 PvrdmaRing *ring; 342 343 cq = rdma_rm_get_cq(&dev->rdma_dev_res, cmd->cq_handle); 344 if (!cq) { 345 rdma_error_report("Got invalid CQ handle"); 346 return -EINVAL; 347 } 348 349 ring = (PvrdmaRing *)cq->opaque; 350 destroy_cq_ring(ring); 351 352 rdma_rm_dealloc_cq(&dev->rdma_dev_res, cmd->cq_handle); 353 354 return 0; 355 } 356 357 static int create_qp_rings(PCIDevice *pci_dev, uint64_t pdir_dma, 358 PvrdmaRing **rings, uint32_t scqe, uint32_t smax_sge, 359 uint32_t spages, uint32_t rcqe, uint32_t rmax_sge, 360 uint32_t rpages, uint8_t is_srq) 361 { 362 uint64_t *dir = NULL, *tbl = NULL; 363 PvrdmaRing *sr, *rr; 364 int rc = -EINVAL; 365 char ring_name[MAX_RING_NAME_SZ]; 366 uint32_t wqe_sz; 367 368 if (!spages || spages > PVRDMA_MAX_FAST_REG_PAGES) { 369 rdma_error_report("Got invalid send page count for QP ring: %d", 370 spages); 371 return rc; 372 } 373 374 if (!is_srq && (!rpages || rpages > PVRDMA_MAX_FAST_REG_PAGES)) { 375 rdma_error_report("Got invalid recv page count for QP ring: %d", 376 rpages); 377 return rc; 378 } 379 380 dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE); 381 if (!dir) { 382 rdma_error_report("Failed to map to QP page directory"); 383 goto out; 384 } 385 386 tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE); 387 if (!tbl) { 388 rdma_error_report("Failed to map to QP page table"); 389 goto out; 390 } 391 392 if (!is_srq) { 393 sr = g_malloc(2 * sizeof(*rr)); 394 rr = &sr[1]; 395 } else { 396 sr = g_malloc(sizeof(*sr)); 397 } 398 399 *rings = sr; 400 401 /* Create send ring */ 402 sr->ring_state = (struct pvrdma_ring *) 403 rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE); 404 if (!sr->ring_state) { 405 rdma_error_report("Failed to map to QP ring state"); 406 goto out_free_sr_mem; 407 } 408 409 wqe_sz = pow2ceil(sizeof(struct pvrdma_sq_wqe_hdr) + 410 sizeof(struct pvrdma_sge) * smax_sge - 1); 411 412 sprintf(ring_name, "qp_sring_%" PRIx64, pdir_dma); 413 rc = pvrdma_ring_init(sr, ring_name, pci_dev, sr->ring_state, 414 scqe, wqe_sz, (dma_addr_t *)&tbl[1], spages); 415 if (rc) { 416 goto out_unmap_ring_state; 417 } 418 419 if (!is_srq) { 420 /* Create recv ring */ 421 rr->ring_state = &sr->ring_state[1]; 422 wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) + 423 sizeof(struct pvrdma_sge) * rmax_sge - 1); 424 sprintf(ring_name, "qp_rring_%" PRIx64, pdir_dma); 425 rc = pvrdma_ring_init(rr, ring_name, pci_dev, rr->ring_state, 426 rcqe, wqe_sz, (dma_addr_t *)&tbl[1 + spages], 427 rpages); 428 if (rc) { 429 goto out_free_sr; 430 } 431 } 432 433 goto out; 434 435 out_free_sr: 436 pvrdma_ring_free(sr); 437 438 out_unmap_ring_state: 439 rdma_pci_dma_unmap(pci_dev, sr->ring_state, TARGET_PAGE_SIZE); 440 441 out_free_sr_mem: 442 g_free(sr); 443 444 out: 445 rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE); 446 rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE); 447 448 return rc; 449 } 450 451 static void destroy_qp_rings(PvrdmaRing *ring, uint8_t is_srq) 452 { 453 pvrdma_ring_free(&ring[0]); 454 if (!is_srq) { 455 pvrdma_ring_free(&ring[1]); 456 } 457 458 rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE); 459 g_free(ring); 460 } 461 462 static int create_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, 463 union pvrdma_cmd_resp *rsp) 464 { 465 struct pvrdma_cmd_create_qp *cmd = &req->create_qp; 466 struct pvrdma_cmd_create_qp_resp *resp = &rsp->create_qp_resp; 467 PvrdmaRing *rings = NULL; 468 int rc; 469 470 memset(resp, 0, sizeof(*resp)); 471 472 rc = create_qp_rings(PCI_DEVICE(dev), cmd->pdir_dma, &rings, 473 cmd->max_send_wr, cmd->max_send_sge, cmd->send_chunks, 474 cmd->max_recv_wr, cmd->max_recv_sge, 475 cmd->total_chunks - cmd->send_chunks - 1, cmd->is_srq); 476 if (rc) { 477 return rc; 478 } 479 480 rc = rdma_rm_alloc_qp(&dev->rdma_dev_res, cmd->pd_handle, cmd->qp_type, 481 cmd->max_send_wr, cmd->max_send_sge, 482 cmd->send_cq_handle, cmd->max_recv_wr, 483 cmd->max_recv_sge, cmd->recv_cq_handle, rings, 484 &resp->qpn, cmd->is_srq, cmd->srq_handle); 485 if (rc) { 486 destroy_qp_rings(rings, cmd->is_srq); 487 return rc; 488 } 489 490 resp->max_send_wr = cmd->max_send_wr; 491 resp->max_recv_wr = cmd->max_recv_wr; 492 resp->max_send_sge = cmd->max_send_sge; 493 resp->max_recv_sge = cmd->max_recv_sge; 494 resp->max_inline_data = cmd->max_inline_data; 495 496 return 0; 497 } 498 499 static int modify_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, 500 union pvrdma_cmd_resp *rsp) 501 { 502 struct pvrdma_cmd_modify_qp *cmd = &req->modify_qp; 503 int rc; 504 505 /* No need to verify sgid_index since it is u8 */ 506 507 rc = rdma_rm_modify_qp(&dev->rdma_dev_res, &dev->backend_dev, 508 cmd->qp_handle, cmd->attr_mask, 509 cmd->attrs.ah_attr.grh.sgid_index, 510 (union ibv_gid *)&cmd->attrs.ah_attr.grh.dgid, 511 cmd->attrs.dest_qp_num, 512 (enum ibv_qp_state)cmd->attrs.qp_state, 513 cmd->attrs.qkey, cmd->attrs.rq_psn, 514 cmd->attrs.sq_psn); 515 516 return rc; 517 } 518 519 static int query_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, 520 union pvrdma_cmd_resp *rsp) 521 { 522 struct pvrdma_cmd_query_qp *cmd = &req->query_qp; 523 struct pvrdma_cmd_query_qp_resp *resp = &rsp->query_qp_resp; 524 struct ibv_qp_init_attr init_attr; 525 int rc; 526 527 memset(resp, 0, sizeof(*resp)); 528 529 rc = rdma_rm_query_qp(&dev->rdma_dev_res, &dev->backend_dev, cmd->qp_handle, 530 (struct ibv_qp_attr *)&resp->attrs, cmd->attr_mask, 531 &init_attr); 532 533 return rc; 534 } 535 536 static int destroy_qp(PVRDMADev *dev, union pvrdma_cmd_req *req, 537 union pvrdma_cmd_resp *rsp) 538 { 539 struct pvrdma_cmd_destroy_qp *cmd = &req->destroy_qp; 540 RdmaRmQP *qp; 541 PvrdmaRing *ring; 542 543 qp = rdma_rm_get_qp(&dev->rdma_dev_res, cmd->qp_handle); 544 if (!qp) { 545 return -EINVAL; 546 } 547 548 ring = (PvrdmaRing *)qp->opaque; 549 destroy_qp_rings(ring, qp->is_srq); 550 rdma_rm_dealloc_qp(&dev->rdma_dev_res, cmd->qp_handle); 551 552 return 0; 553 } 554 555 static int create_bind(PVRDMADev *dev, union pvrdma_cmd_req *req, 556 union pvrdma_cmd_resp *rsp) 557 { 558 struct pvrdma_cmd_create_bind *cmd = &req->create_bind; 559 int rc; 560 union ibv_gid *gid = (union ibv_gid *)&cmd->new_gid; 561 562 if (cmd->index >= MAX_PORT_GIDS) { 563 return -EINVAL; 564 } 565 566 rc = rdma_rm_add_gid(&dev->rdma_dev_res, &dev->backend_dev, 567 dev->backend_eth_device_name, gid, cmd->index); 568 569 return rc; 570 } 571 572 static int destroy_bind(PVRDMADev *dev, union pvrdma_cmd_req *req, 573 union pvrdma_cmd_resp *rsp) 574 { 575 int rc; 576 577 struct pvrdma_cmd_destroy_bind *cmd = &req->destroy_bind; 578 579 if (cmd->index >= MAX_PORT_GIDS) { 580 return -EINVAL; 581 } 582 583 rc = rdma_rm_del_gid(&dev->rdma_dev_res, &dev->backend_dev, 584 dev->backend_eth_device_name, cmd->index); 585 586 return rc; 587 } 588 589 static int create_uc(PVRDMADev *dev, union pvrdma_cmd_req *req, 590 union pvrdma_cmd_resp *rsp) 591 { 592 struct pvrdma_cmd_create_uc *cmd = &req->create_uc; 593 struct pvrdma_cmd_create_uc_resp *resp = &rsp->create_uc_resp; 594 int rc; 595 596 memset(resp, 0, sizeof(*resp)); 597 rc = rdma_rm_alloc_uc(&dev->rdma_dev_res, cmd->pfn, &resp->ctx_handle); 598 599 return rc; 600 } 601 602 static int destroy_uc(PVRDMADev *dev, union pvrdma_cmd_req *req, 603 union pvrdma_cmd_resp *rsp) 604 { 605 struct pvrdma_cmd_destroy_uc *cmd = &req->destroy_uc; 606 607 rdma_rm_dealloc_uc(&dev->rdma_dev_res, cmd->ctx_handle); 608 609 return 0; 610 } 611 612 static int create_srq_ring(PCIDevice *pci_dev, PvrdmaRing **ring, 613 uint64_t pdir_dma, uint32_t max_wr, 614 uint32_t max_sge, uint32_t nchunks) 615 { 616 uint64_t *dir = NULL, *tbl = NULL; 617 PvrdmaRing *r; 618 int rc = -EINVAL; 619 char ring_name[MAX_RING_NAME_SZ]; 620 uint32_t wqe_sz; 621 622 if (!nchunks || nchunks > PVRDMA_MAX_FAST_REG_PAGES) { 623 rdma_error_report("Got invalid page count for SRQ ring: %d", 624 nchunks); 625 return rc; 626 } 627 628 dir = rdma_pci_dma_map(pci_dev, pdir_dma, TARGET_PAGE_SIZE); 629 if (!dir) { 630 rdma_error_report("Failed to map to SRQ page directory"); 631 goto out; 632 } 633 634 tbl = rdma_pci_dma_map(pci_dev, dir[0], TARGET_PAGE_SIZE); 635 if (!tbl) { 636 rdma_error_report("Failed to map to SRQ page table"); 637 goto out; 638 } 639 640 r = g_malloc(sizeof(*r)); 641 *ring = r; 642 643 r->ring_state = (struct pvrdma_ring *) 644 rdma_pci_dma_map(pci_dev, tbl[0], TARGET_PAGE_SIZE); 645 if (!r->ring_state) { 646 rdma_error_report("Failed to map tp SRQ ring state"); 647 goto out_free_ring_mem; 648 } 649 650 wqe_sz = pow2ceil(sizeof(struct pvrdma_rq_wqe_hdr) + 651 sizeof(struct pvrdma_sge) * max_sge - 1); 652 sprintf(ring_name, "srq_ring_%" PRIx64, pdir_dma); 653 rc = pvrdma_ring_init(r, ring_name, pci_dev, &r->ring_state[1], max_wr, 654 wqe_sz, (dma_addr_t *)&tbl[1], nchunks - 1); 655 if (rc) { 656 goto out_unmap_ring_state; 657 } 658 659 goto out; 660 661 out_unmap_ring_state: 662 rdma_pci_dma_unmap(pci_dev, r->ring_state, TARGET_PAGE_SIZE); 663 664 out_free_ring_mem: 665 g_free(r); 666 667 out: 668 rdma_pci_dma_unmap(pci_dev, tbl, TARGET_PAGE_SIZE); 669 rdma_pci_dma_unmap(pci_dev, dir, TARGET_PAGE_SIZE); 670 671 return rc; 672 } 673 674 static void destroy_srq_ring(PvrdmaRing *ring) 675 { 676 pvrdma_ring_free(ring); 677 rdma_pci_dma_unmap(ring->dev, ring->ring_state, TARGET_PAGE_SIZE); 678 g_free(ring); 679 } 680 681 static int create_srq(PVRDMADev *dev, union pvrdma_cmd_req *req, 682 union pvrdma_cmd_resp *rsp) 683 { 684 struct pvrdma_cmd_create_srq *cmd = &req->create_srq; 685 struct pvrdma_cmd_create_srq_resp *resp = &rsp->create_srq_resp; 686 PvrdmaRing *ring = NULL; 687 int rc; 688 689 memset(resp, 0, sizeof(*resp)); 690 691 rc = create_srq_ring(PCI_DEVICE(dev), &ring, cmd->pdir_dma, 692 cmd->attrs.max_wr, cmd->attrs.max_sge, 693 cmd->nchunks); 694 if (rc) { 695 return rc; 696 } 697 698 rc = rdma_rm_alloc_srq(&dev->rdma_dev_res, cmd->pd_handle, 699 cmd->attrs.max_wr, cmd->attrs.max_sge, 700 cmd->attrs.srq_limit, &resp->srqn, ring); 701 if (rc) { 702 destroy_srq_ring(ring); 703 return rc; 704 } 705 706 return 0; 707 } 708 709 static int query_srq(PVRDMADev *dev, union pvrdma_cmd_req *req, 710 union pvrdma_cmd_resp *rsp) 711 { 712 struct pvrdma_cmd_query_srq *cmd = &req->query_srq; 713 struct pvrdma_cmd_query_srq_resp *resp = &rsp->query_srq_resp; 714 715 memset(resp, 0, sizeof(*resp)); 716 717 return rdma_rm_query_srq(&dev->rdma_dev_res, cmd->srq_handle, 718 (struct ibv_srq_attr *)&resp->attrs); 719 } 720 721 static int modify_srq(PVRDMADev *dev, union pvrdma_cmd_req *req, 722 union pvrdma_cmd_resp *rsp) 723 { 724 struct pvrdma_cmd_modify_srq *cmd = &req->modify_srq; 725 726 /* Only support SRQ limit */ 727 if (!(cmd->attr_mask & IBV_SRQ_LIMIT) || 728 (cmd->attr_mask & IBV_SRQ_MAX_WR)) 729 return -EINVAL; 730 731 return rdma_rm_modify_srq(&dev->rdma_dev_res, cmd->srq_handle, 732 (struct ibv_srq_attr *)&cmd->attrs, 733 cmd->attr_mask); 734 } 735 736 static int destroy_srq(PVRDMADev *dev, union pvrdma_cmd_req *req, 737 union pvrdma_cmd_resp *rsp) 738 { 739 struct pvrdma_cmd_destroy_srq *cmd = &req->destroy_srq; 740 RdmaRmSRQ *srq; 741 PvrdmaRing *ring; 742 743 srq = rdma_rm_get_srq(&dev->rdma_dev_res, cmd->srq_handle); 744 if (!srq) { 745 return -EINVAL; 746 } 747 748 ring = (PvrdmaRing *)srq->opaque; 749 destroy_srq_ring(ring); 750 rdma_rm_dealloc_srq(&dev->rdma_dev_res, cmd->srq_handle); 751 752 return 0; 753 } 754 755 struct cmd_handler { 756 uint32_t cmd; 757 uint32_t ack; 758 int (*exec)(PVRDMADev *dev, union pvrdma_cmd_req *req, 759 union pvrdma_cmd_resp *rsp); 760 }; 761 762 static struct cmd_handler cmd_handlers[] = { 763 {PVRDMA_CMD_QUERY_PORT, PVRDMA_CMD_QUERY_PORT_RESP, query_port}, 764 {PVRDMA_CMD_QUERY_PKEY, PVRDMA_CMD_QUERY_PKEY_RESP, query_pkey}, 765 {PVRDMA_CMD_CREATE_PD, PVRDMA_CMD_CREATE_PD_RESP, create_pd}, 766 {PVRDMA_CMD_DESTROY_PD, PVRDMA_CMD_DESTROY_PD_RESP_NOOP, destroy_pd}, 767 {PVRDMA_CMD_CREATE_MR, PVRDMA_CMD_CREATE_MR_RESP, create_mr}, 768 {PVRDMA_CMD_DESTROY_MR, PVRDMA_CMD_DESTROY_MR_RESP_NOOP, destroy_mr}, 769 {PVRDMA_CMD_CREATE_CQ, PVRDMA_CMD_CREATE_CQ_RESP, create_cq}, 770 {PVRDMA_CMD_RESIZE_CQ, PVRDMA_CMD_RESIZE_CQ_RESP, NULL}, 771 {PVRDMA_CMD_DESTROY_CQ, PVRDMA_CMD_DESTROY_CQ_RESP_NOOP, destroy_cq}, 772 {PVRDMA_CMD_CREATE_QP, PVRDMA_CMD_CREATE_QP_RESP, create_qp}, 773 {PVRDMA_CMD_MODIFY_QP, PVRDMA_CMD_MODIFY_QP_RESP, modify_qp}, 774 {PVRDMA_CMD_QUERY_QP, PVRDMA_CMD_QUERY_QP_RESP, query_qp}, 775 {PVRDMA_CMD_DESTROY_QP, PVRDMA_CMD_DESTROY_QP_RESP, destroy_qp}, 776 {PVRDMA_CMD_CREATE_UC, PVRDMA_CMD_CREATE_UC_RESP, create_uc}, 777 {PVRDMA_CMD_DESTROY_UC, PVRDMA_CMD_DESTROY_UC_RESP_NOOP, destroy_uc}, 778 {PVRDMA_CMD_CREATE_BIND, PVRDMA_CMD_CREATE_BIND_RESP_NOOP, create_bind}, 779 {PVRDMA_CMD_DESTROY_BIND, PVRDMA_CMD_DESTROY_BIND_RESP_NOOP, destroy_bind}, 780 {PVRDMA_CMD_CREATE_SRQ, PVRDMA_CMD_CREATE_SRQ_RESP, create_srq}, 781 {PVRDMA_CMD_QUERY_SRQ, PVRDMA_CMD_QUERY_SRQ_RESP, query_srq}, 782 {PVRDMA_CMD_MODIFY_SRQ, PVRDMA_CMD_MODIFY_SRQ_RESP, modify_srq}, 783 {PVRDMA_CMD_DESTROY_SRQ, PVRDMA_CMD_DESTROY_SRQ_RESP, destroy_srq}, 784 }; 785 786 int pvrdma_exec_cmd(PVRDMADev *dev) 787 { 788 int err = 0xFFFF; 789 DSRInfo *dsr_info; 790 791 dsr_info = &dev->dsr_info; 792 793 if (dsr_info->req->hdr.cmd >= sizeof(cmd_handlers) / 794 sizeof(struct cmd_handler)) { 795 rdma_error_report("Unsupported command"); 796 goto out; 797 } 798 799 if (!cmd_handlers[dsr_info->req->hdr.cmd].exec) { 800 rdma_error_report("Unsupported command (not implemented yet)"); 801 goto out; 802 } 803 804 err = cmd_handlers[dsr_info->req->hdr.cmd].exec(dev, dsr_info->req, 805 dsr_info->rsp); 806 dsr_info->rsp->hdr.response = dsr_info->req->hdr.response; 807 dsr_info->rsp->hdr.ack = cmd_handlers[dsr_info->req->hdr.cmd].ack; 808 dsr_info->rsp->hdr.err = err < 0 ? -err : 0; 809 810 trace_pvrdma_exec_cmd(dsr_info->req->hdr.cmd, dsr_info->rsp->hdr.err); 811 812 dev->stats.commands++; 813 814 out: 815 set_reg_val(dev, PVRDMA_REG_ERR, err); 816 post_interrupt(dev, INTR_VEC_CMD_RING); 817 818 return (err == 0) ? 0 : -EINVAL; 819 } 820