1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 /* 3 * Copyright(c) 2016 - 2018 Intel Corporation. 4 */ 5 6 #include <linux/module.h> 7 #include <linux/kernel.h> 8 #include <linux/dma-mapping.h> 9 #include "vt.h" 10 #include "cq.h" 11 #include "trace.h" 12 13 #define RVT_UVERBS_ABI_VERSION 2 14 15 MODULE_LICENSE("Dual BSD/GPL"); 16 MODULE_DESCRIPTION("RDMA Verbs Transport Library"); 17 18 static int __init rvt_init(void) 19 { 20 int ret = rvt_driver_cq_init(); 21 22 if (ret) 23 pr_err("Error in driver CQ init.\n"); 24 25 return ret; 26 } 27 module_init(rvt_init); 28 29 static void __exit rvt_cleanup(void) 30 { 31 rvt_cq_exit(); 32 } 33 module_exit(rvt_cleanup); 34 35 /** 36 * rvt_alloc_device - allocate rdi 37 * @size: how big of a structure to allocate 38 * @nports: number of ports to allocate array slots for 39 * 40 * Use IB core device alloc to allocate space for the rdi which is assumed to be 41 * inside of the ib_device. Any extra space that drivers require should be 42 * included in size. 43 * 44 * We also allocate a port array based on the number of ports. 45 * 46 * Return: pointer to allocated rdi 47 */ 48 struct rvt_dev_info *rvt_alloc_device(size_t size, int nports) 49 { 50 struct rvt_dev_info *rdi; 51 52 rdi = container_of(_ib_alloc_device(size), struct rvt_dev_info, ibdev); 53 if (!rdi) 54 return rdi; 55 56 rdi->ports = kcalloc(nports, sizeof(*rdi->ports), GFP_KERNEL); 57 if (!rdi->ports) 58 ib_dealloc_device(&rdi->ibdev); 59 60 return rdi; 61 } 62 EXPORT_SYMBOL(rvt_alloc_device); 63 64 /** 65 * rvt_dealloc_device - deallocate rdi 66 * @rdi: structure to free 67 * 68 * Free a structure allocated with rvt_alloc_device() 69 */ 70 void rvt_dealloc_device(struct rvt_dev_info *rdi) 71 { 72 kfree(rdi->ports); 73 ib_dealloc_device(&rdi->ibdev); 74 } 75 EXPORT_SYMBOL(rvt_dealloc_device); 76 77 static int rvt_query_device(struct ib_device *ibdev, 78 struct ib_device_attr *props, 79 struct ib_udata *uhw) 80 { 81 struct rvt_dev_info *rdi = ib_to_rvt(ibdev); 82 83 if (uhw->inlen || uhw->outlen) 84 return -EINVAL; 85 /* 86 * Return rvt_dev_info.dparms.props contents 87 */ 88 *props = rdi->dparms.props; 89 return 0; 90 } 91 92 static int rvt_get_numa_node(struct ib_device *ibdev) 93 { 94 struct rvt_dev_info *rdi = ib_to_rvt(ibdev); 95 96 return rdi->dparms.node; 97 } 98 99 static int rvt_modify_device(struct ib_device *device, 100 int device_modify_mask, 101 struct ib_device_modify *device_modify) 102 { 103 /* 104 * There is currently no need to supply this based on qib and hfi1. 105 * Future drivers may need to implement this though. 106 */ 107 108 return -EOPNOTSUPP; 109 } 110 111 /** 112 * rvt_query_port - Passes the query port call to the driver 113 * @ibdev: Verbs IB dev 114 * @port_num: port number, 1 based from ib core 115 * @props: structure to hold returned properties 116 * 117 * Return: 0 on success 118 */ 119 static int rvt_query_port(struct ib_device *ibdev, u32 port_num, 120 struct ib_port_attr *props) 121 { 122 struct rvt_dev_info *rdi = ib_to_rvt(ibdev); 123 struct rvt_ibport *rvp; 124 u32 port_index = ibport_num_to_idx(ibdev, port_num); 125 126 rvp = rdi->ports[port_index]; 127 /* props being zeroed by the caller, avoid zeroing it here */ 128 props->sm_lid = rvp->sm_lid; 129 props->sm_sl = rvp->sm_sl; 130 props->port_cap_flags = rvp->port_cap_flags; 131 props->max_msg_sz = 0x80000000; 132 props->pkey_tbl_len = rvt_get_npkeys(rdi); 133 props->bad_pkey_cntr = rvp->pkey_violations; 134 props->qkey_viol_cntr = rvp->qkey_violations; 135 props->subnet_timeout = rvp->subnet_timeout; 136 props->init_type_reply = 0; 137 138 /* Populate the remaining ib_port_attr elements */ 139 return rdi->driver_f.query_port_state(rdi, port_num, props); 140 } 141 142 /** 143 * rvt_modify_port - modify port 144 * @ibdev: Verbs IB dev 145 * @port_num: Port number, 1 based from ib core 146 * @port_modify_mask: How to change the port 147 * @props: Structure to fill in 148 * 149 * Return: 0 on success 150 */ 151 static int rvt_modify_port(struct ib_device *ibdev, u32 port_num, 152 int port_modify_mask, struct ib_port_modify *props) 153 { 154 struct rvt_dev_info *rdi = ib_to_rvt(ibdev); 155 struct rvt_ibport *rvp; 156 int ret = 0; 157 u32 port_index = ibport_num_to_idx(ibdev, port_num); 158 159 rvp = rdi->ports[port_index]; 160 if (port_modify_mask & IB_PORT_OPA_MASK_CHG) { 161 rvp->port_cap3_flags |= props->set_port_cap_mask; 162 rvp->port_cap3_flags &= ~props->clr_port_cap_mask; 163 } else { 164 rvp->port_cap_flags |= props->set_port_cap_mask; 165 rvp->port_cap_flags &= ~props->clr_port_cap_mask; 166 } 167 168 if (props->set_port_cap_mask || props->clr_port_cap_mask) 169 rdi->driver_f.cap_mask_chg(rdi, port_num); 170 if (port_modify_mask & IB_PORT_SHUTDOWN) 171 ret = rdi->driver_f.shut_down_port(rdi, port_num); 172 if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR) 173 rvp->qkey_violations = 0; 174 175 return ret; 176 } 177 178 /** 179 * rvt_query_pkey - Return a pkey from the table at a given index 180 * @ibdev: Verbs IB dev 181 * @port_num: Port number, 1 based from ib core 182 * @index: Index into pkey table 183 * @pkey: returned pkey from the port pkey table 184 * 185 * Return: 0 on failure pkey otherwise 186 */ 187 static int rvt_query_pkey(struct ib_device *ibdev, u32 port_num, u16 index, 188 u16 *pkey) 189 { 190 /* 191 * Driver will be responsible for keeping rvt_dev_info.pkey_table up to 192 * date. This function will just return that value. There is no need to 193 * lock, if a stale value is read and sent to the user so be it there is 194 * no way to protect against that anyway. 195 */ 196 struct rvt_dev_info *rdi = ib_to_rvt(ibdev); 197 u32 port_index; 198 199 port_index = ibport_num_to_idx(ibdev, port_num); 200 201 if (index >= rvt_get_npkeys(rdi)) 202 return -EINVAL; 203 204 *pkey = rvt_get_pkey(rdi, port_index, index); 205 return 0; 206 } 207 208 /** 209 * rvt_query_gid - Return a gid from the table 210 * @ibdev: Verbs IB dev 211 * @port_num: Port number, 1 based from ib core 212 * @guid_index: Index in table 213 * @gid: Gid to return 214 * 215 * Return: 0 on success 216 */ 217 static int rvt_query_gid(struct ib_device *ibdev, u32 port_num, 218 int guid_index, union ib_gid *gid) 219 { 220 struct rvt_dev_info *rdi; 221 struct rvt_ibport *rvp; 222 u32 port_index; 223 224 /* 225 * Driver is responsible for updating the guid table. Which will be used 226 * to craft the return value. This will work similar to how query_pkey() 227 * is being done. 228 */ 229 port_index = ibport_num_to_idx(ibdev, port_num); 230 231 rdi = ib_to_rvt(ibdev); 232 rvp = rdi->ports[port_index]; 233 234 gid->global.subnet_prefix = rvp->gid_prefix; 235 236 return rdi->driver_f.get_guid_be(rdi, rvp, guid_index, 237 &gid->global.interface_id); 238 } 239 240 /** 241 * rvt_alloc_ucontext - Allocate a user context 242 * @uctx: Verbs context 243 * @udata: User data allocated 244 */ 245 static int rvt_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 246 { 247 return 0; 248 } 249 250 /** 251 * rvt_dealloc_ucontext - Free a user context 252 * @context: Unused 253 */ 254 static void rvt_dealloc_ucontext(struct ib_ucontext *context) 255 { 256 return; 257 } 258 259 static int rvt_get_port_immutable(struct ib_device *ibdev, u32 port_num, 260 struct ib_port_immutable *immutable) 261 { 262 struct rvt_dev_info *rdi = ib_to_rvt(ibdev); 263 struct ib_port_attr attr; 264 int err; 265 266 immutable->core_cap_flags = rdi->dparms.core_cap_flags; 267 268 err = ib_query_port(ibdev, port_num, &attr); 269 if (err) 270 return err; 271 272 immutable->pkey_tbl_len = attr.pkey_tbl_len; 273 immutable->gid_tbl_len = attr.gid_tbl_len; 274 immutable->max_mad_size = rdi->dparms.max_mad_size; 275 276 return 0; 277 } 278 279 enum { 280 MISC, 281 QUERY_DEVICE, 282 MODIFY_DEVICE, 283 QUERY_PORT, 284 MODIFY_PORT, 285 QUERY_PKEY, 286 QUERY_GID, 287 ALLOC_UCONTEXT, 288 DEALLOC_UCONTEXT, 289 GET_PORT_IMMUTABLE, 290 CREATE_QP, 291 MODIFY_QP, 292 DESTROY_QP, 293 QUERY_QP, 294 POST_SEND, 295 POST_RECV, 296 POST_SRQ_RECV, 297 CREATE_AH, 298 DESTROY_AH, 299 MODIFY_AH, 300 QUERY_AH, 301 CREATE_SRQ, 302 MODIFY_SRQ, 303 DESTROY_SRQ, 304 QUERY_SRQ, 305 ATTACH_MCAST, 306 DETACH_MCAST, 307 GET_DMA_MR, 308 REG_USER_MR, 309 DEREG_MR, 310 ALLOC_MR, 311 MAP_MR_SG, 312 ALLOC_FMR, 313 MAP_PHYS_FMR, 314 UNMAP_FMR, 315 DEALLOC_FMR, 316 MMAP, 317 CREATE_CQ, 318 DESTROY_CQ, 319 POLL_CQ, 320 REQ_NOTFIY_CQ, 321 RESIZE_CQ, 322 ALLOC_PD, 323 DEALLOC_PD, 324 _VERB_IDX_MAX /* Must always be last! */ 325 }; 326 327 static const struct ib_device_ops rvt_dev_ops = { 328 .uverbs_abi_ver = RVT_UVERBS_ABI_VERSION, 329 330 .alloc_mr = rvt_alloc_mr, 331 .alloc_pd = rvt_alloc_pd, 332 .alloc_ucontext = rvt_alloc_ucontext, 333 .attach_mcast = rvt_attach_mcast, 334 .create_ah = rvt_create_ah, 335 .create_cq = rvt_create_cq, 336 .create_qp = rvt_create_qp, 337 .create_srq = rvt_create_srq, 338 .create_user_ah = rvt_create_ah, 339 .dealloc_pd = rvt_dealloc_pd, 340 .dealloc_ucontext = rvt_dealloc_ucontext, 341 .dereg_mr = rvt_dereg_mr, 342 .destroy_ah = rvt_destroy_ah, 343 .destroy_cq = rvt_destroy_cq, 344 .destroy_qp = rvt_destroy_qp, 345 .destroy_srq = rvt_destroy_srq, 346 .detach_mcast = rvt_detach_mcast, 347 .get_dma_mr = rvt_get_dma_mr, 348 .get_numa_node = rvt_get_numa_node, 349 .get_port_immutable = rvt_get_port_immutable, 350 .map_mr_sg = rvt_map_mr_sg, 351 .mmap = rvt_mmap, 352 .modify_ah = rvt_modify_ah, 353 .modify_device = rvt_modify_device, 354 .modify_port = rvt_modify_port, 355 .modify_qp = rvt_modify_qp, 356 .modify_srq = rvt_modify_srq, 357 .poll_cq = rvt_poll_cq, 358 .post_recv = rvt_post_recv, 359 .post_send = rvt_post_send, 360 .post_srq_recv = rvt_post_srq_recv, 361 .query_ah = rvt_query_ah, 362 .query_device = rvt_query_device, 363 .query_gid = rvt_query_gid, 364 .query_pkey = rvt_query_pkey, 365 .query_port = rvt_query_port, 366 .query_qp = rvt_query_qp, 367 .query_srq = rvt_query_srq, 368 .reg_user_mr = rvt_reg_user_mr, 369 .req_notify_cq = rvt_req_notify_cq, 370 .resize_cq = rvt_resize_cq, 371 372 INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah), 373 INIT_RDMA_OBJ_SIZE(ib_cq, rvt_cq, ibcq), 374 INIT_RDMA_OBJ_SIZE(ib_pd, rvt_pd, ibpd), 375 INIT_RDMA_OBJ_SIZE(ib_qp, rvt_qp, ibqp), 376 INIT_RDMA_OBJ_SIZE(ib_srq, rvt_srq, ibsrq), 377 INIT_RDMA_OBJ_SIZE(ib_ucontext, rvt_ucontext, ibucontext), 378 }; 379 380 static noinline int check_support(struct rvt_dev_info *rdi, int verb) 381 { 382 switch (verb) { 383 case MISC: 384 /* 385 * These functions are not part of verbs specifically but are 386 * required for rdmavt to function. 387 */ 388 if ((!rdi->ibdev.ops.port_groups) || 389 (!rdi->driver_f.get_pci_dev)) 390 return -EINVAL; 391 break; 392 393 case MODIFY_DEVICE: 394 /* 395 * rdmavt does not support modify device currently drivers must 396 * provide. 397 */ 398 if (!rdi->ibdev.ops.modify_device) 399 return -EOPNOTSUPP; 400 break; 401 402 case QUERY_PORT: 403 if (!rdi->ibdev.ops.query_port) 404 if (!rdi->driver_f.query_port_state) 405 return -EINVAL; 406 break; 407 408 case MODIFY_PORT: 409 if (!rdi->ibdev.ops.modify_port) 410 if (!rdi->driver_f.cap_mask_chg || 411 !rdi->driver_f.shut_down_port) 412 return -EINVAL; 413 break; 414 415 case QUERY_GID: 416 if (!rdi->ibdev.ops.query_gid) 417 if (!rdi->driver_f.get_guid_be) 418 return -EINVAL; 419 break; 420 421 case CREATE_QP: 422 if (!rdi->ibdev.ops.create_qp) 423 if (!rdi->driver_f.qp_priv_alloc || 424 !rdi->driver_f.qp_priv_free || 425 !rdi->driver_f.notify_qp_reset || 426 !rdi->driver_f.flush_qp_waiters || 427 !rdi->driver_f.stop_send_queue || 428 !rdi->driver_f.quiesce_qp) 429 return -EINVAL; 430 break; 431 432 case MODIFY_QP: 433 if (!rdi->ibdev.ops.modify_qp) 434 if (!rdi->driver_f.notify_qp_reset || 435 !rdi->driver_f.schedule_send || 436 !rdi->driver_f.get_pmtu_from_attr || 437 !rdi->driver_f.flush_qp_waiters || 438 !rdi->driver_f.stop_send_queue || 439 !rdi->driver_f.quiesce_qp || 440 !rdi->driver_f.notify_error_qp || 441 !rdi->driver_f.mtu_from_qp || 442 !rdi->driver_f.mtu_to_path_mtu) 443 return -EINVAL; 444 break; 445 446 case DESTROY_QP: 447 if (!rdi->ibdev.ops.destroy_qp) 448 if (!rdi->driver_f.qp_priv_free || 449 !rdi->driver_f.notify_qp_reset || 450 !rdi->driver_f.flush_qp_waiters || 451 !rdi->driver_f.stop_send_queue || 452 !rdi->driver_f.quiesce_qp) 453 return -EINVAL; 454 break; 455 456 case POST_SEND: 457 if (!rdi->ibdev.ops.post_send) 458 if (!rdi->driver_f.schedule_send || 459 !rdi->driver_f.do_send || 460 !rdi->post_parms) 461 return -EINVAL; 462 break; 463 464 } 465 466 return 0; 467 } 468 469 /** 470 * rvt_register_device - register a driver 471 * @rdi: main dev structure for all of rdmavt operations 472 * 473 * It is up to drivers to allocate the rdi and fill in the appropriate 474 * information. 475 * 476 * Return: 0 on success otherwise an errno. 477 */ 478 int rvt_register_device(struct rvt_dev_info *rdi) 479 { 480 int ret = 0, i; 481 482 if (!rdi) 483 return -EINVAL; 484 485 /* 486 * Check to ensure drivers have setup the required helpers for the verbs 487 * they want rdmavt to handle 488 */ 489 for (i = 0; i < _VERB_IDX_MAX; i++) 490 if (check_support(rdi, i)) { 491 pr_err("Driver support req not met at %d\n", i); 492 return -EINVAL; 493 } 494 495 ib_set_device_ops(&rdi->ibdev, &rvt_dev_ops); 496 497 /* Once we get past here we can use rvt_pr macros and tracepoints */ 498 trace_rvt_dbg(rdi, "Driver attempting registration"); 499 rvt_mmap_init(rdi); 500 501 /* Queue Pairs */ 502 ret = rvt_driver_qp_init(rdi); 503 if (ret) { 504 pr_err("Error in driver QP init.\n"); 505 return -EINVAL; 506 } 507 508 /* Address Handle */ 509 spin_lock_init(&rdi->n_ahs_lock); 510 rdi->n_ahs_allocated = 0; 511 512 /* Shared Receive Queue */ 513 rvt_driver_srq_init(rdi); 514 515 /* Multicast */ 516 rvt_driver_mcast_init(rdi); 517 518 /* Mem Region */ 519 ret = rvt_driver_mr_init(rdi); 520 if (ret) { 521 pr_err("Error in driver MR init.\n"); 522 goto bail_no_mr; 523 } 524 525 /* Memory Working Set Size */ 526 ret = rvt_wss_init(rdi); 527 if (ret) { 528 rvt_pr_err(rdi, "Error in WSS init.\n"); 529 goto bail_mr; 530 } 531 532 /* Completion queues */ 533 spin_lock_init(&rdi->n_cqs_lock); 534 535 /* Protection Domain */ 536 spin_lock_init(&rdi->n_pds_lock); 537 rdi->n_pds_allocated = 0; 538 539 /* 540 * There are some things which could be set by underlying drivers but 541 * really should be up to rdmavt to set. For instance drivers can't know 542 * exactly which functions rdmavt supports, nor do they know the ABI 543 * version, so we do all of this sort of stuff here. 544 */ 545 rdi->ibdev.uverbs_cmd_mask |= 546 (1ull << IB_USER_VERBS_CMD_POLL_CQ) | 547 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 548 (1ull << IB_USER_VERBS_CMD_POST_SEND) | 549 (1ull << IB_USER_VERBS_CMD_POST_RECV) | 550 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV); 551 rdi->ibdev.node_type = RDMA_NODE_IB_CA; 552 if (!rdi->ibdev.num_comp_vectors) 553 rdi->ibdev.num_comp_vectors = 1; 554 555 /* We are now good to announce we exist */ 556 ret = ib_register_device(&rdi->ibdev, dev_name(&rdi->ibdev.dev), NULL); 557 if (ret) { 558 rvt_pr_err(rdi, "Failed to register driver with ib core.\n"); 559 goto bail_wss; 560 } 561 562 rvt_create_mad_agents(rdi); 563 564 rvt_pr_info(rdi, "Registration with rdmavt done.\n"); 565 return ret; 566 567 bail_wss: 568 rvt_wss_exit(rdi); 569 bail_mr: 570 rvt_mr_exit(rdi); 571 572 bail_no_mr: 573 rvt_qp_exit(rdi); 574 575 return ret; 576 } 577 EXPORT_SYMBOL(rvt_register_device); 578 579 /** 580 * rvt_unregister_device - remove a driver 581 * @rdi: rvt dev struct 582 */ 583 void rvt_unregister_device(struct rvt_dev_info *rdi) 584 { 585 trace_rvt_dbg(rdi, "Driver is unregistering."); 586 if (!rdi) 587 return; 588 589 rvt_free_mad_agents(rdi); 590 591 ib_unregister_device(&rdi->ibdev); 592 rvt_wss_exit(rdi); 593 rvt_mr_exit(rdi); 594 rvt_qp_exit(rdi); 595 } 596 EXPORT_SYMBOL(rvt_unregister_device); 597 598 /** 599 * rvt_init_port - init internal data for driver port 600 * @rdi: rvt_dev_info struct 601 * @port: rvt port 602 * @port_index: 0 based index of ports, different from IB core port num 603 * @pkey_table: pkey_table for @port 604 * 605 * Keep track of a list of ports. No need to have a detach port. 606 * They persist until the driver goes away. 607 * 608 * Return: always 0 609 */ 610 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port, 611 int port_index, u16 *pkey_table) 612 { 613 614 rdi->ports[port_index] = port; 615 rdi->ports[port_index]->pkey_table = pkey_table; 616 617 return 0; 618 } 619 EXPORT_SYMBOL(rvt_init_port); 620