1 /* 2 * Copyright (c) 2005 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. 5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/completion.h> 37 #include <linux/in.h> 38 #include <linux/in6.h> 39 #include <linux/mutex.h> 40 #include <linux/random.h> 41 #include <linux/idr.h> 42 #include <linux/inetdevice.h> 43 #include <linux/slab.h> 44 45 #include <net/tcp.h> 46 #include <net/ipv6.h> 47 48 #include <rdma/rdma_cm.h> 49 #include <rdma/rdma_cm_ib.h> 50 #include <rdma/ib_cache.h> 51 #include <rdma/ib_cm.h> 52 #include <rdma/ib_sa.h> 53 #include <rdma/iw_cm.h> 54 55 MODULE_AUTHOR("Sean Hefty"); 56 MODULE_DESCRIPTION("Generic RDMA CM Agent"); 57 MODULE_LICENSE("Dual BSD/GPL"); 58 59 #define CMA_CM_RESPONSE_TIMEOUT 20 60 #define CMA_MAX_CM_RETRIES 15 61 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 62 #define CMA_IBOE_PACKET_LIFETIME 18 63 64 static void cma_add_one(struct ib_device *device); 65 static void cma_remove_one(struct ib_device *device); 66 67 static struct ib_client cma_client = { 68 .name = "cma", 69 .add = cma_add_one, 70 .remove = cma_remove_one 71 }; 72 73 static struct ib_sa_client sa_client; 74 static struct rdma_addr_client addr_client; 75 static LIST_HEAD(dev_list); 76 static LIST_HEAD(listen_any_list); 77 static DEFINE_MUTEX(lock); 78 static struct workqueue_struct *cma_wq; 79 static DEFINE_IDR(sdp_ps); 80 static DEFINE_IDR(tcp_ps); 81 static DEFINE_IDR(udp_ps); 82 static DEFINE_IDR(ipoib_ps); 83 84 struct cma_device { 85 struct list_head list; 86 struct ib_device *device; 87 struct completion comp; 88 atomic_t refcount; 89 struct list_head id_list; 90 }; 91 92 enum cma_state { 93 CMA_IDLE, 94 CMA_ADDR_QUERY, 95 CMA_ADDR_RESOLVED, 96 CMA_ROUTE_QUERY, 97 CMA_ROUTE_RESOLVED, 98 CMA_CONNECT, 99 CMA_DISCONNECT, 100 CMA_ADDR_BOUND, 101 CMA_LISTEN, 102 CMA_DEVICE_REMOVAL, 103 CMA_DESTROYING 104 }; 105 106 struct rdma_bind_list { 107 struct idr *ps; 108 struct hlist_head owners; 109 unsigned short port; 110 }; 111 112 /* 113 * Device removal can occur at anytime, so we need extra handling to 114 * serialize notifying the user of device removal with other callbacks. 115 * We do this by disabling removal notification while a callback is in process, 116 * and reporting it after the callback completes. 117 */ 118 struct rdma_id_private { 119 struct rdma_cm_id id; 120 121 struct rdma_bind_list *bind_list; 122 struct hlist_node node; 123 struct list_head list; /* listen_any_list or cma_device.list */ 124 struct list_head listen_list; /* per device listens */ 125 struct cma_device *cma_dev; 126 struct list_head mc_list; 127 128 int internal_id; 129 enum cma_state state; 130 spinlock_t lock; 131 struct mutex qp_mutex; 132 133 struct completion comp; 134 atomic_t refcount; 135 struct mutex handler_mutex; 136 137 int backlog; 138 int timeout_ms; 139 struct ib_sa_query *query; 140 int query_id; 141 union { 142 struct ib_cm_id *ib; 143 struct iw_cm_id *iw; 144 } cm_id; 145 146 u32 seq_num; 147 u32 qkey; 148 u32 qp_num; 149 u8 srq; 150 u8 tos; 151 }; 152 153 struct cma_multicast { 154 struct rdma_id_private *id_priv; 155 union { 156 struct ib_sa_multicast *ib; 157 } multicast; 158 struct list_head list; 159 void *context; 160 struct sockaddr_storage addr; 161 struct kref mcref; 162 }; 163 164 struct cma_work { 165 struct work_struct work; 166 struct rdma_id_private *id; 167 enum cma_state old_state; 168 enum cma_state new_state; 169 struct rdma_cm_event event; 170 }; 171 172 struct cma_ndev_work { 173 struct work_struct work; 174 struct rdma_id_private *id; 175 struct rdma_cm_event event; 176 }; 177 178 struct iboe_mcast_work { 179 struct work_struct work; 180 struct rdma_id_private *id; 181 struct cma_multicast *mc; 182 }; 183 184 union cma_ip_addr { 185 struct in6_addr ip6; 186 struct { 187 __be32 pad[3]; 188 __be32 addr; 189 } ip4; 190 }; 191 192 struct cma_hdr { 193 u8 cma_version; 194 u8 ip_version; /* IP version: 7:4 */ 195 __be16 port; 196 union cma_ip_addr src_addr; 197 union cma_ip_addr dst_addr; 198 }; 199 200 struct sdp_hh { 201 u8 bsdh[16]; 202 u8 sdp_version; /* Major version: 7:4 */ 203 u8 ip_version; /* IP version: 7:4 */ 204 u8 sdp_specific1[10]; 205 __be16 port; 206 __be16 sdp_specific2; 207 union cma_ip_addr src_addr; 208 union cma_ip_addr dst_addr; 209 }; 210 211 struct sdp_hah { 212 u8 bsdh[16]; 213 u8 sdp_version; 214 }; 215 216 #define CMA_VERSION 0x00 217 #define SDP_MAJ_VERSION 0x2 218 219 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp) 220 { 221 unsigned long flags; 222 int ret; 223 224 spin_lock_irqsave(&id_priv->lock, flags); 225 ret = (id_priv->state == comp); 226 spin_unlock_irqrestore(&id_priv->lock, flags); 227 return ret; 228 } 229 230 static int cma_comp_exch(struct rdma_id_private *id_priv, 231 enum cma_state comp, enum cma_state exch) 232 { 233 unsigned long flags; 234 int ret; 235 236 spin_lock_irqsave(&id_priv->lock, flags); 237 if ((ret = (id_priv->state == comp))) 238 id_priv->state = exch; 239 spin_unlock_irqrestore(&id_priv->lock, flags); 240 return ret; 241 } 242 243 static enum cma_state cma_exch(struct rdma_id_private *id_priv, 244 enum cma_state exch) 245 { 246 unsigned long flags; 247 enum cma_state old; 248 249 spin_lock_irqsave(&id_priv->lock, flags); 250 old = id_priv->state; 251 id_priv->state = exch; 252 spin_unlock_irqrestore(&id_priv->lock, flags); 253 return old; 254 } 255 256 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr) 257 { 258 return hdr->ip_version >> 4; 259 } 260 261 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 262 { 263 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 264 } 265 266 static inline u8 sdp_get_majv(u8 sdp_version) 267 { 268 return sdp_version >> 4; 269 } 270 271 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh) 272 { 273 return hh->ip_version >> 4; 274 } 275 276 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver) 277 { 278 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF); 279 } 280 281 static inline int cma_is_ud_ps(enum rdma_port_space ps) 282 { 283 return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB); 284 } 285 286 static void cma_attach_to_dev(struct rdma_id_private *id_priv, 287 struct cma_device *cma_dev) 288 { 289 atomic_inc(&cma_dev->refcount); 290 id_priv->cma_dev = cma_dev; 291 id_priv->id.device = cma_dev->device; 292 id_priv->id.route.addr.dev_addr.transport = 293 rdma_node_get_transport(cma_dev->device->node_type); 294 list_add_tail(&id_priv->list, &cma_dev->id_list); 295 } 296 297 static inline void cma_deref_dev(struct cma_device *cma_dev) 298 { 299 if (atomic_dec_and_test(&cma_dev->refcount)) 300 complete(&cma_dev->comp); 301 } 302 303 static inline void release_mc(struct kref *kref) 304 { 305 struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); 306 307 kfree(mc->multicast.ib); 308 kfree(mc); 309 } 310 311 static void cma_detach_from_dev(struct rdma_id_private *id_priv) 312 { 313 list_del(&id_priv->list); 314 cma_deref_dev(id_priv->cma_dev); 315 id_priv->cma_dev = NULL; 316 } 317 318 static int cma_set_qkey(struct rdma_id_private *id_priv) 319 { 320 struct ib_sa_mcmember_rec rec; 321 int ret = 0; 322 323 if (id_priv->qkey) 324 return 0; 325 326 switch (id_priv->id.ps) { 327 case RDMA_PS_UDP: 328 id_priv->qkey = RDMA_UDP_QKEY; 329 break; 330 case RDMA_PS_IPOIB: 331 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); 332 ret = ib_sa_get_mcmember_rec(id_priv->id.device, 333 id_priv->id.port_num, &rec.mgid, 334 &rec); 335 if (!ret) 336 id_priv->qkey = be32_to_cpu(rec.qkey); 337 break; 338 default: 339 break; 340 } 341 return ret; 342 } 343 344 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num) 345 { 346 int i; 347 int err; 348 struct ib_port_attr props; 349 union ib_gid tmp; 350 351 err = ib_query_port(device, port_num, &props); 352 if (err) 353 return 1; 354 355 for (i = 0; i < props.gid_tbl_len; ++i) { 356 err = ib_query_gid(device, port_num, i, &tmp); 357 if (err) 358 return 1; 359 if (!memcmp(&tmp, gid, sizeof tmp)) 360 return 0; 361 } 362 363 return -EAGAIN; 364 } 365 366 static int cma_acquire_dev(struct rdma_id_private *id_priv) 367 { 368 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 369 struct cma_device *cma_dev; 370 union ib_gid gid, iboe_gid; 371 int ret = -ENODEV; 372 u8 port; 373 enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ? 374 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET; 375 376 iboe_addr_get_sgid(dev_addr, &iboe_gid); 377 memcpy(&gid, dev_addr->src_dev_addr + 378 rdma_addr_gid_offset(dev_addr), sizeof gid); 379 list_for_each_entry(cma_dev, &dev_list, list) { 380 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) { 381 if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) { 382 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB && 383 rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET) 384 ret = find_gid_port(cma_dev->device, &iboe_gid, port); 385 else 386 ret = find_gid_port(cma_dev->device, &gid, port); 387 388 if (!ret) { 389 id_priv->id.port_num = port; 390 goto out; 391 } else if (ret == 1) 392 break; 393 } 394 } 395 } 396 397 out: 398 if (!ret) 399 cma_attach_to_dev(id_priv, cma_dev); 400 401 return ret; 402 } 403 404 static void cma_deref_id(struct rdma_id_private *id_priv) 405 { 406 if (atomic_dec_and_test(&id_priv->refcount)) 407 complete(&id_priv->comp); 408 } 409 410 static int cma_disable_callback(struct rdma_id_private *id_priv, 411 enum cma_state state) 412 { 413 mutex_lock(&id_priv->handler_mutex); 414 if (id_priv->state != state) { 415 mutex_unlock(&id_priv->handler_mutex); 416 return -EINVAL; 417 } 418 return 0; 419 } 420 421 static int cma_has_cm_dev(struct rdma_id_private *id_priv) 422 { 423 return (id_priv->id.device && id_priv->cm_id.ib); 424 } 425 426 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler, 427 void *context, enum rdma_port_space ps) 428 { 429 struct rdma_id_private *id_priv; 430 431 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 432 if (!id_priv) 433 return ERR_PTR(-ENOMEM); 434 435 id_priv->state = CMA_IDLE; 436 id_priv->id.context = context; 437 id_priv->id.event_handler = event_handler; 438 id_priv->id.ps = ps; 439 spin_lock_init(&id_priv->lock); 440 mutex_init(&id_priv->qp_mutex); 441 init_completion(&id_priv->comp); 442 atomic_set(&id_priv->refcount, 1); 443 mutex_init(&id_priv->handler_mutex); 444 INIT_LIST_HEAD(&id_priv->listen_list); 445 INIT_LIST_HEAD(&id_priv->mc_list); 446 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 447 448 return &id_priv->id; 449 } 450 EXPORT_SYMBOL(rdma_create_id); 451 452 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 453 { 454 struct ib_qp_attr qp_attr; 455 int qp_attr_mask, ret; 456 457 qp_attr.qp_state = IB_QPS_INIT; 458 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 459 if (ret) 460 return ret; 461 462 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 463 if (ret) 464 return ret; 465 466 qp_attr.qp_state = IB_QPS_RTR; 467 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 468 if (ret) 469 return ret; 470 471 qp_attr.qp_state = IB_QPS_RTS; 472 qp_attr.sq_psn = 0; 473 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 474 475 return ret; 476 } 477 478 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 479 { 480 struct ib_qp_attr qp_attr; 481 int qp_attr_mask, ret; 482 483 qp_attr.qp_state = IB_QPS_INIT; 484 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 485 if (ret) 486 return ret; 487 488 return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 489 } 490 491 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 492 struct ib_qp_init_attr *qp_init_attr) 493 { 494 struct rdma_id_private *id_priv; 495 struct ib_qp *qp; 496 int ret; 497 498 id_priv = container_of(id, struct rdma_id_private, id); 499 if (id->device != pd->device) 500 return -EINVAL; 501 502 qp = ib_create_qp(pd, qp_init_attr); 503 if (IS_ERR(qp)) 504 return PTR_ERR(qp); 505 506 if (cma_is_ud_ps(id_priv->id.ps)) 507 ret = cma_init_ud_qp(id_priv, qp); 508 else 509 ret = cma_init_conn_qp(id_priv, qp); 510 if (ret) 511 goto err; 512 513 id->qp = qp; 514 id_priv->qp_num = qp->qp_num; 515 id_priv->srq = (qp->srq != NULL); 516 return 0; 517 err: 518 ib_destroy_qp(qp); 519 return ret; 520 } 521 EXPORT_SYMBOL(rdma_create_qp); 522 523 void rdma_destroy_qp(struct rdma_cm_id *id) 524 { 525 struct rdma_id_private *id_priv; 526 527 id_priv = container_of(id, struct rdma_id_private, id); 528 mutex_lock(&id_priv->qp_mutex); 529 ib_destroy_qp(id_priv->id.qp); 530 id_priv->id.qp = NULL; 531 mutex_unlock(&id_priv->qp_mutex); 532 } 533 EXPORT_SYMBOL(rdma_destroy_qp); 534 535 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, 536 struct rdma_conn_param *conn_param) 537 { 538 struct ib_qp_attr qp_attr; 539 int qp_attr_mask, ret; 540 541 mutex_lock(&id_priv->qp_mutex); 542 if (!id_priv->id.qp) { 543 ret = 0; 544 goto out; 545 } 546 547 /* Need to update QP attributes from default values. */ 548 qp_attr.qp_state = IB_QPS_INIT; 549 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 550 if (ret) 551 goto out; 552 553 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 554 if (ret) 555 goto out; 556 557 qp_attr.qp_state = IB_QPS_RTR; 558 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 559 if (ret) 560 goto out; 561 562 if (conn_param) 563 qp_attr.max_dest_rd_atomic = conn_param->responder_resources; 564 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 565 out: 566 mutex_unlock(&id_priv->qp_mutex); 567 return ret; 568 } 569 570 static int cma_modify_qp_rts(struct rdma_id_private *id_priv, 571 struct rdma_conn_param *conn_param) 572 { 573 struct ib_qp_attr qp_attr; 574 int qp_attr_mask, ret; 575 576 mutex_lock(&id_priv->qp_mutex); 577 if (!id_priv->id.qp) { 578 ret = 0; 579 goto out; 580 } 581 582 qp_attr.qp_state = IB_QPS_RTS; 583 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 584 if (ret) 585 goto out; 586 587 if (conn_param) 588 qp_attr.max_rd_atomic = conn_param->initiator_depth; 589 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 590 out: 591 mutex_unlock(&id_priv->qp_mutex); 592 return ret; 593 } 594 595 static int cma_modify_qp_err(struct rdma_id_private *id_priv) 596 { 597 struct ib_qp_attr qp_attr; 598 int ret; 599 600 mutex_lock(&id_priv->qp_mutex); 601 if (!id_priv->id.qp) { 602 ret = 0; 603 goto out; 604 } 605 606 qp_attr.qp_state = IB_QPS_ERR; 607 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); 608 out: 609 mutex_unlock(&id_priv->qp_mutex); 610 return ret; 611 } 612 613 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 614 struct ib_qp_attr *qp_attr, int *qp_attr_mask) 615 { 616 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 617 int ret; 618 u16 pkey; 619 620 if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) == 621 IB_LINK_LAYER_INFINIBAND) 622 pkey = ib_addr_get_pkey(dev_addr); 623 else 624 pkey = 0xffff; 625 626 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 627 pkey, &qp_attr->pkey_index); 628 if (ret) 629 return ret; 630 631 qp_attr->port_num = id_priv->id.port_num; 632 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 633 634 if (cma_is_ud_ps(id_priv->id.ps)) { 635 ret = cma_set_qkey(id_priv); 636 if (ret) 637 return ret; 638 639 qp_attr->qkey = id_priv->qkey; 640 *qp_attr_mask |= IB_QP_QKEY; 641 } else { 642 qp_attr->qp_access_flags = 0; 643 *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 644 } 645 return 0; 646 } 647 648 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 649 int *qp_attr_mask) 650 { 651 struct rdma_id_private *id_priv; 652 int ret = 0; 653 654 id_priv = container_of(id, struct rdma_id_private, id); 655 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 656 case RDMA_TRANSPORT_IB: 657 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps)) 658 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 659 else 660 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 661 qp_attr_mask); 662 if (qp_attr->qp_state == IB_QPS_RTR) 663 qp_attr->rq_psn = id_priv->seq_num; 664 break; 665 case RDMA_TRANSPORT_IWARP: 666 if (!id_priv->cm_id.iw) { 667 qp_attr->qp_access_flags = 0; 668 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 669 } else 670 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 671 qp_attr_mask); 672 break; 673 default: 674 ret = -ENOSYS; 675 break; 676 } 677 678 return ret; 679 } 680 EXPORT_SYMBOL(rdma_init_qp_attr); 681 682 static inline int cma_zero_addr(struct sockaddr *addr) 683 { 684 struct in6_addr *ip6; 685 686 if (addr->sa_family == AF_INET) 687 return ipv4_is_zeronet( 688 ((struct sockaddr_in *)addr)->sin_addr.s_addr); 689 else { 690 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; 691 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | 692 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; 693 } 694 } 695 696 static inline int cma_loopback_addr(struct sockaddr *addr) 697 { 698 if (addr->sa_family == AF_INET) 699 return ipv4_is_loopback( 700 ((struct sockaddr_in *) addr)->sin_addr.s_addr); 701 else 702 return ipv6_addr_loopback( 703 &((struct sockaddr_in6 *) addr)->sin6_addr); 704 } 705 706 static inline int cma_any_addr(struct sockaddr *addr) 707 { 708 return cma_zero_addr(addr) || cma_loopback_addr(addr); 709 } 710 711 static inline __be16 cma_port(struct sockaddr *addr) 712 { 713 if (addr->sa_family == AF_INET) 714 return ((struct sockaddr_in *) addr)->sin_port; 715 else 716 return ((struct sockaddr_in6 *) addr)->sin6_port; 717 } 718 719 static inline int cma_any_port(struct sockaddr *addr) 720 { 721 return !cma_port(addr); 722 } 723 724 static int cma_get_net_info(void *hdr, enum rdma_port_space ps, 725 u8 *ip_ver, __be16 *port, 726 union cma_ip_addr **src, union cma_ip_addr **dst) 727 { 728 switch (ps) { 729 case RDMA_PS_SDP: 730 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) != 731 SDP_MAJ_VERSION) 732 return -EINVAL; 733 734 *ip_ver = sdp_get_ip_ver(hdr); 735 *port = ((struct sdp_hh *) hdr)->port; 736 *src = &((struct sdp_hh *) hdr)->src_addr; 737 *dst = &((struct sdp_hh *) hdr)->dst_addr; 738 break; 739 default: 740 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION) 741 return -EINVAL; 742 743 *ip_ver = cma_get_ip_ver(hdr); 744 *port = ((struct cma_hdr *) hdr)->port; 745 *src = &((struct cma_hdr *) hdr)->src_addr; 746 *dst = &((struct cma_hdr *) hdr)->dst_addr; 747 break; 748 } 749 750 if (*ip_ver != 4 && *ip_ver != 6) 751 return -EINVAL; 752 return 0; 753 } 754 755 static void cma_save_net_info(struct rdma_addr *addr, 756 struct rdma_addr *listen_addr, 757 u8 ip_ver, __be16 port, 758 union cma_ip_addr *src, union cma_ip_addr *dst) 759 { 760 struct sockaddr_in *listen4, *ip4; 761 struct sockaddr_in6 *listen6, *ip6; 762 763 switch (ip_ver) { 764 case 4: 765 listen4 = (struct sockaddr_in *) &listen_addr->src_addr; 766 ip4 = (struct sockaddr_in *) &addr->src_addr; 767 ip4->sin_family = listen4->sin_family; 768 ip4->sin_addr.s_addr = dst->ip4.addr; 769 ip4->sin_port = listen4->sin_port; 770 771 ip4 = (struct sockaddr_in *) &addr->dst_addr; 772 ip4->sin_family = listen4->sin_family; 773 ip4->sin_addr.s_addr = src->ip4.addr; 774 ip4->sin_port = port; 775 break; 776 case 6: 777 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr; 778 ip6 = (struct sockaddr_in6 *) &addr->src_addr; 779 ip6->sin6_family = listen6->sin6_family; 780 ip6->sin6_addr = dst->ip6; 781 ip6->sin6_port = listen6->sin6_port; 782 783 ip6 = (struct sockaddr_in6 *) &addr->dst_addr; 784 ip6->sin6_family = listen6->sin6_family; 785 ip6->sin6_addr = src->ip6; 786 ip6->sin6_port = port; 787 break; 788 default: 789 break; 790 } 791 } 792 793 static inline int cma_user_data_offset(enum rdma_port_space ps) 794 { 795 switch (ps) { 796 case RDMA_PS_SDP: 797 return 0; 798 default: 799 return sizeof(struct cma_hdr); 800 } 801 } 802 803 static void cma_cancel_route(struct rdma_id_private *id_priv) 804 { 805 switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) { 806 case IB_LINK_LAYER_INFINIBAND: 807 if (id_priv->query) 808 ib_sa_cancel_query(id_priv->query_id, id_priv->query); 809 break; 810 default: 811 break; 812 } 813 } 814 815 static void cma_cancel_listens(struct rdma_id_private *id_priv) 816 { 817 struct rdma_id_private *dev_id_priv; 818 819 /* 820 * Remove from listen_any_list to prevent added devices from spawning 821 * additional listen requests. 822 */ 823 mutex_lock(&lock); 824 list_del(&id_priv->list); 825 826 while (!list_empty(&id_priv->listen_list)) { 827 dev_id_priv = list_entry(id_priv->listen_list.next, 828 struct rdma_id_private, listen_list); 829 /* sync with device removal to avoid duplicate destruction */ 830 list_del_init(&dev_id_priv->list); 831 list_del(&dev_id_priv->listen_list); 832 mutex_unlock(&lock); 833 834 rdma_destroy_id(&dev_id_priv->id); 835 mutex_lock(&lock); 836 } 837 mutex_unlock(&lock); 838 } 839 840 static void cma_cancel_operation(struct rdma_id_private *id_priv, 841 enum cma_state state) 842 { 843 switch (state) { 844 case CMA_ADDR_QUERY: 845 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 846 break; 847 case CMA_ROUTE_QUERY: 848 cma_cancel_route(id_priv); 849 break; 850 case CMA_LISTEN: 851 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr) 852 && !id_priv->cma_dev) 853 cma_cancel_listens(id_priv); 854 break; 855 default: 856 break; 857 } 858 } 859 860 static void cma_release_port(struct rdma_id_private *id_priv) 861 { 862 struct rdma_bind_list *bind_list = id_priv->bind_list; 863 864 if (!bind_list) 865 return; 866 867 mutex_lock(&lock); 868 hlist_del(&id_priv->node); 869 if (hlist_empty(&bind_list->owners)) { 870 idr_remove(bind_list->ps, bind_list->port); 871 kfree(bind_list); 872 } 873 mutex_unlock(&lock); 874 } 875 876 static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 877 { 878 struct cma_multicast *mc; 879 880 while (!list_empty(&id_priv->mc_list)) { 881 mc = container_of(id_priv->mc_list.next, 882 struct cma_multicast, list); 883 list_del(&mc->list); 884 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) { 885 case IB_LINK_LAYER_INFINIBAND: 886 ib_sa_free_multicast(mc->multicast.ib); 887 kfree(mc); 888 break; 889 case IB_LINK_LAYER_ETHERNET: 890 kref_put(&mc->mcref, release_mc); 891 break; 892 default: 893 break; 894 } 895 } 896 } 897 898 void rdma_destroy_id(struct rdma_cm_id *id) 899 { 900 struct rdma_id_private *id_priv; 901 enum cma_state state; 902 903 id_priv = container_of(id, struct rdma_id_private, id); 904 state = cma_exch(id_priv, CMA_DESTROYING); 905 cma_cancel_operation(id_priv, state); 906 907 mutex_lock(&lock); 908 if (id_priv->cma_dev) { 909 mutex_unlock(&lock); 910 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 911 case RDMA_TRANSPORT_IB: 912 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) 913 ib_destroy_cm_id(id_priv->cm_id.ib); 914 break; 915 case RDMA_TRANSPORT_IWARP: 916 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw)) 917 iw_destroy_cm_id(id_priv->cm_id.iw); 918 break; 919 default: 920 break; 921 } 922 cma_leave_mc_groups(id_priv); 923 mutex_lock(&lock); 924 cma_detach_from_dev(id_priv); 925 } 926 mutex_unlock(&lock); 927 928 cma_release_port(id_priv); 929 cma_deref_id(id_priv); 930 wait_for_completion(&id_priv->comp); 931 932 if (id_priv->internal_id) 933 cma_deref_id(id_priv->id.context); 934 935 kfree(id_priv->id.route.path_rec); 936 kfree(id_priv); 937 } 938 EXPORT_SYMBOL(rdma_destroy_id); 939 940 static int cma_rep_recv(struct rdma_id_private *id_priv) 941 { 942 int ret; 943 944 ret = cma_modify_qp_rtr(id_priv, NULL); 945 if (ret) 946 goto reject; 947 948 ret = cma_modify_qp_rts(id_priv, NULL); 949 if (ret) 950 goto reject; 951 952 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 953 if (ret) 954 goto reject; 955 956 return 0; 957 reject: 958 cma_modify_qp_err(id_priv); 959 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 960 NULL, 0, NULL, 0); 961 return ret; 962 } 963 964 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data) 965 { 966 if (id_priv->id.ps == RDMA_PS_SDP && 967 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) != 968 SDP_MAJ_VERSION) 969 return -EINVAL; 970 971 return 0; 972 } 973 974 static void cma_set_rep_event_data(struct rdma_cm_event *event, 975 struct ib_cm_rep_event_param *rep_data, 976 void *private_data) 977 { 978 event->param.conn.private_data = private_data; 979 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 980 event->param.conn.responder_resources = rep_data->responder_resources; 981 event->param.conn.initiator_depth = rep_data->initiator_depth; 982 event->param.conn.flow_control = rep_data->flow_control; 983 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 984 event->param.conn.srq = rep_data->srq; 985 event->param.conn.qp_num = rep_data->remote_qpn; 986 } 987 988 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 989 { 990 struct rdma_id_private *id_priv = cm_id->context; 991 struct rdma_cm_event event; 992 int ret = 0; 993 994 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && 995 cma_disable_callback(id_priv, CMA_CONNECT)) || 996 (ib_event->event == IB_CM_TIMEWAIT_EXIT && 997 cma_disable_callback(id_priv, CMA_DISCONNECT))) 998 return 0; 999 1000 memset(&event, 0, sizeof event); 1001 switch (ib_event->event) { 1002 case IB_CM_REQ_ERROR: 1003 case IB_CM_REP_ERROR: 1004 event.event = RDMA_CM_EVENT_UNREACHABLE; 1005 event.status = -ETIMEDOUT; 1006 break; 1007 case IB_CM_REP_RECEIVED: 1008 event.status = cma_verify_rep(id_priv, ib_event->private_data); 1009 if (event.status) 1010 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1011 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) { 1012 event.status = cma_rep_recv(id_priv); 1013 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 1014 RDMA_CM_EVENT_ESTABLISHED; 1015 } else 1016 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 1017 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 1018 ib_event->private_data); 1019 break; 1020 case IB_CM_RTU_RECEIVED: 1021 case IB_CM_USER_ESTABLISHED: 1022 event.event = RDMA_CM_EVENT_ESTABLISHED; 1023 break; 1024 case IB_CM_DREQ_ERROR: 1025 event.status = -ETIMEDOUT; /* fall through */ 1026 case IB_CM_DREQ_RECEIVED: 1027 case IB_CM_DREP_RECEIVED: 1028 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT)) 1029 goto out; 1030 event.event = RDMA_CM_EVENT_DISCONNECTED; 1031 break; 1032 case IB_CM_TIMEWAIT_EXIT: 1033 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; 1034 break; 1035 case IB_CM_MRA_RECEIVED: 1036 /* ignore event */ 1037 goto out; 1038 case IB_CM_REJ_RECEIVED: 1039 cma_modify_qp_err(id_priv); 1040 event.status = ib_event->param.rej_rcvd.reason; 1041 event.event = RDMA_CM_EVENT_REJECTED; 1042 event.param.conn.private_data = ib_event->private_data; 1043 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 1044 break; 1045 default: 1046 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", 1047 ib_event->event); 1048 goto out; 1049 } 1050 1051 ret = id_priv->id.event_handler(&id_priv->id, &event); 1052 if (ret) { 1053 /* Destroy the CM ID by returning a non-zero value. */ 1054 id_priv->cm_id.ib = NULL; 1055 cma_exch(id_priv, CMA_DESTROYING); 1056 mutex_unlock(&id_priv->handler_mutex); 1057 rdma_destroy_id(&id_priv->id); 1058 return ret; 1059 } 1060 out: 1061 mutex_unlock(&id_priv->handler_mutex); 1062 return ret; 1063 } 1064 1065 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, 1066 struct ib_cm_event *ib_event) 1067 { 1068 struct rdma_id_private *id_priv; 1069 struct rdma_cm_id *id; 1070 struct rdma_route *rt; 1071 union cma_ip_addr *src, *dst; 1072 __be16 port; 1073 u8 ip_ver; 1074 int ret; 1075 1076 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 1077 &ip_ver, &port, &src, &dst)) 1078 goto err; 1079 1080 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1081 listen_id->ps); 1082 if (IS_ERR(id)) 1083 goto err; 1084 1085 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1086 ip_ver, port, src, dst); 1087 1088 rt = &id->route; 1089 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 1090 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, 1091 GFP_KERNEL); 1092 if (!rt->path_rec) 1093 goto destroy_id; 1094 1095 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; 1096 if (rt->num_paths == 2) 1097 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 1098 1099 if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) { 1100 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; 1101 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 1102 ib_addr_set_pkey(&rt->addr.dev_addr, rt->path_rec[0].pkey); 1103 } else { 1104 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr, 1105 &rt->addr.dev_addr); 1106 if (ret) 1107 goto destroy_id; 1108 } 1109 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 1110 1111 id_priv = container_of(id, struct rdma_id_private, id); 1112 id_priv->state = CMA_CONNECT; 1113 return id_priv; 1114 1115 destroy_id: 1116 rdma_destroy_id(id); 1117 err: 1118 return NULL; 1119 } 1120 1121 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, 1122 struct ib_cm_event *ib_event) 1123 { 1124 struct rdma_id_private *id_priv; 1125 struct rdma_cm_id *id; 1126 union cma_ip_addr *src, *dst; 1127 __be16 port; 1128 u8 ip_ver; 1129 int ret; 1130 1131 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1132 listen_id->ps); 1133 if (IS_ERR(id)) 1134 return NULL; 1135 1136 1137 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 1138 &ip_ver, &port, &src, &dst)) 1139 goto err; 1140 1141 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1142 ip_ver, port, src, dst); 1143 1144 if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) { 1145 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr, 1146 &id->route.addr.dev_addr); 1147 if (ret) 1148 goto err; 1149 } 1150 1151 id_priv = container_of(id, struct rdma_id_private, id); 1152 id_priv->state = CMA_CONNECT; 1153 return id_priv; 1154 err: 1155 rdma_destroy_id(id); 1156 return NULL; 1157 } 1158 1159 static void cma_set_req_event_data(struct rdma_cm_event *event, 1160 struct ib_cm_req_event_param *req_data, 1161 void *private_data, int offset) 1162 { 1163 event->param.conn.private_data = private_data + offset; 1164 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 1165 event->param.conn.responder_resources = req_data->responder_resources; 1166 event->param.conn.initiator_depth = req_data->initiator_depth; 1167 event->param.conn.flow_control = req_data->flow_control; 1168 event->param.conn.retry_count = req_data->retry_count; 1169 event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 1170 event->param.conn.srq = req_data->srq; 1171 event->param.conn.qp_num = req_data->remote_qpn; 1172 } 1173 1174 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1175 { 1176 struct rdma_id_private *listen_id, *conn_id; 1177 struct rdma_cm_event event; 1178 int offset, ret; 1179 1180 listen_id = cm_id->context; 1181 if (cma_disable_callback(listen_id, CMA_LISTEN)) 1182 return -ECONNABORTED; 1183 1184 memset(&event, 0, sizeof event); 1185 offset = cma_user_data_offset(listen_id->id.ps); 1186 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1187 if (cma_is_ud_ps(listen_id->id.ps)) { 1188 conn_id = cma_new_udp_id(&listen_id->id, ib_event); 1189 event.param.ud.private_data = ib_event->private_data + offset; 1190 event.param.ud.private_data_len = 1191 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 1192 } else { 1193 conn_id = cma_new_conn_id(&listen_id->id, ib_event); 1194 cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 1195 ib_event->private_data, offset); 1196 } 1197 if (!conn_id) { 1198 ret = -ENOMEM; 1199 goto out; 1200 } 1201 1202 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1203 mutex_lock(&lock); 1204 ret = cma_acquire_dev(conn_id); 1205 mutex_unlock(&lock); 1206 if (ret) 1207 goto release_conn_id; 1208 1209 conn_id->cm_id.ib = cm_id; 1210 cm_id->context = conn_id; 1211 cm_id->cm_handler = cma_ib_handler; 1212 1213 ret = conn_id->id.event_handler(&conn_id->id, &event); 1214 if (!ret) { 1215 /* 1216 * Acquire mutex to prevent user executing rdma_destroy_id() 1217 * while we're accessing the cm_id. 1218 */ 1219 mutex_lock(&lock); 1220 if (cma_comp(conn_id, CMA_CONNECT) && 1221 !cma_is_ud_ps(conn_id->id.ps)) 1222 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 1223 mutex_unlock(&lock); 1224 mutex_unlock(&conn_id->handler_mutex); 1225 goto out; 1226 } 1227 1228 /* Destroy the CM ID by returning a non-zero value. */ 1229 conn_id->cm_id.ib = NULL; 1230 1231 release_conn_id: 1232 cma_exch(conn_id, CMA_DESTROYING); 1233 mutex_unlock(&conn_id->handler_mutex); 1234 rdma_destroy_id(&conn_id->id); 1235 1236 out: 1237 mutex_unlock(&listen_id->handler_mutex); 1238 return ret; 1239 } 1240 1241 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr) 1242 { 1243 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr))); 1244 } 1245 1246 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr, 1247 struct ib_cm_compare_data *compare) 1248 { 1249 struct cma_hdr *cma_data, *cma_mask; 1250 struct sdp_hh *sdp_data, *sdp_mask; 1251 __be32 ip4_addr; 1252 struct in6_addr ip6_addr; 1253 1254 memset(compare, 0, sizeof *compare); 1255 cma_data = (void *) compare->data; 1256 cma_mask = (void *) compare->mask; 1257 sdp_data = (void *) compare->data; 1258 sdp_mask = (void *) compare->mask; 1259 1260 switch (addr->sa_family) { 1261 case AF_INET: 1262 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr; 1263 if (ps == RDMA_PS_SDP) { 1264 sdp_set_ip_ver(sdp_data, 4); 1265 sdp_set_ip_ver(sdp_mask, 0xF); 1266 sdp_data->dst_addr.ip4.addr = ip4_addr; 1267 sdp_mask->dst_addr.ip4.addr = htonl(~0); 1268 } else { 1269 cma_set_ip_ver(cma_data, 4); 1270 cma_set_ip_ver(cma_mask, 0xF); 1271 cma_data->dst_addr.ip4.addr = ip4_addr; 1272 cma_mask->dst_addr.ip4.addr = htonl(~0); 1273 } 1274 break; 1275 case AF_INET6: 1276 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr; 1277 if (ps == RDMA_PS_SDP) { 1278 sdp_set_ip_ver(sdp_data, 6); 1279 sdp_set_ip_ver(sdp_mask, 0xF); 1280 sdp_data->dst_addr.ip6 = ip6_addr; 1281 memset(&sdp_mask->dst_addr.ip6, 0xFF, 1282 sizeof sdp_mask->dst_addr.ip6); 1283 } else { 1284 cma_set_ip_ver(cma_data, 6); 1285 cma_set_ip_ver(cma_mask, 0xF); 1286 cma_data->dst_addr.ip6 = ip6_addr; 1287 memset(&cma_mask->dst_addr.ip6, 0xFF, 1288 sizeof cma_mask->dst_addr.ip6); 1289 } 1290 break; 1291 default: 1292 break; 1293 } 1294 } 1295 1296 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 1297 { 1298 struct rdma_id_private *id_priv = iw_id->context; 1299 struct rdma_cm_event event; 1300 struct sockaddr_in *sin; 1301 int ret = 0; 1302 1303 if (cma_disable_callback(id_priv, CMA_CONNECT)) 1304 return 0; 1305 1306 memset(&event, 0, sizeof event); 1307 switch (iw_event->event) { 1308 case IW_CM_EVENT_CLOSE: 1309 event.event = RDMA_CM_EVENT_DISCONNECTED; 1310 break; 1311 case IW_CM_EVENT_CONNECT_REPLY: 1312 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1313 *sin = iw_event->local_addr; 1314 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr; 1315 *sin = iw_event->remote_addr; 1316 switch (iw_event->status) { 1317 case 0: 1318 event.event = RDMA_CM_EVENT_ESTABLISHED; 1319 break; 1320 case -ECONNRESET: 1321 case -ECONNREFUSED: 1322 event.event = RDMA_CM_EVENT_REJECTED; 1323 break; 1324 case -ETIMEDOUT: 1325 event.event = RDMA_CM_EVENT_UNREACHABLE; 1326 break; 1327 default: 1328 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1329 break; 1330 } 1331 break; 1332 case IW_CM_EVENT_ESTABLISHED: 1333 event.event = RDMA_CM_EVENT_ESTABLISHED; 1334 break; 1335 default: 1336 BUG_ON(1); 1337 } 1338 1339 event.status = iw_event->status; 1340 event.param.conn.private_data = iw_event->private_data; 1341 event.param.conn.private_data_len = iw_event->private_data_len; 1342 ret = id_priv->id.event_handler(&id_priv->id, &event); 1343 if (ret) { 1344 /* Destroy the CM ID by returning a non-zero value. */ 1345 id_priv->cm_id.iw = NULL; 1346 cma_exch(id_priv, CMA_DESTROYING); 1347 mutex_unlock(&id_priv->handler_mutex); 1348 rdma_destroy_id(&id_priv->id); 1349 return ret; 1350 } 1351 1352 mutex_unlock(&id_priv->handler_mutex); 1353 return ret; 1354 } 1355 1356 static int iw_conn_req_handler(struct iw_cm_id *cm_id, 1357 struct iw_cm_event *iw_event) 1358 { 1359 struct rdma_cm_id *new_cm_id; 1360 struct rdma_id_private *listen_id, *conn_id; 1361 struct sockaddr_in *sin; 1362 struct net_device *dev = NULL; 1363 struct rdma_cm_event event; 1364 int ret; 1365 struct ib_device_attr attr; 1366 1367 listen_id = cm_id->context; 1368 if (cma_disable_callback(listen_id, CMA_LISTEN)) 1369 return -ECONNABORTED; 1370 1371 /* Create a new RDMA id for the new IW CM ID */ 1372 new_cm_id = rdma_create_id(listen_id->id.event_handler, 1373 listen_id->id.context, 1374 RDMA_PS_TCP); 1375 if (IS_ERR(new_cm_id)) { 1376 ret = -ENOMEM; 1377 goto out; 1378 } 1379 conn_id = container_of(new_cm_id, struct rdma_id_private, id); 1380 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1381 conn_id->state = CMA_CONNECT; 1382 1383 dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr); 1384 if (!dev) { 1385 ret = -EADDRNOTAVAIL; 1386 mutex_unlock(&conn_id->handler_mutex); 1387 rdma_destroy_id(new_cm_id); 1388 goto out; 1389 } 1390 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL); 1391 if (ret) { 1392 mutex_unlock(&conn_id->handler_mutex); 1393 rdma_destroy_id(new_cm_id); 1394 goto out; 1395 } 1396 1397 mutex_lock(&lock); 1398 ret = cma_acquire_dev(conn_id); 1399 mutex_unlock(&lock); 1400 if (ret) { 1401 mutex_unlock(&conn_id->handler_mutex); 1402 rdma_destroy_id(new_cm_id); 1403 goto out; 1404 } 1405 1406 conn_id->cm_id.iw = cm_id; 1407 cm_id->context = conn_id; 1408 cm_id->cm_handler = cma_iw_handler; 1409 1410 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr; 1411 *sin = iw_event->local_addr; 1412 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr; 1413 *sin = iw_event->remote_addr; 1414 1415 ret = ib_query_device(conn_id->id.device, &attr); 1416 if (ret) { 1417 mutex_unlock(&conn_id->handler_mutex); 1418 rdma_destroy_id(new_cm_id); 1419 goto out; 1420 } 1421 1422 memset(&event, 0, sizeof event); 1423 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1424 event.param.conn.private_data = iw_event->private_data; 1425 event.param.conn.private_data_len = iw_event->private_data_len; 1426 event.param.conn.initiator_depth = attr.max_qp_init_rd_atom; 1427 event.param.conn.responder_resources = attr.max_qp_rd_atom; 1428 ret = conn_id->id.event_handler(&conn_id->id, &event); 1429 if (ret) { 1430 /* User wants to destroy the CM ID */ 1431 conn_id->cm_id.iw = NULL; 1432 cma_exch(conn_id, CMA_DESTROYING); 1433 mutex_unlock(&conn_id->handler_mutex); 1434 rdma_destroy_id(&conn_id->id); 1435 goto out; 1436 } 1437 1438 mutex_unlock(&conn_id->handler_mutex); 1439 1440 out: 1441 if (dev) 1442 dev_put(dev); 1443 mutex_unlock(&listen_id->handler_mutex); 1444 return ret; 1445 } 1446 1447 static int cma_ib_listen(struct rdma_id_private *id_priv) 1448 { 1449 struct ib_cm_compare_data compare_data; 1450 struct sockaddr *addr; 1451 __be64 svc_id; 1452 int ret; 1453 1454 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler, 1455 id_priv); 1456 if (IS_ERR(id_priv->cm_id.ib)) 1457 return PTR_ERR(id_priv->cm_id.ib); 1458 1459 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr; 1460 svc_id = cma_get_service_id(id_priv->id.ps, addr); 1461 if (cma_any_addr(addr)) 1462 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL); 1463 else { 1464 cma_set_compare_data(id_priv->id.ps, addr, &compare_data); 1465 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data); 1466 } 1467 1468 if (ret) { 1469 ib_destroy_cm_id(id_priv->cm_id.ib); 1470 id_priv->cm_id.ib = NULL; 1471 } 1472 1473 return ret; 1474 } 1475 1476 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 1477 { 1478 int ret; 1479 struct sockaddr_in *sin; 1480 1481 id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, 1482 iw_conn_req_handler, 1483 id_priv); 1484 if (IS_ERR(id_priv->cm_id.iw)) 1485 return PTR_ERR(id_priv->cm_id.iw); 1486 1487 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1488 id_priv->cm_id.iw->local_addr = *sin; 1489 1490 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 1491 1492 if (ret) { 1493 iw_destroy_cm_id(id_priv->cm_id.iw); 1494 id_priv->cm_id.iw = NULL; 1495 } 1496 1497 return ret; 1498 } 1499 1500 static int cma_listen_handler(struct rdma_cm_id *id, 1501 struct rdma_cm_event *event) 1502 { 1503 struct rdma_id_private *id_priv = id->context; 1504 1505 id->context = id_priv->id.context; 1506 id->event_handler = id_priv->id.event_handler; 1507 return id_priv->id.event_handler(id, event); 1508 } 1509 1510 static void cma_listen_on_dev(struct rdma_id_private *id_priv, 1511 struct cma_device *cma_dev) 1512 { 1513 struct rdma_id_private *dev_id_priv; 1514 struct rdma_cm_id *id; 1515 int ret; 1516 1517 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps); 1518 if (IS_ERR(id)) 1519 return; 1520 1521 dev_id_priv = container_of(id, struct rdma_id_private, id); 1522 1523 dev_id_priv->state = CMA_ADDR_BOUND; 1524 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr, 1525 ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr)); 1526 1527 cma_attach_to_dev(dev_id_priv, cma_dev); 1528 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); 1529 atomic_inc(&id_priv->refcount); 1530 dev_id_priv->internal_id = 1; 1531 1532 ret = rdma_listen(id, id_priv->backlog); 1533 if (ret) 1534 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, " 1535 "listening on device %s\n", ret, cma_dev->device->name); 1536 } 1537 1538 static void cma_listen_on_all(struct rdma_id_private *id_priv) 1539 { 1540 struct cma_device *cma_dev; 1541 1542 mutex_lock(&lock); 1543 list_add_tail(&id_priv->list, &listen_any_list); 1544 list_for_each_entry(cma_dev, &dev_list, list) 1545 cma_listen_on_dev(id_priv, cma_dev); 1546 mutex_unlock(&lock); 1547 } 1548 1549 int rdma_listen(struct rdma_cm_id *id, int backlog) 1550 { 1551 struct rdma_id_private *id_priv; 1552 int ret; 1553 1554 id_priv = container_of(id, struct rdma_id_private, id); 1555 if (id_priv->state == CMA_IDLE) { 1556 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET; 1557 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr); 1558 if (ret) 1559 return ret; 1560 } 1561 1562 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN)) 1563 return -EINVAL; 1564 1565 id_priv->backlog = backlog; 1566 if (id->device) { 1567 switch (rdma_node_get_transport(id->device->node_type)) { 1568 case RDMA_TRANSPORT_IB: 1569 ret = cma_ib_listen(id_priv); 1570 if (ret) 1571 goto err; 1572 break; 1573 case RDMA_TRANSPORT_IWARP: 1574 ret = cma_iw_listen(id_priv, backlog); 1575 if (ret) 1576 goto err; 1577 break; 1578 default: 1579 ret = -ENOSYS; 1580 goto err; 1581 } 1582 } else 1583 cma_listen_on_all(id_priv); 1584 1585 return 0; 1586 err: 1587 id_priv->backlog = 0; 1588 cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND); 1589 return ret; 1590 } 1591 EXPORT_SYMBOL(rdma_listen); 1592 1593 void rdma_set_service_type(struct rdma_cm_id *id, int tos) 1594 { 1595 struct rdma_id_private *id_priv; 1596 1597 id_priv = container_of(id, struct rdma_id_private, id); 1598 id_priv->tos = (u8) tos; 1599 } 1600 EXPORT_SYMBOL(rdma_set_service_type); 1601 1602 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, 1603 void *context) 1604 { 1605 struct cma_work *work = context; 1606 struct rdma_route *route; 1607 1608 route = &work->id->id.route; 1609 1610 if (!status) { 1611 route->num_paths = 1; 1612 *route->path_rec = *path_rec; 1613 } else { 1614 work->old_state = CMA_ROUTE_QUERY; 1615 work->new_state = CMA_ADDR_RESOLVED; 1616 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 1617 work->event.status = status; 1618 } 1619 1620 queue_work(cma_wq, &work->work); 1621 } 1622 1623 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, 1624 struct cma_work *work) 1625 { 1626 struct rdma_addr *addr = &id_priv->id.route.addr; 1627 struct ib_sa_path_rec path_rec; 1628 ib_sa_comp_mask comp_mask; 1629 struct sockaddr_in6 *sin6; 1630 1631 memset(&path_rec, 0, sizeof path_rec); 1632 rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid); 1633 rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid); 1634 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr)); 1635 path_rec.numb_path = 1; 1636 path_rec.reversible = 1; 1637 path_rec.service_id = cma_get_service_id(id_priv->id.ps, 1638 (struct sockaddr *) &addr->dst_addr); 1639 1640 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 1641 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 1642 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; 1643 1644 if (addr->src_addr.ss_family == AF_INET) { 1645 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); 1646 comp_mask |= IB_SA_PATH_REC_QOS_CLASS; 1647 } else { 1648 sin6 = (struct sockaddr_in6 *) &addr->src_addr; 1649 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); 1650 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 1651 } 1652 1653 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 1654 id_priv->id.port_num, &path_rec, 1655 comp_mask, timeout_ms, 1656 GFP_KERNEL, cma_query_handler, 1657 work, &id_priv->query); 1658 1659 return (id_priv->query_id < 0) ? id_priv->query_id : 0; 1660 } 1661 1662 static void cma_work_handler(struct work_struct *_work) 1663 { 1664 struct cma_work *work = container_of(_work, struct cma_work, work); 1665 struct rdma_id_private *id_priv = work->id; 1666 int destroy = 0; 1667 1668 mutex_lock(&id_priv->handler_mutex); 1669 if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 1670 goto out; 1671 1672 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 1673 cma_exch(id_priv, CMA_DESTROYING); 1674 destroy = 1; 1675 } 1676 out: 1677 mutex_unlock(&id_priv->handler_mutex); 1678 cma_deref_id(id_priv); 1679 if (destroy) 1680 rdma_destroy_id(&id_priv->id); 1681 kfree(work); 1682 } 1683 1684 static void cma_ndev_work_handler(struct work_struct *_work) 1685 { 1686 struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); 1687 struct rdma_id_private *id_priv = work->id; 1688 int destroy = 0; 1689 1690 mutex_lock(&id_priv->handler_mutex); 1691 if (id_priv->state == CMA_DESTROYING || 1692 id_priv->state == CMA_DEVICE_REMOVAL) 1693 goto out; 1694 1695 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 1696 cma_exch(id_priv, CMA_DESTROYING); 1697 destroy = 1; 1698 } 1699 1700 out: 1701 mutex_unlock(&id_priv->handler_mutex); 1702 cma_deref_id(id_priv); 1703 if (destroy) 1704 rdma_destroy_id(&id_priv->id); 1705 kfree(work); 1706 } 1707 1708 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) 1709 { 1710 struct rdma_route *route = &id_priv->id.route; 1711 struct cma_work *work; 1712 int ret; 1713 1714 work = kzalloc(sizeof *work, GFP_KERNEL); 1715 if (!work) 1716 return -ENOMEM; 1717 1718 work->id = id_priv; 1719 INIT_WORK(&work->work, cma_work_handler); 1720 work->old_state = CMA_ROUTE_QUERY; 1721 work->new_state = CMA_ROUTE_RESOLVED; 1722 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1723 1724 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 1725 if (!route->path_rec) { 1726 ret = -ENOMEM; 1727 goto err1; 1728 } 1729 1730 ret = cma_query_ib_route(id_priv, timeout_ms, work); 1731 if (ret) 1732 goto err2; 1733 1734 return 0; 1735 err2: 1736 kfree(route->path_rec); 1737 route->path_rec = NULL; 1738 err1: 1739 kfree(work); 1740 return ret; 1741 } 1742 1743 int rdma_set_ib_paths(struct rdma_cm_id *id, 1744 struct ib_sa_path_rec *path_rec, int num_paths) 1745 { 1746 struct rdma_id_private *id_priv; 1747 int ret; 1748 1749 id_priv = container_of(id, struct rdma_id_private, id); 1750 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED)) 1751 return -EINVAL; 1752 1753 id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths, 1754 GFP_KERNEL); 1755 if (!id->route.path_rec) { 1756 ret = -ENOMEM; 1757 goto err; 1758 } 1759 1760 id->route.num_paths = num_paths; 1761 return 0; 1762 err: 1763 cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED); 1764 return ret; 1765 } 1766 EXPORT_SYMBOL(rdma_set_ib_paths); 1767 1768 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) 1769 { 1770 struct cma_work *work; 1771 1772 work = kzalloc(sizeof *work, GFP_KERNEL); 1773 if (!work) 1774 return -ENOMEM; 1775 1776 work->id = id_priv; 1777 INIT_WORK(&work->work, cma_work_handler); 1778 work->old_state = CMA_ROUTE_QUERY; 1779 work->new_state = CMA_ROUTE_RESOLVED; 1780 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1781 queue_work(cma_wq, &work->work); 1782 return 0; 1783 } 1784 1785 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) 1786 { 1787 struct rdma_route *route = &id_priv->id.route; 1788 struct rdma_addr *addr = &route->addr; 1789 struct cma_work *work; 1790 int ret; 1791 struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr; 1792 struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr; 1793 struct net_device *ndev = NULL; 1794 u16 vid; 1795 1796 if (src_addr->sin_family != dst_addr->sin_family) 1797 return -EINVAL; 1798 1799 work = kzalloc(sizeof *work, GFP_KERNEL); 1800 if (!work) 1801 return -ENOMEM; 1802 1803 work->id = id_priv; 1804 INIT_WORK(&work->work, cma_work_handler); 1805 1806 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); 1807 if (!route->path_rec) { 1808 ret = -ENOMEM; 1809 goto err1; 1810 } 1811 1812 route->num_paths = 1; 1813 1814 if (addr->dev_addr.bound_dev_if) 1815 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if); 1816 if (!ndev) { 1817 ret = -ENODEV; 1818 goto err2; 1819 } 1820 1821 vid = rdma_vlan_dev_vlan_id(ndev); 1822 1823 iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid); 1824 iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid); 1825 1826 route->path_rec->hop_limit = 1; 1827 route->path_rec->reversible = 1; 1828 route->path_rec->pkey = cpu_to_be16(0xffff); 1829 route->path_rec->mtu_selector = IB_SA_EQ; 1830 route->path_rec->sl = id_priv->tos >> 5; 1831 1832 route->path_rec->mtu = iboe_get_mtu(ndev->mtu); 1833 route->path_rec->rate_selector = IB_SA_EQ; 1834 route->path_rec->rate = iboe_get_rate(ndev); 1835 dev_put(ndev); 1836 route->path_rec->packet_life_time_selector = IB_SA_EQ; 1837 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME; 1838 if (!route->path_rec->mtu) { 1839 ret = -EINVAL; 1840 goto err2; 1841 } 1842 1843 work->old_state = CMA_ROUTE_QUERY; 1844 work->new_state = CMA_ROUTE_RESOLVED; 1845 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1846 work->event.status = 0; 1847 1848 queue_work(cma_wq, &work->work); 1849 1850 return 0; 1851 1852 err2: 1853 kfree(route->path_rec); 1854 route->path_rec = NULL; 1855 err1: 1856 kfree(work); 1857 return ret; 1858 } 1859 1860 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) 1861 { 1862 struct rdma_id_private *id_priv; 1863 int ret; 1864 1865 id_priv = container_of(id, struct rdma_id_private, id); 1866 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY)) 1867 return -EINVAL; 1868 1869 atomic_inc(&id_priv->refcount); 1870 switch (rdma_node_get_transport(id->device->node_type)) { 1871 case RDMA_TRANSPORT_IB: 1872 switch (rdma_port_get_link_layer(id->device, id->port_num)) { 1873 case IB_LINK_LAYER_INFINIBAND: 1874 ret = cma_resolve_ib_route(id_priv, timeout_ms); 1875 break; 1876 case IB_LINK_LAYER_ETHERNET: 1877 ret = cma_resolve_iboe_route(id_priv); 1878 break; 1879 default: 1880 ret = -ENOSYS; 1881 } 1882 break; 1883 case RDMA_TRANSPORT_IWARP: 1884 ret = cma_resolve_iw_route(id_priv, timeout_ms); 1885 break; 1886 default: 1887 ret = -ENOSYS; 1888 break; 1889 } 1890 if (ret) 1891 goto err; 1892 1893 return 0; 1894 err: 1895 cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED); 1896 cma_deref_id(id_priv); 1897 return ret; 1898 } 1899 EXPORT_SYMBOL(rdma_resolve_route); 1900 1901 static int cma_bind_loopback(struct rdma_id_private *id_priv) 1902 { 1903 struct cma_device *cma_dev; 1904 struct ib_port_attr port_attr; 1905 union ib_gid gid; 1906 u16 pkey; 1907 int ret; 1908 u8 p; 1909 1910 mutex_lock(&lock); 1911 if (list_empty(&dev_list)) { 1912 ret = -ENODEV; 1913 goto out; 1914 } 1915 list_for_each_entry(cma_dev, &dev_list, list) 1916 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p) 1917 if (!ib_query_port(cma_dev->device, p, &port_attr) && 1918 port_attr.state == IB_PORT_ACTIVE) 1919 goto port_found; 1920 1921 p = 1; 1922 cma_dev = list_entry(dev_list.next, struct cma_device, list); 1923 1924 port_found: 1925 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid); 1926 if (ret) 1927 goto out; 1928 1929 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 1930 if (ret) 1931 goto out; 1932 1933 id_priv->id.route.addr.dev_addr.dev_type = 1934 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ? 1935 ARPHRD_INFINIBAND : ARPHRD_ETHER; 1936 1937 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 1938 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 1939 id_priv->id.port_num = p; 1940 cma_attach_to_dev(id_priv, cma_dev); 1941 out: 1942 mutex_unlock(&lock); 1943 return ret; 1944 } 1945 1946 static void addr_handler(int status, struct sockaddr *src_addr, 1947 struct rdma_dev_addr *dev_addr, void *context) 1948 { 1949 struct rdma_id_private *id_priv = context; 1950 struct rdma_cm_event event; 1951 1952 memset(&event, 0, sizeof event); 1953 mutex_lock(&id_priv->handler_mutex); 1954 1955 /* 1956 * Grab mutex to block rdma_destroy_id() from removing the device while 1957 * we're trying to acquire it. 1958 */ 1959 mutex_lock(&lock); 1960 if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) { 1961 mutex_unlock(&lock); 1962 goto out; 1963 } 1964 1965 if (!status && !id_priv->cma_dev) 1966 status = cma_acquire_dev(id_priv); 1967 mutex_unlock(&lock); 1968 1969 if (status) { 1970 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND)) 1971 goto out; 1972 event.event = RDMA_CM_EVENT_ADDR_ERROR; 1973 event.status = status; 1974 } else { 1975 memcpy(&id_priv->id.route.addr.src_addr, src_addr, 1976 ip_addr_size(src_addr)); 1977 event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 1978 } 1979 1980 if (id_priv->id.event_handler(&id_priv->id, &event)) { 1981 cma_exch(id_priv, CMA_DESTROYING); 1982 mutex_unlock(&id_priv->handler_mutex); 1983 cma_deref_id(id_priv); 1984 rdma_destroy_id(&id_priv->id); 1985 return; 1986 } 1987 out: 1988 mutex_unlock(&id_priv->handler_mutex); 1989 cma_deref_id(id_priv); 1990 } 1991 1992 static int cma_resolve_loopback(struct rdma_id_private *id_priv) 1993 { 1994 struct cma_work *work; 1995 struct sockaddr *src, *dst; 1996 union ib_gid gid; 1997 int ret; 1998 1999 work = kzalloc(sizeof *work, GFP_KERNEL); 2000 if (!work) 2001 return -ENOMEM; 2002 2003 if (!id_priv->cma_dev) { 2004 ret = cma_bind_loopback(id_priv); 2005 if (ret) 2006 goto err; 2007 } 2008 2009 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 2010 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 2011 2012 src = (struct sockaddr *) &id_priv->id.route.addr.src_addr; 2013 if (cma_zero_addr(src)) { 2014 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr; 2015 if ((src->sa_family = dst->sa_family) == AF_INET) { 2016 ((struct sockaddr_in *) src)->sin_addr.s_addr = 2017 ((struct sockaddr_in *) dst)->sin_addr.s_addr; 2018 } else { 2019 ipv6_addr_copy(&((struct sockaddr_in6 *) src)->sin6_addr, 2020 &((struct sockaddr_in6 *) dst)->sin6_addr); 2021 } 2022 } 2023 2024 work->id = id_priv; 2025 INIT_WORK(&work->work, cma_work_handler); 2026 work->old_state = CMA_ADDR_QUERY; 2027 work->new_state = CMA_ADDR_RESOLVED; 2028 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2029 queue_work(cma_wq, &work->work); 2030 return 0; 2031 err: 2032 kfree(work); 2033 return ret; 2034 } 2035 2036 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 2037 struct sockaddr *dst_addr) 2038 { 2039 if (!src_addr || !src_addr->sa_family) { 2040 src_addr = (struct sockaddr *) &id->route.addr.src_addr; 2041 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) { 2042 ((struct sockaddr_in6 *) src_addr)->sin6_scope_id = 2043 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id; 2044 } 2045 } 2046 return rdma_bind_addr(id, src_addr); 2047 } 2048 2049 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 2050 struct sockaddr *dst_addr, int timeout_ms) 2051 { 2052 struct rdma_id_private *id_priv; 2053 int ret; 2054 2055 id_priv = container_of(id, struct rdma_id_private, id); 2056 if (id_priv->state == CMA_IDLE) { 2057 ret = cma_bind_addr(id, src_addr, dst_addr); 2058 if (ret) 2059 return ret; 2060 } 2061 2062 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY)) 2063 return -EINVAL; 2064 2065 atomic_inc(&id_priv->refcount); 2066 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr)); 2067 if (cma_any_addr(dst_addr)) 2068 ret = cma_resolve_loopback(id_priv); 2069 else 2070 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr, 2071 dst_addr, &id->route.addr.dev_addr, 2072 timeout_ms, addr_handler, id_priv); 2073 if (ret) 2074 goto err; 2075 2076 return 0; 2077 err: 2078 cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND); 2079 cma_deref_id(id_priv); 2080 return ret; 2081 } 2082 EXPORT_SYMBOL(rdma_resolve_addr); 2083 2084 static void cma_bind_port(struct rdma_bind_list *bind_list, 2085 struct rdma_id_private *id_priv) 2086 { 2087 struct sockaddr_in *sin; 2088 2089 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 2090 sin->sin_port = htons(bind_list->port); 2091 id_priv->bind_list = bind_list; 2092 hlist_add_head(&id_priv->node, &bind_list->owners); 2093 } 2094 2095 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv, 2096 unsigned short snum) 2097 { 2098 struct rdma_bind_list *bind_list; 2099 int port, ret; 2100 2101 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 2102 if (!bind_list) 2103 return -ENOMEM; 2104 2105 do { 2106 ret = idr_get_new_above(ps, bind_list, snum, &port); 2107 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 2108 2109 if (ret) 2110 goto err1; 2111 2112 if (port != snum) { 2113 ret = -EADDRNOTAVAIL; 2114 goto err2; 2115 } 2116 2117 bind_list->ps = ps; 2118 bind_list->port = (unsigned short) port; 2119 cma_bind_port(bind_list, id_priv); 2120 return 0; 2121 err2: 2122 idr_remove(ps, port); 2123 err1: 2124 kfree(bind_list); 2125 return ret; 2126 } 2127 2128 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv) 2129 { 2130 static unsigned int last_used_port; 2131 int low, high, remaining; 2132 unsigned int rover; 2133 2134 inet_get_local_port_range(&low, &high); 2135 remaining = (high - low) + 1; 2136 rover = net_random() % remaining + low; 2137 retry: 2138 if (last_used_port != rover && 2139 !idr_find(ps, (unsigned short) rover)) { 2140 int ret = cma_alloc_port(ps, id_priv, rover); 2141 /* 2142 * Remember previously used port number in order to avoid 2143 * re-using same port immediately after it is closed. 2144 */ 2145 if (!ret) 2146 last_used_port = rover; 2147 if (ret != -EADDRNOTAVAIL) 2148 return ret; 2149 } 2150 if (--remaining) { 2151 rover++; 2152 if ((rover < low) || (rover > high)) 2153 rover = low; 2154 goto retry; 2155 } 2156 return -EADDRNOTAVAIL; 2157 } 2158 2159 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv) 2160 { 2161 struct rdma_id_private *cur_id; 2162 struct sockaddr_in *sin, *cur_sin; 2163 struct rdma_bind_list *bind_list; 2164 struct hlist_node *node; 2165 unsigned short snum; 2166 2167 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 2168 snum = ntohs(sin->sin_port); 2169 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 2170 return -EACCES; 2171 2172 bind_list = idr_find(ps, snum); 2173 if (!bind_list) 2174 return cma_alloc_port(ps, id_priv, snum); 2175 2176 /* 2177 * We don't support binding to any address if anyone is bound to 2178 * a specific address on the same port. 2179 */ 2180 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)) 2181 return -EADDRNOTAVAIL; 2182 2183 hlist_for_each_entry(cur_id, node, &bind_list->owners, node) { 2184 if (cma_any_addr((struct sockaddr *) &cur_id->id.route.addr.src_addr)) 2185 return -EADDRNOTAVAIL; 2186 2187 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr; 2188 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr) 2189 return -EADDRINUSE; 2190 } 2191 2192 cma_bind_port(bind_list, id_priv); 2193 return 0; 2194 } 2195 2196 static int cma_get_port(struct rdma_id_private *id_priv) 2197 { 2198 struct idr *ps; 2199 int ret; 2200 2201 switch (id_priv->id.ps) { 2202 case RDMA_PS_SDP: 2203 ps = &sdp_ps; 2204 break; 2205 case RDMA_PS_TCP: 2206 ps = &tcp_ps; 2207 break; 2208 case RDMA_PS_UDP: 2209 ps = &udp_ps; 2210 break; 2211 case RDMA_PS_IPOIB: 2212 ps = &ipoib_ps; 2213 break; 2214 default: 2215 return -EPROTONOSUPPORT; 2216 } 2217 2218 mutex_lock(&lock); 2219 if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr)) 2220 ret = cma_alloc_any_port(ps, id_priv); 2221 else 2222 ret = cma_use_port(ps, id_priv); 2223 mutex_unlock(&lock); 2224 2225 return ret; 2226 } 2227 2228 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, 2229 struct sockaddr *addr) 2230 { 2231 #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) 2232 struct sockaddr_in6 *sin6; 2233 2234 if (addr->sa_family != AF_INET6) 2235 return 0; 2236 2237 sin6 = (struct sockaddr_in6 *) addr; 2238 if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) && 2239 !sin6->sin6_scope_id) 2240 return -EINVAL; 2241 2242 dev_addr->bound_dev_if = sin6->sin6_scope_id; 2243 #endif 2244 return 0; 2245 } 2246 2247 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 2248 { 2249 struct rdma_id_private *id_priv; 2250 int ret; 2251 2252 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) 2253 return -EAFNOSUPPORT; 2254 2255 id_priv = container_of(id, struct rdma_id_private, id); 2256 if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND)) 2257 return -EINVAL; 2258 2259 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); 2260 if (ret) 2261 goto err1; 2262 2263 if (!cma_any_addr(addr)) { 2264 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr); 2265 if (ret) 2266 goto err1; 2267 2268 mutex_lock(&lock); 2269 ret = cma_acquire_dev(id_priv); 2270 mutex_unlock(&lock); 2271 if (ret) 2272 goto err1; 2273 } 2274 2275 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr)); 2276 ret = cma_get_port(id_priv); 2277 if (ret) 2278 goto err2; 2279 2280 return 0; 2281 err2: 2282 if (id_priv->cma_dev) { 2283 mutex_lock(&lock); 2284 cma_detach_from_dev(id_priv); 2285 mutex_unlock(&lock); 2286 } 2287 err1: 2288 cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE); 2289 return ret; 2290 } 2291 EXPORT_SYMBOL(rdma_bind_addr); 2292 2293 static int cma_format_hdr(void *hdr, enum rdma_port_space ps, 2294 struct rdma_route *route) 2295 { 2296 struct cma_hdr *cma_hdr; 2297 struct sdp_hh *sdp_hdr; 2298 2299 if (route->addr.src_addr.ss_family == AF_INET) { 2300 struct sockaddr_in *src4, *dst4; 2301 2302 src4 = (struct sockaddr_in *) &route->addr.src_addr; 2303 dst4 = (struct sockaddr_in *) &route->addr.dst_addr; 2304 2305 switch (ps) { 2306 case RDMA_PS_SDP: 2307 sdp_hdr = hdr; 2308 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) 2309 return -EINVAL; 2310 sdp_set_ip_ver(sdp_hdr, 4); 2311 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2312 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2313 sdp_hdr->port = src4->sin_port; 2314 break; 2315 default: 2316 cma_hdr = hdr; 2317 cma_hdr->cma_version = CMA_VERSION; 2318 cma_set_ip_ver(cma_hdr, 4); 2319 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2320 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2321 cma_hdr->port = src4->sin_port; 2322 break; 2323 } 2324 } else { 2325 struct sockaddr_in6 *src6, *dst6; 2326 2327 src6 = (struct sockaddr_in6 *) &route->addr.src_addr; 2328 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr; 2329 2330 switch (ps) { 2331 case RDMA_PS_SDP: 2332 sdp_hdr = hdr; 2333 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) 2334 return -EINVAL; 2335 sdp_set_ip_ver(sdp_hdr, 6); 2336 sdp_hdr->src_addr.ip6 = src6->sin6_addr; 2337 sdp_hdr->dst_addr.ip6 = dst6->sin6_addr; 2338 sdp_hdr->port = src6->sin6_port; 2339 break; 2340 default: 2341 cma_hdr = hdr; 2342 cma_hdr->cma_version = CMA_VERSION; 2343 cma_set_ip_ver(cma_hdr, 6); 2344 cma_hdr->src_addr.ip6 = src6->sin6_addr; 2345 cma_hdr->dst_addr.ip6 = dst6->sin6_addr; 2346 cma_hdr->port = src6->sin6_port; 2347 break; 2348 } 2349 } 2350 return 0; 2351 } 2352 2353 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 2354 struct ib_cm_event *ib_event) 2355 { 2356 struct rdma_id_private *id_priv = cm_id->context; 2357 struct rdma_cm_event event; 2358 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; 2359 int ret = 0; 2360 2361 if (cma_disable_callback(id_priv, CMA_CONNECT)) 2362 return 0; 2363 2364 memset(&event, 0, sizeof event); 2365 switch (ib_event->event) { 2366 case IB_CM_SIDR_REQ_ERROR: 2367 event.event = RDMA_CM_EVENT_UNREACHABLE; 2368 event.status = -ETIMEDOUT; 2369 break; 2370 case IB_CM_SIDR_REP_RECEIVED: 2371 event.param.ud.private_data = ib_event->private_data; 2372 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 2373 if (rep->status != IB_SIDR_SUCCESS) { 2374 event.event = RDMA_CM_EVENT_UNREACHABLE; 2375 event.status = ib_event->param.sidr_rep_rcvd.status; 2376 break; 2377 } 2378 ret = cma_set_qkey(id_priv); 2379 if (ret) { 2380 event.event = RDMA_CM_EVENT_ADDR_ERROR; 2381 event.status = -EINVAL; 2382 break; 2383 } 2384 if (id_priv->qkey != rep->qkey) { 2385 event.event = RDMA_CM_EVENT_UNREACHABLE; 2386 event.status = -EINVAL; 2387 break; 2388 } 2389 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, 2390 id_priv->id.route.path_rec, 2391 &event.param.ud.ah_attr); 2392 event.param.ud.qp_num = rep->qpn; 2393 event.param.ud.qkey = rep->qkey; 2394 event.event = RDMA_CM_EVENT_ESTABLISHED; 2395 event.status = 0; 2396 break; 2397 default: 2398 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", 2399 ib_event->event); 2400 goto out; 2401 } 2402 2403 ret = id_priv->id.event_handler(&id_priv->id, &event); 2404 if (ret) { 2405 /* Destroy the CM ID by returning a non-zero value. */ 2406 id_priv->cm_id.ib = NULL; 2407 cma_exch(id_priv, CMA_DESTROYING); 2408 mutex_unlock(&id_priv->handler_mutex); 2409 rdma_destroy_id(&id_priv->id); 2410 return ret; 2411 } 2412 out: 2413 mutex_unlock(&id_priv->handler_mutex); 2414 return ret; 2415 } 2416 2417 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 2418 struct rdma_conn_param *conn_param) 2419 { 2420 struct ib_cm_sidr_req_param req; 2421 struct rdma_route *route; 2422 int ret; 2423 2424 req.private_data_len = sizeof(struct cma_hdr) + 2425 conn_param->private_data_len; 2426 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2427 if (!req.private_data) 2428 return -ENOMEM; 2429 2430 if (conn_param->private_data && conn_param->private_data_len) 2431 memcpy((void *) req.private_data + sizeof(struct cma_hdr), 2432 conn_param->private_data, conn_param->private_data_len); 2433 2434 route = &id_priv->id.route; 2435 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route); 2436 if (ret) 2437 goto out; 2438 2439 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, 2440 cma_sidr_rep_handler, id_priv); 2441 if (IS_ERR(id_priv->cm_id.ib)) { 2442 ret = PTR_ERR(id_priv->cm_id.ib); 2443 goto out; 2444 } 2445 2446 req.path = route->path_rec; 2447 req.service_id = cma_get_service_id(id_priv->id.ps, 2448 (struct sockaddr *) &route->addr.dst_addr); 2449 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); 2450 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2451 2452 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 2453 if (ret) { 2454 ib_destroy_cm_id(id_priv->cm_id.ib); 2455 id_priv->cm_id.ib = NULL; 2456 } 2457 out: 2458 kfree(req.private_data); 2459 return ret; 2460 } 2461 2462 static int cma_connect_ib(struct rdma_id_private *id_priv, 2463 struct rdma_conn_param *conn_param) 2464 { 2465 struct ib_cm_req_param req; 2466 struct rdma_route *route; 2467 void *private_data; 2468 int offset, ret; 2469 2470 memset(&req, 0, sizeof req); 2471 offset = cma_user_data_offset(id_priv->id.ps); 2472 req.private_data_len = offset + conn_param->private_data_len; 2473 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2474 if (!private_data) 2475 return -ENOMEM; 2476 2477 if (conn_param->private_data && conn_param->private_data_len) 2478 memcpy(private_data + offset, conn_param->private_data, 2479 conn_param->private_data_len); 2480 2481 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler, 2482 id_priv); 2483 if (IS_ERR(id_priv->cm_id.ib)) { 2484 ret = PTR_ERR(id_priv->cm_id.ib); 2485 goto out; 2486 } 2487 2488 route = &id_priv->id.route; 2489 ret = cma_format_hdr(private_data, id_priv->id.ps, route); 2490 if (ret) 2491 goto out; 2492 req.private_data = private_data; 2493 2494 req.primary_path = &route->path_rec[0]; 2495 if (route->num_paths == 2) 2496 req.alternate_path = &route->path_rec[1]; 2497 2498 req.service_id = cma_get_service_id(id_priv->id.ps, 2499 (struct sockaddr *) &route->addr.dst_addr); 2500 req.qp_num = id_priv->qp_num; 2501 req.qp_type = IB_QPT_RC; 2502 req.starting_psn = id_priv->seq_num; 2503 req.responder_resources = conn_param->responder_resources; 2504 req.initiator_depth = conn_param->initiator_depth; 2505 req.flow_control = conn_param->flow_control; 2506 req.retry_count = conn_param->retry_count; 2507 req.rnr_retry_count = conn_param->rnr_retry_count; 2508 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 2509 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 2510 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2511 req.srq = id_priv->srq ? 1 : 0; 2512 2513 ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 2514 out: 2515 if (ret && !IS_ERR(id_priv->cm_id.ib)) { 2516 ib_destroy_cm_id(id_priv->cm_id.ib); 2517 id_priv->cm_id.ib = NULL; 2518 } 2519 2520 kfree(private_data); 2521 return ret; 2522 } 2523 2524 static int cma_connect_iw(struct rdma_id_private *id_priv, 2525 struct rdma_conn_param *conn_param) 2526 { 2527 struct iw_cm_id *cm_id; 2528 struct sockaddr_in* sin; 2529 int ret; 2530 struct iw_cm_conn_param iw_param; 2531 2532 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); 2533 if (IS_ERR(cm_id)) { 2534 ret = PTR_ERR(cm_id); 2535 goto out; 2536 } 2537 2538 id_priv->cm_id.iw = cm_id; 2539 2540 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr; 2541 cm_id->local_addr = *sin; 2542 2543 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr; 2544 cm_id->remote_addr = *sin; 2545 2546 ret = cma_modify_qp_rtr(id_priv, conn_param); 2547 if (ret) 2548 goto out; 2549 2550 iw_param.ord = conn_param->initiator_depth; 2551 iw_param.ird = conn_param->responder_resources; 2552 iw_param.private_data = conn_param->private_data; 2553 iw_param.private_data_len = conn_param->private_data_len; 2554 if (id_priv->id.qp) 2555 iw_param.qpn = id_priv->qp_num; 2556 else 2557 iw_param.qpn = conn_param->qp_num; 2558 ret = iw_cm_connect(cm_id, &iw_param); 2559 out: 2560 if (ret && !IS_ERR(cm_id)) { 2561 iw_destroy_cm_id(cm_id); 2562 id_priv->cm_id.iw = NULL; 2563 } 2564 return ret; 2565 } 2566 2567 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2568 { 2569 struct rdma_id_private *id_priv; 2570 int ret; 2571 2572 id_priv = container_of(id, struct rdma_id_private, id); 2573 if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT)) 2574 return -EINVAL; 2575 2576 if (!id->qp) { 2577 id_priv->qp_num = conn_param->qp_num; 2578 id_priv->srq = conn_param->srq; 2579 } 2580 2581 switch (rdma_node_get_transport(id->device->node_type)) { 2582 case RDMA_TRANSPORT_IB: 2583 if (cma_is_ud_ps(id->ps)) 2584 ret = cma_resolve_ib_udp(id_priv, conn_param); 2585 else 2586 ret = cma_connect_ib(id_priv, conn_param); 2587 break; 2588 case RDMA_TRANSPORT_IWARP: 2589 ret = cma_connect_iw(id_priv, conn_param); 2590 break; 2591 default: 2592 ret = -ENOSYS; 2593 break; 2594 } 2595 if (ret) 2596 goto err; 2597 2598 return 0; 2599 err: 2600 cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED); 2601 return ret; 2602 } 2603 EXPORT_SYMBOL(rdma_connect); 2604 2605 static int cma_accept_ib(struct rdma_id_private *id_priv, 2606 struct rdma_conn_param *conn_param) 2607 { 2608 struct ib_cm_rep_param rep; 2609 int ret; 2610 2611 ret = cma_modify_qp_rtr(id_priv, conn_param); 2612 if (ret) 2613 goto out; 2614 2615 ret = cma_modify_qp_rts(id_priv, conn_param); 2616 if (ret) 2617 goto out; 2618 2619 memset(&rep, 0, sizeof rep); 2620 rep.qp_num = id_priv->qp_num; 2621 rep.starting_psn = id_priv->seq_num; 2622 rep.private_data = conn_param->private_data; 2623 rep.private_data_len = conn_param->private_data_len; 2624 rep.responder_resources = conn_param->responder_resources; 2625 rep.initiator_depth = conn_param->initiator_depth; 2626 rep.failover_accepted = 0; 2627 rep.flow_control = conn_param->flow_control; 2628 rep.rnr_retry_count = conn_param->rnr_retry_count; 2629 rep.srq = id_priv->srq ? 1 : 0; 2630 2631 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 2632 out: 2633 return ret; 2634 } 2635 2636 static int cma_accept_iw(struct rdma_id_private *id_priv, 2637 struct rdma_conn_param *conn_param) 2638 { 2639 struct iw_cm_conn_param iw_param; 2640 int ret; 2641 2642 ret = cma_modify_qp_rtr(id_priv, conn_param); 2643 if (ret) 2644 return ret; 2645 2646 iw_param.ord = conn_param->initiator_depth; 2647 iw_param.ird = conn_param->responder_resources; 2648 iw_param.private_data = conn_param->private_data; 2649 iw_param.private_data_len = conn_param->private_data_len; 2650 if (id_priv->id.qp) { 2651 iw_param.qpn = id_priv->qp_num; 2652 } else 2653 iw_param.qpn = conn_param->qp_num; 2654 2655 return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 2656 } 2657 2658 static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 2659 enum ib_cm_sidr_status status, 2660 const void *private_data, int private_data_len) 2661 { 2662 struct ib_cm_sidr_rep_param rep; 2663 int ret; 2664 2665 memset(&rep, 0, sizeof rep); 2666 rep.status = status; 2667 if (status == IB_SIDR_SUCCESS) { 2668 ret = cma_set_qkey(id_priv); 2669 if (ret) 2670 return ret; 2671 rep.qp_num = id_priv->qp_num; 2672 rep.qkey = id_priv->qkey; 2673 } 2674 rep.private_data = private_data; 2675 rep.private_data_len = private_data_len; 2676 2677 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 2678 } 2679 2680 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2681 { 2682 struct rdma_id_private *id_priv; 2683 int ret; 2684 2685 id_priv = container_of(id, struct rdma_id_private, id); 2686 if (!cma_comp(id_priv, CMA_CONNECT)) 2687 return -EINVAL; 2688 2689 if (!id->qp && conn_param) { 2690 id_priv->qp_num = conn_param->qp_num; 2691 id_priv->srq = conn_param->srq; 2692 } 2693 2694 switch (rdma_node_get_transport(id->device->node_type)) { 2695 case RDMA_TRANSPORT_IB: 2696 if (cma_is_ud_ps(id->ps)) 2697 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 2698 conn_param->private_data, 2699 conn_param->private_data_len); 2700 else if (conn_param) 2701 ret = cma_accept_ib(id_priv, conn_param); 2702 else 2703 ret = cma_rep_recv(id_priv); 2704 break; 2705 case RDMA_TRANSPORT_IWARP: 2706 ret = cma_accept_iw(id_priv, conn_param); 2707 break; 2708 default: 2709 ret = -ENOSYS; 2710 break; 2711 } 2712 2713 if (ret) 2714 goto reject; 2715 2716 return 0; 2717 reject: 2718 cma_modify_qp_err(id_priv); 2719 rdma_reject(id, NULL, 0); 2720 return ret; 2721 } 2722 EXPORT_SYMBOL(rdma_accept); 2723 2724 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 2725 { 2726 struct rdma_id_private *id_priv; 2727 int ret; 2728 2729 id_priv = container_of(id, struct rdma_id_private, id); 2730 if (!cma_has_cm_dev(id_priv)) 2731 return -EINVAL; 2732 2733 switch (id->device->node_type) { 2734 case RDMA_NODE_IB_CA: 2735 ret = ib_cm_notify(id_priv->cm_id.ib, event); 2736 break; 2737 default: 2738 ret = 0; 2739 break; 2740 } 2741 return ret; 2742 } 2743 EXPORT_SYMBOL(rdma_notify); 2744 2745 int rdma_reject(struct rdma_cm_id *id, const void *private_data, 2746 u8 private_data_len) 2747 { 2748 struct rdma_id_private *id_priv; 2749 int ret; 2750 2751 id_priv = container_of(id, struct rdma_id_private, id); 2752 if (!cma_has_cm_dev(id_priv)) 2753 return -EINVAL; 2754 2755 switch (rdma_node_get_transport(id->device->node_type)) { 2756 case RDMA_TRANSPORT_IB: 2757 if (cma_is_ud_ps(id->ps)) 2758 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 2759 private_data, private_data_len); 2760 else 2761 ret = ib_send_cm_rej(id_priv->cm_id.ib, 2762 IB_CM_REJ_CONSUMER_DEFINED, NULL, 2763 0, private_data, private_data_len); 2764 break; 2765 case RDMA_TRANSPORT_IWARP: 2766 ret = iw_cm_reject(id_priv->cm_id.iw, 2767 private_data, private_data_len); 2768 break; 2769 default: 2770 ret = -ENOSYS; 2771 break; 2772 } 2773 return ret; 2774 } 2775 EXPORT_SYMBOL(rdma_reject); 2776 2777 int rdma_disconnect(struct rdma_cm_id *id) 2778 { 2779 struct rdma_id_private *id_priv; 2780 int ret; 2781 2782 id_priv = container_of(id, struct rdma_id_private, id); 2783 if (!cma_has_cm_dev(id_priv)) 2784 return -EINVAL; 2785 2786 switch (rdma_node_get_transport(id->device->node_type)) { 2787 case RDMA_TRANSPORT_IB: 2788 ret = cma_modify_qp_err(id_priv); 2789 if (ret) 2790 goto out; 2791 /* Initiate or respond to a disconnect. */ 2792 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) 2793 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); 2794 break; 2795 case RDMA_TRANSPORT_IWARP: 2796 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 2797 break; 2798 default: 2799 ret = -EINVAL; 2800 break; 2801 } 2802 out: 2803 return ret; 2804 } 2805 EXPORT_SYMBOL(rdma_disconnect); 2806 2807 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 2808 { 2809 struct rdma_id_private *id_priv; 2810 struct cma_multicast *mc = multicast->context; 2811 struct rdma_cm_event event; 2812 int ret; 2813 2814 id_priv = mc->id_priv; 2815 if (cma_disable_callback(id_priv, CMA_ADDR_BOUND) && 2816 cma_disable_callback(id_priv, CMA_ADDR_RESOLVED)) 2817 return 0; 2818 2819 mutex_lock(&id_priv->qp_mutex); 2820 if (!status && id_priv->id.qp) 2821 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, 2822 multicast->rec.mlid); 2823 mutex_unlock(&id_priv->qp_mutex); 2824 2825 memset(&event, 0, sizeof event); 2826 event.status = status; 2827 event.param.ud.private_data = mc->context; 2828 if (!status) { 2829 event.event = RDMA_CM_EVENT_MULTICAST_JOIN; 2830 ib_init_ah_from_mcmember(id_priv->id.device, 2831 id_priv->id.port_num, &multicast->rec, 2832 &event.param.ud.ah_attr); 2833 event.param.ud.qp_num = 0xFFFFFF; 2834 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 2835 } else 2836 event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 2837 2838 ret = id_priv->id.event_handler(&id_priv->id, &event); 2839 if (ret) { 2840 cma_exch(id_priv, CMA_DESTROYING); 2841 mutex_unlock(&id_priv->handler_mutex); 2842 rdma_destroy_id(&id_priv->id); 2843 return 0; 2844 } 2845 2846 mutex_unlock(&id_priv->handler_mutex); 2847 return 0; 2848 } 2849 2850 static void cma_set_mgid(struct rdma_id_private *id_priv, 2851 struct sockaddr *addr, union ib_gid *mgid) 2852 { 2853 unsigned char mc_map[MAX_ADDR_LEN]; 2854 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2855 struct sockaddr_in *sin = (struct sockaddr_in *) addr; 2856 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 2857 2858 if (cma_any_addr(addr)) { 2859 memset(mgid, 0, sizeof *mgid); 2860 } else if ((addr->sa_family == AF_INET6) && 2861 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 2862 0xFF10A01B)) { 2863 /* IPv6 address is an SA assigned MGID. */ 2864 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 2865 } else if ((addr->sa_family == AF_INET6)) { 2866 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); 2867 if (id_priv->id.ps == RDMA_PS_UDP) 2868 mc_map[7] = 0x01; /* Use RDMA CM signature */ 2869 *mgid = *(union ib_gid *) (mc_map + 4); 2870 } else { 2871 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); 2872 if (id_priv->id.ps == RDMA_PS_UDP) 2873 mc_map[7] = 0x01; /* Use RDMA CM signature */ 2874 *mgid = *(union ib_gid *) (mc_map + 4); 2875 } 2876 } 2877 2878 static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 2879 struct cma_multicast *mc) 2880 { 2881 struct ib_sa_mcmember_rec rec; 2882 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2883 ib_sa_comp_mask comp_mask; 2884 int ret; 2885 2886 ib_addr_get_mgid(dev_addr, &rec.mgid); 2887 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 2888 &rec.mgid, &rec); 2889 if (ret) 2890 return ret; 2891 2892 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); 2893 if (id_priv->id.ps == RDMA_PS_UDP) 2894 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 2895 rdma_addr_get_sgid(dev_addr, &rec.port_gid); 2896 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 2897 rec.join_state = 1; 2898 2899 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 2900 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 2901 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 2902 IB_SA_MCMEMBER_REC_FLOW_LABEL | 2903 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 2904 2905 if (id_priv->id.ps == RDMA_PS_IPOIB) 2906 comp_mask |= IB_SA_MCMEMBER_REC_RATE | 2907 IB_SA_MCMEMBER_REC_RATE_SELECTOR; 2908 2909 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, 2910 id_priv->id.port_num, &rec, 2911 comp_mask, GFP_KERNEL, 2912 cma_ib_mc_handler, mc); 2913 if (IS_ERR(mc->multicast.ib)) 2914 return PTR_ERR(mc->multicast.ib); 2915 2916 return 0; 2917 } 2918 2919 static void iboe_mcast_work_handler(struct work_struct *work) 2920 { 2921 struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); 2922 struct cma_multicast *mc = mw->mc; 2923 struct ib_sa_multicast *m = mc->multicast.ib; 2924 2925 mc->multicast.ib->context = mc; 2926 cma_ib_mc_handler(0, m); 2927 kref_put(&mc->mcref, release_mc); 2928 kfree(mw); 2929 } 2930 2931 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid) 2932 { 2933 struct sockaddr_in *sin = (struct sockaddr_in *)addr; 2934 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; 2935 2936 if (cma_any_addr(addr)) { 2937 memset(mgid, 0, sizeof *mgid); 2938 } else if (addr->sa_family == AF_INET6) { 2939 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 2940 } else { 2941 mgid->raw[0] = 0xff; 2942 mgid->raw[1] = 0x0e; 2943 mgid->raw[2] = 0; 2944 mgid->raw[3] = 0; 2945 mgid->raw[4] = 0; 2946 mgid->raw[5] = 0; 2947 mgid->raw[6] = 0; 2948 mgid->raw[7] = 0; 2949 mgid->raw[8] = 0; 2950 mgid->raw[9] = 0; 2951 mgid->raw[10] = 0xff; 2952 mgid->raw[11] = 0xff; 2953 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; 2954 } 2955 } 2956 2957 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, 2958 struct cma_multicast *mc) 2959 { 2960 struct iboe_mcast_work *work; 2961 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2962 int err; 2963 struct sockaddr *addr = (struct sockaddr *)&mc->addr; 2964 struct net_device *ndev = NULL; 2965 2966 if (cma_zero_addr((struct sockaddr *)&mc->addr)) 2967 return -EINVAL; 2968 2969 work = kzalloc(sizeof *work, GFP_KERNEL); 2970 if (!work) 2971 return -ENOMEM; 2972 2973 mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); 2974 if (!mc->multicast.ib) { 2975 err = -ENOMEM; 2976 goto out1; 2977 } 2978 2979 cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid); 2980 2981 mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); 2982 if (id_priv->id.ps == RDMA_PS_UDP) 2983 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 2984 2985 if (dev_addr->bound_dev_if) 2986 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if); 2987 if (!ndev) { 2988 err = -ENODEV; 2989 goto out2; 2990 } 2991 mc->multicast.ib->rec.rate = iboe_get_rate(ndev); 2992 mc->multicast.ib->rec.hop_limit = 1; 2993 mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu); 2994 dev_put(ndev); 2995 if (!mc->multicast.ib->rec.mtu) { 2996 err = -EINVAL; 2997 goto out2; 2998 } 2999 iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid); 3000 work->id = id_priv; 3001 work->mc = mc; 3002 INIT_WORK(&work->work, iboe_mcast_work_handler); 3003 kref_get(&mc->mcref); 3004 queue_work(cma_wq, &work->work); 3005 3006 return 0; 3007 3008 out2: 3009 kfree(mc->multicast.ib); 3010 out1: 3011 kfree(work); 3012 return err; 3013 } 3014 3015 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 3016 void *context) 3017 { 3018 struct rdma_id_private *id_priv; 3019 struct cma_multicast *mc; 3020 int ret; 3021 3022 id_priv = container_of(id, struct rdma_id_private, id); 3023 if (!cma_comp(id_priv, CMA_ADDR_BOUND) && 3024 !cma_comp(id_priv, CMA_ADDR_RESOLVED)) 3025 return -EINVAL; 3026 3027 mc = kmalloc(sizeof *mc, GFP_KERNEL); 3028 if (!mc) 3029 return -ENOMEM; 3030 3031 memcpy(&mc->addr, addr, ip_addr_size(addr)); 3032 mc->context = context; 3033 mc->id_priv = id_priv; 3034 3035 spin_lock(&id_priv->lock); 3036 list_add(&mc->list, &id_priv->mc_list); 3037 spin_unlock(&id_priv->lock); 3038 3039 switch (rdma_node_get_transport(id->device->node_type)) { 3040 case RDMA_TRANSPORT_IB: 3041 switch (rdma_port_get_link_layer(id->device, id->port_num)) { 3042 case IB_LINK_LAYER_INFINIBAND: 3043 ret = cma_join_ib_multicast(id_priv, mc); 3044 break; 3045 case IB_LINK_LAYER_ETHERNET: 3046 kref_init(&mc->mcref); 3047 ret = cma_iboe_join_multicast(id_priv, mc); 3048 break; 3049 default: 3050 ret = -EINVAL; 3051 } 3052 break; 3053 default: 3054 ret = -ENOSYS; 3055 break; 3056 } 3057 3058 if (ret) { 3059 spin_lock_irq(&id_priv->lock); 3060 list_del(&mc->list); 3061 spin_unlock_irq(&id_priv->lock); 3062 kfree(mc); 3063 } 3064 return ret; 3065 } 3066 EXPORT_SYMBOL(rdma_join_multicast); 3067 3068 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 3069 { 3070 struct rdma_id_private *id_priv; 3071 struct cma_multicast *mc; 3072 3073 id_priv = container_of(id, struct rdma_id_private, id); 3074 spin_lock_irq(&id_priv->lock); 3075 list_for_each_entry(mc, &id_priv->mc_list, list) { 3076 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) { 3077 list_del(&mc->list); 3078 spin_unlock_irq(&id_priv->lock); 3079 3080 if (id->qp) 3081 ib_detach_mcast(id->qp, 3082 &mc->multicast.ib->rec.mgid, 3083 mc->multicast.ib->rec.mlid); 3084 if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) { 3085 switch (rdma_port_get_link_layer(id->device, id->port_num)) { 3086 case IB_LINK_LAYER_INFINIBAND: 3087 ib_sa_free_multicast(mc->multicast.ib); 3088 kfree(mc); 3089 break; 3090 case IB_LINK_LAYER_ETHERNET: 3091 kref_put(&mc->mcref, release_mc); 3092 break; 3093 default: 3094 break; 3095 } 3096 } 3097 return; 3098 } 3099 } 3100 spin_unlock_irq(&id_priv->lock); 3101 } 3102 EXPORT_SYMBOL(rdma_leave_multicast); 3103 3104 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) 3105 { 3106 struct rdma_dev_addr *dev_addr; 3107 struct cma_ndev_work *work; 3108 3109 dev_addr = &id_priv->id.route.addr.dev_addr; 3110 3111 if ((dev_addr->bound_dev_if == ndev->ifindex) && 3112 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { 3113 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", 3114 ndev->name, &id_priv->id); 3115 work = kzalloc(sizeof *work, GFP_KERNEL); 3116 if (!work) 3117 return -ENOMEM; 3118 3119 INIT_WORK(&work->work, cma_ndev_work_handler); 3120 work->id = id_priv; 3121 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; 3122 atomic_inc(&id_priv->refcount); 3123 queue_work(cma_wq, &work->work); 3124 } 3125 3126 return 0; 3127 } 3128 3129 static int cma_netdev_callback(struct notifier_block *self, unsigned long event, 3130 void *ctx) 3131 { 3132 struct net_device *ndev = (struct net_device *)ctx; 3133 struct cma_device *cma_dev; 3134 struct rdma_id_private *id_priv; 3135 int ret = NOTIFY_DONE; 3136 3137 if (dev_net(ndev) != &init_net) 3138 return NOTIFY_DONE; 3139 3140 if (event != NETDEV_BONDING_FAILOVER) 3141 return NOTIFY_DONE; 3142 3143 if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING)) 3144 return NOTIFY_DONE; 3145 3146 mutex_lock(&lock); 3147 list_for_each_entry(cma_dev, &dev_list, list) 3148 list_for_each_entry(id_priv, &cma_dev->id_list, list) { 3149 ret = cma_netdev_change(ndev, id_priv); 3150 if (ret) 3151 goto out; 3152 } 3153 3154 out: 3155 mutex_unlock(&lock); 3156 return ret; 3157 } 3158 3159 static struct notifier_block cma_nb = { 3160 .notifier_call = cma_netdev_callback 3161 }; 3162 3163 static void cma_add_one(struct ib_device *device) 3164 { 3165 struct cma_device *cma_dev; 3166 struct rdma_id_private *id_priv; 3167 3168 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); 3169 if (!cma_dev) 3170 return; 3171 3172 cma_dev->device = device; 3173 3174 init_completion(&cma_dev->comp); 3175 atomic_set(&cma_dev->refcount, 1); 3176 INIT_LIST_HEAD(&cma_dev->id_list); 3177 ib_set_client_data(device, &cma_client, cma_dev); 3178 3179 mutex_lock(&lock); 3180 list_add_tail(&cma_dev->list, &dev_list); 3181 list_for_each_entry(id_priv, &listen_any_list, list) 3182 cma_listen_on_dev(id_priv, cma_dev); 3183 mutex_unlock(&lock); 3184 } 3185 3186 static int cma_remove_id_dev(struct rdma_id_private *id_priv) 3187 { 3188 struct rdma_cm_event event; 3189 enum cma_state state; 3190 int ret = 0; 3191 3192 /* Record that we want to remove the device */ 3193 state = cma_exch(id_priv, CMA_DEVICE_REMOVAL); 3194 if (state == CMA_DESTROYING) 3195 return 0; 3196 3197 cma_cancel_operation(id_priv, state); 3198 mutex_lock(&id_priv->handler_mutex); 3199 3200 /* Check for destruction from another callback. */ 3201 if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL)) 3202 goto out; 3203 3204 memset(&event, 0, sizeof event); 3205 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; 3206 ret = id_priv->id.event_handler(&id_priv->id, &event); 3207 out: 3208 mutex_unlock(&id_priv->handler_mutex); 3209 return ret; 3210 } 3211 3212 static void cma_process_remove(struct cma_device *cma_dev) 3213 { 3214 struct rdma_id_private *id_priv; 3215 int ret; 3216 3217 mutex_lock(&lock); 3218 while (!list_empty(&cma_dev->id_list)) { 3219 id_priv = list_entry(cma_dev->id_list.next, 3220 struct rdma_id_private, list); 3221 3222 list_del(&id_priv->listen_list); 3223 list_del_init(&id_priv->list); 3224 atomic_inc(&id_priv->refcount); 3225 mutex_unlock(&lock); 3226 3227 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); 3228 cma_deref_id(id_priv); 3229 if (ret) 3230 rdma_destroy_id(&id_priv->id); 3231 3232 mutex_lock(&lock); 3233 } 3234 mutex_unlock(&lock); 3235 3236 cma_deref_dev(cma_dev); 3237 wait_for_completion(&cma_dev->comp); 3238 } 3239 3240 static void cma_remove_one(struct ib_device *device) 3241 { 3242 struct cma_device *cma_dev; 3243 3244 cma_dev = ib_get_client_data(device, &cma_client); 3245 if (!cma_dev) 3246 return; 3247 3248 mutex_lock(&lock); 3249 list_del(&cma_dev->list); 3250 mutex_unlock(&lock); 3251 3252 cma_process_remove(cma_dev); 3253 kfree(cma_dev); 3254 } 3255 3256 static int __init cma_init(void) 3257 { 3258 int ret; 3259 3260 cma_wq = create_singlethread_workqueue("rdma_cm"); 3261 if (!cma_wq) 3262 return -ENOMEM; 3263 3264 ib_sa_register_client(&sa_client); 3265 rdma_addr_register_client(&addr_client); 3266 register_netdevice_notifier(&cma_nb); 3267 3268 ret = ib_register_client(&cma_client); 3269 if (ret) 3270 goto err; 3271 return 0; 3272 3273 err: 3274 unregister_netdevice_notifier(&cma_nb); 3275 rdma_addr_unregister_client(&addr_client); 3276 ib_sa_unregister_client(&sa_client); 3277 destroy_workqueue(cma_wq); 3278 return ret; 3279 } 3280 3281 static void __exit cma_cleanup(void) 3282 { 3283 ib_unregister_client(&cma_client); 3284 unregister_netdevice_notifier(&cma_nb); 3285 rdma_addr_unregister_client(&addr_client); 3286 ib_sa_unregister_client(&sa_client); 3287 destroy_workqueue(cma_wq); 3288 idr_destroy(&sdp_ps); 3289 idr_destroy(&tcp_ps); 3290 idr_destroy(&udp_ps); 3291 idr_destroy(&ipoib_ps); 3292 } 3293 3294 module_init(cma_init); 3295 module_exit(cma_cleanup); 3296