1 /* 2 * Copyright (c) 2005 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. 5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/completion.h> 37 #include <linux/in.h> 38 #include <linux/in6.h> 39 #include <linux/mutex.h> 40 #include <linux/random.h> 41 #include <linux/idr.h> 42 #include <linux/inetdevice.h> 43 #include <linux/slab.h> 44 #include <linux/module.h> 45 #include <net/route.h> 46 47 #include <net/net_namespace.h> 48 #include <net/netns/generic.h> 49 #include <net/tcp.h> 50 #include <net/ipv6.h> 51 #include <net/ip_fib.h> 52 #include <net/ip6_route.h> 53 54 #include <rdma/rdma_cm.h> 55 #include <rdma/rdma_cm_ib.h> 56 #include <rdma/rdma_netlink.h> 57 #include <rdma/ib.h> 58 #include <rdma/ib_cache.h> 59 #include <rdma/ib_cm.h> 60 #include <rdma/ib_sa.h> 61 #include <rdma/iw_cm.h> 62 63 MODULE_AUTHOR("Sean Hefty"); 64 MODULE_DESCRIPTION("Generic RDMA CM Agent"); 65 MODULE_LICENSE("Dual BSD/GPL"); 66 67 #define CMA_CM_RESPONSE_TIMEOUT 20 68 #define CMA_MAX_CM_RETRIES 15 69 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 70 #define CMA_IBOE_PACKET_LIFETIME 18 71 72 static const char * const cma_events[] = { 73 [RDMA_CM_EVENT_ADDR_RESOLVED] = "address resolved", 74 [RDMA_CM_EVENT_ADDR_ERROR] = "address error", 75 [RDMA_CM_EVENT_ROUTE_RESOLVED] = "route resolved ", 76 [RDMA_CM_EVENT_ROUTE_ERROR] = "route error", 77 [RDMA_CM_EVENT_CONNECT_REQUEST] = "connect request", 78 [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response", 79 [RDMA_CM_EVENT_CONNECT_ERROR] = "connect error", 80 [RDMA_CM_EVENT_UNREACHABLE] = "unreachable", 81 [RDMA_CM_EVENT_REJECTED] = "rejected", 82 [RDMA_CM_EVENT_ESTABLISHED] = "established", 83 [RDMA_CM_EVENT_DISCONNECTED] = "disconnected", 84 [RDMA_CM_EVENT_DEVICE_REMOVAL] = "device removal", 85 [RDMA_CM_EVENT_MULTICAST_JOIN] = "multicast join", 86 [RDMA_CM_EVENT_MULTICAST_ERROR] = "multicast error", 87 [RDMA_CM_EVENT_ADDR_CHANGE] = "address change", 88 [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit", 89 }; 90 91 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event) 92 { 93 size_t index = event; 94 95 return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ? 96 cma_events[index] : "unrecognized event"; 97 } 98 EXPORT_SYMBOL(rdma_event_msg); 99 100 static void cma_add_one(struct ib_device *device); 101 static void cma_remove_one(struct ib_device *device, void *client_data); 102 103 static struct ib_client cma_client = { 104 .name = "cma", 105 .add = cma_add_one, 106 .remove = cma_remove_one 107 }; 108 109 static struct ib_sa_client sa_client; 110 static struct rdma_addr_client addr_client; 111 static LIST_HEAD(dev_list); 112 static LIST_HEAD(listen_any_list); 113 static DEFINE_MUTEX(lock); 114 static struct workqueue_struct *cma_wq; 115 static int cma_pernet_id; 116 117 struct cma_pernet { 118 struct idr tcp_ps; 119 struct idr udp_ps; 120 struct idr ipoib_ps; 121 struct idr ib_ps; 122 }; 123 124 static struct cma_pernet *cma_pernet(struct net *net) 125 { 126 return net_generic(net, cma_pernet_id); 127 } 128 129 static struct idr *cma_pernet_idr(struct net *net, enum rdma_port_space ps) 130 { 131 struct cma_pernet *pernet = cma_pernet(net); 132 133 switch (ps) { 134 case RDMA_PS_TCP: 135 return &pernet->tcp_ps; 136 case RDMA_PS_UDP: 137 return &pernet->udp_ps; 138 case RDMA_PS_IPOIB: 139 return &pernet->ipoib_ps; 140 case RDMA_PS_IB: 141 return &pernet->ib_ps; 142 default: 143 return NULL; 144 } 145 } 146 147 struct cma_device { 148 struct list_head list; 149 struct ib_device *device; 150 struct completion comp; 151 atomic_t refcount; 152 struct list_head id_list; 153 }; 154 155 struct rdma_bind_list { 156 enum rdma_port_space ps; 157 struct hlist_head owners; 158 unsigned short port; 159 }; 160 161 static int cma_ps_alloc(struct net *net, enum rdma_port_space ps, 162 struct rdma_bind_list *bind_list, int snum) 163 { 164 struct idr *idr = cma_pernet_idr(net, ps); 165 166 return idr_alloc(idr, bind_list, snum, snum + 1, GFP_KERNEL); 167 } 168 169 static struct rdma_bind_list *cma_ps_find(struct net *net, 170 enum rdma_port_space ps, int snum) 171 { 172 struct idr *idr = cma_pernet_idr(net, ps); 173 174 return idr_find(idr, snum); 175 } 176 177 static void cma_ps_remove(struct net *net, enum rdma_port_space ps, int snum) 178 { 179 struct idr *idr = cma_pernet_idr(net, ps); 180 181 idr_remove(idr, snum); 182 } 183 184 enum { 185 CMA_OPTION_AFONLY, 186 }; 187 188 /* 189 * Device removal can occur at anytime, so we need extra handling to 190 * serialize notifying the user of device removal with other callbacks. 191 * We do this by disabling removal notification while a callback is in process, 192 * and reporting it after the callback completes. 193 */ 194 struct rdma_id_private { 195 struct rdma_cm_id id; 196 197 struct rdma_bind_list *bind_list; 198 struct hlist_node node; 199 struct list_head list; /* listen_any_list or cma_device.list */ 200 struct list_head listen_list; /* per device listens */ 201 struct cma_device *cma_dev; 202 struct list_head mc_list; 203 204 int internal_id; 205 enum rdma_cm_state state; 206 spinlock_t lock; 207 struct mutex qp_mutex; 208 209 struct completion comp; 210 atomic_t refcount; 211 struct mutex handler_mutex; 212 213 int backlog; 214 int timeout_ms; 215 struct ib_sa_query *query; 216 int query_id; 217 union { 218 struct ib_cm_id *ib; 219 struct iw_cm_id *iw; 220 } cm_id; 221 222 u32 seq_num; 223 u32 qkey; 224 u32 qp_num; 225 pid_t owner; 226 u32 options; 227 u8 srq; 228 u8 tos; 229 u8 reuseaddr; 230 u8 afonly; 231 }; 232 233 struct cma_multicast { 234 struct rdma_id_private *id_priv; 235 union { 236 struct ib_sa_multicast *ib; 237 } multicast; 238 struct list_head list; 239 void *context; 240 struct sockaddr_storage addr; 241 struct kref mcref; 242 }; 243 244 struct cma_work { 245 struct work_struct work; 246 struct rdma_id_private *id; 247 enum rdma_cm_state old_state; 248 enum rdma_cm_state new_state; 249 struct rdma_cm_event event; 250 }; 251 252 struct cma_ndev_work { 253 struct work_struct work; 254 struct rdma_id_private *id; 255 struct rdma_cm_event event; 256 }; 257 258 struct iboe_mcast_work { 259 struct work_struct work; 260 struct rdma_id_private *id; 261 struct cma_multicast *mc; 262 }; 263 264 union cma_ip_addr { 265 struct in6_addr ip6; 266 struct { 267 __be32 pad[3]; 268 __be32 addr; 269 } ip4; 270 }; 271 272 struct cma_hdr { 273 u8 cma_version; 274 u8 ip_version; /* IP version: 7:4 */ 275 __be16 port; 276 union cma_ip_addr src_addr; 277 union cma_ip_addr dst_addr; 278 }; 279 280 #define CMA_VERSION 0x00 281 282 struct cma_req_info { 283 struct ib_device *device; 284 int port; 285 union ib_gid local_gid; 286 __be64 service_id; 287 u16 pkey; 288 bool has_gid:1; 289 }; 290 291 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp) 292 { 293 unsigned long flags; 294 int ret; 295 296 spin_lock_irqsave(&id_priv->lock, flags); 297 ret = (id_priv->state == comp); 298 spin_unlock_irqrestore(&id_priv->lock, flags); 299 return ret; 300 } 301 302 static int cma_comp_exch(struct rdma_id_private *id_priv, 303 enum rdma_cm_state comp, enum rdma_cm_state exch) 304 { 305 unsigned long flags; 306 int ret; 307 308 spin_lock_irqsave(&id_priv->lock, flags); 309 if ((ret = (id_priv->state == comp))) 310 id_priv->state = exch; 311 spin_unlock_irqrestore(&id_priv->lock, flags); 312 return ret; 313 } 314 315 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv, 316 enum rdma_cm_state exch) 317 { 318 unsigned long flags; 319 enum rdma_cm_state old; 320 321 spin_lock_irqsave(&id_priv->lock, flags); 322 old = id_priv->state; 323 id_priv->state = exch; 324 spin_unlock_irqrestore(&id_priv->lock, flags); 325 return old; 326 } 327 328 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr) 329 { 330 return hdr->ip_version >> 4; 331 } 332 333 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 334 { 335 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 336 } 337 338 static void cma_attach_to_dev(struct rdma_id_private *id_priv, 339 struct cma_device *cma_dev) 340 { 341 atomic_inc(&cma_dev->refcount); 342 id_priv->cma_dev = cma_dev; 343 id_priv->id.device = cma_dev->device; 344 id_priv->id.route.addr.dev_addr.transport = 345 rdma_node_get_transport(cma_dev->device->node_type); 346 list_add_tail(&id_priv->list, &cma_dev->id_list); 347 } 348 349 static inline void cma_deref_dev(struct cma_device *cma_dev) 350 { 351 if (atomic_dec_and_test(&cma_dev->refcount)) 352 complete(&cma_dev->comp); 353 } 354 355 static inline void release_mc(struct kref *kref) 356 { 357 struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); 358 359 kfree(mc->multicast.ib); 360 kfree(mc); 361 } 362 363 static void cma_release_dev(struct rdma_id_private *id_priv) 364 { 365 mutex_lock(&lock); 366 list_del(&id_priv->list); 367 cma_deref_dev(id_priv->cma_dev); 368 id_priv->cma_dev = NULL; 369 mutex_unlock(&lock); 370 } 371 372 static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv) 373 { 374 return (struct sockaddr *) &id_priv->id.route.addr.src_addr; 375 } 376 377 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv) 378 { 379 return (struct sockaddr *) &id_priv->id.route.addr.dst_addr; 380 } 381 382 static inline unsigned short cma_family(struct rdma_id_private *id_priv) 383 { 384 return id_priv->id.route.addr.src_addr.ss_family; 385 } 386 387 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey) 388 { 389 struct ib_sa_mcmember_rec rec; 390 int ret = 0; 391 392 if (id_priv->qkey) { 393 if (qkey && id_priv->qkey != qkey) 394 return -EINVAL; 395 return 0; 396 } 397 398 if (qkey) { 399 id_priv->qkey = qkey; 400 return 0; 401 } 402 403 switch (id_priv->id.ps) { 404 case RDMA_PS_UDP: 405 case RDMA_PS_IB: 406 id_priv->qkey = RDMA_UDP_QKEY; 407 break; 408 case RDMA_PS_IPOIB: 409 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); 410 ret = ib_sa_get_mcmember_rec(id_priv->id.device, 411 id_priv->id.port_num, &rec.mgid, 412 &rec); 413 if (!ret) 414 id_priv->qkey = be32_to_cpu(rec.qkey); 415 break; 416 default: 417 break; 418 } 419 return ret; 420 } 421 422 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr) 423 { 424 dev_addr->dev_type = ARPHRD_INFINIBAND; 425 rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr); 426 ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey)); 427 } 428 429 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr) 430 { 431 int ret; 432 433 if (addr->sa_family != AF_IB) { 434 ret = rdma_translate_ip(addr, dev_addr, NULL); 435 } else { 436 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr); 437 ret = 0; 438 } 439 440 return ret; 441 } 442 443 static inline int cma_validate_port(struct ib_device *device, u8 port, 444 union ib_gid *gid, int dev_type, 445 int bound_if_index) 446 { 447 int ret = -ENODEV; 448 struct net_device *ndev = NULL; 449 450 if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port)) 451 return ret; 452 453 if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port)) 454 return ret; 455 456 if (dev_type == ARPHRD_ETHER) 457 ndev = dev_get_by_index(&init_net, bound_if_index); 458 459 ret = ib_find_cached_gid_by_port(device, gid, port, ndev, NULL); 460 461 if (ndev) 462 dev_put(ndev); 463 464 return ret; 465 } 466 467 static int cma_acquire_dev(struct rdma_id_private *id_priv, 468 struct rdma_id_private *listen_id_priv) 469 { 470 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 471 struct cma_device *cma_dev; 472 union ib_gid gid, iboe_gid, *gidp; 473 int ret = -ENODEV; 474 u8 port; 475 476 if (dev_addr->dev_type != ARPHRD_INFINIBAND && 477 id_priv->id.ps == RDMA_PS_IPOIB) 478 return -EINVAL; 479 480 mutex_lock(&lock); 481 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 482 &iboe_gid); 483 484 memcpy(&gid, dev_addr->src_dev_addr + 485 rdma_addr_gid_offset(dev_addr), sizeof gid); 486 487 if (listen_id_priv) { 488 cma_dev = listen_id_priv->cma_dev; 489 port = listen_id_priv->id.port_num; 490 gidp = rdma_protocol_roce(cma_dev->device, port) ? 491 &iboe_gid : &gid; 492 493 ret = cma_validate_port(cma_dev->device, port, gidp, 494 dev_addr->dev_type, 495 dev_addr->bound_dev_if); 496 if (!ret) { 497 id_priv->id.port_num = port; 498 goto out; 499 } 500 } 501 502 list_for_each_entry(cma_dev, &dev_list, list) { 503 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) { 504 if (listen_id_priv && 505 listen_id_priv->cma_dev == cma_dev && 506 listen_id_priv->id.port_num == port) 507 continue; 508 509 gidp = rdma_protocol_roce(cma_dev->device, port) ? 510 &iboe_gid : &gid; 511 512 ret = cma_validate_port(cma_dev->device, port, gidp, 513 dev_addr->dev_type, 514 dev_addr->bound_dev_if); 515 if (!ret) { 516 id_priv->id.port_num = port; 517 goto out; 518 } 519 } 520 } 521 522 out: 523 if (!ret) 524 cma_attach_to_dev(id_priv, cma_dev); 525 526 mutex_unlock(&lock); 527 return ret; 528 } 529 530 /* 531 * Select the source IB device and address to reach the destination IB address. 532 */ 533 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) 534 { 535 struct cma_device *cma_dev, *cur_dev; 536 struct sockaddr_ib *addr; 537 union ib_gid gid, sgid, *dgid; 538 u16 pkey, index; 539 u8 p; 540 int i; 541 542 cma_dev = NULL; 543 addr = (struct sockaddr_ib *) cma_dst_addr(id_priv); 544 dgid = (union ib_gid *) &addr->sib_addr; 545 pkey = ntohs(addr->sib_pkey); 546 547 list_for_each_entry(cur_dev, &dev_list, list) { 548 for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) { 549 if (!rdma_cap_af_ib(cur_dev->device, p)) 550 continue; 551 552 if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index)) 553 continue; 554 555 for (i = 0; !ib_get_cached_gid(cur_dev->device, p, i, 556 &gid, NULL); 557 i++) { 558 if (!memcmp(&gid, dgid, sizeof(gid))) { 559 cma_dev = cur_dev; 560 sgid = gid; 561 id_priv->id.port_num = p; 562 goto found; 563 } 564 565 if (!cma_dev && (gid.global.subnet_prefix == 566 dgid->global.subnet_prefix)) { 567 cma_dev = cur_dev; 568 sgid = gid; 569 id_priv->id.port_num = p; 570 } 571 } 572 } 573 } 574 575 if (!cma_dev) 576 return -ENODEV; 577 578 found: 579 cma_attach_to_dev(id_priv, cma_dev); 580 addr = (struct sockaddr_ib *) cma_src_addr(id_priv); 581 memcpy(&addr->sib_addr, &sgid, sizeof sgid); 582 cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr); 583 return 0; 584 } 585 586 static void cma_deref_id(struct rdma_id_private *id_priv) 587 { 588 if (atomic_dec_and_test(&id_priv->refcount)) 589 complete(&id_priv->comp); 590 } 591 592 static int cma_disable_callback(struct rdma_id_private *id_priv, 593 enum rdma_cm_state state) 594 { 595 mutex_lock(&id_priv->handler_mutex); 596 if (id_priv->state != state) { 597 mutex_unlock(&id_priv->handler_mutex); 598 return -EINVAL; 599 } 600 return 0; 601 } 602 603 struct rdma_cm_id *rdma_create_id(struct net *net, 604 rdma_cm_event_handler event_handler, 605 void *context, enum rdma_port_space ps, 606 enum ib_qp_type qp_type) 607 { 608 struct rdma_id_private *id_priv; 609 610 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 611 if (!id_priv) 612 return ERR_PTR(-ENOMEM); 613 614 id_priv->owner = task_pid_nr(current); 615 id_priv->state = RDMA_CM_IDLE; 616 id_priv->id.context = context; 617 id_priv->id.event_handler = event_handler; 618 id_priv->id.ps = ps; 619 id_priv->id.qp_type = qp_type; 620 spin_lock_init(&id_priv->lock); 621 mutex_init(&id_priv->qp_mutex); 622 init_completion(&id_priv->comp); 623 atomic_set(&id_priv->refcount, 1); 624 mutex_init(&id_priv->handler_mutex); 625 INIT_LIST_HEAD(&id_priv->listen_list); 626 INIT_LIST_HEAD(&id_priv->mc_list); 627 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 628 id_priv->id.route.addr.dev_addr.net = get_net(net); 629 630 return &id_priv->id; 631 } 632 EXPORT_SYMBOL(rdma_create_id); 633 634 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 635 { 636 struct ib_qp_attr qp_attr; 637 int qp_attr_mask, ret; 638 639 qp_attr.qp_state = IB_QPS_INIT; 640 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 641 if (ret) 642 return ret; 643 644 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 645 if (ret) 646 return ret; 647 648 qp_attr.qp_state = IB_QPS_RTR; 649 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 650 if (ret) 651 return ret; 652 653 qp_attr.qp_state = IB_QPS_RTS; 654 qp_attr.sq_psn = 0; 655 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 656 657 return ret; 658 } 659 660 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 661 { 662 struct ib_qp_attr qp_attr; 663 int qp_attr_mask, ret; 664 665 qp_attr.qp_state = IB_QPS_INIT; 666 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 667 if (ret) 668 return ret; 669 670 return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 671 } 672 673 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 674 struct ib_qp_init_attr *qp_init_attr) 675 { 676 struct rdma_id_private *id_priv; 677 struct ib_qp *qp; 678 int ret; 679 680 id_priv = container_of(id, struct rdma_id_private, id); 681 if (id->device != pd->device) 682 return -EINVAL; 683 684 qp = ib_create_qp(pd, qp_init_attr); 685 if (IS_ERR(qp)) 686 return PTR_ERR(qp); 687 688 if (id->qp_type == IB_QPT_UD) 689 ret = cma_init_ud_qp(id_priv, qp); 690 else 691 ret = cma_init_conn_qp(id_priv, qp); 692 if (ret) 693 goto err; 694 695 id->qp = qp; 696 id_priv->qp_num = qp->qp_num; 697 id_priv->srq = (qp->srq != NULL); 698 return 0; 699 err: 700 ib_destroy_qp(qp); 701 return ret; 702 } 703 EXPORT_SYMBOL(rdma_create_qp); 704 705 void rdma_destroy_qp(struct rdma_cm_id *id) 706 { 707 struct rdma_id_private *id_priv; 708 709 id_priv = container_of(id, struct rdma_id_private, id); 710 mutex_lock(&id_priv->qp_mutex); 711 ib_destroy_qp(id_priv->id.qp); 712 id_priv->id.qp = NULL; 713 mutex_unlock(&id_priv->qp_mutex); 714 } 715 EXPORT_SYMBOL(rdma_destroy_qp); 716 717 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, 718 struct rdma_conn_param *conn_param) 719 { 720 struct ib_qp_attr qp_attr; 721 int qp_attr_mask, ret; 722 union ib_gid sgid; 723 724 mutex_lock(&id_priv->qp_mutex); 725 if (!id_priv->id.qp) { 726 ret = 0; 727 goto out; 728 } 729 730 /* Need to update QP attributes from default values. */ 731 qp_attr.qp_state = IB_QPS_INIT; 732 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 733 if (ret) 734 goto out; 735 736 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 737 if (ret) 738 goto out; 739 740 qp_attr.qp_state = IB_QPS_RTR; 741 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 742 if (ret) 743 goto out; 744 745 ret = ib_query_gid(id_priv->id.device, id_priv->id.port_num, 746 qp_attr.ah_attr.grh.sgid_index, &sgid, NULL); 747 if (ret) 748 goto out; 749 750 BUG_ON(id_priv->cma_dev->device != id_priv->id.device); 751 752 if (conn_param) 753 qp_attr.max_dest_rd_atomic = conn_param->responder_resources; 754 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 755 out: 756 mutex_unlock(&id_priv->qp_mutex); 757 return ret; 758 } 759 760 static int cma_modify_qp_rts(struct rdma_id_private *id_priv, 761 struct rdma_conn_param *conn_param) 762 { 763 struct ib_qp_attr qp_attr; 764 int qp_attr_mask, ret; 765 766 mutex_lock(&id_priv->qp_mutex); 767 if (!id_priv->id.qp) { 768 ret = 0; 769 goto out; 770 } 771 772 qp_attr.qp_state = IB_QPS_RTS; 773 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 774 if (ret) 775 goto out; 776 777 if (conn_param) 778 qp_attr.max_rd_atomic = conn_param->initiator_depth; 779 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 780 out: 781 mutex_unlock(&id_priv->qp_mutex); 782 return ret; 783 } 784 785 static int cma_modify_qp_err(struct rdma_id_private *id_priv) 786 { 787 struct ib_qp_attr qp_attr; 788 int ret; 789 790 mutex_lock(&id_priv->qp_mutex); 791 if (!id_priv->id.qp) { 792 ret = 0; 793 goto out; 794 } 795 796 qp_attr.qp_state = IB_QPS_ERR; 797 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); 798 out: 799 mutex_unlock(&id_priv->qp_mutex); 800 return ret; 801 } 802 803 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 804 struct ib_qp_attr *qp_attr, int *qp_attr_mask) 805 { 806 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 807 int ret; 808 u16 pkey; 809 810 if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num)) 811 pkey = 0xffff; 812 else 813 pkey = ib_addr_get_pkey(dev_addr); 814 815 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 816 pkey, &qp_attr->pkey_index); 817 if (ret) 818 return ret; 819 820 qp_attr->port_num = id_priv->id.port_num; 821 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 822 823 if (id_priv->id.qp_type == IB_QPT_UD) { 824 ret = cma_set_qkey(id_priv, 0); 825 if (ret) 826 return ret; 827 828 qp_attr->qkey = id_priv->qkey; 829 *qp_attr_mask |= IB_QP_QKEY; 830 } else { 831 qp_attr->qp_access_flags = 0; 832 *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 833 } 834 return 0; 835 } 836 837 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 838 int *qp_attr_mask) 839 { 840 struct rdma_id_private *id_priv; 841 int ret = 0; 842 843 id_priv = container_of(id, struct rdma_id_private, id); 844 if (rdma_cap_ib_cm(id->device, id->port_num)) { 845 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD)) 846 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 847 else 848 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 849 qp_attr_mask); 850 851 if (qp_attr->qp_state == IB_QPS_RTR) 852 qp_attr->rq_psn = id_priv->seq_num; 853 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 854 if (!id_priv->cm_id.iw) { 855 qp_attr->qp_access_flags = 0; 856 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 857 } else 858 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 859 qp_attr_mask); 860 } else 861 ret = -ENOSYS; 862 863 return ret; 864 } 865 EXPORT_SYMBOL(rdma_init_qp_attr); 866 867 static inline int cma_zero_addr(struct sockaddr *addr) 868 { 869 switch (addr->sa_family) { 870 case AF_INET: 871 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr); 872 case AF_INET6: 873 return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr); 874 case AF_IB: 875 return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr); 876 default: 877 return 0; 878 } 879 } 880 881 static inline int cma_loopback_addr(struct sockaddr *addr) 882 { 883 switch (addr->sa_family) { 884 case AF_INET: 885 return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr); 886 case AF_INET6: 887 return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr); 888 case AF_IB: 889 return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr); 890 default: 891 return 0; 892 } 893 } 894 895 static inline int cma_any_addr(struct sockaddr *addr) 896 { 897 return cma_zero_addr(addr) || cma_loopback_addr(addr); 898 } 899 900 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst) 901 { 902 if (src->sa_family != dst->sa_family) 903 return -1; 904 905 switch (src->sa_family) { 906 case AF_INET: 907 return ((struct sockaddr_in *) src)->sin_addr.s_addr != 908 ((struct sockaddr_in *) dst)->sin_addr.s_addr; 909 case AF_INET6: 910 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr, 911 &((struct sockaddr_in6 *) dst)->sin6_addr); 912 default: 913 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr, 914 &((struct sockaddr_ib *) dst)->sib_addr); 915 } 916 } 917 918 static __be16 cma_port(struct sockaddr *addr) 919 { 920 struct sockaddr_ib *sib; 921 922 switch (addr->sa_family) { 923 case AF_INET: 924 return ((struct sockaddr_in *) addr)->sin_port; 925 case AF_INET6: 926 return ((struct sockaddr_in6 *) addr)->sin6_port; 927 case AF_IB: 928 sib = (struct sockaddr_ib *) addr; 929 return htons((u16) (be64_to_cpu(sib->sib_sid) & 930 be64_to_cpu(sib->sib_sid_mask))); 931 default: 932 return 0; 933 } 934 } 935 936 static inline int cma_any_port(struct sockaddr *addr) 937 { 938 return !cma_port(addr); 939 } 940 941 static void cma_save_ib_info(struct sockaddr *src_addr, 942 struct sockaddr *dst_addr, 943 struct rdma_cm_id *listen_id, 944 struct ib_sa_path_rec *path) 945 { 946 struct sockaddr_ib *listen_ib, *ib; 947 948 listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr; 949 if (src_addr) { 950 ib = (struct sockaddr_ib *)src_addr; 951 ib->sib_family = AF_IB; 952 if (path) { 953 ib->sib_pkey = path->pkey; 954 ib->sib_flowinfo = path->flow_label; 955 memcpy(&ib->sib_addr, &path->sgid, 16); 956 ib->sib_sid = path->service_id; 957 ib->sib_scope_id = 0; 958 } else { 959 ib->sib_pkey = listen_ib->sib_pkey; 960 ib->sib_flowinfo = listen_ib->sib_flowinfo; 961 ib->sib_addr = listen_ib->sib_addr; 962 ib->sib_sid = listen_ib->sib_sid; 963 ib->sib_scope_id = listen_ib->sib_scope_id; 964 } 965 ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL); 966 } 967 if (dst_addr) { 968 ib = (struct sockaddr_ib *)dst_addr; 969 ib->sib_family = AF_IB; 970 if (path) { 971 ib->sib_pkey = path->pkey; 972 ib->sib_flowinfo = path->flow_label; 973 memcpy(&ib->sib_addr, &path->dgid, 16); 974 } 975 } 976 } 977 978 static void cma_save_ip4_info(struct sockaddr *src_addr, 979 struct sockaddr *dst_addr, 980 struct cma_hdr *hdr, 981 __be16 local_port) 982 { 983 struct sockaddr_in *ip4; 984 985 if (src_addr) { 986 ip4 = (struct sockaddr_in *)src_addr; 987 ip4->sin_family = AF_INET; 988 ip4->sin_addr.s_addr = hdr->dst_addr.ip4.addr; 989 ip4->sin_port = local_port; 990 } 991 992 if (dst_addr) { 993 ip4 = (struct sockaddr_in *)dst_addr; 994 ip4->sin_family = AF_INET; 995 ip4->sin_addr.s_addr = hdr->src_addr.ip4.addr; 996 ip4->sin_port = hdr->port; 997 } 998 } 999 1000 static void cma_save_ip6_info(struct sockaddr *src_addr, 1001 struct sockaddr *dst_addr, 1002 struct cma_hdr *hdr, 1003 __be16 local_port) 1004 { 1005 struct sockaddr_in6 *ip6; 1006 1007 if (src_addr) { 1008 ip6 = (struct sockaddr_in6 *)src_addr; 1009 ip6->sin6_family = AF_INET6; 1010 ip6->sin6_addr = hdr->dst_addr.ip6; 1011 ip6->sin6_port = local_port; 1012 } 1013 1014 if (dst_addr) { 1015 ip6 = (struct sockaddr_in6 *)dst_addr; 1016 ip6->sin6_family = AF_INET6; 1017 ip6->sin6_addr = hdr->src_addr.ip6; 1018 ip6->sin6_port = hdr->port; 1019 } 1020 } 1021 1022 static u16 cma_port_from_service_id(__be64 service_id) 1023 { 1024 return (u16)be64_to_cpu(service_id); 1025 } 1026 1027 static int cma_save_ip_info(struct sockaddr *src_addr, 1028 struct sockaddr *dst_addr, 1029 struct ib_cm_event *ib_event, 1030 __be64 service_id) 1031 { 1032 struct cma_hdr *hdr; 1033 __be16 port; 1034 1035 hdr = ib_event->private_data; 1036 if (hdr->cma_version != CMA_VERSION) 1037 return -EINVAL; 1038 1039 port = htons(cma_port_from_service_id(service_id)); 1040 1041 switch (cma_get_ip_ver(hdr)) { 1042 case 4: 1043 cma_save_ip4_info(src_addr, dst_addr, hdr, port); 1044 break; 1045 case 6: 1046 cma_save_ip6_info(src_addr, dst_addr, hdr, port); 1047 break; 1048 default: 1049 return -EAFNOSUPPORT; 1050 } 1051 1052 return 0; 1053 } 1054 1055 static int cma_save_net_info(struct sockaddr *src_addr, 1056 struct sockaddr *dst_addr, 1057 struct rdma_cm_id *listen_id, 1058 struct ib_cm_event *ib_event, 1059 sa_family_t sa_family, __be64 service_id) 1060 { 1061 if (sa_family == AF_IB) { 1062 if (ib_event->event == IB_CM_REQ_RECEIVED) 1063 cma_save_ib_info(src_addr, dst_addr, listen_id, 1064 ib_event->param.req_rcvd.primary_path); 1065 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) 1066 cma_save_ib_info(src_addr, dst_addr, listen_id, NULL); 1067 return 0; 1068 } 1069 1070 return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id); 1071 } 1072 1073 static int cma_save_req_info(const struct ib_cm_event *ib_event, 1074 struct cma_req_info *req) 1075 { 1076 const struct ib_cm_req_event_param *req_param = 1077 &ib_event->param.req_rcvd; 1078 const struct ib_cm_sidr_req_event_param *sidr_param = 1079 &ib_event->param.sidr_req_rcvd; 1080 1081 switch (ib_event->event) { 1082 case IB_CM_REQ_RECEIVED: 1083 req->device = req_param->listen_id->device; 1084 req->port = req_param->port; 1085 memcpy(&req->local_gid, &req_param->primary_path->sgid, 1086 sizeof(req->local_gid)); 1087 req->has_gid = true; 1088 req->service_id = req_param->primary_path->service_id; 1089 req->pkey = be16_to_cpu(req_param->primary_path->pkey); 1090 break; 1091 case IB_CM_SIDR_REQ_RECEIVED: 1092 req->device = sidr_param->listen_id->device; 1093 req->port = sidr_param->port; 1094 req->has_gid = false; 1095 req->service_id = sidr_param->service_id; 1096 req->pkey = sidr_param->pkey; 1097 break; 1098 default: 1099 return -EINVAL; 1100 } 1101 1102 return 0; 1103 } 1104 1105 static bool validate_ipv4_net_dev(struct net_device *net_dev, 1106 const struct sockaddr_in *dst_addr, 1107 const struct sockaddr_in *src_addr) 1108 { 1109 __be32 daddr = dst_addr->sin_addr.s_addr, 1110 saddr = src_addr->sin_addr.s_addr; 1111 struct fib_result res; 1112 struct flowi4 fl4; 1113 int err; 1114 bool ret; 1115 1116 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1117 ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) || 1118 ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) || 1119 ipv4_is_loopback(saddr)) 1120 return false; 1121 1122 memset(&fl4, 0, sizeof(fl4)); 1123 fl4.flowi4_iif = net_dev->ifindex; 1124 fl4.daddr = daddr; 1125 fl4.saddr = saddr; 1126 1127 rcu_read_lock(); 1128 err = fib_lookup(dev_net(net_dev), &fl4, &res, 0); 1129 if (err) 1130 return false; 1131 1132 ret = FIB_RES_DEV(res) == net_dev; 1133 rcu_read_unlock(); 1134 1135 return ret; 1136 } 1137 1138 static bool validate_ipv6_net_dev(struct net_device *net_dev, 1139 const struct sockaddr_in6 *dst_addr, 1140 const struct sockaddr_in6 *src_addr) 1141 { 1142 #if IS_ENABLED(CONFIG_IPV6) 1143 const int strict = ipv6_addr_type(&dst_addr->sin6_addr) & 1144 IPV6_ADDR_LINKLOCAL; 1145 struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr, 1146 &src_addr->sin6_addr, net_dev->ifindex, 1147 strict); 1148 bool ret; 1149 1150 if (!rt) 1151 return false; 1152 1153 ret = rt->rt6i_idev->dev == net_dev; 1154 ip6_rt_put(rt); 1155 1156 return ret; 1157 #else 1158 return false; 1159 #endif 1160 } 1161 1162 static bool validate_net_dev(struct net_device *net_dev, 1163 const struct sockaddr *daddr, 1164 const struct sockaddr *saddr) 1165 { 1166 const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr; 1167 const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr; 1168 const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr; 1169 const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr; 1170 1171 switch (daddr->sa_family) { 1172 case AF_INET: 1173 return saddr->sa_family == AF_INET && 1174 validate_ipv4_net_dev(net_dev, daddr4, saddr4); 1175 1176 case AF_INET6: 1177 return saddr->sa_family == AF_INET6 && 1178 validate_ipv6_net_dev(net_dev, daddr6, saddr6); 1179 1180 default: 1181 return false; 1182 } 1183 } 1184 1185 static struct net_device *cma_get_net_dev(struct ib_cm_event *ib_event, 1186 const struct cma_req_info *req) 1187 { 1188 struct sockaddr_storage listen_addr_storage, src_addr_storage; 1189 struct sockaddr *listen_addr = (struct sockaddr *)&listen_addr_storage, 1190 *src_addr = (struct sockaddr *)&src_addr_storage; 1191 struct net_device *net_dev; 1192 const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL; 1193 int err; 1194 1195 err = cma_save_ip_info(listen_addr, src_addr, ib_event, 1196 req->service_id); 1197 if (err) 1198 return ERR_PTR(err); 1199 1200 net_dev = ib_get_net_dev_by_params(req->device, req->port, req->pkey, 1201 gid, listen_addr); 1202 if (!net_dev) 1203 return ERR_PTR(-ENODEV); 1204 1205 if (!validate_net_dev(net_dev, listen_addr, src_addr)) { 1206 dev_put(net_dev); 1207 return ERR_PTR(-EHOSTUNREACH); 1208 } 1209 1210 return net_dev; 1211 } 1212 1213 static enum rdma_port_space rdma_ps_from_service_id(__be64 service_id) 1214 { 1215 return (be64_to_cpu(service_id) >> 16) & 0xffff; 1216 } 1217 1218 static bool cma_match_private_data(struct rdma_id_private *id_priv, 1219 const struct cma_hdr *hdr) 1220 { 1221 struct sockaddr *addr = cma_src_addr(id_priv); 1222 __be32 ip4_addr; 1223 struct in6_addr ip6_addr; 1224 1225 if (cma_any_addr(addr) && !id_priv->afonly) 1226 return true; 1227 1228 switch (addr->sa_family) { 1229 case AF_INET: 1230 ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr; 1231 if (cma_get_ip_ver(hdr) != 4) 1232 return false; 1233 if (!cma_any_addr(addr) && 1234 hdr->dst_addr.ip4.addr != ip4_addr) 1235 return false; 1236 break; 1237 case AF_INET6: 1238 ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr; 1239 if (cma_get_ip_ver(hdr) != 6) 1240 return false; 1241 if (!cma_any_addr(addr) && 1242 memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr))) 1243 return false; 1244 break; 1245 case AF_IB: 1246 return true; 1247 default: 1248 return false; 1249 } 1250 1251 return true; 1252 } 1253 1254 static bool cma_protocol_roce_dev_port(struct ib_device *device, int port_num) 1255 { 1256 enum rdma_link_layer ll = rdma_port_get_link_layer(device, port_num); 1257 enum rdma_transport_type transport = 1258 rdma_node_get_transport(device->node_type); 1259 1260 return ll == IB_LINK_LAYER_ETHERNET && transport == RDMA_TRANSPORT_IB; 1261 } 1262 1263 static bool cma_protocol_roce(const struct rdma_cm_id *id) 1264 { 1265 struct ib_device *device = id->device; 1266 const int port_num = id->port_num ?: rdma_start_port(device); 1267 1268 return cma_protocol_roce_dev_port(device, port_num); 1269 } 1270 1271 static bool cma_match_net_dev(const struct rdma_id_private *id_priv, 1272 const struct net_device *net_dev) 1273 { 1274 const struct rdma_addr *addr = &id_priv->id.route.addr; 1275 1276 if (!net_dev) 1277 /* This request is an AF_IB request or a RoCE request */ 1278 return addr->src_addr.ss_family == AF_IB || 1279 cma_protocol_roce(&id_priv->id); 1280 1281 return !addr->dev_addr.bound_dev_if || 1282 (net_eq(dev_net(net_dev), addr->dev_addr.net) && 1283 addr->dev_addr.bound_dev_if == net_dev->ifindex); 1284 } 1285 1286 static struct rdma_id_private *cma_find_listener( 1287 const struct rdma_bind_list *bind_list, 1288 const struct ib_cm_id *cm_id, 1289 const struct ib_cm_event *ib_event, 1290 const struct cma_req_info *req, 1291 const struct net_device *net_dev) 1292 { 1293 struct rdma_id_private *id_priv, *id_priv_dev; 1294 1295 if (!bind_list) 1296 return ERR_PTR(-EINVAL); 1297 1298 hlist_for_each_entry(id_priv, &bind_list->owners, node) { 1299 if (cma_match_private_data(id_priv, ib_event->private_data)) { 1300 if (id_priv->id.device == cm_id->device && 1301 cma_match_net_dev(id_priv, net_dev)) 1302 return id_priv; 1303 list_for_each_entry(id_priv_dev, 1304 &id_priv->listen_list, 1305 listen_list) { 1306 if (id_priv_dev->id.device == cm_id->device && 1307 cma_match_net_dev(id_priv_dev, net_dev)) 1308 return id_priv_dev; 1309 } 1310 } 1311 } 1312 1313 return ERR_PTR(-EINVAL); 1314 } 1315 1316 static struct rdma_id_private *cma_id_from_event(struct ib_cm_id *cm_id, 1317 struct ib_cm_event *ib_event, 1318 struct net_device **net_dev) 1319 { 1320 struct cma_req_info req; 1321 struct rdma_bind_list *bind_list; 1322 struct rdma_id_private *id_priv; 1323 int err; 1324 1325 err = cma_save_req_info(ib_event, &req); 1326 if (err) 1327 return ERR_PTR(err); 1328 1329 *net_dev = cma_get_net_dev(ib_event, &req); 1330 if (IS_ERR(*net_dev)) { 1331 if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) { 1332 /* Assuming the protocol is AF_IB */ 1333 *net_dev = NULL; 1334 } else if (cma_protocol_roce_dev_port(req.device, req.port)) { 1335 /* TODO find the net dev matching the request parameters 1336 * through the RoCE GID table */ 1337 *net_dev = NULL; 1338 } else { 1339 return ERR_CAST(*net_dev); 1340 } 1341 } 1342 1343 bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net, 1344 rdma_ps_from_service_id(req.service_id), 1345 cma_port_from_service_id(req.service_id)); 1346 id_priv = cma_find_listener(bind_list, cm_id, ib_event, &req, *net_dev); 1347 if (IS_ERR(id_priv) && *net_dev) { 1348 dev_put(*net_dev); 1349 *net_dev = NULL; 1350 } 1351 1352 return id_priv; 1353 } 1354 1355 static inline int cma_user_data_offset(struct rdma_id_private *id_priv) 1356 { 1357 return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr); 1358 } 1359 1360 static void cma_cancel_route(struct rdma_id_private *id_priv) 1361 { 1362 if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) { 1363 if (id_priv->query) 1364 ib_sa_cancel_query(id_priv->query_id, id_priv->query); 1365 } 1366 } 1367 1368 static void cma_cancel_listens(struct rdma_id_private *id_priv) 1369 { 1370 struct rdma_id_private *dev_id_priv; 1371 1372 /* 1373 * Remove from listen_any_list to prevent added devices from spawning 1374 * additional listen requests. 1375 */ 1376 mutex_lock(&lock); 1377 list_del(&id_priv->list); 1378 1379 while (!list_empty(&id_priv->listen_list)) { 1380 dev_id_priv = list_entry(id_priv->listen_list.next, 1381 struct rdma_id_private, listen_list); 1382 /* sync with device removal to avoid duplicate destruction */ 1383 list_del_init(&dev_id_priv->list); 1384 list_del(&dev_id_priv->listen_list); 1385 mutex_unlock(&lock); 1386 1387 rdma_destroy_id(&dev_id_priv->id); 1388 mutex_lock(&lock); 1389 } 1390 mutex_unlock(&lock); 1391 } 1392 1393 static void cma_cancel_operation(struct rdma_id_private *id_priv, 1394 enum rdma_cm_state state) 1395 { 1396 switch (state) { 1397 case RDMA_CM_ADDR_QUERY: 1398 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 1399 break; 1400 case RDMA_CM_ROUTE_QUERY: 1401 cma_cancel_route(id_priv); 1402 break; 1403 case RDMA_CM_LISTEN: 1404 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev) 1405 cma_cancel_listens(id_priv); 1406 break; 1407 default: 1408 break; 1409 } 1410 } 1411 1412 static void cma_release_port(struct rdma_id_private *id_priv) 1413 { 1414 struct rdma_bind_list *bind_list = id_priv->bind_list; 1415 struct net *net = id_priv->id.route.addr.dev_addr.net; 1416 1417 if (!bind_list) 1418 return; 1419 1420 mutex_lock(&lock); 1421 hlist_del(&id_priv->node); 1422 if (hlist_empty(&bind_list->owners)) { 1423 cma_ps_remove(net, bind_list->ps, bind_list->port); 1424 kfree(bind_list); 1425 } 1426 mutex_unlock(&lock); 1427 } 1428 1429 static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 1430 { 1431 struct cma_multicast *mc; 1432 1433 while (!list_empty(&id_priv->mc_list)) { 1434 mc = container_of(id_priv->mc_list.next, 1435 struct cma_multicast, list); 1436 list_del(&mc->list); 1437 if (rdma_cap_ib_mcast(id_priv->cma_dev->device, 1438 id_priv->id.port_num)) { 1439 ib_sa_free_multicast(mc->multicast.ib); 1440 kfree(mc); 1441 } else 1442 kref_put(&mc->mcref, release_mc); 1443 } 1444 } 1445 1446 void rdma_destroy_id(struct rdma_cm_id *id) 1447 { 1448 struct rdma_id_private *id_priv; 1449 enum rdma_cm_state state; 1450 1451 id_priv = container_of(id, struct rdma_id_private, id); 1452 state = cma_exch(id_priv, RDMA_CM_DESTROYING); 1453 cma_cancel_operation(id_priv, state); 1454 1455 /* 1456 * Wait for any active callback to finish. New callbacks will find 1457 * the id_priv state set to destroying and abort. 1458 */ 1459 mutex_lock(&id_priv->handler_mutex); 1460 mutex_unlock(&id_priv->handler_mutex); 1461 1462 if (id_priv->cma_dev) { 1463 if (rdma_cap_ib_cm(id_priv->id.device, 1)) { 1464 if (id_priv->cm_id.ib) 1465 ib_destroy_cm_id(id_priv->cm_id.ib); 1466 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) { 1467 if (id_priv->cm_id.iw) 1468 iw_destroy_cm_id(id_priv->cm_id.iw); 1469 } 1470 cma_leave_mc_groups(id_priv); 1471 cma_release_dev(id_priv); 1472 } 1473 1474 cma_release_port(id_priv); 1475 cma_deref_id(id_priv); 1476 wait_for_completion(&id_priv->comp); 1477 1478 if (id_priv->internal_id) 1479 cma_deref_id(id_priv->id.context); 1480 1481 kfree(id_priv->id.route.path_rec); 1482 put_net(id_priv->id.route.addr.dev_addr.net); 1483 kfree(id_priv); 1484 } 1485 EXPORT_SYMBOL(rdma_destroy_id); 1486 1487 static int cma_rep_recv(struct rdma_id_private *id_priv) 1488 { 1489 int ret; 1490 1491 ret = cma_modify_qp_rtr(id_priv, NULL); 1492 if (ret) 1493 goto reject; 1494 1495 ret = cma_modify_qp_rts(id_priv, NULL); 1496 if (ret) 1497 goto reject; 1498 1499 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 1500 if (ret) 1501 goto reject; 1502 1503 return 0; 1504 reject: 1505 cma_modify_qp_err(id_priv); 1506 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 1507 NULL, 0, NULL, 0); 1508 return ret; 1509 } 1510 1511 static void cma_set_rep_event_data(struct rdma_cm_event *event, 1512 struct ib_cm_rep_event_param *rep_data, 1513 void *private_data) 1514 { 1515 event->param.conn.private_data = private_data; 1516 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 1517 event->param.conn.responder_resources = rep_data->responder_resources; 1518 event->param.conn.initiator_depth = rep_data->initiator_depth; 1519 event->param.conn.flow_control = rep_data->flow_control; 1520 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 1521 event->param.conn.srq = rep_data->srq; 1522 event->param.conn.qp_num = rep_data->remote_qpn; 1523 } 1524 1525 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1526 { 1527 struct rdma_id_private *id_priv = cm_id->context; 1528 struct rdma_cm_event event; 1529 int ret = 0; 1530 1531 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && 1532 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) || 1533 (ib_event->event == IB_CM_TIMEWAIT_EXIT && 1534 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT))) 1535 return 0; 1536 1537 memset(&event, 0, sizeof event); 1538 switch (ib_event->event) { 1539 case IB_CM_REQ_ERROR: 1540 case IB_CM_REP_ERROR: 1541 event.event = RDMA_CM_EVENT_UNREACHABLE; 1542 event.status = -ETIMEDOUT; 1543 break; 1544 case IB_CM_REP_RECEIVED: 1545 if (id_priv->id.qp) { 1546 event.status = cma_rep_recv(id_priv); 1547 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 1548 RDMA_CM_EVENT_ESTABLISHED; 1549 } else { 1550 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 1551 } 1552 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 1553 ib_event->private_data); 1554 break; 1555 case IB_CM_RTU_RECEIVED: 1556 case IB_CM_USER_ESTABLISHED: 1557 event.event = RDMA_CM_EVENT_ESTABLISHED; 1558 break; 1559 case IB_CM_DREQ_ERROR: 1560 event.status = -ETIMEDOUT; /* fall through */ 1561 case IB_CM_DREQ_RECEIVED: 1562 case IB_CM_DREP_RECEIVED: 1563 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT, 1564 RDMA_CM_DISCONNECT)) 1565 goto out; 1566 event.event = RDMA_CM_EVENT_DISCONNECTED; 1567 break; 1568 case IB_CM_TIMEWAIT_EXIT: 1569 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; 1570 break; 1571 case IB_CM_MRA_RECEIVED: 1572 /* ignore event */ 1573 goto out; 1574 case IB_CM_REJ_RECEIVED: 1575 cma_modify_qp_err(id_priv); 1576 event.status = ib_event->param.rej_rcvd.reason; 1577 event.event = RDMA_CM_EVENT_REJECTED; 1578 event.param.conn.private_data = ib_event->private_data; 1579 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 1580 break; 1581 default: 1582 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", 1583 ib_event->event); 1584 goto out; 1585 } 1586 1587 ret = id_priv->id.event_handler(&id_priv->id, &event); 1588 if (ret) { 1589 /* Destroy the CM ID by returning a non-zero value. */ 1590 id_priv->cm_id.ib = NULL; 1591 cma_exch(id_priv, RDMA_CM_DESTROYING); 1592 mutex_unlock(&id_priv->handler_mutex); 1593 rdma_destroy_id(&id_priv->id); 1594 return ret; 1595 } 1596 out: 1597 mutex_unlock(&id_priv->handler_mutex); 1598 return ret; 1599 } 1600 1601 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, 1602 struct ib_cm_event *ib_event, 1603 struct net_device *net_dev) 1604 { 1605 struct rdma_id_private *id_priv; 1606 struct rdma_cm_id *id; 1607 struct rdma_route *rt; 1608 const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; 1609 const __be64 service_id = 1610 ib_event->param.req_rcvd.primary_path->service_id; 1611 int ret; 1612 1613 id = rdma_create_id(listen_id->route.addr.dev_addr.net, 1614 listen_id->event_handler, listen_id->context, 1615 listen_id->ps, ib_event->param.req_rcvd.qp_type); 1616 if (IS_ERR(id)) 1617 return NULL; 1618 1619 id_priv = container_of(id, struct rdma_id_private, id); 1620 if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, 1621 (struct sockaddr *)&id->route.addr.dst_addr, 1622 listen_id, ib_event, ss_family, service_id)) 1623 goto err; 1624 1625 rt = &id->route; 1626 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 1627 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, 1628 GFP_KERNEL); 1629 if (!rt->path_rec) 1630 goto err; 1631 1632 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; 1633 if (rt->num_paths == 2) 1634 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 1635 1636 if (net_dev) { 1637 ret = rdma_copy_addr(&rt->addr.dev_addr, net_dev, NULL); 1638 if (ret) 1639 goto err; 1640 } else { 1641 if (!cma_protocol_roce(listen_id) && 1642 cma_any_addr(cma_src_addr(id_priv))) { 1643 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; 1644 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 1645 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey)); 1646 } else if (!cma_any_addr(cma_src_addr(id_priv))) { 1647 ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr); 1648 if (ret) 1649 goto err; 1650 } 1651 } 1652 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 1653 1654 id_priv->state = RDMA_CM_CONNECT; 1655 return id_priv; 1656 1657 err: 1658 rdma_destroy_id(id); 1659 return NULL; 1660 } 1661 1662 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, 1663 struct ib_cm_event *ib_event, 1664 struct net_device *net_dev) 1665 { 1666 struct rdma_id_private *id_priv; 1667 struct rdma_cm_id *id; 1668 const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; 1669 struct net *net = listen_id->route.addr.dev_addr.net; 1670 int ret; 1671 1672 id = rdma_create_id(net, listen_id->event_handler, listen_id->context, 1673 listen_id->ps, IB_QPT_UD); 1674 if (IS_ERR(id)) 1675 return NULL; 1676 1677 id_priv = container_of(id, struct rdma_id_private, id); 1678 if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, 1679 (struct sockaddr *)&id->route.addr.dst_addr, 1680 listen_id, ib_event, ss_family, 1681 ib_event->param.sidr_req_rcvd.service_id)) 1682 goto err; 1683 1684 if (net_dev) { 1685 ret = rdma_copy_addr(&id->route.addr.dev_addr, net_dev, NULL); 1686 if (ret) 1687 goto err; 1688 } else { 1689 if (!cma_any_addr(cma_src_addr(id_priv))) { 1690 ret = cma_translate_addr(cma_src_addr(id_priv), 1691 &id->route.addr.dev_addr); 1692 if (ret) 1693 goto err; 1694 } 1695 } 1696 1697 id_priv->state = RDMA_CM_CONNECT; 1698 return id_priv; 1699 err: 1700 rdma_destroy_id(id); 1701 return NULL; 1702 } 1703 1704 static void cma_set_req_event_data(struct rdma_cm_event *event, 1705 struct ib_cm_req_event_param *req_data, 1706 void *private_data, int offset) 1707 { 1708 event->param.conn.private_data = private_data + offset; 1709 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 1710 event->param.conn.responder_resources = req_data->responder_resources; 1711 event->param.conn.initiator_depth = req_data->initiator_depth; 1712 event->param.conn.flow_control = req_data->flow_control; 1713 event->param.conn.retry_count = req_data->retry_count; 1714 event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 1715 event->param.conn.srq = req_data->srq; 1716 event->param.conn.qp_num = req_data->remote_qpn; 1717 } 1718 1719 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event) 1720 { 1721 return (((ib_event->event == IB_CM_REQ_RECEIVED) && 1722 (ib_event->param.req_rcvd.qp_type == id->qp_type)) || 1723 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) && 1724 (id->qp_type == IB_QPT_UD)) || 1725 (!id->qp_type)); 1726 } 1727 1728 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1729 { 1730 struct rdma_id_private *listen_id, *conn_id; 1731 struct rdma_cm_event event; 1732 struct net_device *net_dev; 1733 int offset, ret; 1734 1735 listen_id = cma_id_from_event(cm_id, ib_event, &net_dev); 1736 if (IS_ERR(listen_id)) 1737 return PTR_ERR(listen_id); 1738 1739 if (!cma_check_req_qp_type(&listen_id->id, ib_event)) { 1740 ret = -EINVAL; 1741 goto net_dev_put; 1742 } 1743 1744 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN)) { 1745 ret = -ECONNABORTED; 1746 goto net_dev_put; 1747 } 1748 1749 memset(&event, 0, sizeof event); 1750 offset = cma_user_data_offset(listen_id); 1751 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1752 if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) { 1753 conn_id = cma_new_udp_id(&listen_id->id, ib_event, net_dev); 1754 event.param.ud.private_data = ib_event->private_data + offset; 1755 event.param.ud.private_data_len = 1756 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 1757 } else { 1758 conn_id = cma_new_conn_id(&listen_id->id, ib_event, net_dev); 1759 cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 1760 ib_event->private_data, offset); 1761 } 1762 if (!conn_id) { 1763 ret = -ENOMEM; 1764 goto err1; 1765 } 1766 1767 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1768 ret = cma_acquire_dev(conn_id, listen_id); 1769 if (ret) 1770 goto err2; 1771 1772 conn_id->cm_id.ib = cm_id; 1773 cm_id->context = conn_id; 1774 cm_id->cm_handler = cma_ib_handler; 1775 1776 /* 1777 * Protect against the user destroying conn_id from another thread 1778 * until we're done accessing it. 1779 */ 1780 atomic_inc(&conn_id->refcount); 1781 ret = conn_id->id.event_handler(&conn_id->id, &event); 1782 if (ret) 1783 goto err3; 1784 /* 1785 * Acquire mutex to prevent user executing rdma_destroy_id() 1786 * while we're accessing the cm_id. 1787 */ 1788 mutex_lock(&lock); 1789 if (cma_comp(conn_id, RDMA_CM_CONNECT) && 1790 (conn_id->id.qp_type != IB_QPT_UD)) 1791 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 1792 mutex_unlock(&lock); 1793 mutex_unlock(&conn_id->handler_mutex); 1794 mutex_unlock(&listen_id->handler_mutex); 1795 cma_deref_id(conn_id); 1796 if (net_dev) 1797 dev_put(net_dev); 1798 return 0; 1799 1800 err3: 1801 cma_deref_id(conn_id); 1802 /* Destroy the CM ID by returning a non-zero value. */ 1803 conn_id->cm_id.ib = NULL; 1804 err2: 1805 cma_exch(conn_id, RDMA_CM_DESTROYING); 1806 mutex_unlock(&conn_id->handler_mutex); 1807 err1: 1808 mutex_unlock(&listen_id->handler_mutex); 1809 if (conn_id) 1810 rdma_destroy_id(&conn_id->id); 1811 1812 net_dev_put: 1813 if (net_dev) 1814 dev_put(net_dev); 1815 1816 return ret; 1817 } 1818 1819 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr) 1820 { 1821 if (addr->sa_family == AF_IB) 1822 return ((struct sockaddr_ib *) addr)->sib_sid; 1823 1824 return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr))); 1825 } 1826 EXPORT_SYMBOL(rdma_get_service_id); 1827 1828 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 1829 { 1830 struct rdma_id_private *id_priv = iw_id->context; 1831 struct rdma_cm_event event; 1832 int ret = 0; 1833 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; 1834 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; 1835 1836 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT)) 1837 return 0; 1838 1839 memset(&event, 0, sizeof event); 1840 switch (iw_event->event) { 1841 case IW_CM_EVENT_CLOSE: 1842 event.event = RDMA_CM_EVENT_DISCONNECTED; 1843 break; 1844 case IW_CM_EVENT_CONNECT_REPLY: 1845 memcpy(cma_src_addr(id_priv), laddr, 1846 rdma_addr_size(laddr)); 1847 memcpy(cma_dst_addr(id_priv), raddr, 1848 rdma_addr_size(raddr)); 1849 switch (iw_event->status) { 1850 case 0: 1851 event.event = RDMA_CM_EVENT_ESTABLISHED; 1852 event.param.conn.initiator_depth = iw_event->ird; 1853 event.param.conn.responder_resources = iw_event->ord; 1854 break; 1855 case -ECONNRESET: 1856 case -ECONNREFUSED: 1857 event.event = RDMA_CM_EVENT_REJECTED; 1858 break; 1859 case -ETIMEDOUT: 1860 event.event = RDMA_CM_EVENT_UNREACHABLE; 1861 break; 1862 default: 1863 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1864 break; 1865 } 1866 break; 1867 case IW_CM_EVENT_ESTABLISHED: 1868 event.event = RDMA_CM_EVENT_ESTABLISHED; 1869 event.param.conn.initiator_depth = iw_event->ird; 1870 event.param.conn.responder_resources = iw_event->ord; 1871 break; 1872 default: 1873 BUG_ON(1); 1874 } 1875 1876 event.status = iw_event->status; 1877 event.param.conn.private_data = iw_event->private_data; 1878 event.param.conn.private_data_len = iw_event->private_data_len; 1879 ret = id_priv->id.event_handler(&id_priv->id, &event); 1880 if (ret) { 1881 /* Destroy the CM ID by returning a non-zero value. */ 1882 id_priv->cm_id.iw = NULL; 1883 cma_exch(id_priv, RDMA_CM_DESTROYING); 1884 mutex_unlock(&id_priv->handler_mutex); 1885 rdma_destroy_id(&id_priv->id); 1886 return ret; 1887 } 1888 1889 mutex_unlock(&id_priv->handler_mutex); 1890 return ret; 1891 } 1892 1893 static int iw_conn_req_handler(struct iw_cm_id *cm_id, 1894 struct iw_cm_event *iw_event) 1895 { 1896 struct rdma_cm_id *new_cm_id; 1897 struct rdma_id_private *listen_id, *conn_id; 1898 struct rdma_cm_event event; 1899 int ret; 1900 struct ib_device_attr attr; 1901 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; 1902 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; 1903 1904 listen_id = cm_id->context; 1905 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN)) 1906 return -ECONNABORTED; 1907 1908 /* Create a new RDMA id for the new IW CM ID */ 1909 new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net, 1910 listen_id->id.event_handler, 1911 listen_id->id.context, 1912 RDMA_PS_TCP, IB_QPT_RC); 1913 if (IS_ERR(new_cm_id)) { 1914 ret = -ENOMEM; 1915 goto out; 1916 } 1917 conn_id = container_of(new_cm_id, struct rdma_id_private, id); 1918 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1919 conn_id->state = RDMA_CM_CONNECT; 1920 1921 ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr, NULL); 1922 if (ret) { 1923 mutex_unlock(&conn_id->handler_mutex); 1924 rdma_destroy_id(new_cm_id); 1925 goto out; 1926 } 1927 1928 ret = cma_acquire_dev(conn_id, listen_id); 1929 if (ret) { 1930 mutex_unlock(&conn_id->handler_mutex); 1931 rdma_destroy_id(new_cm_id); 1932 goto out; 1933 } 1934 1935 conn_id->cm_id.iw = cm_id; 1936 cm_id->context = conn_id; 1937 cm_id->cm_handler = cma_iw_handler; 1938 1939 memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr)); 1940 memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr)); 1941 1942 ret = ib_query_device(conn_id->id.device, &attr); 1943 if (ret) { 1944 mutex_unlock(&conn_id->handler_mutex); 1945 rdma_destroy_id(new_cm_id); 1946 goto out; 1947 } 1948 1949 memset(&event, 0, sizeof event); 1950 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1951 event.param.conn.private_data = iw_event->private_data; 1952 event.param.conn.private_data_len = iw_event->private_data_len; 1953 event.param.conn.initiator_depth = iw_event->ird; 1954 event.param.conn.responder_resources = iw_event->ord; 1955 1956 /* 1957 * Protect against the user destroying conn_id from another thread 1958 * until we're done accessing it. 1959 */ 1960 atomic_inc(&conn_id->refcount); 1961 ret = conn_id->id.event_handler(&conn_id->id, &event); 1962 if (ret) { 1963 /* User wants to destroy the CM ID */ 1964 conn_id->cm_id.iw = NULL; 1965 cma_exch(conn_id, RDMA_CM_DESTROYING); 1966 mutex_unlock(&conn_id->handler_mutex); 1967 cma_deref_id(conn_id); 1968 rdma_destroy_id(&conn_id->id); 1969 goto out; 1970 } 1971 1972 mutex_unlock(&conn_id->handler_mutex); 1973 cma_deref_id(conn_id); 1974 1975 out: 1976 mutex_unlock(&listen_id->handler_mutex); 1977 return ret; 1978 } 1979 1980 static int cma_ib_listen(struct rdma_id_private *id_priv) 1981 { 1982 struct sockaddr *addr; 1983 struct ib_cm_id *id; 1984 __be64 svc_id; 1985 1986 addr = cma_src_addr(id_priv); 1987 svc_id = rdma_get_service_id(&id_priv->id, addr); 1988 id = ib_cm_insert_listen(id_priv->id.device, cma_req_handler, svc_id); 1989 if (IS_ERR(id)) 1990 return PTR_ERR(id); 1991 id_priv->cm_id.ib = id; 1992 1993 return 0; 1994 } 1995 1996 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 1997 { 1998 int ret; 1999 struct iw_cm_id *id; 2000 2001 id = iw_create_cm_id(id_priv->id.device, 2002 iw_conn_req_handler, 2003 id_priv); 2004 if (IS_ERR(id)) 2005 return PTR_ERR(id); 2006 2007 id->tos = id_priv->tos; 2008 id_priv->cm_id.iw = id; 2009 2010 memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv), 2011 rdma_addr_size(cma_src_addr(id_priv))); 2012 2013 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 2014 2015 if (ret) { 2016 iw_destroy_cm_id(id_priv->cm_id.iw); 2017 id_priv->cm_id.iw = NULL; 2018 } 2019 2020 return ret; 2021 } 2022 2023 static int cma_listen_handler(struct rdma_cm_id *id, 2024 struct rdma_cm_event *event) 2025 { 2026 struct rdma_id_private *id_priv = id->context; 2027 2028 id->context = id_priv->id.context; 2029 id->event_handler = id_priv->id.event_handler; 2030 return id_priv->id.event_handler(id, event); 2031 } 2032 2033 static void cma_listen_on_dev(struct rdma_id_private *id_priv, 2034 struct cma_device *cma_dev) 2035 { 2036 struct rdma_id_private *dev_id_priv; 2037 struct rdma_cm_id *id; 2038 struct net *net = id_priv->id.route.addr.dev_addr.net; 2039 int ret; 2040 2041 if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1)) 2042 return; 2043 2044 id = rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps, 2045 id_priv->id.qp_type); 2046 if (IS_ERR(id)) 2047 return; 2048 2049 dev_id_priv = container_of(id, struct rdma_id_private, id); 2050 2051 dev_id_priv->state = RDMA_CM_ADDR_BOUND; 2052 memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv), 2053 rdma_addr_size(cma_src_addr(id_priv))); 2054 2055 cma_attach_to_dev(dev_id_priv, cma_dev); 2056 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); 2057 atomic_inc(&id_priv->refcount); 2058 dev_id_priv->internal_id = 1; 2059 dev_id_priv->afonly = id_priv->afonly; 2060 2061 ret = rdma_listen(id, id_priv->backlog); 2062 if (ret) 2063 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, " 2064 "listening on device %s\n", ret, cma_dev->device->name); 2065 } 2066 2067 static void cma_listen_on_all(struct rdma_id_private *id_priv) 2068 { 2069 struct cma_device *cma_dev; 2070 2071 mutex_lock(&lock); 2072 list_add_tail(&id_priv->list, &listen_any_list); 2073 list_for_each_entry(cma_dev, &dev_list, list) 2074 cma_listen_on_dev(id_priv, cma_dev); 2075 mutex_unlock(&lock); 2076 } 2077 2078 void rdma_set_service_type(struct rdma_cm_id *id, int tos) 2079 { 2080 struct rdma_id_private *id_priv; 2081 2082 id_priv = container_of(id, struct rdma_id_private, id); 2083 id_priv->tos = (u8) tos; 2084 } 2085 EXPORT_SYMBOL(rdma_set_service_type); 2086 2087 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, 2088 void *context) 2089 { 2090 struct cma_work *work = context; 2091 struct rdma_route *route; 2092 2093 route = &work->id->id.route; 2094 2095 if (!status) { 2096 route->num_paths = 1; 2097 *route->path_rec = *path_rec; 2098 } else { 2099 work->old_state = RDMA_CM_ROUTE_QUERY; 2100 work->new_state = RDMA_CM_ADDR_RESOLVED; 2101 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 2102 work->event.status = status; 2103 } 2104 2105 queue_work(cma_wq, &work->work); 2106 } 2107 2108 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, 2109 struct cma_work *work) 2110 { 2111 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2112 struct ib_sa_path_rec path_rec; 2113 ib_sa_comp_mask comp_mask; 2114 struct sockaddr_in6 *sin6; 2115 struct sockaddr_ib *sib; 2116 2117 memset(&path_rec, 0, sizeof path_rec); 2118 rdma_addr_get_sgid(dev_addr, &path_rec.sgid); 2119 rdma_addr_get_dgid(dev_addr, &path_rec.dgid); 2120 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 2121 path_rec.numb_path = 1; 2122 path_rec.reversible = 1; 2123 path_rec.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 2124 2125 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 2126 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 2127 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; 2128 2129 switch (cma_family(id_priv)) { 2130 case AF_INET: 2131 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); 2132 comp_mask |= IB_SA_PATH_REC_QOS_CLASS; 2133 break; 2134 case AF_INET6: 2135 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); 2136 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); 2137 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 2138 break; 2139 case AF_IB: 2140 sib = (struct sockaddr_ib *) cma_src_addr(id_priv); 2141 path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20); 2142 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 2143 break; 2144 } 2145 2146 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 2147 id_priv->id.port_num, &path_rec, 2148 comp_mask, timeout_ms, 2149 GFP_KERNEL, cma_query_handler, 2150 work, &id_priv->query); 2151 2152 return (id_priv->query_id < 0) ? id_priv->query_id : 0; 2153 } 2154 2155 static void cma_work_handler(struct work_struct *_work) 2156 { 2157 struct cma_work *work = container_of(_work, struct cma_work, work); 2158 struct rdma_id_private *id_priv = work->id; 2159 int destroy = 0; 2160 2161 mutex_lock(&id_priv->handler_mutex); 2162 if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 2163 goto out; 2164 2165 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 2166 cma_exch(id_priv, RDMA_CM_DESTROYING); 2167 destroy = 1; 2168 } 2169 out: 2170 mutex_unlock(&id_priv->handler_mutex); 2171 cma_deref_id(id_priv); 2172 if (destroy) 2173 rdma_destroy_id(&id_priv->id); 2174 kfree(work); 2175 } 2176 2177 static void cma_ndev_work_handler(struct work_struct *_work) 2178 { 2179 struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); 2180 struct rdma_id_private *id_priv = work->id; 2181 int destroy = 0; 2182 2183 mutex_lock(&id_priv->handler_mutex); 2184 if (id_priv->state == RDMA_CM_DESTROYING || 2185 id_priv->state == RDMA_CM_DEVICE_REMOVAL) 2186 goto out; 2187 2188 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 2189 cma_exch(id_priv, RDMA_CM_DESTROYING); 2190 destroy = 1; 2191 } 2192 2193 out: 2194 mutex_unlock(&id_priv->handler_mutex); 2195 cma_deref_id(id_priv); 2196 if (destroy) 2197 rdma_destroy_id(&id_priv->id); 2198 kfree(work); 2199 } 2200 2201 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) 2202 { 2203 struct rdma_route *route = &id_priv->id.route; 2204 struct cma_work *work; 2205 int ret; 2206 2207 work = kzalloc(sizeof *work, GFP_KERNEL); 2208 if (!work) 2209 return -ENOMEM; 2210 2211 work->id = id_priv; 2212 INIT_WORK(&work->work, cma_work_handler); 2213 work->old_state = RDMA_CM_ROUTE_QUERY; 2214 work->new_state = RDMA_CM_ROUTE_RESOLVED; 2215 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 2216 2217 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 2218 if (!route->path_rec) { 2219 ret = -ENOMEM; 2220 goto err1; 2221 } 2222 2223 ret = cma_query_ib_route(id_priv, timeout_ms, work); 2224 if (ret) 2225 goto err2; 2226 2227 return 0; 2228 err2: 2229 kfree(route->path_rec); 2230 route->path_rec = NULL; 2231 err1: 2232 kfree(work); 2233 return ret; 2234 } 2235 2236 int rdma_set_ib_paths(struct rdma_cm_id *id, 2237 struct ib_sa_path_rec *path_rec, int num_paths) 2238 { 2239 struct rdma_id_private *id_priv; 2240 int ret; 2241 2242 id_priv = container_of(id, struct rdma_id_private, id); 2243 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, 2244 RDMA_CM_ROUTE_RESOLVED)) 2245 return -EINVAL; 2246 2247 id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths, 2248 GFP_KERNEL); 2249 if (!id->route.path_rec) { 2250 ret = -ENOMEM; 2251 goto err; 2252 } 2253 2254 id->route.num_paths = num_paths; 2255 return 0; 2256 err: 2257 cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED); 2258 return ret; 2259 } 2260 EXPORT_SYMBOL(rdma_set_ib_paths); 2261 2262 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) 2263 { 2264 struct cma_work *work; 2265 2266 work = kzalloc(sizeof *work, GFP_KERNEL); 2267 if (!work) 2268 return -ENOMEM; 2269 2270 work->id = id_priv; 2271 INIT_WORK(&work->work, cma_work_handler); 2272 work->old_state = RDMA_CM_ROUTE_QUERY; 2273 work->new_state = RDMA_CM_ROUTE_RESOLVED; 2274 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 2275 queue_work(cma_wq, &work->work); 2276 return 0; 2277 } 2278 2279 static int iboe_tos_to_sl(struct net_device *ndev, int tos) 2280 { 2281 int prio; 2282 struct net_device *dev; 2283 2284 prio = rt_tos2priority(tos); 2285 dev = ndev->priv_flags & IFF_802_1Q_VLAN ? 2286 vlan_dev_real_dev(ndev) : ndev; 2287 2288 if (dev->num_tc) 2289 return netdev_get_prio_tc_map(dev, prio); 2290 2291 #if IS_ENABLED(CONFIG_VLAN_8021Q) 2292 if (ndev->priv_flags & IFF_802_1Q_VLAN) 2293 return (vlan_dev_get_egress_qos_mask(ndev, prio) & 2294 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 2295 #endif 2296 return 0; 2297 } 2298 2299 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) 2300 { 2301 struct rdma_route *route = &id_priv->id.route; 2302 struct rdma_addr *addr = &route->addr; 2303 struct cma_work *work; 2304 int ret; 2305 struct net_device *ndev = NULL; 2306 2307 2308 work = kzalloc(sizeof *work, GFP_KERNEL); 2309 if (!work) 2310 return -ENOMEM; 2311 2312 work->id = id_priv; 2313 INIT_WORK(&work->work, cma_work_handler); 2314 2315 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); 2316 if (!route->path_rec) { 2317 ret = -ENOMEM; 2318 goto err1; 2319 } 2320 2321 route->num_paths = 1; 2322 2323 if (addr->dev_addr.bound_dev_if) { 2324 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if); 2325 route->path_rec->net = &init_net; 2326 route->path_rec->ifindex = addr->dev_addr.bound_dev_if; 2327 } 2328 if (!ndev) { 2329 ret = -ENODEV; 2330 goto err2; 2331 } 2332 2333 memcpy(route->path_rec->dmac, addr->dev_addr.dst_dev_addr, ETH_ALEN); 2334 2335 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 2336 &route->path_rec->sgid); 2337 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr, 2338 &route->path_rec->dgid); 2339 2340 route->path_rec->hop_limit = 1; 2341 route->path_rec->reversible = 1; 2342 route->path_rec->pkey = cpu_to_be16(0xffff); 2343 route->path_rec->mtu_selector = IB_SA_EQ; 2344 route->path_rec->sl = iboe_tos_to_sl(ndev, id_priv->tos); 2345 route->path_rec->mtu = iboe_get_mtu(ndev->mtu); 2346 route->path_rec->rate_selector = IB_SA_EQ; 2347 route->path_rec->rate = iboe_get_rate(ndev); 2348 dev_put(ndev); 2349 route->path_rec->packet_life_time_selector = IB_SA_EQ; 2350 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME; 2351 if (!route->path_rec->mtu) { 2352 ret = -EINVAL; 2353 goto err2; 2354 } 2355 2356 work->old_state = RDMA_CM_ROUTE_QUERY; 2357 work->new_state = RDMA_CM_ROUTE_RESOLVED; 2358 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 2359 work->event.status = 0; 2360 2361 queue_work(cma_wq, &work->work); 2362 2363 return 0; 2364 2365 err2: 2366 kfree(route->path_rec); 2367 route->path_rec = NULL; 2368 err1: 2369 kfree(work); 2370 return ret; 2371 } 2372 2373 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) 2374 { 2375 struct rdma_id_private *id_priv; 2376 int ret; 2377 2378 id_priv = container_of(id, struct rdma_id_private, id); 2379 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY)) 2380 return -EINVAL; 2381 2382 atomic_inc(&id_priv->refcount); 2383 if (rdma_cap_ib_sa(id->device, id->port_num)) 2384 ret = cma_resolve_ib_route(id_priv, timeout_ms); 2385 else if (rdma_protocol_roce(id->device, id->port_num)) 2386 ret = cma_resolve_iboe_route(id_priv); 2387 else if (rdma_protocol_iwarp(id->device, id->port_num)) 2388 ret = cma_resolve_iw_route(id_priv, timeout_ms); 2389 else 2390 ret = -ENOSYS; 2391 2392 if (ret) 2393 goto err; 2394 2395 return 0; 2396 err: 2397 cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED); 2398 cma_deref_id(id_priv); 2399 return ret; 2400 } 2401 EXPORT_SYMBOL(rdma_resolve_route); 2402 2403 static void cma_set_loopback(struct sockaddr *addr) 2404 { 2405 switch (addr->sa_family) { 2406 case AF_INET: 2407 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); 2408 break; 2409 case AF_INET6: 2410 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr, 2411 0, 0, 0, htonl(1)); 2412 break; 2413 default: 2414 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr, 2415 0, 0, 0, htonl(1)); 2416 break; 2417 } 2418 } 2419 2420 static int cma_bind_loopback(struct rdma_id_private *id_priv) 2421 { 2422 struct cma_device *cma_dev, *cur_dev; 2423 struct ib_port_attr port_attr; 2424 union ib_gid gid; 2425 u16 pkey; 2426 int ret; 2427 u8 p; 2428 2429 cma_dev = NULL; 2430 mutex_lock(&lock); 2431 list_for_each_entry(cur_dev, &dev_list, list) { 2432 if (cma_family(id_priv) == AF_IB && 2433 !rdma_cap_ib_cm(cur_dev->device, 1)) 2434 continue; 2435 2436 if (!cma_dev) 2437 cma_dev = cur_dev; 2438 2439 for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) { 2440 if (!ib_query_port(cur_dev->device, p, &port_attr) && 2441 port_attr.state == IB_PORT_ACTIVE) { 2442 cma_dev = cur_dev; 2443 goto port_found; 2444 } 2445 } 2446 } 2447 2448 if (!cma_dev) { 2449 ret = -ENODEV; 2450 goto out; 2451 } 2452 2453 p = 1; 2454 2455 port_found: 2456 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid, NULL); 2457 if (ret) 2458 goto out; 2459 2460 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 2461 if (ret) 2462 goto out; 2463 2464 id_priv->id.route.addr.dev_addr.dev_type = 2465 (rdma_protocol_ib(cma_dev->device, p)) ? 2466 ARPHRD_INFINIBAND : ARPHRD_ETHER; 2467 2468 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 2469 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 2470 id_priv->id.port_num = p; 2471 cma_attach_to_dev(id_priv, cma_dev); 2472 cma_set_loopback(cma_src_addr(id_priv)); 2473 out: 2474 mutex_unlock(&lock); 2475 return ret; 2476 } 2477 2478 static void addr_handler(int status, struct sockaddr *src_addr, 2479 struct rdma_dev_addr *dev_addr, void *context) 2480 { 2481 struct rdma_id_private *id_priv = context; 2482 struct rdma_cm_event event; 2483 2484 memset(&event, 0, sizeof event); 2485 mutex_lock(&id_priv->handler_mutex); 2486 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, 2487 RDMA_CM_ADDR_RESOLVED)) 2488 goto out; 2489 2490 memcpy(cma_src_addr(id_priv), src_addr, rdma_addr_size(src_addr)); 2491 if (!status && !id_priv->cma_dev) 2492 status = cma_acquire_dev(id_priv, NULL); 2493 2494 if (status) { 2495 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, 2496 RDMA_CM_ADDR_BOUND)) 2497 goto out; 2498 event.event = RDMA_CM_EVENT_ADDR_ERROR; 2499 event.status = status; 2500 } else 2501 event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2502 2503 if (id_priv->id.event_handler(&id_priv->id, &event)) { 2504 cma_exch(id_priv, RDMA_CM_DESTROYING); 2505 mutex_unlock(&id_priv->handler_mutex); 2506 cma_deref_id(id_priv); 2507 rdma_destroy_id(&id_priv->id); 2508 return; 2509 } 2510 out: 2511 mutex_unlock(&id_priv->handler_mutex); 2512 cma_deref_id(id_priv); 2513 } 2514 2515 static int cma_resolve_loopback(struct rdma_id_private *id_priv) 2516 { 2517 struct cma_work *work; 2518 union ib_gid gid; 2519 int ret; 2520 2521 work = kzalloc(sizeof *work, GFP_KERNEL); 2522 if (!work) 2523 return -ENOMEM; 2524 2525 if (!id_priv->cma_dev) { 2526 ret = cma_bind_loopback(id_priv); 2527 if (ret) 2528 goto err; 2529 } 2530 2531 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 2532 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 2533 2534 work->id = id_priv; 2535 INIT_WORK(&work->work, cma_work_handler); 2536 work->old_state = RDMA_CM_ADDR_QUERY; 2537 work->new_state = RDMA_CM_ADDR_RESOLVED; 2538 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2539 queue_work(cma_wq, &work->work); 2540 return 0; 2541 err: 2542 kfree(work); 2543 return ret; 2544 } 2545 2546 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) 2547 { 2548 struct cma_work *work; 2549 int ret; 2550 2551 work = kzalloc(sizeof *work, GFP_KERNEL); 2552 if (!work) 2553 return -ENOMEM; 2554 2555 if (!id_priv->cma_dev) { 2556 ret = cma_resolve_ib_dev(id_priv); 2557 if (ret) 2558 goto err; 2559 } 2560 2561 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) 2562 &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); 2563 2564 work->id = id_priv; 2565 INIT_WORK(&work->work, cma_work_handler); 2566 work->old_state = RDMA_CM_ADDR_QUERY; 2567 work->new_state = RDMA_CM_ADDR_RESOLVED; 2568 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2569 queue_work(cma_wq, &work->work); 2570 return 0; 2571 err: 2572 kfree(work); 2573 return ret; 2574 } 2575 2576 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 2577 struct sockaddr *dst_addr) 2578 { 2579 if (!src_addr || !src_addr->sa_family) { 2580 src_addr = (struct sockaddr *) &id->route.addr.src_addr; 2581 src_addr->sa_family = dst_addr->sa_family; 2582 if (dst_addr->sa_family == AF_INET6) { 2583 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr; 2584 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr; 2585 src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; 2586 if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL) 2587 id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id; 2588 } else if (dst_addr->sa_family == AF_IB) { 2589 ((struct sockaddr_ib *) src_addr)->sib_pkey = 2590 ((struct sockaddr_ib *) dst_addr)->sib_pkey; 2591 } 2592 } 2593 return rdma_bind_addr(id, src_addr); 2594 } 2595 2596 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 2597 struct sockaddr *dst_addr, int timeout_ms) 2598 { 2599 struct rdma_id_private *id_priv; 2600 int ret; 2601 2602 id_priv = container_of(id, struct rdma_id_private, id); 2603 if (id_priv->state == RDMA_CM_IDLE) { 2604 ret = cma_bind_addr(id, src_addr, dst_addr); 2605 if (ret) 2606 return ret; 2607 } 2608 2609 if (cma_family(id_priv) != dst_addr->sa_family) 2610 return -EINVAL; 2611 2612 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) 2613 return -EINVAL; 2614 2615 atomic_inc(&id_priv->refcount); 2616 memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); 2617 if (cma_any_addr(dst_addr)) { 2618 ret = cma_resolve_loopback(id_priv); 2619 } else { 2620 if (dst_addr->sa_family == AF_IB) { 2621 ret = cma_resolve_ib_addr(id_priv); 2622 } else { 2623 ret = rdma_resolve_ip(&addr_client, cma_src_addr(id_priv), 2624 dst_addr, &id->route.addr.dev_addr, 2625 timeout_ms, addr_handler, id_priv); 2626 } 2627 } 2628 if (ret) 2629 goto err; 2630 2631 return 0; 2632 err: 2633 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 2634 cma_deref_id(id_priv); 2635 return ret; 2636 } 2637 EXPORT_SYMBOL(rdma_resolve_addr); 2638 2639 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse) 2640 { 2641 struct rdma_id_private *id_priv; 2642 unsigned long flags; 2643 int ret; 2644 2645 id_priv = container_of(id, struct rdma_id_private, id); 2646 spin_lock_irqsave(&id_priv->lock, flags); 2647 if (reuse || id_priv->state == RDMA_CM_IDLE) { 2648 id_priv->reuseaddr = reuse; 2649 ret = 0; 2650 } else { 2651 ret = -EINVAL; 2652 } 2653 spin_unlock_irqrestore(&id_priv->lock, flags); 2654 return ret; 2655 } 2656 EXPORT_SYMBOL(rdma_set_reuseaddr); 2657 2658 int rdma_set_afonly(struct rdma_cm_id *id, int afonly) 2659 { 2660 struct rdma_id_private *id_priv; 2661 unsigned long flags; 2662 int ret; 2663 2664 id_priv = container_of(id, struct rdma_id_private, id); 2665 spin_lock_irqsave(&id_priv->lock, flags); 2666 if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) { 2667 id_priv->options |= (1 << CMA_OPTION_AFONLY); 2668 id_priv->afonly = afonly; 2669 ret = 0; 2670 } else { 2671 ret = -EINVAL; 2672 } 2673 spin_unlock_irqrestore(&id_priv->lock, flags); 2674 return ret; 2675 } 2676 EXPORT_SYMBOL(rdma_set_afonly); 2677 2678 static void cma_bind_port(struct rdma_bind_list *bind_list, 2679 struct rdma_id_private *id_priv) 2680 { 2681 struct sockaddr *addr; 2682 struct sockaddr_ib *sib; 2683 u64 sid, mask; 2684 __be16 port; 2685 2686 addr = cma_src_addr(id_priv); 2687 port = htons(bind_list->port); 2688 2689 switch (addr->sa_family) { 2690 case AF_INET: 2691 ((struct sockaddr_in *) addr)->sin_port = port; 2692 break; 2693 case AF_INET6: 2694 ((struct sockaddr_in6 *) addr)->sin6_port = port; 2695 break; 2696 case AF_IB: 2697 sib = (struct sockaddr_ib *) addr; 2698 sid = be64_to_cpu(sib->sib_sid); 2699 mask = be64_to_cpu(sib->sib_sid_mask); 2700 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port)); 2701 sib->sib_sid_mask = cpu_to_be64(~0ULL); 2702 break; 2703 } 2704 id_priv->bind_list = bind_list; 2705 hlist_add_head(&id_priv->node, &bind_list->owners); 2706 } 2707 2708 static int cma_alloc_port(enum rdma_port_space ps, 2709 struct rdma_id_private *id_priv, unsigned short snum) 2710 { 2711 struct rdma_bind_list *bind_list; 2712 int ret; 2713 2714 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 2715 if (!bind_list) 2716 return -ENOMEM; 2717 2718 ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list, 2719 snum); 2720 if (ret < 0) 2721 goto err; 2722 2723 bind_list->ps = ps; 2724 bind_list->port = (unsigned short)ret; 2725 cma_bind_port(bind_list, id_priv); 2726 return 0; 2727 err: 2728 kfree(bind_list); 2729 return ret == -ENOSPC ? -EADDRNOTAVAIL : ret; 2730 } 2731 2732 static int cma_alloc_any_port(enum rdma_port_space ps, 2733 struct rdma_id_private *id_priv) 2734 { 2735 static unsigned int last_used_port; 2736 int low, high, remaining; 2737 unsigned int rover; 2738 struct net *net = id_priv->id.route.addr.dev_addr.net; 2739 2740 inet_get_local_port_range(net, &low, &high); 2741 remaining = (high - low) + 1; 2742 rover = prandom_u32() % remaining + low; 2743 retry: 2744 if (last_used_port != rover && 2745 !cma_ps_find(net, ps, (unsigned short)rover)) { 2746 int ret = cma_alloc_port(ps, id_priv, rover); 2747 /* 2748 * Remember previously used port number in order to avoid 2749 * re-using same port immediately after it is closed. 2750 */ 2751 if (!ret) 2752 last_used_port = rover; 2753 if (ret != -EADDRNOTAVAIL) 2754 return ret; 2755 } 2756 if (--remaining) { 2757 rover++; 2758 if ((rover < low) || (rover > high)) 2759 rover = low; 2760 goto retry; 2761 } 2762 return -EADDRNOTAVAIL; 2763 } 2764 2765 /* 2766 * Check that the requested port is available. This is called when trying to 2767 * bind to a specific port, or when trying to listen on a bound port. In 2768 * the latter case, the provided id_priv may already be on the bind_list, but 2769 * we still need to check that it's okay to start listening. 2770 */ 2771 static int cma_check_port(struct rdma_bind_list *bind_list, 2772 struct rdma_id_private *id_priv, uint8_t reuseaddr) 2773 { 2774 struct rdma_id_private *cur_id; 2775 struct sockaddr *addr, *cur_addr; 2776 2777 addr = cma_src_addr(id_priv); 2778 hlist_for_each_entry(cur_id, &bind_list->owners, node) { 2779 if (id_priv == cur_id) 2780 continue; 2781 2782 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr && 2783 cur_id->reuseaddr) 2784 continue; 2785 2786 cur_addr = cma_src_addr(cur_id); 2787 if (id_priv->afonly && cur_id->afonly && 2788 (addr->sa_family != cur_addr->sa_family)) 2789 continue; 2790 2791 if (cma_any_addr(addr) || cma_any_addr(cur_addr)) 2792 return -EADDRNOTAVAIL; 2793 2794 if (!cma_addr_cmp(addr, cur_addr)) 2795 return -EADDRINUSE; 2796 } 2797 return 0; 2798 } 2799 2800 static int cma_use_port(enum rdma_port_space ps, 2801 struct rdma_id_private *id_priv) 2802 { 2803 struct rdma_bind_list *bind_list; 2804 unsigned short snum; 2805 int ret; 2806 2807 snum = ntohs(cma_port(cma_src_addr(id_priv))); 2808 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 2809 return -EACCES; 2810 2811 bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum); 2812 if (!bind_list) { 2813 ret = cma_alloc_port(ps, id_priv, snum); 2814 } else { 2815 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr); 2816 if (!ret) 2817 cma_bind_port(bind_list, id_priv); 2818 } 2819 return ret; 2820 } 2821 2822 static int cma_bind_listen(struct rdma_id_private *id_priv) 2823 { 2824 struct rdma_bind_list *bind_list = id_priv->bind_list; 2825 int ret = 0; 2826 2827 mutex_lock(&lock); 2828 if (bind_list->owners.first->next) 2829 ret = cma_check_port(bind_list, id_priv, 0); 2830 mutex_unlock(&lock); 2831 return ret; 2832 } 2833 2834 static enum rdma_port_space cma_select_inet_ps( 2835 struct rdma_id_private *id_priv) 2836 { 2837 switch (id_priv->id.ps) { 2838 case RDMA_PS_TCP: 2839 case RDMA_PS_UDP: 2840 case RDMA_PS_IPOIB: 2841 case RDMA_PS_IB: 2842 return id_priv->id.ps; 2843 default: 2844 2845 return 0; 2846 } 2847 } 2848 2849 static enum rdma_port_space cma_select_ib_ps(struct rdma_id_private *id_priv) 2850 { 2851 enum rdma_port_space ps = 0; 2852 struct sockaddr_ib *sib; 2853 u64 sid_ps, mask, sid; 2854 2855 sib = (struct sockaddr_ib *) cma_src_addr(id_priv); 2856 mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK; 2857 sid = be64_to_cpu(sib->sib_sid) & mask; 2858 2859 if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) { 2860 sid_ps = RDMA_IB_IP_PS_IB; 2861 ps = RDMA_PS_IB; 2862 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) && 2863 (sid == (RDMA_IB_IP_PS_TCP & mask))) { 2864 sid_ps = RDMA_IB_IP_PS_TCP; 2865 ps = RDMA_PS_TCP; 2866 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) && 2867 (sid == (RDMA_IB_IP_PS_UDP & mask))) { 2868 sid_ps = RDMA_IB_IP_PS_UDP; 2869 ps = RDMA_PS_UDP; 2870 } 2871 2872 if (ps) { 2873 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib))); 2874 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK | 2875 be64_to_cpu(sib->sib_sid_mask)); 2876 } 2877 return ps; 2878 } 2879 2880 static int cma_get_port(struct rdma_id_private *id_priv) 2881 { 2882 enum rdma_port_space ps; 2883 int ret; 2884 2885 if (cma_family(id_priv) != AF_IB) 2886 ps = cma_select_inet_ps(id_priv); 2887 else 2888 ps = cma_select_ib_ps(id_priv); 2889 if (!ps) 2890 return -EPROTONOSUPPORT; 2891 2892 mutex_lock(&lock); 2893 if (cma_any_port(cma_src_addr(id_priv))) 2894 ret = cma_alloc_any_port(ps, id_priv); 2895 else 2896 ret = cma_use_port(ps, id_priv); 2897 mutex_unlock(&lock); 2898 2899 return ret; 2900 } 2901 2902 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, 2903 struct sockaddr *addr) 2904 { 2905 #if IS_ENABLED(CONFIG_IPV6) 2906 struct sockaddr_in6 *sin6; 2907 2908 if (addr->sa_family != AF_INET6) 2909 return 0; 2910 2911 sin6 = (struct sockaddr_in6 *) addr; 2912 2913 if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL)) 2914 return 0; 2915 2916 if (!sin6->sin6_scope_id) 2917 return -EINVAL; 2918 2919 dev_addr->bound_dev_if = sin6->sin6_scope_id; 2920 #endif 2921 return 0; 2922 } 2923 2924 int rdma_listen(struct rdma_cm_id *id, int backlog) 2925 { 2926 struct rdma_id_private *id_priv; 2927 int ret; 2928 2929 id_priv = container_of(id, struct rdma_id_private, id); 2930 if (id_priv->state == RDMA_CM_IDLE) { 2931 id->route.addr.src_addr.ss_family = AF_INET; 2932 ret = rdma_bind_addr(id, cma_src_addr(id_priv)); 2933 if (ret) 2934 return ret; 2935 } 2936 2937 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) 2938 return -EINVAL; 2939 2940 if (id_priv->reuseaddr) { 2941 ret = cma_bind_listen(id_priv); 2942 if (ret) 2943 goto err; 2944 } 2945 2946 id_priv->backlog = backlog; 2947 if (id->device) { 2948 if (rdma_cap_ib_cm(id->device, 1)) { 2949 ret = cma_ib_listen(id_priv); 2950 if (ret) 2951 goto err; 2952 } else if (rdma_cap_iw_cm(id->device, 1)) { 2953 ret = cma_iw_listen(id_priv, backlog); 2954 if (ret) 2955 goto err; 2956 } else { 2957 ret = -ENOSYS; 2958 goto err; 2959 } 2960 } else 2961 cma_listen_on_all(id_priv); 2962 2963 return 0; 2964 err: 2965 id_priv->backlog = 0; 2966 cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND); 2967 return ret; 2968 } 2969 EXPORT_SYMBOL(rdma_listen); 2970 2971 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 2972 { 2973 struct rdma_id_private *id_priv; 2974 int ret; 2975 2976 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 && 2977 addr->sa_family != AF_IB) 2978 return -EAFNOSUPPORT; 2979 2980 id_priv = container_of(id, struct rdma_id_private, id); 2981 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND)) 2982 return -EINVAL; 2983 2984 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); 2985 if (ret) 2986 goto err1; 2987 2988 memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr)); 2989 if (!cma_any_addr(addr)) { 2990 ret = cma_translate_addr(addr, &id->route.addr.dev_addr); 2991 if (ret) 2992 goto err1; 2993 2994 ret = cma_acquire_dev(id_priv, NULL); 2995 if (ret) 2996 goto err1; 2997 } 2998 2999 if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) { 3000 if (addr->sa_family == AF_INET) 3001 id_priv->afonly = 1; 3002 #if IS_ENABLED(CONFIG_IPV6) 3003 else if (addr->sa_family == AF_INET6) { 3004 struct net *net = id_priv->id.route.addr.dev_addr.net; 3005 3006 id_priv->afonly = net->ipv6.sysctl.bindv6only; 3007 } 3008 #endif 3009 } 3010 ret = cma_get_port(id_priv); 3011 if (ret) 3012 goto err2; 3013 3014 return 0; 3015 err2: 3016 if (id_priv->cma_dev) 3017 cma_release_dev(id_priv); 3018 err1: 3019 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE); 3020 return ret; 3021 } 3022 EXPORT_SYMBOL(rdma_bind_addr); 3023 3024 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv) 3025 { 3026 struct cma_hdr *cma_hdr; 3027 3028 cma_hdr = hdr; 3029 cma_hdr->cma_version = CMA_VERSION; 3030 if (cma_family(id_priv) == AF_INET) { 3031 struct sockaddr_in *src4, *dst4; 3032 3033 src4 = (struct sockaddr_in *) cma_src_addr(id_priv); 3034 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv); 3035 3036 cma_set_ip_ver(cma_hdr, 4); 3037 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 3038 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 3039 cma_hdr->port = src4->sin_port; 3040 } else if (cma_family(id_priv) == AF_INET6) { 3041 struct sockaddr_in6 *src6, *dst6; 3042 3043 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); 3044 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv); 3045 3046 cma_set_ip_ver(cma_hdr, 6); 3047 cma_hdr->src_addr.ip6 = src6->sin6_addr; 3048 cma_hdr->dst_addr.ip6 = dst6->sin6_addr; 3049 cma_hdr->port = src6->sin6_port; 3050 } 3051 return 0; 3052 } 3053 3054 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 3055 struct ib_cm_event *ib_event) 3056 { 3057 struct rdma_id_private *id_priv = cm_id->context; 3058 struct rdma_cm_event event; 3059 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; 3060 int ret = 0; 3061 3062 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT)) 3063 return 0; 3064 3065 memset(&event, 0, sizeof event); 3066 switch (ib_event->event) { 3067 case IB_CM_SIDR_REQ_ERROR: 3068 event.event = RDMA_CM_EVENT_UNREACHABLE; 3069 event.status = -ETIMEDOUT; 3070 break; 3071 case IB_CM_SIDR_REP_RECEIVED: 3072 event.param.ud.private_data = ib_event->private_data; 3073 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 3074 if (rep->status != IB_SIDR_SUCCESS) { 3075 event.event = RDMA_CM_EVENT_UNREACHABLE; 3076 event.status = ib_event->param.sidr_rep_rcvd.status; 3077 break; 3078 } 3079 ret = cma_set_qkey(id_priv, rep->qkey); 3080 if (ret) { 3081 event.event = RDMA_CM_EVENT_ADDR_ERROR; 3082 event.status = ret; 3083 break; 3084 } 3085 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, 3086 id_priv->id.route.path_rec, 3087 &event.param.ud.ah_attr); 3088 event.param.ud.qp_num = rep->qpn; 3089 event.param.ud.qkey = rep->qkey; 3090 event.event = RDMA_CM_EVENT_ESTABLISHED; 3091 event.status = 0; 3092 break; 3093 default: 3094 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", 3095 ib_event->event); 3096 goto out; 3097 } 3098 3099 ret = id_priv->id.event_handler(&id_priv->id, &event); 3100 if (ret) { 3101 /* Destroy the CM ID by returning a non-zero value. */ 3102 id_priv->cm_id.ib = NULL; 3103 cma_exch(id_priv, RDMA_CM_DESTROYING); 3104 mutex_unlock(&id_priv->handler_mutex); 3105 rdma_destroy_id(&id_priv->id); 3106 return ret; 3107 } 3108 out: 3109 mutex_unlock(&id_priv->handler_mutex); 3110 return ret; 3111 } 3112 3113 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 3114 struct rdma_conn_param *conn_param) 3115 { 3116 struct ib_cm_sidr_req_param req; 3117 struct ib_cm_id *id; 3118 void *private_data; 3119 int offset, ret; 3120 3121 memset(&req, 0, sizeof req); 3122 offset = cma_user_data_offset(id_priv); 3123 req.private_data_len = offset + conn_param->private_data_len; 3124 if (req.private_data_len < conn_param->private_data_len) 3125 return -EINVAL; 3126 3127 if (req.private_data_len) { 3128 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 3129 if (!private_data) 3130 return -ENOMEM; 3131 } else { 3132 private_data = NULL; 3133 } 3134 3135 if (conn_param->private_data && conn_param->private_data_len) 3136 memcpy(private_data + offset, conn_param->private_data, 3137 conn_param->private_data_len); 3138 3139 if (private_data) { 3140 ret = cma_format_hdr(private_data, id_priv); 3141 if (ret) 3142 goto out; 3143 req.private_data = private_data; 3144 } 3145 3146 id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler, 3147 id_priv); 3148 if (IS_ERR(id)) { 3149 ret = PTR_ERR(id); 3150 goto out; 3151 } 3152 id_priv->cm_id.ib = id; 3153 3154 req.path = id_priv->id.route.path_rec; 3155 req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 3156 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); 3157 req.max_cm_retries = CMA_MAX_CM_RETRIES; 3158 3159 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 3160 if (ret) { 3161 ib_destroy_cm_id(id_priv->cm_id.ib); 3162 id_priv->cm_id.ib = NULL; 3163 } 3164 out: 3165 kfree(private_data); 3166 return ret; 3167 } 3168 3169 static int cma_connect_ib(struct rdma_id_private *id_priv, 3170 struct rdma_conn_param *conn_param) 3171 { 3172 struct ib_cm_req_param req; 3173 struct rdma_route *route; 3174 void *private_data; 3175 struct ib_cm_id *id; 3176 int offset, ret; 3177 3178 memset(&req, 0, sizeof req); 3179 offset = cma_user_data_offset(id_priv); 3180 req.private_data_len = offset + conn_param->private_data_len; 3181 if (req.private_data_len < conn_param->private_data_len) 3182 return -EINVAL; 3183 3184 if (req.private_data_len) { 3185 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 3186 if (!private_data) 3187 return -ENOMEM; 3188 } else { 3189 private_data = NULL; 3190 } 3191 3192 if (conn_param->private_data && conn_param->private_data_len) 3193 memcpy(private_data + offset, conn_param->private_data, 3194 conn_param->private_data_len); 3195 3196 id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv); 3197 if (IS_ERR(id)) { 3198 ret = PTR_ERR(id); 3199 goto out; 3200 } 3201 id_priv->cm_id.ib = id; 3202 3203 route = &id_priv->id.route; 3204 if (private_data) { 3205 ret = cma_format_hdr(private_data, id_priv); 3206 if (ret) 3207 goto out; 3208 req.private_data = private_data; 3209 } 3210 3211 req.primary_path = &route->path_rec[0]; 3212 if (route->num_paths == 2) 3213 req.alternate_path = &route->path_rec[1]; 3214 3215 req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 3216 req.qp_num = id_priv->qp_num; 3217 req.qp_type = id_priv->id.qp_type; 3218 req.starting_psn = id_priv->seq_num; 3219 req.responder_resources = conn_param->responder_resources; 3220 req.initiator_depth = conn_param->initiator_depth; 3221 req.flow_control = conn_param->flow_control; 3222 req.retry_count = min_t(u8, 7, conn_param->retry_count); 3223 req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); 3224 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 3225 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 3226 req.max_cm_retries = CMA_MAX_CM_RETRIES; 3227 req.srq = id_priv->srq ? 1 : 0; 3228 3229 ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 3230 out: 3231 if (ret && !IS_ERR(id)) { 3232 ib_destroy_cm_id(id); 3233 id_priv->cm_id.ib = NULL; 3234 } 3235 3236 kfree(private_data); 3237 return ret; 3238 } 3239 3240 static int cma_connect_iw(struct rdma_id_private *id_priv, 3241 struct rdma_conn_param *conn_param) 3242 { 3243 struct iw_cm_id *cm_id; 3244 int ret; 3245 struct iw_cm_conn_param iw_param; 3246 3247 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); 3248 if (IS_ERR(cm_id)) 3249 return PTR_ERR(cm_id); 3250 3251 cm_id->tos = id_priv->tos; 3252 id_priv->cm_id.iw = cm_id; 3253 3254 memcpy(&cm_id->local_addr, cma_src_addr(id_priv), 3255 rdma_addr_size(cma_src_addr(id_priv))); 3256 memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv), 3257 rdma_addr_size(cma_dst_addr(id_priv))); 3258 3259 ret = cma_modify_qp_rtr(id_priv, conn_param); 3260 if (ret) 3261 goto out; 3262 3263 if (conn_param) { 3264 iw_param.ord = conn_param->initiator_depth; 3265 iw_param.ird = conn_param->responder_resources; 3266 iw_param.private_data = conn_param->private_data; 3267 iw_param.private_data_len = conn_param->private_data_len; 3268 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num; 3269 } else { 3270 memset(&iw_param, 0, sizeof iw_param); 3271 iw_param.qpn = id_priv->qp_num; 3272 } 3273 ret = iw_cm_connect(cm_id, &iw_param); 3274 out: 3275 if (ret) { 3276 iw_destroy_cm_id(cm_id); 3277 id_priv->cm_id.iw = NULL; 3278 } 3279 return ret; 3280 } 3281 3282 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 3283 { 3284 struct rdma_id_private *id_priv; 3285 int ret; 3286 3287 id_priv = container_of(id, struct rdma_id_private, id); 3288 if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT)) 3289 return -EINVAL; 3290 3291 if (!id->qp) { 3292 id_priv->qp_num = conn_param->qp_num; 3293 id_priv->srq = conn_param->srq; 3294 } 3295 3296 if (rdma_cap_ib_cm(id->device, id->port_num)) { 3297 if (id->qp_type == IB_QPT_UD) 3298 ret = cma_resolve_ib_udp(id_priv, conn_param); 3299 else 3300 ret = cma_connect_ib(id_priv, conn_param); 3301 } else if (rdma_cap_iw_cm(id->device, id->port_num)) 3302 ret = cma_connect_iw(id_priv, conn_param); 3303 else 3304 ret = -ENOSYS; 3305 if (ret) 3306 goto err; 3307 3308 return 0; 3309 err: 3310 cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED); 3311 return ret; 3312 } 3313 EXPORT_SYMBOL(rdma_connect); 3314 3315 static int cma_accept_ib(struct rdma_id_private *id_priv, 3316 struct rdma_conn_param *conn_param) 3317 { 3318 struct ib_cm_rep_param rep; 3319 int ret; 3320 3321 ret = cma_modify_qp_rtr(id_priv, conn_param); 3322 if (ret) 3323 goto out; 3324 3325 ret = cma_modify_qp_rts(id_priv, conn_param); 3326 if (ret) 3327 goto out; 3328 3329 memset(&rep, 0, sizeof rep); 3330 rep.qp_num = id_priv->qp_num; 3331 rep.starting_psn = id_priv->seq_num; 3332 rep.private_data = conn_param->private_data; 3333 rep.private_data_len = conn_param->private_data_len; 3334 rep.responder_resources = conn_param->responder_resources; 3335 rep.initiator_depth = conn_param->initiator_depth; 3336 rep.failover_accepted = 0; 3337 rep.flow_control = conn_param->flow_control; 3338 rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); 3339 rep.srq = id_priv->srq ? 1 : 0; 3340 3341 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 3342 out: 3343 return ret; 3344 } 3345 3346 static int cma_accept_iw(struct rdma_id_private *id_priv, 3347 struct rdma_conn_param *conn_param) 3348 { 3349 struct iw_cm_conn_param iw_param; 3350 int ret; 3351 3352 ret = cma_modify_qp_rtr(id_priv, conn_param); 3353 if (ret) 3354 return ret; 3355 3356 iw_param.ord = conn_param->initiator_depth; 3357 iw_param.ird = conn_param->responder_resources; 3358 iw_param.private_data = conn_param->private_data; 3359 iw_param.private_data_len = conn_param->private_data_len; 3360 if (id_priv->id.qp) { 3361 iw_param.qpn = id_priv->qp_num; 3362 } else 3363 iw_param.qpn = conn_param->qp_num; 3364 3365 return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 3366 } 3367 3368 static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 3369 enum ib_cm_sidr_status status, u32 qkey, 3370 const void *private_data, int private_data_len) 3371 { 3372 struct ib_cm_sidr_rep_param rep; 3373 int ret; 3374 3375 memset(&rep, 0, sizeof rep); 3376 rep.status = status; 3377 if (status == IB_SIDR_SUCCESS) { 3378 ret = cma_set_qkey(id_priv, qkey); 3379 if (ret) 3380 return ret; 3381 rep.qp_num = id_priv->qp_num; 3382 rep.qkey = id_priv->qkey; 3383 } 3384 rep.private_data = private_data; 3385 rep.private_data_len = private_data_len; 3386 3387 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 3388 } 3389 3390 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 3391 { 3392 struct rdma_id_private *id_priv; 3393 int ret; 3394 3395 id_priv = container_of(id, struct rdma_id_private, id); 3396 3397 id_priv->owner = task_pid_nr(current); 3398 3399 if (!cma_comp(id_priv, RDMA_CM_CONNECT)) 3400 return -EINVAL; 3401 3402 if (!id->qp && conn_param) { 3403 id_priv->qp_num = conn_param->qp_num; 3404 id_priv->srq = conn_param->srq; 3405 } 3406 3407 if (rdma_cap_ib_cm(id->device, id->port_num)) { 3408 if (id->qp_type == IB_QPT_UD) { 3409 if (conn_param) 3410 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 3411 conn_param->qkey, 3412 conn_param->private_data, 3413 conn_param->private_data_len); 3414 else 3415 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 3416 0, NULL, 0); 3417 } else { 3418 if (conn_param) 3419 ret = cma_accept_ib(id_priv, conn_param); 3420 else 3421 ret = cma_rep_recv(id_priv); 3422 } 3423 } else if (rdma_cap_iw_cm(id->device, id->port_num)) 3424 ret = cma_accept_iw(id_priv, conn_param); 3425 else 3426 ret = -ENOSYS; 3427 3428 if (ret) 3429 goto reject; 3430 3431 return 0; 3432 reject: 3433 cma_modify_qp_err(id_priv); 3434 rdma_reject(id, NULL, 0); 3435 return ret; 3436 } 3437 EXPORT_SYMBOL(rdma_accept); 3438 3439 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 3440 { 3441 struct rdma_id_private *id_priv; 3442 int ret; 3443 3444 id_priv = container_of(id, struct rdma_id_private, id); 3445 if (!id_priv->cm_id.ib) 3446 return -EINVAL; 3447 3448 switch (id->device->node_type) { 3449 case RDMA_NODE_IB_CA: 3450 ret = ib_cm_notify(id_priv->cm_id.ib, event); 3451 break; 3452 default: 3453 ret = 0; 3454 break; 3455 } 3456 return ret; 3457 } 3458 EXPORT_SYMBOL(rdma_notify); 3459 3460 int rdma_reject(struct rdma_cm_id *id, const void *private_data, 3461 u8 private_data_len) 3462 { 3463 struct rdma_id_private *id_priv; 3464 int ret; 3465 3466 id_priv = container_of(id, struct rdma_id_private, id); 3467 if (!id_priv->cm_id.ib) 3468 return -EINVAL; 3469 3470 if (rdma_cap_ib_cm(id->device, id->port_num)) { 3471 if (id->qp_type == IB_QPT_UD) 3472 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0, 3473 private_data, private_data_len); 3474 else 3475 ret = ib_send_cm_rej(id_priv->cm_id.ib, 3476 IB_CM_REJ_CONSUMER_DEFINED, NULL, 3477 0, private_data, private_data_len); 3478 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 3479 ret = iw_cm_reject(id_priv->cm_id.iw, 3480 private_data, private_data_len); 3481 } else 3482 ret = -ENOSYS; 3483 3484 return ret; 3485 } 3486 EXPORT_SYMBOL(rdma_reject); 3487 3488 int rdma_disconnect(struct rdma_cm_id *id) 3489 { 3490 struct rdma_id_private *id_priv; 3491 int ret; 3492 3493 id_priv = container_of(id, struct rdma_id_private, id); 3494 if (!id_priv->cm_id.ib) 3495 return -EINVAL; 3496 3497 if (rdma_cap_ib_cm(id->device, id->port_num)) { 3498 ret = cma_modify_qp_err(id_priv); 3499 if (ret) 3500 goto out; 3501 /* Initiate or respond to a disconnect. */ 3502 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) 3503 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); 3504 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 3505 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 3506 } else 3507 ret = -EINVAL; 3508 3509 out: 3510 return ret; 3511 } 3512 EXPORT_SYMBOL(rdma_disconnect); 3513 3514 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 3515 { 3516 struct rdma_id_private *id_priv; 3517 struct cma_multicast *mc = multicast->context; 3518 struct rdma_cm_event event; 3519 int ret; 3520 3521 id_priv = mc->id_priv; 3522 if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) && 3523 cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED)) 3524 return 0; 3525 3526 if (!status) 3527 status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey)); 3528 mutex_lock(&id_priv->qp_mutex); 3529 if (!status && id_priv->id.qp) 3530 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, 3531 be16_to_cpu(multicast->rec.mlid)); 3532 mutex_unlock(&id_priv->qp_mutex); 3533 3534 memset(&event, 0, sizeof event); 3535 event.status = status; 3536 event.param.ud.private_data = mc->context; 3537 if (!status) { 3538 event.event = RDMA_CM_EVENT_MULTICAST_JOIN; 3539 ib_init_ah_from_mcmember(id_priv->id.device, 3540 id_priv->id.port_num, &multicast->rec, 3541 &event.param.ud.ah_attr); 3542 event.param.ud.qp_num = 0xFFFFFF; 3543 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 3544 } else 3545 event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 3546 3547 ret = id_priv->id.event_handler(&id_priv->id, &event); 3548 if (ret) { 3549 cma_exch(id_priv, RDMA_CM_DESTROYING); 3550 mutex_unlock(&id_priv->handler_mutex); 3551 rdma_destroy_id(&id_priv->id); 3552 return 0; 3553 } 3554 3555 mutex_unlock(&id_priv->handler_mutex); 3556 return 0; 3557 } 3558 3559 static void cma_set_mgid(struct rdma_id_private *id_priv, 3560 struct sockaddr *addr, union ib_gid *mgid) 3561 { 3562 unsigned char mc_map[MAX_ADDR_LEN]; 3563 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 3564 struct sockaddr_in *sin = (struct sockaddr_in *) addr; 3565 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 3566 3567 if (cma_any_addr(addr)) { 3568 memset(mgid, 0, sizeof *mgid); 3569 } else if ((addr->sa_family == AF_INET6) && 3570 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 3571 0xFF10A01B)) { 3572 /* IPv6 address is an SA assigned MGID. */ 3573 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 3574 } else if (addr->sa_family == AF_IB) { 3575 memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid); 3576 } else if ((addr->sa_family == AF_INET6)) { 3577 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); 3578 if (id_priv->id.ps == RDMA_PS_UDP) 3579 mc_map[7] = 0x01; /* Use RDMA CM signature */ 3580 *mgid = *(union ib_gid *) (mc_map + 4); 3581 } else { 3582 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); 3583 if (id_priv->id.ps == RDMA_PS_UDP) 3584 mc_map[7] = 0x01; /* Use RDMA CM signature */ 3585 *mgid = *(union ib_gid *) (mc_map + 4); 3586 } 3587 } 3588 3589 static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 3590 struct cma_multicast *mc) 3591 { 3592 struct ib_sa_mcmember_rec rec; 3593 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 3594 ib_sa_comp_mask comp_mask; 3595 int ret; 3596 3597 ib_addr_get_mgid(dev_addr, &rec.mgid); 3598 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 3599 &rec.mgid, &rec); 3600 if (ret) 3601 return ret; 3602 3603 ret = cma_set_qkey(id_priv, 0); 3604 if (ret) 3605 return ret; 3606 3607 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); 3608 rec.qkey = cpu_to_be32(id_priv->qkey); 3609 rdma_addr_get_sgid(dev_addr, &rec.port_gid); 3610 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 3611 rec.join_state = 1; 3612 3613 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 3614 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 3615 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 3616 IB_SA_MCMEMBER_REC_FLOW_LABEL | 3617 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 3618 3619 if (id_priv->id.ps == RDMA_PS_IPOIB) 3620 comp_mask |= IB_SA_MCMEMBER_REC_RATE | 3621 IB_SA_MCMEMBER_REC_RATE_SELECTOR | 3622 IB_SA_MCMEMBER_REC_MTU_SELECTOR | 3623 IB_SA_MCMEMBER_REC_MTU | 3624 IB_SA_MCMEMBER_REC_HOP_LIMIT; 3625 3626 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, 3627 id_priv->id.port_num, &rec, 3628 comp_mask, GFP_KERNEL, 3629 cma_ib_mc_handler, mc); 3630 return PTR_ERR_OR_ZERO(mc->multicast.ib); 3631 } 3632 3633 static void iboe_mcast_work_handler(struct work_struct *work) 3634 { 3635 struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); 3636 struct cma_multicast *mc = mw->mc; 3637 struct ib_sa_multicast *m = mc->multicast.ib; 3638 3639 mc->multicast.ib->context = mc; 3640 cma_ib_mc_handler(0, m); 3641 kref_put(&mc->mcref, release_mc); 3642 kfree(mw); 3643 } 3644 3645 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid) 3646 { 3647 struct sockaddr_in *sin = (struct sockaddr_in *)addr; 3648 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; 3649 3650 if (cma_any_addr(addr)) { 3651 memset(mgid, 0, sizeof *mgid); 3652 } else if (addr->sa_family == AF_INET6) { 3653 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 3654 } else { 3655 mgid->raw[0] = 0xff; 3656 mgid->raw[1] = 0x0e; 3657 mgid->raw[2] = 0; 3658 mgid->raw[3] = 0; 3659 mgid->raw[4] = 0; 3660 mgid->raw[5] = 0; 3661 mgid->raw[6] = 0; 3662 mgid->raw[7] = 0; 3663 mgid->raw[8] = 0; 3664 mgid->raw[9] = 0; 3665 mgid->raw[10] = 0xff; 3666 mgid->raw[11] = 0xff; 3667 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; 3668 } 3669 } 3670 3671 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, 3672 struct cma_multicast *mc) 3673 { 3674 struct iboe_mcast_work *work; 3675 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 3676 int err; 3677 struct sockaddr *addr = (struct sockaddr *)&mc->addr; 3678 struct net_device *ndev = NULL; 3679 3680 if (cma_zero_addr((struct sockaddr *)&mc->addr)) 3681 return -EINVAL; 3682 3683 work = kzalloc(sizeof *work, GFP_KERNEL); 3684 if (!work) 3685 return -ENOMEM; 3686 3687 mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); 3688 if (!mc->multicast.ib) { 3689 err = -ENOMEM; 3690 goto out1; 3691 } 3692 3693 cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid); 3694 3695 mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); 3696 if (id_priv->id.ps == RDMA_PS_UDP) 3697 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 3698 3699 if (dev_addr->bound_dev_if) 3700 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if); 3701 if (!ndev) { 3702 err = -ENODEV; 3703 goto out2; 3704 } 3705 mc->multicast.ib->rec.rate = iboe_get_rate(ndev); 3706 mc->multicast.ib->rec.hop_limit = 1; 3707 mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu); 3708 dev_put(ndev); 3709 if (!mc->multicast.ib->rec.mtu) { 3710 err = -EINVAL; 3711 goto out2; 3712 } 3713 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 3714 &mc->multicast.ib->rec.port_gid); 3715 work->id = id_priv; 3716 work->mc = mc; 3717 INIT_WORK(&work->work, iboe_mcast_work_handler); 3718 kref_get(&mc->mcref); 3719 queue_work(cma_wq, &work->work); 3720 3721 return 0; 3722 3723 out2: 3724 kfree(mc->multicast.ib); 3725 out1: 3726 kfree(work); 3727 return err; 3728 } 3729 3730 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 3731 void *context) 3732 { 3733 struct rdma_id_private *id_priv; 3734 struct cma_multicast *mc; 3735 int ret; 3736 3737 id_priv = container_of(id, struct rdma_id_private, id); 3738 if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) && 3739 !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED)) 3740 return -EINVAL; 3741 3742 mc = kmalloc(sizeof *mc, GFP_KERNEL); 3743 if (!mc) 3744 return -ENOMEM; 3745 3746 memcpy(&mc->addr, addr, rdma_addr_size(addr)); 3747 mc->context = context; 3748 mc->id_priv = id_priv; 3749 3750 spin_lock(&id_priv->lock); 3751 list_add(&mc->list, &id_priv->mc_list); 3752 spin_unlock(&id_priv->lock); 3753 3754 if (rdma_protocol_roce(id->device, id->port_num)) { 3755 kref_init(&mc->mcref); 3756 ret = cma_iboe_join_multicast(id_priv, mc); 3757 } else if (rdma_cap_ib_mcast(id->device, id->port_num)) 3758 ret = cma_join_ib_multicast(id_priv, mc); 3759 else 3760 ret = -ENOSYS; 3761 3762 if (ret) { 3763 spin_lock_irq(&id_priv->lock); 3764 list_del(&mc->list); 3765 spin_unlock_irq(&id_priv->lock); 3766 kfree(mc); 3767 } 3768 return ret; 3769 } 3770 EXPORT_SYMBOL(rdma_join_multicast); 3771 3772 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 3773 { 3774 struct rdma_id_private *id_priv; 3775 struct cma_multicast *mc; 3776 3777 id_priv = container_of(id, struct rdma_id_private, id); 3778 spin_lock_irq(&id_priv->lock); 3779 list_for_each_entry(mc, &id_priv->mc_list, list) { 3780 if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) { 3781 list_del(&mc->list); 3782 spin_unlock_irq(&id_priv->lock); 3783 3784 if (id->qp) 3785 ib_detach_mcast(id->qp, 3786 &mc->multicast.ib->rec.mgid, 3787 be16_to_cpu(mc->multicast.ib->rec.mlid)); 3788 3789 BUG_ON(id_priv->cma_dev->device != id->device); 3790 3791 if (rdma_cap_ib_mcast(id->device, id->port_num)) { 3792 ib_sa_free_multicast(mc->multicast.ib); 3793 kfree(mc); 3794 } else if (rdma_protocol_roce(id->device, id->port_num)) 3795 kref_put(&mc->mcref, release_mc); 3796 3797 return; 3798 } 3799 } 3800 spin_unlock_irq(&id_priv->lock); 3801 } 3802 EXPORT_SYMBOL(rdma_leave_multicast); 3803 3804 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) 3805 { 3806 struct rdma_dev_addr *dev_addr; 3807 struct cma_ndev_work *work; 3808 3809 dev_addr = &id_priv->id.route.addr.dev_addr; 3810 3811 if ((dev_addr->bound_dev_if == ndev->ifindex) && 3812 (net_eq(dev_net(ndev), dev_addr->net)) && 3813 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { 3814 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", 3815 ndev->name, &id_priv->id); 3816 work = kzalloc(sizeof *work, GFP_KERNEL); 3817 if (!work) 3818 return -ENOMEM; 3819 3820 INIT_WORK(&work->work, cma_ndev_work_handler); 3821 work->id = id_priv; 3822 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; 3823 atomic_inc(&id_priv->refcount); 3824 queue_work(cma_wq, &work->work); 3825 } 3826 3827 return 0; 3828 } 3829 3830 static int cma_netdev_callback(struct notifier_block *self, unsigned long event, 3831 void *ptr) 3832 { 3833 struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 3834 struct cma_device *cma_dev; 3835 struct rdma_id_private *id_priv; 3836 int ret = NOTIFY_DONE; 3837 3838 if (event != NETDEV_BONDING_FAILOVER) 3839 return NOTIFY_DONE; 3840 3841 if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING)) 3842 return NOTIFY_DONE; 3843 3844 mutex_lock(&lock); 3845 list_for_each_entry(cma_dev, &dev_list, list) 3846 list_for_each_entry(id_priv, &cma_dev->id_list, list) { 3847 ret = cma_netdev_change(ndev, id_priv); 3848 if (ret) 3849 goto out; 3850 } 3851 3852 out: 3853 mutex_unlock(&lock); 3854 return ret; 3855 } 3856 3857 static struct notifier_block cma_nb = { 3858 .notifier_call = cma_netdev_callback 3859 }; 3860 3861 static void cma_add_one(struct ib_device *device) 3862 { 3863 struct cma_device *cma_dev; 3864 struct rdma_id_private *id_priv; 3865 3866 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); 3867 if (!cma_dev) 3868 return; 3869 3870 cma_dev->device = device; 3871 3872 init_completion(&cma_dev->comp); 3873 atomic_set(&cma_dev->refcount, 1); 3874 INIT_LIST_HEAD(&cma_dev->id_list); 3875 ib_set_client_data(device, &cma_client, cma_dev); 3876 3877 mutex_lock(&lock); 3878 list_add_tail(&cma_dev->list, &dev_list); 3879 list_for_each_entry(id_priv, &listen_any_list, list) 3880 cma_listen_on_dev(id_priv, cma_dev); 3881 mutex_unlock(&lock); 3882 } 3883 3884 static int cma_remove_id_dev(struct rdma_id_private *id_priv) 3885 { 3886 struct rdma_cm_event event; 3887 enum rdma_cm_state state; 3888 int ret = 0; 3889 3890 /* Record that we want to remove the device */ 3891 state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL); 3892 if (state == RDMA_CM_DESTROYING) 3893 return 0; 3894 3895 cma_cancel_operation(id_priv, state); 3896 mutex_lock(&id_priv->handler_mutex); 3897 3898 /* Check for destruction from another callback. */ 3899 if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL)) 3900 goto out; 3901 3902 memset(&event, 0, sizeof event); 3903 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; 3904 ret = id_priv->id.event_handler(&id_priv->id, &event); 3905 out: 3906 mutex_unlock(&id_priv->handler_mutex); 3907 return ret; 3908 } 3909 3910 static void cma_process_remove(struct cma_device *cma_dev) 3911 { 3912 struct rdma_id_private *id_priv; 3913 int ret; 3914 3915 mutex_lock(&lock); 3916 while (!list_empty(&cma_dev->id_list)) { 3917 id_priv = list_entry(cma_dev->id_list.next, 3918 struct rdma_id_private, list); 3919 3920 list_del(&id_priv->listen_list); 3921 list_del_init(&id_priv->list); 3922 atomic_inc(&id_priv->refcount); 3923 mutex_unlock(&lock); 3924 3925 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); 3926 cma_deref_id(id_priv); 3927 if (ret) 3928 rdma_destroy_id(&id_priv->id); 3929 3930 mutex_lock(&lock); 3931 } 3932 mutex_unlock(&lock); 3933 3934 cma_deref_dev(cma_dev); 3935 wait_for_completion(&cma_dev->comp); 3936 } 3937 3938 static void cma_remove_one(struct ib_device *device, void *client_data) 3939 { 3940 struct cma_device *cma_dev = client_data; 3941 3942 if (!cma_dev) 3943 return; 3944 3945 mutex_lock(&lock); 3946 list_del(&cma_dev->list); 3947 mutex_unlock(&lock); 3948 3949 cma_process_remove(cma_dev); 3950 kfree(cma_dev); 3951 } 3952 3953 static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb) 3954 { 3955 struct nlmsghdr *nlh; 3956 struct rdma_cm_id_stats *id_stats; 3957 struct rdma_id_private *id_priv; 3958 struct rdma_cm_id *id = NULL; 3959 struct cma_device *cma_dev; 3960 int i_dev = 0, i_id = 0; 3961 3962 /* 3963 * We export all of the IDs as a sequence of messages. Each 3964 * ID gets its own netlink message. 3965 */ 3966 mutex_lock(&lock); 3967 3968 list_for_each_entry(cma_dev, &dev_list, list) { 3969 if (i_dev < cb->args[0]) { 3970 i_dev++; 3971 continue; 3972 } 3973 3974 i_id = 0; 3975 list_for_each_entry(id_priv, &cma_dev->id_list, list) { 3976 if (i_id < cb->args[1]) { 3977 i_id++; 3978 continue; 3979 } 3980 3981 id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq, 3982 sizeof *id_stats, RDMA_NL_RDMA_CM, 3983 RDMA_NL_RDMA_CM_ID_STATS, 3984 NLM_F_MULTI); 3985 if (!id_stats) 3986 goto out; 3987 3988 memset(id_stats, 0, sizeof *id_stats); 3989 id = &id_priv->id; 3990 id_stats->node_type = id->route.addr.dev_addr.dev_type; 3991 id_stats->port_num = id->port_num; 3992 id_stats->bound_dev_if = 3993 id->route.addr.dev_addr.bound_dev_if; 3994 3995 if (ibnl_put_attr(skb, nlh, 3996 rdma_addr_size(cma_src_addr(id_priv)), 3997 cma_src_addr(id_priv), 3998 RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) 3999 goto out; 4000 if (ibnl_put_attr(skb, nlh, 4001 rdma_addr_size(cma_src_addr(id_priv)), 4002 cma_dst_addr(id_priv), 4003 RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) 4004 goto out; 4005 4006 id_stats->pid = id_priv->owner; 4007 id_stats->port_space = id->ps; 4008 id_stats->cm_state = id_priv->state; 4009 id_stats->qp_num = id_priv->qp_num; 4010 id_stats->qp_type = id->qp_type; 4011 4012 i_id++; 4013 } 4014 4015 cb->args[1] = 0; 4016 i_dev++; 4017 } 4018 4019 out: 4020 mutex_unlock(&lock); 4021 cb->args[0] = i_dev; 4022 cb->args[1] = i_id; 4023 4024 return skb->len; 4025 } 4026 4027 static const struct ibnl_client_cbs cma_cb_table[] = { 4028 [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats, 4029 .module = THIS_MODULE }, 4030 }; 4031 4032 static int cma_init_net(struct net *net) 4033 { 4034 struct cma_pernet *pernet = cma_pernet(net); 4035 4036 idr_init(&pernet->tcp_ps); 4037 idr_init(&pernet->udp_ps); 4038 idr_init(&pernet->ipoib_ps); 4039 idr_init(&pernet->ib_ps); 4040 4041 return 0; 4042 } 4043 4044 static void cma_exit_net(struct net *net) 4045 { 4046 struct cma_pernet *pernet = cma_pernet(net); 4047 4048 idr_destroy(&pernet->tcp_ps); 4049 idr_destroy(&pernet->udp_ps); 4050 idr_destroy(&pernet->ipoib_ps); 4051 idr_destroy(&pernet->ib_ps); 4052 } 4053 4054 static struct pernet_operations cma_pernet_operations = { 4055 .init = cma_init_net, 4056 .exit = cma_exit_net, 4057 .id = &cma_pernet_id, 4058 .size = sizeof(struct cma_pernet), 4059 }; 4060 4061 static int __init cma_init(void) 4062 { 4063 int ret; 4064 4065 cma_wq = create_singlethread_workqueue("rdma_cm"); 4066 if (!cma_wq) 4067 return -ENOMEM; 4068 4069 ret = register_pernet_subsys(&cma_pernet_operations); 4070 if (ret) 4071 goto err_wq; 4072 4073 ib_sa_register_client(&sa_client); 4074 rdma_addr_register_client(&addr_client); 4075 register_netdevice_notifier(&cma_nb); 4076 4077 ret = ib_register_client(&cma_client); 4078 if (ret) 4079 goto err; 4080 4081 if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table)) 4082 printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n"); 4083 4084 return 0; 4085 4086 err: 4087 unregister_netdevice_notifier(&cma_nb); 4088 rdma_addr_unregister_client(&addr_client); 4089 ib_sa_unregister_client(&sa_client); 4090 err_wq: 4091 destroy_workqueue(cma_wq); 4092 return ret; 4093 } 4094 4095 static void __exit cma_cleanup(void) 4096 { 4097 ibnl_remove_client(RDMA_NL_RDMA_CM); 4098 ib_unregister_client(&cma_client); 4099 unregister_netdevice_notifier(&cma_nb); 4100 rdma_addr_unregister_client(&addr_client); 4101 ib_sa_unregister_client(&sa_client); 4102 unregister_pernet_subsys(&cma_pernet_operations); 4103 destroy_workqueue(cma_wq); 4104 } 4105 4106 module_init(cma_init); 4107 module_exit(cma_cleanup); 4108