1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2005 Voltaire Inc. All rights reserved. 4 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 5 * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved. 6 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 7 */ 8 9 #include <linux/completion.h> 10 #include <linux/in.h> 11 #include <linux/in6.h> 12 #include <linux/mutex.h> 13 #include <linux/random.h> 14 #include <linux/rbtree.h> 15 #include <linux/igmp.h> 16 #include <linux/xarray.h> 17 #include <linux/inetdevice.h> 18 #include <linux/slab.h> 19 #include <linux/module.h> 20 #include <net/route.h> 21 22 #include <net/net_namespace.h> 23 #include <net/netns/generic.h> 24 #include <net/netevent.h> 25 #include <net/tcp.h> 26 #include <net/ipv6.h> 27 #include <net/ip_fib.h> 28 #include <net/ip6_route.h> 29 30 #include <rdma/rdma_cm.h> 31 #include <rdma/rdma_cm_ib.h> 32 #include <rdma/rdma_netlink.h> 33 #include <rdma/ib.h> 34 #include <rdma/ib_cache.h> 35 #include <rdma/ib_cm.h> 36 #include <rdma/ib_sa.h> 37 #include <rdma/iw_cm.h> 38 39 #include "core_priv.h" 40 #include "cma_priv.h" 41 #include "cma_trace.h" 42 43 MODULE_AUTHOR("Sean Hefty"); 44 MODULE_DESCRIPTION("Generic RDMA CM Agent"); 45 MODULE_LICENSE("Dual BSD/GPL"); 46 47 #define CMA_CM_RESPONSE_TIMEOUT 20 48 #define CMA_MAX_CM_RETRIES 15 49 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 50 #define CMA_IBOE_PACKET_LIFETIME 18 51 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP 52 53 static const char * const cma_events[] = { 54 [RDMA_CM_EVENT_ADDR_RESOLVED] = "address resolved", 55 [RDMA_CM_EVENT_ADDR_ERROR] = "address error", 56 [RDMA_CM_EVENT_ROUTE_RESOLVED] = "route resolved ", 57 [RDMA_CM_EVENT_ROUTE_ERROR] = "route error", 58 [RDMA_CM_EVENT_CONNECT_REQUEST] = "connect request", 59 [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response", 60 [RDMA_CM_EVENT_CONNECT_ERROR] = "connect error", 61 [RDMA_CM_EVENT_UNREACHABLE] = "unreachable", 62 [RDMA_CM_EVENT_REJECTED] = "rejected", 63 [RDMA_CM_EVENT_ESTABLISHED] = "established", 64 [RDMA_CM_EVENT_DISCONNECTED] = "disconnected", 65 [RDMA_CM_EVENT_DEVICE_REMOVAL] = "device removal", 66 [RDMA_CM_EVENT_MULTICAST_JOIN] = "multicast join", 67 [RDMA_CM_EVENT_MULTICAST_ERROR] = "multicast error", 68 [RDMA_CM_EVENT_ADDR_CHANGE] = "address change", 69 [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit", 70 }; 71 72 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, 73 enum ib_gid_type gid_type); 74 75 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event) 76 { 77 size_t index = event; 78 79 return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ? 80 cma_events[index] : "unrecognized event"; 81 } 82 EXPORT_SYMBOL(rdma_event_msg); 83 84 const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id, 85 int reason) 86 { 87 if (rdma_ib_or_roce(id->device, id->port_num)) 88 return ibcm_reject_msg(reason); 89 90 if (rdma_protocol_iwarp(id->device, id->port_num)) 91 return iwcm_reject_msg(reason); 92 93 WARN_ON_ONCE(1); 94 return "unrecognized transport"; 95 } 96 EXPORT_SYMBOL(rdma_reject_msg); 97 98 /** 99 * rdma_is_consumer_reject - return true if the consumer rejected the connect 100 * request. 101 * @id: Communication identifier that received the REJECT event. 102 * @reason: Value returned in the REJECT event status field. 103 */ 104 static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason) 105 { 106 if (rdma_ib_or_roce(id->device, id->port_num)) 107 return reason == IB_CM_REJ_CONSUMER_DEFINED; 108 109 if (rdma_protocol_iwarp(id->device, id->port_num)) 110 return reason == -ECONNREFUSED; 111 112 WARN_ON_ONCE(1); 113 return false; 114 } 115 116 const void *rdma_consumer_reject_data(struct rdma_cm_id *id, 117 struct rdma_cm_event *ev, u8 *data_len) 118 { 119 const void *p; 120 121 if (rdma_is_consumer_reject(id, ev->status)) { 122 *data_len = ev->param.conn.private_data_len; 123 p = ev->param.conn.private_data; 124 } else { 125 *data_len = 0; 126 p = NULL; 127 } 128 return p; 129 } 130 EXPORT_SYMBOL(rdma_consumer_reject_data); 131 132 /** 133 * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id. 134 * @id: Communication Identifier 135 */ 136 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id) 137 { 138 struct rdma_id_private *id_priv; 139 140 id_priv = container_of(id, struct rdma_id_private, id); 141 if (id->device->node_type == RDMA_NODE_RNIC) 142 return id_priv->cm_id.iw; 143 return NULL; 144 } 145 EXPORT_SYMBOL(rdma_iw_cm_id); 146 147 /** 148 * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack. 149 * @res: rdma resource tracking entry pointer 150 */ 151 struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res) 152 { 153 struct rdma_id_private *id_priv = 154 container_of(res, struct rdma_id_private, res); 155 156 return &id_priv->id; 157 } 158 EXPORT_SYMBOL(rdma_res_to_id); 159 160 static int cma_add_one(struct ib_device *device); 161 static void cma_remove_one(struct ib_device *device, void *client_data); 162 163 static struct ib_client cma_client = { 164 .name = "cma", 165 .add = cma_add_one, 166 .remove = cma_remove_one 167 }; 168 169 static struct ib_sa_client sa_client; 170 static LIST_HEAD(dev_list); 171 static LIST_HEAD(listen_any_list); 172 static DEFINE_MUTEX(lock); 173 static struct rb_root id_table = RB_ROOT; 174 /* Serialize operations of id_table tree */ 175 static DEFINE_SPINLOCK(id_table_lock); 176 static struct workqueue_struct *cma_wq; 177 static unsigned int cma_pernet_id; 178 179 struct cma_pernet { 180 struct xarray tcp_ps; 181 struct xarray udp_ps; 182 struct xarray ipoib_ps; 183 struct xarray ib_ps; 184 }; 185 186 static struct cma_pernet *cma_pernet(struct net *net) 187 { 188 return net_generic(net, cma_pernet_id); 189 } 190 191 static 192 struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps) 193 { 194 struct cma_pernet *pernet = cma_pernet(net); 195 196 switch (ps) { 197 case RDMA_PS_TCP: 198 return &pernet->tcp_ps; 199 case RDMA_PS_UDP: 200 return &pernet->udp_ps; 201 case RDMA_PS_IPOIB: 202 return &pernet->ipoib_ps; 203 case RDMA_PS_IB: 204 return &pernet->ib_ps; 205 default: 206 return NULL; 207 } 208 } 209 210 struct id_table_entry { 211 struct list_head id_list; 212 struct rb_node rb_node; 213 }; 214 215 struct cma_device { 216 struct list_head list; 217 struct ib_device *device; 218 struct completion comp; 219 refcount_t refcount; 220 struct list_head id_list; 221 enum ib_gid_type *default_gid_type; 222 u8 *default_roce_tos; 223 }; 224 225 struct rdma_bind_list { 226 enum rdma_ucm_port_space ps; 227 struct hlist_head owners; 228 unsigned short port; 229 }; 230 231 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps, 232 struct rdma_bind_list *bind_list, int snum) 233 { 234 struct xarray *xa = cma_pernet_xa(net, ps); 235 236 return xa_insert(xa, snum, bind_list, GFP_KERNEL); 237 } 238 239 static struct rdma_bind_list *cma_ps_find(struct net *net, 240 enum rdma_ucm_port_space ps, int snum) 241 { 242 struct xarray *xa = cma_pernet_xa(net, ps); 243 244 return xa_load(xa, snum); 245 } 246 247 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps, 248 int snum) 249 { 250 struct xarray *xa = cma_pernet_xa(net, ps); 251 252 xa_erase(xa, snum); 253 } 254 255 enum { 256 CMA_OPTION_AFONLY, 257 }; 258 259 void cma_dev_get(struct cma_device *cma_dev) 260 { 261 refcount_inc(&cma_dev->refcount); 262 } 263 264 void cma_dev_put(struct cma_device *cma_dev) 265 { 266 if (refcount_dec_and_test(&cma_dev->refcount)) 267 complete(&cma_dev->comp); 268 } 269 270 struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter, 271 void *cookie) 272 { 273 struct cma_device *cma_dev; 274 struct cma_device *found_cma_dev = NULL; 275 276 mutex_lock(&lock); 277 278 list_for_each_entry(cma_dev, &dev_list, list) 279 if (filter(cma_dev->device, cookie)) { 280 found_cma_dev = cma_dev; 281 break; 282 } 283 284 if (found_cma_dev) 285 cma_dev_get(found_cma_dev); 286 mutex_unlock(&lock); 287 return found_cma_dev; 288 } 289 290 int cma_get_default_gid_type(struct cma_device *cma_dev, 291 u32 port) 292 { 293 if (!rdma_is_port_valid(cma_dev->device, port)) 294 return -EINVAL; 295 296 return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)]; 297 } 298 299 int cma_set_default_gid_type(struct cma_device *cma_dev, 300 u32 port, 301 enum ib_gid_type default_gid_type) 302 { 303 unsigned long supported_gids; 304 305 if (!rdma_is_port_valid(cma_dev->device, port)) 306 return -EINVAL; 307 308 if (default_gid_type == IB_GID_TYPE_IB && 309 rdma_protocol_roce_eth_encap(cma_dev->device, port)) 310 default_gid_type = IB_GID_TYPE_ROCE; 311 312 supported_gids = roce_gid_type_mask_support(cma_dev->device, port); 313 314 if (!(supported_gids & 1 << default_gid_type)) 315 return -EINVAL; 316 317 cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] = 318 default_gid_type; 319 320 return 0; 321 } 322 323 int cma_get_default_roce_tos(struct cma_device *cma_dev, u32 port) 324 { 325 if (!rdma_is_port_valid(cma_dev->device, port)) 326 return -EINVAL; 327 328 return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)]; 329 } 330 331 int cma_set_default_roce_tos(struct cma_device *cma_dev, u32 port, 332 u8 default_roce_tos) 333 { 334 if (!rdma_is_port_valid(cma_dev->device, port)) 335 return -EINVAL; 336 337 cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] = 338 default_roce_tos; 339 340 return 0; 341 } 342 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev) 343 { 344 return cma_dev->device; 345 } 346 347 /* 348 * Device removal can occur at anytime, so we need extra handling to 349 * serialize notifying the user of device removal with other callbacks. 350 * We do this by disabling removal notification while a callback is in process, 351 * and reporting it after the callback completes. 352 */ 353 354 struct cma_multicast { 355 struct rdma_id_private *id_priv; 356 union { 357 struct ib_sa_multicast *sa_mc; 358 struct { 359 struct work_struct work; 360 struct rdma_cm_event event; 361 } iboe_join; 362 }; 363 struct list_head list; 364 void *context; 365 struct sockaddr_storage addr; 366 u8 join_state; 367 }; 368 369 struct cma_work { 370 struct work_struct work; 371 struct rdma_id_private *id; 372 enum rdma_cm_state old_state; 373 enum rdma_cm_state new_state; 374 struct rdma_cm_event event; 375 }; 376 377 union cma_ip_addr { 378 struct in6_addr ip6; 379 struct { 380 __be32 pad[3]; 381 __be32 addr; 382 } ip4; 383 }; 384 385 struct cma_hdr { 386 u8 cma_version; 387 u8 ip_version; /* IP version: 7:4 */ 388 __be16 port; 389 union cma_ip_addr src_addr; 390 union cma_ip_addr dst_addr; 391 }; 392 393 #define CMA_VERSION 0x00 394 395 struct cma_req_info { 396 struct sockaddr_storage listen_addr_storage; 397 struct sockaddr_storage src_addr_storage; 398 struct ib_device *device; 399 union ib_gid local_gid; 400 __be64 service_id; 401 int port; 402 bool has_gid; 403 u16 pkey; 404 }; 405 406 static int cma_comp_exch(struct rdma_id_private *id_priv, 407 enum rdma_cm_state comp, enum rdma_cm_state exch) 408 { 409 unsigned long flags; 410 int ret; 411 412 /* 413 * The FSM uses a funny double locking where state is protected by both 414 * the handler_mutex and the spinlock. State is not allowed to change 415 * to/from a handler_mutex protected value without also holding 416 * handler_mutex. 417 */ 418 if (comp == RDMA_CM_CONNECT || exch == RDMA_CM_CONNECT) 419 lockdep_assert_held(&id_priv->handler_mutex); 420 421 spin_lock_irqsave(&id_priv->lock, flags); 422 if ((ret = (id_priv->state == comp))) 423 id_priv->state = exch; 424 spin_unlock_irqrestore(&id_priv->lock, flags); 425 return ret; 426 } 427 428 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr) 429 { 430 return hdr->ip_version >> 4; 431 } 432 433 static void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 434 { 435 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 436 } 437 438 static struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv) 439 { 440 return (struct sockaddr *)&id_priv->id.route.addr.src_addr; 441 } 442 443 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv) 444 { 445 return (struct sockaddr *)&id_priv->id.route.addr.dst_addr; 446 } 447 448 static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join) 449 { 450 struct in_device *in_dev = NULL; 451 452 if (ndev) { 453 rtnl_lock(); 454 in_dev = __in_dev_get_rtnl(ndev); 455 if (in_dev) { 456 if (join) 457 ip_mc_inc_group(in_dev, 458 *(__be32 *)(mgid->raw + 12)); 459 else 460 ip_mc_dec_group(in_dev, 461 *(__be32 *)(mgid->raw + 12)); 462 } 463 rtnl_unlock(); 464 } 465 return (in_dev) ? 0 : -ENODEV; 466 } 467 468 static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa, 469 struct id_table_entry *entry_b) 470 { 471 struct rdma_id_private *id_priv = list_first_entry( 472 &entry_b->id_list, struct rdma_id_private, id_list_entry); 473 int ifindex_b = id_priv->id.route.addr.dev_addr.bound_dev_if; 474 struct sockaddr *sb = cma_dst_addr(id_priv); 475 476 if (ifindex_a != ifindex_b) 477 return (ifindex_a > ifindex_b) ? 1 : -1; 478 479 if (sa->sa_family != sb->sa_family) 480 return sa->sa_family - sb->sa_family; 481 482 if (sa->sa_family == AF_INET) 483 return memcmp((char *)&((struct sockaddr_in *)sa)->sin_addr, 484 (char *)&((struct sockaddr_in *)sb)->sin_addr, 485 sizeof(((struct sockaddr_in *)sa)->sin_addr)); 486 487 return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr, 488 &((struct sockaddr_in6 *)sb)->sin6_addr); 489 } 490 491 static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv) 492 { 493 struct rb_node **new, *parent = NULL; 494 struct id_table_entry *this, *node; 495 unsigned long flags; 496 int result; 497 498 node = kzalloc(sizeof(*node), GFP_KERNEL); 499 if (!node) 500 return -ENOMEM; 501 502 spin_lock_irqsave(&id_table_lock, flags); 503 new = &id_table.rb_node; 504 while (*new) { 505 this = container_of(*new, struct id_table_entry, rb_node); 506 result = compare_netdev_and_ip( 507 node_id_priv->id.route.addr.dev_addr.bound_dev_if, 508 cma_dst_addr(node_id_priv), this); 509 510 parent = *new; 511 if (result < 0) 512 new = &((*new)->rb_left); 513 else if (result > 0) 514 new = &((*new)->rb_right); 515 else { 516 list_add_tail(&node_id_priv->id_list_entry, 517 &this->id_list); 518 kfree(node); 519 goto unlock; 520 } 521 } 522 523 INIT_LIST_HEAD(&node->id_list); 524 list_add_tail(&node_id_priv->id_list_entry, &node->id_list); 525 526 rb_link_node(&node->rb_node, parent, new); 527 rb_insert_color(&node->rb_node, &id_table); 528 529 unlock: 530 spin_unlock_irqrestore(&id_table_lock, flags); 531 return 0; 532 } 533 534 static struct id_table_entry * 535 node_from_ndev_ip(struct rb_root *root, int ifindex, struct sockaddr *sa) 536 { 537 struct rb_node *node = root->rb_node; 538 struct id_table_entry *data; 539 int result; 540 541 while (node) { 542 data = container_of(node, struct id_table_entry, rb_node); 543 result = compare_netdev_and_ip(ifindex, sa, data); 544 if (result < 0) 545 node = node->rb_left; 546 else if (result > 0) 547 node = node->rb_right; 548 else 549 return data; 550 } 551 552 return NULL; 553 } 554 555 static void cma_remove_id_from_tree(struct rdma_id_private *id_priv) 556 { 557 struct id_table_entry *data; 558 unsigned long flags; 559 560 spin_lock_irqsave(&id_table_lock, flags); 561 if (list_empty(&id_priv->id_list_entry)) 562 goto out; 563 564 data = node_from_ndev_ip(&id_table, 565 id_priv->id.route.addr.dev_addr.bound_dev_if, 566 cma_dst_addr(id_priv)); 567 if (!data) 568 goto out; 569 570 list_del_init(&id_priv->id_list_entry); 571 if (list_empty(&data->id_list)) { 572 rb_erase(&data->rb_node, &id_table); 573 kfree(data); 574 } 575 out: 576 spin_unlock_irqrestore(&id_table_lock, flags); 577 } 578 579 static void _cma_attach_to_dev(struct rdma_id_private *id_priv, 580 struct cma_device *cma_dev) 581 { 582 cma_dev_get(cma_dev); 583 id_priv->cma_dev = cma_dev; 584 id_priv->id.device = cma_dev->device; 585 id_priv->id.route.addr.dev_addr.transport = 586 rdma_node_get_transport(cma_dev->device->node_type); 587 list_add_tail(&id_priv->device_item, &cma_dev->id_list); 588 589 trace_cm_id_attach(id_priv, cma_dev->device); 590 } 591 592 static void cma_attach_to_dev(struct rdma_id_private *id_priv, 593 struct cma_device *cma_dev) 594 { 595 _cma_attach_to_dev(id_priv, cma_dev); 596 id_priv->gid_type = 597 cma_dev->default_gid_type[id_priv->id.port_num - 598 rdma_start_port(cma_dev->device)]; 599 } 600 601 static void cma_release_dev(struct rdma_id_private *id_priv) 602 { 603 mutex_lock(&lock); 604 list_del_init(&id_priv->device_item); 605 cma_dev_put(id_priv->cma_dev); 606 id_priv->cma_dev = NULL; 607 id_priv->id.device = NULL; 608 if (id_priv->id.route.addr.dev_addr.sgid_attr) { 609 rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr); 610 id_priv->id.route.addr.dev_addr.sgid_attr = NULL; 611 } 612 mutex_unlock(&lock); 613 } 614 615 static inline unsigned short cma_family(struct rdma_id_private *id_priv) 616 { 617 return id_priv->id.route.addr.src_addr.ss_family; 618 } 619 620 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey) 621 { 622 struct ib_sa_mcmember_rec rec; 623 int ret = 0; 624 625 if (id_priv->qkey) { 626 if (qkey && id_priv->qkey != qkey) 627 return -EINVAL; 628 return 0; 629 } 630 631 if (qkey) { 632 id_priv->qkey = qkey; 633 return 0; 634 } 635 636 switch (id_priv->id.ps) { 637 case RDMA_PS_UDP: 638 case RDMA_PS_IB: 639 id_priv->qkey = RDMA_UDP_QKEY; 640 break; 641 case RDMA_PS_IPOIB: 642 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); 643 ret = ib_sa_get_mcmember_rec(id_priv->id.device, 644 id_priv->id.port_num, &rec.mgid, 645 &rec); 646 if (!ret) 647 id_priv->qkey = be32_to_cpu(rec.qkey); 648 break; 649 default: 650 break; 651 } 652 return ret; 653 } 654 655 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr) 656 { 657 dev_addr->dev_type = ARPHRD_INFINIBAND; 658 rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr); 659 ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey)); 660 } 661 662 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr) 663 { 664 int ret; 665 666 if (addr->sa_family != AF_IB) { 667 ret = rdma_translate_ip(addr, dev_addr); 668 } else { 669 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr); 670 ret = 0; 671 } 672 673 return ret; 674 } 675 676 static const struct ib_gid_attr * 677 cma_validate_port(struct ib_device *device, u32 port, 678 enum ib_gid_type gid_type, 679 union ib_gid *gid, 680 struct rdma_id_private *id_priv) 681 { 682 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 683 int bound_if_index = dev_addr->bound_dev_if; 684 const struct ib_gid_attr *sgid_attr; 685 int dev_type = dev_addr->dev_type; 686 struct net_device *ndev = NULL; 687 688 if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net)) 689 return ERR_PTR(-ENODEV); 690 691 if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port)) 692 return ERR_PTR(-ENODEV); 693 694 if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port)) 695 return ERR_PTR(-ENODEV); 696 697 if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) { 698 ndev = dev_get_by_index(dev_addr->net, bound_if_index); 699 if (!ndev) 700 return ERR_PTR(-ENODEV); 701 } else { 702 gid_type = IB_GID_TYPE_IB; 703 } 704 705 sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev); 706 if (ndev) 707 dev_put(ndev); 708 return sgid_attr; 709 } 710 711 static void cma_bind_sgid_attr(struct rdma_id_private *id_priv, 712 const struct ib_gid_attr *sgid_attr) 713 { 714 WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr); 715 id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr; 716 } 717 718 /** 719 * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute 720 * based on source ip address. 721 * @id_priv: cm_id which should be bound to cma device 722 * 723 * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute 724 * based on source IP address. It returns 0 on success or error code otherwise. 725 * It is applicable to active and passive side cm_id. 726 */ 727 static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv) 728 { 729 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 730 const struct ib_gid_attr *sgid_attr; 731 union ib_gid gid, iboe_gid, *gidp; 732 struct cma_device *cma_dev; 733 enum ib_gid_type gid_type; 734 int ret = -ENODEV; 735 u32 port; 736 737 if (dev_addr->dev_type != ARPHRD_INFINIBAND && 738 id_priv->id.ps == RDMA_PS_IPOIB) 739 return -EINVAL; 740 741 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 742 &iboe_gid); 743 744 memcpy(&gid, dev_addr->src_dev_addr + 745 rdma_addr_gid_offset(dev_addr), sizeof(gid)); 746 747 mutex_lock(&lock); 748 list_for_each_entry(cma_dev, &dev_list, list) { 749 rdma_for_each_port (cma_dev->device, port) { 750 gidp = rdma_protocol_roce(cma_dev->device, port) ? 751 &iboe_gid : &gid; 752 gid_type = cma_dev->default_gid_type[port - 1]; 753 sgid_attr = cma_validate_port(cma_dev->device, port, 754 gid_type, gidp, id_priv); 755 if (!IS_ERR(sgid_attr)) { 756 id_priv->id.port_num = port; 757 cma_bind_sgid_attr(id_priv, sgid_attr); 758 cma_attach_to_dev(id_priv, cma_dev); 759 ret = 0; 760 goto out; 761 } 762 } 763 } 764 out: 765 mutex_unlock(&lock); 766 return ret; 767 } 768 769 /** 770 * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute 771 * @id_priv: cm id to bind to cma device 772 * @listen_id_priv: listener cm id to match against 773 * @req: Pointer to req structure containaining incoming 774 * request information 775 * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when 776 * rdma device matches for listen_id and incoming request. It also verifies 777 * that a GID table entry is present for the source address. 778 * Returns 0 on success, or returns error code otherwise. 779 */ 780 static int cma_ib_acquire_dev(struct rdma_id_private *id_priv, 781 const struct rdma_id_private *listen_id_priv, 782 struct cma_req_info *req) 783 { 784 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 785 const struct ib_gid_attr *sgid_attr; 786 enum ib_gid_type gid_type; 787 union ib_gid gid; 788 789 if (dev_addr->dev_type != ARPHRD_INFINIBAND && 790 id_priv->id.ps == RDMA_PS_IPOIB) 791 return -EINVAL; 792 793 if (rdma_protocol_roce(req->device, req->port)) 794 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 795 &gid); 796 else 797 memcpy(&gid, dev_addr->src_dev_addr + 798 rdma_addr_gid_offset(dev_addr), sizeof(gid)); 799 800 gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1]; 801 sgid_attr = cma_validate_port(req->device, req->port, 802 gid_type, &gid, id_priv); 803 if (IS_ERR(sgid_attr)) 804 return PTR_ERR(sgid_attr); 805 806 id_priv->id.port_num = req->port; 807 cma_bind_sgid_attr(id_priv, sgid_attr); 808 /* Need to acquire lock to protect against reader 809 * of cma_dev->id_list such as cma_netdev_callback() and 810 * cma_process_remove(). 811 */ 812 mutex_lock(&lock); 813 cma_attach_to_dev(id_priv, listen_id_priv->cma_dev); 814 mutex_unlock(&lock); 815 rdma_restrack_add(&id_priv->res); 816 return 0; 817 } 818 819 static int cma_iw_acquire_dev(struct rdma_id_private *id_priv, 820 const struct rdma_id_private *listen_id_priv) 821 { 822 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 823 const struct ib_gid_attr *sgid_attr; 824 struct cma_device *cma_dev; 825 enum ib_gid_type gid_type; 826 int ret = -ENODEV; 827 union ib_gid gid; 828 u32 port; 829 830 if (dev_addr->dev_type != ARPHRD_INFINIBAND && 831 id_priv->id.ps == RDMA_PS_IPOIB) 832 return -EINVAL; 833 834 memcpy(&gid, dev_addr->src_dev_addr + 835 rdma_addr_gid_offset(dev_addr), sizeof(gid)); 836 837 mutex_lock(&lock); 838 839 cma_dev = listen_id_priv->cma_dev; 840 port = listen_id_priv->id.port_num; 841 gid_type = listen_id_priv->gid_type; 842 sgid_attr = cma_validate_port(cma_dev->device, port, 843 gid_type, &gid, id_priv); 844 if (!IS_ERR(sgid_attr)) { 845 id_priv->id.port_num = port; 846 cma_bind_sgid_attr(id_priv, sgid_attr); 847 ret = 0; 848 goto out; 849 } 850 851 list_for_each_entry(cma_dev, &dev_list, list) { 852 rdma_for_each_port (cma_dev->device, port) { 853 if (listen_id_priv->cma_dev == cma_dev && 854 listen_id_priv->id.port_num == port) 855 continue; 856 857 gid_type = cma_dev->default_gid_type[port - 1]; 858 sgid_attr = cma_validate_port(cma_dev->device, port, 859 gid_type, &gid, id_priv); 860 if (!IS_ERR(sgid_attr)) { 861 id_priv->id.port_num = port; 862 cma_bind_sgid_attr(id_priv, sgid_attr); 863 ret = 0; 864 goto out; 865 } 866 } 867 } 868 869 out: 870 if (!ret) { 871 cma_attach_to_dev(id_priv, cma_dev); 872 rdma_restrack_add(&id_priv->res); 873 } 874 875 mutex_unlock(&lock); 876 return ret; 877 } 878 879 /* 880 * Select the source IB device and address to reach the destination IB address. 881 */ 882 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) 883 { 884 struct cma_device *cma_dev, *cur_dev; 885 struct sockaddr_ib *addr; 886 union ib_gid gid, sgid, *dgid; 887 unsigned int p; 888 u16 pkey, index; 889 enum ib_port_state port_state; 890 int ret; 891 int i; 892 893 cma_dev = NULL; 894 addr = (struct sockaddr_ib *) cma_dst_addr(id_priv); 895 dgid = (union ib_gid *) &addr->sib_addr; 896 pkey = ntohs(addr->sib_pkey); 897 898 mutex_lock(&lock); 899 list_for_each_entry(cur_dev, &dev_list, list) { 900 rdma_for_each_port (cur_dev->device, p) { 901 if (!rdma_cap_af_ib(cur_dev->device, p)) 902 continue; 903 904 if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index)) 905 continue; 906 907 if (ib_get_cached_port_state(cur_dev->device, p, &port_state)) 908 continue; 909 910 for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len; 911 ++i) { 912 ret = rdma_query_gid(cur_dev->device, p, i, 913 &gid); 914 if (ret) 915 continue; 916 917 if (!memcmp(&gid, dgid, sizeof(gid))) { 918 cma_dev = cur_dev; 919 sgid = gid; 920 id_priv->id.port_num = p; 921 goto found; 922 } 923 924 if (!cma_dev && (gid.global.subnet_prefix == 925 dgid->global.subnet_prefix) && 926 port_state == IB_PORT_ACTIVE) { 927 cma_dev = cur_dev; 928 sgid = gid; 929 id_priv->id.port_num = p; 930 goto found; 931 } 932 } 933 } 934 } 935 mutex_unlock(&lock); 936 return -ENODEV; 937 938 found: 939 cma_attach_to_dev(id_priv, cma_dev); 940 rdma_restrack_add(&id_priv->res); 941 mutex_unlock(&lock); 942 addr = (struct sockaddr_ib *)cma_src_addr(id_priv); 943 memcpy(&addr->sib_addr, &sgid, sizeof(sgid)); 944 cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr); 945 return 0; 946 } 947 948 static void cma_id_get(struct rdma_id_private *id_priv) 949 { 950 refcount_inc(&id_priv->refcount); 951 } 952 953 static void cma_id_put(struct rdma_id_private *id_priv) 954 { 955 if (refcount_dec_and_test(&id_priv->refcount)) 956 complete(&id_priv->comp); 957 } 958 959 static struct rdma_id_private * 960 __rdma_create_id(struct net *net, rdma_cm_event_handler event_handler, 961 void *context, enum rdma_ucm_port_space ps, 962 enum ib_qp_type qp_type, const struct rdma_id_private *parent) 963 { 964 struct rdma_id_private *id_priv; 965 966 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 967 if (!id_priv) 968 return ERR_PTR(-ENOMEM); 969 970 id_priv->state = RDMA_CM_IDLE; 971 id_priv->id.context = context; 972 id_priv->id.event_handler = event_handler; 973 id_priv->id.ps = ps; 974 id_priv->id.qp_type = qp_type; 975 id_priv->tos_set = false; 976 id_priv->timeout_set = false; 977 id_priv->min_rnr_timer_set = false; 978 id_priv->gid_type = IB_GID_TYPE_IB; 979 spin_lock_init(&id_priv->lock); 980 mutex_init(&id_priv->qp_mutex); 981 init_completion(&id_priv->comp); 982 refcount_set(&id_priv->refcount, 1); 983 mutex_init(&id_priv->handler_mutex); 984 INIT_LIST_HEAD(&id_priv->device_item); 985 INIT_LIST_HEAD(&id_priv->id_list_entry); 986 INIT_LIST_HEAD(&id_priv->listen_list); 987 INIT_LIST_HEAD(&id_priv->mc_list); 988 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 989 id_priv->id.route.addr.dev_addr.net = get_net(net); 990 id_priv->seq_num &= 0x00ffffff; 991 992 rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID); 993 if (parent) 994 rdma_restrack_parent_name(&id_priv->res, &parent->res); 995 996 return id_priv; 997 } 998 999 struct rdma_cm_id * 1000 __rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler, 1001 void *context, enum rdma_ucm_port_space ps, 1002 enum ib_qp_type qp_type, const char *caller) 1003 { 1004 struct rdma_id_private *ret; 1005 1006 ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL); 1007 if (IS_ERR(ret)) 1008 return ERR_CAST(ret); 1009 1010 rdma_restrack_set_name(&ret->res, caller); 1011 return &ret->id; 1012 } 1013 EXPORT_SYMBOL(__rdma_create_kernel_id); 1014 1015 struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler, 1016 void *context, 1017 enum rdma_ucm_port_space ps, 1018 enum ib_qp_type qp_type) 1019 { 1020 struct rdma_id_private *ret; 1021 1022 ret = __rdma_create_id(current->nsproxy->net_ns, event_handler, context, 1023 ps, qp_type, NULL); 1024 if (IS_ERR(ret)) 1025 return ERR_CAST(ret); 1026 1027 rdma_restrack_set_name(&ret->res, NULL); 1028 return &ret->id; 1029 } 1030 EXPORT_SYMBOL(rdma_create_user_id); 1031 1032 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 1033 { 1034 struct ib_qp_attr qp_attr; 1035 int qp_attr_mask, ret; 1036 1037 qp_attr.qp_state = IB_QPS_INIT; 1038 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1039 if (ret) 1040 return ret; 1041 1042 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 1043 if (ret) 1044 return ret; 1045 1046 qp_attr.qp_state = IB_QPS_RTR; 1047 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 1048 if (ret) 1049 return ret; 1050 1051 qp_attr.qp_state = IB_QPS_RTS; 1052 qp_attr.sq_psn = 0; 1053 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 1054 1055 return ret; 1056 } 1057 1058 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 1059 { 1060 struct ib_qp_attr qp_attr; 1061 int qp_attr_mask, ret; 1062 1063 qp_attr.qp_state = IB_QPS_INIT; 1064 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1065 if (ret) 1066 return ret; 1067 1068 return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 1069 } 1070 1071 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 1072 struct ib_qp_init_attr *qp_init_attr) 1073 { 1074 struct rdma_id_private *id_priv; 1075 struct ib_qp *qp; 1076 int ret; 1077 1078 id_priv = container_of(id, struct rdma_id_private, id); 1079 if (id->device != pd->device) { 1080 ret = -EINVAL; 1081 goto out_err; 1082 } 1083 1084 qp_init_attr->port_num = id->port_num; 1085 qp = ib_create_qp(pd, qp_init_attr); 1086 if (IS_ERR(qp)) { 1087 ret = PTR_ERR(qp); 1088 goto out_err; 1089 } 1090 1091 if (id->qp_type == IB_QPT_UD) 1092 ret = cma_init_ud_qp(id_priv, qp); 1093 else 1094 ret = cma_init_conn_qp(id_priv, qp); 1095 if (ret) 1096 goto out_destroy; 1097 1098 id->qp = qp; 1099 id_priv->qp_num = qp->qp_num; 1100 id_priv->srq = (qp->srq != NULL); 1101 trace_cm_qp_create(id_priv, pd, qp_init_attr, 0); 1102 return 0; 1103 out_destroy: 1104 ib_destroy_qp(qp); 1105 out_err: 1106 trace_cm_qp_create(id_priv, pd, qp_init_attr, ret); 1107 return ret; 1108 } 1109 EXPORT_SYMBOL(rdma_create_qp); 1110 1111 void rdma_destroy_qp(struct rdma_cm_id *id) 1112 { 1113 struct rdma_id_private *id_priv; 1114 1115 id_priv = container_of(id, struct rdma_id_private, id); 1116 trace_cm_qp_destroy(id_priv); 1117 mutex_lock(&id_priv->qp_mutex); 1118 ib_destroy_qp(id_priv->id.qp); 1119 id_priv->id.qp = NULL; 1120 mutex_unlock(&id_priv->qp_mutex); 1121 } 1122 EXPORT_SYMBOL(rdma_destroy_qp); 1123 1124 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, 1125 struct rdma_conn_param *conn_param) 1126 { 1127 struct ib_qp_attr qp_attr; 1128 int qp_attr_mask, ret; 1129 1130 mutex_lock(&id_priv->qp_mutex); 1131 if (!id_priv->id.qp) { 1132 ret = 0; 1133 goto out; 1134 } 1135 1136 /* Need to update QP attributes from default values. */ 1137 qp_attr.qp_state = IB_QPS_INIT; 1138 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1139 if (ret) 1140 goto out; 1141 1142 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 1143 if (ret) 1144 goto out; 1145 1146 qp_attr.qp_state = IB_QPS_RTR; 1147 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1148 if (ret) 1149 goto out; 1150 1151 BUG_ON(id_priv->cma_dev->device != id_priv->id.device); 1152 1153 if (conn_param) 1154 qp_attr.max_dest_rd_atomic = conn_param->responder_resources; 1155 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 1156 out: 1157 mutex_unlock(&id_priv->qp_mutex); 1158 return ret; 1159 } 1160 1161 static int cma_modify_qp_rts(struct rdma_id_private *id_priv, 1162 struct rdma_conn_param *conn_param) 1163 { 1164 struct ib_qp_attr qp_attr; 1165 int qp_attr_mask, ret; 1166 1167 mutex_lock(&id_priv->qp_mutex); 1168 if (!id_priv->id.qp) { 1169 ret = 0; 1170 goto out; 1171 } 1172 1173 qp_attr.qp_state = IB_QPS_RTS; 1174 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1175 if (ret) 1176 goto out; 1177 1178 if (conn_param) 1179 qp_attr.max_rd_atomic = conn_param->initiator_depth; 1180 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 1181 out: 1182 mutex_unlock(&id_priv->qp_mutex); 1183 return ret; 1184 } 1185 1186 static int cma_modify_qp_err(struct rdma_id_private *id_priv) 1187 { 1188 struct ib_qp_attr qp_attr; 1189 int ret; 1190 1191 mutex_lock(&id_priv->qp_mutex); 1192 if (!id_priv->id.qp) { 1193 ret = 0; 1194 goto out; 1195 } 1196 1197 qp_attr.qp_state = IB_QPS_ERR; 1198 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); 1199 out: 1200 mutex_unlock(&id_priv->qp_mutex); 1201 return ret; 1202 } 1203 1204 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 1205 struct ib_qp_attr *qp_attr, int *qp_attr_mask) 1206 { 1207 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 1208 int ret; 1209 u16 pkey; 1210 1211 if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num)) 1212 pkey = 0xffff; 1213 else 1214 pkey = ib_addr_get_pkey(dev_addr); 1215 1216 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 1217 pkey, &qp_attr->pkey_index); 1218 if (ret) 1219 return ret; 1220 1221 qp_attr->port_num = id_priv->id.port_num; 1222 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 1223 1224 if (id_priv->id.qp_type == IB_QPT_UD) { 1225 ret = cma_set_qkey(id_priv, 0); 1226 if (ret) 1227 return ret; 1228 1229 qp_attr->qkey = id_priv->qkey; 1230 *qp_attr_mask |= IB_QP_QKEY; 1231 } else { 1232 qp_attr->qp_access_flags = 0; 1233 *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 1234 } 1235 return 0; 1236 } 1237 1238 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 1239 int *qp_attr_mask) 1240 { 1241 struct rdma_id_private *id_priv; 1242 int ret = 0; 1243 1244 id_priv = container_of(id, struct rdma_id_private, id); 1245 if (rdma_cap_ib_cm(id->device, id->port_num)) { 1246 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD)) 1247 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 1248 else 1249 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 1250 qp_attr_mask); 1251 1252 if (qp_attr->qp_state == IB_QPS_RTR) 1253 qp_attr->rq_psn = id_priv->seq_num; 1254 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 1255 if (!id_priv->cm_id.iw) { 1256 qp_attr->qp_access_flags = 0; 1257 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 1258 } else 1259 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 1260 qp_attr_mask); 1261 qp_attr->port_num = id_priv->id.port_num; 1262 *qp_attr_mask |= IB_QP_PORT; 1263 } else { 1264 ret = -ENOSYS; 1265 } 1266 1267 if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set) 1268 qp_attr->timeout = id_priv->timeout; 1269 1270 if ((*qp_attr_mask & IB_QP_MIN_RNR_TIMER) && id_priv->min_rnr_timer_set) 1271 qp_attr->min_rnr_timer = id_priv->min_rnr_timer; 1272 1273 return ret; 1274 } 1275 EXPORT_SYMBOL(rdma_init_qp_attr); 1276 1277 static inline bool cma_zero_addr(const struct sockaddr *addr) 1278 { 1279 switch (addr->sa_family) { 1280 case AF_INET: 1281 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr); 1282 case AF_INET6: 1283 return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr); 1284 case AF_IB: 1285 return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr); 1286 default: 1287 return false; 1288 } 1289 } 1290 1291 static inline bool cma_loopback_addr(const struct sockaddr *addr) 1292 { 1293 switch (addr->sa_family) { 1294 case AF_INET: 1295 return ipv4_is_loopback( 1296 ((struct sockaddr_in *)addr)->sin_addr.s_addr); 1297 case AF_INET6: 1298 return ipv6_addr_loopback( 1299 &((struct sockaddr_in6 *)addr)->sin6_addr); 1300 case AF_IB: 1301 return ib_addr_loopback( 1302 &((struct sockaddr_ib *)addr)->sib_addr); 1303 default: 1304 return false; 1305 } 1306 } 1307 1308 static inline bool cma_any_addr(const struct sockaddr *addr) 1309 { 1310 return cma_zero_addr(addr) || cma_loopback_addr(addr); 1311 } 1312 1313 static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst) 1314 { 1315 if (src->sa_family != dst->sa_family) 1316 return -1; 1317 1318 switch (src->sa_family) { 1319 case AF_INET: 1320 return ((struct sockaddr_in *)src)->sin_addr.s_addr != 1321 ((struct sockaddr_in *)dst)->sin_addr.s_addr; 1322 case AF_INET6: { 1323 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src; 1324 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst; 1325 bool link_local; 1326 1327 if (ipv6_addr_cmp(&src_addr6->sin6_addr, 1328 &dst_addr6->sin6_addr)) 1329 return 1; 1330 link_local = ipv6_addr_type(&dst_addr6->sin6_addr) & 1331 IPV6_ADDR_LINKLOCAL; 1332 /* Link local must match their scope_ids */ 1333 return link_local ? (src_addr6->sin6_scope_id != 1334 dst_addr6->sin6_scope_id) : 1335 0; 1336 } 1337 1338 default: 1339 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr, 1340 &((struct sockaddr_ib *) dst)->sib_addr); 1341 } 1342 } 1343 1344 static __be16 cma_port(const struct sockaddr *addr) 1345 { 1346 struct sockaddr_ib *sib; 1347 1348 switch (addr->sa_family) { 1349 case AF_INET: 1350 return ((struct sockaddr_in *) addr)->sin_port; 1351 case AF_INET6: 1352 return ((struct sockaddr_in6 *) addr)->sin6_port; 1353 case AF_IB: 1354 sib = (struct sockaddr_ib *) addr; 1355 return htons((u16) (be64_to_cpu(sib->sib_sid) & 1356 be64_to_cpu(sib->sib_sid_mask))); 1357 default: 1358 return 0; 1359 } 1360 } 1361 1362 static inline int cma_any_port(const struct sockaddr *addr) 1363 { 1364 return !cma_port(addr); 1365 } 1366 1367 static void cma_save_ib_info(struct sockaddr *src_addr, 1368 struct sockaddr *dst_addr, 1369 const struct rdma_cm_id *listen_id, 1370 const struct sa_path_rec *path) 1371 { 1372 struct sockaddr_ib *listen_ib, *ib; 1373 1374 listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr; 1375 if (src_addr) { 1376 ib = (struct sockaddr_ib *)src_addr; 1377 ib->sib_family = AF_IB; 1378 if (path) { 1379 ib->sib_pkey = path->pkey; 1380 ib->sib_flowinfo = path->flow_label; 1381 memcpy(&ib->sib_addr, &path->sgid, 16); 1382 ib->sib_sid = path->service_id; 1383 ib->sib_scope_id = 0; 1384 } else { 1385 ib->sib_pkey = listen_ib->sib_pkey; 1386 ib->sib_flowinfo = listen_ib->sib_flowinfo; 1387 ib->sib_addr = listen_ib->sib_addr; 1388 ib->sib_sid = listen_ib->sib_sid; 1389 ib->sib_scope_id = listen_ib->sib_scope_id; 1390 } 1391 ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL); 1392 } 1393 if (dst_addr) { 1394 ib = (struct sockaddr_ib *)dst_addr; 1395 ib->sib_family = AF_IB; 1396 if (path) { 1397 ib->sib_pkey = path->pkey; 1398 ib->sib_flowinfo = path->flow_label; 1399 memcpy(&ib->sib_addr, &path->dgid, 16); 1400 } 1401 } 1402 } 1403 1404 static void cma_save_ip4_info(struct sockaddr_in *src_addr, 1405 struct sockaddr_in *dst_addr, 1406 struct cma_hdr *hdr, 1407 __be16 local_port) 1408 { 1409 if (src_addr) { 1410 *src_addr = (struct sockaddr_in) { 1411 .sin_family = AF_INET, 1412 .sin_addr.s_addr = hdr->dst_addr.ip4.addr, 1413 .sin_port = local_port, 1414 }; 1415 } 1416 1417 if (dst_addr) { 1418 *dst_addr = (struct sockaddr_in) { 1419 .sin_family = AF_INET, 1420 .sin_addr.s_addr = hdr->src_addr.ip4.addr, 1421 .sin_port = hdr->port, 1422 }; 1423 } 1424 } 1425 1426 static void cma_save_ip6_info(struct sockaddr_in6 *src_addr, 1427 struct sockaddr_in6 *dst_addr, 1428 struct cma_hdr *hdr, 1429 __be16 local_port) 1430 { 1431 if (src_addr) { 1432 *src_addr = (struct sockaddr_in6) { 1433 .sin6_family = AF_INET6, 1434 .sin6_addr = hdr->dst_addr.ip6, 1435 .sin6_port = local_port, 1436 }; 1437 } 1438 1439 if (dst_addr) { 1440 *dst_addr = (struct sockaddr_in6) { 1441 .sin6_family = AF_INET6, 1442 .sin6_addr = hdr->src_addr.ip6, 1443 .sin6_port = hdr->port, 1444 }; 1445 } 1446 } 1447 1448 static u16 cma_port_from_service_id(__be64 service_id) 1449 { 1450 return (u16)be64_to_cpu(service_id); 1451 } 1452 1453 static int cma_save_ip_info(struct sockaddr *src_addr, 1454 struct sockaddr *dst_addr, 1455 const struct ib_cm_event *ib_event, 1456 __be64 service_id) 1457 { 1458 struct cma_hdr *hdr; 1459 __be16 port; 1460 1461 hdr = ib_event->private_data; 1462 if (hdr->cma_version != CMA_VERSION) 1463 return -EINVAL; 1464 1465 port = htons(cma_port_from_service_id(service_id)); 1466 1467 switch (cma_get_ip_ver(hdr)) { 1468 case 4: 1469 cma_save_ip4_info((struct sockaddr_in *)src_addr, 1470 (struct sockaddr_in *)dst_addr, hdr, port); 1471 break; 1472 case 6: 1473 cma_save_ip6_info((struct sockaddr_in6 *)src_addr, 1474 (struct sockaddr_in6 *)dst_addr, hdr, port); 1475 break; 1476 default: 1477 return -EAFNOSUPPORT; 1478 } 1479 1480 return 0; 1481 } 1482 1483 static int cma_save_net_info(struct sockaddr *src_addr, 1484 struct sockaddr *dst_addr, 1485 const struct rdma_cm_id *listen_id, 1486 const struct ib_cm_event *ib_event, 1487 sa_family_t sa_family, __be64 service_id) 1488 { 1489 if (sa_family == AF_IB) { 1490 if (ib_event->event == IB_CM_REQ_RECEIVED) 1491 cma_save_ib_info(src_addr, dst_addr, listen_id, 1492 ib_event->param.req_rcvd.primary_path); 1493 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) 1494 cma_save_ib_info(src_addr, dst_addr, listen_id, NULL); 1495 return 0; 1496 } 1497 1498 return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id); 1499 } 1500 1501 static int cma_save_req_info(const struct ib_cm_event *ib_event, 1502 struct cma_req_info *req) 1503 { 1504 const struct ib_cm_req_event_param *req_param = 1505 &ib_event->param.req_rcvd; 1506 const struct ib_cm_sidr_req_event_param *sidr_param = 1507 &ib_event->param.sidr_req_rcvd; 1508 1509 switch (ib_event->event) { 1510 case IB_CM_REQ_RECEIVED: 1511 req->device = req_param->listen_id->device; 1512 req->port = req_param->port; 1513 memcpy(&req->local_gid, &req_param->primary_path->sgid, 1514 sizeof(req->local_gid)); 1515 req->has_gid = true; 1516 req->service_id = req_param->primary_path->service_id; 1517 req->pkey = be16_to_cpu(req_param->primary_path->pkey); 1518 if (req->pkey != req_param->bth_pkey) 1519 pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n" 1520 "RDMA CMA: in the future this may cause the request to be dropped\n", 1521 req_param->bth_pkey, req->pkey); 1522 break; 1523 case IB_CM_SIDR_REQ_RECEIVED: 1524 req->device = sidr_param->listen_id->device; 1525 req->port = sidr_param->port; 1526 req->has_gid = false; 1527 req->service_id = sidr_param->service_id; 1528 req->pkey = sidr_param->pkey; 1529 if (req->pkey != sidr_param->bth_pkey) 1530 pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n" 1531 "RDMA CMA: in the future this may cause the request to be dropped\n", 1532 sidr_param->bth_pkey, req->pkey); 1533 break; 1534 default: 1535 return -EINVAL; 1536 } 1537 1538 return 0; 1539 } 1540 1541 static bool validate_ipv4_net_dev(struct net_device *net_dev, 1542 const struct sockaddr_in *dst_addr, 1543 const struct sockaddr_in *src_addr) 1544 { 1545 __be32 daddr = dst_addr->sin_addr.s_addr, 1546 saddr = src_addr->sin_addr.s_addr; 1547 struct fib_result res; 1548 struct flowi4 fl4; 1549 int err; 1550 bool ret; 1551 1552 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1553 ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) || 1554 ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) || 1555 ipv4_is_loopback(saddr)) 1556 return false; 1557 1558 memset(&fl4, 0, sizeof(fl4)); 1559 fl4.flowi4_iif = net_dev->ifindex; 1560 fl4.daddr = daddr; 1561 fl4.saddr = saddr; 1562 1563 rcu_read_lock(); 1564 err = fib_lookup(dev_net(net_dev), &fl4, &res, 0); 1565 ret = err == 0 && FIB_RES_DEV(res) == net_dev; 1566 rcu_read_unlock(); 1567 1568 return ret; 1569 } 1570 1571 static bool validate_ipv6_net_dev(struct net_device *net_dev, 1572 const struct sockaddr_in6 *dst_addr, 1573 const struct sockaddr_in6 *src_addr) 1574 { 1575 #if IS_ENABLED(CONFIG_IPV6) 1576 const int strict = ipv6_addr_type(&dst_addr->sin6_addr) & 1577 IPV6_ADDR_LINKLOCAL; 1578 struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr, 1579 &src_addr->sin6_addr, net_dev->ifindex, 1580 NULL, strict); 1581 bool ret; 1582 1583 if (!rt) 1584 return false; 1585 1586 ret = rt->rt6i_idev->dev == net_dev; 1587 ip6_rt_put(rt); 1588 1589 return ret; 1590 #else 1591 return false; 1592 #endif 1593 } 1594 1595 static bool validate_net_dev(struct net_device *net_dev, 1596 const struct sockaddr *daddr, 1597 const struct sockaddr *saddr) 1598 { 1599 const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr; 1600 const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr; 1601 const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr; 1602 const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr; 1603 1604 switch (daddr->sa_family) { 1605 case AF_INET: 1606 return saddr->sa_family == AF_INET && 1607 validate_ipv4_net_dev(net_dev, daddr4, saddr4); 1608 1609 case AF_INET6: 1610 return saddr->sa_family == AF_INET6 && 1611 validate_ipv6_net_dev(net_dev, daddr6, saddr6); 1612 1613 default: 1614 return false; 1615 } 1616 } 1617 1618 static struct net_device * 1619 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event) 1620 { 1621 const struct ib_gid_attr *sgid_attr = NULL; 1622 struct net_device *ndev; 1623 1624 if (ib_event->event == IB_CM_REQ_RECEIVED) 1625 sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr; 1626 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) 1627 sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr; 1628 1629 if (!sgid_attr) 1630 return NULL; 1631 1632 rcu_read_lock(); 1633 ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr); 1634 if (IS_ERR(ndev)) 1635 ndev = NULL; 1636 else 1637 dev_hold(ndev); 1638 rcu_read_unlock(); 1639 return ndev; 1640 } 1641 1642 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event, 1643 struct cma_req_info *req) 1644 { 1645 struct sockaddr *listen_addr = 1646 (struct sockaddr *)&req->listen_addr_storage; 1647 struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage; 1648 struct net_device *net_dev; 1649 const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL; 1650 int err; 1651 1652 err = cma_save_ip_info(listen_addr, src_addr, ib_event, 1653 req->service_id); 1654 if (err) 1655 return ERR_PTR(err); 1656 1657 if (rdma_protocol_roce(req->device, req->port)) 1658 net_dev = roce_get_net_dev_by_cm_event(ib_event); 1659 else 1660 net_dev = ib_get_net_dev_by_params(req->device, req->port, 1661 req->pkey, 1662 gid, listen_addr); 1663 if (!net_dev) 1664 return ERR_PTR(-ENODEV); 1665 1666 return net_dev; 1667 } 1668 1669 static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id) 1670 { 1671 return (be64_to_cpu(service_id) >> 16) & 0xffff; 1672 } 1673 1674 static bool cma_match_private_data(struct rdma_id_private *id_priv, 1675 const struct cma_hdr *hdr) 1676 { 1677 struct sockaddr *addr = cma_src_addr(id_priv); 1678 __be32 ip4_addr; 1679 struct in6_addr ip6_addr; 1680 1681 if (cma_any_addr(addr) && !id_priv->afonly) 1682 return true; 1683 1684 switch (addr->sa_family) { 1685 case AF_INET: 1686 ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr; 1687 if (cma_get_ip_ver(hdr) != 4) 1688 return false; 1689 if (!cma_any_addr(addr) && 1690 hdr->dst_addr.ip4.addr != ip4_addr) 1691 return false; 1692 break; 1693 case AF_INET6: 1694 ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr; 1695 if (cma_get_ip_ver(hdr) != 6) 1696 return false; 1697 if (!cma_any_addr(addr) && 1698 memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr))) 1699 return false; 1700 break; 1701 case AF_IB: 1702 return true; 1703 default: 1704 return false; 1705 } 1706 1707 return true; 1708 } 1709 1710 static bool cma_protocol_roce(const struct rdma_cm_id *id) 1711 { 1712 struct ib_device *device = id->device; 1713 const u32 port_num = id->port_num ?: rdma_start_port(device); 1714 1715 return rdma_protocol_roce(device, port_num); 1716 } 1717 1718 static bool cma_is_req_ipv6_ll(const struct cma_req_info *req) 1719 { 1720 const struct sockaddr *daddr = 1721 (const struct sockaddr *)&req->listen_addr_storage; 1722 const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr; 1723 1724 /* Returns true if the req is for IPv6 link local */ 1725 return (daddr->sa_family == AF_INET6 && 1726 (ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)); 1727 } 1728 1729 static bool cma_match_net_dev(const struct rdma_cm_id *id, 1730 const struct net_device *net_dev, 1731 const struct cma_req_info *req) 1732 { 1733 const struct rdma_addr *addr = &id->route.addr; 1734 1735 if (!net_dev) 1736 /* This request is an AF_IB request */ 1737 return (!id->port_num || id->port_num == req->port) && 1738 (addr->src_addr.ss_family == AF_IB); 1739 1740 /* 1741 * If the request is not for IPv6 link local, allow matching 1742 * request to any netdevice of the one or multiport rdma device. 1743 */ 1744 if (!cma_is_req_ipv6_ll(req)) 1745 return true; 1746 /* 1747 * Net namespaces must match, and if the listner is listening 1748 * on a specific netdevice than netdevice must match as well. 1749 */ 1750 if (net_eq(dev_net(net_dev), addr->dev_addr.net) && 1751 (!!addr->dev_addr.bound_dev_if == 1752 (addr->dev_addr.bound_dev_if == net_dev->ifindex))) 1753 return true; 1754 else 1755 return false; 1756 } 1757 1758 static struct rdma_id_private *cma_find_listener( 1759 const struct rdma_bind_list *bind_list, 1760 const struct ib_cm_id *cm_id, 1761 const struct ib_cm_event *ib_event, 1762 const struct cma_req_info *req, 1763 const struct net_device *net_dev) 1764 { 1765 struct rdma_id_private *id_priv, *id_priv_dev; 1766 1767 lockdep_assert_held(&lock); 1768 1769 if (!bind_list) 1770 return ERR_PTR(-EINVAL); 1771 1772 hlist_for_each_entry(id_priv, &bind_list->owners, node) { 1773 if (cma_match_private_data(id_priv, ib_event->private_data)) { 1774 if (id_priv->id.device == cm_id->device && 1775 cma_match_net_dev(&id_priv->id, net_dev, req)) 1776 return id_priv; 1777 list_for_each_entry(id_priv_dev, 1778 &id_priv->listen_list, 1779 listen_item) { 1780 if (id_priv_dev->id.device == cm_id->device && 1781 cma_match_net_dev(&id_priv_dev->id, 1782 net_dev, req)) 1783 return id_priv_dev; 1784 } 1785 } 1786 } 1787 1788 return ERR_PTR(-EINVAL); 1789 } 1790 1791 static struct rdma_id_private * 1792 cma_ib_id_from_event(struct ib_cm_id *cm_id, 1793 const struct ib_cm_event *ib_event, 1794 struct cma_req_info *req, 1795 struct net_device **net_dev) 1796 { 1797 struct rdma_bind_list *bind_list; 1798 struct rdma_id_private *id_priv; 1799 int err; 1800 1801 err = cma_save_req_info(ib_event, req); 1802 if (err) 1803 return ERR_PTR(err); 1804 1805 *net_dev = cma_get_net_dev(ib_event, req); 1806 if (IS_ERR(*net_dev)) { 1807 if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) { 1808 /* Assuming the protocol is AF_IB */ 1809 *net_dev = NULL; 1810 } else { 1811 return ERR_CAST(*net_dev); 1812 } 1813 } 1814 1815 mutex_lock(&lock); 1816 /* 1817 * Net namespace might be getting deleted while route lookup, 1818 * cm_id lookup is in progress. Therefore, perform netdevice 1819 * validation, cm_id lookup under rcu lock. 1820 * RCU lock along with netdevice state check, synchronizes with 1821 * netdevice migrating to different net namespace and also avoids 1822 * case where net namespace doesn't get deleted while lookup is in 1823 * progress. 1824 * If the device state is not IFF_UP, its properties such as ifindex 1825 * and nd_net cannot be trusted to remain valid without rcu lock. 1826 * net/core/dev.c change_net_namespace() ensures to synchronize with 1827 * ongoing operations on net device after device is closed using 1828 * synchronize_net(). 1829 */ 1830 rcu_read_lock(); 1831 if (*net_dev) { 1832 /* 1833 * If netdevice is down, it is likely that it is administratively 1834 * down or it might be migrating to different namespace. 1835 * In that case avoid further processing, as the net namespace 1836 * or ifindex may change. 1837 */ 1838 if (((*net_dev)->flags & IFF_UP) == 0) { 1839 id_priv = ERR_PTR(-EHOSTUNREACH); 1840 goto err; 1841 } 1842 1843 if (!validate_net_dev(*net_dev, 1844 (struct sockaddr *)&req->listen_addr_storage, 1845 (struct sockaddr *)&req->src_addr_storage)) { 1846 id_priv = ERR_PTR(-EHOSTUNREACH); 1847 goto err; 1848 } 1849 } 1850 1851 bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net, 1852 rdma_ps_from_service_id(req->service_id), 1853 cma_port_from_service_id(req->service_id)); 1854 id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev); 1855 err: 1856 rcu_read_unlock(); 1857 mutex_unlock(&lock); 1858 if (IS_ERR(id_priv) && *net_dev) { 1859 dev_put(*net_dev); 1860 *net_dev = NULL; 1861 } 1862 return id_priv; 1863 } 1864 1865 static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv) 1866 { 1867 return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr); 1868 } 1869 1870 static void cma_cancel_route(struct rdma_id_private *id_priv) 1871 { 1872 if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) { 1873 if (id_priv->query) 1874 ib_sa_cancel_query(id_priv->query_id, id_priv->query); 1875 } 1876 } 1877 1878 static void _cma_cancel_listens(struct rdma_id_private *id_priv) 1879 { 1880 struct rdma_id_private *dev_id_priv; 1881 1882 lockdep_assert_held(&lock); 1883 1884 /* 1885 * Remove from listen_any_list to prevent added devices from spawning 1886 * additional listen requests. 1887 */ 1888 list_del_init(&id_priv->listen_any_item); 1889 1890 while (!list_empty(&id_priv->listen_list)) { 1891 dev_id_priv = 1892 list_first_entry(&id_priv->listen_list, 1893 struct rdma_id_private, listen_item); 1894 /* sync with device removal to avoid duplicate destruction */ 1895 list_del_init(&dev_id_priv->device_item); 1896 list_del_init(&dev_id_priv->listen_item); 1897 mutex_unlock(&lock); 1898 1899 rdma_destroy_id(&dev_id_priv->id); 1900 mutex_lock(&lock); 1901 } 1902 } 1903 1904 static void cma_cancel_listens(struct rdma_id_private *id_priv) 1905 { 1906 mutex_lock(&lock); 1907 _cma_cancel_listens(id_priv); 1908 mutex_unlock(&lock); 1909 } 1910 1911 static void cma_cancel_operation(struct rdma_id_private *id_priv, 1912 enum rdma_cm_state state) 1913 { 1914 switch (state) { 1915 case RDMA_CM_ADDR_QUERY: 1916 /* 1917 * We can avoid doing the rdma_addr_cancel() based on state, 1918 * only RDMA_CM_ADDR_QUERY has a work that could still execute. 1919 * Notice that the addr_handler work could still be exiting 1920 * outside this state, however due to the interaction with the 1921 * handler_mutex the work is guaranteed not to touch id_priv 1922 * during exit. 1923 */ 1924 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 1925 break; 1926 case RDMA_CM_ROUTE_QUERY: 1927 cma_cancel_route(id_priv); 1928 break; 1929 case RDMA_CM_LISTEN: 1930 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev) 1931 cma_cancel_listens(id_priv); 1932 break; 1933 default: 1934 break; 1935 } 1936 } 1937 1938 static void cma_release_port(struct rdma_id_private *id_priv) 1939 { 1940 struct rdma_bind_list *bind_list = id_priv->bind_list; 1941 struct net *net = id_priv->id.route.addr.dev_addr.net; 1942 1943 if (!bind_list) 1944 return; 1945 1946 mutex_lock(&lock); 1947 hlist_del(&id_priv->node); 1948 if (hlist_empty(&bind_list->owners)) { 1949 cma_ps_remove(net, bind_list->ps, bind_list->port); 1950 kfree(bind_list); 1951 } 1952 mutex_unlock(&lock); 1953 } 1954 1955 static void destroy_mc(struct rdma_id_private *id_priv, 1956 struct cma_multicast *mc) 1957 { 1958 bool send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN); 1959 1960 if (rdma_cap_ib_mcast(id_priv->id.device, id_priv->id.port_num)) 1961 ib_sa_free_multicast(mc->sa_mc); 1962 1963 if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) { 1964 struct rdma_dev_addr *dev_addr = 1965 &id_priv->id.route.addr.dev_addr; 1966 struct net_device *ndev = NULL; 1967 1968 if (dev_addr->bound_dev_if) 1969 ndev = dev_get_by_index(dev_addr->net, 1970 dev_addr->bound_dev_if); 1971 if (ndev && !send_only) { 1972 enum ib_gid_type gid_type; 1973 union ib_gid mgid; 1974 1975 gid_type = id_priv->cma_dev->default_gid_type 1976 [id_priv->id.port_num - 1977 rdma_start_port( 1978 id_priv->cma_dev->device)]; 1979 cma_iboe_set_mgid((struct sockaddr *)&mc->addr, &mgid, 1980 gid_type); 1981 cma_igmp_send(ndev, &mgid, false); 1982 } 1983 dev_put(ndev); 1984 1985 cancel_work_sync(&mc->iboe_join.work); 1986 } 1987 kfree(mc); 1988 } 1989 1990 static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 1991 { 1992 struct cma_multicast *mc; 1993 1994 while (!list_empty(&id_priv->mc_list)) { 1995 mc = list_first_entry(&id_priv->mc_list, struct cma_multicast, 1996 list); 1997 list_del(&mc->list); 1998 destroy_mc(id_priv, mc); 1999 } 2000 } 2001 2002 static void _destroy_id(struct rdma_id_private *id_priv, 2003 enum rdma_cm_state state) 2004 { 2005 cma_cancel_operation(id_priv, state); 2006 2007 rdma_restrack_del(&id_priv->res); 2008 cma_remove_id_from_tree(id_priv); 2009 if (id_priv->cma_dev) { 2010 if (rdma_cap_ib_cm(id_priv->id.device, 1)) { 2011 if (id_priv->cm_id.ib) 2012 ib_destroy_cm_id(id_priv->cm_id.ib); 2013 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) { 2014 if (id_priv->cm_id.iw) 2015 iw_destroy_cm_id(id_priv->cm_id.iw); 2016 } 2017 cma_leave_mc_groups(id_priv); 2018 cma_release_dev(id_priv); 2019 } 2020 2021 cma_release_port(id_priv); 2022 cma_id_put(id_priv); 2023 wait_for_completion(&id_priv->comp); 2024 2025 if (id_priv->internal_id) 2026 cma_id_put(id_priv->id.context); 2027 2028 kfree(id_priv->id.route.path_rec); 2029 kfree(id_priv->id.route.path_rec_inbound); 2030 kfree(id_priv->id.route.path_rec_outbound); 2031 2032 put_net(id_priv->id.route.addr.dev_addr.net); 2033 kfree(id_priv); 2034 } 2035 2036 /* 2037 * destroy an ID from within the handler_mutex. This ensures that no other 2038 * handlers can start running concurrently. 2039 */ 2040 static void destroy_id_handler_unlock(struct rdma_id_private *id_priv) 2041 __releases(&idprv->handler_mutex) 2042 { 2043 enum rdma_cm_state state; 2044 unsigned long flags; 2045 2046 trace_cm_id_destroy(id_priv); 2047 2048 /* 2049 * Setting the state to destroyed under the handler mutex provides a 2050 * fence against calling handler callbacks. If this is invoked due to 2051 * the failure of a handler callback then it guarentees that no future 2052 * handlers will be called. 2053 */ 2054 lockdep_assert_held(&id_priv->handler_mutex); 2055 spin_lock_irqsave(&id_priv->lock, flags); 2056 state = id_priv->state; 2057 id_priv->state = RDMA_CM_DESTROYING; 2058 spin_unlock_irqrestore(&id_priv->lock, flags); 2059 mutex_unlock(&id_priv->handler_mutex); 2060 _destroy_id(id_priv, state); 2061 } 2062 2063 void rdma_destroy_id(struct rdma_cm_id *id) 2064 { 2065 struct rdma_id_private *id_priv = 2066 container_of(id, struct rdma_id_private, id); 2067 2068 mutex_lock(&id_priv->handler_mutex); 2069 destroy_id_handler_unlock(id_priv); 2070 } 2071 EXPORT_SYMBOL(rdma_destroy_id); 2072 2073 static int cma_rep_recv(struct rdma_id_private *id_priv) 2074 { 2075 int ret; 2076 2077 ret = cma_modify_qp_rtr(id_priv, NULL); 2078 if (ret) 2079 goto reject; 2080 2081 ret = cma_modify_qp_rts(id_priv, NULL); 2082 if (ret) 2083 goto reject; 2084 2085 trace_cm_send_rtu(id_priv); 2086 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 2087 if (ret) 2088 goto reject; 2089 2090 return 0; 2091 reject: 2092 pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret); 2093 cma_modify_qp_err(id_priv); 2094 trace_cm_send_rej(id_priv); 2095 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 2096 NULL, 0, NULL, 0); 2097 return ret; 2098 } 2099 2100 static void cma_set_rep_event_data(struct rdma_cm_event *event, 2101 const struct ib_cm_rep_event_param *rep_data, 2102 void *private_data) 2103 { 2104 event->param.conn.private_data = private_data; 2105 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 2106 event->param.conn.responder_resources = rep_data->responder_resources; 2107 event->param.conn.initiator_depth = rep_data->initiator_depth; 2108 event->param.conn.flow_control = rep_data->flow_control; 2109 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 2110 event->param.conn.srq = rep_data->srq; 2111 event->param.conn.qp_num = rep_data->remote_qpn; 2112 2113 event->ece.vendor_id = rep_data->ece.vendor_id; 2114 event->ece.attr_mod = rep_data->ece.attr_mod; 2115 } 2116 2117 static int cma_cm_event_handler(struct rdma_id_private *id_priv, 2118 struct rdma_cm_event *event) 2119 { 2120 int ret; 2121 2122 lockdep_assert_held(&id_priv->handler_mutex); 2123 2124 trace_cm_event_handler(id_priv, event); 2125 ret = id_priv->id.event_handler(&id_priv->id, event); 2126 trace_cm_event_done(id_priv, event, ret); 2127 return ret; 2128 } 2129 2130 static int cma_ib_handler(struct ib_cm_id *cm_id, 2131 const struct ib_cm_event *ib_event) 2132 { 2133 struct rdma_id_private *id_priv = cm_id->context; 2134 struct rdma_cm_event event = {}; 2135 enum rdma_cm_state state; 2136 int ret; 2137 2138 mutex_lock(&id_priv->handler_mutex); 2139 state = READ_ONCE(id_priv->state); 2140 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && 2141 state != RDMA_CM_CONNECT) || 2142 (ib_event->event == IB_CM_TIMEWAIT_EXIT && 2143 state != RDMA_CM_DISCONNECT)) 2144 goto out; 2145 2146 switch (ib_event->event) { 2147 case IB_CM_REQ_ERROR: 2148 case IB_CM_REP_ERROR: 2149 event.event = RDMA_CM_EVENT_UNREACHABLE; 2150 event.status = -ETIMEDOUT; 2151 break; 2152 case IB_CM_REP_RECEIVED: 2153 if (state == RDMA_CM_CONNECT && 2154 (id_priv->id.qp_type != IB_QPT_UD)) { 2155 trace_cm_send_mra(id_priv); 2156 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 2157 } 2158 if (id_priv->id.qp) { 2159 event.status = cma_rep_recv(id_priv); 2160 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 2161 RDMA_CM_EVENT_ESTABLISHED; 2162 } else { 2163 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 2164 } 2165 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 2166 ib_event->private_data); 2167 break; 2168 case IB_CM_RTU_RECEIVED: 2169 case IB_CM_USER_ESTABLISHED: 2170 event.event = RDMA_CM_EVENT_ESTABLISHED; 2171 break; 2172 case IB_CM_DREQ_ERROR: 2173 event.status = -ETIMEDOUT; 2174 fallthrough; 2175 case IB_CM_DREQ_RECEIVED: 2176 case IB_CM_DREP_RECEIVED: 2177 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT, 2178 RDMA_CM_DISCONNECT)) 2179 goto out; 2180 event.event = RDMA_CM_EVENT_DISCONNECTED; 2181 break; 2182 case IB_CM_TIMEWAIT_EXIT: 2183 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; 2184 break; 2185 case IB_CM_MRA_RECEIVED: 2186 /* ignore event */ 2187 goto out; 2188 case IB_CM_REJ_RECEIVED: 2189 pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id, 2190 ib_event->param.rej_rcvd.reason)); 2191 cma_modify_qp_err(id_priv); 2192 event.status = ib_event->param.rej_rcvd.reason; 2193 event.event = RDMA_CM_EVENT_REJECTED; 2194 event.param.conn.private_data = ib_event->private_data; 2195 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 2196 break; 2197 default: 2198 pr_err("RDMA CMA: unexpected IB CM event: %d\n", 2199 ib_event->event); 2200 goto out; 2201 } 2202 2203 ret = cma_cm_event_handler(id_priv, &event); 2204 if (ret) { 2205 /* Destroy the CM ID by returning a non-zero value. */ 2206 id_priv->cm_id.ib = NULL; 2207 destroy_id_handler_unlock(id_priv); 2208 return ret; 2209 } 2210 out: 2211 mutex_unlock(&id_priv->handler_mutex); 2212 return 0; 2213 } 2214 2215 static struct rdma_id_private * 2216 cma_ib_new_conn_id(const struct rdma_cm_id *listen_id, 2217 const struct ib_cm_event *ib_event, 2218 struct net_device *net_dev) 2219 { 2220 struct rdma_id_private *listen_id_priv; 2221 struct rdma_id_private *id_priv; 2222 struct rdma_cm_id *id; 2223 struct rdma_route *rt; 2224 const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; 2225 struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path; 2226 const __be64 service_id = 2227 ib_event->param.req_rcvd.primary_path->service_id; 2228 int ret; 2229 2230 listen_id_priv = container_of(listen_id, struct rdma_id_private, id); 2231 id_priv = __rdma_create_id(listen_id->route.addr.dev_addr.net, 2232 listen_id->event_handler, listen_id->context, 2233 listen_id->ps, 2234 ib_event->param.req_rcvd.qp_type, 2235 listen_id_priv); 2236 if (IS_ERR(id_priv)) 2237 return NULL; 2238 2239 id = &id_priv->id; 2240 if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, 2241 (struct sockaddr *)&id->route.addr.dst_addr, 2242 listen_id, ib_event, ss_family, service_id)) 2243 goto err; 2244 2245 rt = &id->route; 2246 rt->num_pri_alt_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 2247 rt->path_rec = kmalloc_array(rt->num_pri_alt_paths, 2248 sizeof(*rt->path_rec), GFP_KERNEL); 2249 if (!rt->path_rec) 2250 goto err; 2251 2252 rt->path_rec[0] = *path; 2253 if (rt->num_pri_alt_paths == 2) 2254 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 2255 2256 if (net_dev) { 2257 rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev); 2258 } else { 2259 if (!cma_protocol_roce(listen_id) && 2260 cma_any_addr(cma_src_addr(id_priv))) { 2261 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; 2262 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 2263 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey)); 2264 } else if (!cma_any_addr(cma_src_addr(id_priv))) { 2265 ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr); 2266 if (ret) 2267 goto err; 2268 } 2269 } 2270 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 2271 2272 id_priv->state = RDMA_CM_CONNECT; 2273 return id_priv; 2274 2275 err: 2276 rdma_destroy_id(id); 2277 return NULL; 2278 } 2279 2280 static struct rdma_id_private * 2281 cma_ib_new_udp_id(const struct rdma_cm_id *listen_id, 2282 const struct ib_cm_event *ib_event, 2283 struct net_device *net_dev) 2284 { 2285 const struct rdma_id_private *listen_id_priv; 2286 struct rdma_id_private *id_priv; 2287 struct rdma_cm_id *id; 2288 const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; 2289 struct net *net = listen_id->route.addr.dev_addr.net; 2290 int ret; 2291 2292 listen_id_priv = container_of(listen_id, struct rdma_id_private, id); 2293 id_priv = __rdma_create_id(net, listen_id->event_handler, 2294 listen_id->context, listen_id->ps, IB_QPT_UD, 2295 listen_id_priv); 2296 if (IS_ERR(id_priv)) 2297 return NULL; 2298 2299 id = &id_priv->id; 2300 if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, 2301 (struct sockaddr *)&id->route.addr.dst_addr, 2302 listen_id, ib_event, ss_family, 2303 ib_event->param.sidr_req_rcvd.service_id)) 2304 goto err; 2305 2306 if (net_dev) { 2307 rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev); 2308 } else { 2309 if (!cma_any_addr(cma_src_addr(id_priv))) { 2310 ret = cma_translate_addr(cma_src_addr(id_priv), 2311 &id->route.addr.dev_addr); 2312 if (ret) 2313 goto err; 2314 } 2315 } 2316 2317 id_priv->state = RDMA_CM_CONNECT; 2318 return id_priv; 2319 err: 2320 rdma_destroy_id(id); 2321 return NULL; 2322 } 2323 2324 static void cma_set_req_event_data(struct rdma_cm_event *event, 2325 const struct ib_cm_req_event_param *req_data, 2326 void *private_data, int offset) 2327 { 2328 event->param.conn.private_data = private_data + offset; 2329 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 2330 event->param.conn.responder_resources = req_data->responder_resources; 2331 event->param.conn.initiator_depth = req_data->initiator_depth; 2332 event->param.conn.flow_control = req_data->flow_control; 2333 event->param.conn.retry_count = req_data->retry_count; 2334 event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 2335 event->param.conn.srq = req_data->srq; 2336 event->param.conn.qp_num = req_data->remote_qpn; 2337 2338 event->ece.vendor_id = req_data->ece.vendor_id; 2339 event->ece.attr_mod = req_data->ece.attr_mod; 2340 } 2341 2342 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id, 2343 const struct ib_cm_event *ib_event) 2344 { 2345 return (((ib_event->event == IB_CM_REQ_RECEIVED) && 2346 (ib_event->param.req_rcvd.qp_type == id->qp_type)) || 2347 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) && 2348 (id->qp_type == IB_QPT_UD)) || 2349 (!id->qp_type)); 2350 } 2351 2352 static int cma_ib_req_handler(struct ib_cm_id *cm_id, 2353 const struct ib_cm_event *ib_event) 2354 { 2355 struct rdma_id_private *listen_id, *conn_id = NULL; 2356 struct rdma_cm_event event = {}; 2357 struct cma_req_info req = {}; 2358 struct net_device *net_dev; 2359 u8 offset; 2360 int ret; 2361 2362 listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev); 2363 if (IS_ERR(listen_id)) 2364 return PTR_ERR(listen_id); 2365 2366 trace_cm_req_handler(listen_id, ib_event->event); 2367 if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) { 2368 ret = -EINVAL; 2369 goto net_dev_put; 2370 } 2371 2372 mutex_lock(&listen_id->handler_mutex); 2373 if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) { 2374 ret = -ECONNABORTED; 2375 goto err_unlock; 2376 } 2377 2378 offset = cma_user_data_offset(listen_id); 2379 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 2380 if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) { 2381 conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev); 2382 event.param.ud.private_data = ib_event->private_data + offset; 2383 event.param.ud.private_data_len = 2384 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 2385 } else { 2386 conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev); 2387 cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 2388 ib_event->private_data, offset); 2389 } 2390 if (!conn_id) { 2391 ret = -ENOMEM; 2392 goto err_unlock; 2393 } 2394 2395 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 2396 ret = cma_ib_acquire_dev(conn_id, listen_id, &req); 2397 if (ret) { 2398 destroy_id_handler_unlock(conn_id); 2399 goto err_unlock; 2400 } 2401 2402 conn_id->cm_id.ib = cm_id; 2403 cm_id->context = conn_id; 2404 cm_id->cm_handler = cma_ib_handler; 2405 2406 ret = cma_cm_event_handler(conn_id, &event); 2407 if (ret) { 2408 /* Destroy the CM ID by returning a non-zero value. */ 2409 conn_id->cm_id.ib = NULL; 2410 mutex_unlock(&listen_id->handler_mutex); 2411 destroy_id_handler_unlock(conn_id); 2412 goto net_dev_put; 2413 } 2414 2415 if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT && 2416 conn_id->id.qp_type != IB_QPT_UD) { 2417 trace_cm_send_mra(cm_id->context); 2418 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 2419 } 2420 mutex_unlock(&conn_id->handler_mutex); 2421 2422 err_unlock: 2423 mutex_unlock(&listen_id->handler_mutex); 2424 2425 net_dev_put: 2426 if (net_dev) 2427 dev_put(net_dev); 2428 2429 return ret; 2430 } 2431 2432 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr) 2433 { 2434 if (addr->sa_family == AF_IB) 2435 return ((struct sockaddr_ib *) addr)->sib_sid; 2436 2437 return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr))); 2438 } 2439 EXPORT_SYMBOL(rdma_get_service_id); 2440 2441 void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid, 2442 union ib_gid *dgid) 2443 { 2444 struct rdma_addr *addr = &cm_id->route.addr; 2445 2446 if (!cm_id->device) { 2447 if (sgid) 2448 memset(sgid, 0, sizeof(*sgid)); 2449 if (dgid) 2450 memset(dgid, 0, sizeof(*dgid)); 2451 return; 2452 } 2453 2454 if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) { 2455 if (sgid) 2456 rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid); 2457 if (dgid) 2458 rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid); 2459 } else { 2460 if (sgid) 2461 rdma_addr_get_sgid(&addr->dev_addr, sgid); 2462 if (dgid) 2463 rdma_addr_get_dgid(&addr->dev_addr, dgid); 2464 } 2465 } 2466 EXPORT_SYMBOL(rdma_read_gids); 2467 2468 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 2469 { 2470 struct rdma_id_private *id_priv = iw_id->context; 2471 struct rdma_cm_event event = {}; 2472 int ret = 0; 2473 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; 2474 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; 2475 2476 mutex_lock(&id_priv->handler_mutex); 2477 if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT) 2478 goto out; 2479 2480 switch (iw_event->event) { 2481 case IW_CM_EVENT_CLOSE: 2482 event.event = RDMA_CM_EVENT_DISCONNECTED; 2483 break; 2484 case IW_CM_EVENT_CONNECT_REPLY: 2485 memcpy(cma_src_addr(id_priv), laddr, 2486 rdma_addr_size(laddr)); 2487 memcpy(cma_dst_addr(id_priv), raddr, 2488 rdma_addr_size(raddr)); 2489 switch (iw_event->status) { 2490 case 0: 2491 event.event = RDMA_CM_EVENT_ESTABLISHED; 2492 event.param.conn.initiator_depth = iw_event->ird; 2493 event.param.conn.responder_resources = iw_event->ord; 2494 break; 2495 case -ECONNRESET: 2496 case -ECONNREFUSED: 2497 event.event = RDMA_CM_EVENT_REJECTED; 2498 break; 2499 case -ETIMEDOUT: 2500 event.event = RDMA_CM_EVENT_UNREACHABLE; 2501 break; 2502 default: 2503 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 2504 break; 2505 } 2506 break; 2507 case IW_CM_EVENT_ESTABLISHED: 2508 event.event = RDMA_CM_EVENT_ESTABLISHED; 2509 event.param.conn.initiator_depth = iw_event->ird; 2510 event.param.conn.responder_resources = iw_event->ord; 2511 break; 2512 default: 2513 goto out; 2514 } 2515 2516 event.status = iw_event->status; 2517 event.param.conn.private_data = iw_event->private_data; 2518 event.param.conn.private_data_len = iw_event->private_data_len; 2519 ret = cma_cm_event_handler(id_priv, &event); 2520 if (ret) { 2521 /* Destroy the CM ID by returning a non-zero value. */ 2522 id_priv->cm_id.iw = NULL; 2523 destroy_id_handler_unlock(id_priv); 2524 return ret; 2525 } 2526 2527 out: 2528 mutex_unlock(&id_priv->handler_mutex); 2529 return ret; 2530 } 2531 2532 static int iw_conn_req_handler(struct iw_cm_id *cm_id, 2533 struct iw_cm_event *iw_event) 2534 { 2535 struct rdma_id_private *listen_id, *conn_id; 2536 struct rdma_cm_event event = {}; 2537 int ret = -ECONNABORTED; 2538 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; 2539 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; 2540 2541 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 2542 event.param.conn.private_data = iw_event->private_data; 2543 event.param.conn.private_data_len = iw_event->private_data_len; 2544 event.param.conn.initiator_depth = iw_event->ird; 2545 event.param.conn.responder_resources = iw_event->ord; 2546 2547 listen_id = cm_id->context; 2548 2549 mutex_lock(&listen_id->handler_mutex); 2550 if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) 2551 goto out; 2552 2553 /* Create a new RDMA id for the new IW CM ID */ 2554 conn_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net, 2555 listen_id->id.event_handler, 2556 listen_id->id.context, RDMA_PS_TCP, 2557 IB_QPT_RC, listen_id); 2558 if (IS_ERR(conn_id)) { 2559 ret = -ENOMEM; 2560 goto out; 2561 } 2562 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 2563 conn_id->state = RDMA_CM_CONNECT; 2564 2565 ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr); 2566 if (ret) { 2567 mutex_unlock(&listen_id->handler_mutex); 2568 destroy_id_handler_unlock(conn_id); 2569 return ret; 2570 } 2571 2572 ret = cma_iw_acquire_dev(conn_id, listen_id); 2573 if (ret) { 2574 mutex_unlock(&listen_id->handler_mutex); 2575 destroy_id_handler_unlock(conn_id); 2576 return ret; 2577 } 2578 2579 conn_id->cm_id.iw = cm_id; 2580 cm_id->context = conn_id; 2581 cm_id->cm_handler = cma_iw_handler; 2582 2583 memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr)); 2584 memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr)); 2585 2586 ret = cma_cm_event_handler(conn_id, &event); 2587 if (ret) { 2588 /* User wants to destroy the CM ID */ 2589 conn_id->cm_id.iw = NULL; 2590 mutex_unlock(&listen_id->handler_mutex); 2591 destroy_id_handler_unlock(conn_id); 2592 return ret; 2593 } 2594 2595 mutex_unlock(&conn_id->handler_mutex); 2596 2597 out: 2598 mutex_unlock(&listen_id->handler_mutex); 2599 return ret; 2600 } 2601 2602 static int cma_ib_listen(struct rdma_id_private *id_priv) 2603 { 2604 struct sockaddr *addr; 2605 struct ib_cm_id *id; 2606 __be64 svc_id; 2607 2608 addr = cma_src_addr(id_priv); 2609 svc_id = rdma_get_service_id(&id_priv->id, addr); 2610 id = ib_cm_insert_listen(id_priv->id.device, 2611 cma_ib_req_handler, svc_id); 2612 if (IS_ERR(id)) 2613 return PTR_ERR(id); 2614 id_priv->cm_id.ib = id; 2615 2616 return 0; 2617 } 2618 2619 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 2620 { 2621 int ret; 2622 struct iw_cm_id *id; 2623 2624 id = iw_create_cm_id(id_priv->id.device, 2625 iw_conn_req_handler, 2626 id_priv); 2627 if (IS_ERR(id)) 2628 return PTR_ERR(id); 2629 2630 mutex_lock(&id_priv->qp_mutex); 2631 id->tos = id_priv->tos; 2632 id->tos_set = id_priv->tos_set; 2633 mutex_unlock(&id_priv->qp_mutex); 2634 id->afonly = id_priv->afonly; 2635 id_priv->cm_id.iw = id; 2636 2637 memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv), 2638 rdma_addr_size(cma_src_addr(id_priv))); 2639 2640 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 2641 2642 if (ret) { 2643 iw_destroy_cm_id(id_priv->cm_id.iw); 2644 id_priv->cm_id.iw = NULL; 2645 } 2646 2647 return ret; 2648 } 2649 2650 static int cma_listen_handler(struct rdma_cm_id *id, 2651 struct rdma_cm_event *event) 2652 { 2653 struct rdma_id_private *id_priv = id->context; 2654 2655 /* Listening IDs are always destroyed on removal */ 2656 if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) 2657 return -1; 2658 2659 id->context = id_priv->id.context; 2660 id->event_handler = id_priv->id.event_handler; 2661 trace_cm_event_handler(id_priv, event); 2662 return id_priv->id.event_handler(id, event); 2663 } 2664 2665 static int cma_listen_on_dev(struct rdma_id_private *id_priv, 2666 struct cma_device *cma_dev, 2667 struct rdma_id_private **to_destroy) 2668 { 2669 struct rdma_id_private *dev_id_priv; 2670 struct net *net = id_priv->id.route.addr.dev_addr.net; 2671 int ret; 2672 2673 lockdep_assert_held(&lock); 2674 2675 *to_destroy = NULL; 2676 if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1)) 2677 return 0; 2678 2679 dev_id_priv = 2680 __rdma_create_id(net, cma_listen_handler, id_priv, 2681 id_priv->id.ps, id_priv->id.qp_type, id_priv); 2682 if (IS_ERR(dev_id_priv)) 2683 return PTR_ERR(dev_id_priv); 2684 2685 dev_id_priv->state = RDMA_CM_ADDR_BOUND; 2686 memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv), 2687 rdma_addr_size(cma_src_addr(id_priv))); 2688 2689 _cma_attach_to_dev(dev_id_priv, cma_dev); 2690 rdma_restrack_add(&dev_id_priv->res); 2691 cma_id_get(id_priv); 2692 dev_id_priv->internal_id = 1; 2693 dev_id_priv->afonly = id_priv->afonly; 2694 mutex_lock(&id_priv->qp_mutex); 2695 dev_id_priv->tos_set = id_priv->tos_set; 2696 dev_id_priv->tos = id_priv->tos; 2697 mutex_unlock(&id_priv->qp_mutex); 2698 2699 ret = rdma_listen(&dev_id_priv->id, id_priv->backlog); 2700 if (ret) 2701 goto err_listen; 2702 list_add_tail(&dev_id_priv->listen_item, &id_priv->listen_list); 2703 return 0; 2704 err_listen: 2705 /* Caller must destroy this after releasing lock */ 2706 *to_destroy = dev_id_priv; 2707 dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret); 2708 return ret; 2709 } 2710 2711 static int cma_listen_on_all(struct rdma_id_private *id_priv) 2712 { 2713 struct rdma_id_private *to_destroy; 2714 struct cma_device *cma_dev; 2715 int ret; 2716 2717 mutex_lock(&lock); 2718 list_add_tail(&id_priv->listen_any_item, &listen_any_list); 2719 list_for_each_entry(cma_dev, &dev_list, list) { 2720 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy); 2721 if (ret) { 2722 /* Prevent racing with cma_process_remove() */ 2723 if (to_destroy) 2724 list_del_init(&to_destroy->device_item); 2725 goto err_listen; 2726 } 2727 } 2728 mutex_unlock(&lock); 2729 return 0; 2730 2731 err_listen: 2732 _cma_cancel_listens(id_priv); 2733 mutex_unlock(&lock); 2734 if (to_destroy) 2735 rdma_destroy_id(&to_destroy->id); 2736 return ret; 2737 } 2738 2739 void rdma_set_service_type(struct rdma_cm_id *id, int tos) 2740 { 2741 struct rdma_id_private *id_priv; 2742 2743 id_priv = container_of(id, struct rdma_id_private, id); 2744 mutex_lock(&id_priv->qp_mutex); 2745 id_priv->tos = (u8) tos; 2746 id_priv->tos_set = true; 2747 mutex_unlock(&id_priv->qp_mutex); 2748 } 2749 EXPORT_SYMBOL(rdma_set_service_type); 2750 2751 /** 2752 * rdma_set_ack_timeout() - Set the ack timeout of QP associated 2753 * with a connection identifier. 2754 * @id: Communication identifier to associated with service type. 2755 * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec. 2756 * 2757 * This function should be called before rdma_connect() on active side, 2758 * and on passive side before rdma_accept(). It is applicable to primary 2759 * path only. The timeout will affect the local side of the QP, it is not 2760 * negotiated with remote side and zero disables the timer. In case it is 2761 * set before rdma_resolve_route, the value will also be used to determine 2762 * PacketLifeTime for RoCE. 2763 * 2764 * Return: 0 for success 2765 */ 2766 int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout) 2767 { 2768 struct rdma_id_private *id_priv; 2769 2770 if (id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_INI) 2771 return -EINVAL; 2772 2773 id_priv = container_of(id, struct rdma_id_private, id); 2774 mutex_lock(&id_priv->qp_mutex); 2775 id_priv->timeout = timeout; 2776 id_priv->timeout_set = true; 2777 mutex_unlock(&id_priv->qp_mutex); 2778 2779 return 0; 2780 } 2781 EXPORT_SYMBOL(rdma_set_ack_timeout); 2782 2783 /** 2784 * rdma_set_min_rnr_timer() - Set the minimum RNR Retry timer of the 2785 * QP associated with a connection identifier. 2786 * @id: Communication identifier to associated with service type. 2787 * @min_rnr_timer: 5-bit value encoded as Table 45: "Encoding for RNR NAK 2788 * Timer Field" in the IBTA specification. 2789 * 2790 * This function should be called before rdma_connect() on active 2791 * side, and on passive side before rdma_accept(). The timer value 2792 * will be associated with the local QP. When it receives a send it is 2793 * not read to handle, typically if the receive queue is empty, an RNR 2794 * Retry NAK is returned to the requester with the min_rnr_timer 2795 * encoded. The requester will then wait at least the time specified 2796 * in the NAK before retrying. The default is zero, which translates 2797 * to a minimum RNR Timer value of 655 ms. 2798 * 2799 * Return: 0 for success 2800 */ 2801 int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer) 2802 { 2803 struct rdma_id_private *id_priv; 2804 2805 /* It is a five-bit value */ 2806 if (min_rnr_timer & 0xe0) 2807 return -EINVAL; 2808 2809 if (WARN_ON(id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_TGT)) 2810 return -EINVAL; 2811 2812 id_priv = container_of(id, struct rdma_id_private, id); 2813 mutex_lock(&id_priv->qp_mutex); 2814 id_priv->min_rnr_timer = min_rnr_timer; 2815 id_priv->min_rnr_timer_set = true; 2816 mutex_unlock(&id_priv->qp_mutex); 2817 2818 return 0; 2819 } 2820 EXPORT_SYMBOL(rdma_set_min_rnr_timer); 2821 2822 static void route_set_path_rec_inbound(struct cma_work *work, 2823 struct sa_path_rec *path_rec) 2824 { 2825 struct rdma_route *route = &work->id->id.route; 2826 2827 if (!route->path_rec_inbound) { 2828 route->path_rec_inbound = 2829 kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL); 2830 if (!route->path_rec_inbound) 2831 return; 2832 } 2833 2834 *route->path_rec_inbound = *path_rec; 2835 } 2836 2837 static void route_set_path_rec_outbound(struct cma_work *work, 2838 struct sa_path_rec *path_rec) 2839 { 2840 struct rdma_route *route = &work->id->id.route; 2841 2842 if (!route->path_rec_outbound) { 2843 route->path_rec_outbound = 2844 kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL); 2845 if (!route->path_rec_outbound) 2846 return; 2847 } 2848 2849 *route->path_rec_outbound = *path_rec; 2850 } 2851 2852 static void cma_query_handler(int status, struct sa_path_rec *path_rec, 2853 int num_prs, void *context) 2854 { 2855 struct cma_work *work = context; 2856 struct rdma_route *route; 2857 int i; 2858 2859 route = &work->id->id.route; 2860 2861 if (status) 2862 goto fail; 2863 2864 for (i = 0; i < num_prs; i++) { 2865 if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP)) 2866 *route->path_rec = path_rec[i]; 2867 else if (path_rec[i].flags & IB_PATH_INBOUND) 2868 route_set_path_rec_inbound(work, &path_rec[i]); 2869 else if (path_rec[i].flags & IB_PATH_OUTBOUND) 2870 route_set_path_rec_outbound(work, &path_rec[i]); 2871 } 2872 if (!route->path_rec) { 2873 status = -EINVAL; 2874 goto fail; 2875 } 2876 2877 route->num_pri_alt_paths = 1; 2878 queue_work(cma_wq, &work->work); 2879 return; 2880 2881 fail: 2882 work->old_state = RDMA_CM_ROUTE_QUERY; 2883 work->new_state = RDMA_CM_ADDR_RESOLVED; 2884 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 2885 work->event.status = status; 2886 pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n", 2887 status); 2888 queue_work(cma_wq, &work->work); 2889 } 2890 2891 static int cma_query_ib_route(struct rdma_id_private *id_priv, 2892 unsigned long timeout_ms, struct cma_work *work) 2893 { 2894 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2895 struct sa_path_rec path_rec; 2896 ib_sa_comp_mask comp_mask; 2897 struct sockaddr_in6 *sin6; 2898 struct sockaddr_ib *sib; 2899 2900 memset(&path_rec, 0, sizeof path_rec); 2901 2902 if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num)) 2903 path_rec.rec_type = SA_PATH_REC_TYPE_OPA; 2904 else 2905 path_rec.rec_type = SA_PATH_REC_TYPE_IB; 2906 rdma_addr_get_sgid(dev_addr, &path_rec.sgid); 2907 rdma_addr_get_dgid(dev_addr, &path_rec.dgid); 2908 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 2909 path_rec.numb_path = 1; 2910 path_rec.reversible = 1; 2911 path_rec.service_id = rdma_get_service_id(&id_priv->id, 2912 cma_dst_addr(id_priv)); 2913 2914 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 2915 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 2916 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; 2917 2918 switch (cma_family(id_priv)) { 2919 case AF_INET: 2920 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); 2921 comp_mask |= IB_SA_PATH_REC_QOS_CLASS; 2922 break; 2923 case AF_INET6: 2924 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); 2925 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); 2926 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 2927 break; 2928 case AF_IB: 2929 sib = (struct sockaddr_ib *) cma_src_addr(id_priv); 2930 path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20); 2931 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 2932 break; 2933 } 2934 2935 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 2936 id_priv->id.port_num, &path_rec, 2937 comp_mask, timeout_ms, 2938 GFP_KERNEL, cma_query_handler, 2939 work, &id_priv->query); 2940 2941 return (id_priv->query_id < 0) ? id_priv->query_id : 0; 2942 } 2943 2944 static void cma_iboe_join_work_handler(struct work_struct *work) 2945 { 2946 struct cma_multicast *mc = 2947 container_of(work, struct cma_multicast, iboe_join.work); 2948 struct rdma_cm_event *event = &mc->iboe_join.event; 2949 struct rdma_id_private *id_priv = mc->id_priv; 2950 int ret; 2951 2952 mutex_lock(&id_priv->handler_mutex); 2953 if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING || 2954 READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL) 2955 goto out_unlock; 2956 2957 ret = cma_cm_event_handler(id_priv, event); 2958 WARN_ON(ret); 2959 2960 out_unlock: 2961 mutex_unlock(&id_priv->handler_mutex); 2962 if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN) 2963 rdma_destroy_ah_attr(&event->param.ud.ah_attr); 2964 } 2965 2966 static void cma_work_handler(struct work_struct *_work) 2967 { 2968 struct cma_work *work = container_of(_work, struct cma_work, work); 2969 struct rdma_id_private *id_priv = work->id; 2970 2971 mutex_lock(&id_priv->handler_mutex); 2972 if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING || 2973 READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL) 2974 goto out_unlock; 2975 if (work->old_state != 0 || work->new_state != 0) { 2976 if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 2977 goto out_unlock; 2978 } 2979 2980 if (cma_cm_event_handler(id_priv, &work->event)) { 2981 cma_id_put(id_priv); 2982 destroy_id_handler_unlock(id_priv); 2983 goto out_free; 2984 } 2985 2986 out_unlock: 2987 mutex_unlock(&id_priv->handler_mutex); 2988 cma_id_put(id_priv); 2989 out_free: 2990 if (work->event.event == RDMA_CM_EVENT_MULTICAST_JOIN) 2991 rdma_destroy_ah_attr(&work->event.param.ud.ah_attr); 2992 kfree(work); 2993 } 2994 2995 static void cma_init_resolve_route_work(struct cma_work *work, 2996 struct rdma_id_private *id_priv) 2997 { 2998 work->id = id_priv; 2999 INIT_WORK(&work->work, cma_work_handler); 3000 work->old_state = RDMA_CM_ROUTE_QUERY; 3001 work->new_state = RDMA_CM_ROUTE_RESOLVED; 3002 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 3003 } 3004 3005 static void enqueue_resolve_addr_work(struct cma_work *work, 3006 struct rdma_id_private *id_priv) 3007 { 3008 /* Balances with cma_id_put() in cma_work_handler */ 3009 cma_id_get(id_priv); 3010 3011 work->id = id_priv; 3012 INIT_WORK(&work->work, cma_work_handler); 3013 work->old_state = RDMA_CM_ADDR_QUERY; 3014 work->new_state = RDMA_CM_ADDR_RESOLVED; 3015 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 3016 3017 queue_work(cma_wq, &work->work); 3018 } 3019 3020 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, 3021 unsigned long timeout_ms) 3022 { 3023 struct rdma_route *route = &id_priv->id.route; 3024 struct cma_work *work; 3025 int ret; 3026 3027 work = kzalloc(sizeof *work, GFP_KERNEL); 3028 if (!work) 3029 return -ENOMEM; 3030 3031 cma_init_resolve_route_work(work, id_priv); 3032 3033 if (!route->path_rec) 3034 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 3035 if (!route->path_rec) { 3036 ret = -ENOMEM; 3037 goto err1; 3038 } 3039 3040 ret = cma_query_ib_route(id_priv, timeout_ms, work); 3041 if (ret) 3042 goto err2; 3043 3044 return 0; 3045 err2: 3046 kfree(route->path_rec); 3047 route->path_rec = NULL; 3048 err1: 3049 kfree(work); 3050 return ret; 3051 } 3052 3053 static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type, 3054 unsigned long supported_gids, 3055 enum ib_gid_type default_gid) 3056 { 3057 if ((network_type == RDMA_NETWORK_IPV4 || 3058 network_type == RDMA_NETWORK_IPV6) && 3059 test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids)) 3060 return IB_GID_TYPE_ROCE_UDP_ENCAP; 3061 3062 return default_gid; 3063 } 3064 3065 /* 3066 * cma_iboe_set_path_rec_l2_fields() is helper function which sets 3067 * path record type based on GID type. 3068 * It also sets up other L2 fields which includes destination mac address 3069 * netdev ifindex, of the path record. 3070 * It returns the netdev of the bound interface for this path record entry. 3071 */ 3072 static struct net_device * 3073 cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv) 3074 { 3075 struct rdma_route *route = &id_priv->id.route; 3076 enum ib_gid_type gid_type = IB_GID_TYPE_ROCE; 3077 struct rdma_addr *addr = &route->addr; 3078 unsigned long supported_gids; 3079 struct net_device *ndev; 3080 3081 if (!addr->dev_addr.bound_dev_if) 3082 return NULL; 3083 3084 ndev = dev_get_by_index(addr->dev_addr.net, 3085 addr->dev_addr.bound_dev_if); 3086 if (!ndev) 3087 return NULL; 3088 3089 supported_gids = roce_gid_type_mask_support(id_priv->id.device, 3090 id_priv->id.port_num); 3091 gid_type = cma_route_gid_type(addr->dev_addr.network, 3092 supported_gids, 3093 id_priv->gid_type); 3094 /* Use the hint from IP Stack to select GID Type */ 3095 if (gid_type < ib_network_to_gid_type(addr->dev_addr.network)) 3096 gid_type = ib_network_to_gid_type(addr->dev_addr.network); 3097 route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type); 3098 3099 route->path_rec->roce.route_resolved = true; 3100 sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr); 3101 return ndev; 3102 } 3103 3104 int rdma_set_ib_path(struct rdma_cm_id *id, 3105 struct sa_path_rec *path_rec) 3106 { 3107 struct rdma_id_private *id_priv; 3108 struct net_device *ndev; 3109 int ret; 3110 3111 id_priv = container_of(id, struct rdma_id_private, id); 3112 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, 3113 RDMA_CM_ROUTE_RESOLVED)) 3114 return -EINVAL; 3115 3116 id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec), 3117 GFP_KERNEL); 3118 if (!id->route.path_rec) { 3119 ret = -ENOMEM; 3120 goto err; 3121 } 3122 3123 if (rdma_protocol_roce(id->device, id->port_num)) { 3124 ndev = cma_iboe_set_path_rec_l2_fields(id_priv); 3125 if (!ndev) { 3126 ret = -ENODEV; 3127 goto err_free; 3128 } 3129 dev_put(ndev); 3130 } 3131 3132 id->route.num_pri_alt_paths = 1; 3133 return 0; 3134 3135 err_free: 3136 kfree(id->route.path_rec); 3137 id->route.path_rec = NULL; 3138 err: 3139 cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED); 3140 return ret; 3141 } 3142 EXPORT_SYMBOL(rdma_set_ib_path); 3143 3144 static int cma_resolve_iw_route(struct rdma_id_private *id_priv) 3145 { 3146 struct cma_work *work; 3147 3148 work = kzalloc(sizeof *work, GFP_KERNEL); 3149 if (!work) 3150 return -ENOMEM; 3151 3152 cma_init_resolve_route_work(work, id_priv); 3153 queue_work(cma_wq, &work->work); 3154 return 0; 3155 } 3156 3157 static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio) 3158 { 3159 struct net_device *dev; 3160 3161 dev = vlan_dev_real_dev(vlan_ndev); 3162 if (dev->num_tc) 3163 return netdev_get_prio_tc_map(dev, prio); 3164 3165 return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) & 3166 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 3167 } 3168 3169 struct iboe_prio_tc_map { 3170 int input_prio; 3171 int output_tc; 3172 bool found; 3173 }; 3174 3175 static int get_lower_vlan_dev_tc(struct net_device *dev, 3176 struct netdev_nested_priv *priv) 3177 { 3178 struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv->data; 3179 3180 if (is_vlan_dev(dev)) 3181 map->output_tc = get_vlan_ndev_tc(dev, map->input_prio); 3182 else if (dev->num_tc) 3183 map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio); 3184 else 3185 map->output_tc = 0; 3186 /* We are interested only in first level VLAN device, so always 3187 * return 1 to stop iterating over next level devices. 3188 */ 3189 map->found = true; 3190 return 1; 3191 } 3192 3193 static int iboe_tos_to_sl(struct net_device *ndev, int tos) 3194 { 3195 struct iboe_prio_tc_map prio_tc_map = {}; 3196 int prio = rt_tos2priority(tos); 3197 struct netdev_nested_priv priv; 3198 3199 /* If VLAN device, get it directly from the VLAN netdev */ 3200 if (is_vlan_dev(ndev)) 3201 return get_vlan_ndev_tc(ndev, prio); 3202 3203 prio_tc_map.input_prio = prio; 3204 priv.data = (void *)&prio_tc_map; 3205 rcu_read_lock(); 3206 netdev_walk_all_lower_dev_rcu(ndev, 3207 get_lower_vlan_dev_tc, 3208 &priv); 3209 rcu_read_unlock(); 3210 /* If map is found from lower device, use it; Otherwise 3211 * continue with the current netdevice to get priority to tc map. 3212 */ 3213 if (prio_tc_map.found) 3214 return prio_tc_map.output_tc; 3215 else if (ndev->num_tc) 3216 return netdev_get_prio_tc_map(ndev, prio); 3217 else 3218 return 0; 3219 } 3220 3221 static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv) 3222 { 3223 struct sockaddr_in6 *addr6; 3224 u16 dport, sport; 3225 u32 hash, fl; 3226 3227 addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv); 3228 fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK; 3229 if ((cma_family(id_priv) != AF_INET6) || !fl) { 3230 dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv))); 3231 sport = be16_to_cpu(cma_port(cma_src_addr(id_priv))); 3232 hash = (u32)sport * 31 + dport; 3233 fl = hash & IB_GRH_FLOWLABEL_MASK; 3234 } 3235 3236 return cpu_to_be32(fl); 3237 } 3238 3239 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) 3240 { 3241 struct rdma_route *route = &id_priv->id.route; 3242 struct rdma_addr *addr = &route->addr; 3243 struct cma_work *work; 3244 int ret; 3245 struct net_device *ndev; 3246 3247 u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num - 3248 rdma_start_port(id_priv->cma_dev->device)]; 3249 u8 tos; 3250 3251 mutex_lock(&id_priv->qp_mutex); 3252 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos; 3253 mutex_unlock(&id_priv->qp_mutex); 3254 3255 work = kzalloc(sizeof *work, GFP_KERNEL); 3256 if (!work) 3257 return -ENOMEM; 3258 3259 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); 3260 if (!route->path_rec) { 3261 ret = -ENOMEM; 3262 goto err1; 3263 } 3264 3265 route->num_pri_alt_paths = 1; 3266 3267 ndev = cma_iboe_set_path_rec_l2_fields(id_priv); 3268 if (!ndev) { 3269 ret = -ENODEV; 3270 goto err2; 3271 } 3272 3273 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 3274 &route->path_rec->sgid); 3275 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr, 3276 &route->path_rec->dgid); 3277 3278 if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB) 3279 /* TODO: get the hoplimit from the inet/inet6 device */ 3280 route->path_rec->hop_limit = addr->dev_addr.hoplimit; 3281 else 3282 route->path_rec->hop_limit = 1; 3283 route->path_rec->reversible = 1; 3284 route->path_rec->pkey = cpu_to_be16(0xffff); 3285 route->path_rec->mtu_selector = IB_SA_EQ; 3286 route->path_rec->sl = iboe_tos_to_sl(ndev, tos); 3287 route->path_rec->traffic_class = tos; 3288 route->path_rec->mtu = iboe_get_mtu(ndev->mtu); 3289 route->path_rec->rate_selector = IB_SA_EQ; 3290 route->path_rec->rate = iboe_get_rate(ndev); 3291 dev_put(ndev); 3292 route->path_rec->packet_life_time_selector = IB_SA_EQ; 3293 /* In case ACK timeout is set, use this value to calculate 3294 * PacketLifeTime. As per IBTA 12.7.34, 3295 * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay). 3296 * Assuming a negligible local ACK delay, we can use 3297 * PacketLifeTime = local ACK timeout/2 3298 * as a reasonable approximation for RoCE networks. 3299 */ 3300 mutex_lock(&id_priv->qp_mutex); 3301 if (id_priv->timeout_set && id_priv->timeout) 3302 route->path_rec->packet_life_time = id_priv->timeout - 1; 3303 else 3304 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME; 3305 mutex_unlock(&id_priv->qp_mutex); 3306 3307 if (!route->path_rec->mtu) { 3308 ret = -EINVAL; 3309 goto err2; 3310 } 3311 3312 if (rdma_protocol_roce_udp_encap(id_priv->id.device, 3313 id_priv->id.port_num)) 3314 route->path_rec->flow_label = 3315 cma_get_roce_udp_flow_label(id_priv); 3316 3317 cma_init_resolve_route_work(work, id_priv); 3318 queue_work(cma_wq, &work->work); 3319 3320 return 0; 3321 3322 err2: 3323 kfree(route->path_rec); 3324 route->path_rec = NULL; 3325 route->num_pri_alt_paths = 0; 3326 err1: 3327 kfree(work); 3328 return ret; 3329 } 3330 3331 int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms) 3332 { 3333 struct rdma_id_private *id_priv; 3334 int ret; 3335 3336 if (!timeout_ms) 3337 return -EINVAL; 3338 3339 id_priv = container_of(id, struct rdma_id_private, id); 3340 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY)) 3341 return -EINVAL; 3342 3343 cma_id_get(id_priv); 3344 if (rdma_cap_ib_sa(id->device, id->port_num)) 3345 ret = cma_resolve_ib_route(id_priv, timeout_ms); 3346 else if (rdma_protocol_roce(id->device, id->port_num)) { 3347 ret = cma_resolve_iboe_route(id_priv); 3348 if (!ret) 3349 cma_add_id_to_tree(id_priv); 3350 } 3351 else if (rdma_protocol_iwarp(id->device, id->port_num)) 3352 ret = cma_resolve_iw_route(id_priv); 3353 else 3354 ret = -ENOSYS; 3355 3356 if (ret) 3357 goto err; 3358 3359 return 0; 3360 err: 3361 cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED); 3362 cma_id_put(id_priv); 3363 return ret; 3364 } 3365 EXPORT_SYMBOL(rdma_resolve_route); 3366 3367 static void cma_set_loopback(struct sockaddr *addr) 3368 { 3369 switch (addr->sa_family) { 3370 case AF_INET: 3371 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); 3372 break; 3373 case AF_INET6: 3374 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr, 3375 0, 0, 0, htonl(1)); 3376 break; 3377 default: 3378 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr, 3379 0, 0, 0, htonl(1)); 3380 break; 3381 } 3382 } 3383 3384 static int cma_bind_loopback(struct rdma_id_private *id_priv) 3385 { 3386 struct cma_device *cma_dev, *cur_dev; 3387 union ib_gid gid; 3388 enum ib_port_state port_state; 3389 unsigned int p; 3390 u16 pkey; 3391 int ret; 3392 3393 cma_dev = NULL; 3394 mutex_lock(&lock); 3395 list_for_each_entry(cur_dev, &dev_list, list) { 3396 if (cma_family(id_priv) == AF_IB && 3397 !rdma_cap_ib_cm(cur_dev->device, 1)) 3398 continue; 3399 3400 if (!cma_dev) 3401 cma_dev = cur_dev; 3402 3403 rdma_for_each_port (cur_dev->device, p) { 3404 if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) && 3405 port_state == IB_PORT_ACTIVE) { 3406 cma_dev = cur_dev; 3407 goto port_found; 3408 } 3409 } 3410 } 3411 3412 if (!cma_dev) { 3413 ret = -ENODEV; 3414 goto out; 3415 } 3416 3417 p = 1; 3418 3419 port_found: 3420 ret = rdma_query_gid(cma_dev->device, p, 0, &gid); 3421 if (ret) 3422 goto out; 3423 3424 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 3425 if (ret) 3426 goto out; 3427 3428 id_priv->id.route.addr.dev_addr.dev_type = 3429 (rdma_protocol_ib(cma_dev->device, p)) ? 3430 ARPHRD_INFINIBAND : ARPHRD_ETHER; 3431 3432 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 3433 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 3434 id_priv->id.port_num = p; 3435 cma_attach_to_dev(id_priv, cma_dev); 3436 rdma_restrack_add(&id_priv->res); 3437 cma_set_loopback(cma_src_addr(id_priv)); 3438 out: 3439 mutex_unlock(&lock); 3440 return ret; 3441 } 3442 3443 static void addr_handler(int status, struct sockaddr *src_addr, 3444 struct rdma_dev_addr *dev_addr, void *context) 3445 { 3446 struct rdma_id_private *id_priv = context; 3447 struct rdma_cm_event event = {}; 3448 struct sockaddr *addr; 3449 struct sockaddr_storage old_addr; 3450 3451 mutex_lock(&id_priv->handler_mutex); 3452 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, 3453 RDMA_CM_ADDR_RESOLVED)) 3454 goto out; 3455 3456 /* 3457 * Store the previous src address, so that if we fail to acquire 3458 * matching rdma device, old address can be restored back, which helps 3459 * to cancel the cma listen operation correctly. 3460 */ 3461 addr = cma_src_addr(id_priv); 3462 memcpy(&old_addr, addr, rdma_addr_size(addr)); 3463 memcpy(addr, src_addr, rdma_addr_size(src_addr)); 3464 if (!status && !id_priv->cma_dev) { 3465 status = cma_acquire_dev_by_src_ip(id_priv); 3466 if (status) 3467 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n", 3468 status); 3469 rdma_restrack_add(&id_priv->res); 3470 } else if (status) { 3471 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status); 3472 } 3473 3474 if (status) { 3475 memcpy(addr, &old_addr, 3476 rdma_addr_size((struct sockaddr *)&old_addr)); 3477 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, 3478 RDMA_CM_ADDR_BOUND)) 3479 goto out; 3480 event.event = RDMA_CM_EVENT_ADDR_ERROR; 3481 event.status = status; 3482 } else 3483 event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 3484 3485 if (cma_cm_event_handler(id_priv, &event)) { 3486 destroy_id_handler_unlock(id_priv); 3487 return; 3488 } 3489 out: 3490 mutex_unlock(&id_priv->handler_mutex); 3491 } 3492 3493 static int cma_resolve_loopback(struct rdma_id_private *id_priv) 3494 { 3495 struct cma_work *work; 3496 union ib_gid gid; 3497 int ret; 3498 3499 work = kzalloc(sizeof *work, GFP_KERNEL); 3500 if (!work) 3501 return -ENOMEM; 3502 3503 if (!id_priv->cma_dev) { 3504 ret = cma_bind_loopback(id_priv); 3505 if (ret) 3506 goto err; 3507 } 3508 3509 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 3510 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 3511 3512 enqueue_resolve_addr_work(work, id_priv); 3513 return 0; 3514 err: 3515 kfree(work); 3516 return ret; 3517 } 3518 3519 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) 3520 { 3521 struct cma_work *work; 3522 int ret; 3523 3524 work = kzalloc(sizeof *work, GFP_KERNEL); 3525 if (!work) 3526 return -ENOMEM; 3527 3528 if (!id_priv->cma_dev) { 3529 ret = cma_resolve_ib_dev(id_priv); 3530 if (ret) 3531 goto err; 3532 } 3533 3534 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) 3535 &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); 3536 3537 enqueue_resolve_addr_work(work, id_priv); 3538 return 0; 3539 err: 3540 kfree(work); 3541 return ret; 3542 } 3543 3544 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 3545 const struct sockaddr *dst_addr) 3546 { 3547 struct sockaddr_storage zero_sock = {}; 3548 3549 if (src_addr && src_addr->sa_family) 3550 return rdma_bind_addr(id, src_addr); 3551 3552 /* 3553 * When the src_addr is not specified, automatically supply an any addr 3554 */ 3555 zero_sock.ss_family = dst_addr->sa_family; 3556 if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) { 3557 struct sockaddr_in6 *src_addr6 = 3558 (struct sockaddr_in6 *)&zero_sock; 3559 struct sockaddr_in6 *dst_addr6 = 3560 (struct sockaddr_in6 *)dst_addr; 3561 3562 src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; 3563 if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL) 3564 id->route.addr.dev_addr.bound_dev_if = 3565 dst_addr6->sin6_scope_id; 3566 } else if (dst_addr->sa_family == AF_IB) { 3567 ((struct sockaddr_ib *)&zero_sock)->sib_pkey = 3568 ((struct sockaddr_ib *)dst_addr)->sib_pkey; 3569 } 3570 return rdma_bind_addr(id, (struct sockaddr *)&zero_sock); 3571 } 3572 3573 /* 3574 * If required, resolve the source address for bind and leave the id_priv in 3575 * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior 3576 * calls made by ULP, a previously bound ID will not be re-bound and src_addr is 3577 * ignored. 3578 */ 3579 static int resolve_prepare_src(struct rdma_id_private *id_priv, 3580 struct sockaddr *src_addr, 3581 const struct sockaddr *dst_addr) 3582 { 3583 int ret; 3584 3585 memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); 3586 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { 3587 /* For a well behaved ULP state will be RDMA_CM_IDLE */ 3588 ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr); 3589 if (ret) 3590 goto err_dst; 3591 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, 3592 RDMA_CM_ADDR_QUERY))) { 3593 ret = -EINVAL; 3594 goto err_dst; 3595 } 3596 } 3597 3598 if (cma_family(id_priv) != dst_addr->sa_family) { 3599 ret = -EINVAL; 3600 goto err_state; 3601 } 3602 return 0; 3603 3604 err_state: 3605 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 3606 err_dst: 3607 memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); 3608 return ret; 3609 } 3610 3611 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 3612 const struct sockaddr *dst_addr, unsigned long timeout_ms) 3613 { 3614 struct rdma_id_private *id_priv = 3615 container_of(id, struct rdma_id_private, id); 3616 int ret; 3617 3618 ret = resolve_prepare_src(id_priv, src_addr, dst_addr); 3619 if (ret) 3620 return ret; 3621 3622 if (cma_any_addr(dst_addr)) { 3623 ret = cma_resolve_loopback(id_priv); 3624 } else { 3625 if (dst_addr->sa_family == AF_IB) { 3626 ret = cma_resolve_ib_addr(id_priv); 3627 } else { 3628 /* 3629 * The FSM can return back to RDMA_CM_ADDR_BOUND after 3630 * rdma_resolve_ip() is called, eg through the error 3631 * path in addr_handler(). If this happens the existing 3632 * request must be canceled before issuing a new one. 3633 * Since canceling a request is a bit slow and this 3634 * oddball path is rare, keep track once a request has 3635 * been issued. The track turns out to be a permanent 3636 * state since this is the only cancel as it is 3637 * immediately before rdma_resolve_ip(). 3638 */ 3639 if (id_priv->used_resolve_ip) 3640 rdma_addr_cancel(&id->route.addr.dev_addr); 3641 else 3642 id_priv->used_resolve_ip = 1; 3643 ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr, 3644 &id->route.addr.dev_addr, 3645 timeout_ms, addr_handler, 3646 false, id_priv); 3647 } 3648 } 3649 if (ret) 3650 goto err; 3651 3652 return 0; 3653 err: 3654 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 3655 return ret; 3656 } 3657 EXPORT_SYMBOL(rdma_resolve_addr); 3658 3659 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse) 3660 { 3661 struct rdma_id_private *id_priv; 3662 unsigned long flags; 3663 int ret; 3664 3665 id_priv = container_of(id, struct rdma_id_private, id); 3666 spin_lock_irqsave(&id_priv->lock, flags); 3667 if ((reuse && id_priv->state != RDMA_CM_LISTEN) || 3668 id_priv->state == RDMA_CM_IDLE) { 3669 id_priv->reuseaddr = reuse; 3670 ret = 0; 3671 } else { 3672 ret = -EINVAL; 3673 } 3674 spin_unlock_irqrestore(&id_priv->lock, flags); 3675 return ret; 3676 } 3677 EXPORT_SYMBOL(rdma_set_reuseaddr); 3678 3679 int rdma_set_afonly(struct rdma_cm_id *id, int afonly) 3680 { 3681 struct rdma_id_private *id_priv; 3682 unsigned long flags; 3683 int ret; 3684 3685 id_priv = container_of(id, struct rdma_id_private, id); 3686 spin_lock_irqsave(&id_priv->lock, flags); 3687 if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) { 3688 id_priv->options |= (1 << CMA_OPTION_AFONLY); 3689 id_priv->afonly = afonly; 3690 ret = 0; 3691 } else { 3692 ret = -EINVAL; 3693 } 3694 spin_unlock_irqrestore(&id_priv->lock, flags); 3695 return ret; 3696 } 3697 EXPORT_SYMBOL(rdma_set_afonly); 3698 3699 static void cma_bind_port(struct rdma_bind_list *bind_list, 3700 struct rdma_id_private *id_priv) 3701 { 3702 struct sockaddr *addr; 3703 struct sockaddr_ib *sib; 3704 u64 sid, mask; 3705 __be16 port; 3706 3707 lockdep_assert_held(&lock); 3708 3709 addr = cma_src_addr(id_priv); 3710 port = htons(bind_list->port); 3711 3712 switch (addr->sa_family) { 3713 case AF_INET: 3714 ((struct sockaddr_in *) addr)->sin_port = port; 3715 break; 3716 case AF_INET6: 3717 ((struct sockaddr_in6 *) addr)->sin6_port = port; 3718 break; 3719 case AF_IB: 3720 sib = (struct sockaddr_ib *) addr; 3721 sid = be64_to_cpu(sib->sib_sid); 3722 mask = be64_to_cpu(sib->sib_sid_mask); 3723 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port)); 3724 sib->sib_sid_mask = cpu_to_be64(~0ULL); 3725 break; 3726 } 3727 id_priv->bind_list = bind_list; 3728 hlist_add_head(&id_priv->node, &bind_list->owners); 3729 } 3730 3731 static int cma_alloc_port(enum rdma_ucm_port_space ps, 3732 struct rdma_id_private *id_priv, unsigned short snum) 3733 { 3734 struct rdma_bind_list *bind_list; 3735 int ret; 3736 3737 lockdep_assert_held(&lock); 3738 3739 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 3740 if (!bind_list) 3741 return -ENOMEM; 3742 3743 ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list, 3744 snum); 3745 if (ret < 0) 3746 goto err; 3747 3748 bind_list->ps = ps; 3749 bind_list->port = snum; 3750 cma_bind_port(bind_list, id_priv); 3751 return 0; 3752 err: 3753 kfree(bind_list); 3754 return ret == -ENOSPC ? -EADDRNOTAVAIL : ret; 3755 } 3756 3757 static int cma_port_is_unique(struct rdma_bind_list *bind_list, 3758 struct rdma_id_private *id_priv) 3759 { 3760 struct rdma_id_private *cur_id; 3761 struct sockaddr *daddr = cma_dst_addr(id_priv); 3762 struct sockaddr *saddr = cma_src_addr(id_priv); 3763 __be16 dport = cma_port(daddr); 3764 3765 lockdep_assert_held(&lock); 3766 3767 hlist_for_each_entry(cur_id, &bind_list->owners, node) { 3768 struct sockaddr *cur_daddr = cma_dst_addr(cur_id); 3769 struct sockaddr *cur_saddr = cma_src_addr(cur_id); 3770 __be16 cur_dport = cma_port(cur_daddr); 3771 3772 if (id_priv == cur_id) 3773 continue; 3774 3775 /* different dest port -> unique */ 3776 if (!cma_any_port(daddr) && 3777 !cma_any_port(cur_daddr) && 3778 (dport != cur_dport)) 3779 continue; 3780 3781 /* different src address -> unique */ 3782 if (!cma_any_addr(saddr) && 3783 !cma_any_addr(cur_saddr) && 3784 cma_addr_cmp(saddr, cur_saddr)) 3785 continue; 3786 3787 /* different dst address -> unique */ 3788 if (!cma_any_addr(daddr) && 3789 !cma_any_addr(cur_daddr) && 3790 cma_addr_cmp(daddr, cur_daddr)) 3791 continue; 3792 3793 return -EADDRNOTAVAIL; 3794 } 3795 return 0; 3796 } 3797 3798 static int cma_alloc_any_port(enum rdma_ucm_port_space ps, 3799 struct rdma_id_private *id_priv) 3800 { 3801 static unsigned int last_used_port; 3802 int low, high, remaining; 3803 unsigned int rover; 3804 struct net *net = id_priv->id.route.addr.dev_addr.net; 3805 3806 lockdep_assert_held(&lock); 3807 3808 inet_get_local_port_range(net, &low, &high); 3809 remaining = (high - low) + 1; 3810 rover = prandom_u32() % remaining + low; 3811 retry: 3812 if (last_used_port != rover) { 3813 struct rdma_bind_list *bind_list; 3814 int ret; 3815 3816 bind_list = cma_ps_find(net, ps, (unsigned short)rover); 3817 3818 if (!bind_list) { 3819 ret = cma_alloc_port(ps, id_priv, rover); 3820 } else { 3821 ret = cma_port_is_unique(bind_list, id_priv); 3822 if (!ret) 3823 cma_bind_port(bind_list, id_priv); 3824 } 3825 /* 3826 * Remember previously used port number in order to avoid 3827 * re-using same port immediately after it is closed. 3828 */ 3829 if (!ret) 3830 last_used_port = rover; 3831 if (ret != -EADDRNOTAVAIL) 3832 return ret; 3833 } 3834 if (--remaining) { 3835 rover++; 3836 if ((rover < low) || (rover > high)) 3837 rover = low; 3838 goto retry; 3839 } 3840 return -EADDRNOTAVAIL; 3841 } 3842 3843 /* 3844 * Check that the requested port is available. This is called when trying to 3845 * bind to a specific port, or when trying to listen on a bound port. In 3846 * the latter case, the provided id_priv may already be on the bind_list, but 3847 * we still need to check that it's okay to start listening. 3848 */ 3849 static int cma_check_port(struct rdma_bind_list *bind_list, 3850 struct rdma_id_private *id_priv, uint8_t reuseaddr) 3851 { 3852 struct rdma_id_private *cur_id; 3853 struct sockaddr *addr, *cur_addr; 3854 3855 lockdep_assert_held(&lock); 3856 3857 addr = cma_src_addr(id_priv); 3858 hlist_for_each_entry(cur_id, &bind_list->owners, node) { 3859 if (id_priv == cur_id) 3860 continue; 3861 3862 if (reuseaddr && cur_id->reuseaddr) 3863 continue; 3864 3865 cur_addr = cma_src_addr(cur_id); 3866 if (id_priv->afonly && cur_id->afonly && 3867 (addr->sa_family != cur_addr->sa_family)) 3868 continue; 3869 3870 if (cma_any_addr(addr) || cma_any_addr(cur_addr)) 3871 return -EADDRNOTAVAIL; 3872 3873 if (!cma_addr_cmp(addr, cur_addr)) 3874 return -EADDRINUSE; 3875 } 3876 return 0; 3877 } 3878 3879 static int cma_use_port(enum rdma_ucm_port_space ps, 3880 struct rdma_id_private *id_priv) 3881 { 3882 struct rdma_bind_list *bind_list; 3883 unsigned short snum; 3884 int ret; 3885 3886 lockdep_assert_held(&lock); 3887 3888 snum = ntohs(cma_port(cma_src_addr(id_priv))); 3889 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 3890 return -EACCES; 3891 3892 bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum); 3893 if (!bind_list) { 3894 ret = cma_alloc_port(ps, id_priv, snum); 3895 } else { 3896 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr); 3897 if (!ret) 3898 cma_bind_port(bind_list, id_priv); 3899 } 3900 return ret; 3901 } 3902 3903 static enum rdma_ucm_port_space 3904 cma_select_inet_ps(struct rdma_id_private *id_priv) 3905 { 3906 switch (id_priv->id.ps) { 3907 case RDMA_PS_TCP: 3908 case RDMA_PS_UDP: 3909 case RDMA_PS_IPOIB: 3910 case RDMA_PS_IB: 3911 return id_priv->id.ps; 3912 default: 3913 3914 return 0; 3915 } 3916 } 3917 3918 static enum rdma_ucm_port_space 3919 cma_select_ib_ps(struct rdma_id_private *id_priv) 3920 { 3921 enum rdma_ucm_port_space ps = 0; 3922 struct sockaddr_ib *sib; 3923 u64 sid_ps, mask, sid; 3924 3925 sib = (struct sockaddr_ib *) cma_src_addr(id_priv); 3926 mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK; 3927 sid = be64_to_cpu(sib->sib_sid) & mask; 3928 3929 if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) { 3930 sid_ps = RDMA_IB_IP_PS_IB; 3931 ps = RDMA_PS_IB; 3932 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) && 3933 (sid == (RDMA_IB_IP_PS_TCP & mask))) { 3934 sid_ps = RDMA_IB_IP_PS_TCP; 3935 ps = RDMA_PS_TCP; 3936 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) && 3937 (sid == (RDMA_IB_IP_PS_UDP & mask))) { 3938 sid_ps = RDMA_IB_IP_PS_UDP; 3939 ps = RDMA_PS_UDP; 3940 } 3941 3942 if (ps) { 3943 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib))); 3944 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK | 3945 be64_to_cpu(sib->sib_sid_mask)); 3946 } 3947 return ps; 3948 } 3949 3950 static int cma_get_port(struct rdma_id_private *id_priv) 3951 { 3952 enum rdma_ucm_port_space ps; 3953 int ret; 3954 3955 if (cma_family(id_priv) != AF_IB) 3956 ps = cma_select_inet_ps(id_priv); 3957 else 3958 ps = cma_select_ib_ps(id_priv); 3959 if (!ps) 3960 return -EPROTONOSUPPORT; 3961 3962 mutex_lock(&lock); 3963 if (cma_any_port(cma_src_addr(id_priv))) 3964 ret = cma_alloc_any_port(ps, id_priv); 3965 else 3966 ret = cma_use_port(ps, id_priv); 3967 mutex_unlock(&lock); 3968 3969 return ret; 3970 } 3971 3972 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, 3973 struct sockaddr *addr) 3974 { 3975 #if IS_ENABLED(CONFIG_IPV6) 3976 struct sockaddr_in6 *sin6; 3977 3978 if (addr->sa_family != AF_INET6) 3979 return 0; 3980 3981 sin6 = (struct sockaddr_in6 *) addr; 3982 3983 if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL)) 3984 return 0; 3985 3986 if (!sin6->sin6_scope_id) 3987 return -EINVAL; 3988 3989 dev_addr->bound_dev_if = sin6->sin6_scope_id; 3990 #endif 3991 return 0; 3992 } 3993 3994 int rdma_listen(struct rdma_cm_id *id, int backlog) 3995 { 3996 struct rdma_id_private *id_priv = 3997 container_of(id, struct rdma_id_private, id); 3998 int ret; 3999 4000 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) { 4001 struct sockaddr_in any_in = { 4002 .sin_family = AF_INET, 4003 .sin_addr.s_addr = htonl(INADDR_ANY), 4004 }; 4005 4006 /* For a well behaved ULP state will be RDMA_CM_IDLE */ 4007 ret = rdma_bind_addr(id, (struct sockaddr *)&any_in); 4008 if (ret) 4009 return ret; 4010 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, 4011 RDMA_CM_LISTEN))) 4012 return -EINVAL; 4013 } 4014 4015 /* 4016 * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable 4017 * any more, and has to be unique in the bind list. 4018 */ 4019 if (id_priv->reuseaddr) { 4020 mutex_lock(&lock); 4021 ret = cma_check_port(id_priv->bind_list, id_priv, 0); 4022 if (!ret) 4023 id_priv->reuseaddr = 0; 4024 mutex_unlock(&lock); 4025 if (ret) 4026 goto err; 4027 } 4028 4029 id_priv->backlog = backlog; 4030 if (id_priv->cma_dev) { 4031 if (rdma_cap_ib_cm(id->device, 1)) { 4032 ret = cma_ib_listen(id_priv); 4033 if (ret) 4034 goto err; 4035 } else if (rdma_cap_iw_cm(id->device, 1)) { 4036 ret = cma_iw_listen(id_priv, backlog); 4037 if (ret) 4038 goto err; 4039 } else { 4040 ret = -ENOSYS; 4041 goto err; 4042 } 4043 } else { 4044 ret = cma_listen_on_all(id_priv); 4045 if (ret) 4046 goto err; 4047 } 4048 4049 return 0; 4050 err: 4051 id_priv->backlog = 0; 4052 /* 4053 * All the failure paths that lead here will not allow the req_handler's 4054 * to have run. 4055 */ 4056 cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND); 4057 return ret; 4058 } 4059 EXPORT_SYMBOL(rdma_listen); 4060 4061 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 4062 { 4063 struct rdma_id_private *id_priv; 4064 int ret; 4065 struct sockaddr *daddr; 4066 4067 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 && 4068 addr->sa_family != AF_IB) 4069 return -EAFNOSUPPORT; 4070 4071 id_priv = container_of(id, struct rdma_id_private, id); 4072 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND)) 4073 return -EINVAL; 4074 4075 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); 4076 if (ret) 4077 goto err1; 4078 4079 memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr)); 4080 if (!cma_any_addr(addr)) { 4081 ret = cma_translate_addr(addr, &id->route.addr.dev_addr); 4082 if (ret) 4083 goto err1; 4084 4085 ret = cma_acquire_dev_by_src_ip(id_priv); 4086 if (ret) 4087 goto err1; 4088 } 4089 4090 if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) { 4091 if (addr->sa_family == AF_INET) 4092 id_priv->afonly = 1; 4093 #if IS_ENABLED(CONFIG_IPV6) 4094 else if (addr->sa_family == AF_INET6) { 4095 struct net *net = id_priv->id.route.addr.dev_addr.net; 4096 4097 id_priv->afonly = net->ipv6.sysctl.bindv6only; 4098 } 4099 #endif 4100 } 4101 daddr = cma_dst_addr(id_priv); 4102 daddr->sa_family = addr->sa_family; 4103 4104 ret = cma_get_port(id_priv); 4105 if (ret) 4106 goto err2; 4107 4108 if (!cma_any_addr(addr)) 4109 rdma_restrack_add(&id_priv->res); 4110 return 0; 4111 err2: 4112 if (id_priv->cma_dev) 4113 cma_release_dev(id_priv); 4114 err1: 4115 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE); 4116 return ret; 4117 } 4118 EXPORT_SYMBOL(rdma_bind_addr); 4119 4120 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv) 4121 { 4122 struct cma_hdr *cma_hdr; 4123 4124 cma_hdr = hdr; 4125 cma_hdr->cma_version = CMA_VERSION; 4126 if (cma_family(id_priv) == AF_INET) { 4127 struct sockaddr_in *src4, *dst4; 4128 4129 src4 = (struct sockaddr_in *) cma_src_addr(id_priv); 4130 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv); 4131 4132 cma_set_ip_ver(cma_hdr, 4); 4133 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 4134 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 4135 cma_hdr->port = src4->sin_port; 4136 } else if (cma_family(id_priv) == AF_INET6) { 4137 struct sockaddr_in6 *src6, *dst6; 4138 4139 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); 4140 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv); 4141 4142 cma_set_ip_ver(cma_hdr, 6); 4143 cma_hdr->src_addr.ip6 = src6->sin6_addr; 4144 cma_hdr->dst_addr.ip6 = dst6->sin6_addr; 4145 cma_hdr->port = src6->sin6_port; 4146 } 4147 return 0; 4148 } 4149 4150 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 4151 const struct ib_cm_event *ib_event) 4152 { 4153 struct rdma_id_private *id_priv = cm_id->context; 4154 struct rdma_cm_event event = {}; 4155 const struct ib_cm_sidr_rep_event_param *rep = 4156 &ib_event->param.sidr_rep_rcvd; 4157 int ret; 4158 4159 mutex_lock(&id_priv->handler_mutex); 4160 if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT) 4161 goto out; 4162 4163 switch (ib_event->event) { 4164 case IB_CM_SIDR_REQ_ERROR: 4165 event.event = RDMA_CM_EVENT_UNREACHABLE; 4166 event.status = -ETIMEDOUT; 4167 break; 4168 case IB_CM_SIDR_REP_RECEIVED: 4169 event.param.ud.private_data = ib_event->private_data; 4170 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 4171 if (rep->status != IB_SIDR_SUCCESS) { 4172 event.event = RDMA_CM_EVENT_UNREACHABLE; 4173 event.status = ib_event->param.sidr_rep_rcvd.status; 4174 pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n", 4175 event.status); 4176 break; 4177 } 4178 ret = cma_set_qkey(id_priv, rep->qkey); 4179 if (ret) { 4180 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret); 4181 event.event = RDMA_CM_EVENT_ADDR_ERROR; 4182 event.status = ret; 4183 break; 4184 } 4185 ib_init_ah_attr_from_path(id_priv->id.device, 4186 id_priv->id.port_num, 4187 id_priv->id.route.path_rec, 4188 &event.param.ud.ah_attr, 4189 rep->sgid_attr); 4190 event.param.ud.qp_num = rep->qpn; 4191 event.param.ud.qkey = rep->qkey; 4192 event.event = RDMA_CM_EVENT_ESTABLISHED; 4193 event.status = 0; 4194 break; 4195 default: 4196 pr_err("RDMA CMA: unexpected IB CM event: %d\n", 4197 ib_event->event); 4198 goto out; 4199 } 4200 4201 ret = cma_cm_event_handler(id_priv, &event); 4202 4203 rdma_destroy_ah_attr(&event.param.ud.ah_attr); 4204 if (ret) { 4205 /* Destroy the CM ID by returning a non-zero value. */ 4206 id_priv->cm_id.ib = NULL; 4207 destroy_id_handler_unlock(id_priv); 4208 return ret; 4209 } 4210 out: 4211 mutex_unlock(&id_priv->handler_mutex); 4212 return 0; 4213 } 4214 4215 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 4216 struct rdma_conn_param *conn_param) 4217 { 4218 struct ib_cm_sidr_req_param req; 4219 struct ib_cm_id *id; 4220 void *private_data; 4221 u8 offset; 4222 int ret; 4223 4224 memset(&req, 0, sizeof req); 4225 offset = cma_user_data_offset(id_priv); 4226 if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len)) 4227 return -EINVAL; 4228 4229 if (req.private_data_len) { 4230 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 4231 if (!private_data) 4232 return -ENOMEM; 4233 } else { 4234 private_data = NULL; 4235 } 4236 4237 if (conn_param->private_data && conn_param->private_data_len) 4238 memcpy(private_data + offset, conn_param->private_data, 4239 conn_param->private_data_len); 4240 4241 if (private_data) { 4242 ret = cma_format_hdr(private_data, id_priv); 4243 if (ret) 4244 goto out; 4245 req.private_data = private_data; 4246 } 4247 4248 id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler, 4249 id_priv); 4250 if (IS_ERR(id)) { 4251 ret = PTR_ERR(id); 4252 goto out; 4253 } 4254 id_priv->cm_id.ib = id; 4255 4256 req.path = id_priv->id.route.path_rec; 4257 req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr; 4258 req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 4259 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); 4260 req.max_cm_retries = CMA_MAX_CM_RETRIES; 4261 4262 trace_cm_send_sidr_req(id_priv); 4263 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 4264 if (ret) { 4265 ib_destroy_cm_id(id_priv->cm_id.ib); 4266 id_priv->cm_id.ib = NULL; 4267 } 4268 out: 4269 kfree(private_data); 4270 return ret; 4271 } 4272 4273 static int cma_connect_ib(struct rdma_id_private *id_priv, 4274 struct rdma_conn_param *conn_param) 4275 { 4276 struct ib_cm_req_param req; 4277 struct rdma_route *route; 4278 void *private_data; 4279 struct ib_cm_id *id; 4280 u8 offset; 4281 int ret; 4282 4283 memset(&req, 0, sizeof req); 4284 offset = cma_user_data_offset(id_priv); 4285 if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len)) 4286 return -EINVAL; 4287 4288 if (req.private_data_len) { 4289 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 4290 if (!private_data) 4291 return -ENOMEM; 4292 } else { 4293 private_data = NULL; 4294 } 4295 4296 if (conn_param->private_data && conn_param->private_data_len) 4297 memcpy(private_data + offset, conn_param->private_data, 4298 conn_param->private_data_len); 4299 4300 id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv); 4301 if (IS_ERR(id)) { 4302 ret = PTR_ERR(id); 4303 goto out; 4304 } 4305 id_priv->cm_id.ib = id; 4306 4307 route = &id_priv->id.route; 4308 if (private_data) { 4309 ret = cma_format_hdr(private_data, id_priv); 4310 if (ret) 4311 goto out; 4312 req.private_data = private_data; 4313 } 4314 4315 req.primary_path = &route->path_rec[0]; 4316 if (route->num_pri_alt_paths == 2) 4317 req.alternate_path = &route->path_rec[1]; 4318 4319 req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr; 4320 /* Alternate path SGID attribute currently unsupported */ 4321 req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 4322 req.qp_num = id_priv->qp_num; 4323 req.qp_type = id_priv->id.qp_type; 4324 req.starting_psn = id_priv->seq_num; 4325 req.responder_resources = conn_param->responder_resources; 4326 req.initiator_depth = conn_param->initiator_depth; 4327 req.flow_control = conn_param->flow_control; 4328 req.retry_count = min_t(u8, 7, conn_param->retry_count); 4329 req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); 4330 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 4331 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 4332 req.max_cm_retries = CMA_MAX_CM_RETRIES; 4333 req.srq = id_priv->srq ? 1 : 0; 4334 req.ece.vendor_id = id_priv->ece.vendor_id; 4335 req.ece.attr_mod = id_priv->ece.attr_mod; 4336 4337 trace_cm_send_req(id_priv); 4338 ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 4339 out: 4340 if (ret && !IS_ERR(id)) { 4341 ib_destroy_cm_id(id); 4342 id_priv->cm_id.ib = NULL; 4343 } 4344 4345 kfree(private_data); 4346 return ret; 4347 } 4348 4349 static int cma_connect_iw(struct rdma_id_private *id_priv, 4350 struct rdma_conn_param *conn_param) 4351 { 4352 struct iw_cm_id *cm_id; 4353 int ret; 4354 struct iw_cm_conn_param iw_param; 4355 4356 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); 4357 if (IS_ERR(cm_id)) 4358 return PTR_ERR(cm_id); 4359 4360 mutex_lock(&id_priv->qp_mutex); 4361 cm_id->tos = id_priv->tos; 4362 cm_id->tos_set = id_priv->tos_set; 4363 mutex_unlock(&id_priv->qp_mutex); 4364 4365 id_priv->cm_id.iw = cm_id; 4366 4367 memcpy(&cm_id->local_addr, cma_src_addr(id_priv), 4368 rdma_addr_size(cma_src_addr(id_priv))); 4369 memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv), 4370 rdma_addr_size(cma_dst_addr(id_priv))); 4371 4372 ret = cma_modify_qp_rtr(id_priv, conn_param); 4373 if (ret) 4374 goto out; 4375 4376 if (conn_param) { 4377 iw_param.ord = conn_param->initiator_depth; 4378 iw_param.ird = conn_param->responder_resources; 4379 iw_param.private_data = conn_param->private_data; 4380 iw_param.private_data_len = conn_param->private_data_len; 4381 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num; 4382 } else { 4383 memset(&iw_param, 0, sizeof iw_param); 4384 iw_param.qpn = id_priv->qp_num; 4385 } 4386 ret = iw_cm_connect(cm_id, &iw_param); 4387 out: 4388 if (ret) { 4389 iw_destroy_cm_id(cm_id); 4390 id_priv->cm_id.iw = NULL; 4391 } 4392 return ret; 4393 } 4394 4395 /** 4396 * rdma_connect_locked - Initiate an active connection request. 4397 * @id: Connection identifier to connect. 4398 * @conn_param: Connection information used for connected QPs. 4399 * 4400 * Same as rdma_connect() but can only be called from the 4401 * RDMA_CM_EVENT_ROUTE_RESOLVED handler callback. 4402 */ 4403 int rdma_connect_locked(struct rdma_cm_id *id, 4404 struct rdma_conn_param *conn_param) 4405 { 4406 struct rdma_id_private *id_priv = 4407 container_of(id, struct rdma_id_private, id); 4408 int ret; 4409 4410 if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT)) 4411 return -EINVAL; 4412 4413 if (!id->qp) { 4414 id_priv->qp_num = conn_param->qp_num; 4415 id_priv->srq = conn_param->srq; 4416 } 4417 4418 if (rdma_cap_ib_cm(id->device, id->port_num)) { 4419 if (id->qp_type == IB_QPT_UD) 4420 ret = cma_resolve_ib_udp(id_priv, conn_param); 4421 else 4422 ret = cma_connect_ib(id_priv, conn_param); 4423 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 4424 ret = cma_connect_iw(id_priv, conn_param); 4425 } else { 4426 ret = -ENOSYS; 4427 } 4428 if (ret) 4429 goto err_state; 4430 return 0; 4431 err_state: 4432 cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED); 4433 return ret; 4434 } 4435 EXPORT_SYMBOL(rdma_connect_locked); 4436 4437 /** 4438 * rdma_connect - Initiate an active connection request. 4439 * @id: Connection identifier to connect. 4440 * @conn_param: Connection information used for connected QPs. 4441 * 4442 * Users must have resolved a route for the rdma_cm_id to connect with by having 4443 * called rdma_resolve_route before calling this routine. 4444 * 4445 * This call will either connect to a remote QP or obtain remote QP information 4446 * for unconnected rdma_cm_id's. The actual operation is based on the 4447 * rdma_cm_id's port space. 4448 */ 4449 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 4450 { 4451 struct rdma_id_private *id_priv = 4452 container_of(id, struct rdma_id_private, id); 4453 int ret; 4454 4455 mutex_lock(&id_priv->handler_mutex); 4456 ret = rdma_connect_locked(id, conn_param); 4457 mutex_unlock(&id_priv->handler_mutex); 4458 return ret; 4459 } 4460 EXPORT_SYMBOL(rdma_connect); 4461 4462 /** 4463 * rdma_connect_ece - Initiate an active connection request with ECE data. 4464 * @id: Connection identifier to connect. 4465 * @conn_param: Connection information used for connected QPs. 4466 * @ece: ECE parameters 4467 * 4468 * See rdma_connect() explanation. 4469 */ 4470 int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, 4471 struct rdma_ucm_ece *ece) 4472 { 4473 struct rdma_id_private *id_priv = 4474 container_of(id, struct rdma_id_private, id); 4475 4476 id_priv->ece.vendor_id = ece->vendor_id; 4477 id_priv->ece.attr_mod = ece->attr_mod; 4478 4479 return rdma_connect(id, conn_param); 4480 } 4481 EXPORT_SYMBOL(rdma_connect_ece); 4482 4483 static int cma_accept_ib(struct rdma_id_private *id_priv, 4484 struct rdma_conn_param *conn_param) 4485 { 4486 struct ib_cm_rep_param rep; 4487 int ret; 4488 4489 ret = cma_modify_qp_rtr(id_priv, conn_param); 4490 if (ret) 4491 goto out; 4492 4493 ret = cma_modify_qp_rts(id_priv, conn_param); 4494 if (ret) 4495 goto out; 4496 4497 memset(&rep, 0, sizeof rep); 4498 rep.qp_num = id_priv->qp_num; 4499 rep.starting_psn = id_priv->seq_num; 4500 rep.private_data = conn_param->private_data; 4501 rep.private_data_len = conn_param->private_data_len; 4502 rep.responder_resources = conn_param->responder_resources; 4503 rep.initiator_depth = conn_param->initiator_depth; 4504 rep.failover_accepted = 0; 4505 rep.flow_control = conn_param->flow_control; 4506 rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); 4507 rep.srq = id_priv->srq ? 1 : 0; 4508 rep.ece.vendor_id = id_priv->ece.vendor_id; 4509 rep.ece.attr_mod = id_priv->ece.attr_mod; 4510 4511 trace_cm_send_rep(id_priv); 4512 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 4513 out: 4514 return ret; 4515 } 4516 4517 static int cma_accept_iw(struct rdma_id_private *id_priv, 4518 struct rdma_conn_param *conn_param) 4519 { 4520 struct iw_cm_conn_param iw_param; 4521 int ret; 4522 4523 if (!conn_param) 4524 return -EINVAL; 4525 4526 ret = cma_modify_qp_rtr(id_priv, conn_param); 4527 if (ret) 4528 return ret; 4529 4530 iw_param.ord = conn_param->initiator_depth; 4531 iw_param.ird = conn_param->responder_resources; 4532 iw_param.private_data = conn_param->private_data; 4533 iw_param.private_data_len = conn_param->private_data_len; 4534 if (id_priv->id.qp) 4535 iw_param.qpn = id_priv->qp_num; 4536 else 4537 iw_param.qpn = conn_param->qp_num; 4538 4539 return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 4540 } 4541 4542 static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 4543 enum ib_cm_sidr_status status, u32 qkey, 4544 const void *private_data, int private_data_len) 4545 { 4546 struct ib_cm_sidr_rep_param rep; 4547 int ret; 4548 4549 memset(&rep, 0, sizeof rep); 4550 rep.status = status; 4551 if (status == IB_SIDR_SUCCESS) { 4552 ret = cma_set_qkey(id_priv, qkey); 4553 if (ret) 4554 return ret; 4555 rep.qp_num = id_priv->qp_num; 4556 rep.qkey = id_priv->qkey; 4557 4558 rep.ece.vendor_id = id_priv->ece.vendor_id; 4559 rep.ece.attr_mod = id_priv->ece.attr_mod; 4560 } 4561 4562 rep.private_data = private_data; 4563 rep.private_data_len = private_data_len; 4564 4565 trace_cm_send_sidr_rep(id_priv); 4566 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 4567 } 4568 4569 /** 4570 * rdma_accept - Called to accept a connection request or response. 4571 * @id: Connection identifier associated with the request. 4572 * @conn_param: Information needed to establish the connection. This must be 4573 * provided if accepting a connection request. If accepting a connection 4574 * response, this parameter must be NULL. 4575 * 4576 * Typically, this routine is only called by the listener to accept a connection 4577 * request. It must also be called on the active side of a connection if the 4578 * user is performing their own QP transitions. 4579 * 4580 * In the case of error, a reject message is sent to the remote side and the 4581 * state of the qp associated with the id is modified to error, such that any 4582 * previously posted receive buffers would be flushed. 4583 * 4584 * This function is for use by kernel ULPs and must be called from under the 4585 * handler callback. 4586 */ 4587 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 4588 { 4589 struct rdma_id_private *id_priv = 4590 container_of(id, struct rdma_id_private, id); 4591 int ret; 4592 4593 lockdep_assert_held(&id_priv->handler_mutex); 4594 4595 if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT) 4596 return -EINVAL; 4597 4598 if (!id->qp && conn_param) { 4599 id_priv->qp_num = conn_param->qp_num; 4600 id_priv->srq = conn_param->srq; 4601 } 4602 4603 if (rdma_cap_ib_cm(id->device, id->port_num)) { 4604 if (id->qp_type == IB_QPT_UD) { 4605 if (conn_param) 4606 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 4607 conn_param->qkey, 4608 conn_param->private_data, 4609 conn_param->private_data_len); 4610 else 4611 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 4612 0, NULL, 0); 4613 } else { 4614 if (conn_param) 4615 ret = cma_accept_ib(id_priv, conn_param); 4616 else 4617 ret = cma_rep_recv(id_priv); 4618 } 4619 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 4620 ret = cma_accept_iw(id_priv, conn_param); 4621 } else { 4622 ret = -ENOSYS; 4623 } 4624 if (ret) 4625 goto reject; 4626 4627 return 0; 4628 reject: 4629 cma_modify_qp_err(id_priv); 4630 rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 4631 return ret; 4632 } 4633 EXPORT_SYMBOL(rdma_accept); 4634 4635 int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, 4636 struct rdma_ucm_ece *ece) 4637 { 4638 struct rdma_id_private *id_priv = 4639 container_of(id, struct rdma_id_private, id); 4640 4641 id_priv->ece.vendor_id = ece->vendor_id; 4642 id_priv->ece.attr_mod = ece->attr_mod; 4643 4644 return rdma_accept(id, conn_param); 4645 } 4646 EXPORT_SYMBOL(rdma_accept_ece); 4647 4648 void rdma_lock_handler(struct rdma_cm_id *id) 4649 { 4650 struct rdma_id_private *id_priv = 4651 container_of(id, struct rdma_id_private, id); 4652 4653 mutex_lock(&id_priv->handler_mutex); 4654 } 4655 EXPORT_SYMBOL(rdma_lock_handler); 4656 4657 void rdma_unlock_handler(struct rdma_cm_id *id) 4658 { 4659 struct rdma_id_private *id_priv = 4660 container_of(id, struct rdma_id_private, id); 4661 4662 mutex_unlock(&id_priv->handler_mutex); 4663 } 4664 EXPORT_SYMBOL(rdma_unlock_handler); 4665 4666 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 4667 { 4668 struct rdma_id_private *id_priv; 4669 int ret; 4670 4671 id_priv = container_of(id, struct rdma_id_private, id); 4672 if (!id_priv->cm_id.ib) 4673 return -EINVAL; 4674 4675 switch (id->device->node_type) { 4676 case RDMA_NODE_IB_CA: 4677 ret = ib_cm_notify(id_priv->cm_id.ib, event); 4678 break; 4679 default: 4680 ret = 0; 4681 break; 4682 } 4683 return ret; 4684 } 4685 EXPORT_SYMBOL(rdma_notify); 4686 4687 int rdma_reject(struct rdma_cm_id *id, const void *private_data, 4688 u8 private_data_len, u8 reason) 4689 { 4690 struct rdma_id_private *id_priv; 4691 int ret; 4692 4693 id_priv = container_of(id, struct rdma_id_private, id); 4694 if (!id_priv->cm_id.ib) 4695 return -EINVAL; 4696 4697 if (rdma_cap_ib_cm(id->device, id->port_num)) { 4698 if (id->qp_type == IB_QPT_UD) { 4699 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0, 4700 private_data, private_data_len); 4701 } else { 4702 trace_cm_send_rej(id_priv); 4703 ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0, 4704 private_data, private_data_len); 4705 } 4706 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 4707 ret = iw_cm_reject(id_priv->cm_id.iw, 4708 private_data, private_data_len); 4709 } else { 4710 ret = -ENOSYS; 4711 } 4712 4713 return ret; 4714 } 4715 EXPORT_SYMBOL(rdma_reject); 4716 4717 int rdma_disconnect(struct rdma_cm_id *id) 4718 { 4719 struct rdma_id_private *id_priv; 4720 int ret; 4721 4722 id_priv = container_of(id, struct rdma_id_private, id); 4723 if (!id_priv->cm_id.ib) 4724 return -EINVAL; 4725 4726 if (rdma_cap_ib_cm(id->device, id->port_num)) { 4727 ret = cma_modify_qp_err(id_priv); 4728 if (ret) 4729 goto out; 4730 /* Initiate or respond to a disconnect. */ 4731 trace_cm_disconnect(id_priv); 4732 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { 4733 if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) 4734 trace_cm_sent_drep(id_priv); 4735 } else { 4736 trace_cm_sent_dreq(id_priv); 4737 } 4738 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 4739 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 4740 } else 4741 ret = -EINVAL; 4742 4743 out: 4744 return ret; 4745 } 4746 EXPORT_SYMBOL(rdma_disconnect); 4747 4748 static void cma_make_mc_event(int status, struct rdma_id_private *id_priv, 4749 struct ib_sa_multicast *multicast, 4750 struct rdma_cm_event *event, 4751 struct cma_multicast *mc) 4752 { 4753 struct rdma_dev_addr *dev_addr; 4754 enum ib_gid_type gid_type; 4755 struct net_device *ndev; 4756 4757 if (!status) 4758 status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey)); 4759 else 4760 pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n", 4761 status); 4762 4763 event->status = status; 4764 event->param.ud.private_data = mc->context; 4765 if (status) { 4766 event->event = RDMA_CM_EVENT_MULTICAST_ERROR; 4767 return; 4768 } 4769 4770 dev_addr = &id_priv->id.route.addr.dev_addr; 4771 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); 4772 gid_type = 4773 id_priv->cma_dev 4774 ->default_gid_type[id_priv->id.port_num - 4775 rdma_start_port( 4776 id_priv->cma_dev->device)]; 4777 4778 event->event = RDMA_CM_EVENT_MULTICAST_JOIN; 4779 if (ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num, 4780 &multicast->rec, ndev, gid_type, 4781 &event->param.ud.ah_attr)) { 4782 event->event = RDMA_CM_EVENT_MULTICAST_ERROR; 4783 goto out; 4784 } 4785 4786 event->param.ud.qp_num = 0xFFFFFF; 4787 event->param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 4788 4789 out: 4790 if (ndev) 4791 dev_put(ndev); 4792 } 4793 4794 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 4795 { 4796 struct cma_multicast *mc = multicast->context; 4797 struct rdma_id_private *id_priv = mc->id_priv; 4798 struct rdma_cm_event event = {}; 4799 int ret = 0; 4800 4801 mutex_lock(&id_priv->handler_mutex); 4802 if (READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL || 4803 READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING) 4804 goto out; 4805 4806 cma_make_mc_event(status, id_priv, multicast, &event, mc); 4807 ret = cma_cm_event_handler(id_priv, &event); 4808 rdma_destroy_ah_attr(&event.param.ud.ah_attr); 4809 WARN_ON(ret); 4810 4811 out: 4812 mutex_unlock(&id_priv->handler_mutex); 4813 return 0; 4814 } 4815 4816 static void cma_set_mgid(struct rdma_id_private *id_priv, 4817 struct sockaddr *addr, union ib_gid *mgid) 4818 { 4819 unsigned char mc_map[MAX_ADDR_LEN]; 4820 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 4821 struct sockaddr_in *sin = (struct sockaddr_in *) addr; 4822 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 4823 4824 if (cma_any_addr(addr)) { 4825 memset(mgid, 0, sizeof *mgid); 4826 } else if ((addr->sa_family == AF_INET6) && 4827 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 4828 0xFF10A01B)) { 4829 /* IPv6 address is an SA assigned MGID. */ 4830 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 4831 } else if (addr->sa_family == AF_IB) { 4832 memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid); 4833 } else if (addr->sa_family == AF_INET6) { 4834 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); 4835 if (id_priv->id.ps == RDMA_PS_UDP) 4836 mc_map[7] = 0x01; /* Use RDMA CM signature */ 4837 *mgid = *(union ib_gid *) (mc_map + 4); 4838 } else { 4839 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); 4840 if (id_priv->id.ps == RDMA_PS_UDP) 4841 mc_map[7] = 0x01; /* Use RDMA CM signature */ 4842 *mgid = *(union ib_gid *) (mc_map + 4); 4843 } 4844 } 4845 4846 static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 4847 struct cma_multicast *mc) 4848 { 4849 struct ib_sa_mcmember_rec rec; 4850 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 4851 ib_sa_comp_mask comp_mask; 4852 int ret; 4853 4854 ib_addr_get_mgid(dev_addr, &rec.mgid); 4855 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 4856 &rec.mgid, &rec); 4857 if (ret) 4858 return ret; 4859 4860 ret = cma_set_qkey(id_priv, 0); 4861 if (ret) 4862 return ret; 4863 4864 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); 4865 rec.qkey = cpu_to_be32(id_priv->qkey); 4866 rdma_addr_get_sgid(dev_addr, &rec.port_gid); 4867 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 4868 rec.join_state = mc->join_state; 4869 4870 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 4871 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 4872 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 4873 IB_SA_MCMEMBER_REC_FLOW_LABEL | 4874 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 4875 4876 if (id_priv->id.ps == RDMA_PS_IPOIB) 4877 comp_mask |= IB_SA_MCMEMBER_REC_RATE | 4878 IB_SA_MCMEMBER_REC_RATE_SELECTOR | 4879 IB_SA_MCMEMBER_REC_MTU_SELECTOR | 4880 IB_SA_MCMEMBER_REC_MTU | 4881 IB_SA_MCMEMBER_REC_HOP_LIMIT; 4882 4883 mc->sa_mc = ib_sa_join_multicast(&sa_client, id_priv->id.device, 4884 id_priv->id.port_num, &rec, comp_mask, 4885 GFP_KERNEL, cma_ib_mc_handler, mc); 4886 return PTR_ERR_OR_ZERO(mc->sa_mc); 4887 } 4888 4889 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, 4890 enum ib_gid_type gid_type) 4891 { 4892 struct sockaddr_in *sin = (struct sockaddr_in *)addr; 4893 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; 4894 4895 if (cma_any_addr(addr)) { 4896 memset(mgid, 0, sizeof *mgid); 4897 } else if (addr->sa_family == AF_INET6) { 4898 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 4899 } else { 4900 mgid->raw[0] = 4901 (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff; 4902 mgid->raw[1] = 4903 (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e; 4904 mgid->raw[2] = 0; 4905 mgid->raw[3] = 0; 4906 mgid->raw[4] = 0; 4907 mgid->raw[5] = 0; 4908 mgid->raw[6] = 0; 4909 mgid->raw[7] = 0; 4910 mgid->raw[8] = 0; 4911 mgid->raw[9] = 0; 4912 mgid->raw[10] = 0xff; 4913 mgid->raw[11] = 0xff; 4914 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; 4915 } 4916 } 4917 4918 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, 4919 struct cma_multicast *mc) 4920 { 4921 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 4922 int err = 0; 4923 struct sockaddr *addr = (struct sockaddr *)&mc->addr; 4924 struct net_device *ndev = NULL; 4925 struct ib_sa_multicast ib; 4926 enum ib_gid_type gid_type; 4927 bool send_only; 4928 4929 send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN); 4930 4931 if (cma_zero_addr(addr)) 4932 return -EINVAL; 4933 4934 gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num - 4935 rdma_start_port(id_priv->cma_dev->device)]; 4936 cma_iboe_set_mgid(addr, &ib.rec.mgid, gid_type); 4937 4938 ib.rec.pkey = cpu_to_be16(0xffff); 4939 if (id_priv->id.ps == RDMA_PS_UDP) 4940 ib.rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 4941 4942 if (dev_addr->bound_dev_if) 4943 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); 4944 if (!ndev) 4945 return -ENODEV; 4946 4947 ib.rec.rate = iboe_get_rate(ndev); 4948 ib.rec.hop_limit = 1; 4949 ib.rec.mtu = iboe_get_mtu(ndev->mtu); 4950 4951 if (addr->sa_family == AF_INET) { 4952 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 4953 ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; 4954 if (!send_only) { 4955 err = cma_igmp_send(ndev, &ib.rec.mgid, 4956 true); 4957 } 4958 } 4959 } else { 4960 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) 4961 err = -ENOTSUPP; 4962 } 4963 dev_put(ndev); 4964 if (err || !ib.rec.mtu) 4965 return err ?: -EINVAL; 4966 4967 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 4968 &ib.rec.port_gid); 4969 INIT_WORK(&mc->iboe_join.work, cma_iboe_join_work_handler); 4970 cma_make_mc_event(0, id_priv, &ib, &mc->iboe_join.event, mc); 4971 queue_work(cma_wq, &mc->iboe_join.work); 4972 return 0; 4973 } 4974 4975 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 4976 u8 join_state, void *context) 4977 { 4978 struct rdma_id_private *id_priv = 4979 container_of(id, struct rdma_id_private, id); 4980 struct cma_multicast *mc; 4981 int ret; 4982 4983 /* Not supported for kernel QPs */ 4984 if (WARN_ON(id->qp)) 4985 return -EINVAL; 4986 4987 /* ULP is calling this wrong. */ 4988 if (!id->device || (READ_ONCE(id_priv->state) != RDMA_CM_ADDR_BOUND && 4989 READ_ONCE(id_priv->state) != RDMA_CM_ADDR_RESOLVED)) 4990 return -EINVAL; 4991 4992 mc = kzalloc(sizeof(*mc), GFP_KERNEL); 4993 if (!mc) 4994 return -ENOMEM; 4995 4996 memcpy(&mc->addr, addr, rdma_addr_size(addr)); 4997 mc->context = context; 4998 mc->id_priv = id_priv; 4999 mc->join_state = join_state; 5000 5001 if (rdma_protocol_roce(id->device, id->port_num)) { 5002 ret = cma_iboe_join_multicast(id_priv, mc); 5003 if (ret) 5004 goto out_err; 5005 } else if (rdma_cap_ib_mcast(id->device, id->port_num)) { 5006 ret = cma_join_ib_multicast(id_priv, mc); 5007 if (ret) 5008 goto out_err; 5009 } else { 5010 ret = -ENOSYS; 5011 goto out_err; 5012 } 5013 5014 spin_lock(&id_priv->lock); 5015 list_add(&mc->list, &id_priv->mc_list); 5016 spin_unlock(&id_priv->lock); 5017 5018 return 0; 5019 out_err: 5020 kfree(mc); 5021 return ret; 5022 } 5023 EXPORT_SYMBOL(rdma_join_multicast); 5024 5025 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 5026 { 5027 struct rdma_id_private *id_priv; 5028 struct cma_multicast *mc; 5029 5030 id_priv = container_of(id, struct rdma_id_private, id); 5031 spin_lock_irq(&id_priv->lock); 5032 list_for_each_entry(mc, &id_priv->mc_list, list) { 5033 if (memcmp(&mc->addr, addr, rdma_addr_size(addr)) != 0) 5034 continue; 5035 list_del(&mc->list); 5036 spin_unlock_irq(&id_priv->lock); 5037 5038 WARN_ON(id_priv->cma_dev->device != id->device); 5039 destroy_mc(id_priv, mc); 5040 return; 5041 } 5042 spin_unlock_irq(&id_priv->lock); 5043 } 5044 EXPORT_SYMBOL(rdma_leave_multicast); 5045 5046 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) 5047 { 5048 struct rdma_dev_addr *dev_addr; 5049 struct cma_work *work; 5050 5051 dev_addr = &id_priv->id.route.addr.dev_addr; 5052 5053 if ((dev_addr->bound_dev_if == ndev->ifindex) && 5054 (net_eq(dev_net(ndev), dev_addr->net)) && 5055 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { 5056 pr_info("RDMA CM addr change for ndev %s used by id %p\n", 5057 ndev->name, &id_priv->id); 5058 work = kzalloc(sizeof *work, GFP_KERNEL); 5059 if (!work) 5060 return -ENOMEM; 5061 5062 INIT_WORK(&work->work, cma_work_handler); 5063 work->id = id_priv; 5064 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; 5065 cma_id_get(id_priv); 5066 queue_work(cma_wq, &work->work); 5067 } 5068 5069 return 0; 5070 } 5071 5072 static int cma_netdev_callback(struct notifier_block *self, unsigned long event, 5073 void *ptr) 5074 { 5075 struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 5076 struct cma_device *cma_dev; 5077 struct rdma_id_private *id_priv; 5078 int ret = NOTIFY_DONE; 5079 5080 if (event != NETDEV_BONDING_FAILOVER) 5081 return NOTIFY_DONE; 5082 5083 if (!netif_is_bond_master(ndev)) 5084 return NOTIFY_DONE; 5085 5086 mutex_lock(&lock); 5087 list_for_each_entry(cma_dev, &dev_list, list) 5088 list_for_each_entry(id_priv, &cma_dev->id_list, device_item) { 5089 ret = cma_netdev_change(ndev, id_priv); 5090 if (ret) 5091 goto out; 5092 } 5093 5094 out: 5095 mutex_unlock(&lock); 5096 return ret; 5097 } 5098 5099 static void cma_netevent_work_handler(struct work_struct *_work) 5100 { 5101 struct rdma_id_private *id_priv = 5102 container_of(_work, struct rdma_id_private, id.net_work); 5103 struct rdma_cm_event event = {}; 5104 5105 mutex_lock(&id_priv->handler_mutex); 5106 5107 if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING || 5108 READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL) 5109 goto out_unlock; 5110 5111 event.event = RDMA_CM_EVENT_UNREACHABLE; 5112 event.status = -ETIMEDOUT; 5113 5114 if (cma_cm_event_handler(id_priv, &event)) { 5115 __acquire(&id_priv->handler_mutex); 5116 id_priv->cm_id.ib = NULL; 5117 cma_id_put(id_priv); 5118 destroy_id_handler_unlock(id_priv); 5119 return; 5120 } 5121 5122 out_unlock: 5123 mutex_unlock(&id_priv->handler_mutex); 5124 cma_id_put(id_priv); 5125 } 5126 5127 static int cma_netevent_callback(struct notifier_block *self, 5128 unsigned long event, void *ctx) 5129 { 5130 struct id_table_entry *ips_node = NULL; 5131 struct rdma_id_private *current_id; 5132 struct neighbour *neigh = ctx; 5133 unsigned long flags; 5134 5135 if (event != NETEVENT_NEIGH_UPDATE) 5136 return NOTIFY_DONE; 5137 5138 spin_lock_irqsave(&id_table_lock, flags); 5139 if (neigh->tbl->family == AF_INET6) { 5140 struct sockaddr_in6 neigh_sock_6; 5141 5142 neigh_sock_6.sin6_family = AF_INET6; 5143 neigh_sock_6.sin6_addr = *(struct in6_addr *)neigh->primary_key; 5144 ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex, 5145 (struct sockaddr *)&neigh_sock_6); 5146 } else if (neigh->tbl->family == AF_INET) { 5147 struct sockaddr_in neigh_sock_4; 5148 5149 neigh_sock_4.sin_family = AF_INET; 5150 neigh_sock_4.sin_addr.s_addr = *(__be32 *)(neigh->primary_key); 5151 ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex, 5152 (struct sockaddr *)&neigh_sock_4); 5153 } else 5154 goto out; 5155 5156 if (!ips_node) 5157 goto out; 5158 5159 list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) { 5160 if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr, 5161 neigh->ha, ETH_ALEN)) 5162 continue; 5163 INIT_WORK(¤t_id->id.net_work, cma_netevent_work_handler); 5164 cma_id_get(current_id); 5165 queue_work(cma_wq, ¤t_id->id.net_work); 5166 } 5167 out: 5168 spin_unlock_irqrestore(&id_table_lock, flags); 5169 return NOTIFY_DONE; 5170 } 5171 5172 static struct notifier_block cma_nb = { 5173 .notifier_call = cma_netdev_callback 5174 }; 5175 5176 static struct notifier_block cma_netevent_cb = { 5177 .notifier_call = cma_netevent_callback 5178 }; 5179 5180 static void cma_send_device_removal_put(struct rdma_id_private *id_priv) 5181 { 5182 struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL }; 5183 enum rdma_cm_state state; 5184 unsigned long flags; 5185 5186 mutex_lock(&id_priv->handler_mutex); 5187 /* Record that we want to remove the device */ 5188 spin_lock_irqsave(&id_priv->lock, flags); 5189 state = id_priv->state; 5190 if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) { 5191 spin_unlock_irqrestore(&id_priv->lock, flags); 5192 mutex_unlock(&id_priv->handler_mutex); 5193 cma_id_put(id_priv); 5194 return; 5195 } 5196 id_priv->state = RDMA_CM_DEVICE_REMOVAL; 5197 spin_unlock_irqrestore(&id_priv->lock, flags); 5198 5199 if (cma_cm_event_handler(id_priv, &event)) { 5200 /* 5201 * At this point the ULP promises it won't call 5202 * rdma_destroy_id() concurrently 5203 */ 5204 cma_id_put(id_priv); 5205 mutex_unlock(&id_priv->handler_mutex); 5206 trace_cm_id_destroy(id_priv); 5207 _destroy_id(id_priv, state); 5208 return; 5209 } 5210 mutex_unlock(&id_priv->handler_mutex); 5211 5212 /* 5213 * If this races with destroy then the thread that first assigns state 5214 * to a destroying does the cancel. 5215 */ 5216 cma_cancel_operation(id_priv, state); 5217 cma_id_put(id_priv); 5218 } 5219 5220 static void cma_process_remove(struct cma_device *cma_dev) 5221 { 5222 mutex_lock(&lock); 5223 while (!list_empty(&cma_dev->id_list)) { 5224 struct rdma_id_private *id_priv = list_first_entry( 5225 &cma_dev->id_list, struct rdma_id_private, device_item); 5226 5227 list_del_init(&id_priv->listen_item); 5228 list_del_init(&id_priv->device_item); 5229 cma_id_get(id_priv); 5230 mutex_unlock(&lock); 5231 5232 cma_send_device_removal_put(id_priv); 5233 5234 mutex_lock(&lock); 5235 } 5236 mutex_unlock(&lock); 5237 5238 cma_dev_put(cma_dev); 5239 wait_for_completion(&cma_dev->comp); 5240 } 5241 5242 static bool cma_supported(struct ib_device *device) 5243 { 5244 u32 i; 5245 5246 rdma_for_each_port(device, i) { 5247 if (rdma_cap_ib_cm(device, i) || rdma_cap_iw_cm(device, i)) 5248 return true; 5249 } 5250 return false; 5251 } 5252 5253 static int cma_add_one(struct ib_device *device) 5254 { 5255 struct rdma_id_private *to_destroy; 5256 struct cma_device *cma_dev; 5257 struct rdma_id_private *id_priv; 5258 unsigned long supported_gids = 0; 5259 int ret; 5260 u32 i; 5261 5262 if (!cma_supported(device)) 5263 return -EOPNOTSUPP; 5264 5265 cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL); 5266 if (!cma_dev) 5267 return -ENOMEM; 5268 5269 cma_dev->device = device; 5270 cma_dev->default_gid_type = kcalloc(device->phys_port_cnt, 5271 sizeof(*cma_dev->default_gid_type), 5272 GFP_KERNEL); 5273 if (!cma_dev->default_gid_type) { 5274 ret = -ENOMEM; 5275 goto free_cma_dev; 5276 } 5277 5278 cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt, 5279 sizeof(*cma_dev->default_roce_tos), 5280 GFP_KERNEL); 5281 if (!cma_dev->default_roce_tos) { 5282 ret = -ENOMEM; 5283 goto free_gid_type; 5284 } 5285 5286 rdma_for_each_port (device, i) { 5287 supported_gids = roce_gid_type_mask_support(device, i); 5288 WARN_ON(!supported_gids); 5289 if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE)) 5290 cma_dev->default_gid_type[i - rdma_start_port(device)] = 5291 CMA_PREFERRED_ROCE_GID_TYPE; 5292 else 5293 cma_dev->default_gid_type[i - rdma_start_port(device)] = 5294 find_first_bit(&supported_gids, BITS_PER_LONG); 5295 cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0; 5296 } 5297 5298 init_completion(&cma_dev->comp); 5299 refcount_set(&cma_dev->refcount, 1); 5300 INIT_LIST_HEAD(&cma_dev->id_list); 5301 ib_set_client_data(device, &cma_client, cma_dev); 5302 5303 mutex_lock(&lock); 5304 list_add_tail(&cma_dev->list, &dev_list); 5305 list_for_each_entry(id_priv, &listen_any_list, listen_any_item) { 5306 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy); 5307 if (ret) 5308 goto free_listen; 5309 } 5310 mutex_unlock(&lock); 5311 5312 trace_cm_add_one(device); 5313 return 0; 5314 5315 free_listen: 5316 list_del(&cma_dev->list); 5317 mutex_unlock(&lock); 5318 5319 /* cma_process_remove() will delete to_destroy */ 5320 cma_process_remove(cma_dev); 5321 kfree(cma_dev->default_roce_tos); 5322 free_gid_type: 5323 kfree(cma_dev->default_gid_type); 5324 5325 free_cma_dev: 5326 kfree(cma_dev); 5327 return ret; 5328 } 5329 5330 static void cma_remove_one(struct ib_device *device, void *client_data) 5331 { 5332 struct cma_device *cma_dev = client_data; 5333 5334 trace_cm_remove_one(device); 5335 5336 mutex_lock(&lock); 5337 list_del(&cma_dev->list); 5338 mutex_unlock(&lock); 5339 5340 cma_process_remove(cma_dev); 5341 kfree(cma_dev->default_roce_tos); 5342 kfree(cma_dev->default_gid_type); 5343 kfree(cma_dev); 5344 } 5345 5346 static int cma_init_net(struct net *net) 5347 { 5348 struct cma_pernet *pernet = cma_pernet(net); 5349 5350 xa_init(&pernet->tcp_ps); 5351 xa_init(&pernet->udp_ps); 5352 xa_init(&pernet->ipoib_ps); 5353 xa_init(&pernet->ib_ps); 5354 5355 return 0; 5356 } 5357 5358 static void cma_exit_net(struct net *net) 5359 { 5360 struct cma_pernet *pernet = cma_pernet(net); 5361 5362 WARN_ON(!xa_empty(&pernet->tcp_ps)); 5363 WARN_ON(!xa_empty(&pernet->udp_ps)); 5364 WARN_ON(!xa_empty(&pernet->ipoib_ps)); 5365 WARN_ON(!xa_empty(&pernet->ib_ps)); 5366 } 5367 5368 static struct pernet_operations cma_pernet_operations = { 5369 .init = cma_init_net, 5370 .exit = cma_exit_net, 5371 .id = &cma_pernet_id, 5372 .size = sizeof(struct cma_pernet), 5373 }; 5374 5375 static int __init cma_init(void) 5376 { 5377 int ret; 5378 5379 /* 5380 * There is a rare lock ordering dependency in cma_netdev_callback() 5381 * that only happens when bonding is enabled. Teach lockdep that rtnl 5382 * must never be nested under lock so it can find these without having 5383 * to test with bonding. 5384 */ 5385 if (IS_ENABLED(CONFIG_LOCKDEP)) { 5386 rtnl_lock(); 5387 mutex_lock(&lock); 5388 mutex_unlock(&lock); 5389 rtnl_unlock(); 5390 } 5391 5392 cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM); 5393 if (!cma_wq) 5394 return -ENOMEM; 5395 5396 ret = register_pernet_subsys(&cma_pernet_operations); 5397 if (ret) 5398 goto err_wq; 5399 5400 ib_sa_register_client(&sa_client); 5401 register_netdevice_notifier(&cma_nb); 5402 register_netevent_notifier(&cma_netevent_cb); 5403 5404 ret = ib_register_client(&cma_client); 5405 if (ret) 5406 goto err; 5407 5408 ret = cma_configfs_init(); 5409 if (ret) 5410 goto err_ib; 5411 5412 return 0; 5413 5414 err_ib: 5415 ib_unregister_client(&cma_client); 5416 err: 5417 unregister_netevent_notifier(&cma_netevent_cb); 5418 unregister_netdevice_notifier(&cma_nb); 5419 ib_sa_unregister_client(&sa_client); 5420 unregister_pernet_subsys(&cma_pernet_operations); 5421 err_wq: 5422 destroy_workqueue(cma_wq); 5423 return ret; 5424 } 5425 5426 static void __exit cma_cleanup(void) 5427 { 5428 cma_configfs_exit(); 5429 ib_unregister_client(&cma_client); 5430 unregister_netevent_notifier(&cma_netevent_cb); 5431 unregister_netdevice_notifier(&cma_nb); 5432 ib_sa_unregister_client(&sa_client); 5433 unregister_pernet_subsys(&cma_pernet_operations); 5434 destroy_workqueue(cma_wq); 5435 } 5436 5437 module_init(cma_init); 5438 module_exit(cma_cleanup); 5439