1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2004-2007 Intel Corporation. All rights reserved. 4 * Copyright (c) 2004 Topspin Corporation. All rights reserved. 5 * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved. 6 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 7 * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. 8 */ 9 10 #include <linux/completion.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/device.h> 13 #include <linux/module.h> 14 #include <linux/err.h> 15 #include <linux/idr.h> 16 #include <linux/interrupt.h> 17 #include <linux/random.h> 18 #include <linux/rbtree.h> 19 #include <linux/spinlock.h> 20 #include <linux/slab.h> 21 #include <linux/sysfs.h> 22 #include <linux/workqueue.h> 23 #include <linux/kdev_t.h> 24 #include <linux/etherdevice.h> 25 26 #include <rdma/ib_cache.h> 27 #include <rdma/ib_cm.h> 28 #include "cm_msgs.h" 29 #include "core_priv.h" 30 31 MODULE_AUTHOR("Sean Hefty"); 32 MODULE_DESCRIPTION("InfiniBand CM"); 33 MODULE_LICENSE("Dual BSD/GPL"); 34 35 static const char * const ibcm_rej_reason_strs[] = { 36 [IB_CM_REJ_NO_QP] = "no QP", 37 [IB_CM_REJ_NO_EEC] = "no EEC", 38 [IB_CM_REJ_NO_RESOURCES] = "no resources", 39 [IB_CM_REJ_TIMEOUT] = "timeout", 40 [IB_CM_REJ_UNSUPPORTED] = "unsupported", 41 [IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID", 42 [IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance", 43 [IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID", 44 [IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type", 45 [IB_CM_REJ_STALE_CONN] = "stale conn", 46 [IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist", 47 [IB_CM_REJ_INVALID_GID] = "invalid GID", 48 [IB_CM_REJ_INVALID_LID] = "invalid LID", 49 [IB_CM_REJ_INVALID_SL] = "invalid SL", 50 [IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class", 51 [IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit", 52 [IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate", 53 [IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID", 54 [IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID", 55 [IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL", 56 [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class", 57 [IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit", 58 [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate", 59 [IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect", 60 [IB_CM_REJ_PORT_REDIRECT] = "port redirect", 61 [IB_CM_REJ_INVALID_MTU] = "invalid MTU", 62 [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources", 63 [IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined", 64 [IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry", 65 [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID", 66 [IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version", 67 [IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label", 68 [IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label", 69 }; 70 71 const char *__attribute_const__ ibcm_reject_msg(int reason) 72 { 73 size_t index = reason; 74 75 if (index < ARRAY_SIZE(ibcm_rej_reason_strs) && 76 ibcm_rej_reason_strs[index]) 77 return ibcm_rej_reason_strs[index]; 78 else 79 return "unrecognized reason"; 80 } 81 EXPORT_SYMBOL(ibcm_reject_msg); 82 83 struct cm_id_private; 84 static void cm_add_one(struct ib_device *device); 85 static void cm_remove_one(struct ib_device *device, void *client_data); 86 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, 87 struct ib_cm_sidr_rep_param *param); 88 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, 89 const void *private_data, u8 private_data_len); 90 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, 91 void *private_data, u8 private_data_len); 92 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, 93 enum ib_cm_rej_reason reason, void *ari, 94 u8 ari_length, const void *private_data, 95 u8 private_data_len); 96 97 static struct ib_client cm_client = { 98 .name = "cm", 99 .add = cm_add_one, 100 .remove = cm_remove_one 101 }; 102 103 static struct ib_cm { 104 spinlock_t lock; 105 struct list_head device_list; 106 rwlock_t device_lock; 107 struct rb_root listen_service_table; 108 u64 listen_service_id; 109 /* struct rb_root peer_service_table; todo: fix peer to peer */ 110 struct rb_root remote_qp_table; 111 struct rb_root remote_id_table; 112 struct rb_root remote_sidr_table; 113 struct xarray local_id_table; 114 u32 local_id_next; 115 __be32 random_id_operand; 116 struct list_head timewait_list; 117 struct workqueue_struct *wq; 118 /* Sync on cm change port state */ 119 spinlock_t state_lock; 120 } cm; 121 122 /* Counter indexes ordered by attribute ID */ 123 enum { 124 CM_REQ_COUNTER, 125 CM_MRA_COUNTER, 126 CM_REJ_COUNTER, 127 CM_REP_COUNTER, 128 CM_RTU_COUNTER, 129 CM_DREQ_COUNTER, 130 CM_DREP_COUNTER, 131 CM_SIDR_REQ_COUNTER, 132 CM_SIDR_REP_COUNTER, 133 CM_LAP_COUNTER, 134 CM_APR_COUNTER, 135 CM_ATTR_COUNT, 136 CM_ATTR_ID_OFFSET = 0x0010, 137 }; 138 139 enum { 140 CM_XMIT, 141 CM_XMIT_RETRIES, 142 CM_RECV, 143 CM_RECV_DUPLICATES, 144 CM_COUNTER_GROUPS 145 }; 146 147 static char const counter_group_names[CM_COUNTER_GROUPS] 148 [sizeof("cm_rx_duplicates")] = { 149 "cm_tx_msgs", "cm_tx_retries", 150 "cm_rx_msgs", "cm_rx_duplicates" 151 }; 152 153 struct cm_counter_group { 154 struct kobject obj; 155 atomic_long_t counter[CM_ATTR_COUNT]; 156 }; 157 158 struct cm_counter_attribute { 159 struct attribute attr; 160 int index; 161 }; 162 163 #define CM_COUNTER_ATTR(_name, _index) \ 164 struct cm_counter_attribute cm_##_name##_counter_attr = { \ 165 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 166 .index = _index \ 167 } 168 169 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER); 170 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER); 171 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER); 172 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER); 173 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER); 174 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER); 175 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER); 176 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER); 177 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER); 178 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER); 179 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER); 180 181 static struct attribute *cm_counter_default_attrs[] = { 182 &cm_req_counter_attr.attr, 183 &cm_mra_counter_attr.attr, 184 &cm_rej_counter_attr.attr, 185 &cm_rep_counter_attr.attr, 186 &cm_rtu_counter_attr.attr, 187 &cm_dreq_counter_attr.attr, 188 &cm_drep_counter_attr.attr, 189 &cm_sidr_req_counter_attr.attr, 190 &cm_sidr_rep_counter_attr.attr, 191 &cm_lap_counter_attr.attr, 192 &cm_apr_counter_attr.attr, 193 NULL 194 }; 195 196 struct cm_port { 197 struct cm_device *cm_dev; 198 struct ib_mad_agent *mad_agent; 199 struct kobject port_obj; 200 u8 port_num; 201 struct list_head cm_priv_prim_list; 202 struct list_head cm_priv_altr_list; 203 struct cm_counter_group counter_group[CM_COUNTER_GROUPS]; 204 }; 205 206 struct cm_device { 207 struct list_head list; 208 struct ib_device *ib_device; 209 u8 ack_delay; 210 int going_down; 211 struct cm_port *port[]; 212 }; 213 214 struct cm_av { 215 struct cm_port *port; 216 union ib_gid dgid; 217 struct rdma_ah_attr ah_attr; 218 u16 pkey_index; 219 u8 timeout; 220 }; 221 222 struct cm_work { 223 struct delayed_work work; 224 struct list_head list; 225 struct cm_port *port; 226 struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */ 227 __be32 local_id; /* Established / timewait */ 228 __be32 remote_id; 229 struct ib_cm_event cm_event; 230 struct sa_path_rec path[]; 231 }; 232 233 struct cm_timewait_info { 234 struct cm_work work; 235 struct list_head list; 236 struct rb_node remote_qp_node; 237 struct rb_node remote_id_node; 238 __be64 remote_ca_guid; 239 __be32 remote_qpn; 240 u8 inserted_remote_qp; 241 u8 inserted_remote_id; 242 }; 243 244 struct cm_id_private { 245 struct ib_cm_id id; 246 247 struct rb_node service_node; 248 struct rb_node sidr_id_node; 249 spinlock_t lock; /* Do not acquire inside cm.lock */ 250 struct completion comp; 251 refcount_t refcount; 252 /* Number of clients sharing this ib_cm_id. Only valid for listeners. 253 * Protected by the cm.lock spinlock. */ 254 int listen_sharecount; 255 struct rcu_head rcu; 256 257 struct ib_mad_send_buf *msg; 258 struct cm_timewait_info *timewait_info; 259 /* todo: use alternate port on send failure */ 260 struct cm_av av; 261 struct cm_av alt_av; 262 263 void *private_data; 264 __be64 tid; 265 __be32 local_qpn; 266 __be32 remote_qpn; 267 enum ib_qp_type qp_type; 268 __be32 sq_psn; 269 __be32 rq_psn; 270 int timeout_ms; 271 enum ib_mtu path_mtu; 272 __be16 pkey; 273 u8 private_data_len; 274 u8 max_cm_retries; 275 u8 responder_resources; 276 u8 initiator_depth; 277 u8 retry_count; 278 u8 rnr_retry_count; 279 u8 service_timeout; 280 u8 target_ack_delay; 281 282 struct list_head prim_list; 283 struct list_head altr_list; 284 /* Indicates that the send port mad is registered and av is set */ 285 int prim_send_port_not_ready; 286 int altr_send_port_not_ready; 287 288 struct list_head work_list; 289 atomic_t work_count; 290 }; 291 292 static void cm_work_handler(struct work_struct *work); 293 294 static inline void cm_deref_id(struct cm_id_private *cm_id_priv) 295 { 296 if (refcount_dec_and_test(&cm_id_priv->refcount)) 297 complete(&cm_id_priv->comp); 298 } 299 300 static int cm_alloc_msg(struct cm_id_private *cm_id_priv, 301 struct ib_mad_send_buf **msg) 302 { 303 struct ib_mad_agent *mad_agent; 304 struct ib_mad_send_buf *m; 305 struct ib_ah *ah; 306 struct cm_av *av; 307 unsigned long flags, flags2; 308 int ret = 0; 309 310 /* don't let the port to be released till the agent is down */ 311 spin_lock_irqsave(&cm.state_lock, flags2); 312 spin_lock_irqsave(&cm.lock, flags); 313 if (!cm_id_priv->prim_send_port_not_ready) 314 av = &cm_id_priv->av; 315 else if (!cm_id_priv->altr_send_port_not_ready && 316 (cm_id_priv->alt_av.port)) 317 av = &cm_id_priv->alt_av; 318 else { 319 pr_info("%s: not valid CM id\n", __func__); 320 ret = -ENODEV; 321 spin_unlock_irqrestore(&cm.lock, flags); 322 goto out; 323 } 324 spin_unlock_irqrestore(&cm.lock, flags); 325 /* Make sure the port haven't released the mad yet */ 326 mad_agent = cm_id_priv->av.port->mad_agent; 327 if (!mad_agent) { 328 pr_info("%s: not a valid MAD agent\n", __func__); 329 ret = -ENODEV; 330 goto out; 331 } 332 ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0); 333 if (IS_ERR(ah)) { 334 ret = PTR_ERR(ah); 335 goto out; 336 } 337 338 m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn, 339 av->pkey_index, 340 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, 341 GFP_ATOMIC, 342 IB_MGMT_BASE_VERSION); 343 if (IS_ERR(m)) { 344 rdma_destroy_ah(ah, 0); 345 ret = PTR_ERR(m); 346 goto out; 347 } 348 349 /* Timeout set by caller if response is expected. */ 350 m->ah = ah; 351 m->retries = cm_id_priv->max_cm_retries; 352 353 refcount_inc(&cm_id_priv->refcount); 354 m->context[0] = cm_id_priv; 355 *msg = m; 356 357 out: 358 spin_unlock_irqrestore(&cm.state_lock, flags2); 359 return ret; 360 } 361 362 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port, 363 struct ib_mad_recv_wc *mad_recv_wc) 364 { 365 return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index, 366 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, 367 GFP_ATOMIC, 368 IB_MGMT_BASE_VERSION); 369 } 370 371 static int cm_create_response_msg_ah(struct cm_port *port, 372 struct ib_mad_recv_wc *mad_recv_wc, 373 struct ib_mad_send_buf *msg) 374 { 375 struct ib_ah *ah; 376 377 ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc, 378 mad_recv_wc->recv_buf.grh, port->port_num); 379 if (IS_ERR(ah)) 380 return PTR_ERR(ah); 381 382 msg->ah = ah; 383 return 0; 384 } 385 386 static void cm_free_msg(struct ib_mad_send_buf *msg) 387 { 388 if (msg->ah) 389 rdma_destroy_ah(msg->ah, 0); 390 if (msg->context[0]) 391 cm_deref_id(msg->context[0]); 392 ib_free_send_mad(msg); 393 } 394 395 static int cm_alloc_response_msg(struct cm_port *port, 396 struct ib_mad_recv_wc *mad_recv_wc, 397 struct ib_mad_send_buf **msg) 398 { 399 struct ib_mad_send_buf *m; 400 int ret; 401 402 m = cm_alloc_response_msg_no_ah(port, mad_recv_wc); 403 if (IS_ERR(m)) 404 return PTR_ERR(m); 405 406 ret = cm_create_response_msg_ah(port, mad_recv_wc, m); 407 if (ret) { 408 cm_free_msg(m); 409 return ret; 410 } 411 412 *msg = m; 413 return 0; 414 } 415 416 static void * cm_copy_private_data(const void *private_data, 417 u8 private_data_len) 418 { 419 void *data; 420 421 if (!private_data || !private_data_len) 422 return NULL; 423 424 data = kmemdup(private_data, private_data_len, GFP_KERNEL); 425 if (!data) 426 return ERR_PTR(-ENOMEM); 427 428 return data; 429 } 430 431 static void cm_set_private_data(struct cm_id_private *cm_id_priv, 432 void *private_data, u8 private_data_len) 433 { 434 if (cm_id_priv->private_data && cm_id_priv->private_data_len) 435 kfree(cm_id_priv->private_data); 436 437 cm_id_priv->private_data = private_data; 438 cm_id_priv->private_data_len = private_data_len; 439 } 440 441 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc, 442 struct ib_grh *grh, struct cm_av *av) 443 { 444 struct rdma_ah_attr new_ah_attr; 445 int ret; 446 447 av->port = port; 448 av->pkey_index = wc->pkey_index; 449 450 /* 451 * av->ah_attr might be initialized based on past wc during incoming 452 * connect request or while sending out connect request. So initialize 453 * a new ah_attr on stack. If initialization fails, old ah_attr is 454 * used for sending any responses. If initialization is successful, 455 * than new ah_attr is used by overwriting old one. 456 */ 457 ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device, 458 port->port_num, wc, 459 grh, &new_ah_attr); 460 if (ret) 461 return ret; 462 463 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); 464 return 0; 465 } 466 467 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc, 468 struct ib_grh *grh, struct cm_av *av) 469 { 470 av->port = port; 471 av->pkey_index = wc->pkey_index; 472 return ib_init_ah_attr_from_wc(port->cm_dev->ib_device, 473 port->port_num, wc, 474 grh, &av->ah_attr); 475 } 476 477 static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv, 478 struct cm_av *av, 479 struct cm_port *port) 480 { 481 unsigned long flags; 482 int ret = 0; 483 484 spin_lock_irqsave(&cm.lock, flags); 485 486 if (&cm_id_priv->av == av) 487 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list); 488 else if (&cm_id_priv->alt_av == av) 489 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list); 490 else 491 ret = -EINVAL; 492 493 spin_unlock_irqrestore(&cm.lock, flags); 494 return ret; 495 } 496 497 static struct cm_port * 498 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr) 499 { 500 struct cm_device *cm_dev; 501 struct cm_port *port = NULL; 502 unsigned long flags; 503 504 if (attr) { 505 read_lock_irqsave(&cm.device_lock, flags); 506 list_for_each_entry(cm_dev, &cm.device_list, list) { 507 if (cm_dev->ib_device == attr->device) { 508 port = cm_dev->port[attr->port_num - 1]; 509 break; 510 } 511 } 512 read_unlock_irqrestore(&cm.device_lock, flags); 513 } else { 514 /* SGID attribute can be NULL in following 515 * conditions. 516 * (a) Alternative path 517 * (b) IB link layer without GRH 518 * (c) LAP send messages 519 */ 520 read_lock_irqsave(&cm.device_lock, flags); 521 list_for_each_entry(cm_dev, &cm.device_list, list) { 522 attr = rdma_find_gid(cm_dev->ib_device, 523 &path->sgid, 524 sa_conv_pathrec_to_gid_type(path), 525 NULL); 526 if (!IS_ERR(attr)) { 527 port = cm_dev->port[attr->port_num - 1]; 528 break; 529 } 530 } 531 read_unlock_irqrestore(&cm.device_lock, flags); 532 if (port) 533 rdma_put_gid_attr(attr); 534 } 535 return port; 536 } 537 538 static int cm_init_av_by_path(struct sa_path_rec *path, 539 const struct ib_gid_attr *sgid_attr, 540 struct cm_av *av, 541 struct cm_id_private *cm_id_priv) 542 { 543 struct rdma_ah_attr new_ah_attr; 544 struct cm_device *cm_dev; 545 struct cm_port *port; 546 int ret; 547 548 port = get_cm_port_from_path(path, sgid_attr); 549 if (!port) 550 return -EINVAL; 551 cm_dev = port->cm_dev; 552 553 ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num, 554 be16_to_cpu(path->pkey), &av->pkey_index); 555 if (ret) 556 return ret; 557 558 av->port = port; 559 560 /* 561 * av->ah_attr might be initialized based on wc or during 562 * request processing time which might have reference to sgid_attr. 563 * So initialize a new ah_attr on stack. 564 * If initialization fails, old ah_attr is used for sending any 565 * responses. If initialization is successful, than new ah_attr 566 * is used by overwriting the old one. So that right ah_attr 567 * can be used to return an error response. 568 */ 569 ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path, 570 &new_ah_attr, sgid_attr); 571 if (ret) 572 return ret; 573 574 av->timeout = path->packet_life_time + 1; 575 576 ret = add_cm_id_to_port_list(cm_id_priv, av, port); 577 if (ret) { 578 rdma_destroy_ah_attr(&new_ah_attr); 579 return ret; 580 } 581 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); 582 return 0; 583 } 584 585 static u32 cm_local_id(__be32 local_id) 586 { 587 return (__force u32) (local_id ^ cm.random_id_operand); 588 } 589 590 static void cm_free_id(__be32 local_id) 591 { 592 xa_erase_irq(&cm.local_id_table, cm_local_id(local_id)); 593 } 594 595 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id) 596 { 597 struct cm_id_private *cm_id_priv; 598 599 rcu_read_lock(); 600 cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id)); 601 if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id || 602 !refcount_inc_not_zero(&cm_id_priv->refcount)) 603 cm_id_priv = NULL; 604 rcu_read_unlock(); 605 606 return cm_id_priv; 607 } 608 609 /* 610 * Trivial helpers to strip endian annotation and compare; the 611 * endianness doesn't actually matter since we just need a stable 612 * order for the RB tree. 613 */ 614 static int be32_lt(__be32 a, __be32 b) 615 { 616 return (__force u32) a < (__force u32) b; 617 } 618 619 static int be32_gt(__be32 a, __be32 b) 620 { 621 return (__force u32) a > (__force u32) b; 622 } 623 624 static int be64_lt(__be64 a, __be64 b) 625 { 626 return (__force u64) a < (__force u64) b; 627 } 628 629 static int be64_gt(__be64 a, __be64 b) 630 { 631 return (__force u64) a > (__force u64) b; 632 } 633 634 /* 635 * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv 636 * if the new ID was inserted, NULL if it could not be inserted due to a 637 * collision, or the existing cm_id_priv ready for shared usage. 638 */ 639 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv, 640 ib_cm_handler shared_handler) 641 { 642 struct rb_node **link = &cm.listen_service_table.rb_node; 643 struct rb_node *parent = NULL; 644 struct cm_id_private *cur_cm_id_priv; 645 __be64 service_id = cm_id_priv->id.service_id; 646 __be64 service_mask = cm_id_priv->id.service_mask; 647 unsigned long flags; 648 649 spin_lock_irqsave(&cm.lock, flags); 650 while (*link) { 651 parent = *link; 652 cur_cm_id_priv = rb_entry(parent, struct cm_id_private, 653 service_node); 654 if ((cur_cm_id_priv->id.service_mask & service_id) == 655 (service_mask & cur_cm_id_priv->id.service_id) && 656 (cm_id_priv->id.device == cur_cm_id_priv->id.device)) { 657 /* 658 * Sharing an ib_cm_id with different handlers is not 659 * supported 660 */ 661 if (cur_cm_id_priv->id.cm_handler != shared_handler || 662 cur_cm_id_priv->id.context || 663 WARN_ON(!cur_cm_id_priv->id.cm_handler)) { 664 spin_unlock_irqrestore(&cm.lock, flags); 665 return NULL; 666 } 667 refcount_inc(&cur_cm_id_priv->refcount); 668 cur_cm_id_priv->listen_sharecount++; 669 spin_unlock_irqrestore(&cm.lock, flags); 670 return cur_cm_id_priv; 671 } 672 673 if (cm_id_priv->id.device < cur_cm_id_priv->id.device) 674 link = &(*link)->rb_left; 675 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device) 676 link = &(*link)->rb_right; 677 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id)) 678 link = &(*link)->rb_left; 679 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id)) 680 link = &(*link)->rb_right; 681 else 682 link = &(*link)->rb_right; 683 } 684 cm_id_priv->listen_sharecount++; 685 rb_link_node(&cm_id_priv->service_node, parent, link); 686 rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table); 687 spin_unlock_irqrestore(&cm.lock, flags); 688 return cm_id_priv; 689 } 690 691 static struct cm_id_private * cm_find_listen(struct ib_device *device, 692 __be64 service_id) 693 { 694 struct rb_node *node = cm.listen_service_table.rb_node; 695 struct cm_id_private *cm_id_priv; 696 697 while (node) { 698 cm_id_priv = rb_entry(node, struct cm_id_private, service_node); 699 if ((cm_id_priv->id.service_mask & service_id) == 700 cm_id_priv->id.service_id && 701 (cm_id_priv->id.device == device)) 702 return cm_id_priv; 703 704 if (device < cm_id_priv->id.device) 705 node = node->rb_left; 706 else if (device > cm_id_priv->id.device) 707 node = node->rb_right; 708 else if (be64_lt(service_id, cm_id_priv->id.service_id)) 709 node = node->rb_left; 710 else if (be64_gt(service_id, cm_id_priv->id.service_id)) 711 node = node->rb_right; 712 else 713 node = node->rb_right; 714 } 715 return NULL; 716 } 717 718 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info 719 *timewait_info) 720 { 721 struct rb_node **link = &cm.remote_id_table.rb_node; 722 struct rb_node *parent = NULL; 723 struct cm_timewait_info *cur_timewait_info; 724 __be64 remote_ca_guid = timewait_info->remote_ca_guid; 725 __be32 remote_id = timewait_info->work.remote_id; 726 727 while (*link) { 728 parent = *link; 729 cur_timewait_info = rb_entry(parent, struct cm_timewait_info, 730 remote_id_node); 731 if (be32_lt(remote_id, cur_timewait_info->work.remote_id)) 732 link = &(*link)->rb_left; 733 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id)) 734 link = &(*link)->rb_right; 735 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 736 link = &(*link)->rb_left; 737 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 738 link = &(*link)->rb_right; 739 else 740 return cur_timewait_info; 741 } 742 timewait_info->inserted_remote_id = 1; 743 rb_link_node(&timewait_info->remote_id_node, parent, link); 744 rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table); 745 return NULL; 746 } 747 748 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid, 749 __be32 remote_id) 750 { 751 struct rb_node *node = cm.remote_id_table.rb_node; 752 struct cm_timewait_info *timewait_info; 753 754 while (node) { 755 timewait_info = rb_entry(node, struct cm_timewait_info, 756 remote_id_node); 757 if (be32_lt(remote_id, timewait_info->work.remote_id)) 758 node = node->rb_left; 759 else if (be32_gt(remote_id, timewait_info->work.remote_id)) 760 node = node->rb_right; 761 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid)) 762 node = node->rb_left; 763 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid)) 764 node = node->rb_right; 765 else 766 return timewait_info; 767 } 768 return NULL; 769 } 770 771 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info 772 *timewait_info) 773 { 774 struct rb_node **link = &cm.remote_qp_table.rb_node; 775 struct rb_node *parent = NULL; 776 struct cm_timewait_info *cur_timewait_info; 777 __be64 remote_ca_guid = timewait_info->remote_ca_guid; 778 __be32 remote_qpn = timewait_info->remote_qpn; 779 780 while (*link) { 781 parent = *link; 782 cur_timewait_info = rb_entry(parent, struct cm_timewait_info, 783 remote_qp_node); 784 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn)) 785 link = &(*link)->rb_left; 786 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn)) 787 link = &(*link)->rb_right; 788 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 789 link = &(*link)->rb_left; 790 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 791 link = &(*link)->rb_right; 792 else 793 return cur_timewait_info; 794 } 795 timewait_info->inserted_remote_qp = 1; 796 rb_link_node(&timewait_info->remote_qp_node, parent, link); 797 rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table); 798 return NULL; 799 } 800 801 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private 802 *cm_id_priv) 803 { 804 struct rb_node **link = &cm.remote_sidr_table.rb_node; 805 struct rb_node *parent = NULL; 806 struct cm_id_private *cur_cm_id_priv; 807 union ib_gid *port_gid = &cm_id_priv->av.dgid; 808 __be32 remote_id = cm_id_priv->id.remote_id; 809 810 while (*link) { 811 parent = *link; 812 cur_cm_id_priv = rb_entry(parent, struct cm_id_private, 813 sidr_id_node); 814 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id)) 815 link = &(*link)->rb_left; 816 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id)) 817 link = &(*link)->rb_right; 818 else { 819 int cmp; 820 cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid, 821 sizeof *port_gid); 822 if (cmp < 0) 823 link = &(*link)->rb_left; 824 else if (cmp > 0) 825 link = &(*link)->rb_right; 826 else 827 return cur_cm_id_priv; 828 } 829 } 830 rb_link_node(&cm_id_priv->sidr_id_node, parent, link); 831 rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); 832 return NULL; 833 } 834 835 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device, 836 ib_cm_handler cm_handler, 837 void *context) 838 { 839 struct cm_id_private *cm_id_priv; 840 u32 id; 841 int ret; 842 843 cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL); 844 if (!cm_id_priv) 845 return ERR_PTR(-ENOMEM); 846 847 cm_id_priv->id.state = IB_CM_IDLE; 848 cm_id_priv->id.device = device; 849 cm_id_priv->id.cm_handler = cm_handler; 850 cm_id_priv->id.context = context; 851 cm_id_priv->id.remote_cm_qpn = 1; 852 853 RB_CLEAR_NODE(&cm_id_priv->service_node); 854 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node); 855 spin_lock_init(&cm_id_priv->lock); 856 init_completion(&cm_id_priv->comp); 857 INIT_LIST_HEAD(&cm_id_priv->work_list); 858 INIT_LIST_HEAD(&cm_id_priv->prim_list); 859 INIT_LIST_HEAD(&cm_id_priv->altr_list); 860 atomic_set(&cm_id_priv->work_count, -1); 861 refcount_set(&cm_id_priv->refcount, 1); 862 863 ret = xa_alloc_cyclic_irq(&cm.local_id_table, &id, NULL, xa_limit_32b, 864 &cm.local_id_next, GFP_KERNEL); 865 if (ret) 866 goto error; 867 cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand; 868 869 return cm_id_priv; 870 871 error: 872 kfree(cm_id_priv); 873 return ERR_PTR(ret); 874 } 875 876 /* 877 * Make the ID visible to the MAD handlers and other threads that use the 878 * xarray. 879 */ 880 static void cm_finalize_id(struct cm_id_private *cm_id_priv) 881 { 882 xa_store_irq(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id), 883 cm_id_priv, GFP_KERNEL); 884 } 885 886 struct ib_cm_id *ib_create_cm_id(struct ib_device *device, 887 ib_cm_handler cm_handler, 888 void *context) 889 { 890 struct cm_id_private *cm_id_priv; 891 892 cm_id_priv = cm_alloc_id_priv(device, cm_handler, context); 893 if (IS_ERR(cm_id_priv)) 894 return ERR_CAST(cm_id_priv); 895 896 cm_finalize_id(cm_id_priv); 897 return &cm_id_priv->id; 898 } 899 EXPORT_SYMBOL(ib_create_cm_id); 900 901 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv) 902 { 903 struct cm_work *work; 904 905 if (list_empty(&cm_id_priv->work_list)) 906 return NULL; 907 908 work = list_entry(cm_id_priv->work_list.next, struct cm_work, list); 909 list_del(&work->list); 910 return work; 911 } 912 913 static void cm_free_work(struct cm_work *work) 914 { 915 if (work->mad_recv_wc) 916 ib_free_recv_mad(work->mad_recv_wc); 917 kfree(work); 918 } 919 920 static inline int cm_convert_to_ms(int iba_time) 921 { 922 /* approximate conversion to ms from 4.096us x 2^iba_time */ 923 return 1 << max(iba_time - 8, 0); 924 } 925 926 /* 927 * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time 928 * Because of how ack_timeout is stored, adding one doubles the timeout. 929 * To avoid large timeouts, select the max(ack_delay, life_time + 1), and 930 * increment it (round up) only if the other is within 50%. 931 */ 932 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time) 933 { 934 int ack_timeout = packet_life_time + 1; 935 936 if (ack_timeout >= ca_ack_delay) 937 ack_timeout += (ca_ack_delay >= (ack_timeout - 1)); 938 else 939 ack_timeout = ca_ack_delay + 940 (ack_timeout >= (ca_ack_delay - 1)); 941 942 return min(31, ack_timeout); 943 } 944 945 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info) 946 { 947 if (timewait_info->inserted_remote_id) { 948 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table); 949 timewait_info->inserted_remote_id = 0; 950 } 951 952 if (timewait_info->inserted_remote_qp) { 953 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table); 954 timewait_info->inserted_remote_qp = 0; 955 } 956 } 957 958 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id) 959 { 960 struct cm_timewait_info *timewait_info; 961 962 timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL); 963 if (!timewait_info) 964 return ERR_PTR(-ENOMEM); 965 966 timewait_info->work.local_id = local_id; 967 INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler); 968 timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT; 969 return timewait_info; 970 } 971 972 static void cm_enter_timewait(struct cm_id_private *cm_id_priv) 973 { 974 int wait_time; 975 unsigned long flags; 976 struct cm_device *cm_dev; 977 978 lockdep_assert_held(&cm_id_priv->lock); 979 980 cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client); 981 if (!cm_dev) 982 return; 983 984 spin_lock_irqsave(&cm.lock, flags); 985 cm_cleanup_timewait(cm_id_priv->timewait_info); 986 list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list); 987 spin_unlock_irqrestore(&cm.lock, flags); 988 989 /* 990 * The cm_id could be destroyed by the user before we exit timewait. 991 * To protect against this, we search for the cm_id after exiting 992 * timewait before notifying the user that we've exited timewait. 993 */ 994 cm_id_priv->id.state = IB_CM_TIMEWAIT; 995 wait_time = cm_convert_to_ms(cm_id_priv->av.timeout); 996 997 /* Check if the device started its remove_one */ 998 spin_lock_irqsave(&cm.lock, flags); 999 if (!cm_dev->going_down) 1000 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work, 1001 msecs_to_jiffies(wait_time)); 1002 spin_unlock_irqrestore(&cm.lock, flags); 1003 1004 cm_id_priv->timewait_info = NULL; 1005 } 1006 1007 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv) 1008 { 1009 unsigned long flags; 1010 1011 lockdep_assert_held(&cm_id_priv->lock); 1012 1013 cm_id_priv->id.state = IB_CM_IDLE; 1014 if (cm_id_priv->timewait_info) { 1015 spin_lock_irqsave(&cm.lock, flags); 1016 cm_cleanup_timewait(cm_id_priv->timewait_info); 1017 spin_unlock_irqrestore(&cm.lock, flags); 1018 kfree(cm_id_priv->timewait_info); 1019 cm_id_priv->timewait_info = NULL; 1020 } 1021 } 1022 1023 static void cm_destroy_id(struct ib_cm_id *cm_id, int err) 1024 { 1025 struct cm_id_private *cm_id_priv; 1026 struct cm_work *work; 1027 1028 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 1029 spin_lock_irq(&cm_id_priv->lock); 1030 retest: 1031 switch (cm_id->state) { 1032 case IB_CM_LISTEN: 1033 spin_lock(&cm.lock); 1034 if (--cm_id_priv->listen_sharecount > 0) { 1035 /* The id is still shared. */ 1036 WARN_ON(refcount_read(&cm_id_priv->refcount) == 1); 1037 spin_unlock(&cm.lock); 1038 spin_unlock_irq(&cm_id_priv->lock); 1039 cm_deref_id(cm_id_priv); 1040 return; 1041 } 1042 cm_id->state = IB_CM_IDLE; 1043 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table); 1044 RB_CLEAR_NODE(&cm_id_priv->service_node); 1045 spin_unlock(&cm.lock); 1046 break; 1047 case IB_CM_SIDR_REQ_SENT: 1048 cm_id->state = IB_CM_IDLE; 1049 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1050 break; 1051 case IB_CM_SIDR_REQ_RCVD: 1052 cm_send_sidr_rep_locked(cm_id_priv, 1053 &(struct ib_cm_sidr_rep_param){ 1054 .status = IB_SIDR_REJECT }); 1055 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */ 1056 cm_id->state = IB_CM_IDLE; 1057 break; 1058 case IB_CM_REQ_SENT: 1059 case IB_CM_MRA_REQ_RCVD: 1060 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1061 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT, 1062 &cm_id_priv->id.device->node_guid, 1063 sizeof(cm_id_priv->id.device->node_guid), 1064 NULL, 0); 1065 break; 1066 case IB_CM_REQ_RCVD: 1067 if (err == -ENOMEM) { 1068 /* Do not reject to allow future retries. */ 1069 cm_reset_to_idle(cm_id_priv); 1070 } else { 1071 cm_send_rej_locked(cm_id_priv, 1072 IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, 1073 NULL, 0); 1074 } 1075 break; 1076 case IB_CM_REP_SENT: 1077 case IB_CM_MRA_REP_RCVD: 1078 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1079 /* Fall through */ 1080 case IB_CM_MRA_REQ_SENT: 1081 case IB_CM_REP_RCVD: 1082 case IB_CM_MRA_REP_SENT: 1083 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL, 1084 0, NULL, 0); 1085 break; 1086 case IB_CM_ESTABLISHED: 1087 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) { 1088 cm_id->state = IB_CM_IDLE; 1089 break; 1090 } 1091 cm_send_dreq_locked(cm_id_priv, NULL, 0); 1092 goto retest; 1093 case IB_CM_DREQ_SENT: 1094 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1095 cm_enter_timewait(cm_id_priv); 1096 goto retest; 1097 case IB_CM_DREQ_RCVD: 1098 cm_send_drep_locked(cm_id_priv, NULL, 0); 1099 WARN_ON(cm_id->state != IB_CM_TIMEWAIT); 1100 goto retest; 1101 case IB_CM_TIMEWAIT: 1102 /* 1103 * The cm_acquire_id in cm_timewait_handler will stop working 1104 * once we do cm_free_id() below, so just move to idle here for 1105 * consistency. 1106 */ 1107 cm_id->state = IB_CM_IDLE; 1108 break; 1109 case IB_CM_IDLE: 1110 break; 1111 } 1112 WARN_ON(cm_id->state != IB_CM_IDLE); 1113 1114 spin_lock(&cm.lock); 1115 /* Required for cleanup paths related cm_req_handler() */ 1116 if (cm_id_priv->timewait_info) { 1117 cm_cleanup_timewait(cm_id_priv->timewait_info); 1118 kfree(cm_id_priv->timewait_info); 1119 cm_id_priv->timewait_info = NULL; 1120 } 1121 if (!list_empty(&cm_id_priv->altr_list) && 1122 (!cm_id_priv->altr_send_port_not_ready)) 1123 list_del(&cm_id_priv->altr_list); 1124 if (!list_empty(&cm_id_priv->prim_list) && 1125 (!cm_id_priv->prim_send_port_not_ready)) 1126 list_del(&cm_id_priv->prim_list); 1127 WARN_ON(cm_id_priv->listen_sharecount); 1128 WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node)); 1129 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) 1130 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); 1131 spin_unlock(&cm.lock); 1132 spin_unlock_irq(&cm_id_priv->lock); 1133 1134 cm_free_id(cm_id->local_id); 1135 cm_deref_id(cm_id_priv); 1136 wait_for_completion(&cm_id_priv->comp); 1137 while ((work = cm_dequeue_work(cm_id_priv)) != NULL) 1138 cm_free_work(work); 1139 1140 rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr); 1141 rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr); 1142 kfree(cm_id_priv->private_data); 1143 kfree_rcu(cm_id_priv, rcu); 1144 } 1145 1146 void ib_destroy_cm_id(struct ib_cm_id *cm_id) 1147 { 1148 cm_destroy_id(cm_id, 0); 1149 } 1150 EXPORT_SYMBOL(ib_destroy_cm_id); 1151 1152 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id, 1153 __be64 service_mask) 1154 { 1155 service_mask = service_mask ? service_mask : ~cpu_to_be64(0); 1156 service_id &= service_mask; 1157 if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID && 1158 (service_id != IB_CM_ASSIGN_SERVICE_ID)) 1159 return -EINVAL; 1160 1161 if (service_id == IB_CM_ASSIGN_SERVICE_ID) { 1162 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++); 1163 cm_id_priv->id.service_mask = ~cpu_to_be64(0); 1164 } else { 1165 cm_id_priv->id.service_id = service_id; 1166 cm_id_priv->id.service_mask = service_mask; 1167 } 1168 return 0; 1169 } 1170 1171 /** 1172 * ib_cm_listen - Initiates listening on the specified service ID for 1173 * connection and service ID resolution requests. 1174 * @cm_id: Connection identifier associated with the listen request. 1175 * @service_id: Service identifier matched against incoming connection 1176 * and service ID resolution requests. The service ID should be specified 1177 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will 1178 * assign a service ID to the caller. 1179 * @service_mask: Mask applied to service ID used to listen across a 1180 * range of service IDs. If set to 0, the service ID is matched 1181 * exactly. This parameter is ignored if %service_id is set to 1182 * IB_CM_ASSIGN_SERVICE_ID. 1183 */ 1184 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask) 1185 { 1186 struct cm_id_private *cm_id_priv = 1187 container_of(cm_id, struct cm_id_private, id); 1188 unsigned long flags; 1189 int ret; 1190 1191 spin_lock_irqsave(&cm_id_priv->lock, flags); 1192 if (cm_id_priv->id.state != IB_CM_IDLE) { 1193 ret = -EINVAL; 1194 goto out; 1195 } 1196 1197 ret = cm_init_listen(cm_id_priv, service_id, service_mask); 1198 if (ret) 1199 goto out; 1200 1201 if (!cm_insert_listen(cm_id_priv, NULL)) { 1202 ret = -EBUSY; 1203 goto out; 1204 } 1205 1206 cm_id_priv->id.state = IB_CM_LISTEN; 1207 ret = 0; 1208 1209 out: 1210 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1211 return ret; 1212 } 1213 EXPORT_SYMBOL(ib_cm_listen); 1214 1215 /** 1216 * Create a new listening ib_cm_id and listen on the given service ID. 1217 * 1218 * If there's an existing ID listening on that same device and service ID, 1219 * return it. 1220 * 1221 * @device: Device associated with the cm_id. All related communication will 1222 * be associated with the specified device. 1223 * @cm_handler: Callback invoked to notify the user of CM events. 1224 * @service_id: Service identifier matched against incoming connection 1225 * and service ID resolution requests. The service ID should be specified 1226 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will 1227 * assign a service ID to the caller. 1228 * 1229 * Callers should call ib_destroy_cm_id when done with the listener ID. 1230 */ 1231 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, 1232 ib_cm_handler cm_handler, 1233 __be64 service_id) 1234 { 1235 struct cm_id_private *listen_id_priv; 1236 struct cm_id_private *cm_id_priv; 1237 int err = 0; 1238 1239 /* Create an ID in advance, since the creation may sleep */ 1240 cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL); 1241 if (IS_ERR(cm_id_priv)) 1242 return ERR_CAST(cm_id_priv); 1243 1244 err = cm_init_listen(cm_id_priv, service_id, 0); 1245 if (err) 1246 return ERR_PTR(err); 1247 1248 spin_lock_irq(&cm_id_priv->lock); 1249 listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler); 1250 if (listen_id_priv != cm_id_priv) { 1251 spin_unlock_irq(&cm_id_priv->lock); 1252 ib_destroy_cm_id(&cm_id_priv->id); 1253 if (!listen_id_priv) 1254 return ERR_PTR(-EINVAL); 1255 return &listen_id_priv->id; 1256 } 1257 cm_id_priv->id.state = IB_CM_LISTEN; 1258 spin_unlock_irq(&cm_id_priv->lock); 1259 1260 /* 1261 * A listen ID does not need to be in the xarray since it does not 1262 * receive mads, is not placed in the remote_id or remote_qpn rbtree, 1263 * and does not enter timewait. 1264 */ 1265 1266 return &cm_id_priv->id; 1267 } 1268 EXPORT_SYMBOL(ib_cm_insert_listen); 1269 1270 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv) 1271 { 1272 u64 hi_tid, low_tid; 1273 1274 hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32; 1275 low_tid = (u64)cm_id_priv->id.local_id; 1276 return cpu_to_be64(hi_tid | low_tid); 1277 } 1278 1279 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr, 1280 __be16 attr_id, __be64 tid) 1281 { 1282 hdr->base_version = IB_MGMT_BASE_VERSION; 1283 hdr->mgmt_class = IB_MGMT_CLASS_CM; 1284 hdr->class_version = IB_CM_CLASS_VERSION; 1285 hdr->method = IB_MGMT_METHOD_SEND; 1286 hdr->attr_id = attr_id; 1287 hdr->tid = tid; 1288 } 1289 1290 static void cm_format_req(struct cm_req_msg *req_msg, 1291 struct cm_id_private *cm_id_priv, 1292 struct ib_cm_req_param *param) 1293 { 1294 struct sa_path_rec *pri_path = param->primary_path; 1295 struct sa_path_rec *alt_path = param->alternate_path; 1296 bool pri_ext = false; 1297 1298 if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA) 1299 pri_ext = opa_is_extended_lid(pri_path->opa.dlid, 1300 pri_path->opa.slid); 1301 1302 cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID, 1303 cm_form_tid(cm_id_priv)); 1304 1305 IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg, 1306 be32_to_cpu(cm_id_priv->id.local_id)); 1307 IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id)); 1308 IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg, 1309 be64_to_cpu(cm_id_priv->id.device->node_guid)); 1310 IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num); 1311 IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth); 1312 IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg, 1313 param->remote_cm_response_timeout); 1314 cm_req_set_qp_type(req_msg, param->qp_type); 1315 IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control); 1316 IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn); 1317 IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg, 1318 param->local_cm_response_timeout); 1319 IBA_SET(CM_REQ_PARTITION_KEY, req_msg, 1320 be16_to_cpu(param->primary_path->pkey)); 1321 IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg, 1322 param->primary_path->mtu); 1323 IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries); 1324 1325 if (param->qp_type != IB_QPT_XRC_INI) { 1326 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg, 1327 param->responder_resources); 1328 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count); 1329 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg, 1330 param->rnr_retry_count); 1331 IBA_SET(CM_REQ_SRQ, req_msg, param->srq); 1332 } 1333 1334 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) = 1335 pri_path->sgid; 1336 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) = 1337 pri_path->dgid; 1338 if (pri_ext) { 1339 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) 1340 ->global.interface_id = 1341 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid)); 1342 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) 1343 ->global.interface_id = 1344 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); 1345 } 1346 if (pri_path->hop_limit <= 1) { 1347 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, 1348 be16_to_cpu(pri_ext ? 0 : 1349 htons(ntohl(sa_path_get_slid( 1350 pri_path))))); 1351 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, 1352 be16_to_cpu(pri_ext ? 0 : 1353 htons(ntohl(sa_path_get_dlid( 1354 pri_path))))); 1355 } else { 1356 /* Work-around until there's a way to obtain remote LID info */ 1357 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, 1358 be16_to_cpu(IB_LID_PERMISSIVE)); 1359 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, 1360 be16_to_cpu(IB_LID_PERMISSIVE)); 1361 } 1362 IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg, 1363 be32_to_cpu(pri_path->flow_label)); 1364 IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate); 1365 IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class); 1366 IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit); 1367 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl); 1368 IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg, 1369 (pri_path->hop_limit <= 1)); 1370 IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg, 1371 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, 1372 pri_path->packet_life_time)); 1373 1374 if (alt_path) { 1375 bool alt_ext = false; 1376 1377 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA) 1378 alt_ext = opa_is_extended_lid(alt_path->opa.dlid, 1379 alt_path->opa.slid); 1380 1381 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) = 1382 alt_path->sgid; 1383 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) = 1384 alt_path->dgid; 1385 if (alt_ext) { 1386 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, 1387 req_msg) 1388 ->global.interface_id = 1389 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid)); 1390 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, 1391 req_msg) 1392 ->global.interface_id = 1393 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); 1394 } 1395 if (alt_path->hop_limit <= 1) { 1396 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, 1397 be16_to_cpu( 1398 alt_ext ? 0 : 1399 htons(ntohl(sa_path_get_slid( 1400 alt_path))))); 1401 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, 1402 be16_to_cpu( 1403 alt_ext ? 0 : 1404 htons(ntohl(sa_path_get_dlid( 1405 alt_path))))); 1406 } else { 1407 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, 1408 be16_to_cpu(IB_LID_PERMISSIVE)); 1409 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, 1410 be16_to_cpu(IB_LID_PERMISSIVE)); 1411 } 1412 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg, 1413 be32_to_cpu(alt_path->flow_label)); 1414 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate); 1415 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg, 1416 alt_path->traffic_class); 1417 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg, 1418 alt_path->hop_limit); 1419 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl); 1420 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg, 1421 (alt_path->hop_limit <= 1)); 1422 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg, 1423 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, 1424 alt_path->packet_life_time)); 1425 } 1426 1427 if (param->private_data && param->private_data_len) 1428 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data, 1429 param->private_data_len); 1430 } 1431 1432 static int cm_validate_req_param(struct ib_cm_req_param *param) 1433 { 1434 if (!param->primary_path) 1435 return -EINVAL; 1436 1437 if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC && 1438 param->qp_type != IB_QPT_XRC_INI) 1439 return -EINVAL; 1440 1441 if (param->private_data && 1442 param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE) 1443 return -EINVAL; 1444 1445 if (param->alternate_path && 1446 (param->alternate_path->pkey != param->primary_path->pkey || 1447 param->alternate_path->mtu != param->primary_path->mtu)) 1448 return -EINVAL; 1449 1450 return 0; 1451 } 1452 1453 int ib_send_cm_req(struct ib_cm_id *cm_id, 1454 struct ib_cm_req_param *param) 1455 { 1456 struct cm_id_private *cm_id_priv; 1457 struct cm_req_msg *req_msg; 1458 unsigned long flags; 1459 int ret; 1460 1461 ret = cm_validate_req_param(param); 1462 if (ret) 1463 return ret; 1464 1465 /* Verify that we're not in timewait. */ 1466 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 1467 spin_lock_irqsave(&cm_id_priv->lock, flags); 1468 if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) { 1469 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1470 ret = -EINVAL; 1471 goto out; 1472 } 1473 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1474 1475 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> 1476 id.local_id); 1477 if (IS_ERR(cm_id_priv->timewait_info)) { 1478 ret = PTR_ERR(cm_id_priv->timewait_info); 1479 goto out; 1480 } 1481 1482 ret = cm_init_av_by_path(param->primary_path, 1483 param->ppath_sgid_attr, &cm_id_priv->av, 1484 cm_id_priv); 1485 if (ret) 1486 goto out; 1487 if (param->alternate_path) { 1488 ret = cm_init_av_by_path(param->alternate_path, NULL, 1489 &cm_id_priv->alt_av, cm_id_priv); 1490 if (ret) 1491 goto out; 1492 } 1493 cm_id->service_id = param->service_id; 1494 cm_id->service_mask = ~cpu_to_be64(0); 1495 cm_id_priv->timeout_ms = cm_convert_to_ms( 1496 param->primary_path->packet_life_time) * 2 + 1497 cm_convert_to_ms( 1498 param->remote_cm_response_timeout); 1499 cm_id_priv->max_cm_retries = param->max_cm_retries; 1500 cm_id_priv->initiator_depth = param->initiator_depth; 1501 cm_id_priv->responder_resources = param->responder_resources; 1502 cm_id_priv->retry_count = param->retry_count; 1503 cm_id_priv->path_mtu = param->primary_path->mtu; 1504 cm_id_priv->pkey = param->primary_path->pkey; 1505 cm_id_priv->qp_type = param->qp_type; 1506 1507 ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg); 1508 if (ret) 1509 goto out; 1510 1511 req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad; 1512 cm_format_req(req_msg, cm_id_priv, param); 1513 cm_id_priv->tid = req_msg->hdr.tid; 1514 cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms; 1515 cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT; 1516 1517 cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); 1518 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); 1519 1520 spin_lock_irqsave(&cm_id_priv->lock, flags); 1521 ret = ib_post_send_mad(cm_id_priv->msg, NULL); 1522 if (ret) { 1523 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1524 goto error2; 1525 } 1526 BUG_ON(cm_id->state != IB_CM_IDLE); 1527 cm_id->state = IB_CM_REQ_SENT; 1528 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1529 return 0; 1530 1531 error2: cm_free_msg(cm_id_priv->msg); 1532 out: return ret; 1533 } 1534 EXPORT_SYMBOL(ib_send_cm_req); 1535 1536 static int cm_issue_rej(struct cm_port *port, 1537 struct ib_mad_recv_wc *mad_recv_wc, 1538 enum ib_cm_rej_reason reason, 1539 enum cm_msg_response msg_rejected, 1540 void *ari, u8 ari_length) 1541 { 1542 struct ib_mad_send_buf *msg = NULL; 1543 struct cm_rej_msg *rej_msg, *rcv_msg; 1544 int ret; 1545 1546 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg); 1547 if (ret) 1548 return ret; 1549 1550 /* We just need common CM header information. Cast to any message. */ 1551 rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad; 1552 rej_msg = (struct cm_rej_msg *) msg->mad; 1553 1554 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid); 1555 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, 1556 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg)); 1557 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1558 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg)); 1559 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected); 1560 IBA_SET(CM_REJ_REASON, rej_msg, reason); 1561 1562 if (ari && ari_length) { 1563 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); 1564 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); 1565 } 1566 1567 ret = ib_post_send_mad(msg, NULL); 1568 if (ret) 1569 cm_free_msg(msg); 1570 1571 return ret; 1572 } 1573 1574 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg) 1575 { 1576 return ((cpu_to_be16( 1577 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) || 1578 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, 1579 req_msg)))); 1580 } 1581 1582 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num, 1583 struct sa_path_rec *path, union ib_gid *gid) 1584 { 1585 if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num)) 1586 path->rec_type = SA_PATH_REC_TYPE_OPA; 1587 else 1588 path->rec_type = SA_PATH_REC_TYPE_IB; 1589 } 1590 1591 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, 1592 struct sa_path_rec *primary_path, 1593 struct sa_path_rec *alt_path) 1594 { 1595 u32 lid; 1596 1597 if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) { 1598 sa_path_set_dlid(primary_path, 1599 IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, 1600 req_msg)); 1601 sa_path_set_slid(primary_path, 1602 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, 1603 req_msg)); 1604 } else { 1605 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1606 CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)); 1607 sa_path_set_dlid(primary_path, lid); 1608 1609 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1610 CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)); 1611 sa_path_set_slid(primary_path, lid); 1612 } 1613 1614 if (!cm_req_has_alt_path(req_msg)) 1615 return; 1616 1617 if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) { 1618 sa_path_set_dlid(alt_path, 1619 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, 1620 req_msg)); 1621 sa_path_set_slid(alt_path, 1622 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, 1623 req_msg)); 1624 } else { 1625 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1626 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg)); 1627 sa_path_set_dlid(alt_path, lid); 1628 1629 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1630 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg)); 1631 sa_path_set_slid(alt_path, lid); 1632 } 1633 } 1634 1635 static void cm_format_paths_from_req(struct cm_req_msg *req_msg, 1636 struct sa_path_rec *primary_path, 1637 struct sa_path_rec *alt_path) 1638 { 1639 primary_path->dgid = 1640 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg); 1641 primary_path->sgid = 1642 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg); 1643 primary_path->flow_label = 1644 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg)); 1645 primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg); 1646 primary_path->traffic_class = 1647 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg); 1648 primary_path->reversible = 1; 1649 primary_path->pkey = 1650 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); 1651 primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg); 1652 primary_path->mtu_selector = IB_SA_EQ; 1653 primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); 1654 primary_path->rate_selector = IB_SA_EQ; 1655 primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg); 1656 primary_path->packet_life_time_selector = IB_SA_EQ; 1657 primary_path->packet_life_time = 1658 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg); 1659 primary_path->packet_life_time -= (primary_path->packet_life_time > 0); 1660 primary_path->service_id = 1661 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); 1662 if (sa_path_is_roce(primary_path)) 1663 primary_path->roce.route_resolved = false; 1664 1665 if (cm_req_has_alt_path(req_msg)) { 1666 alt_path->dgid = *IBA_GET_MEM_PTR( 1667 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg); 1668 alt_path->sgid = *IBA_GET_MEM_PTR( 1669 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg); 1670 alt_path->flow_label = cpu_to_be32( 1671 IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg)); 1672 alt_path->hop_limit = 1673 IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg); 1674 alt_path->traffic_class = 1675 IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg); 1676 alt_path->reversible = 1; 1677 alt_path->pkey = 1678 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); 1679 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg); 1680 alt_path->mtu_selector = IB_SA_EQ; 1681 alt_path->mtu = 1682 IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); 1683 alt_path->rate_selector = IB_SA_EQ; 1684 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg); 1685 alt_path->packet_life_time_selector = IB_SA_EQ; 1686 alt_path->packet_life_time = 1687 IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg); 1688 alt_path->packet_life_time -= (alt_path->packet_life_time > 0); 1689 alt_path->service_id = 1690 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); 1691 1692 if (sa_path_is_roce(alt_path)) 1693 alt_path->roce.route_resolved = false; 1694 } 1695 cm_format_path_lid_from_req(req_msg, primary_path, alt_path); 1696 } 1697 1698 static u16 cm_get_bth_pkey(struct cm_work *work) 1699 { 1700 struct ib_device *ib_dev = work->port->cm_dev->ib_device; 1701 u8 port_num = work->port->port_num; 1702 u16 pkey_index = work->mad_recv_wc->wc->pkey_index; 1703 u16 pkey; 1704 int ret; 1705 1706 ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey); 1707 if (ret) { 1708 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n", 1709 port_num, pkey_index, ret); 1710 return 0; 1711 } 1712 1713 return pkey; 1714 } 1715 1716 /** 1717 * Convert OPA SGID to IB SGID 1718 * ULPs (such as IPoIB) do not understand OPA GIDs and will 1719 * reject them as the local_gid will not match the sgid. Therefore, 1720 * change the pathrec's SGID to an IB SGID. 1721 * 1722 * @work: Work completion 1723 * @path: Path record 1724 */ 1725 static void cm_opa_to_ib_sgid(struct cm_work *work, 1726 struct sa_path_rec *path) 1727 { 1728 struct ib_device *dev = work->port->cm_dev->ib_device; 1729 u8 port_num = work->port->port_num; 1730 1731 if (rdma_cap_opa_ah(dev, port_num) && 1732 (ib_is_opa_gid(&path->sgid))) { 1733 union ib_gid sgid; 1734 1735 if (rdma_query_gid(dev, port_num, 0, &sgid)) { 1736 dev_warn(&dev->dev, 1737 "Error updating sgid in CM request\n"); 1738 return; 1739 } 1740 1741 path->sgid = sgid; 1742 } 1743 } 1744 1745 static void cm_format_req_event(struct cm_work *work, 1746 struct cm_id_private *cm_id_priv, 1747 struct ib_cm_id *listen_id) 1748 { 1749 struct cm_req_msg *req_msg; 1750 struct ib_cm_req_event_param *param; 1751 1752 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; 1753 param = &work->cm_event.param.req_rcvd; 1754 param->listen_id = listen_id; 1755 param->bth_pkey = cm_get_bth_pkey(work); 1756 param->port = cm_id_priv->av.port->port_num; 1757 param->primary_path = &work->path[0]; 1758 cm_opa_to_ib_sgid(work, param->primary_path); 1759 if (cm_req_has_alt_path(req_msg)) { 1760 param->alternate_path = &work->path[1]; 1761 cm_opa_to_ib_sgid(work, param->alternate_path); 1762 } else { 1763 param->alternate_path = NULL; 1764 } 1765 param->remote_ca_guid = 1766 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); 1767 param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg); 1768 param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg); 1769 param->qp_type = cm_req_get_qp_type(req_msg); 1770 param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg); 1771 param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); 1772 param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); 1773 param->local_cm_response_timeout = 1774 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg); 1775 param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg); 1776 param->remote_cm_response_timeout = 1777 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg); 1778 param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); 1779 param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); 1780 param->srq = IBA_GET(CM_REQ_SRQ, req_msg); 1781 param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; 1782 work->cm_event.private_data = 1783 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg); 1784 } 1785 1786 static void cm_process_work(struct cm_id_private *cm_id_priv, 1787 struct cm_work *work) 1788 { 1789 int ret; 1790 1791 /* We will typically only have the current event to report. */ 1792 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event); 1793 cm_free_work(work); 1794 1795 while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) { 1796 spin_lock_irq(&cm_id_priv->lock); 1797 work = cm_dequeue_work(cm_id_priv); 1798 spin_unlock_irq(&cm_id_priv->lock); 1799 if (!work) 1800 return; 1801 1802 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, 1803 &work->cm_event); 1804 cm_free_work(work); 1805 } 1806 cm_deref_id(cm_id_priv); 1807 if (ret) 1808 cm_destroy_id(&cm_id_priv->id, ret); 1809 } 1810 1811 static void cm_format_mra(struct cm_mra_msg *mra_msg, 1812 struct cm_id_private *cm_id_priv, 1813 enum cm_msg_response msg_mraed, u8 service_timeout, 1814 const void *private_data, u8 private_data_len) 1815 { 1816 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); 1817 IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed); 1818 IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg, 1819 be32_to_cpu(cm_id_priv->id.local_id)); 1820 IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg, 1821 be32_to_cpu(cm_id_priv->id.remote_id)); 1822 IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); 1823 1824 if (private_data && private_data_len) 1825 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data, 1826 private_data_len); 1827 } 1828 1829 static void cm_format_rej(struct cm_rej_msg *rej_msg, 1830 struct cm_id_private *cm_id_priv, 1831 enum ib_cm_rej_reason reason, 1832 void *ari, 1833 u8 ari_length, 1834 const void *private_data, 1835 u8 private_data_len) 1836 { 1837 lockdep_assert_held(&cm_id_priv->lock); 1838 1839 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid); 1840 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, 1841 be32_to_cpu(cm_id_priv->id.remote_id)); 1842 1843 switch(cm_id_priv->id.state) { 1844 case IB_CM_REQ_RCVD: 1845 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0)); 1846 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); 1847 break; 1848 case IB_CM_MRA_REQ_SENT: 1849 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1850 be32_to_cpu(cm_id_priv->id.local_id)); 1851 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); 1852 break; 1853 case IB_CM_REP_RCVD: 1854 case IB_CM_MRA_REP_SENT: 1855 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1856 be32_to_cpu(cm_id_priv->id.local_id)); 1857 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP); 1858 break; 1859 default: 1860 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1861 be32_to_cpu(cm_id_priv->id.local_id)); 1862 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, 1863 CM_MSG_RESPONSE_OTHER); 1864 break; 1865 } 1866 1867 IBA_SET(CM_REJ_REASON, rej_msg, reason); 1868 if (ari && ari_length) { 1869 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); 1870 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); 1871 } 1872 1873 if (private_data && private_data_len) 1874 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data, 1875 private_data_len); 1876 } 1877 1878 static void cm_dup_req_handler(struct cm_work *work, 1879 struct cm_id_private *cm_id_priv) 1880 { 1881 struct ib_mad_send_buf *msg = NULL; 1882 int ret; 1883 1884 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 1885 counter[CM_REQ_COUNTER]); 1886 1887 /* Quick state check to discard duplicate REQs. */ 1888 spin_lock_irq(&cm_id_priv->lock); 1889 if (cm_id_priv->id.state == IB_CM_REQ_RCVD) { 1890 spin_unlock_irq(&cm_id_priv->lock); 1891 return; 1892 } 1893 spin_unlock_irq(&cm_id_priv->lock); 1894 1895 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg); 1896 if (ret) 1897 return; 1898 1899 spin_lock_irq(&cm_id_priv->lock); 1900 switch (cm_id_priv->id.state) { 1901 case IB_CM_MRA_REQ_SENT: 1902 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 1903 CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout, 1904 cm_id_priv->private_data, 1905 cm_id_priv->private_data_len); 1906 break; 1907 case IB_CM_TIMEWAIT: 1908 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv, 1909 IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0); 1910 break; 1911 default: 1912 goto unlock; 1913 } 1914 spin_unlock_irq(&cm_id_priv->lock); 1915 1916 ret = ib_post_send_mad(msg, NULL); 1917 if (ret) 1918 goto free; 1919 return; 1920 1921 unlock: spin_unlock_irq(&cm_id_priv->lock); 1922 free: cm_free_msg(msg); 1923 } 1924 1925 static struct cm_id_private * cm_match_req(struct cm_work *work, 1926 struct cm_id_private *cm_id_priv) 1927 { 1928 struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv; 1929 struct cm_timewait_info *timewait_info; 1930 struct cm_req_msg *req_msg; 1931 struct ib_cm_id *cm_id; 1932 1933 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; 1934 1935 /* Check for possible duplicate REQ. */ 1936 spin_lock_irq(&cm.lock); 1937 timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info); 1938 if (timewait_info) { 1939 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 1940 timewait_info->work.remote_id); 1941 spin_unlock_irq(&cm.lock); 1942 if (cur_cm_id_priv) { 1943 cm_dup_req_handler(work, cur_cm_id_priv); 1944 cm_deref_id(cur_cm_id_priv); 1945 } 1946 return NULL; 1947 } 1948 1949 /* Check for stale connections. */ 1950 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); 1951 if (timewait_info) { 1952 cm_cleanup_timewait(cm_id_priv->timewait_info); 1953 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 1954 timewait_info->work.remote_id); 1955 1956 spin_unlock_irq(&cm.lock); 1957 cm_issue_rej(work->port, work->mad_recv_wc, 1958 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ, 1959 NULL, 0); 1960 if (cur_cm_id_priv) { 1961 cm_id = &cur_cm_id_priv->id; 1962 ib_send_cm_dreq(cm_id, NULL, 0); 1963 cm_deref_id(cur_cm_id_priv); 1964 } 1965 return NULL; 1966 } 1967 1968 /* Find matching listen request. */ 1969 listen_cm_id_priv = cm_find_listen( 1970 cm_id_priv->id.device, 1971 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg))); 1972 if (!listen_cm_id_priv) { 1973 cm_cleanup_timewait(cm_id_priv->timewait_info); 1974 spin_unlock_irq(&cm.lock); 1975 cm_issue_rej(work->port, work->mad_recv_wc, 1976 IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ, 1977 NULL, 0); 1978 return NULL; 1979 } 1980 refcount_inc(&listen_cm_id_priv->refcount); 1981 spin_unlock_irq(&cm.lock); 1982 return listen_cm_id_priv; 1983 } 1984 1985 /* 1986 * Work-around for inter-subnet connections. If the LIDs are permissive, 1987 * we need to override the LID/SL data in the REQ with the LID information 1988 * in the work completion. 1989 */ 1990 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) 1991 { 1992 if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) { 1993 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, 1994 req_msg)) == IB_LID_PERMISSIVE) { 1995 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, 1996 be16_to_cpu(ib_lid_be16(wc->slid))); 1997 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl); 1998 } 1999 2000 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, 2001 req_msg)) == IB_LID_PERMISSIVE) 2002 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, 2003 wc->dlid_path_bits); 2004 } 2005 2006 if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) { 2007 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, 2008 req_msg)) == IB_LID_PERMISSIVE) { 2009 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, 2010 be16_to_cpu(ib_lid_be16(wc->slid))); 2011 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl); 2012 } 2013 2014 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, 2015 req_msg)) == IB_LID_PERMISSIVE) 2016 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, 2017 wc->dlid_path_bits); 2018 } 2019 } 2020 2021 static int cm_req_handler(struct cm_work *work) 2022 { 2023 struct cm_id_private *cm_id_priv, *listen_cm_id_priv; 2024 struct cm_req_msg *req_msg; 2025 const struct ib_global_route *grh; 2026 const struct ib_gid_attr *gid_attr; 2027 int ret; 2028 2029 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; 2030 2031 cm_id_priv = 2032 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL); 2033 if (IS_ERR(cm_id_priv)) 2034 return PTR_ERR(cm_id_priv); 2035 2036 cm_id_priv->id.remote_id = 2037 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); 2038 cm_id_priv->id.service_id = 2039 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); 2040 cm_id_priv->id.service_mask = ~cpu_to_be64(0); 2041 cm_id_priv->tid = req_msg->hdr.tid; 2042 cm_id_priv->timeout_ms = cm_convert_to_ms( 2043 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg)); 2044 cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg); 2045 cm_id_priv->remote_qpn = 2046 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); 2047 cm_id_priv->initiator_depth = 2048 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); 2049 cm_id_priv->responder_resources = 2050 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); 2051 cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); 2052 cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); 2053 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); 2054 cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); 2055 cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); 2056 cm_id_priv->qp_type = cm_req_get_qp_type(req_msg); 2057 2058 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, 2059 work->mad_recv_wc->recv_buf.grh, 2060 &cm_id_priv->av); 2061 if (ret) 2062 goto destroy; 2063 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> 2064 id.local_id); 2065 if (IS_ERR(cm_id_priv->timewait_info)) { 2066 ret = PTR_ERR(cm_id_priv->timewait_info); 2067 goto destroy; 2068 } 2069 cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id; 2070 cm_id_priv->timewait_info->remote_ca_guid = 2071 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); 2072 cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn; 2073 2074 /* 2075 * Note that the ID pointer is not in the xarray at this point, 2076 * so this set is only visible to the local thread. 2077 */ 2078 cm_id_priv->id.state = IB_CM_REQ_RCVD; 2079 2080 listen_cm_id_priv = cm_match_req(work, cm_id_priv); 2081 if (!listen_cm_id_priv) { 2082 pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__, 2083 be32_to_cpu(cm_id_priv->id.local_id)); 2084 cm_id_priv->id.state = IB_CM_IDLE; 2085 ret = -EINVAL; 2086 goto destroy; 2087 } 2088 2089 cm_process_routed_req(req_msg, work->mad_recv_wc->wc); 2090 2091 memset(&work->path[0], 0, sizeof(work->path[0])); 2092 if (cm_req_has_alt_path(req_msg)) 2093 memset(&work->path[1], 0, sizeof(work->path[1])); 2094 grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr); 2095 gid_attr = grh->sgid_attr; 2096 2097 if (gid_attr && 2098 rdma_protocol_roce(work->port->cm_dev->ib_device, 2099 work->port->port_num)) { 2100 work->path[0].rec_type = 2101 sa_conv_gid_to_pathrec_type(gid_attr->gid_type); 2102 } else { 2103 cm_path_set_rec_type( 2104 work->port->cm_dev->ib_device, work->port->port_num, 2105 &work->path[0], 2106 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, 2107 req_msg)); 2108 } 2109 if (cm_req_has_alt_path(req_msg)) 2110 work->path[1].rec_type = work->path[0].rec_type; 2111 cm_format_paths_from_req(req_msg, &work->path[0], 2112 &work->path[1]); 2113 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) 2114 sa_path_set_dmac(&work->path[0], 2115 cm_id_priv->av.ah_attr.roce.dmac); 2116 work->path[0].hop_limit = grh->hop_limit; 2117 ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av, 2118 cm_id_priv); 2119 if (ret) { 2120 int err; 2121 2122 err = rdma_query_gid(work->port->cm_dev->ib_device, 2123 work->port->port_num, 0, 2124 &work->path[0].sgid); 2125 if (err) 2126 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID, 2127 NULL, 0, NULL, 0); 2128 else 2129 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID, 2130 &work->path[0].sgid, 2131 sizeof(work->path[0].sgid), 2132 NULL, 0); 2133 goto rejected; 2134 } 2135 if (cm_req_has_alt_path(req_msg)) { 2136 ret = cm_init_av_by_path(&work->path[1], NULL, 2137 &cm_id_priv->alt_av, cm_id_priv); 2138 if (ret) { 2139 ib_send_cm_rej(&cm_id_priv->id, 2140 IB_CM_REJ_INVALID_ALT_GID, 2141 &work->path[0].sgid, 2142 sizeof(work->path[0].sgid), NULL, 0); 2143 goto rejected; 2144 } 2145 } 2146 2147 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; 2148 cm_id_priv->id.context = listen_cm_id_priv->id.context; 2149 cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id); 2150 2151 /* Now MAD handlers can see the new ID */ 2152 spin_lock_irq(&cm_id_priv->lock); 2153 cm_finalize_id(cm_id_priv); 2154 2155 /* Refcount belongs to the event, pairs with cm_process_work() */ 2156 refcount_inc(&cm_id_priv->refcount); 2157 atomic_inc(&cm_id_priv->work_count); 2158 spin_unlock_irq(&cm_id_priv->lock); 2159 cm_process_work(cm_id_priv, work); 2160 /* 2161 * Since this ID was just created and was not made visible to other MAD 2162 * handlers until the cm_finalize_id() above we know that the 2163 * cm_process_work() will deliver the event and the listen_cm_id 2164 * embedded in the event can be derefed here. 2165 */ 2166 cm_deref_id(listen_cm_id_priv); 2167 return 0; 2168 2169 rejected: 2170 cm_deref_id(listen_cm_id_priv); 2171 destroy: 2172 ib_destroy_cm_id(&cm_id_priv->id); 2173 return ret; 2174 } 2175 2176 static void cm_format_rep(struct cm_rep_msg *rep_msg, 2177 struct cm_id_private *cm_id_priv, 2178 struct ib_cm_rep_param *param) 2179 { 2180 cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid); 2181 IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg, 2182 be32_to_cpu(cm_id_priv->id.local_id)); 2183 IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg, 2184 be32_to_cpu(cm_id_priv->id.remote_id)); 2185 IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn); 2186 IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg, 2187 param->responder_resources); 2188 IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg, 2189 cm_id_priv->av.port->cm_dev->ack_delay); 2190 IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted); 2191 IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count); 2192 IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg, 2193 be64_to_cpu(cm_id_priv->id.device->node_guid)); 2194 2195 if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) { 2196 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg, 2197 param->initiator_depth); 2198 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg, 2199 param->flow_control); 2200 IBA_SET(CM_REP_SRQ, rep_msg, param->srq); 2201 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num); 2202 } else { 2203 IBA_SET(CM_REP_SRQ, rep_msg, 1); 2204 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num); 2205 } 2206 2207 if (param->private_data && param->private_data_len) 2208 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data, 2209 param->private_data_len); 2210 } 2211 2212 int ib_send_cm_rep(struct ib_cm_id *cm_id, 2213 struct ib_cm_rep_param *param) 2214 { 2215 struct cm_id_private *cm_id_priv; 2216 struct ib_mad_send_buf *msg; 2217 struct cm_rep_msg *rep_msg; 2218 unsigned long flags; 2219 int ret; 2220 2221 if (param->private_data && 2222 param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE) 2223 return -EINVAL; 2224 2225 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 2226 spin_lock_irqsave(&cm_id_priv->lock, flags); 2227 if (cm_id->state != IB_CM_REQ_RCVD && 2228 cm_id->state != IB_CM_MRA_REQ_SENT) { 2229 pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__, 2230 be32_to_cpu(cm_id_priv->id.local_id), cm_id->state); 2231 ret = -EINVAL; 2232 goto out; 2233 } 2234 2235 ret = cm_alloc_msg(cm_id_priv, &msg); 2236 if (ret) 2237 goto out; 2238 2239 rep_msg = (struct cm_rep_msg *) msg->mad; 2240 cm_format_rep(rep_msg, cm_id_priv, param); 2241 msg->timeout_ms = cm_id_priv->timeout_ms; 2242 msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT; 2243 2244 ret = ib_post_send_mad(msg, NULL); 2245 if (ret) { 2246 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2247 cm_free_msg(msg); 2248 return ret; 2249 } 2250 2251 cm_id->state = IB_CM_REP_SENT; 2252 cm_id_priv->msg = msg; 2253 cm_id_priv->initiator_depth = param->initiator_depth; 2254 cm_id_priv->responder_resources = param->responder_resources; 2255 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); 2256 WARN_ONCE(param->qp_num & 0xFF000000, 2257 "IBTA declares QPN to be 24 bits, but it is 0x%X\n", 2258 param->qp_num); 2259 cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF); 2260 2261 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2262 return ret; 2263 } 2264 EXPORT_SYMBOL(ib_send_cm_rep); 2265 2266 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, 2267 struct cm_id_private *cm_id_priv, 2268 const void *private_data, 2269 u8 private_data_len) 2270 { 2271 cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid); 2272 IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg, 2273 be32_to_cpu(cm_id_priv->id.local_id)); 2274 IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg, 2275 be32_to_cpu(cm_id_priv->id.remote_id)); 2276 2277 if (private_data && private_data_len) 2278 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data, 2279 private_data_len); 2280 } 2281 2282 int ib_send_cm_rtu(struct ib_cm_id *cm_id, 2283 const void *private_data, 2284 u8 private_data_len) 2285 { 2286 struct cm_id_private *cm_id_priv; 2287 struct ib_mad_send_buf *msg; 2288 unsigned long flags; 2289 void *data; 2290 int ret; 2291 2292 if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE) 2293 return -EINVAL; 2294 2295 data = cm_copy_private_data(private_data, private_data_len); 2296 if (IS_ERR(data)) 2297 return PTR_ERR(data); 2298 2299 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 2300 spin_lock_irqsave(&cm_id_priv->lock, flags); 2301 if (cm_id->state != IB_CM_REP_RCVD && 2302 cm_id->state != IB_CM_MRA_REP_SENT) { 2303 pr_debug("%s: local_id %d, cm_id->state %d\n", __func__, 2304 be32_to_cpu(cm_id->local_id), cm_id->state); 2305 ret = -EINVAL; 2306 goto error; 2307 } 2308 2309 ret = cm_alloc_msg(cm_id_priv, &msg); 2310 if (ret) 2311 goto error; 2312 2313 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, 2314 private_data, private_data_len); 2315 2316 ret = ib_post_send_mad(msg, NULL); 2317 if (ret) { 2318 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2319 cm_free_msg(msg); 2320 kfree(data); 2321 return ret; 2322 } 2323 2324 cm_id->state = IB_CM_ESTABLISHED; 2325 cm_set_private_data(cm_id_priv, data, private_data_len); 2326 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2327 return 0; 2328 2329 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2330 kfree(data); 2331 return ret; 2332 } 2333 EXPORT_SYMBOL(ib_send_cm_rtu); 2334 2335 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) 2336 { 2337 struct cm_rep_msg *rep_msg; 2338 struct ib_cm_rep_event_param *param; 2339 2340 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; 2341 param = &work->cm_event.param.rep_rcvd; 2342 param->remote_ca_guid = 2343 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); 2344 param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg); 2345 param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type)); 2346 param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg); 2347 param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); 2348 param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); 2349 param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); 2350 param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg); 2351 param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg); 2352 param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); 2353 param->srq = IBA_GET(CM_REP_SRQ, rep_msg); 2354 work->cm_event.private_data = 2355 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg); 2356 } 2357 2358 static void cm_dup_rep_handler(struct cm_work *work) 2359 { 2360 struct cm_id_private *cm_id_priv; 2361 struct cm_rep_msg *rep_msg; 2362 struct ib_mad_send_buf *msg = NULL; 2363 int ret; 2364 2365 rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad; 2366 cm_id_priv = cm_acquire_id( 2367 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 2368 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg))); 2369 if (!cm_id_priv) 2370 return; 2371 2372 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2373 counter[CM_REP_COUNTER]); 2374 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg); 2375 if (ret) 2376 goto deref; 2377 2378 spin_lock_irq(&cm_id_priv->lock); 2379 if (cm_id_priv->id.state == IB_CM_ESTABLISHED) 2380 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, 2381 cm_id_priv->private_data, 2382 cm_id_priv->private_data_len); 2383 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT) 2384 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 2385 CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout, 2386 cm_id_priv->private_data, 2387 cm_id_priv->private_data_len); 2388 else 2389 goto unlock; 2390 spin_unlock_irq(&cm_id_priv->lock); 2391 2392 ret = ib_post_send_mad(msg, NULL); 2393 if (ret) 2394 goto free; 2395 goto deref; 2396 2397 unlock: spin_unlock_irq(&cm_id_priv->lock); 2398 free: cm_free_msg(msg); 2399 deref: cm_deref_id(cm_id_priv); 2400 } 2401 2402 static int cm_rep_handler(struct cm_work *work) 2403 { 2404 struct cm_id_private *cm_id_priv; 2405 struct cm_rep_msg *rep_msg; 2406 int ret; 2407 struct cm_id_private *cur_cm_id_priv; 2408 struct ib_cm_id *cm_id; 2409 struct cm_timewait_info *timewait_info; 2410 2411 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; 2412 cm_id_priv = cm_acquire_id( 2413 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0); 2414 if (!cm_id_priv) { 2415 cm_dup_rep_handler(work); 2416 pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__, 2417 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2418 return -EINVAL; 2419 } 2420 2421 cm_format_rep_event(work, cm_id_priv->qp_type); 2422 2423 spin_lock_irq(&cm_id_priv->lock); 2424 switch (cm_id_priv->id.state) { 2425 case IB_CM_REQ_SENT: 2426 case IB_CM_MRA_REQ_RCVD: 2427 break; 2428 default: 2429 ret = -EINVAL; 2430 pr_debug( 2431 "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", 2432 __func__, cm_id_priv->id.state, 2433 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), 2434 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2435 spin_unlock_irq(&cm_id_priv->lock); 2436 goto error; 2437 } 2438 2439 cm_id_priv->timewait_info->work.remote_id = 2440 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); 2441 cm_id_priv->timewait_info->remote_ca_guid = 2442 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); 2443 cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); 2444 2445 spin_lock(&cm.lock); 2446 /* Check for duplicate REP. */ 2447 if (cm_insert_remote_id(cm_id_priv->timewait_info)) { 2448 spin_unlock(&cm.lock); 2449 spin_unlock_irq(&cm_id_priv->lock); 2450 ret = -EINVAL; 2451 pr_debug("%s: Failed to insert remote id %d\n", __func__, 2452 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2453 goto error; 2454 } 2455 /* Check for a stale connection. */ 2456 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); 2457 if (timewait_info) { 2458 rb_erase(&cm_id_priv->timewait_info->remote_id_node, 2459 &cm.remote_id_table); 2460 cm_id_priv->timewait_info->inserted_remote_id = 0; 2461 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 2462 timewait_info->work.remote_id); 2463 2464 spin_unlock(&cm.lock); 2465 spin_unlock_irq(&cm_id_priv->lock); 2466 cm_issue_rej(work->port, work->mad_recv_wc, 2467 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, 2468 NULL, 0); 2469 ret = -EINVAL; 2470 pr_debug( 2471 "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", 2472 __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), 2473 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2474 2475 if (cur_cm_id_priv) { 2476 cm_id = &cur_cm_id_priv->id; 2477 ib_send_cm_dreq(cm_id, NULL, 0); 2478 cm_deref_id(cur_cm_id_priv); 2479 } 2480 2481 goto error; 2482 } 2483 spin_unlock(&cm.lock); 2484 2485 cm_id_priv->id.state = IB_CM_REP_RCVD; 2486 cm_id_priv->id.remote_id = 2487 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); 2488 cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); 2489 cm_id_priv->initiator_depth = 2490 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); 2491 cm_id_priv->responder_resources = 2492 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); 2493 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); 2494 cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); 2495 cm_id_priv->target_ack_delay = 2496 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); 2497 cm_id_priv->av.timeout = 2498 cm_ack_timeout(cm_id_priv->target_ack_delay, 2499 cm_id_priv->av.timeout - 1); 2500 cm_id_priv->alt_av.timeout = 2501 cm_ack_timeout(cm_id_priv->target_ack_delay, 2502 cm_id_priv->alt_av.timeout - 1); 2503 2504 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2505 ret = atomic_inc_and_test(&cm_id_priv->work_count); 2506 if (!ret) 2507 list_add_tail(&work->list, &cm_id_priv->work_list); 2508 spin_unlock_irq(&cm_id_priv->lock); 2509 2510 if (ret) 2511 cm_process_work(cm_id_priv, work); 2512 else 2513 cm_deref_id(cm_id_priv); 2514 return 0; 2515 2516 error: 2517 cm_deref_id(cm_id_priv); 2518 return ret; 2519 } 2520 2521 static int cm_establish_handler(struct cm_work *work) 2522 { 2523 struct cm_id_private *cm_id_priv; 2524 int ret; 2525 2526 /* See comment in cm_establish about lookup. */ 2527 cm_id_priv = cm_acquire_id(work->local_id, work->remote_id); 2528 if (!cm_id_priv) 2529 return -EINVAL; 2530 2531 spin_lock_irq(&cm_id_priv->lock); 2532 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) { 2533 spin_unlock_irq(&cm_id_priv->lock); 2534 goto out; 2535 } 2536 2537 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2538 ret = atomic_inc_and_test(&cm_id_priv->work_count); 2539 if (!ret) 2540 list_add_tail(&work->list, &cm_id_priv->work_list); 2541 spin_unlock_irq(&cm_id_priv->lock); 2542 2543 if (ret) 2544 cm_process_work(cm_id_priv, work); 2545 else 2546 cm_deref_id(cm_id_priv); 2547 return 0; 2548 out: 2549 cm_deref_id(cm_id_priv); 2550 return -EINVAL; 2551 } 2552 2553 static int cm_rtu_handler(struct cm_work *work) 2554 { 2555 struct cm_id_private *cm_id_priv; 2556 struct cm_rtu_msg *rtu_msg; 2557 int ret; 2558 2559 rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad; 2560 cm_id_priv = cm_acquire_id( 2561 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)), 2562 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg))); 2563 if (!cm_id_priv) 2564 return -EINVAL; 2565 2566 work->cm_event.private_data = 2567 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg); 2568 2569 spin_lock_irq(&cm_id_priv->lock); 2570 if (cm_id_priv->id.state != IB_CM_REP_SENT && 2571 cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) { 2572 spin_unlock_irq(&cm_id_priv->lock); 2573 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2574 counter[CM_RTU_COUNTER]); 2575 goto out; 2576 } 2577 cm_id_priv->id.state = IB_CM_ESTABLISHED; 2578 2579 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2580 ret = atomic_inc_and_test(&cm_id_priv->work_count); 2581 if (!ret) 2582 list_add_tail(&work->list, &cm_id_priv->work_list); 2583 spin_unlock_irq(&cm_id_priv->lock); 2584 2585 if (ret) 2586 cm_process_work(cm_id_priv, work); 2587 else 2588 cm_deref_id(cm_id_priv); 2589 return 0; 2590 out: 2591 cm_deref_id(cm_id_priv); 2592 return -EINVAL; 2593 } 2594 2595 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, 2596 struct cm_id_private *cm_id_priv, 2597 const void *private_data, 2598 u8 private_data_len) 2599 { 2600 cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID, 2601 cm_form_tid(cm_id_priv)); 2602 IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg, 2603 be32_to_cpu(cm_id_priv->id.local_id)); 2604 IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg, 2605 be32_to_cpu(cm_id_priv->id.remote_id)); 2606 IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg, 2607 be32_to_cpu(cm_id_priv->remote_qpn)); 2608 2609 if (private_data && private_data_len) 2610 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data, 2611 private_data_len); 2612 } 2613 2614 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, 2615 const void *private_data, u8 private_data_len) 2616 { 2617 struct ib_mad_send_buf *msg; 2618 int ret; 2619 2620 lockdep_assert_held(&cm_id_priv->lock); 2621 2622 if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE) 2623 return -EINVAL; 2624 2625 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) { 2626 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, 2627 be32_to_cpu(cm_id_priv->id.local_id), 2628 cm_id_priv->id.state); 2629 return -EINVAL; 2630 } 2631 2632 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT || 2633 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) 2634 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2635 2636 ret = cm_alloc_msg(cm_id_priv, &msg); 2637 if (ret) { 2638 cm_enter_timewait(cm_id_priv); 2639 return ret; 2640 } 2641 2642 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv, 2643 private_data, private_data_len); 2644 msg->timeout_ms = cm_id_priv->timeout_ms; 2645 msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT; 2646 2647 ret = ib_post_send_mad(msg, NULL); 2648 if (ret) { 2649 cm_enter_timewait(cm_id_priv); 2650 cm_free_msg(msg); 2651 return ret; 2652 } 2653 2654 cm_id_priv->id.state = IB_CM_DREQ_SENT; 2655 cm_id_priv->msg = msg; 2656 return 0; 2657 } 2658 2659 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data, 2660 u8 private_data_len) 2661 { 2662 struct cm_id_private *cm_id_priv = 2663 container_of(cm_id, struct cm_id_private, id); 2664 unsigned long flags; 2665 int ret; 2666 2667 spin_lock_irqsave(&cm_id_priv->lock, flags); 2668 ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len); 2669 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2670 return ret; 2671 } 2672 EXPORT_SYMBOL(ib_send_cm_dreq); 2673 2674 static void cm_format_drep(struct cm_drep_msg *drep_msg, 2675 struct cm_id_private *cm_id_priv, 2676 const void *private_data, 2677 u8 private_data_len) 2678 { 2679 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid); 2680 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, 2681 be32_to_cpu(cm_id_priv->id.local_id)); 2682 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, 2683 be32_to_cpu(cm_id_priv->id.remote_id)); 2684 2685 if (private_data && private_data_len) 2686 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data, 2687 private_data_len); 2688 } 2689 2690 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, 2691 void *private_data, u8 private_data_len) 2692 { 2693 struct ib_mad_send_buf *msg; 2694 int ret; 2695 2696 lockdep_assert_held(&cm_id_priv->lock); 2697 2698 if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE) 2699 return -EINVAL; 2700 2701 if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) { 2702 pr_debug( 2703 "%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n", 2704 __func__, be32_to_cpu(cm_id_priv->id.local_id), 2705 cm_id_priv->id.state); 2706 kfree(private_data); 2707 return -EINVAL; 2708 } 2709 2710 cm_set_private_data(cm_id_priv, private_data, private_data_len); 2711 cm_enter_timewait(cm_id_priv); 2712 2713 ret = cm_alloc_msg(cm_id_priv, &msg); 2714 if (ret) 2715 return ret; 2716 2717 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, 2718 private_data, private_data_len); 2719 2720 ret = ib_post_send_mad(msg, NULL); 2721 if (ret) { 2722 cm_free_msg(msg); 2723 return ret; 2724 } 2725 return 0; 2726 } 2727 2728 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data, 2729 u8 private_data_len) 2730 { 2731 struct cm_id_private *cm_id_priv = 2732 container_of(cm_id, struct cm_id_private, id); 2733 unsigned long flags; 2734 void *data; 2735 int ret; 2736 2737 data = cm_copy_private_data(private_data, private_data_len); 2738 if (IS_ERR(data)) 2739 return PTR_ERR(data); 2740 2741 spin_lock_irqsave(&cm_id_priv->lock, flags); 2742 ret = cm_send_drep_locked(cm_id_priv, data, private_data_len); 2743 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2744 return ret; 2745 } 2746 EXPORT_SYMBOL(ib_send_cm_drep); 2747 2748 static int cm_issue_drep(struct cm_port *port, 2749 struct ib_mad_recv_wc *mad_recv_wc) 2750 { 2751 struct ib_mad_send_buf *msg = NULL; 2752 struct cm_dreq_msg *dreq_msg; 2753 struct cm_drep_msg *drep_msg; 2754 int ret; 2755 2756 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg); 2757 if (ret) 2758 return ret; 2759 2760 dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad; 2761 drep_msg = (struct cm_drep_msg *) msg->mad; 2762 2763 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid); 2764 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, 2765 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)); 2766 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, 2767 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); 2768 2769 ret = ib_post_send_mad(msg, NULL); 2770 if (ret) 2771 cm_free_msg(msg); 2772 2773 return ret; 2774 } 2775 2776 static int cm_dreq_handler(struct cm_work *work) 2777 { 2778 struct cm_id_private *cm_id_priv; 2779 struct cm_dreq_msg *dreq_msg; 2780 struct ib_mad_send_buf *msg = NULL; 2781 int ret; 2782 2783 dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad; 2784 cm_id_priv = cm_acquire_id( 2785 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)), 2786 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg))); 2787 if (!cm_id_priv) { 2788 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2789 counter[CM_DREQ_COUNTER]); 2790 cm_issue_drep(work->port, work->mad_recv_wc); 2791 pr_debug( 2792 "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", 2793 __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), 2794 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); 2795 return -EINVAL; 2796 } 2797 2798 work->cm_event.private_data = 2799 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg); 2800 2801 spin_lock_irq(&cm_id_priv->lock); 2802 if (cm_id_priv->local_qpn != 2803 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg))) 2804 goto unlock; 2805 2806 switch (cm_id_priv->id.state) { 2807 case IB_CM_REP_SENT: 2808 case IB_CM_DREQ_SENT: 2809 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2810 break; 2811 case IB_CM_ESTABLISHED: 2812 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT || 2813 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) 2814 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2815 break; 2816 case IB_CM_MRA_REP_RCVD: 2817 break; 2818 case IB_CM_TIMEWAIT: 2819 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2820 counter[CM_DREQ_COUNTER]); 2821 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc); 2822 if (IS_ERR(msg)) 2823 goto unlock; 2824 2825 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, 2826 cm_id_priv->private_data, 2827 cm_id_priv->private_data_len); 2828 spin_unlock_irq(&cm_id_priv->lock); 2829 2830 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) || 2831 ib_post_send_mad(msg, NULL)) 2832 cm_free_msg(msg); 2833 goto deref; 2834 case IB_CM_DREQ_RCVD: 2835 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2836 counter[CM_DREQ_COUNTER]); 2837 goto unlock; 2838 default: 2839 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 2840 __func__, be32_to_cpu(cm_id_priv->id.local_id), 2841 cm_id_priv->id.state); 2842 goto unlock; 2843 } 2844 cm_id_priv->id.state = IB_CM_DREQ_RCVD; 2845 cm_id_priv->tid = dreq_msg->hdr.tid; 2846 ret = atomic_inc_and_test(&cm_id_priv->work_count); 2847 if (!ret) 2848 list_add_tail(&work->list, &cm_id_priv->work_list); 2849 spin_unlock_irq(&cm_id_priv->lock); 2850 2851 if (ret) 2852 cm_process_work(cm_id_priv, work); 2853 else 2854 cm_deref_id(cm_id_priv); 2855 return 0; 2856 2857 unlock: spin_unlock_irq(&cm_id_priv->lock); 2858 deref: cm_deref_id(cm_id_priv); 2859 return -EINVAL; 2860 } 2861 2862 static int cm_drep_handler(struct cm_work *work) 2863 { 2864 struct cm_id_private *cm_id_priv; 2865 struct cm_drep_msg *drep_msg; 2866 int ret; 2867 2868 drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad; 2869 cm_id_priv = cm_acquire_id( 2870 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)), 2871 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg))); 2872 if (!cm_id_priv) 2873 return -EINVAL; 2874 2875 work->cm_event.private_data = 2876 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg); 2877 2878 spin_lock_irq(&cm_id_priv->lock); 2879 if (cm_id_priv->id.state != IB_CM_DREQ_SENT && 2880 cm_id_priv->id.state != IB_CM_DREQ_RCVD) { 2881 spin_unlock_irq(&cm_id_priv->lock); 2882 goto out; 2883 } 2884 cm_enter_timewait(cm_id_priv); 2885 2886 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2887 ret = atomic_inc_and_test(&cm_id_priv->work_count); 2888 if (!ret) 2889 list_add_tail(&work->list, &cm_id_priv->work_list); 2890 spin_unlock_irq(&cm_id_priv->lock); 2891 2892 if (ret) 2893 cm_process_work(cm_id_priv, work); 2894 else 2895 cm_deref_id(cm_id_priv); 2896 return 0; 2897 out: 2898 cm_deref_id(cm_id_priv); 2899 return -EINVAL; 2900 } 2901 2902 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, 2903 enum ib_cm_rej_reason reason, void *ari, 2904 u8 ari_length, const void *private_data, 2905 u8 private_data_len) 2906 { 2907 struct ib_mad_send_buf *msg; 2908 int ret; 2909 2910 lockdep_assert_held(&cm_id_priv->lock); 2911 2912 if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) || 2913 (ari && ari_length > IB_CM_REJ_ARI_LENGTH)) 2914 return -EINVAL; 2915 2916 switch (cm_id_priv->id.state) { 2917 case IB_CM_REQ_SENT: 2918 case IB_CM_MRA_REQ_RCVD: 2919 case IB_CM_REQ_RCVD: 2920 case IB_CM_MRA_REQ_SENT: 2921 case IB_CM_REP_RCVD: 2922 case IB_CM_MRA_REP_SENT: 2923 cm_reset_to_idle(cm_id_priv); 2924 ret = cm_alloc_msg(cm_id_priv, &msg); 2925 if (ret) 2926 return ret; 2927 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason, 2928 ari, ari_length, private_data, private_data_len); 2929 break; 2930 case IB_CM_REP_SENT: 2931 case IB_CM_MRA_REP_RCVD: 2932 cm_enter_timewait(cm_id_priv); 2933 ret = cm_alloc_msg(cm_id_priv, &msg); 2934 if (ret) 2935 return ret; 2936 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason, 2937 ari, ari_length, private_data, private_data_len); 2938 break; 2939 default: 2940 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, 2941 be32_to_cpu(cm_id_priv->id.local_id), 2942 cm_id_priv->id.state); 2943 return -EINVAL; 2944 } 2945 2946 ret = ib_post_send_mad(msg, NULL); 2947 if (ret) { 2948 cm_free_msg(msg); 2949 return ret; 2950 } 2951 2952 return 0; 2953 } 2954 2955 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason, 2956 void *ari, u8 ari_length, const void *private_data, 2957 u8 private_data_len) 2958 { 2959 struct cm_id_private *cm_id_priv = 2960 container_of(cm_id, struct cm_id_private, id); 2961 unsigned long flags; 2962 int ret; 2963 2964 spin_lock_irqsave(&cm_id_priv->lock, flags); 2965 ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length, 2966 private_data, private_data_len); 2967 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2968 return ret; 2969 } 2970 EXPORT_SYMBOL(ib_send_cm_rej); 2971 2972 static void cm_format_rej_event(struct cm_work *work) 2973 { 2974 struct cm_rej_msg *rej_msg; 2975 struct ib_cm_rej_event_param *param; 2976 2977 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; 2978 param = &work->cm_event.param.rej_rcvd; 2979 param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg); 2980 param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg); 2981 param->reason = IBA_GET(CM_REJ_REASON, rej_msg); 2982 work->cm_event.private_data = 2983 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg); 2984 } 2985 2986 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) 2987 { 2988 struct cm_timewait_info *timewait_info; 2989 struct cm_id_private *cm_id_priv; 2990 __be32 remote_id; 2991 2992 remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg)); 2993 2994 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) { 2995 spin_lock_irq(&cm.lock); 2996 timewait_info = cm_find_remote_id( 2997 *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)), 2998 remote_id); 2999 if (!timewait_info) { 3000 spin_unlock_irq(&cm.lock); 3001 return NULL; 3002 } 3003 cm_id_priv = 3004 cm_acquire_id(timewait_info->work.local_id, remote_id); 3005 spin_unlock_irq(&cm.lock); 3006 } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) == 3007 CM_MSG_RESPONSE_REQ) 3008 cm_id_priv = cm_acquire_id( 3009 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), 3010 0); 3011 else 3012 cm_id_priv = cm_acquire_id( 3013 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), 3014 remote_id); 3015 3016 return cm_id_priv; 3017 } 3018 3019 static int cm_rej_handler(struct cm_work *work) 3020 { 3021 struct cm_id_private *cm_id_priv; 3022 struct cm_rej_msg *rej_msg; 3023 int ret; 3024 3025 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; 3026 cm_id_priv = cm_acquire_rejected_id(rej_msg); 3027 if (!cm_id_priv) 3028 return -EINVAL; 3029 3030 cm_format_rej_event(work); 3031 3032 spin_lock_irq(&cm_id_priv->lock); 3033 switch (cm_id_priv->id.state) { 3034 case IB_CM_REQ_SENT: 3035 case IB_CM_MRA_REQ_RCVD: 3036 case IB_CM_REP_SENT: 3037 case IB_CM_MRA_REP_RCVD: 3038 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3039 /* fall through */ 3040 case IB_CM_REQ_RCVD: 3041 case IB_CM_MRA_REQ_SENT: 3042 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN) 3043 cm_enter_timewait(cm_id_priv); 3044 else 3045 cm_reset_to_idle(cm_id_priv); 3046 break; 3047 case IB_CM_DREQ_SENT: 3048 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3049 /* fall through */ 3050 case IB_CM_REP_RCVD: 3051 case IB_CM_MRA_REP_SENT: 3052 cm_enter_timewait(cm_id_priv); 3053 break; 3054 case IB_CM_ESTABLISHED: 3055 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT || 3056 cm_id_priv->id.lap_state == IB_CM_LAP_SENT) { 3057 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT) 3058 ib_cancel_mad(cm_id_priv->av.port->mad_agent, 3059 cm_id_priv->msg); 3060 cm_enter_timewait(cm_id_priv); 3061 break; 3062 } 3063 /* fall through */ 3064 default: 3065 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 3066 __func__, be32_to_cpu(cm_id_priv->id.local_id), 3067 cm_id_priv->id.state); 3068 spin_unlock_irq(&cm_id_priv->lock); 3069 ret = -EINVAL; 3070 goto out; 3071 } 3072 3073 ret = atomic_inc_and_test(&cm_id_priv->work_count); 3074 if (!ret) 3075 list_add_tail(&work->list, &cm_id_priv->work_list); 3076 spin_unlock_irq(&cm_id_priv->lock); 3077 3078 if (ret) 3079 cm_process_work(cm_id_priv, work); 3080 else 3081 cm_deref_id(cm_id_priv); 3082 return 0; 3083 out: 3084 cm_deref_id(cm_id_priv); 3085 return -EINVAL; 3086 } 3087 3088 int ib_send_cm_mra(struct ib_cm_id *cm_id, 3089 u8 service_timeout, 3090 const void *private_data, 3091 u8 private_data_len) 3092 { 3093 struct cm_id_private *cm_id_priv; 3094 struct ib_mad_send_buf *msg; 3095 enum ib_cm_state cm_state; 3096 enum ib_cm_lap_state lap_state; 3097 enum cm_msg_response msg_response; 3098 void *data; 3099 unsigned long flags; 3100 int ret; 3101 3102 if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE) 3103 return -EINVAL; 3104 3105 data = cm_copy_private_data(private_data, private_data_len); 3106 if (IS_ERR(data)) 3107 return PTR_ERR(data); 3108 3109 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3110 3111 spin_lock_irqsave(&cm_id_priv->lock, flags); 3112 switch(cm_id_priv->id.state) { 3113 case IB_CM_REQ_RCVD: 3114 cm_state = IB_CM_MRA_REQ_SENT; 3115 lap_state = cm_id->lap_state; 3116 msg_response = CM_MSG_RESPONSE_REQ; 3117 break; 3118 case IB_CM_REP_RCVD: 3119 cm_state = IB_CM_MRA_REP_SENT; 3120 lap_state = cm_id->lap_state; 3121 msg_response = CM_MSG_RESPONSE_REP; 3122 break; 3123 case IB_CM_ESTABLISHED: 3124 if (cm_id->lap_state == IB_CM_LAP_RCVD) { 3125 cm_state = cm_id->state; 3126 lap_state = IB_CM_MRA_LAP_SENT; 3127 msg_response = CM_MSG_RESPONSE_OTHER; 3128 break; 3129 } 3130 /* fall through */ 3131 default: 3132 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 3133 __func__, be32_to_cpu(cm_id_priv->id.local_id), 3134 cm_id_priv->id.state); 3135 ret = -EINVAL; 3136 goto error1; 3137 } 3138 3139 if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) { 3140 ret = cm_alloc_msg(cm_id_priv, &msg); 3141 if (ret) 3142 goto error1; 3143 3144 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 3145 msg_response, service_timeout, 3146 private_data, private_data_len); 3147 ret = ib_post_send_mad(msg, NULL); 3148 if (ret) 3149 goto error2; 3150 } 3151 3152 cm_id->state = cm_state; 3153 cm_id->lap_state = lap_state; 3154 cm_id_priv->service_timeout = service_timeout; 3155 cm_set_private_data(cm_id_priv, data, private_data_len); 3156 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3157 return 0; 3158 3159 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3160 kfree(data); 3161 return ret; 3162 3163 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3164 kfree(data); 3165 cm_free_msg(msg); 3166 return ret; 3167 } 3168 EXPORT_SYMBOL(ib_send_cm_mra); 3169 3170 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) 3171 { 3172 switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) { 3173 case CM_MSG_RESPONSE_REQ: 3174 return cm_acquire_id( 3175 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), 3176 0); 3177 case CM_MSG_RESPONSE_REP: 3178 case CM_MSG_RESPONSE_OTHER: 3179 return cm_acquire_id( 3180 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), 3181 cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg))); 3182 default: 3183 return NULL; 3184 } 3185 } 3186 3187 static int cm_mra_handler(struct cm_work *work) 3188 { 3189 struct cm_id_private *cm_id_priv; 3190 struct cm_mra_msg *mra_msg; 3191 int timeout, ret; 3192 3193 mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad; 3194 cm_id_priv = cm_acquire_mraed_id(mra_msg); 3195 if (!cm_id_priv) 3196 return -EINVAL; 3197 3198 work->cm_event.private_data = 3199 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg); 3200 work->cm_event.param.mra_rcvd.service_timeout = 3201 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg); 3202 timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) + 3203 cm_convert_to_ms(cm_id_priv->av.timeout); 3204 3205 spin_lock_irq(&cm_id_priv->lock); 3206 switch (cm_id_priv->id.state) { 3207 case IB_CM_REQ_SENT: 3208 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != 3209 CM_MSG_RESPONSE_REQ || 3210 ib_modify_mad(cm_id_priv->av.port->mad_agent, 3211 cm_id_priv->msg, timeout)) 3212 goto out; 3213 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD; 3214 break; 3215 case IB_CM_REP_SENT: 3216 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != 3217 CM_MSG_RESPONSE_REP || 3218 ib_modify_mad(cm_id_priv->av.port->mad_agent, 3219 cm_id_priv->msg, timeout)) 3220 goto out; 3221 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD; 3222 break; 3223 case IB_CM_ESTABLISHED: 3224 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != 3225 CM_MSG_RESPONSE_OTHER || 3226 cm_id_priv->id.lap_state != IB_CM_LAP_SENT || 3227 ib_modify_mad(cm_id_priv->av.port->mad_agent, 3228 cm_id_priv->msg, timeout)) { 3229 if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) 3230 atomic_long_inc(&work->port-> 3231 counter_group[CM_RECV_DUPLICATES]. 3232 counter[CM_MRA_COUNTER]); 3233 goto out; 3234 } 3235 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD; 3236 break; 3237 case IB_CM_MRA_REQ_RCVD: 3238 case IB_CM_MRA_REP_RCVD: 3239 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3240 counter[CM_MRA_COUNTER]); 3241 /* fall through */ 3242 default: 3243 pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n", 3244 __func__, be32_to_cpu(cm_id_priv->id.local_id), 3245 cm_id_priv->id.state); 3246 goto out; 3247 } 3248 3249 cm_id_priv->msg->context[1] = (void *) (unsigned long) 3250 cm_id_priv->id.state; 3251 ret = atomic_inc_and_test(&cm_id_priv->work_count); 3252 if (!ret) 3253 list_add_tail(&work->list, &cm_id_priv->work_list); 3254 spin_unlock_irq(&cm_id_priv->lock); 3255 3256 if (ret) 3257 cm_process_work(cm_id_priv, work); 3258 else 3259 cm_deref_id(cm_id_priv); 3260 return 0; 3261 out: 3262 spin_unlock_irq(&cm_id_priv->lock); 3263 cm_deref_id(cm_id_priv); 3264 return -EINVAL; 3265 } 3266 3267 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, 3268 struct sa_path_rec *path) 3269 { 3270 u32 lid; 3271 3272 if (path->rec_type != SA_PATH_REC_TYPE_OPA) { 3273 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID, 3274 lap_msg)); 3275 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID, 3276 lap_msg)); 3277 } else { 3278 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 3279 CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg)); 3280 sa_path_set_dlid(path, lid); 3281 3282 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 3283 CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg)); 3284 sa_path_set_slid(path, lid); 3285 } 3286 } 3287 3288 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, 3289 struct sa_path_rec *path, 3290 struct cm_lap_msg *lap_msg) 3291 { 3292 path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg); 3293 path->sgid = 3294 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg); 3295 path->flow_label = 3296 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg)); 3297 path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg); 3298 path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg); 3299 path->reversible = 1; 3300 path->pkey = cm_id_priv->pkey; 3301 path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg); 3302 path->mtu_selector = IB_SA_EQ; 3303 path->mtu = cm_id_priv->path_mtu; 3304 path->rate_selector = IB_SA_EQ; 3305 path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg); 3306 path->packet_life_time_selector = IB_SA_EQ; 3307 path->packet_life_time = 3308 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg); 3309 path->packet_life_time -= (path->packet_life_time > 0); 3310 cm_format_path_lid_from_lap(lap_msg, path); 3311 } 3312 3313 static int cm_lap_handler(struct cm_work *work) 3314 { 3315 struct cm_id_private *cm_id_priv; 3316 struct cm_lap_msg *lap_msg; 3317 struct ib_cm_lap_event_param *param; 3318 struct ib_mad_send_buf *msg = NULL; 3319 int ret; 3320 3321 /* Currently Alternate path messages are not supported for 3322 * RoCE link layer. 3323 */ 3324 if (rdma_protocol_roce(work->port->cm_dev->ib_device, 3325 work->port->port_num)) 3326 return -EINVAL; 3327 3328 /* todo: verify LAP request and send reject APR if invalid. */ 3329 lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad; 3330 cm_id_priv = cm_acquire_id( 3331 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)), 3332 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg))); 3333 if (!cm_id_priv) 3334 return -EINVAL; 3335 3336 param = &work->cm_event.param.lap_rcvd; 3337 memset(&work->path[0], 0, sizeof(work->path[1])); 3338 cm_path_set_rec_type(work->port->cm_dev->ib_device, 3339 work->port->port_num, &work->path[0], 3340 IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, 3341 lap_msg)); 3342 param->alternate_path = &work->path[0]; 3343 cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg); 3344 work->cm_event.private_data = 3345 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg); 3346 3347 spin_lock_irq(&cm_id_priv->lock); 3348 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) 3349 goto unlock; 3350 3351 switch (cm_id_priv->id.lap_state) { 3352 case IB_CM_LAP_UNINIT: 3353 case IB_CM_LAP_IDLE: 3354 break; 3355 case IB_CM_MRA_LAP_SENT: 3356 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3357 counter[CM_LAP_COUNTER]); 3358 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc); 3359 if (IS_ERR(msg)) 3360 goto unlock; 3361 3362 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 3363 CM_MSG_RESPONSE_OTHER, 3364 cm_id_priv->service_timeout, 3365 cm_id_priv->private_data, 3366 cm_id_priv->private_data_len); 3367 spin_unlock_irq(&cm_id_priv->lock); 3368 3369 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) || 3370 ib_post_send_mad(msg, NULL)) 3371 cm_free_msg(msg); 3372 goto deref; 3373 case IB_CM_LAP_RCVD: 3374 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3375 counter[CM_LAP_COUNTER]); 3376 goto unlock; 3377 default: 3378 goto unlock; 3379 } 3380 3381 ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc, 3382 work->mad_recv_wc->recv_buf.grh, 3383 &cm_id_priv->av); 3384 if (ret) 3385 goto unlock; 3386 3387 ret = cm_init_av_by_path(param->alternate_path, NULL, 3388 &cm_id_priv->alt_av, cm_id_priv); 3389 if (ret) 3390 goto unlock; 3391 3392 cm_id_priv->id.lap_state = IB_CM_LAP_RCVD; 3393 cm_id_priv->tid = lap_msg->hdr.tid; 3394 ret = atomic_inc_and_test(&cm_id_priv->work_count); 3395 if (!ret) 3396 list_add_tail(&work->list, &cm_id_priv->work_list); 3397 spin_unlock_irq(&cm_id_priv->lock); 3398 3399 if (ret) 3400 cm_process_work(cm_id_priv, work); 3401 else 3402 cm_deref_id(cm_id_priv); 3403 return 0; 3404 3405 unlock: spin_unlock_irq(&cm_id_priv->lock); 3406 deref: cm_deref_id(cm_id_priv); 3407 return -EINVAL; 3408 } 3409 3410 static int cm_apr_handler(struct cm_work *work) 3411 { 3412 struct cm_id_private *cm_id_priv; 3413 struct cm_apr_msg *apr_msg; 3414 int ret; 3415 3416 /* Currently Alternate path messages are not supported for 3417 * RoCE link layer. 3418 */ 3419 if (rdma_protocol_roce(work->port->cm_dev->ib_device, 3420 work->port->port_num)) 3421 return -EINVAL; 3422 3423 apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad; 3424 cm_id_priv = cm_acquire_id( 3425 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)), 3426 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg))); 3427 if (!cm_id_priv) 3428 return -EINVAL; /* Unmatched reply. */ 3429 3430 work->cm_event.param.apr_rcvd.ap_status = 3431 IBA_GET(CM_APR_AR_STATUS, apr_msg); 3432 work->cm_event.param.apr_rcvd.apr_info = 3433 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg); 3434 work->cm_event.param.apr_rcvd.info_len = 3435 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg); 3436 work->cm_event.private_data = 3437 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg); 3438 3439 spin_lock_irq(&cm_id_priv->lock); 3440 if (cm_id_priv->id.state != IB_CM_ESTABLISHED || 3441 (cm_id_priv->id.lap_state != IB_CM_LAP_SENT && 3442 cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) { 3443 spin_unlock_irq(&cm_id_priv->lock); 3444 goto out; 3445 } 3446 cm_id_priv->id.lap_state = IB_CM_LAP_IDLE; 3447 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3448 cm_id_priv->msg = NULL; 3449 3450 ret = atomic_inc_and_test(&cm_id_priv->work_count); 3451 if (!ret) 3452 list_add_tail(&work->list, &cm_id_priv->work_list); 3453 spin_unlock_irq(&cm_id_priv->lock); 3454 3455 if (ret) 3456 cm_process_work(cm_id_priv, work); 3457 else 3458 cm_deref_id(cm_id_priv); 3459 return 0; 3460 out: 3461 cm_deref_id(cm_id_priv); 3462 return -EINVAL; 3463 } 3464 3465 static int cm_timewait_handler(struct cm_work *work) 3466 { 3467 struct cm_timewait_info *timewait_info; 3468 struct cm_id_private *cm_id_priv; 3469 int ret; 3470 3471 timewait_info = container_of(work, struct cm_timewait_info, work); 3472 spin_lock_irq(&cm.lock); 3473 list_del(&timewait_info->list); 3474 spin_unlock_irq(&cm.lock); 3475 3476 cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 3477 timewait_info->work.remote_id); 3478 if (!cm_id_priv) 3479 return -EINVAL; 3480 3481 spin_lock_irq(&cm_id_priv->lock); 3482 if (cm_id_priv->id.state != IB_CM_TIMEWAIT || 3483 cm_id_priv->remote_qpn != timewait_info->remote_qpn) { 3484 spin_unlock_irq(&cm_id_priv->lock); 3485 goto out; 3486 } 3487 cm_id_priv->id.state = IB_CM_IDLE; 3488 ret = atomic_inc_and_test(&cm_id_priv->work_count); 3489 if (!ret) 3490 list_add_tail(&work->list, &cm_id_priv->work_list); 3491 spin_unlock_irq(&cm_id_priv->lock); 3492 3493 if (ret) 3494 cm_process_work(cm_id_priv, work); 3495 else 3496 cm_deref_id(cm_id_priv); 3497 return 0; 3498 out: 3499 cm_deref_id(cm_id_priv); 3500 return -EINVAL; 3501 } 3502 3503 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, 3504 struct cm_id_private *cm_id_priv, 3505 struct ib_cm_sidr_req_param *param) 3506 { 3507 cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID, 3508 cm_form_tid(cm_id_priv)); 3509 IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg, 3510 be32_to_cpu(cm_id_priv->id.local_id)); 3511 IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg, 3512 be16_to_cpu(param->path->pkey)); 3513 IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg, 3514 be64_to_cpu(param->service_id)); 3515 3516 if (param->private_data && param->private_data_len) 3517 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg, 3518 param->private_data, param->private_data_len); 3519 } 3520 3521 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, 3522 struct ib_cm_sidr_req_param *param) 3523 { 3524 struct cm_id_private *cm_id_priv; 3525 struct ib_mad_send_buf *msg; 3526 unsigned long flags; 3527 int ret; 3528 3529 if (!param->path || (param->private_data && 3530 param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE)) 3531 return -EINVAL; 3532 3533 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3534 ret = cm_init_av_by_path(param->path, param->sgid_attr, 3535 &cm_id_priv->av, 3536 cm_id_priv); 3537 if (ret) 3538 goto out; 3539 3540 cm_id->service_id = param->service_id; 3541 cm_id->service_mask = ~cpu_to_be64(0); 3542 cm_id_priv->timeout_ms = param->timeout_ms; 3543 cm_id_priv->max_cm_retries = param->max_cm_retries; 3544 ret = cm_alloc_msg(cm_id_priv, &msg); 3545 if (ret) 3546 goto out; 3547 3548 cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv, 3549 param); 3550 msg->timeout_ms = cm_id_priv->timeout_ms; 3551 msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT; 3552 3553 spin_lock_irqsave(&cm_id_priv->lock, flags); 3554 if (cm_id->state == IB_CM_IDLE) 3555 ret = ib_post_send_mad(msg, NULL); 3556 else 3557 ret = -EINVAL; 3558 3559 if (ret) { 3560 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3561 cm_free_msg(msg); 3562 goto out; 3563 } 3564 cm_id->state = IB_CM_SIDR_REQ_SENT; 3565 cm_id_priv->msg = msg; 3566 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3567 out: 3568 return ret; 3569 } 3570 EXPORT_SYMBOL(ib_send_cm_sidr_req); 3571 3572 static void cm_format_sidr_req_event(struct cm_work *work, 3573 const struct cm_id_private *rx_cm_id, 3574 struct ib_cm_id *listen_id) 3575 { 3576 struct cm_sidr_req_msg *sidr_req_msg; 3577 struct ib_cm_sidr_req_event_param *param; 3578 3579 sidr_req_msg = (struct cm_sidr_req_msg *) 3580 work->mad_recv_wc->recv_buf.mad; 3581 param = &work->cm_event.param.sidr_req_rcvd; 3582 param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg); 3583 param->listen_id = listen_id; 3584 param->service_id = 3585 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); 3586 param->bth_pkey = cm_get_bth_pkey(work); 3587 param->port = work->port->port_num; 3588 param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr; 3589 work->cm_event.private_data = 3590 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg); 3591 } 3592 3593 static int cm_sidr_req_handler(struct cm_work *work) 3594 { 3595 struct cm_id_private *cm_id_priv, *listen_cm_id_priv; 3596 struct cm_sidr_req_msg *sidr_req_msg; 3597 struct ib_wc *wc; 3598 int ret; 3599 3600 cm_id_priv = 3601 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL); 3602 if (IS_ERR(cm_id_priv)) 3603 return PTR_ERR(cm_id_priv); 3604 3605 /* Record SGID/SLID and request ID for lookup. */ 3606 sidr_req_msg = (struct cm_sidr_req_msg *) 3607 work->mad_recv_wc->recv_buf.mad; 3608 3609 cm_id_priv->id.remote_id = 3610 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg)); 3611 cm_id_priv->id.service_id = 3612 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); 3613 cm_id_priv->id.service_mask = ~cpu_to_be64(0); 3614 cm_id_priv->tid = sidr_req_msg->hdr.tid; 3615 3616 wc = work->mad_recv_wc->wc; 3617 cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid); 3618 cm_id_priv->av.dgid.global.interface_id = 0; 3619 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, 3620 work->mad_recv_wc->recv_buf.grh, 3621 &cm_id_priv->av); 3622 if (ret) 3623 goto out; 3624 3625 spin_lock_irq(&cm.lock); 3626 listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv); 3627 if (listen_cm_id_priv) { 3628 spin_unlock_irq(&cm.lock); 3629 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3630 counter[CM_SIDR_REQ_COUNTER]); 3631 goto out; /* Duplicate message. */ 3632 } 3633 cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD; 3634 listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device, 3635 cm_id_priv->id.service_id); 3636 if (!listen_cm_id_priv) { 3637 spin_unlock_irq(&cm.lock); 3638 ib_send_cm_sidr_rep(&cm_id_priv->id, 3639 &(struct ib_cm_sidr_rep_param){ 3640 .status = IB_SIDR_UNSUPPORTED }); 3641 goto out; /* No match. */ 3642 } 3643 refcount_inc(&listen_cm_id_priv->refcount); 3644 spin_unlock_irq(&cm.lock); 3645 3646 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; 3647 cm_id_priv->id.context = listen_cm_id_priv->id.context; 3648 3649 /* 3650 * A SIDR ID does not need to be in the xarray since it does not receive 3651 * mads, is not placed in the remote_id or remote_qpn rbtree, and does 3652 * not enter timewait. 3653 */ 3654 3655 cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id); 3656 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event); 3657 cm_free_work(work); 3658 /* 3659 * A pointer to the listen_cm_id is held in the event, so this deref 3660 * must be after the event is delivered above. 3661 */ 3662 cm_deref_id(listen_cm_id_priv); 3663 if (ret) 3664 cm_destroy_id(&cm_id_priv->id, ret); 3665 return 0; 3666 out: 3667 ib_destroy_cm_id(&cm_id_priv->id); 3668 return -EINVAL; 3669 } 3670 3671 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, 3672 struct cm_id_private *cm_id_priv, 3673 struct ib_cm_sidr_rep_param *param) 3674 { 3675 cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID, 3676 cm_id_priv->tid); 3677 IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg, 3678 be32_to_cpu(cm_id_priv->id.remote_id)); 3679 IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status); 3680 IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num); 3681 IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg, 3682 be64_to_cpu(cm_id_priv->id.service_id)); 3683 IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey); 3684 3685 if (param->info && param->info_length) 3686 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg, 3687 param->info, param->info_length); 3688 3689 if (param->private_data && param->private_data_len) 3690 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg, 3691 param->private_data, param->private_data_len); 3692 } 3693 3694 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, 3695 struct ib_cm_sidr_rep_param *param) 3696 { 3697 struct ib_mad_send_buf *msg; 3698 int ret; 3699 3700 lockdep_assert_held(&cm_id_priv->lock); 3701 3702 if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) || 3703 (param->private_data && 3704 param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE)) 3705 return -EINVAL; 3706 3707 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD) 3708 return -EINVAL; 3709 3710 ret = cm_alloc_msg(cm_id_priv, &msg); 3711 if (ret) 3712 return ret; 3713 3714 cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv, 3715 param); 3716 ret = ib_post_send_mad(msg, NULL); 3717 if (ret) { 3718 cm_free_msg(msg); 3719 return ret; 3720 } 3721 cm_id_priv->id.state = IB_CM_IDLE; 3722 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) { 3723 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); 3724 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node); 3725 } 3726 return 0; 3727 } 3728 3729 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, 3730 struct ib_cm_sidr_rep_param *param) 3731 { 3732 struct cm_id_private *cm_id_priv = 3733 container_of(cm_id, struct cm_id_private, id); 3734 unsigned long flags; 3735 int ret; 3736 3737 spin_lock_irqsave(&cm_id_priv->lock, flags); 3738 ret = cm_send_sidr_rep_locked(cm_id_priv, param); 3739 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3740 return ret; 3741 } 3742 EXPORT_SYMBOL(ib_send_cm_sidr_rep); 3743 3744 static void cm_format_sidr_rep_event(struct cm_work *work, 3745 const struct cm_id_private *cm_id_priv) 3746 { 3747 struct cm_sidr_rep_msg *sidr_rep_msg; 3748 struct ib_cm_sidr_rep_event_param *param; 3749 3750 sidr_rep_msg = (struct cm_sidr_rep_msg *) 3751 work->mad_recv_wc->recv_buf.mad; 3752 param = &work->cm_event.param.sidr_rep_rcvd; 3753 param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg); 3754 param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg); 3755 param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg); 3756 param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION, 3757 sidr_rep_msg); 3758 param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, 3759 sidr_rep_msg); 3760 param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; 3761 work->cm_event.private_data = 3762 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg); 3763 } 3764 3765 static int cm_sidr_rep_handler(struct cm_work *work) 3766 { 3767 struct cm_sidr_rep_msg *sidr_rep_msg; 3768 struct cm_id_private *cm_id_priv; 3769 3770 sidr_rep_msg = (struct cm_sidr_rep_msg *) 3771 work->mad_recv_wc->recv_buf.mad; 3772 cm_id_priv = cm_acquire_id( 3773 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0); 3774 if (!cm_id_priv) 3775 return -EINVAL; /* Unmatched reply. */ 3776 3777 spin_lock_irq(&cm_id_priv->lock); 3778 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) { 3779 spin_unlock_irq(&cm_id_priv->lock); 3780 goto out; 3781 } 3782 cm_id_priv->id.state = IB_CM_IDLE; 3783 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3784 spin_unlock_irq(&cm_id_priv->lock); 3785 3786 cm_format_sidr_rep_event(work, cm_id_priv); 3787 cm_process_work(cm_id_priv, work); 3788 return 0; 3789 out: 3790 cm_deref_id(cm_id_priv); 3791 return -EINVAL; 3792 } 3793 3794 static void cm_process_send_error(struct ib_mad_send_buf *msg, 3795 enum ib_wc_status wc_status) 3796 { 3797 struct cm_id_private *cm_id_priv; 3798 struct ib_cm_event cm_event; 3799 enum ib_cm_state state; 3800 int ret; 3801 3802 memset(&cm_event, 0, sizeof cm_event); 3803 cm_id_priv = msg->context[0]; 3804 3805 /* Discard old sends or ones without a response. */ 3806 spin_lock_irq(&cm_id_priv->lock); 3807 state = (enum ib_cm_state) (unsigned long) msg->context[1]; 3808 if (msg != cm_id_priv->msg || state != cm_id_priv->id.state) 3809 goto discard; 3810 3811 pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n", 3812 state, ib_wc_status_msg(wc_status)); 3813 switch (state) { 3814 case IB_CM_REQ_SENT: 3815 case IB_CM_MRA_REQ_RCVD: 3816 cm_reset_to_idle(cm_id_priv); 3817 cm_event.event = IB_CM_REQ_ERROR; 3818 break; 3819 case IB_CM_REP_SENT: 3820 case IB_CM_MRA_REP_RCVD: 3821 cm_reset_to_idle(cm_id_priv); 3822 cm_event.event = IB_CM_REP_ERROR; 3823 break; 3824 case IB_CM_DREQ_SENT: 3825 cm_enter_timewait(cm_id_priv); 3826 cm_event.event = IB_CM_DREQ_ERROR; 3827 break; 3828 case IB_CM_SIDR_REQ_SENT: 3829 cm_id_priv->id.state = IB_CM_IDLE; 3830 cm_event.event = IB_CM_SIDR_REQ_ERROR; 3831 break; 3832 default: 3833 goto discard; 3834 } 3835 spin_unlock_irq(&cm_id_priv->lock); 3836 cm_event.param.send_status = wc_status; 3837 3838 /* No other events can occur on the cm_id at this point. */ 3839 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event); 3840 cm_free_msg(msg); 3841 if (ret) 3842 ib_destroy_cm_id(&cm_id_priv->id); 3843 return; 3844 discard: 3845 spin_unlock_irq(&cm_id_priv->lock); 3846 cm_free_msg(msg); 3847 } 3848 3849 static void cm_send_handler(struct ib_mad_agent *mad_agent, 3850 struct ib_mad_send_wc *mad_send_wc) 3851 { 3852 struct ib_mad_send_buf *msg = mad_send_wc->send_buf; 3853 struct cm_port *port; 3854 u16 attr_index; 3855 3856 port = mad_agent->context; 3857 attr_index = be16_to_cpu(((struct ib_mad_hdr *) 3858 msg->mad)->attr_id) - CM_ATTR_ID_OFFSET; 3859 3860 /* 3861 * If the send was in response to a received message (context[0] is not 3862 * set to a cm_id), and is not a REJ, then it is a send that was 3863 * manually retried. 3864 */ 3865 if (!msg->context[0] && (attr_index != CM_REJ_COUNTER)) 3866 msg->retries = 1; 3867 3868 atomic_long_add(1 + msg->retries, 3869 &port->counter_group[CM_XMIT].counter[attr_index]); 3870 if (msg->retries) 3871 atomic_long_add(msg->retries, 3872 &port->counter_group[CM_XMIT_RETRIES]. 3873 counter[attr_index]); 3874 3875 switch (mad_send_wc->status) { 3876 case IB_WC_SUCCESS: 3877 case IB_WC_WR_FLUSH_ERR: 3878 cm_free_msg(msg); 3879 break; 3880 default: 3881 if (msg->context[0] && msg->context[1]) 3882 cm_process_send_error(msg, mad_send_wc->status); 3883 else 3884 cm_free_msg(msg); 3885 break; 3886 } 3887 } 3888 3889 static void cm_work_handler(struct work_struct *_work) 3890 { 3891 struct cm_work *work = container_of(_work, struct cm_work, work.work); 3892 int ret; 3893 3894 switch (work->cm_event.event) { 3895 case IB_CM_REQ_RECEIVED: 3896 ret = cm_req_handler(work); 3897 break; 3898 case IB_CM_MRA_RECEIVED: 3899 ret = cm_mra_handler(work); 3900 break; 3901 case IB_CM_REJ_RECEIVED: 3902 ret = cm_rej_handler(work); 3903 break; 3904 case IB_CM_REP_RECEIVED: 3905 ret = cm_rep_handler(work); 3906 break; 3907 case IB_CM_RTU_RECEIVED: 3908 ret = cm_rtu_handler(work); 3909 break; 3910 case IB_CM_USER_ESTABLISHED: 3911 ret = cm_establish_handler(work); 3912 break; 3913 case IB_CM_DREQ_RECEIVED: 3914 ret = cm_dreq_handler(work); 3915 break; 3916 case IB_CM_DREP_RECEIVED: 3917 ret = cm_drep_handler(work); 3918 break; 3919 case IB_CM_SIDR_REQ_RECEIVED: 3920 ret = cm_sidr_req_handler(work); 3921 break; 3922 case IB_CM_SIDR_REP_RECEIVED: 3923 ret = cm_sidr_rep_handler(work); 3924 break; 3925 case IB_CM_LAP_RECEIVED: 3926 ret = cm_lap_handler(work); 3927 break; 3928 case IB_CM_APR_RECEIVED: 3929 ret = cm_apr_handler(work); 3930 break; 3931 case IB_CM_TIMEWAIT_EXIT: 3932 ret = cm_timewait_handler(work); 3933 break; 3934 default: 3935 pr_debug("cm_event.event: 0x%x\n", work->cm_event.event); 3936 ret = -EINVAL; 3937 break; 3938 } 3939 if (ret) 3940 cm_free_work(work); 3941 } 3942 3943 static int cm_establish(struct ib_cm_id *cm_id) 3944 { 3945 struct cm_id_private *cm_id_priv; 3946 struct cm_work *work; 3947 unsigned long flags; 3948 int ret = 0; 3949 struct cm_device *cm_dev; 3950 3951 cm_dev = ib_get_client_data(cm_id->device, &cm_client); 3952 if (!cm_dev) 3953 return -ENODEV; 3954 3955 work = kmalloc(sizeof *work, GFP_ATOMIC); 3956 if (!work) 3957 return -ENOMEM; 3958 3959 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3960 spin_lock_irqsave(&cm_id_priv->lock, flags); 3961 switch (cm_id->state) 3962 { 3963 case IB_CM_REP_SENT: 3964 case IB_CM_MRA_REP_RCVD: 3965 cm_id->state = IB_CM_ESTABLISHED; 3966 break; 3967 case IB_CM_ESTABLISHED: 3968 ret = -EISCONN; 3969 break; 3970 default: 3971 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, 3972 be32_to_cpu(cm_id->local_id), cm_id->state); 3973 ret = -EINVAL; 3974 break; 3975 } 3976 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3977 3978 if (ret) { 3979 kfree(work); 3980 goto out; 3981 } 3982 3983 /* 3984 * The CM worker thread may try to destroy the cm_id before it 3985 * can execute this work item. To prevent potential deadlock, 3986 * we need to find the cm_id once we're in the context of the 3987 * worker thread, rather than holding a reference on it. 3988 */ 3989 INIT_DELAYED_WORK(&work->work, cm_work_handler); 3990 work->local_id = cm_id->local_id; 3991 work->remote_id = cm_id->remote_id; 3992 work->mad_recv_wc = NULL; 3993 work->cm_event.event = IB_CM_USER_ESTABLISHED; 3994 3995 /* Check if the device started its remove_one */ 3996 spin_lock_irqsave(&cm.lock, flags); 3997 if (!cm_dev->going_down) { 3998 queue_delayed_work(cm.wq, &work->work, 0); 3999 } else { 4000 kfree(work); 4001 ret = -ENODEV; 4002 } 4003 spin_unlock_irqrestore(&cm.lock, flags); 4004 4005 out: 4006 return ret; 4007 } 4008 4009 static int cm_migrate(struct ib_cm_id *cm_id) 4010 { 4011 struct cm_id_private *cm_id_priv; 4012 struct cm_av tmp_av; 4013 unsigned long flags; 4014 int tmp_send_port_not_ready; 4015 int ret = 0; 4016 4017 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 4018 spin_lock_irqsave(&cm_id_priv->lock, flags); 4019 if (cm_id->state == IB_CM_ESTABLISHED && 4020 (cm_id->lap_state == IB_CM_LAP_UNINIT || 4021 cm_id->lap_state == IB_CM_LAP_IDLE)) { 4022 cm_id->lap_state = IB_CM_LAP_IDLE; 4023 /* Swap address vector */ 4024 tmp_av = cm_id_priv->av; 4025 cm_id_priv->av = cm_id_priv->alt_av; 4026 cm_id_priv->alt_av = tmp_av; 4027 /* Swap port send ready state */ 4028 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready; 4029 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready; 4030 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready; 4031 } else 4032 ret = -EINVAL; 4033 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4034 4035 return ret; 4036 } 4037 4038 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event) 4039 { 4040 int ret; 4041 4042 switch (event) { 4043 case IB_EVENT_COMM_EST: 4044 ret = cm_establish(cm_id); 4045 break; 4046 case IB_EVENT_PATH_MIG: 4047 ret = cm_migrate(cm_id); 4048 break; 4049 default: 4050 ret = -EINVAL; 4051 } 4052 return ret; 4053 } 4054 EXPORT_SYMBOL(ib_cm_notify); 4055 4056 static void cm_recv_handler(struct ib_mad_agent *mad_agent, 4057 struct ib_mad_send_buf *send_buf, 4058 struct ib_mad_recv_wc *mad_recv_wc) 4059 { 4060 struct cm_port *port = mad_agent->context; 4061 struct cm_work *work; 4062 enum ib_cm_event_type event; 4063 bool alt_path = false; 4064 u16 attr_id; 4065 int paths = 0; 4066 int going_down = 0; 4067 4068 switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) { 4069 case CM_REQ_ATTR_ID: 4070 alt_path = cm_req_has_alt_path((struct cm_req_msg *) 4071 mad_recv_wc->recv_buf.mad); 4072 paths = 1 + (alt_path != 0); 4073 event = IB_CM_REQ_RECEIVED; 4074 break; 4075 case CM_MRA_ATTR_ID: 4076 event = IB_CM_MRA_RECEIVED; 4077 break; 4078 case CM_REJ_ATTR_ID: 4079 event = IB_CM_REJ_RECEIVED; 4080 break; 4081 case CM_REP_ATTR_ID: 4082 event = IB_CM_REP_RECEIVED; 4083 break; 4084 case CM_RTU_ATTR_ID: 4085 event = IB_CM_RTU_RECEIVED; 4086 break; 4087 case CM_DREQ_ATTR_ID: 4088 event = IB_CM_DREQ_RECEIVED; 4089 break; 4090 case CM_DREP_ATTR_ID: 4091 event = IB_CM_DREP_RECEIVED; 4092 break; 4093 case CM_SIDR_REQ_ATTR_ID: 4094 event = IB_CM_SIDR_REQ_RECEIVED; 4095 break; 4096 case CM_SIDR_REP_ATTR_ID: 4097 event = IB_CM_SIDR_REP_RECEIVED; 4098 break; 4099 case CM_LAP_ATTR_ID: 4100 paths = 1; 4101 event = IB_CM_LAP_RECEIVED; 4102 break; 4103 case CM_APR_ATTR_ID: 4104 event = IB_CM_APR_RECEIVED; 4105 break; 4106 default: 4107 ib_free_recv_mad(mad_recv_wc); 4108 return; 4109 } 4110 4111 attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id); 4112 atomic_long_inc(&port->counter_group[CM_RECV]. 4113 counter[attr_id - CM_ATTR_ID_OFFSET]); 4114 4115 work = kmalloc(struct_size(work, path, paths), GFP_KERNEL); 4116 if (!work) { 4117 ib_free_recv_mad(mad_recv_wc); 4118 return; 4119 } 4120 4121 INIT_DELAYED_WORK(&work->work, cm_work_handler); 4122 work->cm_event.event = event; 4123 work->mad_recv_wc = mad_recv_wc; 4124 work->port = port; 4125 4126 /* Check if the device started its remove_one */ 4127 spin_lock_irq(&cm.lock); 4128 if (!port->cm_dev->going_down) 4129 queue_delayed_work(cm.wq, &work->work, 0); 4130 else 4131 going_down = 1; 4132 spin_unlock_irq(&cm.lock); 4133 4134 if (going_down) { 4135 kfree(work); 4136 ib_free_recv_mad(mad_recv_wc); 4137 } 4138 } 4139 4140 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, 4141 struct ib_qp_attr *qp_attr, 4142 int *qp_attr_mask) 4143 { 4144 unsigned long flags; 4145 int ret; 4146 4147 spin_lock_irqsave(&cm_id_priv->lock, flags); 4148 switch (cm_id_priv->id.state) { 4149 case IB_CM_REQ_SENT: 4150 case IB_CM_MRA_REQ_RCVD: 4151 case IB_CM_REQ_RCVD: 4152 case IB_CM_MRA_REQ_SENT: 4153 case IB_CM_REP_RCVD: 4154 case IB_CM_MRA_REP_SENT: 4155 case IB_CM_REP_SENT: 4156 case IB_CM_MRA_REP_RCVD: 4157 case IB_CM_ESTABLISHED: 4158 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS | 4159 IB_QP_PKEY_INDEX | IB_QP_PORT; 4160 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE; 4161 if (cm_id_priv->responder_resources) 4162 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ | 4163 IB_ACCESS_REMOTE_ATOMIC; 4164 qp_attr->pkey_index = cm_id_priv->av.pkey_index; 4165 qp_attr->port_num = cm_id_priv->av.port->port_num; 4166 ret = 0; 4167 break; 4168 default: 4169 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 4170 __func__, be32_to_cpu(cm_id_priv->id.local_id), 4171 cm_id_priv->id.state); 4172 ret = -EINVAL; 4173 break; 4174 } 4175 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4176 return ret; 4177 } 4178 4179 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, 4180 struct ib_qp_attr *qp_attr, 4181 int *qp_attr_mask) 4182 { 4183 unsigned long flags; 4184 int ret; 4185 4186 spin_lock_irqsave(&cm_id_priv->lock, flags); 4187 switch (cm_id_priv->id.state) { 4188 case IB_CM_REQ_RCVD: 4189 case IB_CM_MRA_REQ_SENT: 4190 case IB_CM_REP_RCVD: 4191 case IB_CM_MRA_REP_SENT: 4192 case IB_CM_REP_SENT: 4193 case IB_CM_MRA_REP_RCVD: 4194 case IB_CM_ESTABLISHED: 4195 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | 4196 IB_QP_DEST_QPN | IB_QP_RQ_PSN; 4197 qp_attr->ah_attr = cm_id_priv->av.ah_attr; 4198 qp_attr->path_mtu = cm_id_priv->path_mtu; 4199 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn); 4200 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn); 4201 if (cm_id_priv->qp_type == IB_QPT_RC || 4202 cm_id_priv->qp_type == IB_QPT_XRC_TGT) { 4203 *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC | 4204 IB_QP_MIN_RNR_TIMER; 4205 qp_attr->max_dest_rd_atomic = 4206 cm_id_priv->responder_resources; 4207 qp_attr->min_rnr_timer = 0; 4208 } 4209 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) { 4210 *qp_attr_mask |= IB_QP_ALT_PATH; 4211 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num; 4212 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index; 4213 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout; 4214 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr; 4215 } 4216 ret = 0; 4217 break; 4218 default: 4219 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 4220 __func__, be32_to_cpu(cm_id_priv->id.local_id), 4221 cm_id_priv->id.state); 4222 ret = -EINVAL; 4223 break; 4224 } 4225 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4226 return ret; 4227 } 4228 4229 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, 4230 struct ib_qp_attr *qp_attr, 4231 int *qp_attr_mask) 4232 { 4233 unsigned long flags; 4234 int ret; 4235 4236 spin_lock_irqsave(&cm_id_priv->lock, flags); 4237 switch (cm_id_priv->id.state) { 4238 /* Allow transition to RTS before sending REP */ 4239 case IB_CM_REQ_RCVD: 4240 case IB_CM_MRA_REQ_SENT: 4241 4242 case IB_CM_REP_RCVD: 4243 case IB_CM_MRA_REP_SENT: 4244 case IB_CM_REP_SENT: 4245 case IB_CM_MRA_REP_RCVD: 4246 case IB_CM_ESTABLISHED: 4247 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) { 4248 *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN; 4249 qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn); 4250 switch (cm_id_priv->qp_type) { 4251 case IB_QPT_RC: 4252 case IB_QPT_XRC_INI: 4253 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | 4254 IB_QP_MAX_QP_RD_ATOMIC; 4255 qp_attr->retry_cnt = cm_id_priv->retry_count; 4256 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count; 4257 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth; 4258 /* fall through */ 4259 case IB_QPT_XRC_TGT: 4260 *qp_attr_mask |= IB_QP_TIMEOUT; 4261 qp_attr->timeout = cm_id_priv->av.timeout; 4262 break; 4263 default: 4264 break; 4265 } 4266 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) { 4267 *qp_attr_mask |= IB_QP_PATH_MIG_STATE; 4268 qp_attr->path_mig_state = IB_MIG_REARM; 4269 } 4270 } else { 4271 *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE; 4272 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num; 4273 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index; 4274 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout; 4275 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr; 4276 qp_attr->path_mig_state = IB_MIG_REARM; 4277 } 4278 ret = 0; 4279 break; 4280 default: 4281 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 4282 __func__, be32_to_cpu(cm_id_priv->id.local_id), 4283 cm_id_priv->id.state); 4284 ret = -EINVAL; 4285 break; 4286 } 4287 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4288 return ret; 4289 } 4290 4291 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, 4292 struct ib_qp_attr *qp_attr, 4293 int *qp_attr_mask) 4294 { 4295 struct cm_id_private *cm_id_priv; 4296 int ret; 4297 4298 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 4299 switch (qp_attr->qp_state) { 4300 case IB_QPS_INIT: 4301 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask); 4302 break; 4303 case IB_QPS_RTR: 4304 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask); 4305 break; 4306 case IB_QPS_RTS: 4307 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask); 4308 break; 4309 default: 4310 ret = -EINVAL; 4311 break; 4312 } 4313 return ret; 4314 } 4315 EXPORT_SYMBOL(ib_cm_init_qp_attr); 4316 4317 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr, 4318 char *buf) 4319 { 4320 struct cm_counter_group *group; 4321 struct cm_counter_attribute *cm_attr; 4322 4323 group = container_of(obj, struct cm_counter_group, obj); 4324 cm_attr = container_of(attr, struct cm_counter_attribute, attr); 4325 4326 return sprintf(buf, "%ld\n", 4327 atomic_long_read(&group->counter[cm_attr->index])); 4328 } 4329 4330 static const struct sysfs_ops cm_counter_ops = { 4331 .show = cm_show_counter 4332 }; 4333 4334 static struct kobj_type cm_counter_obj_type = { 4335 .sysfs_ops = &cm_counter_ops, 4336 .default_attrs = cm_counter_default_attrs 4337 }; 4338 4339 static char *cm_devnode(struct device *dev, umode_t *mode) 4340 { 4341 if (mode) 4342 *mode = 0666; 4343 return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev)); 4344 } 4345 4346 struct class cm_class = { 4347 .owner = THIS_MODULE, 4348 .name = "infiniband_cm", 4349 .devnode = cm_devnode, 4350 }; 4351 EXPORT_SYMBOL(cm_class); 4352 4353 static int cm_create_port_fs(struct cm_port *port) 4354 { 4355 int i, ret; 4356 4357 for (i = 0; i < CM_COUNTER_GROUPS; i++) { 4358 ret = ib_port_register_module_stat(port->cm_dev->ib_device, 4359 port->port_num, 4360 &port->counter_group[i].obj, 4361 &cm_counter_obj_type, 4362 counter_group_names[i]); 4363 if (ret) 4364 goto error; 4365 } 4366 4367 return 0; 4368 4369 error: 4370 while (i--) 4371 ib_port_unregister_module_stat(&port->counter_group[i].obj); 4372 return ret; 4373 4374 } 4375 4376 static void cm_remove_port_fs(struct cm_port *port) 4377 { 4378 int i; 4379 4380 for (i = 0; i < CM_COUNTER_GROUPS; i++) 4381 ib_port_unregister_module_stat(&port->counter_group[i].obj); 4382 4383 } 4384 4385 static void cm_add_one(struct ib_device *ib_device) 4386 { 4387 struct cm_device *cm_dev; 4388 struct cm_port *port; 4389 struct ib_mad_reg_req reg_req = { 4390 .mgmt_class = IB_MGMT_CLASS_CM, 4391 .mgmt_class_version = IB_CM_CLASS_VERSION, 4392 }; 4393 struct ib_port_modify port_modify = { 4394 .set_port_cap_mask = IB_PORT_CM_SUP 4395 }; 4396 unsigned long flags; 4397 int ret; 4398 int count = 0; 4399 u8 i; 4400 4401 cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt), 4402 GFP_KERNEL); 4403 if (!cm_dev) 4404 return; 4405 4406 cm_dev->ib_device = ib_device; 4407 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay; 4408 cm_dev->going_down = 0; 4409 4410 set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask); 4411 for (i = 1; i <= ib_device->phys_port_cnt; i++) { 4412 if (!rdma_cap_ib_cm(ib_device, i)) 4413 continue; 4414 4415 port = kzalloc(sizeof *port, GFP_KERNEL); 4416 if (!port) 4417 goto error1; 4418 4419 cm_dev->port[i-1] = port; 4420 port->cm_dev = cm_dev; 4421 port->port_num = i; 4422 4423 INIT_LIST_HEAD(&port->cm_priv_prim_list); 4424 INIT_LIST_HEAD(&port->cm_priv_altr_list); 4425 4426 ret = cm_create_port_fs(port); 4427 if (ret) 4428 goto error1; 4429 4430 port->mad_agent = ib_register_mad_agent(ib_device, i, 4431 IB_QPT_GSI, 4432 ®_req, 4433 0, 4434 cm_send_handler, 4435 cm_recv_handler, 4436 port, 4437 0); 4438 if (IS_ERR(port->mad_agent)) 4439 goto error2; 4440 4441 ret = ib_modify_port(ib_device, i, 0, &port_modify); 4442 if (ret) 4443 goto error3; 4444 4445 count++; 4446 } 4447 4448 if (!count) 4449 goto free; 4450 4451 ib_set_client_data(ib_device, &cm_client, cm_dev); 4452 4453 write_lock_irqsave(&cm.device_lock, flags); 4454 list_add_tail(&cm_dev->list, &cm.device_list); 4455 write_unlock_irqrestore(&cm.device_lock, flags); 4456 return; 4457 4458 error3: 4459 ib_unregister_mad_agent(port->mad_agent); 4460 error2: 4461 cm_remove_port_fs(port); 4462 error1: 4463 port_modify.set_port_cap_mask = 0; 4464 port_modify.clr_port_cap_mask = IB_PORT_CM_SUP; 4465 kfree(port); 4466 while (--i) { 4467 if (!rdma_cap_ib_cm(ib_device, i)) 4468 continue; 4469 4470 port = cm_dev->port[i-1]; 4471 ib_modify_port(ib_device, port->port_num, 0, &port_modify); 4472 ib_unregister_mad_agent(port->mad_agent); 4473 cm_remove_port_fs(port); 4474 kfree(port); 4475 } 4476 free: 4477 kfree(cm_dev); 4478 } 4479 4480 static void cm_remove_one(struct ib_device *ib_device, void *client_data) 4481 { 4482 struct cm_device *cm_dev = client_data; 4483 struct cm_port *port; 4484 struct cm_id_private *cm_id_priv; 4485 struct ib_mad_agent *cur_mad_agent; 4486 struct ib_port_modify port_modify = { 4487 .clr_port_cap_mask = IB_PORT_CM_SUP 4488 }; 4489 unsigned long flags; 4490 int i; 4491 4492 if (!cm_dev) 4493 return; 4494 4495 write_lock_irqsave(&cm.device_lock, flags); 4496 list_del(&cm_dev->list); 4497 write_unlock_irqrestore(&cm.device_lock, flags); 4498 4499 spin_lock_irq(&cm.lock); 4500 cm_dev->going_down = 1; 4501 spin_unlock_irq(&cm.lock); 4502 4503 for (i = 1; i <= ib_device->phys_port_cnt; i++) { 4504 if (!rdma_cap_ib_cm(ib_device, i)) 4505 continue; 4506 4507 port = cm_dev->port[i-1]; 4508 ib_modify_port(ib_device, port->port_num, 0, &port_modify); 4509 /* Mark all the cm_id's as not valid */ 4510 spin_lock_irq(&cm.lock); 4511 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list) 4512 cm_id_priv->altr_send_port_not_ready = 1; 4513 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list) 4514 cm_id_priv->prim_send_port_not_ready = 1; 4515 spin_unlock_irq(&cm.lock); 4516 /* 4517 * We flush the queue here after the going_down set, this 4518 * verify that no new works will be queued in the recv handler, 4519 * after that we can call the unregister_mad_agent 4520 */ 4521 flush_workqueue(cm.wq); 4522 spin_lock_irq(&cm.state_lock); 4523 cur_mad_agent = port->mad_agent; 4524 port->mad_agent = NULL; 4525 spin_unlock_irq(&cm.state_lock); 4526 ib_unregister_mad_agent(cur_mad_agent); 4527 cm_remove_port_fs(port); 4528 kfree(port); 4529 } 4530 4531 kfree(cm_dev); 4532 } 4533 4534 static int __init ib_cm_init(void) 4535 { 4536 int ret; 4537 4538 INIT_LIST_HEAD(&cm.device_list); 4539 rwlock_init(&cm.device_lock); 4540 spin_lock_init(&cm.lock); 4541 spin_lock_init(&cm.state_lock); 4542 cm.listen_service_table = RB_ROOT; 4543 cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID); 4544 cm.remote_id_table = RB_ROOT; 4545 cm.remote_qp_table = RB_ROOT; 4546 cm.remote_sidr_table = RB_ROOT; 4547 xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); 4548 get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand); 4549 INIT_LIST_HEAD(&cm.timewait_list); 4550 4551 ret = class_register(&cm_class); 4552 if (ret) { 4553 ret = -ENOMEM; 4554 goto error1; 4555 } 4556 4557 cm.wq = alloc_workqueue("ib_cm", 0, 1); 4558 if (!cm.wq) { 4559 ret = -ENOMEM; 4560 goto error2; 4561 } 4562 4563 ret = ib_register_client(&cm_client); 4564 if (ret) 4565 goto error3; 4566 4567 return 0; 4568 error3: 4569 destroy_workqueue(cm.wq); 4570 error2: 4571 class_unregister(&cm_class); 4572 error1: 4573 return ret; 4574 } 4575 4576 static void __exit ib_cm_cleanup(void) 4577 { 4578 struct cm_timewait_info *timewait_info, *tmp; 4579 4580 spin_lock_irq(&cm.lock); 4581 list_for_each_entry(timewait_info, &cm.timewait_list, list) 4582 cancel_delayed_work(&timewait_info->work.work); 4583 spin_unlock_irq(&cm.lock); 4584 4585 ib_unregister_client(&cm_client); 4586 destroy_workqueue(cm.wq); 4587 4588 list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) { 4589 list_del(&timewait_info->list); 4590 kfree(timewait_info); 4591 } 4592 4593 class_unregister(&cm_class); 4594 WARN_ON(!xa_empty(&cm.local_id_table)); 4595 } 4596 4597 module_init(ib_cm_init); 4598 module_exit(ib_cm_cleanup); 4599