xref: /openbmc/linux/drivers/infiniband/core/cm.c (revision 55fd7e02)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
4  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
5  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
6  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
8  */
9 
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/device.h>
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/idr.h>
16 #include <linux/interrupt.h>
17 #include <linux/random.h>
18 #include <linux/rbtree.h>
19 #include <linux/spinlock.h>
20 #include <linux/slab.h>
21 #include <linux/sysfs.h>
22 #include <linux/workqueue.h>
23 #include <linux/kdev_t.h>
24 #include <linux/etherdevice.h>
25 
26 #include <rdma/ib_cache.h>
27 #include <rdma/ib_cm.h>
28 #include "cm_msgs.h"
29 #include "core_priv.h"
30 
31 MODULE_AUTHOR("Sean Hefty");
32 MODULE_DESCRIPTION("InfiniBand CM");
33 MODULE_LICENSE("Dual BSD/GPL");
34 
35 static const char * const ibcm_rej_reason_strs[] = {
36 	[IB_CM_REJ_NO_QP]			= "no QP",
37 	[IB_CM_REJ_NO_EEC]			= "no EEC",
38 	[IB_CM_REJ_NO_RESOURCES]		= "no resources",
39 	[IB_CM_REJ_TIMEOUT]			= "timeout",
40 	[IB_CM_REJ_UNSUPPORTED]			= "unsupported",
41 	[IB_CM_REJ_INVALID_COMM_ID]		= "invalid comm ID",
42 	[IB_CM_REJ_INVALID_COMM_INSTANCE]	= "invalid comm instance",
43 	[IB_CM_REJ_INVALID_SERVICE_ID]		= "invalid service ID",
44 	[IB_CM_REJ_INVALID_TRANSPORT_TYPE]	= "invalid transport type",
45 	[IB_CM_REJ_STALE_CONN]			= "stale conn",
46 	[IB_CM_REJ_RDC_NOT_EXIST]		= "RDC not exist",
47 	[IB_CM_REJ_INVALID_GID]			= "invalid GID",
48 	[IB_CM_REJ_INVALID_LID]			= "invalid LID",
49 	[IB_CM_REJ_INVALID_SL]			= "invalid SL",
50 	[IB_CM_REJ_INVALID_TRAFFIC_CLASS]	= "invalid traffic class",
51 	[IB_CM_REJ_INVALID_HOP_LIMIT]		= "invalid hop limit",
52 	[IB_CM_REJ_INVALID_PACKET_RATE]		= "invalid packet rate",
53 	[IB_CM_REJ_INVALID_ALT_GID]		= "invalid alt GID",
54 	[IB_CM_REJ_INVALID_ALT_LID]		= "invalid alt LID",
55 	[IB_CM_REJ_INVALID_ALT_SL]		= "invalid alt SL",
56 	[IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]	= "invalid alt traffic class",
57 	[IB_CM_REJ_INVALID_ALT_HOP_LIMIT]	= "invalid alt hop limit",
58 	[IB_CM_REJ_INVALID_ALT_PACKET_RATE]	= "invalid alt packet rate",
59 	[IB_CM_REJ_PORT_CM_REDIRECT]		= "port CM redirect",
60 	[IB_CM_REJ_PORT_REDIRECT]		= "port redirect",
61 	[IB_CM_REJ_INVALID_MTU]			= "invalid MTU",
62 	[IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES]	= "insufficient resp resources",
63 	[IB_CM_REJ_CONSUMER_DEFINED]		= "consumer defined",
64 	[IB_CM_REJ_INVALID_RNR_RETRY]		= "invalid RNR retry",
65 	[IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]	= "duplicate local comm ID",
66 	[IB_CM_REJ_INVALID_CLASS_VERSION]	= "invalid class version",
67 	[IB_CM_REJ_INVALID_FLOW_LABEL]		= "invalid flow label",
68 	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL]	= "invalid alt flow label",
69 	[IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
70 		"vendor option is not supported",
71 };
72 
73 const char *__attribute_const__ ibcm_reject_msg(int reason)
74 {
75 	size_t index = reason;
76 
77 	if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
78 	    ibcm_rej_reason_strs[index])
79 		return ibcm_rej_reason_strs[index];
80 	else
81 		return "unrecognized reason";
82 }
83 EXPORT_SYMBOL(ibcm_reject_msg);
84 
85 struct cm_id_private;
86 struct cm_work;
87 static int cm_add_one(struct ib_device *device);
88 static void cm_remove_one(struct ib_device *device, void *client_data);
89 static void cm_process_work(struct cm_id_private *cm_id_priv,
90 			    struct cm_work *work);
91 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
92 				   struct ib_cm_sidr_rep_param *param);
93 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
94 			       const void *private_data, u8 private_data_len);
95 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
96 			       void *private_data, u8 private_data_len);
97 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
98 			      enum ib_cm_rej_reason reason, void *ari,
99 			      u8 ari_length, const void *private_data,
100 			      u8 private_data_len);
101 
102 static struct ib_client cm_client = {
103 	.name   = "cm",
104 	.add    = cm_add_one,
105 	.remove = cm_remove_one
106 };
107 
108 static struct ib_cm {
109 	spinlock_t lock;
110 	struct list_head device_list;
111 	rwlock_t device_lock;
112 	struct rb_root listen_service_table;
113 	u64 listen_service_id;
114 	/* struct rb_root peer_service_table; todo: fix peer to peer */
115 	struct rb_root remote_qp_table;
116 	struct rb_root remote_id_table;
117 	struct rb_root remote_sidr_table;
118 	struct xarray local_id_table;
119 	u32 local_id_next;
120 	__be32 random_id_operand;
121 	struct list_head timewait_list;
122 	struct workqueue_struct *wq;
123 	/* Sync on cm change port state */
124 	spinlock_t state_lock;
125 } cm;
126 
127 /* Counter indexes ordered by attribute ID */
128 enum {
129 	CM_REQ_COUNTER,
130 	CM_MRA_COUNTER,
131 	CM_REJ_COUNTER,
132 	CM_REP_COUNTER,
133 	CM_RTU_COUNTER,
134 	CM_DREQ_COUNTER,
135 	CM_DREP_COUNTER,
136 	CM_SIDR_REQ_COUNTER,
137 	CM_SIDR_REP_COUNTER,
138 	CM_LAP_COUNTER,
139 	CM_APR_COUNTER,
140 	CM_ATTR_COUNT,
141 	CM_ATTR_ID_OFFSET = 0x0010,
142 };
143 
144 enum {
145 	CM_XMIT,
146 	CM_XMIT_RETRIES,
147 	CM_RECV,
148 	CM_RECV_DUPLICATES,
149 	CM_COUNTER_GROUPS
150 };
151 
152 static char const counter_group_names[CM_COUNTER_GROUPS]
153 				     [sizeof("cm_rx_duplicates")] = {
154 	"cm_tx_msgs", "cm_tx_retries",
155 	"cm_rx_msgs", "cm_rx_duplicates"
156 };
157 
158 struct cm_counter_group {
159 	struct kobject obj;
160 	atomic_long_t counter[CM_ATTR_COUNT];
161 };
162 
163 struct cm_counter_attribute {
164 	struct attribute attr;
165 	int index;
166 };
167 
168 #define CM_COUNTER_ATTR(_name, _index) \
169 struct cm_counter_attribute cm_##_name##_counter_attr = { \
170 	.attr = { .name = __stringify(_name), .mode = 0444 }, \
171 	.index = _index \
172 }
173 
174 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
175 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
176 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
177 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
178 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
179 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
180 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
181 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
182 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
183 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
184 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
185 
186 static struct attribute *cm_counter_default_attrs[] = {
187 	&cm_req_counter_attr.attr,
188 	&cm_mra_counter_attr.attr,
189 	&cm_rej_counter_attr.attr,
190 	&cm_rep_counter_attr.attr,
191 	&cm_rtu_counter_attr.attr,
192 	&cm_dreq_counter_attr.attr,
193 	&cm_drep_counter_attr.attr,
194 	&cm_sidr_req_counter_attr.attr,
195 	&cm_sidr_rep_counter_attr.attr,
196 	&cm_lap_counter_attr.attr,
197 	&cm_apr_counter_attr.attr,
198 	NULL
199 };
200 
201 struct cm_port {
202 	struct cm_device *cm_dev;
203 	struct ib_mad_agent *mad_agent;
204 	struct kobject port_obj;
205 	u8 port_num;
206 	struct list_head cm_priv_prim_list;
207 	struct list_head cm_priv_altr_list;
208 	struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
209 };
210 
211 struct cm_device {
212 	struct list_head list;
213 	struct ib_device *ib_device;
214 	u8 ack_delay;
215 	int going_down;
216 	struct cm_port *port[];
217 };
218 
219 struct cm_av {
220 	struct cm_port *port;
221 	union ib_gid dgid;
222 	struct rdma_ah_attr ah_attr;
223 	u16 pkey_index;
224 	u8 timeout;
225 };
226 
227 struct cm_work {
228 	struct delayed_work work;
229 	struct list_head list;
230 	struct cm_port *port;
231 	struct ib_mad_recv_wc *mad_recv_wc;	/* Received MADs */
232 	__be32 local_id;			/* Established / timewait */
233 	__be32 remote_id;
234 	struct ib_cm_event cm_event;
235 	struct sa_path_rec path[];
236 };
237 
238 struct cm_timewait_info {
239 	struct cm_work work;
240 	struct list_head list;
241 	struct rb_node remote_qp_node;
242 	struct rb_node remote_id_node;
243 	__be64 remote_ca_guid;
244 	__be32 remote_qpn;
245 	u8 inserted_remote_qp;
246 	u8 inserted_remote_id;
247 };
248 
249 struct cm_id_private {
250 	struct ib_cm_id	id;
251 
252 	struct rb_node service_node;
253 	struct rb_node sidr_id_node;
254 	spinlock_t lock;	/* Do not acquire inside cm.lock */
255 	struct completion comp;
256 	refcount_t refcount;
257 	/* Number of clients sharing this ib_cm_id. Only valid for listeners.
258 	 * Protected by the cm.lock spinlock. */
259 	int listen_sharecount;
260 	struct rcu_head rcu;
261 
262 	struct ib_mad_send_buf *msg;
263 	struct cm_timewait_info *timewait_info;
264 	/* todo: use alternate port on send failure */
265 	struct cm_av av;
266 	struct cm_av alt_av;
267 
268 	void *private_data;
269 	__be64 tid;
270 	__be32 local_qpn;
271 	__be32 remote_qpn;
272 	enum ib_qp_type qp_type;
273 	__be32 sq_psn;
274 	__be32 rq_psn;
275 	int timeout_ms;
276 	enum ib_mtu path_mtu;
277 	__be16 pkey;
278 	u8 private_data_len;
279 	u8 max_cm_retries;
280 	u8 responder_resources;
281 	u8 initiator_depth;
282 	u8 retry_count;
283 	u8 rnr_retry_count;
284 	u8 service_timeout;
285 	u8 target_ack_delay;
286 
287 	struct list_head prim_list;
288 	struct list_head altr_list;
289 	/* Indicates that the send port mad is registered and av is set */
290 	int prim_send_port_not_ready;
291 	int altr_send_port_not_ready;
292 
293 	struct list_head work_list;
294 	atomic_t work_count;
295 
296 	struct rdma_ucm_ece ece;
297 };
298 
299 static void cm_work_handler(struct work_struct *work);
300 
301 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
302 {
303 	if (refcount_dec_and_test(&cm_id_priv->refcount))
304 		complete(&cm_id_priv->comp);
305 }
306 
307 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
308 			struct ib_mad_send_buf **msg)
309 {
310 	struct ib_mad_agent *mad_agent;
311 	struct ib_mad_send_buf *m;
312 	struct ib_ah *ah;
313 	struct cm_av *av;
314 	unsigned long flags, flags2;
315 	int ret = 0;
316 
317 	/* don't let the port to be released till the agent is down */
318 	spin_lock_irqsave(&cm.state_lock, flags2);
319 	spin_lock_irqsave(&cm.lock, flags);
320 	if (!cm_id_priv->prim_send_port_not_ready)
321 		av = &cm_id_priv->av;
322 	else if (!cm_id_priv->altr_send_port_not_ready &&
323 		 (cm_id_priv->alt_av.port))
324 		av = &cm_id_priv->alt_av;
325 	else {
326 		pr_info("%s: not valid CM id\n", __func__);
327 		ret = -ENODEV;
328 		spin_unlock_irqrestore(&cm.lock, flags);
329 		goto out;
330 	}
331 	spin_unlock_irqrestore(&cm.lock, flags);
332 	/* Make sure the port haven't released the mad yet */
333 	mad_agent = cm_id_priv->av.port->mad_agent;
334 	if (!mad_agent) {
335 		pr_info("%s: not a valid MAD agent\n", __func__);
336 		ret = -ENODEV;
337 		goto out;
338 	}
339 	ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0);
340 	if (IS_ERR(ah)) {
341 		ret = PTR_ERR(ah);
342 		goto out;
343 	}
344 
345 	m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
346 			       av->pkey_index,
347 			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
348 			       GFP_ATOMIC,
349 			       IB_MGMT_BASE_VERSION);
350 	if (IS_ERR(m)) {
351 		rdma_destroy_ah(ah, 0);
352 		ret = PTR_ERR(m);
353 		goto out;
354 	}
355 
356 	/* Timeout set by caller if response is expected. */
357 	m->ah = ah;
358 	m->retries = cm_id_priv->max_cm_retries;
359 
360 	refcount_inc(&cm_id_priv->refcount);
361 	m->context[0] = cm_id_priv;
362 	*msg = m;
363 
364 out:
365 	spin_unlock_irqrestore(&cm.state_lock, flags2);
366 	return ret;
367 }
368 
369 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
370 							   struct ib_mad_recv_wc *mad_recv_wc)
371 {
372 	return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
373 				  0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
374 				  GFP_ATOMIC,
375 				  IB_MGMT_BASE_VERSION);
376 }
377 
378 static int cm_create_response_msg_ah(struct cm_port *port,
379 				     struct ib_mad_recv_wc *mad_recv_wc,
380 				     struct ib_mad_send_buf *msg)
381 {
382 	struct ib_ah *ah;
383 
384 	ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
385 				  mad_recv_wc->recv_buf.grh, port->port_num);
386 	if (IS_ERR(ah))
387 		return PTR_ERR(ah);
388 
389 	msg->ah = ah;
390 	return 0;
391 }
392 
393 static void cm_free_msg(struct ib_mad_send_buf *msg)
394 {
395 	if (msg->ah)
396 		rdma_destroy_ah(msg->ah, 0);
397 	if (msg->context[0])
398 		cm_deref_id(msg->context[0]);
399 	ib_free_send_mad(msg);
400 }
401 
402 static int cm_alloc_response_msg(struct cm_port *port,
403 				 struct ib_mad_recv_wc *mad_recv_wc,
404 				 struct ib_mad_send_buf **msg)
405 {
406 	struct ib_mad_send_buf *m;
407 	int ret;
408 
409 	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
410 	if (IS_ERR(m))
411 		return PTR_ERR(m);
412 
413 	ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
414 	if (ret) {
415 		cm_free_msg(m);
416 		return ret;
417 	}
418 
419 	*msg = m;
420 	return 0;
421 }
422 
423 static void * cm_copy_private_data(const void *private_data,
424 				   u8 private_data_len)
425 {
426 	void *data;
427 
428 	if (!private_data || !private_data_len)
429 		return NULL;
430 
431 	data = kmemdup(private_data, private_data_len, GFP_KERNEL);
432 	if (!data)
433 		return ERR_PTR(-ENOMEM);
434 
435 	return data;
436 }
437 
438 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
439 				 void *private_data, u8 private_data_len)
440 {
441 	if (cm_id_priv->private_data && cm_id_priv->private_data_len)
442 		kfree(cm_id_priv->private_data);
443 
444 	cm_id_priv->private_data = private_data;
445 	cm_id_priv->private_data_len = private_data_len;
446 }
447 
448 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
449 			      struct ib_grh *grh, struct cm_av *av)
450 {
451 	struct rdma_ah_attr new_ah_attr;
452 	int ret;
453 
454 	av->port = port;
455 	av->pkey_index = wc->pkey_index;
456 
457 	/*
458 	 * av->ah_attr might be initialized based on past wc during incoming
459 	 * connect request or while sending out connect request. So initialize
460 	 * a new ah_attr on stack. If initialization fails, old ah_attr is
461 	 * used for sending any responses. If initialization is successful,
462 	 * than new ah_attr is used by overwriting old one.
463 	 */
464 	ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
465 				      port->port_num, wc,
466 				      grh, &new_ah_attr);
467 	if (ret)
468 		return ret;
469 
470 	rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
471 	return 0;
472 }
473 
474 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
475 				   struct ib_grh *grh, struct cm_av *av)
476 {
477 	av->port = port;
478 	av->pkey_index = wc->pkey_index;
479 	return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
480 				       port->port_num, wc,
481 				       grh, &av->ah_attr);
482 }
483 
484 static void add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
485 				   struct cm_av *av, struct cm_port *port)
486 {
487 	unsigned long flags;
488 
489 	spin_lock_irqsave(&cm.lock, flags);
490 	if (&cm_id_priv->av == av)
491 		list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
492 	else if (&cm_id_priv->alt_av == av)
493 		list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
494 	else
495 		WARN_ON(true);
496 	spin_unlock_irqrestore(&cm.lock, flags);
497 }
498 
499 static struct cm_port *
500 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
501 {
502 	struct cm_device *cm_dev;
503 	struct cm_port *port = NULL;
504 	unsigned long flags;
505 
506 	if (attr) {
507 		read_lock_irqsave(&cm.device_lock, flags);
508 		list_for_each_entry(cm_dev, &cm.device_list, list) {
509 			if (cm_dev->ib_device == attr->device) {
510 				port = cm_dev->port[attr->port_num - 1];
511 				break;
512 			}
513 		}
514 		read_unlock_irqrestore(&cm.device_lock, flags);
515 	} else {
516 		/* SGID attribute can be NULL in following
517 		 * conditions.
518 		 * (a) Alternative path
519 		 * (b) IB link layer without GRH
520 		 * (c) LAP send messages
521 		 */
522 		read_lock_irqsave(&cm.device_lock, flags);
523 		list_for_each_entry(cm_dev, &cm.device_list, list) {
524 			attr = rdma_find_gid(cm_dev->ib_device,
525 					     &path->sgid,
526 					     sa_conv_pathrec_to_gid_type(path),
527 					     NULL);
528 			if (!IS_ERR(attr)) {
529 				port = cm_dev->port[attr->port_num - 1];
530 				break;
531 			}
532 		}
533 		read_unlock_irqrestore(&cm.device_lock, flags);
534 		if (port)
535 			rdma_put_gid_attr(attr);
536 	}
537 	return port;
538 }
539 
540 static int cm_init_av_by_path(struct sa_path_rec *path,
541 			      const struct ib_gid_attr *sgid_attr,
542 			      struct cm_av *av,
543 			      struct cm_id_private *cm_id_priv)
544 {
545 	struct rdma_ah_attr new_ah_attr;
546 	struct cm_device *cm_dev;
547 	struct cm_port *port;
548 	int ret;
549 
550 	port = get_cm_port_from_path(path, sgid_attr);
551 	if (!port)
552 		return -EINVAL;
553 	cm_dev = port->cm_dev;
554 
555 	ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
556 				  be16_to_cpu(path->pkey), &av->pkey_index);
557 	if (ret)
558 		return ret;
559 
560 	av->port = port;
561 
562 	/*
563 	 * av->ah_attr might be initialized based on wc or during
564 	 * request processing time which might have reference to sgid_attr.
565 	 * So initialize a new ah_attr on stack.
566 	 * If initialization fails, old ah_attr is used for sending any
567 	 * responses. If initialization is successful, than new ah_attr
568 	 * is used by overwriting the old one. So that right ah_attr
569 	 * can be used to return an error response.
570 	 */
571 	ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
572 					&new_ah_attr, sgid_attr);
573 	if (ret)
574 		return ret;
575 
576 	av->timeout = path->packet_life_time + 1;
577 	add_cm_id_to_port_list(cm_id_priv, av, port);
578 	rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
579 	return 0;
580 }
581 
582 static u32 cm_local_id(__be32 local_id)
583 {
584 	return (__force u32) (local_id ^ cm.random_id_operand);
585 }
586 
587 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
588 {
589 	struct cm_id_private *cm_id_priv;
590 
591 	rcu_read_lock();
592 	cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
593 	if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
594 	    !refcount_inc_not_zero(&cm_id_priv->refcount))
595 		cm_id_priv = NULL;
596 	rcu_read_unlock();
597 
598 	return cm_id_priv;
599 }
600 
601 /*
602  * Trivial helpers to strip endian annotation and compare; the
603  * endianness doesn't actually matter since we just need a stable
604  * order for the RB tree.
605  */
606 static int be32_lt(__be32 a, __be32 b)
607 {
608 	return (__force u32) a < (__force u32) b;
609 }
610 
611 static int be32_gt(__be32 a, __be32 b)
612 {
613 	return (__force u32) a > (__force u32) b;
614 }
615 
616 static int be64_lt(__be64 a, __be64 b)
617 {
618 	return (__force u64) a < (__force u64) b;
619 }
620 
621 static int be64_gt(__be64 a, __be64 b)
622 {
623 	return (__force u64) a > (__force u64) b;
624 }
625 
626 /*
627  * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
628  * if the new ID was inserted, NULL if it could not be inserted due to a
629  * collision, or the existing cm_id_priv ready for shared usage.
630  */
631 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
632 					      ib_cm_handler shared_handler)
633 {
634 	struct rb_node **link = &cm.listen_service_table.rb_node;
635 	struct rb_node *parent = NULL;
636 	struct cm_id_private *cur_cm_id_priv;
637 	__be64 service_id = cm_id_priv->id.service_id;
638 	__be64 service_mask = cm_id_priv->id.service_mask;
639 	unsigned long flags;
640 
641 	spin_lock_irqsave(&cm.lock, flags);
642 	while (*link) {
643 		parent = *link;
644 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
645 					  service_node);
646 		if ((cur_cm_id_priv->id.service_mask & service_id) ==
647 		    (service_mask & cur_cm_id_priv->id.service_id) &&
648 		    (cm_id_priv->id.device == cur_cm_id_priv->id.device)) {
649 			/*
650 			 * Sharing an ib_cm_id with different handlers is not
651 			 * supported
652 			 */
653 			if (cur_cm_id_priv->id.cm_handler != shared_handler ||
654 			    cur_cm_id_priv->id.context ||
655 			    WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
656 				spin_unlock_irqrestore(&cm.lock, flags);
657 				return NULL;
658 			}
659 			refcount_inc(&cur_cm_id_priv->refcount);
660 			cur_cm_id_priv->listen_sharecount++;
661 			spin_unlock_irqrestore(&cm.lock, flags);
662 			return cur_cm_id_priv;
663 		}
664 
665 		if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
666 			link = &(*link)->rb_left;
667 		else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
668 			link = &(*link)->rb_right;
669 		else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
670 			link = &(*link)->rb_left;
671 		else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
672 			link = &(*link)->rb_right;
673 		else
674 			link = &(*link)->rb_right;
675 	}
676 	cm_id_priv->listen_sharecount++;
677 	rb_link_node(&cm_id_priv->service_node, parent, link);
678 	rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
679 	spin_unlock_irqrestore(&cm.lock, flags);
680 	return cm_id_priv;
681 }
682 
683 static struct cm_id_private * cm_find_listen(struct ib_device *device,
684 					     __be64 service_id)
685 {
686 	struct rb_node *node = cm.listen_service_table.rb_node;
687 	struct cm_id_private *cm_id_priv;
688 
689 	while (node) {
690 		cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
691 		if ((cm_id_priv->id.service_mask & service_id) ==
692 		     cm_id_priv->id.service_id &&
693 		    (cm_id_priv->id.device == device)) {
694 			refcount_inc(&cm_id_priv->refcount);
695 			return cm_id_priv;
696 		}
697 		if (device < cm_id_priv->id.device)
698 			node = node->rb_left;
699 		else if (device > cm_id_priv->id.device)
700 			node = node->rb_right;
701 		else if (be64_lt(service_id, cm_id_priv->id.service_id))
702 			node = node->rb_left;
703 		else if (be64_gt(service_id, cm_id_priv->id.service_id))
704 			node = node->rb_right;
705 		else
706 			node = node->rb_right;
707 	}
708 	return NULL;
709 }
710 
711 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
712 						     *timewait_info)
713 {
714 	struct rb_node **link = &cm.remote_id_table.rb_node;
715 	struct rb_node *parent = NULL;
716 	struct cm_timewait_info *cur_timewait_info;
717 	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
718 	__be32 remote_id = timewait_info->work.remote_id;
719 
720 	while (*link) {
721 		parent = *link;
722 		cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
723 					     remote_id_node);
724 		if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
725 			link = &(*link)->rb_left;
726 		else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
727 			link = &(*link)->rb_right;
728 		else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
729 			link = &(*link)->rb_left;
730 		else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
731 			link = &(*link)->rb_right;
732 		else
733 			return cur_timewait_info;
734 	}
735 	timewait_info->inserted_remote_id = 1;
736 	rb_link_node(&timewait_info->remote_id_node, parent, link);
737 	rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
738 	return NULL;
739 }
740 
741 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
742 					       __be32 remote_id)
743 {
744 	struct rb_node *node = cm.remote_id_table.rb_node;
745 	struct cm_timewait_info *timewait_info;
746 	struct cm_id_private *res = NULL;
747 
748 	spin_lock_irq(&cm.lock);
749 	while (node) {
750 		timewait_info = rb_entry(node, struct cm_timewait_info,
751 					 remote_id_node);
752 		if (be32_lt(remote_id, timewait_info->work.remote_id))
753 			node = node->rb_left;
754 		else if (be32_gt(remote_id, timewait_info->work.remote_id))
755 			node = node->rb_right;
756 		else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
757 			node = node->rb_left;
758 		else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
759 			node = node->rb_right;
760 		else {
761 			res = cm_acquire_id(timewait_info->work.local_id,
762 					     timewait_info->work.remote_id);
763 			break;
764 		}
765 	}
766 	spin_unlock_irq(&cm.lock);
767 	return res;
768 }
769 
770 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
771 						      *timewait_info)
772 {
773 	struct rb_node **link = &cm.remote_qp_table.rb_node;
774 	struct rb_node *parent = NULL;
775 	struct cm_timewait_info *cur_timewait_info;
776 	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
777 	__be32 remote_qpn = timewait_info->remote_qpn;
778 
779 	while (*link) {
780 		parent = *link;
781 		cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
782 					     remote_qp_node);
783 		if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
784 			link = &(*link)->rb_left;
785 		else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
786 			link = &(*link)->rb_right;
787 		else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
788 			link = &(*link)->rb_left;
789 		else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
790 			link = &(*link)->rb_right;
791 		else
792 			return cur_timewait_info;
793 	}
794 	timewait_info->inserted_remote_qp = 1;
795 	rb_link_node(&timewait_info->remote_qp_node, parent, link);
796 	rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
797 	return NULL;
798 }
799 
800 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
801 						    *cm_id_priv)
802 {
803 	struct rb_node **link = &cm.remote_sidr_table.rb_node;
804 	struct rb_node *parent = NULL;
805 	struct cm_id_private *cur_cm_id_priv;
806 	union ib_gid *port_gid = &cm_id_priv->av.dgid;
807 	__be32 remote_id = cm_id_priv->id.remote_id;
808 
809 	while (*link) {
810 		parent = *link;
811 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
812 					  sidr_id_node);
813 		if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
814 			link = &(*link)->rb_left;
815 		else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
816 			link = &(*link)->rb_right;
817 		else {
818 			int cmp;
819 			cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
820 				     sizeof *port_gid);
821 			if (cmp < 0)
822 				link = &(*link)->rb_left;
823 			else if (cmp > 0)
824 				link = &(*link)->rb_right;
825 			else
826 				return cur_cm_id_priv;
827 		}
828 	}
829 	rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
830 	rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
831 	return NULL;
832 }
833 
834 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
835 					      ib_cm_handler cm_handler,
836 					      void *context)
837 {
838 	struct cm_id_private *cm_id_priv;
839 	u32 id;
840 	int ret;
841 
842 	cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
843 	if (!cm_id_priv)
844 		return ERR_PTR(-ENOMEM);
845 
846 	cm_id_priv->id.state = IB_CM_IDLE;
847 	cm_id_priv->id.device = device;
848 	cm_id_priv->id.cm_handler = cm_handler;
849 	cm_id_priv->id.context = context;
850 	cm_id_priv->id.remote_cm_qpn = 1;
851 
852 	RB_CLEAR_NODE(&cm_id_priv->service_node);
853 	RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
854 	spin_lock_init(&cm_id_priv->lock);
855 	init_completion(&cm_id_priv->comp);
856 	INIT_LIST_HEAD(&cm_id_priv->work_list);
857 	INIT_LIST_HEAD(&cm_id_priv->prim_list);
858 	INIT_LIST_HEAD(&cm_id_priv->altr_list);
859 	atomic_set(&cm_id_priv->work_count, -1);
860 	refcount_set(&cm_id_priv->refcount, 1);
861 
862 	ret = xa_alloc_cyclic_irq(&cm.local_id_table, &id, NULL, xa_limit_32b,
863 				  &cm.local_id_next, GFP_KERNEL);
864 	if (ret < 0)
865 		goto error;
866 	cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
867 
868 	return cm_id_priv;
869 
870 error:
871 	kfree(cm_id_priv);
872 	return ERR_PTR(ret);
873 }
874 
875 /*
876  * Make the ID visible to the MAD handlers and other threads that use the
877  * xarray.
878  */
879 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
880 {
881 	xa_store_irq(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
882 		     cm_id_priv, GFP_KERNEL);
883 }
884 
885 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
886 				 ib_cm_handler cm_handler,
887 				 void *context)
888 {
889 	struct cm_id_private *cm_id_priv;
890 
891 	cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
892 	if (IS_ERR(cm_id_priv))
893 		return ERR_CAST(cm_id_priv);
894 
895 	cm_finalize_id(cm_id_priv);
896 	return &cm_id_priv->id;
897 }
898 EXPORT_SYMBOL(ib_create_cm_id);
899 
900 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
901 {
902 	struct cm_work *work;
903 
904 	if (list_empty(&cm_id_priv->work_list))
905 		return NULL;
906 
907 	work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
908 	list_del(&work->list);
909 	return work;
910 }
911 
912 static void cm_free_work(struct cm_work *work)
913 {
914 	if (work->mad_recv_wc)
915 		ib_free_recv_mad(work->mad_recv_wc);
916 	kfree(work);
917 }
918 
919 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
920 				 struct cm_work *work)
921 	__releases(&cm_id_priv->lock)
922 {
923 	bool immediate;
924 
925 	/*
926 	 * To deliver the event to the user callback we have the drop the
927 	 * spinlock, however, we need to ensure that the user callback is single
928 	 * threaded and receives events in the temporal order. If there are
929 	 * already events being processed then thread new events onto a list,
930 	 * the thread currently processing will pick them up.
931 	 */
932 	immediate = atomic_inc_and_test(&cm_id_priv->work_count);
933 	if (!immediate) {
934 		list_add_tail(&work->list, &cm_id_priv->work_list);
935 		/*
936 		 * This routine always consumes incoming reference. Once queued
937 		 * to the work_list then a reference is held by the thread
938 		 * currently running cm_process_work() and this reference is not
939 		 * needed.
940 		 */
941 		cm_deref_id(cm_id_priv);
942 	}
943 	spin_unlock_irq(&cm_id_priv->lock);
944 
945 	if (immediate)
946 		cm_process_work(cm_id_priv, work);
947 }
948 
949 static inline int cm_convert_to_ms(int iba_time)
950 {
951 	/* approximate conversion to ms from 4.096us x 2^iba_time */
952 	return 1 << max(iba_time - 8, 0);
953 }
954 
955 /*
956  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
957  * Because of how ack_timeout is stored, adding one doubles the timeout.
958  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
959  * increment it (round up) only if the other is within 50%.
960  */
961 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
962 {
963 	int ack_timeout = packet_life_time + 1;
964 
965 	if (ack_timeout >= ca_ack_delay)
966 		ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
967 	else
968 		ack_timeout = ca_ack_delay +
969 			      (ack_timeout >= (ca_ack_delay - 1));
970 
971 	return min(31, ack_timeout);
972 }
973 
974 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
975 {
976 	struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
977 
978 	if (timewait_info->inserted_remote_id) {
979 		rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
980 		timewait_info->inserted_remote_id = 0;
981 	}
982 
983 	if (timewait_info->inserted_remote_qp) {
984 		rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
985 		timewait_info->inserted_remote_qp = 0;
986 	}
987 }
988 
989 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
990 {
991 	struct cm_timewait_info *timewait_info;
992 
993 	timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
994 	if (!timewait_info)
995 		return ERR_PTR(-ENOMEM);
996 
997 	timewait_info->work.local_id = local_id;
998 	INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
999 	timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
1000 	return timewait_info;
1001 }
1002 
1003 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
1004 {
1005 	int wait_time;
1006 	unsigned long flags;
1007 	struct cm_device *cm_dev;
1008 
1009 	lockdep_assert_held(&cm_id_priv->lock);
1010 
1011 	cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
1012 	if (!cm_dev)
1013 		return;
1014 
1015 	spin_lock_irqsave(&cm.lock, flags);
1016 	cm_remove_remote(cm_id_priv);
1017 	list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
1018 	spin_unlock_irqrestore(&cm.lock, flags);
1019 
1020 	/*
1021 	 * The cm_id could be destroyed by the user before we exit timewait.
1022 	 * To protect against this, we search for the cm_id after exiting
1023 	 * timewait before notifying the user that we've exited timewait.
1024 	 */
1025 	cm_id_priv->id.state = IB_CM_TIMEWAIT;
1026 	wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
1027 
1028 	/* Check if the device started its remove_one */
1029 	spin_lock_irqsave(&cm.lock, flags);
1030 	if (!cm_dev->going_down)
1031 		queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1032 				   msecs_to_jiffies(wait_time));
1033 	spin_unlock_irqrestore(&cm.lock, flags);
1034 
1035 	/*
1036 	 * The timewait_info is converted into a work and gets freed during
1037 	 * cm_free_work() in cm_timewait_handler().
1038 	 */
1039 	BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1040 	cm_id_priv->timewait_info = NULL;
1041 }
1042 
1043 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1044 {
1045 	unsigned long flags;
1046 
1047 	lockdep_assert_held(&cm_id_priv->lock);
1048 
1049 	cm_id_priv->id.state = IB_CM_IDLE;
1050 	if (cm_id_priv->timewait_info) {
1051 		spin_lock_irqsave(&cm.lock, flags);
1052 		cm_remove_remote(cm_id_priv);
1053 		spin_unlock_irqrestore(&cm.lock, flags);
1054 		kfree(cm_id_priv->timewait_info);
1055 		cm_id_priv->timewait_info = NULL;
1056 	}
1057 }
1058 
1059 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1060 {
1061 	struct cm_id_private *cm_id_priv;
1062 	struct cm_work *work;
1063 
1064 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1065 	spin_lock_irq(&cm_id_priv->lock);
1066 retest:
1067 	switch (cm_id->state) {
1068 	case IB_CM_LISTEN:
1069 		spin_lock(&cm.lock);
1070 		if (--cm_id_priv->listen_sharecount > 0) {
1071 			/* The id is still shared. */
1072 			WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1073 			spin_unlock(&cm.lock);
1074 			spin_unlock_irq(&cm_id_priv->lock);
1075 			cm_deref_id(cm_id_priv);
1076 			return;
1077 		}
1078 		cm_id->state = IB_CM_IDLE;
1079 		rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1080 		RB_CLEAR_NODE(&cm_id_priv->service_node);
1081 		spin_unlock(&cm.lock);
1082 		break;
1083 	case IB_CM_SIDR_REQ_SENT:
1084 		cm_id->state = IB_CM_IDLE;
1085 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1086 		break;
1087 	case IB_CM_SIDR_REQ_RCVD:
1088 		cm_send_sidr_rep_locked(cm_id_priv,
1089 					&(struct ib_cm_sidr_rep_param){
1090 						.status = IB_SIDR_REJECT });
1091 		/* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1092 		cm_id->state = IB_CM_IDLE;
1093 		break;
1094 	case IB_CM_REQ_SENT:
1095 	case IB_CM_MRA_REQ_RCVD:
1096 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1097 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1098 				   &cm_id_priv->id.device->node_guid,
1099 				   sizeof(cm_id_priv->id.device->node_guid),
1100 				   NULL, 0);
1101 		break;
1102 	case IB_CM_REQ_RCVD:
1103 		if (err == -ENOMEM) {
1104 			/* Do not reject to allow future retries. */
1105 			cm_reset_to_idle(cm_id_priv);
1106 		} else {
1107 			cm_send_rej_locked(cm_id_priv,
1108 					   IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1109 					   NULL, 0);
1110 		}
1111 		break;
1112 	case IB_CM_REP_SENT:
1113 	case IB_CM_MRA_REP_RCVD:
1114 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1115 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1116 				   0, NULL, 0);
1117 		goto retest;
1118 	case IB_CM_MRA_REQ_SENT:
1119 	case IB_CM_REP_RCVD:
1120 	case IB_CM_MRA_REP_SENT:
1121 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1122 				   0, NULL, 0);
1123 		break;
1124 	case IB_CM_ESTABLISHED:
1125 		if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1126 			cm_id->state = IB_CM_IDLE;
1127 			break;
1128 		}
1129 		cm_send_dreq_locked(cm_id_priv, NULL, 0);
1130 		goto retest;
1131 	case IB_CM_DREQ_SENT:
1132 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1133 		cm_enter_timewait(cm_id_priv);
1134 		goto retest;
1135 	case IB_CM_DREQ_RCVD:
1136 		cm_send_drep_locked(cm_id_priv, NULL, 0);
1137 		WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1138 		goto retest;
1139 	case IB_CM_TIMEWAIT:
1140 		/*
1141 		 * The cm_acquire_id in cm_timewait_handler will stop working
1142 		 * once we do xa_erase below, so just move to idle here for
1143 		 * consistency.
1144 		 */
1145 		cm_id->state = IB_CM_IDLE;
1146 		break;
1147 	case IB_CM_IDLE:
1148 		break;
1149 	}
1150 	WARN_ON(cm_id->state != IB_CM_IDLE);
1151 
1152 	spin_lock(&cm.lock);
1153 	/* Required for cleanup paths related cm_req_handler() */
1154 	if (cm_id_priv->timewait_info) {
1155 		cm_remove_remote(cm_id_priv);
1156 		kfree(cm_id_priv->timewait_info);
1157 		cm_id_priv->timewait_info = NULL;
1158 	}
1159 	if (!list_empty(&cm_id_priv->altr_list) &&
1160 	    (!cm_id_priv->altr_send_port_not_ready))
1161 		list_del(&cm_id_priv->altr_list);
1162 	if (!list_empty(&cm_id_priv->prim_list) &&
1163 	    (!cm_id_priv->prim_send_port_not_ready))
1164 		list_del(&cm_id_priv->prim_list);
1165 	WARN_ON(cm_id_priv->listen_sharecount);
1166 	WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1167 	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1168 		rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1169 	spin_unlock(&cm.lock);
1170 	spin_unlock_irq(&cm_id_priv->lock);
1171 
1172 	xa_erase_irq(&cm.local_id_table, cm_local_id(cm_id->local_id));
1173 	cm_deref_id(cm_id_priv);
1174 	wait_for_completion(&cm_id_priv->comp);
1175 	while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1176 		cm_free_work(work);
1177 
1178 	rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr);
1179 	rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr);
1180 	kfree(cm_id_priv->private_data);
1181 	kfree_rcu(cm_id_priv, rcu);
1182 }
1183 
1184 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1185 {
1186 	cm_destroy_id(cm_id, 0);
1187 }
1188 EXPORT_SYMBOL(ib_destroy_cm_id);
1189 
1190 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id,
1191 			  __be64 service_mask)
1192 {
1193 	service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1194 	service_id &= service_mask;
1195 	if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1196 	    (service_id != IB_CM_ASSIGN_SERVICE_ID))
1197 		return -EINVAL;
1198 
1199 	if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1200 		cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1201 		cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1202 	} else {
1203 		cm_id_priv->id.service_id = service_id;
1204 		cm_id_priv->id.service_mask = service_mask;
1205 	}
1206 	return 0;
1207 }
1208 
1209 /**
1210  * ib_cm_listen - Initiates listening on the specified service ID for
1211  *   connection and service ID resolution requests.
1212  * @cm_id: Connection identifier associated with the listen request.
1213  * @service_id: Service identifier matched against incoming connection
1214  *   and service ID resolution requests.  The service ID should be specified
1215  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1216  *   assign a service ID to the caller.
1217  * @service_mask: Mask applied to service ID used to listen across a
1218  *   range of service IDs.  If set to 0, the service ID is matched
1219  *   exactly.  This parameter is ignored if %service_id is set to
1220  *   IB_CM_ASSIGN_SERVICE_ID.
1221  */
1222 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1223 {
1224 	struct cm_id_private *cm_id_priv =
1225 		container_of(cm_id, struct cm_id_private, id);
1226 	unsigned long flags;
1227 	int ret;
1228 
1229 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1230 	if (cm_id_priv->id.state != IB_CM_IDLE) {
1231 		ret = -EINVAL;
1232 		goto out;
1233 	}
1234 
1235 	ret = cm_init_listen(cm_id_priv, service_id, service_mask);
1236 	if (ret)
1237 		goto out;
1238 
1239 	if (!cm_insert_listen(cm_id_priv, NULL)) {
1240 		ret = -EBUSY;
1241 		goto out;
1242 	}
1243 
1244 	cm_id_priv->id.state = IB_CM_LISTEN;
1245 	ret = 0;
1246 
1247 out:
1248 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1249 	return ret;
1250 }
1251 EXPORT_SYMBOL(ib_cm_listen);
1252 
1253 /**
1254  * Create a new listening ib_cm_id and listen on the given service ID.
1255  *
1256  * If there's an existing ID listening on that same device and service ID,
1257  * return it.
1258  *
1259  * @device: Device associated with the cm_id.  All related communication will
1260  * be associated with the specified device.
1261  * @cm_handler: Callback invoked to notify the user of CM events.
1262  * @service_id: Service identifier matched against incoming connection
1263  *   and service ID resolution requests.  The service ID should be specified
1264  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1265  *   assign a service ID to the caller.
1266  *
1267  * Callers should call ib_destroy_cm_id when done with the listener ID.
1268  */
1269 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1270 				     ib_cm_handler cm_handler,
1271 				     __be64 service_id)
1272 {
1273 	struct cm_id_private *listen_id_priv;
1274 	struct cm_id_private *cm_id_priv;
1275 	int err = 0;
1276 
1277 	/* Create an ID in advance, since the creation may sleep */
1278 	cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1279 	if (IS_ERR(cm_id_priv))
1280 		return ERR_CAST(cm_id_priv);
1281 
1282 	err = cm_init_listen(cm_id_priv, service_id, 0);
1283 	if (err)
1284 		return ERR_PTR(err);
1285 
1286 	spin_lock_irq(&cm_id_priv->lock);
1287 	listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1288 	if (listen_id_priv != cm_id_priv) {
1289 		spin_unlock_irq(&cm_id_priv->lock);
1290 		ib_destroy_cm_id(&cm_id_priv->id);
1291 		if (!listen_id_priv)
1292 			return ERR_PTR(-EINVAL);
1293 		return &listen_id_priv->id;
1294 	}
1295 	cm_id_priv->id.state = IB_CM_LISTEN;
1296 	spin_unlock_irq(&cm_id_priv->lock);
1297 
1298 	/*
1299 	 * A listen ID does not need to be in the xarray since it does not
1300 	 * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1301 	 * and does not enter timewait.
1302 	 */
1303 
1304 	return &cm_id_priv->id;
1305 }
1306 EXPORT_SYMBOL(ib_cm_insert_listen);
1307 
1308 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1309 {
1310 	u64 hi_tid, low_tid;
1311 
1312 	hi_tid   = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1313 	low_tid  = (u64)cm_id_priv->id.local_id;
1314 	return cpu_to_be64(hi_tid | low_tid);
1315 }
1316 
1317 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1318 			      __be16 attr_id, __be64 tid)
1319 {
1320 	hdr->base_version  = IB_MGMT_BASE_VERSION;
1321 	hdr->mgmt_class	   = IB_MGMT_CLASS_CM;
1322 	hdr->class_version = IB_CM_CLASS_VERSION;
1323 	hdr->method	   = IB_MGMT_METHOD_SEND;
1324 	hdr->attr_id	   = attr_id;
1325 	hdr->tid	   = tid;
1326 }
1327 
1328 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1329 				  __be64 tid, u32 attr_mod)
1330 {
1331 	cm_format_mad_hdr(hdr, attr_id, tid);
1332 	hdr->attr_mod = cpu_to_be32(attr_mod);
1333 }
1334 
1335 static void cm_format_req(struct cm_req_msg *req_msg,
1336 			  struct cm_id_private *cm_id_priv,
1337 			  struct ib_cm_req_param *param)
1338 {
1339 	struct sa_path_rec *pri_path = param->primary_path;
1340 	struct sa_path_rec *alt_path = param->alternate_path;
1341 	bool pri_ext = false;
1342 
1343 	if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1344 		pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1345 					      pri_path->opa.slid);
1346 
1347 	cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1348 			      cm_form_tid(cm_id_priv), param->ece.attr_mod);
1349 
1350 	IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1351 		be32_to_cpu(cm_id_priv->id.local_id));
1352 	IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1353 	IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1354 		be64_to_cpu(cm_id_priv->id.device->node_guid));
1355 	IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1356 	IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1357 	IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1358 		param->remote_cm_response_timeout);
1359 	cm_req_set_qp_type(req_msg, param->qp_type);
1360 	IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1361 	IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1362 	IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1363 		param->local_cm_response_timeout);
1364 	IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1365 		be16_to_cpu(param->primary_path->pkey));
1366 	IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1367 		param->primary_path->mtu);
1368 	IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1369 
1370 	if (param->qp_type != IB_QPT_XRC_INI) {
1371 		IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1372 			param->responder_resources);
1373 		IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1374 		IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1375 			param->rnr_retry_count);
1376 		IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1377 	}
1378 
1379 	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1380 		pri_path->sgid;
1381 	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1382 		pri_path->dgid;
1383 	if (pri_ext) {
1384 		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1385 			->global.interface_id =
1386 			OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1387 		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1388 			->global.interface_id =
1389 			OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1390 	}
1391 	if (pri_path->hop_limit <= 1) {
1392 		IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1393 			be16_to_cpu(pri_ext ? 0 :
1394 					      htons(ntohl(sa_path_get_slid(
1395 						      pri_path)))));
1396 		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1397 			be16_to_cpu(pri_ext ? 0 :
1398 					      htons(ntohl(sa_path_get_dlid(
1399 						      pri_path)))));
1400 	} else {
1401 		/* Work-around until there's a way to obtain remote LID info */
1402 		IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1403 			be16_to_cpu(IB_LID_PERMISSIVE));
1404 		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1405 			be16_to_cpu(IB_LID_PERMISSIVE));
1406 	}
1407 	IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1408 		be32_to_cpu(pri_path->flow_label));
1409 	IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1410 	IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1411 	IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1412 	IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1413 	IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1414 		(pri_path->hop_limit <= 1));
1415 	IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1416 		cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1417 			       pri_path->packet_life_time));
1418 
1419 	if (alt_path) {
1420 		bool alt_ext = false;
1421 
1422 		if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1423 			alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1424 						      alt_path->opa.slid);
1425 
1426 		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1427 			alt_path->sgid;
1428 		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1429 			alt_path->dgid;
1430 		if (alt_ext) {
1431 			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1432 					req_msg)
1433 				->global.interface_id =
1434 				OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1435 			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1436 					req_msg)
1437 				->global.interface_id =
1438 				OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1439 		}
1440 		if (alt_path->hop_limit <= 1) {
1441 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1442 				be16_to_cpu(
1443 					alt_ext ? 0 :
1444 						  htons(ntohl(sa_path_get_slid(
1445 							  alt_path)))));
1446 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1447 				be16_to_cpu(
1448 					alt_ext ? 0 :
1449 						  htons(ntohl(sa_path_get_dlid(
1450 							  alt_path)))));
1451 		} else {
1452 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1453 				be16_to_cpu(IB_LID_PERMISSIVE));
1454 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1455 				be16_to_cpu(IB_LID_PERMISSIVE));
1456 		}
1457 		IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1458 			be32_to_cpu(alt_path->flow_label));
1459 		IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1460 		IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1461 			alt_path->traffic_class);
1462 		IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1463 			alt_path->hop_limit);
1464 		IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1465 		IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1466 			(alt_path->hop_limit <= 1));
1467 		IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1468 			cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1469 				       alt_path->packet_life_time));
1470 	}
1471 	IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1472 
1473 	if (param->private_data && param->private_data_len)
1474 		IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1475 			    param->private_data_len);
1476 }
1477 
1478 static int cm_validate_req_param(struct ib_cm_req_param *param)
1479 {
1480 	if (!param->primary_path)
1481 		return -EINVAL;
1482 
1483 	if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1484 	    param->qp_type != IB_QPT_XRC_INI)
1485 		return -EINVAL;
1486 
1487 	if (param->private_data &&
1488 	    param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1489 		return -EINVAL;
1490 
1491 	if (param->alternate_path &&
1492 	    (param->alternate_path->pkey != param->primary_path->pkey ||
1493 	     param->alternate_path->mtu != param->primary_path->mtu))
1494 		return -EINVAL;
1495 
1496 	return 0;
1497 }
1498 
1499 int ib_send_cm_req(struct ib_cm_id *cm_id,
1500 		   struct ib_cm_req_param *param)
1501 {
1502 	struct cm_id_private *cm_id_priv;
1503 	struct cm_req_msg *req_msg;
1504 	unsigned long flags;
1505 	int ret;
1506 
1507 	ret = cm_validate_req_param(param);
1508 	if (ret)
1509 		return ret;
1510 
1511 	/* Verify that we're not in timewait. */
1512 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1513 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1514 	if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1515 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1516 		ret = -EINVAL;
1517 		goto out;
1518 	}
1519 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1520 
1521 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1522 							    id.local_id);
1523 	if (IS_ERR(cm_id_priv->timewait_info)) {
1524 		ret = PTR_ERR(cm_id_priv->timewait_info);
1525 		goto out;
1526 	}
1527 
1528 	ret = cm_init_av_by_path(param->primary_path,
1529 				 param->ppath_sgid_attr, &cm_id_priv->av,
1530 				 cm_id_priv);
1531 	if (ret)
1532 		goto out;
1533 	if (param->alternate_path) {
1534 		ret = cm_init_av_by_path(param->alternate_path, NULL,
1535 					 &cm_id_priv->alt_av, cm_id_priv);
1536 		if (ret)
1537 			goto out;
1538 	}
1539 	cm_id->service_id = param->service_id;
1540 	cm_id->service_mask = ~cpu_to_be64(0);
1541 	cm_id_priv->timeout_ms = cm_convert_to_ms(
1542 				    param->primary_path->packet_life_time) * 2 +
1543 				 cm_convert_to_ms(
1544 				    param->remote_cm_response_timeout);
1545 	cm_id_priv->max_cm_retries = param->max_cm_retries;
1546 	cm_id_priv->initiator_depth = param->initiator_depth;
1547 	cm_id_priv->responder_resources = param->responder_resources;
1548 	cm_id_priv->retry_count = param->retry_count;
1549 	cm_id_priv->path_mtu = param->primary_path->mtu;
1550 	cm_id_priv->pkey = param->primary_path->pkey;
1551 	cm_id_priv->qp_type = param->qp_type;
1552 
1553 	ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1554 	if (ret)
1555 		goto out;
1556 
1557 	req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1558 	cm_format_req(req_msg, cm_id_priv, param);
1559 	cm_id_priv->tid = req_msg->hdr.tid;
1560 	cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1561 	cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1562 
1563 	cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1564 	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1565 
1566 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1567 	ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1568 	if (ret) {
1569 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1570 		goto error2;
1571 	}
1572 	BUG_ON(cm_id->state != IB_CM_IDLE);
1573 	cm_id->state = IB_CM_REQ_SENT;
1574 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1575 	return 0;
1576 
1577 error2:	cm_free_msg(cm_id_priv->msg);
1578 out:	return ret;
1579 }
1580 EXPORT_SYMBOL(ib_send_cm_req);
1581 
1582 static int cm_issue_rej(struct cm_port *port,
1583 			struct ib_mad_recv_wc *mad_recv_wc,
1584 			enum ib_cm_rej_reason reason,
1585 			enum cm_msg_response msg_rejected,
1586 			void *ari, u8 ari_length)
1587 {
1588 	struct ib_mad_send_buf *msg = NULL;
1589 	struct cm_rej_msg *rej_msg, *rcv_msg;
1590 	int ret;
1591 
1592 	ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1593 	if (ret)
1594 		return ret;
1595 
1596 	/* We just need common CM header information.  Cast to any message. */
1597 	rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1598 	rej_msg = (struct cm_rej_msg *) msg->mad;
1599 
1600 	cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1601 	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1602 		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1603 	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1604 		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1605 	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1606 	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1607 
1608 	if (ari && ari_length) {
1609 		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1610 		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1611 	}
1612 
1613 	ret = ib_post_send_mad(msg, NULL);
1614 	if (ret)
1615 		cm_free_msg(msg);
1616 
1617 	return ret;
1618 }
1619 
1620 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1621 {
1622 	return ((cpu_to_be16(
1623 			IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1624 		(ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1625 					       req_msg))));
1626 }
1627 
1628 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
1629 				 struct sa_path_rec *path, union ib_gid *gid)
1630 {
1631 	if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1632 		path->rec_type = SA_PATH_REC_TYPE_OPA;
1633 	else
1634 		path->rec_type = SA_PATH_REC_TYPE_IB;
1635 }
1636 
1637 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1638 					struct sa_path_rec *primary_path,
1639 					struct sa_path_rec *alt_path)
1640 {
1641 	u32 lid;
1642 
1643 	if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1644 		sa_path_set_dlid(primary_path,
1645 				 IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
1646 					 req_msg));
1647 		sa_path_set_slid(primary_path,
1648 				 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1649 					 req_msg));
1650 	} else {
1651 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1652 			CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1653 		sa_path_set_dlid(primary_path, lid);
1654 
1655 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1656 			CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1657 		sa_path_set_slid(primary_path, lid);
1658 	}
1659 
1660 	if (!cm_req_has_alt_path(req_msg))
1661 		return;
1662 
1663 	if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1664 		sa_path_set_dlid(alt_path,
1665 				 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1666 					 req_msg));
1667 		sa_path_set_slid(alt_path,
1668 				 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1669 					 req_msg));
1670 	} else {
1671 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1672 			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1673 		sa_path_set_dlid(alt_path, lid);
1674 
1675 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1676 			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1677 		sa_path_set_slid(alt_path, lid);
1678 	}
1679 }
1680 
1681 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1682 				     struct sa_path_rec *primary_path,
1683 				     struct sa_path_rec *alt_path)
1684 {
1685 	primary_path->dgid =
1686 		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1687 	primary_path->sgid =
1688 		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1689 	primary_path->flow_label =
1690 		cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1691 	primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1692 	primary_path->traffic_class =
1693 		IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1694 	primary_path->reversible = 1;
1695 	primary_path->pkey =
1696 		cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1697 	primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1698 	primary_path->mtu_selector = IB_SA_EQ;
1699 	primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1700 	primary_path->rate_selector = IB_SA_EQ;
1701 	primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1702 	primary_path->packet_life_time_selector = IB_SA_EQ;
1703 	primary_path->packet_life_time =
1704 		IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1705 	primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1706 	primary_path->service_id =
1707 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1708 	if (sa_path_is_roce(primary_path))
1709 		primary_path->roce.route_resolved = false;
1710 
1711 	if (cm_req_has_alt_path(req_msg)) {
1712 		alt_path->dgid = *IBA_GET_MEM_PTR(
1713 			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1714 		alt_path->sgid = *IBA_GET_MEM_PTR(
1715 			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1716 		alt_path->flow_label = cpu_to_be32(
1717 			IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1718 		alt_path->hop_limit =
1719 			IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1720 		alt_path->traffic_class =
1721 			IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1722 		alt_path->reversible = 1;
1723 		alt_path->pkey =
1724 			cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1725 		alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1726 		alt_path->mtu_selector = IB_SA_EQ;
1727 		alt_path->mtu =
1728 			IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1729 		alt_path->rate_selector = IB_SA_EQ;
1730 		alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1731 		alt_path->packet_life_time_selector = IB_SA_EQ;
1732 		alt_path->packet_life_time =
1733 			IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1734 		alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1735 		alt_path->service_id =
1736 			cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1737 
1738 		if (sa_path_is_roce(alt_path))
1739 			alt_path->roce.route_resolved = false;
1740 	}
1741 	cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
1742 }
1743 
1744 static u16 cm_get_bth_pkey(struct cm_work *work)
1745 {
1746 	struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1747 	u8 port_num = work->port->port_num;
1748 	u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1749 	u16 pkey;
1750 	int ret;
1751 
1752 	ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1753 	if (ret) {
1754 		dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",
1755 				     port_num, pkey_index, ret);
1756 		return 0;
1757 	}
1758 
1759 	return pkey;
1760 }
1761 
1762 /**
1763  * Convert OPA SGID to IB SGID
1764  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1765  * reject them as the local_gid will not match the sgid. Therefore,
1766  * change the pathrec's SGID to an IB SGID.
1767  *
1768  * @work: Work completion
1769  * @path: Path record
1770  */
1771 static void cm_opa_to_ib_sgid(struct cm_work *work,
1772 			      struct sa_path_rec *path)
1773 {
1774 	struct ib_device *dev = work->port->cm_dev->ib_device;
1775 	u8 port_num = work->port->port_num;
1776 
1777 	if (rdma_cap_opa_ah(dev, port_num) &&
1778 	    (ib_is_opa_gid(&path->sgid))) {
1779 		union ib_gid sgid;
1780 
1781 		if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1782 			dev_warn(&dev->dev,
1783 				 "Error updating sgid in CM request\n");
1784 			return;
1785 		}
1786 
1787 		path->sgid = sgid;
1788 	}
1789 }
1790 
1791 static void cm_format_req_event(struct cm_work *work,
1792 				struct cm_id_private *cm_id_priv,
1793 				struct ib_cm_id *listen_id)
1794 {
1795 	struct cm_req_msg *req_msg;
1796 	struct ib_cm_req_event_param *param;
1797 
1798 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1799 	param = &work->cm_event.param.req_rcvd;
1800 	param->listen_id = listen_id;
1801 	param->bth_pkey = cm_get_bth_pkey(work);
1802 	param->port = cm_id_priv->av.port->port_num;
1803 	param->primary_path = &work->path[0];
1804 	cm_opa_to_ib_sgid(work, param->primary_path);
1805 	if (cm_req_has_alt_path(req_msg)) {
1806 		param->alternate_path = &work->path[1];
1807 		cm_opa_to_ib_sgid(work, param->alternate_path);
1808 	} else {
1809 		param->alternate_path = NULL;
1810 	}
1811 	param->remote_ca_guid =
1812 		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1813 	param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1814 	param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1815 	param->qp_type = cm_req_get_qp_type(req_msg);
1816 	param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1817 	param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1818 	param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1819 	param->local_cm_response_timeout =
1820 		IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1821 	param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1822 	param->remote_cm_response_timeout =
1823 		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1824 	param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1825 	param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1826 	param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1827 	param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1828 	param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1829 	param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1830 
1831 	work->cm_event.private_data =
1832 		IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1833 }
1834 
1835 static void cm_process_work(struct cm_id_private *cm_id_priv,
1836 			    struct cm_work *work)
1837 {
1838 	int ret;
1839 
1840 	/* We will typically only have the current event to report. */
1841 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1842 	cm_free_work(work);
1843 
1844 	while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1845 		spin_lock_irq(&cm_id_priv->lock);
1846 		work = cm_dequeue_work(cm_id_priv);
1847 		spin_unlock_irq(&cm_id_priv->lock);
1848 		if (!work)
1849 			return;
1850 
1851 		ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1852 						&work->cm_event);
1853 		cm_free_work(work);
1854 	}
1855 	cm_deref_id(cm_id_priv);
1856 	if (ret)
1857 		cm_destroy_id(&cm_id_priv->id, ret);
1858 }
1859 
1860 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1861 			  struct cm_id_private *cm_id_priv,
1862 			  enum cm_msg_response msg_mraed, u8 service_timeout,
1863 			  const void *private_data, u8 private_data_len)
1864 {
1865 	cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1866 	IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1867 	IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1868 		be32_to_cpu(cm_id_priv->id.local_id));
1869 	IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1870 		be32_to_cpu(cm_id_priv->id.remote_id));
1871 	IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1872 
1873 	if (private_data && private_data_len)
1874 		IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1875 			    private_data_len);
1876 }
1877 
1878 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1879 			  struct cm_id_private *cm_id_priv,
1880 			  enum ib_cm_rej_reason reason, void *ari,
1881 			  u8 ari_length, const void *private_data,
1882 			  u8 private_data_len, enum ib_cm_state state)
1883 {
1884 	lockdep_assert_held(&cm_id_priv->lock);
1885 
1886 	cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1887 	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1888 		be32_to_cpu(cm_id_priv->id.remote_id));
1889 
1890 	switch (state) {
1891 	case IB_CM_REQ_RCVD:
1892 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1893 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1894 		break;
1895 	case IB_CM_MRA_REQ_SENT:
1896 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1897 			be32_to_cpu(cm_id_priv->id.local_id));
1898 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1899 		break;
1900 	case IB_CM_REP_RCVD:
1901 	case IB_CM_MRA_REP_SENT:
1902 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1903 			be32_to_cpu(cm_id_priv->id.local_id));
1904 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1905 		break;
1906 	default:
1907 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1908 			be32_to_cpu(cm_id_priv->id.local_id));
1909 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1910 			CM_MSG_RESPONSE_OTHER);
1911 		break;
1912 	}
1913 
1914 	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1915 	if (ari && ari_length) {
1916 		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1917 		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1918 	}
1919 
1920 	if (private_data && private_data_len)
1921 		IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1922 			    private_data_len);
1923 }
1924 
1925 static void cm_dup_req_handler(struct cm_work *work,
1926 			       struct cm_id_private *cm_id_priv)
1927 {
1928 	struct ib_mad_send_buf *msg = NULL;
1929 	int ret;
1930 
1931 	atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1932 			counter[CM_REQ_COUNTER]);
1933 
1934 	/* Quick state check to discard duplicate REQs. */
1935 	spin_lock_irq(&cm_id_priv->lock);
1936 	if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1937 		spin_unlock_irq(&cm_id_priv->lock);
1938 		return;
1939 	}
1940 	spin_unlock_irq(&cm_id_priv->lock);
1941 
1942 	ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1943 	if (ret)
1944 		return;
1945 
1946 	spin_lock_irq(&cm_id_priv->lock);
1947 	switch (cm_id_priv->id.state) {
1948 	case IB_CM_MRA_REQ_SENT:
1949 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1950 			      CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1951 			      cm_id_priv->private_data,
1952 			      cm_id_priv->private_data_len);
1953 		break;
1954 	case IB_CM_TIMEWAIT:
1955 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1956 			      IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1957 			      IB_CM_TIMEWAIT);
1958 		break;
1959 	default:
1960 		goto unlock;
1961 	}
1962 	spin_unlock_irq(&cm_id_priv->lock);
1963 
1964 	ret = ib_post_send_mad(msg, NULL);
1965 	if (ret)
1966 		goto free;
1967 	return;
1968 
1969 unlock:	spin_unlock_irq(&cm_id_priv->lock);
1970 free:	cm_free_msg(msg);
1971 }
1972 
1973 static struct cm_id_private * cm_match_req(struct cm_work *work,
1974 					   struct cm_id_private *cm_id_priv)
1975 {
1976 	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1977 	struct cm_timewait_info *timewait_info;
1978 	struct cm_req_msg *req_msg;
1979 
1980 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1981 
1982 	/* Check for possible duplicate REQ. */
1983 	spin_lock_irq(&cm.lock);
1984 	timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1985 	if (timewait_info) {
1986 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1987 					   timewait_info->work.remote_id);
1988 		spin_unlock_irq(&cm.lock);
1989 		if (cur_cm_id_priv) {
1990 			cm_dup_req_handler(work, cur_cm_id_priv);
1991 			cm_deref_id(cur_cm_id_priv);
1992 		}
1993 		return NULL;
1994 	}
1995 
1996 	/* Check for stale connections. */
1997 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1998 	if (timewait_info) {
1999 		cm_remove_remote(cm_id_priv);
2000 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2001 					   timewait_info->work.remote_id);
2002 
2003 		spin_unlock_irq(&cm.lock);
2004 		cm_issue_rej(work->port, work->mad_recv_wc,
2005 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2006 			     NULL, 0);
2007 		if (cur_cm_id_priv) {
2008 			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2009 			cm_deref_id(cur_cm_id_priv);
2010 		}
2011 		return NULL;
2012 	}
2013 
2014 	/* Find matching listen request. */
2015 	listen_cm_id_priv = cm_find_listen(
2016 		cm_id_priv->id.device,
2017 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2018 	if (!listen_cm_id_priv) {
2019 		cm_remove_remote(cm_id_priv);
2020 		spin_unlock_irq(&cm.lock);
2021 		cm_issue_rej(work->port, work->mad_recv_wc,
2022 			     IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2023 			     NULL, 0);
2024 		return NULL;
2025 	}
2026 	spin_unlock_irq(&cm.lock);
2027 	return listen_cm_id_priv;
2028 }
2029 
2030 /*
2031  * Work-around for inter-subnet connections.  If the LIDs are permissive,
2032  * we need to override the LID/SL data in the REQ with the LID information
2033  * in the work completion.
2034  */
2035 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2036 {
2037 	if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2038 		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2039 					req_msg)) == IB_LID_PERMISSIVE) {
2040 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2041 				be16_to_cpu(ib_lid_be16(wc->slid)));
2042 			IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2043 		}
2044 
2045 		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2046 					req_msg)) == IB_LID_PERMISSIVE)
2047 			IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2048 				wc->dlid_path_bits);
2049 	}
2050 
2051 	if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2052 		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2053 					req_msg)) == IB_LID_PERMISSIVE) {
2054 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2055 				be16_to_cpu(ib_lid_be16(wc->slid)));
2056 			IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2057 		}
2058 
2059 		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2060 					req_msg)) == IB_LID_PERMISSIVE)
2061 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2062 				wc->dlid_path_bits);
2063 	}
2064 }
2065 
2066 static int cm_req_handler(struct cm_work *work)
2067 {
2068 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2069 	struct cm_req_msg *req_msg;
2070 	const struct ib_global_route *grh;
2071 	const struct ib_gid_attr *gid_attr;
2072 	int ret;
2073 
2074 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2075 
2076 	cm_id_priv =
2077 		cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2078 	if (IS_ERR(cm_id_priv))
2079 		return PTR_ERR(cm_id_priv);
2080 
2081 	cm_id_priv->id.remote_id =
2082 		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2083 	cm_id_priv->id.service_id =
2084 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2085 	cm_id_priv->id.service_mask = ~cpu_to_be64(0);
2086 	cm_id_priv->tid = req_msg->hdr.tid;
2087 	cm_id_priv->timeout_ms = cm_convert_to_ms(
2088 		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2089 	cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2090 	cm_id_priv->remote_qpn =
2091 		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2092 	cm_id_priv->initiator_depth =
2093 		IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2094 	cm_id_priv->responder_resources =
2095 		IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2096 	cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2097 	cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2098 	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2099 	cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2100 	cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2101 	cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2102 
2103 	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2104 				      work->mad_recv_wc->recv_buf.grh,
2105 				      &cm_id_priv->av);
2106 	if (ret)
2107 		goto destroy;
2108 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2109 							    id.local_id);
2110 	if (IS_ERR(cm_id_priv->timewait_info)) {
2111 		ret = PTR_ERR(cm_id_priv->timewait_info);
2112 		goto destroy;
2113 	}
2114 	cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2115 	cm_id_priv->timewait_info->remote_ca_guid =
2116 		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2117 	cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2118 
2119 	/*
2120 	 * Note that the ID pointer is not in the xarray at this point,
2121 	 * so this set is only visible to the local thread.
2122 	 */
2123 	cm_id_priv->id.state = IB_CM_REQ_RCVD;
2124 
2125 	listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2126 	if (!listen_cm_id_priv) {
2127 		pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__,
2128 			 be32_to_cpu(cm_id_priv->id.local_id));
2129 		cm_id_priv->id.state = IB_CM_IDLE;
2130 		ret = -EINVAL;
2131 		goto destroy;
2132 	}
2133 
2134 	cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2135 
2136 	memset(&work->path[0], 0, sizeof(work->path[0]));
2137 	if (cm_req_has_alt_path(req_msg))
2138 		memset(&work->path[1], 0, sizeof(work->path[1]));
2139 	grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2140 	gid_attr = grh->sgid_attr;
2141 
2142 	if (gid_attr &&
2143 	    rdma_protocol_roce(work->port->cm_dev->ib_device,
2144 			       work->port->port_num)) {
2145 		work->path[0].rec_type =
2146 			sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2147 	} else {
2148 		cm_path_set_rec_type(
2149 			work->port->cm_dev->ib_device, work->port->port_num,
2150 			&work->path[0],
2151 			IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2152 					req_msg));
2153 	}
2154 	if (cm_req_has_alt_path(req_msg))
2155 		work->path[1].rec_type = work->path[0].rec_type;
2156 	cm_format_paths_from_req(req_msg, &work->path[0],
2157 				 &work->path[1]);
2158 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2159 		sa_path_set_dmac(&work->path[0],
2160 				 cm_id_priv->av.ah_attr.roce.dmac);
2161 	work->path[0].hop_limit = grh->hop_limit;
2162 	ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av,
2163 				 cm_id_priv);
2164 	if (ret) {
2165 		int err;
2166 
2167 		err = rdma_query_gid(work->port->cm_dev->ib_device,
2168 				     work->port->port_num, 0,
2169 				     &work->path[0].sgid);
2170 		if (err)
2171 			ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2172 				       NULL, 0, NULL, 0);
2173 		else
2174 			ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2175 				       &work->path[0].sgid,
2176 				       sizeof(work->path[0].sgid),
2177 				       NULL, 0);
2178 		goto rejected;
2179 	}
2180 	if (cm_req_has_alt_path(req_msg)) {
2181 		ret = cm_init_av_by_path(&work->path[1], NULL,
2182 					 &cm_id_priv->alt_av, cm_id_priv);
2183 		if (ret) {
2184 			ib_send_cm_rej(&cm_id_priv->id,
2185 				       IB_CM_REJ_INVALID_ALT_GID,
2186 				       &work->path[0].sgid,
2187 				       sizeof(work->path[0].sgid), NULL, 0);
2188 			goto rejected;
2189 		}
2190 	}
2191 
2192 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2193 	cm_id_priv->id.context = listen_cm_id_priv->id.context;
2194 	cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2195 
2196 	/* Now MAD handlers can see the new ID */
2197 	spin_lock_irq(&cm_id_priv->lock);
2198 	cm_finalize_id(cm_id_priv);
2199 
2200 	/* Refcount belongs to the event, pairs with cm_process_work() */
2201 	refcount_inc(&cm_id_priv->refcount);
2202 	cm_queue_work_unlock(cm_id_priv, work);
2203 	/*
2204 	 * Since this ID was just created and was not made visible to other MAD
2205 	 * handlers until the cm_finalize_id() above we know that the
2206 	 * cm_process_work() will deliver the event and the listen_cm_id
2207 	 * embedded in the event can be derefed here.
2208 	 */
2209 	cm_deref_id(listen_cm_id_priv);
2210 	return 0;
2211 
2212 rejected:
2213 	cm_deref_id(listen_cm_id_priv);
2214 destroy:
2215 	ib_destroy_cm_id(&cm_id_priv->id);
2216 	return ret;
2217 }
2218 
2219 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2220 			  struct cm_id_private *cm_id_priv,
2221 			  struct ib_cm_rep_param *param)
2222 {
2223 	cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2224 			      param->ece.attr_mod);
2225 	IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2226 		be32_to_cpu(cm_id_priv->id.local_id));
2227 	IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2228 		be32_to_cpu(cm_id_priv->id.remote_id));
2229 	IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2230 	IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2231 		param->responder_resources);
2232 	IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2233 		cm_id_priv->av.port->cm_dev->ack_delay);
2234 	IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2235 	IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2236 	IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2237 		be64_to_cpu(cm_id_priv->id.device->node_guid));
2238 
2239 	if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2240 		IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2241 			param->initiator_depth);
2242 		IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2243 			param->flow_control);
2244 		IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2245 		IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2246 	} else {
2247 		IBA_SET(CM_REP_SRQ, rep_msg, 1);
2248 		IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2249 	}
2250 
2251 	IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2252 	IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2253 	IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2254 
2255 	if (param->private_data && param->private_data_len)
2256 		IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2257 			    param->private_data_len);
2258 }
2259 
2260 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2261 		   struct ib_cm_rep_param *param)
2262 {
2263 	struct cm_id_private *cm_id_priv;
2264 	struct ib_mad_send_buf *msg;
2265 	struct cm_rep_msg *rep_msg;
2266 	unsigned long flags;
2267 	int ret;
2268 
2269 	if (param->private_data &&
2270 	    param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2271 		return -EINVAL;
2272 
2273 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2274 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2275 	if (cm_id->state != IB_CM_REQ_RCVD &&
2276 	    cm_id->state != IB_CM_MRA_REQ_SENT) {
2277 		pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__,
2278 			 be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
2279 		ret = -EINVAL;
2280 		goto out;
2281 	}
2282 
2283 	ret = cm_alloc_msg(cm_id_priv, &msg);
2284 	if (ret)
2285 		goto out;
2286 
2287 	rep_msg = (struct cm_rep_msg *) msg->mad;
2288 	cm_format_rep(rep_msg, cm_id_priv, param);
2289 	msg->timeout_ms = cm_id_priv->timeout_ms;
2290 	msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2291 
2292 	ret = ib_post_send_mad(msg, NULL);
2293 	if (ret) {
2294 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2295 		cm_free_msg(msg);
2296 		return ret;
2297 	}
2298 
2299 	cm_id->state = IB_CM_REP_SENT;
2300 	cm_id_priv->msg = msg;
2301 	cm_id_priv->initiator_depth = param->initiator_depth;
2302 	cm_id_priv->responder_resources = param->responder_resources;
2303 	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2304 	WARN_ONCE(param->qp_num & 0xFF000000,
2305 		  "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2306 		  param->qp_num);
2307 	cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2308 
2309 out:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2310 	return ret;
2311 }
2312 EXPORT_SYMBOL(ib_send_cm_rep);
2313 
2314 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2315 			  struct cm_id_private *cm_id_priv,
2316 			  const void *private_data,
2317 			  u8 private_data_len)
2318 {
2319 	cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2320 	IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2321 		be32_to_cpu(cm_id_priv->id.local_id));
2322 	IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2323 		be32_to_cpu(cm_id_priv->id.remote_id));
2324 
2325 	if (private_data && private_data_len)
2326 		IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2327 			    private_data_len);
2328 }
2329 
2330 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2331 		   const void *private_data,
2332 		   u8 private_data_len)
2333 {
2334 	struct cm_id_private *cm_id_priv;
2335 	struct ib_mad_send_buf *msg;
2336 	unsigned long flags;
2337 	void *data;
2338 	int ret;
2339 
2340 	if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2341 		return -EINVAL;
2342 
2343 	data = cm_copy_private_data(private_data, private_data_len);
2344 	if (IS_ERR(data))
2345 		return PTR_ERR(data);
2346 
2347 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2348 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2349 	if (cm_id->state != IB_CM_REP_RCVD &&
2350 	    cm_id->state != IB_CM_MRA_REP_SENT) {
2351 		pr_debug("%s: local_id %d, cm_id->state %d\n", __func__,
2352 			 be32_to_cpu(cm_id->local_id), cm_id->state);
2353 		ret = -EINVAL;
2354 		goto error;
2355 	}
2356 
2357 	ret = cm_alloc_msg(cm_id_priv, &msg);
2358 	if (ret)
2359 		goto error;
2360 
2361 	cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2362 		      private_data, private_data_len);
2363 
2364 	ret = ib_post_send_mad(msg, NULL);
2365 	if (ret) {
2366 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2367 		cm_free_msg(msg);
2368 		kfree(data);
2369 		return ret;
2370 	}
2371 
2372 	cm_id->state = IB_CM_ESTABLISHED;
2373 	cm_set_private_data(cm_id_priv, data, private_data_len);
2374 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2375 	return 0;
2376 
2377 error:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2378 	kfree(data);
2379 	return ret;
2380 }
2381 EXPORT_SYMBOL(ib_send_cm_rtu);
2382 
2383 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2384 {
2385 	struct cm_rep_msg *rep_msg;
2386 	struct ib_cm_rep_event_param *param;
2387 
2388 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2389 	param = &work->cm_event.param.rep_rcvd;
2390 	param->remote_ca_guid =
2391 		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2392 	param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2393 	param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2394 	param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2395 	param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2396 	param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2397 	param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2398 	param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2399 	param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2400 	param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2401 	param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2402 	param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2403 	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2404 	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2405 	param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2406 
2407 	work->cm_event.private_data =
2408 		IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2409 }
2410 
2411 static void cm_dup_rep_handler(struct cm_work *work)
2412 {
2413 	struct cm_id_private *cm_id_priv;
2414 	struct cm_rep_msg *rep_msg;
2415 	struct ib_mad_send_buf *msg = NULL;
2416 	int ret;
2417 
2418 	rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2419 	cm_id_priv = cm_acquire_id(
2420 		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2421 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2422 	if (!cm_id_priv)
2423 		return;
2424 
2425 	atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2426 			counter[CM_REP_COUNTER]);
2427 	ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2428 	if (ret)
2429 		goto deref;
2430 
2431 	spin_lock_irq(&cm_id_priv->lock);
2432 	if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2433 		cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2434 			      cm_id_priv->private_data,
2435 			      cm_id_priv->private_data_len);
2436 	else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2437 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2438 			      CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2439 			      cm_id_priv->private_data,
2440 			      cm_id_priv->private_data_len);
2441 	else
2442 		goto unlock;
2443 	spin_unlock_irq(&cm_id_priv->lock);
2444 
2445 	ret = ib_post_send_mad(msg, NULL);
2446 	if (ret)
2447 		goto free;
2448 	goto deref;
2449 
2450 unlock:	spin_unlock_irq(&cm_id_priv->lock);
2451 free:	cm_free_msg(msg);
2452 deref:	cm_deref_id(cm_id_priv);
2453 }
2454 
2455 static int cm_rep_handler(struct cm_work *work)
2456 {
2457 	struct cm_id_private *cm_id_priv;
2458 	struct cm_rep_msg *rep_msg;
2459 	int ret;
2460 	struct cm_id_private *cur_cm_id_priv;
2461 	struct cm_timewait_info *timewait_info;
2462 
2463 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2464 	cm_id_priv = cm_acquire_id(
2465 		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2466 	if (!cm_id_priv) {
2467 		cm_dup_rep_handler(work);
2468 		pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__,
2469 			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2470 		return -EINVAL;
2471 	}
2472 
2473 	cm_format_rep_event(work, cm_id_priv->qp_type);
2474 
2475 	spin_lock_irq(&cm_id_priv->lock);
2476 	switch (cm_id_priv->id.state) {
2477 	case IB_CM_REQ_SENT:
2478 	case IB_CM_MRA_REQ_RCVD:
2479 		break;
2480 	default:
2481 		ret = -EINVAL;
2482 		pr_debug(
2483 			"%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n",
2484 			__func__, cm_id_priv->id.state,
2485 			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2486 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2487 		spin_unlock_irq(&cm_id_priv->lock);
2488 		goto error;
2489 	}
2490 
2491 	cm_id_priv->timewait_info->work.remote_id =
2492 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2493 	cm_id_priv->timewait_info->remote_ca_guid =
2494 		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2495 	cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2496 
2497 	spin_lock(&cm.lock);
2498 	/* Check for duplicate REP. */
2499 	if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2500 		spin_unlock(&cm.lock);
2501 		spin_unlock_irq(&cm_id_priv->lock);
2502 		ret = -EINVAL;
2503 		pr_debug("%s: Failed to insert remote id %d\n", __func__,
2504 			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2505 		goto error;
2506 	}
2507 	/* Check for a stale connection. */
2508 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2509 	if (timewait_info) {
2510 		cm_remove_remote(cm_id_priv);
2511 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2512 					   timewait_info->work.remote_id);
2513 
2514 		spin_unlock(&cm.lock);
2515 		spin_unlock_irq(&cm_id_priv->lock);
2516 		cm_issue_rej(work->port, work->mad_recv_wc,
2517 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2518 			     NULL, 0);
2519 		ret = -EINVAL;
2520 		pr_debug(
2521 			"%s: Stale connection. local_comm_id %d, remote_comm_id %d\n",
2522 			__func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2523 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2524 
2525 		if (cur_cm_id_priv) {
2526 			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2527 			cm_deref_id(cur_cm_id_priv);
2528 		}
2529 
2530 		goto error;
2531 	}
2532 	spin_unlock(&cm.lock);
2533 
2534 	cm_id_priv->id.state = IB_CM_REP_RCVD;
2535 	cm_id_priv->id.remote_id =
2536 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2537 	cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2538 	cm_id_priv->initiator_depth =
2539 		IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2540 	cm_id_priv->responder_resources =
2541 		IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2542 	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2543 	cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2544 	cm_id_priv->target_ack_delay =
2545 		IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2546 	cm_id_priv->av.timeout =
2547 			cm_ack_timeout(cm_id_priv->target_ack_delay,
2548 				       cm_id_priv->av.timeout - 1);
2549 	cm_id_priv->alt_av.timeout =
2550 			cm_ack_timeout(cm_id_priv->target_ack_delay,
2551 				       cm_id_priv->alt_av.timeout - 1);
2552 
2553 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2554 	cm_queue_work_unlock(cm_id_priv, work);
2555 	return 0;
2556 
2557 error:
2558 	cm_deref_id(cm_id_priv);
2559 	return ret;
2560 }
2561 
2562 static int cm_establish_handler(struct cm_work *work)
2563 {
2564 	struct cm_id_private *cm_id_priv;
2565 
2566 	/* See comment in cm_establish about lookup. */
2567 	cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2568 	if (!cm_id_priv)
2569 		return -EINVAL;
2570 
2571 	spin_lock_irq(&cm_id_priv->lock);
2572 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2573 		spin_unlock_irq(&cm_id_priv->lock);
2574 		goto out;
2575 	}
2576 
2577 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2578 	cm_queue_work_unlock(cm_id_priv, work);
2579 	return 0;
2580 out:
2581 	cm_deref_id(cm_id_priv);
2582 	return -EINVAL;
2583 }
2584 
2585 static int cm_rtu_handler(struct cm_work *work)
2586 {
2587 	struct cm_id_private *cm_id_priv;
2588 	struct cm_rtu_msg *rtu_msg;
2589 
2590 	rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2591 	cm_id_priv = cm_acquire_id(
2592 		cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2593 		cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2594 	if (!cm_id_priv)
2595 		return -EINVAL;
2596 
2597 	work->cm_event.private_data =
2598 		IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2599 
2600 	spin_lock_irq(&cm_id_priv->lock);
2601 	if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2602 	    cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2603 		spin_unlock_irq(&cm_id_priv->lock);
2604 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2605 				counter[CM_RTU_COUNTER]);
2606 		goto out;
2607 	}
2608 	cm_id_priv->id.state = IB_CM_ESTABLISHED;
2609 
2610 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2611 	cm_queue_work_unlock(cm_id_priv, work);
2612 	return 0;
2613 out:
2614 	cm_deref_id(cm_id_priv);
2615 	return -EINVAL;
2616 }
2617 
2618 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2619 			  struct cm_id_private *cm_id_priv,
2620 			  const void *private_data,
2621 			  u8 private_data_len)
2622 {
2623 	cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2624 			  cm_form_tid(cm_id_priv));
2625 	IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2626 		be32_to_cpu(cm_id_priv->id.local_id));
2627 	IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2628 		be32_to_cpu(cm_id_priv->id.remote_id));
2629 	IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2630 		be32_to_cpu(cm_id_priv->remote_qpn));
2631 
2632 	if (private_data && private_data_len)
2633 		IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2634 			    private_data_len);
2635 }
2636 
2637 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2638 			       const void *private_data, u8 private_data_len)
2639 {
2640 	struct ib_mad_send_buf *msg;
2641 	int ret;
2642 
2643 	lockdep_assert_held(&cm_id_priv->lock);
2644 
2645 	if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2646 		return -EINVAL;
2647 
2648 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2649 		pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2650 			 be32_to_cpu(cm_id_priv->id.local_id),
2651 			 cm_id_priv->id.state);
2652 		return -EINVAL;
2653 	}
2654 
2655 	if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2656 	    cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2657 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2658 
2659 	ret = cm_alloc_msg(cm_id_priv, &msg);
2660 	if (ret) {
2661 		cm_enter_timewait(cm_id_priv);
2662 		return ret;
2663 	}
2664 
2665 	cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2666 		       private_data, private_data_len);
2667 	msg->timeout_ms = cm_id_priv->timeout_ms;
2668 	msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2669 
2670 	ret = ib_post_send_mad(msg, NULL);
2671 	if (ret) {
2672 		cm_enter_timewait(cm_id_priv);
2673 		cm_free_msg(msg);
2674 		return ret;
2675 	}
2676 
2677 	cm_id_priv->id.state = IB_CM_DREQ_SENT;
2678 	cm_id_priv->msg = msg;
2679 	return 0;
2680 }
2681 
2682 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2683 		    u8 private_data_len)
2684 {
2685 	struct cm_id_private *cm_id_priv =
2686 		container_of(cm_id, struct cm_id_private, id);
2687 	unsigned long flags;
2688 	int ret;
2689 
2690 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2691 	ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2692 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2693 	return ret;
2694 }
2695 EXPORT_SYMBOL(ib_send_cm_dreq);
2696 
2697 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2698 			  struct cm_id_private *cm_id_priv,
2699 			  const void *private_data,
2700 			  u8 private_data_len)
2701 {
2702 	cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2703 	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2704 		be32_to_cpu(cm_id_priv->id.local_id));
2705 	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2706 		be32_to_cpu(cm_id_priv->id.remote_id));
2707 
2708 	if (private_data && private_data_len)
2709 		IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2710 			    private_data_len);
2711 }
2712 
2713 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2714 			       void *private_data, u8 private_data_len)
2715 {
2716 	struct ib_mad_send_buf *msg;
2717 	int ret;
2718 
2719 	lockdep_assert_held(&cm_id_priv->lock);
2720 
2721 	if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2722 		return -EINVAL;
2723 
2724 	if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2725 		pr_debug(
2726 			"%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n",
2727 			__func__, be32_to_cpu(cm_id_priv->id.local_id),
2728 			cm_id_priv->id.state);
2729 		kfree(private_data);
2730 		return -EINVAL;
2731 	}
2732 
2733 	cm_set_private_data(cm_id_priv, private_data, private_data_len);
2734 	cm_enter_timewait(cm_id_priv);
2735 
2736 	ret = cm_alloc_msg(cm_id_priv, &msg);
2737 	if (ret)
2738 		return ret;
2739 
2740 	cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2741 		       private_data, private_data_len);
2742 
2743 	ret = ib_post_send_mad(msg, NULL);
2744 	if (ret) {
2745 		cm_free_msg(msg);
2746 		return ret;
2747 	}
2748 	return 0;
2749 }
2750 
2751 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2752 		    u8 private_data_len)
2753 {
2754 	struct cm_id_private *cm_id_priv =
2755 		container_of(cm_id, struct cm_id_private, id);
2756 	unsigned long flags;
2757 	void *data;
2758 	int ret;
2759 
2760 	data = cm_copy_private_data(private_data, private_data_len);
2761 	if (IS_ERR(data))
2762 		return PTR_ERR(data);
2763 
2764 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2765 	ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2766 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2767 	return ret;
2768 }
2769 EXPORT_SYMBOL(ib_send_cm_drep);
2770 
2771 static int cm_issue_drep(struct cm_port *port,
2772 			 struct ib_mad_recv_wc *mad_recv_wc)
2773 {
2774 	struct ib_mad_send_buf *msg = NULL;
2775 	struct cm_dreq_msg *dreq_msg;
2776 	struct cm_drep_msg *drep_msg;
2777 	int ret;
2778 
2779 	ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2780 	if (ret)
2781 		return ret;
2782 
2783 	dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2784 	drep_msg = (struct cm_drep_msg *) msg->mad;
2785 
2786 	cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2787 	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2788 		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2789 	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2790 		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2791 
2792 	ret = ib_post_send_mad(msg, NULL);
2793 	if (ret)
2794 		cm_free_msg(msg);
2795 
2796 	return ret;
2797 }
2798 
2799 static int cm_dreq_handler(struct cm_work *work)
2800 {
2801 	struct cm_id_private *cm_id_priv;
2802 	struct cm_dreq_msg *dreq_msg;
2803 	struct ib_mad_send_buf *msg = NULL;
2804 
2805 	dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2806 	cm_id_priv = cm_acquire_id(
2807 		cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2808 		cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2809 	if (!cm_id_priv) {
2810 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2811 				counter[CM_DREQ_COUNTER]);
2812 		cm_issue_drep(work->port, work->mad_recv_wc);
2813 		pr_debug(
2814 			"%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n",
2815 			__func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2816 			IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2817 		return -EINVAL;
2818 	}
2819 
2820 	work->cm_event.private_data =
2821 		IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2822 
2823 	spin_lock_irq(&cm_id_priv->lock);
2824 	if (cm_id_priv->local_qpn !=
2825 	    cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2826 		goto unlock;
2827 
2828 	switch (cm_id_priv->id.state) {
2829 	case IB_CM_REP_SENT:
2830 	case IB_CM_DREQ_SENT:
2831 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2832 		break;
2833 	case IB_CM_ESTABLISHED:
2834 		if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2835 		    cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2836 			ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2837 		break;
2838 	case IB_CM_MRA_REP_RCVD:
2839 		break;
2840 	case IB_CM_TIMEWAIT:
2841 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2842 				counter[CM_DREQ_COUNTER]);
2843 		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2844 		if (IS_ERR(msg))
2845 			goto unlock;
2846 
2847 		cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2848 			       cm_id_priv->private_data,
2849 			       cm_id_priv->private_data_len);
2850 		spin_unlock_irq(&cm_id_priv->lock);
2851 
2852 		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2853 		    ib_post_send_mad(msg, NULL))
2854 			cm_free_msg(msg);
2855 		goto deref;
2856 	case IB_CM_DREQ_RCVD:
2857 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2858 				counter[CM_DREQ_COUNTER]);
2859 		goto unlock;
2860 	default:
2861 		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2862 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
2863 			 cm_id_priv->id.state);
2864 		goto unlock;
2865 	}
2866 	cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2867 	cm_id_priv->tid = dreq_msg->hdr.tid;
2868 	cm_queue_work_unlock(cm_id_priv, work);
2869 	return 0;
2870 
2871 unlock:	spin_unlock_irq(&cm_id_priv->lock);
2872 deref:	cm_deref_id(cm_id_priv);
2873 	return -EINVAL;
2874 }
2875 
2876 static int cm_drep_handler(struct cm_work *work)
2877 {
2878 	struct cm_id_private *cm_id_priv;
2879 	struct cm_drep_msg *drep_msg;
2880 
2881 	drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2882 	cm_id_priv = cm_acquire_id(
2883 		cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2884 		cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2885 	if (!cm_id_priv)
2886 		return -EINVAL;
2887 
2888 	work->cm_event.private_data =
2889 		IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2890 
2891 	spin_lock_irq(&cm_id_priv->lock);
2892 	if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2893 	    cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2894 		spin_unlock_irq(&cm_id_priv->lock);
2895 		goto out;
2896 	}
2897 	cm_enter_timewait(cm_id_priv);
2898 
2899 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2900 	cm_queue_work_unlock(cm_id_priv, work);
2901 	return 0;
2902 out:
2903 	cm_deref_id(cm_id_priv);
2904 	return -EINVAL;
2905 }
2906 
2907 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2908 			      enum ib_cm_rej_reason reason, void *ari,
2909 			      u8 ari_length, const void *private_data,
2910 			      u8 private_data_len)
2911 {
2912 	enum ib_cm_state state = cm_id_priv->id.state;
2913 	struct ib_mad_send_buf *msg;
2914 	int ret;
2915 
2916 	lockdep_assert_held(&cm_id_priv->lock);
2917 
2918 	if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2919 	    (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2920 		return -EINVAL;
2921 
2922 	switch (state) {
2923 	case IB_CM_REQ_SENT:
2924 	case IB_CM_MRA_REQ_RCVD:
2925 	case IB_CM_REQ_RCVD:
2926 	case IB_CM_MRA_REQ_SENT:
2927 	case IB_CM_REP_RCVD:
2928 	case IB_CM_MRA_REP_SENT:
2929 		cm_reset_to_idle(cm_id_priv);
2930 		ret = cm_alloc_msg(cm_id_priv, &msg);
2931 		if (ret)
2932 			return ret;
2933 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2934 			      ari, ari_length, private_data, private_data_len,
2935 			      state);
2936 		break;
2937 	case IB_CM_REP_SENT:
2938 	case IB_CM_MRA_REP_RCVD:
2939 		cm_enter_timewait(cm_id_priv);
2940 		ret = cm_alloc_msg(cm_id_priv, &msg);
2941 		if (ret)
2942 			return ret;
2943 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2944 			      ari, ari_length, private_data, private_data_len,
2945 			      state);
2946 		break;
2947 	default:
2948 		pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2949 			 be32_to_cpu(cm_id_priv->id.local_id),
2950 			 cm_id_priv->id.state);
2951 		return -EINVAL;
2952 	}
2953 
2954 	ret = ib_post_send_mad(msg, NULL);
2955 	if (ret) {
2956 		cm_free_msg(msg);
2957 		return ret;
2958 	}
2959 
2960 	return 0;
2961 }
2962 
2963 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2964 		   void *ari, u8 ari_length, const void *private_data,
2965 		   u8 private_data_len)
2966 {
2967 	struct cm_id_private *cm_id_priv =
2968 		container_of(cm_id, struct cm_id_private, id);
2969 	unsigned long flags;
2970 	int ret;
2971 
2972 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2973 	ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2974 				 private_data, private_data_len);
2975 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2976 	return ret;
2977 }
2978 EXPORT_SYMBOL(ib_send_cm_rej);
2979 
2980 static void cm_format_rej_event(struct cm_work *work)
2981 {
2982 	struct cm_rej_msg *rej_msg;
2983 	struct ib_cm_rej_event_param *param;
2984 
2985 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2986 	param = &work->cm_event.param.rej_rcvd;
2987 	param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
2988 	param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
2989 	param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
2990 	work->cm_event.private_data =
2991 		IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
2992 }
2993 
2994 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2995 {
2996 	struct cm_id_private *cm_id_priv;
2997 	__be32 remote_id;
2998 
2999 	remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3000 
3001 	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3002 		cm_id_priv = cm_find_remote_id(
3003 			*((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3004 			remote_id);
3005 	} else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3006 		   CM_MSG_RESPONSE_REQ)
3007 		cm_id_priv = cm_acquire_id(
3008 			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3009 			0);
3010 	else
3011 		cm_id_priv = cm_acquire_id(
3012 			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3013 			remote_id);
3014 
3015 	return cm_id_priv;
3016 }
3017 
3018 static int cm_rej_handler(struct cm_work *work)
3019 {
3020 	struct cm_id_private *cm_id_priv;
3021 	struct cm_rej_msg *rej_msg;
3022 
3023 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3024 	cm_id_priv = cm_acquire_rejected_id(rej_msg);
3025 	if (!cm_id_priv)
3026 		return -EINVAL;
3027 
3028 	cm_format_rej_event(work);
3029 
3030 	spin_lock_irq(&cm_id_priv->lock);
3031 	switch (cm_id_priv->id.state) {
3032 	case IB_CM_REQ_SENT:
3033 	case IB_CM_MRA_REQ_RCVD:
3034 	case IB_CM_REP_SENT:
3035 	case IB_CM_MRA_REP_RCVD:
3036 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3037 		/* fall through */
3038 	case IB_CM_REQ_RCVD:
3039 	case IB_CM_MRA_REQ_SENT:
3040 		if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3041 			cm_enter_timewait(cm_id_priv);
3042 		else
3043 			cm_reset_to_idle(cm_id_priv);
3044 		break;
3045 	case IB_CM_DREQ_SENT:
3046 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3047 		/* fall through */
3048 	case IB_CM_REP_RCVD:
3049 	case IB_CM_MRA_REP_SENT:
3050 		cm_enter_timewait(cm_id_priv);
3051 		break;
3052 	case IB_CM_ESTABLISHED:
3053 		if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3054 		    cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3055 			if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3056 				ib_cancel_mad(cm_id_priv->av.port->mad_agent,
3057 					      cm_id_priv->msg);
3058 			cm_enter_timewait(cm_id_priv);
3059 			break;
3060 		}
3061 		/* fall through */
3062 	default:
3063 		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
3064 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
3065 			 cm_id_priv->id.state);
3066 		spin_unlock_irq(&cm_id_priv->lock);
3067 		goto out;
3068 	}
3069 
3070 	cm_queue_work_unlock(cm_id_priv, work);
3071 	return 0;
3072 out:
3073 	cm_deref_id(cm_id_priv);
3074 	return -EINVAL;
3075 }
3076 
3077 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3078 		   u8 service_timeout,
3079 		   const void *private_data,
3080 		   u8 private_data_len)
3081 {
3082 	struct cm_id_private *cm_id_priv;
3083 	struct ib_mad_send_buf *msg;
3084 	enum ib_cm_state cm_state;
3085 	enum ib_cm_lap_state lap_state;
3086 	enum cm_msg_response msg_response;
3087 	void *data;
3088 	unsigned long flags;
3089 	int ret;
3090 
3091 	if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3092 		return -EINVAL;
3093 
3094 	data = cm_copy_private_data(private_data, private_data_len);
3095 	if (IS_ERR(data))
3096 		return PTR_ERR(data);
3097 
3098 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3099 
3100 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3101 	switch(cm_id_priv->id.state) {
3102 	case IB_CM_REQ_RCVD:
3103 		cm_state = IB_CM_MRA_REQ_SENT;
3104 		lap_state = cm_id->lap_state;
3105 		msg_response = CM_MSG_RESPONSE_REQ;
3106 		break;
3107 	case IB_CM_REP_RCVD:
3108 		cm_state = IB_CM_MRA_REP_SENT;
3109 		lap_state = cm_id->lap_state;
3110 		msg_response = CM_MSG_RESPONSE_REP;
3111 		break;
3112 	case IB_CM_ESTABLISHED:
3113 		if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3114 			cm_state = cm_id->state;
3115 			lap_state = IB_CM_MRA_LAP_SENT;
3116 			msg_response = CM_MSG_RESPONSE_OTHER;
3117 			break;
3118 		}
3119 		/* fall through */
3120 	default:
3121 		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
3122 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
3123 			 cm_id_priv->id.state);
3124 		ret = -EINVAL;
3125 		goto error1;
3126 	}
3127 
3128 	if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3129 		ret = cm_alloc_msg(cm_id_priv, &msg);
3130 		if (ret)
3131 			goto error1;
3132 
3133 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3134 			      msg_response, service_timeout,
3135 			      private_data, private_data_len);
3136 		ret = ib_post_send_mad(msg, NULL);
3137 		if (ret)
3138 			goto error2;
3139 	}
3140 
3141 	cm_id->state = cm_state;
3142 	cm_id->lap_state = lap_state;
3143 	cm_id_priv->service_timeout = service_timeout;
3144 	cm_set_private_data(cm_id_priv, data, private_data_len);
3145 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3146 	return 0;
3147 
3148 error1:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3149 	kfree(data);
3150 	return ret;
3151 
3152 error2:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3153 	kfree(data);
3154 	cm_free_msg(msg);
3155 	return ret;
3156 }
3157 EXPORT_SYMBOL(ib_send_cm_mra);
3158 
3159 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3160 {
3161 	switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3162 	case CM_MSG_RESPONSE_REQ:
3163 		return cm_acquire_id(
3164 			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3165 			0);
3166 	case CM_MSG_RESPONSE_REP:
3167 	case CM_MSG_RESPONSE_OTHER:
3168 		return cm_acquire_id(
3169 			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3170 			cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3171 	default:
3172 		return NULL;
3173 	}
3174 }
3175 
3176 static int cm_mra_handler(struct cm_work *work)
3177 {
3178 	struct cm_id_private *cm_id_priv;
3179 	struct cm_mra_msg *mra_msg;
3180 	int timeout;
3181 
3182 	mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3183 	cm_id_priv = cm_acquire_mraed_id(mra_msg);
3184 	if (!cm_id_priv)
3185 		return -EINVAL;
3186 
3187 	work->cm_event.private_data =
3188 		IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3189 	work->cm_event.param.mra_rcvd.service_timeout =
3190 		IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3191 	timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3192 		  cm_convert_to_ms(cm_id_priv->av.timeout);
3193 
3194 	spin_lock_irq(&cm_id_priv->lock);
3195 	switch (cm_id_priv->id.state) {
3196 	case IB_CM_REQ_SENT:
3197 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3198 			    CM_MSG_RESPONSE_REQ ||
3199 		    ib_modify_mad(cm_id_priv->av.port->mad_agent,
3200 				  cm_id_priv->msg, timeout))
3201 			goto out;
3202 		cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3203 		break;
3204 	case IB_CM_REP_SENT:
3205 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3206 			    CM_MSG_RESPONSE_REP ||
3207 		    ib_modify_mad(cm_id_priv->av.port->mad_agent,
3208 				  cm_id_priv->msg, timeout))
3209 			goto out;
3210 		cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3211 		break;
3212 	case IB_CM_ESTABLISHED:
3213 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3214 			    CM_MSG_RESPONSE_OTHER ||
3215 		    cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3216 		    ib_modify_mad(cm_id_priv->av.port->mad_agent,
3217 				  cm_id_priv->msg, timeout)) {
3218 			if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3219 				atomic_long_inc(&work->port->
3220 						counter_group[CM_RECV_DUPLICATES].
3221 						counter[CM_MRA_COUNTER]);
3222 			goto out;
3223 		}
3224 		cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3225 		break;
3226 	case IB_CM_MRA_REQ_RCVD:
3227 	case IB_CM_MRA_REP_RCVD:
3228 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3229 				counter[CM_MRA_COUNTER]);
3230 		/* fall through */
3231 	default:
3232 		pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n",
3233 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
3234 			 cm_id_priv->id.state);
3235 		goto out;
3236 	}
3237 
3238 	cm_id_priv->msg->context[1] = (void *) (unsigned long)
3239 				      cm_id_priv->id.state;
3240 	cm_queue_work_unlock(cm_id_priv, work);
3241 	return 0;
3242 out:
3243 	spin_unlock_irq(&cm_id_priv->lock);
3244 	cm_deref_id(cm_id_priv);
3245 	return -EINVAL;
3246 }
3247 
3248 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3249 					struct sa_path_rec *path)
3250 {
3251 	u32 lid;
3252 
3253 	if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3254 		sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3255 					       lap_msg));
3256 		sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3257 					       lap_msg));
3258 	} else {
3259 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3260 			CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3261 		sa_path_set_dlid(path, lid);
3262 
3263 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3264 			CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3265 		sa_path_set_slid(path, lid);
3266 	}
3267 }
3268 
3269 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3270 				    struct sa_path_rec *path,
3271 				    struct cm_lap_msg *lap_msg)
3272 {
3273 	path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3274 	path->sgid =
3275 		*IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3276 	path->flow_label =
3277 		cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3278 	path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3279 	path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3280 	path->reversible = 1;
3281 	path->pkey = cm_id_priv->pkey;
3282 	path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3283 	path->mtu_selector = IB_SA_EQ;
3284 	path->mtu = cm_id_priv->path_mtu;
3285 	path->rate_selector = IB_SA_EQ;
3286 	path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3287 	path->packet_life_time_selector = IB_SA_EQ;
3288 	path->packet_life_time =
3289 		IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3290 	path->packet_life_time -= (path->packet_life_time > 0);
3291 	cm_format_path_lid_from_lap(lap_msg, path);
3292 }
3293 
3294 static int cm_lap_handler(struct cm_work *work)
3295 {
3296 	struct cm_id_private *cm_id_priv;
3297 	struct cm_lap_msg *lap_msg;
3298 	struct ib_cm_lap_event_param *param;
3299 	struct ib_mad_send_buf *msg = NULL;
3300 	int ret;
3301 
3302 	/* Currently Alternate path messages are not supported for
3303 	 * RoCE link layer.
3304 	 */
3305 	if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3306 			       work->port->port_num))
3307 		return -EINVAL;
3308 
3309 	/* todo: verify LAP request and send reject APR if invalid. */
3310 	lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3311 	cm_id_priv = cm_acquire_id(
3312 		cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3313 		cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3314 	if (!cm_id_priv)
3315 		return -EINVAL;
3316 
3317 	param = &work->cm_event.param.lap_rcvd;
3318 	memset(&work->path[0], 0, sizeof(work->path[1]));
3319 	cm_path_set_rec_type(work->port->cm_dev->ib_device,
3320 			     work->port->port_num, &work->path[0],
3321 			     IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3322 					     lap_msg));
3323 	param->alternate_path = &work->path[0];
3324 	cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3325 	work->cm_event.private_data =
3326 		IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3327 
3328 	spin_lock_irq(&cm_id_priv->lock);
3329 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3330 		goto unlock;
3331 
3332 	switch (cm_id_priv->id.lap_state) {
3333 	case IB_CM_LAP_UNINIT:
3334 	case IB_CM_LAP_IDLE:
3335 		break;
3336 	case IB_CM_MRA_LAP_SENT:
3337 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3338 				counter[CM_LAP_COUNTER]);
3339 		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3340 		if (IS_ERR(msg))
3341 			goto unlock;
3342 
3343 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3344 			      CM_MSG_RESPONSE_OTHER,
3345 			      cm_id_priv->service_timeout,
3346 			      cm_id_priv->private_data,
3347 			      cm_id_priv->private_data_len);
3348 		spin_unlock_irq(&cm_id_priv->lock);
3349 
3350 		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3351 		    ib_post_send_mad(msg, NULL))
3352 			cm_free_msg(msg);
3353 		goto deref;
3354 	case IB_CM_LAP_RCVD:
3355 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3356 				counter[CM_LAP_COUNTER]);
3357 		goto unlock;
3358 	default:
3359 		goto unlock;
3360 	}
3361 
3362 	ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3363 				 work->mad_recv_wc->recv_buf.grh,
3364 				 &cm_id_priv->av);
3365 	if (ret)
3366 		goto unlock;
3367 
3368 	ret = cm_init_av_by_path(param->alternate_path, NULL,
3369 				 &cm_id_priv->alt_av, cm_id_priv);
3370 	if (ret)
3371 		goto unlock;
3372 
3373 	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3374 	cm_id_priv->tid = lap_msg->hdr.tid;
3375 	cm_queue_work_unlock(cm_id_priv, work);
3376 	return 0;
3377 
3378 unlock:	spin_unlock_irq(&cm_id_priv->lock);
3379 deref:	cm_deref_id(cm_id_priv);
3380 	return -EINVAL;
3381 }
3382 
3383 static int cm_apr_handler(struct cm_work *work)
3384 {
3385 	struct cm_id_private *cm_id_priv;
3386 	struct cm_apr_msg *apr_msg;
3387 
3388 	/* Currently Alternate path messages are not supported for
3389 	 * RoCE link layer.
3390 	 */
3391 	if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3392 			       work->port->port_num))
3393 		return -EINVAL;
3394 
3395 	apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3396 	cm_id_priv = cm_acquire_id(
3397 		cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3398 		cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3399 	if (!cm_id_priv)
3400 		return -EINVAL; /* Unmatched reply. */
3401 
3402 	work->cm_event.param.apr_rcvd.ap_status =
3403 		IBA_GET(CM_APR_AR_STATUS, apr_msg);
3404 	work->cm_event.param.apr_rcvd.apr_info =
3405 		IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3406 	work->cm_event.param.apr_rcvd.info_len =
3407 		IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3408 	work->cm_event.private_data =
3409 		IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3410 
3411 	spin_lock_irq(&cm_id_priv->lock);
3412 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3413 	    (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3414 	     cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3415 		spin_unlock_irq(&cm_id_priv->lock);
3416 		goto out;
3417 	}
3418 	cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3419 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3420 	cm_id_priv->msg = NULL;
3421 	cm_queue_work_unlock(cm_id_priv, work);
3422 	return 0;
3423 out:
3424 	cm_deref_id(cm_id_priv);
3425 	return -EINVAL;
3426 }
3427 
3428 static int cm_timewait_handler(struct cm_work *work)
3429 {
3430 	struct cm_timewait_info *timewait_info;
3431 	struct cm_id_private *cm_id_priv;
3432 
3433 	timewait_info = container_of(work, struct cm_timewait_info, work);
3434 	spin_lock_irq(&cm.lock);
3435 	list_del(&timewait_info->list);
3436 	spin_unlock_irq(&cm.lock);
3437 
3438 	cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3439 				   timewait_info->work.remote_id);
3440 	if (!cm_id_priv)
3441 		return -EINVAL;
3442 
3443 	spin_lock_irq(&cm_id_priv->lock);
3444 	if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3445 	    cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3446 		spin_unlock_irq(&cm_id_priv->lock);
3447 		goto out;
3448 	}
3449 	cm_id_priv->id.state = IB_CM_IDLE;
3450 	cm_queue_work_unlock(cm_id_priv, work);
3451 	return 0;
3452 out:
3453 	cm_deref_id(cm_id_priv);
3454 	return -EINVAL;
3455 }
3456 
3457 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3458 			       struct cm_id_private *cm_id_priv,
3459 			       struct ib_cm_sidr_req_param *param)
3460 {
3461 	cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3462 			  cm_form_tid(cm_id_priv));
3463 	IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3464 		be32_to_cpu(cm_id_priv->id.local_id));
3465 	IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3466 		be16_to_cpu(param->path->pkey));
3467 	IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3468 		be64_to_cpu(param->service_id));
3469 
3470 	if (param->private_data && param->private_data_len)
3471 		IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3472 			    param->private_data, param->private_data_len);
3473 }
3474 
3475 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3476 			struct ib_cm_sidr_req_param *param)
3477 {
3478 	struct cm_id_private *cm_id_priv;
3479 	struct ib_mad_send_buf *msg;
3480 	unsigned long flags;
3481 	int ret;
3482 
3483 	if (!param->path || (param->private_data &&
3484 	     param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3485 		return -EINVAL;
3486 
3487 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3488 	ret = cm_init_av_by_path(param->path, param->sgid_attr,
3489 				 &cm_id_priv->av,
3490 				 cm_id_priv);
3491 	if (ret)
3492 		goto out;
3493 
3494 	cm_id->service_id = param->service_id;
3495 	cm_id->service_mask = ~cpu_to_be64(0);
3496 	cm_id_priv->timeout_ms = param->timeout_ms;
3497 	cm_id_priv->max_cm_retries = param->max_cm_retries;
3498 	ret = cm_alloc_msg(cm_id_priv, &msg);
3499 	if (ret)
3500 		goto out;
3501 
3502 	cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
3503 			   param);
3504 	msg->timeout_ms = cm_id_priv->timeout_ms;
3505 	msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
3506 
3507 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3508 	if (cm_id->state == IB_CM_IDLE)
3509 		ret = ib_post_send_mad(msg, NULL);
3510 	else
3511 		ret = -EINVAL;
3512 
3513 	if (ret) {
3514 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3515 		cm_free_msg(msg);
3516 		goto out;
3517 	}
3518 	cm_id->state = IB_CM_SIDR_REQ_SENT;
3519 	cm_id_priv->msg = msg;
3520 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3521 out:
3522 	return ret;
3523 }
3524 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3525 
3526 static void cm_format_sidr_req_event(struct cm_work *work,
3527 				     const struct cm_id_private *rx_cm_id,
3528 				     struct ib_cm_id *listen_id)
3529 {
3530 	struct cm_sidr_req_msg *sidr_req_msg;
3531 	struct ib_cm_sidr_req_event_param *param;
3532 
3533 	sidr_req_msg = (struct cm_sidr_req_msg *)
3534 				work->mad_recv_wc->recv_buf.mad;
3535 	param = &work->cm_event.param.sidr_req_rcvd;
3536 	param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3537 	param->listen_id = listen_id;
3538 	param->service_id =
3539 		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3540 	param->bth_pkey = cm_get_bth_pkey(work);
3541 	param->port = work->port->port_num;
3542 	param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3543 	work->cm_event.private_data =
3544 		IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3545 }
3546 
3547 static int cm_sidr_req_handler(struct cm_work *work)
3548 {
3549 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3550 	struct cm_sidr_req_msg *sidr_req_msg;
3551 	struct ib_wc *wc;
3552 	int ret;
3553 
3554 	cm_id_priv =
3555 		cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3556 	if (IS_ERR(cm_id_priv))
3557 		return PTR_ERR(cm_id_priv);
3558 
3559 	/* Record SGID/SLID and request ID for lookup. */
3560 	sidr_req_msg = (struct cm_sidr_req_msg *)
3561 				work->mad_recv_wc->recv_buf.mad;
3562 
3563 	cm_id_priv->id.remote_id =
3564 		cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3565 	cm_id_priv->id.service_id =
3566 		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3567 	cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3568 	cm_id_priv->tid = sidr_req_msg->hdr.tid;
3569 
3570 	wc = work->mad_recv_wc->wc;
3571 	cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
3572 	cm_id_priv->av.dgid.global.interface_id = 0;
3573 	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3574 				      work->mad_recv_wc->recv_buf.grh,
3575 				      &cm_id_priv->av);
3576 	if (ret)
3577 		goto out;
3578 
3579 	spin_lock_irq(&cm.lock);
3580 	listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3581 	if (listen_cm_id_priv) {
3582 		spin_unlock_irq(&cm.lock);
3583 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3584 				counter[CM_SIDR_REQ_COUNTER]);
3585 		goto out; /* Duplicate message. */
3586 	}
3587 	cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3588 	listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3589 					   cm_id_priv->id.service_id);
3590 	if (!listen_cm_id_priv) {
3591 		spin_unlock_irq(&cm.lock);
3592 		ib_send_cm_sidr_rep(&cm_id_priv->id,
3593 				    &(struct ib_cm_sidr_rep_param){
3594 					    .status = IB_SIDR_UNSUPPORTED });
3595 		goto out; /* No match. */
3596 	}
3597 	spin_unlock_irq(&cm.lock);
3598 
3599 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3600 	cm_id_priv->id.context = listen_cm_id_priv->id.context;
3601 
3602 	/*
3603 	 * A SIDR ID does not need to be in the xarray since it does not receive
3604 	 * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3605 	 * not enter timewait.
3606 	 */
3607 
3608 	cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3609 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3610 	cm_free_work(work);
3611 	/*
3612 	 * A pointer to the listen_cm_id is held in the event, so this deref
3613 	 * must be after the event is delivered above.
3614 	 */
3615 	cm_deref_id(listen_cm_id_priv);
3616 	if (ret)
3617 		cm_destroy_id(&cm_id_priv->id, ret);
3618 	return 0;
3619 out:
3620 	ib_destroy_cm_id(&cm_id_priv->id);
3621 	return -EINVAL;
3622 }
3623 
3624 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3625 			       struct cm_id_private *cm_id_priv,
3626 			       struct ib_cm_sidr_rep_param *param)
3627 {
3628 	cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3629 			      cm_id_priv->tid, param->ece.attr_mod);
3630 	IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3631 		be32_to_cpu(cm_id_priv->id.remote_id));
3632 	IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3633 	IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3634 	IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3635 		be64_to_cpu(cm_id_priv->id.service_id));
3636 	IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3637 	IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3638 		param->ece.vendor_id & 0xFF);
3639 	IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3640 		(param->ece.vendor_id >> 8) & 0xFF);
3641 
3642 	if (param->info && param->info_length)
3643 		IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3644 			    param->info, param->info_length);
3645 
3646 	if (param->private_data && param->private_data_len)
3647 		IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3648 			    param->private_data, param->private_data_len);
3649 }
3650 
3651 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3652 				   struct ib_cm_sidr_rep_param *param)
3653 {
3654 	struct ib_mad_send_buf *msg;
3655 	int ret;
3656 
3657 	lockdep_assert_held(&cm_id_priv->lock);
3658 
3659 	if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3660 	    (param->private_data &&
3661 	     param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3662 		return -EINVAL;
3663 
3664 	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3665 		return -EINVAL;
3666 
3667 	ret = cm_alloc_msg(cm_id_priv, &msg);
3668 	if (ret)
3669 		return ret;
3670 
3671 	cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3672 			   param);
3673 	ret = ib_post_send_mad(msg, NULL);
3674 	if (ret) {
3675 		cm_free_msg(msg);
3676 		return ret;
3677 	}
3678 	cm_id_priv->id.state = IB_CM_IDLE;
3679 	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3680 		rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3681 		RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3682 	}
3683 	return 0;
3684 }
3685 
3686 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3687 			struct ib_cm_sidr_rep_param *param)
3688 {
3689 	struct cm_id_private *cm_id_priv =
3690 		container_of(cm_id, struct cm_id_private, id);
3691 	unsigned long flags;
3692 	int ret;
3693 
3694 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3695 	ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3696 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3697 	return ret;
3698 }
3699 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3700 
3701 static void cm_format_sidr_rep_event(struct cm_work *work,
3702 				     const struct cm_id_private *cm_id_priv)
3703 {
3704 	struct cm_sidr_rep_msg *sidr_rep_msg;
3705 	struct ib_cm_sidr_rep_event_param *param;
3706 
3707 	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3708 				work->mad_recv_wc->recv_buf.mad;
3709 	param = &work->cm_event.param.sidr_rep_rcvd;
3710 	param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3711 	param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3712 	param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3713 	param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3714 				      sidr_rep_msg);
3715 	param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3716 				  sidr_rep_msg);
3717 	param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3718 	work->cm_event.private_data =
3719 		IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3720 }
3721 
3722 static int cm_sidr_rep_handler(struct cm_work *work)
3723 {
3724 	struct cm_sidr_rep_msg *sidr_rep_msg;
3725 	struct cm_id_private *cm_id_priv;
3726 
3727 	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3728 				work->mad_recv_wc->recv_buf.mad;
3729 	cm_id_priv = cm_acquire_id(
3730 		cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3731 	if (!cm_id_priv)
3732 		return -EINVAL; /* Unmatched reply. */
3733 
3734 	spin_lock_irq(&cm_id_priv->lock);
3735 	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3736 		spin_unlock_irq(&cm_id_priv->lock);
3737 		goto out;
3738 	}
3739 	cm_id_priv->id.state = IB_CM_IDLE;
3740 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3741 	spin_unlock_irq(&cm_id_priv->lock);
3742 
3743 	cm_format_sidr_rep_event(work, cm_id_priv);
3744 	cm_process_work(cm_id_priv, work);
3745 	return 0;
3746 out:
3747 	cm_deref_id(cm_id_priv);
3748 	return -EINVAL;
3749 }
3750 
3751 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3752 				  enum ib_wc_status wc_status)
3753 {
3754 	struct cm_id_private *cm_id_priv;
3755 	struct ib_cm_event cm_event;
3756 	enum ib_cm_state state;
3757 	int ret;
3758 
3759 	memset(&cm_event, 0, sizeof cm_event);
3760 	cm_id_priv = msg->context[0];
3761 
3762 	/* Discard old sends or ones without a response. */
3763 	spin_lock_irq(&cm_id_priv->lock);
3764 	state = (enum ib_cm_state) (unsigned long) msg->context[1];
3765 	if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3766 		goto discard;
3767 
3768 	pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n",
3769 			     state, ib_wc_status_msg(wc_status));
3770 	switch (state) {
3771 	case IB_CM_REQ_SENT:
3772 	case IB_CM_MRA_REQ_RCVD:
3773 		cm_reset_to_idle(cm_id_priv);
3774 		cm_event.event = IB_CM_REQ_ERROR;
3775 		break;
3776 	case IB_CM_REP_SENT:
3777 	case IB_CM_MRA_REP_RCVD:
3778 		cm_reset_to_idle(cm_id_priv);
3779 		cm_event.event = IB_CM_REP_ERROR;
3780 		break;
3781 	case IB_CM_DREQ_SENT:
3782 		cm_enter_timewait(cm_id_priv);
3783 		cm_event.event = IB_CM_DREQ_ERROR;
3784 		break;
3785 	case IB_CM_SIDR_REQ_SENT:
3786 		cm_id_priv->id.state = IB_CM_IDLE;
3787 		cm_event.event = IB_CM_SIDR_REQ_ERROR;
3788 		break;
3789 	default:
3790 		goto discard;
3791 	}
3792 	spin_unlock_irq(&cm_id_priv->lock);
3793 	cm_event.param.send_status = wc_status;
3794 
3795 	/* No other events can occur on the cm_id at this point. */
3796 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3797 	cm_free_msg(msg);
3798 	if (ret)
3799 		ib_destroy_cm_id(&cm_id_priv->id);
3800 	return;
3801 discard:
3802 	spin_unlock_irq(&cm_id_priv->lock);
3803 	cm_free_msg(msg);
3804 }
3805 
3806 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3807 			    struct ib_mad_send_wc *mad_send_wc)
3808 {
3809 	struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3810 	struct cm_port *port;
3811 	u16 attr_index;
3812 
3813 	port = mad_agent->context;
3814 	attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3815 				  msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3816 
3817 	/*
3818 	 * If the send was in response to a received message (context[0] is not
3819 	 * set to a cm_id), and is not a REJ, then it is a send that was
3820 	 * manually retried.
3821 	 */
3822 	if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3823 		msg->retries = 1;
3824 
3825 	atomic_long_add(1 + msg->retries,
3826 			&port->counter_group[CM_XMIT].counter[attr_index]);
3827 	if (msg->retries)
3828 		atomic_long_add(msg->retries,
3829 				&port->counter_group[CM_XMIT_RETRIES].
3830 				counter[attr_index]);
3831 
3832 	switch (mad_send_wc->status) {
3833 	case IB_WC_SUCCESS:
3834 	case IB_WC_WR_FLUSH_ERR:
3835 		cm_free_msg(msg);
3836 		break;
3837 	default:
3838 		if (msg->context[0] && msg->context[1])
3839 			cm_process_send_error(msg, mad_send_wc->status);
3840 		else
3841 			cm_free_msg(msg);
3842 		break;
3843 	}
3844 }
3845 
3846 static void cm_work_handler(struct work_struct *_work)
3847 {
3848 	struct cm_work *work = container_of(_work, struct cm_work, work.work);
3849 	int ret;
3850 
3851 	switch (work->cm_event.event) {
3852 	case IB_CM_REQ_RECEIVED:
3853 		ret = cm_req_handler(work);
3854 		break;
3855 	case IB_CM_MRA_RECEIVED:
3856 		ret = cm_mra_handler(work);
3857 		break;
3858 	case IB_CM_REJ_RECEIVED:
3859 		ret = cm_rej_handler(work);
3860 		break;
3861 	case IB_CM_REP_RECEIVED:
3862 		ret = cm_rep_handler(work);
3863 		break;
3864 	case IB_CM_RTU_RECEIVED:
3865 		ret = cm_rtu_handler(work);
3866 		break;
3867 	case IB_CM_USER_ESTABLISHED:
3868 		ret = cm_establish_handler(work);
3869 		break;
3870 	case IB_CM_DREQ_RECEIVED:
3871 		ret = cm_dreq_handler(work);
3872 		break;
3873 	case IB_CM_DREP_RECEIVED:
3874 		ret = cm_drep_handler(work);
3875 		break;
3876 	case IB_CM_SIDR_REQ_RECEIVED:
3877 		ret = cm_sidr_req_handler(work);
3878 		break;
3879 	case IB_CM_SIDR_REP_RECEIVED:
3880 		ret = cm_sidr_rep_handler(work);
3881 		break;
3882 	case IB_CM_LAP_RECEIVED:
3883 		ret = cm_lap_handler(work);
3884 		break;
3885 	case IB_CM_APR_RECEIVED:
3886 		ret = cm_apr_handler(work);
3887 		break;
3888 	case IB_CM_TIMEWAIT_EXIT:
3889 		ret = cm_timewait_handler(work);
3890 		break;
3891 	default:
3892 		pr_debug("cm_event.event: 0x%x\n", work->cm_event.event);
3893 		ret = -EINVAL;
3894 		break;
3895 	}
3896 	if (ret)
3897 		cm_free_work(work);
3898 }
3899 
3900 static int cm_establish(struct ib_cm_id *cm_id)
3901 {
3902 	struct cm_id_private *cm_id_priv;
3903 	struct cm_work *work;
3904 	unsigned long flags;
3905 	int ret = 0;
3906 	struct cm_device *cm_dev;
3907 
3908 	cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3909 	if (!cm_dev)
3910 		return -ENODEV;
3911 
3912 	work = kmalloc(sizeof *work, GFP_ATOMIC);
3913 	if (!work)
3914 		return -ENOMEM;
3915 
3916 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3917 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3918 	switch (cm_id->state)
3919 	{
3920 	case IB_CM_REP_SENT:
3921 	case IB_CM_MRA_REP_RCVD:
3922 		cm_id->state = IB_CM_ESTABLISHED;
3923 		break;
3924 	case IB_CM_ESTABLISHED:
3925 		ret = -EISCONN;
3926 		break;
3927 	default:
3928 		pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
3929 			 be32_to_cpu(cm_id->local_id), cm_id->state);
3930 		ret = -EINVAL;
3931 		break;
3932 	}
3933 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3934 
3935 	if (ret) {
3936 		kfree(work);
3937 		goto out;
3938 	}
3939 
3940 	/*
3941 	 * The CM worker thread may try to destroy the cm_id before it
3942 	 * can execute this work item.  To prevent potential deadlock,
3943 	 * we need to find the cm_id once we're in the context of the
3944 	 * worker thread, rather than holding a reference on it.
3945 	 */
3946 	INIT_DELAYED_WORK(&work->work, cm_work_handler);
3947 	work->local_id = cm_id->local_id;
3948 	work->remote_id = cm_id->remote_id;
3949 	work->mad_recv_wc = NULL;
3950 	work->cm_event.event = IB_CM_USER_ESTABLISHED;
3951 
3952 	/* Check if the device started its remove_one */
3953 	spin_lock_irqsave(&cm.lock, flags);
3954 	if (!cm_dev->going_down) {
3955 		queue_delayed_work(cm.wq, &work->work, 0);
3956 	} else {
3957 		kfree(work);
3958 		ret = -ENODEV;
3959 	}
3960 	spin_unlock_irqrestore(&cm.lock, flags);
3961 
3962 out:
3963 	return ret;
3964 }
3965 
3966 static int cm_migrate(struct ib_cm_id *cm_id)
3967 {
3968 	struct cm_id_private *cm_id_priv;
3969 	struct cm_av tmp_av;
3970 	unsigned long flags;
3971 	int tmp_send_port_not_ready;
3972 	int ret = 0;
3973 
3974 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3975 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3976 	if (cm_id->state == IB_CM_ESTABLISHED &&
3977 	    (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3978 	     cm_id->lap_state == IB_CM_LAP_IDLE)) {
3979 		cm_id->lap_state = IB_CM_LAP_IDLE;
3980 		/* Swap address vector */
3981 		tmp_av = cm_id_priv->av;
3982 		cm_id_priv->av = cm_id_priv->alt_av;
3983 		cm_id_priv->alt_av = tmp_av;
3984 		/* Swap port send ready state */
3985 		tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
3986 		cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
3987 		cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
3988 	} else
3989 		ret = -EINVAL;
3990 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3991 
3992 	return ret;
3993 }
3994 
3995 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3996 {
3997 	int ret;
3998 
3999 	switch (event) {
4000 	case IB_EVENT_COMM_EST:
4001 		ret = cm_establish(cm_id);
4002 		break;
4003 	case IB_EVENT_PATH_MIG:
4004 		ret = cm_migrate(cm_id);
4005 		break;
4006 	default:
4007 		ret = -EINVAL;
4008 	}
4009 	return ret;
4010 }
4011 EXPORT_SYMBOL(ib_cm_notify);
4012 
4013 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
4014 			    struct ib_mad_send_buf *send_buf,
4015 			    struct ib_mad_recv_wc *mad_recv_wc)
4016 {
4017 	struct cm_port *port = mad_agent->context;
4018 	struct cm_work *work;
4019 	enum ib_cm_event_type event;
4020 	bool alt_path = false;
4021 	u16 attr_id;
4022 	int paths = 0;
4023 	int going_down = 0;
4024 
4025 	switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4026 	case CM_REQ_ATTR_ID:
4027 		alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4028 						mad_recv_wc->recv_buf.mad);
4029 		paths = 1 + (alt_path != 0);
4030 		event = IB_CM_REQ_RECEIVED;
4031 		break;
4032 	case CM_MRA_ATTR_ID:
4033 		event = IB_CM_MRA_RECEIVED;
4034 		break;
4035 	case CM_REJ_ATTR_ID:
4036 		event = IB_CM_REJ_RECEIVED;
4037 		break;
4038 	case CM_REP_ATTR_ID:
4039 		event = IB_CM_REP_RECEIVED;
4040 		break;
4041 	case CM_RTU_ATTR_ID:
4042 		event = IB_CM_RTU_RECEIVED;
4043 		break;
4044 	case CM_DREQ_ATTR_ID:
4045 		event = IB_CM_DREQ_RECEIVED;
4046 		break;
4047 	case CM_DREP_ATTR_ID:
4048 		event = IB_CM_DREP_RECEIVED;
4049 		break;
4050 	case CM_SIDR_REQ_ATTR_ID:
4051 		event = IB_CM_SIDR_REQ_RECEIVED;
4052 		break;
4053 	case CM_SIDR_REP_ATTR_ID:
4054 		event = IB_CM_SIDR_REP_RECEIVED;
4055 		break;
4056 	case CM_LAP_ATTR_ID:
4057 		paths = 1;
4058 		event = IB_CM_LAP_RECEIVED;
4059 		break;
4060 	case CM_APR_ATTR_ID:
4061 		event = IB_CM_APR_RECEIVED;
4062 		break;
4063 	default:
4064 		ib_free_recv_mad(mad_recv_wc);
4065 		return;
4066 	}
4067 
4068 	attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4069 	atomic_long_inc(&port->counter_group[CM_RECV].
4070 			counter[attr_id - CM_ATTR_ID_OFFSET]);
4071 
4072 	work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4073 	if (!work) {
4074 		ib_free_recv_mad(mad_recv_wc);
4075 		return;
4076 	}
4077 
4078 	INIT_DELAYED_WORK(&work->work, cm_work_handler);
4079 	work->cm_event.event = event;
4080 	work->mad_recv_wc = mad_recv_wc;
4081 	work->port = port;
4082 
4083 	/* Check if the device started its remove_one */
4084 	spin_lock_irq(&cm.lock);
4085 	if (!port->cm_dev->going_down)
4086 		queue_delayed_work(cm.wq, &work->work, 0);
4087 	else
4088 		going_down = 1;
4089 	spin_unlock_irq(&cm.lock);
4090 
4091 	if (going_down) {
4092 		kfree(work);
4093 		ib_free_recv_mad(mad_recv_wc);
4094 	}
4095 }
4096 
4097 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4098 				struct ib_qp_attr *qp_attr,
4099 				int *qp_attr_mask)
4100 {
4101 	unsigned long flags;
4102 	int ret;
4103 
4104 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4105 	switch (cm_id_priv->id.state) {
4106 	case IB_CM_REQ_SENT:
4107 	case IB_CM_MRA_REQ_RCVD:
4108 	case IB_CM_REQ_RCVD:
4109 	case IB_CM_MRA_REQ_SENT:
4110 	case IB_CM_REP_RCVD:
4111 	case IB_CM_MRA_REP_SENT:
4112 	case IB_CM_REP_SENT:
4113 	case IB_CM_MRA_REP_RCVD:
4114 	case IB_CM_ESTABLISHED:
4115 		*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4116 				IB_QP_PKEY_INDEX | IB_QP_PORT;
4117 		qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4118 		if (cm_id_priv->responder_resources)
4119 			qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4120 						    IB_ACCESS_REMOTE_ATOMIC;
4121 		qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4122 		qp_attr->port_num = cm_id_priv->av.port->port_num;
4123 		ret = 0;
4124 		break;
4125 	default:
4126 		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4127 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
4128 			 cm_id_priv->id.state);
4129 		ret = -EINVAL;
4130 		break;
4131 	}
4132 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4133 	return ret;
4134 }
4135 
4136 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4137 			       struct ib_qp_attr *qp_attr,
4138 			       int *qp_attr_mask)
4139 {
4140 	unsigned long flags;
4141 	int ret;
4142 
4143 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4144 	switch (cm_id_priv->id.state) {
4145 	case IB_CM_REQ_RCVD:
4146 	case IB_CM_MRA_REQ_SENT:
4147 	case IB_CM_REP_RCVD:
4148 	case IB_CM_MRA_REP_SENT:
4149 	case IB_CM_REP_SENT:
4150 	case IB_CM_MRA_REP_RCVD:
4151 	case IB_CM_ESTABLISHED:
4152 		*qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4153 				IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4154 		qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4155 		qp_attr->path_mtu = cm_id_priv->path_mtu;
4156 		qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4157 		qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4158 		if (cm_id_priv->qp_type == IB_QPT_RC ||
4159 		    cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4160 			*qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4161 					 IB_QP_MIN_RNR_TIMER;
4162 			qp_attr->max_dest_rd_atomic =
4163 					cm_id_priv->responder_resources;
4164 			qp_attr->min_rnr_timer = 0;
4165 		}
4166 		if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4167 			*qp_attr_mask |= IB_QP_ALT_PATH;
4168 			qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4169 			qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4170 			qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4171 			qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4172 		}
4173 		ret = 0;
4174 		break;
4175 	default:
4176 		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4177 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
4178 			 cm_id_priv->id.state);
4179 		ret = -EINVAL;
4180 		break;
4181 	}
4182 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4183 	return ret;
4184 }
4185 
4186 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4187 			       struct ib_qp_attr *qp_attr,
4188 			       int *qp_attr_mask)
4189 {
4190 	unsigned long flags;
4191 	int ret;
4192 
4193 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4194 	switch (cm_id_priv->id.state) {
4195 	/* Allow transition to RTS before sending REP */
4196 	case IB_CM_REQ_RCVD:
4197 	case IB_CM_MRA_REQ_SENT:
4198 
4199 	case IB_CM_REP_RCVD:
4200 	case IB_CM_MRA_REP_SENT:
4201 	case IB_CM_REP_SENT:
4202 	case IB_CM_MRA_REP_RCVD:
4203 	case IB_CM_ESTABLISHED:
4204 		if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4205 			*qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4206 			qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4207 			switch (cm_id_priv->qp_type) {
4208 			case IB_QPT_RC:
4209 			case IB_QPT_XRC_INI:
4210 				*qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4211 						 IB_QP_MAX_QP_RD_ATOMIC;
4212 				qp_attr->retry_cnt = cm_id_priv->retry_count;
4213 				qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4214 				qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4215 				/* fall through */
4216 			case IB_QPT_XRC_TGT:
4217 				*qp_attr_mask |= IB_QP_TIMEOUT;
4218 				qp_attr->timeout = cm_id_priv->av.timeout;
4219 				break;
4220 			default:
4221 				break;
4222 			}
4223 			if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4224 				*qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4225 				qp_attr->path_mig_state = IB_MIG_REARM;
4226 			}
4227 		} else {
4228 			*qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4229 			qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4230 			qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4231 			qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4232 			qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4233 			qp_attr->path_mig_state = IB_MIG_REARM;
4234 		}
4235 		ret = 0;
4236 		break;
4237 	default:
4238 		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4239 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
4240 			 cm_id_priv->id.state);
4241 		ret = -EINVAL;
4242 		break;
4243 	}
4244 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4245 	return ret;
4246 }
4247 
4248 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4249 		       struct ib_qp_attr *qp_attr,
4250 		       int *qp_attr_mask)
4251 {
4252 	struct cm_id_private *cm_id_priv;
4253 	int ret;
4254 
4255 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4256 	switch (qp_attr->qp_state) {
4257 	case IB_QPS_INIT:
4258 		ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4259 		break;
4260 	case IB_QPS_RTR:
4261 		ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4262 		break;
4263 	case IB_QPS_RTS:
4264 		ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4265 		break;
4266 	default:
4267 		ret = -EINVAL;
4268 		break;
4269 	}
4270 	return ret;
4271 }
4272 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4273 
4274 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
4275 			       char *buf)
4276 {
4277 	struct cm_counter_group *group;
4278 	struct cm_counter_attribute *cm_attr;
4279 
4280 	group = container_of(obj, struct cm_counter_group, obj);
4281 	cm_attr = container_of(attr, struct cm_counter_attribute, attr);
4282 
4283 	return sprintf(buf, "%ld\n",
4284 		       atomic_long_read(&group->counter[cm_attr->index]));
4285 }
4286 
4287 static const struct sysfs_ops cm_counter_ops = {
4288 	.show = cm_show_counter
4289 };
4290 
4291 static struct kobj_type cm_counter_obj_type = {
4292 	.sysfs_ops = &cm_counter_ops,
4293 	.default_attrs = cm_counter_default_attrs
4294 };
4295 
4296 static char *cm_devnode(struct device *dev, umode_t *mode)
4297 {
4298 	if (mode)
4299 		*mode = 0666;
4300 	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
4301 }
4302 
4303 struct class cm_class = {
4304 	.owner   = THIS_MODULE,
4305 	.name    = "infiniband_cm",
4306 	.devnode = cm_devnode,
4307 };
4308 EXPORT_SYMBOL(cm_class);
4309 
4310 static int cm_create_port_fs(struct cm_port *port)
4311 {
4312 	int i, ret;
4313 
4314 	for (i = 0; i < CM_COUNTER_GROUPS; i++) {
4315 		ret = ib_port_register_module_stat(port->cm_dev->ib_device,
4316 						   port->port_num,
4317 						   &port->counter_group[i].obj,
4318 						   &cm_counter_obj_type,
4319 						   counter_group_names[i]);
4320 		if (ret)
4321 			goto error;
4322 	}
4323 
4324 	return 0;
4325 
4326 error:
4327 	while (i--)
4328 		ib_port_unregister_module_stat(&port->counter_group[i].obj);
4329 	return ret;
4330 
4331 }
4332 
4333 static void cm_remove_port_fs(struct cm_port *port)
4334 {
4335 	int i;
4336 
4337 	for (i = 0; i < CM_COUNTER_GROUPS; i++)
4338 		ib_port_unregister_module_stat(&port->counter_group[i].obj);
4339 
4340 }
4341 
4342 static int cm_add_one(struct ib_device *ib_device)
4343 {
4344 	struct cm_device *cm_dev;
4345 	struct cm_port *port;
4346 	struct ib_mad_reg_req reg_req = {
4347 		.mgmt_class = IB_MGMT_CLASS_CM,
4348 		.mgmt_class_version = IB_CM_CLASS_VERSION,
4349 	};
4350 	struct ib_port_modify port_modify = {
4351 		.set_port_cap_mask = IB_PORT_CM_SUP
4352 	};
4353 	unsigned long flags;
4354 	int ret;
4355 	int count = 0;
4356 	u8 i;
4357 
4358 	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4359 			 GFP_KERNEL);
4360 	if (!cm_dev)
4361 		return -ENOMEM;
4362 
4363 	cm_dev->ib_device = ib_device;
4364 	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4365 	cm_dev->going_down = 0;
4366 
4367 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4368 	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4369 		if (!rdma_cap_ib_cm(ib_device, i))
4370 			continue;
4371 
4372 		port = kzalloc(sizeof *port, GFP_KERNEL);
4373 		if (!port) {
4374 			ret = -ENOMEM;
4375 			goto error1;
4376 		}
4377 
4378 		cm_dev->port[i-1] = port;
4379 		port->cm_dev = cm_dev;
4380 		port->port_num = i;
4381 
4382 		INIT_LIST_HEAD(&port->cm_priv_prim_list);
4383 		INIT_LIST_HEAD(&port->cm_priv_altr_list);
4384 
4385 		ret = cm_create_port_fs(port);
4386 		if (ret)
4387 			goto error1;
4388 
4389 		port->mad_agent = ib_register_mad_agent(ib_device, i,
4390 							IB_QPT_GSI,
4391 							&reg_req,
4392 							0,
4393 							cm_send_handler,
4394 							cm_recv_handler,
4395 							port,
4396 							0);
4397 		if (IS_ERR(port->mad_agent)) {
4398 			ret = PTR_ERR(port->mad_agent);
4399 			goto error2;
4400 		}
4401 
4402 		ret = ib_modify_port(ib_device, i, 0, &port_modify);
4403 		if (ret)
4404 			goto error3;
4405 
4406 		count++;
4407 	}
4408 
4409 	if (!count) {
4410 		ret = -EOPNOTSUPP;
4411 		goto free;
4412 	}
4413 
4414 	ib_set_client_data(ib_device, &cm_client, cm_dev);
4415 
4416 	write_lock_irqsave(&cm.device_lock, flags);
4417 	list_add_tail(&cm_dev->list, &cm.device_list);
4418 	write_unlock_irqrestore(&cm.device_lock, flags);
4419 	return 0;
4420 
4421 error3:
4422 	ib_unregister_mad_agent(port->mad_agent);
4423 error2:
4424 	cm_remove_port_fs(port);
4425 error1:
4426 	port_modify.set_port_cap_mask = 0;
4427 	port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4428 	kfree(port);
4429 	while (--i) {
4430 		if (!rdma_cap_ib_cm(ib_device, i))
4431 			continue;
4432 
4433 		port = cm_dev->port[i-1];
4434 		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4435 		ib_unregister_mad_agent(port->mad_agent);
4436 		cm_remove_port_fs(port);
4437 		kfree(port);
4438 	}
4439 free:
4440 	kfree(cm_dev);
4441 	return ret;
4442 }
4443 
4444 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4445 {
4446 	struct cm_device *cm_dev = client_data;
4447 	struct cm_port *port;
4448 	struct cm_id_private *cm_id_priv;
4449 	struct ib_mad_agent *cur_mad_agent;
4450 	struct ib_port_modify port_modify = {
4451 		.clr_port_cap_mask = IB_PORT_CM_SUP
4452 	};
4453 	unsigned long flags;
4454 	int i;
4455 
4456 	write_lock_irqsave(&cm.device_lock, flags);
4457 	list_del(&cm_dev->list);
4458 	write_unlock_irqrestore(&cm.device_lock, flags);
4459 
4460 	spin_lock_irq(&cm.lock);
4461 	cm_dev->going_down = 1;
4462 	spin_unlock_irq(&cm.lock);
4463 
4464 	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4465 		if (!rdma_cap_ib_cm(ib_device, i))
4466 			continue;
4467 
4468 		port = cm_dev->port[i-1];
4469 		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4470 		/* Mark all the cm_id's as not valid */
4471 		spin_lock_irq(&cm.lock);
4472 		list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4473 			cm_id_priv->altr_send_port_not_ready = 1;
4474 		list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4475 			cm_id_priv->prim_send_port_not_ready = 1;
4476 		spin_unlock_irq(&cm.lock);
4477 		/*
4478 		 * We flush the queue here after the going_down set, this
4479 		 * verify that no new works will be queued in the recv handler,
4480 		 * after that we can call the unregister_mad_agent
4481 		 */
4482 		flush_workqueue(cm.wq);
4483 		spin_lock_irq(&cm.state_lock);
4484 		cur_mad_agent = port->mad_agent;
4485 		port->mad_agent = NULL;
4486 		spin_unlock_irq(&cm.state_lock);
4487 		ib_unregister_mad_agent(cur_mad_agent);
4488 		cm_remove_port_fs(port);
4489 		kfree(port);
4490 	}
4491 
4492 	kfree(cm_dev);
4493 }
4494 
4495 static int __init ib_cm_init(void)
4496 {
4497 	int ret;
4498 
4499 	INIT_LIST_HEAD(&cm.device_list);
4500 	rwlock_init(&cm.device_lock);
4501 	spin_lock_init(&cm.lock);
4502 	spin_lock_init(&cm.state_lock);
4503 	cm.listen_service_table = RB_ROOT;
4504 	cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4505 	cm.remote_id_table = RB_ROOT;
4506 	cm.remote_qp_table = RB_ROOT;
4507 	cm.remote_sidr_table = RB_ROOT;
4508 	xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
4509 	get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4510 	INIT_LIST_HEAD(&cm.timewait_list);
4511 
4512 	ret = class_register(&cm_class);
4513 	if (ret) {
4514 		ret = -ENOMEM;
4515 		goto error1;
4516 	}
4517 
4518 	cm.wq = alloc_workqueue("ib_cm", 0, 1);
4519 	if (!cm.wq) {
4520 		ret = -ENOMEM;
4521 		goto error2;
4522 	}
4523 
4524 	ret = ib_register_client(&cm_client);
4525 	if (ret)
4526 		goto error3;
4527 
4528 	return 0;
4529 error3:
4530 	destroy_workqueue(cm.wq);
4531 error2:
4532 	class_unregister(&cm_class);
4533 error1:
4534 	return ret;
4535 }
4536 
4537 static void __exit ib_cm_cleanup(void)
4538 {
4539 	struct cm_timewait_info *timewait_info, *tmp;
4540 
4541 	spin_lock_irq(&cm.lock);
4542 	list_for_each_entry(timewait_info, &cm.timewait_list, list)
4543 		cancel_delayed_work(&timewait_info->work.work);
4544 	spin_unlock_irq(&cm.lock);
4545 
4546 	ib_unregister_client(&cm_client);
4547 	destroy_workqueue(cm.wq);
4548 
4549 	list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4550 		list_del(&timewait_info->list);
4551 		kfree(timewait_info);
4552 	}
4553 
4554 	class_unregister(&cm_class);
4555 	WARN_ON(!xa_empty(&cm.local_id_table));
4556 }
4557 
4558 module_init(ib_cm_init);
4559 module_exit(ib_cm_cleanup);
4560