xref: /openbmc/linux/drivers/infiniband/core/cm.c (revision b4285282)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
4  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
5  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
6  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
8  */
9 
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/device.h>
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/idr.h>
16 #include <linux/interrupt.h>
17 #include <linux/random.h>
18 #include <linux/rbtree.h>
19 #include <linux/spinlock.h>
20 #include <linux/slab.h>
21 #include <linux/sysfs.h>
22 #include <linux/workqueue.h>
23 #include <linux/kdev_t.h>
24 #include <linux/etherdevice.h>
25 
26 #include <rdma/ib_cache.h>
27 #include <rdma/ib_cm.h>
28 #include <rdma/ib_sysfs.h>
29 #include "cm_msgs.h"
30 #include "core_priv.h"
31 #include "cm_trace.h"
32 
33 MODULE_AUTHOR("Sean Hefty");
34 MODULE_DESCRIPTION("InfiniBand CM");
35 MODULE_LICENSE("Dual BSD/GPL");
36 
37 #define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */
38 static const char * const ibcm_rej_reason_strs[] = {
39 	[IB_CM_REJ_NO_QP]			= "no QP",
40 	[IB_CM_REJ_NO_EEC]			= "no EEC",
41 	[IB_CM_REJ_NO_RESOURCES]		= "no resources",
42 	[IB_CM_REJ_TIMEOUT]			= "timeout",
43 	[IB_CM_REJ_UNSUPPORTED]			= "unsupported",
44 	[IB_CM_REJ_INVALID_COMM_ID]		= "invalid comm ID",
45 	[IB_CM_REJ_INVALID_COMM_INSTANCE]	= "invalid comm instance",
46 	[IB_CM_REJ_INVALID_SERVICE_ID]		= "invalid service ID",
47 	[IB_CM_REJ_INVALID_TRANSPORT_TYPE]	= "invalid transport type",
48 	[IB_CM_REJ_STALE_CONN]			= "stale conn",
49 	[IB_CM_REJ_RDC_NOT_EXIST]		= "RDC not exist",
50 	[IB_CM_REJ_INVALID_GID]			= "invalid GID",
51 	[IB_CM_REJ_INVALID_LID]			= "invalid LID",
52 	[IB_CM_REJ_INVALID_SL]			= "invalid SL",
53 	[IB_CM_REJ_INVALID_TRAFFIC_CLASS]	= "invalid traffic class",
54 	[IB_CM_REJ_INVALID_HOP_LIMIT]		= "invalid hop limit",
55 	[IB_CM_REJ_INVALID_PACKET_RATE]		= "invalid packet rate",
56 	[IB_CM_REJ_INVALID_ALT_GID]		= "invalid alt GID",
57 	[IB_CM_REJ_INVALID_ALT_LID]		= "invalid alt LID",
58 	[IB_CM_REJ_INVALID_ALT_SL]		= "invalid alt SL",
59 	[IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]	= "invalid alt traffic class",
60 	[IB_CM_REJ_INVALID_ALT_HOP_LIMIT]	= "invalid alt hop limit",
61 	[IB_CM_REJ_INVALID_ALT_PACKET_RATE]	= "invalid alt packet rate",
62 	[IB_CM_REJ_PORT_CM_REDIRECT]		= "port CM redirect",
63 	[IB_CM_REJ_PORT_REDIRECT]		= "port redirect",
64 	[IB_CM_REJ_INVALID_MTU]			= "invalid MTU",
65 	[IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES]	= "insufficient resp resources",
66 	[IB_CM_REJ_CONSUMER_DEFINED]		= "consumer defined",
67 	[IB_CM_REJ_INVALID_RNR_RETRY]		= "invalid RNR retry",
68 	[IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]	= "duplicate local comm ID",
69 	[IB_CM_REJ_INVALID_CLASS_VERSION]	= "invalid class version",
70 	[IB_CM_REJ_INVALID_FLOW_LABEL]		= "invalid flow label",
71 	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL]	= "invalid alt flow label",
72 	[IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
73 		"vendor option is not supported",
74 };
75 
ibcm_reject_msg(int reason)76 const char *__attribute_const__ ibcm_reject_msg(int reason)
77 {
78 	size_t index = reason;
79 
80 	if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
81 	    ibcm_rej_reason_strs[index])
82 		return ibcm_rej_reason_strs[index];
83 	else
84 		return "unrecognized reason";
85 }
86 EXPORT_SYMBOL(ibcm_reject_msg);
87 
88 struct cm_id_private;
89 struct cm_work;
90 static int cm_add_one(struct ib_device *device);
91 static void cm_remove_one(struct ib_device *device, void *client_data);
92 static void cm_process_work(struct cm_id_private *cm_id_priv,
93 			    struct cm_work *work);
94 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
95 				   struct ib_cm_sidr_rep_param *param);
96 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
97 			       const void *private_data, u8 private_data_len);
98 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
99 			       void *private_data, u8 private_data_len);
100 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
101 			      enum ib_cm_rej_reason reason, void *ari,
102 			      u8 ari_length, const void *private_data,
103 			      u8 private_data_len);
104 
105 static struct ib_client cm_client = {
106 	.name   = "cm",
107 	.add    = cm_add_one,
108 	.remove = cm_remove_one
109 };
110 
111 static struct ib_cm {
112 	spinlock_t lock;
113 	struct list_head device_list;
114 	rwlock_t device_lock;
115 	struct rb_root listen_service_table;
116 	u64 listen_service_id;
117 	/* struct rb_root peer_service_table; todo: fix peer to peer */
118 	struct rb_root remote_qp_table;
119 	struct rb_root remote_id_table;
120 	struct rb_root remote_sidr_table;
121 	struct xarray local_id_table;
122 	u32 local_id_next;
123 	__be32 random_id_operand;
124 	struct list_head timewait_list;
125 	struct workqueue_struct *wq;
126 } cm;
127 
128 /* Counter indexes ordered by attribute ID */
129 enum {
130 	CM_REQ_COUNTER,
131 	CM_MRA_COUNTER,
132 	CM_REJ_COUNTER,
133 	CM_REP_COUNTER,
134 	CM_RTU_COUNTER,
135 	CM_DREQ_COUNTER,
136 	CM_DREP_COUNTER,
137 	CM_SIDR_REQ_COUNTER,
138 	CM_SIDR_REP_COUNTER,
139 	CM_LAP_COUNTER,
140 	CM_APR_COUNTER,
141 	CM_ATTR_COUNT,
142 	CM_ATTR_ID_OFFSET = 0x0010,
143 };
144 
145 enum {
146 	CM_XMIT,
147 	CM_XMIT_RETRIES,
148 	CM_RECV,
149 	CM_RECV_DUPLICATES,
150 	CM_COUNTER_GROUPS
151 };
152 
153 struct cm_counter_attribute {
154 	struct ib_port_attribute attr;
155 	unsigned short group;
156 	unsigned short index;
157 };
158 
159 struct cm_port {
160 	struct cm_device *cm_dev;
161 	struct ib_mad_agent *mad_agent;
162 	u32 port_num;
163 	atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
164 };
165 
166 struct cm_device {
167 	struct kref kref;
168 	struct list_head list;
169 	spinlock_t mad_agent_lock;
170 	struct ib_device *ib_device;
171 	u8 ack_delay;
172 	int going_down;
173 	struct cm_port *port[];
174 };
175 
176 struct cm_av {
177 	struct cm_port *port;
178 	struct rdma_ah_attr ah_attr;
179 	u16 dlid_datapath;
180 	u16 pkey_index;
181 	u8 timeout;
182 };
183 
184 struct cm_work {
185 	struct delayed_work work;
186 	struct list_head list;
187 	struct cm_port *port;
188 	struct ib_mad_recv_wc *mad_recv_wc;	/* Received MADs */
189 	__be32 local_id;			/* Established / timewait */
190 	__be32 remote_id;
191 	struct ib_cm_event cm_event;
192 	struct sa_path_rec path[];
193 };
194 
195 struct cm_timewait_info {
196 	struct cm_work work;
197 	struct list_head list;
198 	struct rb_node remote_qp_node;
199 	struct rb_node remote_id_node;
200 	__be64 remote_ca_guid;
201 	__be32 remote_qpn;
202 	u8 inserted_remote_qp;
203 	u8 inserted_remote_id;
204 };
205 
206 struct cm_id_private {
207 	struct ib_cm_id	id;
208 
209 	struct rb_node service_node;
210 	struct rb_node sidr_id_node;
211 	u32 sidr_slid;
212 	spinlock_t lock;	/* Do not acquire inside cm.lock */
213 	struct completion comp;
214 	refcount_t refcount;
215 	/* Number of clients sharing this ib_cm_id. Only valid for listeners.
216 	 * Protected by the cm.lock spinlock.
217 	 */
218 	int listen_sharecount;
219 	struct rcu_head rcu;
220 
221 	struct ib_mad_send_buf *msg;
222 	struct cm_timewait_info *timewait_info;
223 	/* todo: use alternate port on send failure */
224 	struct cm_av av;
225 	struct cm_av alt_av;
226 
227 	void *private_data;
228 	__be64 tid;
229 	__be32 local_qpn;
230 	__be32 remote_qpn;
231 	enum ib_qp_type qp_type;
232 	__be32 sq_psn;
233 	__be32 rq_psn;
234 	int timeout_ms;
235 	enum ib_mtu path_mtu;
236 	__be16 pkey;
237 	u8 private_data_len;
238 	u8 max_cm_retries;
239 	u8 responder_resources;
240 	u8 initiator_depth;
241 	u8 retry_count;
242 	u8 rnr_retry_count;
243 	u8 service_timeout;
244 	u8 target_ack_delay;
245 
246 	struct list_head work_list;
247 	atomic_t work_count;
248 
249 	struct rdma_ucm_ece ece;
250 };
251 
cm_dev_release(struct kref * kref)252 static void cm_dev_release(struct kref *kref)
253 {
254 	struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
255 	u32 i;
256 
257 	rdma_for_each_port(cm_dev->ib_device, i)
258 		kfree(cm_dev->port[i - 1]);
259 
260 	kfree(cm_dev);
261 }
262 
cm_device_put(struct cm_device * cm_dev)263 static void cm_device_put(struct cm_device *cm_dev)
264 {
265 	kref_put(&cm_dev->kref, cm_dev_release);
266 }
267 
268 static void cm_work_handler(struct work_struct *work);
269 
cm_deref_id(struct cm_id_private * cm_id_priv)270 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
271 {
272 	if (refcount_dec_and_test(&cm_id_priv->refcount))
273 		complete(&cm_id_priv->comp);
274 }
275 
cm_alloc_msg(struct cm_id_private * cm_id_priv)276 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
277 {
278 	struct ib_mad_agent *mad_agent;
279 	struct ib_mad_send_buf *m;
280 	struct ib_ah *ah;
281 
282 	lockdep_assert_held(&cm_id_priv->lock);
283 
284 	if (!cm_id_priv->av.port)
285 		return ERR_PTR(-EINVAL);
286 
287 	spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
288 	mad_agent = cm_id_priv->av.port->mad_agent;
289 	if (!mad_agent) {
290 		m = ERR_PTR(-EINVAL);
291 		goto out;
292 	}
293 
294 	ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
295 	if (IS_ERR(ah)) {
296 		m = ERR_CAST(ah);
297 		goto out;
298 	}
299 
300 	m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
301 			       cm_id_priv->av.pkey_index,
302 			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
303 			       GFP_ATOMIC,
304 			       IB_MGMT_BASE_VERSION);
305 	if (IS_ERR(m)) {
306 		rdma_destroy_ah(ah, 0);
307 		goto out;
308 	}
309 
310 	/* Timeout set by caller if response is expected. */
311 	m->ah = ah;
312 	m->retries = cm_id_priv->max_cm_retries;
313 
314 	refcount_inc(&cm_id_priv->refcount);
315 	m->context[0] = cm_id_priv;
316 
317 out:
318 	spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
319 	return m;
320 }
321 
cm_free_msg(struct ib_mad_send_buf * msg)322 static void cm_free_msg(struct ib_mad_send_buf *msg)
323 {
324 	struct cm_id_private *cm_id_priv = msg->context[0];
325 
326 	if (msg->ah)
327 		rdma_destroy_ah(msg->ah, 0);
328 	cm_deref_id(cm_id_priv);
329 	ib_free_send_mad(msg);
330 }
331 
332 static struct ib_mad_send_buf *
cm_alloc_priv_msg(struct cm_id_private * cm_id_priv)333 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv)
334 {
335 	struct ib_mad_send_buf *msg;
336 
337 	lockdep_assert_held(&cm_id_priv->lock);
338 
339 	msg = cm_alloc_msg(cm_id_priv);
340 	if (IS_ERR(msg))
341 		return msg;
342 	cm_id_priv->msg = msg;
343 	return msg;
344 }
345 
cm_free_priv_msg(struct ib_mad_send_buf * msg)346 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
347 {
348 	struct cm_id_private *cm_id_priv = msg->context[0];
349 
350 	lockdep_assert_held(&cm_id_priv->lock);
351 
352 	if (!WARN_ON(cm_id_priv->msg != msg))
353 		cm_id_priv->msg = NULL;
354 
355 	if (msg->ah)
356 		rdma_destroy_ah(msg->ah, 0);
357 	cm_deref_id(cm_id_priv);
358 	ib_free_send_mad(msg);
359 }
360 
cm_alloc_response_msg_no_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)361 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
362 							   struct ib_mad_recv_wc *mad_recv_wc)
363 {
364 	return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
365 				  0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
366 				  GFP_ATOMIC,
367 				  IB_MGMT_BASE_VERSION);
368 }
369 
cm_create_response_msg_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf * msg)370 static int cm_create_response_msg_ah(struct cm_port *port,
371 				     struct ib_mad_recv_wc *mad_recv_wc,
372 				     struct ib_mad_send_buf *msg)
373 {
374 	struct ib_ah *ah;
375 
376 	ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
377 				  mad_recv_wc->recv_buf.grh, port->port_num);
378 	if (IS_ERR(ah))
379 		return PTR_ERR(ah);
380 
381 	msg->ah = ah;
382 	return 0;
383 }
384 
cm_alloc_response_msg(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf ** msg)385 static int cm_alloc_response_msg(struct cm_port *port,
386 				 struct ib_mad_recv_wc *mad_recv_wc,
387 				 struct ib_mad_send_buf **msg)
388 {
389 	struct ib_mad_send_buf *m;
390 	int ret;
391 
392 	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
393 	if (IS_ERR(m))
394 		return PTR_ERR(m);
395 
396 	ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
397 	if (ret) {
398 		ib_free_send_mad(m);
399 		return ret;
400 	}
401 
402 	*msg = m;
403 	return 0;
404 }
405 
cm_free_response_msg(struct ib_mad_send_buf * msg)406 static void cm_free_response_msg(struct ib_mad_send_buf *msg)
407 {
408 	if (msg->ah)
409 		rdma_destroy_ah(msg->ah, 0);
410 	ib_free_send_mad(msg);
411 }
412 
cm_copy_private_data(const void * private_data,u8 private_data_len)413 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
414 {
415 	void *data;
416 
417 	if (!private_data || !private_data_len)
418 		return NULL;
419 
420 	data = kmemdup(private_data, private_data_len, GFP_KERNEL);
421 	if (!data)
422 		return ERR_PTR(-ENOMEM);
423 
424 	return data;
425 }
426 
cm_set_private_data(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)427 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
428 				 void *private_data, u8 private_data_len)
429 {
430 	if (cm_id_priv->private_data && cm_id_priv->private_data_len)
431 		kfree(cm_id_priv->private_data);
432 
433 	cm_id_priv->private_data = private_data;
434 	cm_id_priv->private_data_len = private_data_len;
435 }
436 
cm_set_av_port(struct cm_av * av,struct cm_port * port)437 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
438 {
439 	struct cm_port *old_port = av->port;
440 
441 	if (old_port == port)
442 		return;
443 
444 	av->port = port;
445 	if (old_port)
446 		cm_device_put(old_port->cm_dev);
447 	if (port)
448 		kref_get(&port->cm_dev->kref);
449 }
450 
cm_init_av_for_lap(struct cm_port * port,struct ib_wc * wc,struct rdma_ah_attr * ah_attr,struct cm_av * av)451 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
452 			       struct rdma_ah_attr *ah_attr, struct cm_av *av)
453 {
454 	cm_set_av_port(av, port);
455 	av->pkey_index = wc->pkey_index;
456 	rdma_move_ah_attr(&av->ah_attr, ah_attr);
457 }
458 
cm_init_av_for_response(struct cm_port * port,struct ib_wc * wc,struct ib_grh * grh,struct cm_av * av)459 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
460 				   struct ib_grh *grh, struct cm_av *av)
461 {
462 	cm_set_av_port(av, port);
463 	av->pkey_index = wc->pkey_index;
464 	return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
465 				       port->port_num, wc,
466 				       grh, &av->ah_attr);
467 }
468 
469 static struct cm_port *
get_cm_port_from_path(struct sa_path_rec * path,const struct ib_gid_attr * attr)470 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
471 {
472 	struct cm_device *cm_dev;
473 	struct cm_port *port = NULL;
474 	unsigned long flags;
475 
476 	if (attr) {
477 		read_lock_irqsave(&cm.device_lock, flags);
478 		list_for_each_entry(cm_dev, &cm.device_list, list) {
479 			if (cm_dev->ib_device == attr->device) {
480 				port = cm_dev->port[attr->port_num - 1];
481 				break;
482 			}
483 		}
484 		read_unlock_irqrestore(&cm.device_lock, flags);
485 	} else {
486 		/* SGID attribute can be NULL in following
487 		 * conditions.
488 		 * (a) Alternative path
489 		 * (b) IB link layer without GRH
490 		 * (c) LAP send messages
491 		 */
492 		read_lock_irqsave(&cm.device_lock, flags);
493 		list_for_each_entry(cm_dev, &cm.device_list, list) {
494 			attr = rdma_find_gid(cm_dev->ib_device,
495 					     &path->sgid,
496 					     sa_conv_pathrec_to_gid_type(path),
497 					     NULL);
498 			if (!IS_ERR(attr)) {
499 				port = cm_dev->port[attr->port_num - 1];
500 				break;
501 			}
502 		}
503 		read_unlock_irqrestore(&cm.device_lock, flags);
504 		if (port)
505 			rdma_put_gid_attr(attr);
506 	}
507 	return port;
508 }
509 
cm_init_av_by_path(struct sa_path_rec * path,const struct ib_gid_attr * sgid_attr,struct cm_av * av)510 static int cm_init_av_by_path(struct sa_path_rec *path,
511 			      const struct ib_gid_attr *sgid_attr,
512 			      struct cm_av *av)
513 {
514 	struct rdma_ah_attr new_ah_attr;
515 	struct cm_device *cm_dev;
516 	struct cm_port *port;
517 	int ret;
518 
519 	port = get_cm_port_from_path(path, sgid_attr);
520 	if (!port)
521 		return -EINVAL;
522 	cm_dev = port->cm_dev;
523 
524 	ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
525 				  be16_to_cpu(path->pkey), &av->pkey_index);
526 	if (ret)
527 		return ret;
528 
529 	cm_set_av_port(av, port);
530 
531 	/*
532 	 * av->ah_attr might be initialized based on wc or during
533 	 * request processing time which might have reference to sgid_attr.
534 	 * So initialize a new ah_attr on stack.
535 	 * If initialization fails, old ah_attr is used for sending any
536 	 * responses. If initialization is successful, than new ah_attr
537 	 * is used by overwriting the old one. So that right ah_attr
538 	 * can be used to return an error response.
539 	 */
540 	ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
541 					&new_ah_attr, sgid_attr);
542 	if (ret)
543 		return ret;
544 
545 	av->timeout = path->packet_life_time + 1;
546 	rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
547 	return 0;
548 }
549 
550 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
cm_move_av_from_path(struct cm_av * dest,struct cm_av * src)551 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
552 {
553 	cm_set_av_port(dest, src->port);
554 	cm_set_av_port(src, NULL);
555 	dest->pkey_index = src->pkey_index;
556 	rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
557 	dest->timeout = src->timeout;
558 }
559 
cm_destroy_av(struct cm_av * av)560 static void cm_destroy_av(struct cm_av *av)
561 {
562 	rdma_destroy_ah_attr(&av->ah_attr);
563 	cm_set_av_port(av, NULL);
564 }
565 
cm_local_id(__be32 local_id)566 static u32 cm_local_id(__be32 local_id)
567 {
568 	return (__force u32) (local_id ^ cm.random_id_operand);
569 }
570 
cm_acquire_id(__be32 local_id,__be32 remote_id)571 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
572 {
573 	struct cm_id_private *cm_id_priv;
574 
575 	rcu_read_lock();
576 	cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
577 	if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
578 	    !refcount_inc_not_zero(&cm_id_priv->refcount))
579 		cm_id_priv = NULL;
580 	rcu_read_unlock();
581 
582 	return cm_id_priv;
583 }
584 
585 /*
586  * Trivial helpers to strip endian annotation and compare; the
587  * endianness doesn't actually matter since we just need a stable
588  * order for the RB tree.
589  */
be32_lt(__be32 a,__be32 b)590 static int be32_lt(__be32 a, __be32 b)
591 {
592 	return (__force u32) a < (__force u32) b;
593 }
594 
be32_gt(__be32 a,__be32 b)595 static int be32_gt(__be32 a, __be32 b)
596 {
597 	return (__force u32) a > (__force u32) b;
598 }
599 
be64_lt(__be64 a,__be64 b)600 static int be64_lt(__be64 a, __be64 b)
601 {
602 	return (__force u64) a < (__force u64) b;
603 }
604 
be64_gt(__be64 a,__be64 b)605 static int be64_gt(__be64 a, __be64 b)
606 {
607 	return (__force u64) a > (__force u64) b;
608 }
609 
610 /*
611  * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
612  * if the new ID was inserted, NULL if it could not be inserted due to a
613  * collision, or the existing cm_id_priv ready for shared usage.
614  */
cm_insert_listen(struct cm_id_private * cm_id_priv,ib_cm_handler shared_handler)615 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
616 					      ib_cm_handler shared_handler)
617 {
618 	struct rb_node **link = &cm.listen_service_table.rb_node;
619 	struct rb_node *parent = NULL;
620 	struct cm_id_private *cur_cm_id_priv;
621 	__be64 service_id = cm_id_priv->id.service_id;
622 	unsigned long flags;
623 
624 	spin_lock_irqsave(&cm.lock, flags);
625 	while (*link) {
626 		parent = *link;
627 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
628 					  service_node);
629 
630 		if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
631 			link = &(*link)->rb_left;
632 		else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
633 			link = &(*link)->rb_right;
634 		else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
635 			link = &(*link)->rb_left;
636 		else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
637 			link = &(*link)->rb_right;
638 		else {
639 			/*
640 			 * Sharing an ib_cm_id with different handlers is not
641 			 * supported
642 			 */
643 			if (cur_cm_id_priv->id.cm_handler != shared_handler ||
644 			    cur_cm_id_priv->id.context ||
645 			    WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
646 				spin_unlock_irqrestore(&cm.lock, flags);
647 				return NULL;
648 			}
649 			refcount_inc(&cur_cm_id_priv->refcount);
650 			cur_cm_id_priv->listen_sharecount++;
651 			spin_unlock_irqrestore(&cm.lock, flags);
652 			return cur_cm_id_priv;
653 		}
654 	}
655 	cm_id_priv->listen_sharecount++;
656 	rb_link_node(&cm_id_priv->service_node, parent, link);
657 	rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
658 	spin_unlock_irqrestore(&cm.lock, flags);
659 	return cm_id_priv;
660 }
661 
cm_find_listen(struct ib_device * device,__be64 service_id)662 static struct cm_id_private *cm_find_listen(struct ib_device *device,
663 					    __be64 service_id)
664 {
665 	struct rb_node *node = cm.listen_service_table.rb_node;
666 	struct cm_id_private *cm_id_priv;
667 
668 	while (node) {
669 		cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
670 
671 		if (device < cm_id_priv->id.device)
672 			node = node->rb_left;
673 		else if (device > cm_id_priv->id.device)
674 			node = node->rb_right;
675 		else if (be64_lt(service_id, cm_id_priv->id.service_id))
676 			node = node->rb_left;
677 		else if (be64_gt(service_id, cm_id_priv->id.service_id))
678 			node = node->rb_right;
679 		else {
680 			refcount_inc(&cm_id_priv->refcount);
681 			return cm_id_priv;
682 		}
683 	}
684 	return NULL;
685 }
686 
687 static struct cm_timewait_info *
cm_insert_remote_id(struct cm_timewait_info * timewait_info)688 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
689 {
690 	struct rb_node **link = &cm.remote_id_table.rb_node;
691 	struct rb_node *parent = NULL;
692 	struct cm_timewait_info *cur_timewait_info;
693 	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
694 	__be32 remote_id = timewait_info->work.remote_id;
695 
696 	while (*link) {
697 		parent = *link;
698 		cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
699 					     remote_id_node);
700 		if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
701 			link = &(*link)->rb_left;
702 		else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
703 			link = &(*link)->rb_right;
704 		else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
705 			link = &(*link)->rb_left;
706 		else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
707 			link = &(*link)->rb_right;
708 		else
709 			return cur_timewait_info;
710 	}
711 	timewait_info->inserted_remote_id = 1;
712 	rb_link_node(&timewait_info->remote_id_node, parent, link);
713 	rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
714 	return NULL;
715 }
716 
cm_find_remote_id(__be64 remote_ca_guid,__be32 remote_id)717 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
718 					       __be32 remote_id)
719 {
720 	struct rb_node *node = cm.remote_id_table.rb_node;
721 	struct cm_timewait_info *timewait_info;
722 	struct cm_id_private *res = NULL;
723 
724 	spin_lock_irq(&cm.lock);
725 	while (node) {
726 		timewait_info = rb_entry(node, struct cm_timewait_info,
727 					 remote_id_node);
728 		if (be32_lt(remote_id, timewait_info->work.remote_id))
729 			node = node->rb_left;
730 		else if (be32_gt(remote_id, timewait_info->work.remote_id))
731 			node = node->rb_right;
732 		else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
733 			node = node->rb_left;
734 		else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
735 			node = node->rb_right;
736 		else {
737 			res = cm_acquire_id(timewait_info->work.local_id,
738 					     timewait_info->work.remote_id);
739 			break;
740 		}
741 	}
742 	spin_unlock_irq(&cm.lock);
743 	return res;
744 }
745 
746 static struct cm_timewait_info *
cm_insert_remote_qpn(struct cm_timewait_info * timewait_info)747 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
748 {
749 	struct rb_node **link = &cm.remote_qp_table.rb_node;
750 	struct rb_node *parent = NULL;
751 	struct cm_timewait_info *cur_timewait_info;
752 	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
753 	__be32 remote_qpn = timewait_info->remote_qpn;
754 
755 	while (*link) {
756 		parent = *link;
757 		cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
758 					     remote_qp_node);
759 		if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
760 			link = &(*link)->rb_left;
761 		else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
762 			link = &(*link)->rb_right;
763 		else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
764 			link = &(*link)->rb_left;
765 		else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
766 			link = &(*link)->rb_right;
767 		else
768 			return cur_timewait_info;
769 	}
770 	timewait_info->inserted_remote_qp = 1;
771 	rb_link_node(&timewait_info->remote_qp_node, parent, link);
772 	rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
773 	return NULL;
774 }
775 
776 static struct cm_id_private *
cm_insert_remote_sidr(struct cm_id_private * cm_id_priv)777 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
778 {
779 	struct rb_node **link = &cm.remote_sidr_table.rb_node;
780 	struct rb_node *parent = NULL;
781 	struct cm_id_private *cur_cm_id_priv;
782 	__be32 remote_id = cm_id_priv->id.remote_id;
783 
784 	while (*link) {
785 		parent = *link;
786 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
787 					  sidr_id_node);
788 		if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
789 			link = &(*link)->rb_left;
790 		else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
791 			link = &(*link)->rb_right;
792 		else {
793 			if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
794 				link = &(*link)->rb_left;
795 			else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
796 				link = &(*link)->rb_right;
797 			else
798 				return cur_cm_id_priv;
799 		}
800 	}
801 	rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
802 	rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
803 	return NULL;
804 }
805 
cm_alloc_id_priv(struct ib_device * device,ib_cm_handler cm_handler,void * context)806 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
807 					      ib_cm_handler cm_handler,
808 					      void *context)
809 {
810 	struct cm_id_private *cm_id_priv;
811 	u32 id;
812 	int ret;
813 
814 	cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
815 	if (!cm_id_priv)
816 		return ERR_PTR(-ENOMEM);
817 
818 	cm_id_priv->id.state = IB_CM_IDLE;
819 	cm_id_priv->id.device = device;
820 	cm_id_priv->id.cm_handler = cm_handler;
821 	cm_id_priv->id.context = context;
822 	cm_id_priv->id.remote_cm_qpn = 1;
823 
824 	RB_CLEAR_NODE(&cm_id_priv->service_node);
825 	RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
826 	spin_lock_init(&cm_id_priv->lock);
827 	init_completion(&cm_id_priv->comp);
828 	INIT_LIST_HEAD(&cm_id_priv->work_list);
829 	atomic_set(&cm_id_priv->work_count, -1);
830 	refcount_set(&cm_id_priv->refcount, 1);
831 
832 	ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
833 			      &cm.local_id_next, GFP_KERNEL);
834 	if (ret < 0)
835 		goto error;
836 	cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
837 
838 	return cm_id_priv;
839 
840 error:
841 	kfree(cm_id_priv);
842 	return ERR_PTR(ret);
843 }
844 
845 /*
846  * Make the ID visible to the MAD handlers and other threads that use the
847  * xarray.
848  */
cm_finalize_id(struct cm_id_private * cm_id_priv)849 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
850 {
851 	xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
852 		 cm_id_priv, GFP_ATOMIC);
853 }
854 
ib_create_cm_id(struct ib_device * device,ib_cm_handler cm_handler,void * context)855 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
856 				 ib_cm_handler cm_handler,
857 				 void *context)
858 {
859 	struct cm_id_private *cm_id_priv;
860 
861 	cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
862 	if (IS_ERR(cm_id_priv))
863 		return ERR_CAST(cm_id_priv);
864 
865 	cm_finalize_id(cm_id_priv);
866 	return &cm_id_priv->id;
867 }
868 EXPORT_SYMBOL(ib_create_cm_id);
869 
cm_dequeue_work(struct cm_id_private * cm_id_priv)870 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
871 {
872 	struct cm_work *work;
873 
874 	if (list_empty(&cm_id_priv->work_list))
875 		return NULL;
876 
877 	work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
878 	list_del(&work->list);
879 	return work;
880 }
881 
cm_free_work(struct cm_work * work)882 static void cm_free_work(struct cm_work *work)
883 {
884 	if (work->mad_recv_wc)
885 		ib_free_recv_mad(work->mad_recv_wc);
886 	kfree(work);
887 }
888 
cm_queue_work_unlock(struct cm_id_private * cm_id_priv,struct cm_work * work)889 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
890 				 struct cm_work *work)
891 	__releases(&cm_id_priv->lock)
892 {
893 	bool immediate;
894 
895 	/*
896 	 * To deliver the event to the user callback we have the drop the
897 	 * spinlock, however, we need to ensure that the user callback is single
898 	 * threaded and receives events in the temporal order. If there are
899 	 * already events being processed then thread new events onto a list,
900 	 * the thread currently processing will pick them up.
901 	 */
902 	immediate = atomic_inc_and_test(&cm_id_priv->work_count);
903 	if (!immediate) {
904 		list_add_tail(&work->list, &cm_id_priv->work_list);
905 		/*
906 		 * This routine always consumes incoming reference. Once queued
907 		 * to the work_list then a reference is held by the thread
908 		 * currently running cm_process_work() and this reference is not
909 		 * needed.
910 		 */
911 		cm_deref_id(cm_id_priv);
912 	}
913 	spin_unlock_irq(&cm_id_priv->lock);
914 
915 	if (immediate)
916 		cm_process_work(cm_id_priv, work);
917 }
918 
cm_convert_to_ms(int iba_time)919 static inline int cm_convert_to_ms(int iba_time)
920 {
921 	/* approximate conversion to ms from 4.096us x 2^iba_time */
922 	return 1 << max(iba_time - 8, 0);
923 }
924 
925 /*
926  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
927  * Because of how ack_timeout is stored, adding one doubles the timeout.
928  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
929  * increment it (round up) only if the other is within 50%.
930  */
cm_ack_timeout(u8 ca_ack_delay,u8 packet_life_time)931 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
932 {
933 	int ack_timeout = packet_life_time + 1;
934 
935 	if (ack_timeout >= ca_ack_delay)
936 		ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
937 	else
938 		ack_timeout = ca_ack_delay +
939 			      (ack_timeout >= (ca_ack_delay - 1));
940 
941 	return min(31, ack_timeout);
942 }
943 
cm_remove_remote(struct cm_id_private * cm_id_priv)944 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
945 {
946 	struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
947 
948 	if (timewait_info->inserted_remote_id) {
949 		rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
950 		timewait_info->inserted_remote_id = 0;
951 	}
952 
953 	if (timewait_info->inserted_remote_qp) {
954 		rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
955 		timewait_info->inserted_remote_qp = 0;
956 	}
957 }
958 
cm_create_timewait_info(__be32 local_id)959 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
960 {
961 	struct cm_timewait_info *timewait_info;
962 
963 	timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
964 	if (!timewait_info)
965 		return ERR_PTR(-ENOMEM);
966 
967 	timewait_info->work.local_id = local_id;
968 	INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
969 	timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
970 	return timewait_info;
971 }
972 
cm_enter_timewait(struct cm_id_private * cm_id_priv)973 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
974 {
975 	int wait_time;
976 	unsigned long flags;
977 	struct cm_device *cm_dev;
978 
979 	lockdep_assert_held(&cm_id_priv->lock);
980 
981 	cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
982 	if (!cm_dev)
983 		return;
984 
985 	spin_lock_irqsave(&cm.lock, flags);
986 	cm_remove_remote(cm_id_priv);
987 	list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
988 	spin_unlock_irqrestore(&cm.lock, flags);
989 
990 	/*
991 	 * The cm_id could be destroyed by the user before we exit timewait.
992 	 * To protect against this, we search for the cm_id after exiting
993 	 * timewait before notifying the user that we've exited timewait.
994 	 */
995 	cm_id_priv->id.state = IB_CM_TIMEWAIT;
996 	wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
997 
998 	/* Check if the device started its remove_one */
999 	spin_lock_irqsave(&cm.lock, flags);
1000 	if (!cm_dev->going_down)
1001 		queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1002 				   msecs_to_jiffies(wait_time));
1003 	spin_unlock_irqrestore(&cm.lock, flags);
1004 
1005 	/*
1006 	 * The timewait_info is converted into a work and gets freed during
1007 	 * cm_free_work() in cm_timewait_handler().
1008 	 */
1009 	BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1010 	cm_id_priv->timewait_info = NULL;
1011 }
1012 
cm_reset_to_idle(struct cm_id_private * cm_id_priv)1013 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1014 {
1015 	unsigned long flags;
1016 
1017 	lockdep_assert_held(&cm_id_priv->lock);
1018 
1019 	cm_id_priv->id.state = IB_CM_IDLE;
1020 	if (cm_id_priv->timewait_info) {
1021 		spin_lock_irqsave(&cm.lock, flags);
1022 		cm_remove_remote(cm_id_priv);
1023 		spin_unlock_irqrestore(&cm.lock, flags);
1024 		kfree(cm_id_priv->timewait_info);
1025 		cm_id_priv->timewait_info = NULL;
1026 	}
1027 }
1028 
cm_destroy_id_wait_timeout(struct ib_cm_id * cm_id,enum ib_cm_state old_state)1029 static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,
1030 						enum ib_cm_state old_state)
1031 {
1032 	struct cm_id_private *cm_id_priv;
1033 
1034 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1035 	pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__,
1036 	       cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount));
1037 }
1038 
cm_destroy_id(struct ib_cm_id * cm_id,int err)1039 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1040 {
1041 	struct cm_id_private *cm_id_priv;
1042 	enum ib_cm_state old_state;
1043 	struct cm_work *work;
1044 	int ret;
1045 
1046 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1047 	spin_lock_irq(&cm_id_priv->lock);
1048 	old_state = cm_id->state;
1049 retest:
1050 	switch (cm_id->state) {
1051 	case IB_CM_LISTEN:
1052 		spin_lock(&cm.lock);
1053 		if (--cm_id_priv->listen_sharecount > 0) {
1054 			/* The id is still shared. */
1055 			WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1056 			spin_unlock(&cm.lock);
1057 			spin_unlock_irq(&cm_id_priv->lock);
1058 			cm_deref_id(cm_id_priv);
1059 			return;
1060 		}
1061 		cm_id->state = IB_CM_IDLE;
1062 		rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1063 		RB_CLEAR_NODE(&cm_id_priv->service_node);
1064 		spin_unlock(&cm.lock);
1065 		break;
1066 	case IB_CM_SIDR_REQ_SENT:
1067 		cm_id->state = IB_CM_IDLE;
1068 		ib_cancel_mad(cm_id_priv->msg);
1069 		break;
1070 	case IB_CM_SIDR_REQ_RCVD:
1071 		cm_send_sidr_rep_locked(cm_id_priv,
1072 					&(struct ib_cm_sidr_rep_param){
1073 						.status = IB_SIDR_REJECT });
1074 		/* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1075 		cm_id->state = IB_CM_IDLE;
1076 		break;
1077 	case IB_CM_REQ_SENT:
1078 	case IB_CM_MRA_REQ_RCVD:
1079 		ib_cancel_mad(cm_id_priv->msg);
1080 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1081 				   &cm_id_priv->id.device->node_guid,
1082 				   sizeof(cm_id_priv->id.device->node_guid),
1083 				   NULL, 0);
1084 		break;
1085 	case IB_CM_REQ_RCVD:
1086 		if (err == -ENOMEM) {
1087 			/* Do not reject to allow future retries. */
1088 			cm_reset_to_idle(cm_id_priv);
1089 		} else {
1090 			cm_send_rej_locked(cm_id_priv,
1091 					   IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1092 					   NULL, 0);
1093 		}
1094 		break;
1095 	case IB_CM_REP_SENT:
1096 	case IB_CM_MRA_REP_RCVD:
1097 		ib_cancel_mad(cm_id_priv->msg);
1098 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1099 				   0, NULL, 0);
1100 		goto retest;
1101 	case IB_CM_MRA_REQ_SENT:
1102 	case IB_CM_REP_RCVD:
1103 	case IB_CM_MRA_REP_SENT:
1104 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1105 				   0, NULL, 0);
1106 		break;
1107 	case IB_CM_ESTABLISHED:
1108 		if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1109 			cm_id->state = IB_CM_IDLE;
1110 			break;
1111 		}
1112 		cm_send_dreq_locked(cm_id_priv, NULL, 0);
1113 		goto retest;
1114 	case IB_CM_DREQ_SENT:
1115 		ib_cancel_mad(cm_id_priv->msg);
1116 		cm_enter_timewait(cm_id_priv);
1117 		goto retest;
1118 	case IB_CM_DREQ_RCVD:
1119 		cm_send_drep_locked(cm_id_priv, NULL, 0);
1120 		WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1121 		goto retest;
1122 	case IB_CM_TIMEWAIT:
1123 		/*
1124 		 * The cm_acquire_id in cm_timewait_handler will stop working
1125 		 * once we do xa_erase below, so just move to idle here for
1126 		 * consistency.
1127 		 */
1128 		cm_id->state = IB_CM_IDLE;
1129 		break;
1130 	case IB_CM_IDLE:
1131 		break;
1132 	}
1133 	WARN_ON(cm_id->state != IB_CM_IDLE);
1134 
1135 	spin_lock(&cm.lock);
1136 	/* Required for cleanup paths related cm_req_handler() */
1137 	if (cm_id_priv->timewait_info) {
1138 		cm_remove_remote(cm_id_priv);
1139 		kfree(cm_id_priv->timewait_info);
1140 		cm_id_priv->timewait_info = NULL;
1141 	}
1142 
1143 	WARN_ON(cm_id_priv->listen_sharecount);
1144 	WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1145 	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1146 		rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1147 	spin_unlock(&cm.lock);
1148 	spin_unlock_irq(&cm_id_priv->lock);
1149 
1150 	xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1151 	cm_deref_id(cm_id_priv);
1152 	do {
1153 		ret = wait_for_completion_timeout(&cm_id_priv->comp,
1154 						  msecs_to_jiffies(
1155 						  CM_DESTROY_ID_WAIT_TIMEOUT));
1156 		if (!ret) /* timeout happened */
1157 			cm_destroy_id_wait_timeout(cm_id, old_state);
1158 	} while (!ret);
1159 
1160 	while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1161 		cm_free_work(work);
1162 
1163 	cm_destroy_av(&cm_id_priv->av);
1164 	cm_destroy_av(&cm_id_priv->alt_av);
1165 	kfree(cm_id_priv->private_data);
1166 	kfree_rcu(cm_id_priv, rcu);
1167 }
1168 
ib_destroy_cm_id(struct ib_cm_id * cm_id)1169 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1170 {
1171 	cm_destroy_id(cm_id, 0);
1172 }
1173 EXPORT_SYMBOL(ib_destroy_cm_id);
1174 
cm_init_listen(struct cm_id_private * cm_id_priv,__be64 service_id)1175 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1176 {
1177 	if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1178 	    (service_id != IB_CM_ASSIGN_SERVICE_ID))
1179 		return -EINVAL;
1180 
1181 	if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1182 		cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1183 	else
1184 		cm_id_priv->id.service_id = service_id;
1185 
1186 	return 0;
1187 }
1188 
1189 /**
1190  * ib_cm_listen - Initiates listening on the specified service ID for
1191  *   connection and service ID resolution requests.
1192  * @cm_id: Connection identifier associated with the listen request.
1193  * @service_id: Service identifier matched against incoming connection
1194  *   and service ID resolution requests.  The service ID should be specified
1195  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1196  *   assign a service ID to the caller.
1197  */
ib_cm_listen(struct ib_cm_id * cm_id,__be64 service_id)1198 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1199 {
1200 	struct cm_id_private *cm_id_priv =
1201 		container_of(cm_id, struct cm_id_private, id);
1202 	unsigned long flags;
1203 	int ret;
1204 
1205 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1206 	if (cm_id_priv->id.state != IB_CM_IDLE) {
1207 		ret = -EINVAL;
1208 		goto out;
1209 	}
1210 
1211 	ret = cm_init_listen(cm_id_priv, service_id);
1212 	if (ret)
1213 		goto out;
1214 
1215 	if (!cm_insert_listen(cm_id_priv, NULL)) {
1216 		ret = -EBUSY;
1217 		goto out;
1218 	}
1219 
1220 	cm_id_priv->id.state = IB_CM_LISTEN;
1221 	ret = 0;
1222 
1223 out:
1224 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1225 	return ret;
1226 }
1227 EXPORT_SYMBOL(ib_cm_listen);
1228 
1229 /**
1230  * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1231  *			 the given service ID.
1232  *
1233  * If there's an existing ID listening on that same device and service ID,
1234  * return it.
1235  *
1236  * @device: Device associated with the cm_id.  All related communication will
1237  * be associated with the specified device.
1238  * @cm_handler: Callback invoked to notify the user of CM events.
1239  * @service_id: Service identifier matched against incoming connection
1240  *   and service ID resolution requests.  The service ID should be specified
1241  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1242  *   assign a service ID to the caller.
1243  *
1244  * Callers should call ib_destroy_cm_id when done with the listener ID.
1245  */
ib_cm_insert_listen(struct ib_device * device,ib_cm_handler cm_handler,__be64 service_id)1246 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1247 				     ib_cm_handler cm_handler,
1248 				     __be64 service_id)
1249 {
1250 	struct cm_id_private *listen_id_priv;
1251 	struct cm_id_private *cm_id_priv;
1252 	int err = 0;
1253 
1254 	/* Create an ID in advance, since the creation may sleep */
1255 	cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1256 	if (IS_ERR(cm_id_priv))
1257 		return ERR_CAST(cm_id_priv);
1258 
1259 	err = cm_init_listen(cm_id_priv, service_id);
1260 	if (err) {
1261 		ib_destroy_cm_id(&cm_id_priv->id);
1262 		return ERR_PTR(err);
1263 	}
1264 
1265 	spin_lock_irq(&cm_id_priv->lock);
1266 	listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1267 	if (listen_id_priv != cm_id_priv) {
1268 		spin_unlock_irq(&cm_id_priv->lock);
1269 		ib_destroy_cm_id(&cm_id_priv->id);
1270 		if (!listen_id_priv)
1271 			return ERR_PTR(-EINVAL);
1272 		return &listen_id_priv->id;
1273 	}
1274 	cm_id_priv->id.state = IB_CM_LISTEN;
1275 	spin_unlock_irq(&cm_id_priv->lock);
1276 
1277 	/*
1278 	 * A listen ID does not need to be in the xarray since it does not
1279 	 * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1280 	 * and does not enter timewait.
1281 	 */
1282 
1283 	return &cm_id_priv->id;
1284 }
1285 EXPORT_SYMBOL(ib_cm_insert_listen);
1286 
cm_form_tid(struct cm_id_private * cm_id_priv)1287 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1288 {
1289 	u64 hi_tid = 0, low_tid;
1290 
1291 	lockdep_assert_held(&cm_id_priv->lock);
1292 
1293 	low_tid = (u64)cm_id_priv->id.local_id;
1294 	if (!cm_id_priv->av.port)
1295 		return cpu_to_be64(low_tid);
1296 
1297 	spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1298 	if (cm_id_priv->av.port->mad_agent)
1299 		hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1300 	spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1301 	return cpu_to_be64(hi_tid | low_tid);
1302 }
1303 
cm_format_mad_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid)1304 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1305 			      __be16 attr_id, __be64 tid)
1306 {
1307 	hdr->base_version  = IB_MGMT_BASE_VERSION;
1308 	hdr->mgmt_class	   = IB_MGMT_CLASS_CM;
1309 	hdr->class_version = IB_CM_CLASS_VERSION;
1310 	hdr->method	   = IB_MGMT_METHOD_SEND;
1311 	hdr->attr_id	   = attr_id;
1312 	hdr->tid	   = tid;
1313 }
1314 
cm_format_mad_ece_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid,u32 attr_mod)1315 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1316 				  __be64 tid, u32 attr_mod)
1317 {
1318 	cm_format_mad_hdr(hdr, attr_id, tid);
1319 	hdr->attr_mod = cpu_to_be32(attr_mod);
1320 }
1321 
cm_format_req(struct cm_req_msg * req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_req_param * param)1322 static void cm_format_req(struct cm_req_msg *req_msg,
1323 			  struct cm_id_private *cm_id_priv,
1324 			  struct ib_cm_req_param *param)
1325 {
1326 	struct sa_path_rec *pri_path = param->primary_path;
1327 	struct sa_path_rec *alt_path = param->alternate_path;
1328 	bool pri_ext = false;
1329 	__be16 lid;
1330 
1331 	if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1332 		pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1333 					      pri_path->opa.slid);
1334 
1335 	cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1336 			      cm_form_tid(cm_id_priv), param->ece.attr_mod);
1337 
1338 	IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1339 		be32_to_cpu(cm_id_priv->id.local_id));
1340 	IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1341 	IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1342 		be64_to_cpu(cm_id_priv->id.device->node_guid));
1343 	IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1344 	IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1345 	IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1346 		param->remote_cm_response_timeout);
1347 	cm_req_set_qp_type(req_msg, param->qp_type);
1348 	IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1349 	IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1350 	IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1351 		param->local_cm_response_timeout);
1352 	IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1353 		be16_to_cpu(param->primary_path->pkey));
1354 	IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1355 		param->primary_path->mtu);
1356 	IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1357 
1358 	if (param->qp_type != IB_QPT_XRC_INI) {
1359 		IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1360 			param->responder_resources);
1361 		IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1362 		IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1363 			param->rnr_retry_count);
1364 		IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1365 	}
1366 
1367 	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1368 		pri_path->sgid;
1369 	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1370 		pri_path->dgid;
1371 	if (pri_ext) {
1372 		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1373 			->global.interface_id =
1374 			OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1375 		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1376 			->global.interface_id =
1377 			OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1378 	}
1379 	if (pri_path->hop_limit <= 1) {
1380 		IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1381 			be16_to_cpu(pri_ext ? 0 :
1382 					      htons(ntohl(sa_path_get_slid(
1383 						      pri_path)))));
1384 		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1385 			be16_to_cpu(pri_ext ? 0 :
1386 					      htons(ntohl(sa_path_get_dlid(
1387 						      pri_path)))));
1388 	} else {
1389 
1390 		if (param->primary_path_inbound) {
1391 			lid = param->primary_path_inbound->ib.dlid;
1392 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1393 				be16_to_cpu(lid));
1394 		} else
1395 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1396 				be16_to_cpu(IB_LID_PERMISSIVE));
1397 
1398 		/* Work-around until there's a way to obtain remote LID info */
1399 		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1400 			be16_to_cpu(IB_LID_PERMISSIVE));
1401 	}
1402 	IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1403 		be32_to_cpu(pri_path->flow_label));
1404 	IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1405 	IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1406 	IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1407 	IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1408 	IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1409 		(pri_path->hop_limit <= 1));
1410 	IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1411 		cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1412 			       pri_path->packet_life_time));
1413 
1414 	if (alt_path) {
1415 		bool alt_ext = false;
1416 
1417 		if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1418 			alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1419 						      alt_path->opa.slid);
1420 
1421 		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1422 			alt_path->sgid;
1423 		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1424 			alt_path->dgid;
1425 		if (alt_ext) {
1426 			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1427 					req_msg)
1428 				->global.interface_id =
1429 				OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1430 			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1431 					req_msg)
1432 				->global.interface_id =
1433 				OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1434 		}
1435 		if (alt_path->hop_limit <= 1) {
1436 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1437 				be16_to_cpu(
1438 					alt_ext ? 0 :
1439 						  htons(ntohl(sa_path_get_slid(
1440 							  alt_path)))));
1441 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1442 				be16_to_cpu(
1443 					alt_ext ? 0 :
1444 						  htons(ntohl(sa_path_get_dlid(
1445 							  alt_path)))));
1446 		} else {
1447 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1448 				be16_to_cpu(IB_LID_PERMISSIVE));
1449 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1450 				be16_to_cpu(IB_LID_PERMISSIVE));
1451 		}
1452 		IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1453 			be32_to_cpu(alt_path->flow_label));
1454 		IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1455 		IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1456 			alt_path->traffic_class);
1457 		IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1458 			alt_path->hop_limit);
1459 		IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1460 		IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1461 			(alt_path->hop_limit <= 1));
1462 		IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1463 			cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1464 				       alt_path->packet_life_time));
1465 	}
1466 	IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1467 
1468 	if (param->private_data && param->private_data_len)
1469 		IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1470 			    param->private_data_len);
1471 }
1472 
cm_validate_req_param(struct ib_cm_req_param * param)1473 static int cm_validate_req_param(struct ib_cm_req_param *param)
1474 {
1475 	if (!param->primary_path)
1476 		return -EINVAL;
1477 
1478 	if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1479 	    param->qp_type != IB_QPT_XRC_INI)
1480 		return -EINVAL;
1481 
1482 	if (param->private_data &&
1483 	    param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1484 		return -EINVAL;
1485 
1486 	if (param->alternate_path &&
1487 	    (param->alternate_path->pkey != param->primary_path->pkey ||
1488 	     param->alternate_path->mtu != param->primary_path->mtu))
1489 		return -EINVAL;
1490 
1491 	return 0;
1492 }
1493 
ib_send_cm_req(struct ib_cm_id * cm_id,struct ib_cm_req_param * param)1494 int ib_send_cm_req(struct ib_cm_id *cm_id,
1495 		   struct ib_cm_req_param *param)
1496 {
1497 	struct cm_av av = {}, alt_av = {};
1498 	struct cm_id_private *cm_id_priv;
1499 	struct ib_mad_send_buf *msg;
1500 	struct cm_req_msg *req_msg;
1501 	unsigned long flags;
1502 	int ret;
1503 
1504 	ret = cm_validate_req_param(param);
1505 	if (ret)
1506 		return ret;
1507 
1508 	/* Verify that we're not in timewait. */
1509 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1510 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1511 	if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1512 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1513 		return -EINVAL;
1514 	}
1515 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1516 
1517 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1518 							    id.local_id);
1519 	if (IS_ERR(cm_id_priv->timewait_info)) {
1520 		ret = PTR_ERR(cm_id_priv->timewait_info);
1521 		cm_id_priv->timewait_info = NULL;
1522 		return ret;
1523 	}
1524 
1525 	ret = cm_init_av_by_path(param->primary_path,
1526 				 param->ppath_sgid_attr, &av);
1527 	if (ret)
1528 		return ret;
1529 	if (param->alternate_path) {
1530 		ret = cm_init_av_by_path(param->alternate_path, NULL,
1531 					 &alt_av);
1532 		if (ret) {
1533 			cm_destroy_av(&av);
1534 			return ret;
1535 		}
1536 	}
1537 	cm_id->service_id = param->service_id;
1538 	cm_id_priv->timeout_ms = cm_convert_to_ms(
1539 				    param->primary_path->packet_life_time) * 2 +
1540 				 cm_convert_to_ms(
1541 				    param->remote_cm_response_timeout);
1542 	cm_id_priv->max_cm_retries = param->max_cm_retries;
1543 	cm_id_priv->initiator_depth = param->initiator_depth;
1544 	cm_id_priv->responder_resources = param->responder_resources;
1545 	cm_id_priv->retry_count = param->retry_count;
1546 	cm_id_priv->path_mtu = param->primary_path->mtu;
1547 	cm_id_priv->pkey = param->primary_path->pkey;
1548 	cm_id_priv->qp_type = param->qp_type;
1549 
1550 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1551 
1552 	cm_move_av_from_path(&cm_id_priv->av, &av);
1553 	if (param->primary_path_outbound)
1554 		cm_id_priv->av.dlid_datapath =
1555 			be16_to_cpu(param->primary_path_outbound->ib.dlid);
1556 
1557 	if (param->alternate_path)
1558 		cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1559 
1560 	msg = cm_alloc_priv_msg(cm_id_priv);
1561 	if (IS_ERR(msg)) {
1562 		ret = PTR_ERR(msg);
1563 		goto out_unlock;
1564 	}
1565 
1566 	req_msg = (struct cm_req_msg *)msg->mad;
1567 	cm_format_req(req_msg, cm_id_priv, param);
1568 	cm_id_priv->tid = req_msg->hdr.tid;
1569 	msg->timeout_ms = cm_id_priv->timeout_ms;
1570 	msg->context[1] = (void *)(unsigned long)IB_CM_REQ_SENT;
1571 
1572 	cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1573 	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1574 
1575 	trace_icm_send_req(&cm_id_priv->id);
1576 	ret = ib_post_send_mad(msg, NULL);
1577 	if (ret)
1578 		goto out_free;
1579 	BUG_ON(cm_id->state != IB_CM_IDLE);
1580 	cm_id->state = IB_CM_REQ_SENT;
1581 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1582 	return 0;
1583 out_free:
1584 	cm_free_priv_msg(msg);
1585 out_unlock:
1586 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1587 	return ret;
1588 }
1589 EXPORT_SYMBOL(ib_send_cm_req);
1590 
cm_issue_rej(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,enum ib_cm_rej_reason reason,enum cm_msg_response msg_rejected,void * ari,u8 ari_length)1591 static int cm_issue_rej(struct cm_port *port,
1592 			struct ib_mad_recv_wc *mad_recv_wc,
1593 			enum ib_cm_rej_reason reason,
1594 			enum cm_msg_response msg_rejected,
1595 			void *ari, u8 ari_length)
1596 {
1597 	struct ib_mad_send_buf *msg = NULL;
1598 	struct cm_rej_msg *rej_msg, *rcv_msg;
1599 	int ret;
1600 
1601 	ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1602 	if (ret)
1603 		return ret;
1604 
1605 	/* We just need common CM header information.  Cast to any message. */
1606 	rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1607 	rej_msg = (struct cm_rej_msg *) msg->mad;
1608 
1609 	cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1610 	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1611 		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1612 	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1613 		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1614 	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1615 	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1616 
1617 	if (ari && ari_length) {
1618 		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1619 		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1620 	}
1621 
1622 	trace_icm_issue_rej(
1623 		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1624 		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1625 	ret = ib_post_send_mad(msg, NULL);
1626 	if (ret)
1627 		cm_free_response_msg(msg);
1628 
1629 	return ret;
1630 }
1631 
cm_req_has_alt_path(struct cm_req_msg * req_msg)1632 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1633 {
1634 	return ((cpu_to_be16(
1635 			IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1636 		(ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1637 					       req_msg))));
1638 }
1639 
cm_path_set_rec_type(struct ib_device * ib_device,u32 port_num,struct sa_path_rec * path,union ib_gid * gid)1640 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1641 				 struct sa_path_rec *path, union ib_gid *gid)
1642 {
1643 	if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1644 		path->rec_type = SA_PATH_REC_TYPE_OPA;
1645 	else
1646 		path->rec_type = SA_PATH_REC_TYPE_IB;
1647 }
1648 
cm_format_path_lid_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1649 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1650 					struct sa_path_rec *primary_path,
1651 					struct sa_path_rec *alt_path,
1652 					struct ib_wc *wc)
1653 {
1654 	u32 lid;
1655 
1656 	if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1657 		sa_path_set_dlid(primary_path, wc->slid);
1658 		sa_path_set_slid(primary_path,
1659 				 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1660 					 req_msg));
1661 	} else {
1662 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1663 			CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1664 		sa_path_set_dlid(primary_path, lid);
1665 
1666 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1667 			CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1668 		sa_path_set_slid(primary_path, lid);
1669 	}
1670 
1671 	if (!cm_req_has_alt_path(req_msg))
1672 		return;
1673 
1674 	if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1675 		sa_path_set_dlid(alt_path,
1676 				 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1677 					 req_msg));
1678 		sa_path_set_slid(alt_path,
1679 				 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1680 					 req_msg));
1681 	} else {
1682 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1683 			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1684 		sa_path_set_dlid(alt_path, lid);
1685 
1686 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1687 			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1688 		sa_path_set_slid(alt_path, lid);
1689 	}
1690 }
1691 
cm_format_paths_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1692 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1693 				     struct sa_path_rec *primary_path,
1694 				     struct sa_path_rec *alt_path,
1695 				     struct ib_wc *wc)
1696 {
1697 	primary_path->dgid =
1698 		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1699 	primary_path->sgid =
1700 		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1701 	primary_path->flow_label =
1702 		cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1703 	primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1704 	primary_path->traffic_class =
1705 		IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1706 	primary_path->reversible = 1;
1707 	primary_path->pkey =
1708 		cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1709 	primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1710 	primary_path->mtu_selector = IB_SA_EQ;
1711 	primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1712 	primary_path->rate_selector = IB_SA_EQ;
1713 	primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1714 	primary_path->packet_life_time_selector = IB_SA_EQ;
1715 	primary_path->packet_life_time =
1716 		IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1717 	primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1718 	primary_path->service_id =
1719 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1720 	if (sa_path_is_roce(primary_path))
1721 		primary_path->roce.route_resolved = false;
1722 
1723 	if (cm_req_has_alt_path(req_msg)) {
1724 		alt_path->dgid = *IBA_GET_MEM_PTR(
1725 			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1726 		alt_path->sgid = *IBA_GET_MEM_PTR(
1727 			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1728 		alt_path->flow_label = cpu_to_be32(
1729 			IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1730 		alt_path->hop_limit =
1731 			IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1732 		alt_path->traffic_class =
1733 			IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1734 		alt_path->reversible = 1;
1735 		alt_path->pkey =
1736 			cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1737 		alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1738 		alt_path->mtu_selector = IB_SA_EQ;
1739 		alt_path->mtu =
1740 			IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1741 		alt_path->rate_selector = IB_SA_EQ;
1742 		alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1743 		alt_path->packet_life_time_selector = IB_SA_EQ;
1744 		alt_path->packet_life_time =
1745 			IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1746 		alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1747 		alt_path->service_id =
1748 			cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1749 
1750 		if (sa_path_is_roce(alt_path))
1751 			alt_path->roce.route_resolved = false;
1752 	}
1753 	cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1754 }
1755 
cm_get_bth_pkey(struct cm_work * work)1756 static u16 cm_get_bth_pkey(struct cm_work *work)
1757 {
1758 	struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1759 	u32 port_num = work->port->port_num;
1760 	u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1761 	u16 pkey;
1762 	int ret;
1763 
1764 	ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1765 	if (ret) {
1766 		dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1767 				     port_num, pkey_index, ret);
1768 		return 0;
1769 	}
1770 
1771 	return pkey;
1772 }
1773 
1774 /**
1775  * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1776  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1777  * reject them as the local_gid will not match the sgid. Therefore,
1778  * change the pathrec's SGID to an IB SGID.
1779  *
1780  * @work: Work completion
1781  * @path: Path record
1782  */
cm_opa_to_ib_sgid(struct cm_work * work,struct sa_path_rec * path)1783 static void cm_opa_to_ib_sgid(struct cm_work *work,
1784 			      struct sa_path_rec *path)
1785 {
1786 	struct ib_device *dev = work->port->cm_dev->ib_device;
1787 	u32 port_num = work->port->port_num;
1788 
1789 	if (rdma_cap_opa_ah(dev, port_num) &&
1790 	    (ib_is_opa_gid(&path->sgid))) {
1791 		union ib_gid sgid;
1792 
1793 		if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1794 			dev_warn(&dev->dev,
1795 				 "Error updating sgid in CM request\n");
1796 			return;
1797 		}
1798 
1799 		path->sgid = sgid;
1800 	}
1801 }
1802 
cm_format_req_event(struct cm_work * work,struct cm_id_private * cm_id_priv,struct ib_cm_id * listen_id)1803 static void cm_format_req_event(struct cm_work *work,
1804 				struct cm_id_private *cm_id_priv,
1805 				struct ib_cm_id *listen_id)
1806 {
1807 	struct cm_req_msg *req_msg;
1808 	struct ib_cm_req_event_param *param;
1809 
1810 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1811 	param = &work->cm_event.param.req_rcvd;
1812 	param->listen_id = listen_id;
1813 	param->bth_pkey = cm_get_bth_pkey(work);
1814 	param->port = cm_id_priv->av.port->port_num;
1815 	param->primary_path = &work->path[0];
1816 	cm_opa_to_ib_sgid(work, param->primary_path);
1817 	if (cm_req_has_alt_path(req_msg)) {
1818 		param->alternate_path = &work->path[1];
1819 		cm_opa_to_ib_sgid(work, param->alternate_path);
1820 	} else {
1821 		param->alternate_path = NULL;
1822 	}
1823 	param->remote_ca_guid =
1824 		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1825 	param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1826 	param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1827 	param->qp_type = cm_req_get_qp_type(req_msg);
1828 	param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1829 	param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1830 	param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1831 	param->local_cm_response_timeout =
1832 		IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1833 	param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1834 	param->remote_cm_response_timeout =
1835 		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1836 	param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1837 	param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1838 	param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1839 	param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1840 	param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1841 	param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1842 
1843 	work->cm_event.private_data =
1844 		IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1845 }
1846 
cm_process_work(struct cm_id_private * cm_id_priv,struct cm_work * work)1847 static void cm_process_work(struct cm_id_private *cm_id_priv,
1848 			    struct cm_work *work)
1849 {
1850 	int ret;
1851 
1852 	/* We will typically only have the current event to report. */
1853 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1854 	cm_free_work(work);
1855 
1856 	while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1857 		spin_lock_irq(&cm_id_priv->lock);
1858 		work = cm_dequeue_work(cm_id_priv);
1859 		spin_unlock_irq(&cm_id_priv->lock);
1860 		if (!work)
1861 			return;
1862 
1863 		ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1864 						&work->cm_event);
1865 		cm_free_work(work);
1866 	}
1867 	cm_deref_id(cm_id_priv);
1868 	if (ret)
1869 		cm_destroy_id(&cm_id_priv->id, ret);
1870 }
1871 
cm_format_mra(struct cm_mra_msg * mra_msg,struct cm_id_private * cm_id_priv,enum cm_msg_response msg_mraed,u8 service_timeout,const void * private_data,u8 private_data_len)1872 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1873 			  struct cm_id_private *cm_id_priv,
1874 			  enum cm_msg_response msg_mraed, u8 service_timeout,
1875 			  const void *private_data, u8 private_data_len)
1876 {
1877 	cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1878 	IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1879 	IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1880 		be32_to_cpu(cm_id_priv->id.local_id));
1881 	IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1882 		be32_to_cpu(cm_id_priv->id.remote_id));
1883 	IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1884 
1885 	if (private_data && private_data_len)
1886 		IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1887 			    private_data_len);
1888 }
1889 
cm_format_rej(struct cm_rej_msg * rej_msg,struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len,enum ib_cm_state state)1890 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1891 			  struct cm_id_private *cm_id_priv,
1892 			  enum ib_cm_rej_reason reason, void *ari,
1893 			  u8 ari_length, const void *private_data,
1894 			  u8 private_data_len, enum ib_cm_state state)
1895 {
1896 	lockdep_assert_held(&cm_id_priv->lock);
1897 
1898 	cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1899 	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1900 		be32_to_cpu(cm_id_priv->id.remote_id));
1901 
1902 	switch (state) {
1903 	case IB_CM_REQ_RCVD:
1904 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1905 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1906 		break;
1907 	case IB_CM_MRA_REQ_SENT:
1908 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1909 			be32_to_cpu(cm_id_priv->id.local_id));
1910 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1911 		break;
1912 	case IB_CM_REP_RCVD:
1913 	case IB_CM_MRA_REP_SENT:
1914 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1915 			be32_to_cpu(cm_id_priv->id.local_id));
1916 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1917 		break;
1918 	default:
1919 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1920 			be32_to_cpu(cm_id_priv->id.local_id));
1921 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1922 			CM_MSG_RESPONSE_OTHER);
1923 		break;
1924 	}
1925 
1926 	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1927 	if (ari && ari_length) {
1928 		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1929 		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1930 	}
1931 
1932 	if (private_data && private_data_len)
1933 		IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1934 			    private_data_len);
1935 }
1936 
cm_dup_req_handler(struct cm_work * work,struct cm_id_private * cm_id_priv)1937 static void cm_dup_req_handler(struct cm_work *work,
1938 			       struct cm_id_private *cm_id_priv)
1939 {
1940 	struct ib_mad_send_buf *msg = NULL;
1941 	int ret;
1942 
1943 	atomic_long_inc(
1944 		&work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1945 
1946 	/* Quick state check to discard duplicate REQs. */
1947 	spin_lock_irq(&cm_id_priv->lock);
1948 	if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1949 		spin_unlock_irq(&cm_id_priv->lock);
1950 		return;
1951 	}
1952 	spin_unlock_irq(&cm_id_priv->lock);
1953 
1954 	ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1955 	if (ret)
1956 		return;
1957 
1958 	spin_lock_irq(&cm_id_priv->lock);
1959 	switch (cm_id_priv->id.state) {
1960 	case IB_CM_MRA_REQ_SENT:
1961 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1962 			      CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1963 			      cm_id_priv->private_data,
1964 			      cm_id_priv->private_data_len);
1965 		break;
1966 	case IB_CM_TIMEWAIT:
1967 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1968 			      IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1969 			      IB_CM_TIMEWAIT);
1970 		break;
1971 	default:
1972 		goto unlock;
1973 	}
1974 	spin_unlock_irq(&cm_id_priv->lock);
1975 
1976 	trace_icm_send_dup_req(&cm_id_priv->id);
1977 	ret = ib_post_send_mad(msg, NULL);
1978 	if (ret)
1979 		goto free;
1980 	return;
1981 
1982 unlock:	spin_unlock_irq(&cm_id_priv->lock);
1983 free:	cm_free_response_msg(msg);
1984 }
1985 
cm_match_req(struct cm_work * work,struct cm_id_private * cm_id_priv)1986 static struct cm_id_private *cm_match_req(struct cm_work *work,
1987 					  struct cm_id_private *cm_id_priv)
1988 {
1989 	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1990 	struct cm_timewait_info *timewait_info;
1991 	struct cm_req_msg *req_msg;
1992 
1993 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1994 
1995 	/* Check for possible duplicate REQ. */
1996 	spin_lock_irq(&cm.lock);
1997 	timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1998 	if (timewait_info) {
1999 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2000 					   timewait_info->work.remote_id);
2001 		spin_unlock_irq(&cm.lock);
2002 		if (cur_cm_id_priv) {
2003 			cm_dup_req_handler(work, cur_cm_id_priv);
2004 			cm_deref_id(cur_cm_id_priv);
2005 		}
2006 		return NULL;
2007 	}
2008 
2009 	/* Check for stale connections. */
2010 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2011 	if (timewait_info) {
2012 		cm_remove_remote(cm_id_priv);
2013 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2014 					   timewait_info->work.remote_id);
2015 
2016 		spin_unlock_irq(&cm.lock);
2017 		cm_issue_rej(work->port, work->mad_recv_wc,
2018 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2019 			     NULL, 0);
2020 		if (cur_cm_id_priv) {
2021 			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2022 			cm_deref_id(cur_cm_id_priv);
2023 		}
2024 		return NULL;
2025 	}
2026 
2027 	/* Find matching listen request. */
2028 	listen_cm_id_priv = cm_find_listen(
2029 		cm_id_priv->id.device,
2030 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2031 	if (!listen_cm_id_priv) {
2032 		cm_remove_remote(cm_id_priv);
2033 		spin_unlock_irq(&cm.lock);
2034 		cm_issue_rej(work->port, work->mad_recv_wc,
2035 			     IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2036 			     NULL, 0);
2037 		return NULL;
2038 	}
2039 	spin_unlock_irq(&cm.lock);
2040 	return listen_cm_id_priv;
2041 }
2042 
2043 /*
2044  * Work-around for inter-subnet connections.  If the LIDs are permissive,
2045  * we need to override the LID/SL data in the REQ with the LID information
2046  * in the work completion.
2047  */
cm_process_routed_req(struct cm_req_msg * req_msg,struct ib_wc * wc)2048 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2049 {
2050 	if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2051 		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2052 					req_msg)) == IB_LID_PERMISSIVE) {
2053 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2054 				be16_to_cpu(ib_lid_be16(wc->slid)));
2055 			IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2056 		}
2057 
2058 		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2059 					req_msg)) == IB_LID_PERMISSIVE)
2060 			IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2061 				wc->dlid_path_bits);
2062 	}
2063 
2064 	if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2065 		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2066 					req_msg)) == IB_LID_PERMISSIVE) {
2067 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2068 				be16_to_cpu(ib_lid_be16(wc->slid)));
2069 			IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2070 		}
2071 
2072 		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2073 					req_msg)) == IB_LID_PERMISSIVE)
2074 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2075 				wc->dlid_path_bits);
2076 	}
2077 }
2078 
cm_req_handler(struct cm_work * work)2079 static int cm_req_handler(struct cm_work *work)
2080 {
2081 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2082 	struct cm_req_msg *req_msg;
2083 	const struct ib_global_route *grh;
2084 	const struct ib_gid_attr *gid_attr;
2085 	int ret;
2086 
2087 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2088 
2089 	cm_id_priv =
2090 		cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2091 	if (IS_ERR(cm_id_priv))
2092 		return PTR_ERR(cm_id_priv);
2093 
2094 	cm_id_priv->id.remote_id =
2095 		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2096 	cm_id_priv->id.service_id =
2097 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2098 	cm_id_priv->tid = req_msg->hdr.tid;
2099 	cm_id_priv->timeout_ms = cm_convert_to_ms(
2100 		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2101 	cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2102 	cm_id_priv->remote_qpn =
2103 		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2104 	cm_id_priv->initiator_depth =
2105 		IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2106 	cm_id_priv->responder_resources =
2107 		IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2108 	cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2109 	cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2110 	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2111 	cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2112 	cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2113 	cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2114 
2115 	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2116 				      work->mad_recv_wc->recv_buf.grh,
2117 				      &cm_id_priv->av);
2118 	if (ret)
2119 		goto destroy;
2120 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2121 							    id.local_id);
2122 	if (IS_ERR(cm_id_priv->timewait_info)) {
2123 		ret = PTR_ERR(cm_id_priv->timewait_info);
2124 		cm_id_priv->timewait_info = NULL;
2125 		goto destroy;
2126 	}
2127 	cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2128 	cm_id_priv->timewait_info->remote_ca_guid =
2129 		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2130 	cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2131 
2132 	/*
2133 	 * Note that the ID pointer is not in the xarray at this point,
2134 	 * so this set is only visible to the local thread.
2135 	 */
2136 	cm_id_priv->id.state = IB_CM_REQ_RCVD;
2137 
2138 	listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2139 	if (!listen_cm_id_priv) {
2140 		trace_icm_no_listener_err(&cm_id_priv->id);
2141 		cm_id_priv->id.state = IB_CM_IDLE;
2142 		ret = -EINVAL;
2143 		goto destroy;
2144 	}
2145 
2146 	memset(&work->path[0], 0, sizeof(work->path[0]));
2147 	if (cm_req_has_alt_path(req_msg))
2148 		memset(&work->path[1], 0, sizeof(work->path[1]));
2149 	grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2150 	gid_attr = grh->sgid_attr;
2151 
2152 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2153 		work->path[0].rec_type =
2154 			sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2155 	} else {
2156 		cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2157 		cm_path_set_rec_type(
2158 			work->port->cm_dev->ib_device, work->port->port_num,
2159 			&work->path[0],
2160 			IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2161 					req_msg));
2162 	}
2163 	if (cm_req_has_alt_path(req_msg))
2164 		work->path[1].rec_type = work->path[0].rec_type;
2165 	cm_format_paths_from_req(req_msg, &work->path[0],
2166 				 &work->path[1], work->mad_recv_wc->wc);
2167 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2168 		sa_path_set_dmac(&work->path[0],
2169 				 cm_id_priv->av.ah_attr.roce.dmac);
2170 	work->path[0].hop_limit = grh->hop_limit;
2171 
2172 	/* This destroy call is needed to pair with cm_init_av_for_response */
2173 	cm_destroy_av(&cm_id_priv->av);
2174 	ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2175 	if (ret) {
2176 		int err;
2177 
2178 		err = rdma_query_gid(work->port->cm_dev->ib_device,
2179 				     work->port->port_num, 0,
2180 				     &work->path[0].sgid);
2181 		if (err)
2182 			ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2183 				       NULL, 0, NULL, 0);
2184 		else
2185 			ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2186 				       &work->path[0].sgid,
2187 				       sizeof(work->path[0].sgid),
2188 				       NULL, 0);
2189 		goto rejected;
2190 	}
2191 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2192 		cm_id_priv->av.dlid_datapath =
2193 			IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2194 
2195 	if (cm_req_has_alt_path(req_msg)) {
2196 		ret = cm_init_av_by_path(&work->path[1], NULL,
2197 					 &cm_id_priv->alt_av);
2198 		if (ret) {
2199 			ib_send_cm_rej(&cm_id_priv->id,
2200 				       IB_CM_REJ_INVALID_ALT_GID,
2201 				       &work->path[0].sgid,
2202 				       sizeof(work->path[0].sgid), NULL, 0);
2203 			goto rejected;
2204 		}
2205 	}
2206 
2207 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2208 	cm_id_priv->id.context = listen_cm_id_priv->id.context;
2209 	cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2210 
2211 	/* Now MAD handlers can see the new ID */
2212 	spin_lock_irq(&cm_id_priv->lock);
2213 	cm_finalize_id(cm_id_priv);
2214 
2215 	/* Refcount belongs to the event, pairs with cm_process_work() */
2216 	refcount_inc(&cm_id_priv->refcount);
2217 	cm_queue_work_unlock(cm_id_priv, work);
2218 	/*
2219 	 * Since this ID was just created and was not made visible to other MAD
2220 	 * handlers until the cm_finalize_id() above we know that the
2221 	 * cm_process_work() will deliver the event and the listen_cm_id
2222 	 * embedded in the event can be derefed here.
2223 	 */
2224 	cm_deref_id(listen_cm_id_priv);
2225 	return 0;
2226 
2227 rejected:
2228 	cm_deref_id(listen_cm_id_priv);
2229 destroy:
2230 	ib_destroy_cm_id(&cm_id_priv->id);
2231 	return ret;
2232 }
2233 
cm_format_rep(struct cm_rep_msg * rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_rep_param * param)2234 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2235 			  struct cm_id_private *cm_id_priv,
2236 			  struct ib_cm_rep_param *param)
2237 {
2238 	cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2239 			      param->ece.attr_mod);
2240 	IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2241 		be32_to_cpu(cm_id_priv->id.local_id));
2242 	IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2243 		be32_to_cpu(cm_id_priv->id.remote_id));
2244 	IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2245 	IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2246 		param->responder_resources);
2247 	IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2248 		cm_id_priv->av.port->cm_dev->ack_delay);
2249 	IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2250 	IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2251 	IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2252 		be64_to_cpu(cm_id_priv->id.device->node_guid));
2253 
2254 	if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2255 		IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2256 			param->initiator_depth);
2257 		IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2258 			param->flow_control);
2259 		IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2260 		IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2261 	} else {
2262 		IBA_SET(CM_REP_SRQ, rep_msg, 1);
2263 		IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2264 	}
2265 
2266 	IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2267 	IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2268 	IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2269 
2270 	if (param->private_data && param->private_data_len)
2271 		IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2272 			    param->private_data_len);
2273 }
2274 
ib_send_cm_rep(struct ib_cm_id * cm_id,struct ib_cm_rep_param * param)2275 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2276 		   struct ib_cm_rep_param *param)
2277 {
2278 	struct cm_id_private *cm_id_priv;
2279 	struct ib_mad_send_buf *msg;
2280 	struct cm_rep_msg *rep_msg;
2281 	unsigned long flags;
2282 	int ret;
2283 
2284 	if (param->private_data &&
2285 	    param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2286 		return -EINVAL;
2287 
2288 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2289 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2290 	if (cm_id->state != IB_CM_REQ_RCVD &&
2291 	    cm_id->state != IB_CM_MRA_REQ_SENT) {
2292 		trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2293 		ret = -EINVAL;
2294 		goto out;
2295 	}
2296 
2297 	msg = cm_alloc_priv_msg(cm_id_priv);
2298 	if (IS_ERR(msg)) {
2299 		ret = PTR_ERR(msg);
2300 		goto out;
2301 	}
2302 
2303 	rep_msg = (struct cm_rep_msg *) msg->mad;
2304 	cm_format_rep(rep_msg, cm_id_priv, param);
2305 	msg->timeout_ms = cm_id_priv->timeout_ms;
2306 	msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2307 
2308 	trace_icm_send_rep(cm_id);
2309 	ret = ib_post_send_mad(msg, NULL);
2310 	if (ret)
2311 		goto out_free;
2312 
2313 	cm_id->state = IB_CM_REP_SENT;
2314 	cm_id_priv->initiator_depth = param->initiator_depth;
2315 	cm_id_priv->responder_resources = param->responder_resources;
2316 	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2317 	WARN_ONCE(param->qp_num & 0xFF000000,
2318 		  "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2319 		  param->qp_num);
2320 	cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2321 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2322 	return 0;
2323 
2324 out_free:
2325 	cm_free_priv_msg(msg);
2326 out:
2327 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2328 	return ret;
2329 }
2330 EXPORT_SYMBOL(ib_send_cm_rep);
2331 
cm_format_rtu(struct cm_rtu_msg * rtu_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2332 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2333 			  struct cm_id_private *cm_id_priv,
2334 			  const void *private_data,
2335 			  u8 private_data_len)
2336 {
2337 	cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2338 	IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2339 		be32_to_cpu(cm_id_priv->id.local_id));
2340 	IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2341 		be32_to_cpu(cm_id_priv->id.remote_id));
2342 
2343 	if (private_data && private_data_len)
2344 		IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2345 			    private_data_len);
2346 }
2347 
ib_send_cm_rtu(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2348 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2349 		   const void *private_data,
2350 		   u8 private_data_len)
2351 {
2352 	struct cm_id_private *cm_id_priv;
2353 	struct ib_mad_send_buf *msg;
2354 	unsigned long flags;
2355 	void *data;
2356 	int ret;
2357 
2358 	if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2359 		return -EINVAL;
2360 
2361 	data = cm_copy_private_data(private_data, private_data_len);
2362 	if (IS_ERR(data))
2363 		return PTR_ERR(data);
2364 
2365 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2366 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2367 	if (cm_id->state != IB_CM_REP_RCVD &&
2368 	    cm_id->state != IB_CM_MRA_REP_SENT) {
2369 		trace_icm_send_cm_rtu_err(cm_id);
2370 		ret = -EINVAL;
2371 		goto error;
2372 	}
2373 
2374 	msg = cm_alloc_msg(cm_id_priv);
2375 	if (IS_ERR(msg)) {
2376 		ret = PTR_ERR(msg);
2377 		goto error;
2378 	}
2379 
2380 	cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2381 		      private_data, private_data_len);
2382 
2383 	trace_icm_send_rtu(cm_id);
2384 	ret = ib_post_send_mad(msg, NULL);
2385 	if (ret) {
2386 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2387 		cm_free_msg(msg);
2388 		kfree(data);
2389 		return ret;
2390 	}
2391 
2392 	cm_id->state = IB_CM_ESTABLISHED;
2393 	cm_set_private_data(cm_id_priv, data, private_data_len);
2394 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2395 	return 0;
2396 
2397 error:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2398 	kfree(data);
2399 	return ret;
2400 }
2401 EXPORT_SYMBOL(ib_send_cm_rtu);
2402 
cm_format_rep_event(struct cm_work * work,enum ib_qp_type qp_type)2403 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2404 {
2405 	struct cm_rep_msg *rep_msg;
2406 	struct ib_cm_rep_event_param *param;
2407 
2408 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2409 	param = &work->cm_event.param.rep_rcvd;
2410 	param->remote_ca_guid =
2411 		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2412 	param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2413 	param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2414 	param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2415 	param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2416 	param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2417 	param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2418 	param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2419 	param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2420 	param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2421 	param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2422 	param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2423 	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2424 	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2425 	param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2426 
2427 	work->cm_event.private_data =
2428 		IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2429 }
2430 
cm_dup_rep_handler(struct cm_work * work)2431 static void cm_dup_rep_handler(struct cm_work *work)
2432 {
2433 	struct cm_id_private *cm_id_priv;
2434 	struct cm_rep_msg *rep_msg;
2435 	struct ib_mad_send_buf *msg = NULL;
2436 	int ret;
2437 
2438 	rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2439 	cm_id_priv = cm_acquire_id(
2440 		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2441 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2442 	if (!cm_id_priv)
2443 		return;
2444 
2445 	atomic_long_inc(
2446 		&work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2447 	ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2448 	if (ret)
2449 		goto deref;
2450 
2451 	spin_lock_irq(&cm_id_priv->lock);
2452 	if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2453 		cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2454 			      cm_id_priv->private_data,
2455 			      cm_id_priv->private_data_len);
2456 	else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2457 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2458 			      CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2459 			      cm_id_priv->private_data,
2460 			      cm_id_priv->private_data_len);
2461 	else
2462 		goto unlock;
2463 	spin_unlock_irq(&cm_id_priv->lock);
2464 
2465 	trace_icm_send_dup_rep(&cm_id_priv->id);
2466 	ret = ib_post_send_mad(msg, NULL);
2467 	if (ret)
2468 		goto free;
2469 	goto deref;
2470 
2471 unlock:	spin_unlock_irq(&cm_id_priv->lock);
2472 free:	cm_free_response_msg(msg);
2473 deref:	cm_deref_id(cm_id_priv);
2474 }
2475 
cm_rep_handler(struct cm_work * work)2476 static int cm_rep_handler(struct cm_work *work)
2477 {
2478 	struct cm_id_private *cm_id_priv;
2479 	struct cm_rep_msg *rep_msg;
2480 	int ret;
2481 	struct cm_id_private *cur_cm_id_priv;
2482 	struct cm_timewait_info *timewait_info;
2483 
2484 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2485 	cm_id_priv = cm_acquire_id(
2486 		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2487 	if (!cm_id_priv) {
2488 		cm_dup_rep_handler(work);
2489 		trace_icm_remote_no_priv_err(
2490 			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2491 		return -EINVAL;
2492 	}
2493 
2494 	cm_format_rep_event(work, cm_id_priv->qp_type);
2495 
2496 	spin_lock_irq(&cm_id_priv->lock);
2497 	switch (cm_id_priv->id.state) {
2498 	case IB_CM_REQ_SENT:
2499 	case IB_CM_MRA_REQ_RCVD:
2500 		break;
2501 	default:
2502 		ret = -EINVAL;
2503 		trace_icm_rep_unknown_err(
2504 			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2505 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2506 			cm_id_priv->id.state);
2507 		spin_unlock_irq(&cm_id_priv->lock);
2508 		goto error;
2509 	}
2510 
2511 	cm_id_priv->timewait_info->work.remote_id =
2512 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2513 	cm_id_priv->timewait_info->remote_ca_guid =
2514 		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2515 	cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2516 
2517 	spin_lock(&cm.lock);
2518 	/* Check for duplicate REP. */
2519 	if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2520 		spin_unlock(&cm.lock);
2521 		spin_unlock_irq(&cm_id_priv->lock);
2522 		ret = -EINVAL;
2523 		trace_icm_insert_failed_err(
2524 			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2525 		goto error;
2526 	}
2527 	/* Check for a stale connection. */
2528 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2529 	if (timewait_info) {
2530 		cm_remove_remote(cm_id_priv);
2531 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2532 					   timewait_info->work.remote_id);
2533 
2534 		spin_unlock(&cm.lock);
2535 		spin_unlock_irq(&cm_id_priv->lock);
2536 		cm_issue_rej(work->port, work->mad_recv_wc,
2537 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2538 			     NULL, 0);
2539 		ret = -EINVAL;
2540 		trace_icm_staleconn_err(
2541 			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2542 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2543 
2544 		if (cur_cm_id_priv) {
2545 			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2546 			cm_deref_id(cur_cm_id_priv);
2547 		}
2548 
2549 		goto error;
2550 	}
2551 	spin_unlock(&cm.lock);
2552 
2553 	cm_id_priv->id.state = IB_CM_REP_RCVD;
2554 	cm_id_priv->id.remote_id =
2555 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2556 	cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2557 	cm_id_priv->initiator_depth =
2558 		IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2559 	cm_id_priv->responder_resources =
2560 		IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2561 	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2562 	cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2563 	cm_id_priv->target_ack_delay =
2564 		IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2565 	cm_id_priv->av.timeout =
2566 			cm_ack_timeout(cm_id_priv->target_ack_delay,
2567 				       cm_id_priv->av.timeout - 1);
2568 	cm_id_priv->alt_av.timeout =
2569 			cm_ack_timeout(cm_id_priv->target_ack_delay,
2570 				       cm_id_priv->alt_av.timeout - 1);
2571 
2572 	ib_cancel_mad(cm_id_priv->msg);
2573 	cm_queue_work_unlock(cm_id_priv, work);
2574 	return 0;
2575 
2576 error:
2577 	cm_deref_id(cm_id_priv);
2578 	return ret;
2579 }
2580 
cm_establish_handler(struct cm_work * work)2581 static int cm_establish_handler(struct cm_work *work)
2582 {
2583 	struct cm_id_private *cm_id_priv;
2584 
2585 	/* See comment in cm_establish about lookup. */
2586 	cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2587 	if (!cm_id_priv)
2588 		return -EINVAL;
2589 
2590 	spin_lock_irq(&cm_id_priv->lock);
2591 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2592 		spin_unlock_irq(&cm_id_priv->lock);
2593 		goto out;
2594 	}
2595 
2596 	ib_cancel_mad(cm_id_priv->msg);
2597 	cm_queue_work_unlock(cm_id_priv, work);
2598 	return 0;
2599 out:
2600 	cm_deref_id(cm_id_priv);
2601 	return -EINVAL;
2602 }
2603 
cm_rtu_handler(struct cm_work * work)2604 static int cm_rtu_handler(struct cm_work *work)
2605 {
2606 	struct cm_id_private *cm_id_priv;
2607 	struct cm_rtu_msg *rtu_msg;
2608 
2609 	rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2610 	cm_id_priv = cm_acquire_id(
2611 		cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2612 		cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2613 	if (!cm_id_priv)
2614 		return -EINVAL;
2615 
2616 	work->cm_event.private_data =
2617 		IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2618 
2619 	spin_lock_irq(&cm_id_priv->lock);
2620 	if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2621 	    cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2622 		spin_unlock_irq(&cm_id_priv->lock);
2623 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2624 						     [CM_RTU_COUNTER]);
2625 		goto out;
2626 	}
2627 	cm_id_priv->id.state = IB_CM_ESTABLISHED;
2628 
2629 	ib_cancel_mad(cm_id_priv->msg);
2630 	cm_queue_work_unlock(cm_id_priv, work);
2631 	return 0;
2632 out:
2633 	cm_deref_id(cm_id_priv);
2634 	return -EINVAL;
2635 }
2636 
cm_format_dreq(struct cm_dreq_msg * dreq_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2637 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2638 			  struct cm_id_private *cm_id_priv,
2639 			  const void *private_data,
2640 			  u8 private_data_len)
2641 {
2642 	cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2643 			  cm_form_tid(cm_id_priv));
2644 	IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2645 		be32_to_cpu(cm_id_priv->id.local_id));
2646 	IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2647 		be32_to_cpu(cm_id_priv->id.remote_id));
2648 	IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2649 		be32_to_cpu(cm_id_priv->remote_qpn));
2650 
2651 	if (private_data && private_data_len)
2652 		IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2653 			    private_data_len);
2654 }
2655 
cm_send_dreq_locked(struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2656 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2657 			       const void *private_data, u8 private_data_len)
2658 {
2659 	struct ib_mad_send_buf *msg;
2660 	int ret;
2661 
2662 	lockdep_assert_held(&cm_id_priv->lock);
2663 
2664 	if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2665 		return -EINVAL;
2666 
2667 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2668 		trace_icm_dreq_skipped(&cm_id_priv->id);
2669 		return -EINVAL;
2670 	}
2671 
2672 	if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2673 	    cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2674 		ib_cancel_mad(cm_id_priv->msg);
2675 
2676 	msg = cm_alloc_priv_msg(cm_id_priv);
2677 	if (IS_ERR(msg)) {
2678 		cm_enter_timewait(cm_id_priv);
2679 		return PTR_ERR(msg);
2680 	}
2681 
2682 	cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2683 		       private_data, private_data_len);
2684 	msg->timeout_ms = cm_id_priv->timeout_ms;
2685 	msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2686 
2687 	trace_icm_send_dreq(&cm_id_priv->id);
2688 	ret = ib_post_send_mad(msg, NULL);
2689 	if (ret) {
2690 		cm_enter_timewait(cm_id_priv);
2691 		cm_free_priv_msg(msg);
2692 		return ret;
2693 	}
2694 
2695 	cm_id_priv->id.state = IB_CM_DREQ_SENT;
2696 	return 0;
2697 }
2698 
ib_send_cm_dreq(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2699 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2700 		    u8 private_data_len)
2701 {
2702 	struct cm_id_private *cm_id_priv =
2703 		container_of(cm_id, struct cm_id_private, id);
2704 	unsigned long flags;
2705 	int ret;
2706 
2707 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2708 	ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2709 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2710 	return ret;
2711 }
2712 EXPORT_SYMBOL(ib_send_cm_dreq);
2713 
cm_format_drep(struct cm_drep_msg * drep_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2714 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2715 			  struct cm_id_private *cm_id_priv,
2716 			  const void *private_data,
2717 			  u8 private_data_len)
2718 {
2719 	cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2720 	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2721 		be32_to_cpu(cm_id_priv->id.local_id));
2722 	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2723 		be32_to_cpu(cm_id_priv->id.remote_id));
2724 
2725 	if (private_data && private_data_len)
2726 		IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2727 			    private_data_len);
2728 }
2729 
cm_send_drep_locked(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)2730 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2731 			       void *private_data, u8 private_data_len)
2732 {
2733 	struct ib_mad_send_buf *msg;
2734 	int ret;
2735 
2736 	lockdep_assert_held(&cm_id_priv->lock);
2737 
2738 	if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2739 		return -EINVAL;
2740 
2741 	if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2742 		trace_icm_send_drep_err(&cm_id_priv->id);
2743 		kfree(private_data);
2744 		return -EINVAL;
2745 	}
2746 
2747 	cm_set_private_data(cm_id_priv, private_data, private_data_len);
2748 	cm_enter_timewait(cm_id_priv);
2749 
2750 	msg = cm_alloc_msg(cm_id_priv);
2751 	if (IS_ERR(msg))
2752 		return PTR_ERR(msg);
2753 
2754 	cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2755 		       private_data, private_data_len);
2756 
2757 	trace_icm_send_drep(&cm_id_priv->id);
2758 	ret = ib_post_send_mad(msg, NULL);
2759 	if (ret) {
2760 		cm_free_msg(msg);
2761 		return ret;
2762 	}
2763 	return 0;
2764 }
2765 
ib_send_cm_drep(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2766 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2767 		    u8 private_data_len)
2768 {
2769 	struct cm_id_private *cm_id_priv =
2770 		container_of(cm_id, struct cm_id_private, id);
2771 	unsigned long flags;
2772 	void *data;
2773 	int ret;
2774 
2775 	data = cm_copy_private_data(private_data, private_data_len);
2776 	if (IS_ERR(data))
2777 		return PTR_ERR(data);
2778 
2779 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2780 	ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2781 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2782 	return ret;
2783 }
2784 EXPORT_SYMBOL(ib_send_cm_drep);
2785 
cm_issue_drep(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)2786 static int cm_issue_drep(struct cm_port *port,
2787 			 struct ib_mad_recv_wc *mad_recv_wc)
2788 {
2789 	struct ib_mad_send_buf *msg = NULL;
2790 	struct cm_dreq_msg *dreq_msg;
2791 	struct cm_drep_msg *drep_msg;
2792 	int ret;
2793 
2794 	ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2795 	if (ret)
2796 		return ret;
2797 
2798 	dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2799 	drep_msg = (struct cm_drep_msg *) msg->mad;
2800 
2801 	cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2802 	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2803 		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2804 	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2805 		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2806 
2807 	trace_icm_issue_drep(
2808 		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2809 		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2810 	ret = ib_post_send_mad(msg, NULL);
2811 	if (ret)
2812 		cm_free_response_msg(msg);
2813 
2814 	return ret;
2815 }
2816 
cm_dreq_handler(struct cm_work * work)2817 static int cm_dreq_handler(struct cm_work *work)
2818 {
2819 	struct cm_id_private *cm_id_priv;
2820 	struct cm_dreq_msg *dreq_msg;
2821 	struct ib_mad_send_buf *msg = NULL;
2822 
2823 	dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2824 	cm_id_priv = cm_acquire_id(
2825 		cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2826 		cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2827 	if (!cm_id_priv) {
2828 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2829 						     [CM_DREQ_COUNTER]);
2830 		cm_issue_drep(work->port, work->mad_recv_wc);
2831 		trace_icm_no_priv_err(
2832 			IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2833 			IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2834 		return -EINVAL;
2835 	}
2836 
2837 	work->cm_event.private_data =
2838 		IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2839 
2840 	spin_lock_irq(&cm_id_priv->lock);
2841 	if (cm_id_priv->local_qpn !=
2842 	    cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2843 		goto unlock;
2844 
2845 	switch (cm_id_priv->id.state) {
2846 	case IB_CM_REP_SENT:
2847 	case IB_CM_DREQ_SENT:
2848 	case IB_CM_MRA_REP_RCVD:
2849 		ib_cancel_mad(cm_id_priv->msg);
2850 		break;
2851 	case IB_CM_ESTABLISHED:
2852 		if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2853 		    cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2854 			ib_cancel_mad(cm_id_priv->msg);
2855 		break;
2856 	case IB_CM_TIMEWAIT:
2857 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2858 						     [CM_DREQ_COUNTER]);
2859 		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2860 		if (IS_ERR(msg))
2861 			goto unlock;
2862 
2863 		cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2864 			       cm_id_priv->private_data,
2865 			       cm_id_priv->private_data_len);
2866 		spin_unlock_irq(&cm_id_priv->lock);
2867 
2868 		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2869 		    ib_post_send_mad(msg, NULL))
2870 			cm_free_response_msg(msg);
2871 		goto deref;
2872 	case IB_CM_DREQ_RCVD:
2873 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2874 						     [CM_DREQ_COUNTER]);
2875 		goto unlock;
2876 	default:
2877 		trace_icm_dreq_unknown_err(&cm_id_priv->id);
2878 		goto unlock;
2879 	}
2880 	cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2881 	cm_id_priv->tid = dreq_msg->hdr.tid;
2882 	cm_queue_work_unlock(cm_id_priv, work);
2883 	return 0;
2884 
2885 unlock:	spin_unlock_irq(&cm_id_priv->lock);
2886 deref:	cm_deref_id(cm_id_priv);
2887 	return -EINVAL;
2888 }
2889 
cm_drep_handler(struct cm_work * work)2890 static int cm_drep_handler(struct cm_work *work)
2891 {
2892 	struct cm_id_private *cm_id_priv;
2893 	struct cm_drep_msg *drep_msg;
2894 
2895 	drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2896 	cm_id_priv = cm_acquire_id(
2897 		cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2898 		cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2899 	if (!cm_id_priv)
2900 		return -EINVAL;
2901 
2902 	work->cm_event.private_data =
2903 		IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2904 
2905 	spin_lock_irq(&cm_id_priv->lock);
2906 	if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2907 	    cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2908 		spin_unlock_irq(&cm_id_priv->lock);
2909 		goto out;
2910 	}
2911 	cm_enter_timewait(cm_id_priv);
2912 
2913 	ib_cancel_mad(cm_id_priv->msg);
2914 	cm_queue_work_unlock(cm_id_priv, work);
2915 	return 0;
2916 out:
2917 	cm_deref_id(cm_id_priv);
2918 	return -EINVAL;
2919 }
2920 
cm_send_rej_locked(struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2921 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2922 			      enum ib_cm_rej_reason reason, void *ari,
2923 			      u8 ari_length, const void *private_data,
2924 			      u8 private_data_len)
2925 {
2926 	enum ib_cm_state state = cm_id_priv->id.state;
2927 	struct ib_mad_send_buf *msg;
2928 	int ret;
2929 
2930 	lockdep_assert_held(&cm_id_priv->lock);
2931 
2932 	if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2933 	    (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2934 		return -EINVAL;
2935 
2936 	trace_icm_send_rej(&cm_id_priv->id, reason);
2937 
2938 	switch (state) {
2939 	case IB_CM_REQ_SENT:
2940 	case IB_CM_MRA_REQ_RCVD:
2941 	case IB_CM_REQ_RCVD:
2942 	case IB_CM_MRA_REQ_SENT:
2943 	case IB_CM_REP_RCVD:
2944 	case IB_CM_MRA_REP_SENT:
2945 		cm_reset_to_idle(cm_id_priv);
2946 		msg = cm_alloc_msg(cm_id_priv);
2947 		if (IS_ERR(msg))
2948 			return PTR_ERR(msg);
2949 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2950 			      ari, ari_length, private_data, private_data_len,
2951 			      state);
2952 		break;
2953 	case IB_CM_REP_SENT:
2954 	case IB_CM_MRA_REP_RCVD:
2955 		cm_enter_timewait(cm_id_priv);
2956 		msg = cm_alloc_msg(cm_id_priv);
2957 		if (IS_ERR(msg))
2958 			return PTR_ERR(msg);
2959 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2960 			      ari, ari_length, private_data, private_data_len,
2961 			      state);
2962 		break;
2963 	default:
2964 		trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2965 		return -EINVAL;
2966 	}
2967 
2968 	ret = ib_post_send_mad(msg, NULL);
2969 	if (ret) {
2970 		cm_free_msg(msg);
2971 		return ret;
2972 	}
2973 
2974 	return 0;
2975 }
2976 
ib_send_cm_rej(struct ib_cm_id * cm_id,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2977 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2978 		   void *ari, u8 ari_length, const void *private_data,
2979 		   u8 private_data_len)
2980 {
2981 	struct cm_id_private *cm_id_priv =
2982 		container_of(cm_id, struct cm_id_private, id);
2983 	unsigned long flags;
2984 	int ret;
2985 
2986 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2987 	ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2988 				 private_data, private_data_len);
2989 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2990 	return ret;
2991 }
2992 EXPORT_SYMBOL(ib_send_cm_rej);
2993 
cm_format_rej_event(struct cm_work * work)2994 static void cm_format_rej_event(struct cm_work *work)
2995 {
2996 	struct cm_rej_msg *rej_msg;
2997 	struct ib_cm_rej_event_param *param;
2998 
2999 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3000 	param = &work->cm_event.param.rej_rcvd;
3001 	param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
3002 	param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
3003 	param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
3004 	work->cm_event.private_data =
3005 		IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
3006 }
3007 
cm_acquire_rejected_id(struct cm_rej_msg * rej_msg)3008 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
3009 {
3010 	struct cm_id_private *cm_id_priv;
3011 	__be32 remote_id;
3012 
3013 	remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3014 
3015 	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3016 		cm_id_priv = cm_find_remote_id(
3017 			*((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3018 			remote_id);
3019 	} else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3020 		   CM_MSG_RESPONSE_REQ)
3021 		cm_id_priv = cm_acquire_id(
3022 			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3023 			0);
3024 	else
3025 		cm_id_priv = cm_acquire_id(
3026 			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3027 			remote_id);
3028 
3029 	return cm_id_priv;
3030 }
3031 
cm_rej_handler(struct cm_work * work)3032 static int cm_rej_handler(struct cm_work *work)
3033 {
3034 	struct cm_id_private *cm_id_priv;
3035 	struct cm_rej_msg *rej_msg;
3036 
3037 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3038 	cm_id_priv = cm_acquire_rejected_id(rej_msg);
3039 	if (!cm_id_priv)
3040 		return -EINVAL;
3041 
3042 	cm_format_rej_event(work);
3043 
3044 	spin_lock_irq(&cm_id_priv->lock);
3045 	switch (cm_id_priv->id.state) {
3046 	case IB_CM_REQ_SENT:
3047 	case IB_CM_MRA_REQ_RCVD:
3048 	case IB_CM_REP_SENT:
3049 	case IB_CM_MRA_REP_RCVD:
3050 		ib_cancel_mad(cm_id_priv->msg);
3051 		fallthrough;
3052 	case IB_CM_REQ_RCVD:
3053 	case IB_CM_MRA_REQ_SENT:
3054 		if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3055 			cm_enter_timewait(cm_id_priv);
3056 		else
3057 			cm_reset_to_idle(cm_id_priv);
3058 		break;
3059 	case IB_CM_DREQ_SENT:
3060 		ib_cancel_mad(cm_id_priv->msg);
3061 		fallthrough;
3062 	case IB_CM_REP_RCVD:
3063 	case IB_CM_MRA_REP_SENT:
3064 		cm_enter_timewait(cm_id_priv);
3065 		break;
3066 	case IB_CM_ESTABLISHED:
3067 		if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3068 		    cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3069 			if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3070 				ib_cancel_mad(cm_id_priv->msg);
3071 			cm_enter_timewait(cm_id_priv);
3072 			break;
3073 		}
3074 		fallthrough;
3075 	default:
3076 		trace_icm_rej_unknown_err(&cm_id_priv->id);
3077 		spin_unlock_irq(&cm_id_priv->lock);
3078 		goto out;
3079 	}
3080 
3081 	cm_queue_work_unlock(cm_id_priv, work);
3082 	return 0;
3083 out:
3084 	cm_deref_id(cm_id_priv);
3085 	return -EINVAL;
3086 }
3087 
ib_send_cm_mra(struct ib_cm_id * cm_id,u8 service_timeout,const void * private_data,u8 private_data_len)3088 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3089 		   u8 service_timeout,
3090 		   const void *private_data,
3091 		   u8 private_data_len)
3092 {
3093 	struct cm_id_private *cm_id_priv;
3094 	struct ib_mad_send_buf *msg;
3095 	enum ib_cm_state cm_state;
3096 	enum ib_cm_lap_state lap_state;
3097 	enum cm_msg_response msg_response;
3098 	void *data;
3099 	unsigned long flags;
3100 	int ret;
3101 
3102 	if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3103 		return -EINVAL;
3104 
3105 	data = cm_copy_private_data(private_data, private_data_len);
3106 	if (IS_ERR(data))
3107 		return PTR_ERR(data);
3108 
3109 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3110 
3111 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3112 	switch (cm_id_priv->id.state) {
3113 	case IB_CM_REQ_RCVD:
3114 		cm_state = IB_CM_MRA_REQ_SENT;
3115 		lap_state = cm_id->lap_state;
3116 		msg_response = CM_MSG_RESPONSE_REQ;
3117 		break;
3118 	case IB_CM_REP_RCVD:
3119 		cm_state = IB_CM_MRA_REP_SENT;
3120 		lap_state = cm_id->lap_state;
3121 		msg_response = CM_MSG_RESPONSE_REP;
3122 		break;
3123 	case IB_CM_ESTABLISHED:
3124 		if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3125 			cm_state = cm_id->state;
3126 			lap_state = IB_CM_MRA_LAP_SENT;
3127 			msg_response = CM_MSG_RESPONSE_OTHER;
3128 			break;
3129 		}
3130 		fallthrough;
3131 	default:
3132 		trace_icm_send_mra_unknown_err(&cm_id_priv->id);
3133 		ret = -EINVAL;
3134 		goto error_unlock;
3135 	}
3136 
3137 	if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3138 		msg = cm_alloc_msg(cm_id_priv);
3139 		if (IS_ERR(msg)) {
3140 			ret = PTR_ERR(msg);
3141 			goto error_unlock;
3142 		}
3143 
3144 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3145 			      msg_response, service_timeout,
3146 			      private_data, private_data_len);
3147 		trace_icm_send_mra(cm_id);
3148 		ret = ib_post_send_mad(msg, NULL);
3149 		if (ret)
3150 			goto error_free_msg;
3151 	}
3152 
3153 	cm_id->state = cm_state;
3154 	cm_id->lap_state = lap_state;
3155 	cm_id_priv->service_timeout = service_timeout;
3156 	cm_set_private_data(cm_id_priv, data, private_data_len);
3157 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3158 	return 0;
3159 
3160 error_free_msg:
3161 	cm_free_msg(msg);
3162 error_unlock:
3163 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3164 	kfree(data);
3165 	return ret;
3166 }
3167 EXPORT_SYMBOL(ib_send_cm_mra);
3168 
cm_acquire_mraed_id(struct cm_mra_msg * mra_msg)3169 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3170 {
3171 	switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3172 	case CM_MSG_RESPONSE_REQ:
3173 		return cm_acquire_id(
3174 			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3175 			0);
3176 	case CM_MSG_RESPONSE_REP:
3177 	case CM_MSG_RESPONSE_OTHER:
3178 		return cm_acquire_id(
3179 			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3180 			cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3181 	default:
3182 		return NULL;
3183 	}
3184 }
3185 
cm_mra_handler(struct cm_work * work)3186 static int cm_mra_handler(struct cm_work *work)
3187 {
3188 	struct cm_id_private *cm_id_priv;
3189 	struct cm_mra_msg *mra_msg;
3190 	int timeout;
3191 
3192 	mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3193 	cm_id_priv = cm_acquire_mraed_id(mra_msg);
3194 	if (!cm_id_priv)
3195 		return -EINVAL;
3196 
3197 	work->cm_event.private_data =
3198 		IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3199 	work->cm_event.param.mra_rcvd.service_timeout =
3200 		IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3201 	timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3202 		  cm_convert_to_ms(cm_id_priv->av.timeout);
3203 
3204 	spin_lock_irq(&cm_id_priv->lock);
3205 	switch (cm_id_priv->id.state) {
3206 	case IB_CM_REQ_SENT:
3207 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3208 			    CM_MSG_RESPONSE_REQ ||
3209 		    ib_modify_mad(cm_id_priv->msg, timeout))
3210 			goto out;
3211 		cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3212 		break;
3213 	case IB_CM_REP_SENT:
3214 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3215 			    CM_MSG_RESPONSE_REP ||
3216 		    ib_modify_mad(cm_id_priv->msg, timeout))
3217 			goto out;
3218 		cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3219 		break;
3220 	case IB_CM_ESTABLISHED:
3221 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3222 			    CM_MSG_RESPONSE_OTHER ||
3223 		    cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3224 		    ib_modify_mad(cm_id_priv->msg, timeout)) {
3225 			if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3226 				atomic_long_inc(
3227 					&work->port->counters[CM_RECV_DUPLICATES]
3228 							     [CM_MRA_COUNTER]);
3229 			goto out;
3230 		}
3231 		cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3232 		break;
3233 	case IB_CM_MRA_REQ_RCVD:
3234 	case IB_CM_MRA_REP_RCVD:
3235 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3236 						     [CM_MRA_COUNTER]);
3237 		fallthrough;
3238 	default:
3239 		trace_icm_mra_unknown_err(&cm_id_priv->id);
3240 		goto out;
3241 	}
3242 
3243 	cm_id_priv->msg->context[1] = (void *) (unsigned long)
3244 				      cm_id_priv->id.state;
3245 	cm_queue_work_unlock(cm_id_priv, work);
3246 	return 0;
3247 out:
3248 	spin_unlock_irq(&cm_id_priv->lock);
3249 	cm_deref_id(cm_id_priv);
3250 	return -EINVAL;
3251 }
3252 
cm_format_path_lid_from_lap(struct cm_lap_msg * lap_msg,struct sa_path_rec * path)3253 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3254 					struct sa_path_rec *path)
3255 {
3256 	u32 lid;
3257 
3258 	if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3259 		sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3260 					       lap_msg));
3261 		sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3262 					       lap_msg));
3263 	} else {
3264 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3265 			CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3266 		sa_path_set_dlid(path, lid);
3267 
3268 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3269 			CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3270 		sa_path_set_slid(path, lid);
3271 	}
3272 }
3273 
cm_format_path_from_lap(struct cm_id_private * cm_id_priv,struct sa_path_rec * path,struct cm_lap_msg * lap_msg)3274 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3275 				    struct sa_path_rec *path,
3276 				    struct cm_lap_msg *lap_msg)
3277 {
3278 	path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3279 	path->sgid =
3280 		*IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3281 	path->flow_label =
3282 		cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3283 	path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3284 	path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3285 	path->reversible = 1;
3286 	path->pkey = cm_id_priv->pkey;
3287 	path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3288 	path->mtu_selector = IB_SA_EQ;
3289 	path->mtu = cm_id_priv->path_mtu;
3290 	path->rate_selector = IB_SA_EQ;
3291 	path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3292 	path->packet_life_time_selector = IB_SA_EQ;
3293 	path->packet_life_time =
3294 		IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3295 	path->packet_life_time -= (path->packet_life_time > 0);
3296 	cm_format_path_lid_from_lap(lap_msg, path);
3297 }
3298 
cm_lap_handler(struct cm_work * work)3299 static int cm_lap_handler(struct cm_work *work)
3300 {
3301 	struct cm_id_private *cm_id_priv;
3302 	struct cm_lap_msg *lap_msg;
3303 	struct ib_cm_lap_event_param *param;
3304 	struct ib_mad_send_buf *msg = NULL;
3305 	struct rdma_ah_attr ah_attr;
3306 	struct cm_av alt_av = {};
3307 	int ret;
3308 
3309 	/* Currently Alternate path messages are not supported for
3310 	 * RoCE link layer.
3311 	 */
3312 	if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3313 			       work->port->port_num))
3314 		return -EINVAL;
3315 
3316 	/* todo: verify LAP request and send reject APR if invalid. */
3317 	lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3318 	cm_id_priv = cm_acquire_id(
3319 		cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3320 		cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3321 	if (!cm_id_priv)
3322 		return -EINVAL;
3323 
3324 	param = &work->cm_event.param.lap_rcvd;
3325 	memset(&work->path[0], 0, sizeof(work->path[1]));
3326 	cm_path_set_rec_type(work->port->cm_dev->ib_device,
3327 			     work->port->port_num, &work->path[0],
3328 			     IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3329 					     lap_msg));
3330 	param->alternate_path = &work->path[0];
3331 	cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3332 	work->cm_event.private_data =
3333 		IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3334 
3335 	ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3336 				      work->port->port_num,
3337 				      work->mad_recv_wc->wc,
3338 				      work->mad_recv_wc->recv_buf.grh,
3339 				      &ah_attr);
3340 	if (ret)
3341 		goto deref;
3342 
3343 	ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3344 	if (ret) {
3345 		rdma_destroy_ah_attr(&ah_attr);
3346 		goto deref;
3347 	}
3348 
3349 	spin_lock_irq(&cm_id_priv->lock);
3350 	cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3351 			   &ah_attr, &cm_id_priv->av);
3352 	cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3353 
3354 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3355 		goto unlock;
3356 
3357 	switch (cm_id_priv->id.lap_state) {
3358 	case IB_CM_LAP_UNINIT:
3359 	case IB_CM_LAP_IDLE:
3360 		break;
3361 	case IB_CM_MRA_LAP_SENT:
3362 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3363 						     [CM_LAP_COUNTER]);
3364 		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3365 		if (IS_ERR(msg))
3366 			goto unlock;
3367 
3368 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3369 			      CM_MSG_RESPONSE_OTHER,
3370 			      cm_id_priv->service_timeout,
3371 			      cm_id_priv->private_data,
3372 			      cm_id_priv->private_data_len);
3373 		spin_unlock_irq(&cm_id_priv->lock);
3374 
3375 		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3376 		    ib_post_send_mad(msg, NULL))
3377 			cm_free_response_msg(msg);
3378 		goto deref;
3379 	case IB_CM_LAP_RCVD:
3380 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3381 						     [CM_LAP_COUNTER]);
3382 		goto unlock;
3383 	default:
3384 		goto unlock;
3385 	}
3386 
3387 	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3388 	cm_id_priv->tid = lap_msg->hdr.tid;
3389 	cm_queue_work_unlock(cm_id_priv, work);
3390 	return 0;
3391 
3392 unlock:	spin_unlock_irq(&cm_id_priv->lock);
3393 deref:	cm_deref_id(cm_id_priv);
3394 	return -EINVAL;
3395 }
3396 
cm_apr_handler(struct cm_work * work)3397 static int cm_apr_handler(struct cm_work *work)
3398 {
3399 	struct cm_id_private *cm_id_priv;
3400 	struct cm_apr_msg *apr_msg;
3401 
3402 	/* Currently Alternate path messages are not supported for
3403 	 * RoCE link layer.
3404 	 */
3405 	if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3406 			       work->port->port_num))
3407 		return -EINVAL;
3408 
3409 	apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3410 	cm_id_priv = cm_acquire_id(
3411 		cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3412 		cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3413 	if (!cm_id_priv)
3414 		return -EINVAL; /* Unmatched reply. */
3415 
3416 	work->cm_event.param.apr_rcvd.ap_status =
3417 		IBA_GET(CM_APR_AR_STATUS, apr_msg);
3418 	work->cm_event.param.apr_rcvd.apr_info =
3419 		IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3420 	work->cm_event.param.apr_rcvd.info_len =
3421 		IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3422 	work->cm_event.private_data =
3423 		IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3424 
3425 	spin_lock_irq(&cm_id_priv->lock);
3426 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3427 	    (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3428 	     cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3429 		spin_unlock_irq(&cm_id_priv->lock);
3430 		goto out;
3431 	}
3432 	cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3433 	ib_cancel_mad(cm_id_priv->msg);
3434 	cm_queue_work_unlock(cm_id_priv, work);
3435 	return 0;
3436 out:
3437 	cm_deref_id(cm_id_priv);
3438 	return -EINVAL;
3439 }
3440 
cm_timewait_handler(struct cm_work * work)3441 static int cm_timewait_handler(struct cm_work *work)
3442 {
3443 	struct cm_timewait_info *timewait_info;
3444 	struct cm_id_private *cm_id_priv;
3445 
3446 	timewait_info = container_of(work, struct cm_timewait_info, work);
3447 	spin_lock_irq(&cm.lock);
3448 	list_del(&timewait_info->list);
3449 	spin_unlock_irq(&cm.lock);
3450 
3451 	cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3452 				   timewait_info->work.remote_id);
3453 	if (!cm_id_priv)
3454 		return -EINVAL;
3455 
3456 	spin_lock_irq(&cm_id_priv->lock);
3457 	if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3458 	    cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3459 		spin_unlock_irq(&cm_id_priv->lock);
3460 		goto out;
3461 	}
3462 	cm_id_priv->id.state = IB_CM_IDLE;
3463 	cm_queue_work_unlock(cm_id_priv, work);
3464 	return 0;
3465 out:
3466 	cm_deref_id(cm_id_priv);
3467 	return -EINVAL;
3468 }
3469 
cm_format_sidr_req(struct cm_sidr_req_msg * sidr_req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_req_param * param)3470 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3471 			       struct cm_id_private *cm_id_priv,
3472 			       struct ib_cm_sidr_req_param *param)
3473 {
3474 	cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3475 			  cm_form_tid(cm_id_priv));
3476 	IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3477 		be32_to_cpu(cm_id_priv->id.local_id));
3478 	IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3479 		be16_to_cpu(param->path->pkey));
3480 	IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3481 		be64_to_cpu(param->service_id));
3482 
3483 	if (param->private_data && param->private_data_len)
3484 		IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3485 			    param->private_data, param->private_data_len);
3486 }
3487 
ib_send_cm_sidr_req(struct ib_cm_id * cm_id,struct ib_cm_sidr_req_param * param)3488 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3489 			struct ib_cm_sidr_req_param *param)
3490 {
3491 	struct cm_id_private *cm_id_priv;
3492 	struct ib_mad_send_buf *msg;
3493 	struct cm_av av = {};
3494 	unsigned long flags;
3495 	int ret;
3496 
3497 	if (!param->path || (param->private_data &&
3498 	     param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3499 		return -EINVAL;
3500 
3501 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3502 	ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3503 	if (ret)
3504 		return ret;
3505 
3506 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3507 	cm_move_av_from_path(&cm_id_priv->av, &av);
3508 	cm_id->service_id = param->service_id;
3509 	cm_id_priv->timeout_ms = param->timeout_ms;
3510 	cm_id_priv->max_cm_retries = param->max_cm_retries;
3511 	if (cm_id->state != IB_CM_IDLE) {
3512 		ret = -EINVAL;
3513 		goto out_unlock;
3514 	}
3515 
3516 	msg = cm_alloc_priv_msg(cm_id_priv);
3517 	if (IS_ERR(msg)) {
3518 		ret = PTR_ERR(msg);
3519 		goto out_unlock;
3520 	}
3521 
3522 	cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3523 			   param);
3524 	msg->timeout_ms = cm_id_priv->timeout_ms;
3525 	msg->context[1] = (void *)(unsigned long)IB_CM_SIDR_REQ_SENT;
3526 
3527 	trace_icm_send_sidr_req(&cm_id_priv->id);
3528 	ret = ib_post_send_mad(msg, NULL);
3529 	if (ret)
3530 		goto out_free;
3531 	cm_id->state = IB_CM_SIDR_REQ_SENT;
3532 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3533 	return 0;
3534 out_free:
3535 	cm_free_priv_msg(msg);
3536 out_unlock:
3537 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3538 	return ret;
3539 }
3540 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3541 
cm_format_sidr_req_event(struct cm_work * work,const struct cm_id_private * rx_cm_id,struct ib_cm_id * listen_id)3542 static void cm_format_sidr_req_event(struct cm_work *work,
3543 				     const struct cm_id_private *rx_cm_id,
3544 				     struct ib_cm_id *listen_id)
3545 {
3546 	struct cm_sidr_req_msg *sidr_req_msg;
3547 	struct ib_cm_sidr_req_event_param *param;
3548 
3549 	sidr_req_msg = (struct cm_sidr_req_msg *)
3550 				work->mad_recv_wc->recv_buf.mad;
3551 	param = &work->cm_event.param.sidr_req_rcvd;
3552 	param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3553 	param->listen_id = listen_id;
3554 	param->service_id =
3555 		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3556 	param->bth_pkey = cm_get_bth_pkey(work);
3557 	param->port = work->port->port_num;
3558 	param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3559 	work->cm_event.private_data =
3560 		IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3561 }
3562 
cm_sidr_req_handler(struct cm_work * work)3563 static int cm_sidr_req_handler(struct cm_work *work)
3564 {
3565 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3566 	struct cm_sidr_req_msg *sidr_req_msg;
3567 	struct ib_wc *wc;
3568 	int ret;
3569 
3570 	cm_id_priv =
3571 		cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3572 	if (IS_ERR(cm_id_priv))
3573 		return PTR_ERR(cm_id_priv);
3574 
3575 	/* Record SGID/SLID and request ID for lookup. */
3576 	sidr_req_msg = (struct cm_sidr_req_msg *)
3577 				work->mad_recv_wc->recv_buf.mad;
3578 
3579 	cm_id_priv->id.remote_id =
3580 		cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3581 	cm_id_priv->id.service_id =
3582 		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3583 	cm_id_priv->tid = sidr_req_msg->hdr.tid;
3584 
3585 	wc = work->mad_recv_wc->wc;
3586 	cm_id_priv->sidr_slid = wc->slid;
3587 	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3588 				      work->mad_recv_wc->recv_buf.grh,
3589 				      &cm_id_priv->av);
3590 	if (ret)
3591 		goto out;
3592 
3593 	spin_lock_irq(&cm.lock);
3594 	listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3595 	if (listen_cm_id_priv) {
3596 		spin_unlock_irq(&cm.lock);
3597 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3598 						     [CM_SIDR_REQ_COUNTER]);
3599 		goto out; /* Duplicate message. */
3600 	}
3601 	cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3602 	listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3603 					   cm_id_priv->id.service_id);
3604 	if (!listen_cm_id_priv) {
3605 		spin_unlock_irq(&cm.lock);
3606 		ib_send_cm_sidr_rep(&cm_id_priv->id,
3607 				    &(struct ib_cm_sidr_rep_param){
3608 					    .status = IB_SIDR_UNSUPPORTED });
3609 		goto out; /* No match. */
3610 	}
3611 	spin_unlock_irq(&cm.lock);
3612 
3613 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3614 	cm_id_priv->id.context = listen_cm_id_priv->id.context;
3615 
3616 	/*
3617 	 * A SIDR ID does not need to be in the xarray since it does not receive
3618 	 * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3619 	 * not enter timewait.
3620 	 */
3621 
3622 	cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3623 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3624 	cm_free_work(work);
3625 	/*
3626 	 * A pointer to the listen_cm_id is held in the event, so this deref
3627 	 * must be after the event is delivered above.
3628 	 */
3629 	cm_deref_id(listen_cm_id_priv);
3630 	if (ret)
3631 		cm_destroy_id(&cm_id_priv->id, ret);
3632 	return 0;
3633 out:
3634 	ib_destroy_cm_id(&cm_id_priv->id);
3635 	return -EINVAL;
3636 }
3637 
cm_format_sidr_rep(struct cm_sidr_rep_msg * sidr_rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3638 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3639 			       struct cm_id_private *cm_id_priv,
3640 			       struct ib_cm_sidr_rep_param *param)
3641 {
3642 	cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3643 			      cm_id_priv->tid, param->ece.attr_mod);
3644 	IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3645 		be32_to_cpu(cm_id_priv->id.remote_id));
3646 	IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3647 	IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3648 	IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3649 		be64_to_cpu(cm_id_priv->id.service_id));
3650 	IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3651 	IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3652 		param->ece.vendor_id & 0xFF);
3653 	IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3654 		(param->ece.vendor_id >> 8) & 0xFF);
3655 
3656 	if (param->info && param->info_length)
3657 		IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3658 			    param->info, param->info_length);
3659 
3660 	if (param->private_data && param->private_data_len)
3661 		IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3662 			    param->private_data, param->private_data_len);
3663 }
3664 
cm_send_sidr_rep_locked(struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3665 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3666 				   struct ib_cm_sidr_rep_param *param)
3667 {
3668 	struct ib_mad_send_buf *msg;
3669 	unsigned long flags;
3670 	int ret;
3671 
3672 	lockdep_assert_held(&cm_id_priv->lock);
3673 
3674 	if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3675 	    (param->private_data &&
3676 	     param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3677 		return -EINVAL;
3678 
3679 	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3680 		return -EINVAL;
3681 
3682 	msg = cm_alloc_msg(cm_id_priv);
3683 	if (IS_ERR(msg))
3684 		return PTR_ERR(msg);
3685 
3686 	cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3687 			   param);
3688 	trace_icm_send_sidr_rep(&cm_id_priv->id);
3689 	ret = ib_post_send_mad(msg, NULL);
3690 	if (ret) {
3691 		cm_free_msg(msg);
3692 		return ret;
3693 	}
3694 	cm_id_priv->id.state = IB_CM_IDLE;
3695 	spin_lock_irqsave(&cm.lock, flags);
3696 	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3697 		rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3698 		RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3699 	}
3700 	spin_unlock_irqrestore(&cm.lock, flags);
3701 	return 0;
3702 }
3703 
ib_send_cm_sidr_rep(struct ib_cm_id * cm_id,struct ib_cm_sidr_rep_param * param)3704 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3705 			struct ib_cm_sidr_rep_param *param)
3706 {
3707 	struct cm_id_private *cm_id_priv =
3708 		container_of(cm_id, struct cm_id_private, id);
3709 	unsigned long flags;
3710 	int ret;
3711 
3712 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3713 	ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3714 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3715 	return ret;
3716 }
3717 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3718 
cm_format_sidr_rep_event(struct cm_work * work,const struct cm_id_private * cm_id_priv)3719 static void cm_format_sidr_rep_event(struct cm_work *work,
3720 				     const struct cm_id_private *cm_id_priv)
3721 {
3722 	struct cm_sidr_rep_msg *sidr_rep_msg;
3723 	struct ib_cm_sidr_rep_event_param *param;
3724 
3725 	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3726 				work->mad_recv_wc->recv_buf.mad;
3727 	param = &work->cm_event.param.sidr_rep_rcvd;
3728 	param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3729 	param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3730 	param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3731 	param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3732 				      sidr_rep_msg);
3733 	param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3734 				  sidr_rep_msg);
3735 	param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3736 	work->cm_event.private_data =
3737 		IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3738 }
3739 
cm_sidr_rep_handler(struct cm_work * work)3740 static int cm_sidr_rep_handler(struct cm_work *work)
3741 {
3742 	struct cm_sidr_rep_msg *sidr_rep_msg;
3743 	struct cm_id_private *cm_id_priv;
3744 
3745 	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3746 				work->mad_recv_wc->recv_buf.mad;
3747 	cm_id_priv = cm_acquire_id(
3748 		cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3749 	if (!cm_id_priv)
3750 		return -EINVAL; /* Unmatched reply. */
3751 
3752 	spin_lock_irq(&cm_id_priv->lock);
3753 	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3754 		spin_unlock_irq(&cm_id_priv->lock);
3755 		goto out;
3756 	}
3757 	cm_id_priv->id.state = IB_CM_IDLE;
3758 	ib_cancel_mad(cm_id_priv->msg);
3759 	spin_unlock_irq(&cm_id_priv->lock);
3760 
3761 	cm_format_sidr_rep_event(work, cm_id_priv);
3762 	cm_process_work(cm_id_priv, work);
3763 	return 0;
3764 out:
3765 	cm_deref_id(cm_id_priv);
3766 	return -EINVAL;
3767 }
3768 
cm_process_send_error(struct cm_id_private * cm_id_priv,struct ib_mad_send_buf * msg,enum ib_cm_state state,enum ib_wc_status wc_status)3769 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3770 				  struct ib_mad_send_buf *msg,
3771 				  enum ib_cm_state state,
3772 				  enum ib_wc_status wc_status)
3773 {
3774 	struct ib_cm_event cm_event = {};
3775 	int ret;
3776 
3777 	/* Discard old sends or ones without a response. */
3778 	spin_lock_irq(&cm_id_priv->lock);
3779 	if (msg != cm_id_priv->msg) {
3780 		spin_unlock_irq(&cm_id_priv->lock);
3781 		cm_free_msg(msg);
3782 		return;
3783 	}
3784 	cm_free_priv_msg(msg);
3785 
3786 	if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3787 	    wc_status == IB_WC_WR_FLUSH_ERR)
3788 		goto out_unlock;
3789 
3790 	trace_icm_mad_send_err(state, wc_status);
3791 	switch (state) {
3792 	case IB_CM_REQ_SENT:
3793 	case IB_CM_MRA_REQ_RCVD:
3794 		cm_reset_to_idle(cm_id_priv);
3795 		cm_event.event = IB_CM_REQ_ERROR;
3796 		break;
3797 	case IB_CM_REP_SENT:
3798 	case IB_CM_MRA_REP_RCVD:
3799 		cm_reset_to_idle(cm_id_priv);
3800 		cm_event.event = IB_CM_REP_ERROR;
3801 		break;
3802 	case IB_CM_DREQ_SENT:
3803 		cm_enter_timewait(cm_id_priv);
3804 		cm_event.event = IB_CM_DREQ_ERROR;
3805 		break;
3806 	case IB_CM_SIDR_REQ_SENT:
3807 		cm_id_priv->id.state = IB_CM_IDLE;
3808 		cm_event.event = IB_CM_SIDR_REQ_ERROR;
3809 		break;
3810 	default:
3811 		goto out_unlock;
3812 	}
3813 	spin_unlock_irq(&cm_id_priv->lock);
3814 	cm_event.param.send_status = wc_status;
3815 
3816 	/* No other events can occur on the cm_id at this point. */
3817 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3818 	if (ret)
3819 		ib_destroy_cm_id(&cm_id_priv->id);
3820 	return;
3821 out_unlock:
3822 	spin_unlock_irq(&cm_id_priv->lock);
3823 }
3824 
cm_send_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_wc * mad_send_wc)3825 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3826 			    struct ib_mad_send_wc *mad_send_wc)
3827 {
3828 	struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3829 	struct cm_id_private *cm_id_priv = msg->context[0];
3830 	enum ib_cm_state state =
3831 		(enum ib_cm_state)(unsigned long)msg->context[1];
3832 	struct cm_port *port;
3833 	u16 attr_index;
3834 
3835 	port = mad_agent->context;
3836 	attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3837 				  msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3838 
3839 	/*
3840 	 * If the send was in response to a received message (context[0] is not
3841 	 * set to a cm_id), and is not a REJ, then it is a send that was
3842 	 * manually retried.
3843 	 */
3844 	if (!cm_id_priv && (attr_index != CM_REJ_COUNTER))
3845 		msg->retries = 1;
3846 
3847 	atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3848 	if (msg->retries)
3849 		atomic_long_add(msg->retries,
3850 				&port->counters[CM_XMIT_RETRIES][attr_index]);
3851 
3852 	if (cm_id_priv)
3853 		cm_process_send_error(cm_id_priv, msg, state,
3854 				      mad_send_wc->status);
3855 	else
3856 		cm_free_response_msg(msg);
3857 }
3858 
cm_work_handler(struct work_struct * _work)3859 static void cm_work_handler(struct work_struct *_work)
3860 {
3861 	struct cm_work *work = container_of(_work, struct cm_work, work.work);
3862 	int ret;
3863 
3864 	switch (work->cm_event.event) {
3865 	case IB_CM_REQ_RECEIVED:
3866 		ret = cm_req_handler(work);
3867 		break;
3868 	case IB_CM_MRA_RECEIVED:
3869 		ret = cm_mra_handler(work);
3870 		break;
3871 	case IB_CM_REJ_RECEIVED:
3872 		ret = cm_rej_handler(work);
3873 		break;
3874 	case IB_CM_REP_RECEIVED:
3875 		ret = cm_rep_handler(work);
3876 		break;
3877 	case IB_CM_RTU_RECEIVED:
3878 		ret = cm_rtu_handler(work);
3879 		break;
3880 	case IB_CM_USER_ESTABLISHED:
3881 		ret = cm_establish_handler(work);
3882 		break;
3883 	case IB_CM_DREQ_RECEIVED:
3884 		ret = cm_dreq_handler(work);
3885 		break;
3886 	case IB_CM_DREP_RECEIVED:
3887 		ret = cm_drep_handler(work);
3888 		break;
3889 	case IB_CM_SIDR_REQ_RECEIVED:
3890 		ret = cm_sidr_req_handler(work);
3891 		break;
3892 	case IB_CM_SIDR_REP_RECEIVED:
3893 		ret = cm_sidr_rep_handler(work);
3894 		break;
3895 	case IB_CM_LAP_RECEIVED:
3896 		ret = cm_lap_handler(work);
3897 		break;
3898 	case IB_CM_APR_RECEIVED:
3899 		ret = cm_apr_handler(work);
3900 		break;
3901 	case IB_CM_TIMEWAIT_EXIT:
3902 		ret = cm_timewait_handler(work);
3903 		break;
3904 	default:
3905 		trace_icm_handler_err(work->cm_event.event);
3906 		ret = -EINVAL;
3907 		break;
3908 	}
3909 	if (ret)
3910 		cm_free_work(work);
3911 }
3912 
cm_establish(struct ib_cm_id * cm_id)3913 static int cm_establish(struct ib_cm_id *cm_id)
3914 {
3915 	struct cm_id_private *cm_id_priv;
3916 	struct cm_work *work;
3917 	unsigned long flags;
3918 	int ret = 0;
3919 	struct cm_device *cm_dev;
3920 
3921 	cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3922 	if (!cm_dev)
3923 		return -ENODEV;
3924 
3925 	work = kmalloc(sizeof *work, GFP_ATOMIC);
3926 	if (!work)
3927 		return -ENOMEM;
3928 
3929 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3930 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3931 	switch (cm_id->state) {
3932 	case IB_CM_REP_SENT:
3933 	case IB_CM_MRA_REP_RCVD:
3934 		cm_id->state = IB_CM_ESTABLISHED;
3935 		break;
3936 	case IB_CM_ESTABLISHED:
3937 		ret = -EISCONN;
3938 		break;
3939 	default:
3940 		trace_icm_establish_err(cm_id);
3941 		ret = -EINVAL;
3942 		break;
3943 	}
3944 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3945 
3946 	if (ret) {
3947 		kfree(work);
3948 		goto out;
3949 	}
3950 
3951 	/*
3952 	 * The CM worker thread may try to destroy the cm_id before it
3953 	 * can execute this work item.  To prevent potential deadlock,
3954 	 * we need to find the cm_id once we're in the context of the
3955 	 * worker thread, rather than holding a reference on it.
3956 	 */
3957 	INIT_DELAYED_WORK(&work->work, cm_work_handler);
3958 	work->local_id = cm_id->local_id;
3959 	work->remote_id = cm_id->remote_id;
3960 	work->mad_recv_wc = NULL;
3961 	work->cm_event.event = IB_CM_USER_ESTABLISHED;
3962 
3963 	/* Check if the device started its remove_one */
3964 	spin_lock_irqsave(&cm.lock, flags);
3965 	if (!cm_dev->going_down) {
3966 		queue_delayed_work(cm.wq, &work->work, 0);
3967 	} else {
3968 		kfree(work);
3969 		ret = -ENODEV;
3970 	}
3971 	spin_unlock_irqrestore(&cm.lock, flags);
3972 
3973 out:
3974 	return ret;
3975 }
3976 
cm_migrate(struct ib_cm_id * cm_id)3977 static int cm_migrate(struct ib_cm_id *cm_id)
3978 {
3979 	struct cm_id_private *cm_id_priv;
3980 	unsigned long flags;
3981 	int ret = 0;
3982 
3983 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3984 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3985 	if (cm_id->state == IB_CM_ESTABLISHED &&
3986 	    (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3987 	     cm_id->lap_state == IB_CM_LAP_IDLE)) {
3988 		cm_id->lap_state = IB_CM_LAP_IDLE;
3989 		cm_id_priv->av = cm_id_priv->alt_av;
3990 	} else
3991 		ret = -EINVAL;
3992 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3993 
3994 	return ret;
3995 }
3996 
ib_cm_notify(struct ib_cm_id * cm_id,enum ib_event_type event)3997 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3998 {
3999 	int ret;
4000 
4001 	switch (event) {
4002 	case IB_EVENT_COMM_EST:
4003 		ret = cm_establish(cm_id);
4004 		break;
4005 	case IB_EVENT_PATH_MIG:
4006 		ret = cm_migrate(cm_id);
4007 		break;
4008 	default:
4009 		ret = -EINVAL;
4010 	}
4011 	return ret;
4012 }
4013 EXPORT_SYMBOL(ib_cm_notify);
4014 
cm_recv_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_buf * send_buf,struct ib_mad_recv_wc * mad_recv_wc)4015 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
4016 			    struct ib_mad_send_buf *send_buf,
4017 			    struct ib_mad_recv_wc *mad_recv_wc)
4018 {
4019 	struct cm_port *port = mad_agent->context;
4020 	struct cm_work *work;
4021 	enum ib_cm_event_type event;
4022 	bool alt_path = false;
4023 	u16 attr_id;
4024 	int paths = 0;
4025 	int going_down = 0;
4026 
4027 	switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4028 	case CM_REQ_ATTR_ID:
4029 		alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4030 						mad_recv_wc->recv_buf.mad);
4031 		paths = 1 + (alt_path != 0);
4032 		event = IB_CM_REQ_RECEIVED;
4033 		break;
4034 	case CM_MRA_ATTR_ID:
4035 		event = IB_CM_MRA_RECEIVED;
4036 		break;
4037 	case CM_REJ_ATTR_ID:
4038 		event = IB_CM_REJ_RECEIVED;
4039 		break;
4040 	case CM_REP_ATTR_ID:
4041 		event = IB_CM_REP_RECEIVED;
4042 		break;
4043 	case CM_RTU_ATTR_ID:
4044 		event = IB_CM_RTU_RECEIVED;
4045 		break;
4046 	case CM_DREQ_ATTR_ID:
4047 		event = IB_CM_DREQ_RECEIVED;
4048 		break;
4049 	case CM_DREP_ATTR_ID:
4050 		event = IB_CM_DREP_RECEIVED;
4051 		break;
4052 	case CM_SIDR_REQ_ATTR_ID:
4053 		event = IB_CM_SIDR_REQ_RECEIVED;
4054 		break;
4055 	case CM_SIDR_REP_ATTR_ID:
4056 		event = IB_CM_SIDR_REP_RECEIVED;
4057 		break;
4058 	case CM_LAP_ATTR_ID:
4059 		paths = 1;
4060 		event = IB_CM_LAP_RECEIVED;
4061 		break;
4062 	case CM_APR_ATTR_ID:
4063 		event = IB_CM_APR_RECEIVED;
4064 		break;
4065 	default:
4066 		ib_free_recv_mad(mad_recv_wc);
4067 		return;
4068 	}
4069 
4070 	attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4071 	atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4072 
4073 	work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4074 	if (!work) {
4075 		ib_free_recv_mad(mad_recv_wc);
4076 		return;
4077 	}
4078 
4079 	INIT_DELAYED_WORK(&work->work, cm_work_handler);
4080 	work->cm_event.event = event;
4081 	work->mad_recv_wc = mad_recv_wc;
4082 	work->port = port;
4083 
4084 	/* Check if the device started its remove_one */
4085 	spin_lock_irq(&cm.lock);
4086 	if (!port->cm_dev->going_down)
4087 		queue_delayed_work(cm.wq, &work->work, 0);
4088 	else
4089 		going_down = 1;
4090 	spin_unlock_irq(&cm.lock);
4091 
4092 	if (going_down) {
4093 		kfree(work);
4094 		ib_free_recv_mad(mad_recv_wc);
4095 	}
4096 }
4097 
cm_init_qp_init_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4098 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4099 				struct ib_qp_attr *qp_attr,
4100 				int *qp_attr_mask)
4101 {
4102 	unsigned long flags;
4103 	int ret;
4104 
4105 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4106 	switch (cm_id_priv->id.state) {
4107 	case IB_CM_REQ_SENT:
4108 	case IB_CM_MRA_REQ_RCVD:
4109 	case IB_CM_REQ_RCVD:
4110 	case IB_CM_MRA_REQ_SENT:
4111 	case IB_CM_REP_RCVD:
4112 	case IB_CM_MRA_REP_SENT:
4113 	case IB_CM_REP_SENT:
4114 	case IB_CM_MRA_REP_RCVD:
4115 	case IB_CM_ESTABLISHED:
4116 		*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4117 				IB_QP_PKEY_INDEX | IB_QP_PORT;
4118 		qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4119 		if (cm_id_priv->responder_resources) {
4120 			struct ib_device *ib_dev = cm_id_priv->id.device;
4121 			u64 support_flush = ib_dev->attrs.device_cap_flags &
4122 			  (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
4123 			u32 flushable = support_flush ?
4124 					(IB_ACCESS_FLUSH_GLOBAL |
4125 					 IB_ACCESS_FLUSH_PERSISTENT) : 0;
4126 
4127 			qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4128 						    IB_ACCESS_REMOTE_ATOMIC |
4129 						    flushable;
4130 		}
4131 		qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4132 		if (cm_id_priv->av.port)
4133 			qp_attr->port_num = cm_id_priv->av.port->port_num;
4134 		ret = 0;
4135 		break;
4136 	default:
4137 		trace_icm_qp_init_err(&cm_id_priv->id);
4138 		ret = -EINVAL;
4139 		break;
4140 	}
4141 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4142 	return ret;
4143 }
4144 
cm_init_qp_rtr_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4145 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4146 			       struct ib_qp_attr *qp_attr,
4147 			       int *qp_attr_mask)
4148 {
4149 	unsigned long flags;
4150 	int ret;
4151 
4152 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4153 	switch (cm_id_priv->id.state) {
4154 	case IB_CM_REQ_RCVD:
4155 	case IB_CM_MRA_REQ_SENT:
4156 	case IB_CM_REP_RCVD:
4157 	case IB_CM_MRA_REP_SENT:
4158 	case IB_CM_REP_SENT:
4159 	case IB_CM_MRA_REP_RCVD:
4160 	case IB_CM_ESTABLISHED:
4161 		*qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4162 				IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4163 		qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4164 		if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4165 		    cm_id_priv->av.dlid_datapath &&
4166 		    (cm_id_priv->av.dlid_datapath != 0xffff))
4167 			qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4168 		qp_attr->path_mtu = cm_id_priv->path_mtu;
4169 		qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4170 		qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4171 		if (cm_id_priv->qp_type == IB_QPT_RC ||
4172 		    cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4173 			*qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4174 					 IB_QP_MIN_RNR_TIMER;
4175 			qp_attr->max_dest_rd_atomic =
4176 					cm_id_priv->responder_resources;
4177 			qp_attr->min_rnr_timer = 0;
4178 		}
4179 		if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4180 		    cm_id_priv->alt_av.port) {
4181 			*qp_attr_mask |= IB_QP_ALT_PATH;
4182 			qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4183 			qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4184 			qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4185 			qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4186 		}
4187 		ret = 0;
4188 		break;
4189 	default:
4190 		trace_icm_qp_rtr_err(&cm_id_priv->id);
4191 		ret = -EINVAL;
4192 		break;
4193 	}
4194 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4195 	return ret;
4196 }
4197 
cm_init_qp_rts_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4198 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4199 			       struct ib_qp_attr *qp_attr,
4200 			       int *qp_attr_mask)
4201 {
4202 	unsigned long flags;
4203 	int ret;
4204 
4205 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4206 	switch (cm_id_priv->id.state) {
4207 	/* Allow transition to RTS before sending REP */
4208 	case IB_CM_REQ_RCVD:
4209 	case IB_CM_MRA_REQ_SENT:
4210 
4211 	case IB_CM_REP_RCVD:
4212 	case IB_CM_MRA_REP_SENT:
4213 	case IB_CM_REP_SENT:
4214 	case IB_CM_MRA_REP_RCVD:
4215 	case IB_CM_ESTABLISHED:
4216 		if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4217 			*qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4218 			qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4219 			switch (cm_id_priv->qp_type) {
4220 			case IB_QPT_RC:
4221 			case IB_QPT_XRC_INI:
4222 				*qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4223 						 IB_QP_MAX_QP_RD_ATOMIC;
4224 				qp_attr->retry_cnt = cm_id_priv->retry_count;
4225 				qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4226 				qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4227 				fallthrough;
4228 			case IB_QPT_XRC_TGT:
4229 				*qp_attr_mask |= IB_QP_TIMEOUT;
4230 				qp_attr->timeout = cm_id_priv->av.timeout;
4231 				break;
4232 			default:
4233 				break;
4234 			}
4235 			if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4236 				*qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4237 				qp_attr->path_mig_state = IB_MIG_REARM;
4238 			}
4239 		} else {
4240 			*qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4241 			if (cm_id_priv->alt_av.port)
4242 				qp_attr->alt_port_num =
4243 					cm_id_priv->alt_av.port->port_num;
4244 			qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4245 			qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4246 			qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4247 			qp_attr->path_mig_state = IB_MIG_REARM;
4248 		}
4249 		ret = 0;
4250 		break;
4251 	default:
4252 		trace_icm_qp_rts_err(&cm_id_priv->id);
4253 		ret = -EINVAL;
4254 		break;
4255 	}
4256 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4257 	return ret;
4258 }
4259 
ib_cm_init_qp_attr(struct ib_cm_id * cm_id,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4260 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4261 		       struct ib_qp_attr *qp_attr,
4262 		       int *qp_attr_mask)
4263 {
4264 	struct cm_id_private *cm_id_priv;
4265 	int ret;
4266 
4267 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4268 	switch (qp_attr->qp_state) {
4269 	case IB_QPS_INIT:
4270 		ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4271 		break;
4272 	case IB_QPS_RTR:
4273 		ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4274 		break;
4275 	case IB_QPS_RTS:
4276 		ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4277 		break;
4278 	default:
4279 		ret = -EINVAL;
4280 		break;
4281 	}
4282 	return ret;
4283 }
4284 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4285 
cm_show_counter(struct ib_device * ibdev,u32 port_num,struct ib_port_attribute * attr,char * buf)4286 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4287 			       struct ib_port_attribute *attr, char *buf)
4288 {
4289 	struct cm_counter_attribute *cm_attr =
4290 		container_of(attr, struct cm_counter_attribute, attr);
4291 	struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4292 
4293 	if (WARN_ON(!cm_dev))
4294 		return -EINVAL;
4295 
4296 	return sysfs_emit(
4297 		buf, "%ld\n",
4298 		atomic_long_read(
4299 			&cm_dev->port[port_num - 1]
4300 				 ->counters[cm_attr->group][cm_attr->index]));
4301 }
4302 
4303 #define CM_COUNTER_ATTR(_name, _group, _index)                                 \
4304 	{                                                                      \
4305 		.attr = __ATTR(_name, 0444, cm_show_counter, NULL),            \
4306 		.group = _group, .index = _index                               \
4307 	}
4308 
4309 #define CM_COUNTER_GROUP(_group, _name)                                        \
4310 	static struct cm_counter_attribute cm_counter_attr_##_group[] = {      \
4311 		CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER),                  \
4312 		CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER),                  \
4313 		CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER),                  \
4314 		CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER),                  \
4315 		CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER),                  \
4316 		CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER),                \
4317 		CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER),                \
4318 		CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER),        \
4319 		CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER),        \
4320 		CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER),                  \
4321 		CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER),                  \
4322 	};                                                                     \
4323 	static struct attribute *cm_counter_attrs_##_group[] = {               \
4324 		&cm_counter_attr_##_group[0].attr.attr,                        \
4325 		&cm_counter_attr_##_group[1].attr.attr,                        \
4326 		&cm_counter_attr_##_group[2].attr.attr,                        \
4327 		&cm_counter_attr_##_group[3].attr.attr,                        \
4328 		&cm_counter_attr_##_group[4].attr.attr,                        \
4329 		&cm_counter_attr_##_group[5].attr.attr,                        \
4330 		&cm_counter_attr_##_group[6].attr.attr,                        \
4331 		&cm_counter_attr_##_group[7].attr.attr,                        \
4332 		&cm_counter_attr_##_group[8].attr.attr,                        \
4333 		&cm_counter_attr_##_group[9].attr.attr,                        \
4334 		&cm_counter_attr_##_group[10].attr.attr,                       \
4335 		NULL,                                                          \
4336 	};                                                                     \
4337 	static const struct attribute_group cm_counter_group_##_group = {      \
4338 		.name = _name,                                                 \
4339 		.attrs = cm_counter_attrs_##_group,                            \
4340 	};
4341 
4342 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4343 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4344 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4345 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4346 
4347 static const struct attribute_group *cm_counter_groups[] = {
4348 	&cm_counter_group_CM_XMIT,
4349 	&cm_counter_group_CM_XMIT_RETRIES,
4350 	&cm_counter_group_CM_RECV,
4351 	&cm_counter_group_CM_RECV_DUPLICATES,
4352 	NULL,
4353 };
4354 
cm_add_one(struct ib_device * ib_device)4355 static int cm_add_one(struct ib_device *ib_device)
4356 {
4357 	struct cm_device *cm_dev;
4358 	struct cm_port *port;
4359 	struct ib_mad_reg_req reg_req = {
4360 		.mgmt_class = IB_MGMT_CLASS_CM,
4361 		.mgmt_class_version = IB_CM_CLASS_VERSION,
4362 	};
4363 	struct ib_port_modify port_modify = {
4364 		.set_port_cap_mask = IB_PORT_CM_SUP
4365 	};
4366 	unsigned long flags;
4367 	int ret;
4368 	int count = 0;
4369 	u32 i;
4370 
4371 	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4372 			 GFP_KERNEL);
4373 	if (!cm_dev)
4374 		return -ENOMEM;
4375 
4376 	kref_init(&cm_dev->kref);
4377 	spin_lock_init(&cm_dev->mad_agent_lock);
4378 	cm_dev->ib_device = ib_device;
4379 	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4380 	cm_dev->going_down = 0;
4381 
4382 	ib_set_client_data(ib_device, &cm_client, cm_dev);
4383 
4384 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4385 	rdma_for_each_port (ib_device, i) {
4386 		if (!rdma_cap_ib_cm(ib_device, i))
4387 			continue;
4388 
4389 		port = kzalloc(sizeof *port, GFP_KERNEL);
4390 		if (!port) {
4391 			ret = -ENOMEM;
4392 			goto error1;
4393 		}
4394 
4395 		cm_dev->port[i-1] = port;
4396 		port->cm_dev = cm_dev;
4397 		port->port_num = i;
4398 
4399 		ret = ib_port_register_client_groups(ib_device, i,
4400 						     cm_counter_groups);
4401 		if (ret)
4402 			goto error1;
4403 
4404 		port->mad_agent = ib_register_mad_agent(ib_device, i,
4405 							IB_QPT_GSI,
4406 							&reg_req,
4407 							0,
4408 							cm_send_handler,
4409 							cm_recv_handler,
4410 							port,
4411 							0);
4412 		if (IS_ERR(port->mad_agent)) {
4413 			ret = PTR_ERR(port->mad_agent);
4414 			goto error2;
4415 		}
4416 
4417 		ret = ib_modify_port(ib_device, i, 0, &port_modify);
4418 		if (ret)
4419 			goto error3;
4420 
4421 		count++;
4422 	}
4423 
4424 	if (!count) {
4425 		ret = -EOPNOTSUPP;
4426 		goto free;
4427 	}
4428 
4429 	write_lock_irqsave(&cm.device_lock, flags);
4430 	list_add_tail(&cm_dev->list, &cm.device_list);
4431 	write_unlock_irqrestore(&cm.device_lock, flags);
4432 	return 0;
4433 
4434 error3:
4435 	ib_unregister_mad_agent(port->mad_agent);
4436 error2:
4437 	ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4438 error1:
4439 	port_modify.set_port_cap_mask = 0;
4440 	port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4441 	while (--i) {
4442 		if (!rdma_cap_ib_cm(ib_device, i))
4443 			continue;
4444 
4445 		port = cm_dev->port[i-1];
4446 		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4447 		ib_unregister_mad_agent(port->mad_agent);
4448 		ib_port_unregister_client_groups(ib_device, i,
4449 						 cm_counter_groups);
4450 	}
4451 free:
4452 	cm_device_put(cm_dev);
4453 	return ret;
4454 }
4455 
cm_remove_one(struct ib_device * ib_device,void * client_data)4456 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4457 {
4458 	struct cm_device *cm_dev = client_data;
4459 	struct cm_port *port;
4460 	struct ib_port_modify port_modify = {
4461 		.clr_port_cap_mask = IB_PORT_CM_SUP
4462 	};
4463 	unsigned long flags;
4464 	u32 i;
4465 
4466 	write_lock_irqsave(&cm.device_lock, flags);
4467 	list_del(&cm_dev->list);
4468 	write_unlock_irqrestore(&cm.device_lock, flags);
4469 
4470 	spin_lock_irq(&cm.lock);
4471 	cm_dev->going_down = 1;
4472 	spin_unlock_irq(&cm.lock);
4473 
4474 	rdma_for_each_port (ib_device, i) {
4475 		struct ib_mad_agent *mad_agent;
4476 
4477 		if (!rdma_cap_ib_cm(ib_device, i))
4478 			continue;
4479 
4480 		port = cm_dev->port[i-1];
4481 		mad_agent = port->mad_agent;
4482 		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4483 		/*
4484 		 * We flush the queue here after the going_down set, this
4485 		 * verify that no new works will be queued in the recv handler,
4486 		 * after that we can call the unregister_mad_agent
4487 		 */
4488 		flush_workqueue(cm.wq);
4489 		/*
4490 		 * The above ensures no call paths from the work are running,
4491 		 * the remaining paths all take the mad_agent_lock.
4492 		 */
4493 		spin_lock(&cm_dev->mad_agent_lock);
4494 		port->mad_agent = NULL;
4495 		spin_unlock(&cm_dev->mad_agent_lock);
4496 		ib_unregister_mad_agent(mad_agent);
4497 		ib_port_unregister_client_groups(ib_device, i,
4498 						 cm_counter_groups);
4499 	}
4500 
4501 	cm_device_put(cm_dev);
4502 }
4503 
ib_cm_init(void)4504 static int __init ib_cm_init(void)
4505 {
4506 	int ret;
4507 
4508 	INIT_LIST_HEAD(&cm.device_list);
4509 	rwlock_init(&cm.device_lock);
4510 	spin_lock_init(&cm.lock);
4511 	cm.listen_service_table = RB_ROOT;
4512 	cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4513 	cm.remote_id_table = RB_ROOT;
4514 	cm.remote_qp_table = RB_ROOT;
4515 	cm.remote_sidr_table = RB_ROOT;
4516 	xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4517 	get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4518 	INIT_LIST_HEAD(&cm.timewait_list);
4519 
4520 	cm.wq = alloc_workqueue("ib_cm", 0, 1);
4521 	if (!cm.wq) {
4522 		ret = -ENOMEM;
4523 		goto error2;
4524 	}
4525 
4526 	ret = ib_register_client(&cm_client);
4527 	if (ret)
4528 		goto error3;
4529 
4530 	return 0;
4531 error3:
4532 	destroy_workqueue(cm.wq);
4533 error2:
4534 	return ret;
4535 }
4536 
ib_cm_cleanup(void)4537 static void __exit ib_cm_cleanup(void)
4538 {
4539 	struct cm_timewait_info *timewait_info, *tmp;
4540 
4541 	spin_lock_irq(&cm.lock);
4542 	list_for_each_entry(timewait_info, &cm.timewait_list, list)
4543 		cancel_delayed_work(&timewait_info->work.work);
4544 	spin_unlock_irq(&cm.lock);
4545 
4546 	ib_unregister_client(&cm_client);
4547 	destroy_workqueue(cm.wq);
4548 
4549 	list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4550 		list_del(&timewait_info->list);
4551 		kfree(timewait_info);
4552 	}
4553 
4554 	WARN_ON(!xa_empty(&cm.local_id_table));
4555 }
4556 
4557 module_init(ib_cm_init);
4558 module_exit(ib_cm_cleanup);
4559