xref: /openbmc/linux/drivers/infiniband/core/cma.c (revision f97769fd)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
4  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
5  * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.
6  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
7  */
8 
9 #include <linux/completion.h>
10 #include <linux/in.h>
11 #include <linux/in6.h>
12 #include <linux/mutex.h>
13 #include <linux/random.h>
14 #include <linux/igmp.h>
15 #include <linux/xarray.h>
16 #include <linux/inetdevice.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <net/route.h>
20 
21 #include <net/net_namespace.h>
22 #include <net/netns/generic.h>
23 #include <net/tcp.h>
24 #include <net/ipv6.h>
25 #include <net/ip_fib.h>
26 #include <net/ip6_route.h>
27 
28 #include <rdma/rdma_cm.h>
29 #include <rdma/rdma_cm_ib.h>
30 #include <rdma/rdma_netlink.h>
31 #include <rdma/ib.h>
32 #include <rdma/ib_cache.h>
33 #include <rdma/ib_cm.h>
34 #include <rdma/ib_sa.h>
35 #include <rdma/iw_cm.h>
36 
37 #include "core_priv.h"
38 #include "cma_priv.h"
39 #include "cma_trace.h"
40 
41 MODULE_AUTHOR("Sean Hefty");
42 MODULE_DESCRIPTION("Generic RDMA CM Agent");
43 MODULE_LICENSE("Dual BSD/GPL");
44 
45 #define CMA_CM_RESPONSE_TIMEOUT 20
46 #define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000
47 #define CMA_MAX_CM_RETRIES 15
48 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
49 #define CMA_IBOE_PACKET_LIFETIME 18
50 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
51 
52 static const char * const cma_events[] = {
53 	[RDMA_CM_EVENT_ADDR_RESOLVED]	 = "address resolved",
54 	[RDMA_CM_EVENT_ADDR_ERROR]	 = "address error",
55 	[RDMA_CM_EVENT_ROUTE_RESOLVED]	 = "route resolved ",
56 	[RDMA_CM_EVENT_ROUTE_ERROR]	 = "route error",
57 	[RDMA_CM_EVENT_CONNECT_REQUEST]	 = "connect request",
58 	[RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
59 	[RDMA_CM_EVENT_CONNECT_ERROR]	 = "connect error",
60 	[RDMA_CM_EVENT_UNREACHABLE]	 = "unreachable",
61 	[RDMA_CM_EVENT_REJECTED]	 = "rejected",
62 	[RDMA_CM_EVENT_ESTABLISHED]	 = "established",
63 	[RDMA_CM_EVENT_DISCONNECTED]	 = "disconnected",
64 	[RDMA_CM_EVENT_DEVICE_REMOVAL]	 = "device removal",
65 	[RDMA_CM_EVENT_MULTICAST_JOIN]	 = "multicast join",
66 	[RDMA_CM_EVENT_MULTICAST_ERROR]	 = "multicast error",
67 	[RDMA_CM_EVENT_ADDR_CHANGE]	 = "address change",
68 	[RDMA_CM_EVENT_TIMEWAIT_EXIT]	 = "timewait exit",
69 };
70 
71 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
72 {
73 	size_t index = event;
74 
75 	return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
76 			cma_events[index] : "unrecognized event";
77 }
78 EXPORT_SYMBOL(rdma_event_msg);
79 
80 const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
81 						int reason)
82 {
83 	if (rdma_ib_or_roce(id->device, id->port_num))
84 		return ibcm_reject_msg(reason);
85 
86 	if (rdma_protocol_iwarp(id->device, id->port_num))
87 		return iwcm_reject_msg(reason);
88 
89 	WARN_ON_ONCE(1);
90 	return "unrecognized transport";
91 }
92 EXPORT_SYMBOL(rdma_reject_msg);
93 
94 /**
95  * rdma_is_consumer_reject - return true if the consumer rejected the connect
96  *                           request.
97  * @id: Communication identifier that received the REJECT event.
98  * @reason: Value returned in the REJECT event status field.
99  */
100 static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
101 {
102 	if (rdma_ib_or_roce(id->device, id->port_num))
103 		return reason == IB_CM_REJ_CONSUMER_DEFINED;
104 
105 	if (rdma_protocol_iwarp(id->device, id->port_num))
106 		return reason == -ECONNREFUSED;
107 
108 	WARN_ON_ONCE(1);
109 	return false;
110 }
111 
112 const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
113 				      struct rdma_cm_event *ev, u8 *data_len)
114 {
115 	const void *p;
116 
117 	if (rdma_is_consumer_reject(id, ev->status)) {
118 		*data_len = ev->param.conn.private_data_len;
119 		p = ev->param.conn.private_data;
120 	} else {
121 		*data_len = 0;
122 		p = NULL;
123 	}
124 	return p;
125 }
126 EXPORT_SYMBOL(rdma_consumer_reject_data);
127 
128 /**
129  * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
130  * @id: Communication Identifier
131  */
132 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
133 {
134 	struct rdma_id_private *id_priv;
135 
136 	id_priv = container_of(id, struct rdma_id_private, id);
137 	if (id->device->node_type == RDMA_NODE_RNIC)
138 		return id_priv->cm_id.iw;
139 	return NULL;
140 }
141 EXPORT_SYMBOL(rdma_iw_cm_id);
142 
143 /**
144  * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.
145  * @res: rdma resource tracking entry pointer
146  */
147 struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
148 {
149 	struct rdma_id_private *id_priv =
150 		container_of(res, struct rdma_id_private, res);
151 
152 	return &id_priv->id;
153 }
154 EXPORT_SYMBOL(rdma_res_to_id);
155 
156 static int cma_add_one(struct ib_device *device);
157 static void cma_remove_one(struct ib_device *device, void *client_data);
158 
159 static struct ib_client cma_client = {
160 	.name   = "cma",
161 	.add    = cma_add_one,
162 	.remove = cma_remove_one
163 };
164 
165 static struct ib_sa_client sa_client;
166 static LIST_HEAD(dev_list);
167 static LIST_HEAD(listen_any_list);
168 static DEFINE_MUTEX(lock);
169 static struct workqueue_struct *cma_wq;
170 static unsigned int cma_pernet_id;
171 
172 struct cma_pernet {
173 	struct xarray tcp_ps;
174 	struct xarray udp_ps;
175 	struct xarray ipoib_ps;
176 	struct xarray ib_ps;
177 };
178 
179 static struct cma_pernet *cma_pernet(struct net *net)
180 {
181 	return net_generic(net, cma_pernet_id);
182 }
183 
184 static
185 struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
186 {
187 	struct cma_pernet *pernet = cma_pernet(net);
188 
189 	switch (ps) {
190 	case RDMA_PS_TCP:
191 		return &pernet->tcp_ps;
192 	case RDMA_PS_UDP:
193 		return &pernet->udp_ps;
194 	case RDMA_PS_IPOIB:
195 		return &pernet->ipoib_ps;
196 	case RDMA_PS_IB:
197 		return &pernet->ib_ps;
198 	default:
199 		return NULL;
200 	}
201 }
202 
203 struct cma_device {
204 	struct list_head	list;
205 	struct ib_device	*device;
206 	struct completion	comp;
207 	refcount_t refcount;
208 	struct list_head	id_list;
209 	enum ib_gid_type	*default_gid_type;
210 	u8			*default_roce_tos;
211 };
212 
213 struct rdma_bind_list {
214 	enum rdma_ucm_port_space ps;
215 	struct hlist_head	owners;
216 	unsigned short		port;
217 };
218 
219 struct class_port_info_context {
220 	struct ib_class_port_info	*class_port_info;
221 	struct ib_device		*device;
222 	struct completion		done;
223 	struct ib_sa_query		*sa_query;
224 	u8				port_num;
225 };
226 
227 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
228 			struct rdma_bind_list *bind_list, int snum)
229 {
230 	struct xarray *xa = cma_pernet_xa(net, ps);
231 
232 	return xa_insert(xa, snum, bind_list, GFP_KERNEL);
233 }
234 
235 static struct rdma_bind_list *cma_ps_find(struct net *net,
236 					  enum rdma_ucm_port_space ps, int snum)
237 {
238 	struct xarray *xa = cma_pernet_xa(net, ps);
239 
240 	return xa_load(xa, snum);
241 }
242 
243 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
244 			  int snum)
245 {
246 	struct xarray *xa = cma_pernet_xa(net, ps);
247 
248 	xa_erase(xa, snum);
249 }
250 
251 enum {
252 	CMA_OPTION_AFONLY,
253 };
254 
255 void cma_dev_get(struct cma_device *cma_dev)
256 {
257 	refcount_inc(&cma_dev->refcount);
258 }
259 
260 void cma_dev_put(struct cma_device *cma_dev)
261 {
262 	if (refcount_dec_and_test(&cma_dev->refcount))
263 		complete(&cma_dev->comp);
264 }
265 
266 struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,
267 					     void		*cookie)
268 {
269 	struct cma_device *cma_dev;
270 	struct cma_device *found_cma_dev = NULL;
271 
272 	mutex_lock(&lock);
273 
274 	list_for_each_entry(cma_dev, &dev_list, list)
275 		if (filter(cma_dev->device, cookie)) {
276 			found_cma_dev = cma_dev;
277 			break;
278 		}
279 
280 	if (found_cma_dev)
281 		cma_dev_get(found_cma_dev);
282 	mutex_unlock(&lock);
283 	return found_cma_dev;
284 }
285 
286 int cma_get_default_gid_type(struct cma_device *cma_dev,
287 			     unsigned int port)
288 {
289 	if (!rdma_is_port_valid(cma_dev->device, port))
290 		return -EINVAL;
291 
292 	return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
293 }
294 
295 int cma_set_default_gid_type(struct cma_device *cma_dev,
296 			     unsigned int port,
297 			     enum ib_gid_type default_gid_type)
298 {
299 	unsigned long supported_gids;
300 
301 	if (!rdma_is_port_valid(cma_dev->device, port))
302 		return -EINVAL;
303 
304 	supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
305 
306 	if (!(supported_gids & 1 << default_gid_type))
307 		return -EINVAL;
308 
309 	cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
310 		default_gid_type;
311 
312 	return 0;
313 }
314 
315 int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port)
316 {
317 	if (!rdma_is_port_valid(cma_dev->device, port))
318 		return -EINVAL;
319 
320 	return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];
321 }
322 
323 int cma_set_default_roce_tos(struct cma_device *cma_dev, unsigned int port,
324 			     u8 default_roce_tos)
325 {
326 	if (!rdma_is_port_valid(cma_dev->device, port))
327 		return -EINVAL;
328 
329 	cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =
330 		 default_roce_tos;
331 
332 	return 0;
333 }
334 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
335 {
336 	return cma_dev->device;
337 }
338 
339 /*
340  * Device removal can occur at anytime, so we need extra handling to
341  * serialize notifying the user of device removal with other callbacks.
342  * We do this by disabling removal notification while a callback is in process,
343  * and reporting it after the callback completes.
344  */
345 
346 struct cma_multicast {
347 	struct rdma_id_private *id_priv;
348 	union {
349 		struct ib_sa_multicast *ib;
350 	} multicast;
351 	struct list_head	list;
352 	void			*context;
353 	struct sockaddr_storage	addr;
354 	struct kref		mcref;
355 	u8			join_state;
356 };
357 
358 struct cma_work {
359 	struct work_struct	work;
360 	struct rdma_id_private	*id;
361 	enum rdma_cm_state	old_state;
362 	enum rdma_cm_state	new_state;
363 	struct rdma_cm_event	event;
364 };
365 
366 struct cma_ndev_work {
367 	struct work_struct	work;
368 	struct rdma_id_private	*id;
369 	struct rdma_cm_event	event;
370 };
371 
372 struct iboe_mcast_work {
373 	struct work_struct	 work;
374 	struct rdma_id_private	*id;
375 	struct cma_multicast	*mc;
376 };
377 
378 union cma_ip_addr {
379 	struct in6_addr ip6;
380 	struct {
381 		__be32 pad[3];
382 		__be32 addr;
383 	} ip4;
384 };
385 
386 struct cma_hdr {
387 	u8 cma_version;
388 	u8 ip_version;	/* IP version: 7:4 */
389 	__be16 port;
390 	union cma_ip_addr src_addr;
391 	union cma_ip_addr dst_addr;
392 };
393 
394 #define CMA_VERSION 0x00
395 
396 struct cma_req_info {
397 	struct sockaddr_storage listen_addr_storage;
398 	struct sockaddr_storage src_addr_storage;
399 	struct ib_device *device;
400 	union ib_gid local_gid;
401 	__be64 service_id;
402 	int port;
403 	bool has_gid;
404 	u16 pkey;
405 };
406 
407 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
408 {
409 	unsigned long flags;
410 	int ret;
411 
412 	spin_lock_irqsave(&id_priv->lock, flags);
413 	ret = (id_priv->state == comp);
414 	spin_unlock_irqrestore(&id_priv->lock, flags);
415 	return ret;
416 }
417 
418 static int cma_comp_exch(struct rdma_id_private *id_priv,
419 			 enum rdma_cm_state comp, enum rdma_cm_state exch)
420 {
421 	unsigned long flags;
422 	int ret;
423 
424 	spin_lock_irqsave(&id_priv->lock, flags);
425 	if ((ret = (id_priv->state == comp)))
426 		id_priv->state = exch;
427 	spin_unlock_irqrestore(&id_priv->lock, flags);
428 	return ret;
429 }
430 
431 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
432 {
433 	return hdr->ip_version >> 4;
434 }
435 
436 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
437 {
438 	hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
439 }
440 
441 static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
442 {
443 	struct in_device *in_dev = NULL;
444 
445 	if (ndev) {
446 		rtnl_lock();
447 		in_dev = __in_dev_get_rtnl(ndev);
448 		if (in_dev) {
449 			if (join)
450 				ip_mc_inc_group(in_dev,
451 						*(__be32 *)(mgid->raw + 12));
452 			else
453 				ip_mc_dec_group(in_dev,
454 						*(__be32 *)(mgid->raw + 12));
455 		}
456 		rtnl_unlock();
457 	}
458 	return (in_dev) ? 0 : -ENODEV;
459 }
460 
461 static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
462 			       struct cma_device *cma_dev)
463 {
464 	cma_dev_get(cma_dev);
465 	id_priv->cma_dev = cma_dev;
466 	id_priv->id.device = cma_dev->device;
467 	id_priv->id.route.addr.dev_addr.transport =
468 		rdma_node_get_transport(cma_dev->device->node_type);
469 	list_add_tail(&id_priv->list, &cma_dev->id_list);
470 	if (id_priv->res.kern_name)
471 		rdma_restrack_kadd(&id_priv->res);
472 	else
473 		rdma_restrack_uadd(&id_priv->res);
474 	trace_cm_id_attach(id_priv, cma_dev->device);
475 }
476 
477 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
478 			      struct cma_device *cma_dev)
479 {
480 	_cma_attach_to_dev(id_priv, cma_dev);
481 	id_priv->gid_type =
482 		cma_dev->default_gid_type[id_priv->id.port_num -
483 					  rdma_start_port(cma_dev->device)];
484 }
485 
486 static inline void release_mc(struct kref *kref)
487 {
488 	struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
489 
490 	kfree(mc->multicast.ib);
491 	kfree(mc);
492 }
493 
494 static void cma_release_dev(struct rdma_id_private *id_priv)
495 {
496 	mutex_lock(&lock);
497 	list_del(&id_priv->list);
498 	cma_dev_put(id_priv->cma_dev);
499 	id_priv->cma_dev = NULL;
500 	mutex_unlock(&lock);
501 }
502 
503 static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
504 {
505 	return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
506 }
507 
508 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
509 {
510 	return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
511 }
512 
513 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
514 {
515 	return id_priv->id.route.addr.src_addr.ss_family;
516 }
517 
518 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
519 {
520 	struct ib_sa_mcmember_rec rec;
521 	int ret = 0;
522 
523 	if (id_priv->qkey) {
524 		if (qkey && id_priv->qkey != qkey)
525 			return -EINVAL;
526 		return 0;
527 	}
528 
529 	if (qkey) {
530 		id_priv->qkey = qkey;
531 		return 0;
532 	}
533 
534 	switch (id_priv->id.ps) {
535 	case RDMA_PS_UDP:
536 	case RDMA_PS_IB:
537 		id_priv->qkey = RDMA_UDP_QKEY;
538 		break;
539 	case RDMA_PS_IPOIB:
540 		ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
541 		ret = ib_sa_get_mcmember_rec(id_priv->id.device,
542 					     id_priv->id.port_num, &rec.mgid,
543 					     &rec);
544 		if (!ret)
545 			id_priv->qkey = be32_to_cpu(rec.qkey);
546 		break;
547 	default:
548 		break;
549 	}
550 	return ret;
551 }
552 
553 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
554 {
555 	dev_addr->dev_type = ARPHRD_INFINIBAND;
556 	rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
557 	ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
558 }
559 
560 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
561 {
562 	int ret;
563 
564 	if (addr->sa_family != AF_IB) {
565 		ret = rdma_translate_ip(addr, dev_addr);
566 	} else {
567 		cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
568 		ret = 0;
569 	}
570 
571 	return ret;
572 }
573 
574 static const struct ib_gid_attr *
575 cma_validate_port(struct ib_device *device, u8 port,
576 		  enum ib_gid_type gid_type,
577 		  union ib_gid *gid,
578 		  struct rdma_id_private *id_priv)
579 {
580 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
581 	int bound_if_index = dev_addr->bound_dev_if;
582 	const struct ib_gid_attr *sgid_attr;
583 	int dev_type = dev_addr->dev_type;
584 	struct net_device *ndev = NULL;
585 
586 	if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
587 		return ERR_PTR(-ENODEV);
588 
589 	if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
590 		return ERR_PTR(-ENODEV);
591 
592 	if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
593 		return ERR_PTR(-ENODEV);
594 
595 	if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
596 		ndev = dev_get_by_index(dev_addr->net, bound_if_index);
597 		if (!ndev)
598 			return ERR_PTR(-ENODEV);
599 	} else {
600 		gid_type = IB_GID_TYPE_IB;
601 	}
602 
603 	sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
604 	if (ndev)
605 		dev_put(ndev);
606 	return sgid_attr;
607 }
608 
609 static void cma_bind_sgid_attr(struct rdma_id_private *id_priv,
610 			       const struct ib_gid_attr *sgid_attr)
611 {
612 	WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr);
613 	id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr;
614 }
615 
616 /**
617  * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute
618  * based on source ip address.
619  * @id_priv:	cm_id which should be bound to cma device
620  *
621  * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute
622  * based on source IP address. It returns 0 on success or error code otherwise.
623  * It is applicable to active and passive side cm_id.
624  */
625 static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
626 {
627 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
628 	const struct ib_gid_attr *sgid_attr;
629 	union ib_gid gid, iboe_gid, *gidp;
630 	struct cma_device *cma_dev;
631 	enum ib_gid_type gid_type;
632 	int ret = -ENODEV;
633 	unsigned int port;
634 
635 	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
636 	    id_priv->id.ps == RDMA_PS_IPOIB)
637 		return -EINVAL;
638 
639 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
640 		    &iboe_gid);
641 
642 	memcpy(&gid, dev_addr->src_dev_addr +
643 	       rdma_addr_gid_offset(dev_addr), sizeof(gid));
644 
645 	mutex_lock(&lock);
646 	list_for_each_entry(cma_dev, &dev_list, list) {
647 		rdma_for_each_port (cma_dev->device, port) {
648 			gidp = rdma_protocol_roce(cma_dev->device, port) ?
649 			       &iboe_gid : &gid;
650 			gid_type = cma_dev->default_gid_type[port - 1];
651 			sgid_attr = cma_validate_port(cma_dev->device, port,
652 						      gid_type, gidp, id_priv);
653 			if (!IS_ERR(sgid_attr)) {
654 				id_priv->id.port_num = port;
655 				cma_bind_sgid_attr(id_priv, sgid_attr);
656 				cma_attach_to_dev(id_priv, cma_dev);
657 				ret = 0;
658 				goto out;
659 			}
660 		}
661 	}
662 out:
663 	mutex_unlock(&lock);
664 	return ret;
665 }
666 
667 /**
668  * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute
669  * @id_priv:		cm id to bind to cma device
670  * @listen_id_priv:	listener cm id to match against
671  * @req:		Pointer to req structure containaining incoming
672  *			request information
673  * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when
674  * rdma device matches for listen_id and incoming request. It also verifies
675  * that a GID table entry is present for the source address.
676  * Returns 0 on success, or returns error code otherwise.
677  */
678 static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,
679 			      const struct rdma_id_private *listen_id_priv,
680 			      struct cma_req_info *req)
681 {
682 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
683 	const struct ib_gid_attr *sgid_attr;
684 	enum ib_gid_type gid_type;
685 	union ib_gid gid;
686 
687 	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
688 	    id_priv->id.ps == RDMA_PS_IPOIB)
689 		return -EINVAL;
690 
691 	if (rdma_protocol_roce(req->device, req->port))
692 		rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
693 			    &gid);
694 	else
695 		memcpy(&gid, dev_addr->src_dev_addr +
696 		       rdma_addr_gid_offset(dev_addr), sizeof(gid));
697 
698 	gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];
699 	sgid_attr = cma_validate_port(req->device, req->port,
700 				      gid_type, &gid, id_priv);
701 	if (IS_ERR(sgid_attr))
702 		return PTR_ERR(sgid_attr);
703 
704 	id_priv->id.port_num = req->port;
705 	cma_bind_sgid_attr(id_priv, sgid_attr);
706 	/* Need to acquire lock to protect against reader
707 	 * of cma_dev->id_list such as cma_netdev_callback() and
708 	 * cma_process_remove().
709 	 */
710 	mutex_lock(&lock);
711 	cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);
712 	mutex_unlock(&lock);
713 	return 0;
714 }
715 
716 static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
717 			      const struct rdma_id_private *listen_id_priv)
718 {
719 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
720 	const struct ib_gid_attr *sgid_attr;
721 	struct cma_device *cma_dev;
722 	enum ib_gid_type gid_type;
723 	int ret = -ENODEV;
724 	unsigned int port;
725 	union ib_gid gid;
726 
727 	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
728 	    id_priv->id.ps == RDMA_PS_IPOIB)
729 		return -EINVAL;
730 
731 	memcpy(&gid, dev_addr->src_dev_addr +
732 	       rdma_addr_gid_offset(dev_addr), sizeof(gid));
733 
734 	mutex_lock(&lock);
735 
736 	cma_dev = listen_id_priv->cma_dev;
737 	port = listen_id_priv->id.port_num;
738 	gid_type = listen_id_priv->gid_type;
739 	sgid_attr = cma_validate_port(cma_dev->device, port,
740 				      gid_type, &gid, id_priv);
741 	if (!IS_ERR(sgid_attr)) {
742 		id_priv->id.port_num = port;
743 		cma_bind_sgid_attr(id_priv, sgid_attr);
744 		ret = 0;
745 		goto out;
746 	}
747 
748 	list_for_each_entry(cma_dev, &dev_list, list) {
749 		rdma_for_each_port (cma_dev->device, port) {
750 			if (listen_id_priv->cma_dev == cma_dev &&
751 			    listen_id_priv->id.port_num == port)
752 				continue;
753 
754 			gid_type = cma_dev->default_gid_type[port - 1];
755 			sgid_attr = cma_validate_port(cma_dev->device, port,
756 						      gid_type, &gid, id_priv);
757 			if (!IS_ERR(sgid_attr)) {
758 				id_priv->id.port_num = port;
759 				cma_bind_sgid_attr(id_priv, sgid_attr);
760 				ret = 0;
761 				goto out;
762 			}
763 		}
764 	}
765 
766 out:
767 	if (!ret)
768 		cma_attach_to_dev(id_priv, cma_dev);
769 
770 	mutex_unlock(&lock);
771 	return ret;
772 }
773 
774 /*
775  * Select the source IB device and address to reach the destination IB address.
776  */
777 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
778 {
779 	struct cma_device *cma_dev, *cur_dev;
780 	struct sockaddr_ib *addr;
781 	union ib_gid gid, sgid, *dgid;
782 	unsigned int p;
783 	u16 pkey, index;
784 	enum ib_port_state port_state;
785 	int i;
786 
787 	cma_dev = NULL;
788 	addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
789 	dgid = (union ib_gid *) &addr->sib_addr;
790 	pkey = ntohs(addr->sib_pkey);
791 
792 	mutex_lock(&lock);
793 	list_for_each_entry(cur_dev, &dev_list, list) {
794 		rdma_for_each_port (cur_dev->device, p) {
795 			if (!rdma_cap_af_ib(cur_dev->device, p))
796 				continue;
797 
798 			if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
799 				continue;
800 
801 			if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
802 				continue;
803 			for (i = 0; !rdma_query_gid(cur_dev->device,
804 						    p, i, &gid);
805 			     i++) {
806 				if (!memcmp(&gid, dgid, sizeof(gid))) {
807 					cma_dev = cur_dev;
808 					sgid = gid;
809 					id_priv->id.port_num = p;
810 					goto found;
811 				}
812 
813 				if (!cma_dev && (gid.global.subnet_prefix ==
814 				    dgid->global.subnet_prefix) &&
815 				    port_state == IB_PORT_ACTIVE) {
816 					cma_dev = cur_dev;
817 					sgid = gid;
818 					id_priv->id.port_num = p;
819 					goto found;
820 				}
821 			}
822 		}
823 	}
824 	mutex_unlock(&lock);
825 	return -ENODEV;
826 
827 found:
828 	cma_attach_to_dev(id_priv, cma_dev);
829 	mutex_unlock(&lock);
830 	addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
831 	memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
832 	cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
833 	return 0;
834 }
835 
836 static void cma_id_get(struct rdma_id_private *id_priv)
837 {
838 	refcount_inc(&id_priv->refcount);
839 }
840 
841 static void cma_id_put(struct rdma_id_private *id_priv)
842 {
843 	if (refcount_dec_and_test(&id_priv->refcount))
844 		complete(&id_priv->comp);
845 }
846 
847 struct rdma_cm_id *__rdma_create_id(struct net *net,
848 				    rdma_cm_event_handler event_handler,
849 				    void *context, enum rdma_ucm_port_space ps,
850 				    enum ib_qp_type qp_type, const char *caller)
851 {
852 	struct rdma_id_private *id_priv;
853 
854 	id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
855 	if (!id_priv)
856 		return ERR_PTR(-ENOMEM);
857 
858 	rdma_restrack_set_task(&id_priv->res, caller);
859 	id_priv->res.type = RDMA_RESTRACK_CM_ID;
860 	id_priv->state = RDMA_CM_IDLE;
861 	id_priv->id.context = context;
862 	id_priv->id.event_handler = event_handler;
863 	id_priv->id.ps = ps;
864 	id_priv->id.qp_type = qp_type;
865 	id_priv->tos_set = false;
866 	id_priv->timeout_set = false;
867 	id_priv->gid_type = IB_GID_TYPE_IB;
868 	spin_lock_init(&id_priv->lock);
869 	mutex_init(&id_priv->qp_mutex);
870 	init_completion(&id_priv->comp);
871 	refcount_set(&id_priv->refcount, 1);
872 	mutex_init(&id_priv->handler_mutex);
873 	INIT_LIST_HEAD(&id_priv->listen_list);
874 	INIT_LIST_HEAD(&id_priv->mc_list);
875 	get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
876 	id_priv->id.route.addr.dev_addr.net = get_net(net);
877 	id_priv->seq_num &= 0x00ffffff;
878 
879 	return &id_priv->id;
880 }
881 EXPORT_SYMBOL(__rdma_create_id);
882 
883 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
884 {
885 	struct ib_qp_attr qp_attr;
886 	int qp_attr_mask, ret;
887 
888 	qp_attr.qp_state = IB_QPS_INIT;
889 	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
890 	if (ret)
891 		return ret;
892 
893 	ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
894 	if (ret)
895 		return ret;
896 
897 	qp_attr.qp_state = IB_QPS_RTR;
898 	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
899 	if (ret)
900 		return ret;
901 
902 	qp_attr.qp_state = IB_QPS_RTS;
903 	qp_attr.sq_psn = 0;
904 	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
905 
906 	return ret;
907 }
908 
909 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
910 {
911 	struct ib_qp_attr qp_attr;
912 	int qp_attr_mask, ret;
913 
914 	qp_attr.qp_state = IB_QPS_INIT;
915 	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
916 	if (ret)
917 		return ret;
918 
919 	return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
920 }
921 
922 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
923 		   struct ib_qp_init_attr *qp_init_attr)
924 {
925 	struct rdma_id_private *id_priv;
926 	struct ib_qp *qp;
927 	int ret;
928 
929 	id_priv = container_of(id, struct rdma_id_private, id);
930 	if (id->device != pd->device) {
931 		ret = -EINVAL;
932 		goto out_err;
933 	}
934 
935 	qp_init_attr->port_num = id->port_num;
936 	qp = ib_create_qp(pd, qp_init_attr);
937 	if (IS_ERR(qp)) {
938 		ret = PTR_ERR(qp);
939 		goto out_err;
940 	}
941 
942 	if (id->qp_type == IB_QPT_UD)
943 		ret = cma_init_ud_qp(id_priv, qp);
944 	else
945 		ret = cma_init_conn_qp(id_priv, qp);
946 	if (ret)
947 		goto out_destroy;
948 
949 	id->qp = qp;
950 	id_priv->qp_num = qp->qp_num;
951 	id_priv->srq = (qp->srq != NULL);
952 	trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);
953 	return 0;
954 out_destroy:
955 	ib_destroy_qp(qp);
956 out_err:
957 	trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);
958 	return ret;
959 }
960 EXPORT_SYMBOL(rdma_create_qp);
961 
962 void rdma_destroy_qp(struct rdma_cm_id *id)
963 {
964 	struct rdma_id_private *id_priv;
965 
966 	id_priv = container_of(id, struct rdma_id_private, id);
967 	trace_cm_qp_destroy(id_priv);
968 	mutex_lock(&id_priv->qp_mutex);
969 	ib_destroy_qp(id_priv->id.qp);
970 	id_priv->id.qp = NULL;
971 	mutex_unlock(&id_priv->qp_mutex);
972 }
973 EXPORT_SYMBOL(rdma_destroy_qp);
974 
975 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
976 			     struct rdma_conn_param *conn_param)
977 {
978 	struct ib_qp_attr qp_attr;
979 	int qp_attr_mask, ret;
980 
981 	mutex_lock(&id_priv->qp_mutex);
982 	if (!id_priv->id.qp) {
983 		ret = 0;
984 		goto out;
985 	}
986 
987 	/* Need to update QP attributes from default values. */
988 	qp_attr.qp_state = IB_QPS_INIT;
989 	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
990 	if (ret)
991 		goto out;
992 
993 	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
994 	if (ret)
995 		goto out;
996 
997 	qp_attr.qp_state = IB_QPS_RTR;
998 	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
999 	if (ret)
1000 		goto out;
1001 
1002 	BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
1003 
1004 	if (conn_param)
1005 		qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
1006 	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1007 out:
1008 	mutex_unlock(&id_priv->qp_mutex);
1009 	return ret;
1010 }
1011 
1012 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
1013 			     struct rdma_conn_param *conn_param)
1014 {
1015 	struct ib_qp_attr qp_attr;
1016 	int qp_attr_mask, ret;
1017 
1018 	mutex_lock(&id_priv->qp_mutex);
1019 	if (!id_priv->id.qp) {
1020 		ret = 0;
1021 		goto out;
1022 	}
1023 
1024 	qp_attr.qp_state = IB_QPS_RTS;
1025 	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1026 	if (ret)
1027 		goto out;
1028 
1029 	if (conn_param)
1030 		qp_attr.max_rd_atomic = conn_param->initiator_depth;
1031 	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1032 out:
1033 	mutex_unlock(&id_priv->qp_mutex);
1034 	return ret;
1035 }
1036 
1037 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
1038 {
1039 	struct ib_qp_attr qp_attr;
1040 	int ret;
1041 
1042 	mutex_lock(&id_priv->qp_mutex);
1043 	if (!id_priv->id.qp) {
1044 		ret = 0;
1045 		goto out;
1046 	}
1047 
1048 	qp_attr.qp_state = IB_QPS_ERR;
1049 	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
1050 out:
1051 	mutex_unlock(&id_priv->qp_mutex);
1052 	return ret;
1053 }
1054 
1055 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
1056 			       struct ib_qp_attr *qp_attr, int *qp_attr_mask)
1057 {
1058 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1059 	int ret;
1060 	u16 pkey;
1061 
1062 	if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
1063 		pkey = 0xffff;
1064 	else
1065 		pkey = ib_addr_get_pkey(dev_addr);
1066 
1067 	ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
1068 				  pkey, &qp_attr->pkey_index);
1069 	if (ret)
1070 		return ret;
1071 
1072 	qp_attr->port_num = id_priv->id.port_num;
1073 	*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
1074 
1075 	if (id_priv->id.qp_type == IB_QPT_UD) {
1076 		ret = cma_set_qkey(id_priv, 0);
1077 		if (ret)
1078 			return ret;
1079 
1080 		qp_attr->qkey = id_priv->qkey;
1081 		*qp_attr_mask |= IB_QP_QKEY;
1082 	} else {
1083 		qp_attr->qp_access_flags = 0;
1084 		*qp_attr_mask |= IB_QP_ACCESS_FLAGS;
1085 	}
1086 	return 0;
1087 }
1088 
1089 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
1090 		       int *qp_attr_mask)
1091 {
1092 	struct rdma_id_private *id_priv;
1093 	int ret = 0;
1094 
1095 	id_priv = container_of(id, struct rdma_id_private, id);
1096 	if (rdma_cap_ib_cm(id->device, id->port_num)) {
1097 		if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
1098 			ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
1099 		else
1100 			ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
1101 						 qp_attr_mask);
1102 
1103 		if (qp_attr->qp_state == IB_QPS_RTR)
1104 			qp_attr->rq_psn = id_priv->seq_num;
1105 	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
1106 		if (!id_priv->cm_id.iw) {
1107 			qp_attr->qp_access_flags = 0;
1108 			*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
1109 		} else
1110 			ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
1111 						 qp_attr_mask);
1112 		qp_attr->port_num = id_priv->id.port_num;
1113 		*qp_attr_mask |= IB_QP_PORT;
1114 	} else
1115 		ret = -ENOSYS;
1116 
1117 	if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set)
1118 		qp_attr->timeout = id_priv->timeout;
1119 
1120 	return ret;
1121 }
1122 EXPORT_SYMBOL(rdma_init_qp_attr);
1123 
1124 static inline bool cma_zero_addr(const struct sockaddr *addr)
1125 {
1126 	switch (addr->sa_family) {
1127 	case AF_INET:
1128 		return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
1129 	case AF_INET6:
1130 		return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr);
1131 	case AF_IB:
1132 		return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr);
1133 	default:
1134 		return false;
1135 	}
1136 }
1137 
1138 static inline bool cma_loopback_addr(const struct sockaddr *addr)
1139 {
1140 	switch (addr->sa_family) {
1141 	case AF_INET:
1142 		return ipv4_is_loopback(
1143 			((struct sockaddr_in *)addr)->sin_addr.s_addr);
1144 	case AF_INET6:
1145 		return ipv6_addr_loopback(
1146 			&((struct sockaddr_in6 *)addr)->sin6_addr);
1147 	case AF_IB:
1148 		return ib_addr_loopback(
1149 			&((struct sockaddr_ib *)addr)->sib_addr);
1150 	default:
1151 		return false;
1152 	}
1153 }
1154 
1155 static inline bool cma_any_addr(const struct sockaddr *addr)
1156 {
1157 	return cma_zero_addr(addr) || cma_loopback_addr(addr);
1158 }
1159 
1160 static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
1161 {
1162 	if (src->sa_family != dst->sa_family)
1163 		return -1;
1164 
1165 	switch (src->sa_family) {
1166 	case AF_INET:
1167 		return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
1168 		       ((struct sockaddr_in *)dst)->sin_addr.s_addr;
1169 	case AF_INET6: {
1170 		struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
1171 		struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
1172 		bool link_local;
1173 
1174 		if (ipv6_addr_cmp(&src_addr6->sin6_addr,
1175 					  &dst_addr6->sin6_addr))
1176 			return 1;
1177 		link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
1178 			     IPV6_ADDR_LINKLOCAL;
1179 		/* Link local must match their scope_ids */
1180 		return link_local ? (src_addr6->sin6_scope_id !=
1181 				     dst_addr6->sin6_scope_id) :
1182 				    0;
1183 	}
1184 
1185 	default:
1186 		return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
1187 				   &((struct sockaddr_ib *) dst)->sib_addr);
1188 	}
1189 }
1190 
1191 static __be16 cma_port(const struct sockaddr *addr)
1192 {
1193 	struct sockaddr_ib *sib;
1194 
1195 	switch (addr->sa_family) {
1196 	case AF_INET:
1197 		return ((struct sockaddr_in *) addr)->sin_port;
1198 	case AF_INET6:
1199 		return ((struct sockaddr_in6 *) addr)->sin6_port;
1200 	case AF_IB:
1201 		sib = (struct sockaddr_ib *) addr;
1202 		return htons((u16) (be64_to_cpu(sib->sib_sid) &
1203 				    be64_to_cpu(sib->sib_sid_mask)));
1204 	default:
1205 		return 0;
1206 	}
1207 }
1208 
1209 static inline int cma_any_port(const struct sockaddr *addr)
1210 {
1211 	return !cma_port(addr);
1212 }
1213 
1214 static void cma_save_ib_info(struct sockaddr *src_addr,
1215 			     struct sockaddr *dst_addr,
1216 			     const struct rdma_cm_id *listen_id,
1217 			     const struct sa_path_rec *path)
1218 {
1219 	struct sockaddr_ib *listen_ib, *ib;
1220 
1221 	listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
1222 	if (src_addr) {
1223 		ib = (struct sockaddr_ib *)src_addr;
1224 		ib->sib_family = AF_IB;
1225 		if (path) {
1226 			ib->sib_pkey = path->pkey;
1227 			ib->sib_flowinfo = path->flow_label;
1228 			memcpy(&ib->sib_addr, &path->sgid, 16);
1229 			ib->sib_sid = path->service_id;
1230 			ib->sib_scope_id = 0;
1231 		} else {
1232 			ib->sib_pkey = listen_ib->sib_pkey;
1233 			ib->sib_flowinfo = listen_ib->sib_flowinfo;
1234 			ib->sib_addr = listen_ib->sib_addr;
1235 			ib->sib_sid = listen_ib->sib_sid;
1236 			ib->sib_scope_id = listen_ib->sib_scope_id;
1237 		}
1238 		ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
1239 	}
1240 	if (dst_addr) {
1241 		ib = (struct sockaddr_ib *)dst_addr;
1242 		ib->sib_family = AF_IB;
1243 		if (path) {
1244 			ib->sib_pkey = path->pkey;
1245 			ib->sib_flowinfo = path->flow_label;
1246 			memcpy(&ib->sib_addr, &path->dgid, 16);
1247 		}
1248 	}
1249 }
1250 
1251 static void cma_save_ip4_info(struct sockaddr_in *src_addr,
1252 			      struct sockaddr_in *dst_addr,
1253 			      struct cma_hdr *hdr,
1254 			      __be16 local_port)
1255 {
1256 	if (src_addr) {
1257 		*src_addr = (struct sockaddr_in) {
1258 			.sin_family = AF_INET,
1259 			.sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1260 			.sin_port = local_port,
1261 		};
1262 	}
1263 
1264 	if (dst_addr) {
1265 		*dst_addr = (struct sockaddr_in) {
1266 			.sin_family = AF_INET,
1267 			.sin_addr.s_addr = hdr->src_addr.ip4.addr,
1268 			.sin_port = hdr->port,
1269 		};
1270 	}
1271 }
1272 
1273 static void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
1274 			      struct sockaddr_in6 *dst_addr,
1275 			      struct cma_hdr *hdr,
1276 			      __be16 local_port)
1277 {
1278 	if (src_addr) {
1279 		*src_addr = (struct sockaddr_in6) {
1280 			.sin6_family = AF_INET6,
1281 			.sin6_addr = hdr->dst_addr.ip6,
1282 			.sin6_port = local_port,
1283 		};
1284 	}
1285 
1286 	if (dst_addr) {
1287 		*dst_addr = (struct sockaddr_in6) {
1288 			.sin6_family = AF_INET6,
1289 			.sin6_addr = hdr->src_addr.ip6,
1290 			.sin6_port = hdr->port,
1291 		};
1292 	}
1293 }
1294 
1295 static u16 cma_port_from_service_id(__be64 service_id)
1296 {
1297 	return (u16)be64_to_cpu(service_id);
1298 }
1299 
1300 static int cma_save_ip_info(struct sockaddr *src_addr,
1301 			    struct sockaddr *dst_addr,
1302 			    const struct ib_cm_event *ib_event,
1303 			    __be64 service_id)
1304 {
1305 	struct cma_hdr *hdr;
1306 	__be16 port;
1307 
1308 	hdr = ib_event->private_data;
1309 	if (hdr->cma_version != CMA_VERSION)
1310 		return -EINVAL;
1311 
1312 	port = htons(cma_port_from_service_id(service_id));
1313 
1314 	switch (cma_get_ip_ver(hdr)) {
1315 	case 4:
1316 		cma_save_ip4_info((struct sockaddr_in *)src_addr,
1317 				  (struct sockaddr_in *)dst_addr, hdr, port);
1318 		break;
1319 	case 6:
1320 		cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
1321 				  (struct sockaddr_in6 *)dst_addr, hdr, port);
1322 		break;
1323 	default:
1324 		return -EAFNOSUPPORT;
1325 	}
1326 
1327 	return 0;
1328 }
1329 
1330 static int cma_save_net_info(struct sockaddr *src_addr,
1331 			     struct sockaddr *dst_addr,
1332 			     const struct rdma_cm_id *listen_id,
1333 			     const struct ib_cm_event *ib_event,
1334 			     sa_family_t sa_family, __be64 service_id)
1335 {
1336 	if (sa_family == AF_IB) {
1337 		if (ib_event->event == IB_CM_REQ_RECEIVED)
1338 			cma_save_ib_info(src_addr, dst_addr, listen_id,
1339 					 ib_event->param.req_rcvd.primary_path);
1340 		else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1341 			cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
1342 		return 0;
1343 	}
1344 
1345 	return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
1346 }
1347 
1348 static int cma_save_req_info(const struct ib_cm_event *ib_event,
1349 			     struct cma_req_info *req)
1350 {
1351 	const struct ib_cm_req_event_param *req_param =
1352 		&ib_event->param.req_rcvd;
1353 	const struct ib_cm_sidr_req_event_param *sidr_param =
1354 		&ib_event->param.sidr_req_rcvd;
1355 
1356 	switch (ib_event->event) {
1357 	case IB_CM_REQ_RECEIVED:
1358 		req->device	= req_param->listen_id->device;
1359 		req->port	= req_param->port;
1360 		memcpy(&req->local_gid, &req_param->primary_path->sgid,
1361 		       sizeof(req->local_gid));
1362 		req->has_gid	= true;
1363 		req->service_id = req_param->primary_path->service_id;
1364 		req->pkey	= be16_to_cpu(req_param->primary_path->pkey);
1365 		if (req->pkey != req_param->bth_pkey)
1366 			pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
1367 					    "RDMA CMA: in the future this may cause the request to be dropped\n",
1368 					    req_param->bth_pkey, req->pkey);
1369 		break;
1370 	case IB_CM_SIDR_REQ_RECEIVED:
1371 		req->device	= sidr_param->listen_id->device;
1372 		req->port	= sidr_param->port;
1373 		req->has_gid	= false;
1374 		req->service_id	= sidr_param->service_id;
1375 		req->pkey	= sidr_param->pkey;
1376 		if (req->pkey != sidr_param->bth_pkey)
1377 			pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
1378 					    "RDMA CMA: in the future this may cause the request to be dropped\n",
1379 					    sidr_param->bth_pkey, req->pkey);
1380 		break;
1381 	default:
1382 		return -EINVAL;
1383 	}
1384 
1385 	return 0;
1386 }
1387 
1388 static bool validate_ipv4_net_dev(struct net_device *net_dev,
1389 				  const struct sockaddr_in *dst_addr,
1390 				  const struct sockaddr_in *src_addr)
1391 {
1392 	__be32 daddr = dst_addr->sin_addr.s_addr,
1393 	       saddr = src_addr->sin_addr.s_addr;
1394 	struct fib_result res;
1395 	struct flowi4 fl4;
1396 	int err;
1397 	bool ret;
1398 
1399 	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1400 	    ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
1401 	    ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
1402 	    ipv4_is_loopback(saddr))
1403 		return false;
1404 
1405 	memset(&fl4, 0, sizeof(fl4));
1406 	fl4.flowi4_iif = net_dev->ifindex;
1407 	fl4.daddr = daddr;
1408 	fl4.saddr = saddr;
1409 
1410 	rcu_read_lock();
1411 	err = fib_lookup(dev_net(net_dev), &fl4, &res, 0);
1412 	ret = err == 0 && FIB_RES_DEV(res) == net_dev;
1413 	rcu_read_unlock();
1414 
1415 	return ret;
1416 }
1417 
1418 static bool validate_ipv6_net_dev(struct net_device *net_dev,
1419 				  const struct sockaddr_in6 *dst_addr,
1420 				  const struct sockaddr_in6 *src_addr)
1421 {
1422 #if IS_ENABLED(CONFIG_IPV6)
1423 	const int strict = ipv6_addr_type(&dst_addr->sin6_addr) &
1424 			   IPV6_ADDR_LINKLOCAL;
1425 	struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr,
1426 					 &src_addr->sin6_addr, net_dev->ifindex,
1427 					 NULL, strict);
1428 	bool ret;
1429 
1430 	if (!rt)
1431 		return false;
1432 
1433 	ret = rt->rt6i_idev->dev == net_dev;
1434 	ip6_rt_put(rt);
1435 
1436 	return ret;
1437 #else
1438 	return false;
1439 #endif
1440 }
1441 
1442 static bool validate_net_dev(struct net_device *net_dev,
1443 			     const struct sockaddr *daddr,
1444 			     const struct sockaddr *saddr)
1445 {
1446 	const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
1447 	const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
1448 	const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1449 	const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
1450 
1451 	switch (daddr->sa_family) {
1452 	case AF_INET:
1453 		return saddr->sa_family == AF_INET &&
1454 		       validate_ipv4_net_dev(net_dev, daddr4, saddr4);
1455 
1456 	case AF_INET6:
1457 		return saddr->sa_family == AF_INET6 &&
1458 		       validate_ipv6_net_dev(net_dev, daddr6, saddr6);
1459 
1460 	default:
1461 		return false;
1462 	}
1463 }
1464 
1465 static struct net_device *
1466 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
1467 {
1468 	const struct ib_gid_attr *sgid_attr = NULL;
1469 	struct net_device *ndev;
1470 
1471 	if (ib_event->event == IB_CM_REQ_RECEIVED)
1472 		sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
1473 	else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1474 		sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr;
1475 
1476 	if (!sgid_attr)
1477 		return NULL;
1478 
1479 	rcu_read_lock();
1480 	ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
1481 	if (IS_ERR(ndev))
1482 		ndev = NULL;
1483 	else
1484 		dev_hold(ndev);
1485 	rcu_read_unlock();
1486 	return ndev;
1487 }
1488 
1489 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
1490 					  struct cma_req_info *req)
1491 {
1492 	struct sockaddr *listen_addr =
1493 			(struct sockaddr *)&req->listen_addr_storage;
1494 	struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage;
1495 	struct net_device *net_dev;
1496 	const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
1497 	int err;
1498 
1499 	err = cma_save_ip_info(listen_addr, src_addr, ib_event,
1500 			       req->service_id);
1501 	if (err)
1502 		return ERR_PTR(err);
1503 
1504 	if (rdma_protocol_roce(req->device, req->port))
1505 		net_dev = roce_get_net_dev_by_cm_event(ib_event);
1506 	else
1507 		net_dev = ib_get_net_dev_by_params(req->device, req->port,
1508 						   req->pkey,
1509 						   gid, listen_addr);
1510 	if (!net_dev)
1511 		return ERR_PTR(-ENODEV);
1512 
1513 	return net_dev;
1514 }
1515 
1516 static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id)
1517 {
1518 	return (be64_to_cpu(service_id) >> 16) & 0xffff;
1519 }
1520 
1521 static bool cma_match_private_data(struct rdma_id_private *id_priv,
1522 				   const struct cma_hdr *hdr)
1523 {
1524 	struct sockaddr *addr = cma_src_addr(id_priv);
1525 	__be32 ip4_addr;
1526 	struct in6_addr ip6_addr;
1527 
1528 	if (cma_any_addr(addr) && !id_priv->afonly)
1529 		return true;
1530 
1531 	switch (addr->sa_family) {
1532 	case AF_INET:
1533 		ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1534 		if (cma_get_ip_ver(hdr) != 4)
1535 			return false;
1536 		if (!cma_any_addr(addr) &&
1537 		    hdr->dst_addr.ip4.addr != ip4_addr)
1538 			return false;
1539 		break;
1540 	case AF_INET6:
1541 		ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1542 		if (cma_get_ip_ver(hdr) != 6)
1543 			return false;
1544 		if (!cma_any_addr(addr) &&
1545 		    memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1546 			return false;
1547 		break;
1548 	case AF_IB:
1549 		return true;
1550 	default:
1551 		return false;
1552 	}
1553 
1554 	return true;
1555 }
1556 
1557 static bool cma_protocol_roce(const struct rdma_cm_id *id)
1558 {
1559 	struct ib_device *device = id->device;
1560 	const int port_num = id->port_num ?: rdma_start_port(device);
1561 
1562 	return rdma_protocol_roce(device, port_num);
1563 }
1564 
1565 static bool cma_is_req_ipv6_ll(const struct cma_req_info *req)
1566 {
1567 	const struct sockaddr *daddr =
1568 			(const struct sockaddr *)&req->listen_addr_storage;
1569 	const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1570 
1571 	/* Returns true if the req is for IPv6 link local */
1572 	return (daddr->sa_family == AF_INET6 &&
1573 		(ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL));
1574 }
1575 
1576 static bool cma_match_net_dev(const struct rdma_cm_id *id,
1577 			      const struct net_device *net_dev,
1578 			      const struct cma_req_info *req)
1579 {
1580 	const struct rdma_addr *addr = &id->route.addr;
1581 
1582 	if (!net_dev)
1583 		/* This request is an AF_IB request */
1584 		return (!id->port_num || id->port_num == req->port) &&
1585 		       (addr->src_addr.ss_family == AF_IB);
1586 
1587 	/*
1588 	 * If the request is not for IPv6 link local, allow matching
1589 	 * request to any netdevice of the one or multiport rdma device.
1590 	 */
1591 	if (!cma_is_req_ipv6_ll(req))
1592 		return true;
1593 	/*
1594 	 * Net namespaces must match, and if the listner is listening
1595 	 * on a specific netdevice than netdevice must match as well.
1596 	 */
1597 	if (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
1598 	    (!!addr->dev_addr.bound_dev_if ==
1599 	     (addr->dev_addr.bound_dev_if == net_dev->ifindex)))
1600 		return true;
1601 	else
1602 		return false;
1603 }
1604 
1605 static struct rdma_id_private *cma_find_listener(
1606 		const struct rdma_bind_list *bind_list,
1607 		const struct ib_cm_id *cm_id,
1608 		const struct ib_cm_event *ib_event,
1609 		const struct cma_req_info *req,
1610 		const struct net_device *net_dev)
1611 {
1612 	struct rdma_id_private *id_priv, *id_priv_dev;
1613 
1614 	lockdep_assert_held(&lock);
1615 
1616 	if (!bind_list)
1617 		return ERR_PTR(-EINVAL);
1618 
1619 	hlist_for_each_entry(id_priv, &bind_list->owners, node) {
1620 		if (cma_match_private_data(id_priv, ib_event->private_data)) {
1621 			if (id_priv->id.device == cm_id->device &&
1622 			    cma_match_net_dev(&id_priv->id, net_dev, req))
1623 				return id_priv;
1624 			list_for_each_entry(id_priv_dev,
1625 					    &id_priv->listen_list,
1626 					    listen_list) {
1627 				if (id_priv_dev->id.device == cm_id->device &&
1628 				    cma_match_net_dev(&id_priv_dev->id,
1629 						      net_dev, req))
1630 					return id_priv_dev;
1631 			}
1632 		}
1633 	}
1634 
1635 	return ERR_PTR(-EINVAL);
1636 }
1637 
1638 static struct rdma_id_private *
1639 cma_ib_id_from_event(struct ib_cm_id *cm_id,
1640 		     const struct ib_cm_event *ib_event,
1641 		     struct cma_req_info *req,
1642 		     struct net_device **net_dev)
1643 {
1644 	struct rdma_bind_list *bind_list;
1645 	struct rdma_id_private *id_priv;
1646 	int err;
1647 
1648 	err = cma_save_req_info(ib_event, req);
1649 	if (err)
1650 		return ERR_PTR(err);
1651 
1652 	*net_dev = cma_get_net_dev(ib_event, req);
1653 	if (IS_ERR(*net_dev)) {
1654 		if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
1655 			/* Assuming the protocol is AF_IB */
1656 			*net_dev = NULL;
1657 		} else {
1658 			return ERR_CAST(*net_dev);
1659 		}
1660 	}
1661 
1662 	mutex_lock(&lock);
1663 	/*
1664 	 * Net namespace might be getting deleted while route lookup,
1665 	 * cm_id lookup is in progress. Therefore, perform netdevice
1666 	 * validation, cm_id lookup under rcu lock.
1667 	 * RCU lock along with netdevice state check, synchronizes with
1668 	 * netdevice migrating to different net namespace and also avoids
1669 	 * case where net namespace doesn't get deleted while lookup is in
1670 	 * progress.
1671 	 * If the device state is not IFF_UP, its properties such as ifindex
1672 	 * and nd_net cannot be trusted to remain valid without rcu lock.
1673 	 * net/core/dev.c change_net_namespace() ensures to synchronize with
1674 	 * ongoing operations on net device after device is closed using
1675 	 * synchronize_net().
1676 	 */
1677 	rcu_read_lock();
1678 	if (*net_dev) {
1679 		/*
1680 		 * If netdevice is down, it is likely that it is administratively
1681 		 * down or it might be migrating to different namespace.
1682 		 * In that case avoid further processing, as the net namespace
1683 		 * or ifindex may change.
1684 		 */
1685 		if (((*net_dev)->flags & IFF_UP) == 0) {
1686 			id_priv = ERR_PTR(-EHOSTUNREACH);
1687 			goto err;
1688 		}
1689 
1690 		if (!validate_net_dev(*net_dev,
1691 				 (struct sockaddr *)&req->listen_addr_storage,
1692 				 (struct sockaddr *)&req->src_addr_storage)) {
1693 			id_priv = ERR_PTR(-EHOSTUNREACH);
1694 			goto err;
1695 		}
1696 	}
1697 
1698 	bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
1699 				rdma_ps_from_service_id(req->service_id),
1700 				cma_port_from_service_id(req->service_id));
1701 	id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);
1702 err:
1703 	rcu_read_unlock();
1704 	mutex_unlock(&lock);
1705 	if (IS_ERR(id_priv) && *net_dev) {
1706 		dev_put(*net_dev);
1707 		*net_dev = NULL;
1708 	}
1709 	return id_priv;
1710 }
1711 
1712 static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv)
1713 {
1714 	return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr);
1715 }
1716 
1717 static void cma_cancel_route(struct rdma_id_private *id_priv)
1718 {
1719 	if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
1720 		if (id_priv->query)
1721 			ib_sa_cancel_query(id_priv->query_id, id_priv->query);
1722 	}
1723 }
1724 
1725 static void cma_cancel_listens(struct rdma_id_private *id_priv)
1726 {
1727 	struct rdma_id_private *dev_id_priv;
1728 
1729 	/*
1730 	 * Remove from listen_any_list to prevent added devices from spawning
1731 	 * additional listen requests.
1732 	 */
1733 	mutex_lock(&lock);
1734 	list_del(&id_priv->list);
1735 
1736 	while (!list_empty(&id_priv->listen_list)) {
1737 		dev_id_priv = list_entry(id_priv->listen_list.next,
1738 					 struct rdma_id_private, listen_list);
1739 		/* sync with device removal to avoid duplicate destruction */
1740 		list_del_init(&dev_id_priv->list);
1741 		list_del(&dev_id_priv->listen_list);
1742 		mutex_unlock(&lock);
1743 
1744 		rdma_destroy_id(&dev_id_priv->id);
1745 		mutex_lock(&lock);
1746 	}
1747 	mutex_unlock(&lock);
1748 }
1749 
1750 static void cma_cancel_operation(struct rdma_id_private *id_priv,
1751 				 enum rdma_cm_state state)
1752 {
1753 	switch (state) {
1754 	case RDMA_CM_ADDR_QUERY:
1755 		rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
1756 		break;
1757 	case RDMA_CM_ROUTE_QUERY:
1758 		cma_cancel_route(id_priv);
1759 		break;
1760 	case RDMA_CM_LISTEN:
1761 		if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
1762 			cma_cancel_listens(id_priv);
1763 		break;
1764 	default:
1765 		break;
1766 	}
1767 }
1768 
1769 static void cma_release_port(struct rdma_id_private *id_priv)
1770 {
1771 	struct rdma_bind_list *bind_list = id_priv->bind_list;
1772 	struct net *net = id_priv->id.route.addr.dev_addr.net;
1773 
1774 	if (!bind_list)
1775 		return;
1776 
1777 	mutex_lock(&lock);
1778 	hlist_del(&id_priv->node);
1779 	if (hlist_empty(&bind_list->owners)) {
1780 		cma_ps_remove(net, bind_list->ps, bind_list->port);
1781 		kfree(bind_list);
1782 	}
1783 	mutex_unlock(&lock);
1784 }
1785 
1786 static void cma_leave_roce_mc_group(struct rdma_id_private *id_priv,
1787 				    struct cma_multicast *mc)
1788 {
1789 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1790 	struct net_device *ndev = NULL;
1791 
1792 	if (dev_addr->bound_dev_if)
1793 		ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
1794 	if (ndev) {
1795 		cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, false);
1796 		dev_put(ndev);
1797 	}
1798 	kref_put(&mc->mcref, release_mc);
1799 }
1800 
1801 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
1802 {
1803 	struct cma_multicast *mc;
1804 
1805 	while (!list_empty(&id_priv->mc_list)) {
1806 		mc = container_of(id_priv->mc_list.next,
1807 				  struct cma_multicast, list);
1808 		list_del(&mc->list);
1809 		if (rdma_cap_ib_mcast(id_priv->cma_dev->device,
1810 				      id_priv->id.port_num)) {
1811 			ib_sa_free_multicast(mc->multicast.ib);
1812 			kfree(mc);
1813 		} else {
1814 			cma_leave_roce_mc_group(id_priv, mc);
1815 		}
1816 	}
1817 }
1818 
1819 static void _destroy_id(struct rdma_id_private *id_priv,
1820 			enum rdma_cm_state state)
1821 {
1822 	cma_cancel_operation(id_priv, state);
1823 
1824 	rdma_restrack_del(&id_priv->res);
1825 	if (id_priv->cma_dev) {
1826 		if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
1827 			if (id_priv->cm_id.ib)
1828 				ib_destroy_cm_id(id_priv->cm_id.ib);
1829 		} else if (rdma_cap_iw_cm(id_priv->id.device, 1)) {
1830 			if (id_priv->cm_id.iw)
1831 				iw_destroy_cm_id(id_priv->cm_id.iw);
1832 		}
1833 		cma_leave_mc_groups(id_priv);
1834 		cma_release_dev(id_priv);
1835 	}
1836 
1837 	cma_release_port(id_priv);
1838 	cma_id_put(id_priv);
1839 	wait_for_completion(&id_priv->comp);
1840 
1841 	if (id_priv->internal_id)
1842 		cma_id_put(id_priv->id.context);
1843 
1844 	kfree(id_priv->id.route.path_rec);
1845 
1846 	if (id_priv->id.route.addr.dev_addr.sgid_attr)
1847 		rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
1848 
1849 	put_net(id_priv->id.route.addr.dev_addr.net);
1850 	kfree(id_priv);
1851 }
1852 
1853 /*
1854  * destroy an ID from within the handler_mutex. This ensures that no other
1855  * handlers can start running concurrently.
1856  */
1857 static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
1858 	__releases(&idprv->handler_mutex)
1859 {
1860 	enum rdma_cm_state state;
1861 	unsigned long flags;
1862 
1863 	trace_cm_id_destroy(id_priv);
1864 
1865 	/*
1866 	 * Setting the state to destroyed under the handler mutex provides a
1867 	 * fence against calling handler callbacks. If this is invoked due to
1868 	 * the failure of a handler callback then it guarentees that no future
1869 	 * handlers will be called.
1870 	 */
1871 	lockdep_assert_held(&id_priv->handler_mutex);
1872 	spin_lock_irqsave(&id_priv->lock, flags);
1873 	state = id_priv->state;
1874 	id_priv->state = RDMA_CM_DESTROYING;
1875 	spin_unlock_irqrestore(&id_priv->lock, flags);
1876 	mutex_unlock(&id_priv->handler_mutex);
1877 	_destroy_id(id_priv, state);
1878 }
1879 
1880 void rdma_destroy_id(struct rdma_cm_id *id)
1881 {
1882 	struct rdma_id_private *id_priv =
1883 		container_of(id, struct rdma_id_private, id);
1884 
1885 	mutex_lock(&id_priv->handler_mutex);
1886 	destroy_id_handler_unlock(id_priv);
1887 }
1888 EXPORT_SYMBOL(rdma_destroy_id);
1889 
1890 static int cma_rep_recv(struct rdma_id_private *id_priv)
1891 {
1892 	int ret;
1893 
1894 	ret = cma_modify_qp_rtr(id_priv, NULL);
1895 	if (ret)
1896 		goto reject;
1897 
1898 	ret = cma_modify_qp_rts(id_priv, NULL);
1899 	if (ret)
1900 		goto reject;
1901 
1902 	trace_cm_send_rtu(id_priv);
1903 	ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1904 	if (ret)
1905 		goto reject;
1906 
1907 	return 0;
1908 reject:
1909 	pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);
1910 	cma_modify_qp_err(id_priv);
1911 	trace_cm_send_rej(id_priv);
1912 	ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1913 		       NULL, 0, NULL, 0);
1914 	return ret;
1915 }
1916 
1917 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1918 				   const struct ib_cm_rep_event_param *rep_data,
1919 				   void *private_data)
1920 {
1921 	event->param.conn.private_data = private_data;
1922 	event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1923 	event->param.conn.responder_resources = rep_data->responder_resources;
1924 	event->param.conn.initiator_depth = rep_data->initiator_depth;
1925 	event->param.conn.flow_control = rep_data->flow_control;
1926 	event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1927 	event->param.conn.srq = rep_data->srq;
1928 	event->param.conn.qp_num = rep_data->remote_qpn;
1929 
1930 	event->ece.vendor_id = rep_data->ece.vendor_id;
1931 	event->ece.attr_mod = rep_data->ece.attr_mod;
1932 }
1933 
1934 static int cma_cm_event_handler(struct rdma_id_private *id_priv,
1935 				struct rdma_cm_event *event)
1936 {
1937 	int ret;
1938 
1939 	lockdep_assert_held(&id_priv->handler_mutex);
1940 
1941 	trace_cm_event_handler(id_priv, event);
1942 	ret = id_priv->id.event_handler(&id_priv->id, event);
1943 	trace_cm_event_done(id_priv, event, ret);
1944 	return ret;
1945 }
1946 
1947 static int cma_ib_handler(struct ib_cm_id *cm_id,
1948 			  const struct ib_cm_event *ib_event)
1949 {
1950 	struct rdma_id_private *id_priv = cm_id->context;
1951 	struct rdma_cm_event event = {};
1952 	int ret;
1953 
1954 	mutex_lock(&id_priv->handler_mutex);
1955 	if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1956 	     id_priv->state != RDMA_CM_CONNECT) ||
1957 	    (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1958 	     id_priv->state != RDMA_CM_DISCONNECT))
1959 		goto out;
1960 
1961 	switch (ib_event->event) {
1962 	case IB_CM_REQ_ERROR:
1963 	case IB_CM_REP_ERROR:
1964 		event.event = RDMA_CM_EVENT_UNREACHABLE;
1965 		event.status = -ETIMEDOUT;
1966 		break;
1967 	case IB_CM_REP_RECEIVED:
1968 		if (cma_comp(id_priv, RDMA_CM_CONNECT) &&
1969 		    (id_priv->id.qp_type != IB_QPT_UD)) {
1970 			trace_cm_send_mra(id_priv);
1971 			ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1972 		}
1973 		if (id_priv->id.qp) {
1974 			event.status = cma_rep_recv(id_priv);
1975 			event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1976 						     RDMA_CM_EVENT_ESTABLISHED;
1977 		} else {
1978 			event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1979 		}
1980 		cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1981 				       ib_event->private_data);
1982 		break;
1983 	case IB_CM_RTU_RECEIVED:
1984 	case IB_CM_USER_ESTABLISHED:
1985 		event.event = RDMA_CM_EVENT_ESTABLISHED;
1986 		break;
1987 	case IB_CM_DREQ_ERROR:
1988 		event.status = -ETIMEDOUT;
1989 		fallthrough;
1990 	case IB_CM_DREQ_RECEIVED:
1991 	case IB_CM_DREP_RECEIVED:
1992 		if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1993 				   RDMA_CM_DISCONNECT))
1994 			goto out;
1995 		event.event = RDMA_CM_EVENT_DISCONNECTED;
1996 		break;
1997 	case IB_CM_TIMEWAIT_EXIT:
1998 		event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1999 		break;
2000 	case IB_CM_MRA_RECEIVED:
2001 		/* ignore event */
2002 		goto out;
2003 	case IB_CM_REJ_RECEIVED:
2004 		pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id,
2005 										ib_event->param.rej_rcvd.reason));
2006 		cma_modify_qp_err(id_priv);
2007 		event.status = ib_event->param.rej_rcvd.reason;
2008 		event.event = RDMA_CM_EVENT_REJECTED;
2009 		event.param.conn.private_data = ib_event->private_data;
2010 		event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
2011 		break;
2012 	default:
2013 		pr_err("RDMA CMA: unexpected IB CM event: %d\n",
2014 		       ib_event->event);
2015 		goto out;
2016 	}
2017 
2018 	ret = cma_cm_event_handler(id_priv, &event);
2019 	if (ret) {
2020 		/* Destroy the CM ID by returning a non-zero value. */
2021 		id_priv->cm_id.ib = NULL;
2022 		destroy_id_handler_unlock(id_priv);
2023 		return ret;
2024 	}
2025 out:
2026 	mutex_unlock(&id_priv->handler_mutex);
2027 	return 0;
2028 }
2029 
2030 static struct rdma_id_private *
2031 cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
2032 		   const struct ib_cm_event *ib_event,
2033 		   struct net_device *net_dev)
2034 {
2035 	struct rdma_id_private *listen_id_priv;
2036 	struct rdma_id_private *id_priv;
2037 	struct rdma_cm_id *id;
2038 	struct rdma_route *rt;
2039 	const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2040 	struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path;
2041 	const __be64 service_id =
2042 		ib_event->param.req_rcvd.primary_path->service_id;
2043 	int ret;
2044 
2045 	listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2046 	id = __rdma_create_id(listen_id->route.addr.dev_addr.net,
2047 			    listen_id->event_handler, listen_id->context,
2048 			    listen_id->ps, ib_event->param.req_rcvd.qp_type,
2049 			    listen_id_priv->res.kern_name);
2050 	if (IS_ERR(id))
2051 		return NULL;
2052 
2053 	id_priv = container_of(id, struct rdma_id_private, id);
2054 	if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2055 			      (struct sockaddr *)&id->route.addr.dst_addr,
2056 			      listen_id, ib_event, ss_family, service_id))
2057 		goto err;
2058 
2059 	rt = &id->route;
2060 	rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
2061 	rt->path_rec = kmalloc_array(rt->num_paths, sizeof(*rt->path_rec),
2062 				     GFP_KERNEL);
2063 	if (!rt->path_rec)
2064 		goto err;
2065 
2066 	rt->path_rec[0] = *path;
2067 	if (rt->num_paths == 2)
2068 		rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
2069 
2070 	if (net_dev) {
2071 		rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
2072 	} else {
2073 		if (!cma_protocol_roce(listen_id) &&
2074 		    cma_any_addr(cma_src_addr(id_priv))) {
2075 			rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
2076 			rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
2077 			ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
2078 		} else if (!cma_any_addr(cma_src_addr(id_priv))) {
2079 			ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
2080 			if (ret)
2081 				goto err;
2082 		}
2083 	}
2084 	rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
2085 
2086 	id_priv->state = RDMA_CM_CONNECT;
2087 	return id_priv;
2088 
2089 err:
2090 	rdma_destroy_id(id);
2091 	return NULL;
2092 }
2093 
2094 static struct rdma_id_private *
2095 cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
2096 		  const struct ib_cm_event *ib_event,
2097 		  struct net_device *net_dev)
2098 {
2099 	const struct rdma_id_private *listen_id_priv;
2100 	struct rdma_id_private *id_priv;
2101 	struct rdma_cm_id *id;
2102 	const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2103 	struct net *net = listen_id->route.addr.dev_addr.net;
2104 	int ret;
2105 
2106 	listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2107 	id = __rdma_create_id(net, listen_id->event_handler, listen_id->context,
2108 			      listen_id->ps, IB_QPT_UD,
2109 			      listen_id_priv->res.kern_name);
2110 	if (IS_ERR(id))
2111 		return NULL;
2112 
2113 	id_priv = container_of(id, struct rdma_id_private, id);
2114 	if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2115 			      (struct sockaddr *)&id->route.addr.dst_addr,
2116 			      listen_id, ib_event, ss_family,
2117 			      ib_event->param.sidr_req_rcvd.service_id))
2118 		goto err;
2119 
2120 	if (net_dev) {
2121 		rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
2122 	} else {
2123 		if (!cma_any_addr(cma_src_addr(id_priv))) {
2124 			ret = cma_translate_addr(cma_src_addr(id_priv),
2125 						 &id->route.addr.dev_addr);
2126 			if (ret)
2127 				goto err;
2128 		}
2129 	}
2130 
2131 	id_priv->state = RDMA_CM_CONNECT;
2132 	return id_priv;
2133 err:
2134 	rdma_destroy_id(id);
2135 	return NULL;
2136 }
2137 
2138 static void cma_set_req_event_data(struct rdma_cm_event *event,
2139 				   const struct ib_cm_req_event_param *req_data,
2140 				   void *private_data, int offset)
2141 {
2142 	event->param.conn.private_data = private_data + offset;
2143 	event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
2144 	event->param.conn.responder_resources = req_data->responder_resources;
2145 	event->param.conn.initiator_depth = req_data->initiator_depth;
2146 	event->param.conn.flow_control = req_data->flow_control;
2147 	event->param.conn.retry_count = req_data->retry_count;
2148 	event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
2149 	event->param.conn.srq = req_data->srq;
2150 	event->param.conn.qp_num = req_data->remote_qpn;
2151 
2152 	event->ece.vendor_id = req_data->ece.vendor_id;
2153 	event->ece.attr_mod = req_data->ece.attr_mod;
2154 }
2155 
2156 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
2157 				    const struct ib_cm_event *ib_event)
2158 {
2159 	return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
2160 		 (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
2161 		((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
2162 		 (id->qp_type == IB_QPT_UD)) ||
2163 		(!id->qp_type));
2164 }
2165 
2166 static int cma_ib_req_handler(struct ib_cm_id *cm_id,
2167 			      const struct ib_cm_event *ib_event)
2168 {
2169 	struct rdma_id_private *listen_id, *conn_id = NULL;
2170 	struct rdma_cm_event event = {};
2171 	struct cma_req_info req = {};
2172 	struct net_device *net_dev;
2173 	u8 offset;
2174 	int ret;
2175 
2176 	listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev);
2177 	if (IS_ERR(listen_id))
2178 		return PTR_ERR(listen_id);
2179 
2180 	trace_cm_req_handler(listen_id, ib_event->event);
2181 	if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) {
2182 		ret = -EINVAL;
2183 		goto net_dev_put;
2184 	}
2185 
2186 	mutex_lock(&listen_id->handler_mutex);
2187 	if (listen_id->state != RDMA_CM_LISTEN) {
2188 		ret = -ECONNABORTED;
2189 		goto err_unlock;
2190 	}
2191 
2192 	offset = cma_user_data_offset(listen_id);
2193 	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2194 	if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
2195 		conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev);
2196 		event.param.ud.private_data = ib_event->private_data + offset;
2197 		event.param.ud.private_data_len =
2198 				IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
2199 	} else {
2200 		conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev);
2201 		cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
2202 				       ib_event->private_data, offset);
2203 	}
2204 	if (!conn_id) {
2205 		ret = -ENOMEM;
2206 		goto err_unlock;
2207 	}
2208 
2209 	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2210 	ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2211 	if (ret) {
2212 		destroy_id_handler_unlock(conn_id);
2213 		goto err_unlock;
2214 	}
2215 
2216 	conn_id->cm_id.ib = cm_id;
2217 	cm_id->context = conn_id;
2218 	cm_id->cm_handler = cma_ib_handler;
2219 
2220 	ret = cma_cm_event_handler(conn_id, &event);
2221 	if (ret) {
2222 		/* Destroy the CM ID by returning a non-zero value. */
2223 		conn_id->cm_id.ib = NULL;
2224 		mutex_unlock(&listen_id->handler_mutex);
2225 		destroy_id_handler_unlock(conn_id);
2226 		goto net_dev_put;
2227 	}
2228 
2229 	if (cma_comp(conn_id, RDMA_CM_CONNECT) &&
2230 	    (conn_id->id.qp_type != IB_QPT_UD)) {
2231 		trace_cm_send_mra(cm_id->context);
2232 		ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2233 	}
2234 	mutex_unlock(&conn_id->handler_mutex);
2235 
2236 err_unlock:
2237 	mutex_unlock(&listen_id->handler_mutex);
2238 
2239 net_dev_put:
2240 	if (net_dev)
2241 		dev_put(net_dev);
2242 
2243 	return ret;
2244 }
2245 
2246 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
2247 {
2248 	if (addr->sa_family == AF_IB)
2249 		return ((struct sockaddr_ib *) addr)->sib_sid;
2250 
2251 	return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr)));
2252 }
2253 EXPORT_SYMBOL(rdma_get_service_id);
2254 
2255 void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
2256 		    union ib_gid *dgid)
2257 {
2258 	struct rdma_addr *addr = &cm_id->route.addr;
2259 
2260 	if (!cm_id->device) {
2261 		if (sgid)
2262 			memset(sgid, 0, sizeof(*sgid));
2263 		if (dgid)
2264 			memset(dgid, 0, sizeof(*dgid));
2265 		return;
2266 	}
2267 
2268 	if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
2269 		if (sgid)
2270 			rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
2271 		if (dgid)
2272 			rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
2273 	} else {
2274 		if (sgid)
2275 			rdma_addr_get_sgid(&addr->dev_addr, sgid);
2276 		if (dgid)
2277 			rdma_addr_get_dgid(&addr->dev_addr, dgid);
2278 	}
2279 }
2280 EXPORT_SYMBOL(rdma_read_gids);
2281 
2282 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
2283 {
2284 	struct rdma_id_private *id_priv = iw_id->context;
2285 	struct rdma_cm_event event = {};
2286 	int ret = 0;
2287 	struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2288 	struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2289 
2290 	mutex_lock(&id_priv->handler_mutex);
2291 	if (id_priv->state != RDMA_CM_CONNECT)
2292 		goto out;
2293 
2294 	switch (iw_event->event) {
2295 	case IW_CM_EVENT_CLOSE:
2296 		event.event = RDMA_CM_EVENT_DISCONNECTED;
2297 		break;
2298 	case IW_CM_EVENT_CONNECT_REPLY:
2299 		memcpy(cma_src_addr(id_priv), laddr,
2300 		       rdma_addr_size(laddr));
2301 		memcpy(cma_dst_addr(id_priv), raddr,
2302 		       rdma_addr_size(raddr));
2303 		switch (iw_event->status) {
2304 		case 0:
2305 			event.event = RDMA_CM_EVENT_ESTABLISHED;
2306 			event.param.conn.initiator_depth = iw_event->ird;
2307 			event.param.conn.responder_resources = iw_event->ord;
2308 			break;
2309 		case -ECONNRESET:
2310 		case -ECONNREFUSED:
2311 			event.event = RDMA_CM_EVENT_REJECTED;
2312 			break;
2313 		case -ETIMEDOUT:
2314 			event.event = RDMA_CM_EVENT_UNREACHABLE;
2315 			break;
2316 		default:
2317 			event.event = RDMA_CM_EVENT_CONNECT_ERROR;
2318 			break;
2319 		}
2320 		break;
2321 	case IW_CM_EVENT_ESTABLISHED:
2322 		event.event = RDMA_CM_EVENT_ESTABLISHED;
2323 		event.param.conn.initiator_depth = iw_event->ird;
2324 		event.param.conn.responder_resources = iw_event->ord;
2325 		break;
2326 	default:
2327 		goto out;
2328 	}
2329 
2330 	event.status = iw_event->status;
2331 	event.param.conn.private_data = iw_event->private_data;
2332 	event.param.conn.private_data_len = iw_event->private_data_len;
2333 	ret = cma_cm_event_handler(id_priv, &event);
2334 	if (ret) {
2335 		/* Destroy the CM ID by returning a non-zero value. */
2336 		id_priv->cm_id.iw = NULL;
2337 		destroy_id_handler_unlock(id_priv);
2338 		return ret;
2339 	}
2340 
2341 out:
2342 	mutex_unlock(&id_priv->handler_mutex);
2343 	return ret;
2344 }
2345 
2346 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
2347 			       struct iw_cm_event *iw_event)
2348 {
2349 	struct rdma_cm_id *new_cm_id;
2350 	struct rdma_id_private *listen_id, *conn_id;
2351 	struct rdma_cm_event event = {};
2352 	int ret = -ECONNABORTED;
2353 	struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2354 	struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2355 
2356 	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2357 	event.param.conn.private_data = iw_event->private_data;
2358 	event.param.conn.private_data_len = iw_event->private_data_len;
2359 	event.param.conn.initiator_depth = iw_event->ird;
2360 	event.param.conn.responder_resources = iw_event->ord;
2361 
2362 	listen_id = cm_id->context;
2363 
2364 	mutex_lock(&listen_id->handler_mutex);
2365 	if (listen_id->state != RDMA_CM_LISTEN)
2366 		goto out;
2367 
2368 	/* Create a new RDMA id for the new IW CM ID */
2369 	new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
2370 				     listen_id->id.event_handler,
2371 				     listen_id->id.context,
2372 				     RDMA_PS_TCP, IB_QPT_RC,
2373 				     listen_id->res.kern_name);
2374 	if (IS_ERR(new_cm_id)) {
2375 		ret = -ENOMEM;
2376 		goto out;
2377 	}
2378 	conn_id = container_of(new_cm_id, struct rdma_id_private, id);
2379 	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2380 	conn_id->state = RDMA_CM_CONNECT;
2381 
2382 	ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2383 	if (ret) {
2384 		mutex_unlock(&listen_id->handler_mutex);
2385 		destroy_id_handler_unlock(conn_id);
2386 		return ret;
2387 	}
2388 
2389 	ret = cma_iw_acquire_dev(conn_id, listen_id);
2390 	if (ret) {
2391 		mutex_unlock(&listen_id->handler_mutex);
2392 		destroy_id_handler_unlock(conn_id);
2393 		return ret;
2394 	}
2395 
2396 	conn_id->cm_id.iw = cm_id;
2397 	cm_id->context = conn_id;
2398 	cm_id->cm_handler = cma_iw_handler;
2399 
2400 	memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2401 	memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2402 
2403 	ret = cma_cm_event_handler(conn_id, &event);
2404 	if (ret) {
2405 		/* User wants to destroy the CM ID */
2406 		conn_id->cm_id.iw = NULL;
2407 		mutex_unlock(&listen_id->handler_mutex);
2408 		destroy_id_handler_unlock(conn_id);
2409 		return ret;
2410 	}
2411 
2412 	mutex_unlock(&conn_id->handler_mutex);
2413 
2414 out:
2415 	mutex_unlock(&listen_id->handler_mutex);
2416 	return ret;
2417 }
2418 
2419 static int cma_ib_listen(struct rdma_id_private *id_priv)
2420 {
2421 	struct sockaddr *addr;
2422 	struct ib_cm_id	*id;
2423 	__be64 svc_id;
2424 
2425 	addr = cma_src_addr(id_priv);
2426 	svc_id = rdma_get_service_id(&id_priv->id, addr);
2427 	id = ib_cm_insert_listen(id_priv->id.device,
2428 				 cma_ib_req_handler, svc_id);
2429 	if (IS_ERR(id))
2430 		return PTR_ERR(id);
2431 	id_priv->cm_id.ib = id;
2432 
2433 	return 0;
2434 }
2435 
2436 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
2437 {
2438 	int ret;
2439 	struct iw_cm_id	*id;
2440 
2441 	id = iw_create_cm_id(id_priv->id.device,
2442 			     iw_conn_req_handler,
2443 			     id_priv);
2444 	if (IS_ERR(id))
2445 		return PTR_ERR(id);
2446 
2447 	id->tos = id_priv->tos;
2448 	id->tos_set = id_priv->tos_set;
2449 	id_priv->cm_id.iw = id;
2450 
2451 	memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
2452 	       rdma_addr_size(cma_src_addr(id_priv)));
2453 
2454 	ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
2455 
2456 	if (ret) {
2457 		iw_destroy_cm_id(id_priv->cm_id.iw);
2458 		id_priv->cm_id.iw = NULL;
2459 	}
2460 
2461 	return ret;
2462 }
2463 
2464 static int cma_listen_handler(struct rdma_cm_id *id,
2465 			      struct rdma_cm_event *event)
2466 {
2467 	struct rdma_id_private *id_priv = id->context;
2468 
2469 	/* Listening IDs are always destroyed on removal */
2470 	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
2471 		return -1;
2472 
2473 	id->context = id_priv->id.context;
2474 	id->event_handler = id_priv->id.event_handler;
2475 	trace_cm_event_handler(id_priv, event);
2476 	return id_priv->id.event_handler(id, event);
2477 }
2478 
2479 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
2480 			      struct cma_device *cma_dev)
2481 {
2482 	struct rdma_id_private *dev_id_priv;
2483 	struct rdma_cm_id *id;
2484 	struct net *net = id_priv->id.route.addr.dev_addr.net;
2485 	int ret;
2486 
2487 	lockdep_assert_held(&lock);
2488 
2489 	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2490 		return;
2491 
2492 	id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps,
2493 			      id_priv->id.qp_type, id_priv->res.kern_name);
2494 	if (IS_ERR(id))
2495 		return;
2496 
2497 	dev_id_priv = container_of(id, struct rdma_id_private, id);
2498 
2499 	dev_id_priv->state = RDMA_CM_ADDR_BOUND;
2500 	memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
2501 	       rdma_addr_size(cma_src_addr(id_priv)));
2502 
2503 	_cma_attach_to_dev(dev_id_priv, cma_dev);
2504 	list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
2505 	cma_id_get(id_priv);
2506 	dev_id_priv->internal_id = 1;
2507 	dev_id_priv->afonly = id_priv->afonly;
2508 	dev_id_priv->tos_set = id_priv->tos_set;
2509 	dev_id_priv->tos = id_priv->tos;
2510 
2511 	ret = rdma_listen(id, id_priv->backlog);
2512 	if (ret)
2513 		dev_warn(&cma_dev->device->dev,
2514 			 "RDMA CMA: cma_listen_on_dev, error %d\n", ret);
2515 }
2516 
2517 static void cma_listen_on_all(struct rdma_id_private *id_priv)
2518 {
2519 	struct cma_device *cma_dev;
2520 
2521 	mutex_lock(&lock);
2522 	list_add_tail(&id_priv->list, &listen_any_list);
2523 	list_for_each_entry(cma_dev, &dev_list, list)
2524 		cma_listen_on_dev(id_priv, cma_dev);
2525 	mutex_unlock(&lock);
2526 }
2527 
2528 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
2529 {
2530 	struct rdma_id_private *id_priv;
2531 
2532 	id_priv = container_of(id, struct rdma_id_private, id);
2533 	id_priv->tos = (u8) tos;
2534 	id_priv->tos_set = true;
2535 }
2536 EXPORT_SYMBOL(rdma_set_service_type);
2537 
2538 /**
2539  * rdma_set_ack_timeout() - Set the ack timeout of QP associated
2540  *                          with a connection identifier.
2541  * @id: Communication identifier to associated with service type.
2542  * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec.
2543  *
2544  * This function should be called before rdma_connect() on active side,
2545  * and on passive side before rdma_accept(). It is applicable to primary
2546  * path only. The timeout will affect the local side of the QP, it is not
2547  * negotiated with remote side and zero disables the timer. In case it is
2548  * set before rdma_resolve_route, the value will also be used to determine
2549  * PacketLifeTime for RoCE.
2550  *
2551  * Return: 0 for success
2552  */
2553 int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
2554 {
2555 	struct rdma_id_private *id_priv;
2556 
2557 	if (id->qp_type != IB_QPT_RC)
2558 		return -EINVAL;
2559 
2560 	id_priv = container_of(id, struct rdma_id_private, id);
2561 	id_priv->timeout = timeout;
2562 	id_priv->timeout_set = true;
2563 
2564 	return 0;
2565 }
2566 EXPORT_SYMBOL(rdma_set_ack_timeout);
2567 
2568 static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2569 			      void *context)
2570 {
2571 	struct cma_work *work = context;
2572 	struct rdma_route *route;
2573 
2574 	route = &work->id->id.route;
2575 
2576 	if (!status) {
2577 		route->num_paths = 1;
2578 		*route->path_rec = *path_rec;
2579 	} else {
2580 		work->old_state = RDMA_CM_ROUTE_QUERY;
2581 		work->new_state = RDMA_CM_ADDR_RESOLVED;
2582 		work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
2583 		work->event.status = status;
2584 		pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",
2585 				     status);
2586 	}
2587 
2588 	queue_work(cma_wq, &work->work);
2589 }
2590 
2591 static int cma_query_ib_route(struct rdma_id_private *id_priv,
2592 			      unsigned long timeout_ms, struct cma_work *work)
2593 {
2594 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2595 	struct sa_path_rec path_rec;
2596 	ib_sa_comp_mask comp_mask;
2597 	struct sockaddr_in6 *sin6;
2598 	struct sockaddr_ib *sib;
2599 
2600 	memset(&path_rec, 0, sizeof path_rec);
2601 
2602 	if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num))
2603 		path_rec.rec_type = SA_PATH_REC_TYPE_OPA;
2604 	else
2605 		path_rec.rec_type = SA_PATH_REC_TYPE_IB;
2606 	rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
2607 	rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
2608 	path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2609 	path_rec.numb_path = 1;
2610 	path_rec.reversible = 1;
2611 	path_rec.service_id = rdma_get_service_id(&id_priv->id,
2612 						  cma_dst_addr(id_priv));
2613 
2614 	comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
2615 		    IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
2616 		    IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
2617 
2618 	switch (cma_family(id_priv)) {
2619 	case AF_INET:
2620 		path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
2621 		comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
2622 		break;
2623 	case AF_INET6:
2624 		sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2625 		path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
2626 		comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2627 		break;
2628 	case AF_IB:
2629 		sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2630 		path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20);
2631 		comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2632 		break;
2633 	}
2634 
2635 	id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
2636 					       id_priv->id.port_num, &path_rec,
2637 					       comp_mask, timeout_ms,
2638 					       GFP_KERNEL, cma_query_handler,
2639 					       work, &id_priv->query);
2640 
2641 	return (id_priv->query_id < 0) ? id_priv->query_id : 0;
2642 }
2643 
2644 static void cma_work_handler(struct work_struct *_work)
2645 {
2646 	struct cma_work *work = container_of(_work, struct cma_work, work);
2647 	struct rdma_id_private *id_priv = work->id;
2648 
2649 	mutex_lock(&id_priv->handler_mutex);
2650 	if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
2651 		goto out_unlock;
2652 
2653 	if (cma_cm_event_handler(id_priv, &work->event)) {
2654 		cma_id_put(id_priv);
2655 		destroy_id_handler_unlock(id_priv);
2656 		goto out_free;
2657 	}
2658 
2659 out_unlock:
2660 	mutex_unlock(&id_priv->handler_mutex);
2661 	cma_id_put(id_priv);
2662 out_free:
2663 	kfree(work);
2664 }
2665 
2666 static void cma_ndev_work_handler(struct work_struct *_work)
2667 {
2668 	struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
2669 	struct rdma_id_private *id_priv = work->id;
2670 
2671 	mutex_lock(&id_priv->handler_mutex);
2672 	if (id_priv->state == RDMA_CM_DESTROYING ||
2673 	    id_priv->state == RDMA_CM_DEVICE_REMOVAL)
2674 		goto out_unlock;
2675 
2676 	if (cma_cm_event_handler(id_priv, &work->event)) {
2677 		cma_id_put(id_priv);
2678 		destroy_id_handler_unlock(id_priv);
2679 		goto out_free;
2680 	}
2681 
2682 out_unlock:
2683 	mutex_unlock(&id_priv->handler_mutex);
2684 	cma_id_put(id_priv);
2685 out_free:
2686 	kfree(work);
2687 }
2688 
2689 static void cma_init_resolve_route_work(struct cma_work *work,
2690 					struct rdma_id_private *id_priv)
2691 {
2692 	work->id = id_priv;
2693 	INIT_WORK(&work->work, cma_work_handler);
2694 	work->old_state = RDMA_CM_ROUTE_QUERY;
2695 	work->new_state = RDMA_CM_ROUTE_RESOLVED;
2696 	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
2697 }
2698 
2699 static void enqueue_resolve_addr_work(struct cma_work *work,
2700 				      struct rdma_id_private *id_priv)
2701 {
2702 	/* Balances with cma_id_put() in cma_work_handler */
2703 	cma_id_get(id_priv);
2704 
2705 	work->id = id_priv;
2706 	INIT_WORK(&work->work, cma_work_handler);
2707 	work->old_state = RDMA_CM_ADDR_QUERY;
2708 	work->new_state = RDMA_CM_ADDR_RESOLVED;
2709 	work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2710 
2711 	queue_work(cma_wq, &work->work);
2712 }
2713 
2714 static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
2715 				unsigned long timeout_ms)
2716 {
2717 	struct rdma_route *route = &id_priv->id.route;
2718 	struct cma_work *work;
2719 	int ret;
2720 
2721 	work = kzalloc(sizeof *work, GFP_KERNEL);
2722 	if (!work)
2723 		return -ENOMEM;
2724 
2725 	cma_init_resolve_route_work(work, id_priv);
2726 
2727 	route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
2728 	if (!route->path_rec) {
2729 		ret = -ENOMEM;
2730 		goto err1;
2731 	}
2732 
2733 	ret = cma_query_ib_route(id_priv, timeout_ms, work);
2734 	if (ret)
2735 		goto err2;
2736 
2737 	return 0;
2738 err2:
2739 	kfree(route->path_rec);
2740 	route->path_rec = NULL;
2741 err1:
2742 	kfree(work);
2743 	return ret;
2744 }
2745 
2746 static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
2747 					   unsigned long supported_gids,
2748 					   enum ib_gid_type default_gid)
2749 {
2750 	if ((network_type == RDMA_NETWORK_IPV4 ||
2751 	     network_type == RDMA_NETWORK_IPV6) &&
2752 	    test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
2753 		return IB_GID_TYPE_ROCE_UDP_ENCAP;
2754 
2755 	return default_gid;
2756 }
2757 
2758 /*
2759  * cma_iboe_set_path_rec_l2_fields() is helper function which sets
2760  * path record type based on GID type.
2761  * It also sets up other L2 fields which includes destination mac address
2762  * netdev ifindex, of the path record.
2763  * It returns the netdev of the bound interface for this path record entry.
2764  */
2765 static struct net_device *
2766 cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
2767 {
2768 	struct rdma_route *route = &id_priv->id.route;
2769 	enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
2770 	struct rdma_addr *addr = &route->addr;
2771 	unsigned long supported_gids;
2772 	struct net_device *ndev;
2773 
2774 	if (!addr->dev_addr.bound_dev_if)
2775 		return NULL;
2776 
2777 	ndev = dev_get_by_index(addr->dev_addr.net,
2778 				addr->dev_addr.bound_dev_if);
2779 	if (!ndev)
2780 		return NULL;
2781 
2782 	supported_gids = roce_gid_type_mask_support(id_priv->id.device,
2783 						    id_priv->id.port_num);
2784 	gid_type = cma_route_gid_type(addr->dev_addr.network,
2785 				      supported_gids,
2786 				      id_priv->gid_type);
2787 	/* Use the hint from IP Stack to select GID Type */
2788 	if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
2789 		gid_type = ib_network_to_gid_type(addr->dev_addr.network);
2790 	route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
2791 
2792 	route->path_rec->roce.route_resolved = true;
2793 	sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
2794 	return ndev;
2795 }
2796 
2797 int rdma_set_ib_path(struct rdma_cm_id *id,
2798 		     struct sa_path_rec *path_rec)
2799 {
2800 	struct rdma_id_private *id_priv;
2801 	struct net_device *ndev;
2802 	int ret;
2803 
2804 	id_priv = container_of(id, struct rdma_id_private, id);
2805 	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2806 			   RDMA_CM_ROUTE_RESOLVED))
2807 		return -EINVAL;
2808 
2809 	id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
2810 				     GFP_KERNEL);
2811 	if (!id->route.path_rec) {
2812 		ret = -ENOMEM;
2813 		goto err;
2814 	}
2815 
2816 	if (rdma_protocol_roce(id->device, id->port_num)) {
2817 		ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
2818 		if (!ndev) {
2819 			ret = -ENODEV;
2820 			goto err_free;
2821 		}
2822 		dev_put(ndev);
2823 	}
2824 
2825 	id->route.num_paths = 1;
2826 	return 0;
2827 
2828 err_free:
2829 	kfree(id->route.path_rec);
2830 	id->route.path_rec = NULL;
2831 err:
2832 	cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
2833 	return ret;
2834 }
2835 EXPORT_SYMBOL(rdma_set_ib_path);
2836 
2837 static int cma_resolve_iw_route(struct rdma_id_private *id_priv)
2838 {
2839 	struct cma_work *work;
2840 
2841 	work = kzalloc(sizeof *work, GFP_KERNEL);
2842 	if (!work)
2843 		return -ENOMEM;
2844 
2845 	cma_init_resolve_route_work(work, id_priv);
2846 	queue_work(cma_wq, &work->work);
2847 	return 0;
2848 }
2849 
2850 static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)
2851 {
2852 	struct net_device *dev;
2853 
2854 	dev = vlan_dev_real_dev(vlan_ndev);
2855 	if (dev->num_tc)
2856 		return netdev_get_prio_tc_map(dev, prio);
2857 
2858 	return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &
2859 		VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
2860 }
2861 
2862 struct iboe_prio_tc_map {
2863 	int input_prio;
2864 	int output_tc;
2865 	bool found;
2866 };
2867 
2868 static int get_lower_vlan_dev_tc(struct net_device *dev, void *data)
2869 {
2870 	struct iboe_prio_tc_map *map = data;
2871 
2872 	if (is_vlan_dev(dev))
2873 		map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
2874 	else if (dev->num_tc)
2875 		map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio);
2876 	else
2877 		map->output_tc = 0;
2878 	/* We are interested only in first level VLAN device, so always
2879 	 * return 1 to stop iterating over next level devices.
2880 	 */
2881 	map->found = true;
2882 	return 1;
2883 }
2884 
2885 static int iboe_tos_to_sl(struct net_device *ndev, int tos)
2886 {
2887 	struct iboe_prio_tc_map prio_tc_map = {};
2888 	int prio = rt_tos2priority(tos);
2889 
2890 	/* If VLAN device, get it directly from the VLAN netdev */
2891 	if (is_vlan_dev(ndev))
2892 		return get_vlan_ndev_tc(ndev, prio);
2893 
2894 	prio_tc_map.input_prio = prio;
2895 	rcu_read_lock();
2896 	netdev_walk_all_lower_dev_rcu(ndev,
2897 				      get_lower_vlan_dev_tc,
2898 				      &prio_tc_map);
2899 	rcu_read_unlock();
2900 	/* If map is found from lower device, use it; Otherwise
2901 	 * continue with the current netdevice to get priority to tc map.
2902 	 */
2903 	if (prio_tc_map.found)
2904 		return prio_tc_map.output_tc;
2905 	else if (ndev->num_tc)
2906 		return netdev_get_prio_tc_map(ndev, prio);
2907 	else
2908 		return 0;
2909 }
2910 
2911 static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)
2912 {
2913 	struct sockaddr_in6 *addr6;
2914 	u16 dport, sport;
2915 	u32 hash, fl;
2916 
2917 	addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);
2918 	fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK;
2919 	if ((cma_family(id_priv) != AF_INET6) || !fl) {
2920 		dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));
2921 		sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));
2922 		hash = (u32)sport * 31 + dport;
2923 		fl = hash & IB_GRH_FLOWLABEL_MASK;
2924 	}
2925 
2926 	return cpu_to_be32(fl);
2927 }
2928 
2929 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
2930 {
2931 	struct rdma_route *route = &id_priv->id.route;
2932 	struct rdma_addr *addr = &route->addr;
2933 	struct cma_work *work;
2934 	int ret;
2935 	struct net_device *ndev;
2936 
2937 	u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
2938 					rdma_start_port(id_priv->cma_dev->device)];
2939 	u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
2940 
2941 
2942 	work = kzalloc(sizeof *work, GFP_KERNEL);
2943 	if (!work)
2944 		return -ENOMEM;
2945 
2946 	route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
2947 	if (!route->path_rec) {
2948 		ret = -ENOMEM;
2949 		goto err1;
2950 	}
2951 
2952 	route->num_paths = 1;
2953 
2954 	ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
2955 	if (!ndev) {
2956 		ret = -ENODEV;
2957 		goto err2;
2958 	}
2959 
2960 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
2961 		    &route->path_rec->sgid);
2962 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
2963 		    &route->path_rec->dgid);
2964 
2965 	if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
2966 		/* TODO: get the hoplimit from the inet/inet6 device */
2967 		route->path_rec->hop_limit = addr->dev_addr.hoplimit;
2968 	else
2969 		route->path_rec->hop_limit = 1;
2970 	route->path_rec->reversible = 1;
2971 	route->path_rec->pkey = cpu_to_be16(0xffff);
2972 	route->path_rec->mtu_selector = IB_SA_EQ;
2973 	route->path_rec->sl = iboe_tos_to_sl(ndev, tos);
2974 	route->path_rec->traffic_class = tos;
2975 	route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
2976 	route->path_rec->rate_selector = IB_SA_EQ;
2977 	route->path_rec->rate = iboe_get_rate(ndev);
2978 	dev_put(ndev);
2979 	route->path_rec->packet_life_time_selector = IB_SA_EQ;
2980 	/* In case ACK timeout is set, use this value to calculate
2981 	 * PacketLifeTime.  As per IBTA 12.7.34,
2982 	 * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).
2983 	 * Assuming a negligible local ACK delay, we can use
2984 	 * PacketLifeTime = local ACK timeout/2
2985 	 * as a reasonable approximation for RoCE networks.
2986 	 */
2987 	route->path_rec->packet_life_time = id_priv->timeout_set ?
2988 		id_priv->timeout - 1 : CMA_IBOE_PACKET_LIFETIME;
2989 
2990 	if (!route->path_rec->mtu) {
2991 		ret = -EINVAL;
2992 		goto err2;
2993 	}
2994 
2995 	if (rdma_protocol_roce_udp_encap(id_priv->id.device,
2996 					 id_priv->id.port_num))
2997 		route->path_rec->flow_label =
2998 			cma_get_roce_udp_flow_label(id_priv);
2999 
3000 	cma_init_resolve_route_work(work, id_priv);
3001 	queue_work(cma_wq, &work->work);
3002 
3003 	return 0;
3004 
3005 err2:
3006 	kfree(route->path_rec);
3007 	route->path_rec = NULL;
3008 	route->num_paths = 0;
3009 err1:
3010 	kfree(work);
3011 	return ret;
3012 }
3013 
3014 int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
3015 {
3016 	struct rdma_id_private *id_priv;
3017 	int ret;
3018 
3019 	id_priv = container_of(id, struct rdma_id_private, id);
3020 	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
3021 		return -EINVAL;
3022 
3023 	cma_id_get(id_priv);
3024 	if (rdma_cap_ib_sa(id->device, id->port_num))
3025 		ret = cma_resolve_ib_route(id_priv, timeout_ms);
3026 	else if (rdma_protocol_roce(id->device, id->port_num))
3027 		ret = cma_resolve_iboe_route(id_priv);
3028 	else if (rdma_protocol_iwarp(id->device, id->port_num))
3029 		ret = cma_resolve_iw_route(id_priv);
3030 	else
3031 		ret = -ENOSYS;
3032 
3033 	if (ret)
3034 		goto err;
3035 
3036 	return 0;
3037 err:
3038 	cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
3039 	cma_id_put(id_priv);
3040 	return ret;
3041 }
3042 EXPORT_SYMBOL(rdma_resolve_route);
3043 
3044 static void cma_set_loopback(struct sockaddr *addr)
3045 {
3046 	switch (addr->sa_family) {
3047 	case AF_INET:
3048 		((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3049 		break;
3050 	case AF_INET6:
3051 		ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
3052 			      0, 0, 0, htonl(1));
3053 		break;
3054 	default:
3055 		ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
3056 			    0, 0, 0, htonl(1));
3057 		break;
3058 	}
3059 }
3060 
3061 static int cma_bind_loopback(struct rdma_id_private *id_priv)
3062 {
3063 	struct cma_device *cma_dev, *cur_dev;
3064 	union ib_gid gid;
3065 	enum ib_port_state port_state;
3066 	unsigned int p;
3067 	u16 pkey;
3068 	int ret;
3069 
3070 	cma_dev = NULL;
3071 	mutex_lock(&lock);
3072 	list_for_each_entry(cur_dev, &dev_list, list) {
3073 		if (cma_family(id_priv) == AF_IB &&
3074 		    !rdma_cap_ib_cm(cur_dev->device, 1))
3075 			continue;
3076 
3077 		if (!cma_dev)
3078 			cma_dev = cur_dev;
3079 
3080 		rdma_for_each_port (cur_dev->device, p) {
3081 			if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
3082 			    port_state == IB_PORT_ACTIVE) {
3083 				cma_dev = cur_dev;
3084 				goto port_found;
3085 			}
3086 		}
3087 	}
3088 
3089 	if (!cma_dev) {
3090 		ret = -ENODEV;
3091 		goto out;
3092 	}
3093 
3094 	p = 1;
3095 
3096 port_found:
3097 	ret = rdma_query_gid(cma_dev->device, p, 0, &gid);
3098 	if (ret)
3099 		goto out;
3100 
3101 	ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
3102 	if (ret)
3103 		goto out;
3104 
3105 	id_priv->id.route.addr.dev_addr.dev_type =
3106 		(rdma_protocol_ib(cma_dev->device, p)) ?
3107 		ARPHRD_INFINIBAND : ARPHRD_ETHER;
3108 
3109 	rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3110 	ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
3111 	id_priv->id.port_num = p;
3112 	cma_attach_to_dev(id_priv, cma_dev);
3113 	cma_set_loopback(cma_src_addr(id_priv));
3114 out:
3115 	mutex_unlock(&lock);
3116 	return ret;
3117 }
3118 
3119 static void addr_handler(int status, struct sockaddr *src_addr,
3120 			 struct rdma_dev_addr *dev_addr, void *context)
3121 {
3122 	struct rdma_id_private *id_priv = context;
3123 	struct rdma_cm_event event = {};
3124 	struct sockaddr *addr;
3125 	struct sockaddr_storage old_addr;
3126 
3127 	mutex_lock(&id_priv->handler_mutex);
3128 	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
3129 			   RDMA_CM_ADDR_RESOLVED))
3130 		goto out;
3131 
3132 	/*
3133 	 * Store the previous src address, so that if we fail to acquire
3134 	 * matching rdma device, old address can be restored back, which helps
3135 	 * to cancel the cma listen operation correctly.
3136 	 */
3137 	addr = cma_src_addr(id_priv);
3138 	memcpy(&old_addr, addr, rdma_addr_size(addr));
3139 	memcpy(addr, src_addr, rdma_addr_size(src_addr));
3140 	if (!status && !id_priv->cma_dev) {
3141 		status = cma_acquire_dev_by_src_ip(id_priv);
3142 		if (status)
3143 			pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",
3144 					     status);
3145 	} else if (status) {
3146 		pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);
3147 	}
3148 
3149 	if (status) {
3150 		memcpy(addr, &old_addr,
3151 		       rdma_addr_size((struct sockaddr *)&old_addr));
3152 		if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3153 				   RDMA_CM_ADDR_BOUND))
3154 			goto out;
3155 		event.event = RDMA_CM_EVENT_ADDR_ERROR;
3156 		event.status = status;
3157 	} else
3158 		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3159 
3160 	if (cma_cm_event_handler(id_priv, &event)) {
3161 		destroy_id_handler_unlock(id_priv);
3162 		return;
3163 	}
3164 out:
3165 	mutex_unlock(&id_priv->handler_mutex);
3166 }
3167 
3168 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
3169 {
3170 	struct cma_work *work;
3171 	union ib_gid gid;
3172 	int ret;
3173 
3174 	work = kzalloc(sizeof *work, GFP_KERNEL);
3175 	if (!work)
3176 		return -ENOMEM;
3177 
3178 	if (!id_priv->cma_dev) {
3179 		ret = cma_bind_loopback(id_priv);
3180 		if (ret)
3181 			goto err;
3182 	}
3183 
3184 	rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3185 	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
3186 
3187 	enqueue_resolve_addr_work(work, id_priv);
3188 	return 0;
3189 err:
3190 	kfree(work);
3191 	return ret;
3192 }
3193 
3194 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
3195 {
3196 	struct cma_work *work;
3197 	int ret;
3198 
3199 	work = kzalloc(sizeof *work, GFP_KERNEL);
3200 	if (!work)
3201 		return -ENOMEM;
3202 
3203 	if (!id_priv->cma_dev) {
3204 		ret = cma_resolve_ib_dev(id_priv);
3205 		if (ret)
3206 			goto err;
3207 	}
3208 
3209 	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
3210 		&(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
3211 
3212 	enqueue_resolve_addr_work(work, id_priv);
3213 	return 0;
3214 err:
3215 	kfree(work);
3216 	return ret;
3217 }
3218 
3219 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3220 			 const struct sockaddr *dst_addr)
3221 {
3222 	if (!src_addr || !src_addr->sa_family) {
3223 		src_addr = (struct sockaddr *) &id->route.addr.src_addr;
3224 		src_addr->sa_family = dst_addr->sa_family;
3225 		if (IS_ENABLED(CONFIG_IPV6) &&
3226 		    dst_addr->sa_family == AF_INET6) {
3227 			struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr;
3228 			struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr;
3229 			src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
3230 			if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
3231 				id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id;
3232 		} else if (dst_addr->sa_family == AF_IB) {
3233 			((struct sockaddr_ib *) src_addr)->sib_pkey =
3234 				((struct sockaddr_ib *) dst_addr)->sib_pkey;
3235 		}
3236 	}
3237 	return rdma_bind_addr(id, src_addr);
3238 }
3239 
3240 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3241 		      const struct sockaddr *dst_addr, unsigned long timeout_ms)
3242 {
3243 	struct rdma_id_private *id_priv;
3244 	int ret;
3245 
3246 	id_priv = container_of(id, struct rdma_id_private, id);
3247 	memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
3248 	if (id_priv->state == RDMA_CM_IDLE) {
3249 		ret = cma_bind_addr(id, src_addr, dst_addr);
3250 		if (ret) {
3251 			memset(cma_dst_addr(id_priv), 0,
3252 			       rdma_addr_size(dst_addr));
3253 			return ret;
3254 		}
3255 	}
3256 
3257 	if (cma_family(id_priv) != dst_addr->sa_family) {
3258 		memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3259 		return -EINVAL;
3260 	}
3261 
3262 	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
3263 		memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3264 		return -EINVAL;
3265 	}
3266 
3267 	if (cma_any_addr(dst_addr)) {
3268 		ret = cma_resolve_loopback(id_priv);
3269 	} else {
3270 		if (dst_addr->sa_family == AF_IB) {
3271 			ret = cma_resolve_ib_addr(id_priv);
3272 		} else {
3273 			ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
3274 					      &id->route.addr.dev_addr,
3275 					      timeout_ms, addr_handler,
3276 					      false, id_priv);
3277 		}
3278 	}
3279 	if (ret)
3280 		goto err;
3281 
3282 	return 0;
3283 err:
3284 	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3285 	return ret;
3286 }
3287 EXPORT_SYMBOL(rdma_resolve_addr);
3288 
3289 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3290 {
3291 	struct rdma_id_private *id_priv;
3292 	unsigned long flags;
3293 	int ret;
3294 
3295 	id_priv = container_of(id, struct rdma_id_private, id);
3296 	spin_lock_irqsave(&id_priv->lock, flags);
3297 	if (reuse || id_priv->state == RDMA_CM_IDLE) {
3298 		id_priv->reuseaddr = reuse;
3299 		ret = 0;
3300 	} else {
3301 		ret = -EINVAL;
3302 	}
3303 	spin_unlock_irqrestore(&id_priv->lock, flags);
3304 	return ret;
3305 }
3306 EXPORT_SYMBOL(rdma_set_reuseaddr);
3307 
3308 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
3309 {
3310 	struct rdma_id_private *id_priv;
3311 	unsigned long flags;
3312 	int ret;
3313 
3314 	id_priv = container_of(id, struct rdma_id_private, id);
3315 	spin_lock_irqsave(&id_priv->lock, flags);
3316 	if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
3317 		id_priv->options |= (1 << CMA_OPTION_AFONLY);
3318 		id_priv->afonly = afonly;
3319 		ret = 0;
3320 	} else {
3321 		ret = -EINVAL;
3322 	}
3323 	spin_unlock_irqrestore(&id_priv->lock, flags);
3324 	return ret;
3325 }
3326 EXPORT_SYMBOL(rdma_set_afonly);
3327 
3328 static void cma_bind_port(struct rdma_bind_list *bind_list,
3329 			  struct rdma_id_private *id_priv)
3330 {
3331 	struct sockaddr *addr;
3332 	struct sockaddr_ib *sib;
3333 	u64 sid, mask;
3334 	__be16 port;
3335 
3336 	lockdep_assert_held(&lock);
3337 
3338 	addr = cma_src_addr(id_priv);
3339 	port = htons(bind_list->port);
3340 
3341 	switch (addr->sa_family) {
3342 	case AF_INET:
3343 		((struct sockaddr_in *) addr)->sin_port = port;
3344 		break;
3345 	case AF_INET6:
3346 		((struct sockaddr_in6 *) addr)->sin6_port = port;
3347 		break;
3348 	case AF_IB:
3349 		sib = (struct sockaddr_ib *) addr;
3350 		sid = be64_to_cpu(sib->sib_sid);
3351 		mask = be64_to_cpu(sib->sib_sid_mask);
3352 		sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
3353 		sib->sib_sid_mask = cpu_to_be64(~0ULL);
3354 		break;
3355 	}
3356 	id_priv->bind_list = bind_list;
3357 	hlist_add_head(&id_priv->node, &bind_list->owners);
3358 }
3359 
3360 static int cma_alloc_port(enum rdma_ucm_port_space ps,
3361 			  struct rdma_id_private *id_priv, unsigned short snum)
3362 {
3363 	struct rdma_bind_list *bind_list;
3364 	int ret;
3365 
3366 	lockdep_assert_held(&lock);
3367 
3368 	bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
3369 	if (!bind_list)
3370 		return -ENOMEM;
3371 
3372 	ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list,
3373 			   snum);
3374 	if (ret < 0)
3375 		goto err;
3376 
3377 	bind_list->ps = ps;
3378 	bind_list->port = snum;
3379 	cma_bind_port(bind_list, id_priv);
3380 	return 0;
3381 err:
3382 	kfree(bind_list);
3383 	return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
3384 }
3385 
3386 static int cma_port_is_unique(struct rdma_bind_list *bind_list,
3387 			      struct rdma_id_private *id_priv)
3388 {
3389 	struct rdma_id_private *cur_id;
3390 	struct sockaddr  *daddr = cma_dst_addr(id_priv);
3391 	struct sockaddr  *saddr = cma_src_addr(id_priv);
3392 	__be16 dport = cma_port(daddr);
3393 
3394 	lockdep_assert_held(&lock);
3395 
3396 	hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3397 		struct sockaddr  *cur_daddr = cma_dst_addr(cur_id);
3398 		struct sockaddr  *cur_saddr = cma_src_addr(cur_id);
3399 		__be16 cur_dport = cma_port(cur_daddr);
3400 
3401 		if (id_priv == cur_id)
3402 			continue;
3403 
3404 		/* different dest port -> unique */
3405 		if (!cma_any_port(daddr) &&
3406 		    !cma_any_port(cur_daddr) &&
3407 		    (dport != cur_dport))
3408 			continue;
3409 
3410 		/* different src address -> unique */
3411 		if (!cma_any_addr(saddr) &&
3412 		    !cma_any_addr(cur_saddr) &&
3413 		    cma_addr_cmp(saddr, cur_saddr))
3414 			continue;
3415 
3416 		/* different dst address -> unique */
3417 		if (!cma_any_addr(daddr) &&
3418 		    !cma_any_addr(cur_daddr) &&
3419 		    cma_addr_cmp(daddr, cur_daddr))
3420 			continue;
3421 
3422 		return -EADDRNOTAVAIL;
3423 	}
3424 	return 0;
3425 }
3426 
3427 static int cma_alloc_any_port(enum rdma_ucm_port_space ps,
3428 			      struct rdma_id_private *id_priv)
3429 {
3430 	static unsigned int last_used_port;
3431 	int low, high, remaining;
3432 	unsigned int rover;
3433 	struct net *net = id_priv->id.route.addr.dev_addr.net;
3434 
3435 	lockdep_assert_held(&lock);
3436 
3437 	inet_get_local_port_range(net, &low, &high);
3438 	remaining = (high - low) + 1;
3439 	rover = prandom_u32() % remaining + low;
3440 retry:
3441 	if (last_used_port != rover) {
3442 		struct rdma_bind_list *bind_list;
3443 		int ret;
3444 
3445 		bind_list = cma_ps_find(net, ps, (unsigned short)rover);
3446 
3447 		if (!bind_list) {
3448 			ret = cma_alloc_port(ps, id_priv, rover);
3449 		} else {
3450 			ret = cma_port_is_unique(bind_list, id_priv);
3451 			if (!ret)
3452 				cma_bind_port(bind_list, id_priv);
3453 		}
3454 		/*
3455 		 * Remember previously used port number in order to avoid
3456 		 * re-using same port immediately after it is closed.
3457 		 */
3458 		if (!ret)
3459 			last_used_port = rover;
3460 		if (ret != -EADDRNOTAVAIL)
3461 			return ret;
3462 	}
3463 	if (--remaining) {
3464 		rover++;
3465 		if ((rover < low) || (rover > high))
3466 			rover = low;
3467 		goto retry;
3468 	}
3469 	return -EADDRNOTAVAIL;
3470 }
3471 
3472 /*
3473  * Check that the requested port is available.  This is called when trying to
3474  * bind to a specific port, or when trying to listen on a bound port.  In
3475  * the latter case, the provided id_priv may already be on the bind_list, but
3476  * we still need to check that it's okay to start listening.
3477  */
3478 static int cma_check_port(struct rdma_bind_list *bind_list,
3479 			  struct rdma_id_private *id_priv, uint8_t reuseaddr)
3480 {
3481 	struct rdma_id_private *cur_id;
3482 	struct sockaddr *addr, *cur_addr;
3483 
3484 	lockdep_assert_held(&lock);
3485 
3486 	addr = cma_src_addr(id_priv);
3487 	hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3488 		if (id_priv == cur_id)
3489 			continue;
3490 
3491 		if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
3492 		    cur_id->reuseaddr)
3493 			continue;
3494 
3495 		cur_addr = cma_src_addr(cur_id);
3496 		if (id_priv->afonly && cur_id->afonly &&
3497 		    (addr->sa_family != cur_addr->sa_family))
3498 			continue;
3499 
3500 		if (cma_any_addr(addr) || cma_any_addr(cur_addr))
3501 			return -EADDRNOTAVAIL;
3502 
3503 		if (!cma_addr_cmp(addr, cur_addr))
3504 			return -EADDRINUSE;
3505 	}
3506 	return 0;
3507 }
3508 
3509 static int cma_use_port(enum rdma_ucm_port_space ps,
3510 			struct rdma_id_private *id_priv)
3511 {
3512 	struct rdma_bind_list *bind_list;
3513 	unsigned short snum;
3514 	int ret;
3515 
3516 	lockdep_assert_held(&lock);
3517 
3518 	snum = ntohs(cma_port(cma_src_addr(id_priv)));
3519 	if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
3520 		return -EACCES;
3521 
3522 	bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum);
3523 	if (!bind_list) {
3524 		ret = cma_alloc_port(ps, id_priv, snum);
3525 	} else {
3526 		ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
3527 		if (!ret)
3528 			cma_bind_port(bind_list, id_priv);
3529 	}
3530 	return ret;
3531 }
3532 
3533 static int cma_bind_listen(struct rdma_id_private *id_priv)
3534 {
3535 	struct rdma_bind_list *bind_list = id_priv->bind_list;
3536 	int ret = 0;
3537 
3538 	mutex_lock(&lock);
3539 	if (bind_list->owners.first->next)
3540 		ret = cma_check_port(bind_list, id_priv, 0);
3541 	mutex_unlock(&lock);
3542 	return ret;
3543 }
3544 
3545 static enum rdma_ucm_port_space
3546 cma_select_inet_ps(struct rdma_id_private *id_priv)
3547 {
3548 	switch (id_priv->id.ps) {
3549 	case RDMA_PS_TCP:
3550 	case RDMA_PS_UDP:
3551 	case RDMA_PS_IPOIB:
3552 	case RDMA_PS_IB:
3553 		return id_priv->id.ps;
3554 	default:
3555 
3556 		return 0;
3557 	}
3558 }
3559 
3560 static enum rdma_ucm_port_space
3561 cma_select_ib_ps(struct rdma_id_private *id_priv)
3562 {
3563 	enum rdma_ucm_port_space ps = 0;
3564 	struct sockaddr_ib *sib;
3565 	u64 sid_ps, mask, sid;
3566 
3567 	sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
3568 	mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
3569 	sid = be64_to_cpu(sib->sib_sid) & mask;
3570 
3571 	if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
3572 		sid_ps = RDMA_IB_IP_PS_IB;
3573 		ps = RDMA_PS_IB;
3574 	} else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
3575 		   (sid == (RDMA_IB_IP_PS_TCP & mask))) {
3576 		sid_ps = RDMA_IB_IP_PS_TCP;
3577 		ps = RDMA_PS_TCP;
3578 	} else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
3579 		   (sid == (RDMA_IB_IP_PS_UDP & mask))) {
3580 		sid_ps = RDMA_IB_IP_PS_UDP;
3581 		ps = RDMA_PS_UDP;
3582 	}
3583 
3584 	if (ps) {
3585 		sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
3586 		sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
3587 						be64_to_cpu(sib->sib_sid_mask));
3588 	}
3589 	return ps;
3590 }
3591 
3592 static int cma_get_port(struct rdma_id_private *id_priv)
3593 {
3594 	enum rdma_ucm_port_space ps;
3595 	int ret;
3596 
3597 	if (cma_family(id_priv) != AF_IB)
3598 		ps = cma_select_inet_ps(id_priv);
3599 	else
3600 		ps = cma_select_ib_ps(id_priv);
3601 	if (!ps)
3602 		return -EPROTONOSUPPORT;
3603 
3604 	mutex_lock(&lock);
3605 	if (cma_any_port(cma_src_addr(id_priv)))
3606 		ret = cma_alloc_any_port(ps, id_priv);
3607 	else
3608 		ret = cma_use_port(ps, id_priv);
3609 	mutex_unlock(&lock);
3610 
3611 	return ret;
3612 }
3613 
3614 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
3615 			       struct sockaddr *addr)
3616 {
3617 #if IS_ENABLED(CONFIG_IPV6)
3618 	struct sockaddr_in6 *sin6;
3619 
3620 	if (addr->sa_family != AF_INET6)
3621 		return 0;
3622 
3623 	sin6 = (struct sockaddr_in6 *) addr;
3624 
3625 	if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL))
3626 		return 0;
3627 
3628 	if (!sin6->sin6_scope_id)
3629 			return -EINVAL;
3630 
3631 	dev_addr->bound_dev_if = sin6->sin6_scope_id;
3632 #endif
3633 	return 0;
3634 }
3635 
3636 int rdma_listen(struct rdma_cm_id *id, int backlog)
3637 {
3638 	struct rdma_id_private *id_priv;
3639 	int ret;
3640 
3641 	id_priv = container_of(id, struct rdma_id_private, id);
3642 	if (id_priv->state == RDMA_CM_IDLE) {
3643 		id->route.addr.src_addr.ss_family = AF_INET;
3644 		ret = rdma_bind_addr(id, cma_src_addr(id_priv));
3645 		if (ret)
3646 			return ret;
3647 	}
3648 
3649 	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
3650 		return -EINVAL;
3651 
3652 	if (id_priv->reuseaddr) {
3653 		ret = cma_bind_listen(id_priv);
3654 		if (ret)
3655 			goto err;
3656 	}
3657 
3658 	id_priv->backlog = backlog;
3659 	if (id->device) {
3660 		if (rdma_cap_ib_cm(id->device, 1)) {
3661 			ret = cma_ib_listen(id_priv);
3662 			if (ret)
3663 				goto err;
3664 		} else if (rdma_cap_iw_cm(id->device, 1)) {
3665 			ret = cma_iw_listen(id_priv, backlog);
3666 			if (ret)
3667 				goto err;
3668 		} else {
3669 			ret = -ENOSYS;
3670 			goto err;
3671 		}
3672 	} else
3673 		cma_listen_on_all(id_priv);
3674 
3675 	return 0;
3676 err:
3677 	id_priv->backlog = 0;
3678 	cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
3679 	return ret;
3680 }
3681 EXPORT_SYMBOL(rdma_listen);
3682 
3683 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
3684 {
3685 	struct rdma_id_private *id_priv;
3686 	int ret;
3687 	struct sockaddr  *daddr;
3688 
3689 	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
3690 	    addr->sa_family != AF_IB)
3691 		return -EAFNOSUPPORT;
3692 
3693 	id_priv = container_of(id, struct rdma_id_private, id);
3694 	if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
3695 		return -EINVAL;
3696 
3697 	ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
3698 	if (ret)
3699 		goto err1;
3700 
3701 	memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
3702 	if (!cma_any_addr(addr)) {
3703 		ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
3704 		if (ret)
3705 			goto err1;
3706 
3707 		ret = cma_acquire_dev_by_src_ip(id_priv);
3708 		if (ret)
3709 			goto err1;
3710 	}
3711 
3712 	if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
3713 		if (addr->sa_family == AF_INET)
3714 			id_priv->afonly = 1;
3715 #if IS_ENABLED(CONFIG_IPV6)
3716 		else if (addr->sa_family == AF_INET6) {
3717 			struct net *net = id_priv->id.route.addr.dev_addr.net;
3718 
3719 			id_priv->afonly = net->ipv6.sysctl.bindv6only;
3720 		}
3721 #endif
3722 	}
3723 	daddr = cma_dst_addr(id_priv);
3724 	daddr->sa_family = addr->sa_family;
3725 
3726 	ret = cma_get_port(id_priv);
3727 	if (ret)
3728 		goto err2;
3729 
3730 	return 0;
3731 err2:
3732 	rdma_restrack_del(&id_priv->res);
3733 	if (id_priv->cma_dev)
3734 		cma_release_dev(id_priv);
3735 err1:
3736 	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
3737 	return ret;
3738 }
3739 EXPORT_SYMBOL(rdma_bind_addr);
3740 
3741 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
3742 {
3743 	struct cma_hdr *cma_hdr;
3744 
3745 	cma_hdr = hdr;
3746 	cma_hdr->cma_version = CMA_VERSION;
3747 	if (cma_family(id_priv) == AF_INET) {
3748 		struct sockaddr_in *src4, *dst4;
3749 
3750 		src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
3751 		dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
3752 
3753 		cma_set_ip_ver(cma_hdr, 4);
3754 		cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
3755 		cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
3756 		cma_hdr->port = src4->sin_port;
3757 	} else if (cma_family(id_priv) == AF_INET6) {
3758 		struct sockaddr_in6 *src6, *dst6;
3759 
3760 		src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
3761 		dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
3762 
3763 		cma_set_ip_ver(cma_hdr, 6);
3764 		cma_hdr->src_addr.ip6 = src6->sin6_addr;
3765 		cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
3766 		cma_hdr->port = src6->sin6_port;
3767 	}
3768 	return 0;
3769 }
3770 
3771 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
3772 				const struct ib_cm_event *ib_event)
3773 {
3774 	struct rdma_id_private *id_priv = cm_id->context;
3775 	struct rdma_cm_event event = {};
3776 	const struct ib_cm_sidr_rep_event_param *rep =
3777 				&ib_event->param.sidr_rep_rcvd;
3778 	int ret;
3779 
3780 	mutex_lock(&id_priv->handler_mutex);
3781 	if (id_priv->state != RDMA_CM_CONNECT)
3782 		goto out;
3783 
3784 	switch (ib_event->event) {
3785 	case IB_CM_SIDR_REQ_ERROR:
3786 		event.event = RDMA_CM_EVENT_UNREACHABLE;
3787 		event.status = -ETIMEDOUT;
3788 		break;
3789 	case IB_CM_SIDR_REP_RECEIVED:
3790 		event.param.ud.private_data = ib_event->private_data;
3791 		event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
3792 		if (rep->status != IB_SIDR_SUCCESS) {
3793 			event.event = RDMA_CM_EVENT_UNREACHABLE;
3794 			event.status = ib_event->param.sidr_rep_rcvd.status;
3795 			pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n",
3796 					     event.status);
3797 			break;
3798 		}
3799 		ret = cma_set_qkey(id_priv, rep->qkey);
3800 		if (ret) {
3801 			pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret);
3802 			event.event = RDMA_CM_EVENT_ADDR_ERROR;
3803 			event.status = ret;
3804 			break;
3805 		}
3806 		ib_init_ah_attr_from_path(id_priv->id.device,
3807 					  id_priv->id.port_num,
3808 					  id_priv->id.route.path_rec,
3809 					  &event.param.ud.ah_attr,
3810 					  rep->sgid_attr);
3811 		event.param.ud.qp_num = rep->qpn;
3812 		event.param.ud.qkey = rep->qkey;
3813 		event.event = RDMA_CM_EVENT_ESTABLISHED;
3814 		event.status = 0;
3815 		break;
3816 	default:
3817 		pr_err("RDMA CMA: unexpected IB CM event: %d\n",
3818 		       ib_event->event);
3819 		goto out;
3820 	}
3821 
3822 	ret = cma_cm_event_handler(id_priv, &event);
3823 
3824 	rdma_destroy_ah_attr(&event.param.ud.ah_attr);
3825 	if (ret) {
3826 		/* Destroy the CM ID by returning a non-zero value. */
3827 		id_priv->cm_id.ib = NULL;
3828 		destroy_id_handler_unlock(id_priv);
3829 		return ret;
3830 	}
3831 out:
3832 	mutex_unlock(&id_priv->handler_mutex);
3833 	return 0;
3834 }
3835 
3836 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
3837 			      struct rdma_conn_param *conn_param)
3838 {
3839 	struct ib_cm_sidr_req_param req;
3840 	struct ib_cm_id	*id;
3841 	void *private_data;
3842 	u8 offset;
3843 	int ret;
3844 
3845 	memset(&req, 0, sizeof req);
3846 	offset = cma_user_data_offset(id_priv);
3847 	req.private_data_len = offset + conn_param->private_data_len;
3848 	if (req.private_data_len < conn_param->private_data_len)
3849 		return -EINVAL;
3850 
3851 	if (req.private_data_len) {
3852 		private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
3853 		if (!private_data)
3854 			return -ENOMEM;
3855 	} else {
3856 		private_data = NULL;
3857 	}
3858 
3859 	if (conn_param->private_data && conn_param->private_data_len)
3860 		memcpy(private_data + offset, conn_param->private_data,
3861 		       conn_param->private_data_len);
3862 
3863 	if (private_data) {
3864 		ret = cma_format_hdr(private_data, id_priv);
3865 		if (ret)
3866 			goto out;
3867 		req.private_data = private_data;
3868 	}
3869 
3870 	id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
3871 			     id_priv);
3872 	if (IS_ERR(id)) {
3873 		ret = PTR_ERR(id);
3874 		goto out;
3875 	}
3876 	id_priv->cm_id.ib = id;
3877 
3878 	req.path = id_priv->id.route.path_rec;
3879 	req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
3880 	req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
3881 	req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
3882 	req.max_cm_retries = CMA_MAX_CM_RETRIES;
3883 
3884 	trace_cm_send_sidr_req(id_priv);
3885 	ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
3886 	if (ret) {
3887 		ib_destroy_cm_id(id_priv->cm_id.ib);
3888 		id_priv->cm_id.ib = NULL;
3889 	}
3890 out:
3891 	kfree(private_data);
3892 	return ret;
3893 }
3894 
3895 static int cma_connect_ib(struct rdma_id_private *id_priv,
3896 			  struct rdma_conn_param *conn_param)
3897 {
3898 	struct ib_cm_req_param req;
3899 	struct rdma_route *route;
3900 	void *private_data;
3901 	struct ib_cm_id	*id;
3902 	u8 offset;
3903 	int ret;
3904 
3905 	memset(&req, 0, sizeof req);
3906 	offset = cma_user_data_offset(id_priv);
3907 	req.private_data_len = offset + conn_param->private_data_len;
3908 	if (req.private_data_len < conn_param->private_data_len)
3909 		return -EINVAL;
3910 
3911 	if (req.private_data_len) {
3912 		private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
3913 		if (!private_data)
3914 			return -ENOMEM;
3915 	} else {
3916 		private_data = NULL;
3917 	}
3918 
3919 	if (conn_param->private_data && conn_param->private_data_len)
3920 		memcpy(private_data + offset, conn_param->private_data,
3921 		       conn_param->private_data_len);
3922 
3923 	id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
3924 	if (IS_ERR(id)) {
3925 		ret = PTR_ERR(id);
3926 		goto out;
3927 	}
3928 	id_priv->cm_id.ib = id;
3929 
3930 	route = &id_priv->id.route;
3931 	if (private_data) {
3932 		ret = cma_format_hdr(private_data, id_priv);
3933 		if (ret)
3934 			goto out;
3935 		req.private_data = private_data;
3936 	}
3937 
3938 	req.primary_path = &route->path_rec[0];
3939 	if (route->num_paths == 2)
3940 		req.alternate_path = &route->path_rec[1];
3941 
3942 	req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
3943 	/* Alternate path SGID attribute currently unsupported */
3944 	req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
3945 	req.qp_num = id_priv->qp_num;
3946 	req.qp_type = id_priv->id.qp_type;
3947 	req.starting_psn = id_priv->seq_num;
3948 	req.responder_resources = conn_param->responder_resources;
3949 	req.initiator_depth = conn_param->initiator_depth;
3950 	req.flow_control = conn_param->flow_control;
3951 	req.retry_count = min_t(u8, 7, conn_param->retry_count);
3952 	req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
3953 	req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
3954 	req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
3955 	req.max_cm_retries = CMA_MAX_CM_RETRIES;
3956 	req.srq = id_priv->srq ? 1 : 0;
3957 	req.ece.vendor_id = id_priv->ece.vendor_id;
3958 	req.ece.attr_mod = id_priv->ece.attr_mod;
3959 
3960 	trace_cm_send_req(id_priv);
3961 	ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
3962 out:
3963 	if (ret && !IS_ERR(id)) {
3964 		ib_destroy_cm_id(id);
3965 		id_priv->cm_id.ib = NULL;
3966 	}
3967 
3968 	kfree(private_data);
3969 	return ret;
3970 }
3971 
3972 static int cma_connect_iw(struct rdma_id_private *id_priv,
3973 			  struct rdma_conn_param *conn_param)
3974 {
3975 	struct iw_cm_id *cm_id;
3976 	int ret;
3977 	struct iw_cm_conn_param iw_param;
3978 
3979 	cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
3980 	if (IS_ERR(cm_id))
3981 		return PTR_ERR(cm_id);
3982 
3983 	cm_id->tos = id_priv->tos;
3984 	cm_id->tos_set = id_priv->tos_set;
3985 	id_priv->cm_id.iw = cm_id;
3986 
3987 	memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
3988 	       rdma_addr_size(cma_src_addr(id_priv)));
3989 	memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
3990 	       rdma_addr_size(cma_dst_addr(id_priv)));
3991 
3992 	ret = cma_modify_qp_rtr(id_priv, conn_param);
3993 	if (ret)
3994 		goto out;
3995 
3996 	if (conn_param) {
3997 		iw_param.ord = conn_param->initiator_depth;
3998 		iw_param.ird = conn_param->responder_resources;
3999 		iw_param.private_data = conn_param->private_data;
4000 		iw_param.private_data_len = conn_param->private_data_len;
4001 		iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
4002 	} else {
4003 		memset(&iw_param, 0, sizeof iw_param);
4004 		iw_param.qpn = id_priv->qp_num;
4005 	}
4006 	ret = iw_cm_connect(cm_id, &iw_param);
4007 out:
4008 	if (ret) {
4009 		iw_destroy_cm_id(cm_id);
4010 		id_priv->cm_id.iw = NULL;
4011 	}
4012 	return ret;
4013 }
4014 
4015 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4016 {
4017 	struct rdma_id_private *id_priv;
4018 	int ret;
4019 
4020 	id_priv = container_of(id, struct rdma_id_private, id);
4021 	if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
4022 		return -EINVAL;
4023 
4024 	if (!id->qp) {
4025 		id_priv->qp_num = conn_param->qp_num;
4026 		id_priv->srq = conn_param->srq;
4027 	}
4028 
4029 	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4030 		if (id->qp_type == IB_QPT_UD)
4031 			ret = cma_resolve_ib_udp(id_priv, conn_param);
4032 		else
4033 			ret = cma_connect_ib(id_priv, conn_param);
4034 	} else if (rdma_cap_iw_cm(id->device, id->port_num))
4035 		ret = cma_connect_iw(id_priv, conn_param);
4036 	else
4037 		ret = -ENOSYS;
4038 	if (ret)
4039 		goto err;
4040 
4041 	return 0;
4042 err:
4043 	cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
4044 	return ret;
4045 }
4046 EXPORT_SYMBOL(rdma_connect);
4047 
4048 /**
4049  * rdma_connect_ece - Initiate an active connection request with ECE data.
4050  * @id: Connection identifier to connect.
4051  * @conn_param: Connection information used for connected QPs.
4052  * @ece: ECE parameters
4053  *
4054  * See rdma_connect() explanation.
4055  */
4056 int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4057 		     struct rdma_ucm_ece *ece)
4058 {
4059 	struct rdma_id_private *id_priv =
4060 		container_of(id, struct rdma_id_private, id);
4061 
4062 	id_priv->ece.vendor_id = ece->vendor_id;
4063 	id_priv->ece.attr_mod = ece->attr_mod;
4064 
4065 	return rdma_connect(id, conn_param);
4066 }
4067 EXPORT_SYMBOL(rdma_connect_ece);
4068 
4069 static int cma_accept_ib(struct rdma_id_private *id_priv,
4070 			 struct rdma_conn_param *conn_param)
4071 {
4072 	struct ib_cm_rep_param rep;
4073 	int ret;
4074 
4075 	ret = cma_modify_qp_rtr(id_priv, conn_param);
4076 	if (ret)
4077 		goto out;
4078 
4079 	ret = cma_modify_qp_rts(id_priv, conn_param);
4080 	if (ret)
4081 		goto out;
4082 
4083 	memset(&rep, 0, sizeof rep);
4084 	rep.qp_num = id_priv->qp_num;
4085 	rep.starting_psn = id_priv->seq_num;
4086 	rep.private_data = conn_param->private_data;
4087 	rep.private_data_len = conn_param->private_data_len;
4088 	rep.responder_resources = conn_param->responder_resources;
4089 	rep.initiator_depth = conn_param->initiator_depth;
4090 	rep.failover_accepted = 0;
4091 	rep.flow_control = conn_param->flow_control;
4092 	rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4093 	rep.srq = id_priv->srq ? 1 : 0;
4094 	rep.ece.vendor_id = id_priv->ece.vendor_id;
4095 	rep.ece.attr_mod = id_priv->ece.attr_mod;
4096 
4097 	trace_cm_send_rep(id_priv);
4098 	ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
4099 out:
4100 	return ret;
4101 }
4102 
4103 static int cma_accept_iw(struct rdma_id_private *id_priv,
4104 		  struct rdma_conn_param *conn_param)
4105 {
4106 	struct iw_cm_conn_param iw_param;
4107 	int ret;
4108 
4109 	if (!conn_param)
4110 		return -EINVAL;
4111 
4112 	ret = cma_modify_qp_rtr(id_priv, conn_param);
4113 	if (ret)
4114 		return ret;
4115 
4116 	iw_param.ord = conn_param->initiator_depth;
4117 	iw_param.ird = conn_param->responder_resources;
4118 	iw_param.private_data = conn_param->private_data;
4119 	iw_param.private_data_len = conn_param->private_data_len;
4120 	if (id_priv->id.qp) {
4121 		iw_param.qpn = id_priv->qp_num;
4122 	} else
4123 		iw_param.qpn = conn_param->qp_num;
4124 
4125 	return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
4126 }
4127 
4128 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
4129 			     enum ib_cm_sidr_status status, u32 qkey,
4130 			     const void *private_data, int private_data_len)
4131 {
4132 	struct ib_cm_sidr_rep_param rep;
4133 	int ret;
4134 
4135 	memset(&rep, 0, sizeof rep);
4136 	rep.status = status;
4137 	if (status == IB_SIDR_SUCCESS) {
4138 		ret = cma_set_qkey(id_priv, qkey);
4139 		if (ret)
4140 			return ret;
4141 		rep.qp_num = id_priv->qp_num;
4142 		rep.qkey = id_priv->qkey;
4143 
4144 		rep.ece.vendor_id = id_priv->ece.vendor_id;
4145 		rep.ece.attr_mod = id_priv->ece.attr_mod;
4146 	}
4147 
4148 	rep.private_data = private_data;
4149 	rep.private_data_len = private_data_len;
4150 
4151 	trace_cm_send_sidr_rep(id_priv);
4152 	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
4153 }
4154 
4155 int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4156 		  const char *caller)
4157 {
4158 	struct rdma_id_private *id_priv;
4159 	int ret;
4160 
4161 	id_priv = container_of(id, struct rdma_id_private, id);
4162 
4163 	rdma_restrack_set_task(&id_priv->res, caller);
4164 
4165 	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
4166 		return -EINVAL;
4167 
4168 	if (!id->qp && conn_param) {
4169 		id_priv->qp_num = conn_param->qp_num;
4170 		id_priv->srq = conn_param->srq;
4171 	}
4172 
4173 	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4174 		if (id->qp_type == IB_QPT_UD) {
4175 			if (conn_param)
4176 				ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4177 							conn_param->qkey,
4178 							conn_param->private_data,
4179 							conn_param->private_data_len);
4180 			else
4181 				ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4182 							0, NULL, 0);
4183 		} else {
4184 			if (conn_param)
4185 				ret = cma_accept_ib(id_priv, conn_param);
4186 			else
4187 				ret = cma_rep_recv(id_priv);
4188 		}
4189 	} else if (rdma_cap_iw_cm(id->device, id->port_num))
4190 		ret = cma_accept_iw(id_priv, conn_param);
4191 	else
4192 		ret = -ENOSYS;
4193 
4194 	if (ret)
4195 		goto reject;
4196 
4197 	return 0;
4198 reject:
4199 	cma_modify_qp_err(id_priv);
4200 	rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
4201 	return ret;
4202 }
4203 EXPORT_SYMBOL(__rdma_accept);
4204 
4205 int __rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4206 		      const char *caller, struct rdma_ucm_ece *ece)
4207 {
4208 	struct rdma_id_private *id_priv =
4209 		container_of(id, struct rdma_id_private, id);
4210 
4211 	id_priv->ece.vendor_id = ece->vendor_id;
4212 	id_priv->ece.attr_mod = ece->attr_mod;
4213 
4214 	return __rdma_accept(id, conn_param, caller);
4215 }
4216 EXPORT_SYMBOL(__rdma_accept_ece);
4217 
4218 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
4219 {
4220 	struct rdma_id_private *id_priv;
4221 	int ret;
4222 
4223 	id_priv = container_of(id, struct rdma_id_private, id);
4224 	if (!id_priv->cm_id.ib)
4225 		return -EINVAL;
4226 
4227 	switch (id->device->node_type) {
4228 	case RDMA_NODE_IB_CA:
4229 		ret = ib_cm_notify(id_priv->cm_id.ib, event);
4230 		break;
4231 	default:
4232 		ret = 0;
4233 		break;
4234 	}
4235 	return ret;
4236 }
4237 EXPORT_SYMBOL(rdma_notify);
4238 
4239 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
4240 		u8 private_data_len, u8 reason)
4241 {
4242 	struct rdma_id_private *id_priv;
4243 	int ret;
4244 
4245 	id_priv = container_of(id, struct rdma_id_private, id);
4246 	if (!id_priv->cm_id.ib)
4247 		return -EINVAL;
4248 
4249 	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4250 		if (id->qp_type == IB_QPT_UD) {
4251 			ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
4252 						private_data, private_data_len);
4253 		} else {
4254 			trace_cm_send_rej(id_priv);
4255 			ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0,
4256 					     private_data, private_data_len);
4257 		}
4258 	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4259 		ret = iw_cm_reject(id_priv->cm_id.iw,
4260 				   private_data, private_data_len);
4261 	} else
4262 		ret = -ENOSYS;
4263 
4264 	return ret;
4265 }
4266 EXPORT_SYMBOL(rdma_reject);
4267 
4268 int rdma_disconnect(struct rdma_cm_id *id)
4269 {
4270 	struct rdma_id_private *id_priv;
4271 	int ret;
4272 
4273 	id_priv = container_of(id, struct rdma_id_private, id);
4274 	if (!id_priv->cm_id.ib)
4275 		return -EINVAL;
4276 
4277 	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4278 		ret = cma_modify_qp_err(id_priv);
4279 		if (ret)
4280 			goto out;
4281 		/* Initiate or respond to a disconnect. */
4282 		trace_cm_disconnect(id_priv);
4283 		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
4284 			if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
4285 				trace_cm_sent_drep(id_priv);
4286 		} else {
4287 			trace_cm_sent_dreq(id_priv);
4288 		}
4289 	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4290 		ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
4291 	} else
4292 		ret = -EINVAL;
4293 
4294 out:
4295 	return ret;
4296 }
4297 EXPORT_SYMBOL(rdma_disconnect);
4298 
4299 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
4300 {
4301 	struct rdma_id_private *id_priv;
4302 	struct cma_multicast *mc = multicast->context;
4303 	struct rdma_cm_event event = {};
4304 	int ret = 0;
4305 
4306 	id_priv = mc->id_priv;
4307 	mutex_lock(&id_priv->handler_mutex);
4308 	if (id_priv->state != RDMA_CM_ADDR_BOUND &&
4309 	    id_priv->state != RDMA_CM_ADDR_RESOLVED)
4310 		goto out;
4311 
4312 	if (!status)
4313 		status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
4314 	else
4315 		pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",
4316 				     status);
4317 	mutex_lock(&id_priv->qp_mutex);
4318 	if (!status && id_priv->id.qp) {
4319 		status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
4320 					 be16_to_cpu(multicast->rec.mlid));
4321 		if (status)
4322 			pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to attach QP. status %d\n",
4323 					     status);
4324 	}
4325 	mutex_unlock(&id_priv->qp_mutex);
4326 
4327 	event.status = status;
4328 	event.param.ud.private_data = mc->context;
4329 	if (!status) {
4330 		struct rdma_dev_addr *dev_addr =
4331 			&id_priv->id.route.addr.dev_addr;
4332 		struct net_device *ndev =
4333 			dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4334 		enum ib_gid_type gid_type =
4335 			id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4336 			rdma_start_port(id_priv->cma_dev->device)];
4337 
4338 		event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
4339 		ret = ib_init_ah_from_mcmember(id_priv->id.device,
4340 					       id_priv->id.port_num,
4341 					       &multicast->rec,
4342 					       ndev, gid_type,
4343 					       &event.param.ud.ah_attr);
4344 		if (ret)
4345 			event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
4346 
4347 		event.param.ud.qp_num = 0xFFFFFF;
4348 		event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
4349 		if (ndev)
4350 			dev_put(ndev);
4351 	} else
4352 		event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
4353 
4354 	ret = cma_cm_event_handler(id_priv, &event);
4355 
4356 	rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4357 	if (ret) {
4358 		destroy_id_handler_unlock(id_priv);
4359 		return 0;
4360 	}
4361 
4362 out:
4363 	mutex_unlock(&id_priv->handler_mutex);
4364 	return 0;
4365 }
4366 
4367 static void cma_set_mgid(struct rdma_id_private *id_priv,
4368 			 struct sockaddr *addr, union ib_gid *mgid)
4369 {
4370 	unsigned char mc_map[MAX_ADDR_LEN];
4371 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4372 	struct sockaddr_in *sin = (struct sockaddr_in *) addr;
4373 	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
4374 
4375 	if (cma_any_addr(addr)) {
4376 		memset(mgid, 0, sizeof *mgid);
4377 	} else if ((addr->sa_family == AF_INET6) &&
4378 		   ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
4379 								 0xFF10A01B)) {
4380 		/* IPv6 address is an SA assigned MGID. */
4381 		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4382 	} else if (addr->sa_family == AF_IB) {
4383 		memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
4384 	} else if (addr->sa_family == AF_INET6) {
4385 		ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
4386 		if (id_priv->id.ps == RDMA_PS_UDP)
4387 			mc_map[7] = 0x01;	/* Use RDMA CM signature */
4388 		*mgid = *(union ib_gid *) (mc_map + 4);
4389 	} else {
4390 		ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
4391 		if (id_priv->id.ps == RDMA_PS_UDP)
4392 			mc_map[7] = 0x01;	/* Use RDMA CM signature */
4393 		*mgid = *(union ib_gid *) (mc_map + 4);
4394 	}
4395 }
4396 
4397 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
4398 				 struct cma_multicast *mc)
4399 {
4400 	struct ib_sa_mcmember_rec rec;
4401 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4402 	ib_sa_comp_mask comp_mask;
4403 	int ret;
4404 
4405 	ib_addr_get_mgid(dev_addr, &rec.mgid);
4406 	ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
4407 				     &rec.mgid, &rec);
4408 	if (ret)
4409 		return ret;
4410 
4411 	ret = cma_set_qkey(id_priv, 0);
4412 	if (ret)
4413 		return ret;
4414 
4415 	cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
4416 	rec.qkey = cpu_to_be32(id_priv->qkey);
4417 	rdma_addr_get_sgid(dev_addr, &rec.port_gid);
4418 	rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
4419 	rec.join_state = mc->join_state;
4420 
4421 	if ((rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) &&
4422 	    (!ib_sa_sendonly_fullmem_support(&sa_client,
4423 					     id_priv->id.device,
4424 					     id_priv->id.port_num))) {
4425 		dev_warn(
4426 			&id_priv->id.device->dev,
4427 			"RDMA CM: port %u Unable to multicast join: SM doesn't support Send Only Full Member option\n",
4428 			id_priv->id.port_num);
4429 		return -EOPNOTSUPP;
4430 	}
4431 
4432 	comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
4433 		    IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
4434 		    IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
4435 		    IB_SA_MCMEMBER_REC_FLOW_LABEL |
4436 		    IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
4437 
4438 	if (id_priv->id.ps == RDMA_PS_IPOIB)
4439 		comp_mask |= IB_SA_MCMEMBER_REC_RATE |
4440 			     IB_SA_MCMEMBER_REC_RATE_SELECTOR |
4441 			     IB_SA_MCMEMBER_REC_MTU_SELECTOR |
4442 			     IB_SA_MCMEMBER_REC_MTU |
4443 			     IB_SA_MCMEMBER_REC_HOP_LIMIT;
4444 
4445 	mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
4446 						id_priv->id.port_num, &rec,
4447 						comp_mask, GFP_KERNEL,
4448 						cma_ib_mc_handler, mc);
4449 	return PTR_ERR_OR_ZERO(mc->multicast.ib);
4450 }
4451 
4452 static void iboe_mcast_work_handler(struct work_struct *work)
4453 {
4454 	struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
4455 	struct cma_multicast *mc = mw->mc;
4456 	struct ib_sa_multicast *m = mc->multicast.ib;
4457 
4458 	mc->multicast.ib->context = mc;
4459 	cma_ib_mc_handler(0, m);
4460 	kref_put(&mc->mcref, release_mc);
4461 	kfree(mw);
4462 }
4463 
4464 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
4465 			      enum ib_gid_type gid_type)
4466 {
4467 	struct sockaddr_in *sin = (struct sockaddr_in *)addr;
4468 	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
4469 
4470 	if (cma_any_addr(addr)) {
4471 		memset(mgid, 0, sizeof *mgid);
4472 	} else if (addr->sa_family == AF_INET6) {
4473 		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4474 	} else {
4475 		mgid->raw[0] =
4476 			(gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
4477 		mgid->raw[1] =
4478 			(gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
4479 		mgid->raw[2] = 0;
4480 		mgid->raw[3] = 0;
4481 		mgid->raw[4] = 0;
4482 		mgid->raw[5] = 0;
4483 		mgid->raw[6] = 0;
4484 		mgid->raw[7] = 0;
4485 		mgid->raw[8] = 0;
4486 		mgid->raw[9] = 0;
4487 		mgid->raw[10] = 0xff;
4488 		mgid->raw[11] = 0xff;
4489 		*(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
4490 	}
4491 }
4492 
4493 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
4494 				   struct cma_multicast *mc)
4495 {
4496 	struct iboe_mcast_work *work;
4497 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4498 	int err = 0;
4499 	struct sockaddr *addr = (struct sockaddr *)&mc->addr;
4500 	struct net_device *ndev = NULL;
4501 	enum ib_gid_type gid_type;
4502 	bool send_only;
4503 
4504 	send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
4505 
4506 	if (cma_zero_addr((struct sockaddr *)&mc->addr))
4507 		return -EINVAL;
4508 
4509 	work = kzalloc(sizeof *work, GFP_KERNEL);
4510 	if (!work)
4511 		return -ENOMEM;
4512 
4513 	mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
4514 	if (!mc->multicast.ib) {
4515 		err = -ENOMEM;
4516 		goto out1;
4517 	}
4518 
4519 	gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4520 		   rdma_start_port(id_priv->cma_dev->device)];
4521 	cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid, gid_type);
4522 
4523 	mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
4524 	if (id_priv->id.ps == RDMA_PS_UDP)
4525 		mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
4526 
4527 	if (dev_addr->bound_dev_if)
4528 		ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4529 	if (!ndev) {
4530 		err = -ENODEV;
4531 		goto out2;
4532 	}
4533 	mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
4534 	mc->multicast.ib->rec.hop_limit = 1;
4535 	mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
4536 
4537 	if (addr->sa_family == AF_INET) {
4538 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
4539 			mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
4540 			if (!send_only) {
4541 				err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid,
4542 						    true);
4543 			}
4544 		}
4545 	} else {
4546 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4547 			err = -ENOTSUPP;
4548 	}
4549 	dev_put(ndev);
4550 	if (err || !mc->multicast.ib->rec.mtu) {
4551 		if (!err)
4552 			err = -EINVAL;
4553 		goto out2;
4554 	}
4555 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
4556 		    &mc->multicast.ib->rec.port_gid);
4557 	work->id = id_priv;
4558 	work->mc = mc;
4559 	INIT_WORK(&work->work, iboe_mcast_work_handler);
4560 	kref_get(&mc->mcref);
4561 	queue_work(cma_wq, &work->work);
4562 
4563 	return 0;
4564 
4565 out2:
4566 	kfree(mc->multicast.ib);
4567 out1:
4568 	kfree(work);
4569 	return err;
4570 }
4571 
4572 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
4573 			u8 join_state, void *context)
4574 {
4575 	struct rdma_id_private *id_priv;
4576 	struct cma_multicast *mc;
4577 	int ret;
4578 
4579 	if (!id->device)
4580 		return -EINVAL;
4581 
4582 	id_priv = container_of(id, struct rdma_id_private, id);
4583 	if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
4584 	    !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
4585 		return -EINVAL;
4586 
4587 	mc = kmalloc(sizeof *mc, GFP_KERNEL);
4588 	if (!mc)
4589 		return -ENOMEM;
4590 
4591 	memcpy(&mc->addr, addr, rdma_addr_size(addr));
4592 	mc->context = context;
4593 	mc->id_priv = id_priv;
4594 	mc->join_state = join_state;
4595 
4596 	if (rdma_protocol_roce(id->device, id->port_num)) {
4597 		kref_init(&mc->mcref);
4598 		ret = cma_iboe_join_multicast(id_priv, mc);
4599 		if (ret)
4600 			goto out_err;
4601 	} else if (rdma_cap_ib_mcast(id->device, id->port_num)) {
4602 		ret = cma_join_ib_multicast(id_priv, mc);
4603 		if (ret)
4604 			goto out_err;
4605 	} else {
4606 		ret = -ENOSYS;
4607 		goto out_err;
4608 	}
4609 
4610 	spin_lock(&id_priv->lock);
4611 	list_add(&mc->list, &id_priv->mc_list);
4612 	spin_unlock(&id_priv->lock);
4613 
4614 	return 0;
4615 out_err:
4616 	kfree(mc);
4617 	return ret;
4618 }
4619 EXPORT_SYMBOL(rdma_join_multicast);
4620 
4621 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
4622 {
4623 	struct rdma_id_private *id_priv;
4624 	struct cma_multicast *mc;
4625 
4626 	id_priv = container_of(id, struct rdma_id_private, id);
4627 	spin_lock_irq(&id_priv->lock);
4628 	list_for_each_entry(mc, &id_priv->mc_list, list) {
4629 		if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) {
4630 			list_del(&mc->list);
4631 			spin_unlock_irq(&id_priv->lock);
4632 
4633 			if (id->qp)
4634 				ib_detach_mcast(id->qp,
4635 						&mc->multicast.ib->rec.mgid,
4636 						be16_to_cpu(mc->multicast.ib->rec.mlid));
4637 
4638 			BUG_ON(id_priv->cma_dev->device != id->device);
4639 
4640 			if (rdma_cap_ib_mcast(id->device, id->port_num)) {
4641 				ib_sa_free_multicast(mc->multicast.ib);
4642 				kfree(mc);
4643 			} else if (rdma_protocol_roce(id->device, id->port_num)) {
4644 				cma_leave_roce_mc_group(id_priv, mc);
4645 			}
4646 			return;
4647 		}
4648 	}
4649 	spin_unlock_irq(&id_priv->lock);
4650 }
4651 EXPORT_SYMBOL(rdma_leave_multicast);
4652 
4653 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
4654 {
4655 	struct rdma_dev_addr *dev_addr;
4656 	struct cma_ndev_work *work;
4657 
4658 	dev_addr = &id_priv->id.route.addr.dev_addr;
4659 
4660 	if ((dev_addr->bound_dev_if == ndev->ifindex) &&
4661 	    (net_eq(dev_net(ndev), dev_addr->net)) &&
4662 	    memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
4663 		pr_info("RDMA CM addr change for ndev %s used by id %p\n",
4664 			ndev->name, &id_priv->id);
4665 		work = kzalloc(sizeof *work, GFP_KERNEL);
4666 		if (!work)
4667 			return -ENOMEM;
4668 
4669 		INIT_WORK(&work->work, cma_ndev_work_handler);
4670 		work->id = id_priv;
4671 		work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
4672 		cma_id_get(id_priv);
4673 		queue_work(cma_wq, &work->work);
4674 	}
4675 
4676 	return 0;
4677 }
4678 
4679 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
4680 			       void *ptr)
4681 {
4682 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
4683 	struct cma_device *cma_dev;
4684 	struct rdma_id_private *id_priv;
4685 	int ret = NOTIFY_DONE;
4686 
4687 	if (event != NETDEV_BONDING_FAILOVER)
4688 		return NOTIFY_DONE;
4689 
4690 	if (!netif_is_bond_master(ndev))
4691 		return NOTIFY_DONE;
4692 
4693 	mutex_lock(&lock);
4694 	list_for_each_entry(cma_dev, &dev_list, list)
4695 		list_for_each_entry(id_priv, &cma_dev->id_list, list) {
4696 			ret = cma_netdev_change(ndev, id_priv);
4697 			if (ret)
4698 				goto out;
4699 		}
4700 
4701 out:
4702 	mutex_unlock(&lock);
4703 	return ret;
4704 }
4705 
4706 static struct notifier_block cma_nb = {
4707 	.notifier_call = cma_netdev_callback
4708 };
4709 
4710 static int cma_add_one(struct ib_device *device)
4711 {
4712 	struct cma_device *cma_dev;
4713 	struct rdma_id_private *id_priv;
4714 	unsigned int i;
4715 	unsigned long supported_gids = 0;
4716 	int ret;
4717 
4718 	cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
4719 	if (!cma_dev)
4720 		return -ENOMEM;
4721 
4722 	cma_dev->device = device;
4723 	cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
4724 					    sizeof(*cma_dev->default_gid_type),
4725 					    GFP_KERNEL);
4726 	if (!cma_dev->default_gid_type) {
4727 		ret = -ENOMEM;
4728 		goto free_cma_dev;
4729 	}
4730 
4731 	cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
4732 					    sizeof(*cma_dev->default_roce_tos),
4733 					    GFP_KERNEL);
4734 	if (!cma_dev->default_roce_tos) {
4735 		ret = -ENOMEM;
4736 		goto free_gid_type;
4737 	}
4738 
4739 	rdma_for_each_port (device, i) {
4740 		supported_gids = roce_gid_type_mask_support(device, i);
4741 		WARN_ON(!supported_gids);
4742 		if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
4743 			cma_dev->default_gid_type[i - rdma_start_port(device)] =
4744 				CMA_PREFERRED_ROCE_GID_TYPE;
4745 		else
4746 			cma_dev->default_gid_type[i - rdma_start_port(device)] =
4747 				find_first_bit(&supported_gids, BITS_PER_LONG);
4748 		cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
4749 	}
4750 
4751 	init_completion(&cma_dev->comp);
4752 	refcount_set(&cma_dev->refcount, 1);
4753 	INIT_LIST_HEAD(&cma_dev->id_list);
4754 	ib_set_client_data(device, &cma_client, cma_dev);
4755 
4756 	mutex_lock(&lock);
4757 	list_add_tail(&cma_dev->list, &dev_list);
4758 	list_for_each_entry(id_priv, &listen_any_list, list)
4759 		cma_listen_on_dev(id_priv, cma_dev);
4760 	mutex_unlock(&lock);
4761 
4762 	trace_cm_add_one(device);
4763 	return 0;
4764 
4765 free_gid_type:
4766 	kfree(cma_dev->default_gid_type);
4767 
4768 free_cma_dev:
4769 	kfree(cma_dev);
4770 	return ret;
4771 }
4772 
4773 static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
4774 {
4775 	struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
4776 	enum rdma_cm_state state;
4777 	unsigned long flags;
4778 
4779 	mutex_lock(&id_priv->handler_mutex);
4780 	/* Record that we want to remove the device */
4781 	spin_lock_irqsave(&id_priv->lock, flags);
4782 	state = id_priv->state;
4783 	if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {
4784 		spin_unlock_irqrestore(&id_priv->lock, flags);
4785 		mutex_unlock(&id_priv->handler_mutex);
4786 		cma_id_put(id_priv);
4787 		return;
4788 	}
4789 	id_priv->state = RDMA_CM_DEVICE_REMOVAL;
4790 	spin_unlock_irqrestore(&id_priv->lock, flags);
4791 
4792 	if (cma_cm_event_handler(id_priv, &event)) {
4793 		/*
4794 		 * At this point the ULP promises it won't call
4795 		 * rdma_destroy_id() concurrently
4796 		 */
4797 		cma_id_put(id_priv);
4798 		mutex_unlock(&id_priv->handler_mutex);
4799 		trace_cm_id_destroy(id_priv);
4800 		_destroy_id(id_priv, state);
4801 		return;
4802 	}
4803 	mutex_unlock(&id_priv->handler_mutex);
4804 
4805 	/*
4806 	 * If this races with destroy then the thread that first assigns state
4807 	 * to a destroying does the cancel.
4808 	 */
4809 	cma_cancel_operation(id_priv, state);
4810 	cma_id_put(id_priv);
4811 }
4812 
4813 static void cma_process_remove(struct cma_device *cma_dev)
4814 {
4815 	mutex_lock(&lock);
4816 	while (!list_empty(&cma_dev->id_list)) {
4817 		struct rdma_id_private *id_priv = list_first_entry(
4818 			&cma_dev->id_list, struct rdma_id_private, list);
4819 
4820 		list_del(&id_priv->listen_list);
4821 		list_del_init(&id_priv->list);
4822 		cma_id_get(id_priv);
4823 		mutex_unlock(&lock);
4824 
4825 		cma_send_device_removal_put(id_priv);
4826 
4827 		mutex_lock(&lock);
4828 	}
4829 	mutex_unlock(&lock);
4830 
4831 	cma_dev_put(cma_dev);
4832 	wait_for_completion(&cma_dev->comp);
4833 }
4834 
4835 static void cma_remove_one(struct ib_device *device, void *client_data)
4836 {
4837 	struct cma_device *cma_dev = client_data;
4838 
4839 	trace_cm_remove_one(device);
4840 
4841 	mutex_lock(&lock);
4842 	list_del(&cma_dev->list);
4843 	mutex_unlock(&lock);
4844 
4845 	cma_process_remove(cma_dev);
4846 	kfree(cma_dev->default_roce_tos);
4847 	kfree(cma_dev->default_gid_type);
4848 	kfree(cma_dev);
4849 }
4850 
4851 static int cma_init_net(struct net *net)
4852 {
4853 	struct cma_pernet *pernet = cma_pernet(net);
4854 
4855 	xa_init(&pernet->tcp_ps);
4856 	xa_init(&pernet->udp_ps);
4857 	xa_init(&pernet->ipoib_ps);
4858 	xa_init(&pernet->ib_ps);
4859 
4860 	return 0;
4861 }
4862 
4863 static void cma_exit_net(struct net *net)
4864 {
4865 	struct cma_pernet *pernet = cma_pernet(net);
4866 
4867 	WARN_ON(!xa_empty(&pernet->tcp_ps));
4868 	WARN_ON(!xa_empty(&pernet->udp_ps));
4869 	WARN_ON(!xa_empty(&pernet->ipoib_ps));
4870 	WARN_ON(!xa_empty(&pernet->ib_ps));
4871 }
4872 
4873 static struct pernet_operations cma_pernet_operations = {
4874 	.init = cma_init_net,
4875 	.exit = cma_exit_net,
4876 	.id = &cma_pernet_id,
4877 	.size = sizeof(struct cma_pernet),
4878 };
4879 
4880 static int __init cma_init(void)
4881 {
4882 	int ret;
4883 
4884 	/*
4885 	 * There is a rare lock ordering dependency in cma_netdev_callback()
4886 	 * that only happens when bonding is enabled. Teach lockdep that rtnl
4887 	 * must never be nested under lock so it can find these without having
4888 	 * to test with bonding.
4889 	 */
4890 	if (IS_ENABLED(CONFIG_LOCKDEP)) {
4891 		rtnl_lock();
4892 		mutex_lock(&lock);
4893 		mutex_unlock(&lock);
4894 		rtnl_unlock();
4895 	}
4896 
4897 	cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
4898 	if (!cma_wq)
4899 		return -ENOMEM;
4900 
4901 	ret = register_pernet_subsys(&cma_pernet_operations);
4902 	if (ret)
4903 		goto err_wq;
4904 
4905 	ib_sa_register_client(&sa_client);
4906 	register_netdevice_notifier(&cma_nb);
4907 
4908 	ret = ib_register_client(&cma_client);
4909 	if (ret)
4910 		goto err;
4911 
4912 	ret = cma_configfs_init();
4913 	if (ret)
4914 		goto err_ib;
4915 
4916 	return 0;
4917 
4918 err_ib:
4919 	ib_unregister_client(&cma_client);
4920 err:
4921 	unregister_netdevice_notifier(&cma_nb);
4922 	ib_sa_unregister_client(&sa_client);
4923 	unregister_pernet_subsys(&cma_pernet_operations);
4924 err_wq:
4925 	destroy_workqueue(cma_wq);
4926 	return ret;
4927 }
4928 
4929 static void __exit cma_cleanup(void)
4930 {
4931 	cma_configfs_exit();
4932 	ib_unregister_client(&cma_client);
4933 	unregister_netdevice_notifier(&cma_nb);
4934 	ib_sa_unregister_client(&sa_client);
4935 	unregister_pernet_subsys(&cma_pernet_operations);
4936 	destroy_workqueue(cma_wq);
4937 }
4938 
4939 module_init(cma_init);
4940 module_exit(cma_cleanup);
4941