xref: /openbmc/linux/net/rds/ib.c (revision 021ed9f5)
1 /*
2  * Copyright (c) 2006, 2019 Oracle and/or its affiliates. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  *
32  */
33 #include <linux/dmapool.h>
34 #include <linux/kernel.h>
35 #include <linux/in.h>
36 #include <linux/if.h>
37 #include <linux/netdevice.h>
38 #include <linux/inetdevice.h>
39 #include <linux/if_arp.h>
40 #include <linux/delay.h>
41 #include <linux/slab.h>
42 #include <linux/module.h>
43 #include <net/addrconf.h>
44 
45 #include "rds_single_path.h"
46 #include "rds.h"
47 #include "ib.h"
48 #include "ib_mr.h"
49 
50 static unsigned int rds_ib_mr_1m_pool_size = RDS_MR_1M_POOL_SIZE;
51 static unsigned int rds_ib_mr_8k_pool_size = RDS_MR_8K_POOL_SIZE;
52 unsigned int rds_ib_retry_count = RDS_IB_DEFAULT_RETRY_COUNT;
53 static atomic_t rds_ib_unloading;
54 
55 module_param(rds_ib_mr_1m_pool_size, int, 0444);
56 MODULE_PARM_DESC(rds_ib_mr_1m_pool_size, " Max number of 1M mr per HCA");
57 module_param(rds_ib_mr_8k_pool_size, int, 0444);
58 MODULE_PARM_DESC(rds_ib_mr_8k_pool_size, " Max number of 8K mr per HCA");
59 module_param(rds_ib_retry_count, int, 0444);
60 MODULE_PARM_DESC(rds_ib_retry_count, " Number of hw retries before reporting an error");
61 
62 /*
63  * we have a clumsy combination of RCU and a rwsem protecting this list
64  * because it is used both in the get_mr fast path and while blocking in
65  * the FMR flushing path.
66  */
67 DECLARE_RWSEM(rds_ib_devices_lock);
68 struct list_head rds_ib_devices;
69 
70 /* NOTE: if also grabbing ibdev lock, grab this first */
71 DEFINE_SPINLOCK(ib_nodev_conns_lock);
72 LIST_HEAD(ib_nodev_conns);
73 
74 static void rds_ib_nodev_connect(void)
75 {
76 	struct rds_ib_connection *ic;
77 
78 	spin_lock(&ib_nodev_conns_lock);
79 	list_for_each_entry(ic, &ib_nodev_conns, ib_node)
80 		rds_conn_connect_if_down(ic->conn);
81 	spin_unlock(&ib_nodev_conns_lock);
82 }
83 
84 static void rds_ib_dev_shutdown(struct rds_ib_device *rds_ibdev)
85 {
86 	struct rds_ib_connection *ic;
87 	unsigned long flags;
88 
89 	spin_lock_irqsave(&rds_ibdev->spinlock, flags);
90 	list_for_each_entry(ic, &rds_ibdev->conn_list, ib_node)
91 		rds_conn_path_drop(&ic->conn->c_path[0], true);
92 	spin_unlock_irqrestore(&rds_ibdev->spinlock, flags);
93 }
94 
95 /*
96  * rds_ib_destroy_mr_pool() blocks on a few things and mrs drop references
97  * from interrupt context so we push freing off into a work struct in krdsd.
98  */
99 static void rds_ib_dev_free(struct work_struct *work)
100 {
101 	struct rds_ib_ipaddr *i_ipaddr, *i_next;
102 	struct rds_ib_device *rds_ibdev = container_of(work,
103 					struct rds_ib_device, free_work);
104 
105 	if (rds_ibdev->mr_8k_pool)
106 		rds_ib_destroy_mr_pool(rds_ibdev->mr_8k_pool);
107 	if (rds_ibdev->mr_1m_pool)
108 		rds_ib_destroy_mr_pool(rds_ibdev->mr_1m_pool);
109 	if (rds_ibdev->pd)
110 		ib_dealloc_pd(rds_ibdev->pd);
111 	dma_pool_destroy(rds_ibdev->rid_hdrs_pool);
112 
113 	list_for_each_entry_safe(i_ipaddr, i_next, &rds_ibdev->ipaddr_list, list) {
114 		list_del(&i_ipaddr->list);
115 		kfree(i_ipaddr);
116 	}
117 
118 	kfree(rds_ibdev->vector_load);
119 
120 	kfree(rds_ibdev);
121 }
122 
123 void rds_ib_dev_put(struct rds_ib_device *rds_ibdev)
124 {
125 	BUG_ON(refcount_read(&rds_ibdev->refcount) == 0);
126 	if (refcount_dec_and_test(&rds_ibdev->refcount))
127 		queue_work(rds_wq, &rds_ibdev->free_work);
128 }
129 
130 static void rds_ib_add_one(struct ib_device *device)
131 {
132 	struct rds_ib_device *rds_ibdev;
133 	bool has_fr, has_fmr;
134 
135 	/* Only handle IB (no iWARP) devices */
136 	if (device->node_type != RDMA_NODE_IB_CA)
137 		return;
138 
139 	rds_ibdev = kzalloc_node(sizeof(struct rds_ib_device), GFP_KERNEL,
140 				 ibdev_to_node(device));
141 	if (!rds_ibdev)
142 		return;
143 
144 	spin_lock_init(&rds_ibdev->spinlock);
145 	refcount_set(&rds_ibdev->refcount, 1);
146 	INIT_WORK(&rds_ibdev->free_work, rds_ib_dev_free);
147 
148 	INIT_LIST_HEAD(&rds_ibdev->ipaddr_list);
149 	INIT_LIST_HEAD(&rds_ibdev->conn_list);
150 
151 	rds_ibdev->max_wrs = device->attrs.max_qp_wr;
152 	rds_ibdev->max_sge = min(device->attrs.max_send_sge, RDS_IB_MAX_SGE);
153 
154 	has_fr = (device->attrs.device_cap_flags &
155 		  IB_DEVICE_MEM_MGT_EXTENSIONS);
156 	has_fmr = (device->ops.alloc_fmr && device->ops.dealloc_fmr &&
157 		   device->ops.map_phys_fmr && device->ops.unmap_fmr);
158 	rds_ibdev->use_fastreg = (has_fr && !has_fmr);
159 	rds_ibdev->odp_capable =
160 		!!(device->attrs.device_cap_flags &
161 		   IB_DEVICE_ON_DEMAND_PAGING) &&
162 		!!(device->attrs.odp_caps.per_transport_caps.rc_odp_caps &
163 		   IB_ODP_SUPPORT_WRITE) &&
164 		!!(device->attrs.odp_caps.per_transport_caps.rc_odp_caps &
165 		   IB_ODP_SUPPORT_READ);
166 
167 	rds_ibdev->fmr_max_remaps = device->attrs.max_map_per_fmr?: 32;
168 	rds_ibdev->max_1m_mrs = device->attrs.max_mr ?
169 		min_t(unsigned int, (device->attrs.max_mr / 2),
170 		      rds_ib_mr_1m_pool_size) : rds_ib_mr_1m_pool_size;
171 
172 	rds_ibdev->max_8k_mrs = device->attrs.max_mr ?
173 		min_t(unsigned int, ((device->attrs.max_mr / 2) * RDS_MR_8K_SCALE),
174 		      rds_ib_mr_8k_pool_size) : rds_ib_mr_8k_pool_size;
175 
176 	rds_ibdev->max_initiator_depth = device->attrs.max_qp_init_rd_atom;
177 	rds_ibdev->max_responder_resources = device->attrs.max_qp_rd_atom;
178 
179 	rds_ibdev->vector_load = kcalloc(device->num_comp_vectors,
180 					 sizeof(int),
181 					 GFP_KERNEL);
182 	if (!rds_ibdev->vector_load) {
183 		pr_err("RDS/IB: %s failed to allocate vector memory\n",
184 			__func__);
185 		goto put_dev;
186 	}
187 
188 	rds_ibdev->dev = device;
189 	rds_ibdev->pd = ib_alloc_pd(device, 0);
190 	if (IS_ERR(rds_ibdev->pd)) {
191 		rds_ibdev->pd = NULL;
192 		goto put_dev;
193 	}
194 	rds_ibdev->rid_hdrs_pool = dma_pool_create(device->name,
195 						   device->dma_device,
196 						   sizeof(struct rds_header),
197 						   L1_CACHE_BYTES, 0);
198 	if (!rds_ibdev->rid_hdrs_pool)
199 		goto put_dev;
200 
201 	rds_ibdev->mr_1m_pool =
202 		rds_ib_create_mr_pool(rds_ibdev, RDS_IB_MR_1M_POOL);
203 	if (IS_ERR(rds_ibdev->mr_1m_pool)) {
204 		rds_ibdev->mr_1m_pool = NULL;
205 		goto put_dev;
206 	}
207 
208 	rds_ibdev->mr_8k_pool =
209 		rds_ib_create_mr_pool(rds_ibdev, RDS_IB_MR_8K_POOL);
210 	if (IS_ERR(rds_ibdev->mr_8k_pool)) {
211 		rds_ibdev->mr_8k_pool = NULL;
212 		goto put_dev;
213 	}
214 
215 	rdsdebug("RDS/IB: max_mr = %d, max_wrs = %d, max_sge = %d, fmr_max_remaps = %d, max_1m_mrs = %d, max_8k_mrs = %d\n",
216 		 device->attrs.max_fmr, rds_ibdev->max_wrs, rds_ibdev->max_sge,
217 		 rds_ibdev->fmr_max_remaps, rds_ibdev->max_1m_mrs,
218 		 rds_ibdev->max_8k_mrs);
219 
220 	pr_info("RDS/IB: %s: %s supported and preferred\n",
221 		device->name,
222 		rds_ibdev->use_fastreg ? "FRMR" : "FMR");
223 
224 	down_write(&rds_ib_devices_lock);
225 	list_add_tail_rcu(&rds_ibdev->list, &rds_ib_devices);
226 	up_write(&rds_ib_devices_lock);
227 	refcount_inc(&rds_ibdev->refcount);
228 
229 	ib_set_client_data(device, &rds_ib_client, rds_ibdev);
230 	refcount_inc(&rds_ibdev->refcount);
231 
232 	rds_ib_nodev_connect();
233 
234 put_dev:
235 	rds_ib_dev_put(rds_ibdev);
236 }
237 
238 /*
239  * New connections use this to find the device to associate with the
240  * connection.  It's not in the fast path so we're not concerned about the
241  * performance of the IB call.  (As of this writing, it uses an interrupt
242  * blocking spinlock to serialize walking a per-device list of all registered
243  * clients.)
244  *
245  * RCU is used to handle incoming connections racing with device teardown.
246  * Rather than use a lock to serialize removal from the client_data and
247  * getting a new reference, we use an RCU grace period.  The destruction
248  * path removes the device from client_data and then waits for all RCU
249  * readers to finish.
250  *
251  * A new connection can get NULL from this if its arriving on a
252  * device that is in the process of being removed.
253  */
254 struct rds_ib_device *rds_ib_get_client_data(struct ib_device *device)
255 {
256 	struct rds_ib_device *rds_ibdev;
257 
258 	rcu_read_lock();
259 	rds_ibdev = ib_get_client_data(device, &rds_ib_client);
260 	if (rds_ibdev)
261 		refcount_inc(&rds_ibdev->refcount);
262 	rcu_read_unlock();
263 	return rds_ibdev;
264 }
265 
266 /*
267  * The IB stack is letting us know that a device is going away.  This can
268  * happen if the underlying HCA driver is removed or if PCI hotplug is removing
269  * the pci function, for example.
270  *
271  * This can be called at any time and can be racing with any other RDS path.
272  */
273 static void rds_ib_remove_one(struct ib_device *device, void *client_data)
274 {
275 	struct rds_ib_device *rds_ibdev = client_data;
276 
277 	if (!rds_ibdev)
278 		return;
279 
280 	rds_ib_dev_shutdown(rds_ibdev);
281 
282 	/* stop connection attempts from getting a reference to this device. */
283 	ib_set_client_data(device, &rds_ib_client, NULL);
284 
285 	down_write(&rds_ib_devices_lock);
286 	list_del_rcu(&rds_ibdev->list);
287 	up_write(&rds_ib_devices_lock);
288 
289 	/*
290 	 * This synchronize rcu is waiting for readers of both the ib
291 	 * client data and the devices list to finish before we drop
292 	 * both of those references.
293 	 */
294 	synchronize_rcu();
295 	rds_ib_dev_put(rds_ibdev);
296 	rds_ib_dev_put(rds_ibdev);
297 }
298 
299 struct ib_client rds_ib_client = {
300 	.name   = "rds_ib",
301 	.add    = rds_ib_add_one,
302 	.remove = rds_ib_remove_one
303 };
304 
305 static int rds_ib_conn_info_visitor(struct rds_connection *conn,
306 				    void *buffer)
307 {
308 	struct rds_info_rdma_connection *iinfo = buffer;
309 	struct rds_ib_connection *ic = conn->c_transport_data;
310 
311 	/* We will only ever look at IB transports */
312 	if (conn->c_trans != &rds_ib_transport)
313 		return 0;
314 	if (conn->c_isv6)
315 		return 0;
316 
317 	iinfo->src_addr = conn->c_laddr.s6_addr32[3];
318 	iinfo->dst_addr = conn->c_faddr.s6_addr32[3];
319 	if (ic) {
320 		iinfo->tos = conn->c_tos;
321 		iinfo->sl = ic->i_sl;
322 	}
323 
324 	memset(&iinfo->src_gid, 0, sizeof(iinfo->src_gid));
325 	memset(&iinfo->dst_gid, 0, sizeof(iinfo->dst_gid));
326 	if (rds_conn_state(conn) == RDS_CONN_UP) {
327 		struct rds_ib_device *rds_ibdev;
328 
329 		rdma_read_gids(ic->i_cm_id, (union ib_gid *)&iinfo->src_gid,
330 			       (union ib_gid *)&iinfo->dst_gid);
331 
332 		rds_ibdev = ic->rds_ibdev;
333 		iinfo->max_send_wr = ic->i_send_ring.w_nr;
334 		iinfo->max_recv_wr = ic->i_recv_ring.w_nr;
335 		iinfo->max_send_sge = rds_ibdev->max_sge;
336 		rds_ib_get_mr_info(rds_ibdev, iinfo);
337 		iinfo->cache_allocs = atomic_read(&ic->i_cache_allocs);
338 	}
339 	return 1;
340 }
341 
342 #if IS_ENABLED(CONFIG_IPV6)
343 /* IPv6 version of rds_ib_conn_info_visitor(). */
344 static int rds6_ib_conn_info_visitor(struct rds_connection *conn,
345 				     void *buffer)
346 {
347 	struct rds6_info_rdma_connection *iinfo6 = buffer;
348 	struct rds_ib_connection *ic = conn->c_transport_data;
349 
350 	/* We will only ever look at IB transports */
351 	if (conn->c_trans != &rds_ib_transport)
352 		return 0;
353 
354 	iinfo6->src_addr = conn->c_laddr;
355 	iinfo6->dst_addr = conn->c_faddr;
356 	if (ic) {
357 		iinfo6->tos = conn->c_tos;
358 		iinfo6->sl = ic->i_sl;
359 	}
360 
361 	memset(&iinfo6->src_gid, 0, sizeof(iinfo6->src_gid));
362 	memset(&iinfo6->dst_gid, 0, sizeof(iinfo6->dst_gid));
363 
364 	if (rds_conn_state(conn) == RDS_CONN_UP) {
365 		struct rds_ib_device *rds_ibdev;
366 
367 		rdma_read_gids(ic->i_cm_id, (union ib_gid *)&iinfo6->src_gid,
368 			       (union ib_gid *)&iinfo6->dst_gid);
369 		rds_ibdev = ic->rds_ibdev;
370 		iinfo6->max_send_wr = ic->i_send_ring.w_nr;
371 		iinfo6->max_recv_wr = ic->i_recv_ring.w_nr;
372 		iinfo6->max_send_sge = rds_ibdev->max_sge;
373 		rds6_ib_get_mr_info(rds_ibdev, iinfo6);
374 		iinfo6->cache_allocs = atomic_read(&ic->i_cache_allocs);
375 	}
376 	return 1;
377 }
378 #endif
379 
380 static void rds_ib_ic_info(struct socket *sock, unsigned int len,
381 			   struct rds_info_iterator *iter,
382 			   struct rds_info_lengths *lens)
383 {
384 	u64 buffer[(sizeof(struct rds_info_rdma_connection) + 7) / 8];
385 
386 	rds_for_each_conn_info(sock, len, iter, lens,
387 				rds_ib_conn_info_visitor,
388 				buffer,
389 				sizeof(struct rds_info_rdma_connection));
390 }
391 
392 #if IS_ENABLED(CONFIG_IPV6)
393 /* IPv6 version of rds_ib_ic_info(). */
394 static void rds6_ib_ic_info(struct socket *sock, unsigned int len,
395 			    struct rds_info_iterator *iter,
396 			    struct rds_info_lengths *lens)
397 {
398 	u64 buffer[(sizeof(struct rds6_info_rdma_connection) + 7) / 8];
399 
400 	rds_for_each_conn_info(sock, len, iter, lens,
401 			       rds6_ib_conn_info_visitor,
402 			       buffer,
403 			       sizeof(struct rds6_info_rdma_connection));
404 }
405 #endif
406 
407 /*
408  * Early RDS/IB was built to only bind to an address if there is an IPoIB
409  * device with that address set.
410  *
411  * If it were me, I'd advocate for something more flexible.  Sending and
412  * receiving should be device-agnostic.  Transports would try and maintain
413  * connections between peers who have messages queued.  Userspace would be
414  * allowed to influence which paths have priority.  We could call userspace
415  * asserting this policy "routing".
416  */
417 static int rds_ib_laddr_check(struct net *net, const struct in6_addr *addr,
418 			      __u32 scope_id)
419 {
420 	int ret;
421 	struct rdma_cm_id *cm_id;
422 #if IS_ENABLED(CONFIG_IPV6)
423 	struct sockaddr_in6 sin6;
424 #endif
425 	struct sockaddr_in sin;
426 	struct sockaddr *sa;
427 	bool isv4;
428 
429 	isv4 = ipv6_addr_v4mapped(addr);
430 	/* Create a CMA ID and try to bind it. This catches both
431 	 * IB and iWARP capable NICs.
432 	 */
433 	cm_id = rdma_create_id(&init_net, rds_rdma_cm_event_handler,
434 			       NULL, RDMA_PS_TCP, IB_QPT_RC);
435 	if (IS_ERR(cm_id))
436 		return PTR_ERR(cm_id);
437 
438 	if (isv4) {
439 		memset(&sin, 0, sizeof(sin));
440 		sin.sin_family = AF_INET;
441 		sin.sin_addr.s_addr = addr->s6_addr32[3];
442 		sa = (struct sockaddr *)&sin;
443 	} else {
444 #if IS_ENABLED(CONFIG_IPV6)
445 		memset(&sin6, 0, sizeof(sin6));
446 		sin6.sin6_family = AF_INET6;
447 		sin6.sin6_addr = *addr;
448 		sin6.sin6_scope_id = scope_id;
449 		sa = (struct sockaddr *)&sin6;
450 
451 		/* XXX Do a special IPv6 link local address check here.  The
452 		 * reason is that rdma_bind_addr() always succeeds with IPv6
453 		 * link local address regardless it is indeed configured in a
454 		 * system.
455 		 */
456 		if (ipv6_addr_type(addr) & IPV6_ADDR_LINKLOCAL) {
457 			struct net_device *dev;
458 
459 			if (scope_id == 0) {
460 				ret = -EADDRNOTAVAIL;
461 				goto out;
462 			}
463 
464 			/* Use init_net for now as RDS is not network
465 			 * name space aware.
466 			 */
467 			dev = dev_get_by_index(&init_net, scope_id);
468 			if (!dev) {
469 				ret = -EADDRNOTAVAIL;
470 				goto out;
471 			}
472 			if (!ipv6_chk_addr(&init_net, addr, dev, 1)) {
473 				dev_put(dev);
474 				ret = -EADDRNOTAVAIL;
475 				goto out;
476 			}
477 			dev_put(dev);
478 		}
479 #else
480 		ret = -EADDRNOTAVAIL;
481 		goto out;
482 #endif
483 	}
484 
485 	/* rdma_bind_addr will only succeed for IB & iWARP devices */
486 	ret = rdma_bind_addr(cm_id, sa);
487 	/* due to this, we will claim to support iWARP devices unless we
488 	   check node_type. */
489 	if (ret || !cm_id->device ||
490 	    cm_id->device->node_type != RDMA_NODE_IB_CA)
491 		ret = -EADDRNOTAVAIL;
492 
493 	rdsdebug("addr %pI6c%%%u ret %d node type %d\n",
494 		 addr, scope_id, ret,
495 		 cm_id->device ? cm_id->device->node_type : -1);
496 
497 out:
498 	rdma_destroy_id(cm_id);
499 
500 	return ret;
501 }
502 
503 static void rds_ib_unregister_client(void)
504 {
505 	ib_unregister_client(&rds_ib_client);
506 	/* wait for rds_ib_dev_free() to complete */
507 	flush_workqueue(rds_wq);
508 }
509 
510 static void rds_ib_set_unloading(void)
511 {
512 	atomic_set(&rds_ib_unloading, 1);
513 }
514 
515 static bool rds_ib_is_unloading(struct rds_connection *conn)
516 {
517 	struct rds_conn_path *cp = &conn->c_path[0];
518 
519 	return (test_bit(RDS_DESTROY_PENDING, &cp->cp_flags) ||
520 		atomic_read(&rds_ib_unloading) != 0);
521 }
522 
523 void rds_ib_exit(void)
524 {
525 	rds_ib_set_unloading();
526 	synchronize_rcu();
527 	rds_info_deregister_func(RDS_INFO_IB_CONNECTIONS, rds_ib_ic_info);
528 #if IS_ENABLED(CONFIG_IPV6)
529 	rds_info_deregister_func(RDS6_INFO_IB_CONNECTIONS, rds6_ib_ic_info);
530 #endif
531 	rds_ib_unregister_client();
532 	rds_ib_destroy_nodev_conns();
533 	rds_ib_sysctl_exit();
534 	rds_ib_recv_exit();
535 	rds_trans_unregister(&rds_ib_transport);
536 	rds_ib_mr_exit();
537 }
538 
539 static u8 rds_ib_get_tos_map(u8 tos)
540 {
541 	/* 1:1 user to transport map for RDMA transport.
542 	 * In future, if custom map is desired, hook can export
543 	 * user configurable map.
544 	 */
545 	return tos;
546 }
547 
548 struct rds_transport rds_ib_transport = {
549 	.laddr_check		= rds_ib_laddr_check,
550 	.xmit_path_complete	= rds_ib_xmit_path_complete,
551 	.xmit			= rds_ib_xmit,
552 	.xmit_rdma		= rds_ib_xmit_rdma,
553 	.xmit_atomic		= rds_ib_xmit_atomic,
554 	.recv_path		= rds_ib_recv_path,
555 	.conn_alloc		= rds_ib_conn_alloc,
556 	.conn_free		= rds_ib_conn_free,
557 	.conn_path_connect	= rds_ib_conn_path_connect,
558 	.conn_path_shutdown	= rds_ib_conn_path_shutdown,
559 	.inc_copy_to_user	= rds_ib_inc_copy_to_user,
560 	.inc_free		= rds_ib_inc_free,
561 	.cm_initiate_connect	= rds_ib_cm_initiate_connect,
562 	.cm_handle_connect	= rds_ib_cm_handle_connect,
563 	.cm_connect_complete	= rds_ib_cm_connect_complete,
564 	.stats_info_copy	= rds_ib_stats_info_copy,
565 	.exit			= rds_ib_exit,
566 	.get_mr			= rds_ib_get_mr,
567 	.sync_mr		= rds_ib_sync_mr,
568 	.free_mr		= rds_ib_free_mr,
569 	.flush_mrs		= rds_ib_flush_mrs,
570 	.get_tos_map		= rds_ib_get_tos_map,
571 	.t_owner		= THIS_MODULE,
572 	.t_name			= "infiniband",
573 	.t_unloading		= rds_ib_is_unloading,
574 	.t_type			= RDS_TRANS_IB
575 };
576 
577 int rds_ib_init(void)
578 {
579 	int ret;
580 
581 	INIT_LIST_HEAD(&rds_ib_devices);
582 
583 	ret = rds_ib_mr_init();
584 	if (ret)
585 		goto out;
586 
587 	ret = ib_register_client(&rds_ib_client);
588 	if (ret)
589 		goto out_mr_exit;
590 
591 	ret = rds_ib_sysctl_init();
592 	if (ret)
593 		goto out_ibreg;
594 
595 	ret = rds_ib_recv_init();
596 	if (ret)
597 		goto out_sysctl;
598 
599 	rds_trans_register(&rds_ib_transport);
600 
601 	rds_info_register_func(RDS_INFO_IB_CONNECTIONS, rds_ib_ic_info);
602 #if IS_ENABLED(CONFIG_IPV6)
603 	rds_info_register_func(RDS6_INFO_IB_CONNECTIONS, rds6_ib_ic_info);
604 #endif
605 
606 	goto out;
607 
608 out_sysctl:
609 	rds_ib_sysctl_exit();
610 out_ibreg:
611 	rds_ib_unregister_client();
612 out_mr_exit:
613 	rds_ib_mr_exit();
614 out:
615 	return ret;
616 }
617 
618 MODULE_LICENSE("GPL");
619