xref: /openbmc/linux/net/rds/ib_cm.c (revision 5aa54bd2)
1 /*
2  * Copyright (c) 2006, 2019 Oracle and/or its affiliates. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  *
32  */
33 #include <linux/kernel.h>
34 #include <linux/in.h>
35 #include <linux/slab.h>
36 #include <linux/vmalloc.h>
37 #include <linux/ratelimit.h>
38 #include <net/addrconf.h>
39 #include <rdma/ib_cm.h>
40 
41 #include "rds_single_path.h"
42 #include "rds.h"
43 #include "ib.h"
44 #include "ib_mr.h"
45 
46 /*
47  * Set the selected protocol version
48  */
rds_ib_set_protocol(struct rds_connection * conn,unsigned int version)49 static void rds_ib_set_protocol(struct rds_connection *conn, unsigned int version)
50 {
51 	conn->c_version = version;
52 }
53 
54 /*
55  * Set up flow control
56  */
rds_ib_set_flow_control(struct rds_connection * conn,u32 credits)57 static void rds_ib_set_flow_control(struct rds_connection *conn, u32 credits)
58 {
59 	struct rds_ib_connection *ic = conn->c_transport_data;
60 
61 	if (rds_ib_sysctl_flow_control && credits != 0) {
62 		/* We're doing flow control */
63 		ic->i_flowctl = 1;
64 		rds_ib_send_add_credits(conn, credits);
65 	} else {
66 		ic->i_flowctl = 0;
67 	}
68 }
69 
70 /*
71  * Connection established.
72  * We get here for both outgoing and incoming connection.
73  */
rds_ib_cm_connect_complete(struct rds_connection * conn,struct rdma_cm_event * event)74 void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event)
75 {
76 	struct rds_ib_connection *ic = conn->c_transport_data;
77 	const union rds_ib_conn_priv *dp = NULL;
78 	__be64 ack_seq = 0;
79 	__be32 credit = 0;
80 	u8 major = 0;
81 	u8 minor = 0;
82 	int err;
83 
84 	dp = event->param.conn.private_data;
85 	if (conn->c_isv6) {
86 		if (event->param.conn.private_data_len >=
87 		    sizeof(struct rds6_ib_connect_private)) {
88 			major = dp->ricp_v6.dp_protocol_major;
89 			minor = dp->ricp_v6.dp_protocol_minor;
90 			credit = dp->ricp_v6.dp_credit;
91 			/* dp structure start is not guaranteed to be 8 bytes
92 			 * aligned.  Since dp_ack_seq is 64-bit extended load
93 			 * operations can be used so go through get_unaligned
94 			 * to avoid unaligned errors.
95 			 */
96 			ack_seq = get_unaligned(&dp->ricp_v6.dp_ack_seq);
97 		}
98 	} else if (event->param.conn.private_data_len >=
99 		   sizeof(struct rds_ib_connect_private)) {
100 		major = dp->ricp_v4.dp_protocol_major;
101 		minor = dp->ricp_v4.dp_protocol_minor;
102 		credit = dp->ricp_v4.dp_credit;
103 		ack_seq = get_unaligned(&dp->ricp_v4.dp_ack_seq);
104 	}
105 
106 	/* make sure it isn't empty data */
107 	if (major) {
108 		rds_ib_set_protocol(conn, RDS_PROTOCOL(major, minor));
109 		rds_ib_set_flow_control(conn, be32_to_cpu(credit));
110 	}
111 
112 	if (conn->c_version < RDS_PROTOCOL_VERSION) {
113 		if (conn->c_version != RDS_PROTOCOL_COMPAT_VERSION) {
114 			pr_notice("RDS/IB: Connection <%pI6c,%pI6c> version %u.%u no longer supported\n",
115 				  &conn->c_laddr, &conn->c_faddr,
116 				  RDS_PROTOCOL_MAJOR(conn->c_version),
117 				  RDS_PROTOCOL_MINOR(conn->c_version));
118 			rds_conn_destroy(conn);
119 			return;
120 		}
121 	}
122 
123 	pr_notice("RDS/IB: %s conn connected <%pI6c,%pI6c,%d> version %u.%u%s\n",
124 		  ic->i_active_side ? "Active" : "Passive",
125 		  &conn->c_laddr, &conn->c_faddr, conn->c_tos,
126 		  RDS_PROTOCOL_MAJOR(conn->c_version),
127 		  RDS_PROTOCOL_MINOR(conn->c_version),
128 		  ic->i_flowctl ? ", flow control" : "");
129 
130 	/* receive sl from the peer */
131 	ic->i_sl = ic->i_cm_id->route.path_rec->sl;
132 
133 	atomic_set(&ic->i_cq_quiesce, 0);
134 
135 	/* Init rings and fill recv. this needs to wait until protocol
136 	 * negotiation is complete, since ring layout is different
137 	 * from 3.1 to 4.1.
138 	 */
139 	rds_ib_send_init_ring(ic);
140 	rds_ib_recv_init_ring(ic);
141 	/* Post receive buffers - as a side effect, this will update
142 	 * the posted credit count. */
143 	rds_ib_recv_refill(conn, 1, GFP_KERNEL);
144 
145 	/* update ib_device with this local ipaddr */
146 	err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);
147 	if (err)
148 		printk(KERN_ERR "rds_ib_update_ipaddr failed (%d)\n",
149 			err);
150 
151 	/* If the peer gave us the last packet it saw, process this as if
152 	 * we had received a regular ACK. */
153 	if (dp) {
154 		if (ack_seq)
155 			rds_send_drop_acked(conn, be64_to_cpu(ack_seq),
156 					    NULL);
157 	}
158 
159 	conn->c_proposed_version = conn->c_version;
160 	rds_connect_complete(conn);
161 }
162 
rds_ib_cm_fill_conn_param(struct rds_connection * conn,struct rdma_conn_param * conn_param,union rds_ib_conn_priv * dp,u32 protocol_version,u32 max_responder_resources,u32 max_initiator_depth,bool isv6)163 static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
164 				      struct rdma_conn_param *conn_param,
165 				      union rds_ib_conn_priv *dp,
166 				      u32 protocol_version,
167 				      u32 max_responder_resources,
168 				      u32 max_initiator_depth,
169 				      bool isv6)
170 {
171 	struct rds_ib_connection *ic = conn->c_transport_data;
172 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
173 
174 	memset(conn_param, 0, sizeof(struct rdma_conn_param));
175 
176 	conn_param->responder_resources =
177 		min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
178 	conn_param->initiator_depth =
179 		min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
180 	conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
181 	conn_param->rnr_retry_count = 7;
182 
183 	if (dp) {
184 		memset(dp, 0, sizeof(*dp));
185 		if (isv6) {
186 			dp->ricp_v6.dp_saddr = conn->c_laddr;
187 			dp->ricp_v6.dp_daddr = conn->c_faddr;
188 			dp->ricp_v6.dp_protocol_major =
189 			    RDS_PROTOCOL_MAJOR(protocol_version);
190 			dp->ricp_v6.dp_protocol_minor =
191 			    RDS_PROTOCOL_MINOR(protocol_version);
192 			dp->ricp_v6.dp_protocol_minor_mask =
193 			    cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
194 			dp->ricp_v6.dp_ack_seq =
195 			    cpu_to_be64(rds_ib_piggyb_ack(ic));
196 			dp->ricp_v6.dp_cmn.ricpc_dp_toss = conn->c_tos;
197 
198 			conn_param->private_data = &dp->ricp_v6;
199 			conn_param->private_data_len = sizeof(dp->ricp_v6);
200 		} else {
201 			dp->ricp_v4.dp_saddr = conn->c_laddr.s6_addr32[3];
202 			dp->ricp_v4.dp_daddr = conn->c_faddr.s6_addr32[3];
203 			dp->ricp_v4.dp_protocol_major =
204 			    RDS_PROTOCOL_MAJOR(protocol_version);
205 			dp->ricp_v4.dp_protocol_minor =
206 			    RDS_PROTOCOL_MINOR(protocol_version);
207 			dp->ricp_v4.dp_protocol_minor_mask =
208 			    cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
209 			dp->ricp_v4.dp_ack_seq =
210 			    cpu_to_be64(rds_ib_piggyb_ack(ic));
211 			dp->ricp_v4.dp_cmn.ricpc_dp_toss = conn->c_tos;
212 
213 			conn_param->private_data = &dp->ricp_v4;
214 			conn_param->private_data_len = sizeof(dp->ricp_v4);
215 		}
216 
217 		/* Advertise flow control */
218 		if (ic->i_flowctl) {
219 			unsigned int credits;
220 
221 			credits = IB_GET_POST_CREDITS
222 				(atomic_read(&ic->i_credits));
223 			if (isv6)
224 				dp->ricp_v6.dp_credit = cpu_to_be32(credits);
225 			else
226 				dp->ricp_v4.dp_credit = cpu_to_be32(credits);
227 			atomic_sub(IB_SET_POST_CREDITS(credits),
228 				   &ic->i_credits);
229 		}
230 	}
231 }
232 
rds_ib_cq_event_handler(struct ib_event * event,void * data)233 static void rds_ib_cq_event_handler(struct ib_event *event, void *data)
234 {
235 	rdsdebug("event %u (%s) data %p\n",
236 		 event->event, ib_event_msg(event->event), data);
237 }
238 
239 /* Plucking the oldest entry from the ring can be done concurrently with
240  * the thread refilling the ring.  Each ring operation is protected by
241  * spinlocks and the transient state of refilling doesn't change the
242  * recording of which entry is oldest.
243  *
244  * This relies on IB only calling one cq comp_handler for each cq so that
245  * there will only be one caller of rds_recv_incoming() per RDS connection.
246  */
rds_ib_cq_comp_handler_recv(struct ib_cq * cq,void * context)247 static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
248 {
249 	struct rds_connection *conn = context;
250 	struct rds_ib_connection *ic = conn->c_transport_data;
251 
252 	rdsdebug("conn %p cq %p\n", conn, cq);
253 
254 	rds_ib_stats_inc(s_ib_evt_handler_call);
255 
256 	tasklet_schedule(&ic->i_recv_tasklet);
257 }
258 
poll_scq(struct rds_ib_connection * ic,struct ib_cq * cq,struct ib_wc * wcs)259 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
260 		     struct ib_wc *wcs)
261 {
262 	int nr, i;
263 	struct ib_wc *wc;
264 
265 	while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
266 		for (i = 0; i < nr; i++) {
267 			wc = wcs + i;
268 			rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
269 				 (unsigned long long)wc->wr_id, wc->status,
270 				 wc->byte_len, be32_to_cpu(wc->ex.imm_data));
271 
272 			if (wc->wr_id <= ic->i_send_ring.w_nr ||
273 			    wc->wr_id == RDS_IB_ACK_WR_ID)
274 				rds_ib_send_cqe_handler(ic, wc);
275 			else
276 				rds_ib_mr_cqe_handler(ic, wc);
277 
278 		}
279 	}
280 }
281 
rds_ib_tasklet_fn_send(unsigned long data)282 static void rds_ib_tasklet_fn_send(unsigned long data)
283 {
284 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
285 	struct rds_connection *conn = ic->conn;
286 
287 	rds_ib_stats_inc(s_ib_tasklet_call);
288 
289 	/* if cq has been already reaped, ignore incoming cq event */
290 	if (atomic_read(&ic->i_cq_quiesce))
291 		return;
292 
293 	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
294 	ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
295 	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
296 
297 	if (rds_conn_up(conn) &&
298 	    (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags) ||
299 	    test_bit(0, &conn->c_map_queued)))
300 		rds_send_xmit(&ic->conn->c_path[0]);
301 }
302 
poll_rcq(struct rds_ib_connection * ic,struct ib_cq * cq,struct ib_wc * wcs,struct rds_ib_ack_state * ack_state)303 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
304 		     struct ib_wc *wcs,
305 		     struct rds_ib_ack_state *ack_state)
306 {
307 	int nr, i;
308 	struct ib_wc *wc;
309 
310 	while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
311 		for (i = 0; i < nr; i++) {
312 			wc = wcs + i;
313 			rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
314 				 (unsigned long long)wc->wr_id, wc->status,
315 				 wc->byte_len, be32_to_cpu(wc->ex.imm_data));
316 
317 			rds_ib_recv_cqe_handler(ic, wc, ack_state);
318 		}
319 	}
320 }
321 
rds_ib_tasklet_fn_recv(unsigned long data)322 static void rds_ib_tasklet_fn_recv(unsigned long data)
323 {
324 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
325 	struct rds_connection *conn = ic->conn;
326 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
327 	struct rds_ib_ack_state state;
328 
329 	if (!rds_ibdev)
330 		rds_conn_drop(conn);
331 
332 	rds_ib_stats_inc(s_ib_tasklet_call);
333 
334 	/* if cq has been already reaped, ignore incoming cq event */
335 	if (atomic_read(&ic->i_cq_quiesce))
336 		return;
337 
338 	memset(&state, 0, sizeof(state));
339 	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
340 	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
341 	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
342 
343 	if (state.ack_next_valid)
344 		rds_ib_set_ack(ic, state.ack_next, state.ack_required);
345 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
346 		rds_send_drop_acked(conn, state.ack_recv, NULL);
347 		ic->i_ack_recv = state.ack_recv;
348 	}
349 
350 	if (rds_conn_up(conn))
351 		rds_ib_attempt_ack(ic);
352 }
353 
rds_ib_qp_event_handler(struct ib_event * event,void * data)354 static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
355 {
356 	struct rds_connection *conn = data;
357 	struct rds_ib_connection *ic = conn->c_transport_data;
358 
359 	rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,
360 		 ib_event_msg(event->event));
361 
362 	switch (event->event) {
363 	case IB_EVENT_COMM_EST:
364 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
365 		break;
366 	default:
367 		rdsdebug("Fatal QP Event %u (%s) - connection %pI6c->%pI6c, reconnecting\n",
368 			 event->event, ib_event_msg(event->event),
369 			 &conn->c_laddr, &conn->c_faddr);
370 		rds_conn_drop(conn);
371 		break;
372 	}
373 }
374 
rds_ib_cq_comp_handler_send(struct ib_cq * cq,void * context)375 static void rds_ib_cq_comp_handler_send(struct ib_cq *cq, void *context)
376 {
377 	struct rds_connection *conn = context;
378 	struct rds_ib_connection *ic = conn->c_transport_data;
379 
380 	rdsdebug("conn %p cq %p\n", conn, cq);
381 
382 	rds_ib_stats_inc(s_ib_evt_handler_call);
383 
384 	tasklet_schedule(&ic->i_send_tasklet);
385 }
386 
ibdev_get_unused_vector(struct rds_ib_device * rds_ibdev)387 static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev)
388 {
389 	int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1];
390 	int index = rds_ibdev->dev->num_comp_vectors - 1;
391 	int i;
392 
393 	for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) {
394 		if (rds_ibdev->vector_load[i] < min) {
395 			index = i;
396 			min = rds_ibdev->vector_load[i];
397 		}
398 	}
399 
400 	rds_ibdev->vector_load[index]++;
401 	return index;
402 }
403 
ibdev_put_vector(struct rds_ib_device * rds_ibdev,int index)404 static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index)
405 {
406 	rds_ibdev->vector_load[index]--;
407 }
408 
rds_dma_hdr_free(struct ib_device * dev,struct rds_header * hdr,dma_addr_t dma_addr,enum dma_data_direction dir)409 static void rds_dma_hdr_free(struct ib_device *dev, struct rds_header *hdr,
410 		dma_addr_t dma_addr, enum dma_data_direction dir)
411 {
412 	ib_dma_unmap_single(dev, dma_addr, sizeof(*hdr), dir);
413 	kfree(hdr);
414 }
415 
rds_dma_hdr_alloc(struct ib_device * dev,dma_addr_t * dma_addr,enum dma_data_direction dir)416 static struct rds_header *rds_dma_hdr_alloc(struct ib_device *dev,
417 		dma_addr_t *dma_addr, enum dma_data_direction dir)
418 {
419 	struct rds_header *hdr;
420 
421 	hdr = kzalloc_node(sizeof(*hdr), GFP_KERNEL, ibdev_to_node(dev));
422 	if (!hdr)
423 		return NULL;
424 
425 	*dma_addr = ib_dma_map_single(dev, hdr, sizeof(*hdr),
426 				      DMA_BIDIRECTIONAL);
427 	if (ib_dma_mapping_error(dev, *dma_addr)) {
428 		kfree(hdr);
429 		return NULL;
430 	}
431 
432 	return hdr;
433 }
434 
435 /* Free the DMA memory used to store struct rds_header.
436  *
437  * @dev: the RDS IB device
438  * @hdrs: pointer to the array storing DMA memory pointers
439  * @dma_addrs: pointer to the array storing DMA addresses
440  * @num_hdars: number of headers to free.
441  */
rds_dma_hdrs_free(struct rds_ib_device * dev,struct rds_header ** hdrs,dma_addr_t * dma_addrs,u32 num_hdrs,enum dma_data_direction dir)442 static void rds_dma_hdrs_free(struct rds_ib_device *dev,
443 		struct rds_header **hdrs, dma_addr_t *dma_addrs, u32 num_hdrs,
444 		enum dma_data_direction dir)
445 {
446 	u32 i;
447 
448 	for (i = 0; i < num_hdrs; i++)
449 		rds_dma_hdr_free(dev->dev, hdrs[i], dma_addrs[i], dir);
450 	kvfree(hdrs);
451 	kvfree(dma_addrs);
452 }
453 
454 
455 /* Allocate DMA coherent memory to be used to store struct rds_header for
456  * sending/receiving packets.  The pointers to the DMA memory and the
457  * associated DMA addresses are stored in two arrays.
458  *
459  * @dev: the RDS IB device
460  * @dma_addrs: pointer to the array for storing DMA addresses
461  * @num_hdrs: number of headers to allocate
462  *
463  * It returns the pointer to the array storing the DMA memory pointers.  On
464  * error, NULL pointer is returned.
465  */
rds_dma_hdrs_alloc(struct rds_ib_device * dev,dma_addr_t ** dma_addrs,u32 num_hdrs,enum dma_data_direction dir)466 static struct rds_header **rds_dma_hdrs_alloc(struct rds_ib_device *dev,
467 		dma_addr_t **dma_addrs, u32 num_hdrs,
468 		enum dma_data_direction dir)
469 {
470 	struct rds_header **hdrs;
471 	dma_addr_t *hdr_daddrs;
472 	u32 i;
473 
474 	hdrs = kvmalloc_node(sizeof(*hdrs) * num_hdrs, GFP_KERNEL,
475 			     ibdev_to_node(dev->dev));
476 	if (!hdrs)
477 		return NULL;
478 
479 	hdr_daddrs = kvmalloc_node(sizeof(*hdr_daddrs) * num_hdrs, GFP_KERNEL,
480 				   ibdev_to_node(dev->dev));
481 	if (!hdr_daddrs) {
482 		kvfree(hdrs);
483 		return NULL;
484 	}
485 
486 	for (i = 0; i < num_hdrs; i++) {
487 		hdrs[i] = rds_dma_hdr_alloc(dev->dev, &hdr_daddrs[i], dir);
488 		if (!hdrs[i]) {
489 			rds_dma_hdrs_free(dev, hdrs, hdr_daddrs, i, dir);
490 			return NULL;
491 		}
492 	}
493 
494 	*dma_addrs = hdr_daddrs;
495 	return hdrs;
496 }
497 
498 /*
499  * This needs to be very careful to not leave IS_ERR pointers around for
500  * cleanup to trip over.
501  */
rds_ib_setup_qp(struct rds_connection * conn)502 static int rds_ib_setup_qp(struct rds_connection *conn)
503 {
504 	struct rds_ib_connection *ic = conn->c_transport_data;
505 	struct ib_device *dev = ic->i_cm_id->device;
506 	struct ib_qp_init_attr attr;
507 	struct ib_cq_init_attr cq_attr = {};
508 	struct rds_ib_device *rds_ibdev;
509 	unsigned long max_wrs;
510 	int ret, fr_queue_space;
511 
512 	/*
513 	 * It's normal to see a null device if an incoming connection races
514 	 * with device removal, so we don't print a warning.
515 	 */
516 	rds_ibdev = rds_ib_get_client_data(dev);
517 	if (!rds_ibdev)
518 		return -EOPNOTSUPP;
519 
520 	/* The fr_queue_space is currently set to 512, to add extra space on
521 	 * completion queue and send queue. This extra space is used for FRWR
522 	 * registration and invalidation work requests
523 	 */
524 	fr_queue_space = RDS_IB_DEFAULT_FR_WR;
525 
526 	/* add the conn now so that connection establishment has the dev */
527 	rds_ib_add_conn(rds_ibdev, conn);
528 
529 	max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_send_wr + 1 ?
530 		rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_send_wr;
531 	if (ic->i_send_ring.w_nr != max_wrs)
532 		rds_ib_ring_resize(&ic->i_send_ring, max_wrs);
533 
534 	max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_recv_wr + 1 ?
535 		rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_recv_wr;
536 	if (ic->i_recv_ring.w_nr != max_wrs)
537 		rds_ib_ring_resize(&ic->i_recv_ring, max_wrs);
538 
539 	/* Protection domain and memory range */
540 	ic->i_pd = rds_ibdev->pd;
541 
542 	ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
543 	cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
544 	cq_attr.comp_vector = ic->i_scq_vector;
545 	ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
546 				     rds_ib_cq_event_handler, conn,
547 				     &cq_attr);
548 	if (IS_ERR(ic->i_send_cq)) {
549 		ret = PTR_ERR(ic->i_send_cq);
550 		ic->i_send_cq = NULL;
551 		ibdev_put_vector(rds_ibdev, ic->i_scq_vector);
552 		rdsdebug("ib_create_cq send failed: %d\n", ret);
553 		goto rds_ibdev_out;
554 	}
555 
556 	ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
557 	cq_attr.cqe = ic->i_recv_ring.w_nr;
558 	cq_attr.comp_vector = ic->i_rcq_vector;
559 	ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
560 				     rds_ib_cq_event_handler, conn,
561 				     &cq_attr);
562 	if (IS_ERR(ic->i_recv_cq)) {
563 		ret = PTR_ERR(ic->i_recv_cq);
564 		ic->i_recv_cq = NULL;
565 		ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);
566 		rdsdebug("ib_create_cq recv failed: %d\n", ret);
567 		goto send_cq_out;
568 	}
569 
570 	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
571 	if (ret) {
572 		rdsdebug("ib_req_notify_cq send failed: %d\n", ret);
573 		goto recv_cq_out;
574 	}
575 
576 	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
577 	if (ret) {
578 		rdsdebug("ib_req_notify_cq recv failed: %d\n", ret);
579 		goto recv_cq_out;
580 	}
581 
582 	/* XXX negotiate max send/recv with remote? */
583 	memset(&attr, 0, sizeof(attr));
584 	attr.event_handler = rds_ib_qp_event_handler;
585 	attr.qp_context = conn;
586 	/* + 1 to allow for the single ack message */
587 	attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;
588 	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
589 	attr.cap.max_send_sge = rds_ibdev->max_sge;
590 	attr.cap.max_recv_sge = RDS_IB_RECV_SGE;
591 	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
592 	attr.qp_type = IB_QPT_RC;
593 	attr.send_cq = ic->i_send_cq;
594 	attr.recv_cq = ic->i_recv_cq;
595 
596 	/*
597 	 * XXX this can fail if max_*_wr is too large?  Are we supposed
598 	 * to back off until we get a value that the hardware can support?
599 	 */
600 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
601 	if (ret) {
602 		rdsdebug("rdma_create_qp failed: %d\n", ret);
603 		goto recv_cq_out;
604 	}
605 
606 	ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma,
607 					     ic->i_send_ring.w_nr,
608 					     DMA_TO_DEVICE);
609 	if (!ic->i_send_hdrs) {
610 		ret = -ENOMEM;
611 		rdsdebug("DMA send hdrs alloc failed\n");
612 		goto qp_out;
613 	}
614 
615 	ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma,
616 					     ic->i_recv_ring.w_nr,
617 					     DMA_FROM_DEVICE);
618 	if (!ic->i_recv_hdrs) {
619 		ret = -ENOMEM;
620 		rdsdebug("DMA recv hdrs alloc failed\n");
621 		goto send_hdrs_dma_out;
622 	}
623 
624 	ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma,
625 				      DMA_TO_DEVICE);
626 	if (!ic->i_ack) {
627 		ret = -ENOMEM;
628 		rdsdebug("DMA ack header alloc failed\n");
629 		goto recv_hdrs_dma_out;
630 	}
631 
632 	ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),
633 					      ic->i_send_ring.w_nr),
634 				   ibdev_to_node(dev));
635 	if (!ic->i_sends) {
636 		ret = -ENOMEM;
637 		rdsdebug("send allocation failed\n");
638 		goto ack_dma_out;
639 	}
640 
641 	ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),
642 					      ic->i_recv_ring.w_nr),
643 				   ibdev_to_node(dev));
644 	if (!ic->i_recvs) {
645 		ret = -ENOMEM;
646 		rdsdebug("recv allocation failed\n");
647 		goto sends_out;
648 	}
649 
650 	rds_ib_recv_init_ack(ic);
651 
652 	rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,
653 		 ic->i_send_cq, ic->i_recv_cq);
654 
655 	goto out;
656 
657 sends_out:
658 	vfree(ic->i_sends);
659 
660 ack_dma_out:
661 	rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma,
662 			 DMA_TO_DEVICE);
663 	ic->i_ack = NULL;
664 
665 recv_hdrs_dma_out:
666 	rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma,
667 			  ic->i_recv_ring.w_nr, DMA_FROM_DEVICE);
668 	ic->i_recv_hdrs = NULL;
669 	ic->i_recv_hdrs_dma = NULL;
670 
671 send_hdrs_dma_out:
672 	rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma,
673 			  ic->i_send_ring.w_nr, DMA_TO_DEVICE);
674 	ic->i_send_hdrs = NULL;
675 	ic->i_send_hdrs_dma = NULL;
676 
677 qp_out:
678 	rdma_destroy_qp(ic->i_cm_id);
679 recv_cq_out:
680 	ib_destroy_cq(ic->i_recv_cq);
681 	ic->i_recv_cq = NULL;
682 send_cq_out:
683 	ib_destroy_cq(ic->i_send_cq);
684 	ic->i_send_cq = NULL;
685 rds_ibdev_out:
686 	rds_ib_remove_conn(rds_ibdev, conn);
687 out:
688 	rds_ib_dev_put(rds_ibdev);
689 
690 	return ret;
691 }
692 
rds_ib_protocol_compatible(struct rdma_cm_event * event,bool isv6)693 static u32 rds_ib_protocol_compatible(struct rdma_cm_event *event, bool isv6)
694 {
695 	const union rds_ib_conn_priv *dp = event->param.conn.private_data;
696 	u8 data_len, major, minor;
697 	u32 version = 0;
698 	__be16 mask;
699 	u16 common;
700 
701 	/*
702 	 * rdma_cm private data is odd - when there is any private data in the
703 	 * request, we will be given a pretty large buffer without telling us the
704 	 * original size. The only way to tell the difference is by looking at
705 	 * the contents, which are initialized to zero.
706 	 * If the protocol version fields aren't set, this is a connection attempt
707 	 * from an older version. This could be 3.0 or 2.0 - we can't tell.
708 	 * We really should have changed this for OFED 1.3 :-(
709 	 */
710 
711 	/* Be paranoid. RDS always has privdata */
712 	if (!event->param.conn.private_data_len) {
713 		printk(KERN_NOTICE "RDS incoming connection has no private data, "
714 			"rejecting\n");
715 		return 0;
716 	}
717 
718 	if (isv6) {
719 		data_len = sizeof(struct rds6_ib_connect_private);
720 		major = dp->ricp_v6.dp_protocol_major;
721 		minor = dp->ricp_v6.dp_protocol_minor;
722 		mask = dp->ricp_v6.dp_protocol_minor_mask;
723 	} else {
724 		data_len = sizeof(struct rds_ib_connect_private);
725 		major = dp->ricp_v4.dp_protocol_major;
726 		minor = dp->ricp_v4.dp_protocol_minor;
727 		mask = dp->ricp_v4.dp_protocol_minor_mask;
728 	}
729 
730 	/* Even if len is crap *now* I still want to check it. -ASG */
731 	if (event->param.conn.private_data_len < data_len || major == 0)
732 		return RDS_PROTOCOL_4_0;
733 
734 	common = be16_to_cpu(mask) & RDS_IB_SUPPORTED_PROTOCOLS;
735 	if (major == 4 && common) {
736 		version = RDS_PROTOCOL_4_0;
737 		while ((common >>= 1) != 0)
738 			version++;
739 	} else if (RDS_PROTOCOL_COMPAT_VERSION ==
740 		   RDS_PROTOCOL(major, minor)) {
741 		version = RDS_PROTOCOL_COMPAT_VERSION;
742 	} else {
743 		if (isv6)
744 			printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI6c using incompatible protocol version %u.%u\n",
745 					   &dp->ricp_v6.dp_saddr, major, minor);
746 		else
747 			printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI4 using incompatible protocol version %u.%u\n",
748 					   &dp->ricp_v4.dp_saddr, major, minor);
749 	}
750 	return version;
751 }
752 
753 #if IS_ENABLED(CONFIG_IPV6)
754 /* Given an IPv6 address, find the net_device which hosts that address and
755  * return its index.  This is used by the rds_ib_cm_handle_connect() code to
756  * find the interface index of where an incoming request comes from when
757  * the request is using a link local address.
758  *
759  * Note one problem in this search.  It is possible that two interfaces have
760  * the same link local address.  Unfortunately, this cannot be solved unless
761  * the underlying layer gives us the interface which an incoming RDMA connect
762  * request comes from.
763  */
__rds_find_ifindex(struct net * net,const struct in6_addr * addr)764 static u32 __rds_find_ifindex(struct net *net, const struct in6_addr *addr)
765 {
766 	struct net_device *dev;
767 	int idx = 0;
768 
769 	rcu_read_lock();
770 	for_each_netdev_rcu(net, dev) {
771 		if (ipv6_chk_addr(net, addr, dev, 1)) {
772 			idx = dev->ifindex;
773 			break;
774 		}
775 	}
776 	rcu_read_unlock();
777 
778 	return idx;
779 }
780 #endif
781 
rds_ib_cm_handle_connect(struct rdma_cm_id * cm_id,struct rdma_cm_event * event,bool isv6)782 int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
783 			     struct rdma_cm_event *event, bool isv6)
784 {
785 	__be64 lguid = cm_id->route.path_rec->sgid.global.interface_id;
786 	__be64 fguid = cm_id->route.path_rec->dgid.global.interface_id;
787 	const struct rds_ib_conn_priv_cmn *dp_cmn;
788 	struct rds_connection *conn = NULL;
789 	struct rds_ib_connection *ic = NULL;
790 	struct rdma_conn_param conn_param;
791 	const union rds_ib_conn_priv *dp;
792 	union rds_ib_conn_priv dp_rep;
793 	struct in6_addr s_mapped_addr;
794 	struct in6_addr d_mapped_addr;
795 	const struct in6_addr *saddr6;
796 	const struct in6_addr *daddr6;
797 	int destroy = 1;
798 	u32 ifindex = 0;
799 	u32 version;
800 	int err = 1;
801 
802 	/* Check whether the remote protocol version matches ours. */
803 	version = rds_ib_protocol_compatible(event, isv6);
804 	if (!version) {
805 		err = RDS_RDMA_REJ_INCOMPAT;
806 		goto out;
807 	}
808 
809 	dp = event->param.conn.private_data;
810 	if (isv6) {
811 #if IS_ENABLED(CONFIG_IPV6)
812 		dp_cmn = &dp->ricp_v6.dp_cmn;
813 		saddr6 = &dp->ricp_v6.dp_saddr;
814 		daddr6 = &dp->ricp_v6.dp_daddr;
815 		/* If either address is link local, need to find the
816 		 * interface index in order to create a proper RDS
817 		 * connection.
818 		 */
819 		if (ipv6_addr_type(daddr6) & IPV6_ADDR_LINKLOCAL) {
820 			/* Using init_net for now ..  */
821 			ifindex = __rds_find_ifindex(&init_net, daddr6);
822 			/* No index found...  Need to bail out. */
823 			if (ifindex == 0) {
824 				err = -EOPNOTSUPP;
825 				goto out;
826 			}
827 		} else if (ipv6_addr_type(saddr6) & IPV6_ADDR_LINKLOCAL) {
828 			/* Use our address to find the correct index. */
829 			ifindex = __rds_find_ifindex(&init_net, daddr6);
830 			/* No index found...  Need to bail out. */
831 			if (ifindex == 0) {
832 				err = -EOPNOTSUPP;
833 				goto out;
834 			}
835 		}
836 #else
837 		err = -EOPNOTSUPP;
838 		goto out;
839 #endif
840 	} else {
841 		dp_cmn = &dp->ricp_v4.dp_cmn;
842 		ipv6_addr_set_v4mapped(dp->ricp_v4.dp_saddr, &s_mapped_addr);
843 		ipv6_addr_set_v4mapped(dp->ricp_v4.dp_daddr, &d_mapped_addr);
844 		saddr6 = &s_mapped_addr;
845 		daddr6 = &d_mapped_addr;
846 	}
847 
848 	rdsdebug("saddr %pI6c daddr %pI6c RDSv%u.%u lguid 0x%llx fguid 0x%llx, tos:%d\n",
849 		 saddr6, daddr6, RDS_PROTOCOL_MAJOR(version),
850 		 RDS_PROTOCOL_MINOR(version),
851 		 (unsigned long long)be64_to_cpu(lguid),
852 		 (unsigned long long)be64_to_cpu(fguid), dp_cmn->ricpc_dp_toss);
853 
854 	/* RDS/IB is not currently netns aware, thus init_net */
855 	conn = rds_conn_create(&init_net, daddr6, saddr6,
856 			       &rds_ib_transport, dp_cmn->ricpc_dp_toss,
857 			       GFP_KERNEL, ifindex);
858 	if (IS_ERR(conn)) {
859 		rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
860 		conn = NULL;
861 		goto out;
862 	}
863 
864 	/*
865 	 * The connection request may occur while the
866 	 * previous connection exist, e.g. in case of failover.
867 	 * But as connections may be initiated simultaneously
868 	 * by both hosts, we have a random backoff mechanism -
869 	 * see the comment above rds_queue_reconnect()
870 	 */
871 	mutex_lock(&conn->c_cm_lock);
872 	if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
873 		if (rds_conn_state(conn) == RDS_CONN_UP) {
874 			rdsdebug("incoming connect while connecting\n");
875 			rds_conn_drop(conn);
876 			rds_ib_stats_inc(s_ib_listen_closed_stale);
877 		} else
878 		if (rds_conn_state(conn) == RDS_CONN_CONNECTING) {
879 			/* Wait and see - our connect may still be succeeding */
880 			rds_ib_stats_inc(s_ib_connect_raced);
881 		}
882 		goto out;
883 	}
884 
885 	ic = conn->c_transport_data;
886 
887 	rds_ib_set_protocol(conn, version);
888 	rds_ib_set_flow_control(conn, be32_to_cpu(dp_cmn->ricpc_credit));
889 
890 	/* If the peer gave us the last packet it saw, process this as if
891 	 * we had received a regular ACK. */
892 	if (dp_cmn->ricpc_ack_seq)
893 		rds_send_drop_acked(conn, be64_to_cpu(dp_cmn->ricpc_ack_seq),
894 				    NULL);
895 
896 	BUG_ON(cm_id->context);
897 	BUG_ON(ic->i_cm_id);
898 
899 	ic->i_cm_id = cm_id;
900 	cm_id->context = conn;
901 
902 	/* We got halfway through setting up the ib_connection, if we
903 	 * fail now, we have to take the long route out of this mess. */
904 	destroy = 0;
905 
906 	err = rds_ib_setup_qp(conn);
907 	if (err) {
908 		rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", err);
909 		goto out;
910 	}
911 
912 	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
913 				  event->param.conn.responder_resources,
914 				  event->param.conn.initiator_depth, isv6);
915 
916 	rdma_set_min_rnr_timer(cm_id, IB_RNR_TIMER_000_32);
917 	/* rdma_accept() calls rdma_reject() internally if it fails */
918 	if (rdma_accept(cm_id, &conn_param))
919 		rds_ib_conn_error(conn, "rdma_accept failed\n");
920 
921 out:
922 	if (conn)
923 		mutex_unlock(&conn->c_cm_lock);
924 	if (err)
925 		rdma_reject(cm_id, &err, sizeof(int),
926 			    IB_CM_REJ_CONSUMER_DEFINED);
927 	return destroy;
928 }
929 
930 
rds_ib_cm_initiate_connect(struct rdma_cm_id * cm_id,bool isv6)931 int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6)
932 {
933 	struct rds_connection *conn = cm_id->context;
934 	struct rds_ib_connection *ic = conn->c_transport_data;
935 	struct rdma_conn_param conn_param;
936 	union rds_ib_conn_priv dp;
937 	int ret;
938 
939 	/* If the peer doesn't do protocol negotiation, we must
940 	 * default to RDSv3.0 */
941 	rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
942 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */
943 
944 	ret = rds_ib_setup_qp(conn);
945 	if (ret) {
946 		rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", ret);
947 		goto out;
948 	}
949 
950 	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp,
951 				  conn->c_proposed_version,
952 				  UINT_MAX, UINT_MAX, isv6);
953 	ret = rdma_connect_locked(cm_id, &conn_param);
954 	if (ret)
955 		rds_ib_conn_error(conn, "rdma_connect_locked failed (%d)\n",
956 				  ret);
957 
958 out:
959 	/* Beware - returning non-zero tells the rdma_cm to destroy
960 	 * the cm_id. We should certainly not do it as long as we still
961 	 * "own" the cm_id. */
962 	if (ret) {
963 		if (ic->i_cm_id == cm_id)
964 			ret = 0;
965 	}
966 	ic->i_active_side = true;
967 	return ret;
968 }
969 
rds_ib_conn_path_connect(struct rds_conn_path * cp)970 int rds_ib_conn_path_connect(struct rds_conn_path *cp)
971 {
972 	struct rds_connection *conn = cp->cp_conn;
973 	struct sockaddr_storage src, dest;
974 	rdma_cm_event_handler handler;
975 	struct rds_ib_connection *ic;
976 	int ret;
977 
978 	ic = conn->c_transport_data;
979 
980 	/* XXX I wonder what affect the port space has */
981 	/* delegate cm event handler to rdma_transport */
982 #if IS_ENABLED(CONFIG_IPV6)
983 	if (conn->c_isv6)
984 		handler = rds6_rdma_cm_event_handler;
985 	else
986 #endif
987 		handler = rds_rdma_cm_event_handler;
988 	ic->i_cm_id = rdma_create_id(&init_net, handler, conn,
989 				     RDMA_PS_TCP, IB_QPT_RC);
990 	if (IS_ERR(ic->i_cm_id)) {
991 		ret = PTR_ERR(ic->i_cm_id);
992 		ic->i_cm_id = NULL;
993 		rdsdebug("rdma_create_id() failed: %d\n", ret);
994 		goto out;
995 	}
996 
997 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);
998 
999 	if (ipv6_addr_v4mapped(&conn->c_faddr)) {
1000 		struct sockaddr_in *sin;
1001 
1002 		sin = (struct sockaddr_in *)&src;
1003 		sin->sin_family = AF_INET;
1004 		sin->sin_addr.s_addr = conn->c_laddr.s6_addr32[3];
1005 		sin->sin_port = 0;
1006 
1007 		sin = (struct sockaddr_in *)&dest;
1008 		sin->sin_family = AF_INET;
1009 		sin->sin_addr.s_addr = conn->c_faddr.s6_addr32[3];
1010 		sin->sin_port = htons(RDS_PORT);
1011 	} else {
1012 		struct sockaddr_in6 *sin6;
1013 
1014 		sin6 = (struct sockaddr_in6 *)&src;
1015 		sin6->sin6_family = AF_INET6;
1016 		sin6->sin6_addr = conn->c_laddr;
1017 		sin6->sin6_port = 0;
1018 		sin6->sin6_scope_id = conn->c_dev_if;
1019 
1020 		sin6 = (struct sockaddr_in6 *)&dest;
1021 		sin6->sin6_family = AF_INET6;
1022 		sin6->sin6_addr = conn->c_faddr;
1023 		sin6->sin6_port = htons(RDS_CM_PORT);
1024 		sin6->sin6_scope_id = conn->c_dev_if;
1025 	}
1026 
1027 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
1028 				(struct sockaddr *)&dest,
1029 				RDS_RDMA_RESOLVE_TIMEOUT_MS);
1030 	if (ret) {
1031 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
1032 			 ret);
1033 		rdma_destroy_id(ic->i_cm_id);
1034 		ic->i_cm_id = NULL;
1035 	}
1036 
1037 out:
1038 	return ret;
1039 }
1040 
1041 /*
1042  * This is so careful about only cleaning up resources that were built up
1043  * so that it can be called at any point during startup.  In fact it
1044  * can be called multiple times for a given connection.
1045  */
rds_ib_conn_path_shutdown(struct rds_conn_path * cp)1046 void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
1047 {
1048 	struct rds_connection *conn = cp->cp_conn;
1049 	struct rds_ib_connection *ic = conn->c_transport_data;
1050 	int err = 0;
1051 
1052 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
1053 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
1054 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);
1055 
1056 	if (ic->i_cm_id) {
1057 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
1058 		err = rdma_disconnect(ic->i_cm_id);
1059 		if (err) {
1060 			/* Actually this may happen quite frequently, when
1061 			 * an outgoing connect raced with an incoming connect.
1062 			 */
1063 			rdsdebug("failed to disconnect, cm: %p err %d\n",
1064 				ic->i_cm_id, err);
1065 		}
1066 
1067 		/* kick off "flush_worker" for all pools in order to reap
1068 		 * all FRMR registrations that are still marked "FRMR_IS_INUSE"
1069 		 */
1070 		rds_ib_flush_mrs();
1071 
1072 		/*
1073 		 * We want to wait for tx and rx completion to finish
1074 		 * before we tear down the connection, but we have to be
1075 		 * careful not to get stuck waiting on a send ring that
1076 		 * only has unsignaled sends in it.  We've shutdown new
1077 		 * sends before getting here so by waiting for signaled
1078 		 * sends to complete we're ensured that there will be no
1079 		 * more tx processing.
1080 		 */
1081 		wait_event(rds_ib_ring_empty_wait,
1082 			   rds_ib_ring_empty(&ic->i_recv_ring) &&
1083 			   (atomic_read(&ic->i_signaled_sends) == 0) &&
1084 			   (atomic_read(&ic->i_fastreg_inuse_count) == 0) &&
1085 			   (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));
1086 		tasklet_kill(&ic->i_send_tasklet);
1087 		tasklet_kill(&ic->i_recv_tasklet);
1088 
1089 		atomic_set(&ic->i_cq_quiesce, 1);
1090 
1091 		/* first destroy the ib state that generates callbacks */
1092 		if (ic->i_cm_id->qp)
1093 			rdma_destroy_qp(ic->i_cm_id);
1094 		if (ic->i_send_cq) {
1095 			if (ic->rds_ibdev)
1096 				ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);
1097 			ib_destroy_cq(ic->i_send_cq);
1098 		}
1099 
1100 		if (ic->i_recv_cq) {
1101 			if (ic->rds_ibdev)
1102 				ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);
1103 			ib_destroy_cq(ic->i_recv_cq);
1104 		}
1105 
1106 		if (ic->rds_ibdev) {
1107 			/* then free the resources that ib callbacks use */
1108 			if (ic->i_send_hdrs) {
1109 				rds_dma_hdrs_free(ic->rds_ibdev,
1110 						  ic->i_send_hdrs,
1111 						  ic->i_send_hdrs_dma,
1112 						  ic->i_send_ring.w_nr,
1113 						  DMA_TO_DEVICE);
1114 				ic->i_send_hdrs = NULL;
1115 				ic->i_send_hdrs_dma = NULL;
1116 			}
1117 
1118 			if (ic->i_recv_hdrs) {
1119 				rds_dma_hdrs_free(ic->rds_ibdev,
1120 						  ic->i_recv_hdrs,
1121 						  ic->i_recv_hdrs_dma,
1122 						  ic->i_recv_ring.w_nr,
1123 						  DMA_FROM_DEVICE);
1124 				ic->i_recv_hdrs = NULL;
1125 				ic->i_recv_hdrs_dma = NULL;
1126 			}
1127 
1128 			if (ic->i_ack) {
1129 				rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack,
1130 						 ic->i_ack_dma, DMA_TO_DEVICE);
1131 				ic->i_ack = NULL;
1132 			}
1133 		} else {
1134 			WARN_ON(ic->i_send_hdrs);
1135 			WARN_ON(ic->i_send_hdrs_dma);
1136 			WARN_ON(ic->i_recv_hdrs);
1137 			WARN_ON(ic->i_recv_hdrs_dma);
1138 			WARN_ON(ic->i_ack);
1139 		}
1140 
1141 		if (ic->i_sends)
1142 			rds_ib_send_clear_ring(ic);
1143 		if (ic->i_recvs)
1144 			rds_ib_recv_clear_ring(ic);
1145 
1146 		rdma_destroy_id(ic->i_cm_id);
1147 
1148 		/*
1149 		 * Move connection back to the nodev list.
1150 		 */
1151 		if (ic->rds_ibdev)
1152 			rds_ib_remove_conn(ic->rds_ibdev, conn);
1153 
1154 		ic->i_cm_id = NULL;
1155 		ic->i_pd = NULL;
1156 		ic->i_send_cq = NULL;
1157 		ic->i_recv_cq = NULL;
1158 	}
1159 	BUG_ON(ic->rds_ibdev);
1160 
1161 	/* Clear pending transmit */
1162 	if (ic->i_data_op) {
1163 		struct rds_message *rm;
1164 
1165 		rm = container_of(ic->i_data_op, struct rds_message, data);
1166 		rds_message_put(rm);
1167 		ic->i_data_op = NULL;
1168 	}
1169 
1170 	/* Clear the ACK state */
1171 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
1172 #ifdef KERNEL_HAS_ATOMIC64
1173 	atomic64_set(&ic->i_ack_next, 0);
1174 #else
1175 	ic->i_ack_next = 0;
1176 #endif
1177 	ic->i_ack_recv = 0;
1178 
1179 	/* Clear flow control state */
1180 	ic->i_flowctl = 0;
1181 	atomic_set(&ic->i_credits, 0);
1182 
1183 	/* Re-init rings, but retain sizes. */
1184 	rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr);
1185 	rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr);
1186 
1187 	if (ic->i_ibinc) {
1188 		rds_inc_put(&ic->i_ibinc->ii_inc);
1189 		ic->i_ibinc = NULL;
1190 	}
1191 
1192 	vfree(ic->i_sends);
1193 	ic->i_sends = NULL;
1194 	vfree(ic->i_recvs);
1195 	ic->i_recvs = NULL;
1196 	ic->i_active_side = false;
1197 }
1198 
rds_ib_conn_alloc(struct rds_connection * conn,gfp_t gfp)1199 int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp)
1200 {
1201 	struct rds_ib_connection *ic;
1202 	unsigned long flags;
1203 	int ret;
1204 
1205 	/* XXX too lazy? */
1206 	ic = kzalloc(sizeof(struct rds_ib_connection), gfp);
1207 	if (!ic)
1208 		return -ENOMEM;
1209 
1210 	ret = rds_ib_recv_alloc_caches(ic, gfp);
1211 	if (ret) {
1212 		kfree(ic);
1213 		return ret;
1214 	}
1215 
1216 	INIT_LIST_HEAD(&ic->ib_node);
1217 	tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,
1218 		     (unsigned long)ic);
1219 	tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,
1220 		     (unsigned long)ic);
1221 	mutex_init(&ic->i_recv_mutex);
1222 #ifndef KERNEL_HAS_ATOMIC64
1223 	spin_lock_init(&ic->i_ack_lock);
1224 #endif
1225 	atomic_set(&ic->i_signaled_sends, 0);
1226 	atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
1227 
1228 	/*
1229 	 * rds_ib_conn_shutdown() waits for these to be emptied so they
1230 	 * must be initialized before it can be called.
1231 	 */
1232 	rds_ib_ring_init(&ic->i_send_ring, 0);
1233 	rds_ib_ring_init(&ic->i_recv_ring, 0);
1234 
1235 	ic->conn = conn;
1236 	conn->c_transport_data = ic;
1237 
1238 	spin_lock_irqsave(&ib_nodev_conns_lock, flags);
1239 	list_add_tail(&ic->ib_node, &ib_nodev_conns);
1240 	spin_unlock_irqrestore(&ib_nodev_conns_lock, flags);
1241 
1242 
1243 	rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);
1244 	return 0;
1245 }
1246 
1247 /*
1248  * Free a connection. Connection must be shut down and not set for reconnect.
1249  */
rds_ib_conn_free(void * arg)1250 void rds_ib_conn_free(void *arg)
1251 {
1252 	struct rds_ib_connection *ic = arg;
1253 	spinlock_t	*lock_ptr;
1254 
1255 	rdsdebug("ic %p\n", ic);
1256 
1257 	/*
1258 	 * Conn is either on a dev's list or on the nodev list.
1259 	 * A race with shutdown() or connect() would cause problems
1260 	 * (since rds_ibdev would change) but that should never happen.
1261 	 */
1262 	lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;
1263 
1264 	spin_lock_irq(lock_ptr);
1265 	list_del(&ic->ib_node);
1266 	spin_unlock_irq(lock_ptr);
1267 
1268 	rds_ib_recv_free_caches(ic);
1269 
1270 	kfree(ic);
1271 }
1272 
1273 
1274 /*
1275  * An error occurred on the connection
1276  */
1277 void
__rds_ib_conn_error(struct rds_connection * conn,const char * fmt,...)1278 __rds_ib_conn_error(struct rds_connection *conn, const char *fmt, ...)
1279 {
1280 	va_list ap;
1281 
1282 	rds_conn_drop(conn);
1283 
1284 	va_start(ap, fmt);
1285 	vprintk(fmt, ap);
1286 	va_end(ap);
1287 }
1288