1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * RDMA Transport Layer
4  *
5  * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
6  * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
7  * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
8  */
9 
10 #undef pr_fmt
11 #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
12 
13 #include <linux/module.h>
14 #include <linux/rculist.h>
15 #include <linux/random.h>
16 
17 #include "rtrs-clt.h"
18 #include "rtrs-log.h"
19 
20 #define RTRS_CONNECT_TIMEOUT_MS 30000
21 /*
22  * Wait a bit before trying to reconnect after a failure
23  * in order to give server time to finish clean up which
24  * leads to "false positives" failed reconnect attempts
25  */
26 #define RTRS_RECONNECT_BACKOFF 1000
27 /*
28  * Wait for additional random time between 0 and 8 seconds
29  * before starting to reconnect to avoid clients reconnecting
30  * all at once in case of a major network outage
31  */
32 #define RTRS_RECONNECT_SEED 8
33 
34 #define FIRST_CONN 0x01
35 /* limit to 128 * 4k = 512k max IO */
36 #define RTRS_MAX_SEGMENTS          128
37 
38 MODULE_DESCRIPTION("RDMA Transport Client");
39 MODULE_LICENSE("GPL");
40 
41 static const struct rtrs_rdma_dev_pd_ops dev_pd_ops;
42 static struct rtrs_rdma_dev_pd dev_pd = {
43 	.ops = &dev_pd_ops
44 };
45 
46 static struct workqueue_struct *rtrs_wq;
47 static struct class *rtrs_clt_dev_class;
48 
49 static inline bool rtrs_clt_is_connected(const struct rtrs_clt_sess *clt)
50 {
51 	struct rtrs_clt_path *clt_path;
52 	bool connected = false;
53 
54 	rcu_read_lock();
55 	list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry)
56 		connected |= READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED;
57 	rcu_read_unlock();
58 
59 	return connected;
60 }
61 
62 static struct rtrs_permit *
63 __rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type)
64 {
65 	size_t max_depth = clt->queue_depth;
66 	struct rtrs_permit *permit;
67 	int bit;
68 
69 	/*
70 	 * Adapted from null_blk get_tag(). Callers from different cpus may
71 	 * grab the same bit, since find_first_zero_bit is not atomic.
72 	 * But then the test_and_set_bit_lock will fail for all the
73 	 * callers but one, so that they will loop again.
74 	 * This way an explicit spinlock is not required.
75 	 */
76 	do {
77 		bit = find_first_zero_bit(clt->permits_map, max_depth);
78 		if (bit >= max_depth)
79 			return NULL;
80 	} while (test_and_set_bit_lock(bit, clt->permits_map));
81 
82 	permit = get_permit(clt, bit);
83 	WARN_ON(permit->mem_id != bit);
84 	permit->cpu_id = raw_smp_processor_id();
85 	permit->con_type = con_type;
86 
87 	return permit;
88 }
89 
90 static inline void __rtrs_put_permit(struct rtrs_clt_sess *clt,
91 				      struct rtrs_permit *permit)
92 {
93 	clear_bit_unlock(permit->mem_id, clt->permits_map);
94 }
95 
96 /**
97  * rtrs_clt_get_permit() - allocates permit for future RDMA operation
98  * @clt:	Current session
99  * @con_type:	Type of connection to use with the permit
100  * @can_wait:	Wait type
101  *
102  * Description:
103  *    Allocates permit for the following RDMA operation.  Permit is used
104  *    to preallocate all resources and to propagate memory pressure
105  *    up earlier.
106  *
107  * Context:
108  *    Can sleep if @wait == RTRS_PERMIT_WAIT
109  */
110 struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt_sess *clt,
111 					  enum rtrs_clt_con_type con_type,
112 					  enum wait_type can_wait)
113 {
114 	struct rtrs_permit *permit;
115 	DEFINE_WAIT(wait);
116 
117 	permit = __rtrs_get_permit(clt, con_type);
118 	if (permit || !can_wait)
119 		return permit;
120 
121 	do {
122 		prepare_to_wait(&clt->permits_wait, &wait,
123 				TASK_UNINTERRUPTIBLE);
124 		permit = __rtrs_get_permit(clt, con_type);
125 		if (permit)
126 			break;
127 
128 		io_schedule();
129 	} while (1);
130 
131 	finish_wait(&clt->permits_wait, &wait);
132 
133 	return permit;
134 }
135 EXPORT_SYMBOL(rtrs_clt_get_permit);
136 
137 /**
138  * rtrs_clt_put_permit() - puts allocated permit
139  * @clt:	Current session
140  * @permit:	Permit to be freed
141  *
142  * Context:
143  *    Does not matter
144  */
145 void rtrs_clt_put_permit(struct rtrs_clt_sess *clt,
146 			 struct rtrs_permit *permit)
147 {
148 	if (WARN_ON(!test_bit(permit->mem_id, clt->permits_map)))
149 		return;
150 
151 	__rtrs_put_permit(clt, permit);
152 
153 	/*
154 	 * rtrs_clt_get_permit() adds itself to the &clt->permits_wait list
155 	 * before calling schedule(). So if rtrs_clt_get_permit() is sleeping
156 	 * it must have added itself to &clt->permits_wait before
157 	 * __rtrs_put_permit() finished.
158 	 * Hence it is safe to guard wake_up() with a waitqueue_active() test.
159 	 */
160 	if (waitqueue_active(&clt->permits_wait))
161 		wake_up(&clt->permits_wait);
162 }
163 EXPORT_SYMBOL(rtrs_clt_put_permit);
164 
165 /**
166  * rtrs_permit_to_clt_con() - returns RDMA connection pointer by the permit
167  * @clt_path: client path pointer
168  * @permit: permit for the allocation of the RDMA buffer
169  * Note:
170  *     IO connection starts from 1.
171  *     0 connection is for user messages.
172  */
173 static
174 struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_path *clt_path,
175 					    struct rtrs_permit *permit)
176 {
177 	int id = 0;
178 
179 	if (permit->con_type == RTRS_IO_CON)
180 		id = (permit->cpu_id % (clt_path->s.irq_con_num - 1)) + 1;
181 
182 	return to_clt_con(clt_path->s.con[id]);
183 }
184 
185 /**
186  * rtrs_clt_change_state() - change the session state through session state
187  * machine.
188  *
189  * @clt_path: client path to change the state of.
190  * @new_state: state to change to.
191  *
192  * returns true if sess's state is changed to new state, otherwise return false.
193  *
194  * Locks:
195  * state_wq lock must be hold.
196  */
197 static bool rtrs_clt_change_state(struct rtrs_clt_path *clt_path,
198 				     enum rtrs_clt_state new_state)
199 {
200 	enum rtrs_clt_state old_state;
201 	bool changed = false;
202 
203 	lockdep_assert_held(&clt_path->state_wq.lock);
204 
205 	old_state = clt_path->state;
206 	switch (new_state) {
207 	case RTRS_CLT_CONNECTING:
208 		switch (old_state) {
209 		case RTRS_CLT_RECONNECTING:
210 			changed = true;
211 			fallthrough;
212 		default:
213 			break;
214 		}
215 		break;
216 	case RTRS_CLT_RECONNECTING:
217 		switch (old_state) {
218 		case RTRS_CLT_CONNECTED:
219 		case RTRS_CLT_CONNECTING_ERR:
220 		case RTRS_CLT_CLOSED:
221 			changed = true;
222 			fallthrough;
223 		default:
224 			break;
225 		}
226 		break;
227 	case RTRS_CLT_CONNECTED:
228 		switch (old_state) {
229 		case RTRS_CLT_CONNECTING:
230 			changed = true;
231 			fallthrough;
232 		default:
233 			break;
234 		}
235 		break;
236 	case RTRS_CLT_CONNECTING_ERR:
237 		switch (old_state) {
238 		case RTRS_CLT_CONNECTING:
239 			changed = true;
240 			fallthrough;
241 		default:
242 			break;
243 		}
244 		break;
245 	case RTRS_CLT_CLOSING:
246 		switch (old_state) {
247 		case RTRS_CLT_CONNECTING:
248 		case RTRS_CLT_CONNECTING_ERR:
249 		case RTRS_CLT_RECONNECTING:
250 		case RTRS_CLT_CONNECTED:
251 			changed = true;
252 			fallthrough;
253 		default:
254 			break;
255 		}
256 		break;
257 	case RTRS_CLT_CLOSED:
258 		switch (old_state) {
259 		case RTRS_CLT_CLOSING:
260 			changed = true;
261 			fallthrough;
262 		default:
263 			break;
264 		}
265 		break;
266 	case RTRS_CLT_DEAD:
267 		switch (old_state) {
268 		case RTRS_CLT_CLOSED:
269 			changed = true;
270 			fallthrough;
271 		default:
272 			break;
273 		}
274 		break;
275 	default:
276 		break;
277 	}
278 	if (changed) {
279 		clt_path->state = new_state;
280 		wake_up_locked(&clt_path->state_wq);
281 	}
282 
283 	return changed;
284 }
285 
286 static bool rtrs_clt_change_state_from_to(struct rtrs_clt_path *clt_path,
287 					   enum rtrs_clt_state old_state,
288 					   enum rtrs_clt_state new_state)
289 {
290 	bool changed = false;
291 
292 	spin_lock_irq(&clt_path->state_wq.lock);
293 	if (clt_path->state == old_state)
294 		changed = rtrs_clt_change_state(clt_path, new_state);
295 	spin_unlock_irq(&clt_path->state_wq.lock);
296 
297 	return changed;
298 }
299 
300 static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path);
301 static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)
302 {
303 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
304 
305 	if (rtrs_clt_change_state_from_to(clt_path,
306 					   RTRS_CLT_CONNECTED,
307 					   RTRS_CLT_RECONNECTING)) {
308 		queue_work(rtrs_wq, &clt_path->err_recovery_work);
309 	} else {
310 		/*
311 		 * Error can happen just on establishing new connection,
312 		 * so notify waiter with error state, waiter is responsible
313 		 * for cleaning the rest and reconnect if needed.
314 		 */
315 		rtrs_clt_change_state_from_to(clt_path,
316 					       RTRS_CLT_CONNECTING,
317 					       RTRS_CLT_CONNECTING_ERR);
318 	}
319 }
320 
321 static void rtrs_clt_fast_reg_done(struct ib_cq *cq, struct ib_wc *wc)
322 {
323 	struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
324 
325 	if (wc->status != IB_WC_SUCCESS) {
326 		rtrs_err(con->c.path, "Failed IB_WR_REG_MR: %s\n",
327 			  ib_wc_status_msg(wc->status));
328 		rtrs_rdma_error_recovery(con);
329 	}
330 }
331 
332 static struct ib_cqe fast_reg_cqe = {
333 	.done = rtrs_clt_fast_reg_done
334 };
335 
336 static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno,
337 			      bool notify, bool can_wait);
338 
339 static void rtrs_clt_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
340 {
341 	struct rtrs_clt_io_req *req =
342 		container_of(wc->wr_cqe, typeof(*req), inv_cqe);
343 	struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
344 
345 	if (wc->status != IB_WC_SUCCESS) {
346 		rtrs_err(con->c.path, "Failed IB_WR_LOCAL_INV: %s\n",
347 			  ib_wc_status_msg(wc->status));
348 		rtrs_rdma_error_recovery(con);
349 	}
350 	req->need_inv = false;
351 	if (req->need_inv_comp)
352 		complete(&req->inv_comp);
353 	else
354 		/* Complete request from INV callback */
355 		complete_rdma_req(req, req->inv_errno, true, false);
356 }
357 
358 static int rtrs_inv_rkey(struct rtrs_clt_io_req *req)
359 {
360 	struct rtrs_clt_con *con = req->con;
361 	struct ib_send_wr wr = {
362 		.opcode		    = IB_WR_LOCAL_INV,
363 		.wr_cqe		    = &req->inv_cqe,
364 		.send_flags	    = IB_SEND_SIGNALED,
365 		.ex.invalidate_rkey = req->mr->rkey,
366 	};
367 	req->inv_cqe.done = rtrs_clt_inv_rkey_done;
368 
369 	return ib_post_send(con->c.qp, &wr, NULL);
370 }
371 
372 static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno,
373 			      bool notify, bool can_wait)
374 {
375 	struct rtrs_clt_con *con = req->con;
376 	struct rtrs_clt_path *clt_path;
377 	int err;
378 
379 	if (WARN_ON(!req->in_use))
380 		return;
381 	if (WARN_ON(!req->con))
382 		return;
383 	clt_path = to_clt_path(con->c.path);
384 
385 	if (req->sg_cnt) {
386 		if (req->dir == DMA_FROM_DEVICE && req->need_inv) {
387 			/*
388 			 * We are here to invalidate read requests
389 			 * ourselves.  In normal scenario server should
390 			 * send INV for all read requests, but
391 			 * we are here, thus two things could happen:
392 			 *
393 			 *    1.  this is failover, when errno != 0
394 			 *        and can_wait == 1,
395 			 *
396 			 *    2.  something totally bad happened and
397 			 *        server forgot to send INV, so we
398 			 *        should do that ourselves.
399 			 */
400 
401 			if (can_wait) {
402 				req->need_inv_comp = true;
403 			} else {
404 				/* This should be IO path, so always notify */
405 				WARN_ON(!notify);
406 				/* Save errno for INV callback */
407 				req->inv_errno = errno;
408 			}
409 
410 			refcount_inc(&req->ref);
411 			err = rtrs_inv_rkey(req);
412 			if (err) {
413 				rtrs_err(con->c.path, "Send INV WR key=%#x: %d\n",
414 					  req->mr->rkey, err);
415 			} else if (can_wait) {
416 				wait_for_completion(&req->inv_comp);
417 			} else {
418 				/*
419 				 * Something went wrong, so request will be
420 				 * completed from INV callback.
421 				 */
422 				WARN_ON_ONCE(1);
423 
424 				return;
425 			}
426 			if (!refcount_dec_and_test(&req->ref))
427 				return;
428 		}
429 		ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist,
430 				req->sg_cnt, req->dir);
431 	}
432 	if (!refcount_dec_and_test(&req->ref))
433 		return;
434 	if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)
435 		atomic_dec(&clt_path->stats->inflight);
436 
437 	req->in_use = false;
438 	req->con = NULL;
439 
440 	if (errno) {
441 		rtrs_err_rl(con->c.path, "IO request failed: error=%d path=%s [%s:%u] notify=%d\n",
442 			    errno, kobject_name(&clt_path->kobj), clt_path->hca_name,
443 			    clt_path->hca_port, notify);
444 	}
445 
446 	if (notify)
447 		req->conf(req->priv, errno);
448 }
449 
450 static int rtrs_post_send_rdma(struct rtrs_clt_con *con,
451 				struct rtrs_clt_io_req *req,
452 				struct rtrs_rbuf *rbuf, u32 off,
453 				u32 imm, struct ib_send_wr *wr)
454 {
455 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
456 	enum ib_send_flags flags;
457 	struct ib_sge sge;
458 
459 	if (!req->sg_size) {
460 		rtrs_wrn(con->c.path,
461 			 "Doing RDMA Write failed, no data supplied\n");
462 		return -EINVAL;
463 	}
464 
465 	/* user data and user message in the first list element */
466 	sge.addr   = req->iu->dma_addr;
467 	sge.length = req->sg_size;
468 	sge.lkey   = clt_path->s.dev->ib_pd->local_dma_lkey;
469 
470 	/*
471 	 * From time to time we have to post signalled sends,
472 	 * or send queue will fill up and only QP reset can help.
473 	 */
474 	flags = atomic_inc_return(&con->c.wr_cnt) % clt_path->s.signal_interval ?
475 			0 : IB_SEND_SIGNALED;
476 
477 	ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev,
478 				      req->iu->dma_addr,
479 				      req->sg_size, DMA_TO_DEVICE);
480 
481 	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,
482 					    rbuf->rkey, rbuf->addr + off,
483 					    imm, flags, wr, NULL);
484 }
485 
486 static void process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id,
487 			   s16 errno, bool w_inval)
488 {
489 	struct rtrs_clt_io_req *req;
490 
491 	if (WARN_ON(msg_id >= clt_path->queue_depth))
492 		return;
493 
494 	req = &clt_path->reqs[msg_id];
495 	/* Drop need_inv if server responded with send with invalidation */
496 	req->need_inv &= !w_inval;
497 	complete_rdma_req(req, errno, true, false);
498 }
499 
500 static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc)
501 {
502 	struct rtrs_iu *iu;
503 	int err;
504 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
505 
506 	WARN_ON((clt_path->flags & RTRS_MSG_NEW_RKEY_F) == 0);
507 	iu = container_of(wc->wr_cqe, struct rtrs_iu,
508 			  cqe);
509 	err = rtrs_iu_post_recv(&con->c, iu);
510 	if (err) {
511 		rtrs_err(con->c.path, "post iu failed %d\n", err);
512 		rtrs_rdma_error_recovery(con);
513 	}
514 }
515 
516 static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc)
517 {
518 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
519 	struct rtrs_msg_rkey_rsp *msg;
520 	u32 imm_type, imm_payload;
521 	bool w_inval = false;
522 	struct rtrs_iu *iu;
523 	u32 buf_id;
524 	int err;
525 
526 	WARN_ON((clt_path->flags & RTRS_MSG_NEW_RKEY_F) == 0);
527 
528 	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
529 
530 	if (wc->byte_len < sizeof(*msg)) {
531 		rtrs_err(con->c.path, "rkey response is malformed: size %d\n",
532 			  wc->byte_len);
533 		goto out;
534 	}
535 	ib_dma_sync_single_for_cpu(clt_path->s.dev->ib_dev, iu->dma_addr,
536 				   iu->size, DMA_FROM_DEVICE);
537 	msg = iu->buf;
538 	if (le16_to_cpu(msg->type) != RTRS_MSG_RKEY_RSP) {
539 		rtrs_err(clt_path->clt,
540 			  "rkey response is malformed: type %d\n",
541 			  le16_to_cpu(msg->type));
542 		goto out;
543 	}
544 	buf_id = le16_to_cpu(msg->buf_id);
545 	if (WARN_ON(buf_id >= clt_path->queue_depth))
546 		goto out;
547 
548 	rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), &imm_type, &imm_payload);
549 	if (imm_type == RTRS_IO_RSP_IMM ||
550 	    imm_type == RTRS_IO_RSP_W_INV_IMM) {
551 		u32 msg_id;
552 
553 		w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM);
554 		rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err);
555 
556 		if (WARN_ON(buf_id != msg_id))
557 			goto out;
558 		clt_path->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey);
559 		process_io_rsp(clt_path, msg_id, err, w_inval);
560 	}
561 	ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, iu->dma_addr,
562 				      iu->size, DMA_FROM_DEVICE);
563 	return rtrs_clt_recv_done(con, wc);
564 out:
565 	rtrs_rdma_error_recovery(con);
566 }
567 
568 static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc);
569 
570 static struct ib_cqe io_comp_cqe = {
571 	.done = rtrs_clt_rdma_done
572 };
573 
574 /*
575  * Post x2 empty WRs: first is for this RDMA with IMM,
576  * second is for RECV with INV, which happened earlier.
577  */
578 static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe)
579 {
580 	struct ib_recv_wr wr_arr[2], *wr;
581 	int i;
582 
583 	memset(wr_arr, 0, sizeof(wr_arr));
584 	for (i = 0; i < ARRAY_SIZE(wr_arr); i++) {
585 		wr = &wr_arr[i];
586 		wr->wr_cqe  = cqe;
587 		if (i)
588 			/* Chain backwards */
589 			wr->next = &wr_arr[i - 1];
590 	}
591 
592 	return ib_post_recv(con->qp, wr, NULL);
593 }
594 
595 static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc)
596 {
597 	struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
598 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
599 	u32 imm_type, imm_payload;
600 	bool w_inval = false;
601 	int err;
602 
603 	if (wc->status != IB_WC_SUCCESS) {
604 		if (wc->status != IB_WC_WR_FLUSH_ERR) {
605 			rtrs_err(clt_path->clt, "RDMA failed: %s\n",
606 				  ib_wc_status_msg(wc->status));
607 			rtrs_rdma_error_recovery(con);
608 		}
609 		return;
610 	}
611 	rtrs_clt_update_wc_stats(con);
612 
613 	switch (wc->opcode) {
614 	case IB_WC_RECV_RDMA_WITH_IMM:
615 		/*
616 		 * post_recv() RDMA write completions of IO reqs (read/write)
617 		 * and hb
618 		 */
619 		if (WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done))
620 			return;
621 		rtrs_from_imm(be32_to_cpu(wc->ex.imm_data),
622 			       &imm_type, &imm_payload);
623 		if (imm_type == RTRS_IO_RSP_IMM ||
624 		    imm_type == RTRS_IO_RSP_W_INV_IMM) {
625 			u32 msg_id;
626 
627 			w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM);
628 			rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err);
629 
630 			process_io_rsp(clt_path, msg_id, err, w_inval);
631 		} else if (imm_type == RTRS_HB_MSG_IMM) {
632 			WARN_ON(con->c.cid);
633 			rtrs_send_hb_ack(&clt_path->s);
634 			if (clt_path->flags & RTRS_MSG_NEW_RKEY_F)
635 				return  rtrs_clt_recv_done(con, wc);
636 		} else if (imm_type == RTRS_HB_ACK_IMM) {
637 			WARN_ON(con->c.cid);
638 			clt_path->s.hb_missed_cnt = 0;
639 			clt_path->s.hb_cur_latency =
640 				ktime_sub(ktime_get(), clt_path->s.hb_last_sent);
641 			if (clt_path->flags & RTRS_MSG_NEW_RKEY_F)
642 				return  rtrs_clt_recv_done(con, wc);
643 		} else {
644 			rtrs_wrn(con->c.path, "Unknown IMM type %u\n",
645 				  imm_type);
646 		}
647 		if (w_inval)
648 			/*
649 			 * Post x2 empty WRs: first is for this RDMA with IMM,
650 			 * second is for RECV with INV, which happened earlier.
651 			 */
652 			err = rtrs_post_recv_empty_x2(&con->c, &io_comp_cqe);
653 		else
654 			err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
655 		if (err) {
656 			rtrs_err(con->c.path, "rtrs_post_recv_empty(): %d\n",
657 				  err);
658 			rtrs_rdma_error_recovery(con);
659 		}
660 		break;
661 	case IB_WC_RECV:
662 		/*
663 		 * Key invalidations from server side
664 		 */
665 		WARN_ON(!(wc->wc_flags & IB_WC_WITH_INVALIDATE ||
666 			  wc->wc_flags & IB_WC_WITH_IMM));
667 		WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done);
668 		if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) {
669 			if (wc->wc_flags & IB_WC_WITH_INVALIDATE)
670 				return  rtrs_clt_recv_done(con, wc);
671 
672 			return  rtrs_clt_rkey_rsp_done(con, wc);
673 		}
674 		break;
675 	case IB_WC_RDMA_WRITE:
676 		/*
677 		 * post_send() RDMA write completions of IO reqs (read/write)
678 		 * and hb.
679 		 */
680 		break;
681 
682 	default:
683 		rtrs_wrn(clt_path->clt, "Unexpected WC type: %d\n", wc->opcode);
684 		return;
685 	}
686 }
687 
688 static int post_recv_io(struct rtrs_clt_con *con, size_t q_size)
689 {
690 	int err, i;
691 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
692 
693 	for (i = 0; i < q_size; i++) {
694 		if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) {
695 			struct rtrs_iu *iu = &con->rsp_ius[i];
696 
697 			err = rtrs_iu_post_recv(&con->c, iu);
698 		} else {
699 			err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
700 		}
701 		if (err)
702 			return err;
703 	}
704 
705 	return 0;
706 }
707 
708 static int post_recv_path(struct rtrs_clt_path *clt_path)
709 {
710 	size_t q_size = 0;
711 	int err, cid;
712 
713 	for (cid = 0; cid < clt_path->s.con_num; cid++) {
714 		if (cid == 0)
715 			q_size = SERVICE_CON_QUEUE_DEPTH;
716 		else
717 			q_size = clt_path->queue_depth;
718 
719 		/*
720 		 * x2 for RDMA read responses + FR key invalidations,
721 		 * RDMA writes do not require any FR registrations.
722 		 */
723 		q_size *= 2;
724 
725 		err = post_recv_io(to_clt_con(clt_path->s.con[cid]), q_size);
726 		if (err) {
727 			rtrs_err(clt_path->clt, "post_recv_io(), err: %d\n",
728 				 err);
729 			return err;
730 		}
731 	}
732 
733 	return 0;
734 }
735 
736 struct path_it {
737 	int i;
738 	struct list_head skip_list;
739 	struct rtrs_clt_sess *clt;
740 	struct rtrs_clt_path *(*next_path)(struct path_it *it);
741 };
742 
743 /**
744  * list_next_or_null_rr_rcu - get next list element in round-robin fashion.
745  * @head:	the head for the list.
746  * @ptr:        the list head to take the next element from.
747  * @type:       the type of the struct this is embedded in.
748  * @memb:       the name of the list_head within the struct.
749  *
750  * Next element returned in round-robin fashion, i.e. head will be skipped,
751  * but if list is observed as empty, NULL will be returned.
752  *
753  * This primitive may safely run concurrently with the _rcu list-mutation
754  * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
755  */
756 #define list_next_or_null_rr_rcu(head, ptr, type, memb) \
757 ({ \
758 	list_next_or_null_rcu(head, ptr, type, memb) ?: \
759 		list_next_or_null_rcu(head, READ_ONCE((ptr)->next), \
760 				      type, memb); \
761 })
762 
763 /**
764  * get_next_path_rr() - Returns path in round-robin fashion.
765  * @it:	the path pointer
766  *
767  * Related to @MP_POLICY_RR
768  *
769  * Locks:
770  *    rcu_read_lock() must be hold.
771  */
772 static struct rtrs_clt_path *get_next_path_rr(struct path_it *it)
773 {
774 	struct rtrs_clt_path __rcu **ppcpu_path;
775 	struct rtrs_clt_path *path;
776 	struct rtrs_clt_sess *clt;
777 
778 	clt = it->clt;
779 
780 	/*
781 	 * Here we use two RCU objects: @paths_list and @pcpu_path
782 	 * pointer.  See rtrs_clt_remove_path_from_arr() for details
783 	 * how that is handled.
784 	 */
785 
786 	ppcpu_path = this_cpu_ptr(clt->pcpu_path);
787 	path = rcu_dereference(*ppcpu_path);
788 	if (!path)
789 		path = list_first_or_null_rcu(&clt->paths_list,
790 					      typeof(*path), s.entry);
791 	else
792 		path = list_next_or_null_rr_rcu(&clt->paths_list,
793 						&path->s.entry,
794 						typeof(*path),
795 						s.entry);
796 	rcu_assign_pointer(*ppcpu_path, path);
797 
798 	return path;
799 }
800 
801 /**
802  * get_next_path_min_inflight() - Returns path with minimal inflight count.
803  * @it:	the path pointer
804  *
805  * Related to @MP_POLICY_MIN_INFLIGHT
806  *
807  * Locks:
808  *    rcu_read_lock() must be hold.
809  */
810 static struct rtrs_clt_path *get_next_path_min_inflight(struct path_it *it)
811 {
812 	struct rtrs_clt_path *min_path = NULL;
813 	struct rtrs_clt_sess *clt = it->clt;
814 	struct rtrs_clt_path *clt_path;
815 	int min_inflight = INT_MAX;
816 	int inflight;
817 
818 	list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) {
819 		if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
820 			continue;
821 
822 		if (!list_empty(raw_cpu_ptr(clt_path->mp_skip_entry)))
823 			continue;
824 
825 		inflight = atomic_read(&clt_path->stats->inflight);
826 
827 		if (inflight < min_inflight) {
828 			min_inflight = inflight;
829 			min_path = clt_path;
830 		}
831 	}
832 
833 	/*
834 	 * add the path to the skip list, so that next time we can get
835 	 * a different one
836 	 */
837 	if (min_path)
838 		list_add(raw_cpu_ptr(min_path->mp_skip_entry), &it->skip_list);
839 
840 	return min_path;
841 }
842 
843 /**
844  * get_next_path_min_latency() - Returns path with minimal latency.
845  * @it:	the path pointer
846  *
847  * Return: a path with the lowest latency or NULL if all paths are tried
848  *
849  * Locks:
850  *    rcu_read_lock() must be hold.
851  *
852  * Related to @MP_POLICY_MIN_LATENCY
853  *
854  * This DOES skip an already-tried path.
855  * There is a skip-list to skip a path if the path has tried but failed.
856  * It will try the minimum latency path and then the second minimum latency
857  * path and so on. Finally it will return NULL if all paths are tried.
858  * Therefore the caller MUST check the returned
859  * path is NULL and trigger the IO error.
860  */
861 static struct rtrs_clt_path *get_next_path_min_latency(struct path_it *it)
862 {
863 	struct rtrs_clt_path *min_path = NULL;
864 	struct rtrs_clt_sess *clt = it->clt;
865 	struct rtrs_clt_path *clt_path;
866 	ktime_t min_latency = KTIME_MAX;
867 	ktime_t latency;
868 
869 	list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) {
870 		if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
871 			continue;
872 
873 		if (!list_empty(raw_cpu_ptr(clt_path->mp_skip_entry)))
874 			continue;
875 
876 		latency = clt_path->s.hb_cur_latency;
877 
878 		if (latency < min_latency) {
879 			min_latency = latency;
880 			min_path = clt_path;
881 		}
882 	}
883 
884 	/*
885 	 * add the path to the skip list, so that next time we can get
886 	 * a different one
887 	 */
888 	if (min_path)
889 		list_add(raw_cpu_ptr(min_path->mp_skip_entry), &it->skip_list);
890 
891 	return min_path;
892 }
893 
894 static inline void path_it_init(struct path_it *it, struct rtrs_clt_sess *clt)
895 {
896 	INIT_LIST_HEAD(&it->skip_list);
897 	it->clt = clt;
898 	it->i = 0;
899 
900 	if (clt->mp_policy == MP_POLICY_RR)
901 		it->next_path = get_next_path_rr;
902 	else if (clt->mp_policy == MP_POLICY_MIN_INFLIGHT)
903 		it->next_path = get_next_path_min_inflight;
904 	else
905 		it->next_path = get_next_path_min_latency;
906 }
907 
908 static inline void path_it_deinit(struct path_it *it)
909 {
910 	struct list_head *skip, *tmp;
911 	/*
912 	 * The skip_list is used only for the MIN_INFLIGHT and MIN_LATENCY policies.
913 	 * We need to remove paths from it, so that next IO can insert
914 	 * paths (->mp_skip_entry) into a skip_list again.
915 	 */
916 	list_for_each_safe(skip, tmp, &it->skip_list)
917 		list_del_init(skip);
918 }
919 
920 /**
921  * rtrs_clt_init_req() - Initialize an rtrs_clt_io_req holding information
922  * about an inflight IO.
923  * The user buffer holding user control message (not data) is copied into
924  * the corresponding buffer of rtrs_iu (req->iu->buf), which later on will
925  * also hold the control message of rtrs.
926  * @req: an io request holding information about IO.
927  * @clt_path: client path
928  * @conf: conformation callback function to notify upper layer.
929  * @permit: permit for allocation of RDMA remote buffer
930  * @priv: private pointer
931  * @vec: kernel vector containing control message
932  * @usr_len: length of the user message
933  * @sg: scater list for IO data
934  * @sg_cnt: number of scater list entries
935  * @data_len: length of the IO data
936  * @dir: direction of the IO.
937  */
938 static void rtrs_clt_init_req(struct rtrs_clt_io_req *req,
939 			      struct rtrs_clt_path *clt_path,
940 			      void (*conf)(void *priv, int errno),
941 			      struct rtrs_permit *permit, void *priv,
942 			      const struct kvec *vec, size_t usr_len,
943 			      struct scatterlist *sg, size_t sg_cnt,
944 			      size_t data_len, int dir)
945 {
946 	struct iov_iter iter;
947 	size_t len;
948 
949 	req->permit = permit;
950 	req->in_use = true;
951 	req->usr_len = usr_len;
952 	req->data_len = data_len;
953 	req->sglist = sg;
954 	req->sg_cnt = sg_cnt;
955 	req->priv = priv;
956 	req->dir = dir;
957 	req->con = rtrs_permit_to_clt_con(clt_path, permit);
958 	req->conf = conf;
959 	req->need_inv = false;
960 	req->need_inv_comp = false;
961 	req->inv_errno = 0;
962 	refcount_set(&req->ref, 1);
963 	req->mp_policy = clt_path->clt->mp_policy;
964 
965 	iov_iter_kvec(&iter, READ, vec, 1, usr_len);
966 	len = _copy_from_iter(req->iu->buf, usr_len, &iter);
967 	WARN_ON(len != usr_len);
968 
969 	reinit_completion(&req->inv_comp);
970 }
971 
972 static struct rtrs_clt_io_req *
973 rtrs_clt_get_req(struct rtrs_clt_path *clt_path,
974 		 void (*conf)(void *priv, int errno),
975 		 struct rtrs_permit *permit, void *priv,
976 		 const struct kvec *vec, size_t usr_len,
977 		 struct scatterlist *sg, size_t sg_cnt,
978 		 size_t data_len, int dir)
979 {
980 	struct rtrs_clt_io_req *req;
981 
982 	req = &clt_path->reqs[permit->mem_id];
983 	rtrs_clt_init_req(req, clt_path, conf, permit, priv, vec, usr_len,
984 			   sg, sg_cnt, data_len, dir);
985 	return req;
986 }
987 
988 static struct rtrs_clt_io_req *
989 rtrs_clt_get_copy_req(struct rtrs_clt_path *alive_path,
990 		       struct rtrs_clt_io_req *fail_req)
991 {
992 	struct rtrs_clt_io_req *req;
993 	struct kvec vec = {
994 		.iov_base = fail_req->iu->buf,
995 		.iov_len  = fail_req->usr_len
996 	};
997 
998 	req = &alive_path->reqs[fail_req->permit->mem_id];
999 	rtrs_clt_init_req(req, alive_path, fail_req->conf, fail_req->permit,
1000 			   fail_req->priv, &vec, fail_req->usr_len,
1001 			   fail_req->sglist, fail_req->sg_cnt,
1002 			   fail_req->data_len, fail_req->dir);
1003 	return req;
1004 }
1005 
1006 static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con,
1007 				   struct rtrs_clt_io_req *req,
1008 				   struct rtrs_rbuf *rbuf, bool fr_en,
1009 				   u32 size, u32 imm, struct ib_send_wr *wr,
1010 				   struct ib_send_wr *tail)
1011 {
1012 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1013 	struct ib_sge *sge = req->sge;
1014 	enum ib_send_flags flags;
1015 	struct scatterlist *sg;
1016 	size_t num_sge;
1017 	int i;
1018 	struct ib_send_wr *ptail = NULL;
1019 
1020 	if (fr_en) {
1021 		i = 0;
1022 		sge[i].addr   = req->mr->iova;
1023 		sge[i].length = req->mr->length;
1024 		sge[i].lkey   = req->mr->lkey;
1025 		i++;
1026 		num_sge = 2;
1027 		ptail = tail;
1028 	} else {
1029 		for_each_sg(req->sglist, sg, req->sg_cnt, i) {
1030 			sge[i].addr   = sg_dma_address(sg);
1031 			sge[i].length = sg_dma_len(sg);
1032 			sge[i].lkey   = clt_path->s.dev->ib_pd->local_dma_lkey;
1033 		}
1034 		num_sge = 1 + req->sg_cnt;
1035 	}
1036 	sge[i].addr   = req->iu->dma_addr;
1037 	sge[i].length = size;
1038 	sge[i].lkey   = clt_path->s.dev->ib_pd->local_dma_lkey;
1039 
1040 	/*
1041 	 * From time to time we have to post signalled sends,
1042 	 * or send queue will fill up and only QP reset can help.
1043 	 */
1044 	flags = atomic_inc_return(&con->c.wr_cnt) % clt_path->s.signal_interval ?
1045 			0 : IB_SEND_SIGNALED;
1046 
1047 	ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev,
1048 				      req->iu->dma_addr,
1049 				      size, DMA_TO_DEVICE);
1050 
1051 	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, sge, num_sge,
1052 					    rbuf->rkey, rbuf->addr, imm,
1053 					    flags, wr, ptail);
1054 }
1055 
1056 static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count)
1057 {
1058 	int nr;
1059 
1060 	/* Align the MR to a 4K page size to match the block virt boundary */
1061 	nr = ib_map_mr_sg(req->mr, req->sglist, count, NULL, SZ_4K);
1062 	if (nr < 0)
1063 		return nr;
1064 	if (nr < req->sg_cnt)
1065 		return -EINVAL;
1066 	ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
1067 
1068 	return nr;
1069 }
1070 
1071 static int rtrs_clt_write_req(struct rtrs_clt_io_req *req)
1072 {
1073 	struct rtrs_clt_con *con = req->con;
1074 	struct rtrs_path *s = con->c.path;
1075 	struct rtrs_clt_path *clt_path = to_clt_path(s);
1076 	struct rtrs_msg_rdma_write *msg;
1077 
1078 	struct rtrs_rbuf *rbuf;
1079 	int ret, count = 0;
1080 	u32 imm, buf_id;
1081 	struct ib_reg_wr rwr;
1082 	struct ib_send_wr inv_wr;
1083 	struct ib_send_wr *wr = NULL;
1084 	bool fr_en = false;
1085 
1086 	const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;
1087 
1088 	if (tsize > clt_path->chunk_size) {
1089 		rtrs_wrn(s, "Write request failed, size too big %zu > %d\n",
1090 			  tsize, clt_path->chunk_size);
1091 		return -EMSGSIZE;
1092 	}
1093 	if (req->sg_cnt) {
1094 		count = ib_dma_map_sg(clt_path->s.dev->ib_dev, req->sglist,
1095 				      req->sg_cnt, req->dir);
1096 		if (!count) {
1097 			rtrs_wrn(s, "Write request failed, map failed\n");
1098 			return -EINVAL;
1099 		}
1100 	}
1101 	/* put rtrs msg after sg and user message */
1102 	msg = req->iu->buf + req->usr_len;
1103 	msg->type = cpu_to_le16(RTRS_MSG_WRITE);
1104 	msg->usr_len = cpu_to_le16(req->usr_len);
1105 
1106 	/* rtrs message on server side will be after user data and message */
1107 	imm = req->permit->mem_off + req->data_len + req->usr_len;
1108 	imm = rtrs_to_io_req_imm(imm);
1109 	buf_id = req->permit->mem_id;
1110 	req->sg_size = tsize;
1111 	rbuf = &clt_path->rbufs[buf_id];
1112 
1113 	if (count) {
1114 		ret = rtrs_map_sg_fr(req, count);
1115 		if (ret < 0) {
1116 			rtrs_err_rl(s,
1117 				    "Write request failed, failed to map fast reg. data, err: %d\n",
1118 				    ret);
1119 			ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist,
1120 					req->sg_cnt, req->dir);
1121 			return ret;
1122 		}
1123 		inv_wr = (struct ib_send_wr) {
1124 			.opcode		    = IB_WR_LOCAL_INV,
1125 			.wr_cqe		    = &req->inv_cqe,
1126 			.send_flags	    = IB_SEND_SIGNALED,
1127 			.ex.invalidate_rkey = req->mr->rkey,
1128 		};
1129 		req->inv_cqe.done = rtrs_clt_inv_rkey_done;
1130 		rwr = (struct ib_reg_wr) {
1131 			.wr.opcode = IB_WR_REG_MR,
1132 			.wr.wr_cqe = &fast_reg_cqe,
1133 			.mr = req->mr,
1134 			.key = req->mr->rkey,
1135 			.access = (IB_ACCESS_LOCAL_WRITE),
1136 		};
1137 		wr = &rwr.wr;
1138 		fr_en = true;
1139 		refcount_inc(&req->ref);
1140 	}
1141 	/*
1142 	 * Update stats now, after request is successfully sent it is not
1143 	 * safe anymore to touch it.
1144 	 */
1145 	rtrs_clt_update_all_stats(req, WRITE);
1146 
1147 	ret = rtrs_post_rdma_write_sg(req->con, req, rbuf, fr_en,
1148 				      req->usr_len + sizeof(*msg),
1149 				      imm, wr, &inv_wr);
1150 	if (ret) {
1151 		rtrs_err_rl(s,
1152 			    "Write request failed: error=%d path=%s [%s:%u]\n",
1153 			    ret, kobject_name(&clt_path->kobj), clt_path->hca_name,
1154 			    clt_path->hca_port);
1155 		if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)
1156 			atomic_dec(&clt_path->stats->inflight);
1157 		if (req->sg_cnt)
1158 			ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist,
1159 					req->sg_cnt, req->dir);
1160 	}
1161 
1162 	return ret;
1163 }
1164 
1165 static int rtrs_clt_read_req(struct rtrs_clt_io_req *req)
1166 {
1167 	struct rtrs_clt_con *con = req->con;
1168 	struct rtrs_path *s = con->c.path;
1169 	struct rtrs_clt_path *clt_path = to_clt_path(s);
1170 	struct rtrs_msg_rdma_read *msg;
1171 	struct rtrs_ib_dev *dev = clt_path->s.dev;
1172 
1173 	struct ib_reg_wr rwr;
1174 	struct ib_send_wr *wr = NULL;
1175 
1176 	int ret, count = 0;
1177 	u32 imm, buf_id;
1178 
1179 	const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;
1180 
1181 	if (tsize > clt_path->chunk_size) {
1182 		rtrs_wrn(s,
1183 			  "Read request failed, message size is %zu, bigger than CHUNK_SIZE %d\n",
1184 			  tsize, clt_path->chunk_size);
1185 		return -EMSGSIZE;
1186 	}
1187 
1188 	if (req->sg_cnt) {
1189 		count = ib_dma_map_sg(dev->ib_dev, req->sglist, req->sg_cnt,
1190 				      req->dir);
1191 		if (!count) {
1192 			rtrs_wrn(s,
1193 				  "Read request failed, dma map failed\n");
1194 			return -EINVAL;
1195 		}
1196 	}
1197 	/* put our message into req->buf after user message*/
1198 	msg = req->iu->buf + req->usr_len;
1199 	msg->type = cpu_to_le16(RTRS_MSG_READ);
1200 	msg->usr_len = cpu_to_le16(req->usr_len);
1201 
1202 	if (count) {
1203 		ret = rtrs_map_sg_fr(req, count);
1204 		if (ret < 0) {
1205 			rtrs_err_rl(s,
1206 				     "Read request failed, failed to map  fast reg. data, err: %d\n",
1207 				     ret);
1208 			ib_dma_unmap_sg(dev->ib_dev, req->sglist, req->sg_cnt,
1209 					req->dir);
1210 			return ret;
1211 		}
1212 		rwr = (struct ib_reg_wr) {
1213 			.wr.opcode = IB_WR_REG_MR,
1214 			.wr.wr_cqe = &fast_reg_cqe,
1215 			.mr = req->mr,
1216 			.key = req->mr->rkey,
1217 			.access = (IB_ACCESS_LOCAL_WRITE |
1218 				   IB_ACCESS_REMOTE_WRITE),
1219 		};
1220 		wr = &rwr.wr;
1221 
1222 		msg->sg_cnt = cpu_to_le16(1);
1223 		msg->flags = cpu_to_le16(RTRS_MSG_NEED_INVAL_F);
1224 
1225 		msg->desc[0].addr = cpu_to_le64(req->mr->iova);
1226 		msg->desc[0].key = cpu_to_le32(req->mr->rkey);
1227 		msg->desc[0].len = cpu_to_le32(req->mr->length);
1228 
1229 		/* Further invalidation is required */
1230 		req->need_inv = !!RTRS_MSG_NEED_INVAL_F;
1231 
1232 	} else {
1233 		msg->sg_cnt = 0;
1234 		msg->flags = 0;
1235 	}
1236 	/*
1237 	 * rtrs message will be after the space reserved for disk data and
1238 	 * user message
1239 	 */
1240 	imm = req->permit->mem_off + req->data_len + req->usr_len;
1241 	imm = rtrs_to_io_req_imm(imm);
1242 	buf_id = req->permit->mem_id;
1243 
1244 	req->sg_size  = sizeof(*msg);
1245 	req->sg_size += le16_to_cpu(msg->sg_cnt) * sizeof(struct rtrs_sg_desc);
1246 	req->sg_size += req->usr_len;
1247 
1248 	/*
1249 	 * Update stats now, after request is successfully sent it is not
1250 	 * safe anymore to touch it.
1251 	 */
1252 	rtrs_clt_update_all_stats(req, READ);
1253 
1254 	ret = rtrs_post_send_rdma(req->con, req, &clt_path->rbufs[buf_id],
1255 				   req->data_len, imm, wr);
1256 	if (ret) {
1257 		rtrs_err_rl(s,
1258 			    "Read request failed: error=%d path=%s [%s:%u]\n",
1259 			    ret, kobject_name(&clt_path->kobj), clt_path->hca_name,
1260 			    clt_path->hca_port);
1261 		if (req->mp_policy == MP_POLICY_MIN_INFLIGHT)
1262 			atomic_dec(&clt_path->stats->inflight);
1263 		req->need_inv = false;
1264 		if (req->sg_cnt)
1265 			ib_dma_unmap_sg(dev->ib_dev, req->sglist,
1266 					req->sg_cnt, req->dir);
1267 	}
1268 
1269 	return ret;
1270 }
1271 
1272 /**
1273  * rtrs_clt_failover_req() - Try to find an active path for a failed request
1274  * @clt: clt context
1275  * @fail_req: a failed io request.
1276  */
1277 static int rtrs_clt_failover_req(struct rtrs_clt_sess *clt,
1278 				 struct rtrs_clt_io_req *fail_req)
1279 {
1280 	struct rtrs_clt_path *alive_path;
1281 	struct rtrs_clt_io_req *req;
1282 	int err = -ECONNABORTED;
1283 	struct path_it it;
1284 
1285 	rcu_read_lock();
1286 	for (path_it_init(&it, clt);
1287 	     (alive_path = it.next_path(&it)) && it.i < it.clt->paths_num;
1288 	     it.i++) {
1289 		if (READ_ONCE(alive_path->state) != RTRS_CLT_CONNECTED)
1290 			continue;
1291 		req = rtrs_clt_get_copy_req(alive_path, fail_req);
1292 		if (req->dir == DMA_TO_DEVICE)
1293 			err = rtrs_clt_write_req(req);
1294 		else
1295 			err = rtrs_clt_read_req(req);
1296 		if (err) {
1297 			req->in_use = false;
1298 			continue;
1299 		}
1300 		/* Success path */
1301 		rtrs_clt_inc_failover_cnt(alive_path->stats);
1302 		break;
1303 	}
1304 	path_it_deinit(&it);
1305 	rcu_read_unlock();
1306 
1307 	return err;
1308 }
1309 
1310 static void fail_all_outstanding_reqs(struct rtrs_clt_path *clt_path)
1311 {
1312 	struct rtrs_clt_sess *clt = clt_path->clt;
1313 	struct rtrs_clt_io_req *req;
1314 	int i, err;
1315 
1316 	if (!clt_path->reqs)
1317 		return;
1318 	for (i = 0; i < clt_path->queue_depth; ++i) {
1319 		req = &clt_path->reqs[i];
1320 		if (!req->in_use)
1321 			continue;
1322 
1323 		/*
1324 		 * Safely (without notification) complete failed request.
1325 		 * After completion this request is still useble and can
1326 		 * be failovered to another path.
1327 		 */
1328 		complete_rdma_req(req, -ECONNABORTED, false, true);
1329 
1330 		err = rtrs_clt_failover_req(clt, req);
1331 		if (err)
1332 			/* Failover failed, notify anyway */
1333 			req->conf(req->priv, err);
1334 	}
1335 }
1336 
1337 static void free_path_reqs(struct rtrs_clt_path *clt_path)
1338 {
1339 	struct rtrs_clt_io_req *req;
1340 	int i;
1341 
1342 	if (!clt_path->reqs)
1343 		return;
1344 	for (i = 0; i < clt_path->queue_depth; ++i) {
1345 		req = &clt_path->reqs[i];
1346 		if (req->mr)
1347 			ib_dereg_mr(req->mr);
1348 		kfree(req->sge);
1349 		rtrs_iu_free(req->iu, clt_path->s.dev->ib_dev, 1);
1350 	}
1351 	kfree(clt_path->reqs);
1352 	clt_path->reqs = NULL;
1353 }
1354 
1355 static int alloc_path_reqs(struct rtrs_clt_path *clt_path)
1356 {
1357 	struct rtrs_clt_io_req *req;
1358 	int i, err = -ENOMEM;
1359 
1360 	clt_path->reqs = kcalloc(clt_path->queue_depth,
1361 				 sizeof(*clt_path->reqs),
1362 				 GFP_KERNEL);
1363 	if (!clt_path->reqs)
1364 		return -ENOMEM;
1365 
1366 	for (i = 0; i < clt_path->queue_depth; ++i) {
1367 		req = &clt_path->reqs[i];
1368 		req->iu = rtrs_iu_alloc(1, clt_path->max_hdr_size, GFP_KERNEL,
1369 					 clt_path->s.dev->ib_dev,
1370 					 DMA_TO_DEVICE,
1371 					 rtrs_clt_rdma_done);
1372 		if (!req->iu)
1373 			goto out;
1374 
1375 		req->sge = kcalloc(2, sizeof(*req->sge), GFP_KERNEL);
1376 		if (!req->sge)
1377 			goto out;
1378 
1379 		req->mr = ib_alloc_mr(clt_path->s.dev->ib_pd,
1380 				      IB_MR_TYPE_MEM_REG,
1381 				      clt_path->max_pages_per_mr);
1382 		if (IS_ERR(req->mr)) {
1383 			err = PTR_ERR(req->mr);
1384 			req->mr = NULL;
1385 			pr_err("Failed to alloc clt_path->max_pages_per_mr %d\n",
1386 			       clt_path->max_pages_per_mr);
1387 			goto out;
1388 		}
1389 
1390 		init_completion(&req->inv_comp);
1391 	}
1392 
1393 	return 0;
1394 
1395 out:
1396 	free_path_reqs(clt_path);
1397 
1398 	return err;
1399 }
1400 
1401 static int alloc_permits(struct rtrs_clt_sess *clt)
1402 {
1403 	unsigned int chunk_bits;
1404 	int err, i;
1405 
1406 	clt->permits_map = kcalloc(BITS_TO_LONGS(clt->queue_depth),
1407 				   sizeof(long), GFP_KERNEL);
1408 	if (!clt->permits_map) {
1409 		err = -ENOMEM;
1410 		goto out_err;
1411 	}
1412 	clt->permits = kcalloc(clt->queue_depth, permit_size(clt), GFP_KERNEL);
1413 	if (!clt->permits) {
1414 		err = -ENOMEM;
1415 		goto err_map;
1416 	}
1417 	chunk_bits = ilog2(clt->queue_depth - 1) + 1;
1418 	for (i = 0; i < clt->queue_depth; i++) {
1419 		struct rtrs_permit *permit;
1420 
1421 		permit = get_permit(clt, i);
1422 		permit->mem_id = i;
1423 		permit->mem_off = i << (MAX_IMM_PAYL_BITS - chunk_bits);
1424 	}
1425 
1426 	return 0;
1427 
1428 err_map:
1429 	kfree(clt->permits_map);
1430 	clt->permits_map = NULL;
1431 out_err:
1432 	return err;
1433 }
1434 
1435 static void free_permits(struct rtrs_clt_sess *clt)
1436 {
1437 	if (clt->permits_map) {
1438 		size_t sz = clt->queue_depth;
1439 
1440 		wait_event(clt->permits_wait,
1441 			   find_first_bit(clt->permits_map, sz) >= sz);
1442 	}
1443 	kfree(clt->permits_map);
1444 	clt->permits_map = NULL;
1445 	kfree(clt->permits);
1446 	clt->permits = NULL;
1447 }
1448 
1449 static void query_fast_reg_mode(struct rtrs_clt_path *clt_path)
1450 {
1451 	struct ib_device *ib_dev;
1452 	u64 max_pages_per_mr;
1453 	int mr_page_shift;
1454 
1455 	ib_dev = clt_path->s.dev->ib_dev;
1456 
1457 	/*
1458 	 * Use the smallest page size supported by the HCA, down to a
1459 	 * minimum of 4096 bytes. We're unlikely to build large sglists
1460 	 * out of smaller entries.
1461 	 */
1462 	mr_page_shift      = max(12, ffs(ib_dev->attrs.page_size_cap) - 1);
1463 	max_pages_per_mr   = ib_dev->attrs.max_mr_size;
1464 	do_div(max_pages_per_mr, (1ull << mr_page_shift));
1465 	clt_path->max_pages_per_mr =
1466 		min3(clt_path->max_pages_per_mr, (u32)max_pages_per_mr,
1467 		     ib_dev->attrs.max_fast_reg_page_list_len);
1468 	clt_path->clt->max_segments =
1469 		min(clt_path->max_pages_per_mr, clt_path->clt->max_segments);
1470 }
1471 
1472 static bool rtrs_clt_change_state_get_old(struct rtrs_clt_path *clt_path,
1473 					   enum rtrs_clt_state new_state,
1474 					   enum rtrs_clt_state *old_state)
1475 {
1476 	bool changed;
1477 
1478 	spin_lock_irq(&clt_path->state_wq.lock);
1479 	if (old_state)
1480 		*old_state = clt_path->state;
1481 	changed = rtrs_clt_change_state(clt_path, new_state);
1482 	spin_unlock_irq(&clt_path->state_wq.lock);
1483 
1484 	return changed;
1485 }
1486 
1487 static void rtrs_clt_hb_err_handler(struct rtrs_con *c)
1488 {
1489 	struct rtrs_clt_con *con = container_of(c, typeof(*con), c);
1490 
1491 	rtrs_rdma_error_recovery(con);
1492 }
1493 
1494 static void rtrs_clt_init_hb(struct rtrs_clt_path *clt_path)
1495 {
1496 	rtrs_init_hb(&clt_path->s, &io_comp_cqe,
1497 		      RTRS_HB_INTERVAL_MS,
1498 		      RTRS_HB_MISSED_MAX,
1499 		      rtrs_clt_hb_err_handler,
1500 		      rtrs_wq);
1501 }
1502 
1503 static void rtrs_clt_reconnect_work(struct work_struct *work);
1504 static void rtrs_clt_close_work(struct work_struct *work);
1505 
1506 static void rtrs_clt_err_recovery_work(struct work_struct *work)
1507 {
1508 	struct rtrs_clt_path *clt_path;
1509 	struct rtrs_clt_sess *clt;
1510 	int delay_ms;
1511 
1512 	clt_path = container_of(work, struct rtrs_clt_path, err_recovery_work);
1513 	clt = clt_path->clt;
1514 	delay_ms = clt->reconnect_delay_sec * 1000;
1515 	rtrs_clt_stop_and_destroy_conns(clt_path);
1516 	queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork,
1517 			   msecs_to_jiffies(delay_ms +
1518 					    prandom_u32() %
1519 					    RTRS_RECONNECT_SEED));
1520 }
1521 
1522 static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt,
1523 					const struct rtrs_addr *path,
1524 					size_t con_num, u32 nr_poll_queues)
1525 {
1526 	struct rtrs_clt_path *clt_path;
1527 	int err = -ENOMEM;
1528 	int cpu;
1529 	size_t total_con;
1530 
1531 	clt_path = kzalloc(sizeof(*clt_path), GFP_KERNEL);
1532 	if (!clt_path)
1533 		goto err;
1534 
1535 	/*
1536 	 * irqmode and poll
1537 	 * +1: Extra connection for user messages
1538 	 */
1539 	total_con = con_num + nr_poll_queues + 1;
1540 	clt_path->s.con = kcalloc(total_con, sizeof(*clt_path->s.con),
1541 				  GFP_KERNEL);
1542 	if (!clt_path->s.con)
1543 		goto err_free_path;
1544 
1545 	clt_path->s.con_num = total_con;
1546 	clt_path->s.irq_con_num = con_num + 1;
1547 
1548 	clt_path->stats = kzalloc(sizeof(*clt_path->stats), GFP_KERNEL);
1549 	if (!clt_path->stats)
1550 		goto err_free_con;
1551 
1552 	mutex_init(&clt_path->init_mutex);
1553 	uuid_gen(&clt_path->s.uuid);
1554 	memcpy(&clt_path->s.dst_addr, path->dst,
1555 	       rdma_addr_size((struct sockaddr *)path->dst));
1556 
1557 	/*
1558 	 * rdma_resolve_addr() passes src_addr to cma_bind_addr, which
1559 	 * checks the sa_family to be non-zero. If user passed src_addr=NULL
1560 	 * the sess->src_addr will contain only zeros, which is then fine.
1561 	 */
1562 	if (path->src)
1563 		memcpy(&clt_path->s.src_addr, path->src,
1564 		       rdma_addr_size((struct sockaddr *)path->src));
1565 	strscpy(clt_path->s.sessname, clt->sessname,
1566 		sizeof(clt_path->s.sessname));
1567 	clt_path->clt = clt;
1568 	clt_path->max_pages_per_mr = RTRS_MAX_SEGMENTS;
1569 	init_waitqueue_head(&clt_path->state_wq);
1570 	clt_path->state = RTRS_CLT_CONNECTING;
1571 	atomic_set(&clt_path->connected_cnt, 0);
1572 	INIT_WORK(&clt_path->close_work, rtrs_clt_close_work);
1573 	INIT_WORK(&clt_path->err_recovery_work, rtrs_clt_err_recovery_work);
1574 	INIT_DELAYED_WORK(&clt_path->reconnect_dwork, rtrs_clt_reconnect_work);
1575 	rtrs_clt_init_hb(clt_path);
1576 
1577 	clt_path->mp_skip_entry = alloc_percpu(typeof(*clt_path->mp_skip_entry));
1578 	if (!clt_path->mp_skip_entry)
1579 		goto err_free_stats;
1580 
1581 	for_each_possible_cpu(cpu)
1582 		INIT_LIST_HEAD(per_cpu_ptr(clt_path->mp_skip_entry, cpu));
1583 
1584 	err = rtrs_clt_init_stats(clt_path->stats);
1585 	if (err)
1586 		goto err_free_percpu;
1587 
1588 	return clt_path;
1589 
1590 err_free_percpu:
1591 	free_percpu(clt_path->mp_skip_entry);
1592 err_free_stats:
1593 	kfree(clt_path->stats);
1594 err_free_con:
1595 	kfree(clt_path->s.con);
1596 err_free_path:
1597 	kfree(clt_path);
1598 err:
1599 	return ERR_PTR(err);
1600 }
1601 
1602 void free_path(struct rtrs_clt_path *clt_path)
1603 {
1604 	free_percpu(clt_path->mp_skip_entry);
1605 	mutex_destroy(&clt_path->init_mutex);
1606 	kfree(clt_path->s.con);
1607 	kfree(clt_path->rbufs);
1608 	kfree(clt_path);
1609 }
1610 
1611 static int create_con(struct rtrs_clt_path *clt_path, unsigned int cid)
1612 {
1613 	struct rtrs_clt_con *con;
1614 
1615 	con = kzalloc(sizeof(*con), GFP_KERNEL);
1616 	if (!con)
1617 		return -ENOMEM;
1618 
1619 	/* Map first two connections to the first CPU */
1620 	con->cpu  = (cid ? cid - 1 : 0) % nr_cpu_ids;
1621 	con->c.cid = cid;
1622 	con->c.path = &clt_path->s;
1623 	/* Align with srv, init as 1 */
1624 	atomic_set(&con->c.wr_cnt, 1);
1625 	mutex_init(&con->con_mutex);
1626 
1627 	clt_path->s.con[cid] = &con->c;
1628 
1629 	return 0;
1630 }
1631 
1632 static void destroy_con(struct rtrs_clt_con *con)
1633 {
1634 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1635 
1636 	clt_path->s.con[con->c.cid] = NULL;
1637 	mutex_destroy(&con->con_mutex);
1638 	kfree(con);
1639 }
1640 
1641 static int create_con_cq_qp(struct rtrs_clt_con *con)
1642 {
1643 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1644 	u32 max_send_wr, max_recv_wr, cq_num, max_send_sge, wr_limit;
1645 	int err, cq_vector;
1646 	struct rtrs_msg_rkey_rsp *rsp;
1647 
1648 	lockdep_assert_held(&con->con_mutex);
1649 	if (con->c.cid == 0) {
1650 		max_send_sge = 1;
1651 		/* We must be the first here */
1652 		if (WARN_ON(clt_path->s.dev))
1653 			return -EINVAL;
1654 
1655 		/*
1656 		 * The whole session uses device from user connection.
1657 		 * Be careful not to close user connection before ib dev
1658 		 * is gracefully put.
1659 		 */
1660 		clt_path->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device,
1661 						       &dev_pd);
1662 		if (!clt_path->s.dev) {
1663 			rtrs_wrn(clt_path->clt,
1664 				  "rtrs_ib_dev_find_get_or_add(): no memory\n");
1665 			return -ENOMEM;
1666 		}
1667 		clt_path->s.dev_ref = 1;
1668 		query_fast_reg_mode(clt_path);
1669 		wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr;
1670 		/*
1671 		 * Two (request + registration) completion for send
1672 		 * Two for recv if always_invalidate is set on server
1673 		 * or one for recv.
1674 		 * + 2 for drain and heartbeat
1675 		 * in case qp gets into error state.
1676 		 */
1677 		max_send_wr =
1678 			min_t(int, wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2);
1679 		max_recv_wr = max_send_wr;
1680 	} else {
1681 		/*
1682 		 * Here we assume that session members are correctly set.
1683 		 * This is always true if user connection (cid == 0) is
1684 		 * established first.
1685 		 */
1686 		if (WARN_ON(!clt_path->s.dev))
1687 			return -EINVAL;
1688 		if (WARN_ON(!clt_path->queue_depth))
1689 			return -EINVAL;
1690 
1691 		wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr;
1692 		/* Shared between connections */
1693 		clt_path->s.dev_ref++;
1694 		max_send_wr = min_t(int, wr_limit,
1695 			      /* QD * (REQ + RSP + FR REGS or INVS) + drain */
1696 			      clt_path->queue_depth * 3 + 1);
1697 		max_recv_wr = min_t(int, wr_limit,
1698 			      clt_path->queue_depth * 3 + 1);
1699 		max_send_sge = 2;
1700 	}
1701 	atomic_set(&con->c.sq_wr_avail, max_send_wr);
1702 	cq_num = max_send_wr + max_recv_wr;
1703 	/* alloc iu to recv new rkey reply when server reports flags set */
1704 	if (clt_path->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {
1705 		con->rsp_ius = rtrs_iu_alloc(cq_num, sizeof(*rsp),
1706 					      GFP_KERNEL,
1707 					      clt_path->s.dev->ib_dev,
1708 					      DMA_FROM_DEVICE,
1709 					      rtrs_clt_rdma_done);
1710 		if (!con->rsp_ius)
1711 			return -ENOMEM;
1712 		con->queue_num = cq_num;
1713 	}
1714 	cq_num = max_send_wr + max_recv_wr;
1715 	cq_vector = con->cpu % clt_path->s.dev->ib_dev->num_comp_vectors;
1716 	if (con->c.cid >= clt_path->s.irq_con_num)
1717 		err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
1718 					cq_vector, cq_num, max_send_wr,
1719 					max_recv_wr, IB_POLL_DIRECT);
1720 	else
1721 		err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge,
1722 					cq_vector, cq_num, max_send_wr,
1723 					max_recv_wr, IB_POLL_SOFTIRQ);
1724 	/*
1725 	 * In case of error we do not bother to clean previous allocations,
1726 	 * since destroy_con_cq_qp() must be called.
1727 	 */
1728 	return err;
1729 }
1730 
1731 static void destroy_con_cq_qp(struct rtrs_clt_con *con)
1732 {
1733 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1734 
1735 	/*
1736 	 * Be careful here: destroy_con_cq_qp() can be called even
1737 	 * create_con_cq_qp() failed, see comments there.
1738 	 */
1739 	lockdep_assert_held(&con->con_mutex);
1740 	rtrs_cq_qp_destroy(&con->c);
1741 	if (con->rsp_ius) {
1742 		rtrs_iu_free(con->rsp_ius, clt_path->s.dev->ib_dev,
1743 			     con->queue_num);
1744 		con->rsp_ius = NULL;
1745 		con->queue_num = 0;
1746 	}
1747 	if (clt_path->s.dev_ref && !--clt_path->s.dev_ref) {
1748 		rtrs_ib_dev_put(clt_path->s.dev);
1749 		clt_path->s.dev = NULL;
1750 	}
1751 }
1752 
1753 static void stop_cm(struct rtrs_clt_con *con)
1754 {
1755 	rdma_disconnect(con->c.cm_id);
1756 	if (con->c.qp)
1757 		ib_drain_qp(con->c.qp);
1758 }
1759 
1760 static void destroy_cm(struct rtrs_clt_con *con)
1761 {
1762 	rdma_destroy_id(con->c.cm_id);
1763 	con->c.cm_id = NULL;
1764 }
1765 
1766 static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con)
1767 {
1768 	struct rtrs_path *s = con->c.path;
1769 	int err;
1770 
1771 	mutex_lock(&con->con_mutex);
1772 	err = create_con_cq_qp(con);
1773 	mutex_unlock(&con->con_mutex);
1774 	if (err) {
1775 		rtrs_err(s, "create_con_cq_qp(), err: %d\n", err);
1776 		return err;
1777 	}
1778 	err = rdma_resolve_route(con->c.cm_id, RTRS_CONNECT_TIMEOUT_MS);
1779 	if (err)
1780 		rtrs_err(s, "Resolving route failed, err: %d\n", err);
1781 
1782 	return err;
1783 }
1784 
1785 static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con)
1786 {
1787 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1788 	struct rtrs_clt_sess *clt = clt_path->clt;
1789 	struct rtrs_msg_conn_req msg;
1790 	struct rdma_conn_param param;
1791 
1792 	int err;
1793 
1794 	param = (struct rdma_conn_param) {
1795 		.retry_count = 7,
1796 		.rnr_retry_count = 7,
1797 		.private_data = &msg,
1798 		.private_data_len = sizeof(msg),
1799 	};
1800 
1801 	msg = (struct rtrs_msg_conn_req) {
1802 		.magic = cpu_to_le16(RTRS_MAGIC),
1803 		.version = cpu_to_le16(RTRS_PROTO_VER),
1804 		.cid = cpu_to_le16(con->c.cid),
1805 		.cid_num = cpu_to_le16(clt_path->s.con_num),
1806 		.recon_cnt = cpu_to_le16(clt_path->s.recon_cnt),
1807 	};
1808 	msg.first_conn = clt_path->for_new_clt ? FIRST_CONN : 0;
1809 	uuid_copy(&msg.sess_uuid, &clt_path->s.uuid);
1810 	uuid_copy(&msg.paths_uuid, &clt->paths_uuid);
1811 
1812 	err = rdma_connect_locked(con->c.cm_id, &param);
1813 	if (err)
1814 		rtrs_err(clt, "rdma_connect_locked(): %d\n", err);
1815 
1816 	return err;
1817 }
1818 
1819 static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,
1820 				       struct rdma_cm_event *ev)
1821 {
1822 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1823 	struct rtrs_clt_sess *clt = clt_path->clt;
1824 	const struct rtrs_msg_conn_rsp *msg;
1825 	u16 version, queue_depth;
1826 	int errno;
1827 	u8 len;
1828 
1829 	msg = ev->param.conn.private_data;
1830 	len = ev->param.conn.private_data_len;
1831 	if (len < sizeof(*msg)) {
1832 		rtrs_err(clt, "Invalid RTRS connection response\n");
1833 		return -ECONNRESET;
1834 	}
1835 	if (le16_to_cpu(msg->magic) != RTRS_MAGIC) {
1836 		rtrs_err(clt, "Invalid RTRS magic\n");
1837 		return -ECONNRESET;
1838 	}
1839 	version = le16_to_cpu(msg->version);
1840 	if (version >> 8 != RTRS_PROTO_VER_MAJOR) {
1841 		rtrs_err(clt, "Unsupported major RTRS version: %d, expected %d\n",
1842 			  version >> 8, RTRS_PROTO_VER_MAJOR);
1843 		return -ECONNRESET;
1844 	}
1845 	errno = le16_to_cpu(msg->errno);
1846 	if (errno) {
1847 		rtrs_err(clt, "Invalid RTRS message: errno %d\n",
1848 			  errno);
1849 		return -ECONNRESET;
1850 	}
1851 	if (con->c.cid == 0) {
1852 		queue_depth = le16_to_cpu(msg->queue_depth);
1853 
1854 		if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) {
1855 			rtrs_err(clt, "Error: queue depth changed\n");
1856 
1857 			/*
1858 			 * Stop any more reconnection attempts
1859 			 */
1860 			clt_path->reconnect_attempts = -1;
1861 			rtrs_err(clt,
1862 				"Disabling auto-reconnect. Trigger a manual reconnect after issue is resolved\n");
1863 			return -ECONNRESET;
1864 		}
1865 
1866 		if (!clt_path->rbufs) {
1867 			clt_path->rbufs = kcalloc(queue_depth,
1868 						  sizeof(*clt_path->rbufs),
1869 						  GFP_KERNEL);
1870 			if (!clt_path->rbufs)
1871 				return -ENOMEM;
1872 		}
1873 		clt_path->queue_depth = queue_depth;
1874 		clt_path->s.signal_interval = min_not_zero(queue_depth,
1875 						(unsigned short) SERVICE_CON_QUEUE_DEPTH);
1876 		clt_path->max_hdr_size = le32_to_cpu(msg->max_hdr_size);
1877 		clt_path->max_io_size = le32_to_cpu(msg->max_io_size);
1878 		clt_path->flags = le32_to_cpu(msg->flags);
1879 		clt_path->chunk_size = clt_path->max_io_size + clt_path->max_hdr_size;
1880 
1881 		/*
1882 		 * Global IO size is always a minimum.
1883 		 * If while a reconnection server sends us a value a bit
1884 		 * higher - client does not care and uses cached minimum.
1885 		 *
1886 		 * Since we can have several sessions (paths) restablishing
1887 		 * connections in parallel, use lock.
1888 		 */
1889 		mutex_lock(&clt->paths_mutex);
1890 		clt->queue_depth = clt_path->queue_depth;
1891 		clt->max_io_size = min_not_zero(clt_path->max_io_size,
1892 						clt->max_io_size);
1893 		mutex_unlock(&clt->paths_mutex);
1894 
1895 		/*
1896 		 * Cache the hca_port and hca_name for sysfs
1897 		 */
1898 		clt_path->hca_port = con->c.cm_id->port_num;
1899 		scnprintf(clt_path->hca_name, sizeof(clt_path->hca_name),
1900 			  clt_path->s.dev->ib_dev->name);
1901 		clt_path->s.src_addr = con->c.cm_id->route.addr.src_addr;
1902 		/* set for_new_clt, to allow future reconnect on any path */
1903 		clt_path->for_new_clt = 1;
1904 	}
1905 
1906 	return 0;
1907 }
1908 
1909 static inline void flag_success_on_conn(struct rtrs_clt_con *con)
1910 {
1911 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
1912 
1913 	atomic_inc(&clt_path->connected_cnt);
1914 	con->cm_err = 1;
1915 }
1916 
1917 static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,
1918 				    struct rdma_cm_event *ev)
1919 {
1920 	struct rtrs_path *s = con->c.path;
1921 	const struct rtrs_msg_conn_rsp *msg;
1922 	const char *rej_msg;
1923 	int status, errno;
1924 	u8 data_len;
1925 
1926 	status = ev->status;
1927 	rej_msg = rdma_reject_msg(con->c.cm_id, status);
1928 	msg = rdma_consumer_reject_data(con->c.cm_id, ev, &data_len);
1929 
1930 	if (msg && data_len >= sizeof(*msg)) {
1931 		errno = (int16_t)le16_to_cpu(msg->errno);
1932 		if (errno == -EBUSY)
1933 			rtrs_err(s,
1934 				  "Previous session is still exists on the server, please reconnect later\n");
1935 		else
1936 			rtrs_err(s,
1937 				  "Connect rejected: status %d (%s), rtrs errno %d\n",
1938 				  status, rej_msg, errno);
1939 	} else {
1940 		rtrs_err(s,
1941 			  "Connect rejected but with malformed message: status %d (%s)\n",
1942 			  status, rej_msg);
1943 	}
1944 
1945 	return -ECONNRESET;
1946 }
1947 
1948 void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait)
1949 {
1950 	if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL))
1951 		queue_work(rtrs_wq, &clt_path->close_work);
1952 	if (wait)
1953 		flush_work(&clt_path->close_work);
1954 }
1955 
1956 static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err)
1957 {
1958 	if (con->cm_err == 1) {
1959 		struct rtrs_clt_path *clt_path;
1960 
1961 		clt_path = to_clt_path(con->c.path);
1962 		if (atomic_dec_and_test(&clt_path->connected_cnt))
1963 
1964 			wake_up(&clt_path->state_wq);
1965 	}
1966 	con->cm_err = cm_err;
1967 }
1968 
1969 static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id,
1970 				     struct rdma_cm_event *ev)
1971 {
1972 	struct rtrs_clt_con *con = cm_id->context;
1973 	struct rtrs_path *s = con->c.path;
1974 	struct rtrs_clt_path *clt_path = to_clt_path(s);
1975 	int cm_err = 0;
1976 
1977 	switch (ev->event) {
1978 	case RDMA_CM_EVENT_ADDR_RESOLVED:
1979 		cm_err = rtrs_rdma_addr_resolved(con);
1980 		break;
1981 	case RDMA_CM_EVENT_ROUTE_RESOLVED:
1982 		cm_err = rtrs_rdma_route_resolved(con);
1983 		break;
1984 	case RDMA_CM_EVENT_ESTABLISHED:
1985 		cm_err = rtrs_rdma_conn_established(con, ev);
1986 		if (!cm_err) {
1987 			/*
1988 			 * Report success and wake up. Here we abuse state_wq,
1989 			 * i.e. wake up without state change, but we set cm_err.
1990 			 */
1991 			flag_success_on_conn(con);
1992 			wake_up(&clt_path->state_wq);
1993 			return 0;
1994 		}
1995 		break;
1996 	case RDMA_CM_EVENT_REJECTED:
1997 		cm_err = rtrs_rdma_conn_rejected(con, ev);
1998 		break;
1999 	case RDMA_CM_EVENT_DISCONNECTED:
2000 		/* No message for disconnecting */
2001 		cm_err = -ECONNRESET;
2002 		break;
2003 	case RDMA_CM_EVENT_CONNECT_ERROR:
2004 	case RDMA_CM_EVENT_UNREACHABLE:
2005 	case RDMA_CM_EVENT_ADDR_CHANGE:
2006 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
2007 		rtrs_wrn(s, "CM error (CM event: %s, err: %d)\n",
2008 			 rdma_event_msg(ev->event), ev->status);
2009 		cm_err = -ECONNRESET;
2010 		break;
2011 	case RDMA_CM_EVENT_ADDR_ERROR:
2012 	case RDMA_CM_EVENT_ROUTE_ERROR:
2013 		rtrs_wrn(s, "CM error (CM event: %s, err: %d)\n",
2014 			 rdma_event_msg(ev->event), ev->status);
2015 		cm_err = -EHOSTUNREACH;
2016 		break;
2017 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
2018 		/*
2019 		 * Device removal is a special case.  Queue close and return 0.
2020 		 */
2021 		rtrs_clt_close_conns(clt_path, false);
2022 		return 0;
2023 	default:
2024 		rtrs_err(s, "Unexpected RDMA CM error (CM event: %s, err: %d)\n",
2025 			 rdma_event_msg(ev->event), ev->status);
2026 		cm_err = -ECONNRESET;
2027 		break;
2028 	}
2029 
2030 	if (cm_err) {
2031 		/*
2032 		 * cm error makes sense only on connection establishing,
2033 		 * in other cases we rely on normal procedure of reconnecting.
2034 		 */
2035 		flag_error_on_conn(con, cm_err);
2036 		rtrs_rdma_error_recovery(con);
2037 	}
2038 
2039 	return 0;
2040 }
2041 
2042 static int create_cm(struct rtrs_clt_con *con)
2043 {
2044 	struct rtrs_path *s = con->c.path;
2045 	struct rtrs_clt_path *clt_path = to_clt_path(s);
2046 	struct rdma_cm_id *cm_id;
2047 	int err;
2048 
2049 	cm_id = rdma_create_id(&init_net, rtrs_clt_rdma_cm_handler, con,
2050 			       clt_path->s.dst_addr.ss_family == AF_IB ?
2051 			       RDMA_PS_IB : RDMA_PS_TCP, IB_QPT_RC);
2052 	if (IS_ERR(cm_id)) {
2053 		err = PTR_ERR(cm_id);
2054 		rtrs_err(s, "Failed to create CM ID, err: %d\n", err);
2055 
2056 		return err;
2057 	}
2058 	con->c.cm_id = cm_id;
2059 	con->cm_err = 0;
2060 	/* allow the port to be reused */
2061 	err = rdma_set_reuseaddr(cm_id, 1);
2062 	if (err != 0) {
2063 		rtrs_err(s, "Set address reuse failed, err: %d\n", err);
2064 		goto destroy_cm;
2065 	}
2066 	err = rdma_resolve_addr(cm_id, (struct sockaddr *)&clt_path->s.src_addr,
2067 				(struct sockaddr *)&clt_path->s.dst_addr,
2068 				RTRS_CONNECT_TIMEOUT_MS);
2069 	if (err) {
2070 		rtrs_err(s, "Failed to resolve address, err: %d\n", err);
2071 		goto destroy_cm;
2072 	}
2073 	/*
2074 	 * Combine connection status and session events. This is needed
2075 	 * for waiting two possible cases: cm_err has something meaningful
2076 	 * or session state was really changed to error by device removal.
2077 	 */
2078 	err = wait_event_interruptible_timeout(
2079 			clt_path->state_wq,
2080 			con->cm_err || clt_path->state != RTRS_CLT_CONNECTING,
2081 			msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));
2082 	if (err == 0 || err == -ERESTARTSYS) {
2083 		if (err == 0)
2084 			err = -ETIMEDOUT;
2085 		/* Timedout or interrupted */
2086 		goto errr;
2087 	}
2088 	if (con->cm_err < 0) {
2089 		err = con->cm_err;
2090 		goto errr;
2091 	}
2092 	if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTING) {
2093 		/* Device removal */
2094 		err = -ECONNABORTED;
2095 		goto errr;
2096 	}
2097 
2098 	return 0;
2099 
2100 errr:
2101 	stop_cm(con);
2102 	mutex_lock(&con->con_mutex);
2103 	destroy_con_cq_qp(con);
2104 	mutex_unlock(&con->con_mutex);
2105 destroy_cm:
2106 	destroy_cm(con);
2107 
2108 	return err;
2109 }
2110 
2111 static void rtrs_clt_path_up(struct rtrs_clt_path *clt_path)
2112 {
2113 	struct rtrs_clt_sess *clt = clt_path->clt;
2114 	int up;
2115 
2116 	/*
2117 	 * We can fire RECONNECTED event only when all paths were
2118 	 * connected on rtrs_clt_open(), then each was disconnected
2119 	 * and the first one connected again.  That's why this nasty
2120 	 * game with counter value.
2121 	 */
2122 
2123 	mutex_lock(&clt->paths_ev_mutex);
2124 	up = ++clt->paths_up;
2125 	/*
2126 	 * Here it is safe to access paths num directly since up counter
2127 	 * is greater than MAX_PATHS_NUM only while rtrs_clt_open() is
2128 	 * in progress, thus paths removals are impossible.
2129 	 */
2130 	if (up > MAX_PATHS_NUM && up == MAX_PATHS_NUM + clt->paths_num)
2131 		clt->paths_up = clt->paths_num;
2132 	else if (up == 1)
2133 		clt->link_ev(clt->priv, RTRS_CLT_LINK_EV_RECONNECTED);
2134 	mutex_unlock(&clt->paths_ev_mutex);
2135 
2136 	/* Mark session as established */
2137 	clt_path->established = true;
2138 	clt_path->reconnect_attempts = 0;
2139 	clt_path->stats->reconnects.successful_cnt++;
2140 }
2141 
2142 static void rtrs_clt_path_down(struct rtrs_clt_path *clt_path)
2143 {
2144 	struct rtrs_clt_sess *clt = clt_path->clt;
2145 
2146 	if (!clt_path->established)
2147 		return;
2148 
2149 	clt_path->established = false;
2150 	mutex_lock(&clt->paths_ev_mutex);
2151 	WARN_ON(!clt->paths_up);
2152 	if (--clt->paths_up == 0)
2153 		clt->link_ev(clt->priv, RTRS_CLT_LINK_EV_DISCONNECTED);
2154 	mutex_unlock(&clt->paths_ev_mutex);
2155 }
2156 
2157 static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path)
2158 {
2159 	struct rtrs_clt_con *con;
2160 	unsigned int cid;
2161 
2162 	WARN_ON(READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED);
2163 
2164 	/*
2165 	 * Possible race with rtrs_clt_open(), when DEVICE_REMOVAL comes
2166 	 * exactly in between.  Start destroying after it finishes.
2167 	 */
2168 	mutex_lock(&clt_path->init_mutex);
2169 	mutex_unlock(&clt_path->init_mutex);
2170 
2171 	/*
2172 	 * All IO paths must observe !CONNECTED state before we
2173 	 * free everything.
2174 	 */
2175 	synchronize_rcu();
2176 
2177 	rtrs_stop_hb(&clt_path->s);
2178 
2179 	/*
2180 	 * The order it utterly crucial: firstly disconnect and complete all
2181 	 * rdma requests with error (thus set in_use=false for requests),
2182 	 * then fail outstanding requests checking in_use for each, and
2183 	 * eventually notify upper layer about session disconnection.
2184 	 */
2185 
2186 	for (cid = 0; cid < clt_path->s.con_num; cid++) {
2187 		if (!clt_path->s.con[cid])
2188 			break;
2189 		con = to_clt_con(clt_path->s.con[cid]);
2190 		stop_cm(con);
2191 	}
2192 	fail_all_outstanding_reqs(clt_path);
2193 	free_path_reqs(clt_path);
2194 	rtrs_clt_path_down(clt_path);
2195 
2196 	/*
2197 	 * Wait for graceful shutdown, namely when peer side invokes
2198 	 * rdma_disconnect(). 'connected_cnt' is decremented only on
2199 	 * CM events, thus if other side had crashed and hb has detected
2200 	 * something is wrong, here we will stuck for exactly timeout ms,
2201 	 * since CM does not fire anything.  That is fine, we are not in
2202 	 * hurry.
2203 	 */
2204 	wait_event_timeout(clt_path->state_wq,
2205 			   !atomic_read(&clt_path->connected_cnt),
2206 			   msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));
2207 
2208 	for (cid = 0; cid < clt_path->s.con_num; cid++) {
2209 		if (!clt_path->s.con[cid])
2210 			break;
2211 		con = to_clt_con(clt_path->s.con[cid]);
2212 		mutex_lock(&con->con_mutex);
2213 		destroy_con_cq_qp(con);
2214 		mutex_unlock(&con->con_mutex);
2215 		destroy_cm(con);
2216 		destroy_con(con);
2217 	}
2218 }
2219 
2220 static inline bool xchg_paths(struct rtrs_clt_path __rcu **rcu_ppcpu_path,
2221 			      struct rtrs_clt_path *clt_path,
2222 			      struct rtrs_clt_path *next)
2223 {
2224 	struct rtrs_clt_path **ppcpu_path;
2225 
2226 	/* Call cmpxchg() without sparse warnings */
2227 	ppcpu_path = (typeof(ppcpu_path))rcu_ppcpu_path;
2228 	return clt_path == cmpxchg(ppcpu_path, clt_path, next);
2229 }
2230 
2231 static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_path *clt_path)
2232 {
2233 	struct rtrs_clt_sess *clt = clt_path->clt;
2234 	struct rtrs_clt_path *next;
2235 	bool wait_for_grace = false;
2236 	int cpu;
2237 
2238 	mutex_lock(&clt->paths_mutex);
2239 	list_del_rcu(&clt_path->s.entry);
2240 
2241 	/* Make sure everybody observes path removal. */
2242 	synchronize_rcu();
2243 
2244 	/*
2245 	 * At this point nobody sees @sess in the list, but still we have
2246 	 * dangling pointer @pcpu_path which _can_ point to @sess.  Since
2247 	 * nobody can observe @sess in the list, we guarantee that IO path
2248 	 * will not assign @sess to @pcpu_path, i.e. @pcpu_path can be equal
2249 	 * to @sess, but can never again become @sess.
2250 	 */
2251 
2252 	/*
2253 	 * Decrement paths number only after grace period, because
2254 	 * caller of do_each_path() must firstly observe list without
2255 	 * path and only then decremented paths number.
2256 	 *
2257 	 * Otherwise there can be the following situation:
2258 	 *    o Two paths exist and IO is coming.
2259 	 *    o One path is removed:
2260 	 *      CPU#0                          CPU#1
2261 	 *      do_each_path():                rtrs_clt_remove_path_from_arr():
2262 	 *          path = get_next_path()
2263 	 *          ^^^                            list_del_rcu(path)
2264 	 *          [!CONNECTED path]              clt->paths_num--
2265 	 *                                              ^^^^^^^^^
2266 	 *          load clt->paths_num                 from 2 to 1
2267 	 *                    ^^^^^^^^^
2268 	 *                    sees 1
2269 	 *
2270 	 *      path is observed as !CONNECTED, but do_each_path() loop
2271 	 *      ends, because expression i < clt->paths_num is false.
2272 	 */
2273 	clt->paths_num--;
2274 
2275 	/*
2276 	 * Get @next connection from current @sess which is going to be
2277 	 * removed.  If @sess is the last element, then @next is NULL.
2278 	 */
2279 	rcu_read_lock();
2280 	next = list_next_or_null_rr_rcu(&clt->paths_list, &clt_path->s.entry,
2281 					typeof(*next), s.entry);
2282 	rcu_read_unlock();
2283 
2284 	/*
2285 	 * @pcpu paths can still point to the path which is going to be
2286 	 * removed, so change the pointer manually.
2287 	 */
2288 	for_each_possible_cpu(cpu) {
2289 		struct rtrs_clt_path __rcu **ppcpu_path;
2290 
2291 		ppcpu_path = per_cpu_ptr(clt->pcpu_path, cpu);
2292 		if (rcu_dereference_protected(*ppcpu_path,
2293 			lockdep_is_held(&clt->paths_mutex)) != clt_path)
2294 			/*
2295 			 * synchronize_rcu() was called just after deleting
2296 			 * entry from the list, thus IO code path cannot
2297 			 * change pointer back to the pointer which is going
2298 			 * to be removed, we are safe here.
2299 			 */
2300 			continue;
2301 
2302 		/*
2303 		 * We race with IO code path, which also changes pointer,
2304 		 * thus we have to be careful not to overwrite it.
2305 		 */
2306 		if (xchg_paths(ppcpu_path, clt_path, next))
2307 			/*
2308 			 * @ppcpu_path was successfully replaced with @next,
2309 			 * that means that someone could also pick up the
2310 			 * @sess and dereferencing it right now, so wait for
2311 			 * a grace period is required.
2312 			 */
2313 			wait_for_grace = true;
2314 	}
2315 	if (wait_for_grace)
2316 		synchronize_rcu();
2317 
2318 	mutex_unlock(&clt->paths_mutex);
2319 }
2320 
2321 static void rtrs_clt_add_path_to_arr(struct rtrs_clt_path *clt_path)
2322 {
2323 	struct rtrs_clt_sess *clt = clt_path->clt;
2324 
2325 	mutex_lock(&clt->paths_mutex);
2326 	clt->paths_num++;
2327 
2328 	list_add_tail_rcu(&clt_path->s.entry, &clt->paths_list);
2329 	mutex_unlock(&clt->paths_mutex);
2330 }
2331 
2332 static void rtrs_clt_close_work(struct work_struct *work)
2333 {
2334 	struct rtrs_clt_path *clt_path;
2335 
2336 	clt_path = container_of(work, struct rtrs_clt_path, close_work);
2337 
2338 	cancel_work_sync(&clt_path->err_recovery_work);
2339 	cancel_delayed_work_sync(&clt_path->reconnect_dwork);
2340 	rtrs_clt_stop_and_destroy_conns(clt_path);
2341 	rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSED, NULL);
2342 }
2343 
2344 static int init_conns(struct rtrs_clt_path *clt_path)
2345 {
2346 	unsigned int cid;
2347 	int err;
2348 
2349 	/*
2350 	 * On every new session connections increase reconnect counter
2351 	 * to avoid clashes with previous sessions not yet closed
2352 	 * sessions on a server side.
2353 	 */
2354 	clt_path->s.recon_cnt++;
2355 
2356 	/* Establish all RDMA connections  */
2357 	for (cid = 0; cid < clt_path->s.con_num; cid++) {
2358 		err = create_con(clt_path, cid);
2359 		if (err)
2360 			goto destroy;
2361 
2362 		err = create_cm(to_clt_con(clt_path->s.con[cid]));
2363 		if (err) {
2364 			destroy_con(to_clt_con(clt_path->s.con[cid]));
2365 			goto destroy;
2366 		}
2367 	}
2368 	err = alloc_path_reqs(clt_path);
2369 	if (err)
2370 		goto destroy;
2371 
2372 	rtrs_start_hb(&clt_path->s);
2373 
2374 	return 0;
2375 
2376 destroy:
2377 	while (cid--) {
2378 		struct rtrs_clt_con *con = to_clt_con(clt_path->s.con[cid]);
2379 
2380 		stop_cm(con);
2381 
2382 		mutex_lock(&con->con_mutex);
2383 		destroy_con_cq_qp(con);
2384 		mutex_unlock(&con->con_mutex);
2385 		destroy_cm(con);
2386 		destroy_con(con);
2387 	}
2388 	/*
2389 	 * If we've never taken async path and got an error, say,
2390 	 * doing rdma_resolve_addr(), switch to CONNECTION_ERR state
2391 	 * manually to keep reconnecting.
2392 	 */
2393 	rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING_ERR, NULL);
2394 
2395 	return err;
2396 }
2397 
2398 static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc)
2399 {
2400 	struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
2401 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
2402 	struct rtrs_iu *iu;
2403 
2404 	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
2405 	rtrs_iu_free(iu, clt_path->s.dev->ib_dev, 1);
2406 
2407 	if (wc->status != IB_WC_SUCCESS) {
2408 		rtrs_err(clt_path->clt, "Path info request send failed: %s\n",
2409 			  ib_wc_status_msg(wc->status));
2410 		rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING_ERR, NULL);
2411 		return;
2412 	}
2413 
2414 	rtrs_clt_update_wc_stats(con);
2415 }
2416 
2417 static int process_info_rsp(struct rtrs_clt_path *clt_path,
2418 			    const struct rtrs_msg_info_rsp *msg)
2419 {
2420 	unsigned int sg_cnt, total_len;
2421 	int i, sgi;
2422 
2423 	sg_cnt = le16_to_cpu(msg->sg_cnt);
2424 	if (!sg_cnt || (clt_path->queue_depth % sg_cnt)) {
2425 		rtrs_err(clt_path->clt,
2426 			  "Incorrect sg_cnt %d, is not multiple\n",
2427 			  sg_cnt);
2428 		return -EINVAL;
2429 	}
2430 
2431 	/*
2432 	 * Check if IB immediate data size is enough to hold the mem_id and
2433 	 * the offset inside the memory chunk.
2434 	 */
2435 	if ((ilog2(sg_cnt - 1) + 1) + (ilog2(clt_path->chunk_size - 1) + 1) >
2436 	    MAX_IMM_PAYL_BITS) {
2437 		rtrs_err(clt_path->clt,
2438 			  "RDMA immediate size (%db) not enough to encode %d buffers of size %dB\n",
2439 			  MAX_IMM_PAYL_BITS, sg_cnt, clt_path->chunk_size);
2440 		return -EINVAL;
2441 	}
2442 	total_len = 0;
2443 	for (sgi = 0, i = 0; sgi < sg_cnt && i < clt_path->queue_depth; sgi++) {
2444 		const struct rtrs_sg_desc *desc = &msg->desc[sgi];
2445 		u32 len, rkey;
2446 		u64 addr;
2447 
2448 		addr = le64_to_cpu(desc->addr);
2449 		rkey = le32_to_cpu(desc->key);
2450 		len  = le32_to_cpu(desc->len);
2451 
2452 		total_len += len;
2453 
2454 		if (!len || (len % clt_path->chunk_size)) {
2455 			rtrs_err(clt_path->clt, "Incorrect [%d].len %d\n",
2456 				  sgi,
2457 				  len);
2458 			return -EINVAL;
2459 		}
2460 		for ( ; len && i < clt_path->queue_depth; i++) {
2461 			clt_path->rbufs[i].addr = addr;
2462 			clt_path->rbufs[i].rkey = rkey;
2463 
2464 			len  -= clt_path->chunk_size;
2465 			addr += clt_path->chunk_size;
2466 		}
2467 	}
2468 	/* Sanity check */
2469 	if (sgi != sg_cnt || i != clt_path->queue_depth) {
2470 		rtrs_err(clt_path->clt,
2471 			 "Incorrect sg vector, not fully mapped\n");
2472 		return -EINVAL;
2473 	}
2474 	if (total_len != clt_path->chunk_size * clt_path->queue_depth) {
2475 		rtrs_err(clt_path->clt, "Incorrect total_len %d\n", total_len);
2476 		return -EINVAL;
2477 	}
2478 
2479 	return 0;
2480 }
2481 
2482 static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)
2483 {
2484 	struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context);
2485 	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
2486 	struct rtrs_msg_info_rsp *msg;
2487 	enum rtrs_clt_state state;
2488 	struct rtrs_iu *iu;
2489 	size_t rx_sz;
2490 	int err;
2491 
2492 	state = RTRS_CLT_CONNECTING_ERR;
2493 
2494 	WARN_ON(con->c.cid);
2495 	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
2496 	if (wc->status != IB_WC_SUCCESS) {
2497 		rtrs_err(clt_path->clt, "Path info response recv failed: %s\n",
2498 			  ib_wc_status_msg(wc->status));
2499 		goto out;
2500 	}
2501 	WARN_ON(wc->opcode != IB_WC_RECV);
2502 
2503 	if (wc->byte_len < sizeof(*msg)) {
2504 		rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n",
2505 			  wc->byte_len);
2506 		goto out;
2507 	}
2508 	ib_dma_sync_single_for_cpu(clt_path->s.dev->ib_dev, iu->dma_addr,
2509 				   iu->size, DMA_FROM_DEVICE);
2510 	msg = iu->buf;
2511 	if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_RSP) {
2512 		rtrs_err(clt_path->clt, "Path info response is malformed: type %d\n",
2513 			  le16_to_cpu(msg->type));
2514 		goto out;
2515 	}
2516 	rx_sz  = sizeof(*msg);
2517 	rx_sz += sizeof(msg->desc[0]) * le16_to_cpu(msg->sg_cnt);
2518 	if (wc->byte_len < rx_sz) {
2519 		rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n",
2520 			  wc->byte_len);
2521 		goto out;
2522 	}
2523 	err = process_info_rsp(clt_path, msg);
2524 	if (err)
2525 		goto out;
2526 
2527 	err = post_recv_path(clt_path);
2528 	if (err)
2529 		goto out;
2530 
2531 	state = RTRS_CLT_CONNECTED;
2532 
2533 out:
2534 	rtrs_clt_update_wc_stats(con);
2535 	rtrs_iu_free(iu, clt_path->s.dev->ib_dev, 1);
2536 	rtrs_clt_change_state_get_old(clt_path, state, NULL);
2537 }
2538 
2539 static int rtrs_send_path_info(struct rtrs_clt_path *clt_path)
2540 {
2541 	struct rtrs_clt_con *usr_con = to_clt_con(clt_path->s.con[0]);
2542 	struct rtrs_msg_info_req *msg;
2543 	struct rtrs_iu *tx_iu, *rx_iu;
2544 	size_t rx_sz;
2545 	int err;
2546 
2547 	rx_sz  = sizeof(struct rtrs_msg_info_rsp);
2548 	rx_sz += sizeof(struct rtrs_sg_desc) * clt_path->queue_depth;
2549 
2550 	tx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), GFP_KERNEL,
2551 			       clt_path->s.dev->ib_dev, DMA_TO_DEVICE,
2552 			       rtrs_clt_info_req_done);
2553 	rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, clt_path->s.dev->ib_dev,
2554 			       DMA_FROM_DEVICE, rtrs_clt_info_rsp_done);
2555 	if (!tx_iu || !rx_iu) {
2556 		err = -ENOMEM;
2557 		goto out;
2558 	}
2559 	/* Prepare for getting info response */
2560 	err = rtrs_iu_post_recv(&usr_con->c, rx_iu);
2561 	if (err) {
2562 		rtrs_err(clt_path->clt, "rtrs_iu_post_recv(), err: %d\n", err);
2563 		goto out;
2564 	}
2565 	rx_iu = NULL;
2566 
2567 	msg = tx_iu->buf;
2568 	msg->type = cpu_to_le16(RTRS_MSG_INFO_REQ);
2569 	memcpy(msg->pathname, clt_path->s.sessname, sizeof(msg->pathname));
2570 
2571 	ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev,
2572 				      tx_iu->dma_addr,
2573 				      tx_iu->size, DMA_TO_DEVICE);
2574 
2575 	/* Send info request */
2576 	err = rtrs_iu_post_send(&usr_con->c, tx_iu, sizeof(*msg), NULL);
2577 	if (err) {
2578 		rtrs_err(clt_path->clt, "rtrs_iu_post_send(), err: %d\n", err);
2579 		goto out;
2580 	}
2581 	tx_iu = NULL;
2582 
2583 	/* Wait for state change */
2584 	wait_event_interruptible_timeout(clt_path->state_wq,
2585 					 clt_path->state != RTRS_CLT_CONNECTING,
2586 					 msecs_to_jiffies(
2587 						 RTRS_CONNECT_TIMEOUT_MS));
2588 	if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) {
2589 		if (READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTING_ERR)
2590 			err = -ECONNRESET;
2591 		else
2592 			err = -ETIMEDOUT;
2593 	}
2594 
2595 out:
2596 	if (tx_iu)
2597 		rtrs_iu_free(tx_iu, clt_path->s.dev->ib_dev, 1);
2598 	if (rx_iu)
2599 		rtrs_iu_free(rx_iu, clt_path->s.dev->ib_dev, 1);
2600 	if (err)
2601 		/* If we've never taken async path because of malloc problems */
2602 		rtrs_clt_change_state_get_old(clt_path,
2603 					      RTRS_CLT_CONNECTING_ERR, NULL);
2604 
2605 	return err;
2606 }
2607 
2608 /**
2609  * init_path() - establishes all path connections and does handshake
2610  * @clt_path: client path.
2611  * In case of error full close or reconnect procedure should be taken,
2612  * because reconnect or close async works can be started.
2613  */
2614 static int init_path(struct rtrs_clt_path *clt_path)
2615 {
2616 	int err;
2617 	char str[NAME_MAX];
2618 	struct rtrs_addr path = {
2619 		.src = &clt_path->s.src_addr,
2620 		.dst = &clt_path->s.dst_addr,
2621 	};
2622 
2623 	rtrs_addr_to_str(&path, str, sizeof(str));
2624 
2625 	mutex_lock(&clt_path->init_mutex);
2626 	err = init_conns(clt_path);
2627 	if (err) {
2628 		rtrs_err(clt_path->clt,
2629 			 "init_conns() failed: err=%d path=%s [%s:%u]\n", err,
2630 			 str, clt_path->hca_name, clt_path->hca_port);
2631 		goto out;
2632 	}
2633 	err = rtrs_send_path_info(clt_path);
2634 	if (err) {
2635 		rtrs_err(clt_path->clt,
2636 			 "rtrs_send_path_info() failed: err=%d path=%s [%s:%u]\n",
2637 			 err, str, clt_path->hca_name, clt_path->hca_port);
2638 		goto out;
2639 	}
2640 	rtrs_clt_path_up(clt_path);
2641 out:
2642 	mutex_unlock(&clt_path->init_mutex);
2643 
2644 	return err;
2645 }
2646 
2647 static void rtrs_clt_reconnect_work(struct work_struct *work)
2648 {
2649 	struct rtrs_clt_path *clt_path;
2650 	struct rtrs_clt_sess *clt;
2651 	int err;
2652 
2653 	clt_path = container_of(to_delayed_work(work), struct rtrs_clt_path,
2654 				reconnect_dwork);
2655 	clt = clt_path->clt;
2656 
2657 	if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING)
2658 		return;
2659 
2660 	if (clt_path->reconnect_attempts >= clt->max_reconnect_attempts) {
2661 		/* Close a path completely if max attempts is reached */
2662 		rtrs_clt_close_conns(clt_path, false);
2663 		return;
2664 	}
2665 	clt_path->reconnect_attempts++;
2666 
2667 	msleep(RTRS_RECONNECT_BACKOFF);
2668 	if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING, NULL)) {
2669 		err = init_path(clt_path);
2670 		if (err)
2671 			goto reconnect_again;
2672 	}
2673 
2674 	return;
2675 
2676 reconnect_again:
2677 	if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_RECONNECTING, NULL)) {
2678 		clt_path->stats->reconnects.fail_cnt++;
2679 		queue_work(rtrs_wq, &clt_path->err_recovery_work);
2680 	}
2681 }
2682 
2683 static void rtrs_clt_dev_release(struct device *dev)
2684 {
2685 	struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess,
2686 						 dev);
2687 
2688 	mutex_destroy(&clt->paths_ev_mutex);
2689 	mutex_destroy(&clt->paths_mutex);
2690 	kfree(clt);
2691 }
2692 
2693 static struct rtrs_clt_sess *alloc_clt(const char *sessname, size_t paths_num,
2694 				  u16 port, size_t pdu_sz, void *priv,
2695 				  void	(*link_ev)(void *priv,
2696 						   enum rtrs_clt_link_ev ev),
2697 				  unsigned int reconnect_delay_sec,
2698 				  unsigned int max_reconnect_attempts)
2699 {
2700 	struct rtrs_clt_sess *clt;
2701 	int err;
2702 
2703 	if (!paths_num || paths_num > MAX_PATHS_NUM)
2704 		return ERR_PTR(-EINVAL);
2705 
2706 	if (strlen(sessname) >= sizeof(clt->sessname))
2707 		return ERR_PTR(-EINVAL);
2708 
2709 	clt = kzalloc(sizeof(*clt), GFP_KERNEL);
2710 	if (!clt)
2711 		return ERR_PTR(-ENOMEM);
2712 
2713 	clt->pcpu_path = alloc_percpu(typeof(*clt->pcpu_path));
2714 	if (!clt->pcpu_path) {
2715 		kfree(clt);
2716 		return ERR_PTR(-ENOMEM);
2717 	}
2718 
2719 	clt->dev.class = rtrs_clt_dev_class;
2720 	clt->dev.release = rtrs_clt_dev_release;
2721 	uuid_gen(&clt->paths_uuid);
2722 	INIT_LIST_HEAD_RCU(&clt->paths_list);
2723 	clt->paths_num = paths_num;
2724 	clt->paths_up = MAX_PATHS_NUM;
2725 	clt->port = port;
2726 	clt->pdu_sz = pdu_sz;
2727 	clt->max_segments = RTRS_MAX_SEGMENTS;
2728 	clt->reconnect_delay_sec = reconnect_delay_sec;
2729 	clt->max_reconnect_attempts = max_reconnect_attempts;
2730 	clt->priv = priv;
2731 	clt->link_ev = link_ev;
2732 	clt->mp_policy = MP_POLICY_MIN_INFLIGHT;
2733 	strscpy(clt->sessname, sessname, sizeof(clt->sessname));
2734 	init_waitqueue_head(&clt->permits_wait);
2735 	mutex_init(&clt->paths_ev_mutex);
2736 	mutex_init(&clt->paths_mutex);
2737 	device_initialize(&clt->dev);
2738 
2739 	err = dev_set_name(&clt->dev, "%s", sessname);
2740 	if (err)
2741 		goto err_put;
2742 
2743 	/*
2744 	 * Suppress user space notification until
2745 	 * sysfs files are created
2746 	 */
2747 	dev_set_uevent_suppress(&clt->dev, true);
2748 	err = device_add(&clt->dev);
2749 	if (err)
2750 		goto err_put;
2751 
2752 	clt->kobj_paths = kobject_create_and_add("paths", &clt->dev.kobj);
2753 	if (!clt->kobj_paths) {
2754 		err = -ENOMEM;
2755 		goto err_del;
2756 	}
2757 	err = rtrs_clt_create_sysfs_root_files(clt);
2758 	if (err) {
2759 		kobject_del(clt->kobj_paths);
2760 		kobject_put(clt->kobj_paths);
2761 		goto err_del;
2762 	}
2763 	dev_set_uevent_suppress(&clt->dev, false);
2764 	kobject_uevent(&clt->dev.kobj, KOBJ_ADD);
2765 
2766 	return clt;
2767 err_del:
2768 	device_del(&clt->dev);
2769 err_put:
2770 	free_percpu(clt->pcpu_path);
2771 	put_device(&clt->dev);
2772 	return ERR_PTR(err);
2773 }
2774 
2775 static void free_clt(struct rtrs_clt_sess *clt)
2776 {
2777 	free_percpu(clt->pcpu_path);
2778 
2779 	/*
2780 	 * release callback will free clt and destroy mutexes in last put
2781 	 */
2782 	device_unregister(&clt->dev);
2783 }
2784 
2785 /**
2786  * rtrs_clt_open() - Open a path to an RTRS server
2787  * @ops: holds the link event callback and the private pointer.
2788  * @pathname: name of the path to an RTRS server
2789  * @paths: Paths to be established defined by their src and dst addresses
2790  * @paths_num: Number of elements in the @paths array
2791  * @port: port to be used by the RTRS session
2792  * @pdu_sz: Size of extra payload which can be accessed after permit allocation.
2793  * @reconnect_delay_sec: time between reconnect tries
2794  * @max_reconnect_attempts: Number of times to reconnect on error before giving
2795  *			    up, 0 for * disabled, -1 for forever
2796  * @nr_poll_queues: number of polling mode connection using IB_POLL_DIRECT flag
2797  *
2798  * Starts session establishment with the rtrs_server. The function can block
2799  * up to ~2000ms before it returns.
2800  *
2801  * Return a valid pointer on success otherwise PTR_ERR.
2802  */
2803 struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops,
2804 				 const char *pathname,
2805 				 const struct rtrs_addr *paths,
2806 				 size_t paths_num, u16 port,
2807 				 size_t pdu_sz, u8 reconnect_delay_sec,
2808 				 s16 max_reconnect_attempts, u32 nr_poll_queues)
2809 {
2810 	struct rtrs_clt_path *clt_path, *tmp;
2811 	struct rtrs_clt_sess *clt;
2812 	int err, i;
2813 
2814 	if (strchr(pathname, '/') || strchr(pathname, '.')) {
2815 		pr_err("pathname cannot contain / and .\n");
2816 		err = -EINVAL;
2817 		goto out;
2818 	}
2819 
2820 	clt = alloc_clt(pathname, paths_num, port, pdu_sz, ops->priv,
2821 			ops->link_ev,
2822 			reconnect_delay_sec,
2823 			max_reconnect_attempts);
2824 	if (IS_ERR(clt)) {
2825 		err = PTR_ERR(clt);
2826 		goto out;
2827 	}
2828 	for (i = 0; i < paths_num; i++) {
2829 		struct rtrs_clt_path *clt_path;
2830 
2831 		clt_path = alloc_path(clt, &paths[i], nr_cpu_ids,
2832 				  nr_poll_queues);
2833 		if (IS_ERR(clt_path)) {
2834 			err = PTR_ERR(clt_path);
2835 			goto close_all_path;
2836 		}
2837 		if (!i)
2838 			clt_path->for_new_clt = 1;
2839 		list_add_tail_rcu(&clt_path->s.entry, &clt->paths_list);
2840 
2841 		err = init_path(clt_path);
2842 		if (err) {
2843 			list_del_rcu(&clt_path->s.entry);
2844 			rtrs_clt_close_conns(clt_path, true);
2845 			free_percpu(clt_path->stats->pcpu_stats);
2846 			kfree(clt_path->stats);
2847 			free_path(clt_path);
2848 			goto close_all_path;
2849 		}
2850 
2851 		err = rtrs_clt_create_path_files(clt_path);
2852 		if (err) {
2853 			list_del_rcu(&clt_path->s.entry);
2854 			rtrs_clt_close_conns(clt_path, true);
2855 			free_percpu(clt_path->stats->pcpu_stats);
2856 			kfree(clt_path->stats);
2857 			free_path(clt_path);
2858 			goto close_all_path;
2859 		}
2860 	}
2861 	err = alloc_permits(clt);
2862 	if (err)
2863 		goto close_all_path;
2864 
2865 	return clt;
2866 
2867 close_all_path:
2868 	list_for_each_entry_safe(clt_path, tmp, &clt->paths_list, s.entry) {
2869 		rtrs_clt_destroy_path_files(clt_path, NULL);
2870 		rtrs_clt_close_conns(clt_path, true);
2871 		kobject_put(&clt_path->kobj);
2872 	}
2873 	rtrs_clt_destroy_sysfs_root(clt);
2874 	free_clt(clt);
2875 
2876 out:
2877 	return ERR_PTR(err);
2878 }
2879 EXPORT_SYMBOL(rtrs_clt_open);
2880 
2881 /**
2882  * rtrs_clt_close() - Close a path
2883  * @clt: Session handle. Session is freed upon return.
2884  */
2885 void rtrs_clt_close(struct rtrs_clt_sess *clt)
2886 {
2887 	struct rtrs_clt_path *clt_path, *tmp;
2888 
2889 	/* Firstly forbid sysfs access */
2890 	rtrs_clt_destroy_sysfs_root(clt);
2891 
2892 	/* Now it is safe to iterate over all paths without locks */
2893 	list_for_each_entry_safe(clt_path, tmp, &clt->paths_list, s.entry) {
2894 		rtrs_clt_close_conns(clt_path, true);
2895 		rtrs_clt_destroy_path_files(clt_path, NULL);
2896 		kobject_put(&clt_path->kobj);
2897 	}
2898 	free_permits(clt);
2899 	free_clt(clt);
2900 }
2901 EXPORT_SYMBOL(rtrs_clt_close);
2902 
2903 int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_path *clt_path)
2904 {
2905 	enum rtrs_clt_state old_state;
2906 	int err = -EBUSY;
2907 	bool changed;
2908 
2909 	changed = rtrs_clt_change_state_get_old(clt_path,
2910 						 RTRS_CLT_RECONNECTING,
2911 						 &old_state);
2912 	if (changed) {
2913 		clt_path->reconnect_attempts = 0;
2914 		rtrs_clt_stop_and_destroy_conns(clt_path);
2915 		queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork, 0);
2916 	}
2917 	if (changed || old_state == RTRS_CLT_RECONNECTING) {
2918 		/*
2919 		 * flush_delayed_work() queues pending work for immediate
2920 		 * execution, so do the flush if we have queued something
2921 		 * right now or work is pending.
2922 		 */
2923 		flush_delayed_work(&clt_path->reconnect_dwork);
2924 		err = (READ_ONCE(clt_path->state) ==
2925 		       RTRS_CLT_CONNECTED ? 0 : -ENOTCONN);
2926 	}
2927 
2928 	return err;
2929 }
2930 
2931 int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_path *clt_path,
2932 				     const struct attribute *sysfs_self)
2933 {
2934 	enum rtrs_clt_state old_state;
2935 	bool changed;
2936 
2937 	/*
2938 	 * Continue stopping path till state was changed to DEAD or
2939 	 * state was observed as DEAD:
2940 	 * 1. State was changed to DEAD - we were fast and nobody
2941 	 *    invoked rtrs_clt_reconnect(), which can again start
2942 	 *    reconnecting.
2943 	 * 2. State was observed as DEAD - we have someone in parallel
2944 	 *    removing the path.
2945 	 */
2946 	do {
2947 		rtrs_clt_close_conns(clt_path, true);
2948 		changed = rtrs_clt_change_state_get_old(clt_path,
2949 							RTRS_CLT_DEAD,
2950 							&old_state);
2951 	} while (!changed && old_state != RTRS_CLT_DEAD);
2952 
2953 	if (changed) {
2954 		rtrs_clt_remove_path_from_arr(clt_path);
2955 		rtrs_clt_destroy_path_files(clt_path, sysfs_self);
2956 		kobject_put(&clt_path->kobj);
2957 	}
2958 
2959 	return 0;
2960 }
2961 
2962 void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt_sess *clt, int value)
2963 {
2964 	clt->max_reconnect_attempts = (unsigned int)value;
2965 }
2966 
2967 int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt_sess *clt)
2968 {
2969 	return (int)clt->max_reconnect_attempts;
2970 }
2971 
2972 /**
2973  * rtrs_clt_request() - Request data transfer to/from server via RDMA.
2974  *
2975  * @dir:	READ/WRITE
2976  * @ops:	callback function to be called as confirmation, and the pointer.
2977  * @clt:	Session
2978  * @permit:	Preallocated permit
2979  * @vec:	Message that is sent to server together with the request.
2980  *		Sum of len of all @vec elements limited to <= IO_MSG_SIZE.
2981  *		Since the msg is copied internally it can be allocated on stack.
2982  * @nr:		Number of elements in @vec.
2983  * @data_len:	length of data sent to/from server
2984  * @sg:		Pages to be sent/received to/from server.
2985  * @sg_cnt:	Number of elements in the @sg
2986  *
2987  * Return:
2988  * 0:		Success
2989  * <0:		Error
2990  *
2991  * On dir=READ rtrs client will request a data transfer from Server to client.
2992  * The data that the server will respond with will be stored in @sg when
2993  * the user receives an %RTRS_CLT_RDMA_EV_RDMA_REQUEST_WRITE_COMPL event.
2994  * On dir=WRITE rtrs client will rdma write data in sg to server side.
2995  */
2996 int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
2997 		     struct rtrs_clt_sess *clt, struct rtrs_permit *permit,
2998 		     const struct kvec *vec, size_t nr, size_t data_len,
2999 		     struct scatterlist *sg, unsigned int sg_cnt)
3000 {
3001 	struct rtrs_clt_io_req *req;
3002 	struct rtrs_clt_path *clt_path;
3003 
3004 	enum dma_data_direction dma_dir;
3005 	int err = -ECONNABORTED, i;
3006 	size_t usr_len, hdr_len;
3007 	struct path_it it;
3008 
3009 	/* Get kvec length */
3010 	for (i = 0, usr_len = 0; i < nr; i++)
3011 		usr_len += vec[i].iov_len;
3012 
3013 	if (dir == READ) {
3014 		hdr_len = sizeof(struct rtrs_msg_rdma_read) +
3015 			  sg_cnt * sizeof(struct rtrs_sg_desc);
3016 		dma_dir = DMA_FROM_DEVICE;
3017 	} else {
3018 		hdr_len = sizeof(struct rtrs_msg_rdma_write);
3019 		dma_dir = DMA_TO_DEVICE;
3020 	}
3021 
3022 	rcu_read_lock();
3023 	for (path_it_init(&it, clt);
3024 	     (clt_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {
3025 		if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
3026 			continue;
3027 
3028 		if (usr_len + hdr_len > clt_path->max_hdr_size) {
3029 			rtrs_wrn_rl(clt_path->clt,
3030 				     "%s request failed, user message size is %zu and header length %zu, but max size is %u\n",
3031 				     dir == READ ? "Read" : "Write",
3032 				     usr_len, hdr_len, clt_path->max_hdr_size);
3033 			err = -EMSGSIZE;
3034 			break;
3035 		}
3036 		req = rtrs_clt_get_req(clt_path, ops->conf_fn, permit, ops->priv,
3037 				       vec, usr_len, sg, sg_cnt, data_len,
3038 				       dma_dir);
3039 		if (dir == READ)
3040 			err = rtrs_clt_read_req(req);
3041 		else
3042 			err = rtrs_clt_write_req(req);
3043 		if (err) {
3044 			req->in_use = false;
3045 			continue;
3046 		}
3047 		/* Success path */
3048 		break;
3049 	}
3050 	path_it_deinit(&it);
3051 	rcu_read_unlock();
3052 
3053 	return err;
3054 }
3055 EXPORT_SYMBOL(rtrs_clt_request);
3056 
3057 int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index)
3058 {
3059 	/* If no path, return -1 for block layer not to try again */
3060 	int cnt = -1;
3061 	struct rtrs_con *con;
3062 	struct rtrs_clt_path *clt_path;
3063 	struct path_it it;
3064 
3065 	rcu_read_lock();
3066 	for (path_it_init(&it, clt);
3067 	     (clt_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {
3068 		if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED)
3069 			continue;
3070 
3071 		con = clt_path->s.con[index + 1];
3072 		cnt = ib_process_cq_direct(con->cq, -1);
3073 		if (cnt)
3074 			break;
3075 	}
3076 	path_it_deinit(&it);
3077 	rcu_read_unlock();
3078 
3079 	return cnt;
3080 }
3081 EXPORT_SYMBOL(rtrs_clt_rdma_cq_direct);
3082 
3083 /**
3084  * rtrs_clt_query() - queries RTRS session attributes
3085  *@clt: session pointer
3086  *@attr: query results for session attributes.
3087  * Returns:
3088  *    0 on success
3089  *    -ECOMM		no connection to the server
3090  */
3091 int rtrs_clt_query(struct rtrs_clt_sess *clt, struct rtrs_attrs *attr)
3092 {
3093 	if (!rtrs_clt_is_connected(clt))
3094 		return -ECOMM;
3095 
3096 	attr->queue_depth      = clt->queue_depth;
3097 	attr->max_segments     = clt->max_segments;
3098 	/* Cap max_io_size to min of remote buffer size and the fr pages */
3099 	attr->max_io_size = min_t(int, clt->max_io_size,
3100 				  clt->max_segments * SZ_4K);
3101 
3102 	return 0;
3103 }
3104 EXPORT_SYMBOL(rtrs_clt_query);
3105 
3106 int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt,
3107 				     struct rtrs_addr *addr)
3108 {
3109 	struct rtrs_clt_path *clt_path;
3110 	int err;
3111 
3112 	clt_path = alloc_path(clt, addr, nr_cpu_ids, 0);
3113 	if (IS_ERR(clt_path))
3114 		return PTR_ERR(clt_path);
3115 
3116 	mutex_lock(&clt->paths_mutex);
3117 	if (clt->paths_num == 0) {
3118 		/*
3119 		 * When all the paths are removed for a session,
3120 		 * the addition of the first path is like a new session for
3121 		 * the storage server
3122 		 */
3123 		clt_path->for_new_clt = 1;
3124 	}
3125 
3126 	mutex_unlock(&clt->paths_mutex);
3127 
3128 	/*
3129 	 * It is totally safe to add path in CONNECTING state: coming
3130 	 * IO will never grab it.  Also it is very important to add
3131 	 * path before init, since init fires LINK_CONNECTED event.
3132 	 */
3133 	rtrs_clt_add_path_to_arr(clt_path);
3134 
3135 	err = init_path(clt_path);
3136 	if (err)
3137 		goto close_path;
3138 
3139 	err = rtrs_clt_create_path_files(clt_path);
3140 	if (err)
3141 		goto close_path;
3142 
3143 	return 0;
3144 
3145 close_path:
3146 	rtrs_clt_remove_path_from_arr(clt_path);
3147 	rtrs_clt_close_conns(clt_path, true);
3148 	free_percpu(clt_path->stats->pcpu_stats);
3149 	kfree(clt_path->stats);
3150 	free_path(clt_path);
3151 
3152 	return err;
3153 }
3154 
3155 static int rtrs_clt_ib_dev_init(struct rtrs_ib_dev *dev)
3156 {
3157 	if (!(dev->ib_dev->attrs.device_cap_flags &
3158 	      IB_DEVICE_MEM_MGT_EXTENSIONS)) {
3159 		pr_err("Memory registrations not supported.\n");
3160 		return -ENOTSUPP;
3161 	}
3162 
3163 	return 0;
3164 }
3165 
3166 static const struct rtrs_rdma_dev_pd_ops dev_pd_ops = {
3167 	.init = rtrs_clt_ib_dev_init
3168 };
3169 
3170 static int __init rtrs_client_init(void)
3171 {
3172 	rtrs_rdma_dev_pd_init(0, &dev_pd);
3173 
3174 	rtrs_clt_dev_class = class_create(THIS_MODULE, "rtrs-client");
3175 	if (IS_ERR(rtrs_clt_dev_class)) {
3176 		pr_err("Failed to create rtrs-client dev class\n");
3177 		return PTR_ERR(rtrs_clt_dev_class);
3178 	}
3179 	rtrs_wq = alloc_workqueue("rtrs_client_wq", 0, 0);
3180 	if (!rtrs_wq) {
3181 		class_destroy(rtrs_clt_dev_class);
3182 		return -ENOMEM;
3183 	}
3184 
3185 	return 0;
3186 }
3187 
3188 static void __exit rtrs_client_exit(void)
3189 {
3190 	destroy_workqueue(rtrs_wq);
3191 	class_destroy(rtrs_clt_dev_class);
3192 	rtrs_rdma_dev_pd_deinit(&dev_pd);
3193 }
3194 
3195 module_init(rtrs_client_init);
3196 module_exit(rtrs_client_exit);
3197