xref: /openbmc/linux/net/smc/smc_core.c (revision 6f52b16c5b29b89d92c0e7236f4655dc8491ad70)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
4  *
5  *  Basic Transport Functions exploiting Infiniband API
6  *
7  *  Copyright IBM Corp. 2016
8  *
9  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
10  */
11 
12 #include <linux/socket.h>
13 #include <linux/if_vlan.h>
14 #include <linux/random.h>
15 #include <linux/workqueue.h>
16 #include <net/tcp.h>
17 #include <net/sock.h>
18 #include <rdma/ib_verbs.h>
19 
20 #include "smc.h"
21 #include "smc_clc.h"
22 #include "smc_core.h"
23 #include "smc_ib.h"
24 #include "smc_wr.h"
25 #include "smc_llc.h"
26 #include "smc_cdc.h"
27 #include "smc_close.h"
28 
29 #define SMC_LGR_NUM_INCR		256
30 #define SMC_LGR_FREE_DELAY_SERV		(600 * HZ)
31 #define SMC_LGR_FREE_DELAY_CLNT		(SMC_LGR_FREE_DELAY_SERV + 10)
32 
33 static u32 smc_lgr_num;			/* unique link group number */
34 
35 /* Register connection's alert token in our lookup structure.
36  * To use rbtrees we have to implement our own insert core.
37  * Requires @conns_lock
38  * @smc		connection to register
39  * Returns 0 on success, != otherwise.
40  */
41 static void smc_lgr_add_alert_token(struct smc_connection *conn)
42 {
43 	struct rb_node **link, *parent = NULL;
44 	u32 token = conn->alert_token_local;
45 
46 	link = &conn->lgr->conns_all.rb_node;
47 	while (*link) {
48 		struct smc_connection *cur = rb_entry(*link,
49 					struct smc_connection, alert_node);
50 
51 		parent = *link;
52 		if (cur->alert_token_local > token)
53 			link = &parent->rb_left;
54 		else
55 			link = &parent->rb_right;
56 	}
57 	/* Put the new node there */
58 	rb_link_node(&conn->alert_node, parent, link);
59 	rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
60 }
61 
62 /* Register connection in link group by assigning an alert token
63  * registered in a search tree.
64  * Requires @conns_lock
65  * Note that '0' is a reserved value and not assigned.
66  */
67 static void smc_lgr_register_conn(struct smc_connection *conn)
68 {
69 	struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
70 	static atomic_t nexttoken = ATOMIC_INIT(0);
71 
72 	/* find a new alert_token_local value not yet used by some connection
73 	 * in this link group
74 	 */
75 	sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */
76 	while (!conn->alert_token_local) {
77 		conn->alert_token_local = atomic_inc_return(&nexttoken);
78 		if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
79 			conn->alert_token_local = 0;
80 	}
81 	smc_lgr_add_alert_token(conn);
82 	conn->lgr->conns_num++;
83 }
84 
85 /* Unregister connection and reset the alert token of the given connection<
86  */
87 static void __smc_lgr_unregister_conn(struct smc_connection *conn)
88 {
89 	struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
90 	struct smc_link_group *lgr = conn->lgr;
91 
92 	rb_erase(&conn->alert_node, &lgr->conns_all);
93 	lgr->conns_num--;
94 	conn->alert_token_local = 0;
95 	conn->lgr = NULL;
96 	sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */
97 }
98 
99 /* Unregister connection and trigger lgr freeing if applicable
100  */
101 static void smc_lgr_unregister_conn(struct smc_connection *conn)
102 {
103 	struct smc_link_group *lgr = conn->lgr;
104 	int reduced = 0;
105 
106 	write_lock_bh(&lgr->conns_lock);
107 	if (conn->alert_token_local) {
108 		reduced = 1;
109 		__smc_lgr_unregister_conn(conn);
110 	}
111 	write_unlock_bh(&lgr->conns_lock);
112 	if (!reduced || lgr->conns_num)
113 		return;
114 	/* client link group creation always follows the server link group
115 	 * creation. For client use a somewhat higher removal delay time,
116 	 * otherwise there is a risk of out-of-sync link groups.
117 	 */
118 	mod_delayed_work(system_wq, &lgr->free_work,
119 			 lgr->role == SMC_CLNT ? SMC_LGR_FREE_DELAY_CLNT :
120 						 SMC_LGR_FREE_DELAY_SERV);
121 }
122 
123 static void smc_lgr_free_work(struct work_struct *work)
124 {
125 	struct smc_link_group *lgr = container_of(to_delayed_work(work),
126 						  struct smc_link_group,
127 						  free_work);
128 	bool conns;
129 
130 	spin_lock_bh(&smc_lgr_list.lock);
131 	read_lock_bh(&lgr->conns_lock);
132 	conns = RB_EMPTY_ROOT(&lgr->conns_all);
133 	read_unlock_bh(&lgr->conns_lock);
134 	if (!conns) { /* number of lgr connections is no longer zero */
135 		spin_unlock_bh(&smc_lgr_list.lock);
136 		return;
137 	}
138 	list_del_init(&lgr->list); /* remove from smc_lgr_list */
139 	spin_unlock_bh(&smc_lgr_list.lock);
140 	smc_lgr_free(lgr);
141 }
142 
143 /* create a new SMC link group */
144 static int smc_lgr_create(struct smc_sock *smc, __be32 peer_in_addr,
145 			  struct smc_ib_device *smcibdev, u8 ibport,
146 			  char *peer_systemid, unsigned short vlan_id)
147 {
148 	struct smc_link_group *lgr;
149 	struct smc_link *lnk;
150 	u8 rndvec[3];
151 	int rc = 0;
152 	int i;
153 
154 	lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
155 	if (!lgr) {
156 		rc = -ENOMEM;
157 		goto out;
158 	}
159 	lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
160 	lgr->sync_err = false;
161 	lgr->daddr = peer_in_addr;
162 	memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN);
163 	lgr->vlan_id = vlan_id;
164 	rwlock_init(&lgr->sndbufs_lock);
165 	rwlock_init(&lgr->rmbs_lock);
166 	for (i = 0; i < SMC_RMBE_SIZES; i++) {
167 		INIT_LIST_HEAD(&lgr->sndbufs[i]);
168 		INIT_LIST_HEAD(&lgr->rmbs[i]);
169 	}
170 	smc_lgr_num += SMC_LGR_NUM_INCR;
171 	memcpy(&lgr->id, (u8 *)&smc_lgr_num, SMC_LGR_ID_SIZE);
172 	INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
173 	lgr->conns_all = RB_ROOT;
174 
175 	lnk = &lgr->lnk[SMC_SINGLE_LINK];
176 	/* initialize link */
177 	lnk->smcibdev = smcibdev;
178 	lnk->ibport = ibport;
179 	lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu;
180 	if (!smcibdev->initialized)
181 		smc_ib_setup_per_ibdev(smcibdev);
182 	get_random_bytes(rndvec, sizeof(rndvec));
183 	lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + (rndvec[2] << 16);
184 	rc = smc_wr_alloc_link_mem(lnk);
185 	if (rc)
186 		goto free_lgr;
187 	rc = smc_ib_create_protection_domain(lnk);
188 	if (rc)
189 		goto free_link_mem;
190 	rc = smc_ib_create_queue_pair(lnk);
191 	if (rc)
192 		goto dealloc_pd;
193 	rc = smc_wr_create_link(lnk);
194 	if (rc)
195 		goto destroy_qp;
196 	init_completion(&lnk->llc_confirm);
197 	init_completion(&lnk->llc_confirm_resp);
198 
199 	smc->conn.lgr = lgr;
200 	rwlock_init(&lgr->conns_lock);
201 	spin_lock_bh(&smc_lgr_list.lock);
202 	list_add(&lgr->list, &smc_lgr_list.list);
203 	spin_unlock_bh(&smc_lgr_list.lock);
204 	return 0;
205 
206 destroy_qp:
207 	smc_ib_destroy_queue_pair(lnk);
208 dealloc_pd:
209 	smc_ib_dealloc_protection_domain(lnk);
210 free_link_mem:
211 	smc_wr_free_link_mem(lnk);
212 free_lgr:
213 	kfree(lgr);
214 out:
215 	return rc;
216 }
217 
218 static void smc_buf_unuse(struct smc_connection *conn)
219 {
220 	if (conn->sndbuf_desc) {
221 		conn->sndbuf_desc->used = 0;
222 		conn->sndbuf_size = 0;
223 	}
224 	if (conn->rmb_desc) {
225 		conn->rmb_desc->reused = true;
226 		conn->rmb_desc->used = 0;
227 		conn->rmbe_size = 0;
228 	}
229 }
230 
231 /* remove a finished connection from its link group */
232 void smc_conn_free(struct smc_connection *conn)
233 {
234 	struct smc_link_group *lgr = conn->lgr;
235 
236 	if (!lgr)
237 		return;
238 	smc_cdc_tx_dismiss_slots(conn);
239 	smc_lgr_unregister_conn(conn);
240 	smc_buf_unuse(conn);
241 }
242 
243 static void smc_link_clear(struct smc_link *lnk)
244 {
245 	lnk->peer_qpn = 0;
246 	smc_ib_modify_qp_reset(lnk);
247 	smc_wr_free_link(lnk);
248 	smc_ib_destroy_queue_pair(lnk);
249 	smc_ib_dealloc_protection_domain(lnk);
250 	smc_wr_free_link_mem(lnk);
251 }
252 
253 static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk,
254 			 bool is_rmb)
255 {
256 	if (is_rmb) {
257 		if (buf_desc->mr_rx[SMC_SINGLE_LINK])
258 			smc_ib_put_memory_region(
259 					buf_desc->mr_rx[SMC_SINGLE_LINK]);
260 		smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
261 				    DMA_FROM_DEVICE);
262 	} else {
263 		smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
264 				    DMA_TO_DEVICE);
265 	}
266 	sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
267 	if (buf_desc->cpu_addr)
268 		free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order);
269 	kfree(buf_desc);
270 }
271 
272 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)
273 {
274 	struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
275 	struct smc_buf_desc *buf_desc, *bf_desc;
276 	struct list_head *buf_list;
277 	int i;
278 
279 	for (i = 0; i < SMC_RMBE_SIZES; i++) {
280 		if (is_rmb)
281 			buf_list = &lgr->rmbs[i];
282 		else
283 			buf_list = &lgr->sndbufs[i];
284 		list_for_each_entry_safe(buf_desc, bf_desc, buf_list,
285 					 list) {
286 			list_del(&buf_desc->list);
287 			smc_buf_free(buf_desc, lnk, is_rmb);
288 		}
289 	}
290 }
291 
292 static void smc_lgr_free_bufs(struct smc_link_group *lgr)
293 {
294 	/* free send buffers */
295 	__smc_lgr_free_bufs(lgr, false);
296 	/* free rmbs */
297 	__smc_lgr_free_bufs(lgr, true);
298 }
299 
300 /* remove a link group */
301 void smc_lgr_free(struct smc_link_group *lgr)
302 {
303 	smc_lgr_free_bufs(lgr);
304 	smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);
305 	kfree(lgr);
306 }
307 
308 /* terminate linkgroup abnormally */
309 void smc_lgr_terminate(struct smc_link_group *lgr)
310 {
311 	struct smc_connection *conn;
312 	struct smc_sock *smc;
313 	struct rb_node *node;
314 
315 	spin_lock_bh(&smc_lgr_list.lock);
316 	if (list_empty(&lgr->list)) {
317 		/* termination already triggered */
318 		spin_unlock_bh(&smc_lgr_list.lock);
319 		return;
320 	}
321 	/* do not use this link group for new connections */
322 	list_del_init(&lgr->list);
323 	spin_unlock_bh(&smc_lgr_list.lock);
324 
325 	write_lock_bh(&lgr->conns_lock);
326 	node = rb_first(&lgr->conns_all);
327 	while (node) {
328 		conn = rb_entry(node, struct smc_connection, alert_node);
329 		smc = container_of(conn, struct smc_sock, conn);
330 		sock_hold(&smc->sk);
331 		__smc_lgr_unregister_conn(conn);
332 		schedule_work(&conn->close_work);
333 		sock_put(&smc->sk);
334 		node = rb_first(&lgr->conns_all);
335 	}
336 	write_unlock_bh(&lgr->conns_lock);
337 }
338 
339 /* Determine vlan of internal TCP socket.
340  * @vlan_id: address to store the determined vlan id into
341  */
342 static int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id)
343 {
344 	struct dst_entry *dst = sk_dst_get(clcsock->sk);
345 	int rc = 0;
346 
347 	*vlan_id = 0;
348 	if (!dst) {
349 		rc = -ENOTCONN;
350 		goto out;
351 	}
352 	if (!dst->dev) {
353 		rc = -ENODEV;
354 		goto out_rel;
355 	}
356 
357 	if (is_vlan_dev(dst->dev))
358 		*vlan_id = vlan_dev_vlan_id(dst->dev);
359 
360 out_rel:
361 	dst_release(dst);
362 out:
363 	return rc;
364 }
365 
366 /* determine the link gid matching the vlan id of the link group */
367 static int smc_link_determine_gid(struct smc_link_group *lgr)
368 {
369 	struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
370 	struct ib_gid_attr gattr;
371 	union ib_gid gid;
372 	int i;
373 
374 	if (!lgr->vlan_id) {
375 		lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1];
376 		return 0;
377 	}
378 
379 	for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len;
380 	     i++) {
381 		if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid,
382 				 &gattr))
383 			continue;
384 		if (gattr.ndev &&
385 		    (vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id)) {
386 			lnk->gid = gid;
387 			return 0;
388 		}
389 	}
390 	return -ENODEV;
391 }
392 
393 /* create a new SMC connection (and a new link group if necessary) */
394 int smc_conn_create(struct smc_sock *smc, __be32 peer_in_addr,
395 		    struct smc_ib_device *smcibdev, u8 ibport,
396 		    struct smc_clc_msg_local *lcl, int srv_first_contact)
397 {
398 	struct smc_connection *conn = &smc->conn;
399 	struct smc_link_group *lgr;
400 	unsigned short vlan_id;
401 	enum smc_lgr_role role;
402 	int local_contact = SMC_FIRST_CONTACT;
403 	int rc = 0;
404 
405 	role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
406 	rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id);
407 	if (rc)
408 		return rc;
409 
410 	if ((role == SMC_CLNT) && srv_first_contact)
411 		/* create new link group as well */
412 		goto create;
413 
414 	/* determine if an existing link group can be reused */
415 	spin_lock_bh(&smc_lgr_list.lock);
416 	list_for_each_entry(lgr, &smc_lgr_list.list, list) {
417 		write_lock_bh(&lgr->conns_lock);
418 		if (!memcmp(lgr->peer_systemid, lcl->id_for_peer,
419 			    SMC_SYSTEMID_LEN) &&
420 		    !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,
421 			    SMC_GID_SIZE) &&
422 		    !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,
423 			    sizeof(lcl->mac)) &&
424 		    !lgr->sync_err &&
425 		    (lgr->role == role) &&
426 		    (lgr->vlan_id == vlan_id) &&
427 		    ((role == SMC_CLNT) ||
428 		     (lgr->conns_num < SMC_RMBS_PER_LGR_MAX))) {
429 			/* link group found */
430 			local_contact = SMC_REUSE_CONTACT;
431 			conn->lgr = lgr;
432 			smc_lgr_register_conn(conn); /* add smc conn to lgr */
433 			write_unlock_bh(&lgr->conns_lock);
434 			break;
435 		}
436 		write_unlock_bh(&lgr->conns_lock);
437 	}
438 	spin_unlock_bh(&smc_lgr_list.lock);
439 
440 	if (role == SMC_CLNT && !srv_first_contact &&
441 	    (local_contact == SMC_FIRST_CONTACT)) {
442 		/* Server reuses a link group, but Client wants to start
443 		 * a new one
444 		 * send out_of_sync decline, reason synchr. error
445 		 */
446 		return -ENOLINK;
447 	}
448 
449 create:
450 	if (local_contact == SMC_FIRST_CONTACT) {
451 		rc = smc_lgr_create(smc, peer_in_addr, smcibdev, ibport,
452 				    lcl->id_for_peer, vlan_id);
453 		if (rc)
454 			goto out;
455 		smc_lgr_register_conn(conn); /* add smc conn to lgr */
456 		rc = smc_link_determine_gid(conn->lgr);
457 	}
458 	conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
459 	conn->local_tx_ctrl.len = sizeof(struct smc_cdc_msg);
460 #ifndef KERNEL_HAS_ATOMIC64
461 	spin_lock_init(&conn->acurs_lock);
462 #endif
463 
464 out:
465 	return rc ? rc : local_contact;
466 }
467 
468 /* try to reuse a sndbuf or rmb description slot for a certain
469  * buffer size; if not available, return NULL
470  */
471 static inline
472 struct smc_buf_desc *smc_buf_get_slot(struct smc_link_group *lgr,
473 				      int compressed_bufsize,
474 				      rwlock_t *lock,
475 				      struct list_head *buf_list)
476 {
477 	struct smc_buf_desc *buf_slot;
478 
479 	read_lock_bh(lock);
480 	list_for_each_entry(buf_slot, buf_list, list) {
481 		if (cmpxchg(&buf_slot->used, 0, 1) == 0) {
482 			read_unlock_bh(lock);
483 			return buf_slot;
484 		}
485 	}
486 	read_unlock_bh(lock);
487 	return NULL;
488 }
489 
490 /* one of the conditions for announcing a receiver's current window size is
491  * that it "results in a minimum increase in the window size of 10% of the
492  * receive buffer space" [RFC7609]
493  */
494 static inline int smc_rmb_wnd_update_limit(int rmbe_size)
495 {
496 	return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
497 }
498 
499 static struct smc_buf_desc *smc_new_buf_create(struct smc_link_group *lgr,
500 					       bool is_rmb, int bufsize)
501 {
502 	struct smc_buf_desc *buf_desc;
503 	struct smc_link *lnk;
504 	int rc;
505 
506 	/* try to alloc a new buffer */
507 	buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
508 	if (!buf_desc)
509 		return ERR_PTR(-ENOMEM);
510 
511 	buf_desc->cpu_addr =
512 		(void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN |
513 					 __GFP_NOMEMALLOC |
514 					 __GFP_NORETRY | __GFP_ZERO,
515 					 get_order(bufsize));
516 	if (!buf_desc->cpu_addr) {
517 		kfree(buf_desc);
518 		return ERR_PTR(-EAGAIN);
519 	}
520 	buf_desc->order = get_order(bufsize);
521 
522 	/* build the sg table from the pages */
523 	lnk = &lgr->lnk[SMC_SINGLE_LINK];
524 	rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1,
525 			    GFP_KERNEL);
526 	if (rc) {
527 		smc_buf_free(buf_desc, lnk, is_rmb);
528 		return ERR_PTR(rc);
529 	}
530 	sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl,
531 		   buf_desc->cpu_addr, bufsize);
532 
533 	/* map sg table to DMA address */
534 	rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc,
535 			       is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
536 	/* SMC protocol depends on mapping to one DMA address only */
537 	if (rc != 1)  {
538 		smc_buf_free(buf_desc, lnk, is_rmb);
539 		return ERR_PTR(-EAGAIN);
540 	}
541 
542 	/* create a new memory region for the RMB */
543 	if (is_rmb) {
544 		rc = smc_ib_get_memory_region(lnk->roce_pd,
545 					      IB_ACCESS_REMOTE_WRITE |
546 					      IB_ACCESS_LOCAL_WRITE,
547 					      buf_desc);
548 		if (rc) {
549 			smc_buf_free(buf_desc, lnk, is_rmb);
550 			return ERR_PTR(rc);
551 		}
552 	}
553 
554 	return buf_desc;
555 }
556 
557 static int __smc_buf_create(struct smc_sock *smc, bool is_rmb)
558 {
559 	struct smc_connection *conn = &smc->conn;
560 	struct smc_link_group *lgr = conn->lgr;
561 	struct smc_buf_desc *buf_desc = NULL;
562 	struct list_head *buf_list;
563 	int bufsize, bufsize_short;
564 	int sk_buf_size;
565 	rwlock_t *lock;
566 
567 	if (is_rmb)
568 		/* use socket recv buffer size (w/o overhead) as start value */
569 		sk_buf_size = smc->sk.sk_rcvbuf / 2;
570 	else
571 		/* use socket send buffer size (w/o overhead) as start value */
572 		sk_buf_size = smc->sk.sk_sndbuf / 2;
573 
574 	for (bufsize_short = smc_compress_bufsize(smc->sk.sk_sndbuf / 2);
575 	     bufsize_short >= 0; bufsize_short--) {
576 
577 		if (is_rmb) {
578 			lock = &lgr->rmbs_lock;
579 			buf_list = &lgr->rmbs[bufsize_short];
580 		} else {
581 			lock = &lgr->sndbufs_lock;
582 			buf_list = &lgr->sndbufs[bufsize_short];
583 		}
584 		bufsize = smc_uncompress_bufsize(bufsize_short);
585 		if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC)
586 			continue;
587 
588 		/* check for reusable slot in the link group */
589 		buf_desc = smc_buf_get_slot(lgr, bufsize_short, lock, buf_list);
590 		if (buf_desc) {
591 			memset(buf_desc->cpu_addr, 0, bufsize);
592 			break; /* found reusable slot */
593 		}
594 
595 		buf_desc = smc_new_buf_create(lgr, is_rmb, bufsize);
596 		if (PTR_ERR(buf_desc) == -ENOMEM)
597 			break;
598 		if (IS_ERR(buf_desc))
599 			continue;
600 
601 		buf_desc->used = 1;
602 		write_lock_bh(lock);
603 		list_add(&buf_desc->list, buf_list);
604 		write_unlock_bh(lock);
605 		break; /* found */
606 	}
607 
608 	if (IS_ERR(buf_desc))
609 		return -ENOMEM;
610 
611 	if (is_rmb) {
612 		conn->rmb_desc = buf_desc;
613 		conn->rmbe_size = bufsize;
614 		conn->rmbe_size_short = bufsize_short;
615 		smc->sk.sk_rcvbuf = bufsize * 2;
616 		atomic_set(&conn->bytes_to_rcv, 0);
617 		conn->rmbe_update_limit = smc_rmb_wnd_update_limit(bufsize);
618 	} else {
619 		conn->sndbuf_desc = buf_desc;
620 		conn->sndbuf_size = bufsize;
621 		smc->sk.sk_sndbuf = bufsize * 2;
622 		atomic_set(&conn->sndbuf_space, bufsize);
623 	}
624 	return 0;
625 }
626 
627 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
628 {
629 	struct smc_link_group *lgr = conn->lgr;
630 
631 	smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
632 			       conn->sndbuf_desc, DMA_TO_DEVICE);
633 }
634 
635 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
636 {
637 	struct smc_link_group *lgr = conn->lgr;
638 
639 	smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
640 				  conn->sndbuf_desc, DMA_TO_DEVICE);
641 }
642 
643 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
644 {
645 	struct smc_link_group *lgr = conn->lgr;
646 
647 	smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
648 			       conn->rmb_desc, DMA_FROM_DEVICE);
649 }
650 
651 void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
652 {
653 	struct smc_link_group *lgr = conn->lgr;
654 
655 	smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
656 				  conn->rmb_desc, DMA_FROM_DEVICE);
657 }
658 
659 /* create the send and receive buffer for an SMC socket;
660  * receive buffers are called RMBs;
661  * (even though the SMC protocol allows more than one RMB-element per RMB,
662  * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
663  * extra RMB for every connection in a link group
664  */
665 int smc_buf_create(struct smc_sock *smc)
666 {
667 	int rc;
668 
669 	/* create send buffer */
670 	rc = __smc_buf_create(smc, false);
671 	if (rc)
672 		return rc;
673 	/* create rmb */
674 	rc = __smc_buf_create(smc, true);
675 	if (rc)
676 		smc_buf_free(smc->conn.sndbuf_desc,
677 			     &smc->conn.lgr->lnk[SMC_SINGLE_LINK], false);
678 	return rc;
679 }
680 
681 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
682 {
683 	int i;
684 
685 	for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
686 		if (!test_and_set_bit(i, lgr->rtokens_used_mask))
687 			return i;
688 	}
689 	return -ENOSPC;
690 }
691 
692 /* save rkey and dma_addr received from peer during clc handshake */
693 int smc_rmb_rtoken_handling(struct smc_connection *conn,
694 			    struct smc_clc_msg_accept_confirm *clc)
695 {
696 	u64 dma_addr = be64_to_cpu(clc->rmb_dma_addr);
697 	struct smc_link_group *lgr = conn->lgr;
698 	u32 rkey = ntohl(clc->rmb_rkey);
699 	int i;
700 
701 	for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
702 		if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&
703 		    (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&
704 		    test_bit(i, lgr->rtokens_used_mask)) {
705 			conn->rtoken_idx = i;
706 			return 0;
707 		}
708 	}
709 	conn->rtoken_idx = smc_rmb_reserve_rtoken_idx(lgr);
710 	if (conn->rtoken_idx < 0)
711 		return conn->rtoken_idx;
712 	lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey = rkey;
713 	lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr = dma_addr;
714 	return 0;
715 }
716