xref: /openbmc/linux/net/smc/smc_llc.c (revision 715f23b6)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
4  *
5  *  Link Layer Control (LLC)
6  *
7  *  Copyright IBM Corp. 2016
8  *
9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
11  */
12 
13 #include <net/tcp.h>
14 #include <rdma/ib_verbs.h>
15 
16 #include "smc.h"
17 #include "smc_core.h"
18 #include "smc_clc.h"
19 #include "smc_llc.h"
20 
21 #define SMC_LLC_DATA_LEN		40
22 
23 struct smc_llc_hdr {
24 	struct smc_wr_rx_hdr common;
25 	u8 length;	/* 44 */
26 #if defined(__BIG_ENDIAN_BITFIELD)
27 	u8 reserved:4,
28 	   add_link_rej_rsn:4;
29 #elif defined(__LITTLE_ENDIAN_BITFIELD)
30 	u8 add_link_rej_rsn:4,
31 	   reserved:4;
32 #endif
33 	u8 flags;
34 };
35 
36 #define SMC_LLC_FLAG_NO_RMBE_EYEC	0x03
37 
38 struct smc_llc_msg_confirm_link {	/* type 0x01 */
39 	struct smc_llc_hdr hd;
40 	u8 sender_mac[ETH_ALEN];
41 	u8 sender_gid[SMC_GID_SIZE];
42 	u8 sender_qp_num[3];
43 	u8 link_num;
44 	u8 link_uid[SMC_LGR_ID_SIZE];
45 	u8 max_links;
46 	u8 reserved[9];
47 };
48 
49 #define SMC_LLC_FLAG_ADD_LNK_REJ	0x40
50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH	1
51 
52 #define SMC_LLC_ADD_LNK_MAX_LINKS	2
53 
54 struct smc_llc_msg_add_link {		/* type 0x02 */
55 	struct smc_llc_hdr hd;
56 	u8 sender_mac[ETH_ALEN];
57 	u8 reserved2[2];
58 	u8 sender_gid[SMC_GID_SIZE];
59 	u8 sender_qp_num[3];
60 	u8 link_num;
61 	u8 flags2;	/* QP mtu */
62 	u8 initial_psn[3];
63 	u8 reserved[8];
64 };
65 
66 #define SMC_LLC_FLAG_DEL_LINK_ALL	0x40
67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY	0x20
68 
69 struct smc_llc_msg_del_link {		/* type 0x04 */
70 	struct smc_llc_hdr hd;
71 	u8 link_num;
72 	__be32 reason;
73 	u8 reserved[35];
74 } __packed;			/* format defined in RFC7609 */
75 
76 struct smc_llc_msg_test_link {		/* type 0x07 */
77 	struct smc_llc_hdr hd;
78 	u8 user_data[16];
79 	u8 reserved[24];
80 };
81 
82 struct smc_rmb_rtoken {
83 	union {
84 		u8 num_rkeys;	/* first rtoken byte of CONFIRM LINK msg */
85 				/* is actually the num of rtokens, first */
86 				/* rtoken is always for the current link */
87 		u8 link_id;	/* link id of the rtoken */
88 	};
89 	__be32 rmb_key;
90 	__be64 rmb_vaddr;
91 } __packed;			/* format defined in RFC7609 */
92 
93 #define SMC_LLC_RKEYS_PER_MSG	3
94 
95 struct smc_llc_msg_confirm_rkey {	/* type 0x06 */
96 	struct smc_llc_hdr hd;
97 	struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
98 	u8 reserved;
99 };
100 
101 struct smc_llc_msg_confirm_rkey_cont {	/* type 0x08 */
102 	struct smc_llc_hdr hd;
103 	u8 num_rkeys;
104 	struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
105 };
106 
107 #define SMC_LLC_DEL_RKEY_MAX	8
108 #define SMC_LLC_FLAG_RKEY_NEG	0x20
109 
110 struct smc_llc_msg_delete_rkey {	/* type 0x09 */
111 	struct smc_llc_hdr hd;
112 	u8 num_rkeys;
113 	u8 err_mask;
114 	u8 reserved[2];
115 	__be32 rkey[8];
116 	u8 reserved2[4];
117 };
118 
119 union smc_llc_msg {
120 	struct smc_llc_msg_confirm_link confirm_link;
121 	struct smc_llc_msg_add_link add_link;
122 	struct smc_llc_msg_del_link delete_link;
123 
124 	struct smc_llc_msg_confirm_rkey confirm_rkey;
125 	struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
126 	struct smc_llc_msg_delete_rkey delete_rkey;
127 
128 	struct smc_llc_msg_test_link test_link;
129 	struct {
130 		struct smc_llc_hdr hdr;
131 		u8 data[SMC_LLC_DATA_LEN];
132 	} raw;
133 };
134 
135 #define SMC_LLC_FLAG_RESP		0x80
136 
137 /********************************** send *************************************/
138 
139 struct smc_llc_tx_pend {
140 };
141 
142 /* handler for send/transmission completion of an LLC msg */
143 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
144 			       struct smc_link *link,
145 			       enum ib_wc_status wc_status)
146 {
147 	/* future work: handle wc_status error for recovery and failover */
148 }
149 
150 /**
151  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
152  * @link: Pointer to SMC link used for sending LLC control message.
153  * @wr_buf: Out variable returning pointer to work request payload buffer.
154  * @pend: Out variable returning pointer to private pending WR tracking.
155  *	  It's the context the transmit complete handler will get.
156  *
157  * Reserves and pre-fills an entry for a pending work request send/tx.
158  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
159  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
160  *
161  * Return: 0 on success, otherwise an error value.
162  */
163 static int smc_llc_add_pending_send(struct smc_link *link,
164 				    struct smc_wr_buf **wr_buf,
165 				    struct smc_wr_tx_pend_priv **pend)
166 {
167 	int rc;
168 
169 	rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
170 				     pend);
171 	if (rc < 0)
172 		return rc;
173 	BUILD_BUG_ON_MSG(
174 		sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
175 		"must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
176 	BUILD_BUG_ON_MSG(
177 		sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
178 		"must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
179 	BUILD_BUG_ON_MSG(
180 		sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
181 		"must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
182 	return 0;
183 }
184 
185 /* high-level API to send LLC confirm link */
186 int smc_llc_send_confirm_link(struct smc_link *link,
187 			      enum smc_llc_reqresp reqresp)
188 {
189 	struct smc_link_group *lgr = smc_get_lgr(link);
190 	struct smc_llc_msg_confirm_link *confllc;
191 	struct smc_wr_tx_pend_priv *pend;
192 	struct smc_wr_buf *wr_buf;
193 	int rc;
194 
195 	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
196 	if (rc)
197 		return rc;
198 	confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
199 	memset(confllc, 0, sizeof(*confllc));
200 	confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
201 	confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
202 	confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
203 	if (reqresp == SMC_LLC_RESP)
204 		confllc->hd.flags |= SMC_LLC_FLAG_RESP;
205 	memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
206 	       ETH_ALEN);
207 	memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
208 	hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
209 	confllc->link_num = link->link_id;
210 	memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
211 	confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
212 	/* send llc message */
213 	rc = smc_wr_tx_send(link, pend);
214 	return rc;
215 }
216 
217 /* send LLC confirm rkey request */
218 static int smc_llc_send_confirm_rkey(struct smc_link *link,
219 				     struct smc_buf_desc *rmb_desc)
220 {
221 	struct smc_llc_msg_confirm_rkey *rkeyllc;
222 	struct smc_wr_tx_pend_priv *pend;
223 	struct smc_wr_buf *wr_buf;
224 	int rc;
225 
226 	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
227 	if (rc)
228 		return rc;
229 	rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
230 	memset(rkeyllc, 0, sizeof(*rkeyllc));
231 	rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
232 	rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
233 	rkeyllc->rtoken[0].rmb_key =
234 		htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
235 	rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
236 		(u64)sg_dma_address(rmb_desc->sgt[SMC_SINGLE_LINK].sgl));
237 	/* send llc message */
238 	rc = smc_wr_tx_send(link, pend);
239 	return rc;
240 }
241 
242 /* send LLC delete rkey request */
243 static int smc_llc_send_delete_rkey(struct smc_link *link,
244 				    struct smc_buf_desc *rmb_desc)
245 {
246 	struct smc_llc_msg_delete_rkey *rkeyllc;
247 	struct smc_wr_tx_pend_priv *pend;
248 	struct smc_wr_buf *wr_buf;
249 	int rc;
250 
251 	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
252 	if (rc)
253 		return rc;
254 	rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
255 	memset(rkeyllc, 0, sizeof(*rkeyllc));
256 	rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
257 	rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
258 	rkeyllc->num_rkeys = 1;
259 	rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
260 	/* send llc message */
261 	rc = smc_wr_tx_send(link, pend);
262 	return rc;
263 }
264 
265 /* prepare an add link message */
266 static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
267 				  struct smc_link *link, u8 mac[], u8 gid[],
268 				  enum smc_llc_reqresp reqresp)
269 {
270 	memset(addllc, 0, sizeof(*addllc));
271 	addllc->hd.common.type = SMC_LLC_ADD_LINK;
272 	addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
273 	if (reqresp == SMC_LLC_RESP) {
274 		addllc->hd.flags |= SMC_LLC_FLAG_RESP;
275 		/* always reject more links for now */
276 		addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
277 		addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
278 	}
279 	memcpy(addllc->sender_mac, mac, ETH_ALEN);
280 	memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
281 }
282 
283 /* send ADD LINK request or response */
284 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
285 			  enum smc_llc_reqresp reqresp)
286 {
287 	struct smc_llc_msg_add_link *addllc;
288 	struct smc_wr_tx_pend_priv *pend;
289 	struct smc_wr_buf *wr_buf;
290 	int rc;
291 
292 	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
293 	if (rc)
294 		return rc;
295 	addllc = (struct smc_llc_msg_add_link *)wr_buf;
296 	smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
297 	/* send llc message */
298 	rc = smc_wr_tx_send(link, pend);
299 	return rc;
300 }
301 
302 /* prepare a delete link message */
303 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
304 				     struct smc_link *link,
305 				     enum smc_llc_reqresp reqresp, bool orderly)
306 {
307 	memset(delllc, 0, sizeof(*delllc));
308 	delllc->hd.common.type = SMC_LLC_DELETE_LINK;
309 	delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
310 	if (reqresp == SMC_LLC_RESP)
311 		delllc->hd.flags |= SMC_LLC_FLAG_RESP;
312 	/* DEL_LINK_ALL because only 1 link supported */
313 	delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
314 	if (orderly)
315 		delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
316 	delllc->link_num = link->link_id;
317 }
318 
319 /* send DELETE LINK request or response */
320 int smc_llc_send_delete_link(struct smc_link *link,
321 			     enum smc_llc_reqresp reqresp, bool orderly)
322 {
323 	struct smc_llc_msg_del_link *delllc;
324 	struct smc_wr_tx_pend_priv *pend;
325 	struct smc_wr_buf *wr_buf;
326 	int rc;
327 
328 	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
329 	if (rc)
330 		return rc;
331 	delllc = (struct smc_llc_msg_del_link *)wr_buf;
332 	smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
333 	/* send llc message */
334 	rc = smc_wr_tx_send(link, pend);
335 	return rc;
336 }
337 
338 /* send LLC test link request */
339 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
340 {
341 	struct smc_llc_msg_test_link *testllc;
342 	struct smc_wr_tx_pend_priv *pend;
343 	struct smc_wr_buf *wr_buf;
344 	int rc;
345 
346 	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
347 	if (rc)
348 		return rc;
349 	testllc = (struct smc_llc_msg_test_link *)wr_buf;
350 	memset(testllc, 0, sizeof(*testllc));
351 	testllc->hd.common.type = SMC_LLC_TEST_LINK;
352 	testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
353 	memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
354 	/* send llc message */
355 	rc = smc_wr_tx_send(link, pend);
356 	return rc;
357 }
358 
359 struct smc_llc_send_work {
360 	struct work_struct work;
361 	struct smc_link *link;
362 	int llclen;
363 	union smc_llc_msg llcbuf;
364 };
365 
366 /* worker that sends a prepared message */
367 static void smc_llc_send_message_work(struct work_struct *work)
368 {
369 	struct smc_llc_send_work *llcwrk = container_of(work,
370 						struct smc_llc_send_work, work);
371 	struct smc_wr_tx_pend_priv *pend;
372 	struct smc_wr_buf *wr_buf;
373 	int rc;
374 
375 	if (llcwrk->link->state == SMC_LNK_INACTIVE)
376 		goto out;
377 	rc = smc_llc_add_pending_send(llcwrk->link, &wr_buf, &pend);
378 	if (rc)
379 		goto out;
380 	memcpy(wr_buf, &llcwrk->llcbuf, llcwrk->llclen);
381 	smc_wr_tx_send(llcwrk->link, pend);
382 out:
383 	kfree(llcwrk);
384 }
385 
386 /* copy llcbuf and schedule an llc send on link */
387 static int smc_llc_send_message(struct smc_link *link, void *llcbuf, int llclen)
388 {
389 	struct smc_llc_send_work *wrk = kmalloc(sizeof(*wrk), GFP_ATOMIC);
390 
391 	if (!wrk)
392 		return -ENOMEM;
393 	INIT_WORK(&wrk->work, smc_llc_send_message_work);
394 	wrk->link = link;
395 	wrk->llclen = llclen;
396 	memcpy(&wrk->llcbuf, llcbuf, llclen);
397 	queue_work(link->llc_wq, &wrk->work);
398 	return 0;
399 }
400 
401 /********************************* receive ***********************************/
402 
403 static void smc_llc_rx_confirm_link(struct smc_link *link,
404 				    struct smc_llc_msg_confirm_link *llc)
405 {
406 	struct smc_link_group *lgr = smc_get_lgr(link);
407 	int conf_rc;
408 
409 	/* RMBE eyecatchers are not supported */
410 	if (llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)
411 		conf_rc = 0;
412 	else
413 		conf_rc = ENOTSUPP;
414 
415 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
416 		if (lgr->role == SMC_SERV &&
417 		    link->state == SMC_LNK_ACTIVATING) {
418 			link->llc_confirm_resp_rc = conf_rc;
419 			complete(&link->llc_confirm_resp);
420 		}
421 	} else {
422 		if (lgr->role == SMC_CLNT &&
423 		    link->state == SMC_LNK_ACTIVATING) {
424 			link->llc_confirm_rc = conf_rc;
425 			link->link_id = llc->link_num;
426 			complete(&link->llc_confirm);
427 		}
428 	}
429 }
430 
431 static void smc_llc_rx_add_link(struct smc_link *link,
432 				struct smc_llc_msg_add_link *llc)
433 {
434 	struct smc_link_group *lgr = smc_get_lgr(link);
435 
436 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
437 		if (link->state == SMC_LNK_ACTIVATING)
438 			complete(&link->llc_add_resp);
439 	} else {
440 		if (link->state == SMC_LNK_ACTIVATING) {
441 			complete(&link->llc_add);
442 			return;
443 		}
444 
445 		if (lgr->role == SMC_SERV) {
446 			smc_llc_prep_add_link(llc, link,
447 					link->smcibdev->mac[link->ibport - 1],
448 					link->gid, SMC_LLC_REQ);
449 
450 		} else {
451 			smc_llc_prep_add_link(llc, link,
452 					link->smcibdev->mac[link->ibport - 1],
453 					link->gid, SMC_LLC_RESP);
454 		}
455 		smc_llc_send_message(link, llc, sizeof(*llc));
456 	}
457 }
458 
459 static void smc_llc_rx_delete_link(struct smc_link *link,
460 				   struct smc_llc_msg_del_link *llc)
461 {
462 	struct smc_link_group *lgr = smc_get_lgr(link);
463 
464 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
465 		if (lgr->role == SMC_SERV)
466 			smc_lgr_schedule_free_work_fast(lgr);
467 	} else {
468 		smc_lgr_forget(lgr);
469 		smc_llc_link_deleting(link);
470 		if (lgr->role == SMC_SERV) {
471 			/* client asks to delete this link, send request */
472 			smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
473 		} else {
474 			/* server requests to delete this link, send response */
475 			smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
476 		}
477 		smc_llc_send_message(link, llc, sizeof(*llc));
478 		smc_lgr_terminate_sched(lgr);
479 	}
480 }
481 
482 static void smc_llc_rx_test_link(struct smc_link *link,
483 				 struct smc_llc_msg_test_link *llc)
484 {
485 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
486 		if (link->state == SMC_LNK_ACTIVE)
487 			complete(&link->llc_testlink_resp);
488 	} else {
489 		llc->hd.flags |= SMC_LLC_FLAG_RESP;
490 		smc_llc_send_message(link, llc, sizeof(*llc));
491 	}
492 }
493 
494 static void smc_llc_rx_confirm_rkey(struct smc_link *link,
495 				    struct smc_llc_msg_confirm_rkey *llc)
496 {
497 	int rc;
498 
499 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
500 		link->llc_confirm_rkey_rc = llc->hd.flags &
501 					    SMC_LLC_FLAG_RKEY_NEG;
502 		complete(&link->llc_confirm_rkey);
503 	} else {
504 		rc = smc_rtoken_add(smc_get_lgr(link),
505 				    llc->rtoken[0].rmb_vaddr,
506 				    llc->rtoken[0].rmb_key);
507 
508 		/* ignore rtokens for other links, we have only one link */
509 
510 		llc->hd.flags |= SMC_LLC_FLAG_RESP;
511 		if (rc < 0)
512 			llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
513 		smc_llc_send_message(link, llc, sizeof(*llc));
514 	}
515 }
516 
517 static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link,
518 				      struct smc_llc_msg_confirm_rkey_cont *llc)
519 {
520 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
521 		/* unused as long as we don't send this type of msg */
522 	} else {
523 		/* ignore rtokens for other links, we have only one link */
524 		llc->hd.flags |= SMC_LLC_FLAG_RESP;
525 		smc_llc_send_message(link, llc, sizeof(*llc));
526 	}
527 }
528 
529 static void smc_llc_rx_delete_rkey(struct smc_link *link,
530 				   struct smc_llc_msg_delete_rkey *llc)
531 {
532 	u8 err_mask = 0;
533 	int i, max;
534 
535 	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
536 		link->llc_delete_rkey_rc = llc->hd.flags &
537 					    SMC_LLC_FLAG_RKEY_NEG;
538 		complete(&link->llc_delete_rkey);
539 	} else {
540 		max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
541 		for (i = 0; i < max; i++) {
542 			if (smc_rtoken_delete(smc_get_lgr(link), llc->rkey[i]))
543 				err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
544 		}
545 
546 		if (err_mask) {
547 			llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
548 			llc->err_mask = err_mask;
549 		}
550 
551 		llc->hd.flags |= SMC_LLC_FLAG_RESP;
552 		smc_llc_send_message(link, llc, sizeof(*llc));
553 	}
554 }
555 
556 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
557 {
558 	struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
559 	union smc_llc_msg *llc = buf;
560 
561 	if (wc->byte_len < sizeof(*llc))
562 		return; /* short message */
563 	if (llc->raw.hdr.length != sizeof(*llc))
564 		return; /* invalid message */
565 	if (link->state == SMC_LNK_INACTIVE)
566 		return; /* link not active, drop msg */
567 
568 	switch (llc->raw.hdr.common.type) {
569 	case SMC_LLC_TEST_LINK:
570 		smc_llc_rx_test_link(link, &llc->test_link);
571 		break;
572 	case SMC_LLC_CONFIRM_LINK:
573 		smc_llc_rx_confirm_link(link, &llc->confirm_link);
574 		break;
575 	case SMC_LLC_ADD_LINK:
576 		smc_llc_rx_add_link(link, &llc->add_link);
577 		break;
578 	case SMC_LLC_DELETE_LINK:
579 		smc_llc_rx_delete_link(link, &llc->delete_link);
580 		break;
581 	case SMC_LLC_CONFIRM_RKEY:
582 		smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);
583 		break;
584 	case SMC_LLC_CONFIRM_RKEY_CONT:
585 		smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont);
586 		break;
587 	case SMC_LLC_DELETE_RKEY:
588 		smc_llc_rx_delete_rkey(link, &llc->delete_rkey);
589 		break;
590 	}
591 }
592 
593 /***************************** worker, utils *********************************/
594 
595 static void smc_llc_testlink_work(struct work_struct *work)
596 {
597 	struct smc_link *link = container_of(to_delayed_work(work),
598 					     struct smc_link, llc_testlink_wrk);
599 	unsigned long next_interval;
600 	unsigned long expire_time;
601 	u8 user_data[16] = { 0 };
602 	int rc;
603 
604 	if (link->state != SMC_LNK_ACTIVE)
605 		return;		/* don't reschedule worker */
606 	expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
607 	if (time_is_after_jiffies(expire_time)) {
608 		next_interval = expire_time - jiffies;
609 		goto out;
610 	}
611 	reinit_completion(&link->llc_testlink_resp);
612 	smc_llc_send_test_link(link, user_data);
613 	/* receive TEST LINK response over RoCE fabric */
614 	rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
615 						       SMC_LLC_WAIT_TIME);
616 	if (rc <= 0) {
617 		smc_lgr_terminate(smc_get_lgr(link), true);
618 		return;
619 	}
620 	next_interval = link->llc_testlink_time;
621 out:
622 	queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk,
623 			   next_interval);
624 }
625 
626 int smc_llc_link_init(struct smc_link *link)
627 {
628 	struct smc_link_group *lgr = smc_get_lgr(link);
629 	link->llc_wq = alloc_ordered_workqueue("llc_wq-%x:%x)", WQ_MEM_RECLAIM,
630 					       *((u32 *)lgr->id),
631 					       link->link_id);
632 	if (!link->llc_wq)
633 		return -ENOMEM;
634 	init_completion(&link->llc_confirm);
635 	init_completion(&link->llc_confirm_resp);
636 	init_completion(&link->llc_add);
637 	init_completion(&link->llc_add_resp);
638 	init_completion(&link->llc_confirm_rkey);
639 	init_completion(&link->llc_delete_rkey);
640 	mutex_init(&link->llc_delete_rkey_mutex);
641 	init_completion(&link->llc_testlink_resp);
642 	INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
643 	return 0;
644 }
645 
646 void smc_llc_link_active(struct smc_link *link, int testlink_time)
647 {
648 	link->state = SMC_LNK_ACTIVE;
649 	if (testlink_time) {
650 		link->llc_testlink_time = testlink_time * HZ;
651 		queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk,
652 				   link->llc_testlink_time);
653 	}
654 }
655 
656 void smc_llc_link_deleting(struct smc_link *link)
657 {
658 	link->state = SMC_LNK_DELETING;
659 	smc_wr_wakeup_tx_wait(link);
660 }
661 
662 /* called in tasklet context */
663 void smc_llc_link_inactive(struct smc_link *link)
664 {
665 	link->state = SMC_LNK_INACTIVE;
666 	cancel_delayed_work(&link->llc_testlink_wrk);
667 	smc_wr_wakeup_reg_wait(link);
668 	smc_wr_wakeup_tx_wait(link);
669 }
670 
671 /* called in worker context */
672 void smc_llc_link_clear(struct smc_link *link)
673 {
674 	flush_workqueue(link->llc_wq);
675 	destroy_workqueue(link->llc_wq);
676 }
677 
678 /* register a new rtoken at the remote peer */
679 int smc_llc_do_confirm_rkey(struct smc_link *link,
680 			    struct smc_buf_desc *rmb_desc)
681 {
682 	int rc;
683 
684 	/* protected by mutex smc_create_lgr_pending */
685 	reinit_completion(&link->llc_confirm_rkey);
686 	rc = smc_llc_send_confirm_rkey(link, rmb_desc);
687 	if (rc)
688 		return rc;
689 	/* receive CONFIRM RKEY response from server over RoCE fabric */
690 	rc = wait_for_completion_interruptible_timeout(&link->llc_confirm_rkey,
691 						       SMC_LLC_WAIT_TIME);
692 	if (rc <= 0 || link->llc_confirm_rkey_rc)
693 		return -EFAULT;
694 	return 0;
695 }
696 
697 /* unregister an rtoken at the remote peer */
698 int smc_llc_do_delete_rkey(struct smc_link *link,
699 			   struct smc_buf_desc *rmb_desc)
700 {
701 	int rc = 0;
702 
703 	mutex_lock(&link->llc_delete_rkey_mutex);
704 	if (link->state != SMC_LNK_ACTIVE)
705 		goto out;
706 	reinit_completion(&link->llc_delete_rkey);
707 	rc = smc_llc_send_delete_rkey(link, rmb_desc);
708 	if (rc)
709 		goto out;
710 	/* receive DELETE RKEY response from server over RoCE fabric */
711 	rc = wait_for_completion_interruptible_timeout(&link->llc_delete_rkey,
712 						       SMC_LLC_WAIT_TIME);
713 	if (rc <= 0 || link->llc_delete_rkey_rc)
714 		rc = -EFAULT;
715 	else
716 		rc = 0;
717 out:
718 	mutex_unlock(&link->llc_delete_rkey_mutex);
719 	return rc;
720 }
721 
722 /***************************** init, exit, misc ******************************/
723 
724 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
725 	{
726 		.handler	= smc_llc_rx_handler,
727 		.type		= SMC_LLC_CONFIRM_LINK
728 	},
729 	{
730 		.handler	= smc_llc_rx_handler,
731 		.type		= SMC_LLC_TEST_LINK
732 	},
733 	{
734 		.handler	= smc_llc_rx_handler,
735 		.type		= SMC_LLC_ADD_LINK
736 	},
737 	{
738 		.handler	= smc_llc_rx_handler,
739 		.type		= SMC_LLC_DELETE_LINK
740 	},
741 	{
742 		.handler	= smc_llc_rx_handler,
743 		.type		= SMC_LLC_CONFIRM_RKEY
744 	},
745 	{
746 		.handler	= smc_llc_rx_handler,
747 		.type		= SMC_LLC_CONFIRM_RKEY_CONT
748 	},
749 	{
750 		.handler	= smc_llc_rx_handler,
751 		.type		= SMC_LLC_DELETE_RKEY
752 	},
753 	{
754 		.handler	= NULL,
755 	}
756 };
757 
758 int __init smc_llc_init(void)
759 {
760 	struct smc_wr_rx_handler *handler;
761 	int rc = 0;
762 
763 	for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
764 		INIT_HLIST_NODE(&handler->list);
765 		rc = smc_wr_rx_register_handler(handler);
766 		if (rc)
767 			break;
768 	}
769 	return rc;
770 }
771