1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13 #include <net/tcp.h>
14 #include <rdma/ib_verbs.h>
15
16 #include "smc.h"
17 #include "smc_core.h"
18 #include "smc_clc.h"
19 #include "smc_llc.h"
20 #include "smc_pnet.h"
21
22 #define SMC_LLC_DATA_LEN 40
23
24 struct smc_llc_hdr {
25 struct smc_wr_rx_hdr common;
26 union {
27 struct {
28 u8 length; /* 44 */
29 #if defined(__BIG_ENDIAN_BITFIELD)
30 u8 reserved:4,
31 add_link_rej_rsn:4;
32 #elif defined(__LITTLE_ENDIAN_BITFIELD)
33 u8 add_link_rej_rsn:4,
34 reserved:4;
35 #endif
36 };
37 u16 length_v2; /* 44 - 8192*/
38 };
39 u8 flags;
40 } __packed; /* format defined in
41 * IBM Shared Memory Communications Version 2
42 * (https://www.ibm.com/support/pages/node/6326337)
43 */
44
45 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
46
47 struct smc_llc_msg_confirm_link { /* type 0x01 */
48 struct smc_llc_hdr hd;
49 u8 sender_mac[ETH_ALEN];
50 u8 sender_gid[SMC_GID_SIZE];
51 u8 sender_qp_num[3];
52 u8 link_num;
53 u8 link_uid[SMC_LGR_ID_SIZE];
54 u8 max_links;
55 u8 max_conns;
56 u8 reserved[8];
57 };
58
59 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
60 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
61
62 struct smc_llc_msg_add_link { /* type 0x02 */
63 struct smc_llc_hdr hd;
64 u8 sender_mac[ETH_ALEN];
65 u8 reserved2[2];
66 u8 sender_gid[SMC_GID_SIZE];
67 u8 sender_qp_num[3];
68 u8 link_num;
69 #if defined(__BIG_ENDIAN_BITFIELD)
70 u8 reserved3 : 4,
71 qp_mtu : 4;
72 #elif defined(__LITTLE_ENDIAN_BITFIELD)
73 u8 qp_mtu : 4,
74 reserved3 : 4;
75 #endif
76 u8 initial_psn[3];
77 u8 reserved[8];
78 };
79
80 struct smc_llc_msg_add_link_cont_rt {
81 __be32 rmb_key;
82 __be32 rmb_key_new;
83 __be64 rmb_vaddr_new;
84 };
85
86 struct smc_llc_msg_add_link_v2_ext {
87 #if defined(__BIG_ENDIAN_BITFIELD)
88 u8 v2_direct : 1,
89 reserved : 7;
90 #elif defined(__LITTLE_ENDIAN_BITFIELD)
91 u8 reserved : 7,
92 v2_direct : 1;
93 #endif
94 u8 reserved2;
95 u8 client_target_gid[SMC_GID_SIZE];
96 u8 reserved3[8];
97 u16 num_rkeys;
98 struct smc_llc_msg_add_link_cont_rt rt[];
99 } __packed; /* format defined in
100 * IBM Shared Memory Communications Version 2
101 * (https://www.ibm.com/support/pages/node/6326337)
102 */
103
104 struct smc_llc_msg_req_add_link_v2 {
105 struct smc_llc_hdr hd;
106 u8 reserved[20];
107 u8 gid_cnt;
108 u8 reserved2[3];
109 u8 gid[][SMC_GID_SIZE];
110 };
111
112 #define SMC_LLC_RKEYS_PER_CONT_MSG 2
113
114 struct smc_llc_msg_add_link_cont { /* type 0x03 */
115 struct smc_llc_hdr hd;
116 u8 link_num;
117 u8 num_rkeys;
118 u8 reserved2[2];
119 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
120 u8 reserved[4];
121 } __packed; /* format defined in RFC7609 */
122
123 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
124 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
125
126 struct smc_llc_msg_del_link { /* type 0x04 */
127 struct smc_llc_hdr hd;
128 u8 link_num;
129 __be32 reason;
130 u8 reserved[35];
131 } __packed; /* format defined in RFC7609 */
132
133 struct smc_llc_msg_test_link { /* type 0x07 */
134 struct smc_llc_hdr hd;
135 u8 user_data[16];
136 u8 reserved[24];
137 };
138
139 struct smc_rmb_rtoken {
140 union {
141 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
142 /* is actually the num of rtokens, first */
143 /* rtoken is always for the current link */
144 u8 link_id; /* link id of the rtoken */
145 };
146 __be32 rmb_key;
147 __be64 rmb_vaddr;
148 } __packed; /* format defined in RFC7609 */
149
150 #define SMC_LLC_RKEYS_PER_MSG 3
151 #define SMC_LLC_RKEYS_PER_MSG_V2 255
152
153 struct smc_llc_msg_confirm_rkey { /* type 0x06 */
154 struct smc_llc_hdr hd;
155 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
156 u8 reserved;
157 };
158
159 #define SMC_LLC_DEL_RKEY_MAX 8
160 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
161 #define SMC_LLC_FLAG_RKEY_NEG 0x20
162
163 struct smc_llc_msg_delete_rkey { /* type 0x09 */
164 struct smc_llc_hdr hd;
165 u8 num_rkeys;
166 u8 err_mask;
167 u8 reserved[2];
168 __be32 rkey[8];
169 u8 reserved2[4];
170 };
171
172 struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */
173 struct smc_llc_hdr hd;
174 u8 num_rkeys;
175 u8 num_inval_rkeys;
176 u8 reserved[2];
177 __be32 rkey[];
178 };
179
180 union smc_llc_msg {
181 struct smc_llc_msg_confirm_link confirm_link;
182 struct smc_llc_msg_add_link add_link;
183 struct smc_llc_msg_req_add_link_v2 req_add_link;
184 struct smc_llc_msg_add_link_cont add_link_cont;
185 struct smc_llc_msg_del_link delete_link;
186
187 struct smc_llc_msg_confirm_rkey confirm_rkey;
188 struct smc_llc_msg_delete_rkey delete_rkey;
189
190 struct smc_llc_msg_test_link test_link;
191 struct {
192 struct smc_llc_hdr hdr;
193 u8 data[SMC_LLC_DATA_LEN];
194 } raw;
195 };
196
197 #define SMC_LLC_FLAG_RESP 0x80
198
199 struct smc_llc_qentry {
200 struct list_head list;
201 struct smc_link *link;
202 union smc_llc_msg msg;
203 };
204
205 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
206
smc_llc_flow_qentry_clr(struct smc_llc_flow * flow)207 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
208 {
209 struct smc_llc_qentry *qentry = flow->qentry;
210
211 flow->qentry = NULL;
212 return qentry;
213 }
214
smc_llc_flow_qentry_del(struct smc_llc_flow * flow)215 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
216 {
217 struct smc_llc_qentry *qentry;
218
219 if (flow->qentry) {
220 qentry = flow->qentry;
221 flow->qentry = NULL;
222 kfree(qentry);
223 }
224 }
225
smc_llc_flow_qentry_set(struct smc_llc_flow * flow,struct smc_llc_qentry * qentry)226 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
227 struct smc_llc_qentry *qentry)
228 {
229 flow->qentry = qentry;
230 }
231
smc_llc_flow_parallel(struct smc_link_group * lgr,u8 flow_type,struct smc_llc_qentry * qentry)232 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
233 struct smc_llc_qentry *qentry)
234 {
235 u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
236
237 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
238 flow_type != msg_type && !lgr->delayed_event) {
239 lgr->delayed_event = qentry;
240 return;
241 }
242 /* drop parallel or already-in-progress llc requests */
243 if (flow_type != msg_type)
244 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel "
245 "LLC msg: msg %d flow %d role %d\n",
246 SMC_LGR_ID_SIZE, &lgr->id,
247 lgr->net->net_cookie,
248 qentry->msg.raw.hdr.common.type,
249 flow_type, lgr->role);
250 kfree(qentry);
251 }
252
253 /* try to start a new llc flow, initiated by an incoming llc msg */
smc_llc_flow_start(struct smc_llc_flow * flow,struct smc_llc_qentry * qentry)254 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
255 struct smc_llc_qentry *qentry)
256 {
257 struct smc_link_group *lgr = qentry->link->lgr;
258
259 spin_lock_bh(&lgr->llc_flow_lock);
260 if (flow->type) {
261 /* a flow is already active */
262 smc_llc_flow_parallel(lgr, flow->type, qentry);
263 spin_unlock_bh(&lgr->llc_flow_lock);
264 return false;
265 }
266 switch (qentry->msg.raw.hdr.common.llc_type) {
267 case SMC_LLC_ADD_LINK:
268 flow->type = SMC_LLC_FLOW_ADD_LINK;
269 break;
270 case SMC_LLC_DELETE_LINK:
271 flow->type = SMC_LLC_FLOW_DEL_LINK;
272 break;
273 case SMC_LLC_CONFIRM_RKEY:
274 case SMC_LLC_DELETE_RKEY:
275 flow->type = SMC_LLC_FLOW_RKEY;
276 break;
277 default:
278 flow->type = SMC_LLC_FLOW_NONE;
279 }
280 smc_llc_flow_qentry_set(flow, qentry);
281 spin_unlock_bh(&lgr->llc_flow_lock);
282 return true;
283 }
284
285 /* start a new local llc flow, wait till current flow finished */
smc_llc_flow_initiate(struct smc_link_group * lgr,enum smc_llc_flowtype type)286 int smc_llc_flow_initiate(struct smc_link_group *lgr,
287 enum smc_llc_flowtype type)
288 {
289 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
290 int rc;
291
292 /* all flows except confirm_rkey and delete_rkey are exclusive,
293 * confirm/delete rkey flows can run concurrently (local and remote)
294 */
295 if (type == SMC_LLC_FLOW_RKEY)
296 allowed_remote = SMC_LLC_FLOW_RKEY;
297 again:
298 if (list_empty(&lgr->list))
299 return -ENODEV;
300 spin_lock_bh(&lgr->llc_flow_lock);
301 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
302 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
303 lgr->llc_flow_rmt.type == allowed_remote)) {
304 lgr->llc_flow_lcl.type = type;
305 spin_unlock_bh(&lgr->llc_flow_lock);
306 return 0;
307 }
308 spin_unlock_bh(&lgr->llc_flow_lock);
309 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
310 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
311 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
312 lgr->llc_flow_rmt.type == allowed_remote))),
313 SMC_LLC_WAIT_TIME * 10);
314 if (!rc)
315 return -ETIMEDOUT;
316 goto again;
317 }
318
319 /* finish the current llc flow */
smc_llc_flow_stop(struct smc_link_group * lgr,struct smc_llc_flow * flow)320 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
321 {
322 spin_lock_bh(&lgr->llc_flow_lock);
323 memset(flow, 0, sizeof(*flow));
324 flow->type = SMC_LLC_FLOW_NONE;
325 spin_unlock_bh(&lgr->llc_flow_lock);
326 if (!list_empty(&lgr->list) && lgr->delayed_event &&
327 flow == &lgr->llc_flow_lcl)
328 schedule_work(&lgr->llc_event_work);
329 else
330 wake_up(&lgr->llc_flow_waiter);
331 }
332
333 /* lnk is optional and used for early wakeup when link goes down, useful in
334 * cases where we wait for a response on the link after we sent a request
335 */
smc_llc_wait(struct smc_link_group * lgr,struct smc_link * lnk,int time_out,u8 exp_msg)336 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
337 struct smc_link *lnk,
338 int time_out, u8 exp_msg)
339 {
340 struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
341 u8 rcv_msg;
342
343 wait_event_timeout(lgr->llc_msg_waiter,
344 (flow->qentry ||
345 (lnk && !smc_link_usable(lnk)) ||
346 list_empty(&lgr->list)),
347 time_out);
348 if (!flow->qentry ||
349 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
350 smc_llc_flow_qentry_del(flow);
351 goto out;
352 }
353 rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
354 if (exp_msg && rcv_msg != exp_msg) {
355 if (exp_msg == SMC_LLC_ADD_LINK &&
356 rcv_msg == SMC_LLC_DELETE_LINK) {
357 /* flow_start will delay the unexpected msg */
358 smc_llc_flow_start(&lgr->llc_flow_lcl,
359 smc_llc_flow_qentry_clr(flow));
360 return NULL;
361 }
362 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: "
363 "msg %d exp %d flow %d role %d flags %x\n",
364 SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,
365 rcv_msg, exp_msg,
366 flow->type, lgr->role,
367 flow->qentry->msg.raw.hdr.flags);
368 smc_llc_flow_qentry_del(flow);
369 }
370 out:
371 return flow->qentry;
372 }
373
374 /********************************** send *************************************/
375
376 struct smc_llc_tx_pend {
377 };
378
379 /* handler for send/transmission completion of an LLC msg */
smc_llc_tx_handler(struct smc_wr_tx_pend_priv * pend,struct smc_link * link,enum ib_wc_status wc_status)380 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
381 struct smc_link *link,
382 enum ib_wc_status wc_status)
383 {
384 /* future work: handle wc_status error for recovery and failover */
385 }
386
387 /**
388 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
389 * @link: Pointer to SMC link used for sending LLC control message.
390 * @wr_buf: Out variable returning pointer to work request payload buffer.
391 * @pend: Out variable returning pointer to private pending WR tracking.
392 * It's the context the transmit complete handler will get.
393 *
394 * Reserves and pre-fills an entry for a pending work request send/tx.
395 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
396 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
397 *
398 * Return: 0 on success, otherwise an error value.
399 */
smc_llc_add_pending_send(struct smc_link * link,struct smc_wr_buf ** wr_buf,struct smc_wr_tx_pend_priv ** pend)400 static int smc_llc_add_pending_send(struct smc_link *link,
401 struct smc_wr_buf **wr_buf,
402 struct smc_wr_tx_pend_priv **pend)
403 {
404 int rc;
405
406 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
407 pend);
408 if (rc < 0)
409 return rc;
410 BUILD_BUG_ON_MSG(
411 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
412 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
413 BUILD_BUG_ON_MSG(
414 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
415 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
416 BUILD_BUG_ON_MSG(
417 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
418 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
419 return 0;
420 }
421
smc_llc_add_pending_send_v2(struct smc_link * link,struct smc_wr_v2_buf ** wr_buf,struct smc_wr_tx_pend_priv ** pend)422 static int smc_llc_add_pending_send_v2(struct smc_link *link,
423 struct smc_wr_v2_buf **wr_buf,
424 struct smc_wr_tx_pend_priv **pend)
425 {
426 int rc;
427
428 rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend);
429 if (rc < 0)
430 return rc;
431 return 0;
432 }
433
smc_llc_init_msg_hdr(struct smc_llc_hdr * hdr,struct smc_link_group * lgr,size_t len)434 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
435 struct smc_link_group *lgr, size_t len)
436 {
437 if (lgr->smc_version == SMC_V2) {
438 hdr->common.llc_version = SMC_V2;
439 hdr->length_v2 = len;
440 } else {
441 hdr->common.llc_version = 0;
442 hdr->length = len;
443 }
444 }
445
446 /* high-level API to send LLC confirm link */
smc_llc_send_confirm_link(struct smc_link * link,enum smc_llc_reqresp reqresp)447 int smc_llc_send_confirm_link(struct smc_link *link,
448 enum smc_llc_reqresp reqresp)
449 {
450 struct smc_llc_msg_confirm_link *confllc;
451 struct smc_wr_tx_pend_priv *pend;
452 struct smc_wr_buf *wr_buf;
453 int rc;
454
455 if (!smc_wr_tx_link_hold(link))
456 return -ENOLINK;
457 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
458 if (rc)
459 goto put_out;
460 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
461 memset(confllc, 0, sizeof(*confllc));
462 confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
463 smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));
464 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
465 if (reqresp == SMC_LLC_RESP)
466 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
467 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
468 ETH_ALEN);
469 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
470 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
471 confllc->link_num = link->link_id;
472 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
473 confllc->max_links = SMC_LINKS_ADD_LNK_MAX;
474 if (link->lgr->smc_version == SMC_V2 &&
475 link->lgr->peer_smc_release >= SMC_RELEASE_1) {
476 confllc->max_conns = link->lgr->max_conns;
477 confllc->max_links = link->lgr->max_links;
478 }
479 /* send llc message */
480 rc = smc_wr_tx_send(link, pend);
481 put_out:
482 smc_wr_tx_link_put(link);
483 return rc;
484 }
485
486 /* send LLC confirm rkey request */
smc_llc_send_confirm_rkey(struct smc_link * send_link,struct smc_buf_desc * rmb_desc)487 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
488 struct smc_buf_desc *rmb_desc)
489 {
490 struct smc_llc_msg_confirm_rkey *rkeyllc;
491 struct smc_wr_tx_pend_priv *pend;
492 struct smc_wr_buf *wr_buf;
493 struct smc_link *link;
494 int i, rc, rtok_ix;
495
496 if (!smc_wr_tx_link_hold(send_link))
497 return -ENOLINK;
498 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
499 if (rc)
500 goto put_out;
501 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
502 memset(rkeyllc, 0, sizeof(*rkeyllc));
503 rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
504 smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));
505
506 rtok_ix = 1;
507 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
508 link = &send_link->lgr->lnk[i];
509 if (smc_link_active(link) && link != send_link) {
510 rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
511 rkeyllc->rtoken[rtok_ix].rmb_key =
512 htonl(rmb_desc->mr[link->link_idx]->rkey);
513 rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ?
514 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
515 cpu_to_be64((u64)sg_dma_address
516 (rmb_desc->sgt[link->link_idx].sgl));
517 rtok_ix++;
518 }
519 }
520 /* rkey of send_link is in rtoken[0] */
521 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
522 rkeyllc->rtoken[0].rmb_key =
523 htonl(rmb_desc->mr[send_link->link_idx]->rkey);
524 rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ?
525 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
526 cpu_to_be64((u64)sg_dma_address
527 (rmb_desc->sgt[send_link->link_idx].sgl));
528 /* send llc message */
529 rc = smc_wr_tx_send(send_link, pend);
530 put_out:
531 smc_wr_tx_link_put(send_link);
532 return rc;
533 }
534
535 /* send LLC delete rkey request */
smc_llc_send_delete_rkey(struct smc_link * link,struct smc_buf_desc * rmb_desc)536 static int smc_llc_send_delete_rkey(struct smc_link *link,
537 struct smc_buf_desc *rmb_desc)
538 {
539 struct smc_llc_msg_delete_rkey *rkeyllc;
540 struct smc_wr_tx_pend_priv *pend;
541 struct smc_wr_buf *wr_buf;
542 int rc;
543
544 if (!smc_wr_tx_link_hold(link))
545 return -ENOLINK;
546 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
547 if (rc)
548 goto put_out;
549 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
550 memset(rkeyllc, 0, sizeof(*rkeyllc));
551 rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
552 smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));
553 rkeyllc->num_rkeys = 1;
554 rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey);
555 /* send llc message */
556 rc = smc_wr_tx_send(link, pend);
557 put_out:
558 smc_wr_tx_link_put(link);
559 return rc;
560 }
561
562 /* return first buffer from any of the next buf lists */
_smc_llc_get_next_rmb(struct smc_link_group * lgr,int * buf_lst)563 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
564 int *buf_lst)
565 {
566 struct smc_buf_desc *buf_pos;
567
568 while (*buf_lst < SMC_RMBE_SIZES) {
569 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
570 struct smc_buf_desc, list);
571 if (buf_pos)
572 return buf_pos;
573 (*buf_lst)++;
574 }
575 return NULL;
576 }
577
578 /* return next rmb from buffer lists */
smc_llc_get_next_rmb(struct smc_link_group * lgr,int * buf_lst,struct smc_buf_desc * buf_pos)579 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
580 int *buf_lst,
581 struct smc_buf_desc *buf_pos)
582 {
583 struct smc_buf_desc *buf_next;
584
585 if (!buf_pos)
586 return _smc_llc_get_next_rmb(lgr, buf_lst);
587
588 if (list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
589 (*buf_lst)++;
590 return _smc_llc_get_next_rmb(lgr, buf_lst);
591 }
592 buf_next = list_next_entry(buf_pos, list);
593 return buf_next;
594 }
595
smc_llc_get_first_rmb(struct smc_link_group * lgr,int * buf_lst)596 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
597 int *buf_lst)
598 {
599 *buf_lst = 0;
600 return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
601 }
602
smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext * ext,struct smc_link * link,struct smc_link * link_new)603 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
604 struct smc_link *link, struct smc_link *link_new)
605 {
606 struct smc_link_group *lgr = link->lgr;
607 struct smc_buf_desc *buf_pos;
608 int prim_lnk_idx, lnk_idx, i;
609 struct smc_buf_desc *rmb;
610 int len = sizeof(*ext);
611 int buf_lst;
612
613 ext->v2_direct = !lgr->uses_gateway;
614 memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
615
616 prim_lnk_idx = link->link_idx;
617 lnk_idx = link_new->link_idx;
618 down_write(&lgr->rmbs_lock);
619 ext->num_rkeys = lgr->conns_num;
620 if (!ext->num_rkeys)
621 goto out;
622 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
623 for (i = 0; i < ext->num_rkeys; i++) {
624 while (buf_pos && !(buf_pos)->used)
625 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
626 if (!buf_pos)
627 break;
628 rmb = buf_pos;
629 ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
630 ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
631 ext->rt[i].rmb_vaddr_new = rmb->is_vm ?
632 cpu_to_be64((uintptr_t)rmb->cpu_addr) :
633 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
634 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
635 }
636 len += i * sizeof(ext->rt[0]);
637 out:
638 up_write(&lgr->rmbs_lock);
639 return len;
640 }
641
642 /* send ADD LINK request or response */
smc_llc_send_add_link(struct smc_link * link,u8 mac[],u8 gid[],struct smc_link * link_new,enum smc_llc_reqresp reqresp)643 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
644 struct smc_link *link_new,
645 enum smc_llc_reqresp reqresp)
646 {
647 struct smc_llc_msg_add_link_v2_ext *ext = NULL;
648 struct smc_llc_msg_add_link *addllc;
649 struct smc_wr_tx_pend_priv *pend;
650 int len = sizeof(*addllc);
651 int rc;
652
653 if (!smc_wr_tx_link_hold(link))
654 return -ENOLINK;
655 if (link->lgr->smc_version == SMC_V2) {
656 struct smc_wr_v2_buf *wr_buf;
657
658 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
659 if (rc)
660 goto put_out;
661 addllc = (struct smc_llc_msg_add_link *)wr_buf;
662 ext = (struct smc_llc_msg_add_link_v2_ext *)
663 &wr_buf->raw[sizeof(*addllc)];
664 memset(ext, 0, SMC_WR_TX_SIZE);
665 } else {
666 struct smc_wr_buf *wr_buf;
667
668 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
669 if (rc)
670 goto put_out;
671 addllc = (struct smc_llc_msg_add_link *)wr_buf;
672 }
673
674 memset(addllc, 0, sizeof(*addllc));
675 addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
676 if (reqresp == SMC_LLC_RESP)
677 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
678 memcpy(addllc->sender_mac, mac, ETH_ALEN);
679 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
680 if (link_new) {
681 addllc->link_num = link_new->link_id;
682 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
683 hton24(addllc->initial_psn, link_new->psn_initial);
684 if (reqresp == SMC_LLC_REQ)
685 addllc->qp_mtu = link_new->path_mtu;
686 else
687 addllc->qp_mtu = min(link_new->path_mtu,
688 link_new->peer_mtu);
689 }
690 if (ext && link_new)
691 len += smc_llc_fill_ext_v2(ext, link, link_new);
692 smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);
693 /* send llc message */
694 if (link->lgr->smc_version == SMC_V2)
695 rc = smc_wr_tx_v2_send(link, pend, len);
696 else
697 rc = smc_wr_tx_send(link, pend);
698 put_out:
699 smc_wr_tx_link_put(link);
700 return rc;
701 }
702
703 /* send DELETE LINK request or response */
smc_llc_send_delete_link(struct smc_link * link,u8 link_del_id,enum smc_llc_reqresp reqresp,bool orderly,u32 reason)704 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
705 enum smc_llc_reqresp reqresp, bool orderly,
706 u32 reason)
707 {
708 struct smc_llc_msg_del_link *delllc;
709 struct smc_wr_tx_pend_priv *pend;
710 struct smc_wr_buf *wr_buf;
711 int rc;
712
713 if (!smc_wr_tx_link_hold(link))
714 return -ENOLINK;
715 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
716 if (rc)
717 goto put_out;
718 delllc = (struct smc_llc_msg_del_link *)wr_buf;
719
720 memset(delllc, 0, sizeof(*delllc));
721 delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
722 smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));
723 if (reqresp == SMC_LLC_RESP)
724 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
725 if (orderly)
726 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
727 if (link_del_id)
728 delllc->link_num = link_del_id;
729 else
730 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
731 delllc->reason = htonl(reason);
732 /* send llc message */
733 rc = smc_wr_tx_send(link, pend);
734 put_out:
735 smc_wr_tx_link_put(link);
736 return rc;
737 }
738
739 /* send LLC test link request */
smc_llc_send_test_link(struct smc_link * link,u8 user_data[16])740 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
741 {
742 struct smc_llc_msg_test_link *testllc;
743 struct smc_wr_tx_pend_priv *pend;
744 struct smc_wr_buf *wr_buf;
745 int rc;
746
747 if (!smc_wr_tx_link_hold(link))
748 return -ENOLINK;
749 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
750 if (rc)
751 goto put_out;
752 testllc = (struct smc_llc_msg_test_link *)wr_buf;
753 memset(testllc, 0, sizeof(*testllc));
754 testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
755 smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));
756 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
757 /* send llc message */
758 rc = smc_wr_tx_send(link, pend);
759 put_out:
760 smc_wr_tx_link_put(link);
761 return rc;
762 }
763
764 /* schedule an llc send on link, may wait for buffers */
smc_llc_send_message(struct smc_link * link,void * llcbuf)765 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
766 {
767 struct smc_wr_tx_pend_priv *pend;
768 struct smc_wr_buf *wr_buf;
769 int rc;
770
771 if (!smc_wr_tx_link_hold(link))
772 return -ENOLINK;
773 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
774 if (rc)
775 goto put_out;
776 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
777 rc = smc_wr_tx_send(link, pend);
778 put_out:
779 smc_wr_tx_link_put(link);
780 return rc;
781 }
782
783 /* schedule an llc send on link, may wait for buffers,
784 * and wait for send completion notification.
785 * @return 0 on success
786 */
smc_llc_send_message_wait(struct smc_link * link,void * llcbuf)787 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
788 {
789 struct smc_wr_tx_pend_priv *pend;
790 struct smc_wr_buf *wr_buf;
791 int rc;
792
793 if (!smc_wr_tx_link_hold(link))
794 return -ENOLINK;
795 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
796 if (rc)
797 goto put_out;
798 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
799 rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
800 put_out:
801 smc_wr_tx_link_put(link);
802 return rc;
803 }
804
805 /********************************* receive ***********************************/
806
smc_llc_alloc_alt_link(struct smc_link_group * lgr,enum smc_lgr_type lgr_new_t)807 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
808 enum smc_lgr_type lgr_new_t)
809 {
810 int i;
811
812 if (lgr->type == SMC_LGR_SYMMETRIC ||
813 (lgr->type != SMC_LGR_SINGLE &&
814 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
815 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
816 return -EMLINK;
817
818 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
819 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
820 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
821 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
822 return i;
823 } else {
824 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
825 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
826 return i;
827 }
828 return -EMLINK;
829 }
830
831 /* send one add_link_continue msg */
smc_llc_add_link_cont(struct smc_link * link,struct smc_link * link_new,u8 * num_rkeys_todo,int * buf_lst,struct smc_buf_desc ** buf_pos)832 static int smc_llc_add_link_cont(struct smc_link *link,
833 struct smc_link *link_new, u8 *num_rkeys_todo,
834 int *buf_lst, struct smc_buf_desc **buf_pos)
835 {
836 struct smc_llc_msg_add_link_cont *addc_llc;
837 struct smc_link_group *lgr = link->lgr;
838 int prim_lnk_idx, lnk_idx, i, rc;
839 struct smc_wr_tx_pend_priv *pend;
840 struct smc_wr_buf *wr_buf;
841 struct smc_buf_desc *rmb;
842 u8 n;
843
844 if (!smc_wr_tx_link_hold(link))
845 return -ENOLINK;
846 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
847 if (rc)
848 goto put_out;
849 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
850 memset(addc_llc, 0, sizeof(*addc_llc));
851
852 prim_lnk_idx = link->link_idx;
853 lnk_idx = link_new->link_idx;
854 addc_llc->link_num = link_new->link_id;
855 addc_llc->num_rkeys = *num_rkeys_todo;
856 n = *num_rkeys_todo;
857 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
858 while (*buf_pos && !(*buf_pos)->used)
859 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
860 if (!*buf_pos) {
861 addc_llc->num_rkeys = addc_llc->num_rkeys -
862 *num_rkeys_todo;
863 *num_rkeys_todo = 0;
864 break;
865 }
866 rmb = *buf_pos;
867
868 addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
869 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
870 addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ?
871 cpu_to_be64((uintptr_t)rmb->cpu_addr) :
872 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
873
874 (*num_rkeys_todo)--;
875 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
876 }
877 addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
878 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
879 if (lgr->role == SMC_CLNT)
880 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
881 rc = smc_wr_tx_send(link, pend);
882 put_out:
883 smc_wr_tx_link_put(link);
884 return rc;
885 }
886
smc_llc_cli_rkey_exchange(struct smc_link * link,struct smc_link * link_new)887 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
888 struct smc_link *link_new)
889 {
890 struct smc_llc_msg_add_link_cont *addc_llc;
891 struct smc_link_group *lgr = link->lgr;
892 u8 max, num_rkeys_send, num_rkeys_recv;
893 struct smc_llc_qentry *qentry;
894 struct smc_buf_desc *buf_pos;
895 int buf_lst;
896 int rc = 0;
897 int i;
898
899 down_write(&lgr->rmbs_lock);
900 num_rkeys_send = lgr->conns_num;
901 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
902 do {
903 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
904 SMC_LLC_ADD_LINK_CONT);
905 if (!qentry) {
906 rc = -ETIMEDOUT;
907 break;
908 }
909 addc_llc = &qentry->msg.add_link_cont;
910 num_rkeys_recv = addc_llc->num_rkeys;
911 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
912 for (i = 0; i < max; i++) {
913 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
914 addc_llc->rt[i].rmb_key,
915 addc_llc->rt[i].rmb_vaddr_new,
916 addc_llc->rt[i].rmb_key_new);
917 num_rkeys_recv--;
918 }
919 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
920 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
921 &buf_lst, &buf_pos);
922 if (rc)
923 break;
924 } while (num_rkeys_send || num_rkeys_recv);
925
926 up_write(&lgr->rmbs_lock);
927 return rc;
928 }
929
930 /* prepare and send an add link reject response */
smc_llc_cli_add_link_reject(struct smc_llc_qentry * qentry)931 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
932 {
933 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
934 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
935 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
936 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
937 sizeof(qentry->msg));
938 return smc_llc_send_message(qentry->link, &qentry->msg);
939 }
940
smc_llc_cli_conf_link(struct smc_link * link,struct smc_init_info * ini,struct smc_link * link_new,enum smc_lgr_type lgr_new_t)941 static int smc_llc_cli_conf_link(struct smc_link *link,
942 struct smc_init_info *ini,
943 struct smc_link *link_new,
944 enum smc_lgr_type lgr_new_t)
945 {
946 struct smc_link_group *lgr = link->lgr;
947 struct smc_llc_qentry *qentry = NULL;
948 int rc = 0;
949
950 /* receive CONFIRM LINK request over RoCE fabric */
951 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
952 if (!qentry) {
953 rc = smc_llc_send_delete_link(link, link_new->link_id,
954 SMC_LLC_REQ, false,
955 SMC_LLC_DEL_LOST_PATH);
956 return -ENOLINK;
957 }
958 if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
959 /* received DELETE_LINK instead */
960 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
961 smc_llc_send_message(link, &qentry->msg);
962 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
963 return -ENOLINK;
964 }
965 smc_llc_save_peer_uid(qentry);
966 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
967
968 rc = smc_ib_modify_qp_rts(link_new);
969 if (rc) {
970 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
971 false, SMC_LLC_DEL_LOST_PATH);
972 return -ENOLINK;
973 }
974 smc_wr_remember_qp_attr(link_new);
975
976 rc = smcr_buf_reg_lgr(link_new);
977 if (rc) {
978 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
979 false, SMC_LLC_DEL_LOST_PATH);
980 return -ENOLINK;
981 }
982
983 /* send CONFIRM LINK response over RoCE fabric */
984 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
985 if (rc) {
986 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
987 false, SMC_LLC_DEL_LOST_PATH);
988 return -ENOLINK;
989 }
990 smc_llc_link_active(link_new);
991 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
992 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
993 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
994 else
995 smcr_lgr_set_type(lgr, lgr_new_t);
996 return 0;
997 }
998
smc_llc_save_add_link_rkeys(struct smc_link * link,struct smc_link * link_new)999 static void smc_llc_save_add_link_rkeys(struct smc_link *link,
1000 struct smc_link *link_new)
1001 {
1002 struct smc_llc_msg_add_link_v2_ext *ext;
1003 struct smc_link_group *lgr = link->lgr;
1004 int max, i;
1005
1006 ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +
1007 SMC_WR_TX_SIZE);
1008 max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1009 down_write(&lgr->rmbs_lock);
1010 for (i = 0; i < max; i++) {
1011 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1012 ext->rt[i].rmb_key,
1013 ext->rt[i].rmb_vaddr_new,
1014 ext->rt[i].rmb_key_new);
1015 }
1016 up_write(&lgr->rmbs_lock);
1017 }
1018
smc_llc_save_add_link_info(struct smc_link * link,struct smc_llc_msg_add_link * add_llc)1019 static void smc_llc_save_add_link_info(struct smc_link *link,
1020 struct smc_llc_msg_add_link *add_llc)
1021 {
1022 link->peer_qpn = ntoh24(add_llc->sender_qp_num);
1023 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1024 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1025 link->peer_psn = ntoh24(add_llc->initial_psn);
1026 link->peer_mtu = add_llc->qp_mtu;
1027 }
1028
1029 /* as an SMC client, process an add link request */
smc_llc_cli_add_link(struct smc_link * link,struct smc_llc_qentry * qentry)1030 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1031 {
1032 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1033 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1034 struct smc_link_group *lgr = smc_get_lgr(link);
1035 struct smc_init_info *ini = NULL;
1036 struct smc_link *lnk_new = NULL;
1037 int lnk_idx, rc = 0;
1038
1039 if (!llc->qp_mtu)
1040 goto out_reject;
1041
1042 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1043 if (!ini) {
1044 rc = -ENOMEM;
1045 goto out_reject;
1046 }
1047
1048 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) {
1049 rc = 0;
1050 goto out_reject;
1051 }
1052
1053 ini->vlan_id = lgr->vlan_id;
1054 if (lgr->smc_version == SMC_V2) {
1055 ini->check_smcrv2 = true;
1056 ini->smcrv2.saddr = lgr->saddr;
1057 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid);
1058 }
1059 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1060 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1061 (lgr->smc_version == SMC_V2 ||
1062 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1063 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
1064 goto out_reject;
1065 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1066 }
1067 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1068 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1069 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1070 ini->smcrv2.ib_port_v2 = link->ibport;
1071 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1072 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1073 ini->ib_dev = link->smcibdev;
1074 ini->ib_port = link->ibport;
1075 }
1076 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1077 if (lnk_idx < 0)
1078 goto out_reject;
1079 lnk_new = &lgr->lnk[lnk_idx];
1080 rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);
1081 if (rc)
1082 goto out_reject;
1083 smc_llc_save_add_link_info(lnk_new, llc);
1084 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */
1085 smc_llc_link_set_uid(lnk_new);
1086
1087 rc = smc_ib_ready_link(lnk_new);
1088 if (rc)
1089 goto out_clear_lnk;
1090
1091 rc = smcr_buf_map_lgr(lnk_new);
1092 if (rc)
1093 goto out_clear_lnk;
1094
1095 rc = smc_llc_send_add_link(link,
1096 lnk_new->smcibdev->mac[lnk_new->ibport - 1],
1097 lnk_new->gid, lnk_new, SMC_LLC_RESP);
1098 if (rc)
1099 goto out_clear_lnk;
1100 if (lgr->smc_version == SMC_V2) {
1101 smc_llc_save_add_link_rkeys(link, lnk_new);
1102 } else {
1103 rc = smc_llc_cli_rkey_exchange(link, lnk_new);
1104 if (rc) {
1105 rc = 0;
1106 goto out_clear_lnk;
1107 }
1108 }
1109 rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t);
1110 if (!rc)
1111 goto out;
1112 out_clear_lnk:
1113 lnk_new->state = SMC_LNK_INACTIVE;
1114 smcr_link_clear(lnk_new, false);
1115 out_reject:
1116 smc_llc_cli_add_link_reject(qentry);
1117 out:
1118 kfree(ini);
1119 kfree(qentry);
1120 return rc;
1121 }
1122
smc_llc_send_request_add_link(struct smc_link * link)1123 static void smc_llc_send_request_add_link(struct smc_link *link)
1124 {
1125 struct smc_llc_msg_req_add_link_v2 *llc;
1126 struct smc_wr_tx_pend_priv *pend;
1127 struct smc_wr_v2_buf *wr_buf;
1128 struct smc_gidlist gidlist;
1129 int rc, len, i;
1130
1131 if (!smc_wr_tx_link_hold(link))
1132 return;
1133 if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1134 link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1135 goto put_out;
1136
1137 smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);
1138 if (gidlist.len <= 1)
1139 goto put_out;
1140
1141 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
1142 if (rc)
1143 goto put_out;
1144 llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1145 memset(llc, 0, SMC_WR_TX_SIZE);
1146
1147 llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1148 for (i = 0; i < gidlist.len; i++)
1149 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1150 llc->gid_cnt = gidlist.len;
1151 len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1152 smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);
1153 rc = smc_wr_tx_v2_send(link, pend, len);
1154 if (!rc)
1155 /* set REQ_ADD_LINK flow and wait for response from peer */
1156 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1157 put_out:
1158 smc_wr_tx_link_put(link);
1159 }
1160
1161 /* as an SMC client, invite server to start the add_link processing */
smc_llc_cli_add_link_invite(struct smc_link * link,struct smc_llc_qentry * qentry)1162 static void smc_llc_cli_add_link_invite(struct smc_link *link,
1163 struct smc_llc_qentry *qentry)
1164 {
1165 struct smc_link_group *lgr = smc_get_lgr(link);
1166 struct smc_init_info *ini = NULL;
1167
1168 if (lgr->smc_version == SMC_V2) {
1169 smc_llc_send_request_add_link(link);
1170 goto out;
1171 }
1172
1173 if (lgr->type == SMC_LGR_SYMMETRIC ||
1174 lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1175 goto out;
1176
1177 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1)
1178 goto out;
1179
1180 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1181 if (!ini)
1182 goto out;
1183
1184 ini->vlan_id = lgr->vlan_id;
1185 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1186 if (!ini->ib_dev)
1187 goto out;
1188
1189 smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1],
1190 ini->ib_gid, NULL, SMC_LLC_REQ);
1191 out:
1192 kfree(ini);
1193 kfree(qentry);
1194 }
1195
smc_llc_is_empty_llc_message(union smc_llc_msg * llc)1196 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1197 {
1198 int i;
1199
1200 for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1201 if (llc->raw.data[i])
1202 return false;
1203 return true;
1204 }
1205
smc_llc_is_local_add_link(union smc_llc_msg * llc)1206 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1207 {
1208 if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
1209 smc_llc_is_empty_llc_message(llc))
1210 return true;
1211 return false;
1212 }
1213
smc_llc_process_cli_add_link(struct smc_link_group * lgr)1214 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1215 {
1216 struct smc_llc_qentry *qentry;
1217
1218 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1219
1220 down_write(&lgr->llc_conf_mutex);
1221 if (smc_llc_is_local_add_link(&qentry->msg))
1222 smc_llc_cli_add_link_invite(qentry->link, qentry);
1223 else
1224 smc_llc_cli_add_link(qentry->link, qentry);
1225 up_write(&lgr->llc_conf_mutex);
1226 }
1227
smc_llc_active_link_count(struct smc_link_group * lgr)1228 static int smc_llc_active_link_count(struct smc_link_group *lgr)
1229 {
1230 int i, link_count = 0;
1231
1232 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1233 if (!smc_link_active(&lgr->lnk[i]))
1234 continue;
1235 link_count++;
1236 }
1237 return link_count;
1238 }
1239
1240 /* find the asymmetric link when 3 links are established */
smc_llc_find_asym_link(struct smc_link_group * lgr)1241 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1242 {
1243 int asym_idx = -ENOENT;
1244 int i, j, k;
1245 bool found;
1246
1247 /* determine asymmetric link */
1248 found = false;
1249 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1250 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1251 if (!smc_link_usable(&lgr->lnk[i]) ||
1252 !smc_link_usable(&lgr->lnk[j]))
1253 continue;
1254 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
1255 SMC_GID_SIZE)) {
1256 found = true; /* asym_lnk is i or j */
1257 break;
1258 }
1259 }
1260 if (found)
1261 break;
1262 }
1263 if (!found)
1264 goto out; /* no asymmetric link */
1265 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1266 if (!smc_link_usable(&lgr->lnk[k]))
1267 continue;
1268 if (k != i &&
1269 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
1270 SMC_GID_SIZE)) {
1271 asym_idx = i;
1272 break;
1273 }
1274 if (k != j &&
1275 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1276 SMC_GID_SIZE)) {
1277 asym_idx = j;
1278 break;
1279 }
1280 }
1281 out:
1282 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1283 }
1284
smc_llc_delete_asym_link(struct smc_link_group * lgr)1285 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1286 {
1287 struct smc_link *lnk_new = NULL, *lnk_asym;
1288 struct smc_llc_qentry *qentry;
1289 int rc;
1290
1291 lnk_asym = smc_llc_find_asym_link(lgr);
1292 if (!lnk_asym)
1293 return; /* no asymmetric link */
1294 if (!smc_link_downing(&lnk_asym->state))
1295 return;
1296 lnk_new = smc_switch_conns(lgr, lnk_asym, false);
1297 smc_wr_tx_wait_no_pending_sends(lnk_asym);
1298 if (!lnk_new)
1299 goto out_free;
1300 /* change flow type from ADD_LINK into DEL_LINK */
1301 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1302 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1303 true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1304 if (rc) {
1305 smcr_link_down_cond(lnk_new);
1306 goto out_free;
1307 }
1308 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1309 SMC_LLC_DELETE_LINK);
1310 if (!qentry) {
1311 smcr_link_down_cond(lnk_new);
1312 goto out_free;
1313 }
1314 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1315 out_free:
1316 smcr_link_clear(lnk_asym, true);
1317 }
1318
smc_llc_srv_rkey_exchange(struct smc_link * link,struct smc_link * link_new)1319 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1320 struct smc_link *link_new)
1321 {
1322 struct smc_llc_msg_add_link_cont *addc_llc;
1323 struct smc_link_group *lgr = link->lgr;
1324 u8 max, num_rkeys_send, num_rkeys_recv;
1325 struct smc_llc_qentry *qentry = NULL;
1326 struct smc_buf_desc *buf_pos;
1327 int buf_lst;
1328 int rc = 0;
1329 int i;
1330
1331 down_write(&lgr->rmbs_lock);
1332 num_rkeys_send = lgr->conns_num;
1333 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1334 do {
1335 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1336 &buf_lst, &buf_pos);
1337 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1338 SMC_LLC_ADD_LINK_CONT);
1339 if (!qentry) {
1340 rc = -ETIMEDOUT;
1341 goto out;
1342 }
1343 addc_llc = &qentry->msg.add_link_cont;
1344 num_rkeys_recv = addc_llc->num_rkeys;
1345 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1346 for (i = 0; i < max; i++) {
1347 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1348 addc_llc->rt[i].rmb_key,
1349 addc_llc->rt[i].rmb_vaddr_new,
1350 addc_llc->rt[i].rmb_key_new);
1351 num_rkeys_recv--;
1352 }
1353 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1354 } while (num_rkeys_send || num_rkeys_recv);
1355 out:
1356 up_write(&lgr->rmbs_lock);
1357 return rc;
1358 }
1359
smc_llc_srv_conf_link(struct smc_link * link,struct smc_link * link_new,enum smc_lgr_type lgr_new_t)1360 static int smc_llc_srv_conf_link(struct smc_link *link,
1361 struct smc_link *link_new,
1362 enum smc_lgr_type lgr_new_t)
1363 {
1364 struct smc_link_group *lgr = link->lgr;
1365 struct smc_llc_qentry *qentry = NULL;
1366 int rc;
1367
1368 /* send CONFIRM LINK request over the RoCE fabric */
1369 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1370 if (rc)
1371 return -ENOLINK;
1372 /* receive CONFIRM LINK response over the RoCE fabric */
1373 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1374 if (!qentry ||
1375 qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1376 /* send DELETE LINK */
1377 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1378 false, SMC_LLC_DEL_LOST_PATH);
1379 if (qentry)
1380 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1381 return -ENOLINK;
1382 }
1383 smc_llc_save_peer_uid(qentry);
1384 smc_llc_link_active(link_new);
1385 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1386 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1387 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1388 else
1389 smcr_lgr_set_type(lgr, lgr_new_t);
1390 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1391 return 0;
1392 }
1393
smc_llc_send_req_add_link_response(struct smc_llc_qentry * qentry)1394 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1395 {
1396 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1397 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
1398 sizeof(qentry->msg));
1399 memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1400 smc_llc_send_message(qentry->link, &qentry->msg);
1401 }
1402
smc_llc_srv_add_link(struct smc_link * link,struct smc_llc_qentry * req_qentry)1403 int smc_llc_srv_add_link(struct smc_link *link,
1404 struct smc_llc_qentry *req_qentry)
1405 {
1406 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1407 struct smc_link_group *lgr = link->lgr;
1408 struct smc_llc_msg_add_link *add_llc;
1409 struct smc_llc_qentry *qentry = NULL;
1410 bool send_req_add_link_resp = false;
1411 struct smc_link *link_new = NULL;
1412 struct smc_init_info *ini = NULL;
1413 int lnk_idx, rc = 0;
1414
1415 if (req_qentry &&
1416 req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1417 send_req_add_link_resp = true;
1418
1419 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1420 if (!ini) {
1421 rc = -ENOMEM;
1422 goto out;
1423 }
1424
1425 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) {
1426 rc = 0;
1427 goto out;
1428 }
1429
1430 /* ignore client add link recommendation, start new flow */
1431 ini->vlan_id = lgr->vlan_id;
1432 if (lgr->smc_version == SMC_V2) {
1433 ini->check_smcrv2 = true;
1434 ini->smcrv2.saddr = lgr->saddr;
1435 if (send_req_add_link_resp) {
1436 struct smc_llc_msg_req_add_link_v2 *req_add =
1437 &req_qentry->msg.req_add_link;
1438
1439 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]);
1440 }
1441 }
1442 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1443 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1444 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1445 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1446 ini->smcrv2.ib_port_v2 = link->ibport;
1447 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1448 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1449 ini->ib_dev = link->smcibdev;
1450 ini->ib_port = link->ibport;
1451 }
1452 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1453 if (lnk_idx < 0) {
1454 rc = 0;
1455 goto out;
1456 }
1457
1458 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);
1459 if (rc)
1460 goto out;
1461 link_new = &lgr->lnk[lnk_idx];
1462
1463 rc = smcr_buf_map_lgr(link_new);
1464 if (rc)
1465 goto out_err;
1466
1467 rc = smc_llc_send_add_link(link,
1468 link_new->smcibdev->mac[link_new->ibport-1],
1469 link_new->gid, link_new, SMC_LLC_REQ);
1470 if (rc)
1471 goto out_err;
1472 send_req_add_link_resp = false;
1473 /* receive ADD LINK response over the RoCE fabric */
1474 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1475 if (!qentry) {
1476 rc = -ETIMEDOUT;
1477 goto out_err;
1478 }
1479 add_llc = &qentry->msg.add_link;
1480 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1481 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1482 rc = -ENOLINK;
1483 goto out_err;
1484 }
1485 if (lgr->type == SMC_LGR_SINGLE &&
1486 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1487 (lgr->smc_version == SMC_V2 ||
1488 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) {
1489 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1490 }
1491 smc_llc_save_add_link_info(link_new, add_llc);
1492 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1493
1494 rc = smc_ib_ready_link(link_new);
1495 if (rc)
1496 goto out_err;
1497 rc = smcr_buf_reg_lgr(link_new);
1498 if (rc)
1499 goto out_err;
1500 if (lgr->smc_version == SMC_V2) {
1501 smc_llc_save_add_link_rkeys(link, link_new);
1502 } else {
1503 rc = smc_llc_srv_rkey_exchange(link, link_new);
1504 if (rc)
1505 goto out_err;
1506 }
1507 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1508 if (rc)
1509 goto out_err;
1510 kfree(ini);
1511 return 0;
1512 out_err:
1513 if (link_new) {
1514 link_new->state = SMC_LNK_INACTIVE;
1515 smcr_link_clear(link_new, false);
1516 }
1517 out:
1518 kfree(ini);
1519 if (send_req_add_link_resp)
1520 smc_llc_send_req_add_link_response(req_qentry);
1521 return rc;
1522 }
1523
smc_llc_process_srv_add_link(struct smc_link_group * lgr)1524 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1525 {
1526 struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1527 struct smc_llc_qentry *qentry;
1528 int rc;
1529
1530 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1531
1532 down_write(&lgr->llc_conf_mutex);
1533 rc = smc_llc_srv_add_link(link, qentry);
1534 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1535 /* delete any asymmetric link */
1536 smc_llc_delete_asym_link(lgr);
1537 }
1538 up_write(&lgr->llc_conf_mutex);
1539 kfree(qentry);
1540 }
1541
1542 /* enqueue a local add_link req to trigger a new add_link flow */
smc_llc_add_link_local(struct smc_link * link)1543 void smc_llc_add_link_local(struct smc_link *link)
1544 {
1545 struct smc_llc_msg_add_link add_llc = {};
1546
1547 add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1548 smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));
1549 /* no dev and port needed */
1550 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1551 }
1552
1553 /* worker to process an add link message */
smc_llc_add_link_work(struct work_struct * work)1554 static void smc_llc_add_link_work(struct work_struct *work)
1555 {
1556 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1557 llc_add_link_work);
1558
1559 if (list_empty(&lgr->list)) {
1560 /* link group is terminating */
1561 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1562 goto out;
1563 }
1564
1565 if (lgr->role == SMC_CLNT)
1566 smc_llc_process_cli_add_link(lgr);
1567 else
1568 smc_llc_process_srv_add_link(lgr);
1569 out:
1570 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1571 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1572 }
1573
1574 /* enqueue a local del_link msg to trigger a new del_link flow,
1575 * called only for role SMC_SERV
1576 */
smc_llc_srv_delete_link_local(struct smc_link * link,u8 del_link_id)1577 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1578 {
1579 struct smc_llc_msg_del_link del_llc = {};
1580
1581 del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1582 smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));
1583 del_llc.link_num = del_link_id;
1584 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1585 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1586 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1587 }
1588
smc_llc_process_cli_delete_link(struct smc_link_group * lgr)1589 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1590 {
1591 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1592 struct smc_llc_msg_del_link *del_llc;
1593 struct smc_llc_qentry *qentry;
1594 int active_links;
1595 int lnk_idx;
1596
1597 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1598 lnk = qentry->link;
1599 del_llc = &qentry->msg.delete_link;
1600
1601 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1602 smc_lgr_terminate_sched(lgr);
1603 goto out;
1604 }
1605 down_write(&lgr->llc_conf_mutex);
1606 /* delete single link */
1607 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1608 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1609 continue;
1610 lnk_del = &lgr->lnk[lnk_idx];
1611 break;
1612 }
1613 del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1614 if (!lnk_del) {
1615 /* link was not found */
1616 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1617 smc_llc_send_message(lnk, &qentry->msg);
1618 goto out_unlock;
1619 }
1620 lnk_asym = smc_llc_find_asym_link(lgr);
1621
1622 del_llc->reason = 0;
1623 smc_llc_send_message(lnk, &qentry->msg); /* response */
1624
1625 if (smc_link_downing(&lnk_del->state))
1626 smc_switch_conns(lgr, lnk_del, false);
1627 smcr_link_clear(lnk_del, true);
1628
1629 active_links = smc_llc_active_link_count(lgr);
1630 if (lnk_del == lnk_asym) {
1631 /* expected deletion of asym link, don't change lgr state */
1632 } else if (active_links == 1) {
1633 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1634 } else if (!active_links) {
1635 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1636 smc_lgr_terminate_sched(lgr);
1637 }
1638 out_unlock:
1639 up_write(&lgr->llc_conf_mutex);
1640 out:
1641 kfree(qentry);
1642 }
1643
1644 /* try to send a DELETE LINK ALL request on any active link,
1645 * waiting for send completion
1646 */
smc_llc_send_link_delete_all(struct smc_link_group * lgr,bool ord,u32 rsn)1647 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1648 {
1649 struct smc_llc_msg_del_link delllc = {};
1650 int i;
1651
1652 delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1653 smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));
1654 if (ord)
1655 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1656 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1657 delllc.reason = htonl(rsn);
1658
1659 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1660 if (!smc_link_sendable(&lgr->lnk[i]))
1661 continue;
1662 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1663 break;
1664 }
1665 }
1666
smc_llc_process_srv_delete_link(struct smc_link_group * lgr)1667 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1668 {
1669 struct smc_llc_msg_del_link *del_llc;
1670 struct smc_link *lnk, *lnk_del;
1671 struct smc_llc_qentry *qentry;
1672 int active_links;
1673 int i;
1674
1675 down_write(&lgr->llc_conf_mutex);
1676 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1677 lnk = qentry->link;
1678 del_llc = &qentry->msg.delete_link;
1679
1680 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1681 /* delete entire lgr */
1682 smc_llc_send_link_delete_all(lgr, true, ntohl(
1683 qentry->msg.delete_link.reason));
1684 smc_lgr_terminate_sched(lgr);
1685 goto out;
1686 }
1687 /* delete single link */
1688 lnk_del = NULL;
1689 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1690 if (lgr->lnk[i].link_id == del_llc->link_num) {
1691 lnk_del = &lgr->lnk[i];
1692 break;
1693 }
1694 }
1695 if (!lnk_del)
1696 goto out; /* asymmetric link already deleted */
1697
1698 if (smc_link_downing(&lnk_del->state)) {
1699 if (smc_switch_conns(lgr, lnk_del, false))
1700 smc_wr_tx_wait_no_pending_sends(lnk_del);
1701 }
1702 if (!list_empty(&lgr->list)) {
1703 /* qentry is either a request from peer (send it back to
1704 * initiate the DELETE_LINK processing), or a locally
1705 * enqueued DELETE_LINK request (forward it)
1706 */
1707 if (!smc_llc_send_message(lnk, &qentry->msg)) {
1708 struct smc_llc_qentry *qentry2;
1709
1710 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1711 SMC_LLC_DELETE_LINK);
1712 if (qentry2)
1713 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1714 }
1715 }
1716 smcr_link_clear(lnk_del, true);
1717
1718 active_links = smc_llc_active_link_count(lgr);
1719 if (active_links == 1) {
1720 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1721 } else if (!active_links) {
1722 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1723 smc_lgr_terminate_sched(lgr);
1724 }
1725
1726 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1727 /* trigger setup of asymm alt link */
1728 smc_llc_add_link_local(lnk);
1729 }
1730 out:
1731 up_write(&lgr->llc_conf_mutex);
1732 kfree(qentry);
1733 }
1734
smc_llc_delete_link_work(struct work_struct * work)1735 static void smc_llc_delete_link_work(struct work_struct *work)
1736 {
1737 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1738 llc_del_link_work);
1739
1740 if (list_empty(&lgr->list)) {
1741 /* link group is terminating */
1742 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1743 goto out;
1744 }
1745
1746 if (lgr->role == SMC_CLNT)
1747 smc_llc_process_cli_delete_link(lgr);
1748 else
1749 smc_llc_process_srv_delete_link(lgr);
1750 out:
1751 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1752 }
1753
1754 /* process a confirm_rkey request from peer, remote flow */
smc_llc_rmt_conf_rkey(struct smc_link_group * lgr)1755 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1756 {
1757 struct smc_llc_msg_confirm_rkey *llc;
1758 struct smc_llc_qentry *qentry;
1759 struct smc_link *link;
1760 int num_entries;
1761 int rk_idx;
1762 int i;
1763
1764 qentry = lgr->llc_flow_rmt.qentry;
1765 llc = &qentry->msg.confirm_rkey;
1766 link = qentry->link;
1767
1768 num_entries = llc->rtoken[0].num_rkeys;
1769 if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1770 goto out_err;
1771 /* first rkey entry is for receiving link */
1772 rk_idx = smc_rtoken_add(link,
1773 llc->rtoken[0].rmb_vaddr,
1774 llc->rtoken[0].rmb_key);
1775 if (rk_idx < 0)
1776 goto out_err;
1777
1778 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1779 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1780 llc->rtoken[i].rmb_vaddr,
1781 llc->rtoken[i].rmb_key);
1782 /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1783 goto out;
1784 out_err:
1785 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1786 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1787 out:
1788 llc->hd.flags |= SMC_LLC_FLAG_RESP;
1789 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1790 smc_llc_send_message(link, &qentry->msg);
1791 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1792 }
1793
1794 /* process a delete_rkey request from peer, remote flow */
smc_llc_rmt_delete_rkey(struct smc_link_group * lgr)1795 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1796 {
1797 struct smc_llc_msg_delete_rkey *llc;
1798 struct smc_llc_qentry *qentry;
1799 struct smc_link *link;
1800 u8 err_mask = 0;
1801 int i, max;
1802
1803 qentry = lgr->llc_flow_rmt.qentry;
1804 llc = &qentry->msg.delete_rkey;
1805 link = qentry->link;
1806
1807 if (lgr->smc_version == SMC_V2) {
1808 struct smc_llc_msg_delete_rkey_v2 *llcv2;
1809
1810 memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));
1811 llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;
1812 llcv2->num_inval_rkeys = 0;
1813
1814 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1815 for (i = 0; i < max; i++) {
1816 if (smc_rtoken_delete(link, llcv2->rkey[i]))
1817 llcv2->num_inval_rkeys++;
1818 }
1819 memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1820 memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1821 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1822 if (llcv2->num_inval_rkeys) {
1823 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1824 llc->err_mask = llcv2->num_inval_rkeys;
1825 }
1826 goto finish;
1827 }
1828
1829 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1830 for (i = 0; i < max; i++) {
1831 if (smc_rtoken_delete(link, llc->rkey[i]))
1832 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1833 }
1834 if (err_mask) {
1835 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1836 llc->err_mask = err_mask;
1837 }
1838 finish:
1839 llc->hd.flags |= SMC_LLC_FLAG_RESP;
1840 smc_llc_send_message(link, &qentry->msg);
1841 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1842 }
1843
smc_llc_protocol_violation(struct smc_link_group * lgr,u8 type)1844 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1845 {
1846 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: "
1847 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,
1848 lgr->net->net_cookie, type);
1849 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1850 smc_lgr_terminate_sched(lgr);
1851 }
1852
1853 /* flush the llc event queue */
smc_llc_event_flush(struct smc_link_group * lgr)1854 static void smc_llc_event_flush(struct smc_link_group *lgr)
1855 {
1856 struct smc_llc_qentry *qentry, *q;
1857
1858 spin_lock_bh(&lgr->llc_event_q_lock);
1859 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1860 list_del_init(&qentry->list);
1861 kfree(qentry);
1862 }
1863 spin_unlock_bh(&lgr->llc_event_q_lock);
1864 }
1865
smc_llc_event_handler(struct smc_llc_qentry * qentry)1866 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1867 {
1868 union smc_llc_msg *llc = &qentry->msg;
1869 struct smc_link *link = qentry->link;
1870 struct smc_link_group *lgr = link->lgr;
1871
1872 if (!smc_link_usable(link))
1873 goto out;
1874
1875 switch (llc->raw.hdr.common.llc_type) {
1876 case SMC_LLC_TEST_LINK:
1877 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1878 smc_llc_send_message(link, llc);
1879 break;
1880 case SMC_LLC_ADD_LINK:
1881 if (list_empty(&lgr->list))
1882 goto out; /* lgr is terminating */
1883 if (lgr->role == SMC_CLNT) {
1884 if (smc_llc_is_local_add_link(llc)) {
1885 if (lgr->llc_flow_lcl.type ==
1886 SMC_LLC_FLOW_ADD_LINK)
1887 break; /* add_link in progress */
1888 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1889 qentry)) {
1890 schedule_work(&lgr->llc_add_link_work);
1891 }
1892 return;
1893 }
1894 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1895 !lgr->llc_flow_lcl.qentry) {
1896 /* a flow is waiting for this message */
1897 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1898 qentry);
1899 wake_up(&lgr->llc_msg_waiter);
1900 return;
1901 }
1902 if (lgr->llc_flow_lcl.type ==
1903 SMC_LLC_FLOW_REQ_ADD_LINK) {
1904 /* server started add_link processing */
1905 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1906 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1907 qentry);
1908 schedule_work(&lgr->llc_add_link_work);
1909 return;
1910 }
1911 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1912 schedule_work(&lgr->llc_add_link_work);
1913 }
1914 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1915 /* as smc server, handle client suggestion */
1916 schedule_work(&lgr->llc_add_link_work);
1917 }
1918 return;
1919 case SMC_LLC_CONFIRM_LINK:
1920 case SMC_LLC_ADD_LINK_CONT:
1921 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1922 /* a flow is waiting for this message */
1923 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1924 wake_up(&lgr->llc_msg_waiter);
1925 return;
1926 }
1927 break;
1928 case SMC_LLC_DELETE_LINK:
1929 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1930 !lgr->llc_flow_lcl.qentry) {
1931 /* DEL LINK REQ during ADD LINK SEQ */
1932 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1933 wake_up(&lgr->llc_msg_waiter);
1934 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1935 schedule_work(&lgr->llc_del_link_work);
1936 }
1937 return;
1938 case SMC_LLC_CONFIRM_RKEY:
1939 /* new request from remote, assign to remote flow */
1940 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1941 /* process here, does not wait for more llc msgs */
1942 smc_llc_rmt_conf_rkey(lgr);
1943 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1944 }
1945 return;
1946 case SMC_LLC_CONFIRM_RKEY_CONT:
1947 /* not used because max links is 3, and 3 rkeys fit into
1948 * one CONFIRM_RKEY message
1949 */
1950 break;
1951 case SMC_LLC_DELETE_RKEY:
1952 /* new request from remote, assign to remote flow */
1953 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1954 /* process here, does not wait for more llc msgs */
1955 smc_llc_rmt_delete_rkey(lgr);
1956 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1957 }
1958 return;
1959 case SMC_LLC_REQ_ADD_LINK:
1960 /* handle response here, smc_llc_flow_stop() cannot be called
1961 * in tasklet context
1962 */
1963 if (lgr->role == SMC_CLNT &&
1964 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
1965 (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
1966 smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);
1967 } else if (lgr->role == SMC_SERV) {
1968 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1969 /* as smc server, handle client suggestion */
1970 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1971 schedule_work(&lgr->llc_add_link_work);
1972 }
1973 return;
1974 }
1975 break;
1976 default:
1977 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1978 break;
1979 }
1980 out:
1981 kfree(qentry);
1982 }
1983
1984 /* worker to process llc messages on the event queue */
smc_llc_event_work(struct work_struct * work)1985 static void smc_llc_event_work(struct work_struct *work)
1986 {
1987 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1988 llc_event_work);
1989 struct smc_llc_qentry *qentry;
1990
1991 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1992 qentry = lgr->delayed_event;
1993 lgr->delayed_event = NULL;
1994 if (smc_link_usable(qentry->link))
1995 smc_llc_event_handler(qentry);
1996 else
1997 kfree(qentry);
1998 }
1999
2000 again:
2001 spin_lock_bh(&lgr->llc_event_q_lock);
2002 if (!list_empty(&lgr->llc_event_q)) {
2003 qentry = list_first_entry(&lgr->llc_event_q,
2004 struct smc_llc_qentry, list);
2005 list_del_init(&qentry->list);
2006 spin_unlock_bh(&lgr->llc_event_q_lock);
2007 smc_llc_event_handler(qentry);
2008 goto again;
2009 }
2010 spin_unlock_bh(&lgr->llc_event_q_lock);
2011 }
2012
2013 /* process llc responses in tasklet context */
smc_llc_rx_response(struct smc_link * link,struct smc_llc_qentry * qentry)2014 static void smc_llc_rx_response(struct smc_link *link,
2015 struct smc_llc_qentry *qentry)
2016 {
2017 enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
2018 struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
2019 u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
2020
2021 switch (llc_type) {
2022 case SMC_LLC_TEST_LINK:
2023 if (smc_link_active(link))
2024 complete(&link->llc_testlink_resp);
2025 break;
2026 case SMC_LLC_ADD_LINK:
2027 case SMC_LLC_ADD_LINK_CONT:
2028 case SMC_LLC_CONFIRM_LINK:
2029 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2030 break; /* drop out-of-flow response */
2031 goto assign;
2032 case SMC_LLC_DELETE_LINK:
2033 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2034 break; /* drop out-of-flow response */
2035 goto assign;
2036 case SMC_LLC_CONFIRM_RKEY:
2037 case SMC_LLC_DELETE_RKEY:
2038 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2039 break; /* drop out-of-flow response */
2040 goto assign;
2041 case SMC_LLC_CONFIRM_RKEY_CONT:
2042 /* not used because max links is 3 */
2043 break;
2044 default:
2045 smc_llc_protocol_violation(link->lgr,
2046 qentry->msg.raw.hdr.common.type);
2047 break;
2048 }
2049 kfree(qentry);
2050 return;
2051 assign:
2052 /* assign responses to the local flow, we requested them */
2053 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
2054 wake_up(&link->lgr->llc_msg_waiter);
2055 }
2056
smc_llc_enqueue(struct smc_link * link,union smc_llc_msg * llc)2057 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
2058 {
2059 struct smc_link_group *lgr = link->lgr;
2060 struct smc_llc_qentry *qentry;
2061 unsigned long flags;
2062
2063 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
2064 if (!qentry)
2065 return;
2066 qentry->link = link;
2067 INIT_LIST_HEAD(&qentry->list);
2068 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
2069
2070 /* process responses immediately */
2071 if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2072 llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
2073 smc_llc_rx_response(link, qentry);
2074 return;
2075 }
2076
2077 /* add requests to event queue */
2078 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2079 list_add_tail(&qentry->list, &lgr->llc_event_q);
2080 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
2081 queue_work(system_highpri_wq, &lgr->llc_event_work);
2082 }
2083
2084 /* copy received msg and add it to the event queue */
smc_llc_rx_handler(struct ib_wc * wc,void * buf)2085 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2086 {
2087 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2088 union smc_llc_msg *llc = buf;
2089
2090 if (wc->byte_len < sizeof(*llc))
2091 return; /* short message */
2092 if (!llc->raw.hdr.common.llc_version) {
2093 if (llc->raw.hdr.length != sizeof(*llc))
2094 return; /* invalid message */
2095 } else {
2096 if (llc->raw.hdr.length_v2 < sizeof(*llc))
2097 return; /* invalid message */
2098 }
2099
2100 smc_llc_enqueue(link, llc);
2101 }
2102
2103 /***************************** worker, utils *********************************/
2104
smc_llc_testlink_work(struct work_struct * work)2105 static void smc_llc_testlink_work(struct work_struct *work)
2106 {
2107 struct smc_link *link = container_of(to_delayed_work(work),
2108 struct smc_link, llc_testlink_wrk);
2109 unsigned long next_interval;
2110 unsigned long expire_time;
2111 u8 user_data[16] = { 0 };
2112 int rc;
2113
2114 if (!smc_link_active(link))
2115 return; /* don't reschedule worker */
2116 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2117 if (time_is_after_jiffies(expire_time)) {
2118 next_interval = expire_time - jiffies;
2119 goto out;
2120 }
2121 reinit_completion(&link->llc_testlink_resp);
2122 smc_llc_send_test_link(link, user_data);
2123 /* receive TEST LINK response over RoCE fabric */
2124 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
2125 SMC_LLC_WAIT_TIME);
2126 if (!smc_link_active(link))
2127 return; /* link state changed */
2128 if (rc <= 0) {
2129 smcr_link_down_cond_sched(link);
2130 return;
2131 }
2132 next_interval = link->llc_testlink_time;
2133 out:
2134 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
2135 }
2136
smc_llc_lgr_init(struct smc_link_group * lgr,struct smc_sock * smc)2137 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2138 {
2139 struct net *net = sock_net(smc->clcsock->sk);
2140
2141 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
2142 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
2143 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
2144 INIT_LIST_HEAD(&lgr->llc_event_q);
2145 spin_lock_init(&lgr->llc_event_q_lock);
2146 spin_lock_init(&lgr->llc_flow_lock);
2147 init_waitqueue_head(&lgr->llc_flow_waiter);
2148 init_waitqueue_head(&lgr->llc_msg_waiter);
2149 init_rwsem(&lgr->llc_conf_mutex);
2150 lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
2151 }
2152
2153 /* called after lgr was removed from lgr_list */
smc_llc_lgr_clear(struct smc_link_group * lgr)2154 void smc_llc_lgr_clear(struct smc_link_group *lgr)
2155 {
2156 smc_llc_event_flush(lgr);
2157 wake_up_all(&lgr->llc_flow_waiter);
2158 wake_up_all(&lgr->llc_msg_waiter);
2159 cancel_work_sync(&lgr->llc_event_work);
2160 cancel_work_sync(&lgr->llc_add_link_work);
2161 cancel_work_sync(&lgr->llc_del_link_work);
2162 if (lgr->delayed_event) {
2163 kfree(lgr->delayed_event);
2164 lgr->delayed_event = NULL;
2165 }
2166 }
2167
smc_llc_link_init(struct smc_link * link)2168 int smc_llc_link_init(struct smc_link *link)
2169 {
2170 init_completion(&link->llc_testlink_resp);
2171 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2172 return 0;
2173 }
2174
smc_llc_link_active(struct smc_link * link)2175 void smc_llc_link_active(struct smc_link *link)
2176 {
2177 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, "
2178 "peerid %*phN, ibdev %s, ibport %d\n",
2179 SMC_LGR_ID_SIZE, &link->lgr->id,
2180 link->lgr->net->net_cookie,
2181 SMC_LGR_ID_SIZE, &link->link_uid,
2182 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2183 link->smcibdev->ibdev->name, link->ibport);
2184 link->state = SMC_LNK_ACTIVE;
2185 if (link->lgr->llc_testlink_time) {
2186 link->llc_testlink_time = link->lgr->llc_testlink_time;
2187 schedule_delayed_work(&link->llc_testlink_wrk,
2188 link->llc_testlink_time);
2189 }
2190 }
2191
2192 /* called in worker context */
smc_llc_link_clear(struct smc_link * link,bool log)2193 void smc_llc_link_clear(struct smc_link *link, bool log)
2194 {
2195 if (log)
2196 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN"
2197 ", peerid %*phN, ibdev %s, ibport %d\n",
2198 SMC_LGR_ID_SIZE, &link->lgr->id,
2199 link->lgr->net->net_cookie,
2200 SMC_LGR_ID_SIZE, &link->link_uid,
2201 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2202 link->smcibdev->ibdev->name, link->ibport);
2203 complete(&link->llc_testlink_resp);
2204 cancel_delayed_work_sync(&link->llc_testlink_wrk);
2205 }
2206
2207 /* register a new rtoken at the remote peer (for all links) */
smc_llc_do_confirm_rkey(struct smc_link * send_link,struct smc_buf_desc * rmb_desc)2208 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
2209 struct smc_buf_desc *rmb_desc)
2210 {
2211 struct smc_link_group *lgr = send_link->lgr;
2212 struct smc_llc_qentry *qentry = NULL;
2213 int rc = 0;
2214
2215 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2216 if (rc)
2217 goto out;
2218 /* receive CONFIRM RKEY response from server over RoCE fabric */
2219 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2220 SMC_LLC_CONFIRM_RKEY);
2221 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2222 rc = -EFAULT;
2223 out:
2224 if (qentry)
2225 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2226 return rc;
2227 }
2228
2229 /* unregister an rtoken at the remote peer */
smc_llc_do_delete_rkey(struct smc_link_group * lgr,struct smc_buf_desc * rmb_desc)2230 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
2231 struct smc_buf_desc *rmb_desc)
2232 {
2233 struct smc_llc_qentry *qentry = NULL;
2234 struct smc_link *send_link;
2235 int rc = 0;
2236
2237 send_link = smc_llc_usable_link(lgr);
2238 if (!send_link)
2239 return -ENOLINK;
2240
2241 /* protected by llc_flow control */
2242 rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
2243 if (rc)
2244 goto out;
2245 /* receive DELETE RKEY response from server over RoCE fabric */
2246 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2247 SMC_LLC_DELETE_RKEY);
2248 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2249 rc = -EFAULT;
2250 out:
2251 if (qentry)
2252 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2253 return rc;
2254 }
2255
smc_llc_link_set_uid(struct smc_link * link)2256 void smc_llc_link_set_uid(struct smc_link *link)
2257 {
2258 __be32 link_uid;
2259
2260 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2261 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2262 }
2263
2264 /* save peers link user id, used for debug purposes */
smc_llc_save_peer_uid(struct smc_llc_qentry * qentry)2265 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2266 {
2267 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2268 SMC_LGR_ID_SIZE);
2269 }
2270
2271 /* evaluate confirm link request or response */
smc_llc_eval_conf_link(struct smc_llc_qentry * qentry,enum smc_llc_reqresp type)2272 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2273 enum smc_llc_reqresp type)
2274 {
2275 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */
2276 qentry->link->link_id = qentry->msg.confirm_link.link_num;
2277 smc_llc_link_set_uid(qentry->link);
2278 }
2279 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2280 return -ENOTSUPP;
2281 return 0;
2282 }
2283
2284 /***************************** init, exit, misc ******************************/
2285
2286 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2287 {
2288 .handler = smc_llc_rx_handler,
2289 .type = SMC_LLC_CONFIRM_LINK
2290 },
2291 {
2292 .handler = smc_llc_rx_handler,
2293 .type = SMC_LLC_TEST_LINK
2294 },
2295 {
2296 .handler = smc_llc_rx_handler,
2297 .type = SMC_LLC_ADD_LINK
2298 },
2299 {
2300 .handler = smc_llc_rx_handler,
2301 .type = SMC_LLC_ADD_LINK_CONT
2302 },
2303 {
2304 .handler = smc_llc_rx_handler,
2305 .type = SMC_LLC_DELETE_LINK
2306 },
2307 {
2308 .handler = smc_llc_rx_handler,
2309 .type = SMC_LLC_CONFIRM_RKEY
2310 },
2311 {
2312 .handler = smc_llc_rx_handler,
2313 .type = SMC_LLC_CONFIRM_RKEY_CONT
2314 },
2315 {
2316 .handler = smc_llc_rx_handler,
2317 .type = SMC_LLC_DELETE_RKEY
2318 },
2319 /* V2 types */
2320 {
2321 .handler = smc_llc_rx_handler,
2322 .type = SMC_LLC_CONFIRM_LINK_V2
2323 },
2324 {
2325 .handler = smc_llc_rx_handler,
2326 .type = SMC_LLC_TEST_LINK_V2
2327 },
2328 {
2329 .handler = smc_llc_rx_handler,
2330 .type = SMC_LLC_ADD_LINK_V2
2331 },
2332 {
2333 .handler = smc_llc_rx_handler,
2334 .type = SMC_LLC_DELETE_LINK_V2
2335 },
2336 {
2337 .handler = smc_llc_rx_handler,
2338 .type = SMC_LLC_REQ_ADD_LINK_V2
2339 },
2340 {
2341 .handler = smc_llc_rx_handler,
2342 .type = SMC_LLC_CONFIRM_RKEY_V2
2343 },
2344 {
2345 .handler = smc_llc_rx_handler,
2346 .type = SMC_LLC_DELETE_RKEY_V2
2347 },
2348 {
2349 .handler = NULL,
2350 }
2351 };
2352
smc_llc_init(void)2353 int __init smc_llc_init(void)
2354 {
2355 struct smc_wr_rx_handler *handler;
2356 int rc = 0;
2357
2358 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2359 INIT_HLIST_NODE(&handler->list);
2360 rc = smc_wr_rx_register_handler(handler);
2361 if (rc)
2362 break;
2363 }
2364 return rc;
2365 }
2366