1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13 #include <net/tcp.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "smc.h" 17 #include "smc_core.h" 18 #include "smc_clc.h" 19 #include "smc_llc.h" 20 #include "smc_pnet.h" 21 22 #define SMC_LLC_DATA_LEN 40 23 24 struct smc_llc_hdr { 25 struct smc_wr_rx_hdr common; 26 u8 length; /* 44 */ 27 #if defined(__BIG_ENDIAN_BITFIELD) 28 u8 reserved:4, 29 add_link_rej_rsn:4; 30 #elif defined(__LITTLE_ENDIAN_BITFIELD) 31 u8 add_link_rej_rsn:4, 32 reserved:4; 33 #endif 34 u8 flags; 35 }; 36 37 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 38 39 struct smc_llc_msg_confirm_link { /* type 0x01 */ 40 struct smc_llc_hdr hd; 41 u8 sender_mac[ETH_ALEN]; 42 u8 sender_gid[SMC_GID_SIZE]; 43 u8 sender_qp_num[3]; 44 u8 link_num; 45 u8 link_uid[SMC_LGR_ID_SIZE]; 46 u8 max_links; 47 u8 reserved[9]; 48 }; 49 50 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 51 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 52 53 #define SMC_LLC_ADD_LNK_MAX_LINKS 2 54 55 struct smc_llc_msg_add_link { /* type 0x02 */ 56 struct smc_llc_hdr hd; 57 u8 sender_mac[ETH_ALEN]; 58 u8 reserved2[2]; 59 u8 sender_gid[SMC_GID_SIZE]; 60 u8 sender_qp_num[3]; 61 u8 link_num; 62 #if defined(__BIG_ENDIAN_BITFIELD) 63 u8 reserved3 : 4, 64 qp_mtu : 4; 65 #elif defined(__LITTLE_ENDIAN_BITFIELD) 66 u8 qp_mtu : 4, 67 reserved3 : 4; 68 #endif 69 u8 initial_psn[3]; 70 u8 reserved[8]; 71 }; 72 73 struct smc_llc_msg_add_link_cont_rt { 74 __be32 rmb_key; 75 __be32 rmb_key_new; 76 __be64 rmb_vaddr_new; 77 }; 78 79 #define SMC_LLC_RKEYS_PER_CONT_MSG 2 80 81 struct smc_llc_msg_add_link_cont { /* type 0x03 */ 82 struct smc_llc_hdr hd; 83 u8 link_num; 84 u8 num_rkeys; 85 u8 reserved2[2]; 86 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 87 u8 reserved[4]; 88 } __packed; /* format defined in RFC7609 */ 89 90 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 91 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 92 93 struct smc_llc_msg_del_link { /* type 0x04 */ 94 struct smc_llc_hdr hd; 95 u8 link_num; 96 __be32 reason; 97 u8 reserved[35]; 98 } __packed; /* format defined in RFC7609 */ 99 100 struct smc_llc_msg_test_link { /* type 0x07 */ 101 struct smc_llc_hdr hd; 102 u8 user_data[16]; 103 u8 reserved[24]; 104 }; 105 106 struct smc_rmb_rtoken { 107 union { 108 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 109 /* is actually the num of rtokens, first */ 110 /* rtoken is always for the current link */ 111 u8 link_id; /* link id of the rtoken */ 112 }; 113 __be32 rmb_key; 114 __be64 rmb_vaddr; 115 } __packed; /* format defined in RFC7609 */ 116 117 #define SMC_LLC_RKEYS_PER_MSG 3 118 119 struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 120 struct smc_llc_hdr hd; 121 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 122 u8 reserved; 123 }; 124 125 #define SMC_LLC_DEL_RKEY_MAX 8 126 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 127 #define SMC_LLC_FLAG_RKEY_NEG 0x20 128 129 struct smc_llc_msg_delete_rkey { /* type 0x09 */ 130 struct smc_llc_hdr hd; 131 u8 num_rkeys; 132 u8 err_mask; 133 u8 reserved[2]; 134 __be32 rkey[8]; 135 u8 reserved2[4]; 136 }; 137 138 union smc_llc_msg { 139 struct smc_llc_msg_confirm_link confirm_link; 140 struct smc_llc_msg_add_link add_link; 141 struct smc_llc_msg_add_link_cont add_link_cont; 142 struct smc_llc_msg_del_link delete_link; 143 144 struct smc_llc_msg_confirm_rkey confirm_rkey; 145 struct smc_llc_msg_delete_rkey delete_rkey; 146 147 struct smc_llc_msg_test_link test_link; 148 struct { 149 struct smc_llc_hdr hdr; 150 u8 data[SMC_LLC_DATA_LEN]; 151 } raw; 152 }; 153 154 #define SMC_LLC_FLAG_RESP 0x80 155 156 struct smc_llc_qentry { 157 struct list_head list; 158 struct smc_link *link; 159 union smc_llc_msg msg; 160 }; 161 162 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 163 164 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 165 { 166 struct smc_llc_qentry *qentry = flow->qentry; 167 168 flow->qentry = NULL; 169 return qentry; 170 } 171 172 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 173 { 174 struct smc_llc_qentry *qentry; 175 176 if (flow->qentry) { 177 qentry = flow->qentry; 178 flow->qentry = NULL; 179 kfree(qentry); 180 } 181 } 182 183 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 184 struct smc_llc_qentry *qentry) 185 { 186 flow->qentry = qentry; 187 } 188 189 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 190 struct smc_llc_qentry *qentry) 191 { 192 u8 msg_type = qentry->msg.raw.hdr.common.type; 193 194 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 195 flow_type != msg_type && !lgr->delayed_event) { 196 lgr->delayed_event = qentry; 197 return; 198 } 199 /* drop parallel or already-in-progress llc requests */ 200 if (flow_type != msg_type) 201 pr_warn_once("smc: SMC-R lg %*phN dropped parallel " 202 "LLC msg: msg %d flow %d role %d\n", 203 SMC_LGR_ID_SIZE, &lgr->id, 204 qentry->msg.raw.hdr.common.type, 205 flow_type, lgr->role); 206 kfree(qentry); 207 } 208 209 /* try to start a new llc flow, initiated by an incoming llc msg */ 210 static bool smc_llc_flow_start(struct smc_llc_flow *flow, 211 struct smc_llc_qentry *qentry) 212 { 213 struct smc_link_group *lgr = qentry->link->lgr; 214 215 spin_lock_bh(&lgr->llc_flow_lock); 216 if (flow->type) { 217 /* a flow is already active */ 218 smc_llc_flow_parallel(lgr, flow->type, qentry); 219 spin_unlock_bh(&lgr->llc_flow_lock); 220 return false; 221 } 222 switch (qentry->msg.raw.hdr.common.type) { 223 case SMC_LLC_ADD_LINK: 224 flow->type = SMC_LLC_FLOW_ADD_LINK; 225 break; 226 case SMC_LLC_DELETE_LINK: 227 flow->type = SMC_LLC_FLOW_DEL_LINK; 228 break; 229 case SMC_LLC_CONFIRM_RKEY: 230 case SMC_LLC_DELETE_RKEY: 231 flow->type = SMC_LLC_FLOW_RKEY; 232 break; 233 default: 234 flow->type = SMC_LLC_FLOW_NONE; 235 } 236 if (qentry == lgr->delayed_event) 237 lgr->delayed_event = NULL; 238 smc_llc_flow_qentry_set(flow, qentry); 239 spin_unlock_bh(&lgr->llc_flow_lock); 240 return true; 241 } 242 243 /* start a new local llc flow, wait till current flow finished */ 244 int smc_llc_flow_initiate(struct smc_link_group *lgr, 245 enum smc_llc_flowtype type) 246 { 247 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 248 int rc; 249 250 /* all flows except confirm_rkey and delete_rkey are exclusive, 251 * confirm/delete rkey flows can run concurrently (local and remote) 252 */ 253 if (type == SMC_LLC_FLOW_RKEY) 254 allowed_remote = SMC_LLC_FLOW_RKEY; 255 again: 256 if (list_empty(&lgr->list)) 257 return -ENODEV; 258 spin_lock_bh(&lgr->llc_flow_lock); 259 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 260 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 261 lgr->llc_flow_rmt.type == allowed_remote)) { 262 lgr->llc_flow_lcl.type = type; 263 spin_unlock_bh(&lgr->llc_flow_lock); 264 return 0; 265 } 266 spin_unlock_bh(&lgr->llc_flow_lock); 267 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 268 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 269 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 270 lgr->llc_flow_rmt.type == allowed_remote))), 271 SMC_LLC_WAIT_TIME * 10); 272 if (!rc) 273 return -ETIMEDOUT; 274 goto again; 275 } 276 277 /* finish the current llc flow */ 278 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 279 { 280 spin_lock_bh(&lgr->llc_flow_lock); 281 memset(flow, 0, sizeof(*flow)); 282 flow->type = SMC_LLC_FLOW_NONE; 283 spin_unlock_bh(&lgr->llc_flow_lock); 284 if (!list_empty(&lgr->list) && lgr->delayed_event && 285 flow == &lgr->llc_flow_lcl) 286 schedule_work(&lgr->llc_event_work); 287 else 288 wake_up(&lgr->llc_flow_waiter); 289 } 290 291 /* lnk is optional and used for early wakeup when link goes down, useful in 292 * cases where we wait for a response on the link after we sent a request 293 */ 294 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 295 struct smc_link *lnk, 296 int time_out, u8 exp_msg) 297 { 298 struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 299 u8 rcv_msg; 300 301 wait_event_timeout(lgr->llc_msg_waiter, 302 (flow->qentry || 303 (lnk && !smc_link_usable(lnk)) || 304 list_empty(&lgr->list)), 305 time_out); 306 if (!flow->qentry || 307 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 308 smc_llc_flow_qentry_del(flow); 309 goto out; 310 } 311 rcv_msg = flow->qentry->msg.raw.hdr.common.type; 312 if (exp_msg && rcv_msg != exp_msg) { 313 if (exp_msg == SMC_LLC_ADD_LINK && 314 rcv_msg == SMC_LLC_DELETE_LINK) { 315 /* flow_start will delay the unexpected msg */ 316 smc_llc_flow_start(&lgr->llc_flow_lcl, 317 smc_llc_flow_qentry_clr(flow)); 318 return NULL; 319 } 320 pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: " 321 "msg %d exp %d flow %d role %d flags %x\n", 322 SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg, 323 flow->type, lgr->role, 324 flow->qentry->msg.raw.hdr.flags); 325 smc_llc_flow_qentry_del(flow); 326 } 327 out: 328 return flow->qentry; 329 } 330 331 /********************************** send *************************************/ 332 333 struct smc_llc_tx_pend { 334 }; 335 336 /* handler for send/transmission completion of an LLC msg */ 337 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 338 struct smc_link *link, 339 enum ib_wc_status wc_status) 340 { 341 /* future work: handle wc_status error for recovery and failover */ 342 } 343 344 /** 345 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 346 * @link: Pointer to SMC link used for sending LLC control message. 347 * @wr_buf: Out variable returning pointer to work request payload buffer. 348 * @pend: Out variable returning pointer to private pending WR tracking. 349 * It's the context the transmit complete handler will get. 350 * 351 * Reserves and pre-fills an entry for a pending work request send/tx. 352 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 353 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 354 * 355 * Return: 0 on success, otherwise an error value. 356 */ 357 static int smc_llc_add_pending_send(struct smc_link *link, 358 struct smc_wr_buf **wr_buf, 359 struct smc_wr_tx_pend_priv **pend) 360 { 361 int rc; 362 363 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 364 pend); 365 if (rc < 0) 366 return rc; 367 BUILD_BUG_ON_MSG( 368 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 369 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 370 BUILD_BUG_ON_MSG( 371 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 372 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 373 BUILD_BUG_ON_MSG( 374 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 375 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 376 return 0; 377 } 378 379 /* high-level API to send LLC confirm link */ 380 int smc_llc_send_confirm_link(struct smc_link *link, 381 enum smc_llc_reqresp reqresp) 382 { 383 struct smc_llc_msg_confirm_link *confllc; 384 struct smc_wr_tx_pend_priv *pend; 385 struct smc_wr_buf *wr_buf; 386 int rc; 387 388 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 389 if (rc) 390 return rc; 391 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 392 memset(confllc, 0, sizeof(*confllc)); 393 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 394 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 395 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 396 if (reqresp == SMC_LLC_RESP) 397 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 398 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 399 ETH_ALEN); 400 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 401 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 402 confllc->link_num = link->link_id; 403 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 404 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 405 /* send llc message */ 406 rc = smc_wr_tx_send(link, pend); 407 return rc; 408 } 409 410 /* send LLC confirm rkey request */ 411 static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 412 struct smc_buf_desc *rmb_desc) 413 { 414 struct smc_llc_msg_confirm_rkey *rkeyllc; 415 struct smc_wr_tx_pend_priv *pend; 416 struct smc_wr_buf *wr_buf; 417 struct smc_link *link; 418 int i, rc, rtok_ix; 419 420 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 421 if (rc) 422 return rc; 423 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 424 memset(rkeyllc, 0, sizeof(*rkeyllc)); 425 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 426 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 427 428 rtok_ix = 1; 429 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 430 link = &send_link->lgr->lnk[i]; 431 if (link->state == SMC_LNK_ACTIVE && link != send_link) { 432 rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 433 rkeyllc->rtoken[rtok_ix].rmb_key = 434 htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 435 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 436 (u64)sg_dma_address( 437 rmb_desc->sgt[link->link_idx].sgl)); 438 rtok_ix++; 439 } 440 } 441 /* rkey of send_link is in rtoken[0] */ 442 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 443 rkeyllc->rtoken[0].rmb_key = 444 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 445 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 446 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 447 /* send llc message */ 448 rc = smc_wr_tx_send(send_link, pend); 449 return rc; 450 } 451 452 /* send LLC delete rkey request */ 453 static int smc_llc_send_delete_rkey(struct smc_link *link, 454 struct smc_buf_desc *rmb_desc) 455 { 456 struct smc_llc_msg_delete_rkey *rkeyllc; 457 struct smc_wr_tx_pend_priv *pend; 458 struct smc_wr_buf *wr_buf; 459 int rc; 460 461 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 462 if (rc) 463 return rc; 464 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 465 memset(rkeyllc, 0, sizeof(*rkeyllc)); 466 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 467 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 468 rkeyllc->num_rkeys = 1; 469 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 470 /* send llc message */ 471 rc = smc_wr_tx_send(link, pend); 472 return rc; 473 } 474 475 /* send ADD LINK request or response */ 476 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 477 struct smc_link *link_new, 478 enum smc_llc_reqresp reqresp) 479 { 480 struct smc_llc_msg_add_link *addllc; 481 struct smc_wr_tx_pend_priv *pend; 482 struct smc_wr_buf *wr_buf; 483 int rc; 484 485 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 486 if (rc) 487 return rc; 488 addllc = (struct smc_llc_msg_add_link *)wr_buf; 489 490 memset(addllc, 0, sizeof(*addllc)); 491 addllc->hd.common.type = SMC_LLC_ADD_LINK; 492 addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 493 if (reqresp == SMC_LLC_RESP) 494 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 495 memcpy(addllc->sender_mac, mac, ETH_ALEN); 496 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 497 if (link_new) { 498 addllc->link_num = link_new->link_id; 499 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 500 hton24(addllc->initial_psn, link_new->psn_initial); 501 if (reqresp == SMC_LLC_REQ) 502 addllc->qp_mtu = link_new->path_mtu; 503 else 504 addllc->qp_mtu = min(link_new->path_mtu, 505 link_new->peer_mtu); 506 } 507 /* send llc message */ 508 rc = smc_wr_tx_send(link, pend); 509 return rc; 510 } 511 512 /* send DELETE LINK request or response */ 513 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 514 enum smc_llc_reqresp reqresp, bool orderly, 515 u32 reason) 516 { 517 struct smc_llc_msg_del_link *delllc; 518 struct smc_wr_tx_pend_priv *pend; 519 struct smc_wr_buf *wr_buf; 520 int rc; 521 522 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 523 if (rc) 524 return rc; 525 delllc = (struct smc_llc_msg_del_link *)wr_buf; 526 527 memset(delllc, 0, sizeof(*delllc)); 528 delllc->hd.common.type = SMC_LLC_DELETE_LINK; 529 delllc->hd.length = sizeof(struct smc_llc_msg_del_link); 530 if (reqresp == SMC_LLC_RESP) 531 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 532 if (orderly) 533 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 534 if (link_del_id) 535 delllc->link_num = link_del_id; 536 else 537 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 538 delllc->reason = htonl(reason); 539 /* send llc message */ 540 rc = smc_wr_tx_send(link, pend); 541 return rc; 542 } 543 544 /* send LLC test link request */ 545 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 546 { 547 struct smc_llc_msg_test_link *testllc; 548 struct smc_wr_tx_pend_priv *pend; 549 struct smc_wr_buf *wr_buf; 550 int rc; 551 552 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 553 if (rc) 554 return rc; 555 testllc = (struct smc_llc_msg_test_link *)wr_buf; 556 memset(testllc, 0, sizeof(*testllc)); 557 testllc->hd.common.type = SMC_LLC_TEST_LINK; 558 testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 559 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 560 /* send llc message */ 561 rc = smc_wr_tx_send(link, pend); 562 return rc; 563 } 564 565 /* schedule an llc send on link, may wait for buffers */ 566 static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 567 { 568 struct smc_wr_tx_pend_priv *pend; 569 struct smc_wr_buf *wr_buf; 570 int rc; 571 572 if (!smc_link_usable(link)) 573 return -ENOLINK; 574 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 575 if (rc) 576 return rc; 577 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 578 return smc_wr_tx_send(link, pend); 579 } 580 581 /* schedule an llc send on link, may wait for buffers, 582 * and wait for send completion notification. 583 * @return 0 on success 584 */ 585 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 586 { 587 struct smc_wr_tx_pend_priv *pend; 588 struct smc_wr_buf *wr_buf; 589 int rc; 590 591 if (!smc_link_usable(link)) 592 return -ENOLINK; 593 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 594 if (rc) 595 return rc; 596 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 597 return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 598 } 599 600 /********************************* receive ***********************************/ 601 602 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 603 enum smc_lgr_type lgr_new_t) 604 { 605 int i; 606 607 if (lgr->type == SMC_LGR_SYMMETRIC || 608 (lgr->type != SMC_LGR_SINGLE && 609 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 610 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 611 return -EMLINK; 612 613 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 614 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 615 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 616 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 617 return i; 618 } else { 619 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 620 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 621 return i; 622 } 623 return -EMLINK; 624 } 625 626 /* return first buffer from any of the next buf lists */ 627 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 628 int *buf_lst) 629 { 630 struct smc_buf_desc *buf_pos; 631 632 while (*buf_lst < SMC_RMBE_SIZES) { 633 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 634 struct smc_buf_desc, list); 635 if (buf_pos) 636 return buf_pos; 637 (*buf_lst)++; 638 } 639 return NULL; 640 } 641 642 /* return next rmb from buffer lists */ 643 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 644 int *buf_lst, 645 struct smc_buf_desc *buf_pos) 646 { 647 struct smc_buf_desc *buf_next; 648 649 if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 650 (*buf_lst)++; 651 return _smc_llc_get_next_rmb(lgr, buf_lst); 652 } 653 buf_next = list_next_entry(buf_pos, list); 654 return buf_next; 655 } 656 657 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 658 int *buf_lst) 659 { 660 *buf_lst = 0; 661 return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 662 } 663 664 /* send one add_link_continue msg */ 665 static int smc_llc_add_link_cont(struct smc_link *link, 666 struct smc_link *link_new, u8 *num_rkeys_todo, 667 int *buf_lst, struct smc_buf_desc **buf_pos) 668 { 669 struct smc_llc_msg_add_link_cont *addc_llc; 670 struct smc_link_group *lgr = link->lgr; 671 int prim_lnk_idx, lnk_idx, i, rc; 672 struct smc_wr_tx_pend_priv *pend; 673 struct smc_wr_buf *wr_buf; 674 struct smc_buf_desc *rmb; 675 u8 n; 676 677 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 678 if (rc) 679 return rc; 680 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 681 memset(addc_llc, 0, sizeof(*addc_llc)); 682 683 prim_lnk_idx = link->link_idx; 684 lnk_idx = link_new->link_idx; 685 addc_llc->link_num = link_new->link_id; 686 addc_llc->num_rkeys = *num_rkeys_todo; 687 n = *num_rkeys_todo; 688 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 689 if (!*buf_pos) { 690 addc_llc->num_rkeys = addc_llc->num_rkeys - 691 *num_rkeys_todo; 692 *num_rkeys_todo = 0; 693 break; 694 } 695 rmb = *buf_pos; 696 697 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 698 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 699 addc_llc->rt[i].rmb_vaddr_new = 700 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 701 702 (*num_rkeys_todo)--; 703 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 704 while (*buf_pos && !(*buf_pos)->used) 705 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 706 } 707 addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; 708 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 709 if (lgr->role == SMC_CLNT) 710 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 711 return smc_wr_tx_send(link, pend); 712 } 713 714 static int smc_llc_cli_rkey_exchange(struct smc_link *link, 715 struct smc_link *link_new) 716 { 717 struct smc_llc_msg_add_link_cont *addc_llc; 718 struct smc_link_group *lgr = link->lgr; 719 u8 max, num_rkeys_send, num_rkeys_recv; 720 struct smc_llc_qentry *qentry; 721 struct smc_buf_desc *buf_pos; 722 int buf_lst; 723 int rc = 0; 724 int i; 725 726 mutex_lock(&lgr->rmbs_lock); 727 num_rkeys_send = lgr->conns_num; 728 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 729 do { 730 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 731 SMC_LLC_ADD_LINK_CONT); 732 if (!qentry) { 733 rc = -ETIMEDOUT; 734 break; 735 } 736 addc_llc = &qentry->msg.add_link_cont; 737 num_rkeys_recv = addc_llc->num_rkeys; 738 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 739 for (i = 0; i < max; i++) { 740 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 741 addc_llc->rt[i].rmb_key, 742 addc_llc->rt[i].rmb_vaddr_new, 743 addc_llc->rt[i].rmb_key_new); 744 num_rkeys_recv--; 745 } 746 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 747 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 748 &buf_lst, &buf_pos); 749 if (rc) 750 break; 751 } while (num_rkeys_send || num_rkeys_recv); 752 753 mutex_unlock(&lgr->rmbs_lock); 754 return rc; 755 } 756 757 /* prepare and send an add link reject response */ 758 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 759 { 760 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 761 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 762 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 763 return smc_llc_send_message(qentry->link, &qentry->msg); 764 } 765 766 static int smc_llc_cli_conf_link(struct smc_link *link, 767 struct smc_init_info *ini, 768 struct smc_link *link_new, 769 enum smc_lgr_type lgr_new_t) 770 { 771 struct smc_link_group *lgr = link->lgr; 772 struct smc_llc_qentry *qentry = NULL; 773 int rc = 0; 774 775 /* receive CONFIRM LINK request over RoCE fabric */ 776 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 777 if (!qentry) { 778 rc = smc_llc_send_delete_link(link, link_new->link_id, 779 SMC_LLC_REQ, false, 780 SMC_LLC_DEL_LOST_PATH); 781 return -ENOLINK; 782 } 783 if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 784 /* received DELETE_LINK instead */ 785 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 786 smc_llc_send_message(link, &qentry->msg); 787 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 788 return -ENOLINK; 789 } 790 smc_llc_save_peer_uid(qentry); 791 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 792 793 rc = smc_ib_modify_qp_rts(link_new); 794 if (rc) { 795 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 796 false, SMC_LLC_DEL_LOST_PATH); 797 return -ENOLINK; 798 } 799 smc_wr_remember_qp_attr(link_new); 800 801 rc = smcr_buf_reg_lgr(link_new); 802 if (rc) { 803 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 804 false, SMC_LLC_DEL_LOST_PATH); 805 return -ENOLINK; 806 } 807 808 /* send CONFIRM LINK response over RoCE fabric */ 809 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 810 if (rc) { 811 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 812 false, SMC_LLC_DEL_LOST_PATH); 813 return -ENOLINK; 814 } 815 smc_llc_link_active(link_new); 816 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 817 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 818 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 819 else 820 smcr_lgr_set_type(lgr, lgr_new_t); 821 return 0; 822 } 823 824 static void smc_llc_save_add_link_info(struct smc_link *link, 825 struct smc_llc_msg_add_link *add_llc) 826 { 827 link->peer_qpn = ntoh24(add_llc->sender_qp_num); 828 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 829 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 830 link->peer_psn = ntoh24(add_llc->initial_psn); 831 link->peer_mtu = add_llc->qp_mtu; 832 } 833 834 /* as an SMC client, process an add link request */ 835 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 836 { 837 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 838 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 839 struct smc_link_group *lgr = smc_get_lgr(link); 840 struct smc_link *lnk_new = NULL; 841 struct smc_init_info ini; 842 int lnk_idx, rc = 0; 843 844 ini.vlan_id = lgr->vlan_id; 845 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 846 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 847 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { 848 if (!ini.ib_dev) 849 goto out_reject; 850 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 851 } 852 if (!ini.ib_dev) { 853 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 854 ini.ib_dev = link->smcibdev; 855 ini.ib_port = link->ibport; 856 } 857 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 858 if (lnk_idx < 0) 859 goto out_reject; 860 lnk_new = &lgr->lnk[lnk_idx]; 861 rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); 862 if (rc) 863 goto out_reject; 864 smc_llc_save_add_link_info(lnk_new, llc); 865 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 866 smc_llc_link_set_uid(lnk_new); 867 868 rc = smc_ib_ready_link(lnk_new); 869 if (rc) 870 goto out_clear_lnk; 871 872 rc = smcr_buf_map_lgr(lnk_new); 873 if (rc) 874 goto out_clear_lnk; 875 876 rc = smc_llc_send_add_link(link, 877 lnk_new->smcibdev->mac[ini.ib_port - 1], 878 lnk_new->gid, lnk_new, SMC_LLC_RESP); 879 if (rc) 880 goto out_clear_lnk; 881 rc = smc_llc_cli_rkey_exchange(link, lnk_new); 882 if (rc) { 883 rc = 0; 884 goto out_clear_lnk; 885 } 886 rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); 887 if (!rc) 888 goto out; 889 out_clear_lnk: 890 smcr_link_clear(lnk_new, false); 891 out_reject: 892 smc_llc_cli_add_link_reject(qentry); 893 out: 894 kfree(qentry); 895 return rc; 896 } 897 898 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 899 { 900 struct smc_llc_qentry *qentry; 901 902 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 903 904 mutex_lock(&lgr->llc_conf_mutex); 905 smc_llc_cli_add_link(qentry->link, qentry); 906 mutex_unlock(&lgr->llc_conf_mutex); 907 } 908 909 static int smc_llc_active_link_count(struct smc_link_group *lgr) 910 { 911 int i, link_count = 0; 912 913 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 914 if (!smc_link_usable(&lgr->lnk[i])) 915 continue; 916 link_count++; 917 } 918 return link_count; 919 } 920 921 /* find the asymmetric link when 3 links are established */ 922 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 923 { 924 int asym_idx = -ENOENT; 925 int i, j, k; 926 bool found; 927 928 /* determine asymmetric link */ 929 found = false; 930 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 931 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 932 if (!smc_link_usable(&lgr->lnk[i]) || 933 !smc_link_usable(&lgr->lnk[j])) 934 continue; 935 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 936 SMC_GID_SIZE)) { 937 found = true; /* asym_lnk is i or j */ 938 break; 939 } 940 } 941 if (found) 942 break; 943 } 944 if (!found) 945 goto out; /* no asymmetric link */ 946 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 947 if (!smc_link_usable(&lgr->lnk[k])) 948 continue; 949 if (k != i && 950 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 951 SMC_GID_SIZE)) { 952 asym_idx = i; 953 break; 954 } 955 if (k != j && 956 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 957 SMC_GID_SIZE)) { 958 asym_idx = j; 959 break; 960 } 961 } 962 out: 963 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 964 } 965 966 static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 967 { 968 struct smc_link *lnk_new = NULL, *lnk_asym; 969 struct smc_llc_qentry *qentry; 970 int rc; 971 972 lnk_asym = smc_llc_find_asym_link(lgr); 973 if (!lnk_asym) 974 return; /* no asymmetric link */ 975 if (!smc_link_downing(&lnk_asym->state)) 976 return; 977 lnk_new = smc_switch_conns(lgr, lnk_asym, false); 978 smc_wr_tx_wait_no_pending_sends(lnk_asym); 979 if (!lnk_new) 980 goto out_free; 981 /* change flow type from ADD_LINK into DEL_LINK */ 982 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 983 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 984 true, SMC_LLC_DEL_NO_ASYM_NEEDED); 985 if (rc) { 986 smcr_link_down_cond(lnk_new); 987 goto out_free; 988 } 989 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 990 SMC_LLC_DELETE_LINK); 991 if (!qentry) { 992 smcr_link_down_cond(lnk_new); 993 goto out_free; 994 } 995 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 996 out_free: 997 smcr_link_clear(lnk_asym, true); 998 } 999 1000 static int smc_llc_srv_rkey_exchange(struct smc_link *link, 1001 struct smc_link *link_new) 1002 { 1003 struct smc_llc_msg_add_link_cont *addc_llc; 1004 struct smc_link_group *lgr = link->lgr; 1005 u8 max, num_rkeys_send, num_rkeys_recv; 1006 struct smc_llc_qentry *qentry = NULL; 1007 struct smc_buf_desc *buf_pos; 1008 int buf_lst; 1009 int rc = 0; 1010 int i; 1011 1012 mutex_lock(&lgr->rmbs_lock); 1013 num_rkeys_send = lgr->conns_num; 1014 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 1015 do { 1016 smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 1017 &buf_lst, &buf_pos); 1018 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 1019 SMC_LLC_ADD_LINK_CONT); 1020 if (!qentry) { 1021 rc = -ETIMEDOUT; 1022 goto out; 1023 } 1024 addc_llc = &qentry->msg.add_link_cont; 1025 num_rkeys_recv = addc_llc->num_rkeys; 1026 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 1027 for (i = 0; i < max; i++) { 1028 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1029 addc_llc->rt[i].rmb_key, 1030 addc_llc->rt[i].rmb_vaddr_new, 1031 addc_llc->rt[i].rmb_key_new); 1032 num_rkeys_recv--; 1033 } 1034 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1035 } while (num_rkeys_send || num_rkeys_recv); 1036 out: 1037 mutex_unlock(&lgr->rmbs_lock); 1038 return rc; 1039 } 1040 1041 static int smc_llc_srv_conf_link(struct smc_link *link, 1042 struct smc_link *link_new, 1043 enum smc_lgr_type lgr_new_t) 1044 { 1045 struct smc_link_group *lgr = link->lgr; 1046 struct smc_llc_qentry *qentry = NULL; 1047 int rc; 1048 1049 /* send CONFIRM LINK request over the RoCE fabric */ 1050 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 1051 if (rc) 1052 return -ENOLINK; 1053 /* receive CONFIRM LINK response over the RoCE fabric */ 1054 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 1055 SMC_LLC_CONFIRM_LINK); 1056 if (!qentry) { 1057 /* send DELETE LINK */ 1058 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1059 false, SMC_LLC_DEL_LOST_PATH); 1060 return -ENOLINK; 1061 } 1062 smc_llc_save_peer_uid(qentry); 1063 smc_llc_link_active(link_new); 1064 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1065 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1066 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1067 else 1068 smcr_lgr_set_type(lgr, lgr_new_t); 1069 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1070 return 0; 1071 } 1072 1073 int smc_llc_srv_add_link(struct smc_link *link) 1074 { 1075 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1076 struct smc_link_group *lgr = link->lgr; 1077 struct smc_llc_msg_add_link *add_llc; 1078 struct smc_llc_qentry *qentry = NULL; 1079 struct smc_link *link_new; 1080 struct smc_init_info ini; 1081 int lnk_idx, rc = 0; 1082 1083 /* ignore client add link recommendation, start new flow */ 1084 ini.vlan_id = lgr->vlan_id; 1085 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 1086 if (!ini.ib_dev) { 1087 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1088 ini.ib_dev = link->smcibdev; 1089 ini.ib_port = link->ibport; 1090 } 1091 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1092 if (lnk_idx < 0) 1093 return 0; 1094 1095 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); 1096 if (rc) 1097 return rc; 1098 link_new = &lgr->lnk[lnk_idx]; 1099 rc = smc_llc_send_add_link(link, 1100 link_new->smcibdev->mac[ini.ib_port - 1], 1101 link_new->gid, link_new, SMC_LLC_REQ); 1102 if (rc) 1103 goto out_err; 1104 /* receive ADD LINK response over the RoCE fabric */ 1105 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 1106 if (!qentry) { 1107 rc = -ETIMEDOUT; 1108 goto out_err; 1109 } 1110 add_llc = &qentry->msg.add_link; 1111 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 1112 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1113 rc = -ENOLINK; 1114 goto out_err; 1115 } 1116 if (lgr->type == SMC_LGR_SINGLE && 1117 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1118 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { 1119 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1120 } 1121 smc_llc_save_add_link_info(link_new, add_llc); 1122 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1123 1124 rc = smc_ib_ready_link(link_new); 1125 if (rc) 1126 goto out_err; 1127 rc = smcr_buf_map_lgr(link_new); 1128 if (rc) 1129 goto out_err; 1130 rc = smcr_buf_reg_lgr(link_new); 1131 if (rc) 1132 goto out_err; 1133 rc = smc_llc_srv_rkey_exchange(link, link_new); 1134 if (rc) 1135 goto out_err; 1136 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 1137 if (rc) 1138 goto out_err; 1139 return 0; 1140 out_err: 1141 smcr_link_clear(link_new, false); 1142 return rc; 1143 } 1144 1145 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 1146 { 1147 struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 1148 int rc; 1149 1150 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1151 1152 mutex_lock(&lgr->llc_conf_mutex); 1153 rc = smc_llc_srv_add_link(link); 1154 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 1155 /* delete any asymmetric link */ 1156 smc_llc_delete_asym_link(lgr); 1157 } 1158 mutex_unlock(&lgr->llc_conf_mutex); 1159 } 1160 1161 /* enqueue a local add_link req to trigger a new add_link flow, only as SERV */ 1162 void smc_llc_srv_add_link_local(struct smc_link *link) 1163 { 1164 struct smc_llc_msg_add_link add_llc = {0}; 1165 1166 add_llc.hd.length = sizeof(add_llc); 1167 add_llc.hd.common.type = SMC_LLC_ADD_LINK; 1168 /* no dev and port needed, we as server ignore client data anyway */ 1169 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 1170 } 1171 1172 /* worker to process an add link message */ 1173 static void smc_llc_add_link_work(struct work_struct *work) 1174 { 1175 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1176 llc_add_link_work); 1177 1178 if (list_empty(&lgr->list)) { 1179 /* link group is terminating */ 1180 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1181 goto out; 1182 } 1183 1184 if (lgr->role == SMC_CLNT) 1185 smc_llc_process_cli_add_link(lgr); 1186 else 1187 smc_llc_process_srv_add_link(lgr); 1188 out: 1189 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1190 } 1191 1192 /* enqueue a local del_link msg to trigger a new del_link flow, 1193 * called only for role SMC_SERV 1194 */ 1195 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 1196 { 1197 struct smc_llc_msg_del_link del_llc = {0}; 1198 1199 del_llc.hd.length = sizeof(del_llc); 1200 del_llc.hd.common.type = SMC_LLC_DELETE_LINK; 1201 del_llc.link_num = del_link_id; 1202 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 1203 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1204 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 1205 } 1206 1207 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 1208 { 1209 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 1210 struct smc_llc_msg_del_link *del_llc; 1211 struct smc_llc_qentry *qentry; 1212 int active_links; 1213 int lnk_idx; 1214 1215 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1216 lnk = qentry->link; 1217 del_llc = &qentry->msg.delete_link; 1218 1219 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1220 smc_lgr_terminate_sched(lgr); 1221 goto out; 1222 } 1223 mutex_lock(&lgr->llc_conf_mutex); 1224 /* delete single link */ 1225 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 1226 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 1227 continue; 1228 lnk_del = &lgr->lnk[lnk_idx]; 1229 break; 1230 } 1231 del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 1232 if (!lnk_del) { 1233 /* link was not found */ 1234 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 1235 smc_llc_send_message(lnk, &qentry->msg); 1236 goto out_unlock; 1237 } 1238 lnk_asym = smc_llc_find_asym_link(lgr); 1239 1240 del_llc->reason = 0; 1241 smc_llc_send_message(lnk, &qentry->msg); /* response */ 1242 1243 if (smc_link_downing(&lnk_del->state)) { 1244 if (smc_switch_conns(lgr, lnk_del, false)) 1245 smc_wr_tx_wait_no_pending_sends(lnk_del); 1246 } 1247 smcr_link_clear(lnk_del, true); 1248 1249 active_links = smc_llc_active_link_count(lgr); 1250 if (lnk_del == lnk_asym) { 1251 /* expected deletion of asym link, don't change lgr state */ 1252 } else if (active_links == 1) { 1253 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1254 } else if (!active_links) { 1255 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1256 smc_lgr_terminate_sched(lgr); 1257 } 1258 out_unlock: 1259 mutex_unlock(&lgr->llc_conf_mutex); 1260 out: 1261 kfree(qentry); 1262 } 1263 1264 /* try to send a DELETE LINK ALL request on any active link, 1265 * waiting for send completion 1266 */ 1267 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1268 { 1269 struct smc_llc_msg_del_link delllc = {0}; 1270 int i; 1271 1272 delllc.hd.common.type = SMC_LLC_DELETE_LINK; 1273 delllc.hd.length = sizeof(delllc); 1274 if (ord) 1275 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1276 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1277 delllc.reason = htonl(rsn); 1278 1279 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1280 if (!smc_link_usable(&lgr->lnk[i])) 1281 continue; 1282 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1283 break; 1284 } 1285 } 1286 1287 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 1288 { 1289 struct smc_llc_msg_del_link *del_llc; 1290 struct smc_link *lnk, *lnk_del; 1291 struct smc_llc_qentry *qentry; 1292 int active_links; 1293 int i; 1294 1295 mutex_lock(&lgr->llc_conf_mutex); 1296 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1297 lnk = qentry->link; 1298 del_llc = &qentry->msg.delete_link; 1299 1300 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1301 /* delete entire lgr */ 1302 smc_llc_send_link_delete_all(lgr, true, ntohl( 1303 qentry->msg.delete_link.reason)); 1304 smc_lgr_terminate_sched(lgr); 1305 goto out; 1306 } 1307 /* delete single link */ 1308 lnk_del = NULL; 1309 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1310 if (lgr->lnk[i].link_id == del_llc->link_num) { 1311 lnk_del = &lgr->lnk[i]; 1312 break; 1313 } 1314 } 1315 if (!lnk_del) 1316 goto out; /* asymmetric link already deleted */ 1317 1318 if (smc_link_downing(&lnk_del->state)) { 1319 if (smc_switch_conns(lgr, lnk_del, false)) 1320 smc_wr_tx_wait_no_pending_sends(lnk_del); 1321 } 1322 if (!list_empty(&lgr->list)) { 1323 /* qentry is either a request from peer (send it back to 1324 * initiate the DELETE_LINK processing), or a locally 1325 * enqueued DELETE_LINK request (forward it) 1326 */ 1327 if (!smc_llc_send_message(lnk, &qentry->msg)) { 1328 struct smc_llc_qentry *qentry2; 1329 1330 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 1331 SMC_LLC_DELETE_LINK); 1332 if (qentry2) 1333 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1334 } 1335 } 1336 smcr_link_clear(lnk_del, true); 1337 1338 active_links = smc_llc_active_link_count(lgr); 1339 if (active_links == 1) { 1340 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1341 } else if (!active_links) { 1342 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1343 smc_lgr_terminate_sched(lgr); 1344 } 1345 1346 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 1347 /* trigger setup of asymm alt link */ 1348 smc_llc_srv_add_link_local(lnk); 1349 } 1350 out: 1351 mutex_unlock(&lgr->llc_conf_mutex); 1352 kfree(qentry); 1353 } 1354 1355 static void smc_llc_delete_link_work(struct work_struct *work) 1356 { 1357 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1358 llc_del_link_work); 1359 1360 if (list_empty(&lgr->list)) { 1361 /* link group is terminating */ 1362 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1363 goto out; 1364 } 1365 1366 if (lgr->role == SMC_CLNT) 1367 smc_llc_process_cli_delete_link(lgr); 1368 else 1369 smc_llc_process_srv_delete_link(lgr); 1370 out: 1371 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1372 } 1373 1374 /* process a confirm_rkey request from peer, remote flow */ 1375 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 1376 { 1377 struct smc_llc_msg_confirm_rkey *llc; 1378 struct smc_llc_qentry *qentry; 1379 struct smc_link *link; 1380 int num_entries; 1381 int rk_idx; 1382 int i; 1383 1384 qentry = lgr->llc_flow_rmt.qentry; 1385 llc = &qentry->msg.confirm_rkey; 1386 link = qentry->link; 1387 1388 num_entries = llc->rtoken[0].num_rkeys; 1389 /* first rkey entry is for receiving link */ 1390 rk_idx = smc_rtoken_add(link, 1391 llc->rtoken[0].rmb_vaddr, 1392 llc->rtoken[0].rmb_key); 1393 if (rk_idx < 0) 1394 goto out_err; 1395 1396 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 1397 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 1398 llc->rtoken[i].rmb_vaddr, 1399 llc->rtoken[i].rmb_key); 1400 /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 1401 goto out; 1402 out_err: 1403 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1404 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 1405 out: 1406 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1407 smc_llc_send_message(link, &qentry->msg); 1408 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1409 } 1410 1411 /* process a delete_rkey request from peer, remote flow */ 1412 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 1413 { 1414 struct smc_llc_msg_delete_rkey *llc; 1415 struct smc_llc_qentry *qentry; 1416 struct smc_link *link; 1417 u8 err_mask = 0; 1418 int i, max; 1419 1420 qentry = lgr->llc_flow_rmt.qentry; 1421 llc = &qentry->msg.delete_rkey; 1422 link = qentry->link; 1423 1424 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 1425 for (i = 0; i < max; i++) { 1426 if (smc_rtoken_delete(link, llc->rkey[i])) 1427 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 1428 } 1429 if (err_mask) { 1430 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1431 llc->err_mask = err_mask; 1432 } 1433 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1434 smc_llc_send_message(link, &qentry->msg); 1435 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1436 } 1437 1438 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 1439 { 1440 pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: " 1441 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type); 1442 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 1443 smc_lgr_terminate_sched(lgr); 1444 } 1445 1446 /* flush the llc event queue */ 1447 static void smc_llc_event_flush(struct smc_link_group *lgr) 1448 { 1449 struct smc_llc_qentry *qentry, *q; 1450 1451 spin_lock_bh(&lgr->llc_event_q_lock); 1452 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 1453 list_del_init(&qentry->list); 1454 kfree(qentry); 1455 } 1456 spin_unlock_bh(&lgr->llc_event_q_lock); 1457 } 1458 1459 static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 1460 { 1461 union smc_llc_msg *llc = &qentry->msg; 1462 struct smc_link *link = qentry->link; 1463 struct smc_link_group *lgr = link->lgr; 1464 1465 if (!smc_link_usable(link)) 1466 goto out; 1467 1468 switch (llc->raw.hdr.common.type) { 1469 case SMC_LLC_TEST_LINK: 1470 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 1471 smc_llc_send_message(link, llc); 1472 break; 1473 case SMC_LLC_ADD_LINK: 1474 if (list_empty(&lgr->list)) 1475 goto out; /* lgr is terminating */ 1476 if (lgr->role == SMC_CLNT) { 1477 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { 1478 /* a flow is waiting for this message */ 1479 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1480 qentry); 1481 wake_up(&lgr->llc_msg_waiter); 1482 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1483 qentry)) { 1484 schedule_work(&lgr->llc_add_link_work); 1485 } 1486 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1487 /* as smc server, handle client suggestion */ 1488 schedule_work(&lgr->llc_add_link_work); 1489 } 1490 return; 1491 case SMC_LLC_CONFIRM_LINK: 1492 case SMC_LLC_ADD_LINK_CONT: 1493 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 1494 /* a flow is waiting for this message */ 1495 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1496 wake_up(&lgr->llc_msg_waiter); 1497 return; 1498 } 1499 break; 1500 case SMC_LLC_DELETE_LINK: 1501 if (lgr->role == SMC_CLNT) { 1502 /* server requests to delete this link, send response */ 1503 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 1504 /* DEL LINK REQ during ADD LINK SEQ */ 1505 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1506 qentry); 1507 wake_up(&lgr->llc_msg_waiter); 1508 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1509 qentry)) { 1510 schedule_work(&lgr->llc_del_link_work); 1511 } 1512 } else { 1513 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1514 !lgr->llc_flow_lcl.qentry) { 1515 /* DEL LINK REQ during ADD LINK SEQ */ 1516 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1517 qentry); 1518 wake_up(&lgr->llc_msg_waiter); 1519 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1520 qentry)) { 1521 schedule_work(&lgr->llc_del_link_work); 1522 } 1523 } 1524 return; 1525 case SMC_LLC_CONFIRM_RKEY: 1526 /* new request from remote, assign to remote flow */ 1527 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1528 /* process here, does not wait for more llc msgs */ 1529 smc_llc_rmt_conf_rkey(lgr); 1530 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1531 } 1532 return; 1533 case SMC_LLC_CONFIRM_RKEY_CONT: 1534 /* not used because max links is 3, and 3 rkeys fit into 1535 * one CONFIRM_RKEY message 1536 */ 1537 break; 1538 case SMC_LLC_DELETE_RKEY: 1539 /* new request from remote, assign to remote flow */ 1540 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1541 /* process here, does not wait for more llc msgs */ 1542 smc_llc_rmt_delete_rkey(lgr); 1543 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1544 } 1545 return; 1546 default: 1547 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 1548 break; 1549 } 1550 out: 1551 kfree(qentry); 1552 } 1553 1554 /* worker to process llc messages on the event queue */ 1555 static void smc_llc_event_work(struct work_struct *work) 1556 { 1557 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1558 llc_event_work); 1559 struct smc_llc_qentry *qentry; 1560 1561 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1562 if (smc_link_usable(lgr->delayed_event->link)) { 1563 smc_llc_event_handler(lgr->delayed_event); 1564 } else { 1565 qentry = lgr->delayed_event; 1566 lgr->delayed_event = NULL; 1567 kfree(qentry); 1568 } 1569 } 1570 1571 again: 1572 spin_lock_bh(&lgr->llc_event_q_lock); 1573 if (!list_empty(&lgr->llc_event_q)) { 1574 qentry = list_first_entry(&lgr->llc_event_q, 1575 struct smc_llc_qentry, list); 1576 list_del_init(&qentry->list); 1577 spin_unlock_bh(&lgr->llc_event_q_lock); 1578 smc_llc_event_handler(qentry); 1579 goto again; 1580 } 1581 spin_unlock_bh(&lgr->llc_event_q_lock); 1582 } 1583 1584 /* process llc responses in tasklet context */ 1585 static void smc_llc_rx_response(struct smc_link *link, 1586 struct smc_llc_qentry *qentry) 1587 { 1588 u8 llc_type = qentry->msg.raw.hdr.common.type; 1589 1590 switch (llc_type) { 1591 case SMC_LLC_TEST_LINK: 1592 if (link->state == SMC_LNK_ACTIVE) 1593 complete(&link->llc_testlink_resp); 1594 break; 1595 case SMC_LLC_ADD_LINK: 1596 case SMC_LLC_DELETE_LINK: 1597 case SMC_LLC_CONFIRM_LINK: 1598 case SMC_LLC_ADD_LINK_CONT: 1599 case SMC_LLC_CONFIRM_RKEY: 1600 case SMC_LLC_DELETE_RKEY: 1601 /* assign responses to the local flow, we requested them */ 1602 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 1603 wake_up(&link->lgr->llc_msg_waiter); 1604 return; 1605 case SMC_LLC_CONFIRM_RKEY_CONT: 1606 /* not used because max links is 3 */ 1607 break; 1608 default: 1609 smc_llc_protocol_violation(link->lgr, llc_type); 1610 break; 1611 } 1612 kfree(qentry); 1613 } 1614 1615 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 1616 { 1617 struct smc_link_group *lgr = link->lgr; 1618 struct smc_llc_qentry *qentry; 1619 unsigned long flags; 1620 1621 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 1622 if (!qentry) 1623 return; 1624 qentry->link = link; 1625 INIT_LIST_HEAD(&qentry->list); 1626 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 1627 1628 /* process responses immediately */ 1629 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 1630 smc_llc_rx_response(link, qentry); 1631 return; 1632 } 1633 1634 /* add requests to event queue */ 1635 spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 1636 list_add_tail(&qentry->list, &lgr->llc_event_q); 1637 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 1638 schedule_work(&lgr->llc_event_work); 1639 } 1640 1641 /* copy received msg and add it to the event queue */ 1642 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 1643 { 1644 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 1645 union smc_llc_msg *llc = buf; 1646 1647 if (wc->byte_len < sizeof(*llc)) 1648 return; /* short message */ 1649 if (llc->raw.hdr.length != sizeof(*llc)) 1650 return; /* invalid message */ 1651 1652 smc_llc_enqueue(link, llc); 1653 } 1654 1655 /***************************** worker, utils *********************************/ 1656 1657 static void smc_llc_testlink_work(struct work_struct *work) 1658 { 1659 struct smc_link *link = container_of(to_delayed_work(work), 1660 struct smc_link, llc_testlink_wrk); 1661 unsigned long next_interval; 1662 unsigned long expire_time; 1663 u8 user_data[16] = { 0 }; 1664 int rc; 1665 1666 if (link->state != SMC_LNK_ACTIVE) 1667 return; /* don't reschedule worker */ 1668 expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 1669 if (time_is_after_jiffies(expire_time)) { 1670 next_interval = expire_time - jiffies; 1671 goto out; 1672 } 1673 reinit_completion(&link->llc_testlink_resp); 1674 smc_llc_send_test_link(link, user_data); 1675 /* receive TEST LINK response over RoCE fabric */ 1676 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 1677 SMC_LLC_WAIT_TIME); 1678 if (link->state != SMC_LNK_ACTIVE) 1679 return; /* link state changed */ 1680 if (rc <= 0) { 1681 smcr_link_down_cond_sched(link); 1682 return; 1683 } 1684 next_interval = link->llc_testlink_time; 1685 out: 1686 schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 1687 } 1688 1689 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 1690 { 1691 struct net *net = sock_net(smc->clcsock->sk); 1692 1693 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 1694 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 1695 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 1696 INIT_LIST_HEAD(&lgr->llc_event_q); 1697 spin_lock_init(&lgr->llc_event_q_lock); 1698 spin_lock_init(&lgr->llc_flow_lock); 1699 init_waitqueue_head(&lgr->llc_flow_waiter); 1700 init_waitqueue_head(&lgr->llc_msg_waiter); 1701 mutex_init(&lgr->llc_conf_mutex); 1702 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 1703 } 1704 1705 /* called after lgr was removed from lgr_list */ 1706 void smc_llc_lgr_clear(struct smc_link_group *lgr) 1707 { 1708 smc_llc_event_flush(lgr); 1709 wake_up_all(&lgr->llc_flow_waiter); 1710 wake_up_all(&lgr->llc_msg_waiter); 1711 cancel_work_sync(&lgr->llc_event_work); 1712 cancel_work_sync(&lgr->llc_add_link_work); 1713 cancel_work_sync(&lgr->llc_del_link_work); 1714 if (lgr->delayed_event) { 1715 kfree(lgr->delayed_event); 1716 lgr->delayed_event = NULL; 1717 } 1718 } 1719 1720 int smc_llc_link_init(struct smc_link *link) 1721 { 1722 init_completion(&link->llc_testlink_resp); 1723 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 1724 return 0; 1725 } 1726 1727 void smc_llc_link_active(struct smc_link *link) 1728 { 1729 pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, " 1730 "peerid %*phN, ibdev %s, ibport %d\n", 1731 SMC_LGR_ID_SIZE, &link->lgr->id, 1732 SMC_LGR_ID_SIZE, &link->link_uid, 1733 SMC_LGR_ID_SIZE, &link->peer_link_uid, 1734 link->smcibdev->ibdev->name, link->ibport); 1735 link->state = SMC_LNK_ACTIVE; 1736 if (link->lgr->llc_testlink_time) { 1737 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 1738 schedule_delayed_work(&link->llc_testlink_wrk, 1739 link->llc_testlink_time); 1740 } 1741 } 1742 1743 /* called in worker context */ 1744 void smc_llc_link_clear(struct smc_link *link, bool log) 1745 { 1746 if (log) 1747 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN" 1748 ", peerid %*phN, ibdev %s, ibport %d\n", 1749 SMC_LGR_ID_SIZE, &link->lgr->id, 1750 SMC_LGR_ID_SIZE, &link->link_uid, 1751 SMC_LGR_ID_SIZE, &link->peer_link_uid, 1752 link->smcibdev->ibdev->name, link->ibport); 1753 complete(&link->llc_testlink_resp); 1754 cancel_delayed_work_sync(&link->llc_testlink_wrk); 1755 smc_wr_wakeup_reg_wait(link); 1756 smc_wr_wakeup_tx_wait(link); 1757 } 1758 1759 /* register a new rtoken at the remote peer (for all links) */ 1760 int smc_llc_do_confirm_rkey(struct smc_link *send_link, 1761 struct smc_buf_desc *rmb_desc) 1762 { 1763 struct smc_link_group *lgr = send_link->lgr; 1764 struct smc_llc_qentry *qentry = NULL; 1765 int rc = 0; 1766 1767 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 1768 if (rc) 1769 goto out; 1770 /* receive CONFIRM RKEY response from server over RoCE fabric */ 1771 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 1772 SMC_LLC_CONFIRM_RKEY); 1773 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 1774 rc = -EFAULT; 1775 out: 1776 if (qentry) 1777 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1778 return rc; 1779 } 1780 1781 /* unregister an rtoken at the remote peer */ 1782 int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 1783 struct smc_buf_desc *rmb_desc) 1784 { 1785 struct smc_llc_qentry *qentry = NULL; 1786 struct smc_link *send_link; 1787 int rc = 0; 1788 1789 send_link = smc_llc_usable_link(lgr); 1790 if (!send_link) 1791 return -ENOLINK; 1792 1793 /* protected by llc_flow control */ 1794 rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 1795 if (rc) 1796 goto out; 1797 /* receive DELETE RKEY response from server over RoCE fabric */ 1798 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 1799 SMC_LLC_DELETE_RKEY); 1800 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 1801 rc = -EFAULT; 1802 out: 1803 if (qentry) 1804 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1805 return rc; 1806 } 1807 1808 void smc_llc_link_set_uid(struct smc_link *link) 1809 { 1810 __be32 link_uid; 1811 1812 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 1813 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 1814 } 1815 1816 /* save peers link user id, used for debug purposes */ 1817 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 1818 { 1819 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 1820 SMC_LGR_ID_SIZE); 1821 } 1822 1823 /* evaluate confirm link request or response */ 1824 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 1825 enum smc_llc_reqresp type) 1826 { 1827 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 1828 qentry->link->link_id = qentry->msg.confirm_link.link_num; 1829 smc_llc_link_set_uid(qentry->link); 1830 } 1831 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 1832 return -ENOTSUPP; 1833 return 0; 1834 } 1835 1836 /***************************** init, exit, misc ******************************/ 1837 1838 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 1839 { 1840 .handler = smc_llc_rx_handler, 1841 .type = SMC_LLC_CONFIRM_LINK 1842 }, 1843 { 1844 .handler = smc_llc_rx_handler, 1845 .type = SMC_LLC_TEST_LINK 1846 }, 1847 { 1848 .handler = smc_llc_rx_handler, 1849 .type = SMC_LLC_ADD_LINK 1850 }, 1851 { 1852 .handler = smc_llc_rx_handler, 1853 .type = SMC_LLC_ADD_LINK_CONT 1854 }, 1855 { 1856 .handler = smc_llc_rx_handler, 1857 .type = SMC_LLC_DELETE_LINK 1858 }, 1859 { 1860 .handler = smc_llc_rx_handler, 1861 .type = SMC_LLC_CONFIRM_RKEY 1862 }, 1863 { 1864 .handler = smc_llc_rx_handler, 1865 .type = SMC_LLC_CONFIRM_RKEY_CONT 1866 }, 1867 { 1868 .handler = smc_llc_rx_handler, 1869 .type = SMC_LLC_DELETE_RKEY 1870 }, 1871 { 1872 .handler = NULL, 1873 } 1874 }; 1875 1876 int __init smc_llc_init(void) 1877 { 1878 struct smc_wr_rx_handler *handler; 1879 int rc = 0; 1880 1881 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 1882 INIT_HLIST_NODE(&handler->list); 1883 rc = smc_wr_rx_register_handler(handler); 1884 if (rc) 1885 break; 1886 } 1887 return rc; 1888 } 1889