1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13 #include <net/tcp.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "smc.h" 17 #include "smc_core.h" 18 #include "smc_clc.h" 19 #include "smc_llc.h" 20 21 #define SMC_LLC_DATA_LEN 40 22 23 struct smc_llc_hdr { 24 struct smc_wr_rx_hdr common; 25 u8 length; /* 44 */ 26 #if defined(__BIG_ENDIAN_BITFIELD) 27 u8 reserved:4, 28 add_link_rej_rsn:4; 29 #elif defined(__LITTLE_ENDIAN_BITFIELD) 30 u8 add_link_rej_rsn:4, 31 reserved:4; 32 #endif 33 u8 flags; 34 }; 35 36 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 37 38 struct smc_llc_msg_confirm_link { /* type 0x01 */ 39 struct smc_llc_hdr hd; 40 u8 sender_mac[ETH_ALEN]; 41 u8 sender_gid[SMC_GID_SIZE]; 42 u8 sender_qp_num[3]; 43 u8 link_num; 44 u8 link_uid[SMC_LGR_ID_SIZE]; 45 u8 max_links; 46 u8 reserved[9]; 47 }; 48 49 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 51 52 #define SMC_LLC_ADD_LNK_MAX_LINKS 2 53 54 struct smc_llc_msg_add_link { /* type 0x02 */ 55 struct smc_llc_hdr hd; 56 u8 sender_mac[ETH_ALEN]; 57 u8 reserved2[2]; 58 u8 sender_gid[SMC_GID_SIZE]; 59 u8 sender_qp_num[3]; 60 u8 link_num; 61 u8 flags2; /* QP mtu */ 62 u8 initial_psn[3]; 63 u8 reserved[8]; 64 }; 65 66 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 68 69 struct smc_llc_msg_del_link { /* type 0x04 */ 70 struct smc_llc_hdr hd; 71 u8 link_num; 72 __be32 reason; 73 u8 reserved[35]; 74 } __packed; /* format defined in RFC7609 */ 75 76 struct smc_llc_msg_test_link { /* type 0x07 */ 77 struct smc_llc_hdr hd; 78 u8 user_data[16]; 79 u8 reserved[24]; 80 }; 81 82 struct smc_rmb_rtoken { 83 union { 84 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 85 /* is actually the num of rtokens, first */ 86 /* rtoken is always for the current link */ 87 u8 link_id; /* link id of the rtoken */ 88 }; 89 __be32 rmb_key; 90 __be64 rmb_vaddr; 91 } __packed; /* format defined in RFC7609 */ 92 93 #define SMC_LLC_RKEYS_PER_MSG 3 94 95 struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 96 struct smc_llc_hdr hd; 97 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 98 u8 reserved; 99 }; 100 101 struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */ 102 struct smc_llc_hdr hd; 103 u8 num_rkeys; 104 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 105 }; 106 107 #define SMC_LLC_DEL_RKEY_MAX 8 108 #define SMC_LLC_FLAG_RKEY_NEG 0x20 109 110 struct smc_llc_msg_delete_rkey { /* type 0x09 */ 111 struct smc_llc_hdr hd; 112 u8 num_rkeys; 113 u8 err_mask; 114 u8 reserved[2]; 115 __be32 rkey[8]; 116 u8 reserved2[4]; 117 }; 118 119 union smc_llc_msg { 120 struct smc_llc_msg_confirm_link confirm_link; 121 struct smc_llc_msg_add_link add_link; 122 struct smc_llc_msg_del_link delete_link; 123 124 struct smc_llc_msg_confirm_rkey confirm_rkey; 125 struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont; 126 struct smc_llc_msg_delete_rkey delete_rkey; 127 128 struct smc_llc_msg_test_link test_link; 129 struct { 130 struct smc_llc_hdr hdr; 131 u8 data[SMC_LLC_DATA_LEN]; 132 } raw; 133 }; 134 135 #define SMC_LLC_FLAG_RESP 0x80 136 137 /********************************** send *************************************/ 138 139 struct smc_llc_tx_pend { 140 }; 141 142 /* handler for send/transmission completion of an LLC msg */ 143 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 144 struct smc_link *link, 145 enum ib_wc_status wc_status) 146 { 147 /* future work: handle wc_status error for recovery and failover */ 148 } 149 150 /** 151 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 152 * @link: Pointer to SMC link used for sending LLC control message. 153 * @wr_buf: Out variable returning pointer to work request payload buffer. 154 * @pend: Out variable returning pointer to private pending WR tracking. 155 * It's the context the transmit complete handler will get. 156 * 157 * Reserves and pre-fills an entry for a pending work request send/tx. 158 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 159 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 160 * 161 * Return: 0 on success, otherwise an error value. 162 */ 163 static int smc_llc_add_pending_send(struct smc_link *link, 164 struct smc_wr_buf **wr_buf, 165 struct smc_wr_tx_pend_priv **pend) 166 { 167 int rc; 168 169 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 170 pend); 171 if (rc < 0) 172 return rc; 173 BUILD_BUG_ON_MSG( 174 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 175 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 176 BUILD_BUG_ON_MSG( 177 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 178 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 179 BUILD_BUG_ON_MSG( 180 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 181 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 182 return 0; 183 } 184 185 /* high-level API to send LLC confirm link */ 186 int smc_llc_send_confirm_link(struct smc_link *link, 187 enum smc_llc_reqresp reqresp) 188 { 189 struct smc_link_group *lgr = smc_get_lgr(link); 190 struct smc_llc_msg_confirm_link *confllc; 191 struct smc_wr_tx_pend_priv *pend; 192 struct smc_wr_buf *wr_buf; 193 int rc; 194 195 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 196 if (rc) 197 return rc; 198 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 199 memset(confllc, 0, sizeof(*confllc)); 200 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 201 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 202 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 203 if (reqresp == SMC_LLC_RESP) 204 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 205 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 206 ETH_ALEN); 207 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 208 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 209 confllc->link_num = link->link_id; 210 memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); 211 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */ 212 /* send llc message */ 213 rc = smc_wr_tx_send(link, pend); 214 return rc; 215 } 216 217 /* send LLC confirm rkey request */ 218 static int smc_llc_send_confirm_rkey(struct smc_link *link, 219 struct smc_buf_desc *rmb_desc) 220 { 221 struct smc_llc_msg_confirm_rkey *rkeyllc; 222 struct smc_wr_tx_pend_priv *pend; 223 struct smc_wr_buf *wr_buf; 224 int rc; 225 226 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 227 if (rc) 228 return rc; 229 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 230 memset(rkeyllc, 0, sizeof(*rkeyllc)); 231 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 232 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 233 rkeyllc->rtoken[0].rmb_key = 234 htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey); 235 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 236 (u64)sg_dma_address(rmb_desc->sgt[SMC_SINGLE_LINK].sgl)); 237 /* send llc message */ 238 rc = smc_wr_tx_send(link, pend); 239 return rc; 240 } 241 242 /* send LLC delete rkey request */ 243 static int smc_llc_send_delete_rkey(struct smc_link *link, 244 struct smc_buf_desc *rmb_desc) 245 { 246 struct smc_llc_msg_delete_rkey *rkeyllc; 247 struct smc_wr_tx_pend_priv *pend; 248 struct smc_wr_buf *wr_buf; 249 int rc; 250 251 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 252 if (rc) 253 return rc; 254 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 255 memset(rkeyllc, 0, sizeof(*rkeyllc)); 256 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 257 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 258 rkeyllc->num_rkeys = 1; 259 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey); 260 /* send llc message */ 261 rc = smc_wr_tx_send(link, pend); 262 return rc; 263 } 264 265 /* prepare an add link message */ 266 static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc, 267 struct smc_link *link, u8 mac[], u8 gid[], 268 enum smc_llc_reqresp reqresp) 269 { 270 memset(addllc, 0, sizeof(*addllc)); 271 addllc->hd.common.type = SMC_LLC_ADD_LINK; 272 addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 273 if (reqresp == SMC_LLC_RESP) { 274 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 275 /* always reject more links for now */ 276 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 277 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 278 } 279 memcpy(addllc->sender_mac, mac, ETH_ALEN); 280 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 281 } 282 283 /* send ADD LINK request or response */ 284 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 285 enum smc_llc_reqresp reqresp) 286 { 287 struct smc_llc_msg_add_link *addllc; 288 struct smc_wr_tx_pend_priv *pend; 289 struct smc_wr_buf *wr_buf; 290 int rc; 291 292 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 293 if (rc) 294 return rc; 295 addllc = (struct smc_llc_msg_add_link *)wr_buf; 296 smc_llc_prep_add_link(addllc, link, mac, gid, reqresp); 297 /* send llc message */ 298 rc = smc_wr_tx_send(link, pend); 299 return rc; 300 } 301 302 /* prepare a delete link message */ 303 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc, 304 struct smc_link *link, 305 enum smc_llc_reqresp reqresp, bool orderly) 306 { 307 memset(delllc, 0, sizeof(*delllc)); 308 delllc->hd.common.type = SMC_LLC_DELETE_LINK; 309 delllc->hd.length = sizeof(struct smc_llc_msg_add_link); 310 if (reqresp == SMC_LLC_RESP) 311 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 312 /* DEL_LINK_ALL because only 1 link supported */ 313 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 314 if (orderly) 315 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 316 delllc->link_num = link->link_id; 317 } 318 319 /* send DELETE LINK request or response */ 320 int smc_llc_send_delete_link(struct smc_link *link, 321 enum smc_llc_reqresp reqresp, bool orderly) 322 { 323 struct smc_llc_msg_del_link *delllc; 324 struct smc_wr_tx_pend_priv *pend; 325 struct smc_wr_buf *wr_buf; 326 int rc; 327 328 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 329 if (rc) 330 return rc; 331 delllc = (struct smc_llc_msg_del_link *)wr_buf; 332 smc_llc_prep_delete_link(delllc, link, reqresp, orderly); 333 /* send llc message */ 334 rc = smc_wr_tx_send(link, pend); 335 return rc; 336 } 337 338 /* send LLC test link request */ 339 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 340 { 341 struct smc_llc_msg_test_link *testllc; 342 struct smc_wr_tx_pend_priv *pend; 343 struct smc_wr_buf *wr_buf; 344 int rc; 345 346 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 347 if (rc) 348 return rc; 349 testllc = (struct smc_llc_msg_test_link *)wr_buf; 350 memset(testllc, 0, sizeof(*testllc)); 351 testllc->hd.common.type = SMC_LLC_TEST_LINK; 352 testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 353 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 354 /* send llc message */ 355 rc = smc_wr_tx_send(link, pend); 356 return rc; 357 } 358 359 struct smc_llc_send_work { 360 struct work_struct work; 361 struct smc_link *link; 362 int llclen; 363 union smc_llc_msg llcbuf; 364 }; 365 366 /* worker that sends a prepared message */ 367 static void smc_llc_send_message_work(struct work_struct *work) 368 { 369 struct smc_llc_send_work *llcwrk = container_of(work, 370 struct smc_llc_send_work, work); 371 struct smc_wr_tx_pend_priv *pend; 372 struct smc_wr_buf *wr_buf; 373 int rc; 374 375 if (llcwrk->link->state == SMC_LNK_INACTIVE) 376 goto out; 377 rc = smc_llc_add_pending_send(llcwrk->link, &wr_buf, &pend); 378 if (rc) 379 goto out; 380 memcpy(wr_buf, &llcwrk->llcbuf, llcwrk->llclen); 381 smc_wr_tx_send(llcwrk->link, pend); 382 out: 383 kfree(llcwrk); 384 } 385 386 /* copy llcbuf and schedule an llc send on link */ 387 static int smc_llc_send_message(struct smc_link *link, void *llcbuf, int llclen) 388 { 389 struct smc_llc_send_work *wrk = kmalloc(sizeof(*wrk), GFP_ATOMIC); 390 391 if (!wrk) 392 return -ENOMEM; 393 INIT_WORK(&wrk->work, smc_llc_send_message_work); 394 wrk->link = link; 395 wrk->llclen = llclen; 396 memcpy(&wrk->llcbuf, llcbuf, llclen); 397 queue_work(link->llc_wq, &wrk->work); 398 return 0; 399 } 400 401 /********************************* receive ***********************************/ 402 403 static void smc_llc_rx_confirm_link(struct smc_link *link, 404 struct smc_llc_msg_confirm_link *llc) 405 { 406 struct smc_link_group *lgr = smc_get_lgr(link); 407 int conf_rc; 408 409 /* RMBE eyecatchers are not supported */ 410 if (llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC) 411 conf_rc = 0; 412 else 413 conf_rc = ENOTSUPP; 414 415 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 416 if (lgr->role == SMC_SERV && 417 link->state == SMC_LNK_ACTIVATING) { 418 link->llc_confirm_resp_rc = conf_rc; 419 complete(&link->llc_confirm_resp); 420 } 421 } else { 422 if (lgr->role == SMC_CLNT && 423 link->state == SMC_LNK_ACTIVATING) { 424 link->llc_confirm_rc = conf_rc; 425 link->link_id = llc->link_num; 426 complete(&link->llc_confirm); 427 } 428 } 429 } 430 431 static void smc_llc_rx_add_link(struct smc_link *link, 432 struct smc_llc_msg_add_link *llc) 433 { 434 struct smc_link_group *lgr = smc_get_lgr(link); 435 436 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 437 if (link->state == SMC_LNK_ACTIVATING) 438 complete(&link->llc_add_resp); 439 } else { 440 if (link->state == SMC_LNK_ACTIVATING) { 441 complete(&link->llc_add); 442 return; 443 } 444 445 if (lgr->role == SMC_SERV) { 446 smc_llc_prep_add_link(llc, link, 447 link->smcibdev->mac[link->ibport - 1], 448 link->gid, SMC_LLC_REQ); 449 450 } else { 451 smc_llc_prep_add_link(llc, link, 452 link->smcibdev->mac[link->ibport - 1], 453 link->gid, SMC_LLC_RESP); 454 } 455 smc_llc_send_message(link, llc, sizeof(*llc)); 456 } 457 } 458 459 static void smc_llc_rx_delete_link(struct smc_link *link, 460 struct smc_llc_msg_del_link *llc) 461 { 462 struct smc_link_group *lgr = smc_get_lgr(link); 463 464 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 465 if (lgr->role == SMC_SERV) 466 smc_lgr_schedule_free_work_fast(lgr); 467 } else { 468 smc_lgr_forget(lgr); 469 smc_llc_link_deleting(link); 470 if (lgr->role == SMC_SERV) { 471 /* client asks to delete this link, send request */ 472 smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true); 473 } else { 474 /* server requests to delete this link, send response */ 475 smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true); 476 } 477 smc_llc_send_message(link, llc, sizeof(*llc)); 478 smc_lgr_terminate_sched(lgr); 479 } 480 } 481 482 static void smc_llc_rx_test_link(struct smc_link *link, 483 struct smc_llc_msg_test_link *llc) 484 { 485 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 486 if (link->state == SMC_LNK_ACTIVE) 487 complete(&link->llc_testlink_resp); 488 } else { 489 llc->hd.flags |= SMC_LLC_FLAG_RESP; 490 smc_llc_send_message(link, llc, sizeof(*llc)); 491 } 492 } 493 494 static void smc_llc_rx_confirm_rkey(struct smc_link *link, 495 struct smc_llc_msg_confirm_rkey *llc) 496 { 497 int rc; 498 499 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 500 link->llc_confirm_rkey_rc = llc->hd.flags & 501 SMC_LLC_FLAG_RKEY_NEG; 502 complete(&link->llc_confirm_rkey); 503 } else { 504 rc = smc_rtoken_add(smc_get_lgr(link), 505 llc->rtoken[0].rmb_vaddr, 506 llc->rtoken[0].rmb_key); 507 508 /* ignore rtokens for other links, we have only one link */ 509 510 llc->hd.flags |= SMC_LLC_FLAG_RESP; 511 if (rc < 0) 512 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 513 smc_llc_send_message(link, llc, sizeof(*llc)); 514 } 515 } 516 517 static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link, 518 struct smc_llc_msg_confirm_rkey_cont *llc) 519 { 520 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 521 /* unused as long as we don't send this type of msg */ 522 } else { 523 /* ignore rtokens for other links, we have only one link */ 524 llc->hd.flags |= SMC_LLC_FLAG_RESP; 525 smc_llc_send_message(link, llc, sizeof(*llc)); 526 } 527 } 528 529 static void smc_llc_rx_delete_rkey(struct smc_link *link, 530 struct smc_llc_msg_delete_rkey *llc) 531 { 532 u8 err_mask = 0; 533 int i, max; 534 535 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 536 link->llc_delete_rkey_rc = llc->hd.flags & 537 SMC_LLC_FLAG_RKEY_NEG; 538 complete(&link->llc_delete_rkey); 539 } else { 540 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 541 for (i = 0; i < max; i++) { 542 if (smc_rtoken_delete(smc_get_lgr(link), llc->rkey[i])) 543 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 544 } 545 546 if (err_mask) { 547 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 548 llc->err_mask = err_mask; 549 } 550 551 llc->hd.flags |= SMC_LLC_FLAG_RESP; 552 smc_llc_send_message(link, llc, sizeof(*llc)); 553 } 554 } 555 556 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 557 { 558 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 559 union smc_llc_msg *llc = buf; 560 561 if (wc->byte_len < sizeof(*llc)) 562 return; /* short message */ 563 if (llc->raw.hdr.length != sizeof(*llc)) 564 return; /* invalid message */ 565 if (link->state == SMC_LNK_INACTIVE) 566 return; /* link not active, drop msg */ 567 568 switch (llc->raw.hdr.common.type) { 569 case SMC_LLC_TEST_LINK: 570 smc_llc_rx_test_link(link, &llc->test_link); 571 break; 572 case SMC_LLC_CONFIRM_LINK: 573 smc_llc_rx_confirm_link(link, &llc->confirm_link); 574 break; 575 case SMC_LLC_ADD_LINK: 576 smc_llc_rx_add_link(link, &llc->add_link); 577 break; 578 case SMC_LLC_DELETE_LINK: 579 smc_llc_rx_delete_link(link, &llc->delete_link); 580 break; 581 case SMC_LLC_CONFIRM_RKEY: 582 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey); 583 break; 584 case SMC_LLC_CONFIRM_RKEY_CONT: 585 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont); 586 break; 587 case SMC_LLC_DELETE_RKEY: 588 smc_llc_rx_delete_rkey(link, &llc->delete_rkey); 589 break; 590 } 591 } 592 593 /***************************** worker, utils *********************************/ 594 595 static void smc_llc_testlink_work(struct work_struct *work) 596 { 597 struct smc_link *link = container_of(to_delayed_work(work), 598 struct smc_link, llc_testlink_wrk); 599 unsigned long next_interval; 600 unsigned long expire_time; 601 u8 user_data[16] = { 0 }; 602 int rc; 603 604 if (link->state != SMC_LNK_ACTIVE) 605 return; /* don't reschedule worker */ 606 expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 607 if (time_is_after_jiffies(expire_time)) { 608 next_interval = expire_time - jiffies; 609 goto out; 610 } 611 reinit_completion(&link->llc_testlink_resp); 612 smc_llc_send_test_link(link, user_data); 613 /* receive TEST LINK response over RoCE fabric */ 614 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 615 SMC_LLC_WAIT_TIME); 616 if (rc <= 0) { 617 smc_lgr_terminate_sched(smc_get_lgr(link)); 618 return; 619 } 620 next_interval = link->llc_testlink_time; 621 out: 622 queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk, 623 next_interval); 624 } 625 626 int smc_llc_link_init(struct smc_link *link) 627 { 628 struct smc_link_group *lgr = smc_get_lgr(link); 629 link->llc_wq = alloc_ordered_workqueue("llc_wq-%x:%x)", WQ_MEM_RECLAIM, 630 *((u32 *)lgr->id), 631 link->link_id); 632 if (!link->llc_wq) 633 return -ENOMEM; 634 init_completion(&link->llc_confirm); 635 init_completion(&link->llc_confirm_resp); 636 init_completion(&link->llc_add); 637 init_completion(&link->llc_add_resp); 638 init_completion(&link->llc_confirm_rkey); 639 init_completion(&link->llc_delete_rkey); 640 mutex_init(&link->llc_delete_rkey_mutex); 641 init_completion(&link->llc_testlink_resp); 642 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 643 return 0; 644 } 645 646 void smc_llc_link_active(struct smc_link *link, int testlink_time) 647 { 648 link->state = SMC_LNK_ACTIVE; 649 if (testlink_time) { 650 link->llc_testlink_time = testlink_time * HZ; 651 queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk, 652 link->llc_testlink_time); 653 } 654 } 655 656 void smc_llc_link_deleting(struct smc_link *link) 657 { 658 link->state = SMC_LNK_DELETING; 659 smc_wr_wakeup_tx_wait(link); 660 } 661 662 /* called in tasklet context */ 663 void smc_llc_link_inactive(struct smc_link *link) 664 { 665 link->state = SMC_LNK_INACTIVE; 666 cancel_delayed_work(&link->llc_testlink_wrk); 667 smc_wr_wakeup_reg_wait(link); 668 smc_wr_wakeup_tx_wait(link); 669 } 670 671 /* called in worker context */ 672 void smc_llc_link_clear(struct smc_link *link) 673 { 674 flush_workqueue(link->llc_wq); 675 destroy_workqueue(link->llc_wq); 676 } 677 678 /* register a new rtoken at the remote peer */ 679 int smc_llc_do_confirm_rkey(struct smc_link *link, 680 struct smc_buf_desc *rmb_desc) 681 { 682 int rc; 683 684 /* protected by mutex smc_create_lgr_pending */ 685 reinit_completion(&link->llc_confirm_rkey); 686 rc = smc_llc_send_confirm_rkey(link, rmb_desc); 687 if (rc) 688 return rc; 689 /* receive CONFIRM RKEY response from server over RoCE fabric */ 690 rc = wait_for_completion_interruptible_timeout(&link->llc_confirm_rkey, 691 SMC_LLC_WAIT_TIME); 692 if (rc <= 0 || link->llc_confirm_rkey_rc) 693 return -EFAULT; 694 return 0; 695 } 696 697 /* unregister an rtoken at the remote peer */ 698 int smc_llc_do_delete_rkey(struct smc_link *link, 699 struct smc_buf_desc *rmb_desc) 700 { 701 int rc = 0; 702 703 mutex_lock(&link->llc_delete_rkey_mutex); 704 if (link->state != SMC_LNK_ACTIVE) 705 goto out; 706 reinit_completion(&link->llc_delete_rkey); 707 rc = smc_llc_send_delete_rkey(link, rmb_desc); 708 if (rc) 709 goto out; 710 /* receive DELETE RKEY response from server over RoCE fabric */ 711 rc = wait_for_completion_interruptible_timeout(&link->llc_delete_rkey, 712 SMC_LLC_WAIT_TIME); 713 if (rc <= 0 || link->llc_delete_rkey_rc) 714 rc = -EFAULT; 715 else 716 rc = 0; 717 out: 718 mutex_unlock(&link->llc_delete_rkey_mutex); 719 return rc; 720 } 721 722 /***************************** init, exit, misc ******************************/ 723 724 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 725 { 726 .handler = smc_llc_rx_handler, 727 .type = SMC_LLC_CONFIRM_LINK 728 }, 729 { 730 .handler = smc_llc_rx_handler, 731 .type = SMC_LLC_TEST_LINK 732 }, 733 { 734 .handler = smc_llc_rx_handler, 735 .type = SMC_LLC_ADD_LINK 736 }, 737 { 738 .handler = smc_llc_rx_handler, 739 .type = SMC_LLC_DELETE_LINK 740 }, 741 { 742 .handler = smc_llc_rx_handler, 743 .type = SMC_LLC_CONFIRM_RKEY 744 }, 745 { 746 .handler = smc_llc_rx_handler, 747 .type = SMC_LLC_CONFIRM_RKEY_CONT 748 }, 749 { 750 .handler = smc_llc_rx_handler, 751 .type = SMC_LLC_DELETE_RKEY 752 }, 753 { 754 .handler = NULL, 755 } 756 }; 757 758 int __init smc_llc_init(void) 759 { 760 struct smc_wr_rx_handler *handler; 761 int rc = 0; 762 763 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 764 INIT_HLIST_NODE(&handler->list); 765 rc = smc_wr_rx_register_handler(handler); 766 if (rc) 767 break; 768 } 769 return rc; 770 } 771