1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13 #include <net/tcp.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "smc.h" 17 #include "smc_core.h" 18 #include "smc_clc.h" 19 #include "smc_llc.h" 20 21 #define SMC_LLC_DATA_LEN 40 22 23 struct smc_llc_hdr { 24 struct smc_wr_rx_hdr common; 25 u8 length; /* 44 */ 26 #if defined(__BIG_ENDIAN_BITFIELD) 27 u8 reserved:4, 28 add_link_rej_rsn:4; 29 #elif defined(__LITTLE_ENDIAN_BITFIELD) 30 u8 add_link_rej_rsn:4, 31 reserved:4; 32 #endif 33 u8 flags; 34 }; 35 36 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 37 38 struct smc_llc_msg_confirm_link { /* type 0x01 */ 39 struct smc_llc_hdr hd; 40 u8 sender_mac[ETH_ALEN]; 41 u8 sender_gid[SMC_GID_SIZE]; 42 u8 sender_qp_num[3]; 43 u8 link_num; 44 u8 link_uid[SMC_LGR_ID_SIZE]; 45 u8 max_links; 46 u8 reserved[9]; 47 }; 48 49 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 51 52 #define SMC_LLC_ADD_LNK_MAX_LINKS 2 53 54 struct smc_llc_msg_add_link { /* type 0x02 */ 55 struct smc_llc_hdr hd; 56 u8 sender_mac[ETH_ALEN]; 57 u8 reserved2[2]; 58 u8 sender_gid[SMC_GID_SIZE]; 59 u8 sender_qp_num[3]; 60 u8 link_num; 61 u8 flags2; /* QP mtu */ 62 u8 initial_psn[3]; 63 u8 reserved[8]; 64 }; 65 66 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 68 69 struct smc_llc_msg_del_link { /* type 0x04 */ 70 struct smc_llc_hdr hd; 71 u8 link_num; 72 __be32 reason; 73 u8 reserved[35]; 74 } __packed; /* format defined in RFC7609 */ 75 76 struct smc_llc_msg_test_link { /* type 0x07 */ 77 struct smc_llc_hdr hd; 78 u8 user_data[16]; 79 u8 reserved[24]; 80 }; 81 82 struct smc_rmb_rtoken { 83 union { 84 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 85 /* is actually the num of rtokens, first */ 86 /* rtoken is always for the current link */ 87 u8 link_id; /* link id of the rtoken */ 88 }; 89 __be32 rmb_key; 90 __be64 rmb_vaddr; 91 } __packed; /* format defined in RFC7609 */ 92 93 #define SMC_LLC_RKEYS_PER_MSG 3 94 95 struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 96 struct smc_llc_hdr hd; 97 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 98 u8 reserved; 99 }; 100 101 struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */ 102 struct smc_llc_hdr hd; 103 u8 num_rkeys; 104 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 105 }; 106 107 #define SMC_LLC_DEL_RKEY_MAX 8 108 #define SMC_LLC_FLAG_RKEY_NEG 0x20 109 110 struct smc_llc_msg_delete_rkey { /* type 0x09 */ 111 struct smc_llc_hdr hd; 112 u8 num_rkeys; 113 u8 err_mask; 114 u8 reserved[2]; 115 __be32 rkey[8]; 116 u8 reserved2[4]; 117 }; 118 119 union smc_llc_msg { 120 struct smc_llc_msg_confirm_link confirm_link; 121 struct smc_llc_msg_add_link add_link; 122 struct smc_llc_msg_del_link delete_link; 123 124 struct smc_llc_msg_confirm_rkey confirm_rkey; 125 struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont; 126 struct smc_llc_msg_delete_rkey delete_rkey; 127 128 struct smc_llc_msg_test_link test_link; 129 struct { 130 struct smc_llc_hdr hdr; 131 u8 data[SMC_LLC_DATA_LEN]; 132 } raw; 133 }; 134 135 #define SMC_LLC_FLAG_RESP 0x80 136 137 /********************************** send *************************************/ 138 139 struct smc_llc_tx_pend { 140 }; 141 142 /* handler for send/transmission completion of an LLC msg */ 143 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 144 struct smc_link *link, 145 enum ib_wc_status wc_status) 146 { 147 /* future work: handle wc_status error for recovery and failover */ 148 } 149 150 /** 151 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 152 * @link: Pointer to SMC link used for sending LLC control message. 153 * @wr_buf: Out variable returning pointer to work request payload buffer. 154 * @pend: Out variable returning pointer to private pending WR tracking. 155 * It's the context the transmit complete handler will get. 156 * 157 * Reserves and pre-fills an entry for a pending work request send/tx. 158 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 159 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 160 * 161 * Return: 0 on success, otherwise an error value. 162 */ 163 static int smc_llc_add_pending_send(struct smc_link *link, 164 struct smc_wr_buf **wr_buf, 165 struct smc_wr_tx_pend_priv **pend) 166 { 167 int rc; 168 169 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, pend); 170 if (rc < 0) 171 return rc; 172 BUILD_BUG_ON_MSG( 173 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 174 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 175 BUILD_BUG_ON_MSG( 176 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 177 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 178 BUILD_BUG_ON_MSG( 179 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 180 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 181 return 0; 182 } 183 184 /* high-level API to send LLC confirm link */ 185 int smc_llc_send_confirm_link(struct smc_link *link, u8 mac[], 186 union ib_gid *gid, 187 enum smc_llc_reqresp reqresp) 188 { 189 struct smc_link_group *lgr = container_of(link, struct smc_link_group, 190 lnk[SMC_SINGLE_LINK]); 191 struct smc_llc_msg_confirm_link *confllc; 192 struct smc_wr_tx_pend_priv *pend; 193 struct smc_wr_buf *wr_buf; 194 int rc; 195 196 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 197 if (rc) 198 return rc; 199 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 200 memset(confllc, 0, sizeof(*confllc)); 201 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 202 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 203 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 204 if (reqresp == SMC_LLC_RESP) 205 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 206 memcpy(confllc->sender_mac, mac, ETH_ALEN); 207 memcpy(confllc->sender_gid, gid, SMC_GID_SIZE); 208 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 209 confllc->link_num = link->link_id; 210 memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); 211 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */ 212 /* send llc message */ 213 rc = smc_wr_tx_send(link, pend); 214 return rc; 215 } 216 217 /* send ADD LINK request or response */ 218 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], 219 union ib_gid *gid, 220 enum smc_llc_reqresp reqresp) 221 { 222 struct smc_llc_msg_add_link *addllc; 223 struct smc_wr_tx_pend_priv *pend; 224 struct smc_wr_buf *wr_buf; 225 int rc; 226 227 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 228 if (rc) 229 return rc; 230 addllc = (struct smc_llc_msg_add_link *)wr_buf; 231 memset(addllc, 0, sizeof(*addllc)); 232 addllc->hd.common.type = SMC_LLC_ADD_LINK; 233 addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 234 if (reqresp == SMC_LLC_RESP) { 235 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 236 /* always reject more links for now */ 237 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 238 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 239 } 240 memcpy(addllc->sender_mac, mac, ETH_ALEN); 241 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 242 /* send llc message */ 243 rc = smc_wr_tx_send(link, pend); 244 return rc; 245 } 246 247 /* send DELETE LINK request or response */ 248 int smc_llc_send_delete_link(struct smc_link *link, 249 enum smc_llc_reqresp reqresp) 250 { 251 struct smc_llc_msg_del_link *delllc; 252 struct smc_wr_tx_pend_priv *pend; 253 struct smc_wr_buf *wr_buf; 254 int rc; 255 256 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 257 if (rc) 258 return rc; 259 delllc = (struct smc_llc_msg_del_link *)wr_buf; 260 memset(delllc, 0, sizeof(*delllc)); 261 delllc->hd.common.type = SMC_LLC_DELETE_LINK; 262 delllc->hd.length = sizeof(struct smc_llc_msg_add_link); 263 if (reqresp == SMC_LLC_RESP) 264 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 265 /* DEL_LINK_ALL because only 1 link supported */ 266 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 267 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 268 delllc->link_num = link->link_id; 269 /* send llc message */ 270 rc = smc_wr_tx_send(link, pend); 271 return rc; 272 } 273 274 /* send LLC test link request or response */ 275 int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16], 276 enum smc_llc_reqresp reqresp) 277 { 278 struct smc_llc_msg_test_link *testllc; 279 struct smc_wr_tx_pend_priv *pend; 280 struct smc_wr_buf *wr_buf; 281 int rc; 282 283 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 284 if (rc) 285 return rc; 286 testllc = (struct smc_llc_msg_test_link *)wr_buf; 287 memset(testllc, 0, sizeof(*testllc)); 288 testllc->hd.common.type = SMC_LLC_TEST_LINK; 289 testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 290 if (reqresp == SMC_LLC_RESP) 291 testllc->hd.flags |= SMC_LLC_FLAG_RESP; 292 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 293 /* send llc message */ 294 rc = smc_wr_tx_send(link, pend); 295 return rc; 296 } 297 298 /* send a prepared message */ 299 static int smc_llc_send_message(struct smc_link *link, void *llcbuf, int llclen) 300 { 301 struct smc_wr_tx_pend_priv *pend; 302 struct smc_wr_buf *wr_buf; 303 int rc; 304 305 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 306 if (rc) 307 return rc; 308 memcpy(wr_buf, llcbuf, llclen); 309 /* send llc message */ 310 rc = smc_wr_tx_send(link, pend); 311 return rc; 312 } 313 314 /********************************* receive ***********************************/ 315 316 static void smc_llc_rx_confirm_link(struct smc_link *link, 317 struct smc_llc_msg_confirm_link *llc) 318 { 319 struct smc_link_group *lgr; 320 int conf_rc; 321 322 lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); 323 324 /* RMBE eyecatchers are not supported */ 325 if (llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC) 326 conf_rc = 0; 327 else 328 conf_rc = ENOTSUPP; 329 330 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 331 if (lgr->role == SMC_SERV && 332 link->state == SMC_LNK_ACTIVATING) { 333 link->llc_confirm_resp_rc = conf_rc; 334 complete(&link->llc_confirm_resp); 335 } 336 } else { 337 if (lgr->role == SMC_CLNT && 338 link->state == SMC_LNK_ACTIVATING) { 339 link->llc_confirm_rc = conf_rc; 340 link->link_id = llc->link_num; 341 complete(&link->llc_confirm); 342 } 343 } 344 } 345 346 static void smc_llc_rx_add_link(struct smc_link *link, 347 struct smc_llc_msg_add_link *llc) 348 { 349 struct smc_link_group *lgr = container_of(link, struct smc_link_group, 350 lnk[SMC_SINGLE_LINK]); 351 352 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 353 if (link->state == SMC_LNK_ACTIVATING) 354 complete(&link->llc_add_resp); 355 } else { 356 if (link->state == SMC_LNK_ACTIVATING) { 357 complete(&link->llc_add); 358 return; 359 } 360 361 if (lgr->role == SMC_SERV) { 362 smc_llc_send_add_link(link, 363 link->smcibdev->mac[link->ibport - 1], 364 &link->smcibdev->gid[link->ibport - 1], 365 SMC_LLC_REQ); 366 367 } else { 368 smc_llc_send_add_link(link, 369 link->smcibdev->mac[link->ibport - 1], 370 &link->smcibdev->gid[link->ibport - 1], 371 SMC_LLC_RESP); 372 } 373 } 374 } 375 376 static void smc_llc_rx_delete_link(struct smc_link *link, 377 struct smc_llc_msg_del_link *llc) 378 { 379 struct smc_link_group *lgr = container_of(link, struct smc_link_group, 380 lnk[SMC_SINGLE_LINK]); 381 382 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 383 if (lgr->role == SMC_SERV) 384 smc_lgr_terminate(lgr); 385 } else { 386 if (lgr->role == SMC_SERV) { 387 smc_lgr_forget(lgr); 388 smc_llc_send_delete_link(link, SMC_LLC_REQ); 389 } else { 390 smc_llc_send_delete_link(link, SMC_LLC_RESP); 391 smc_lgr_terminate(lgr); 392 } 393 } 394 } 395 396 static void smc_llc_rx_test_link(struct smc_link *link, 397 struct smc_llc_msg_test_link *llc) 398 { 399 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 400 /* unused as long as we don't send this type of msg */ 401 } else { 402 smc_llc_send_test_link(link, llc->user_data, SMC_LLC_RESP); 403 } 404 } 405 406 static void smc_llc_rx_confirm_rkey(struct smc_link *link, 407 struct smc_llc_msg_confirm_rkey *llc) 408 { 409 struct smc_link_group *lgr; 410 int rc; 411 412 lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); 413 414 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 415 /* unused as long as we don't send this type of msg */ 416 } else { 417 rc = smc_rtoken_add(lgr, 418 llc->rtoken[0].rmb_vaddr, 419 llc->rtoken[0].rmb_key); 420 421 /* ignore rtokens for other links, we have only one link */ 422 423 llc->hd.flags |= SMC_LLC_FLAG_RESP; 424 if (rc < 0) 425 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 426 smc_llc_send_message(link, (void *)llc, sizeof(*llc)); 427 } 428 } 429 430 static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link, 431 struct smc_llc_msg_confirm_rkey_cont *llc) 432 { 433 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 434 /* unused as long as we don't send this type of msg */ 435 } else { 436 /* ignore rtokens for other links, we have only one link */ 437 llc->hd.flags |= SMC_LLC_FLAG_RESP; 438 smc_llc_send_message(link, (void *)llc, sizeof(*llc)); 439 } 440 } 441 442 static void smc_llc_rx_delete_rkey(struct smc_link *link, 443 struct smc_llc_msg_delete_rkey *llc) 444 { 445 struct smc_link_group *lgr; 446 u8 err_mask = 0; 447 int i, max; 448 449 lgr = container_of(link, struct smc_link_group, lnk[SMC_SINGLE_LINK]); 450 451 if (llc->hd.flags & SMC_LLC_FLAG_RESP) { 452 /* unused as long as we don't send this type of msg */ 453 } else { 454 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 455 for (i = 0; i < max; i++) { 456 if (smc_rtoken_delete(lgr, llc->rkey[i])) 457 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 458 } 459 460 if (err_mask) { 461 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 462 llc->err_mask = err_mask; 463 } 464 465 llc->hd.flags |= SMC_LLC_FLAG_RESP; 466 smc_llc_send_message(link, (void *)llc, sizeof(*llc)); 467 } 468 } 469 470 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 471 { 472 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 473 union smc_llc_msg *llc = buf; 474 475 if (wc->byte_len < sizeof(*llc)) 476 return; /* short message */ 477 if (llc->raw.hdr.length != sizeof(*llc)) 478 return; /* invalid message */ 479 480 switch (llc->raw.hdr.common.type) { 481 case SMC_LLC_TEST_LINK: 482 smc_llc_rx_test_link(link, &llc->test_link); 483 break; 484 case SMC_LLC_CONFIRM_LINK: 485 smc_llc_rx_confirm_link(link, &llc->confirm_link); 486 break; 487 case SMC_LLC_ADD_LINK: 488 smc_llc_rx_add_link(link, &llc->add_link); 489 break; 490 case SMC_LLC_DELETE_LINK: 491 smc_llc_rx_delete_link(link, &llc->delete_link); 492 break; 493 case SMC_LLC_CONFIRM_RKEY: 494 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey); 495 break; 496 case SMC_LLC_CONFIRM_RKEY_CONT: 497 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont); 498 break; 499 case SMC_LLC_DELETE_RKEY: 500 smc_llc_rx_delete_rkey(link, &llc->delete_rkey); 501 break; 502 } 503 } 504 505 /***************************** init, exit, misc ******************************/ 506 507 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 508 { 509 .handler = smc_llc_rx_handler, 510 .type = SMC_LLC_CONFIRM_LINK 511 }, 512 { 513 .handler = smc_llc_rx_handler, 514 .type = SMC_LLC_TEST_LINK 515 }, 516 { 517 .handler = smc_llc_rx_handler, 518 .type = SMC_LLC_ADD_LINK 519 }, 520 { 521 .handler = smc_llc_rx_handler, 522 .type = SMC_LLC_DELETE_LINK 523 }, 524 { 525 .handler = smc_llc_rx_handler, 526 .type = SMC_LLC_CONFIRM_RKEY 527 }, 528 { 529 .handler = smc_llc_rx_handler, 530 .type = SMC_LLC_CONFIRM_RKEY_CONT 531 }, 532 { 533 .handler = smc_llc_rx_handler, 534 .type = SMC_LLC_DELETE_RKEY 535 }, 536 { 537 .handler = NULL, 538 } 539 }; 540 541 int __init smc_llc_init(void) 542 { 543 struct smc_wr_rx_handler *handler; 544 int rc = 0; 545 546 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 547 INIT_HLIST_NODE(&handler->list); 548 rc = smc_wr_rx_register_handler(handler); 549 if (rc) 550 break; 551 } 552 return rc; 553 } 554